Skip to content

Commit 53145f7

Browse files
ichiriacichiriac
authored
Merge pull request glayzzle#472 from glayzzle/numeric_literal_separator
Numeric literal separator
2 parents 60c9d91 + 5813bde commit 53145f7

File tree

8 files changed

+527
-166
lines changed

8 files changed

+527
-166
lines changed

‎package-lock.json

Lines changed: 54 additions & 54 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/lexer/numbers.js

Lines changed: 61 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -16,58 +16,92 @@ if (process.arch == "x64") {
1616
module.exports = {
1717
consume_NUM: function() {
1818
let ch = this.yytext[0];
19-
let hasPoint = this.yytext[0] === ".";
19+
let hasPoint = ch === ".";
2020
if (ch === "0") {
2121
ch = this.input();
2222
// check if hexa
2323
if (ch === "x" || ch === "X") {
2424
ch = this.input();
25-
if (this.is_HEX()) {
25+
if (ch !== "_" && this.is_HEX()) {
2626
return this.consume_HNUM();
2727
} else {
2828
this.unput(ch ? 2 : 1);
2929
}
30+
// check binary notation
3031
} else if (ch === "b" || ch === "B") {
3132
ch = this.input();
32-
if (ch === "0" || ch === "1") {
33+
if ((ch !== "_" && ch === "0") || ch === "1") {
3334
return this.consume_BNUM();
3435
} else {
3536
this.unput(ch ? 2 : 1);
3637
}
38+
// @fixme check octal notation ? not usefull
3739
} else if (!this.is_NUM()) {
3840
if (ch) this.unput(1);
3941
}
4042
}
4143

4244
while (this.offset < this.size) {
45+
const prev = ch;
4346
ch = this.input();
44-
if (!this.is_NUM()) {
45-
if (ch === "." && !hasPoint) {
46-
hasPoint = true;
47-
} else if (ch === "e" || ch === "E") {
48-
ch = this.input();
49-
if (ch === "+" || ch === "-") {
50-
ch = this.input();
51-
if (this.is_NUM()) {
52-
this.consume_LNUM();
53-
return this.tok.T_DNUMBER;
54-
} else {
55-
this.unput(ch ? 3 : 2);
56-
break;
57-
}
58-
} else if (this.is_NUM()) {
59-
this.consume_LNUM();
60-
return this.tok.T_DNUMBER;
61-
} else {
62-
this.unput(ch ? 2 : 1);
63-
break;
64-
}
65-
} else {
66-
if (ch) this.unput(1);
47+
48+
if (ch === "_") {
49+
if (prev === "_") {
50+
// restriction : next to underscore / 1__1;
51+
this.unput(2); // keep 1
52+
break;
53+
}
54+
if (prev === ".") {
55+
// next to decimal point "1._0"
56+
this.unput(1); // keep 1.
57+
break;
58+
}
59+
if (prev === "e" || prev === "E") {
60+
// next to e "1e_10"
61+
this.unput(2); // keep 1
62+
break;
63+
}
64+
} else if (ch === ".") {
65+
if (hasPoint) {
66+
// no multiple points "1.0.5"
67+
this.unput(1); // keep 1.0
68+
break;
69+
}
70+
if (prev === "_") {
71+
// next to decimal point "1_.0"
72+
this.unput(2); // keep 1
6773
break;
6874
}
75+
hasPoint = true;
76+
continue;
77+
} else if (ch === "e" || ch === "E") {
78+
if (prev === "_") {
79+
// next to e "1_e10"
80+
this.unput(1);
81+
break;
82+
}
83+
let undo = 2;
84+
ch = this.input();
85+
if (ch === "+" || ch === "-") {
86+
// 1e-5
87+
undo = 3;
88+
ch = this.input();
89+
}
90+
if (this.is_NUM_START()) {
91+
this.consume_LNUM();
92+
return this.tok.T_DNUMBER;
93+
}
94+
this.unput(ch ? undo : undo - 1); // keep only 1
95+
break;
96+
}
97+
98+
if (!this.is_NUM()) {
99+
// example : 10.0a
100+
if (ch) this.unput(1); // keep 10.0
101+
break;
69102
}
70103
}
104+
71105
if (hasPoint) {
72106
return this.tok.T_DNUMBER;
73107
} else if (this.yytext.length < MAX_LENGTH_OF_LONG - 1) {
@@ -110,7 +144,7 @@ module.exports = {
110144
let ch;
111145
while (this.offset < this.size) {
112146
ch = this.input();
113-
if (ch !== "0" && ch !== "1") {
147+
if (ch !== "0" && ch !== "1" && ch !== "_") {
114148
if (ch) this.unput(1);
115149
break;
116150
}

‎src/lexer/property.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ module.exports = {
5454
},
5555
matchST_VAR_OFFSET: function() {
5656
const ch = this.input();
57-
if (this.is_NUM()) {
57+
if (this.is_NUM_START()) {
5858
this.consume_NUM();
5959
return this.tok.T_NUM_STRING;
6060
} else if (ch === "]") {

‎src/lexer/scripting.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,13 +69,13 @@ module.exports = {
6969
default:
7070
if (ch === ".") {
7171
ch = this.input();
72-
if (this.is_NUM()) {
72+
if (this.is_NUM_START()) {
7373
return this.consume_NUM();
7474
} else {
7575
if (ch) this.unput(1);
7676
}
7777
}
78-
if (this.is_NUM()) {
78+
if (this.is_NUM_START()) {
7979
return this.consume_NUM();
8080
} else if (this.is_LABEL_START()) {
8181
return this.consume_LABEL().T_STRING();

‎src/lexer/utils.js

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,12 @@ const tokens = ";:,.\\[]()|^&+-/*=%!~$<>?@";
1010
module.exports = {
1111
// check if the char can be a numeric
1212
is_NUM: function() {
13+
const ch = this._input.charCodeAt(this.offset - 1);
14+
return (ch > 47 && ch < 58) || ch === 95;
15+
},
16+
17+
// check if the char can be a numeric
18+
is_NUM_START: function() {
1319
const ch = this._input.charCodeAt(this.offset - 1);
1420
return ch > 47 && ch < 58;
1521
},
@@ -29,9 +35,16 @@ module.exports = {
2935
// check if current char can be a label
3036
is_LABEL_START: function() {
3137
const ch = this._input.charCodeAt(this.offset - 1);
32-
return (
33-
(ch > 96 && ch < 123) || (ch > 64 && ch < 91) || ch === 95 || ch > 126
34-
);
38+
// A - Z
39+
if (ch > 64 && ch < 91) return true;
40+
// a - z
41+
if (ch > 96 && ch < 123) return true;
42+
// _ (95)
43+
if (ch === 95) return true;
44+
// utf8 / extended
45+
if (ch > 126) return true;
46+
// else
47+
return false;
3548
},
3649

3750
// reads each char of the label
@@ -75,8 +88,15 @@ module.exports = {
7588
// check if current char can be a hexadecimal number
7689
is_HEX: function() {
7790
const ch = this._input.charCodeAt(this.offset - 1);
78-
return (
79-
(ch > 47 && ch < 58) || (ch > 64 && ch < 71) || (ch > 96 && ch < 103)
80-
);
91+
// 0 - 9
92+
if (ch > 47 && ch < 58) return true;
93+
// A - F
94+
if (ch > 64 && ch < 71) return true;
95+
// a - f
96+
if (ch > 96 && ch < 103) return true;
97+
// _ (code 95)
98+
if (ch === 95) return true;
99+
// else
100+
return false;
81101
}
82102
};

‎test/debug.js

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,8 @@ const parser = require("../src/index");
1919
const ast = parser.parseCode(
2020
`
2121
<?php
22-
function iter() {
23-
yield 'ator' => $foo;
24-
yield from iter(50);
25-
}
26-
$a = fn($n) => $n * $factor;
22+
$a = 100_00;
23+
$bad = 100__00;
2724
`,
2825
{
2926
parser: {

0 commit comments

Comments
 (0)