Skip to content

Commit 14d643d

Browse files
ichiriacichiriac
committed
glayzzle#138 - implement the extractTokens options
1 parent 82233e9 commit 14d643d

File tree

7 files changed

+119
-47
lines changed

7 files changed

+119
-47
lines changed

‎dist/php-parser.js

Lines changed: 36 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
/*! php-parser - BSD3 License - 2018-03-23 */
1+
/*! php-parser - BSD3 License - 2018-03-25 */
22

33
require=(function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);var f=new Error("Cannot find module '"+o+"'");throw f.code="MODULE_NOT_FOUND",f}var l=n[o]={exports:{}};t[o][0].call(l.exports,function(e){var n=t[o][1][e];return s(n?n:e)},l,l.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o<r.length;o++)s(r[o]);return s})({1:[function(require,module,exports){
44
/*!
@@ -2319,13 +2319,17 @@ var KIND = "program";
23192319
* @extends {Block}
23202320
* @property {Error[]} errors
23212321
* @property {Doc[]?} comments
2322+
* @property {String[]?} tokens
23222323
*/
2323-
var Program = Block.extends(function Program(children, errors, comments, docs, location) {
2324+
var Program = Block.extends(function Program(children, errors, comments, docs, tokens, location) {
23242325
Block.apply(this, [KIND, children, docs, location]);
23252326
this.errors = errors;
23262327
if (comments) {
23272328
this.comments = comments;
23282329
}
2330+
if (tokens) {
2331+
this.tokens = tokens;
2332+
}
23292333
});
23302334

23312335
module.exports = Program;
@@ -4680,6 +4684,7 @@ function isNumber(n) {
46804684
* @property {AST} ast - the AST factory instance
46814685
* @property {Integer|String} token - current token
46824686
* @property {Boolean} extractDoc - should extract documentation as AST node
4687+
* @property {Boolean} extractTokens - should extract each token
46834688
* @property {Boolean} suppressErrors - should ignore parsing errors and continue
46844689
* @property {Boolean} debug - should output debug informations
46854690
*/
@@ -4693,6 +4698,7 @@ var parser = function parser(lexer, ast) {
46934698
this.debug = false;
46944699
this.php7 = true;
46954700
this.extractDoc = false;
4701+
this.extractTokens = false;
46964702
this.suppressErrors = false;
46974703
var mapIt = function mapIt(item) {
46984704
return [item, null];
@@ -4737,6 +4743,11 @@ parser.prototype.parse = function (code, filename) {
47374743
} else {
47384744
this._docs = null;
47394745
}
4746+
if (this.extractTokens) {
4747+
this._tokens = [];
4748+
} else {
4749+
this._tokens = null;
4750+
}
47404751
this._docIndex = 0;
47414752
this.lexer.setInput(code);
47424753
this.lexer.comment_tokens = this.extractDoc;
@@ -4755,7 +4766,7 @@ parser.prototype.parse = function (code, filename) {
47554766
}
47564767
}
47574768
}
4758-
return program(childs, this._errors, this._docs);
4769+
return program(childs, this._errors, this._docs, this._tokens);
47594770
};
47604771

47614772
/**
@@ -4890,11 +4901,8 @@ parser.prototype.text = function () {
48904901

48914902
/** consume the next token **/
48924903
parser.prototype.next = function () {
4893-
// prepare the back command
4894-
this.prev = [this.lexer.yylloc.first_line, this.lexer.yylloc.first_column, this.lexer.offset];
4895-
48964904
// eating the token
4897-
this.token = this.lexer.lex() || this.EOF;
4905+
this.lex();
48984906

48994907
// showing the debug
49004908
if (this.debug) {
@@ -4916,6 +4924,27 @@ parser.prototype.next = function () {
49164924
return this;
49174925
};
49184926

4927+
/**
4928+
* Eating a token
4929+
*/
4930+
parser.prototype.lex = function () {
4931+
// prepare the back command
4932+
this.prev = [this.lexer.yylloc.first_line, this.lexer.yylloc.first_column, this.lexer.offset];
4933+
4934+
// the token
4935+
this.token = this.lexer.lex() || this.EOF;
4936+
4937+
// append on token stack
4938+
if (this.extractTokens && this.token !== this.EOF) {
4939+
var entry = this.lexer.yytext;
4940+
if (this.lexer.engine.tokens.values.hasOwnProperty(this.token)) {
4941+
entry = [this.lexer.engine.tokens.values[this.token], entry, this.lexer.yylloc.first_line];
4942+
}
4943+
this._tokens.push(entry);
4944+
}
4945+
return this;
4946+
};
4947+
49194948
/**
49204949
* Check if token is of specified type
49214950
*/

‎dist/php-parser.min.js

Lines changed: 30 additions & 30 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎dist/php-parser.min.js.map

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎docs/AST.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -480,6 +480,10 @@ Defines a do/while statement
480480

481481
Defines system based call
482482

483+
**Properties**
484+
485+
- `shortForm` **[boolean](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Boolean)**
486+
483487
## Empty
484488

485489
**Extends Sys**

‎docs/parser.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ Type: Parser
1717
- `ast` **AST** the AST factory instance
1818
- `token` **(Integer | [String](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String))** current token
1919
- `extractDoc` **[Boolean](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** should extract documentation as AST node
20+
- `extractTokens` **[Boolean](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** should extract each token
2021
- `suppressErrors` **[Boolean](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** should ignore parsing errors and continue
2122
- `debug` **[Boolean](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** should output debug informations
2223

@@ -99,6 +100,10 @@ Returns **[String](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Refer
99100

100101
consume the next token \*
101102

103+
### lex
104+
105+
Eating a token
106+
102107
### is
103108

104109
Check if token is of specified type

‎src/ast/program.js

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,19 +13,24 @@ const KIND = "program";
1313
* @extends {Block}
1414
* @property {Error[]} errors
1515
* @property {Doc[]?} comments
16+
* @property {String[]?} tokens
1617
*/
1718
const Program = Block.extends(function Program(
1819
children,
1920
errors,
2021
comments,
2122
docs,
23+
tokens,
2224
location
2325
) {
2426
Block.apply(this, [KIND, children, docs, location]);
2527
this.errors = errors;
2628
if (comments) {
2729
this.comments = comments;
2830
}
31+
if (tokens) {
32+
this.tokens = tokens;
33+
}
2934
});
3035

3136
module.exports = Program;

‎src/parser.js

Lines changed: 38 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ function isNumber(n) {
1818
* @property {AST} ast - the AST factory instance
1919
* @property {Integer|String} token - current token
2020
* @property {Boolean} extractDoc - should extract documentation as AST node
21+
* @property {Boolean} extractTokens - should extract each token
2122
* @property {Boolean} suppressErrors - should ignore parsing errors and continue
2223
* @property {Boolean} debug - should output debug informations
2324
*/
@@ -31,6 +32,7 @@ const parser = function(lexer, ast) {
3132
this.debug = false;
3233
this.php7 = true;
3334
this.extractDoc = false;
35+
this.extractTokens = false;
3436
this.suppressErrors = false;
3537
const mapIt = function(item) {
3638
return [item, null];
@@ -252,6 +254,11 @@ parser.prototype.parse = function(code, filename) {
252254
} else {
253255
this._docs = null;
254256
}
257+
if (this.extractTokens) {
258+
this._tokens = [];
259+
} else {
260+
this._tokens = null;
261+
}
255262
this._docIndex = 0;
256263
this.lexer.setInput(code);
257264
this.lexer.comment_tokens = this.extractDoc;
@@ -270,7 +277,7 @@ parser.prototype.parse = function(code, filename) {
270277
}
271278
}
272279
}
273-
return program(childs, this._errors, this._docs);
280+
return program(childs, this._errors, this._docs, this._tokens);
274281
};
275282

276283
/**
@@ -424,15 +431,8 @@ parser.prototype.text = function() {
424431

425432
/** consume the next token **/
426433
parser.prototype.next = function() {
427-
// prepare the back command
428-
this.prev = [
429-
this.lexer.yylloc.first_line,
430-
this.lexer.yylloc.first_column,
431-
this.lexer.offset
432-
];
433-
434434
// eating the token
435-
this.token = this.lexer.lex() || this.EOF;
435+
this.lex();
436436

437437
// showing the debug
438438
if (this.debug) {
@@ -457,6 +457,35 @@ parser.prototype.next = function() {
457457
return this;
458458
};
459459

460+
/**
461+
* Eating a token
462+
*/
463+
parser.prototype.lex = function() {
464+
// prepare the back command
465+
this.prev = [
466+
this.lexer.yylloc.first_line,
467+
this.lexer.yylloc.first_column,
468+
this.lexer.offset
469+
];
470+
471+
// the token
472+
this.token = this.lexer.lex() || this.EOF;
473+
474+
// append on token stack
475+
if (this.extractTokens && this.token !== this.EOF) {
476+
let entry = this.lexer.yytext;
477+
if (this.lexer.engine.tokens.values.hasOwnProperty(this.token)) {
478+
entry = [
479+
this.lexer.engine.tokens.values[this.token],
480+
entry,
481+
this.lexer.yylloc.first_line
482+
];
483+
}
484+
this._tokens.push(entry);
485+
}
486+
return this;
487+
};
488+
460489
/**
461490
* Check if token is of specified type
462491
*/

0 commit comments

Comments
 (0)