Experiment: Use native php tests

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "test/php-src"]
+	path = php-src
+	url = https://github.com/php/php-src

bin/generate-php-src-tests.js

@@ -0,0 +1,190 @@
+#!/usr/bin/env node
+const fs = require("fs");
+const glob = require("glob");
+
+// Note: Ideally we'd check for "--EXPECTF--" in the phpt file, but that also
+// described run-time errors, rather than just parse errors.
+const testsThatExpectParseErrors = [
+  "Zend/tests/attributes/019_variable_attribute_name.phpt",
+  "Zend/tests/bug43343.phpt",
+  "Zend/tests/bug60099.phpt",
+  "Zend/tests/bug64660.phpt",
+  "Zend/tests/bug70430.phpt",
+  "Zend/tests/bug76439_2.phpt",
+  "Zend/tests/bug77993.phpt",
+  "Zend/tests/bug78363.phpt",
+  "Zend/tests/bug78454_1.phpt",
+  "Zend/tests/bug78454_2.phpt",
+  "Zend/tests/ctor_promotion_additional_modifiers.phpt",
+  "Zend/tests/flexible-heredoc-error1.phpt",
+  "Zend/tests/flexible-heredoc-error10.phpt",
+  "Zend/tests/flexible-heredoc-error11.phpt",
+  "Zend/tests/flexible-heredoc-error12.phpt",
+  "Zend/tests/flexible-heredoc-error13.phpt",
+  "Zend/tests/flexible-heredoc-error2.phpt",
+  "Zend/tests/flexible-heredoc-error3.phpt",
+  "Zend/tests/flexible-heredoc-error4.phpt",
+  "Zend/tests/flexible-heredoc-error5.phpt",
+  "Zend/tests/flexible-heredoc-error6.phpt",
+  "Zend/tests/flexible-heredoc-error7.phpt",
+  "Zend/tests/flexible-heredoc-error8.phpt",
+  "Zend/tests/flexible-heredoc-error9.phpt",
+  "Zend/tests/flexible-nowdoc-error1.phpt",
+  "Zend/tests/flexible-nowdoc-error2.phpt",
+  "Zend/tests/flexible-nowdoc-error3.phpt",
+  "Zend/tests/flexible-nowdoc-error4.phpt",
+  "Zend/tests/flexible-nowdoc-error5.phpt",
+  "Zend/tests/flexible-nowdoc-error6.phpt",
+  "Zend/tests/flexible-nowdoc-error7.phpt",
+  "Zend/tests/flexible-nowdoc-error8.phpt",
+  "Zend/tests/function_arguments/call_with_leading_comma_error.phpt",
+  "Zend/tests/function_arguments/call_with_multi_inner_comma_error.phpt",
+  "Zend/tests/function_arguments/call_with_multi_trailing_comma_error.phpt",
+  "Zend/tests/function_arguments/call_with_only_comma_error.phpt",
+  "Zend/tests/function_arguments_001.phpt",
+  "Zend/tests/function_arguments_002.phpt",
+  "Zend/tests/grammar/bug78441.phpt",
+  "Zend/tests/grammar/regression_004.phpt",
+  "Zend/tests/grammar/regression_005.phpt",
+  "Zend/tests/grammar/regression_010.phpt",
+  "Zend/tests/grammar/regression_012.phpt",
+  "Zend/tests/heredoc_005.phpt",
+  "Zend/tests/heredoc_013.phpt",
+  "Zend/tests/heredoc_014.phpt",
+  "Zend/tests/list_011.phpt",
+  "Zend/tests/lsb_008.phpt",
+  "Zend/tests/lsb_009.phpt",
+  "Zend/tests/namespace_name_namespace_start.phpt",
+  "Zend/tests/namespaced_name_whitespace.phpt",
+  "Zend/tests/ns_088.phpt",
+  "Zend/tests/ns_094.phpt",
+  "Zend/tests/ns_096.phpt",
+  "Zend/tests/ns_trailing_comma_error_01.phpt",
+  "Zend/tests/ns_trailing_comma_error_02.phpt",
+  "Zend/tests/ns_trailing_comma_error_03.phpt",
+  "Zend/tests/ns_trailing_comma_error_04.phpt",
+  "Zend/tests/ns_trailing_comma_error_05.phpt",
+  "Zend/tests/ns_trailing_comma_error_06.phpt",
+  "Zend/tests/ns_trailing_comma_error_07.phpt",
+  "Zend/tests/ns_trailing_comma_error_08.phpt",
+  "Zend/tests/numeric_literal_separator_002.phpt",
+  "Zend/tests/numeric_literal_separator_003.phpt",
+  "Zend/tests/numeric_literal_separator_004.phpt",
+  "Zend/tests/numeric_literal_separator_005.phpt",
+  "Zend/tests/numeric_literal_separator_006.phpt",
+  "Zend/tests/numeric_literal_separator_007.phpt",
+  "Zend/tests/numeric_literal_separator_008.phpt",
+  "Zend/tests/numeric_literal_separator_009.phpt",
+  "Zend/tests/oct_whitespace.phpt",
+  "Zend/tests/readonly_classes/readonly_enum.phpt",
+  "Zend/tests/readonly_classes/readonly_interface.phpt",
+  "Zend/tests/readonly_classes/readonly_trait.phpt",
+  "Zend/tests/short_echo_as_identifier.phpt",
+  "Zend/tests/traits/bug55524.phpt",
+  "Zend/tests/traits/bugs/interfaces.phpt",
+  "Zend/tests/type_declarations/intersection_types/invalid_types/invalid_nullable_type.phpt",
+  "Zend/tests/type_declarations/intersection_types/parse_error.phpt",
+  "Zend/tests/type_declarations/intersection_types/parse_error.phpt",
+  "Zend/tests/type_declarations/mixed/casting/mixed_cast_error.phpt",
+  "Zend/tests/type_declarations/static_type_param.phpt",
+  "Zend/tests/type_declarations/static_type_property.phpt",
+  "Zend/tests/type_declarations/typed_properties_025.phpt",
+  "Zend/tests/unterminated_comment.phpt",
+  "Zend/tests/varSyntax/globalNonSimpleVariableError.phpt",
+  "tests/classes/constants_error_006.phpt",
+  "tests/classes/constants_error_007.phpt",
+  "tests/lang/019.phpt",
+  "tests/lang/bug21669.phpt",
+  "tests/lang/bug21820.phpt",
+  "tests/lang/bug71897.phpt",
+  "tests/lang/invalid_octal.phpt",
+  "tests/lang/string/unicode_escape_empty.phpt",
+  "tests/lang/string/unicode_escape_incomplete.phpt",
+  "tests/lang/string/unicode_escape_large_codepoint.phpt",
+  "tests/lang/string/unicode_escape_sign.phpt",
+  "tests/lang/string/unicode_escape_sign2.phpt",
+  "tests/lang/string/unicode_escape_whitespace.phpt",
+
+  // These should also throw parse errors according to PHP 8.1, but we also
+  // support older versions.
+  // "Zend/tests/real_cast.phpt",
+
+  // These are not parse errors in PHP strictly speaking, but they test syntax
+  // that is still invalid (i.e. it generates a fatal error).
+  "Zend/tests/enum/case-in-class.phpt",
+  "Zend/tests/enum/no-name-property.phpt",
+  "Zend/tests/enum/no-properties.phpt",
+  "Zend/tests/errmsg_001.phpt",
+  "Zend/tests/readonly_props/readonly_const.phpt",
+  "Zend/tests/readonly_props/readonly_method.phpt",
+  "Zend/tests/readonly_props/readonly_method_trait.phpt",
+  "Zend/tests/static_in_trait_insteadof_list.phpt",
+  "Zend/tests/variadic/only_last_error.phpt",
+];
+
+const header = `// eslint-disable prettier/prettier
+const parser = require("../main");
+
+describe("php-src tests", function () {`;
+
+const footer = `});
+`;
+
+for (const absolutePath of glob.sync(
+  __dirname + "/../php-src/{tests,Zend,ext/standard}/**/*.phpt"
+)) {
+  const relativePath = absolutePath.substring(
+    absolutePath.indexOf("php-src") + "php-src/".length
+  );
+  const testName = [];
+  const testContents = [];
+  const expectError = testsThatExpectParseErrors.includes(relativePath);
+  let testFileName = relativePath.replace(/[/\\]/g, "-");
+  testFileName = testFileName.substring(0, testFileName.lastIndexOf("."));
+
+  let section = null;
+  for (const line of fs
+    .readFileSync(absolutePath)
+    .toString()
+    .split(/[\r\n]+/)) {
+    if (/^--\w/.test(line)) {
+      if (line === "--TEST--") {
+        section = "name";
+      } else if (line === "--FILE--") {
+        section = "file";
+      } else {
+        section = null;
+      }
+      continue;
+    }
+
+    if (section === "name") {
+      testName.push(line);
+    } else if (section === "file") {
+      testContents.push(line);
+    }
+    // TODO: Handle --FILE_EXTERNAL--
+  }
+
+  if (testContents.length === 0) {
+    continue;
+  }
+
+  const contents = `${header}
+  // ${relativePath}
+  it(${JSON.stringify(testName.join("\n"))}, function () {
+    expect(${expectError ? "() => " : ""}parser.parseCode(${JSON.stringify(
+    testContents.join("\n")
+  )})).${expectError ? "toThrowErrorMatchingSnapshot" : "toMatchSnapshot"}();
+  });
+${footer}`;
+
+  fs.writeFile(
+    `${__dirname}/../test/php-src/${testFileName}.test.js`,
+    contents,
+    (err) => {
+      if (err) throw err;
+      console.log(`Written ${testFileName}`);
+    }
+  );
+}

jest.config.js

@@ -25,6 +25,8 @@ module.exports = {
       testPathIgnorePatterns: [
         "<rootDir>/node_modules/",
         "<rootDir>/coverage/",
+        "<rootDir>/php-src/",
+        "<rootDir>/test/php-src/",
       ],
     },
   ],

package.json

@@ -1,6 +1,6 @@
 {
   "name": "php-parser",
-  "version": "3.1.0-beta.8",
+  "version": "3.1.0-beta.10",
   "description": "Parse PHP code from JS and returns its AST",
   "main": "src/index.js",
   "browser": "dist/php-parser.js",
@@ -62,23 +62,24 @@
   ],
   "license": "BSD-3-Clause",
   "devDependencies": {
-    "@babel/core": "^7.18.0",
-    "@babel/preset-env": "^7.18.0",
+    "@babel/core": "^7.18.2",
+    "@babel/preset-env": "^7.18.2",
     "babel-loader": "^8.0.5",
     "benchmark": "^2.1.4",
     "coveralls": "^3.0.3",
-    "eslint": "^8.16.0",
-    "eslint-plugin-jest": "^26.2.2",
+    "eslint": "^8.17.0",
+    "eslint-plugin-jest": "^26.5.3",
     "eslint-plugin-prettier": "^4.0.0",
+    "glob": "^8.0.3",
     "husky": "^8.0.1",
-    "jest": "^28.1.0",
+    "jest": "^28.1.1",
     "jest-runner-eslint": "^1.0.1",
     "jsdoc": "^3.6.10",
     "jsdoc-template": "^1.2.0",
     "lodash.template": ">=4.5.0",
     "prettier": "^2.6.2",
     "tsd-jsdoc": "^2.5.0",
-    "webpack": "^5.72.1",
+    "webpack": "^5.73.0",
     "webpack-cli": "^4.9.2",
     "yarpm": "^1.1.1"
   }

src/lexer.js

@@ -211,6 +211,23 @@ Lexer.prototype.input = function () {
   return ch;
 };
 
+/**
+ * peeks the next non-whitespace character, or null if we
+ * reach the end of the input
+ * @function Lexer#peekNonWhitespace
+ * @memberOf module:php-parser
+ */
+Lexer.prototype.peekNonWhitespace = function () {
+  for (let i = this.offset; i < this._input.length; i++) {
+    const ch = this._input[i];
+    if (ch !== " " && ch !== "\r" && ch !== "\n") {
+      return ch;
+    }
+  }
+
+  return null;
+};
+
 /**
  * revert eating specified size
  * @function Lexer#unput

src/lexer/comments.js

@@ -55,9 +55,11 @@ module.exports = {
       ch = this.input();
       if (ch === "*" && this._input[this.offset] === "/") {
         this.input();
-        break;
+        return token;
       }
     }
-    return token;
+
+    /* istanbul ignore next */
+    throw new Error(`Unterminated comment starting line ${this.yylineno}`);
   },
 };

src/lexer/numbers.js

@@ -35,7 +35,13 @@ module.exports = {
         } else {
           this.unput(ch ? 2 : 1);
         }
-        // @fixme check octal notation ? not usefull
+      } else if (ch === "o" || ch === "O") {
+        ch = this.input();
+        if (ch !== "_" && this.is_OCTAL()) {
+          return this.consume_ONUM();
+        } else {
+          this.unput(ch ? 2 : 1);
+        }
       } else if (!this.is_NUM()) {
         if (ch) this.unput(1);
       }
@@ -151,4 +157,15 @@ module.exports = {
     }
     return this.tok.T_LNUMBER;
   },
+  // read an octal number
+  consume_ONUM: function () {
+    while (this.offset < this.size) {
+      const ch = this.input();
+      if (!this.is_OCTAL()) {
+        if (ch) this.unput(1);
+        break;
+      }
+    }
+    return this.tok.T_LNUMBER;
+  },
 };

src/lexer/tokens.js

@@ -64,9 +64,8 @@ module.exports = {
             ? this.tok.T_NAME_RELATIVE
             : this.tok.T_NAME_QUALIFIED;
         do {
-          if (this._input[this.offset] === "{") {
+          if (this.peekNonWhitespace() === "{") {
             // e.g. when using group use statements, the last '\\' is followed by a '{'
-            this.input();
             break;
           }
 
@@ -125,9 +124,8 @@ module.exports = {
         if (this.is_LABEL_START()) {
           let ch;
           do {
-            if (this._input[this.offset] === "{") {
+            if (this.peekNonWhitespace() === "{") {
               // e.g. when using group use statements, the last '\\' is followed by a '{'
-              this.input();
               break;
             }
 

src/lexer/utils.js

@@ -99,4 +99,14 @@ module.exports = {
     // else
     return false;
   },
+  // check if current char can be an octal number
+  is_OCTAL: function () {
+    const ch = this._input.charCodeAt(this.offset - 1);
+    // 0 - 7
+    if (ch > 47 && ch < 56) return true;
+    // _ (code 95)
+    if (ch === 95) return true;
+    // else
+    return false;
+  },
 };

src/parser.js

@@ -184,6 +184,15 @@ const Parser = function (lexer, ast) {
       ].map(mapIt)
     ),
     EOS: new Map([";", this.EOF, this.tok.T_INLINE_HTML].map(mapIt)),
+    CLASS_NAME: new Map(
+      [
+        this.tok.T_STATIC,
+        this.tok.T_STRING,
+        this.tok.T_NAME_QUALIFIED,
+        this.tok.T_NAME_FULLY_QUALIFIED,
+        this.tok.T_NAME_RELATIVE,
+      ].map(mapIt)
+    ),
     EXPR: new Map(
       [
         "@",

src/parser/class.js

@@ -79,6 +79,10 @@ module.exports = {
         continue;
       }
 
+      if (this.token === this.tok.T_ATTRIBUTE) {
+        attrs = this.read_attr_list();
+      }
+
       // check enum cases
       if (allow_enum_cases && this.token === this.tok.T_CASE) {
         const enumcase = this.read_enum_case();
@@ -89,10 +93,6 @@ module.exports = {
         continue;
       }
 
-      if (this.token === this.tok.T_ATTRIBUTE) {
-        attrs = this.read_attr_list();
-      }
-
       const locStart = this.position();
 
       // read member flags