diff --git a/.github/no-response.yml b/.github/no-response.yml new file mode 100644 index 00000000..1c8799d1 --- /dev/null +++ b/.github/no-response.yml @@ -0,0 +1,15 @@ +# Configuration for probot-no-response - https://github.com/probot/no-response + +# Number of days of inactivity before an issue is closed for lack of response +daysUntilClose: 28 + +# Label requiring a response +responseRequiredLabel: more-information-needed + +# Comment to post when closing an issue for lack of response. Set to `false` to disable. +closeComment: > + This issue has been automatically closed because there has been no response + to our request for more information from the original author. With only the + information that is currently in the issue, we don't have enough information + to take action. Please reach out if you have or find the answers we need so + that we can investigate further. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..432a2dce --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,45 @@ +name: ci +on: + - pull_request + - push + +jobs: + Test: + if: "!contains(github.event.head_commit.message, '[skip ci]')" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - macos-latest + - windows-latest + atom_channel: + - stable + - nightly + steps: + - uses: actions/checkout@v2 + - name: Cache + uses: actions/cache@v2 + with: + path: | + 'node_modules' + 'C:/Program Files (x86)/MSBuild/Microsoft.Cpp/v4.0/v140' + key: ${{ runner.os }}-${{ matrix.atom_channel }}-${{ hashFiles('package.json') }} + + - uses: UziTech/action-setup-atom@v1 + with: + channel: ${{ matrix.atom_channel }} + + - name: Install dependencies + run: apm install + + - name: Run tests + run: apm test + + Skip: + if: contains(github.event.head_commit.message, '[skip ci]') + runs-on: ubuntu-latest + steps: + - name: Skip CI 🚫 + run: echo skip ci diff --git a/.gitignore b/.gitignore index 3c3629e6..d5f19d89 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ node_modules +package-lock.json diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index a6b772f1..00000000 --- a/.travis.yml +++ /dev/null @@ -1,12 +0,0 @@ -language: objective-c - -notifications: - email: - on_success: never - on_failure: change - -script: 'curl -s https://raw.githubusercontent.com/atom/ci/master/build-package.sh | sh' - -branches: - only: - - master diff --git a/README.md b/README.md index 42d9ef46..d80d4475 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ -# JavaScript language support in Atom -[![macOS Build Status](https://travis-ci.org/atom/language-javascript.svg?branch=master)](https://travis-ci.org/atom/language-javascript) -[![Windows Build Status](https://ci.appveyor.com/api/projects/status/ktooccwna96ssiyr/branch/master?svg=true)](https://ci.appveyor.com/project/Atom/language-javascript-dijf8/branch/master) +##### Atom and all repositories under Atom will be archived on December 15, 2022. Learn more in our [official announcement](https://github.blog/2022-06-08-sunsetting-atom/) + # JavaScript language support in Atom + +![ci](https://github.com/atom/language-javascript/workflows/ci/badge.svg) [![Dependency Status](https://david-dm.org/atom/language-javascript.svg)](https://david-dm.org/atom/language-javascript) Adds syntax highlighting and snippets for JavaScript in Atom. diff --git a/appveyor.yml b/appveyor.yml index 2b0fde43..795da41c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,27 +1,6 @@ -version: "{build}" - -platform: x64 +# empty appveyor +build: off branches: - only: - - master - -clone_depth: 10 - -skip_tags: true - -environment: - APM_TEST_PACKAGES: - - matrix: - - ATOM_CHANNEL: stable - - ATOM_CHANNEL: beta - -install: - - ps: Install-Product node 4 - -build_script: - - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/atom/ci/master/build-package.ps1')) - -test: off -deploy: off + only: + - non-existing diff --git a/grammars/javascript.cson b/grammars/javascript.cson index 085b8e57..62011cef 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -2,18 +2,23 @@ 'fileTypes': [ 'js' '_js' + 'cjs' 'es' 'es6' 'gs' 'htc' 'jscad' + 'jscript' + 'jse' 'jslib' 'jsm' 'json5' 'jspre' + 'mjs' 'pac' 'pjs' 'sjs' + 'snap' 'xsjs' 'xsjslib' ] @@ -172,53 +177,6 @@ { 'include': '#numbers' } - { - # { member1 , member2 as alias2 , [...] } inside re-export - 'begin': '{(?=.*\\bfrom\\b)' - 'beginCaptures': - 0: - 'name': 'punctuation.definition.modules.begin.js' - 'end': '}' - 'endCaptures': - 0: - 'name': 'punctuation.definition.modules.end.js' - 'patterns': [ - { - # (default|name) as alias - 'match': '''(?x) - (?: \\b(default)\\b | \\b([a-zA-Z_$][\\w$]*)\\b) - \\s* - (\\b as \\b) - \\s* - (?: \\b(default)\\b | (\\*) | \\b([a-zA-Z_$][\\w$]*)\\b) - ''' - 'captures': - '1': - 'name': 'variable.language.default.js' - '2': - 'name': 'variable.other.module.js' - '3': - 'name': 'keyword.control.js' - '4': - 'name': 'variable.language.default.js' - '5': - 'name': 'invalid.illegal.js' - '6': - 'name': 'variable.other.module-alias.js' - } - { - 'match': ',' - 'name': 'meta.delimiter.object.comma.js' - } - { - 'include': '#comments' - } - { - 'match': '\\b([a-zA-Z_$][\\w$]*)\\b' - 'name': 'variable.other.module.js' - } - ] - } { # { member1 , member2 as alias2 , [...] } 'begin': '(?![a-zA-Z_$0-9]){' @@ -234,7 +192,7 @@ # name as (default|alias) 'captures': '1': - 'name': 'invalid.illegal.js' + 'name': 'variable.language.default.js' '2': 'name': 'variable.other.module.js' '3': @@ -466,7 +424,7 @@ (?= (?! (break|case|catch|continue|do|else|finally|for|function|if| - package|return|switch|throw|try|while|with) + return|switch|throw|try|while|with) [\\s\\(] ) ( @@ -860,6 +818,39 @@ } ] } + { + # Promise + 'begin': '(?|&&|\\|\\|)\\s*(/)(?![/*+?])(?=.*/)' + 'begin': '(?<=[\\[{=(?:+*,!~-]|^|return|=>|&&|\\|\\|)\\s*(/)(?![/*+?])(?=.*/)' 'beginCaptures': '1': 'name': 'punctuation.definition.string.begin.js' - 'end': '(/)[gimuy]*' + 'end': '(/)([gimsuy]*)' 'endCaptures': '1': 'name': 'punctuation.definition.string.end.js' + '2': + 'name': 'meta.flag.regexp' 'name': 'string.regexp.js' 'patterns': [ { @@ -1248,45 +1245,70 @@ 'numbers': 'patterns': [ { - 'match': '\\b(? "(?:(?:\\*(?!/))|(?:\\\\(?!"))|[^*\\\\])*?" | # [foo="bar"] Double-quoted '(?:(?:\\*(?!/))|(?:\\\\(?!'))|[^*\\\\])*?' | # [foo='bar'] Single-quoted \\[ (?:(?:\\*(?!/))|[^*])*? \\] | # [foo=[1,2]] Array literal - (?:(?:\\*(?!/))|[^*])*? # Everything else + (?:(?:\\*(?!/))|\\s(?!\\s*\\])|\\[.*?(?:\\]|(?=\\*/))|[^*\\s\\[\\]])* # Everything else (sorry) )* ) )? @@ -311,7 +311,15 @@ { # Tags followed by a type expression # - @ {type} - 'begin': '((@)(?:define|enum|exception|implements|modifies|namespace|private|protected|returns?|suppress|throws|type))\\s+(?={)' + 'begin': '''(?x) + ( + (@) + (?:define|enum|exception|export|extends|lends|implements|modifies + |namespace|private|protected|returns?|suppress|this|throws|type + |yields?) + ) + \\s+(?={) + ''' 'beginCaptures': '1': 'name': 'storage.type.class.jsdoc' @@ -399,12 +407,13 @@ |callback|chainable|class|classdesc|code|config|const|constant|constructor|constructs|copyright |default|defaultvalue|define|deprecated|desc|description|dict|emits|enum|event|example|exception |exports?|extends|extension(?:_?for)?|external|externs|file|fileoverview|final|fires|for|func - |function|global|host|ignore|implements|implicitCast|inherit[Dd]oc|inner|instance|interface - |internal|kind|lends|license|listens|main|member|memberof!?|method|mixes|mixins?|modifies|module - |name|namespace|noalias|nocollapse|nocompile|nosideeffects|override|overview|package|param|preserve - |private|prop|property|protected|public|read[Oo]nly|record|require[ds]|returns?|see|since|static - |struct|submodule|summary|suppress|template|this|throws|todo|tutorial|type|typedef|unrestricted - |uses|var|variation|version|virtual|writeOnce) + |function|generator|global|hideconstructor|host|ignore|implements|implicitCast|inherit[Dd]oc + |inner|instance|interface|internal|kind|lends|license|listens|main|member|memberof!?|method + |mixes|mixins?|modifies|module|name|namespace|noalias|nocollapse|nocompile|nosideeffects + |override|overview|package|param|polymer(?:Behavior)?|preserve|private|prop|property|protected + |public|read[Oo]nly|record|require[ds]|returns?|see|since|static|struct|submodule|summary + |suppress|template|this|throws|todo|tutorial|type|typedef|unrestricted|uses|var|variation + |version|virtual|writeOnce|yields?) \\b ''' 'name': 'storage.type.class.jsdoc' diff --git a/grammars/regular expression replacement (javascript).cson b/grammars/regular expression replacement (javascript).cson index 596b8279..eeaa8d70 100644 --- a/grammars/regular expression replacement (javascript).cson +++ b/grammars/regular expression replacement (javascript).cson @@ -1,6 +1,4 @@ 'scopeName': 'source.js.regexp.replacement' -'name': 'Regular Expression Replacement (JavaScript)' -'fileTypes': [] 'patterns': [ { 'include': '#regexp-replacement' diff --git a/grammars/regular expressions (javascript).cson b/grammars/regular expressions (javascript).cson index 92368e8f..1de6f54b 100644 --- a/grammars/regular expressions (javascript).cson +++ b/grammars/regular expressions (javascript).cson @@ -1,6 +1,4 @@ 'scopeName': 'source.js.regexp' -'name': 'Regular Expressions (JavaScript)' -'fileTypes': [] 'patterns': [ { 'include': '#regexp' @@ -33,7 +31,7 @@ 'name': 'keyword.control.anchor.regexp' } { - 'match': '\\\\[1-9]\\d*' + 'match': '\\\\[1-9]\\d*|\\\\k<[a-zA-Z_$][\\w$]*>' 'name': 'keyword.other.back-reference.regexp' } { @@ -45,14 +43,18 @@ 'name': 'keyword.operator.or.regexp' } { - 'begin': '(\\()((\\?=)|(\\?!))' + 'begin': '(\\()(?:(\\?=)|(\\?!)|(\\?<=)|(\\?))?' 'beginCaptures': '0': 'name': 'punctuation.definition.group.regexp' diff --git a/grammars/tree-sitter-javascript.cson b/grammars/tree-sitter-javascript.cson new file mode 100644 index 00000000..86919162 --- /dev/null +++ b/grammars/tree-sitter-javascript.cson @@ -0,0 +1,262 @@ +name: 'JavaScript' +scopeName: 'source.js' +type: 'tree-sitter' +parser: 'tree-sitter-javascript' + +fileTypes: ['js', 'jsx'] + +injectionRegex: '^js$|^JS$|javascript|JavaScript' + +firstLineRegex: [ + # shebang line + '^#!.*\\b(node)\\r?\\n' + + # vim modeline + 'vim\\b.*\\bset\\b.*\\b(filetype|ft|syntax)=(js|javascript)' +] + +folds: [ + { + type: 'comment' + } + { + type: ['jsx_element', 'template_string'], + start: {index: 0} + end: {index: -1} + } + { + type: 'jsx_self_closing_element' + start: {index: 1} + end: {index: -2} + } + { + start: {index: 0, type: '{'} + end: {index: -1, type: '}'} + } + { + start: {index: 0, type: '['} + end: {index: -1, type: ']'} + } + { + start: {index: 0, type: '('} + end: {index: -1, type: ')'} + } + { + type: ['switch_case', 'switch_default'] + start: {index: 0} + end: {type: 'break_statement', index: -1} + } + { + type: ['switch_case', 'switch_default'] + start: {index: 0} + } +] + +comments: + start: '// ' + +scopes: + 'program': 'source.js' + + 'property_identifier': [ + { + match: '^[\$A-Z_]+$', + scopes: 'constant.other.property.js' + } + + 'variable.other.object.property' + ] + + 'member_expression > property_identifier': 'variable.other.object.property.unquoted' + + 'formal_parameters > identifier': 'variable.parameter.function' + 'formal_parameters > rest_parameter > identifier': 'variable.parameter.rest.function' + + 'shorthand_property_identifier': [ + { + match: '^[\$A-Z_]{2,}$', + scopes: 'constant.other' + } + ] + + ' + class > identifier, + new_expression > identifier + ': 'meta.class' + + ' + jsx_opening_element > identifier, + jsx_closing_element > identifier, + jsx_self_closing_element > identifier + ': [ + { + match: '^[A-Z]', + scopes: 'meta.class.component.jsx' + } + ] + + 'call_expression > identifier': {match: '^[A-Z]', scopes: 'meta.class'} + 'arrow_function > identifier:nth-child(0)': 'variable.parameter.function' + + 'function > identifier': 'entity.name.function' + 'function_declaration > identifier': 'entity.name.function' + 'generator_function > identifier': 'entity.name.function' + + 'call_expression > identifier': [ + {match: '^require$', scopes: 'support.function'}, + 'entity.name.function' + ] + + 'call_expression > super': 'support.function.super' + + 'method_definition > property_identifier': 'entity.name.function' + 'call_expression > member_expression > property_identifier': 'entity.name.function' + + 'identifier': [ + { + match: '^(global|globalThis|module|exports|__filename|__dirname)$', + scopes: 'support.variable' + }, + { + match: '^(window|self|frames|event|document|performance|screen|navigator|console)$' + scopes: 'support.variable.dom' + }, + { + exact: 'require', + scopes: 'support.function' + }, + { + match: '^[\$A-Z_]{2,}$', + scopes: 'constant.other' + }, + { + match: '^[A-Z]', + scopes: 'meta.class' + }, + ] + + 'number': 'constant.numeric' + 'string': 'string.quoted' + 'regex': 'string.regexp' + 'escape_sequence': 'constant.character.escape' + 'template_string': 'string.quoted.template' + 'undefined': 'constant.language' + 'null': 'constant.language.null' + 'true': 'constant.language.boolean.true' + 'false': 'constant.language.boolean.false' + 'comment': [ + { + match: "^//", + scopes: 'comment.line' + }, + 'comment.block' + ] + 'hash_bang_line': 'comment.block' + + ' + jsx_expression > "{", + jsx_expression > "}", + template_substitution > "${", + template_substitution > "}" + ': 'punctuation.section.embedded' + 'template_substitution': 'embedded.source' + + '"("': 'punctuation.definition.parameters.begin.bracket.round' + '")"': 'punctuation.definition.parameters.end.bracket.round' + '"{"': 'punctuation.definition.function.body.begin.bracket.curly' + '"}"': 'punctuation.definition.function.body.end.bracket.curly' + '";"': 'punctuation.terminator.statement.semicolon' + '"["': 'punctuation.definition.array.begin.bracket.square' + '"]"': 'punctuation.definition.array.end.bracket.square' + + '"var"': 'storage.type' + '"let"': 'storage.type' + '"class"': 'storage.type' + '"extends"': 'storage.modifier' + '"const"': 'storage.modifier' + '"static"': 'storage.modifier' + '"function"': 'storage.type.function' + '"=>"': 'storage.type.function.arrow' + + '"="': 'keyword.operator.js' + '"+="': 'keyword.operator.js' + '"-="': 'keyword.operator.js' + '"*="': 'keyword.operator.js' + '"/="': 'keyword.operator.js' + '"%="': 'keyword.operator.js' + '"<<="': 'keyword.operator.js' + '">>="': 'keyword.operator.js' + '">>>="': 'keyword.operator.js' + '"&="': 'keyword.operator.js' + '"^="': 'keyword.operator.js' + '"|="': 'keyword.operator.js' + '"!"': 'keyword.operator.js' + '"+"': 'keyword.operator.js' + '"-"': 'keyword.operator.js' + '"*"': 'keyword.operator.js' + '"/"': 'keyword.operator.js' + '"%"': 'keyword.operator.js' + '"=="': 'keyword.operator.js' + '"==="': 'keyword.operator.js' + '"!="': 'keyword.operator.js' + '"!=="': 'keyword.operator.js' + '">="': 'keyword.operator.js' + '"<="': 'keyword.operator.js' + '">"': 'keyword.operator.js' + '"<"': 'keyword.operator.js' + '":"': 'keyword.operator.js' + '"?"': 'keyword.operator.js' + '"&&"': 'keyword.operator.js' + '"||"': 'keyword.operator.js' + '"&"': 'keyword.operator.js' + '"~"': 'keyword.operator.js' + '"^"': 'keyword.operator.js' + '">>"': 'keyword.operator.js' + '">>>"': 'keyword.operator.js' + '"<<"': 'keyword.operator.js' + '"|"': 'keyword.operator.js' + '"++"': 'keyword.operator.js' + '"--"': 'keyword.operator.js' + '"..."': 'keyword.operator.spread.js' + + '"in"': 'keyword.operator.in' + '"instanceof"': 'keyword.operator.instanceof' + '"of"': 'keyword.operator.of' + '"new"': 'keyword.operator.new' + '"typeof"': 'keyword.operator.typeof' + + '"get"': 'keyword.operator.setter' + '"set"': 'keyword.operator.setter' + + '"."': 'meta.delimiter.period' + '","': 'meta.delimiter.comma' + + '"as"': 'keyword.control' + '"if"': 'keyword.control' + '"do"': 'keyword.control' + '"else"': 'keyword.control' + '"while"': 'keyword.control' + '"for"': 'keyword.control' + '"return"': 'keyword.control' + '"break"': 'keyword.control' + '"continue"': 'keyword.control' + '"throw"': 'keyword.control' + '"try"': 'keyword.control' + '"catch"': 'keyword.control' + '"finally"': 'keyword.control' + '"switch"': 'keyword.control' + '"case"': 'keyword.control' + '"default"': 'keyword.control' + '"export"': 'keyword.control' + '"import"': 'keyword.control' + '"from"': 'keyword.control' + '"yield"': 'keyword.control' + '"async"': 'keyword.control' + '"await"': 'keyword.control' + '"debugger"': 'keyword.control' + '"delete"': 'keyword.control' + + 'jsx_attribute > property_identifier': 'entity.other.attribute-name' + 'jsx_opening_element > identifier': 'entity.name.tag' + 'jsx_closing_element > identifier': 'entity.name.tag' + 'jsx_self_closing_element > identifier': 'entity.name.tag' diff --git a/grammars/tree-sitter-jsdoc.cson b/grammars/tree-sitter-jsdoc.cson new file mode 100644 index 00000000..a9f8a679 --- /dev/null +++ b/grammars/tree-sitter-jsdoc.cson @@ -0,0 +1,16 @@ +name: 'JSDoc' +scopeName: 'source.jsdoc' +type: 'tree-sitter' +parser: 'tree-sitter-jsdoc' + +injectionRegex: '^jsdoc$' + +scopes: + 'tag_name': 'keyword.control' + 'identifier': 'variable.other.jsdoc' + 'type': 'support.type' + 'path_expression > identifier': 'string' + '"."': 'meta.delimiter.period' + '":"': 'meta.delimiter.colon' + '"/"': 'meta.delimiter.slash' + '"#", "~"': 'meta.delimiter' diff --git a/grammars/tree-sitter-regex.cson b/grammars/tree-sitter-regex.cson new file mode 100644 index 00000000..812bd378 --- /dev/null +++ b/grammars/tree-sitter-regex.cson @@ -0,0 +1,17 @@ +name: 'Javascript RegExp' +scopeName: 'source.js.regexp' +type: 'tree-sitter' +parser: 'tree-sitter-regex' + +injectionRegex: 'regex' + +scopes: + 'pattern': 'string.quoted', + 'group_name': 'variable.other.object.property' + 'identity_escape, control_letter_escape, control_escape, class_escape': 'constant.character.character-class.regexp' + 'start_assertion, end_assertion, boundary_assertion, non_boundary_assertion': 'constant.character.character-class.regexp' + 'count_quantifier, one_or_more, zero_or_more, optional': 'storage.modifier' + 'character_class': 'string.regexp' + '"(", "(?", "(?:", "(?<"': 'punctuation.definition.parameters.begin.bracket.round' + '">", ")"': 'punctuation.definition.parameters.end.bracket.round' + '"=", "<=", "!", "~!#@%^&*|+=[]{}`?-…' 'commentStart': '// ' 'foldEndPattern': '^\\s*\\}|^\\s*\\]|^\\s*\\)' 'increaseIndentPattern': '(?x) - \\{ [^}"\']* $ - | \\[ [^\\]"\']* $ - | \\( [^)"\']* $ + \\{ [^}"\']*(//.*)? $ + | \\[ [^\\]"\']*(//.*)? $ + | \\( [^)"\']*(//.*)? $ ' 'decreaseIndentPattern': '(?x) ^ \\s* (\\s* /[*] .* [*]/ \\s*)* [}\\])] diff --git a/snippets/language-javascript.cson b/snippets/language-javascript.cson index c8064854..24fdbb61 100644 --- a/snippets/language-javascript.cson +++ b/snippets/language-javascript.cson @@ -1,4 +1,4 @@ -'.source.js': +'.source.js, .source.flow': 'Object Method': 'prefix': 'kf' 'body': '${1:methodName}: function (${2:attribute}) {\n\t$3\n}${4:,}' @@ -34,7 +34,10 @@ 'body': 'for (var ${1:variable} in ${2:object}) {\n\t${3:if (${2:object}.hasOwnProperty(${1:variable})) {\n\t\t$4\n\t\\}}\n}' 'for of': 'prefix': 'forof' - 'body': 'for (${1:variable} of ${2:iterable}) {\n\t$3\n}' + 'body': 'for (var ${1:variable} of ${2:iterable}) {\n\t$3\n}' + 'forEach': + 'prefix' : 'foreach' + 'body' : 'forEach((${1:item}, ${2:i}) => {\n\t$3\n});\n' 'Function': 'prefix': 'fun' 'body': 'function ${1:functionName}($2) {\n\t$0\n}' diff --git a/spec/javascript-spec.coffee b/spec/javascript-spec.coffee index 7b938fc3..192850ec 100644 --- a/spec/javascript-spec.coffee +++ b/spec/javascript-spec.coffee @@ -12,6 +12,8 @@ describe "JavaScript grammar", -> grammar = null beforeEach -> + atom.config.set('core.useTreeSitterParsers', false) + waitsForPromise -> atom.packages.activatePackage("language-javascript") @@ -49,7 +51,6 @@ describe "JavaScript grammar", -> line3 """ + delim expect(lines[0][0]).toEqual value: delim, scopes: ['source.js', scope, 'punctuation.definition.string.begin.js'] - expect(lines[0][1]).toEqual value: 'line1', scopes: ['source.js', scope, 'invalid.illegal.string.js'] expect(lines[1][0]).toEqual value: 'line2\\', scopes: ['source.js', scope] expect(lines[2][0]).toEqual value: 'line3', scopes: ['source.js', scope] expect(lines[2][1]).toEqual value: delim, scopes: ['source.js', scope, 'punctuation.definition.string.end.js'] @@ -120,6 +121,14 @@ describe "JavaScript grammar", -> expect(tokens[0]).toEqual value: "debugger", scopes: ['source.js', 'keyword.other.debugger.js'] expect(tokens[1]).toEqual value: ";", scopes: ['source.js', 'punctuation.terminator.statement.js'] + it "tokenises an `await` keyword after a spread operator", -> + {tokens} = grammar.tokenizeLine("...await stuff()") + expect(tokens[0]).toEqual value: '...', scopes: ['source.js', 'keyword.operator.spread.js'] + expect(tokens[1]).toEqual value: 'await', scopes: ['source.js', 'keyword.control.js'] + expect(tokens[3]).toEqual value: 'stuff', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(tokens[4]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[5]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + describe "built-in globals", -> it "tokenizes built-in classes", -> {tokens} = grammar.tokenizeLine('window') @@ -178,6 +187,74 @@ describe "JavaScript grammar", -> expect(tokens[1]).toEqual value: 'test', scopes: ['source.js', 'string.regexp.js'] expect(tokens[2]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + {tokens} = grammar.tokenizeLine('/random/g') + expect(tokens[0]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: 'random', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[2]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + expect(tokens[3]).toEqual value: 'g', scopes: ['source.js', 'string.regexp.js', 'meta.flag.regexp'] + + {tokens} = grammar.tokenizeLine('/rock(et)?/is') + expect(tokens[0]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: 'rock', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[2]).toEqual value: '(', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[3]).toEqual value: 'et', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp'] + expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[5]).toEqual value: '?', scopes: ['source.js', 'string.regexp.js', 'keyword.operator.quantifier.regexp'] + expect(tokens[6]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + expect(tokens[7]).toEqual value: 'is', scopes: ['source.js', 'string.regexp.js', 'meta.flag.regexp'] + + {tokens} = grammar.tokenizeLine('/(foo)bar\\1/') + expect(tokens[0]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[2]).toEqual value: 'foo', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp'] + expect(tokens[3]).toEqual value: ')', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[4]).toEqual value: 'bar', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[5]).toEqual value: '\\1', scopes: ['source.js', 'string.regexp.js', 'keyword.other.back-reference.regexp'] + expect(tokens[6]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + + {tokens} = grammar.tokenizeLine('/(?foo)bar\\k/') + expect(tokens[0]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: '(?', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[2]).toEqual value: 'foo', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp'] + expect(tokens[3]).toEqual value: ')', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[4]).toEqual value: 'bar', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[5]).toEqual value: '\\k', scopes: ['source.js', 'string.regexp.js', 'keyword.other.back-reference.regexp'] + expect(tokens[6]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + + {tokens} = grammar.tokenizeLine('/(?:foo)bar/') + expect(tokens[0]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: '(?:', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[2]).toEqual value: 'foo', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp'] + expect(tokens[3]).toEqual value: ')', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[4]).toEqual value: 'bar', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[5]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + + {tokens} = grammar.tokenizeLine('/(?<=foo)test(?=bar)/') + expect(tokens[0]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'string.regexp.js', 'meta.group.assertion.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[2]).toEqual value: '?<=', scopes: ['source.js', 'string.regexp.js', 'meta.group.assertion.regexp', 'meta.assertion.look-behind.regexp'] + expect(tokens[3]).toEqual value: 'foo', scopes: ['source.js', 'string.regexp.js', 'meta.group.assertion.regexp'] + expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'string.regexp.js', 'meta.group.assertion.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[5]).toEqual value: 'test', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[6]).toEqual value: '(', scopes: ['source.js', 'string.regexp.js', 'meta.group.assertion.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[7]).toEqual value: '?=', scopes: ['source.js', 'string.regexp.js', 'meta.group.assertion.regexp', 'meta.assertion.look-ahead.regexp'] + expect(tokens[8]).toEqual value: 'bar', scopes: ['source.js', 'string.regexp.js', 'meta.group.assertion.regexp'] + expect(tokens[9]).toEqual value: ')', scopes: ['source.js', 'string.regexp.js', 'meta.group.assertion.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[10]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + + {tokens} = grammar.tokenizeLine('/(? expect(tokens[6]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] expect(tokens[7]).toEqual value: ']', scopes: ['source.js', 'meta.brace.square.js'] + it "tokenizes regular expressions inside curly brackets", -> + {tokens} = grammar.tokenizeLine('{/test/}') + expect(tokens[0]).toEqual value: '{', scopes: ['source.js', 'meta.brace.curly.js'] + expect(tokens[1]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[2]).toEqual value: 'test', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[3]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + expect(tokens[4]).toEqual value: '}', scopes: ['source.js', 'meta.brace.curly.js'] + it "tokenizes regular expressions inside arrow function expressions", -> {tokens} = grammar.tokenizeLine('getRegex = () => /^helloworld$/;') expect(tokens[9]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] @@ -232,6 +317,17 @@ describe "JavaScript grammar", -> {tokens} = grammar.tokenizeLine('0X1D306') expect(tokens[0]).toEqual value: '0X1D306', scopes: ['source.js', 'constant.numeric.hex.js'] + {tokens} = grammar.tokenizeLine('0x1D306n') + expect(tokens[0]).toEqual value: '0x1D306n', scopes: ['source.js', 'constant.numeric.hex.js'] + + {tokens} = grammar.tokenizeLine('0X1D306n') + expect(tokens[0]).toEqual value: '0X1D306n', scopes: ['source.js', 'constant.numeric.hex.js'] + + {tokens} = grammar.tokenizeLine('0X1D30_69A3') + expect(tokens[0]).toEqual value: '0X1D30', scopes: ['source.js', 'constant.numeric.hex.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.hex.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '69A3', scopes: ['source.js', 'constant.numeric.hex.js'] + it "tokenizes binary literals", -> {tokens} = grammar.tokenizeLine('0b011101110111010001100110') expect(tokens[0]).toEqual value: '0b011101110111010001100110', scopes: ['source.js', 'constant.numeric.binary.js'] @@ -239,6 +335,25 @@ describe "JavaScript grammar", -> {tokens} = grammar.tokenizeLine('0B011101110111010001100110') expect(tokens[0]).toEqual value: '0B011101110111010001100110', scopes: ['source.js', 'constant.numeric.binary.js'] + {tokens} = grammar.tokenizeLine('0b011101110111010001100110n') + expect(tokens[0]).toEqual value: '0b011101110111010001100110n', scopes: ['source.js', 'constant.numeric.binary.js'] + + {tokens} = grammar.tokenizeLine('0B011101110111010001100110n') + expect(tokens[0]).toEqual value: '0B011101110111010001100110n', scopes: ['source.js', 'constant.numeric.binary.js'] + + {tokens} = grammar.tokenizeLine('0B0111_0111_0111_0100_0110_0110') + expect(tokens[0]).toEqual value: '0B0111', scopes: ['source.js', 'constant.numeric.binary.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.binary.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '0111', scopes: ['source.js', 'constant.numeric.binary.js'] + expect(tokens[3]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.binary.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[4]).toEqual value: '0111', scopes: ['source.js', 'constant.numeric.binary.js'] + expect(tokens[5]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.binary.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[6]).toEqual value: '0100', scopes: ['source.js', 'constant.numeric.binary.js'] + expect(tokens[7]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.binary.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[8]).toEqual value: '0110', scopes: ['source.js', 'constant.numeric.binary.js'] + expect(tokens[9]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.binary.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[10]).toEqual value: '0110', scopes: ['source.js', 'constant.numeric.binary.js'] + it "tokenizes octal literals", -> {tokens} = grammar.tokenizeLine('0o1411') expect(tokens[0]).toEqual value: '0o1411', scopes: ['source.js', 'constant.numeric.octal.js'] @@ -246,13 +361,41 @@ describe "JavaScript grammar", -> {tokens} = grammar.tokenizeLine('0O1411') expect(tokens[0]).toEqual value: '0O1411', scopes: ['source.js', 'constant.numeric.octal.js'] + {tokens} = grammar.tokenizeLine('0o1411n') + expect(tokens[0]).toEqual value: '0o1411n', scopes: ['source.js', 'constant.numeric.octal.js'] + + {tokens} = grammar.tokenizeLine('0O1411n') + expect(tokens[0]).toEqual value: '0O1411n', scopes: ['source.js', 'constant.numeric.octal.js'] + {tokens} = grammar.tokenizeLine('0010') expect(tokens[0]).toEqual value: '0010', scopes: ['source.js', 'constant.numeric.octal.js'] + {tokens} = grammar.tokenizeLine('0010_7201_5112') + expect(tokens[0]).toEqual value: '0010', scopes: ['source.js', 'constant.numeric.octal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.octal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '7201', scopes: ['source.js', 'constant.numeric.octal.js'] + expect(tokens[3]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.octal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[4]).toEqual value: '5112', scopes: ['source.js', 'constant.numeric.octal.js'] + + {tokens} = grammar.tokenizeLine('0O1411_1236') + expect(tokens[0]).toEqual value: '0O1411', scopes: ['source.js', 'constant.numeric.octal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.octal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '1236', scopes: ['source.js', 'constant.numeric.octal.js'] + it "tokenizes decimals", -> {tokens} = grammar.tokenizeLine('1234') expect(tokens[0]).toEqual value: '1234', scopes: ['source.js', 'constant.numeric.decimal.js'] + {tokens} = grammar.tokenizeLine('123456789n') + expect(tokens[0]).toEqual value: '123456789n', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('123_456_789n') + expect(tokens[0]).toEqual value: '123', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '456', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[3]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[4]).toEqual value: '789n', scopes: ['source.js', 'constant.numeric.decimal.js'] + {tokens} = grammar.tokenizeLine('5e-10') expect(tokens[0]).toEqual value: '5e-10', scopes: ['source.js', 'constant.numeric.decimal.js'] @@ -263,24 +406,92 @@ describe "JavaScript grammar", -> expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] + {tokens} = grammar.tokenizeLine('9_9.') + expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[3]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] + {tokens} = grammar.tokenizeLine('.9') expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] expect(tokens[1]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + {tokens} = grammar.tokenizeLine('.9_9') + expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] + expect(tokens[1]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[2]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[3]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + {tokens} = grammar.tokenizeLine('9.9') expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] expect(tokens[2]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + {tokens} = grammar.tokenizeLine('9_9.9_9') + expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[3]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] + expect(tokens[4]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[5]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[6]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + {tokens} = grammar.tokenizeLine('.1e-23') expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] expect(tokens[1]).toEqual value: '1e-23', scopes: ['source.js', 'constant.numeric.decimal.js'] + {tokens} = grammar.tokenizeLine('.1_1E+1_1') + expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] + expect(tokens[1]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[2]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[3]).toEqual value: '1E+1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[4]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[5]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + {tokens} = grammar.tokenizeLine('1.E3') expect(tokens[0]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] expect(tokens[2]).toEqual value: 'E3', scopes: ['source.js', 'constant.numeric.decimal.js'] + {tokens} = grammar.tokenizeLine('1_1.E-1_1') + expect(tokens[0]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[3]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] + expect(tokens[4]).toEqual value: 'E-1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[5]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[6]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('1_1.1_1E1_1') + expect(tokens[0]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[3]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] + expect(tokens[4]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[5]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[6]).toEqual value: '1E1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[7]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[8]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('9_9') + expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('9_9_9') + expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[3]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[4]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('999_999_999') + expect(tokens[0]).toEqual value: '999', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '999', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[3]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[4]).toEqual value: '999', scopes: ['source.js', 'constant.numeric.decimal.js'] + it "does not tokenize numbers that are part of a variable", -> {tokens} = grammar.tokenizeLine('hi$1') expect(tokens[0]).toEqual value: 'hi$1', scopes: ['source.js'] @@ -288,6 +499,20 @@ describe "JavaScript grammar", -> {tokens} = grammar.tokenizeLine('hi_1') expect(tokens[0]).toEqual value: 'hi_1', scopes: ['source.js'] + {tokens} = grammar.tokenizeLine('_1') + expect(tokens[0]).toEqual value: '_1', scopes: ['source.js', 'constant.other.js'] + + {tokens} = grammar.tokenizeLine('1_') + expect(tokens[0]).toEqual value: '1_', scopes: ['source.js', 'invalid.illegal.identifier.js'] + + {tokens} = grammar.tokenizeLine('1_._1') + expect(tokens[0]).toEqual value: '1_', scopes: ['source.js', 'invalid.illegal.identifier.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.delimiter.property.period.js'] + expect(tokens[2]).toEqual value: '_1', scopes: ['source.js', 'variable.other.property.js'] + + {tokens} = grammar.tokenizeLine('1__1') + expect(tokens[0]).toEqual value: '1__1', scopes: ['source.js', 'invalid.illegal.identifier.js'] + describe "operators", -> it "tokenizes them", -> operators = ["delete", "in", "of", "instanceof", "new", "typeof", "void"] @@ -613,6 +838,15 @@ describe "JavaScript grammar", -> expect(tokens[14]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.js', 'punctuation.definition.string.end.js'] describe "HTML template strings", -> + # TODO: Remove after Atom 1.21 is released + [tagScope, entityScope] = [] + if parseFloat(atom.getVersion()) <= 1.21 + tagScope = 'meta.tag.inline.any.html' + entityScope = 'entity.name.tag.inline.any.html' + else + tagScope = 'meta.tag.inline.b.html' + entityScope = 'entity.name.tag.inline.b.html' + beforeEach -> waitsForPromise -> atom.packages.activatePackage("language-html") @@ -622,15 +856,15 @@ describe "JavaScript grammar", -> expect(tokens[0]).toEqual value: 'html', scopes: ['source.js', 'string.quoted.template.html.js', 'entity.name.function.js'] expect(tokens[1]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.begin.js'] expect(tokens[2]).toEqual value: 'hey ', scopes: ['source.js', 'string.quoted.template.html.js'] - expect(tokens[3]).toEqual value: '<', scopes: ['source.js', 'string.quoted.template.html.js', 'meta.tag.inline.any.html', 'punctuation.definition.tag.begin.html'] - expect(tokens[4]).toEqual value: 'b', scopes: ['source.js', 'string.quoted.template.html.js', 'meta.tag.inline.any.html', 'entity.name.tag.inline.any.html'] - expect(tokens[5]).toEqual value: '>', scopes: ['source.js', 'string.quoted.template.html.js', 'meta.tag.inline.any.html', 'punctuation.definition.tag.end.html'] + expect(tokens[3]).toEqual value: '<', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.begin.html'] + expect(tokens[4]).toEqual value: 'b', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, entityScope] + expect(tokens[5]).toEqual value: '>', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.end.html'] expect(tokens[6]).toEqual value: '${', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] expect(tokens[7]).toEqual value: 'name', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source'] expect(tokens[8]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] - expect(tokens[9]).toEqual value: '', scopes: ['source.js', 'string.quoted.template.html.js', 'meta.tag.inline.any.html', 'punctuation.definition.tag.end.html'] + expect(tokens[9]).toEqual value: '', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.end.html'] expect(tokens[12]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.end.js'] it "tokenizes innerHTML attribute declarations with string template tags", -> @@ -643,15 +877,15 @@ describe "JavaScript grammar", -> expect(tokens[5]).toEqual value: ' ', scopes: ['source.js'] expect(tokens[6]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.begin.js'] expect(tokens[7]).toEqual value: 'hey ', scopes: ['source.js', 'string.quoted.template.html.js'] - expect(tokens[8]).toEqual value: '<', scopes: ['source.js', 'string.quoted.template.html.js', 'meta.tag.inline.any.html', 'punctuation.definition.tag.begin.html'] - expect(tokens[9]).toEqual value: 'b', scopes: ['source.js', 'string.quoted.template.html.js', 'meta.tag.inline.any.html', 'entity.name.tag.inline.any.html'] - expect(tokens[10]).toEqual value: '>', scopes: ['source.js', 'string.quoted.template.html.js', 'meta.tag.inline.any.html', 'punctuation.definition.tag.end.html'] + expect(tokens[8]).toEqual value: '<', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.begin.html'] + expect(tokens[9]).toEqual value: 'b', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, entityScope] + expect(tokens[10]).toEqual value: '>', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.end.html'] expect(tokens[11]).toEqual value: '${', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] expect(tokens[12]).toEqual value: 'name', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source'] expect(tokens[13]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] - expect(tokens[14]).toEqual value: '', scopes: ['source.js', 'string.quoted.template.html.js', 'meta.tag.inline.any.html', 'punctuation.definition.tag.end.html'] + expect(tokens[14]).toEqual value: '', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.end.html'] expect(tokens[17]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.end.js'] it "tokenizes ES6 tagged HTML string templates with expanded function name", -> @@ -659,15 +893,15 @@ describe "JavaScript grammar", -> expect(tokens[0]).toEqual value: 'escapeHTML', scopes: ['source.js', 'string.quoted.template.html.js', 'entity.name.function.js'] expect(tokens[1]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.begin.js'] expect(tokens[2]).toEqual value: 'hey ', scopes: ['source.js', 'string.quoted.template.html.js'] - expect(tokens[3]).toEqual value: '<', scopes: ['source.js', 'string.quoted.template.html.js', 'meta.tag.inline.any.html', 'punctuation.definition.tag.begin.html'] - expect(tokens[4]).toEqual value: 'b', scopes: ['source.js', 'string.quoted.template.html.js', 'meta.tag.inline.any.html', 'entity.name.tag.inline.any.html'] - expect(tokens[5]).toEqual value: '>', scopes: ['source.js', 'string.quoted.template.html.js', 'meta.tag.inline.any.html', 'punctuation.definition.tag.end.html'] + expect(tokens[3]).toEqual value: '<', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.begin.html'] + expect(tokens[4]).toEqual value: 'b', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, entityScope] + expect(tokens[5]).toEqual value: '>', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.end.html'] expect(tokens[6]).toEqual value: '${', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] expect(tokens[7]).toEqual value: 'name', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source'] expect(tokens[8]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] - expect(tokens[9]).toEqual value: '', scopes: ['source.js', 'string.quoted.template.html.js', 'meta.tag.inline.any.html', 'punctuation.definition.tag.end.html'] + expect(tokens[9]).toEqual value: '', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.end.html'] expect(tokens[12]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.end.js'] it "tokenizes ES6 tagged HTML string templates with expanded function name and white space", -> @@ -676,15 +910,15 @@ describe "JavaScript grammar", -> expect(tokens[1]).toEqual value: ' ', scopes: ['source.js', 'string.quoted.template.html.js'] expect(tokens[2]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.begin.js'] expect(tokens[3]).toEqual value: 'hey ', scopes: ['source.js', 'string.quoted.template.html.js'] - expect(tokens[4]).toEqual value: '<', scopes: ['source.js', 'string.quoted.template.html.js', 'meta.tag.inline.any.html', 'punctuation.definition.tag.begin.html'] - expect(tokens[5]).toEqual value: 'b', scopes: ['source.js', 'string.quoted.template.html.js', 'meta.tag.inline.any.html', 'entity.name.tag.inline.any.html'] - expect(tokens[6]).toEqual value: '>', scopes: ['source.js', 'string.quoted.template.html.js', 'meta.tag.inline.any.html', 'punctuation.definition.tag.end.html'] + expect(tokens[4]).toEqual value: '<', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.begin.html'] + expect(tokens[5]).toEqual value: 'b', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, entityScope] + expect(tokens[6]).toEqual value: '>', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.end.html'] expect(tokens[7]).toEqual value: '${', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] expect(tokens[8]).toEqual value: 'name', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source'] expect(tokens[9]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] - expect(tokens[10]).toEqual value: '', scopes: ['source.js', 'string.quoted.template.html.js', 'meta.tag.inline.any.html', 'punctuation.definition.tag.end.html'] + expect(tokens[10]).toEqual value: '', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.end.html'] expect(tokens[13]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.end.js'] describe "ES6 tagged Relay.QL string templates", -> @@ -707,6 +941,14 @@ describe "JavaScript grammar", -> expect(tokens[6]).toEqual value: ' }', scopes: ['source.js', 'string.quoted.template.graphql.js'] expect(tokens[7]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.end.js'] + describe "ES6 tagged gql string templates", -> + it "tokenizes them as strings", -> + {tokens} = grammar.tokenizeLine('gql`fragment on Foo { id }`') + expect(tokens[0]).toEqual value: 'gql', scopes: ['source.js', 'string.quoted.template.graphql.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.begin.js'] + expect(tokens[2]).toEqual value: 'fragment on Foo { id }', scopes: ['source.js', 'string.quoted.template.graphql.js'] + expect(tokens[3]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.end.js'] + describe "ES6 tagged SQL string templates", -> it "tokenizes them as strings", -> {tokens} = grammar.tokenizeLine('SQL`SELECT foo FROM bar WHERE id = :id`') @@ -1014,6 +1256,29 @@ describe "JavaScript grammar", -> expect(tokens[10]).toEqual value: '}', scopes: ['source.js', 'meta.export.js', 'punctuation.definition.modules.end.js'] expect(tokens[12]).toEqual value: 'from', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + it "tokenizes multiline re-export", -> + lines = grammar.tokenizeLines ''' + export { + default as alias, + member1 as alias1, + member2, + } from "module-name"; + ''' + expect(lines[0][0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(lines[0][2]).toEqual value: '{', scopes: ['source.js', 'meta.export.js', 'punctuation.definition.modules.begin.js'] + expect(lines[1][1]).toEqual value: 'default', scopes: ['source.js', 'meta.export.js', 'variable.language.default.js'] + expect(lines[1][3]).toEqual value: 'as', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(lines[1][5]).toEqual value: 'alias', scopes: ['source.js', 'meta.export.js', 'variable.other.module-alias.js'] + expect(lines[1][6]).toEqual value: ',', scopes: ['source.js', 'meta.export.js', 'meta.delimiter.object.comma.js'] + expect(lines[2][1]).toEqual value: 'member1', scopes: ['source.js', 'meta.export.js', 'variable.other.module.js'] + expect(lines[2][3]).toEqual value: 'as', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(lines[2][5]).toEqual value: 'alias1', scopes: ['source.js', 'meta.export.js', 'variable.other.module-alias.js'] + expect(lines[2][6]).toEqual value: ',', scopes: ['source.js', 'meta.export.js', 'meta.delimiter.object.comma.js'] + expect(lines[3][1]).toEqual value: 'member2', scopes: ['source.js', 'meta.export.js', 'variable.other.module.js'] + expect(lines[3][2]).toEqual value: ',', scopes: ['source.js', 'meta.export.js', 'meta.delimiter.object.comma.js'] + expect(lines[4][0]).toEqual value: '}', scopes: ['source.js', 'meta.export.js', 'punctuation.definition.modules.end.js'] + expect(lines[4][2]).toEqual value: 'from', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + describe "ES6 yield", -> it "tokenizes yield", -> {tokens} = grammar.tokenizeLine('yield next') @@ -1852,6 +2117,41 @@ describe "JavaScript grammar", -> expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] expect(tokens[5]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + describe "promise", -> + it "tokenizes the promise object", -> + {tokens} = grammar.tokenizeLine('Promise;') + expect(tokens[0]).toEqual value: 'Promise', scopes: ['source.js', 'support.class.promise.js'] + expect(tokens[1]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + it "tokenizes promise support functions", -> + {tokens} = grammar.tokenizeLine('Promise.race();') + expect(tokens[0]).toEqual value: 'Promise', scopes: ['source.js', 'support.class.promise.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(tokens[2]).toEqual value: 'race', scopes: ['source.js', 'meta.method-call.js', 'support.function.promise.js'] + expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(tokens[5]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + lines = grammar.tokenizeLines ''' + Promise + .resolve(); + ''' + expect(lines[0][0]).toEqual value: 'Promise', scopes: ['source.js', 'support.class.promise.js'] + expect(lines[1][0]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(lines[1][1]).toEqual value: 'resolve', scopes: ['source.js', 'meta.method-call.js', 'support.function.promise.js'] + expect(lines[1][2]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(lines[1][3]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(lines[1][4]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + it "tokenizes promise custom functions", -> + {tokens} = grammar.tokenizeLine('Promise.anExtraFunction();') + expect(tokens[0]).toEqual value: 'Promise', scopes: ['source.js', 'support.class.promise.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(tokens[2]).toEqual value: 'anExtraFunction', scopes: ['source.js', 'meta.method-call.js', 'entity.name.function.js'] + expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(tokens[5]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + describe "object literals", -> keywords = ['super', 'this', 'null', 'true', 'false', 'debugger', 'exports', '__filename'] @@ -2042,7 +2342,7 @@ describe "JavaScript grammar", -> it "indents non-allman-style curly braces", -> expectPreservedIndentation """ if (true) { - for (;;) { + for (;;) { // " while (true) { x(); } @@ -2066,7 +2366,7 @@ describe "JavaScript grammar", -> it "indents collection literals", -> expectPreservedIndentation """ - [ + [ // " { a: b, c: d @@ -2079,7 +2379,7 @@ describe "JavaScript grammar", -> it "indents function arguments", -> expectPreservedIndentation """ f( - g( + g( // " h, i ), diff --git a/spec/jsdoc-spec.coffee b/spec/jsdoc-spec.coffee index c904f5c3..8baf4e28 100644 --- a/spec/jsdoc-spec.coffee +++ b/spec/jsdoc-spec.coffee @@ -2,6 +2,8 @@ describe "JSDoc grammar", -> grammar = null beforeEach -> + atom.config.set('core.useTreeSitterParsers', false) + waitsForPromise -> atom.packages.activatePackage("language-javascript") @@ -1467,7 +1469,7 @@ describe "JSDoc grammar", -> expect(tokens[7]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] expect(tokens[8]).toEqual value: 'oo', scopes: ['source.js'] expect(tokens[9]).toEqual value: '"', scopes: ['source.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] - expect(tokens[10]).toEqual value: '} bar', scopes: ['source.js', 'string.quoted.double.js', 'invalid.illegal.string.js'] + expect(tokens[10]).toEqual value: '} bar', scopes: ['source.js', 'string.quoted.double.js'] it "terminates any embedded JavaScript code", -> lines = grammar.tokenizeLines """ @@ -1527,3 +1529,9 @@ describe "JSDoc grammar", -> expect(tokens[27]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] expect(tokens[29]).toEqual value: '20', scopes: ['source.js', 'constant.numeric.decimal.js'] expect(tokens[30]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + describe "when the line ends without a closing bracket", -> + it "does not attempt to match the optional value (regression)", -> + {tokens} = grammar.tokenizeLine('/** @param {array} [bar = "x" REMOVE THE CLOSE BRACKET HERE.') + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: ' = "x" REMOVE THE CLOSE BRACKET HERE.', scopes: ['source.js', 'comment.block.documentation.js'] diff --git a/spec/regular-expression-replacement-spec.coffee b/spec/regular-expression-replacement-spec.coffee index 5f2c682b..735aa823 100644 --- a/spec/regular-expression-replacement-spec.coffee +++ b/spec/regular-expression-replacement-spec.coffee @@ -2,6 +2,8 @@ describe "Regular Expression Replacement grammar", -> grammar = null beforeEach -> + atom.config.set('core.useTreeSitterParsers', false) + waitsForPromise -> atom.packages.activatePackage("language-javascript")