diff --git a/.github/no-response.yml b/.github/no-response.yml new file mode 100644 index 00000000..1c8799d1 --- /dev/null +++ b/.github/no-response.yml @@ -0,0 +1,15 @@ +# Configuration for probot-no-response - https://github.com/probot/no-response + +# Number of days of inactivity before an issue is closed for lack of response +daysUntilClose: 28 + +# Label requiring a response +responseRequiredLabel: more-information-needed + +# Comment to post when closing an issue for lack of response. Set to `false` to disable. +closeComment: > + This issue has been automatically closed because there has been no response + to our request for more information from the original author. With only the + information that is currently in the issue, we don't have enough information + to take action. Please reach out if you have or find the answers we need so + that we can investigate further. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..432a2dce --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,45 @@ +name: ci +on: + - pull_request + - push + +jobs: + Test: + if: "!contains(github.event.head_commit.message, '[skip ci]')" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - macos-latest + - windows-latest + atom_channel: + - stable + - nightly + steps: + - uses: actions/checkout@v2 + - name: Cache + uses: actions/cache@v2 + with: + path: | + 'node_modules' + 'C:/Program Files (x86)/MSBuild/Microsoft.Cpp/v4.0/v140' + key: ${{ runner.os }}-${{ matrix.atom_channel }}-${{ hashFiles('package.json') }} + + - uses: UziTech/action-setup-atom@v1 + with: + channel: ${{ matrix.atom_channel }} + + - name: Install dependencies + run: apm install + + - name: Run tests + run: apm test + + Skip: + if: contains(github.event.head_commit.message, '[skip ci]') + runs-on: ubuntu-latest + steps: + - name: Skip CI 🚫 + run: echo skip ci diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..d5f19d89 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +node_modules +package-lock.json diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e70782fb..0fd0ad69 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1 +1 @@ -See the [Atom contributing guide](https://atom.io/docs/latest/contributing) +See the [Atom contributing guide](https://github.com/atom/atom/blob/master/CONTRIBUTING.md) diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md new file mode 100644 index 00000000..b60bb86c --- /dev/null +++ b/ISSUE_TEMPLATE.md @@ -0,0 +1,40 @@ + + +### Prerequisites + +* [ ] Put an X between the brackets on this line if you have done all of the following: + * Reproduced the problem in Safe Mode: http://flight-manual.atom.io/hacking-atom/sections/debugging/#using-safe-mode + * Followed all applicable steps in the debugging guide: http://flight-manual.atom.io/hacking-atom/sections/debugging/ + * Checked the FAQs on the message board for common solutions: https://discuss.atom.io/c/faq + * Checked that your issue isn't already filed: https://github.com/issues?utf8=✓&q=is%3Aissue+user%3Aatom + * Checked that there is not already an Atom package that provides the described functionality: https://atom.io/packages + +### Description + +[Description of the issue] + +### Steps to Reproduce + +1. [First Step] +2. [Second Step] +3. [and so on...] + +**Expected behavior:** [What you expect to happen] + +**Actual behavior:** [What actually happens] + +**Reproduces how often:** [What percentage of the time does it reproduce?] + +### Versions + +You can get this information from copy and pasting the output of `atom --version` and `apm --version` from the command line. Also, please include the OS and what version of the OS you're running. + +### Additional Information + +Any additional information, configuration or data that might be necessary to reproduce the issue. diff --git a/LICENSE.md b/LICENSE.md index 124955a4..caa758ef 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,4 +1,4 @@ -Copyright (c) 2014 GitHub Inc. +Copyright (c) 2015 GitHub Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..cdaa94a8 --- /dev/null +++ b/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,28 @@ +### Requirements + +* Filling out the template is required. Any pull request that does not include enough information to be reviewed in a timely manner may be closed at the maintainers' discretion. +* All new code requires tests to ensure against regressions + +### Description of the Change + + + +### Alternate Designs + + + +### Benefits + + + +### Possible Drawbacks + + + +### Applicable Issues + + diff --git a/README.md b/README.md index 7972dd85..d80d4475 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,12 @@ -# JavaScript language support in Atom +##### Atom and all repositories under Atom will be archived on December 15, 2022. Learn more in our [official announcement](https://github.blog/2022-06-08-sunsetting-atom/) + # JavaScript language support in Atom -Adds syntax highlighting and snippets to JavaScript file in Atom. +![ci](https://github.com/atom/language-javascript/workflows/ci/badge.svg) +[![Dependency Status](https://david-dm.org/atom/language-javascript.svg)](https://david-dm.org/atom/language-javascript) -Originally [converted](http://atom.io/docs/latest/converting-a-text-mate-bundle) +Adds syntax highlighting and snippets for JavaScript in Atom. + +Originally [converted](http://flight-manual.atom.io/hacking-atom/sections/converting-from-textmate) from the [JavaScript TextMate bundle](https://github.com/textmate/javascript.tmbundle). Contributions are greatly appreciated. Please fork this repository and open a diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 00000000..795da41c --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,6 @@ +# empty appveyor +build: off + +branches: + only: + - non-existing diff --git a/coffeelint.json b/coffeelint.json new file mode 100644 index 00000000..a5dd715e --- /dev/null +++ b/coffeelint.json @@ -0,0 +1,37 @@ +{ + "max_line_length": { + "level": "ignore" + }, + "no_empty_param_list": { + "level": "error" + }, + "arrow_spacing": { + "level": "error" + }, + "no_interpolation_in_single_quotes": { + "level": "error" + }, + "no_debugger": { + "level": "error" + }, + "prefer_english_operator": { + "level": "error" + }, + "colon_assignment_spacing": { + "spacing": { + "left": 0, + "right": 1 + }, + "level": "error" + }, + "braces_spacing": { + "spaces": 0, + "level": "error" + }, + "spacing_after_comma": { + "level": "error" + }, + "no_stand_alone_at": { + "level": "error" + } +} diff --git a/grammars/javascript.cson b/grammars/javascript.cson index 4729e01f..62011cef 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -1,382 +1,1135 @@ 'scopeName': 'source.js' 'fileTypes': [ 'js' - 'htc' - 'jsx' '_js' + 'cjs' + 'es' 'es6' + 'gs' + 'htc' + 'jscad' + 'jscript' + 'jse' + 'jslib' + 'jsm' + 'json5' + 'jspre' + 'mjs' + 'pac' + 'pjs' + 'sjs' + 'snap' + 'xsjs' + 'xsjslib' ] -'firstLineMatch': '^#!.*\\bnode' +'firstLineMatch': '''(?x) + # Hashbang + ^\\#!.*(?:\\s|\\/|(?<=!)\\b) + (?:node|iojs|JavaScript) + (?:$|\\s) + | + # Modeline + (?i: + # Emacs + -\\*-(?:\\s*(?=[^:;\\s]+\\s*-\\*-)|(?:.*?[;\\s]|(?<=-\\*-))mode\\s*:\\s*) + (?:js|javascript) + (?=[\\s;]|(?]?\\d+|m)?|\\sex)(?=:(?=\\s*set?\\s[^\\n:]+:)|:(?!\\s*set?\\s))(?:(?:\\s|\\s*:\\s*)\\w*(?:\\s*=(?:[^\\n\\\\\\s]|\\\\.)*)?)*[\\s:](?:filetype|ft|syntax)\\s*= + javascript + (?=\\s|:|$) + ) +''' 'name': 'JavaScript' 'patterns': [ { - 'captures': - '1': - 'name': 'support.class.js' - '2': - 'name': 'support.constant.js' - '3': - 'name': 'keyword.operator.js' - 'comment': 'match stuff like: Sound.prototype = { … } when extending an object' - 'match': '([a-zA-Z_?.$][\\w?.$]*)\\.(prototype)\\s*(=)\\s*' - 'name': 'meta.class.js' - } - { - 'begin': '([a-zA-Z_?.$][\\w?.$]*)\\.(prototype)\\.([a-zA-Z_?.$][\\w?.$]*)\\s*(=)\\s*(function\\*?)?\\s*(\\*?)(\\()' + # ES6 import + 'begin': '(? [expression|{statements}] + 'begin': '''(?x) + (?= + (? + ) + ''' + 'end': '''(?x) + (?<=})| + ((?! + \\s*{| + \\G\\(| + \\G[\\w$]+| + \\s*/\\*|\\s*// + )(?=\\s*\\S)) + ''' 'patterns': [ { - 'begin': '//' - 'beginCaptures': - '0': - 'name': 'punctuation.definition.comment.js' - 'end': '\\n' - 'name': 'comment.line.double-slash.js' + 'include': '#comments' + } + { + 'include': '#function_body' + } + { + 'begin': '\\G' + 'end': '(?<=(=>))' + 'name': 'meta.function.arrow.js' + 'patterns': [ + { + 'include': '#arrow_function_innards' + } + ] } ] } { - 'captures': - '0': - 'name': 'punctuation.definition.comment.html.js' - '2': - 'name': 'punctuation.definition.comment.html.js' - 'match': '()' - 'name': 'comment.block.html.js' + # [.]foo = ... => ... + 'begin': '''(?x) + (?= + (\\.)?[a-zA-Z_$][\\w$]* + \\s*(=)\\s* + ((\\(([^\\(\\)]*)?\\))|[\\w$]+) + \\s*=> + ) + ''' + 'end': '''(?x) + (?<=})| + ((?! + \\s*{| + \\G(\\.)?[a-zA-Z_$][\\w$]*\\s*(=)\\s*\\(| + \\G(\\.)?[a-zA-Z_$][\\w$]*\\s*(=)\\s*[\\w$]+| + \\s*/\\*|\\s*// + )(?=\\s*\\S)) + ''' + 'patterns': [ + { + 'include': '#comments' + } + { + 'include': '#function_body' + } + { + 'begin': '\\G' + 'end': '(?<=(=>))' + 'name': 'meta.function.arrow.js' + 'patterns': [ + { + 'match': '\\G(\\.)?([a-zA-Z_$][\\w$]*)\\s*(=)' + 'captures': + '1': + 'name': 'meta.delimiter.method.period.js' + '2': + 'name': 'entity.name.function.js' + '3': + 'name': 'keyword.operator.assignment.js' + } + { + 'include': '#arrow_function_innards' + } + ] + } + ] + } + { + # foo: ... => ... + 'begin': '''(?x) + (?= + \\b[a-zA-Z_$][\\w$]* + \\s*:\\s* + ((\\(([^\\(\\)]*)?\\))|[\\w$]+) + \\s*=> + ) + ''' + 'end': '''(?x) + (?<=})| + ((?! + \\s*{| + \\G[\\w$]+\\s*:| + \\s*/\\*|\\s*// + )(?=\\s*\\S)) + ''' + 'patterns': [ + { + 'include': '#comments' + } + { + 'include': '#function_body' + } + { + 'begin': '\\G' + 'end': '(?<=(=>))' + 'name': 'meta.function.arrow.json.js' + 'patterns': [ + { + 'match': '\\b([a-zA-Z_$][\\w$]*)\\s*(:)\\s*' + 'captures': + '1': + 'name': 'entity.name.function.js' + '2': + 'name': 'keyword.operator.assignment.js' + } + { + 'include': '#arrow_function_innards' + } + ] + } + ] + } + { + # "foo": ... => ... + 'begin': '''(?x) + (?= + ((\'[^\']*?\')|("[^"]*?")) + \\s*:\\s* + ((\\(([^\\(\\)]*)?\\))|[\\w$]+) + \\s*=> + ) + ''' + 'end': '''(?x) + (?<=})| + ((?! + \\G((\'[^\']*?\')|("[^"]*?"))| + \\s*{| + \\s*/\\*|\\s*// + )(?=\\s*\\S)) + ''' + 'patterns': [ + { + 'include': '#comments' + } + { + 'include': '#function_body' + } + { + 'begin': '\\G' + 'end': '(?<=(=>))' + 'name': 'meta.function.arrow.json.js' + 'patterns': [ + { + 'match': '(?:((\')([^\']*?)(\'))|((")([^"]*?)(")))\\s*(:)' + 'captures': + '1': + 'name': 'string.quoted.single.js' + '2': + 'name': 'punctuation.definition.string.begin.js' + '3': + 'name': 'entity.name.function.js' + '4': + 'name': 'punctuation.definition.string.end.js' + '5': + 'name': 'string.quoted.double.js' + '6': + 'name': 'punctuation.definition.string.begin.js' + '7': + 'name': 'entity.name.function.js' + '8': + 'name': 'punctuation.definition.string.end.js' + '9': + 'name': 'keyword.operator.assignment.js' + } + { + 'include': '#arrow_function_innards' + } + ] + } + ] + } + { + 'match': '(=>)' + 'captures': + '0': + 'name': 'meta.function.arrow.js' + '1': + 'name': 'storage.type.function.arrow.js' + } + { + 'match': '''(?x) + \\b(class) + (?: + (?:\\s+(extends)\\s+([a-zA-Z_$][\\w$]*)) + | + (?: + (?:\\s+([a-zA-Z_$][\\w$]*)) + (?:\\s+(extends)\\s+([a-zA-Z_$][\\w$]*))? + ) + ) + ''' + 'captures': + '1': + 'name': 'storage.type.class.js' + '2': + 'name': 'storage.modifier.js' + '3': + 'name': 'entity.other.inherited-class.js' + '4': + 'name': 'entity.name.type.class.js' + '5': + 'name': 'storage.modifier.js' + '6': + 'name': 'entity.other.inherited-class.js' + 'name': 'meta.class.js' + } + { + 'match': '(new)\\s+([\\w$]+[\\w.$]*)' + 'name': 'meta.class.instance.constructor.js' + 'captures': + '1': + 'name': 'keyword.operator.new.js' + '2': + 'name': 'entity.name.type.instance.js' + 'patterns': [ + { + 'match': '\\.' + 'name': 'meta.delimiter.property.period.js' + } + ] + } + { + # console + # console.log(arg1, "arg2", [...]) + 'begin': '(?)' + 'captures': + '0': + 'name': 'punctuation.definition.comment.html.js' + '2': + 'name': 'punctuation.definition.comment.html.js' + 'name': 'comment.block.html.js' } { - 'match': '(?=|<<=|>>=|>>>=|<>|<|>|!|&&|\\|\\||\\?|\\:|\\*=|(?|&&|\\|\\|)\\s*(/)(?![/*+?])(?=.*/)' 'beginCaptures': '1': 'name': 'punctuation.definition.string.begin.js' - 'end': '(/)[igm]*' + 'end': '(/)([gimsuy]*)' 'endCaptures': '1': 'name': 'punctuation.definition.string.end.js' + '2': + 'name': 'meta.flag.regexp' 'name': 'string.regexp.js' 'patterns': [ { @@ -384,12 +1137,56 @@ } ] } + { + 'begin': '\\?' + 'beginCaptures': + '0': + 'name': 'keyword.operator.ternary.js' + 'end': ':' + 'endCaptures': + '0': + 'name': 'keyword.operator.ternary.js' + 'patterns': [ + { + 'include': '#prevent_object_keys_matching' + } + { + 'include': '$self' + } + ] + } + { + 'include': '#operators' + } + { + 'include': '#method_calls' + } + { + 'include': '#function_calls' + } + { + 'include': '#numbers' + } + { + 'include': '#objects' + } + { + 'include': '#properties' + } + { + 'match': '((?>=|>>>=|\\|=' + 'name': 'keyword.operator.assignment.compound.bitwise.js' + } + { + 'match': '<<|>>>|>>' + 'name': 'keyword.operator.bitwise.shift.js' + } + { + 'match': '!==|!=|<=|>=|===|==|<|>' + 'name': 'keyword.operator.comparison.js' + } + { + 'match': '&&|!!|!|\\|\\|' + 'name': 'keyword.operator.logical.js' + } + { + 'match': '&|\\||\\^|~' + 'name': 'keyword.operator.bitwise.js' + } + { + 'match': '=|:' + 'name': 'keyword.operator.assignment.js' + } + { + 'match': '--' + 'name': 'keyword.operator.decrement.js' + } + { + 'match': '\\+\\+' + 'name': 'keyword.operator.increment.js' + } + { + 'match': '%|\\*|/|-|\\+' + 'name': 'keyword.operator.js' + } + ] + 'strings': + 'patterns': [ + { + 'begin': '\'' + 'beginCaptures': + '0': + 'name': 'punctuation.definition.string.begin.js' + 'end': '\'' + 'endCaptures': + '0': + 'name': 'punctuation.definition.string.end.js' + 'name': 'string.quoted.single.js' 'patterns': [ { - 'match': '\\G[$_a-zA-Z][$_a-zA-Z0-9]*' + 'include': '#string_escapes' + } + ] + } + { + 'begin': '"' + 'beginCaptures': + '0': + 'name': 'punctuation.definition.string.begin.js' + 'end': '"' + 'endCaptures': + '0': + 'name': 'punctuation.definition.string.end.js' + 'name': 'string.quoted.double.js' + 'patterns': [ + { + 'include': '#string_escapes' + } + ] + } + { + 'begin': '((\\w+)?(html|HTML|Html))\\s*(`)' + 'beginCaptures': + '1': + 'name': 'entity.name.function.js' + '4': + 'name': 'punctuation.definition.string.begin.js' + 'end': '`' + 'endCaptures': + '0': + 'name': 'punctuation.definition.string.end.js' + 'name': 'string.quoted.template.html.js' + 'patterns': [ + { + 'include': '#string_escapes' + } + { + 'include': '#interpolated_js' + } + { + 'include': 'text.html.basic' + } + ] + } + { + 'begin': '(?<=innerHTML)\\s*(\\+?=)\\s*(?=`)' + 'beginCaptures': + '1': + 'name': 'keyword.operator.assignment.js' + 'end': '(?<=`)' + 'endCaptures': + '0': + 'name': 'punctuation.definition.string.end.js' + 'contentName': 'string.quoted.template.html.js' + 'patterns': [ + { + 'begin': '`' + 'beginCaptures': + '0': + 'name': 'punctuation.definition.string.begin.js' + 'end': '`' + 'endCaptures': + '0': + 'name': 'punctuation.definition.string.end.js' + 'patterns': [ + { + 'include': '#string_escapes' + } + { + 'include': '#interpolated_js' + } + { + 'include': 'text.html.basic' + } + ] + } + ] + } + { + 'begin': '(Relay\\.QL|gql)\\s*(`)' + 'beginCaptures': + '1': + 'name': 'entity.name.function.js' + '2': + 'name': 'punctuation.definition.string.begin.js' + 'end': '`' + 'endCaptures': + '0': + 'name': 'punctuation.definition.string.end.js' + 'name': 'string.quoted.template.graphql.js' + 'patterns': [ + { + 'include': '#string_escapes' + } + { + 'include': '#interpolated_js' + } + { + 'include': 'source.graphql' + } + ] + } + { + 'begin': '(sql|SQL|Sql)\\s*(`)' + 'beginCaptures': + '1': + 'name': 'entity.name.function.js' + '2': + 'name': 'punctuation.definition.string.begin.js' + 'end': '`' + 'endCaptures': + '0': + 'name': 'punctuation.definition.string.end.js' + 'name': 'string.quoted.template.sql.js' + 'patterns': [ + { + 'include': '#string_escapes' + } + { + 'include': '#interpolated_js' + } + { + 'include': 'source.sql' + } + ] + } + { + 'begin': '`' + 'beginCaptures': + '0': + 'name': 'punctuation.definition.string.begin.js' + 'end': '`' + 'endCaptures': + '0': + 'name': 'punctuation.definition.string.end.js' + 'name': 'string.quoted.template.js' + 'patterns': [ + { + 'include': '#string_escapes' + } + { + 'include': '#interpolated_js' + } + ] + } + ] + 'string_escapes': + 'patterns': [ + { + 'match': '\\\\u(?![A-Fa-f0-9]{4}|{[A-Fa-f0-9]+})[^\'"]*' + 'name': 'invalid.illegal.unicode-escape.js' + } + { + 'match': '\\\\u(?:[A-Fa-f0-9]{4}|({)([A-Fa-f0-9]+)(}))' + 'name': 'constant.character.escape.js' + 'captures': + '1': + 'name': 'punctuation.definition.unicode-escape.begin.bracket.curly.js' + '2': + 'patterns': [ + { + # Max codepoint: \u{10FFFF} + 'match': '[A-Fa-f\\d]{7,}|(?!10)[A-Fa-f\\d]{6}' + 'name': 'invalid.illegal.unicode-escape.js' + } + ] + '3': + 'name': 'punctuation.definition.unicode-escape.end.bracket.curly.js' + } + { + 'match': '\\\\(x[0-9A-Fa-f]{2}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.)' + 'name': 'constant.character.escape.js' + } + ] + 'function_params': + 'patterns': [ + { + 'begin': '\\(' + 'beginCaptures': + '0': + 'name': 'punctuation.definition.parameters.begin.bracket.round.js' + 'end': '\\)' + 'endCaptures': + '0': + 'name': 'punctuation.definition.parameters.end.bracket.round.js' + 'name': 'meta.parameters.js' + 'patterns': [ + { + 'match': '(\\.\\.\\.)([a-zA-Z_$][\\w$]*)' + 'captures': + '1': + 'name': 'keyword.operator.spread.js' + '2': + 'name': 'variable.parameter.rest.function.js' + } + { + 'include': '$self' + } + { + 'match': '[a-zA-Z_$][\\w$]*' 'name': 'variable.parameter.function.js' } + ] + } + ] + 'function_body': + 'patterns': [ + { + 'begin': '{' + 'beginCaptures': + '0': + 'name': 'punctuation.definition.function.body.begin.bracket.curly.js' + 'end': '}' + 'endCaptures': + '0': + 'name': 'punctuation.definition.function.body.end.bracket.curly.js' + 'patterns': [ { 'include': '$self' } ] } ] + 'function_innards': + 'patterns': [ + { + 'match': '(?:\\b(async)\\b\\s*)?\\b(function)\\b(?:\\s*(\\*))?' + 'captures': + '1': + 'name': 'storage.modifier.async.js' + '2': + 'name': 'storage.type.function.js' + '3': + 'name': 'storage.modifier.generator.js' + } + { + 'match': '[a-zA-Z_$][\\w$]*(?=\\s*\\()' + 'name': 'entity.name.function.js' + } + { + 'include': '#function_params' + } + { + 'include': '#comments' + } + ] + 'arrow_function_innards': + 'patterns': [ + { + 'match': '=>' + 'name': 'storage.type.function.arrow.js' + } + { + 'include': '#function_params' + } + { + 'match': '([a-zA-Z_$][\\w$]*)(?=\\s*=>)' + 'captures': + '0': + 'name': 'meta.parameters.js' + '1': + 'name': 'variable.parameter.function.js' + } + { + 'match': '(\\d[\\w$]*)' + 'captures': + '0': + 'name': 'meta.parameters.js' + '1': + 'name': 'invalid.illegal.identifier.js' + } + ] + 'arguments': + 'patterns': [ + { + 'begin': '\\(' + 'beginCaptures': + '0': + 'name': 'punctuation.definition.arguments.begin.bracket.round.js' + 'end': '\\)' + 'endCaptures': + '0': + 'name': 'punctuation.definition.arguments.end.bracket.round.js' + 'name': 'meta.arguments.js' + 'patterns': [ + { + 'include': '$self' + } + ] + } + ] + 'method_calls': + 'patterns': [ + { + # .methodCall(arg1, "arg2", [...]) + 'begin': '(\\.)\\s*([\\w$]+)\\s*(?=\\()' + 'beginCaptures': + '1': + 'name': 'meta.delimiter.method.period.js' + '2': + 'patterns': [ + { + 'match': '''(?x) + \\bon(Rowsinserted|Rowsdelete|Rowenter|Rowexit|Resize|Resizestart|Resizeend|Reset| + Readystatechange|Mouseout|Mouseover|Mousedown|Mouseup|Mousemove| + Before(cut|deactivate|unload|update|paste|print|editfocus|activate)| + Blur|Scrolltop|Submit|Select|Selectstart|Selectionchange|Hover|Help| + Change|Contextmenu|Controlselect|Cut|Cellchange|Clock|Close|Deactivate| + Datasetchanged|Datasetcomplete|Dataavailable|Drop|Drag|Dragstart|Dragover| + Dragdrop|Dragenter|Dragend|Dragleave|Dblclick|Unload|Paste|Propertychange|Error| + Errorupdate|Keydown|Keyup|Keypress|Focus|Load|Activate|Afterupdate|Afterprint|Abort)\\b + ''' + 'name': 'support.function.event-handler.js' + } + { + 'match': '''(?x) + \\b(catch|finally|then|shift|showModelessDialog|showModalDialog|showHelp|scroll|scrollX|scrollByPages| + scrollByLines|scrollY|scrollTo|stop|strike|sizeToContent|sidebar|signText|sort| + sup|sub|substr|substring|splice|split|send|set(Milliseconds|Seconds|Minutes|Hours| + Month|Year|FullYear|Date|UTC(Milliseconds|Seconds|Minutes|Hours|Month|FullYear|Date)| + Time|Hotkeys|Cursor|ZOptions|Active|Resizable|RequestHeader)|search|slice| + savePreferences|small|home|handleEvent|navigate|char|charCodeAt|charAt|concat| + contextual|confirm|compile|clear|captureEvents|call|createStyleSheet|createPopup| + createEventObject|to(GMTString|UTCString|String|Source|UpperCase|LowerCase|LocaleString)| + test|taint|taintEnabled|indexOf|italics|disableExternalCapture|dump|detachEvent|unshift| + untaint|unwatch|updateCommands|join|javaEnabled|pop|push|plugins.refresh|paddings|parse|stringify| + print|prompt|preference|enableExternalCapture|exec|execScript|valueOf|UTC|find|file| + fileModifiedDate|fileSize|fileCreatedDate|fileUpdatedDate|fixed|fontsize|fontcolor| + forward|fromCharCode|watch|link|load|lastIndexOf|anchor|attachEvent|atob|apply|alert| + abort|routeEvents|resize|resizeBy|resizeTo|recalc|returnValue|replace|reverse|reload| + releaseCapture|releaseEvents|go|get(Milliseconds|Seconds|Minutes|Hours|Month|Day|Year|FullYear| + Time|Date|TimezoneOffset|UTC(Milliseconds|Seconds|Minutes|Hours|Day|Month|FullYear|Date)| + Attention|Selection|ResponseHeader|AllResponseHeaders)|moveBy|moveBelow|moveTo| + moveToAbsolute|moveAbove|mergeAttributes|match|margins|btoa|big|bold|borderWidths|blink|back)\\b + ''' + 'name': 'support.function.js' + } + { + 'match': '''(?x) + \\b(acceptNode|add|addEventListener|addTextTrack|adoptNode|after|animate|append| + appendChild|appendData|before|blur|canPlayType|captureStream| + caretPositionFromPoint|caretRangeFromPoint|checkValidity|clear|click| + cloneContents|cloneNode|cloneRange|close|closest|collapse| + compareBoundaryPoints|compareDocumentPosition|comparePoint|contains| + convertPointFromNode|convertQuadFromNode|convertRectFromNode|createAttribute| + createAttributeNS|createCaption|createCDATASection|createComment| + createContextualFragment|createDocument|createDocumentFragment| + createDocumentType|createElement|createElementNS|createEntityReference| + createEvent|createExpression|createHTMLDocument|createNodeIterator| + createNSResolver|createProcessingInstruction|createRange|createShadowRoot| + createTBody|createTextNode|createTFoot|createTHead|createTreeWalker|delete| + deleteCaption|deleteCell|deleteContents|deleteData|deleteRow|deleteTFoot| + deleteTHead|detach|disconnect|dispatchEvent|elementFromPoint|elementsFromPoint| + enableStyleSheetsForSet|entries|evaluate|execCommand|exitFullscreen| + exitPointerLock|expand|extractContents|fastSeek|firstChild|focus|forEach|get| + getAll|getAnimations|getAttribute|getAttributeNames|getAttributeNode| + getAttributeNodeNS|getAttributeNS|getBoundingClientRect|getBoxQuads| + getClientRects|getContext|getDestinationInsertionPoints|getElementById| + getElementsByClassName|getElementsByName|getElementsByTagName| + getElementsByTagNameNS|getItem|getNamedItem|getSelection|getStartDate| + getVideoPlaybackQuality|has|hasAttribute|hasAttributeNS|hasAttributes| + hasChildNodes|hasFeature|hasFocus|importNode|initEvent|insertAdjacentElement| + insertAdjacentHTML|insertAdjacentText|insertBefore|insertCell|insertData| + insertNode|insertRow|intersectsNode|isDefaultNamespace|isEqualNode| + isPointInRange|isSameNode|item|key|keys|lastChild|load|lookupNamespaceURI| + lookupPrefix|matches|move|moveAttribute|moveAttributeNode|moveChild| + moveNamedItem|namedItem|nextNode|nextSibling|normalize|observe|open| + parentNode|pause|play|postMessage|prepend|preventDefault|previousNode| + previousSibling|probablySupportsContext|queryCommandEnabled| + queryCommandIndeterm|queryCommandState|queryCommandSupported|queryCommandValue| + querySelector|querySelectorAll|registerContentHandler|registerElement| + registerProtocolHandler|releaseCapture|releaseEvents|remove|removeAttribute| + removeAttributeNode|removeAttributeNS|removeChild|removeEventListener| + removeItem|replace|replaceChild|replaceData|replaceWith|reportValidity| + requestFullscreen|requestPointerLock|reset|scroll|scrollBy|scrollIntoView| + scrollTo|seekToNextFrame|select|selectNode|selectNodeContents|set|setAttribute| + setAttributeNode|setAttributeNodeNS|setAttributeNS|setCapture| + setCustomValidity|setEnd|setEndAfter|setEndBefore|setItem|setNamedItem| + setRangeText|setSelectionRange|setSinkId|setStart|setStartAfter|setStartBefore| + slice|splitText|stepDown|stepUp|stopImmediatePropagation|stopPropagation| + submit|substringData|supports|surroundContents|takeRecords|terminate|toBlob| + toDataURL|toggle|toString|values|write|writeln)\\b + ''' + 'name': 'support.function.dom.js' + } + { + 'match': "[a-zA-Z_$][\\w$]*" + 'name': 'entity.name.function.js' + } + { + 'match': '\\d[\\w$]*' + 'name': 'invalid.illegal.identifier.js' + } + ] + 'end': '(?<=\\))' + 'name': 'meta.method-call.js' + 'patterns': [ + { + 'include': '#arguments' + } + ] + } + ] + 'function_calls': + 'patterns': [ + { + # functionCall(arg1, "arg2", [...]) + 'begin': '([\\w$]+)\\s*(?=\\()' + 'beginCaptures': + '1': + 'patterns': [ + { + 'match': '''(?x) + \\b(isNaN|isFinite|eval|uneval|parseInt|parseFloat|decodeURI| + decodeURIComponent|encodeURI|encodeURIComponent|escape|unescape| + require|set(Interval|Timeout)|clear(Interval|Timeout))\\b + ''' + 'name': 'support.function.js' + } + { + 'match': "[a-zA-Z_$][\\w$]*" + 'name': 'entity.name.function.js' + } + { + 'match': '\\d[\\w$]*' + 'name': 'invalid.illegal.identifier.js' + } + ] + 'end': '(?<=\\))' + 'name': 'meta.function-call.js' + 'patterns': [ + { + 'include': '#arguments' + } + ] + } + ] + 'objects': + 'patterns': [ + { + # OBJ in OBJ.prop, OBJ.methodCall() + 'match': '[A-Z][A-Z0-9_$]*(?=\\s*\\.\\s*[a-zA-Z_$]\\w*)' + 'name': 'constant.other.object.js' + } + { + # obj in obj.prop, obj.methodCall() + 'match': '[a-zA-Z_$][\\w$]*(?=\\s*\\.\\s*[a-zA-Z_$]\\w*)' + 'name': 'variable.other.object.js' + } + ] + 'properties': + 'patterns': [ + { + # PROP1 in obj.PROP1.prop2, func().PROP1.prop2 + 'match': '(\\.)\\s*([A-Z][A-Z0-9_$]*\\b\\$*)(?=\\s*\\.\\s*[a-zA-Z_$]\\w*)' + 'captures': + '1': + 'name': 'meta.delimiter.property.period.js' + '2': + 'name': 'constant.other.object.property.js' + } + { + # prop1 in obj.prop1.prop2, func().prop1.prop2 + 'match': '(\\.)\\s*(\\$*[a-zA-Z_$][\\w$]*)(?=\\s*\\.\\s*[a-zA-Z_$]\\w*)' + 'captures': + '1': + 'name': 'meta.delimiter.property.period.js' + '2': + 'name': 'variable.other.object.property.js' + } + { + # PROP in obj.PROP, func().PROP + 'match': '(\\.)\\s*([A-Z][A-Z0-9_$]*\\b\\$*)' + 'captures': + '1': + 'name': 'meta.delimiter.property.period.js' + '2': + 'name': 'constant.other.property.js' + } + { + # prop in obj.prop, func().prop + 'match': '(\\.)\\s*(\\$*[a-zA-Z_$][\\w$]*)' + 'captures': + '1': + 'name': 'meta.delimiter.property.period.js' + '2': + 'name': 'variable.other.property.js' + } + { + # 123illegal in obj.123illegal, func().123illegal + 'match': '(\\.)\\s*([0-9][\\w$]*)' + 'captures': + '1': + 'name': 'meta.delimiter.property.period.js' + '2': + 'name': 'invalid.illegal.identifier.js' + } + ] + 'interpolated_js': + 'patterns': [ + { + 'begin': '\\${' + 'captures': + '0': + 'name': 'punctuation.section.embedded.js' + 'end': '}' + 'name': 'source.js.embedded.source' + 'patterns': [ + { + 'begin': '{' + 'beginCaptures': + '0': + 'name': 'meta.brace.curly.js' + 'end': '}' + 'endCaptures': + '0': + 'name': 'meta.brace.curly.js' + 'patterns': [ + { + 'include': '$self' + } + ] + } + { + 'include': '$self' + } + ] + } + ] + 'comments': + 'patterns': [ + { + 'captures': + '1': + 'name': 'punctuation.definition.comment.begin.js' + '2': + 'name': 'punctuation.definition.comment.end.js' + 'match': '(/\\*)(\\*/)' + 'name': 'comment.block.empty.js' + } + { + 'begin': '/\\*\\*' + 'beginCaptures': + '0': + 'name': 'punctuation.definition.comment.begin.js' + 'end': '\\*/' + 'endCaptures': + '0': + 'name': 'punctuation.definition.comment.end.js' + 'name': 'comment.block.documentation.js' + 'patterns': [ + { + 'include': 'source.jsdoc' + } + ] + } + { + 'begin': '/\\*' + 'beginCaptures': + '0': + 'name': 'punctuation.definition.comment.begin.js' + 'end': '\\*/' + 'endCaptures': + '0': + 'name': 'punctuation.definition.comment.end.js' + 'name': 'comment.block.js' + } + { + 'begin': '//' + 'beginCaptures': + '0': + 'name': 'punctuation.definition.comment.js' + 'end': '$' + 'name': 'comment.line.double-slash.js' + } + ] + 'switch_statement': + 'patterns': [ + { + # switch(expression) {...} + 'begin': '\\bswitch\\b' + 'beginCaptures': + '0': + 'name': 'keyword.control.switch.js' + 'end': '}' + 'endCaptures': + '0': + 'name': 'punctuation.definition.section.switch-block.end.bracket.curly.js' + 'name': 'meta.switch-statement.js' + 'patterns': [ + { + 'begin': '\\(' + 'beginCaptures': + '0': + 'name': 'punctuation.definition.switch-expression.begin.bracket.round.js' + 'end': '\\)' + 'endCaptures': + '0': + 'name': 'punctuation.definition.switch-expression.end.bracket.round.js' + 'patterns': [ + 'include': '$self' + ] + } + { + 'begin': '{' + 'beginCaptures': + '0': + 'name': 'punctuation.definition.section.switch-block.begin.bracket.curly.js' + 'end': '(?=})' + 'patterns': [ + { + 'begin': '\\bcase\\b' + 'beginCaptures': + '0': + 'name': 'keyword.control.case.js' + 'end': ':' + 'endCaptures': + '0': + 'name': 'punctuation.definition.section.case-statement.js' + 'patterns': [ + { + 'include': '#prevent_object_keys_matching' + } + { + 'include': '$self' + } + ] + } + { + 'match': '(?:^\\s*)?\\b(default)\\b\\s*(:)' + 'captures': + '1': + 'name': 'keyword.control.default.js' + '2': + 'name': 'punctuation.definition.section.case-statement.js' + } + { + 'include': '$self' + } + ] + } + ] + } + ] + 'prevent_object_keys_matching': + 'patterns': [ + { + # e.g. don't treat null as an object key in + # ? null : + # case null: + 'match': '(\\w+)(?=\\s*:)' + 'captures': + '1': + 'patterns': [ + 'include': '$self' + ] + } + ] diff --git a/grammars/jsdoc.cson b/grammars/jsdoc.cson new file mode 100644 index 00000000..f199cacf --- /dev/null +++ b/grammars/jsdoc.cson @@ -0,0 +1,524 @@ +'scopeName': 'source.jsdoc' +'name': 'JSDoc' +'patterns': [ + { + # @access private|protected|public + 'captures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + '3': + 'name': 'constant.language.access-type.jsdoc' + 'match': '''(?x) + ((@)(?:access|api)) + \\s+ + (private|protected|public) + \\b + ''' + } + { + # @author name [] + 'match': '''(?x) + ((@)author) + \\s+ + ( + [^@\\s<>*/] + (?:[^@<>*/]|\\*[^/])* + ) + (?: + \\s* + (<) + ([^>\\s]+) + (>) + )? + ''' + 'captures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + '3': + 'name': 'entity.name.type.instance.jsdoc' + '4': + 'name': 'punctuation.definition.bracket.angle.begin.jsdoc' + '5': + 'name': 'constant.other.email.link.underline.jsdoc' + '6': + 'name': 'punctuation.definition.bracket.angle.end.jsdoc' + } + { + # @borrows as + 'match': '''(?x) + ((@)borrows) \\s+ + ((?:[^@\\s*/]|\\*[^/])+) # + \\s+ (as) \\s+ # as + ((?:[^@\\s*/]|\\*[^/])+) # + ''' + 'captures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + '3': + 'name': 'entity.name.type.instance.jsdoc' + '4': + 'name': 'keyword.operator.control.jsdoc' + '5': + 'name': 'entity.name.type.instance.jsdoc' + } + { + # @example text(); + 'name': 'meta.example.jsdoc' + 'begin': '((@)example)\\s+' + 'end': '(?=@|\\*/)' + 'beginCaptures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + 'patterns': [ + { + # Match to prevent leading asterisk being highlighted as JS + 'match': '^\\s\\*\\s+' + } + { + # Leading … before example + 'begin': '\\G(<)caption(>)' + 'beginCaptures': + '0': + 'name': 'entity.name.tag.inline.jsdoc' + '1': + 'name': 'punctuation.definition.bracket.angle.begin.jsdoc' + '2': + 'name': 'punctuation.definition.bracket.angle.end.jsdoc' + 'contentName': 'constant.other.description.jsdoc' + 'end': '()|(?=\\*/)' + 'endCaptures': + '0': + 'name': 'entity.name.tag.inline.jsdoc' + '1': + 'name': 'punctuation.definition.bracket.angle.begin.jsdoc' + '2': + 'name': 'punctuation.definition.bracket.angle.end.jsdoc' + } + { + # Highlighted JavaScript example + 'match': '[^\\s@*](?:[^*]|\\*[^/])*' + 'captures': + '0': + 'name': 'source.embedded.js' + 'patterns': [ + { + 'include': 'source.js' + } + ] + } + ] + } + { + # @kind type + 'captures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + '3': + 'name': 'constant.language.symbol-type.jsdoc' + 'match': '''(?x) + ((@)kind) + \\s+ + (class|constant|event|external|file|function|member|mixin|module|namespace|typedef) + \\b + ''' + } + { + # @see namepathOrURL + 'captures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + '3': + 'name': 'variable.other.link.underline.jsdoc' + '4': + 'name': 'entity.name.type.instance.jsdoc' + 'match': '''(?x) + ((@)see) + \\s+ + (?: + # URL + ( + (?=https?://) + (?:[^\\s*]|\\*[^/])+ + ) + | + # JSDoc namepath + ( + (?! + # Avoid matching bare URIs (also acceptable as links) + https?:// + | + # Avoid matching {@inline tags}; we match those below + (?:\\[[^\\[\\]]*\\])? # Possible description [preceding]{@tag} + {@(?:link|linkcode|linkplain|tutorial)\\b + ) + # Matched namepath + (?:[^@\\s*/]|\\*[^/])+ + ) + ) + ''' + } + { + # @template Foo,Bar + 'captures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + '3': + 'name': 'variable.other.jsdoc' + 'patterns': [ + { + 'match': ',' + 'name': 'punctuation.delimiter.object.comma.jsdoc' + } + ] + 'match': '''(?x) + ((@)template) + \\s+ + # One or more valid identifiers + ( + [A-Za-z_$] # First character: non-numeric word character + [\\w$.\\[\\]]* # Rest of identifier + (?: # Possible list of additional identifiers + \\s* , \\s* + [A-Za-z_$] + [\\w$.\\[\\]]* + )* + ) + ''' + } + { + # Tags followed by an identifier token + # - @ identifier + 'captures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + '3': + 'name': 'variable.other.jsdoc' + 'match': '''(?x) + ( + (@) + (?:arg|argument|const|constant|member|namespace|param|var) + ) + \\s+ + ( + [A-Za-z_$] + [\\w$.\\[\\]]* + ) + ''' + } + { + # Tags followed by a type expression, then a namepath + # - @ {type} namepath + 'begin': '((@)typedef)\\s+(?={)' + 'beginCaptures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + 'end': '(?=\\s|\\*/|[^{}\\[\\]A-Za-z_$])' + 'patterns': [ + { + 'include': '#type' + } + { + 'name': 'entity.name.type.instance.jsdoc' + 'match': '(?:[^@\\s*/]|\\*[^/])+' + } + ] + } + { + # Tags followed by a type expression, then an identifier + # - @ {type} identifier + 'begin': '((@)(?:arg|argument|const|constant|member|namespace|param|prop|property|var))\\s+(?={)' + 'beginCaptures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + 'end': '(?=\\s|\\*/|[^{}\\[\\]A-Za-z_$])' + 'patterns': [ + { + 'include': '#type' + } + { + 'match': '([A-Za-z_$][\\w$.\\[\\]]*)' + 'name': 'variable.other.jsdoc' + } + { + # Optional value + 'name': 'variable.other.jsdoc' + 'match': '''(?x) + (\\[)\\s* + [\\w$]+ + (?: + (?:\\[\\])? # Foo[].bar properties within an array + \\. # Foo.Bar namespaced parameter + [\\w$]+ + )* + (?: + \\s* + (=) # [foo=bar] Default parameter value + \\s* + ( + # The inner regexes are to stop the match early at */ and to not stop at escaped quotes + (?> + "(?:(?:\\*(?!/))|(?:\\\\(?!"))|[^*\\\\])*?" | # [foo="bar"] Double-quoted + '(?:(?:\\*(?!/))|(?:\\\\(?!'))|[^*\\\\])*?' | # [foo='bar'] Single-quoted + \\[ (?:(?:\\*(?!/))|[^*])*? \\] | # [foo=[1,2]] Array literal + (?:(?:\\*(?!/))|\\s(?!\\s*\\])|\\[.*?(?:\\]|(?=\\*/))|[^*\\s\\[\\]])* # Everything else (sorry) + )* + ) + )? + \\s*(?:(\\])((?:[^*\\s]|\\*[^\\s/])+)?|(?=\\*/)) + ''' + 'captures': + '1': + 'name': 'punctuation.definition.optional-value.begin.bracket.square.jsdoc' + '2': + 'name': 'keyword.operator.assignment.jsdoc' + '3': + 'name': 'source.embedded.js' + 'patterns': [ + { + 'include': '#inline-tags' + } + { + 'include': 'source.js' + } + ] + '4': + 'name': 'punctuation.definition.optional-value.end.bracket.square.jsdoc' + '5': + 'name': 'invalid.illegal.syntax.jsdoc' + } + ] + } + { + # Tags followed by a type expression + # - @ {type} + 'begin': '''(?x) + ( + (@) + (?:define|enum|exception|export|extends|lends|implements|modifies + |namespace|private|protected|returns?|suppress|this|throws|type + |yields?) + ) + \\s+(?={) + ''' + 'beginCaptures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + 'end': '(?=\\s|\\*/|[^{}\\[\\]A-Za-z_$])' + 'patterns': [ + { + 'include': '#type' + } + ] + } + { + # Tags followed by a namepath + # - @ namepath + 'captures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + '3': + 'name': 'entity.name.type.instance.jsdoc' + 'match': '''(?x) + ( + (@) + (?:alias|augments|callback|constructs|emits|event|fires|exports? + |extends|external|function|func|host|lends|listens|interface|memberof!? + |method|module|mixes|mixin|name|requires|see|this|typedef|uses) + ) + \\s+ + ( + (?: + [^{}@\\s*] | \\*[^/] + )+ + ) + ''' + } + { + # Tags followed by a quoted arbitrary value + # - @ "Quoted value" + 'begin': '((@)(?:default(?:value)?|license|version))\\s+(([\'"]))' + 'beginCaptures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + '3': + 'name': 'variable.other.jsdoc' + '4': + 'name': 'punctuation.definition.string.begin.jsdoc' + 'contentName': 'variable.other.jsdoc' + 'end': '(\\3)|(?=$|\\*/)' + 'endCaptures': + '0': + 'name': 'variable.other.jsdoc' + '1': + 'name': 'punctuation.definition.string.end.jsdoc' + } + { + # Tags followed by an arbitrary value + # - @ value + 'captures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + '3': + 'name': 'variable.other.jsdoc' + 'match': '((@)(?:default(?:value)?|license|tutorial|variation|version))\\s+([^\\s*]+)' + } + { + # Tags without arguments, or a tag without expected arguments. Because JSDoc permits + # tags to be spread across lines, we should at least highlight the opening tag for + # stuff like this: + # + # /** + # * @param + # * {type} + # * name + 'captures': + '1': + 'name': 'punctuation.definition.block.tag.jsdoc' + 'match': '''(?x) (@) + (?:abstract|access|alias|api|arg|argument|async|attribute|augments|author|beta|borrows|bubbles + |callback|chainable|class|classdesc|code|config|const|constant|constructor|constructs|copyright + |default|defaultvalue|define|deprecated|desc|description|dict|emits|enum|event|example|exception + |exports?|extends|extension(?:_?for)?|external|externs|file|fileoverview|final|fires|for|func + |function|generator|global|hideconstructor|host|ignore|implements|implicitCast|inherit[Dd]oc + |inner|instance|interface|internal|kind|lends|license|listens|main|member|memberof!?|method + |mixes|mixins?|modifies|module|name|namespace|noalias|nocollapse|nocompile|nosideeffects + |override|overview|package|param|polymer(?:Behavior)?|preserve|private|prop|property|protected + |public|read[Oo]nly|record|require[ds]|returns?|see|since|static|struct|submodule|summary + |suppress|template|this|throws|todo|tutorial|type|typedef|unrestricted|uses|var|variation + |version|virtual|writeOnce|yields?) + \\b + ''' + 'name': 'storage.type.class.jsdoc' + } + { + 'include': '#inline-tags' + } +] + +'repository': + # Balanced brackets (square or curly) + 'brackets': + 'patterns': [ + { + 'begin': '{' + 'end': '}|(?=\\*/)' + 'patterns': [ + { + 'include': '#brackets' + } + ] + } + { + 'begin': '\\[' + 'end': '\\]|(?=\\*/)' + 'patterns': [ + { + 'include': '#brackets' + } + ] + } + ] + 'inline-tags': + 'patterns': [ + { + # Description preceding {@inline tag} + 'captures': + '1': + 'name': 'punctuation.definition.bracket.square.begin.jsdoc' + '2': + 'name': 'punctuation.definition.bracket.square.end.jsdoc' + 'match': '(\\[)[^\\]]+(\\])(?={@(?:link|linkcode|linkplain|tutorial))' + 'name': 'constant.other.description.jsdoc' + } + { + # {@link|@tutorial …} + 'begin': '({)((@)(?:link(?:code|plain)?|tutorial))\\s*' + 'beginCaptures': + '1': + 'name': 'punctuation.definition.bracket.curly.begin.jsdoc' + '2': + 'name': 'storage.type.class.jsdoc' + '3': + 'name': 'punctuation.definition.inline.tag.jsdoc' + 'end': '}|(?=\\*/)' + 'endCaptures': + '0': + 'name': 'punctuation.definition.bracket.curly.end.jsdoc' + 'name': 'entity.name.type.instance.jsdoc' + 'patterns': [ + { + 'captures': + '1': + 'name': 'variable.other.link.underline.jsdoc' + '2': + 'name': 'punctuation.separator.pipe.jsdoc' + 'match': '\\G((?=https?://)(?:[^|}\\s*]|\\*[/])+)(\\|)?' + } + { + 'captures': + '1': + 'name': 'variable.other.description.jsdoc' + '2': + 'name': 'punctuation.separator.pipe.jsdoc' + 'match': '\\G((?:[^{}@\\s|*]|\\*[^/])+)(\\|)?' + } + ] + } + ] + # {type} + 'type': + 'patterns': [ + { + # {unclosed + 'match': '\\G{(?:[^}*]|\\*[^/}])+$' + 'name': 'invalid.illegal.type.jsdoc' + } + { + 'begin': '\\G({)' + 'beginCaptures': + '0': + 'name': 'entity.name.type.instance.jsdoc' + '1': + 'name': 'punctuation.definition.bracket.curly.begin.jsdoc' + 'contentName': 'entity.name.type.instance.jsdoc' + 'end': '((}))\\s*|(?=\\*/)' + 'endCaptures': + '1': + 'name': 'entity.name.type.instance.jsdoc' + '2': + 'name': 'punctuation.definition.bracket.curly.end.jsdoc' + 'patterns': [ + { + 'include': '#brackets' + } + ] + } + ] diff --git a/grammars/regular expression replacement (javascript).cson b/grammars/regular expression replacement (javascript).cson new file mode 100644 index 00000000..eeaa8d70 --- /dev/null +++ b/grammars/regular expression replacement (javascript).cson @@ -0,0 +1,22 @@ +'scopeName': 'source.js.regexp.replacement' +'patterns': [ + { + 'include': '#regexp-replacement' + } +] +'repository': + 'regexp-replacement': + 'patterns': [ + { + 'match': '\\$([1-9][0-9]|[1-9]|0[1-9]|[&`\'])' + 'name': 'variable.regexp.replacement' + } + { + 'match': '\\$\\$' + 'name': 'constant.character.escape.dollar.regexp.replacement' + } + { + 'match': '\\\\[^$]' + 'name': 'constant.character.escape.backslash.regexp.replacement' + } + ] diff --git a/grammars/regular expressions (javascript).cson b/grammars/regular expressions (javascript).cson index 522d502c..1de6f54b 100644 --- a/grammars/regular expressions (javascript).cson +++ b/grammars/regular expressions (javascript).cson @@ -1,6 +1,4 @@ -'fileTypes': [] -'hideFromUser': true -'name': 'Regular Expressions (JavaScript)' +'scopeName': 'source.js.regexp' 'patterns': [ { 'include': '#regexp' @@ -10,11 +8,11 @@ 'regex-character-class': 'patterns': [ { - 'match': '\\\\[wWsSdD]|\\.' + 'match': '\\\\[wWsSdDtrnvf]|\\.' 'name': 'constant.character.character-class.regexp' } { - 'match': '\\\\([0-7]{3}|x\\h\\h|u\\h\\h\\h\\h)' + 'match': '\\\\([0-7]{3}|x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4})' 'name': 'constant.character.numeric.regexp' } { @@ -33,7 +31,7 @@ 'name': 'keyword.control.anchor.regexp' } { - 'match': '\\\\[1-9]\\d*' + 'match': '\\\\[1-9]\\d*|\\\\k<[a-zA-Z_$][\\w$]*>' 'name': 'keyword.other.back-reference.regexp' } { @@ -45,14 +43,18 @@ 'name': 'keyword.operator.or.regexp' } { - 'begin': '(\\()((\\?=)|(\\?!))' + 'begin': '(\\()(?:(\\?=)|(\\?!)|(\\?<=)|(\\?))?' 'beginCaptures': '0': 'name': 'punctuation.definition.group.regexp' @@ -107,7 +109,7 @@ 'name': 'constant.character.control.regexp' '6': 'name': 'constant.character.escape.backslash.regexp' - 'match': '(?:.|(\\\\(?:[0-7]{3}|x\\h\\h|u\\h\\h\\h\\h))|(\\\\c[A-Z])|(\\\\.))\\-(?:[^\\]\\\\]|(\\\\(?:[0-7]{3}|x\\h\\h|u\\h\\h\\h\\h))|(\\\\c[A-Z])|(\\\\.))' + 'match': '(?:.|(\\\\(?:[0-7]{3}|x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}))|(\\\\c[A-Z])|(\\\\.))\\-(?:[^\\]\\\\]|(\\\\(?:[0-7]{3}|x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}))|(\\\\c[A-Z])|(\\\\.))' 'name': 'constant.other.character-class.range.regexp' } { @@ -119,4 +121,3 @@ 'include': '#regex-character-class' } ] -'scopeName': 'source.js.regexp' diff --git a/grammars/tree-sitter-javascript.cson b/grammars/tree-sitter-javascript.cson new file mode 100644 index 00000000..86919162 --- /dev/null +++ b/grammars/tree-sitter-javascript.cson @@ -0,0 +1,262 @@ +name: 'JavaScript' +scopeName: 'source.js' +type: 'tree-sitter' +parser: 'tree-sitter-javascript' + +fileTypes: ['js', 'jsx'] + +injectionRegex: '^js$|^JS$|javascript|JavaScript' + +firstLineRegex: [ + # shebang line + '^#!.*\\b(node)\\r?\\n' + + # vim modeline + 'vim\\b.*\\bset\\b.*\\b(filetype|ft|syntax)=(js|javascript)' +] + +folds: [ + { + type: 'comment' + } + { + type: ['jsx_element', 'template_string'], + start: {index: 0} + end: {index: -1} + } + { + type: 'jsx_self_closing_element' + start: {index: 1} + end: {index: -2} + } + { + start: {index: 0, type: '{'} + end: {index: -1, type: '}'} + } + { + start: {index: 0, type: '['} + end: {index: -1, type: ']'} + } + { + start: {index: 0, type: '('} + end: {index: -1, type: ')'} + } + { + type: ['switch_case', 'switch_default'] + start: {index: 0} + end: {type: 'break_statement', index: -1} + } + { + type: ['switch_case', 'switch_default'] + start: {index: 0} + } +] + +comments: + start: '// ' + +scopes: + 'program': 'source.js' + + 'property_identifier': [ + { + match: '^[\$A-Z_]+$', + scopes: 'constant.other.property.js' + } + + 'variable.other.object.property' + ] + + 'member_expression > property_identifier': 'variable.other.object.property.unquoted' + + 'formal_parameters > identifier': 'variable.parameter.function' + 'formal_parameters > rest_parameter > identifier': 'variable.parameter.rest.function' + + 'shorthand_property_identifier': [ + { + match: '^[\$A-Z_]{2,}$', + scopes: 'constant.other' + } + ] + + ' + class > identifier, + new_expression > identifier + ': 'meta.class' + + ' + jsx_opening_element > identifier, + jsx_closing_element > identifier, + jsx_self_closing_element > identifier + ': [ + { + match: '^[A-Z]', + scopes: 'meta.class.component.jsx' + } + ] + + 'call_expression > identifier': {match: '^[A-Z]', scopes: 'meta.class'} + 'arrow_function > identifier:nth-child(0)': 'variable.parameter.function' + + 'function > identifier': 'entity.name.function' + 'function_declaration > identifier': 'entity.name.function' + 'generator_function > identifier': 'entity.name.function' + + 'call_expression > identifier': [ + {match: '^require$', scopes: 'support.function'}, + 'entity.name.function' + ] + + 'call_expression > super': 'support.function.super' + + 'method_definition > property_identifier': 'entity.name.function' + 'call_expression > member_expression > property_identifier': 'entity.name.function' + + 'identifier': [ + { + match: '^(global|globalThis|module|exports|__filename|__dirname)$', + scopes: 'support.variable' + }, + { + match: '^(window|self|frames|event|document|performance|screen|navigator|console)$' + scopes: 'support.variable.dom' + }, + { + exact: 'require', + scopes: 'support.function' + }, + { + match: '^[\$A-Z_]{2,}$', + scopes: 'constant.other' + }, + { + match: '^[A-Z]', + scopes: 'meta.class' + }, + ] + + 'number': 'constant.numeric' + 'string': 'string.quoted' + 'regex': 'string.regexp' + 'escape_sequence': 'constant.character.escape' + 'template_string': 'string.quoted.template' + 'undefined': 'constant.language' + 'null': 'constant.language.null' + 'true': 'constant.language.boolean.true' + 'false': 'constant.language.boolean.false' + 'comment': [ + { + match: "^//", + scopes: 'comment.line' + }, + 'comment.block' + ] + 'hash_bang_line': 'comment.block' + + ' + jsx_expression > "{", + jsx_expression > "}", + template_substitution > "${", + template_substitution > "}" + ': 'punctuation.section.embedded' + 'template_substitution': 'embedded.source' + + '"("': 'punctuation.definition.parameters.begin.bracket.round' + '")"': 'punctuation.definition.parameters.end.bracket.round' + '"{"': 'punctuation.definition.function.body.begin.bracket.curly' + '"}"': 'punctuation.definition.function.body.end.bracket.curly' + '";"': 'punctuation.terminator.statement.semicolon' + '"["': 'punctuation.definition.array.begin.bracket.square' + '"]"': 'punctuation.definition.array.end.bracket.square' + + '"var"': 'storage.type' + '"let"': 'storage.type' + '"class"': 'storage.type' + '"extends"': 'storage.modifier' + '"const"': 'storage.modifier' + '"static"': 'storage.modifier' + '"function"': 'storage.type.function' + '"=>"': 'storage.type.function.arrow' + + '"="': 'keyword.operator.js' + '"+="': 'keyword.operator.js' + '"-="': 'keyword.operator.js' + '"*="': 'keyword.operator.js' + '"/="': 'keyword.operator.js' + '"%="': 'keyword.operator.js' + '"<<="': 'keyword.operator.js' + '">>="': 'keyword.operator.js' + '">>>="': 'keyword.operator.js' + '"&="': 'keyword.operator.js' + '"^="': 'keyword.operator.js' + '"|="': 'keyword.operator.js' + '"!"': 'keyword.operator.js' + '"+"': 'keyword.operator.js' + '"-"': 'keyword.operator.js' + '"*"': 'keyword.operator.js' + '"/"': 'keyword.operator.js' + '"%"': 'keyword.operator.js' + '"=="': 'keyword.operator.js' + '"==="': 'keyword.operator.js' + '"!="': 'keyword.operator.js' + '"!=="': 'keyword.operator.js' + '">="': 'keyword.operator.js' + '"<="': 'keyword.operator.js' + '">"': 'keyword.operator.js' + '"<"': 'keyword.operator.js' + '":"': 'keyword.operator.js' + '"?"': 'keyword.operator.js' + '"&&"': 'keyword.operator.js' + '"||"': 'keyword.operator.js' + '"&"': 'keyword.operator.js' + '"~"': 'keyword.operator.js' + '"^"': 'keyword.operator.js' + '">>"': 'keyword.operator.js' + '">>>"': 'keyword.operator.js' + '"<<"': 'keyword.operator.js' + '"|"': 'keyword.operator.js' + '"++"': 'keyword.operator.js' + '"--"': 'keyword.operator.js' + '"..."': 'keyword.operator.spread.js' + + '"in"': 'keyword.operator.in' + '"instanceof"': 'keyword.operator.instanceof' + '"of"': 'keyword.operator.of' + '"new"': 'keyword.operator.new' + '"typeof"': 'keyword.operator.typeof' + + '"get"': 'keyword.operator.setter' + '"set"': 'keyword.operator.setter' + + '"."': 'meta.delimiter.period' + '","': 'meta.delimiter.comma' + + '"as"': 'keyword.control' + '"if"': 'keyword.control' + '"do"': 'keyword.control' + '"else"': 'keyword.control' + '"while"': 'keyword.control' + '"for"': 'keyword.control' + '"return"': 'keyword.control' + '"break"': 'keyword.control' + '"continue"': 'keyword.control' + '"throw"': 'keyword.control' + '"try"': 'keyword.control' + '"catch"': 'keyword.control' + '"finally"': 'keyword.control' + '"switch"': 'keyword.control' + '"case"': 'keyword.control' + '"default"': 'keyword.control' + '"export"': 'keyword.control' + '"import"': 'keyword.control' + '"from"': 'keyword.control' + '"yield"': 'keyword.control' + '"async"': 'keyword.control' + '"await"': 'keyword.control' + '"debugger"': 'keyword.control' + '"delete"': 'keyword.control' + + 'jsx_attribute > property_identifier': 'entity.other.attribute-name' + 'jsx_opening_element > identifier': 'entity.name.tag' + 'jsx_closing_element > identifier': 'entity.name.tag' + 'jsx_self_closing_element > identifier': 'entity.name.tag' diff --git a/grammars/tree-sitter-jsdoc.cson b/grammars/tree-sitter-jsdoc.cson new file mode 100644 index 00000000..a9f8a679 --- /dev/null +++ b/grammars/tree-sitter-jsdoc.cson @@ -0,0 +1,16 @@ +name: 'JSDoc' +scopeName: 'source.jsdoc' +type: 'tree-sitter' +parser: 'tree-sitter-jsdoc' + +injectionRegex: '^jsdoc$' + +scopes: + 'tag_name': 'keyword.control' + 'identifier': 'variable.other.jsdoc' + 'type': 'support.type' + 'path_expression > identifier': 'string' + '"."': 'meta.delimiter.period' + '":"': 'meta.delimiter.colon' + '"/"': 'meta.delimiter.slash' + '"#", "~"': 'meta.delimiter' diff --git a/grammars/tree-sitter-regex.cson b/grammars/tree-sitter-regex.cson new file mode 100644 index 00000000..812bd378 --- /dev/null +++ b/grammars/tree-sitter-regex.cson @@ -0,0 +1,17 @@ +name: 'Javascript RegExp' +scopeName: 'source.js.regexp' +type: 'tree-sitter' +parser: 'tree-sitter-regex' + +injectionRegex: 'regex' + +scopes: + 'pattern': 'string.quoted', + 'group_name': 'variable.other.object.property' + 'identity_escape, control_letter_escape, control_escape, class_escape': 'constant.character.character-class.regexp' + 'start_assertion, end_assertion, boundary_assertion, non_boundary_assertion': 'constant.character.character-class.regexp' + 'count_quantifier, one_or_more, zero_or_more, optional': 'storage.modifier' + 'character_class': 'string.regexp' + '"(", "(?", "(?:", "(?<"': 'punctuation.definition.parameters.begin.bracket.round' + '">", ")"': 'punctuation.definition.parameters.end.bracket.round' + '"=", "<=", "!", "~!#@%^&*|+=[]{}`?-…' + 'commentStart': '// ' + 'foldEndPattern': '^\\s*\\}|^\\s*\\]|^\\s*\\)' + 'increaseIndentPattern': '(?x) + \\{ [^}"\']*(//.*)? $ + | \\[ [^\\]"\']*(//.*)? $ + | \\( [^)"\']*(//.*)? $ + ' + 'decreaseIndentPattern': '(?x) + ^ \\s* (\\s* /[*] .* [*]/ \\s*)* [}\\])] + ' diff --git a/snippets/language-javascript.cson b/snippets/language-javascript.cson index b68286c7..24fdbb61 100644 --- a/snippets/language-javascript.cson +++ b/snippets/language-javascript.cson @@ -1,73 +1,130 @@ -'.source.js': +'.source.js, .source.flow': 'Object Method': - 'prefix': ':f' - 'body': '${1:method_name}: function (${2:attribute}) {\n\t$3\n}${4:,}' + 'prefix': 'kf' + 'body': '${1:methodName}: function (${2:attribute}) {\n\t$3\n}${4:,}' 'Object key — key: "value"': - 'prefix': ':' - 'body': '${1:key}: ${2:"${3:value}"}${4:, }' + 'prefix': 'kv' + 'body': '${1:key}: ${2:\'${3:value}\'}${4:, }' 'Prototype': 'prefix': 'proto' - 'body': '${1:class_name}.prototype.${2:method_name} = function (${3:first_argument}) {\n\t${0:// body...}\n};' + 'body': '${1:ClassName}.prototype.${2:methodName} = function ($3) {\n\t$0\n};' 'do': 'prefix': 'do' 'body': 'do {\n\t$2\n} while (${1:true});' + 'condition ? true : false': + 'prefix': 'tern' + 'body': '${1:condition} ? ${2:true} : ${3:false}' + 'if': + 'prefix': 'if' + 'body': 'if (${1:true}) {\n\t$2\n}' + 'if … else': + 'prefix': 'ife' + 'body': 'if (${1:true}) {\n\t$2\n} else {\n\t$3\n}' 'else': 'prefix': 'else' 'body': 'else {\n\t$1\n}' + 'else if': + 'prefix': 'elseif' + 'body': 'else if (${1:true}) {\n\t$2\n}' 'for': 'prefix' : 'for' - 'body' : 'for (var i = 0; i < ${1:array}.length; i++) {\n\t${2:array}[i]\n}' + 'body' : 'for (var ${2:i} = 0; ${2:i} < ${1:array}.length; ${2:i}++) {\n\t${1:array}[${2:i}]$3\n}' 'for in': 'prefix': 'forin' - 'body': 'for (${1:variable} in ${2:object}) {\n\t${5:if (${3:object}.hasOwnProperty(${4:variable})) {\n\t\t$6\n\t}}\n}' + 'body': 'for (var ${1:variable} in ${2:object}) {\n\t${3:if (${2:object}.hasOwnProperty(${1:variable})) {\n\t\t$4\n\t\\}}\n}' + 'for of': + 'prefix': 'forof' + 'body': 'for (var ${1:variable} of ${2:iterable}) {\n\t$3\n}' + 'forEach': + 'prefix' : 'foreach' + 'body' : 'forEach((${1:item}, ${2:i}) => {\n\t$3\n});\n' 'Function': 'prefix': 'fun' - 'body': 'function ${1:function_name}(${2:argument}) {\n\t${0:// body...}\n}' + 'body': 'function ${1:functionName}($2) {\n\t$0\n}' 'Anonymous Function': 'prefix': 'f' 'body': 'function ($1) {\n\t$2\n}' - 'getElementById': - 'prefix': 'get' - 'body': 'getElementById(${1:\'${2:id}\'})' + 'Arrow Function': + 'prefix': 'af' + 'body': '($1) => {\n\t$2\n}' + 'Generator': + 'prefix': 'gen', + 'body': 'function* ${1:functionName}($2) {\n\t$0\n}' + 'Anonymous Generator': + 'prefix': 'g' + 'body': 'function* ($1) {\n\t$2\n}' 'getElementsByClassName': 'prefix': 'get' - 'body': 'getElementsByClassName(${1:\'${2:className}\'})' + 'body': 'getElementsByClassName(${1:\'${2:className}\'})$3' 'getElementsByName': - 'prefix': 'get' - 'body': 'getElementsByName(${1:\'${2:name}\'})' + 'prefix': 'getn' + 'body': 'getElementsByName(${1:\'${2:name}\'})$3' 'getElementsByTagName': - 'prefix': 'get' - 'body': 'getElementsByTagName(${1:\'${2:tagName}\'})' + 'prefix': 'gett' + 'body': 'getElementsByTagName(${1:\'${2:tagName}\'})$3' + 'getElementById': + 'prefix': 'geti' + 'body': 'getElementById(${1:\'${2:id}\'})$3' + 'querySelector': + 'prefix': 'qs' + 'body': 'querySelector(${1:\'${2:query}\'})$3' + 'querySelectorAll': + 'prefix': 'qsa' + 'body': 'querySelectorAll(${1:\'${2:query}\'})$3' 'Immediately-Invoked Function Expression': 'prefix': 'iife' - 'body': '(function() {\n${1:\t"use strict";\n}\t$2\n}());' - 'if … else': - 'prefix': 'ife' - 'body': 'if (${1:true}) {\n\t$2\n} else {\n\t$3\n}' - 'if': - 'prefix': 'if' - 'body': 'if (${1:true}) {\n\t$2\n}' + 'body': '(function() {\n\t${1:\'use strict\';\n}\t$2\n}());' 'log': 'prefix': 'log' 'body': 'console.log($1);$0' + 'dir': + 'prefix': 'dir' + 'body': 'console.dir($1);$0' 'warn': 'prefix': 'warn' 'body': 'console.warn($1);$0' 'error': 'prefix': 'error' 'body': 'console.error($1);$0' + 'inspect': + 'prefix': 'inspect' + 'body': 'console.log(require(\'util\').inspect($0, { depth: null }));' + 'new Promise': + 'prefix': 'prom' + 'body': 'new Promise(function(resolve, reject) {\n\t$1\n});$0' 'setInterval function': 'prefix': 'interval' - 'body': 'setInterval(${2:function () {\n\t$3\n}}, ${1:10});' + 'body': 'setInterval(${2:function () {\n\t$3\n\\}}, ${1:10});' 'setTimeout function': 'prefix': 'timeout' - 'body': 'setTimeout(${2:function () {\n\t$3\n}}, ${1:10});' + 'body': 'setTimeout(${2:function () {\n\t$3\n\\}}, ${1:10});' 'switch': 'prefix': 'switch' - 'body': 'switch (${1:expression}) {\ncase ${2:expression}:\n\t$4\n\tbreak;$5\ndefault:\n\t$3\n}' + 'body': 'switch (${1:expression}) {\n\tcase ${2:expression}:\n\t\t$4\n\t\tbreak;$5\n\tdefault:\n\t\t$3\n}' + 'case': + 'prefix': 'case' + 'body': 'case ${1:expression}:\n\t$2\n\tbreak;' 'try': 'prefix': 'try' - 'body': 'try {\n\t${1:statements}\n} catch (${2:variable}) {\n\t${3:statements}\n}${4: finally {\n\t${5:statements}\n}}' + 'body': 'try {\n\t$1\n} catch (${2:e}) {\n\t$3\n}${4: finally {\n\t$5\n\\}}' 'while': 'prefix': 'while' 'body': 'while (${1:true}) {\n\t$2\n}' + 'Start Docblock': + 'prefix': '/**' + 'body': '/**\n * $1\n */$0' + 'CommonJS require': + 'prefix': 'req' + 'body': 'const ${1:module} = require(\'${1:module}\');' + 'Class': + 'prefix': 'class' + 'body': 'class ${1:ClassName} {\n\tconstructor($2) {\n\t\t$3\n\t}\n}' + 'export function': + 'prefix': 'expfun' + 'body': 'exports.${1:functionName} = function ($2) {\n\t${3:// body...}\n};' + 'export module': + 'prefix': 'expmod' + 'body': 'module.exports = ${1:name};' + 'return': + 'prefix': 'ret' + 'body': 'return $1;$0' diff --git a/spec/javascript-spec.coffee b/spec/javascript-spec.coffee new file mode 100644 index 00000000..192850ec --- /dev/null +++ b/spec/javascript-spec.coffee @@ -0,0 +1,2501 @@ +fs = require 'fs' +path = require 'path' +TextEditor = null +buildTextEditor = (params) -> + if atom.workspace.buildTextEditor? + atom.workspace.buildTextEditor(params) + else + TextEditor ?= require('atom').TextEditor + new TextEditor(params) + +describe "JavaScript grammar", -> + grammar = null + + beforeEach -> + atom.config.set('core.useTreeSitterParsers', false) + + waitsForPromise -> + atom.packages.activatePackage("language-javascript") + + runs -> + grammar = atom.grammars.grammarForScopeName("source.js") + + it "parses the grammar", -> + expect(grammar).toBeTruthy() + expect(grammar.scopeName).toBe "source.js" + + describe "strings", -> + it "tokenizes single-line strings", -> + delimsByScope = + "string.quoted.double.js": '"' + "string.quoted.single.js": "'" + + for scope, delim of delimsByScope + {tokens} = grammar.tokenizeLine(delim + "x" + delim) + expect(tokens[0].value).toEqual delim + expect(tokens[0].scopes).toEqual ["source.js", scope, "punctuation.definition.string.begin.js"] + expect(tokens[1].value).toEqual "x" + expect(tokens[1].scopes).toEqual ["source.js", scope] + expect(tokens[2].value).toEqual delim + expect(tokens[2].scopes).toEqual ["source.js", scope, "punctuation.definition.string.end.js"] + + it "tokenizes invalid multiline strings", -> + delimsByScope = + "string.quoted.double.js": '"' + "string.quoted.single.js": "'" + + for scope, delim of delimsByScope + lines = grammar.tokenizeLines delim + """ + line1 + line2\\ + line3 + """ + delim + expect(lines[0][0]).toEqual value: delim, scopes: ['source.js', scope, 'punctuation.definition.string.begin.js'] + expect(lines[1][0]).toEqual value: 'line2\\', scopes: ['source.js', scope] + expect(lines[2][0]).toEqual value: 'line3', scopes: ['source.js', scope] + expect(lines[2][1]).toEqual value: delim, scopes: ['source.js', scope, 'punctuation.definition.string.end.js'] + + describe "Unicode escape sequences", -> + bracketScopes = [ + 'punctuation.definition.unicode-escape.begin.bracket.curly.js', + 'punctuation.definition.unicode-escape.end.bracket.curly.js' + ] + delimsByScope = + "string.quoted.double.js": '"' + "string.quoted.single.js": "'" + + it "tokenises 2-digit sequences", -> + for scope, quote of delimsByScope + {tokens} = grammar.tokenizeLine(quote + '\\x2011' + quote) + expect(tokens[0]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: '\\x20', scopes: ['source.js', scope, 'constant.character.escape.js'] + expect(tokens[2]).toEqual value: '11', scopes: ['source.js', scope] + expect(tokens[3]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.end.js'] + + it "tokenises 4-digit sequences", -> + for scope, quote of delimsByScope + {tokens} = grammar.tokenizeLine(quote + '\\u2011' + quote) + expect(tokens[0]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: '\\u2011', scopes: ['source.js', scope, 'constant.character.escape.js'] + expect(tokens[2]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.end.js'] + + it "tokenises variable-length sequences", -> + for scope, quote of delimsByScope + {tokens} = grammar.tokenizeLine(quote + '\\u{2000}' + quote) + expect(tokens[0]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: '\\u', scopes: ['source.js', scope, 'constant.character.escape.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', scope, 'constant.character.escape.js', bracketScopes[0]] + expect(tokens[3]).toEqual value: '2000', scopes: ['source.js', scope, 'constant.character.escape.js'] + expect(tokens[4]).toEqual value: '}', scopes: ['source.js', scope, 'constant.character.escape.js', bracketScopes[1]] + expect(tokens[5]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.end.js'] + + it "highlights sequences with invalid syntax", -> + for invalid in ['\\u', '\\u{2000', '\\u{G}'] + {tokens} = grammar.tokenizeLine('"' + invalid + '"') + expect(tokens[1]).toEqual value: invalid, scopes: ['source.js', 'string.quoted.double.js', 'invalid.illegal.unicode-escape.js'] + + it "highlights sequences with invalid codepoints", -> + maxCodepoint = 0x10FFFF + for codepoint in [0x5000, 0x11FFFF, 0x1000000, maxCodepoint] + pointStr = codepoint.toString(16).toUpperCase().replace(/^0x/, "") + {tokens} = grammar.tokenizeLine('"\\u{' + pointStr + '}"') + pointScopes = ['source.js', 'string.quoted.double.js', 'constant.character.escape.js'] + if codepoint > maxCodepoint then pointScopes.push 'invalid.illegal.unicode-escape.js' + expect(tokens[0]).toEqual value: '"', scopes: ['source.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: '\\u', scopes: ['source.js', 'string.quoted.double.js', 'constant.character.escape.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'string.quoted.double.js', 'constant.character.escape.js', bracketScopes[0]] + expect(tokens[3]).toEqual value: pointStr, scopes: pointScopes + expect(tokens[4]).toEqual value: '}', scopes: ['source.js', 'string.quoted.double.js', 'constant.character.escape.js', bracketScopes[1]] + expect(tokens[5]).toEqual value: '"', scopes: ['source.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + + describe "keywords", -> + keywords = ['await', 'break', 'catch', 'continue', 'do'] + + for keyword in keywords + it "tokenizes the #{keyword} keyword", -> + {tokens} = grammar.tokenizeLine(keyword) + expect(tokens[0]).toEqual value: keyword, scopes: ['source.js', 'keyword.control.js'] + + it "tokenizes the debugger statement", -> + {tokens} = grammar.tokenizeLine("debugger;") + expect(tokens[0]).toEqual value: "debugger", scopes: ['source.js', 'keyword.other.debugger.js'] + expect(tokens[1]).toEqual value: ";", scopes: ['source.js', 'punctuation.terminator.statement.js'] + + it "tokenises an `await` keyword after a spread operator", -> + {tokens} = grammar.tokenizeLine("...await stuff()") + expect(tokens[0]).toEqual value: '...', scopes: ['source.js', 'keyword.operator.spread.js'] + expect(tokens[1]).toEqual value: 'await', scopes: ['source.js', 'keyword.control.js'] + expect(tokens[3]).toEqual value: 'stuff', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(tokens[4]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[5]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + + describe "built-in globals", -> + it "tokenizes built-in classes", -> + {tokens} = grammar.tokenizeLine('window') + expect(tokens[0]).toEqual value: 'window', scopes: ['source.js', 'support.variable.dom.js'] + + {tokens} = grammar.tokenizeLine('window.name') + expect(tokens[0]).toEqual value: 'window', scopes: ['source.js', 'support.variable.dom.js'] + + {tokens} = grammar.tokenizeLine('$window') + expect(tokens[0]).toEqual value: '$window', scopes: ['source.js'] + + it "tokenizes built-in variables", -> + {tokens} = grammar.tokenizeLine('module') + expect(tokens[0]).toEqual value: 'module', scopes: ['source.js', 'support.variable.js'] + + {tokens} = grammar.tokenizeLine('module.prop') + expect(tokens[0]).toEqual value: 'module', scopes: ['source.js', 'support.variable.js'] + + describe "instantiation", -> + it "tokenizes the new keyword and instance entities", -> + {tokens} = grammar.tokenizeLine('new something') + expect(tokens[0]).toEqual value: 'new', scopes: ['source.js', 'meta.class.instance.constructor.js', 'keyword.operator.new.js'] + expect(tokens[1]).toEqual value: ' ', scopes: ['source.js', 'meta.class.instance.constructor.js'] + expect(tokens[2]).toEqual value: 'something', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js'] + + {tokens} = grammar.tokenizeLine('new Something') + expect(tokens[0]).toEqual value: 'new', scopes: ['source.js', 'meta.class.instance.constructor.js', 'keyword.operator.new.js'] + expect(tokens[1]).toEqual value: ' ', scopes: ['source.js', 'meta.class.instance.constructor.js'] + expect(tokens[2]).toEqual value: 'Something', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js'] + + {tokens} = grammar.tokenizeLine('new $something') + expect(tokens[0]).toEqual value: 'new', scopes: ['source.js', 'meta.class.instance.constructor.js', 'keyword.operator.new.js'] + expect(tokens[1]).toEqual value: ' ', scopes: ['source.js', 'meta.class.instance.constructor.js'] + expect(tokens[2]).toEqual value: '$something', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js'] + + {tokens} = grammar.tokenizeLine('var instance = new obj.ct.Cla$s();') + expect(tokens[0]).toEqual value: 'var', scopes: ['source.js', 'storage.type.var.js'] + expect(tokens[1]).toEqual value: ' instance ', scopes: ['source.js'] + expect(tokens[2]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js'] + expect(tokens[3]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[4]).toEqual value: 'new', scopes: ['source.js', 'meta.class.instance.constructor.js', 'keyword.operator.new.js'] + expect(tokens[5]).toEqual value: ' ', scopes: ['source.js', 'meta.class.instance.constructor.js'] + expect(tokens[6]).toEqual value: 'obj', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js'] + expect(tokens[7]).toEqual value: '.', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js', 'meta.delimiter.property.period.js'] + expect(tokens[8]).toEqual value: 'ct', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js'] + expect(tokens[9]).toEqual value: '.', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js', 'meta.delimiter.property.period.js'] + expect(tokens[10]).toEqual value: 'Cla$s', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js'] + expect(tokens[11]).toEqual value: '(', scopes: ['source.js', 'meta.brace.round.js'] + expect(tokens[12]).toEqual value: ')', scopes: ['source.js', 'meta.brace.round.js'] + expect(tokens[13]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + describe "regular expressions", -> + it "tokenizes regular expressions", -> + {tokens} = grammar.tokenizeLine('/test/') + expect(tokens[0]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: 'test', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[2]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + + {tokens} = grammar.tokenizeLine('/random/g') + expect(tokens[0]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: 'random', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[2]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + expect(tokens[3]).toEqual value: 'g', scopes: ['source.js', 'string.regexp.js', 'meta.flag.regexp'] + + {tokens} = grammar.tokenizeLine('/rock(et)?/is') + expect(tokens[0]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: 'rock', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[2]).toEqual value: '(', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[3]).toEqual value: 'et', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp'] + expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[5]).toEqual value: '?', scopes: ['source.js', 'string.regexp.js', 'keyword.operator.quantifier.regexp'] + expect(tokens[6]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + expect(tokens[7]).toEqual value: 'is', scopes: ['source.js', 'string.regexp.js', 'meta.flag.regexp'] + + {tokens} = grammar.tokenizeLine('/(foo)bar\\1/') + expect(tokens[0]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[2]).toEqual value: 'foo', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp'] + expect(tokens[3]).toEqual value: ')', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[4]).toEqual value: 'bar', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[5]).toEqual value: '\\1', scopes: ['source.js', 'string.regexp.js', 'keyword.other.back-reference.regexp'] + expect(tokens[6]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + + {tokens} = grammar.tokenizeLine('/(?foo)bar\\k/') + expect(tokens[0]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: '(?', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[2]).toEqual value: 'foo', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp'] + expect(tokens[3]).toEqual value: ')', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[4]).toEqual value: 'bar', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[5]).toEqual value: '\\k', scopes: ['source.js', 'string.regexp.js', 'keyword.other.back-reference.regexp'] + expect(tokens[6]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + + {tokens} = grammar.tokenizeLine('/(?:foo)bar/') + expect(tokens[0]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: '(?:', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[2]).toEqual value: 'foo', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp'] + expect(tokens[3]).toEqual value: ')', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[4]).toEqual value: 'bar', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[5]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + + {tokens} = grammar.tokenizeLine('/(?<=foo)test(?=bar)/') + expect(tokens[0]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'string.regexp.js', 'meta.group.assertion.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[2]).toEqual value: '?<=', scopes: ['source.js', 'string.regexp.js', 'meta.group.assertion.regexp', 'meta.assertion.look-behind.regexp'] + expect(tokens[3]).toEqual value: 'foo', scopes: ['source.js', 'string.regexp.js', 'meta.group.assertion.regexp'] + expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'string.regexp.js', 'meta.group.assertion.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[5]).toEqual value: 'test', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[6]).toEqual value: '(', scopes: ['source.js', 'string.regexp.js', 'meta.group.assertion.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[7]).toEqual value: '?=', scopes: ['source.js', 'string.regexp.js', 'meta.group.assertion.regexp', 'meta.assertion.look-ahead.regexp'] + expect(tokens[8]).toEqual value: 'bar', scopes: ['source.js', 'string.regexp.js', 'meta.group.assertion.regexp'] + expect(tokens[9]).toEqual value: ')', scopes: ['source.js', 'string.regexp.js', 'meta.group.assertion.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[10]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + + {tokens} = grammar.tokenizeLine('/(? + {tokens} = grammar.tokenizeLine('[/test/]') + expect(tokens[0]).toEqual value: '[', scopes: ['source.js', 'meta.brace.square.js'] + expect(tokens[1]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[2]).toEqual value: 'test', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[3]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + expect(tokens[4]).toEqual value: ']', scopes: ['source.js', 'meta.brace.square.js'] + + {tokens} = grammar.tokenizeLine('[1, /test/]') + expect(tokens[0]).toEqual value: '[', scopes: ['source.js', 'meta.brace.square.js'] + expect(tokens[1]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[2]).toEqual value: ',', scopes: ['source.js', 'meta.delimiter.object.comma.js'] + expect(tokens[3]).toEqual value: ' ', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[4]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[5]).toEqual value: 'test', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[6]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + expect(tokens[7]).toEqual value: ']', scopes: ['source.js', 'meta.brace.square.js'] + + it "tokenizes regular expressions inside curly brackets", -> + {tokens} = grammar.tokenizeLine('{/test/}') + expect(tokens[0]).toEqual value: '{', scopes: ['source.js', 'meta.brace.curly.js'] + expect(tokens[1]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[2]).toEqual value: 'test', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[3]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + expect(tokens[4]).toEqual value: '}', scopes: ['source.js', 'meta.brace.curly.js'] + + it "tokenizes regular expressions inside arrow function expressions", -> + {tokens} = grammar.tokenizeLine('getRegex = () => /^helloworld$/;') + expect(tokens[9]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[10]).toEqual value: '^', scopes: ['source.js', 'string.regexp.js', 'keyword.control.anchor.regexp'] + expect(tokens[11]).toEqual value: 'helloworld', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[12]).toEqual value: '$', scopes: ['source.js', 'string.regexp.js', 'keyword.control.anchor.regexp'] + expect(tokens[13]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + expect(tokens[14]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + it "verifies that regular expressions have explicit count modifiers", -> + source = fs.readFileSync(path.resolve(__dirname, '..', 'grammars', 'javascript.cson'), 'utf8') + expect(source.search /{,/).toEqual -1 + + source = fs.readFileSync(path.resolve(__dirname, '..', 'grammars', 'regular expressions (javascript).cson'), 'utf8') + expect(source.search /{,/).toEqual -1 + + describe "numbers", -> + it "tokenizes hexadecimals", -> + {tokens} = grammar.tokenizeLine('0x1D306') + expect(tokens[0]).toEqual value: '0x1D306', scopes: ['source.js', 'constant.numeric.hex.js'] + + {tokens} = grammar.tokenizeLine('0X1D306') + expect(tokens[0]).toEqual value: '0X1D306', scopes: ['source.js', 'constant.numeric.hex.js'] + + {tokens} = grammar.tokenizeLine('0x1D306n') + expect(tokens[0]).toEqual value: '0x1D306n', scopes: ['source.js', 'constant.numeric.hex.js'] + + {tokens} = grammar.tokenizeLine('0X1D306n') + expect(tokens[0]).toEqual value: '0X1D306n', scopes: ['source.js', 'constant.numeric.hex.js'] + + {tokens} = grammar.tokenizeLine('0X1D30_69A3') + expect(tokens[0]).toEqual value: '0X1D30', scopes: ['source.js', 'constant.numeric.hex.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.hex.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '69A3', scopes: ['source.js', 'constant.numeric.hex.js'] + + it "tokenizes binary literals", -> + {tokens} = grammar.tokenizeLine('0b011101110111010001100110') + expect(tokens[0]).toEqual value: '0b011101110111010001100110', scopes: ['source.js', 'constant.numeric.binary.js'] + + {tokens} = grammar.tokenizeLine('0B011101110111010001100110') + expect(tokens[0]).toEqual value: '0B011101110111010001100110', scopes: ['source.js', 'constant.numeric.binary.js'] + + {tokens} = grammar.tokenizeLine('0b011101110111010001100110n') + expect(tokens[0]).toEqual value: '0b011101110111010001100110n', scopes: ['source.js', 'constant.numeric.binary.js'] + + {tokens} = grammar.tokenizeLine('0B011101110111010001100110n') + expect(tokens[0]).toEqual value: '0B011101110111010001100110n', scopes: ['source.js', 'constant.numeric.binary.js'] + + {tokens} = grammar.tokenizeLine('0B0111_0111_0111_0100_0110_0110') + expect(tokens[0]).toEqual value: '0B0111', scopes: ['source.js', 'constant.numeric.binary.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.binary.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '0111', scopes: ['source.js', 'constant.numeric.binary.js'] + expect(tokens[3]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.binary.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[4]).toEqual value: '0111', scopes: ['source.js', 'constant.numeric.binary.js'] + expect(tokens[5]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.binary.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[6]).toEqual value: '0100', scopes: ['source.js', 'constant.numeric.binary.js'] + expect(tokens[7]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.binary.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[8]).toEqual value: '0110', scopes: ['source.js', 'constant.numeric.binary.js'] + expect(tokens[9]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.binary.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[10]).toEqual value: '0110', scopes: ['source.js', 'constant.numeric.binary.js'] + + it "tokenizes octal literals", -> + {tokens} = grammar.tokenizeLine('0o1411') + expect(tokens[0]).toEqual value: '0o1411', scopes: ['source.js', 'constant.numeric.octal.js'] + + {tokens} = grammar.tokenizeLine('0O1411') + expect(tokens[0]).toEqual value: '0O1411', scopes: ['source.js', 'constant.numeric.octal.js'] + + {tokens} = grammar.tokenizeLine('0o1411n') + expect(tokens[0]).toEqual value: '0o1411n', scopes: ['source.js', 'constant.numeric.octal.js'] + + {tokens} = grammar.tokenizeLine('0O1411n') + expect(tokens[0]).toEqual value: '0O1411n', scopes: ['source.js', 'constant.numeric.octal.js'] + + {tokens} = grammar.tokenizeLine('0010') + expect(tokens[0]).toEqual value: '0010', scopes: ['source.js', 'constant.numeric.octal.js'] + + {tokens} = grammar.tokenizeLine('0010_7201_5112') + expect(tokens[0]).toEqual value: '0010', scopes: ['source.js', 'constant.numeric.octal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.octal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '7201', scopes: ['source.js', 'constant.numeric.octal.js'] + expect(tokens[3]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.octal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[4]).toEqual value: '5112', scopes: ['source.js', 'constant.numeric.octal.js'] + + {tokens} = grammar.tokenizeLine('0O1411_1236') + expect(tokens[0]).toEqual value: '0O1411', scopes: ['source.js', 'constant.numeric.octal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.octal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '1236', scopes: ['source.js', 'constant.numeric.octal.js'] + + it "tokenizes decimals", -> + {tokens} = grammar.tokenizeLine('1234') + expect(tokens[0]).toEqual value: '1234', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('123456789n') + expect(tokens[0]).toEqual value: '123456789n', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('123_456_789n') + expect(tokens[0]).toEqual value: '123', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '456', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[3]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[4]).toEqual value: '789n', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('5e-10') + expect(tokens[0]).toEqual value: '5e-10', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('5E+5') + expect(tokens[0]).toEqual value: '5E+5', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('9.') + expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] + + {tokens} = grammar.tokenizeLine('9_9.') + expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[3]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] + + {tokens} = grammar.tokenizeLine('.9') + expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] + expect(tokens[1]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('.9_9') + expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] + expect(tokens[1]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[2]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[3]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('9.9') + expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] + expect(tokens[2]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('9_9.9_9') + expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[3]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] + expect(tokens[4]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[5]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[6]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('.1e-23') + expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] + expect(tokens[1]).toEqual value: '1e-23', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('.1_1E+1_1') + expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] + expect(tokens[1]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[2]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[3]).toEqual value: '1E+1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[4]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[5]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('1.E3') + expect(tokens[0]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] + expect(tokens[2]).toEqual value: 'E3', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('1_1.E-1_1') + expect(tokens[0]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[3]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] + expect(tokens[4]).toEqual value: 'E-1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[5]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[6]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('1_1.1_1E1_1') + expect(tokens[0]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[3]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] + expect(tokens[4]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[5]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[6]).toEqual value: '1E1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[7]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[8]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('9_9') + expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('9_9_9') + expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[3]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[4]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('999_999_999') + expect(tokens[0]).toEqual value: '999', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '999', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[3]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[4]).toEqual value: '999', scopes: ['source.js', 'constant.numeric.decimal.js'] + + it "does not tokenize numbers that are part of a variable", -> + {tokens} = grammar.tokenizeLine('hi$1') + expect(tokens[0]).toEqual value: 'hi$1', scopes: ['source.js'] + + {tokens} = grammar.tokenizeLine('hi_1') + expect(tokens[0]).toEqual value: 'hi_1', scopes: ['source.js'] + + {tokens} = grammar.tokenizeLine('_1') + expect(tokens[0]).toEqual value: '_1', scopes: ['source.js', 'constant.other.js'] + + {tokens} = grammar.tokenizeLine('1_') + expect(tokens[0]).toEqual value: '1_', scopes: ['source.js', 'invalid.illegal.identifier.js'] + + {tokens} = grammar.tokenizeLine('1_._1') + expect(tokens[0]).toEqual value: '1_', scopes: ['source.js', 'invalid.illegal.identifier.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.delimiter.property.period.js'] + expect(tokens[2]).toEqual value: '_1', scopes: ['source.js', 'variable.other.property.js'] + + {tokens} = grammar.tokenizeLine('1__1') + expect(tokens[0]).toEqual value: '1__1', scopes: ['source.js', 'invalid.illegal.identifier.js'] + + describe "operators", -> + it "tokenizes them", -> + operators = ["delete", "in", "of", "instanceof", "new", "typeof", "void"] + + for operator in operators + {tokens} = grammar.tokenizeLine(operator) + expect(tokens[0]).toEqual value: operator, scopes: ['source.js', 'keyword.operator.' + operator + '.js'] + + it "tokenizes spread operator", -> + {tokens} = grammar.tokenizeLine('myFunction(...args);') + expect(tokens[2]).toEqual value: '...', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'keyword.operator.spread.js'] + expect(tokens[3]).toEqual value: 'args', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js'] + + {tokens} = grammar.tokenizeLine('[...iterableObj]') + expect(tokens[1]).toEqual value: '...', scopes: ['source.js', 'keyword.operator.spread.js'] + expect(tokens[2]).toEqual value: 'iterableObj', scopes: ['source.js'] + + {tokens} = grammar.tokenizeLine('...arguments') + expect(tokens[0]).toEqual value: '...', scopes: ['source.js', 'keyword.operator.spread.js'] + expect(tokens[1]).toEqual value: 'arguments', scopes: ['source.js', 'variable.language.js'] + + {tokens} = grammar.tokenizeLine('...super') + expect(tokens[0]).toEqual value: '...', scopes: ['source.js', 'keyword.operator.spread.js'] + expect(tokens[1]).toEqual value: 'super', scopes: ['source.js', 'variable.language.js'] + + {tokens} = grammar.tokenizeLine('...this') + expect(tokens[0]).toEqual value: '...', scopes: ['source.js', 'keyword.operator.spread.js'] + expect(tokens[1]).toEqual value: 'this', scopes: ['source.js', 'variable.language.js'] + + describe "increment, decrement", -> + it "tokenizes increment", -> + {tokens} = grammar.tokenizeLine('i++') + expect(tokens[0]).toEqual value: 'i', scopes: ['source.js'] + expect(tokens[1]).toEqual value: '++', scopes: ['source.js', 'keyword.operator.increment.js'] + + it "tokenizes decrement", -> + {tokens} = grammar.tokenizeLine('i--') + expect(tokens[0]).toEqual value: 'i', scopes: ['source.js'] + expect(tokens[1]).toEqual value: '--', scopes: ['source.js', 'keyword.operator.decrement.js'] + + describe "logical", -> + operators = ["&&", "||", "!"] + + it "tokenizes them", -> + for operator in operators + {tokens} = grammar.tokenizeLine('a ' + operator + ' b') + expect(tokens[0]).toEqual value: 'a ', scopes: ['source.js'] + expect(tokens[1]).toEqual value: operator, scopes: ['source.js', 'keyword.operator.logical.js'] + expect(tokens[2]).toEqual value: ' b', scopes: ['source.js'] + + describe "comparison", -> + operators = ["<=", ">=", "!=", "!==", "===", "==", "<", ">" ] + + it "tokenizes them", -> + for operator in operators + {tokens} = grammar.tokenizeLine('a ' + operator + ' b') + expect(tokens[0]).toEqual value: 'a ', scopes: ['source.js'] + expect(tokens[1]).toEqual value: operator, scopes: ['source.js', 'keyword.operator.comparison.js'] + expect(tokens[2]).toEqual value: ' b', scopes: ['source.js'] + + describe "bitwise", -> + it "tokenizes bitwise 'not'", -> + {tokens} = grammar.tokenizeLine('~a') + expect(tokens[0]).toEqual value: '~', scopes: ['source.js', 'keyword.operator.bitwise.js'] + expect(tokens[1]).toEqual value: 'a', scopes: ['source.js'] + + it "tokenizes bitwise shift operators", -> + operators = ["<<", ">>", ">>>"] + + for operator in operators + {tokens} = grammar.tokenizeLine('a ' + operator + ' b') + expect(tokens[0]).toEqual value: 'a ', scopes: ['source.js'] + expect(tokens[1]).toEqual value: operator, scopes: ['source.js', 'keyword.operator.bitwise.shift.js'] + expect(tokens[2]).toEqual value: ' b', scopes: ['source.js'] + + it "tokenizes them", -> + operators = ["|", "^", "&"] + + for operator in operators + {tokens} = grammar.tokenizeLine('a ' + operator + ' b') + expect(tokens[0]).toEqual value: 'a ', scopes: ['source.js'] + expect(tokens[1]).toEqual value: operator, scopes: ['source.js', 'keyword.operator.bitwise.js'] + expect(tokens[2]).toEqual value: ' b', scopes: ['source.js'] + + describe "arithmetic", -> + operators = ["*", "/", "-", "%", "+"] + + it "tokenizes them", -> + for operator in operators + {tokens} = grammar.tokenizeLine('a ' + operator + ' b') + expect(tokens[0]).toEqual value: 'a ', scopes: ['source.js'] + expect(tokens[1]).toEqual value: operator, scopes: ['source.js', 'keyword.operator.js'] + expect(tokens[2]).toEqual value: ' b', scopes: ['source.js'] + + it "tokenizes the arithmetic operators when separated by newlines", -> + for operator in operators + lines = grammar.tokenizeLines '1\n' + operator + ' 2' + expect(lines[0][0]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(lines[1][0]).toEqual value: operator, scopes: ['source.js', 'keyword.operator.js'] + expect(lines[1][2]).toEqual value: '2', scopes: ['source.js', 'constant.numeric.decimal.js'] + + describe "assignment", -> + it "tokenizes '=' operator", -> + {tokens} = grammar.tokenizeLine('a = b') + expect(tokens[0]).toEqual value: 'a ', scopes: ['source.js'] + expect(tokens[1]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js'] + expect(tokens[2]).toEqual value: ' b', scopes: ['source.js'] + + describe "compound", -> + it "tokenizes them", -> + operators = ["+=", "-=", "*=", "/=", "%="] + for operator in operators + {tokens} = grammar.tokenizeLine('a ' + operator + ' b') + expect(tokens[0]).toEqual value: 'a ', scopes: ['source.js'] + expect(tokens[1]).toEqual value: operator, scopes: ['source.js', 'keyword.operator.assignment.compound.js'] + expect(tokens[2]).toEqual value: ' b', scopes: ['source.js'] + + describe "bitwise", -> + it "tokenizes them", -> + operators = ["<<=", ">>=", ">>>=", "&=", "^=", "|="] + for operator in operators + {tokens} = grammar.tokenizeLine('a ' + operator + ' b') + expect(tokens[0]).toEqual value: 'a ', scopes: ['source.js'] + expect(tokens[1]).toEqual value: operator, scopes: ['source.js', 'keyword.operator.assignment.compound.bitwise.js'] + expect(tokens[2]).toEqual value: ' b', scopes: ['source.js'] + + describe "constants", -> + it "tokenizes ALL_CAPS variables as constants", -> + {tokens} = grammar.tokenizeLine('var MY_COOL_VAR = 42;') + expect(tokens[0]).toEqual value: 'var', scopes: ['source.js', 'storage.type.var.js'] + expect(tokens[1]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[2]).toEqual value: 'MY_COOL_VAR', scopes: ['source.js', 'constant.other.js'] + expect(tokens[3]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js'] + expect(tokens[5]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[6]).toEqual value: '42', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[7]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + {tokens} = grammar.tokenizeLine('something = MY_COOL_VAR * 1;') + expect(tokens[0]).toEqual value: 'something ', scopes: ['source.js'] + expect(tokens[1]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js'] + expect(tokens[2]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[3]).toEqual value: 'MY_COOL_VAR', scopes: ['source.js', 'constant.other.js'] + expect(tokens[4]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[5]).toEqual value: '*', scopes: ['source.js', 'keyword.operator.js'] + expect(tokens[6]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[7]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[8]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + {tokens} = grammar.tokenizeLine('OBJ.prop') + expect(tokens[0]).toEqual value: 'OBJ', scopes: ['source.js', 'constant.other.object.js'] + + it "tokenises constants with dollar signs", -> + constants = ['CON$TANT', 'ABC$', 'ABC$D', '$_ALL', 'ALL_$', 'ANGULAR$$$$$$$$$$$'] + for constant in constants + {tokens} = grammar.tokenizeLine(constant) + expect(tokens[0]).toEqual value: constant, scopes: ['source.js', 'constant.other.js'] + + it "doesn't tokenise constants without alphabetic characters", -> + for name in ['$_', '$', '_', '$_$_$_$___$___$____$'] + {tokens} = grammar.tokenizeLine(name) + expect(tokens[0]).toEqual value: name, scopes: ['source.js'] + + it "tokenizes variables declared using `const` as constants", -> + {tokens} = grammar.tokenizeLine('const myCoolVar = 42;') + expect(tokens[0]).toEqual value: 'const', scopes: ['source.js', 'storage.type.const.js'] + expect(tokens[1]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[2]).toEqual value: 'myCoolVar', scopes: ['source.js', 'constant.other.js'] + expect(tokens[3]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js'] + expect(tokens[5]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[6]).toEqual value: '42', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[7]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + lines = grammar.tokenizeLines """ + const a, + b, + c + if(a) + """ + expect(lines[0][0]).toEqual value: 'const', scopes: ['source.js', 'storage.type.const.js'] + expect(lines[0][1]).toEqual value: ' ', scopes: ['source.js'] + expect(lines[0][2]).toEqual value: 'a', scopes: ['source.js', 'constant.other.js'] + expect(lines[0][3]).toEqual value: ',', scopes: ['source.js', 'meta.delimiter.object.comma.js'] + expect(lines[1][0]).toEqual value: 'b', scopes: ['source.js', 'constant.other.js'] + expect(lines[1][1]).toEqual value: ',', scopes: ['source.js', 'meta.delimiter.object.comma.js'] + expect(lines[2][0]).toEqual value: 'c', scopes: ['source.js', 'constant.other.js'] + expect(lines[3][0]).toEqual value: 'if', scopes: ['source.js', 'keyword.control.js'] + expect(lines[3][1]).toEqual value: '(', scopes: ['source.js', 'meta.brace.round.js'] + expect(lines[3][2]).toEqual value: 'a', scopes: ['source.js'] + expect(lines[3][3]).toEqual value: ')', scopes: ['source.js', 'meta.brace.round.js'] + + lines = grammar.tokenizeLines """ + const { + a, + b, + c, + } = foo + """ + expect(lines[0][0]).toEqual value: 'const', scopes: ['source.js', 'storage.type.const.js'] + expect(lines[0][1]).toEqual value: ' ', scopes: ['source.js'] + expect(lines[0][2]).toEqual value: '{', scopes: ['source.js', 'meta.brace.curly.js'] + expect(lines[1][0]).toEqual value: ' ', scopes: ['source.js'] + expect(lines[1][1]).toEqual value: 'a', scopes: ['source.js', 'constant.other.js'] + expect(lines[1][2]).toEqual value: ',', scopes: ['source.js', 'meta.delimiter.object.comma.js'] + expect(lines[2][0]).toEqual value: ' ', scopes: ['source.js'] + expect(lines[2][1]).toEqual value: 'b', scopes: ['source.js', 'constant.other.js'] + expect(lines[2][2]).toEqual value: ',', scopes: ['source.js', 'meta.delimiter.object.comma.js'] + expect(lines[3][0]).toEqual value: ' ', scopes: ['source.js'] + expect(lines[3][1]).toEqual value: 'c', scopes: ['source.js', 'constant.other.js'] + expect(lines[4][0]).toEqual value: '}', scopes: ['source.js', 'meta.brace.curly.js'] + expect(lines[4][1]).toEqual value: ' ', scopes: ['source.js'] + expect(lines[4][2]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js'] + expect(lines[4][3]).toEqual value: ' foo', scopes: ['source.js'] + + {tokens} = grammar.tokenizeLine('(const hi);') + expect(tokens[0]).toEqual value: '(', scopes: ['source.js', 'meta.brace.round.js'] + expect(tokens[1]).toEqual value: 'const', scopes: ['source.js', 'storage.type.const.js'] + expect(tokens[2]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[3]).toEqual value: 'hi', scopes: ['source.js', 'constant.other.js'] + expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'meta.brace.round.js'] + expect(tokens[5]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + {tokens} = grammar.tokenizeLine('const {first:f,second,...rest} = obj;') + expect(tokens[0]).toEqual value: 'const', scopes: ['source.js', 'storage.type.const.js'] + expect(tokens[1]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'meta.brace.curly.js'] + expect(tokens[3]).toEqual value: 'first', scopes: ['source.js'] + expect(tokens[4]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.assignment.js'] + expect(tokens[5]).toEqual value: 'f', scopes: ['source.js', 'constant.other.js'] + expect(tokens[6]).toEqual value: ',', scopes: ['source.js', 'meta.delimiter.object.comma.js'] + expect(tokens[7]).toEqual value: 'second', scopes: ['source.js', 'constant.other.js'] + expect(tokens[8]).toEqual value: ',', scopes: ['source.js', 'meta.delimiter.object.comma.js'] + expect(tokens[9]).toEqual value: '...', scopes: ['source.js', 'keyword.operator.spread.js'] + expect(tokens[10]).toEqual value: 'rest', scopes: ['source.js', 'constant.other.js'] + expect(tokens[11]).toEqual value: '}', scopes: ['source.js', 'meta.brace.curly.js'] + expect(tokens[12]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[13]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js'] + expect(tokens[14]).toEqual value: ' obj', scopes: ['source.js'] + expect(tokens[15]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + {tokens} = grammar.tokenizeLine('const c = /regex/;') + expect(tokens[0]).toEqual value: 'const', scopes: ['source.js', 'storage.type.const.js'] + expect(tokens[1]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[2]).toEqual value: 'c', scopes: ['source.js', 'constant.other.js'] + expect(tokens[3]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js'] + expect(tokens[5]).toEqual value: ' ', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[6]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[7]).toEqual value: 'regex', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[8]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + expect(tokens[9]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + it "tokenizes variables declared with `const` in for-in and for-of loops", -> + {tokens} = grammar.tokenizeLine 'for (const elem of array) {' + expect(tokens[0]).toEqual value: 'for', scopes: ['source.js', 'keyword.control.js'] + expect(tokens[1]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[2]).toEqual value: '(', scopes: ['source.js', 'meta.brace.round.js'] + expect(tokens[3]).toEqual value: 'const', scopes: ['source.js', 'storage.type.const.js'] + expect(tokens[4]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[5]).toEqual value: 'elem', scopes: ['source.js', 'constant.other.js'] + expect(tokens[6]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[7]).toEqual value: 'of', scopes: ['source.js', 'keyword.operator.of.js'] + expect(tokens[8]).toEqual value: ' array', scopes: ['source.js'] + expect(tokens[9]).toEqual value: ')', scopes: ['source.js', 'meta.brace.round.js'] + + {tokens} = grammar.tokenizeLine 'for (const name in object) {' + expect(tokens[5]).toEqual value: 'name', scopes: ['source.js', 'constant.other.js'] + expect(tokens[6]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[7]).toEqual value: 'in', scopes: ['source.js', 'keyword.operator.in.js'] + expect(tokens[8]).toEqual value: ' object', scopes: ['source.js'] + + {tokens} = grammar.tokenizeLine 'const index = 0;' + expect(tokens[0]).toEqual value: 'const', scopes: ['source.js', 'storage.type.const.js'] + expect(tokens[2]).toEqual value: 'index', scopes: ['source.js', 'constant.other.js'] + expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js'] + + {tokens} = grammar.tokenizeLine 'const offset = 0;' + expect(tokens[0]).toEqual value: 'const', scopes: ['source.js', 'storage.type.const.js'] + expect(tokens[2]).toEqual value: 'offset', scopes: ['source.js', 'constant.other.js'] + expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js'] + + it "tokenizes support constants", -> + {tokens} = grammar.tokenizeLine('awesome = cool.EPSILON;') + expect(tokens[0]).toEqual value: 'awesome ', scopes: ['source.js'] + expect(tokens[1]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js'] + expect(tokens[3]).toEqual value: 'cool', scopes: ['source.js', 'variable.other.object.js'] + expect(tokens[4]).toEqual value: '.', scopes: ['source.js', 'meta.delimiter.property.period.js'] + expect(tokens[5]).toEqual value: 'EPSILON', scopes: ['source.js', 'support.constant.js'] + expect(tokens[6]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + it "tokenizes constants in the middle of ternary expressions", -> + {tokens} = grammar.tokenizeLine('a ? FOO : b') + expect(tokens[3]).toEqual value: 'FOO', scopes: ['source.js', 'constant.other.js'] + + it "tokenizes constants at the end of ternary expressions", -> + {tokens} = grammar.tokenizeLine('a ? b : FOO') + expect(tokens[7]).toEqual value: 'FOO', scopes: ['source.js', 'constant.other.js'] + + describe "ES6 string templates", -> + it "tokenizes them as strings", -> + {tokens} = grammar.tokenizeLine('`hey ${name}`') + expect(tokens[0]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: 'hey ', scopes: ['source.js', 'string.quoted.template.js'] + expect(tokens[2]).toEqual value: '${', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[3]).toEqual value: 'name', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source'] + expect(tokens[4]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[5]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.js', 'punctuation.definition.string.end.js'] + + {tokens} = grammar.tokenizeLine('`hey ${() => {return hi;}}`') + expect(tokens[0]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: 'hey ', scopes: ['source.js', 'string.quoted.template.js'] + expect(tokens[2]).toEqual value: '${', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'meta.function.arrow.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'meta.function.arrow.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(tokens[6]).toEqual value: '=>', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] + expect(tokens[8]).toEqual value: '{', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'punctuation.definition.function.body.begin.bracket.curly.js'] + expect(tokens[9]).toEqual value: 'return', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'keyword.control.js'] + expect(tokens[10]).toEqual value: ' hi', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source'] + expect(tokens[11]).toEqual value: ';', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'punctuation.terminator.statement.js'] + expect(tokens[12]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'punctuation.definition.function.body.end.bracket.curly.js'] + expect(tokens[13]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[14]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.js', 'punctuation.definition.string.end.js'] + + describe "HTML template strings", -> + # TODO: Remove after Atom 1.21 is released + [tagScope, entityScope] = [] + if parseFloat(atom.getVersion()) <= 1.21 + tagScope = 'meta.tag.inline.any.html' + entityScope = 'entity.name.tag.inline.any.html' + else + tagScope = 'meta.tag.inline.b.html' + entityScope = 'entity.name.tag.inline.b.html' + + beforeEach -> + waitsForPromise -> + atom.packages.activatePackage("language-html") + + it "tokenizes ES6 tagged HTML string templates", -> + {tokens} = grammar.tokenizeLine('html`hey ${name}`') + expect(tokens[0]).toEqual value: 'html', scopes: ['source.js', 'string.quoted.template.html.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.begin.js'] + expect(tokens[2]).toEqual value: 'hey ', scopes: ['source.js', 'string.quoted.template.html.js'] + expect(tokens[3]).toEqual value: '<', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.begin.html'] + expect(tokens[4]).toEqual value: 'b', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, entityScope] + expect(tokens[5]).toEqual value: '>', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.end.html'] + expect(tokens[6]).toEqual value: '${', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[7]).toEqual value: 'name', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source'] + expect(tokens[8]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[9]).toEqual value: '', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.end.html'] + expect(tokens[12]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.end.js'] + + it "tokenizes innerHTML attribute declarations with string template tags", -> + {tokens} = grammar.tokenizeLine('text.innerHTML = `hey ${name}`') + expect(tokens[0]).toEqual value: 'text', scopes: ['source.js', 'variable.other.object.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.delimiter.property.period.js'] + expect(tokens[2]).toEqual value: 'innerHTML', scopes: ['source.js', 'variable.other.property.js'] + expect(tokens[3]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js'] + expect(tokens[5]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[6]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.begin.js'] + expect(tokens[7]).toEqual value: 'hey ', scopes: ['source.js', 'string.quoted.template.html.js'] + expect(tokens[8]).toEqual value: '<', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.begin.html'] + expect(tokens[9]).toEqual value: 'b', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, entityScope] + expect(tokens[10]).toEqual value: '>', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.end.html'] + expect(tokens[11]).toEqual value: '${', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[12]).toEqual value: 'name', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source'] + expect(tokens[13]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[14]).toEqual value: '', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.end.html'] + expect(tokens[17]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.end.js'] + + it "tokenizes ES6 tagged HTML string templates with expanded function name", -> + {tokens} = grammar.tokenizeLine('escapeHTML`hey ${name}`') + expect(tokens[0]).toEqual value: 'escapeHTML', scopes: ['source.js', 'string.quoted.template.html.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.begin.js'] + expect(tokens[2]).toEqual value: 'hey ', scopes: ['source.js', 'string.quoted.template.html.js'] + expect(tokens[3]).toEqual value: '<', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.begin.html'] + expect(tokens[4]).toEqual value: 'b', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, entityScope] + expect(tokens[5]).toEqual value: '>', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.end.html'] + expect(tokens[6]).toEqual value: '${', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[7]).toEqual value: 'name', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source'] + expect(tokens[8]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[9]).toEqual value: '', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.end.html'] + expect(tokens[12]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.end.js'] + + it "tokenizes ES6 tagged HTML string templates with expanded function name and white space", -> + {tokens} = grammar.tokenizeLine('escapeHTML `hey ${name}`') + expect(tokens[0]).toEqual value: 'escapeHTML', scopes: ['source.js', 'string.quoted.template.html.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: ' ', scopes: ['source.js', 'string.quoted.template.html.js'] + expect(tokens[2]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.begin.js'] + expect(tokens[3]).toEqual value: 'hey ', scopes: ['source.js', 'string.quoted.template.html.js'] + expect(tokens[4]).toEqual value: '<', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.begin.html'] + expect(tokens[5]).toEqual value: 'b', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, entityScope] + expect(tokens[6]).toEqual value: '>', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.end.html'] + expect(tokens[7]).toEqual value: '${', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[8]).toEqual value: 'name', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source'] + expect(tokens[9]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[10]).toEqual value: '', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.end.html'] + expect(tokens[13]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.end.js'] + + describe "ES6 tagged Relay.QL string templates", -> + it "tokenizes them as strings", -> + {tokens} = grammar.tokenizeLine('Relay.QL`fragment on Foo { id }`') + expect(tokens[0]).toEqual value: 'Relay.QL', scopes: ['source.js', 'string.quoted.template.graphql.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.begin.js'] + expect(tokens[2]).toEqual value: 'fragment on Foo { id }', scopes: ['source.js', 'string.quoted.template.graphql.js'] + expect(tokens[3]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.end.js'] + + describe "ES6 tagged Relay.QL string templates with interpolation", -> + it "tokenizes them as strings", -> + {tokens} = grammar.tokenizeLine('Relay.QL`fragment on Foo { ${myFragment} }`') + expect(tokens[0]).toEqual value: 'Relay.QL', scopes: ['source.js', 'string.quoted.template.graphql.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.begin.js'] + expect(tokens[2]).toEqual value: 'fragment on Foo { ', scopes: ['source.js', 'string.quoted.template.graphql.js'] + expect(tokens[3]).toEqual value: '${', scopes: ['source.js', 'string.quoted.template.graphql.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[4]).toEqual value: 'myFragment', scopes: ['source.js', 'string.quoted.template.graphql.js', 'source.js.embedded.source'] + expect(tokens[5]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.graphql.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[6]).toEqual value: ' }', scopes: ['source.js', 'string.quoted.template.graphql.js'] + expect(tokens[7]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.end.js'] + + describe "ES6 tagged gql string templates", -> + it "tokenizes them as strings", -> + {tokens} = grammar.tokenizeLine('gql`fragment on Foo { id }`') + expect(tokens[0]).toEqual value: 'gql', scopes: ['source.js', 'string.quoted.template.graphql.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.begin.js'] + expect(tokens[2]).toEqual value: 'fragment on Foo { id }', scopes: ['source.js', 'string.quoted.template.graphql.js'] + expect(tokens[3]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.end.js'] + + describe "ES6 tagged SQL string templates", -> + it "tokenizes them as strings", -> + {tokens} = grammar.tokenizeLine('SQL`SELECT foo FROM bar WHERE id = :id`') + expect(tokens[0]).toEqual value: 'SQL', scopes: ['source.js', 'string.quoted.template.sql.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.sql.js', 'punctuation.definition.string.begin.js'] + expect(tokens[2]).toEqual value: 'SELECT foo FROM bar WHERE id = :id', scopes: ['source.js', 'string.quoted.template.sql.js'] + expect(tokens[3]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.sql.js', 'punctuation.definition.string.end.js'] + + describe "ES6 tagged SQL string templates with interpolation", -> + it "tokenizes them as strings", -> + {tokens} = grammar.tokenizeLine('SQL`SELECT foo FROM bar WHERE id = ${id}`') + expect(tokens[0]).toEqual value: 'SQL', scopes: ['source.js', 'string.quoted.template.sql.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.sql.js', 'punctuation.definition.string.begin.js'] + expect(tokens[2]).toEqual value: 'SELECT foo FROM bar WHERE id = ', scopes: ['source.js', 'string.quoted.template.sql.js'] + expect(tokens[3]).toEqual value: '${', scopes: ['source.js', 'string.quoted.template.sql.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[4]).toEqual value: 'id', scopes: ['source.js', 'string.quoted.template.sql.js', 'source.js.embedded.source'] + expect(tokens[5]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.sql.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[6]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.sql.js', 'punctuation.definition.string.end.js'] + + describe "ES6 class", -> + it "tokenizes class", -> + {tokens} = grammar.tokenizeLine('class MyClass') + expect(tokens[0]).toEqual value: 'class', scopes: ['source.js', 'meta.class.js', 'storage.type.class.js'] + expect(tokens[2]).toEqual value: 'MyClass', scopes: ['source.js', 'meta.class.js', 'entity.name.type.class.js'] + + {tokens} = grammar.tokenizeLine('class $abc$') + expect(tokens[2]).toEqual value: '$abc$', scopes: ['source.js', 'meta.class.js', 'entity.name.type.class.js'] + + {tokens} = grammar.tokenizeLine('class $$') + expect(tokens[2]).toEqual value: '$$', scopes: ['source.js', 'meta.class.js', 'entity.name.type.class.js'] + + it "tokenizes class...extends", -> + {tokens} = grammar.tokenizeLine('class MyClass extends SomeClass') + expect(tokens[0]).toEqual value: 'class', scopes: ['source.js', 'meta.class.js', 'storage.type.class.js'] + expect(tokens[2]).toEqual value: 'MyClass', scopes: ['source.js', 'meta.class.js', 'entity.name.type.class.js'] + expect(tokens[4]).toEqual value: 'extends', scopes: ['source.js', 'meta.class.js', 'storage.modifier.js'] + expect(tokens[6]).toEqual value: 'SomeClass', scopes: ['source.js', 'meta.class.js', 'entity.other.inherited-class.js'] + + {tokens} = grammar.tokenizeLine('class MyClass extends $abc$') + expect(tokens[6]).toEqual value: '$abc$', scopes: ['source.js', 'meta.class.js', 'entity.other.inherited-class.js'] + + {tokens} = grammar.tokenizeLine('class MyClass extends $$') + expect(tokens[6]).toEqual value: '$$', scopes: ['source.js', 'meta.class.js', 'entity.other.inherited-class.js'] + + it "tokenizes anonymous class", -> + {tokens} = grammar.tokenizeLine('class extends SomeClass') + expect(tokens[0]).toEqual value: 'class', scopes: ['source.js', 'meta.class.js', 'storage.type.class.js'] + expect(tokens[2]).toEqual value: 'extends', scopes: ['source.js', 'meta.class.js', 'storage.modifier.js'] + expect(tokens[4]).toEqual value: 'SomeClass', scopes: ['source.js', 'meta.class.js', 'entity.other.inherited-class.js'] + + {tokens} = grammar.tokenizeLine('class extends $abc$') + expect(tokens[4]).toEqual value: '$abc$', scopes: ['source.js', 'meta.class.js', 'entity.other.inherited-class.js'] + + {tokens} = grammar.tokenizeLine('class extends $$') + expect(tokens[4]).toEqual value: '$$', scopes: ['source.js', 'meta.class.js', 'entity.other.inherited-class.js'] + + describe "ES6 import", -> + it "tokenizes import", -> + {tokens} = grammar.tokenizeLine('import "module-name";') + expect(tokens[0]).toEqual value: 'import', scopes: ['source.js', 'meta.import.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: '"', scopes: ['source.js', 'meta.import.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[3]).toEqual value: 'module-name', scopes: ['source.js', 'meta.import.js', 'string.quoted.double.js'] + expect(tokens[4]).toEqual value: '"', scopes: ['source.js', 'meta.import.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[5]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + it "tokenizes default import", -> + {tokens} = grammar.tokenizeLine('import defaultMember from "module-name";') + expect(tokens[0]).toEqual value: 'import', scopes: ['source.js', 'meta.import.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: 'defaultMember', scopes: ['source.js', 'meta.import.js', 'variable.other.module.js'] + expect(tokens[4]).toEqual value: 'from', scopes: ['source.js', 'meta.import.js', 'keyword.control.js'] + + it "tokenizes default named import", -> + {tokens} = grammar.tokenizeLine('import { default as defaultMember } from "module-name";') + expect(tokens[0]).toEqual value: 'import', scopes: ['source.js', 'meta.import.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'meta.import.js', 'punctuation.definition.modules.begin.js'] + expect(tokens[4]).toEqual value: 'default', scopes: ['source.js', 'meta.import.js', 'variable.language.default.js'] + expect(tokens[6]).toEqual value: 'as', scopes: ['source.js', 'meta.import.js', 'keyword.control.js'] + expect(tokens[8]).toEqual value: 'defaultMember', scopes: ['source.js', 'meta.import.js', 'variable.other.module-alias.js'] + expect(tokens[10]).toEqual value: '}', scopes: ['source.js', 'meta.import.js', 'punctuation.definition.modules.end.js'] + expect(tokens[12]).toEqual value: 'from', scopes: ['source.js', 'meta.import.js', 'keyword.control.js'] + + it "tokenizes named import", -> + {tokens} = grammar.tokenizeLine('import { member } from "module-name";') + expect(tokens[0]).toEqual value: 'import', scopes: ['source.js', 'meta.import.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'meta.import.js', 'punctuation.definition.modules.begin.js'] + expect(tokens[4]).toEqual value: 'member', scopes: ['source.js', 'meta.import.js', 'variable.other.module.js'] + expect(tokens[6]).toEqual value: '}', scopes: ['source.js', 'meta.import.js', 'punctuation.definition.modules.end.js'] + expect(tokens[8]).toEqual value: 'from', scopes: ['source.js', 'meta.import.js', 'keyword.control.js'] + + {tokens} = grammar.tokenizeLine('import { member1 , member2 as alias2 } from "module-name";') + expect(tokens[0]).toEqual value: 'import', scopes: ['source.js', 'meta.import.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'meta.import.js', 'punctuation.definition.modules.begin.js'] + expect(tokens[4]).toEqual value: 'member1', scopes: ['source.js', 'meta.import.js', 'variable.other.module.js'] + expect(tokens[6]).toEqual value: ',', scopes: ['source.js', 'meta.import.js', 'meta.delimiter.object.comma.js'] + expect(tokens[8]).toEqual value: 'member2', scopes: ['source.js', 'meta.import.js', 'variable.other.module.js'] + expect(tokens[10]).toEqual value: 'as', scopes: ['source.js', 'meta.import.js', 'keyword.control.js'] + expect(tokens[12]).toEqual value: 'alias2', scopes: ['source.js', 'meta.import.js', 'variable.other.module-alias.js'] + expect(tokens[14]).toEqual value: '}', scopes: ['source.js', 'meta.import.js', 'punctuation.definition.modules.end.js'] + expect(tokens[16]).toEqual value: 'from', scopes: ['source.js', 'meta.import.js', 'keyword.control.js'] + + it "tokenizes entire module import", -> + {tokens} = grammar.tokenizeLine('import * as name from "module-name";') + expect(tokens[0]).toEqual value: 'import', scopes: ['source.js', 'meta.import.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: '*', scopes: ['source.js', 'meta.import.js', 'variable.language.import-all.js'] + expect(tokens[4]).toEqual value: 'as', scopes: ['source.js', 'meta.import.js', 'keyword.control.js'] + expect(tokens[6]).toEqual value: 'name', scopes: ['source.js', 'meta.import.js', 'variable.other.module-alias.js'] + expect(tokens[8]).toEqual value: 'from', scopes: ['source.js', 'meta.import.js', 'keyword.control.js'] + + it "tokenizes `import defaultMember, { member } from 'module-name';`", -> + {tokens} = grammar.tokenizeLine('import defaultMember, { member } from "module-name";') + expect(tokens[0]).toEqual value: 'import', scopes: ['source.js', 'meta.import.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: 'defaultMember', scopes: ['source.js', 'meta.import.js', 'variable.other.module.js'] + expect(tokens[3]).toEqual value: ',', scopes: ['source.js', 'meta.import.js', 'meta.delimiter.object.comma.js'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'meta.import.js', 'punctuation.definition.modules.begin.js'] + expect(tokens[7]).toEqual value: 'member', scopes: ['source.js', 'meta.import.js', 'variable.other.module.js'] + expect(tokens[9]).toEqual value: '}', scopes: ['source.js', 'meta.import.js', 'punctuation.definition.modules.end.js'] + expect(tokens[11]).toEqual value: 'from', scopes: ['source.js', 'meta.import.js', 'keyword.control.js'] + + it "tokenizes `import defaultMember, * as alias from 'module-name';", -> + {tokens} = grammar.tokenizeLine('import defaultMember, * as alias from "module-name";') + expect(tokens[0]).toEqual value: 'import', scopes: ['source.js', 'meta.import.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: 'defaultMember', scopes: ['source.js', 'meta.import.js', 'variable.other.module.js'] + expect(tokens[3]).toEqual value: ',', scopes: ['source.js', 'meta.import.js', 'meta.delimiter.object.comma.js'] + expect(tokens[5]).toEqual value: '*', scopes: ['source.js', 'meta.import.js', 'variable.language.import-all.js'] + expect(tokens[7]).toEqual value: 'as', scopes: ['source.js', 'meta.import.js', 'keyword.control.js'] + expect(tokens[9]).toEqual value: 'alias', scopes: ['source.js', 'meta.import.js', 'variable.other.module-alias.js'] + expect(tokens[11]).toEqual value: 'from', scopes: ['source.js', 'meta.import.js', 'keyword.control.js'] + + it "tokenizes comments in statement", -> + lines = grammar.tokenizeLines ''' + import /* comment */ { + member1, // comment + /* comment */ + member2 + } from "module-name"; + ''' + expect(lines[0][2]).toEqual value: '/*', scopes: ['source.js', 'meta.import.js', 'comment.block.js', 'punctuation.definition.comment.begin.js'] + expect(lines[0][3]).toEqual value: ' comment ', scopes: ['source.js', 'meta.import.js', 'comment.block.js'] + expect(lines[0][4]).toEqual value: '*/', scopes: ['source.js', 'meta.import.js', 'comment.block.js', 'punctuation.definition.comment.end.js'] + expect(lines[1][4]).toEqual value: '//', scopes: ['source.js', 'meta.import.js', 'comment.line.double-slash.js', 'punctuation.definition.comment.js'] + expect(lines[1][5]).toEqual value: ' comment', scopes: ['source.js', 'meta.import.js', 'comment.line.double-slash.js'] + expect(lines[2][1]).toEqual value: '/*', scopes: ['source.js', 'meta.import.js', 'comment.block.js', 'punctuation.definition.comment.begin.js'] + expect(lines[2][2]).toEqual value: ' comment ', scopes: ['source.js', 'meta.import.js', 'comment.block.js'] + expect(lines[2][3]).toEqual value: '*/', scopes: ['source.js', 'meta.import.js', 'comment.block.js', 'punctuation.definition.comment.end.js'] + + # https://github.com/atom/language-javascript/issues/485 + lines = grammar.tokenizeLines ''' + import a from 'a' // + import b from 'b' + ''' + expect(lines[0][10]).toEqual value: '//', scopes: ['source.js', 'meta.import.js', 'comment.line.double-slash.js', 'punctuation.definition.comment.js'] + expect(lines[1][0]).toEqual value: 'import', scopes: ['source.js', 'meta.import.js', 'keyword.control.js'] + + describe "ES6 export", -> + it "tokenizes named export", -> + {tokens} = grammar.tokenizeLine('export var x = 0;') + expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: 'var', scopes: ['source.js', 'storage.type.var.js'] + expect(tokens[3]).toEqual value: ' x ', scopes: ['source.js'] + expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js'] + + {tokens} = grammar.tokenizeLine('export let scopedVariable = 0;') + expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: 'let', scopes: ['source.js', 'storage.type.var.js'] + expect(tokens[3]).toEqual value: ' scopedVariable ', scopes: ['source.js'] + expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js'] + + {tokens} = grammar.tokenizeLine('export const CONSTANT = 0;') + expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: 'const', scopes: ['source.js', 'storage.type.const.js'] + expect(tokens[4]).toEqual value: 'CONSTANT', scopes: ['source.js', 'constant.other.js'] + expect(tokens[6]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js'] + + it "tokenizes named function export", -> + {tokens} = grammar.tokenizeLine('export function func(p1, p2){}') + expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] + expect(tokens[4]).toEqual value: 'func', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + + it "tokenizes named class export", -> + {tokens} = grammar.tokenizeLine('export class Foo {}') + expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: 'class', scopes: ['source.js', 'meta.class.js', 'storage.type.class.js'] + expect(tokens[4]).toEqual value: 'Foo', scopes: ['source.js', 'meta.class.js', 'entity.name.type.class.js'] + + it "tokenizes existing variable export", -> + {tokens} = grammar.tokenizeLine('export { bar };') + expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'meta.export.js', 'punctuation.definition.modules.begin.js'] + expect(tokens[4]).toEqual value: 'bar', scopes: ['source.js', 'meta.export.js', 'variable.other.module.js'] + expect(tokens[6]).toEqual value: '}', scopes: ['source.js', 'meta.export.js', 'punctuation.definition.modules.end.js'] + + {tokens} = grammar.tokenizeLine('export { bar, foo as alias };') + expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'meta.export.js', 'punctuation.definition.modules.begin.js'] + expect(tokens[4]).toEqual value: 'bar', scopes: ['source.js', 'meta.export.js', 'variable.other.module.js'] + expect(tokens[5]).toEqual value: ',', scopes: ['source.js', 'meta.export.js', 'meta.delimiter.object.comma.js'] + expect(tokens[7]).toEqual value: 'foo', scopes: ['source.js', 'meta.export.js', 'variable.other.module.js'] + expect(tokens[9]).toEqual value: 'as', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(tokens[11]).toEqual value: 'alias', scopes: ['source.js', 'meta.export.js', 'variable.other.module-alias.js'] + expect(tokens[13]).toEqual value: '}', scopes: ['source.js', 'meta.export.js', 'punctuation.definition.modules.end.js'] + + it "tokenizes default export", -> + {tokens} = grammar.tokenizeLine('export default 123;') + expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: 'default', scopes: ['source.js', 'meta.export.js', 'variable.language.default.js'] + expect(tokens[4]).toEqual value: '123', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('export default name;') + expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: 'default', scopes: ['source.js', 'meta.export.js', 'variable.language.default.js'] + expect(tokens[4]).toEqual value: 'name', scopes: ['source.js', 'meta.export.js', 'variable.other.module.js'] + + {tokens} = grammar.tokenizeLine('export { foo as default };') + expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'meta.export.js', 'punctuation.definition.modules.begin.js'] + expect(tokens[4]).toEqual value: 'foo', scopes: ['source.js', 'meta.export.js', 'variable.other.module.js'] + expect(tokens[6]).toEqual value: 'as', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(tokens[8]).toEqual value: 'default', scopes: ['source.js', 'meta.export.js', 'variable.language.default.js'] + expect(tokens[10]).toEqual value: '}', scopes: ['source.js', 'meta.export.js', 'punctuation.definition.modules.end.js'] + + {tokens} = grammar.tokenizeLine(''' + export default { + 'prop': 'value' + }; + ''') + expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: 'default', scopes: ['source.js', 'meta.export.js', 'variable.language.default.js'] + expect(tokens[4]).toEqual value: '{', scopes: ['source.js', 'meta.brace.curly.js'] + expect(tokens[6]).toEqual value: "'", scopes: ['source.js', 'string.quoted.single.js', 'punctuation.definition.string.begin.js'] + expect(tokens[7]).toEqual value: "prop", scopes: ['source.js', 'string.quoted.single.js'] + expect(tokens[8]).toEqual value: "'", scopes: ['source.js', 'string.quoted.single.js', 'punctuation.definition.string.end.js'] + expect(tokens[9]).toEqual value: ":", scopes: ['source.js', 'keyword.operator.assignment.js'] + expect(tokens[11]).toEqual value: "'", scopes: ['source.js', 'string.quoted.single.js', 'punctuation.definition.string.begin.js'] + expect(tokens[12]).toEqual value: "value", scopes: ['source.js', 'string.quoted.single.js'] + expect(tokens[13]).toEqual value: "'", scopes: ['source.js', 'string.quoted.single.js', 'punctuation.definition.string.end.js'] + expect(tokens[15]).toEqual value: '}', scopes: ['source.js', 'meta.brace.curly.js'] + + it "tokenizes default function export", -> + {tokens} = grammar.tokenizeLine('export default function () {}') + expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: 'default', scopes: ['source.js', 'meta.export.js', 'variable.language.default.js'] + expect(tokens[4]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] + + {tokens} = grammar.tokenizeLine('export default function func() {}') + expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: 'default', scopes: ['source.js', 'meta.export.js', 'variable.language.default.js'] + expect(tokens[4]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] + + it "tokenizes comments in statement", -> + lines = grammar.tokenizeLines ''' + export { + member1, // comment + /* comment */ + member2 + }; + ''' + expect(lines[1][4]).toEqual value: '//', scopes: ['source.js', 'meta.export.js', 'comment.line.double-slash.js', 'punctuation.definition.comment.js'] + expect(lines[1][5]).toEqual value: ' comment', scopes: ['source.js', 'meta.export.js', 'comment.line.double-slash.js'] + expect(lines[2][1]).toEqual value: '/*', scopes: ['source.js', 'meta.export.js', 'comment.block.js', 'punctuation.definition.comment.begin.js'] + expect(lines[2][2]).toEqual value: ' comment ', scopes: ['source.js', 'meta.export.js', 'comment.block.js'] + expect(lines[2][3]).toEqual value: '*/', scopes: ['source.js', 'meta.export.js', 'comment.block.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('export {member1, /* comment */ member2} /* comment */ from "module";') + expect(tokens[6]).toEqual value: '/*', scopes: ['source.js', 'meta.export.js', 'comment.block.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[7]).toEqual value: ' comment ', scopes: ['source.js', 'meta.export.js', 'comment.block.js'] + expect(tokens[8]).toEqual value: '*/', scopes: ['source.js', 'meta.export.js', 'comment.block.js', 'punctuation.definition.comment.end.js'] + expect(tokens[13]).toEqual value: '/*', scopes: ['source.js', 'meta.export.js', 'comment.block.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[14]).toEqual value: ' comment ', scopes: ['source.js', 'meta.export.js', 'comment.block.js'] + expect(tokens[15]).toEqual value: '*/', scopes: ['source.js', 'meta.export.js', 'comment.block.js', 'punctuation.definition.comment.end.js'] + + it "tokenizes default class export", -> + {tokens} = grammar.tokenizeLine('export default class {}') + expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: 'default', scopes: ['source.js', 'meta.export.js', 'variable.language.default.js'] + expect(tokens[4]).toEqual value: 'class', scopes: ['source.js', 'storage.type.js'] + expect(tokens[6]).toEqual value: '{', scopes: ['source.js', 'punctuation.section.scope.begin.js'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'punctuation.section.scope.end.js'] + + {tokens} = grammar.tokenizeLine('export default class Foo {}') + expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: 'default', scopes: ['source.js', 'meta.export.js', 'variable.language.default.js'] + expect(tokens[4]).toEqual value: 'class', scopes: ['source.js', 'meta.class.js', 'storage.type.class.js'] + expect(tokens[6]).toEqual value: 'Foo', scopes: ['source.js', 'meta.class.js', 'entity.name.type.class.js'] + + it "tokenizes re-export", -> + {tokens} = grammar.tokenizeLine('export { name } from "module-name";') + expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'meta.export.js', 'punctuation.definition.modules.begin.js'] + expect(tokens[4]).toEqual value: 'name', scopes: ['source.js', 'meta.export.js', 'variable.other.module.js'] + expect(tokens[6]).toEqual value: '}', scopes: ['source.js', 'meta.export.js', 'punctuation.definition.modules.end.js'] + expect(tokens[8]).toEqual value: 'from', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + + {tokens} = grammar.tokenizeLine('export * from "module-name";') + expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: '*', scopes: ['source.js', 'meta.export.js', 'variable.language.import-all.js'] + expect(tokens[4]).toEqual value: 'from', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + + {tokens} = grammar.tokenizeLine('export { default as alias } from "module-name";') + expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'meta.export.js', 'punctuation.definition.modules.begin.js'] + expect(tokens[4]).toEqual value: 'default', scopes: ['source.js', 'meta.export.js', 'variable.language.default.js'] + expect(tokens[6]).toEqual value: 'as', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(tokens[8]).toEqual value: 'alias', scopes: ['source.js', 'meta.export.js', 'variable.other.module-alias.js'] + expect(tokens[10]).toEqual value: '}', scopes: ['source.js', 'meta.export.js', 'punctuation.definition.modules.end.js'] + expect(tokens[12]).toEqual value: 'from', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + + it "tokenizes multiline re-export", -> + lines = grammar.tokenizeLines ''' + export { + default as alias, + member1 as alias1, + member2, + } from "module-name"; + ''' + expect(lines[0][0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(lines[0][2]).toEqual value: '{', scopes: ['source.js', 'meta.export.js', 'punctuation.definition.modules.begin.js'] + expect(lines[1][1]).toEqual value: 'default', scopes: ['source.js', 'meta.export.js', 'variable.language.default.js'] + expect(lines[1][3]).toEqual value: 'as', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(lines[1][5]).toEqual value: 'alias', scopes: ['source.js', 'meta.export.js', 'variable.other.module-alias.js'] + expect(lines[1][6]).toEqual value: ',', scopes: ['source.js', 'meta.export.js', 'meta.delimiter.object.comma.js'] + expect(lines[2][1]).toEqual value: 'member1', scopes: ['source.js', 'meta.export.js', 'variable.other.module.js'] + expect(lines[2][3]).toEqual value: 'as', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(lines[2][5]).toEqual value: 'alias1', scopes: ['source.js', 'meta.export.js', 'variable.other.module-alias.js'] + expect(lines[2][6]).toEqual value: ',', scopes: ['source.js', 'meta.export.js', 'meta.delimiter.object.comma.js'] + expect(lines[3][1]).toEqual value: 'member2', scopes: ['source.js', 'meta.export.js', 'variable.other.module.js'] + expect(lines[3][2]).toEqual value: ',', scopes: ['source.js', 'meta.export.js', 'meta.delimiter.object.comma.js'] + expect(lines[4][0]).toEqual value: '}', scopes: ['source.js', 'meta.export.js', 'punctuation.definition.modules.end.js'] + expect(lines[4][2]).toEqual value: 'from', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + + describe "ES6 yield", -> + it "tokenizes yield", -> + {tokens} = grammar.tokenizeLine('yield next') + expect(tokens[0]).toEqual value: 'yield', scopes: ['source.js', 'meta.control.yield.js', 'keyword.control.js'] + + it "tokenizes yield*", -> + {tokens} = grammar.tokenizeLine('yield * next') + expect(tokens[0]).toEqual value: 'yield', scopes: ['source.js', 'meta.control.yield.js', 'keyword.control.js'] + expect(tokens[2]).toEqual value: '*', scopes: ['source.js', 'meta.control.yield.js', 'storage.modifier.js'] + + describe "functions", -> + it "tokenizes regular function declarations", -> + {tokens} = grammar.tokenizeLine('function foo(){}') + expect(tokens[0]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] + expect(tokens[2]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + expect(tokens[6]).toEqual value: '}', scopes: ['source.js', 'punctuation.definition.function.body.end.bracket.curly.js'] + + lines = grammar.tokenizeLines ''' + function foo() { + if(something){ } + } + ''' + expect(lines[0][0]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] + expect(lines[0][2]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + expect(lines[0][3]).toEqual value: '(', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(lines[0][4]).toEqual value: ')', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(lines[0][6]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + expect(lines[1][1]).toEqual value: 'if', scopes: ['source.js', 'keyword.control.js'] + expect(lines[1][5]).toEqual value: '{', scopes: ['source.js', 'meta.brace.curly.js'] + expect(lines[1][7]).toEqual value: '}', scopes: ['source.js', 'meta.brace.curly.js'] + expect(lines[2][0]).toEqual value: '}', scopes: ['source.js', 'punctuation.definition.function.body.end.bracket.curly.js'] + + {tokens} = grammar.tokenizeLine('function $abc$(){}') + expect(tokens[2]).toEqual value: '$abc$', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + + {tokens} = grammar.tokenizeLine('function $$(){}') + expect(tokens[2]).toEqual value: '$$', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + + it "tokenizes anonymous functions", -> + {tokens} = grammar.tokenizeLine('function (){}') + expect(tokens[0]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] + expect(tokens[2]).toEqual value: '(', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[3]).toEqual value: ')', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(tokens[4]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + expect(tokens[5]).toEqual value: '}', scopes: ['source.js', 'punctuation.definition.function.body.end.bracket.curly.js'] + + it "tokenizes async functions", -> + {tokens} = grammar.tokenizeLine('async function foo(){}') + expect(tokens[0]).toEqual value: 'async', scopes: ['source.js', 'meta.function.js', 'storage.modifier.async.js'] + expect(tokens[2]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] + expect(tokens[4]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + + it "tokenizes functions as object properties", -> + {tokens} = grammar.tokenizeLine('obj.method = function foo(') + expect(tokens[0]).toEqual value: 'obj', scopes: ['source.js', 'variable.other.object.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.function.js', 'meta.delimiter.method.period.js'] + expect(tokens[2]).toEqual value: 'method', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'meta.function.js', 'keyword.operator.assignment.js'] + expect(tokens[6]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] + expect(tokens[8]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + expect(tokens[9]).toEqual value: '(', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + + {tokens} = grammar.tokenizeLine('this.register = function(') + expect(tokens[0]).toEqual value: 'this', scopes: ['source.js', 'variable.language.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.function.js', 'meta.delimiter.method.period.js'] + expect(tokens[2]).toEqual value: 'register', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + expect(tokens[6]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] + + {tokens} = grammar.tokenizeLine('document.getElementById("foo").onclick = function(') + expect(tokens[8]).toEqual value: '.', scopes: ['source.js', 'meta.function.js', 'meta.delimiter.method.period.js'] + expect(tokens[9]).toEqual value: 'onclick', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + expect(tokens[13]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] + + it "tokenises getter/setter keywords", -> + {tokens} = grammar.tokenizeLine('get name(){ }') + expect(tokens[0]).toEqual value: 'get', scopes: ['source.js', 'meta.function.method.definition.js', 'keyword.operator.getter.js'] + expect(tokens[2]).toEqual value: 'name', scopes: ['source.js', 'meta.function.method.definition.js', 'entity.name.function.js'] + expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'punctuation.definition.function.body.end.bracket.curly.js'] + + it "tokenizes ES6 method definitions", -> + {tokens} = grammar.tokenizeLine('f(a, b) {}') + expect(tokens[0]).toEqual value: 'f', scopes: ['source.js', 'meta.function.method.definition.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[2]).toEqual value: 'a', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[3]).toEqual value: ',', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.delimiter.object.comma.js'] + expect(tokens[5]).toEqual value: 'b', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[6]).toEqual value: ')', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + + {tokens} = grammar.tokenizeLine('async foo(){}') + expect(tokens[0]).toEqual value: 'async', scopes: ['source.js', 'storage.modifier.js'] + expect(tokens[2]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.method.definition.js', 'entity.name.function.js'] + + {tokens} = grammar.tokenizeLine('hi({host, root = "./", plugins = [a, "b", "c", d]}) {}') + expect(tokens[0]).toEqual value: 'hi', scopes: ['source.js', 'meta.function.method.definition.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.brace.curly.js'] + expect(tokens[3]).toEqual value: 'host', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js'] + expect(tokens[4]).toEqual value: ',', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.delimiter.object.comma.js'] + expect(tokens[5]).toEqual value: ' root ', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js'] + expect(tokens[6]).toEqual value: '=', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'keyword.operator.assignment.js'] + expect(tokens[8]).toEqual value: '"', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[9]).toEqual value: './', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'string.quoted.double.js'] + expect(tokens[10]).toEqual value: '"', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[11]).toEqual value: ',', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.delimiter.object.comma.js'] + expect(tokens[12]).toEqual value: ' plugins ', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js'] + expect(tokens[13]).toEqual value: '=', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'keyword.operator.assignment.js'] + expect(tokens[15]).toEqual value: '[', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.brace.square.js'] + expect(tokens[16]).toEqual value: 'a', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js'] + expect(tokens[17]).toEqual value: ',', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.delimiter.object.comma.js'] + expect(tokens[19]).toEqual value: '"', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[22]).toEqual value: ',', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.delimiter.object.comma.js'] + expect(tokens[24]).toEqual value: '"', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[28]).toEqual value: ' d', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js'] + expect(tokens[29]).toEqual value: ']', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.brace.square.js'] + expect(tokens[30]).toEqual value: '}', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'meta.brace.curly.js'] + expect(tokens[31]).toEqual value: ')', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + + {tokens} = grammar.tokenizeLine('write("){");') + expect(tokens[0]).toEqual value: 'write', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[2]).toEqual value: '"', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[3]).toEqual value: '){', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js'] + expect(tokens[4]).toEqual value: '"', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[5]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(tokens[6]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + {tokens} = grammar.tokenizeLine('import (x) { return "Yeah"; }') + expect(tokens[0]).toEqual value: 'import', scopes: ['source.js', 'meta.function.method.definition.js', 'entity.name.function.js'] + expect(tokens[11]).toEqual value: 'Yeah', scopes: ['source.js', 'string.quoted.double.js'] + + {tokens} = grammar.tokenizeLine('export (x) { return "Nah"; }') + expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.function.method.definition.js', 'entity.name.function.js'] + expect(tokens[11]).toEqual value: 'Nah', scopes: ['source.js', 'string.quoted.double.js'] + + it "tokenises ES6 methods with computed names", -> + {tokens} = grammar.tokenizeLine('[ foo ] () { }') + expect(tokens[0]).toEqual value: '[', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.computed-key.js', 'punctuation.definition.computed-key.begin.bracket.square.js'] + expect(tokens[2]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.computed-key.js', 'variable.parameter.property.js'] + expect(tokens[4]).toEqual value: ']', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.computed-key.js', 'punctuation.definition.computed-key.end.bracket.square.js'] + expect(tokens[6]).toEqual value: '(', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[7]).toEqual value: ')', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(tokens[9]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + expect(tokens[11]).toEqual value: '}', scopes: ['source.js', 'punctuation.definition.function.body.end.bracket.curly.js'] + + {tokens} = grammar.tokenizeLine('[ "delet" + this ] (gun) { }') + expect(tokens[0]).toEqual value: '[', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.computed-key.js', 'punctuation.definition.computed-key.begin.bracket.square.js'] + expect(tokens[2]).toEqual value: '"', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.computed-key.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[3]).toEqual value: 'delet', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.computed-key.js', 'string.quoted.double.js'] + expect(tokens[4]).toEqual value: '"', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.computed-key.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[6]).toEqual value: '+', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.computed-key.js', 'keyword.operator.js'] + expect(tokens[8]).toEqual value: 'this', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.computed-key.js', 'variable.language.js'] + expect(tokens[10]).toEqual value: ']', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.computed-key.js', 'punctuation.definition.computed-key.end.bracket.square.js'] + expect(tokens[12]).toEqual value: '(', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[13]).toEqual value: 'gun', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[14]).toEqual value: ')', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(tokens[16]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + expect(tokens[18]).toEqual value: '}', scopes: ['source.js', 'punctuation.definition.function.body.end.bracket.curly.js'] + + {tokens} = grammar.tokenizeLine('get [ foo ] () { }') + expect(tokens[0]).toEqual value: 'get', scopes: ['source.js', 'meta.function.method.definition.js', 'keyword.operator.getter.js'] + expect(tokens[2]).toEqual value: '[', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.computed-key.js', 'punctuation.definition.computed-key.begin.bracket.square.js'] + expect(tokens[4]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.computed-key.js', 'variable.parameter.property.js'] + expect(tokens[6]).toEqual value: ']', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.computed-key.js', 'punctuation.definition.computed-key.end.bracket.square.js'] + expect(tokens[8]).toEqual value: '(', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[9]).toEqual value: ')', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(tokens[11]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + expect(tokens[13]).toEqual value: '}', scopes: ['source.js', 'punctuation.definition.function.body.end.bracket.curly.js'] + + it "tokenizes constructors", -> + {tokens} = grammar.tokenizeLine('constructor(p1, p2) { this.p1 = p1; }') + expect(tokens[0]).toEqual value: 'constructor', scopes: ['source.js', 'meta.function.js', 'entity.name.function.constructor.js'] + expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[2]).toEqual value: 'p1', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[3]).toEqual value: ',', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'meta.delimiter.object.comma.js'] + expect(tokens[5]).toEqual value: 'p2', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[6]).toEqual value: ')', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(tokens[10]).toEqual value: 'this', scopes: ['source.js', 'variable.language.js'] + + {tokens} = grammar.tokenizeLine('constructorABC: {}') + expect(tokens[0]).not.toEqual value: 'constructor', scopes: ['source.js', 'meta.function.js', 'entity.name.function.constructor.js'] + + it "tokenizes named function expressions", -> + {tokens} = grammar.tokenizeLine('var func = function foo(){}') + expect(tokens[0]).toEqual value: 'var', scopes: ['source.js', 'storage.type.var.js'] + expect(tokens[2]).toEqual value: 'func', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'meta.function.js', 'keyword.operator.assignment.js'] + expect(tokens[6]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] + expect(tokens[8]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + expect(tokens[9]).toEqual value: '(', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + + it "tokenizes anonymous function expressions", -> + {tokens} = grammar.tokenizeLine('var func = function(){}') + expect(tokens[0]).toEqual value: 'var', scopes: ['source.js', 'storage.type.var.js'] + expect(tokens[2]).toEqual value: 'func', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'meta.function.js', 'keyword.operator.assignment.js'] + expect(tokens[6]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] + expect(tokens[7]).toEqual value: '(', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + + it "tokenizes functions in object literals", -> + {tokens} = grammar.tokenizeLine('func: function foo(') + expect(tokens[0]).toEqual value: 'func', scopes: ['source.js', 'meta.function.json.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'meta.function.json.js', 'keyword.operator.assignment.js'] + expect(tokens[3]).toEqual value: 'function', scopes: ['source.js', 'meta.function.json.js', 'storage.type.function.js'] + expect(tokens[5]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.json.js', 'entity.name.function.js'] + expect(tokens[6]).toEqual value: '(', scopes: ['source.js', 'meta.function.json.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + + {tokens} = grammar.tokenizeLine('"func": function foo(') + expect(tokens[1]).toEqual value: 'func', scopes: ['source.js', 'meta.function.json.js', 'string.quoted.double.js', 'entity.name.function.js'] + expect(tokens[3]).toEqual value: ':', scopes: ['source.js', 'meta.function.json.js', 'keyword.operator.assignment.js'] + expect(tokens[5]).toEqual value: 'function', scopes: ['source.js', 'meta.function.json.js', 'storage.type.function.js'] + expect(tokens[7]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.json.js', 'entity.name.function.js'] + expect(tokens[8]).toEqual value: '(', scopes: ['source.js', 'meta.function.json.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + + {tokens} = grammar.tokenizeLine('function : a => a') + expect(tokens[0]).toEqual value: 'function', scopes: ['source.js', 'meta.function.arrow.json.js', 'entity.name.function.js'] + expect(tokens[2]).toEqual value: ':', scopes: ['source.js', 'meta.function.arrow.json.js', 'keyword.operator.assignment.js'] + expect(tokens[4]).toEqual value: 'a', scopes: ['source.js', 'meta.function.arrow.json.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[6]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.json.js', 'storage.type.function.arrow.js'] + expect(tokens[7]).toEqual value: ' a', scopes: ['source.js'] + + {tokens} = grammar.tokenizeLine('"func": a => a') + expect(tokens[1]).toEqual value: 'func', scopes: ['source.js', 'meta.function.arrow.json.js', 'string.quoted.double.js', 'entity.name.function.js'] + expect(tokens[3]).toEqual value: ':', scopes: ['source.js', 'meta.function.arrow.json.js', 'keyword.operator.assignment.js'] + expect(tokens[5]).toEqual value: 'a', scopes: ['source.js', 'meta.function.arrow.json.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[7]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.json.js', 'storage.type.function.arrow.js'] + expect(tokens[8]).toEqual value: ' a', scopes: ['source.js'] + + {tokens} = grammar.tokenizeLine('"func" : a => a') + expect(tokens[8]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.json.js', 'storage.type.function.arrow.js'] + + it "tokenizes generator functions", -> + {tokens} = grammar.tokenizeLine('function* foo(){}') + expect(tokens[0]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] + expect(tokens[1]).toEqual value: '*', scopes: ['source.js', 'meta.function.js', 'storage.modifier.generator.js'] + expect(tokens[3]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + expect(tokens[4]).toEqual value: '(', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[5]).toEqual value: ')', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(tokens[6]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'punctuation.definition.function.body.end.bracket.curly.js'] + + {tokens} = grammar.tokenizeLine('function *foo(){}') + expect(tokens[2]).toEqual value: '*', scopes: ['source.js', 'meta.function.js', 'storage.modifier.generator.js'] + + {tokens} = grammar.tokenizeLine('function *(){}') + expect(tokens[2]).toEqual value: '*', scopes: ['source.js', 'meta.function.js', 'storage.modifier.generator.js'] + + it "tokenizes arrow functions", -> + {tokens} = grammar.tokenizeLine('x => x * x') + expect(tokens[0]).toEqual value: 'x', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[2]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] + expect(tokens[3]).toEqual value: ' x ', scopes: ['source.js'] + + {tokens} = grammar.tokenizeLine('() => {}') + expect(tokens[0]).toEqual value: '(', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[1]).toEqual value: ')', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(tokens[3]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + expect(tokens[6]).toEqual value: '}', scopes: ['source.js', 'punctuation.definition.function.body.end.bracket.curly.js'] + + {tokens} = grammar.tokenizeLine('(p1, p2) => {}') + expect(tokens[0]).toEqual value: '(', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[1]).toEqual value: 'p1', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[2]).toEqual value: ',', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'meta.delimiter.object.comma.js'] + expect(tokens[4]).toEqual value: 'p2', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[5]).toEqual value: ')', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(tokens[7]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] + expect(tokens[9]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + expect(tokens[10]).toEqual value: '}', scopes: ['source.js', 'punctuation.definition.function.body.end.bracket.curly.js'] + + lines = grammar.tokenizeLines """ + a = (x, + y) => {} + """ + expect(lines[1][3]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] + + it "tokenizes stored arrow functions", -> + {tokens} = grammar.tokenizeLine('var func = (p1, p2) => {}') + expect(tokens[0]).toEqual value: 'var', scopes: ['source.js', 'storage.type.var.js'] + expect(tokens[2]).toEqual value: 'func', scopes: ['source.js', 'meta.function.arrow.js', 'entity.name.function.js'] + expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'meta.function.arrow.js', 'keyword.operator.assignment.js'] + expect(tokens[11]).toEqual value: ')', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(tokens[13]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] + + it "tokenizes arrow functions as object properties", -> + {tokens} = grammar.tokenizeLine('Utils.isEmpty = (p1, p2) => {}') + expect(tokens[0]).toEqual value: 'Utils', scopes: ['source.js', 'variable.other.object.js'] + expect(tokens[2]).toEqual value: 'isEmpty', scopes: ['source.js', 'meta.function.arrow.js', 'entity.name.function.js'] + expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'meta.function.arrow.js', 'keyword.operator.assignment.js'] + expect(tokens[11]).toEqual value: ')', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(tokens[13]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] + + it "tokenizes arrow functions in object literals", -> + {tokens} = grammar.tokenizeLine('foo: param => {}') + expect(tokens[0]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.arrow.json.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'meta.function.arrow.json.js', 'keyword.operator.assignment.js'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'meta.function.arrow.json.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[5]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.json.js', 'storage.type.function.arrow.js'] + + {tokens} = grammar.tokenizeLine('"foo": param => {}') + expect(tokens[1]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.arrow.json.js', 'string.quoted.double.js', 'entity.name.function.js'] + expect(tokens[3]).toEqual value: ':', scopes: ['source.js', 'meta.function.arrow.json.js', 'keyword.operator.assignment.js'] + expect(tokens[5]).toEqual value: 'param', scopes: ['source.js', 'meta.function.arrow.json.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[7]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.json.js', 'storage.type.function.arrow.js'] + + it "tokenizes default parameters", -> + {tokens} = grammar.tokenizeLine('function multiply(a, b = 1){}') + expect(tokens[7]).toEqual value: 'b', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[9]).toEqual value: '=', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'keyword.operator.assignment.js'] + expect(tokens[11]).toEqual value: '1', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('function callSomething(thing = this.something()) {}') + expect(tokens[4]).toEqual value: 'thing', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[6]).toEqual value: '=', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'keyword.operator.assignment.js'] + expect(tokens[8]).toEqual value: 'this', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'variable.language.js'] + expect(tokens[9]).toEqual value: '.', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(tokens[10]).toEqual value: 'something', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'meta.method-call.js', 'entity.name.function.js'] + + it "tokenizes the rest parameter", -> + {tokens} = grammar.tokenizeLine('(...args) => args[0]') + expect(tokens[1]).toEqual value: '...', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'keyword.operator.spread.js'] + expect(tokens[2]).toEqual value: 'args', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.rest.function.js'] + + it "tokenizes illegal parameters", -> + {tokens} = grammar.tokenizeLine('0abc => {}') + expect(tokens[0]).toEqual value: '0abc', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'invalid.illegal.identifier.js'] + expect(tokens[2]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] + + {tokens} = grammar.tokenizeLine('(0abc) => {}') + expect(tokens[1]).toEqual value: '0abc', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'invalid.illegal.identifier.js'] + expect(tokens[4]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] + + describe "variables", -> + it "tokenizes 'this'", -> + {tokens} = grammar.tokenizeLine('this') + expect(tokens[0]).toEqual value: 'this', scopes: ['source.js', 'variable.language.js'] + + {tokens} = grammar.tokenizeLine('this.obj.prototype = new El()') + expect(tokens[0]).toEqual value: 'this', scopes: ['source.js', 'variable.language.js'] + + {tokens} = grammar.tokenizeLine('$this') + expect(tokens[0]).toEqual value: '$this', scopes: ['source.js'] + + {tokens} = grammar.tokenizeLine('this$') + expect(tokens[0]).toEqual value: 'this$', scopes: ['source.js'] + + it "tokenizes 'super'", -> + {tokens} = grammar.tokenizeLine('super') + expect(tokens[0]).toEqual value: 'super', scopes: ['source.js', 'variable.language.js'] + + it "tokenizes 'arguments'", -> + {tokens} = grammar.tokenizeLine('arguments') + expect(tokens[0]).toEqual value: 'arguments', scopes: ['source.js', 'variable.language.js'] + + {tokens} = grammar.tokenizeLine('arguments[0]') + expect(tokens[0]).toEqual value: 'arguments', scopes: ['source.js', 'variable.language.js'] + + {tokens} = grammar.tokenizeLine('arguments.length') + expect(tokens[0]).toEqual value: 'arguments', scopes: ['source.js', 'variable.language.js'] + + it "tokenizes illegal identifiers", -> + {tokens} = grammar.tokenizeLine('0illegal') + expect(tokens[0]).toEqual value: '0illegal', scopes: ['source.js', 'invalid.illegal.identifier.js'] + + {tokens} = grammar.tokenizeLine('123illegal') + expect(tokens[0]).toEqual value: '123illegal', scopes: ['source.js', 'invalid.illegal.identifier.js'] + + {tokens} = grammar.tokenizeLine('123$illegal') + expect(tokens[0]).toEqual value: '123$illegal', scopes: ['source.js', 'invalid.illegal.identifier.js'] + + describe "objects", -> + it "tokenizes them", -> + {tokens} = grammar.tokenizeLine('obj.prop') + expect(tokens[0]).toEqual value: 'obj', scopes: ['source.js', 'variable.other.object.js'] + + {tokens} = grammar.tokenizeLine('$abc$.prop') + expect(tokens[0]).toEqual value: '$abc$', scopes: ['source.js', 'variable.other.object.js'] + + {tokens} = grammar.tokenizeLine('$$.prop') + expect(tokens[0]).toEqual value: '$$', scopes: ['source.js', 'variable.other.object.js'] + + it "tokenizes illegal objects", -> + {tokens} = grammar.tokenizeLine('1.prop') + expect(tokens[0]).toEqual value: '1', scopes: ['source.js', 'invalid.illegal.identifier.js'] + + {tokens} = grammar.tokenizeLine('123.prop') + expect(tokens[0]).toEqual value: '123', scopes: ['source.js', 'invalid.illegal.identifier.js'] + + {tokens} = grammar.tokenizeLine('123a.prop') + expect(tokens[0]).toEqual value: '123a', scopes: ['source.js', 'invalid.illegal.identifier.js'] + + describe "function calls", -> + it "tokenizes function calls", -> + {tokens} = grammar.tokenizeLine('functionCall()') + expect(tokens[0]).toEqual value: 'functionCall', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[2]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + + {tokens} = grammar.tokenizeLine('functionCall(arg1, "test", {a: 123})') + expect(tokens[0]).toEqual value: 'functionCall', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[2]).toEqual value: 'arg1', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js'] + expect(tokens[3]).toEqual value: ',', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.delimiter.object.comma.js'] + expect(tokens[5]).toEqual value: '"', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[6]).toEqual value: 'test', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js'] + expect(tokens[7]).toEqual value: '"', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[8]).toEqual value: ',', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.delimiter.object.comma.js'] + expect(tokens[10]).toEqual value: '{', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.brace.curly.js'] + expect(tokens[11]).toEqual value: 'a', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js'] + expect(tokens[12]).toEqual value: ':', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'keyword.operator.assignment.js'] + expect(tokens[14]).toEqual value: '123', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'constant.numeric.decimal.js'] + expect(tokens[15]).toEqual value: '}', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.brace.curly.js'] + expect(tokens[16]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + + {tokens} = grammar.tokenizeLine('functionCall((123).toString())') + expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[2]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.brace.round.js'] + expect(tokens[3]).toEqual value: '123', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'constant.numeric.decimal.js'] + expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.brace.round.js'] + expect(tokens[9]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + + {tokens} = grammar.tokenizeLine('$abc$()') + expect(tokens[0]).toEqual value: '$abc$', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js'] + + {tokens} = grammar.tokenizeLine('$$()') + expect(tokens[0]).toEqual value: '$$', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js'] + + {tokens} = grammar.tokenizeLine('ABC()') + expect(tokens[0]).toEqual value: 'ABC', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js'] + + {tokens} = grammar.tokenizeLine('$ABC$()') + expect(tokens[0]).toEqual value: '$ABC$', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js'] + + it "tokenizes function calls when they are arguments", -> + {tokens} = grammar.tokenizeLine('a(b(c))') + expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[2]).toEqual value: 'b', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[4]).toEqual value: 'c', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.function-call.js', 'meta.arguments.js'] + expect(tokens[5]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(tokens[6]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + + it "tokenizes illegal function calls", -> + {tokens} = grammar.tokenizeLine('0illegal()') + expect(tokens[0]).toEqual value: '0illegal', scopes: ['source.js', 'meta.function-call.js', 'invalid.illegal.identifier.js'] + expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[2]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + + it "tokenizes illegal arguments", -> + {tokens} = grammar.tokenizeLine('a(1a)') + expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[2]).toEqual value: '1a', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'invalid.illegal.identifier.js'] + expect(tokens[3]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + + {tokens} = grammar.tokenizeLine('a(123a)') + expect(tokens[2]).toEqual value: '123a', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'invalid.illegal.identifier.js'] + + {tokens} = grammar.tokenizeLine('a(1.prop)') + expect(tokens[2]).toEqual value: '1', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'invalid.illegal.identifier.js'] + expect(tokens[3]).toEqual value: '.', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.delimiter.property.period.js'] + expect(tokens[4]).toEqual value: 'prop', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'variable.other.property.js'] + + it "tokenizes function declaration as an argument", -> + {tokens} = grammar.tokenizeLine('a(function b(p) { return p; })') + expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[2]).toEqual value: 'function', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.function.js', 'storage.type.function.js'] + expect(tokens[4]).toEqual value: 'b', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.function.js', 'entity.name.function.js'] + expect(tokens[5]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[6]).toEqual value: 'p', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.function.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[7]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(tokens[9]).toEqual value: '{', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + expect(tokens[11]).toEqual value: 'return', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'keyword.control.js'] + expect(tokens[12]).toEqual value: ' p', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js'] + expect(tokens[13]).toEqual value: ';', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.terminator.statement.js'] + expect(tokens[15]).toEqual value: '}', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.function.body.end.bracket.curly.js'] + expect(tokens[16]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + + describe "method calls", -> + it "tokenizes method calls", -> + {tokens} = grammar.tokenizeLine('a.b(1+1)') + expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'variable.other.object.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(tokens[2]).toEqual value: 'b', scopes: ['source.js', 'meta.method-call.js', 'entity.name.function.js'] + expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[4]).toEqual value: '1', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'constant.numeric.decimal.js'] + expect(tokens[5]).toEqual value: '+', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'keyword.operator.js'] + expect(tokens[6]).toEqual value: '1', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'constant.numeric.decimal.js'] + expect(tokens[7]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + + {tokens} = grammar.tokenizeLine('a . b(1+1)') + expect(tokens[2]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(tokens[4]).toEqual value: 'b', scopes: ['source.js', 'meta.method-call.js', 'entity.name.function.js'] + expect(tokens[5]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + + {tokens} = grammar.tokenizeLine('a.$abc$()') + expect(tokens[2]).toEqual value: '$abc$', scopes: ['source.js', 'meta.method-call.js', 'entity.name.function.js'] + + {tokens} = grammar.tokenizeLine('a.$$()') + expect(tokens[2]).toEqual value: '$$', scopes: ['source.js', 'meta.method-call.js', 'entity.name.function.js'] + + lines = grammar.tokenizeLines """ + gulp.src("./*.js") + .pipe(minify()) + .pipe(gulp.dest("build")) + """ + expect(lines[0][0]).toEqual value: 'gulp', scopes: ['source.js', 'variable.other.object.js'] + expect(lines[0][1]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(lines[0][2]).toEqual value: 'src', scopes: ['source.js', 'meta.method-call.js', 'entity.name.function.js'] + expect(lines[0][3]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(lines[0][4]).toEqual value: '"', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(lines[0][5]).toEqual value: './*.js', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'string.quoted.double.js'] + expect(lines[0][6]).toEqual value: '"', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(lines[0][7]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(lines[1][1]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(lines[1][2]).toEqual value: 'pipe', scopes: ['source.js', 'meta.method-call.js', 'entity.name.function.js'] + expect(lines[1][3]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(lines[1][4]).toEqual value: 'minify', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(lines[1][5]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(lines[1][6]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(lines[1][7]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(lines[2][1]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(lines[2][2]).toEqual value: 'pipe', scopes: ['source.js', 'meta.method-call.js', 'entity.name.function.js'] + expect(lines[2][3]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(lines[2][4]).toEqual value: 'gulp', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'variable.other.object.js'] + expect(lines[2][5]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(lines[2][6]).toEqual value: 'dest', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'meta.method-call.js', 'entity.name.function.js'] + expect(lines[2][7]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(lines[2][8]).toEqual value: '"', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'meta.method-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(lines[2][9]).toEqual value: 'build', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'meta.method-call.js', 'meta.arguments.js', 'string.quoted.double.js'] + expect(lines[2][10]).toEqual value: '"', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'meta.method-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(lines[2][11]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(lines[2][12]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + + describe "built-in methods", -> + methods = ["require", "parseInt", "parseFloat", "print"] + domMethods = ["substringData", "submit", "splitText", "setNamedItem", "setAttribute"] + + for method in methods + it "tokenizes '#{method}'", -> + {tokens} = grammar.tokenizeLine('.' + method + '()') + expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(tokens[1]).toEqual value: method, scopes: ['source.js', 'meta.method-call.js', 'support.function.js'] + expect(tokens[2]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[3]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + + for domMethod in domMethods + it "tokenizes '#{domMethod}'", -> + {tokens} = grammar.tokenizeLine('.' + domMethod + '()') + expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(tokens[1]).toEqual value: domMethod, scopes: ['source.js', 'meta.method-call.js', 'support.function.dom.js'] + expect(tokens[2]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[3]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + + describe "properties", -> + it "tokenizes properties", -> + {tokens} = grammar.tokenizeLine('obj.property') + expect(tokens[0]).toEqual value: 'obj', scopes: ['source.js', 'variable.other.object.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.delimiter.property.period.js'] + expect(tokens[2]).toEqual value: 'property', scopes: ['source.js', 'variable.other.property.js'] + + {tokens} = grammar.tokenizeLine('obj.property.property') + expect(tokens[0]).toEqual value: 'obj', scopes: ['source.js', 'variable.other.object.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.delimiter.property.period.js'] + expect(tokens[2]).toEqual value: 'property', scopes: ['source.js', 'variable.other.object.property.js'] + + {tokens} = grammar.tokenizeLine('obj.Property') + expect(tokens[0]).toEqual value: 'obj', scopes: ['source.js', 'variable.other.object.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.delimiter.property.period.js'] + expect(tokens[2]).toEqual value: 'Property', scopes: ['source.js', 'variable.other.property.js'] + + {tokens} = grammar.tokenizeLine('obj.$abc$') + expect(tokens[2]).toEqual value: '$abc$', scopes: ['source.js', 'variable.other.property.js'] + + {tokens} = grammar.tokenizeLine('obj.$$') + expect(tokens[2]).toEqual value: '$$', scopes: ['source.js', 'variable.other.property.js'] + + {tokens} = grammar.tokenizeLine('a().b') + expect(tokens[2]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(tokens[3]).toEqual value: '.', scopes: ['source.js', 'meta.delimiter.property.period.js'] + expect(tokens[4]).toEqual value: 'b', scopes: ['source.js', 'variable.other.property.js'] + + {tokens} = grammar.tokenizeLine('a.123illegal') + expect(tokens[0]).toEqual value: 'a', scopes: ['source.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.delimiter.property.period.js'] + expect(tokens[2]).toEqual value: '123illegal', scopes: ['source.js', 'invalid.illegal.identifier.js'] + + it "tokenizes constant properties", -> + {tokens} = grammar.tokenizeLine('obj.MY_CONSTANT') + expect(tokens[0]).toEqual value: 'obj', scopes: ['source.js', 'variable.other.object.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.delimiter.property.period.js'] + expect(tokens[2]).toEqual value: 'MY_CONSTANT', scopes: ['source.js', 'constant.other.property.js'] + + {tokens} = grammar.tokenizeLine('obj.MY_CONSTANT.prop') + expect(tokens[0]).toEqual value: 'obj', scopes: ['source.js', 'variable.other.object.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.delimiter.property.period.js'] + expect(tokens[2]).toEqual value: 'MY_CONSTANT', scopes: ['source.js', 'constant.other.object.property.js'] + + {tokens} = grammar.tokenizeLine('a.C') + expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'variable.other.object.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.delimiter.property.period.js'] + expect(tokens[2]).toEqual value: 'C', scopes: ['source.js', 'constant.other.property.js'] + + describe "strings and functions", -> + it "doesn't confuse them", -> + {tokens} = grammar.tokenizeLine("'a'.b(':c(d)')") + expect(tokens[0]).toEqual value: "'", scopes: ['source.js', 'string.quoted.single.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: "a", scopes: ['source.js', 'string.quoted.single.js'] + expect(tokens[2]).toEqual value: "'", scopes: ['source.js', 'string.quoted.single.js', 'punctuation.definition.string.end.js'] + expect(tokens[3]).toEqual value: ".", scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(tokens[4]).toEqual value: "b", scopes: ['source.js', 'meta.method-call.js', 'entity.name.function.js'] + expect(tokens[5]).toEqual value: "(", scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[6]).toEqual value: "'", scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'string.quoted.single.js', 'punctuation.definition.string.begin.js'] + expect(tokens[7]).toEqual value: ":c(d)", scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'string.quoted.single.js'] + expect(tokens[8]).toEqual value: "'", scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'string.quoted.single.js', 'punctuation.definition.string.end.js'] + expect(tokens[9]).toEqual value: ")", scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + + delimsByScope = + "string.quoted.double.js": '"' + "string.quoted.single.js": "'" + + for scope, delim of delimsByScope + {tokens} = grammar.tokenizeLine('a.push(' + delim + 'x' + delim + ' + y + ' + delim + ':function()' + delim + ');') + expect(tokens[2]).toEqual value: 'push', scopes: ['source.js', 'meta.method-call.js', 'support.function.js'] + expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[4]).toEqual value: delim, scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', scope, 'punctuation.definition.string.begin.js'] + expect(tokens[5]).toEqual value: 'x', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', scope] + expect(tokens[6]).toEqual value: delim, scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', scope, 'punctuation.definition.string.end.js'] + expect(tokens[8]).toEqual value: '+', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'keyword.operator.js'] + expect(tokens[9]).toEqual value: ' y ', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js'] + expect(tokens[10]).toEqual value: '+', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'keyword.operator.js'] + expect(tokens[12]).toEqual value: delim, scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', scope, 'punctuation.definition.string.begin.js'] + expect(tokens[13]).toEqual value: ':function()', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', scope] + expect(tokens[14]).toEqual value: delim, scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', scope, 'punctuation.definition.string.end.js'] + expect(tokens[15]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + + describe "comments", -> + it "tokenizes // comments", -> + {tokens} = grammar.tokenizeLine '//' + expect(tokens[0]).toEqual value: '//', scopes: ['source.js', 'comment.line.double-slash.js', 'punctuation.definition.comment.js'] + + {tokens} = grammar.tokenizeLine '// stuff' + expect(tokens[0]).toEqual value: '//', scopes: ['source.js', 'comment.line.double-slash.js', 'punctuation.definition.comment.js'] + expect(tokens[1]).toEqual value: ' stuff', scopes: ['source.js', 'comment.line.double-slash.js'] + + it "tokenizes /* */ comments", -> + {tokens} = grammar.tokenizeLine('/**/') + expect(tokens[0]).toEqual value: '/*', scopes: ['source.js', 'comment.block.empty.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[1]).toEqual value: '*/', scopes: ['source.js', 'comment.block.empty.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/* foo */') + expect(tokens[0]).toEqual value: '/*', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[1]).toEqual value: ' foo ', scopes: ['source.js', 'comment.block.js'] + expect(tokens[2]).toEqual value: '*/', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.end.js'] + + it "tokenizes /** */ comments", -> + {tokens} = grammar.tokenizeLine('/***/') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[1]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** foo */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[1]).toEqual value: ' foo ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[2]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenizes // comments", -> + {tokens} = grammar.tokenizeLine('// comment') + expect(tokens[0]).toEqual value: '//', scopes: ['source.js', 'comment.line.double-slash.js', 'punctuation.definition.comment.js'] + expect(tokens[1]).toEqual value: ' comment', scopes: ['source.js', 'comment.line.double-slash.js'] + + it "tokenizes comments inside constant definitions", -> + {tokens} = grammar.tokenizeLine('const a, // comment') + expect(tokens[0]).toEqual value: 'const', scopes: ['source.js', 'storage.type.const.js'] + expect(tokens[2]).toEqual value: 'a', scopes: ['source.js', 'constant.other.js'] + expect(tokens[3]).toEqual value: ',', scopes: ['source.js', 'meta.delimiter.object.comma.js'] + expect(tokens[5]).toEqual value: '//', scopes: ['source.js', 'comment.line.double-slash.js', 'punctuation.definition.comment.js'] + expect(tokens[6]).toEqual value: ' comment', scopes: ['source.js', 'comment.line.double-slash.js'] + + it "tokenizes comments inside function declarations", -> + {tokens} = grammar.tokenizeLine('function /* */ foo() /* */ {}') + expect(tokens[0]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] + expect(tokens[2]).toEqual value: '/*', scopes: ['source.js', 'meta.function.js', 'comment.block.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[4]).toEqual value: '*/', scopes: ['source.js', 'meta.function.js', 'comment.block.js', 'punctuation.definition.comment.end.js'] + expect(tokens[6]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + expect(tokens[10]).toEqual value: '/*', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[12]).toEqual value: '*/', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('x => /* */ {}') + expect(tokens[0]).toEqual value: 'x', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[2]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] + expect(tokens[4]).toEqual value: '/*', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[6]).toEqual value: '*/', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.end.js'] + expect(tokens[8]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + + {tokens} = grammar.tokenizeLine('.foo = x => /* */ {}') + expect(tokens[1]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.arrow.js', 'entity.name.function.js'] + expect(tokens[5]).toEqual value: 'x', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[7]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] + expect(tokens[9]).toEqual value: '/*', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.end.js'] + expect(tokens[13]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + + lines = grammar.tokenizeLines ''' + function + // comment + foo() {} + ''' + expect(lines[0][0]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] + expect(lines[1][0]).toEqual value: '//', scopes: ['source.js', 'meta.function.js', 'comment.line.double-slash.js', 'punctuation.definition.comment.js'] + expect(lines[1][1]).toEqual value: ' comment', scopes: ['source.js', 'meta.function.js', 'comment.line.double-slash.js'] + expect(lines[2][0]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] + + lines = grammar.tokenizeLines ''' + x => + // comment + {} + ''' + expect(lines[0][0]).toEqual value: 'x', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(lines[0][2]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] + expect(lines[1][1]).toEqual value: '//', scopes: ['source.js', 'comment.line.double-slash.js', 'punctuation.definition.comment.js'] + expect(lines[1][2]).toEqual value: ' comment', scopes: ['source.js', 'comment.line.double-slash.js'] + expect(lines[2][0]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + + it "tokenizes comments inside function parameters correctly", -> + {tokens} = grammar.tokenizeLine('function test(p1 /*, p2 */) {}') + expect(tokens[6]).toEqual value: '/*', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'comment.block.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[7]).toEqual value: ', p2 ', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'comment.block.js'] + expect(tokens[8]).toEqual value: '*/', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'comment.block.js', 'punctuation.definition.comment.end.js'] + + describe "console", -> + it "tokenizes the console keyword", -> + {tokens} = grammar.tokenizeLine('console;') + expect(tokens[0]).toEqual value: 'console', scopes: ['source.js', 'entity.name.type.object.console.js'] + expect(tokens[1]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + {tokens} = grammar.tokenizeLine('console$') + expect(tokens[0]).not.toEqual value: 'console', scopes: ['source.js', 'entity.name.type.object.console.js'] + + {tokens} = grammar.tokenizeLine('$console') + expect(tokens[1]).not.toEqual value: 'console', scopes: ['source.js', 'entity.name.type.object.console.js'] + + it "tokenizes console support functions", -> + {tokens} = grammar.tokenizeLine('console.log().log()') + expect(tokens[0]).toEqual value: 'console', scopes: ['source.js', 'entity.name.type.object.console.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(tokens[2]).toEqual value: 'log', scopes: ['source.js', 'meta.method-call.js', 'support.function.console.js'] + expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(tokens[6]).not.toEqual value: 'log', scopes: ['source.js', 'meta.method-call.js', 'support.function.console.js'] + + {tokens} = grammar.tokenizeLine('console/**/.log()') + expect(tokens[0]).toEqual value: 'console', scopes: ['source.js', 'entity.name.type.object.console.js'] + expect(tokens[1]).toEqual value: '/*', scopes: ['source.js', 'comment.block.empty.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '*/', scopes: ['source.js', 'comment.block.empty.js', 'punctuation.definition.comment.end.js'] + expect(tokens[3]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(tokens[4]).toEqual value: 'log', scopes: ['source.js', 'meta.method-call.js', 'support.function.console.js'] + expect(tokens[5]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[6]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + + lines = grammar.tokenizeLines ''' + console + .log(); + ''' + expect(lines[0][0]).toEqual value: 'console', scopes: ['source.js', 'entity.name.type.object.console.js'] + expect(lines[1][0]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(lines[1][1]).toEqual value: 'log', scopes: ['source.js', 'meta.method-call.js', 'support.function.console.js'] + expect(lines[1][2]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(lines[1][3]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(lines[1][4]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + {tokens} = grammar.tokenizeLine('console . log();') + expect(tokens[0]).toEqual value: 'console', scopes: ['source.js', 'entity.name.type.object.console.js'] + expect(tokens[2]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(tokens[4]).toEqual value: 'log', scopes: ['source.js', 'meta.method-call.js', 'support.function.console.js'] + expect(tokens[5]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[6]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(tokens[7]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + it "tokenizes console custom functions", -> + {tokens} = grammar.tokenizeLine('console.foo();') + expect(tokens[0]).toEqual value: 'console', scopes: ['source.js', 'entity.name.type.object.console.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(tokens[2]).toEqual value: 'foo', scopes: ['source.js', 'meta.method-call.js', 'entity.name.function.js'] + expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(tokens[5]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + {tokens} = grammar.tokenizeLine('console .foo();') + expect(tokens[3]).toEqual value: 'foo', scopes: ['source.js', 'meta.method-call.js', 'entity.name.function.js'] + + describe "math", -> + it "tokenizes the math object", -> + {tokens} = grammar.tokenizeLine('Math;') + expect(tokens[0]).toEqual value: 'Math', scopes: ['source.js', 'support.class.math.js'] + expect(tokens[1]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + it "tokenizes math support functions/properties", -> + {tokens} = grammar.tokenizeLine('Math.random();') + expect(tokens[0]).toEqual value: 'Math', scopes: ['source.js', 'support.class.math.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(tokens[2]).toEqual value: 'random', scopes: ['source.js', 'meta.method-call.js', 'support.function.math.js'] + expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(tokens[5]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + lines = grammar.tokenizeLines ''' + Math + .random(); + ''' + expect(lines[0][0]).toEqual value: 'Math', scopes: ['source.js', 'support.class.math.js'] + expect(lines[1][0]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(lines[1][1]).toEqual value: 'random', scopes: ['source.js', 'meta.method-call.js', 'support.function.math.js'] + expect(lines[1][2]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(lines[1][3]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(lines[1][4]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + {tokens} = grammar.tokenizeLine('Math.PI;') + expect(tokens[0]).toEqual value: 'Math', scopes: ['source.js', 'support.class.math.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.delimiter.property.period.js'] + expect(tokens[2]).toEqual value: 'PI', scopes: ['source.js', 'support.constant.property.math.js'] + expect(tokens[3]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + it "tokenizes math custom functions", -> + {tokens} = grammar.tokenizeLine('Math.PI();') + expect(tokens[0]).toEqual value: 'Math', scopes: ['source.js', 'support.class.math.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(tokens[2]).toEqual value: 'PI', scopes: ['source.js', 'meta.method-call.js', 'entity.name.function.js'] + expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(tokens[5]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + describe "promise", -> + it "tokenizes the promise object", -> + {tokens} = grammar.tokenizeLine('Promise;') + expect(tokens[0]).toEqual value: 'Promise', scopes: ['source.js', 'support.class.promise.js'] + expect(tokens[1]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + it "tokenizes promise support functions", -> + {tokens} = grammar.tokenizeLine('Promise.race();') + expect(tokens[0]).toEqual value: 'Promise', scopes: ['source.js', 'support.class.promise.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(tokens[2]).toEqual value: 'race', scopes: ['source.js', 'meta.method-call.js', 'support.function.promise.js'] + expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(tokens[5]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + lines = grammar.tokenizeLines ''' + Promise + .resolve(); + ''' + expect(lines[0][0]).toEqual value: 'Promise', scopes: ['source.js', 'support.class.promise.js'] + expect(lines[1][0]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(lines[1][1]).toEqual value: 'resolve', scopes: ['source.js', 'meta.method-call.js', 'support.function.promise.js'] + expect(lines[1][2]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(lines[1][3]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(lines[1][4]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + it "tokenizes promise custom functions", -> + {tokens} = grammar.tokenizeLine('Promise.anExtraFunction();') + expect(tokens[0]).toEqual value: 'Promise', scopes: ['source.js', 'support.class.promise.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(tokens[2]).toEqual value: 'anExtraFunction', scopes: ['source.js', 'meta.method-call.js', 'entity.name.function.js'] + expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(tokens[5]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + describe "object literals", -> + keywords = ['super', 'this', 'null', 'true', 'false', 'debugger', 'exports', '__filename'] + + for keyword in keywords + it "tokenizes the #{keyword} keyword when it is an object key", -> + {tokens} = grammar.tokenizeLine("#{keyword}: 1") + expect(tokens[0]).toEqual value: keyword, scopes: ['source.js'] + expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.assignment.js'] + + it "tokenizes object keys", -> + {tokens} = grammar.tokenizeLine('foo: 1') + expect(tokens[0]).toEqual value: 'foo', scopes: ['source.js'] + expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.assignment.js'] + + {tokens} = grammar.tokenizeLine('$abc$: 1') + expect(tokens[0]).toEqual value: '$abc$', scopes: ['source.js'] + expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.assignment.js'] + + {tokens} = grammar.tokenizeLine('0abc: 1') + expect(tokens[0]).toEqual value: '0abc', scopes: ['source.js', 'invalid.illegal.identifier.js'] + expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.assignment.js'] + + {tokens} = grammar.tokenizeLine('"key": 1') + expect(tokens[0]).toEqual value: '"', scopes: ['source.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: 'key', scopes: ['source.js', 'string.quoted.double.js'] + expect(tokens[2]).toEqual value: '"', scopes: ['source.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[3]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.assignment.js'] + + it "tokenizes numbers when they are object keys", -> + {tokens} = grammar.tokenizeLine('123: 1') + expect(tokens[0]).toEqual value: '123', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.assignment.js'] + + it "tokenizes constants when they are object keys", -> + {tokens} = grammar.tokenizeLine('FOO: 1') + expect(tokens[0]).toEqual value: 'FOO', scopes: ['source.js', 'constant.other.js'] + expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.assignment.js'] + + describe "ternary expressions", -> + map = + FOO: 'constant.other.js' + super: 'variable.language.js' + this: 'variable.language.js' + null: 'constant.language.null.js' + true: 'constant.language.boolean.true.js' + false: 'constant.language.boolean.false.js' + exports: 'support.variable.js' + __filename: 'support.variable.js' + + for keyword, scope of map + do (keyword, scope) -> + it "tokenizes `#{keyword}` in the middle of ternary expressions", -> + {tokens} = grammar.tokenizeLine("a ? #{keyword} : b") + expect(tokens[3]).toEqual value: keyword, scopes: ['source.js', scope] + + it "tokenizes `#{keyword}` at the end of ternary expressions", -> + {tokens} = grammar.tokenizeLine("a ? b : #{keyword}") + expect(tokens[7]).toEqual value: keyword, scopes: ['source.js', scope] + + it "tokenizes yield at the end of ternary expressions", -> + {tokens} = grammar.tokenizeLine('a ? b : yield') + expect(tokens[7]).toEqual value: 'yield', scopes: ['source.js', 'meta.control.yield.js', 'keyword.control.js'] + + it "tokenizes yield in the middle of ternary expressions", -> + {tokens} = grammar.tokenizeLine('a ? yield : b') + expect(tokens[3]).toEqual value: 'yield', scopes: ['source.js', 'meta.control.yield.js', 'keyword.control.js'] + + it "tokenizes regular expressions inside ternary expressions", -> + {tokens} = grammar.tokenizeLine('a ? /b/ : /c/') + expect(tokens[0]).toEqual value: 'a ', scopes: ['source.js'] + expect(tokens[1]).toEqual value: '?', scopes: ['source.js', 'keyword.operator.ternary.js'] + expect(tokens[2]).toEqual value: ' ', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[3]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[4]).toEqual value: 'b', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[5]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + expect(tokens[6]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[7]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.ternary.js'] + expect(tokens[8]).toEqual value: ' ', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[9]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[10]).toEqual value: 'c', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[11]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + + it "tokenizes object literals in the middle of ternary expressions", -> + {tokens} = grammar.tokenizeLine('a ? {key: value} : b') + expect(tokens[1]).toEqual value: '?', scopes: ['source.js', 'keyword.operator.ternary.js'] + expect(tokens[9]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.ternary.js'] + + it "tokenizes arrow functions inside ternary expressions", -> + {tokens} = grammar.tokenizeLine('result = condition ? something : (a, b) => a + b') + expect(tokens[3]).toEqual value: '?', scopes: ['source.js', 'keyword.operator.ternary.js'] + expect(tokens[7]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.ternary.js'] + expect(tokens[16]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] + + {tokens} = grammar.tokenizeLine('result = condition ? (a, b) => a + b : something') + expect(tokens[3]).toEqual value: '?', scopes: ['source.js', 'keyword.operator.ternary.js'] + expect(tokens[12]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] + expect(tokens[18]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.ternary.js'] + + describe "switch statements", -> + it "tokenizes the switch keyword", -> + {tokens} = grammar.tokenizeLine('switch(){}') + expect(tokens[0]).toEqual value: 'switch', scopes: ['source.js', 'meta.switch-statement.js', 'keyword.control.switch.js'] + + it "tokenizes switch expression", -> + {tokens} = grammar.tokenizeLine('switch(foo + bar){}') + expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.switch-statement.js', 'punctuation.definition.switch-expression.begin.bracket.round.js'] + expect(tokens[2]).toEqual value: 'foo ', scopes: ['source.js', 'meta.switch-statement.js'] + expect(tokens[3]).toEqual value: '+', scopes: ['source.js', 'meta.switch-statement.js', 'keyword.operator.js'] + expect(tokens[4]).toEqual value: ' bar', scopes: ['source.js', 'meta.switch-statement.js'] + expect(tokens[5]).toEqual value: ')', scopes: ['source.js', 'meta.switch-statement.js', 'punctuation.definition.switch-expression.end.bracket.round.js'] + + it "tokenizes switch block", -> + lines = grammar.tokenizeLines ''' + switch (foo()) + { + case abc: + case 1+1: + 2+2 + break; + case null: + default: + } + ''' + expect(lines[1][0]).toEqual value: '{', scopes: ['source.js', 'meta.switch-statement.js', 'punctuation.definition.section.switch-block.begin.bracket.curly.js'] + expect(lines[2][1]).toEqual value: 'case', scopes: ['source.js', 'meta.switch-statement.js', 'keyword.control.case.js'] + expect(lines[2][3]).toEqual value: 'abc', scopes: ['source.js', 'meta.switch-statement.js'] + expect(lines[2][4]).toEqual value: ':', scopes: ['source.js', 'meta.switch-statement.js', 'punctuation.definition.section.case-statement.js'] + expect(lines[3][1]).toEqual value: 'case', scopes: ['source.js', 'meta.switch-statement.js', 'keyword.control.case.js'] + expect(lines[3][3]).toEqual value: '1', scopes: ['source.js', 'meta.switch-statement.js', 'constant.numeric.decimal.js'] + expect(lines[3][4]).toEqual value: '+', scopes: ['source.js', 'meta.switch-statement.js', 'keyword.operator.js'] + expect(lines[3][5]).toEqual value: '1', scopes: ['source.js', 'meta.switch-statement.js', 'constant.numeric.decimal.js'] + expect(lines[3][6]).toEqual value: ':', scopes: ['source.js', 'meta.switch-statement.js', 'punctuation.definition.section.case-statement.js'] + expect(lines[4][1]).toEqual value: '2', scopes: ['source.js', 'meta.switch-statement.js', 'constant.numeric.decimal.js'] + expect(lines[4][2]).toEqual value: '+', scopes: ['source.js', 'meta.switch-statement.js', 'keyword.operator.js'] + expect(lines[4][3]).toEqual value: '2', scopes: ['source.js', 'meta.switch-statement.js', 'constant.numeric.decimal.js'] + expect(lines[5][1]).toEqual value: 'break', scopes: ['source.js', 'meta.switch-statement.js', 'keyword.control.js'] + expect(lines[5][2]).toEqual value: ';', scopes: ['source.js', 'meta.switch-statement.js', 'punctuation.terminator.statement.js'] + expect(lines[6][1]).toEqual value: 'case', scopes: ['source.js', 'meta.switch-statement.js', 'keyword.control.case.js'] + expect(lines[6][3]).toEqual value: 'null', scopes: ['source.js', 'meta.switch-statement.js', 'constant.language.null.js'] + expect(lines[6][4]).toEqual value: ':', scopes: ['source.js', 'meta.switch-statement.js', 'punctuation.definition.section.case-statement.js'] + expect(lines[7][1]).toEqual value: 'default', scopes: ['source.js', 'meta.switch-statement.js', 'keyword.control.default.js'] + expect(lines[7][2]).toEqual value: ':', scopes: ['source.js', 'meta.switch-statement.js', 'punctuation.definition.section.case-statement.js'] + expect(lines[8][0]).toEqual value: '}', scopes: ['source.js', 'meta.switch-statement.js', 'punctuation.definition.section.switch-block.end.bracket.curly.js'] + + describe "indentation", -> + editor = null + + beforeEach -> + editor = buildTextEditor() + editor.setGrammar(grammar) + + expectPreservedIndentation = (text) -> + editor.setText(text) + editor.autoIndentBufferRows(0, editor.getLineCount() - 1) + + expectedLines = text.split("\n") + actualLines = editor.getText().split("\n") + for actualLine, i in actualLines + expect([ + actualLine, + editor.indentLevelForLine(actualLine) + ]).toEqual([ + expectedLines[i], + editor.indentLevelForLine(expectedLines[i]) + ]) + + it "indents allman-style curly braces", -> + expectPreservedIndentation """ + if (true) + { + for (;;) + { + while (true) + { + x(); + } + } + } + else + { + do + { + y(); + } while (true); + } + """ + + it "indents non-allman-style curly braces", -> + expectPreservedIndentation """ + if (true) { + for (;;) { // " + while (true) { + x(); + } + } + } else { + do { + y(); + } while (true); + } + """ + + it "doesn't indent case statements, because it wouldn't know when to outdent", -> + expectPreservedIndentation """ + switch (e) { + case 5: + something(); + case 6: + somethingElse(); + } + """ + + it "indents collection literals", -> + expectPreservedIndentation """ + [ // " + { + a: b, + c: d + }, + e, + f + ] + """ + + it "indents function arguments", -> + expectPreservedIndentation """ + f( + g( // " + h, + i + ), + j + ); + """ + + describe "firstLineMatch", -> + it "recognises interpreter directives", -> + valid = """ + #!/usr/sbin/node foo + #!/usr/bin/iojs foo=bar/ + #!/usr/sbin/node + #!/usr/sbin/node foo bar baz + #!/usr/bin/node perl + #!/usr/bin/node bin/perl + #!/usr/bin/node + #!/bin/node + #!/usr/bin/node --script=usr/bin + #! /usr/bin/env A=003 B=149 C=150 D=xzd E=base64 F=tar G=gz H=head I=tail node + #!\t/usr/bin/env --foo=bar node --quu=quux + #! /usr/bin/node + #!/usr/bin/env node + """ + for line in valid.split /\n/ + expect(grammar.firstLineRegex.scanner.findNextMatchSync(line)).not.toBeNull() + + invalid = """ + \x20#!/usr/sbin/node + \t#!/usr/sbin/node + #!/usr/bin/env-node/node-env/ + #!/usr/bin/env-node + #! /usr/binnode + #!\t/usr/bin/env --node=bar + """ + for line in invalid.split /\n/ + expect(grammar.firstLineRegex.scanner.findNextMatchSync(line)).toBeNull() + + it "recognises Emacs modelines", -> + valid = """ + #-*-js-*- + #-*-mode:js-*- + /* -*-js-*- */ + // -*- JavaScript -*- + /* -*- mode:js -*- */ + // -*- font:bar;mode:JS -*- + // -*- font:bar;mode:JavaScript;foo:bar; -*- + // -*-font:mode;mode:JS-*- + // -*- foo:bar mode: js bar:baz -*- + " -*-foo:bar;mode:JS;bar:foo-*- "; + " -*-font-mode:foo;mode:JavaScript;foo-bar:quux-*-" + "-*-font:x;foo:bar; mode : js;bar:foo;foooooo:baaaaar;fo:ba;-*-"; + "-*- font:x;foo : bar ; mode : jS ; bar : foo ; foooooo:baaaaar;fo:ba-*-"; + """ + for line in valid.split /\n/ + expect(grammar.firstLineRegex.scanner.findNextMatchSync(line)).not.toBeNull() + + invalid = """ + /* --*js-*- */ + /* -*-- js -*- + /* -*- -- js -*- + /* -*- javascripts -;- -*- + // -*- iJS -*- + // -*- JS; -*- + // -*- js-stuff -*- + /* -*- model:js -*- + /* -*- indent-mode:js -*- + // -*- font:mode;JS -*- + // -*- mode: -*- JS + // -*- mode: grok-with-js -*- + // -*-font:mode;mode:js--*- + """ + for line in invalid.split /\n/ + expect(grammar.firstLineRegex.scanner.findNextMatchSync(line)).toBeNull() + + it "recognises Vim modelines", -> + valid = """ + vim: se filetype=javascript: + # vim: se ft=javascript: + # vim: set ft=javascript: + # vim: set filetype=JavaScript: + # vim: ft=javascript + # vim: syntax=jAvaScRIPT + # vim: se syntax=JAVASCRIPT: + # ex: syntax=javascript + # vim:ft=javascript + # vim600: ft=javascript + # vim>600: set ft=javascript: + # vi:noai:sw=3 ts=6 ft=javascript + # vi::::::::::noai:::::::::::: ft=javascript + # vim:ts=4:sts=4:sw=4:noexpandtab:ft=javascript + # vi:: noai : : : : sw =3 ts =6 ft =javascript + # vim: ts=4: pi sts=4: ft=javascript: noexpandtab: sw=4: + # vim: ts=4 sts=4: ft=javascript noexpandtab: + # vim:noexpandtab sts=4 ft=javascript ts=4 + # vim:noexpandtab:ft=javascript + # vim:ts=4:sts=4 ft=javascript:noexpandtab:\x20 + # vim:noexpandtab titlestring=hi\|there\\\\ ft=javascript ts=4 + """ + for line in valid.split /\n/ + expect(grammar.firstLineRegex.scanner.findNextMatchSync(line)).not.toBeNull() + + invalid = """ + ex: se filetype=javascript: + _vi: se filetype=javascript: + vi: se filetype=javascript + # vim set ft=javascripts + # vim: soft=javascript + # vim: hairy-syntax=javascript: + # vim set ft=javascript: + # vim: setft=javascript: + # vim: se ft=javascript backupdir=tmp + # vim: set ft=javascript set cmdheight=1 + # vim:noexpandtab sts:4 ft:javascript ts:4 + # vim:noexpandtab titlestring=hi\\|there\\ ft=javascript ts=4 + # vim:noexpandtab titlestring=hi\\|there\\\\\\ ft=javascript ts=4 + """ + for line in invalid.split /\n/ + expect(grammar.firstLineRegex.scanner.findNextMatchSync(line)).toBeNull() diff --git a/spec/jsdoc-spec.coffee b/spec/jsdoc-spec.coffee new file mode 100644 index 00000000..8baf4e28 --- /dev/null +++ b/spec/jsdoc-spec.coffee @@ -0,0 +1,1537 @@ +describe "JSDoc grammar", -> + grammar = null + + beforeEach -> + atom.config.set('core.useTreeSitterParsers', false) + + waitsForPromise -> + atom.packages.activatePackage("language-javascript") + + runs -> + grammar = atom.grammars.grammarForScopeName("source.js") + + describe "inline tags", -> + it "tokenises tags without descriptions", -> + {tokens} = grammar.tokenizeLine('/** Text {@link target} text */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[1]).toEqual value: ' Text ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[3]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.inline.tag.jsdoc'] + expect(tokens[4]).toEqual value: 'link', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc'] + expect(tokens[6]).toEqual value: 'target', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'variable.other.description.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[8]).toEqual value: ' text ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[9]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises tags with an embedded trailing description", -> + {tokens} = grammar.tokenizeLine('/** Text {@linkplain target|Description text} */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[1]).toEqual value: ' Text ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[3]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.inline.tag.jsdoc'] + expect(tokens[4]).toEqual value: 'linkplain', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc'] + expect(tokens[6]).toEqual value: 'target', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'variable.other.description.jsdoc'] + expect(tokens[7]).toEqual value: '|', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.separator.pipe.jsdoc'] + expect(tokens[8]).toEqual value: 'Description text', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[9]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** Text {@linkcode target Description text} */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[1]).toEqual value: ' Text ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[3]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.inline.tag.jsdoc'] + expect(tokens[4]).toEqual value: 'linkcode', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc'] + expect(tokens[6]).toEqual value: 'target', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'variable.other.description.jsdoc'] + expect(tokens[7]).toEqual value: ' Description text', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[8]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[10]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises tags with a preceding description", -> + {tokens} = grammar.tokenizeLine('/** Text [Description text]{@link target} */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[1]).toEqual value: ' Text ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[2]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'constant.other.description.jsdoc', 'punctuation.definition.bracket.square.begin.jsdoc'] + expect(tokens[3]).toEqual value: 'Description text', scopes: ['source.js', 'comment.block.documentation.js', 'constant.other.description.jsdoc'] + expect(tokens[4]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'constant.other.description.jsdoc', 'punctuation.definition.bracket.square.end.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.inline.tag.jsdoc'] + expect(tokens[7]).toEqual value: 'link', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc'] + expect(tokens[9]).toEqual value: 'target', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'variable.other.description.jsdoc'] + expect(tokens[10]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[12]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** Text [Description text]{@tutorial target|Description} */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[1]).toEqual value: ' Text ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[2]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'constant.other.description.jsdoc', 'punctuation.definition.bracket.square.begin.jsdoc'] + expect(tokens[3]).toEqual value: 'Description text', scopes: ['source.js', 'comment.block.documentation.js', 'constant.other.description.jsdoc'] + expect(tokens[4]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'constant.other.description.jsdoc', 'punctuation.definition.bracket.square.end.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.inline.tag.jsdoc'] + expect(tokens[7]).toEqual value: 'tutorial', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc'] + expect(tokens[9]).toEqual value: 'target', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'variable.other.description.jsdoc'] + expect(tokens[10]).toEqual value: '|', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.separator.pipe.jsdoc'] + expect(tokens[11]).toEqual value: 'Description', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[12]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[14]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises inline tags which follow block tags", -> + {tokens} = grammar.tokenizeLine('/** @param {object} variable - this is a {@link linked} description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' - this is a ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[12]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.inline.tag.jsdoc'] + expect(tokens[13]).toEqual value: 'link', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc'] + expect(tokens[15]).toEqual value: 'linked', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'variable.other.description.jsdoc'] + expect(tokens[16]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[17]).toEqual value: ' description ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[18]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} variable - this is a {@link linked#description}. */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' - this is a ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[12]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.inline.tag.jsdoc'] + expect(tokens[13]).toEqual value: 'link', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc'] + expect(tokens[15]).toEqual value: 'linked#description', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'variable.other.description.jsdoc'] + expect(tokens[16]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[17]).toEqual value: '. ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[18]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} variable - this is a [description with a]{@link example}. */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' - this is a ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'constant.other.description.jsdoc', 'punctuation.definition.bracket.square.begin.jsdoc'] + expect(tokens[12]).toEqual value: 'description with a', scopes: ['source.js', 'comment.block.documentation.js', 'constant.other.description.jsdoc'] + expect(tokens[13]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'constant.other.description.jsdoc', 'punctuation.definition.bracket.square.end.jsdoc'] + expect(tokens[14]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[15]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.inline.tag.jsdoc'] + expect(tokens[16]).toEqual value: 'link', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc'] + expect(tokens[18]).toEqual value: 'example', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'variable.other.description.jsdoc'] + expect(tokens[19]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[20]).toEqual value: '. ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[21]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises inline tags within default @param values", -> + {tokens} = grammar.tokenizeLine('/** @param {EntityType} [typeHint={@link EntityType.FILE}] */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'EntityType', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'typeHint', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[13]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.inline.tag.jsdoc'] + expect(tokens[14]).toEqual value: 'link', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc'] + expect(tokens[16]).toEqual value: 'EntityType.FILE', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'entity.name.type.instance.jsdoc', 'variable.other.description.jsdoc'] + expect(tokens[17]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[18]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[20]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + describe "block tags", -> + it "tokenises simple tags", -> + {tokens} = grammar.tokenizeLine('/** @mixins */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'mixins', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[4]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[5]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @global @static */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'global', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[4]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[5]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[6]).toEqual value: 'static', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[7]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[8]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises @see tags with basic links", -> + {tokens} = grammar.tokenizeLine('/** @see name#path */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'see', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[4]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[5]).toEqual value: 'name#path', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[6]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[7]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @see http://atom.io/ */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'see', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[4]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[5]).toEqual value: 'http://atom.io/', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.link.underline.jsdoc'] + expect(tokens[6]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[7]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises @see tags with {@link} tags", -> + {tokens} = grammar.tokenizeLine('/** @see {@link text|Description} */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'see', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.inline.tag.jsdoc'] + expect(tokens[7]).toEqual value: 'link', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc'] + expect(tokens[9]).toEqual value: 'text', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'variable.other.description.jsdoc'] + expect(tokens[10]).toEqual value: '|', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.separator.pipe.jsdoc'] + expect(tokens[11]).toEqual value: 'Description', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[12]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[14]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @see [Description]{@link name#path} */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'see', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '[Description]', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[6]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[7]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.inline.tag.jsdoc'] + expect(tokens[8]).toEqual value: 'link', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc'] + expect(tokens[10]).toEqual value: 'name#path', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'variable.other.description.jsdoc'] + expect(tokens[11]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[13]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises tags with type expressions", -> + {tokens} = grammar.tokenizeLine('/** @const {object} */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'const', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[4]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[8]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[9]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @define {object} */') + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'define', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + it "tokenises unnamed @param tags", -> + {tokens} = grammar.tokenizeLine('/** @param {object} */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[1]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[8]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[9]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises @param tags", -> + {tokens} = grammar.tokenizeLine('/** @param {object} variable */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises @param tags with a description", -> + {tokens} = grammar.tokenizeLine('/** @param {object} variable this is the description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @arg {object} variable this is the description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'arg', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @argument {object} variable this is the description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'argument', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} variable - this is the description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' - this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} $variable this is the description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '$variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises @param tags marked optional", -> + {tokens} = grammar.tokenizeLine('/** @param {object} [variable] this is the description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[12]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[13]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} [ variable ] this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: ' variable ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[12]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + describe "default @param values", -> + it "tokenises unquoted values", -> + {tokens} = grammar.tokenizeLine('/** @param {object} [variable=default value] this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: 'default value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[13]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[14]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} [variable = default value] this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'variable ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[13]).toEqual value: 'default value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[14]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[15]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} [ variable = default value ] this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: ' variable ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[13]).toEqual value: 'default value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[14]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[15]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[16]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} [variable=default.value] this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: 'default', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'variable.other.object.js'] + expect(tokens[13]).toEqual value: '.', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.delimiter.property.period.js'] + expect(tokens[14]).toEqual value: 'value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'support.variable.property.dom.js'] + expect(tokens[15]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[16]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + it "tokenises quoted values", -> + {tokens} = grammar.tokenizeLine('/** @param {object} [variable="default value"] this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[13]).toEqual value: 'default value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js'] + expect(tokens[14]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[15]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[16]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} [variable = "default value"] this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'variable ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[13]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[14]).toEqual value: 'default value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js'] + expect(tokens[15]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[16]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[17]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} [ variable = " default value " ] this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: ' variable ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[13]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[14]).toEqual value: ' default value ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js'] + expect(tokens[15]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[17]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[18]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine("/** @param {object} [variable='default value'] this is the description */") + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '\'', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js', 'punctuation.definition.string.begin.js'] + expect(tokens[13]).toEqual value: 'default value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js'] + expect(tokens[14]).toEqual value: '\'', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js', 'punctuation.definition.string.end.js'] + expect(tokens[15]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[16]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine("/** @param {object} [variable = 'default value'] this is the description */") + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'variable ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[13]).toEqual value: '\'', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js', 'punctuation.definition.string.begin.js'] + expect(tokens[14]).toEqual value: 'default value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js'] + expect(tokens[15]).toEqual value: '\'', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js', 'punctuation.definition.string.end.js'] + expect(tokens[16]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[17]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine("/** @param {object} [ variable = ' default value ' ] this is the description */") + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: ' variable ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[13]).toEqual value: '\'', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js', 'punctuation.definition.string.begin.js'] + expect(tokens[14]).toEqual value: ' default value ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js'] + expect(tokens[15]).toEqual value: '\'', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js', 'punctuation.definition.string.end.js'] + expect(tokens[17]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[18]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + it "tokenises object literals", -> + {tokens} = grammar.tokenizeLine('/** @param {Object} [variable={a: "b"}] - An object */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[13]).toEqual value: 'a', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[14]).toEqual value: ':', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'keyword.operator.assignment.js'] + expect(tokens[16]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[17]).toEqual value: 'b', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js'] + expect(tokens[18]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[19]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[20]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[21]).toEqual value: ' - An object ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Object} [ variable = { a : "b" } ] - An object */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: ' variable ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[13]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[14]).toEqual value: ' a ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[15]).toEqual value: ':', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'keyword.operator.assignment.js'] + expect(tokens[17]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[18]).toEqual value: 'b', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js'] + expect(tokens[19]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[21]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[23]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[24]).toEqual value: ' - An object ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Object} [variable={}] - Empty object */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'punctuation.section.scope.begin.js'] + expect(tokens[13]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'punctuation.section.scope.end.js'] + expect(tokens[14]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[15]).toEqual value: ' - Empty object ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Object} [ variable = { } ] - Empty object */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: ' variable ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[13]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[15]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[17]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[18]).toEqual value: ' - Empty object ', scopes: ['source.js', 'comment.block.documentation.js'] + + it "tokenises arrays", -> + {tokens} = grammar.tokenizeLine('/** @param {Array} [variable=[1,2,3]] - An array */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Array', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[13]).toEqual value: '1', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'constant.numeric.decimal.js'] + expect(tokens[14]).toEqual value: ',', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.delimiter.object.comma.js'] + expect(tokens[15]).toEqual value: '2', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'constant.numeric.decimal.js'] + expect(tokens[16]).toEqual value: ',', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.delimiter.object.comma.js'] + expect(tokens[17]).toEqual value: '3', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'constant.numeric.decimal.js'] + expect(tokens[18]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[19]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[20]).toEqual value: ' - An array ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Array} [ variable = [ 1 , 2 , 3 ] ] - An array */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Array', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: ' variable ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[13]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[15]).toEqual value: '1', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'constant.numeric.decimal.js'] + expect(tokens[17]).toEqual value: ',', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.delimiter.object.comma.js'] + expect(tokens[19]).toEqual value: '2', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'constant.numeric.decimal.js'] + expect(tokens[21]).toEqual value: ',', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.delimiter.object.comma.js'] + expect(tokens[23]).toEqual value: '3', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'constant.numeric.decimal.js'] + expect(tokens[25]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[27]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[28]).toEqual value: ' - An array ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Array} [variable=[]] - Empty array */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Array', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[13]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[14]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[15]).toEqual value: ' - Empty array ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Array} [ variable = [ ] ] - Empty array */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Array', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: ' variable ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[13]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[15]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[17]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[18]).toEqual value: ' - Empty array ', scopes: ['source.js', 'comment.block.documentation.js'] + + it "tokenizes arrays inside object literals", -> + {tokens} = grammar.tokenizeLine('/** @param {Object} [thing={a: [], b: [0, 2], c: "String"}] [Not Highlighted] [] [] [] */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'thing', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[13]).toEqual value: 'a', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[14]).toEqual value: ':', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'keyword.operator.assignment.js'] + expect(tokens[16]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[17]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[18]).toEqual value: ',', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.delimiter.object.comma.js'] + expect(tokens[19]).toEqual value: ' b', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[20]).toEqual value: ':', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'keyword.operator.assignment.js'] + expect(tokens[22]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[23]).toEqual value: '0', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'constant.numeric.decimal.js'] + expect(tokens[24]).toEqual value: ',', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.delimiter.object.comma.js'] + expect(tokens[26]).toEqual value: '2', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'constant.numeric.decimal.js'] + expect(tokens[27]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[28]).toEqual value: ',', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.delimiter.object.comma.js'] + expect(tokens[29]).toEqual value: ' c', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[30]).toEqual value: ':', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'keyword.operator.assignment.js'] + expect(tokens[32]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[33]).toEqual value: 'String', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js'] + expect(tokens[34]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[35]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[36]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[37]).toEqual value: ' [Not Highlighted] [] [] [] ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[38]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "flags any description text touching the closing bracket", -> + {tokens} = grammar.tokenizeLine('/** @param {Object} [thing={a: [], b: [0, 2], c: "String"}][Bad Description] [] [] [] */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'thing', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[13]).toEqual value: 'a', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[14]).toEqual value: ':', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'keyword.operator.assignment.js'] + expect(tokens[16]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[17]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[18]).toEqual value: ',', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.delimiter.object.comma.js'] + expect(tokens[19]).toEqual value: ' b', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[20]).toEqual value: ':', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'keyword.operator.assignment.js'] + expect(tokens[22]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[23]).toEqual value: '0', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'constant.numeric.decimal.js'] + expect(tokens[24]).toEqual value: ',', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.delimiter.object.comma.js'] + expect(tokens[26]).toEqual value: '2', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'constant.numeric.decimal.js'] + expect(tokens[27]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[28]).toEqual value: ',', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.delimiter.object.comma.js'] + expect(tokens[29]).toEqual value: ' c', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[30]).toEqual value: ':', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'keyword.operator.assignment.js'] + expect(tokens[32]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[33]).toEqual value: 'String', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js'] + expect(tokens[34]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[35]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[36]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[37]).toEqual value: '[Bad', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'invalid.illegal.syntax.jsdoc'] + expect(tokens[38]).toEqual value: ' Description] [] [] [] ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[39]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "does not tokenise arrays inside strings", -> + {tokens} = grammar.tokenizeLine('/** @param {Object} [thing=\'{a: [], b: [0, 2], c: "String"}][Quoted Description] [\'] [Unquoted Description] */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'thing', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: "'", scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js', 'punctuation.definition.string.begin.js'] + expect(tokens[13]).toEqual value: '{a: [], b: [0, 2], c: "String"}][Quoted Description] [', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js'] + expect(tokens[14]).toEqual value: "'", scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js', 'punctuation.definition.string.end.js'] + expect(tokens[15]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[16]).toEqual value: ' [Unquoted Description] ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[17]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Object} [thing="{a: [], b: [0, 2], c: \'String\'}][Quoted Description] ["] [] [] */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'thing', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[13]).toEqual value: '{a: [], b: [0, 2], c: \'String\'}][Quoted Description] [', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js'] + expect(tokens[14]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[15]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[16]).toEqual value: ' [] [] ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[17]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Object} [thing="{a: [], b: [0, 2], c: \'String\'}][Quoted Description] ["][Bad Description] [] */') + expect(tokens[14]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[15]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[16]).toEqual value: '[Bad', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'invalid.illegal.syntax.jsdoc'] + expect(tokens[17]).toEqual value: ' Description] [] ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Object} [thing=\'{a: [], b: [0, 2], c: "String"}][Quoted Description] [\'][Bad_Unquoted Description] */') + expect(tokens[14]).toEqual value: "'", scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js', 'punctuation.definition.string.end.js'] + expect(tokens[15]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[16]).toEqual value: '[Bad_Unquoted', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'invalid.illegal.syntax.jsdoc'] + expect(tokens[17]).toEqual value: ' Description] ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[18]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises escape sequences inside strings", -> + {tokens} = grammar.tokenizeLine('/** @param {String} [key="a[\\"]z"] */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'String', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'key', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[13]).toEqual value: 'a[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js'] + expect(tokens[14]).toEqual value: '\\"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'constant.character.escape.js'] + expect(tokens[15]).toEqual value: ']z', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js'] + expect(tokens[16]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[17]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[19]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine("/** @param {String} [key='a[\\']z'] */") + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'String', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'key', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '\'', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js', 'punctuation.definition.string.begin.js'] + expect(tokens[13]).toEqual value: 'a[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js'] + expect(tokens[14]).toEqual value: '\\\'', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js', 'constant.character.escape.js'] + expect(tokens[15]).toEqual value: ']z', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js'] + expect(tokens[16]).toEqual value: '\'', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js', 'punctuation.definition.string.end.js'] + expect(tokens[17]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[19]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises @param tags with accessor-style names", -> + {tokens} = grammar.tokenizeLine('/** @param {object} parameter.property this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'parameter.property', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} [parameter.property] this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'parameter.property', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[12]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} [ parameter.property ] this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: ' parameter.property ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[12]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} [parameter.property=default value] this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'parameter.property', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: 'default value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[13]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[14]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} [parameter.property = default value] this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'parameter.property ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[13]).toEqual value: 'default value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[14]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[15]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} [ parameter.property = default value ] this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: ' parameter.property ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[13]).toEqual value: 'default value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[14]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[15]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[16]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + it "tokenises @param tags with wildcard types", -> + {tokens} = grammar.tokenizeLine('/** @param {*} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '*', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + {tokens} = grammar.tokenizeLine('/** @param {?} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '?', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + it "tokenises @param tags with qualified types", -> + {tokens} = grammar.tokenizeLine('/** @param {myNamespace.MyClass} variable this is the description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'myNamespace.MyClass', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Foo~cb} variable this is the description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Foo~cb', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises @param tags with multiple types", -> + {tokens} = grammar.tokenizeLine('/** @param {function|string} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function|string', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {string[]|number} variable this is the description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'string', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[8]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[9]).toEqual value: '|number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[10]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[12]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[13]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {string|number[]} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'string|number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[8]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[9]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[11]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {(number|function)} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '(number|function)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {(string[]|number)} variable this is the description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '(string', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[8]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[9]).toEqual value: '|number)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[10]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[12]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[13]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[14]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @param {(string|number[])} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '(string|number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[8]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[9]).toEqual value: ')', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[10]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[12]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[13]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + it "tokenises @param tags marked nullable or non-nullable", -> + {tokens} = grammar.tokenizeLine('/** @param {?number} variable this is the description */') + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '?number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {!number} variable this is the description */') + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '!number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + it "tokenises @param tags marked as variable-length", -> + {tokens} = grammar.tokenizeLine('/** @param {...number} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '...number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {...*} remainder */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '...*', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'remainder', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @param {...?} remainder */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '...?', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'remainder', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises @param tags using Google Closure Compiler syntax", -> + {tokens} = grammar.tokenizeLine('/** @param {number=} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'number=', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {number[]} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[8]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[9]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[11]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {Foo[].bar} variable this is the description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Foo', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[8]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[9]).toEqual value: '.bar', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[10]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[12]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[13]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[14]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Array} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Array', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {Array.} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Array.', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Array|Array} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Array|Array', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Array.|Array.} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Array.|Array.', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {(Array|Array)} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '(Array|Array)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {(Array.|Array.)} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '(Array.|Array.)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Object} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Object.} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Object.', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Object|Array} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Object|Array', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Object.|Array.} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Object.|Array.', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {(Array|Object)} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '(Array|Object)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {(Array.|Object.)} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '(Array.|Object.)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function()} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function()', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function ()} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function ()', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function ( )} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function ( )', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function(string)} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function(string, number)} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string, number)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function(...string)} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(...string)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function(string, ...number)} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string, ...number)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function(string, number, ...number)} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string, number, ...number)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function(!string)} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(!string)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function(?string, !number)} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(?string, !number)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function(string[], number=)} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[8]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[9]).toEqual value: ', number=)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[10]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[12]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function():number} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function():number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function(string): number} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string): number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function(string) : number} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string) : number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + it "tokenises @return tags without descriptions", -> + {tokens} = grammar.tokenizeLine('/** @return {object} */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'return', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @returns {object} */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'returns', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises @return tags with trailing descriptions", -> + {tokens} = grammar.tokenizeLine('/** @returns {object} this is the description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'returns', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'this', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[10]).toEqual value: ' is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @return {object} this is the description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'return', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'this', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[10]).toEqual value: ' is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @returns {(Something)} */') + expect(tokens[3]).toEqual value: 'returns', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '(Something)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + it "tokenises @return tags with multiple types", -> + {tokens} = grammar.tokenizeLine('/** @return {Some|Thing} Something to return */') + expect(tokens[3]).toEqual value: 'return', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Some|Thing', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {(String[]|Number[])} Description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'return', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '(String', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[8]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[9]).toEqual value: '|Number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[10]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[11]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[12]).toEqual value: ')', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[13]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[15]).toEqual value: 'Description', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[17]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises function-type @return tags", -> + {tokens} = grammar.tokenizeLine('/** @return {function()} this is the description */') + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'return', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function()', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {function ()} this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function ()', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {function ( )} this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function ( )', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {function(string)} this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {function(string, number)} this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string, number)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {function(...string)} this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(...string)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {function(string, ...number)} this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string, ...number)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {function(string, number, ...number)} this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string, number, ...number)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {function(!string)} this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(!string)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {function(?string, !number)} this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(?string, !number)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {function(string[], number=)} this is the description */') + expect(tokens[3]).toEqual value: 'return', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[8]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[9]).toEqual value: ', number=)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[10]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {function():number} this is the description */') + expect(tokens[3]).toEqual value: 'return', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function():number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {function(string): number} this is the description */') + expect(tokens[3]).toEqual value: 'return', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string): number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {function(string) : number} this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string) : number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + describe "highlighted JavaScript examples", -> + it "highlights JavaScript after an @example tag", -> + lines = grammar.tokenizeLines """ + /** + * @example foo("bar"); + */ + """ + expect(lines[1][0]).toEqual value: ' * ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(lines[1][1]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(lines[1][2]).toEqual value: 'example', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'storage.type.class.jsdoc'] + expect(lines[1][4]).toEqual value: 'foo', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(lines[1][5]).toEqual value: '(', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(lines[1][6]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(lines[1][7]).toEqual value: 'bar', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js'] + expect(lines[1][8]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(lines[1][9]).toEqual value: ')', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(lines[1][10]).toEqual value: ';', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'punctuation.terminator.statement.js'] + expect(lines[2][1]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + lines = grammar.tokenizeLines """ + /** + * @example + * foo("bar"); + */ + """ + expect(lines[1][0]).toEqual value: ' * ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(lines[1][1]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(lines[1][2]).toEqual value: 'example', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'storage.type.class.jsdoc'] + expect(lines[1][3]).toEqual value: '', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc'] + expect(lines[2][0]).toEqual value: ' * ', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc'] + expect(lines[2][1]).toEqual value: 'foo', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(lines[2][2]).toEqual value: '(', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(lines[2][3]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(lines[2][4]).toEqual value: 'bar', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js'] + expect(lines[2][5]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(lines[2][6]).toEqual value: ')', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(lines[2][7]).toEqual value: ';', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'punctuation.terminator.statement.js'] + expect(lines[3][1]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + lines = grammar.tokenizeLines """ + /** + * @example foo("bar"); + * foo(foo("bar")); // Comment + * 4 + 50; + */ + """ + expect(lines[1][0]).toEqual value: ' * ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(lines[1][1]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(lines[1][2]).toEqual value: 'example', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'storage.type.class.jsdoc'] + expect(lines[1][4]).toEqual value: 'foo', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(lines[1][5]).toEqual value: '(', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(lines[1][6]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(lines[1][7]).toEqual value: 'bar', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js'] + expect(lines[1][8]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(lines[1][9]).toEqual value: ')', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(lines[1][10]).toEqual value: ';', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'punctuation.terminator.statement.js'] + expect(lines[2][0]).toEqual value: ' * ', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc'] + expect(lines[2][1]).toEqual value: 'foo', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(lines[2][2]).toEqual value: '(', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(lines[2][3]).toEqual value: 'foo', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(lines[2][4]).toEqual value: '(', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(lines[2][5]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(lines[2][6]).toEqual value: 'bar', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js'] + expect(lines[2][7]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(lines[2][8]).toEqual value: ')', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(lines[2][9]).toEqual value: ')', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(lines[2][10]).toEqual value: ';', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'punctuation.terminator.statement.js'] + expect(lines[2][12]).toEqual value: '//', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'comment.line.double-slash.js', 'punctuation.definition.comment.js'] + expect(lines[2][13]).toEqual value: ' Comment', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'comment.line.double-slash.js'] + expect(lines[3][0]).toEqual value: ' * ', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc'] + expect(lines[3][1]).toEqual value: '4', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'constant.numeric.decimal.js'] + expect(lines[3][3]).toEqual value: '+', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'keyword.operator.js'] + expect(lines[3][5]).toEqual value: '50', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'constant.numeric.decimal.js'] + expect(lines[3][6]).toEqual value: ';', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'punctuation.terminator.statement.js'] + expect(lines[4][1]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises tags at the start of an example block", -> + {tokens} = grammar.tokenizeLine('/** @example Text */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'example', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '<', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'entity.name.tag.inline.jsdoc', 'punctuation.definition.bracket.angle.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'caption', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'entity.name.tag.inline.jsdoc'] + expect(tokens[7]).toEqual value: '>', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'entity.name.tag.inline.jsdoc', 'punctuation.definition.bracket.angle.end.jsdoc'] + expect(tokens[8]).toEqual value: 'Text', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'constant.other.description.jsdoc'] + expect(tokens[9]).toEqual value: '', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'entity.name.tag.inline.jsdoc', 'punctuation.definition.bracket.angle.end.jsdoc'] + expect(tokens[13]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + lines = grammar.tokenizeLines """ + /** + * @example Text + * foo("bar"); + * @return {String} + */ + """ + expect(lines[0][0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(lines[1][0]).toEqual value: ' * ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(lines[1][1]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(lines[1][2]).toEqual value: 'example', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'storage.type.class.jsdoc'] + expect(lines[1][4]).toEqual value: '<', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'entity.name.tag.inline.jsdoc', 'punctuation.definition.bracket.angle.begin.jsdoc'] + expect(lines[1][5]).toEqual value: 'caption', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'entity.name.tag.inline.jsdoc'] + expect(lines[1][6]).toEqual value: '>', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'entity.name.tag.inline.jsdoc', 'punctuation.definition.bracket.angle.end.jsdoc'] + expect(lines[1][7]).toEqual value: 'Text', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'constant.other.description.jsdoc'] + expect(lines[1][8]).toEqual value: '', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'entity.name.tag.inline.jsdoc', 'punctuation.definition.bracket.angle.end.jsdoc'] + expect(lines[2][0]).toEqual value: ' * ', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc'] + expect(lines[2][1]).toEqual value: 'foo', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(lines[2][2]).toEqual value: '(', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(lines[2][3]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(lines[2][4]).toEqual value: 'bar', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js'] + expect(lines[2][5]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(lines[2][6]).toEqual value: ')', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(lines[2][7]).toEqual value: ';', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'punctuation.terminator.statement.js'] + expect(lines[3][0]).toEqual value: ' * ', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc'] + expect(lines[3][1]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(lines[3][2]).toEqual value: 'return', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(lines[3][4]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(lines[3][5]).toEqual value: 'String', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(lines[3][6]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(lines[3][7]).toEqual value: '', scopes: ['source.js', 'comment.block.documentation.js'] + expect(lines[4][1]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + + describe "when the containing comment ends unexpectedly", -> + it "terminates any unclosed tags", -> + {tokens} = grammar.tokenizeLine('/** @param {String */ aa') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'String ', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + expect(tokens[8]).toEqual value: ' aa', scopes: ['source.js'] + + {tokens} = grammar.tokenizeLine('/** @param {*} [name={value: {a:[{*/}}]}] */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '*', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'name', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[13]).toEqual value: 'value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[14]).toEqual value: ':', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'keyword.operator.assignment.js'] + expect(tokens[16]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[17]).toEqual value: 'a', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[18]).toEqual value: ':', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'keyword.operator.assignment.js'] + expect(tokens[19]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[20]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[21]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + expect(tokens[22]).toEqual value: '}}', scopes: ['source.js'] + expect(tokens[23]).toEqual value: ']', scopes: ['source.js', 'meta.brace.square.js'] + expect(tokens[24]).toEqual value: '}', scopes: ['source.js'] + expect(tokens[25]).toEqual value: ']', scopes: ['source.js', 'meta.brace.square.js'] + expect(tokens[27]).toEqual value: '*', scopes: ['source.js', 'keyword.operator.js'] + expect(tokens[28]).toEqual value: '/', scopes: ['source.js', 'keyword.operator.js'] + + {tokens} = grammar.tokenizeLine('/** @param {string="Foo*/oo"} bar') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'string="Foo', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + expect(tokens[8]).toEqual value: 'oo', scopes: ['source.js'] + expect(tokens[9]).toEqual value: '"', scopes: ['source.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[10]).toEqual value: '} bar', scopes: ['source.js', 'string.quoted.double.js'] + + it "terminates any embedded JavaScript code", -> + lines = grammar.tokenizeLines """ + /** + * @example + * foo("bar"); /* Comment */ + * @return {String} + */ + """ + expect(lines[0][0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(lines[1][0]).toEqual value: ' * ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(lines[1][1]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(lines[1][2]).toEqual value: 'example', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'storage.type.class.jsdoc'] + expect(lines[1][3]).toEqual value: '', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc'] + expect(lines[2][0]).toEqual value: ' * ', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc'] + expect(lines[2][1]).toEqual value: 'foo', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(lines[2][2]).toEqual value: '(', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(lines[2][3]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(lines[2][4]).toEqual value: 'bar', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js'] + expect(lines[2][5]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(lines[2][6]).toEqual value: ')', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(lines[2][7]).toEqual value: ';', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'punctuation.terminator.statement.js'] + expect(lines[2][9]).toEqual value: '/*', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'comment.block.js', 'punctuation.definition.comment.begin.js'] + expect(lines[2][10]).toEqual value: ' Comment ', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'comment.block.js'] + expect(lines[2][11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + expect(lines[3][1]).toEqual value: '*', scopes: ['source.js', 'keyword.operator.js'] + expect(lines[3][2]).toEqual value: ' @', scopes: ['source.js'] + expect(lines[3][3]).toEqual value: 'return', scopes: ['source.js', 'keyword.control.js'] + expect(lines[3][5]).toEqual value: '{', scopes: ['source.js', 'meta.brace.curly.js'] + expect(lines[3][6]).toEqual value: 'String', scopes: ['source.js', 'support.class.js'] + expect(lines[3][7]).toEqual value: '}', scopes: ['source.js', 'meta.brace.curly.js'] + expect(lines[4][1]).toEqual value: '*', scopes: ['source.js', 'keyword.operator.js'] + expect(lines[4][2]).toEqual value: '/', scopes: ['source.js', 'keyword.operator.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Something} [value={key: [value, ["22"}] */ 20;') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Something', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[13]).toEqual value: 'key', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[14]).toEqual value: ':', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'keyword.operator.assignment.js'] + expect(tokens[16]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[17]).toEqual value: 'value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[18]).toEqual value: ',', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.delimiter.object.comma.js'] + expect(tokens[20]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[21]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[22]).toEqual value: '22', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js'] + expect(tokens[23]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[24]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[25]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[27]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + expect(tokens[29]).toEqual value: '20', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[30]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + describe "when the line ends without a closing bracket", -> + it "does not attempt to match the optional value (regression)", -> + {tokens} = grammar.tokenizeLine('/** @param {array} [bar = "x" REMOVE THE CLOSE BRACKET HERE.') + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: ' = "x" REMOVE THE CLOSE BRACKET HERE.', scopes: ['source.js', 'comment.block.documentation.js'] diff --git a/spec/regular-expression-replacement-spec.coffee b/spec/regular-expression-replacement-spec.coffee new file mode 100644 index 00000000..735aa823 --- /dev/null +++ b/spec/regular-expression-replacement-spec.coffee @@ -0,0 +1,87 @@ +describe "Regular Expression Replacement grammar", -> + grammar = null + + beforeEach -> + atom.config.set('core.useTreeSitterParsers', false) + + waitsForPromise -> + atom.packages.activatePackage("language-javascript") + + runs -> + grammar = atom.grammars.grammarForScopeName("source.js.regexp.replacement") + + it "parses the grammar", -> + expect(grammar).toBeTruthy() + expect(grammar.scopeName).toBe "source.js.regexp.replacement" + + describe "basic strings", -> + it "tokenizes with no extra scopes", -> + {tokens} = grammar.tokenizeLine('Hello [world]. (hi to you)') + expect(tokens[0]).toEqual value: 'Hello [world]. (hi to you)', scopes: ['source.js.regexp.replacement'] + + describe "escaped characters", -> + it "tokenizes with as an escape character", -> + {tokens} = grammar.tokenizeLine('\\n') + expect(tokens[0]).toEqual value: '\\n', scopes: ['source.js.regexp.replacement', 'constant.character.escape.backslash.regexp.replacement'] + + it "tokenizes '$$' as an escaped '$' character", -> + {tokens} = grammar.tokenizeLine('$$') + expect(tokens[0]).toEqual value: '$$', scopes: ['source.js.regexp.replacement', 'constant.character.escape.dollar.regexp.replacement'] + + it "doesn't treat '\\$' as an escaped '$' character", -> + {tokens} = grammar.tokenizeLine('\\$') + expect(tokens[0]).toEqual value: '\\$', scopes: ['source.js.regexp.replacement'] + + it "tokenizes '$$1' as an escaped '$' character followed by a '1' character", -> + {tokens} = grammar.tokenizeLine('$$1') + expect(tokens[0]).toEqual value: '$$', scopes: ['source.js.regexp.replacement', 'constant.character.escape.dollar.regexp.replacement'] + expect(tokens[1]).toEqual value: '1', scopes: ['source.js.regexp.replacement'] + + describe "Numeric placeholders", -> + it "doesn't tokenize $0 as a variable", -> + {tokens} = grammar.tokenizeLine('$0') + expect(tokens[0]).toEqual value: '$0', scopes: ['source.js.regexp.replacement'] + + it "doesn't tokenize $00 as a variable", -> + {tokens} = grammar.tokenizeLine('$00') + expect(tokens[0]).toEqual value: '$00', scopes: ['source.js.regexp.replacement'] + + it "tokenizes $1 as a variable", -> + {tokens} = grammar.tokenizeLine('$1') + expect(tokens[0]).toEqual value: '$1', scopes: ['source.js.regexp.replacement', 'variable.regexp.replacement'] + + it "tokenizes $01 as a variable", -> + {tokens} = grammar.tokenizeLine('$01') + expect(tokens[0]).toEqual value: '$01', scopes: ['source.js.regexp.replacement', 'variable.regexp.replacement'] + + it "tokenizes $3 as a variable", -> + {tokens} = grammar.tokenizeLine('$3') + expect(tokens[0]).toEqual value: '$3', scopes: ['source.js.regexp.replacement', 'variable.regexp.replacement'] + + it "tokenizes $10 as a variable", -> + {tokens} = grammar.tokenizeLine('$10') + expect(tokens[0]).toEqual value: '$10', scopes: ['source.js.regexp.replacement', 'variable.regexp.replacement'] + + it "tokenizes $99 as a variable", -> + {tokens} = grammar.tokenizeLine('$99') + expect(tokens[0]).toEqual value: '$99', scopes: ['source.js.regexp.replacement', 'variable.regexp.replacement'] + + it "doesn't tokenize the third numberic character in '$100' as a variable", -> + {tokens} = grammar.tokenizeLine('$100') + expect(tokens[0]).toEqual value: '$10', scopes: ['source.js.regexp.replacement', 'variable.regexp.replacement'] + expect(tokens[1]).toEqual value: '0', scopes: ['source.js.regexp.replacement'] + + describe "Matched sub-string placeholder", -> + it "tokenizes $& as a variable", -> + {tokens} = grammar.tokenizeLine('$&') + expect(tokens[0]).toEqual value: '$&', scopes: ['source.js.regexp.replacement', 'variable.regexp.replacement'] + + describe "Preceeding portion placeholder", -> + it "tokenizes $` as a variable", -> + {tokens} = grammar.tokenizeLine('$`') + expect(tokens[0]).toEqual value: '$`', scopes: ['source.js.regexp.replacement', 'variable.regexp.replacement'] + + describe "Following portion placeholder", -> + it "tokenizes $' as a variable", -> + {tokens} = grammar.tokenizeLine('$\'') + expect(tokens[0]).toEqual value: '$\'', scopes: ['source.js.regexp.replacement', 'variable.regexp.replacement']