diff --git a/.coffeelintignore b/.coffeelintignore deleted file mode 100644 index 1db51fed..00000000 --- a/.coffeelintignore +++ /dev/null @@ -1 +0,0 @@ -spec/fixtures diff --git a/.github/no-response.yml b/.github/no-response.yml new file mode 100644 index 00000000..1c8799d1 --- /dev/null +++ b/.github/no-response.yml @@ -0,0 +1,15 @@ +# Configuration for probot-no-response - https://github.com/probot/no-response + +# Number of days of inactivity before an issue is closed for lack of response +daysUntilClose: 28 + +# Label requiring a response +responseRequiredLabel: more-information-needed + +# Comment to post when closing an issue for lack of response. Set to `false` to disable. +closeComment: > + This issue has been automatically closed because there has been no response + to our request for more information from the original author. With only the + information that is currently in the issue, we don't have enough information + to take action. Please reach out if you have or find the answers we need so + that we can investigate further. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..432a2dce --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,45 @@ +name: ci +on: + - pull_request + - push + +jobs: + Test: + if: "!contains(github.event.head_commit.message, '[skip ci]')" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - macos-latest + - windows-latest + atom_channel: + - stable + - nightly + steps: + - uses: actions/checkout@v2 + - name: Cache + uses: actions/cache@v2 + with: + path: | + 'node_modules' + 'C:/Program Files (x86)/MSBuild/Microsoft.Cpp/v4.0/v140' + key: ${{ runner.os }}-${{ matrix.atom_channel }}-${{ hashFiles('package.json') }} + + - uses: UziTech/action-setup-atom@v1 + with: + channel: ${{ matrix.atom_channel }} + + - name: Install dependencies + run: apm install + + - name: Run tests + run: apm test + + Skip: + if: contains(github.event.head_commit.message, '[skip ci]') + runs-on: ubuntu-latest + steps: + - name: Skip CI 🚫 + run: echo skip ci diff --git a/.gitignore b/.gitignore index 3c3629e6..d5f19d89 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ node_modules +package-lock.json diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index a6b772f1..00000000 --- a/.travis.yml +++ /dev/null @@ -1,12 +0,0 @@ -language: objective-c - -notifications: - email: - on_success: never - on_failure: change - -script: 'curl -s https://raw.githubusercontent.com/atom/ci/master/build-package.sh | sh' - -branches: - only: - - master diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e70782fb..0fd0ad69 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1 +1 @@ -See the [Atom contributing guide](https://atom.io/docs/latest/contributing) +See the [Atom contributing guide](https://github.com/atom/atom/blob/master/CONTRIBUTING.md) diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md new file mode 100644 index 00000000..b60bb86c --- /dev/null +++ b/ISSUE_TEMPLATE.md @@ -0,0 +1,40 @@ + + +### Prerequisites + +* [ ] Put an X between the brackets on this line if you have done all of the following: + * Reproduced the problem in Safe Mode: http://flight-manual.atom.io/hacking-atom/sections/debugging/#using-safe-mode + * Followed all applicable steps in the debugging guide: http://flight-manual.atom.io/hacking-atom/sections/debugging/ + * Checked the FAQs on the message board for common solutions: https://discuss.atom.io/c/faq + * Checked that your issue isn't already filed: https://github.com/issues?utf8=✓&q=is%3Aissue+user%3Aatom + * Checked that there is not already an Atom package that provides the described functionality: https://atom.io/packages + +### Description + +[Description of the issue] + +### Steps to Reproduce + +1. [First Step] +2. [Second Step] +3. [and so on...] + +**Expected behavior:** [What you expect to happen] + +**Actual behavior:** [What actually happens] + +**Reproduces how often:** [What percentage of the time does it reproduce?] + +### Versions + +You can get this information from copy and pasting the output of `atom --version` and `apm --version` from the command line. Also, please include the OS and what version of the OS you're running. + +### Additional Information + +Any additional information, configuration or data that might be necessary to reproduce the issue. diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..cdaa94a8 --- /dev/null +++ b/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,28 @@ +### Requirements + +* Filling out the template is required. Any pull request that does not include enough information to be reviewed in a timely manner may be closed at the maintainers' discretion. +* All new code requires tests to ensure against regressions + +### Description of the Change + + + +### Alternate Designs + + + +### Benefits + + + +### Possible Drawbacks + + + +### Applicable Issues + + diff --git a/README.md b/README.md index 04da220f..d80d4475 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,12 @@ -# JavaScript language support in Atom [![Build Status](https://travis-ci.org/atom/language-javascript.svg?branch=master)](https://travis-ci.org/atom/language-javascript) +##### Atom and all repositories under Atom will be archived on December 15, 2022. Learn more in our [official announcement](https://github.blog/2022-06-08-sunsetting-atom/) + # JavaScript language support in Atom + +![ci](https://github.com/atom/language-javascript/workflows/ci/badge.svg) +[![Dependency Status](https://david-dm.org/atom/language-javascript.svg)](https://david-dm.org/atom/language-javascript) Adds syntax highlighting and snippets for JavaScript in Atom. -Originally [converted](http://atom.io/docs/latest/converting-a-text-mate-bundle) +Originally [converted](http://flight-manual.atom.io/hacking-atom/sections/converting-from-textmate) from the [JavaScript TextMate bundle](https://github.com/textmate/javascript.tmbundle). Contributions are greatly appreciated. Please fork this repository and open a diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 00000000..795da41c --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,6 @@ +# empty appveyor +build: off + +branches: + only: + - non-existing diff --git a/grammars/javascript.cson b/grammars/javascript.cson index f3f138c1..62011cef 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -2,43 +2,76 @@ 'fileTypes': [ 'js' '_js' + 'cjs' 'es' 'es6' 'gs' 'htc' 'jscad' + 'jscript' + 'jse' 'jslib' 'jsm' 'json5' 'jspre' + 'mjs' 'pac' 'pjs' + 'sjs' + 'snap' 'xsjs' 'xsjslib' ] -'firstLineMatch': '^#!.*\\b(node|iojs|JavaScript)' +'firstLineMatch': '''(?x) + # Hashbang + ^\\#!.*(?:\\s|\\/|(?<=!)\\b) + (?:node|iojs|JavaScript) + (?:$|\\s) + | + # Modeline + (?i: + # Emacs + -\\*-(?:\\s*(?=[^:;\\s]+\\s*-\\*-)|(?:.*?[;\\s]|(?<=-\\*-))mode\\s*:\\s*) + (?:js|javascript) + (?=[\\s;]|(?]?\\d+|m)?|\\sex)(?=:(?=\\s*set?\\s[^\\n:]+:)|:(?!\\s*set?\\s))(?:(?:\\s|\\s*:\\s*)\\w*(?:\\s*=(?:[^\\n\\\\\\s]|\\\\.)*)?)*[\\s:](?:filetype|ft|syntax)\\s*= + javascript + (?=\\s|:|$) + ) +''' 'name': 'JavaScript' 'patterns': [ { - 'comment': 'ES6 import' - 'begin': '(?)' 'captures': '0': 'name': 'punctuation.definition.comment.html.js' '2': 'name': 'punctuation.definition.comment.html.js' - 'match': '()' 'name': 'comment.block.html.js' } { @@ -828,7 +882,7 @@ 'begin': '(?|&&|\\|\\|)\\s*(/)(?![/*+{}?])(?=.*/)' + 'begin': '(?<=[\\[{=(?:+*,!~-]|^|return|=>|&&|\\|\\|)\\s*(/)(?![/*+?])(?=.*/)' 'beginCaptures': '1': 'name': 'punctuation.definition.string.begin.js' - 'end': '(/)[gimuy]*' + 'end': '(/)([gimsuy]*)' 'endCaptures': '1': 'name': 'punctuation.definition.string.end.js' + '2': + 'name': 'meta.flag.regexp' 'name': 'string.regexp.js' 'patterns': [ { @@ -982,6 +1137,24 @@ } ] } + { + 'begin': '\\?' + 'beginCaptures': + '0': + 'name': 'keyword.operator.ternary.js' + 'end': ':' + 'endCaptures': + '0': + 'name': 'keyword.operator.ternary.js' + 'patterns': [ + { + 'include': '#prevent_object_keys_matching' + } + { + 'include': '$self' + } + ] + } { 'include': '#operators' } @@ -1001,12 +1174,8 @@ 'include': '#properties' } { - 'match': '(?>=|>>>=|\\|=' 'name': 'keyword.operator.assignment.compound.bitwise.js' } + { + 'match': '<<|>>>|>>' + 'name': 'keyword.operator.bitwise.shift.js' + } { 'match': '!==|!=|<=|>=|===|==|<|>' 'name': 'keyword.operator.comparison.js' } { - 'match': '&&|!|\\|\\|' + 'match': '&&|!!|!|\\|\\|' 'name': 'keyword.operator.logical.js' } { @@ -1140,11 +1338,7 @@ 'name': 'keyword.operator.bitwise.js' } { - 'match': '\\?|:' - 'name': 'keyword.operator.js' - } - { - 'match': '=' + 'match': '=|:' 'name': 'keyword.operator.assignment.js' } { @@ -1174,12 +1368,7 @@ 'name': 'string.quoted.single.js' 'patterns': [ { - 'match': '\\\\(x\\h{2}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.)' - 'name': 'constant.character.escape.js' - } - { - 'match': "[^']*[^\\n\\r'\\\\]$" - 'name': 'invalid.illegal.string.js' + 'include': '#string_escapes' } ] } @@ -1195,12 +1384,7 @@ 'name': 'string.quoted.double.js' 'patterns': [ { - 'match': '\\\\(x\\h{2}|[0-2][0-7]{0,2}|3[0-6][0-7]|37[0-7]?|[4-7][0-7]?|.)' - 'name': 'constant.character.escape.js' - } - { - 'match': '[^"]*[^\\n\\r"\\\\]$' - 'name': 'invalid.illegal.string.js' + 'include': '#string_escapes' } ] } @@ -1218,8 +1402,7 @@ 'name': 'string.quoted.template.html.js' 'patterns': [ { - 'match': '\\\\(x\\h{2}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.)' - 'name': 'constant.character.escape.js' + 'include': '#string_escapes' } { 'include': '#interpolated_js' @@ -1230,7 +1413,41 @@ ] } { - 'begin': '(Relay\\.QL)\\s*(`)' + 'begin': '(?<=innerHTML)\\s*(\\+?=)\\s*(?=`)' + 'beginCaptures': + '1': + 'name': 'keyword.operator.assignment.js' + 'end': '(?<=`)' + 'endCaptures': + '0': + 'name': 'punctuation.definition.string.end.js' + 'contentName': 'string.quoted.template.html.js' + 'patterns': [ + { + 'begin': '`' + 'beginCaptures': + '0': + 'name': 'punctuation.definition.string.begin.js' + 'end': '`' + 'endCaptures': + '0': + 'name': 'punctuation.definition.string.end.js' + 'patterns': [ + { + 'include': '#string_escapes' + } + { + 'include': '#interpolated_js' + } + { + 'include': 'text.html.basic' + } + ] + } + ] + } + { + 'begin': '(Relay\\.QL|gql)\\s*(`)' 'beginCaptures': '1': 'name': 'entity.name.function.js' @@ -1243,8 +1460,7 @@ 'name': 'string.quoted.template.graphql.js' 'patterns': [ { - 'match': '\\\\(x\\h{2}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.)' - 'name': 'constant.character.escape.js' + 'include': '#string_escapes' } { 'include': '#interpolated_js' @@ -1254,6 +1470,30 @@ } ] } + { + 'begin': '(sql|SQL|Sql)\\s*(`)' + 'beginCaptures': + '1': + 'name': 'entity.name.function.js' + '2': + 'name': 'punctuation.definition.string.begin.js' + 'end': '`' + 'endCaptures': + '0': + 'name': 'punctuation.definition.string.end.js' + 'name': 'string.quoted.template.sql.js' + 'patterns': [ + { + 'include': '#string_escapes' + } + { + 'include': '#interpolated_js' + } + { + 'include': 'source.sql' + } + ] + } { 'begin': '`' 'beginCaptures': @@ -1266,8 +1506,7 @@ 'name': 'string.quoted.template.js' 'patterns': [ { - 'match': '\\\\(x\\h{2}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.)' - 'name': 'constant.character.escape.js' + 'include': '#string_escapes' } { 'include': '#interpolated_js' @@ -1275,6 +1514,34 @@ ] } ] + 'string_escapes': + 'patterns': [ + { + 'match': '\\\\u(?![A-Fa-f0-9]{4}|{[A-Fa-f0-9]+})[^\'"]*' + 'name': 'invalid.illegal.unicode-escape.js' + } + { + 'match': '\\\\u(?:[A-Fa-f0-9]{4}|({)([A-Fa-f0-9]+)(}))' + 'name': 'constant.character.escape.js' + 'captures': + '1': + 'name': 'punctuation.definition.unicode-escape.begin.bracket.curly.js' + '2': + 'patterns': [ + { + # Max codepoint: \u{10FFFF} + 'match': '[A-Fa-f\\d]{7,}|(?!10)[A-Fa-f\\d]{6}' + 'name': 'invalid.illegal.unicode-escape.js' + } + ] + '3': + 'name': 'punctuation.definition.unicode-escape.end.bracket.curly.js' + } + { + 'match': '\\\\(x[0-9A-Fa-f]{2}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.)' + 'name': 'constant.character.escape.js' + } + ] 'function_params': 'patterns': [ { @@ -1417,7 +1684,7 @@ } { 'match': '''(?x) - \\b(shift|showModelessDialog|showModalDialog|showHelp|scroll|scrollX|scrollByPages| + \\b(catch|finally|then|shift|showModelessDialog|showModalDialog|showHelp|scroll|scrollX|scrollByPages| scrollByLines|scrollY|scrollTo|stop|strike|sizeToContent|sidebar|signText|sort| sup|sub|substr|substring|splice|split|send|set(Milliseconds|Seconds|Minutes|Hours| Month|Year|FullYear|Date|UTC(Milliseconds|Seconds|Minutes|Hours|Month|FullYear|Date)| @@ -1426,14 +1693,12 @@ contextual|confirm|compile|clear|captureEvents|call|createStyleSheet|createPopup| createEventObject|to(GMTString|UTCString|String|Source|UpperCase|LowerCase|LocaleString)| test|taint|taintEnabled|indexOf|italics|disableExternalCapture|dump|detachEvent|unshift| - untaint|unwatch|updateCommands|join|javaEnabled|pop|push|plugins.refresh|paddings|parse| - print|prompt|preference|enableExternalCapture|elementFromPoint|exec|execScript| - execCommand|valueOf|UTC|queryCommandState|queryCommandIndeterm|queryCommandEnabled| - queryCommandValue|find|file|fileModifiedDate|fileSize|fileCreatedDate|fileUpdatedDate| - fixed|fontsize|fontcolor|forward|fromCharCode|watch|link|load|lastIndexOf| - anchor|attachEvent|atob|apply|alert|abort|routeEvents| - resize|resizeBy|resizeTo|recalc|returnValue|replace|reverse|reload|releaseCapture| - releaseEvents|go|get(Milliseconds|Seconds|Minutes|Hours|Month|Day|Year|FullYear| + untaint|unwatch|updateCommands|join|javaEnabled|pop|push|plugins.refresh|paddings|parse|stringify| + print|prompt|preference|enableExternalCapture|exec|execScript|valueOf|UTC|find|file| + fileModifiedDate|fileSize|fileCreatedDate|fileUpdatedDate|fixed|fontsize|fontcolor| + forward|fromCharCode|watch|link|load|lastIndexOf|anchor|attachEvent|atob|apply|alert| + abort|routeEvents|resize|resizeBy|resizeTo|recalc|returnValue|replace|reverse|reload| + releaseCapture|releaseEvents|go|get(Milliseconds|Seconds|Minutes|Hours|Month|Day|Year|FullYear| Time|Date|TimezoneOffset|UTC(Milliseconds|Seconds|Minutes|Hours|Day|Month|FullYear|Date)| Attention|Selection|ResponseHeader|AllResponseHeaders)|moveBy|moveBelow|moveTo| moveToAbsolute|moveAbove|mergeAttributes|match|margins|btoa|big|bold|borderWidths|blink|back)\\b @@ -1442,15 +1707,49 @@ } { 'match': '''(?x) - \\b(substringData|submit|splitText|setNamedItem|setAttribute|setAttributeNode|select| - hasChildNodes|hasFeature|namedItem|click|close|cloneNode|createComment|createCDATASection| - createCaption|createTHead|createTextNode|createTFoot|createDocumentFragment| - createProcessingInstruction|createEntityReference|createElement|createAttribute| - tabIndex|insertRow|insertBefore|insertCell|insertData|item|open|deleteRow|deleteCell| - deleteCaption|deleteTHead|deleteTFoot|deleteData|focus|write|writeln|add|appendChild| - appendData|reset|replaceChild|replaceData|move|moveNamedItem|moveChild|moveAttribute| - moveAttributeNode|getNamedItem|getElementsByName|getElementsByTagName|getElementById| - getAttribute|getAttributeNode|blur)\\b + \\b(acceptNode|add|addEventListener|addTextTrack|adoptNode|after|animate|append| + appendChild|appendData|before|blur|canPlayType|captureStream| + caretPositionFromPoint|caretRangeFromPoint|checkValidity|clear|click| + cloneContents|cloneNode|cloneRange|close|closest|collapse| + compareBoundaryPoints|compareDocumentPosition|comparePoint|contains| + convertPointFromNode|convertQuadFromNode|convertRectFromNode|createAttribute| + createAttributeNS|createCaption|createCDATASection|createComment| + createContextualFragment|createDocument|createDocumentFragment| + createDocumentType|createElement|createElementNS|createEntityReference| + createEvent|createExpression|createHTMLDocument|createNodeIterator| + createNSResolver|createProcessingInstruction|createRange|createShadowRoot| + createTBody|createTextNode|createTFoot|createTHead|createTreeWalker|delete| + deleteCaption|deleteCell|deleteContents|deleteData|deleteRow|deleteTFoot| + deleteTHead|detach|disconnect|dispatchEvent|elementFromPoint|elementsFromPoint| + enableStyleSheetsForSet|entries|evaluate|execCommand|exitFullscreen| + exitPointerLock|expand|extractContents|fastSeek|firstChild|focus|forEach|get| + getAll|getAnimations|getAttribute|getAttributeNames|getAttributeNode| + getAttributeNodeNS|getAttributeNS|getBoundingClientRect|getBoxQuads| + getClientRects|getContext|getDestinationInsertionPoints|getElementById| + getElementsByClassName|getElementsByName|getElementsByTagName| + getElementsByTagNameNS|getItem|getNamedItem|getSelection|getStartDate| + getVideoPlaybackQuality|has|hasAttribute|hasAttributeNS|hasAttributes| + hasChildNodes|hasFeature|hasFocus|importNode|initEvent|insertAdjacentElement| + insertAdjacentHTML|insertAdjacentText|insertBefore|insertCell|insertData| + insertNode|insertRow|intersectsNode|isDefaultNamespace|isEqualNode| + isPointInRange|isSameNode|item|key|keys|lastChild|load|lookupNamespaceURI| + lookupPrefix|matches|move|moveAttribute|moveAttributeNode|moveChild| + moveNamedItem|namedItem|nextNode|nextSibling|normalize|observe|open| + parentNode|pause|play|postMessage|prepend|preventDefault|previousNode| + previousSibling|probablySupportsContext|queryCommandEnabled| + queryCommandIndeterm|queryCommandState|queryCommandSupported|queryCommandValue| + querySelector|querySelectorAll|registerContentHandler|registerElement| + registerProtocolHandler|releaseCapture|releaseEvents|remove|removeAttribute| + removeAttributeNode|removeAttributeNS|removeChild|removeEventListener| + removeItem|replace|replaceChild|replaceData|replaceWith|reportValidity| + requestFullscreen|requestPointerLock|reset|scroll|scrollBy|scrollIntoView| + scrollTo|seekToNextFrame|select|selectNode|selectNodeContents|set|setAttribute| + setAttributeNode|setAttributeNodeNS|setAttributeNS|setCapture| + setCustomValidity|setEnd|setEndAfter|setEndBefore|setItem|setNamedItem| + setRangeText|setSelectionRange|setSinkId|setStart|setStartAfter|setStartBefore| + slice|splitText|stepDown|stepUp|stopImmediatePropagation|stopPropagation| + submit|substringData|supports|surroundContents|takeRecords|terminate|toBlob| + toDataURL|toggle|toString|values|write|writeln)\\b ''' 'name': 'support.function.dom.js' } @@ -1570,11 +1869,11 @@ 'interpolated_js': 'patterns': [ { - 'begin': '\\$\\{' + 'begin': '\\${' 'captures': '0': 'name': 'punctuation.section.embedded.js' - 'end': '\\}' + 'end': '}' 'name': 'source.js.embedded.source' 'patterns': [ { @@ -1598,63 +1897,132 @@ ] } ] - 'docblock': + 'comments': 'patterns': [ { - 'match': '(?] + 'match': '''(?x) + ((@)author) + \\s+ + ( + [^@\\s<>*/] + (?:[^@<>*/]|\\*[^/])* + ) + (?: + \\s* + (<) + ([^>\\s]+) + (>) + )? + ''' + 'captures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + '3': + 'name': 'entity.name.type.instance.jsdoc' + '4': + 'name': 'punctuation.definition.bracket.angle.begin.jsdoc' + '5': + 'name': 'constant.other.email.link.underline.jsdoc' + '6': + 'name': 'punctuation.definition.bracket.angle.end.jsdoc' + } + { + # @borrows as + 'match': '''(?x) + ((@)borrows) \\s+ + ((?:[^@\\s*/]|\\*[^/])+) # + \\s+ (as) \\s+ # as + ((?:[^@\\s*/]|\\*[^/])+) # + ''' + 'captures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + '3': + 'name': 'entity.name.type.instance.jsdoc' + '4': + 'name': 'keyword.operator.control.jsdoc' + '5': + 'name': 'entity.name.type.instance.jsdoc' + } + { + # @example text(); + 'name': 'meta.example.jsdoc' + 'begin': '((@)example)\\s+' + 'end': '(?=@|\\*/)' + 'beginCaptures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + 'patterns': [ + { + # Match to prevent leading asterisk being highlighted as JS + 'match': '^\\s\\*\\s+' + } + { + # Leading … before example + 'begin': '\\G(<)caption(>)' + 'beginCaptures': + '0': + 'name': 'entity.name.tag.inline.jsdoc' + '1': + 'name': 'punctuation.definition.bracket.angle.begin.jsdoc' + '2': + 'name': 'punctuation.definition.bracket.angle.end.jsdoc' + 'contentName': 'constant.other.description.jsdoc' + 'end': '()|(?=\\*/)' + 'endCaptures': + '0': + 'name': 'entity.name.tag.inline.jsdoc' + '1': + 'name': 'punctuation.definition.bracket.angle.begin.jsdoc' + '2': + 'name': 'punctuation.definition.bracket.angle.end.jsdoc' + } + { + # Highlighted JavaScript example + 'match': '[^\\s@*](?:[^*]|\\*[^/])*' + 'captures': + '0': + 'name': 'source.embedded.js' + 'patterns': [ + { + 'include': 'source.js' + } + ] + } + ] + } + { + # @kind type + 'captures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + '3': + 'name': 'constant.language.symbol-type.jsdoc' + 'match': '''(?x) + ((@)kind) + \\s+ + (class|constant|event|external|file|function|member|mixin|module|namespace|typedef) + \\b + ''' + } + { + # @see namepathOrURL + 'captures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + '3': + 'name': 'variable.other.link.underline.jsdoc' + '4': + 'name': 'entity.name.type.instance.jsdoc' + 'match': '''(?x) + ((@)see) + \\s+ + (?: + # URL + ( + (?=https?://) + (?:[^\\s*]|\\*[^/])+ + ) + | + # JSDoc namepath + ( + (?! + # Avoid matching bare URIs (also acceptable as links) + https?:// + | + # Avoid matching {@inline tags}; we match those below + (?:\\[[^\\[\\]]*\\])? # Possible description [preceding]{@tag} + {@(?:link|linkcode|linkplain|tutorial)\\b + ) + # Matched namepath + (?:[^@\\s*/]|\\*[^/])+ + ) + ) + ''' + } + { + # @template Foo,Bar + 'captures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + '3': + 'name': 'variable.other.jsdoc' + 'patterns': [ + { + 'match': ',' + 'name': 'punctuation.delimiter.object.comma.jsdoc' + } + ] + 'match': '''(?x) + ((@)template) + \\s+ + # One or more valid identifiers + ( + [A-Za-z_$] # First character: non-numeric word character + [\\w$.\\[\\]]* # Rest of identifier + (?: # Possible list of additional identifiers + \\s* , \\s* + [A-Za-z_$] + [\\w$.\\[\\]]* + )* + ) + ''' + } + { + # Tags followed by an identifier token + # - @ identifier + 'captures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + '3': + 'name': 'variable.other.jsdoc' + 'match': '''(?x) + ( + (@) + (?:arg|argument|const|constant|member|namespace|param|var) + ) + \\s+ + ( + [A-Za-z_$] + [\\w$.\\[\\]]* + ) + ''' + } + { + # Tags followed by a type expression, then a namepath + # - @ {type} namepath + 'begin': '((@)typedef)\\s+(?={)' + 'beginCaptures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + 'end': '(?=\\s|\\*/|[^{}\\[\\]A-Za-z_$])' + 'patterns': [ + { + 'include': '#type' + } + { + 'name': 'entity.name.type.instance.jsdoc' + 'match': '(?:[^@\\s*/]|\\*[^/])+' + } + ] + } + { + # Tags followed by a type expression, then an identifier + # - @ {type} identifier + 'begin': '((@)(?:arg|argument|const|constant|member|namespace|param|prop|property|var))\\s+(?={)' + 'beginCaptures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + 'end': '(?=\\s|\\*/|[^{}\\[\\]A-Za-z_$])' + 'patterns': [ + { + 'include': '#type' + } + { + 'match': '([A-Za-z_$][\\w$.\\[\\]]*)' + 'name': 'variable.other.jsdoc' + } + { + # Optional value + 'name': 'variable.other.jsdoc' + 'match': '''(?x) + (\\[)\\s* + [\\w$]+ + (?: + (?:\\[\\])? # Foo[].bar properties within an array + \\. # Foo.Bar namespaced parameter + [\\w$]+ + )* + (?: + \\s* + (=) # [foo=bar] Default parameter value + \\s* + ( + # The inner regexes are to stop the match early at */ and to not stop at escaped quotes + (?> + "(?:(?:\\*(?!/))|(?:\\\\(?!"))|[^*\\\\])*?" | # [foo="bar"] Double-quoted + '(?:(?:\\*(?!/))|(?:\\\\(?!'))|[^*\\\\])*?' | # [foo='bar'] Single-quoted + \\[ (?:(?:\\*(?!/))|[^*])*? \\] | # [foo=[1,2]] Array literal + (?:(?:\\*(?!/))|\\s(?!\\s*\\])|\\[.*?(?:\\]|(?=\\*/))|[^*\\s\\[\\]])* # Everything else (sorry) + )* + ) + )? + \\s*(?:(\\])((?:[^*\\s]|\\*[^\\s/])+)?|(?=\\*/)) + ''' + 'captures': + '1': + 'name': 'punctuation.definition.optional-value.begin.bracket.square.jsdoc' + '2': + 'name': 'keyword.operator.assignment.jsdoc' + '3': + 'name': 'source.embedded.js' + 'patterns': [ + { + 'include': '#inline-tags' + } + { + 'include': 'source.js' + } + ] + '4': + 'name': 'punctuation.definition.optional-value.end.bracket.square.jsdoc' + '5': + 'name': 'invalid.illegal.syntax.jsdoc' + } + ] + } + { + # Tags followed by a type expression + # - @ {type} + 'begin': '''(?x) + ( + (@) + (?:define|enum|exception|export|extends|lends|implements|modifies + |namespace|private|protected|returns?|suppress|this|throws|type + |yields?) + ) + \\s+(?={) + ''' + 'beginCaptures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + 'end': '(?=\\s|\\*/|[^{}\\[\\]A-Za-z_$])' + 'patterns': [ + { + 'include': '#type' + } + ] + } + { + # Tags followed by a namepath + # - @ namepath + 'captures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + '3': + 'name': 'entity.name.type.instance.jsdoc' + 'match': '''(?x) + ( + (@) + (?:alias|augments|callback|constructs|emits|event|fires|exports? + |extends|external|function|func|host|lends|listens|interface|memberof!? + |method|module|mixes|mixin|name|requires|see|this|typedef|uses) + ) + \\s+ + ( + (?: + [^{}@\\s*] | \\*[^/] + )+ + ) + ''' + } + { + # Tags followed by a quoted arbitrary value + # - @ "Quoted value" + 'begin': '((@)(?:default(?:value)?|license|version))\\s+(([\'"]))' + 'beginCaptures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + '3': + 'name': 'variable.other.jsdoc' + '4': + 'name': 'punctuation.definition.string.begin.jsdoc' + 'contentName': 'variable.other.jsdoc' + 'end': '(\\3)|(?=$|\\*/)' + 'endCaptures': + '0': + 'name': 'variable.other.jsdoc' + '1': + 'name': 'punctuation.definition.string.end.jsdoc' + } + { + # Tags followed by an arbitrary value + # - @ value + 'captures': + '1': + 'name': 'storage.type.class.jsdoc' + '2': + 'name': 'punctuation.definition.block.tag.jsdoc' + '3': + 'name': 'variable.other.jsdoc' + 'match': '((@)(?:default(?:value)?|license|tutorial|variation|version))\\s+([^\\s*]+)' + } + { + # Tags without arguments, or a tag without expected arguments. Because JSDoc permits + # tags to be spread across lines, we should at least highlight the opening tag for + # stuff like this: + # + # /** + # * @param + # * {type} + # * name + 'captures': + '1': + 'name': 'punctuation.definition.block.tag.jsdoc' + 'match': '''(?x) (@) + (?:abstract|access|alias|api|arg|argument|async|attribute|augments|author|beta|borrows|bubbles + |callback|chainable|class|classdesc|code|config|const|constant|constructor|constructs|copyright + |default|defaultvalue|define|deprecated|desc|description|dict|emits|enum|event|example|exception + |exports?|extends|extension(?:_?for)?|external|externs|file|fileoverview|final|fires|for|func + |function|generator|global|hideconstructor|host|ignore|implements|implicitCast|inherit[Dd]oc + |inner|instance|interface|internal|kind|lends|license|listens|main|member|memberof!?|method + |mixes|mixins?|modifies|module|name|namespace|noalias|nocollapse|nocompile|nosideeffects + |override|overview|package|param|polymer(?:Behavior)?|preserve|private|prop|property|protected + |public|read[Oo]nly|record|require[ds]|returns?|see|since|static|struct|submodule|summary + |suppress|template|this|throws|todo|tutorial|type|typedef|unrestricted|uses|var|variation + |version|virtual|writeOnce|yields?) + \\b + ''' + 'name': 'storage.type.class.jsdoc' + } + { + 'include': '#inline-tags' + } +] + +'repository': + # Balanced brackets (square or curly) + 'brackets': + 'patterns': [ + { + 'begin': '{' + 'end': '}|(?=\\*/)' + 'patterns': [ + { + 'include': '#brackets' + } + ] + } + { + 'begin': '\\[' + 'end': '\\]|(?=\\*/)' + 'patterns': [ + { + 'include': '#brackets' + } + ] + } + ] + 'inline-tags': + 'patterns': [ + { + # Description preceding {@inline tag} + 'captures': + '1': + 'name': 'punctuation.definition.bracket.square.begin.jsdoc' + '2': + 'name': 'punctuation.definition.bracket.square.end.jsdoc' + 'match': '(\\[)[^\\]]+(\\])(?={@(?:link|linkcode|linkplain|tutorial))' + 'name': 'constant.other.description.jsdoc' + } + { + # {@link|@tutorial …} + 'begin': '({)((@)(?:link(?:code|plain)?|tutorial))\\s*' + 'beginCaptures': + '1': + 'name': 'punctuation.definition.bracket.curly.begin.jsdoc' + '2': + 'name': 'storage.type.class.jsdoc' + '3': + 'name': 'punctuation.definition.inline.tag.jsdoc' + 'end': '}|(?=\\*/)' + 'endCaptures': + '0': + 'name': 'punctuation.definition.bracket.curly.end.jsdoc' + 'name': 'entity.name.type.instance.jsdoc' + 'patterns': [ + { + 'captures': + '1': + 'name': 'variable.other.link.underline.jsdoc' + '2': + 'name': 'punctuation.separator.pipe.jsdoc' + 'match': '\\G((?=https?://)(?:[^|}\\s*]|\\*[/])+)(\\|)?' + } + { + 'captures': + '1': + 'name': 'variable.other.description.jsdoc' + '2': + 'name': 'punctuation.separator.pipe.jsdoc' + 'match': '\\G((?:[^{}@\\s|*]|\\*[^/])+)(\\|)?' + } + ] + } + ] + # {type} + 'type': + 'patterns': [ + { + # {unclosed + 'match': '\\G{(?:[^}*]|\\*[^/}])+$' + 'name': 'invalid.illegal.type.jsdoc' + } + { + 'begin': '\\G({)' + 'beginCaptures': + '0': + 'name': 'entity.name.type.instance.jsdoc' + '1': + 'name': 'punctuation.definition.bracket.curly.begin.jsdoc' + 'contentName': 'entity.name.type.instance.jsdoc' + 'end': '((}))\\s*|(?=\\*/)' + 'endCaptures': + '1': + 'name': 'entity.name.type.instance.jsdoc' + '2': + 'name': 'punctuation.definition.bracket.curly.end.jsdoc' + 'patterns': [ + { + 'include': '#brackets' + } + ] + } + ] diff --git a/grammars/regular expression replacement (javascript).cson b/grammars/regular expression replacement (javascript).cson index 596b8279..eeaa8d70 100644 --- a/grammars/regular expression replacement (javascript).cson +++ b/grammars/regular expression replacement (javascript).cson @@ -1,6 +1,4 @@ 'scopeName': 'source.js.regexp.replacement' -'name': 'Regular Expression Replacement (JavaScript)' -'fileTypes': [] 'patterns': [ { 'include': '#regexp-replacement' diff --git a/grammars/regular expressions (javascript).cson b/grammars/regular expressions (javascript).cson index 213b8363..1de6f54b 100644 --- a/grammars/regular expressions (javascript).cson +++ b/grammars/regular expressions (javascript).cson @@ -1,6 +1,4 @@ 'scopeName': 'source.js.regexp' -'name': 'Regular Expressions (JavaScript)' -'fileTypes': [] 'patterns': [ { 'include': '#regexp' @@ -10,11 +8,11 @@ 'regex-character-class': 'patterns': [ { - 'match': '\\\\[wWsSdD]|\\.' + 'match': '\\\\[wWsSdDtrnvf]|\\.' 'name': 'constant.character.character-class.regexp' } { - 'match': '\\\\([0-7]{3}|x\\h\\h|u\\h\\h\\h\\h)' + 'match': '\\\\([0-7]{3}|x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4})' 'name': 'constant.character.numeric.regexp' } { @@ -33,7 +31,7 @@ 'name': 'keyword.control.anchor.regexp' } { - 'match': '\\\\[1-9]\\d*' + 'match': '\\\\[1-9]\\d*|\\\\k<[a-zA-Z_$][\\w$]*>' 'name': 'keyword.other.back-reference.regexp' } { @@ -45,14 +43,18 @@ 'name': 'keyword.operator.or.regexp' } { - 'begin': '(\\()((\\?=)|(\\?!))' + 'begin': '(\\()(?:(\\?=)|(\\?!)|(\\?<=)|(\\?))?' 'beginCaptures': '0': 'name': 'punctuation.definition.group.regexp' @@ -107,7 +109,7 @@ 'name': 'constant.character.control.regexp' '6': 'name': 'constant.character.escape.backslash.regexp' - 'match': '(?:.|(\\\\(?:[0-7]{3}|x\\h\\h|u\\h\\h\\h\\h))|(\\\\c[A-Z])|(\\\\.))\\-(?:[^\\]\\\\]|(\\\\(?:[0-7]{3}|x\\h\\h|u\\h\\h\\h\\h))|(\\\\c[A-Z])|(\\\\.))' + 'match': '(?:.|(\\\\(?:[0-7]{3}|x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}))|(\\\\c[A-Z])|(\\\\.))\\-(?:[^\\]\\\\]|(\\\\(?:[0-7]{3}|x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}))|(\\\\c[A-Z])|(\\\\.))' 'name': 'constant.other.character-class.range.regexp' } { diff --git a/grammars/tree-sitter-javascript.cson b/grammars/tree-sitter-javascript.cson new file mode 100644 index 00000000..86919162 --- /dev/null +++ b/grammars/tree-sitter-javascript.cson @@ -0,0 +1,262 @@ +name: 'JavaScript' +scopeName: 'source.js' +type: 'tree-sitter' +parser: 'tree-sitter-javascript' + +fileTypes: ['js', 'jsx'] + +injectionRegex: '^js$|^JS$|javascript|JavaScript' + +firstLineRegex: [ + # shebang line + '^#!.*\\b(node)\\r?\\n' + + # vim modeline + 'vim\\b.*\\bset\\b.*\\b(filetype|ft|syntax)=(js|javascript)' +] + +folds: [ + { + type: 'comment' + } + { + type: ['jsx_element', 'template_string'], + start: {index: 0} + end: {index: -1} + } + { + type: 'jsx_self_closing_element' + start: {index: 1} + end: {index: -2} + } + { + start: {index: 0, type: '{'} + end: {index: -1, type: '}'} + } + { + start: {index: 0, type: '['} + end: {index: -1, type: ']'} + } + { + start: {index: 0, type: '('} + end: {index: -1, type: ')'} + } + { + type: ['switch_case', 'switch_default'] + start: {index: 0} + end: {type: 'break_statement', index: -1} + } + { + type: ['switch_case', 'switch_default'] + start: {index: 0} + } +] + +comments: + start: '// ' + +scopes: + 'program': 'source.js' + + 'property_identifier': [ + { + match: '^[\$A-Z_]+$', + scopes: 'constant.other.property.js' + } + + 'variable.other.object.property' + ] + + 'member_expression > property_identifier': 'variable.other.object.property.unquoted' + + 'formal_parameters > identifier': 'variable.parameter.function' + 'formal_parameters > rest_parameter > identifier': 'variable.parameter.rest.function' + + 'shorthand_property_identifier': [ + { + match: '^[\$A-Z_]{2,}$', + scopes: 'constant.other' + } + ] + + ' + class > identifier, + new_expression > identifier + ': 'meta.class' + + ' + jsx_opening_element > identifier, + jsx_closing_element > identifier, + jsx_self_closing_element > identifier + ': [ + { + match: '^[A-Z]', + scopes: 'meta.class.component.jsx' + } + ] + + 'call_expression > identifier': {match: '^[A-Z]', scopes: 'meta.class'} + 'arrow_function > identifier:nth-child(0)': 'variable.parameter.function' + + 'function > identifier': 'entity.name.function' + 'function_declaration > identifier': 'entity.name.function' + 'generator_function > identifier': 'entity.name.function' + + 'call_expression > identifier': [ + {match: '^require$', scopes: 'support.function'}, + 'entity.name.function' + ] + + 'call_expression > super': 'support.function.super' + + 'method_definition > property_identifier': 'entity.name.function' + 'call_expression > member_expression > property_identifier': 'entity.name.function' + + 'identifier': [ + { + match: '^(global|globalThis|module|exports|__filename|__dirname)$', + scopes: 'support.variable' + }, + { + match: '^(window|self|frames|event|document|performance|screen|navigator|console)$' + scopes: 'support.variable.dom' + }, + { + exact: 'require', + scopes: 'support.function' + }, + { + match: '^[\$A-Z_]{2,}$', + scopes: 'constant.other' + }, + { + match: '^[A-Z]', + scopes: 'meta.class' + }, + ] + + 'number': 'constant.numeric' + 'string': 'string.quoted' + 'regex': 'string.regexp' + 'escape_sequence': 'constant.character.escape' + 'template_string': 'string.quoted.template' + 'undefined': 'constant.language' + 'null': 'constant.language.null' + 'true': 'constant.language.boolean.true' + 'false': 'constant.language.boolean.false' + 'comment': [ + { + match: "^//", + scopes: 'comment.line' + }, + 'comment.block' + ] + 'hash_bang_line': 'comment.block' + + ' + jsx_expression > "{", + jsx_expression > "}", + template_substitution > "${", + template_substitution > "}" + ': 'punctuation.section.embedded' + 'template_substitution': 'embedded.source' + + '"("': 'punctuation.definition.parameters.begin.bracket.round' + '")"': 'punctuation.definition.parameters.end.bracket.round' + '"{"': 'punctuation.definition.function.body.begin.bracket.curly' + '"}"': 'punctuation.definition.function.body.end.bracket.curly' + '";"': 'punctuation.terminator.statement.semicolon' + '"["': 'punctuation.definition.array.begin.bracket.square' + '"]"': 'punctuation.definition.array.end.bracket.square' + + '"var"': 'storage.type' + '"let"': 'storage.type' + '"class"': 'storage.type' + '"extends"': 'storage.modifier' + '"const"': 'storage.modifier' + '"static"': 'storage.modifier' + '"function"': 'storage.type.function' + '"=>"': 'storage.type.function.arrow' + + '"="': 'keyword.operator.js' + '"+="': 'keyword.operator.js' + '"-="': 'keyword.operator.js' + '"*="': 'keyword.operator.js' + '"/="': 'keyword.operator.js' + '"%="': 'keyword.operator.js' + '"<<="': 'keyword.operator.js' + '">>="': 'keyword.operator.js' + '">>>="': 'keyword.operator.js' + '"&="': 'keyword.operator.js' + '"^="': 'keyword.operator.js' + '"|="': 'keyword.operator.js' + '"!"': 'keyword.operator.js' + '"+"': 'keyword.operator.js' + '"-"': 'keyword.operator.js' + '"*"': 'keyword.operator.js' + '"/"': 'keyword.operator.js' + '"%"': 'keyword.operator.js' + '"=="': 'keyword.operator.js' + '"==="': 'keyword.operator.js' + '"!="': 'keyword.operator.js' + '"!=="': 'keyword.operator.js' + '">="': 'keyword.operator.js' + '"<="': 'keyword.operator.js' + '">"': 'keyword.operator.js' + '"<"': 'keyword.operator.js' + '":"': 'keyword.operator.js' + '"?"': 'keyword.operator.js' + '"&&"': 'keyword.operator.js' + '"||"': 'keyword.operator.js' + '"&"': 'keyword.operator.js' + '"~"': 'keyword.operator.js' + '"^"': 'keyword.operator.js' + '">>"': 'keyword.operator.js' + '">>>"': 'keyword.operator.js' + '"<<"': 'keyword.operator.js' + '"|"': 'keyword.operator.js' + '"++"': 'keyword.operator.js' + '"--"': 'keyword.operator.js' + '"..."': 'keyword.operator.spread.js' + + '"in"': 'keyword.operator.in' + '"instanceof"': 'keyword.operator.instanceof' + '"of"': 'keyword.operator.of' + '"new"': 'keyword.operator.new' + '"typeof"': 'keyword.operator.typeof' + + '"get"': 'keyword.operator.setter' + '"set"': 'keyword.operator.setter' + + '"."': 'meta.delimiter.period' + '","': 'meta.delimiter.comma' + + '"as"': 'keyword.control' + '"if"': 'keyword.control' + '"do"': 'keyword.control' + '"else"': 'keyword.control' + '"while"': 'keyword.control' + '"for"': 'keyword.control' + '"return"': 'keyword.control' + '"break"': 'keyword.control' + '"continue"': 'keyword.control' + '"throw"': 'keyword.control' + '"try"': 'keyword.control' + '"catch"': 'keyword.control' + '"finally"': 'keyword.control' + '"switch"': 'keyword.control' + '"case"': 'keyword.control' + '"default"': 'keyword.control' + '"export"': 'keyword.control' + '"import"': 'keyword.control' + '"from"': 'keyword.control' + '"yield"': 'keyword.control' + '"async"': 'keyword.control' + '"await"': 'keyword.control' + '"debugger"': 'keyword.control' + '"delete"': 'keyword.control' + + 'jsx_attribute > property_identifier': 'entity.other.attribute-name' + 'jsx_opening_element > identifier': 'entity.name.tag' + 'jsx_closing_element > identifier': 'entity.name.tag' + 'jsx_self_closing_element > identifier': 'entity.name.tag' diff --git a/grammars/tree-sitter-jsdoc.cson b/grammars/tree-sitter-jsdoc.cson new file mode 100644 index 00000000..a9f8a679 --- /dev/null +++ b/grammars/tree-sitter-jsdoc.cson @@ -0,0 +1,16 @@ +name: 'JSDoc' +scopeName: 'source.jsdoc' +type: 'tree-sitter' +parser: 'tree-sitter-jsdoc' + +injectionRegex: '^jsdoc$' + +scopes: + 'tag_name': 'keyword.control' + 'identifier': 'variable.other.jsdoc' + 'type': 'support.type' + 'path_expression > identifier': 'string' + '"."': 'meta.delimiter.period' + '":"': 'meta.delimiter.colon' + '"/"': 'meta.delimiter.slash' + '"#", "~"': 'meta.delimiter' diff --git a/grammars/tree-sitter-regex.cson b/grammars/tree-sitter-regex.cson new file mode 100644 index 00000000..812bd378 --- /dev/null +++ b/grammars/tree-sitter-regex.cson @@ -0,0 +1,17 @@ +name: 'Javascript RegExp' +scopeName: 'source.js.regexp' +type: 'tree-sitter' +parser: 'tree-sitter-regex' + +injectionRegex: 'regex' + +scopes: + 'pattern': 'string.quoted', + 'group_name': 'variable.other.object.property' + 'identity_escape, control_letter_escape, control_escape, class_escape': 'constant.character.character-class.regexp' + 'start_assertion, end_assertion, boundary_assertion, non_boundary_assertion': 'constant.character.character-class.regexp' + 'count_quantifier, one_or_more, zero_or_more, optional': 'storage.modifier' + 'character_class': 'string.regexp' + '"(", "(?", "(?:", "(?<"': 'punctuation.definition.parameters.begin.bracket.round' + '">", ")"': 'punctuation.definition.parameters.end.bracket.round' + '"=", "<=", "!", "~!#@%^&*|+=[]{}`?-…' 'commentStart': '// ' 'foldEndPattern': '^\\s*\\}|^\\s*\\]|^\\s*\\)' 'increaseIndentPattern': '(?x) - \\{ [^}"\']* $ - | \\[ [^\\]"\']* $ - | \\( [^)"\']* $ + \\{ [^}"\']*(//.*)? $ + | \\[ [^\\]"\']*(//.*)? $ + | \\( [^)"\']*(//.*)? $ ' 'decreaseIndentPattern': '(?x) ^ \\s* (\\s* /[*] .* [*]/ \\s*)* [}\\])] diff --git a/snippets/language-javascript.cson b/snippets/language-javascript.cson index 172b7048..24fdbb61 100644 --- a/snippets/language-javascript.cson +++ b/snippets/language-javascript.cson @@ -1,4 +1,4 @@ -'.source.js': +'.source.js, .source.flow': 'Object Method': 'prefix': 'kf' 'body': '${1:methodName}: function (${2:attribute}) {\n\t$3\n}${4:,}' @@ -11,6 +11,9 @@ 'do': 'prefix': 'do' 'body': 'do {\n\t$2\n} while (${1:true});' + 'condition ? true : false': + 'prefix': 'tern' + 'body': '${1:condition} ? ${2:true} : ${3:false}' 'if': 'prefix': 'if' 'body': 'if (${1:true}) {\n\t$2\n}' @@ -31,13 +34,19 @@ 'body': 'for (var ${1:variable} in ${2:object}) {\n\t${3:if (${2:object}.hasOwnProperty(${1:variable})) {\n\t\t$4\n\t\\}}\n}' 'for of': 'prefix': 'forof' - 'body': 'for (${1:variable} of ${2:iterable}) {\n\t$3\n}' + 'body': 'for (var ${1:variable} of ${2:iterable}) {\n\t$3\n}' + 'forEach': + 'prefix' : 'foreach' + 'body' : 'forEach((${1:item}, ${2:i}) => {\n\t$3\n});\n' 'Function': 'prefix': 'fun' 'body': 'function ${1:functionName}($2) {\n\t$0\n}' 'Anonymous Function': 'prefix': 'f' 'body': 'function ($1) {\n\t$2\n}' + 'Arrow Function': + 'prefix': 'af' + 'body': '($1) => {\n\t$2\n}' 'Generator': 'prefix': 'gen', 'body': 'function* ${1:functionName}($2) {\n\t$0\n}' @@ -68,6 +77,9 @@ 'log': 'prefix': 'log' 'body': 'console.log($1);$0' + 'dir': + 'prefix': 'dir' + 'body': 'console.dir($1);$0' 'warn': 'prefix': 'warn' 'body': 'console.warn($1);$0' @@ -113,3 +125,6 @@ 'export module': 'prefix': 'expmod' 'body': 'module.exports = ${1:name};' + 'return': + 'prefix': 'ret' + 'body': 'return $1;$0' diff --git a/spec/javascript-spec.coffee b/spec/javascript-spec.coffee index da4c209e..192850ec 100644 --- a/spec/javascript-spec.coffee +++ b/spec/javascript-spec.coffee @@ -8,10 +8,12 @@ buildTextEditor = (params) -> TextEditor ?= require('atom').TextEditor new TextEditor(params) -describe "Javascript grammar", -> +describe "JavaScript grammar", -> grammar = null beforeEach -> + atom.config.set('core.useTreeSitterParsers', false) + waitsForPromise -> atom.packages.activatePackage("language-javascript") @@ -49,50 +51,91 @@ describe "Javascript grammar", -> line3 """ + delim expect(lines[0][0]).toEqual value: delim, scopes: ['source.js', scope, 'punctuation.definition.string.begin.js'] - expect(lines[0][1]).toEqual value: 'line1', scopes: ['source.js', scope, 'invalid.illegal.string.js'] expect(lines[1][0]).toEqual value: 'line2\\', scopes: ['source.js', scope] expect(lines[2][0]).toEqual value: 'line3', scopes: ['source.js', scope] expect(lines[2][1]).toEqual value: delim, scopes: ['source.js', scope, 'punctuation.definition.string.end.js'] - describe "keywords", -> - it "tokenizes with as a keyword", -> - {tokens} = grammar.tokenizeLine('with') - expect(tokens[0]).toEqual value: 'with', scopes: ['source.js', 'keyword.control.js'] - - map = - super: 'variable.language.js' - this: 'variable.language.js' - null: 'constant.language.null.js' - true: 'constant.language.boolean.true.js' - false: 'constant.language.boolean.false.js' - debugger: 'keyword.other.js' - exports: 'support.variable.js' - __filename: 'support.variable.js' - - for keyword, scope of map - do (keyword, scope) -> - it "does not tokenize `#{keyword}` when it is an object key", -> - {tokens} = grammar.tokenizeLine("#{keyword}: 1") - expect(tokens[0]).toEqual value: keyword, scopes: ['source.js'] - expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.js'] + describe "Unicode escape sequences", -> + bracketScopes = [ + 'punctuation.definition.unicode-escape.begin.bracket.curly.js', + 'punctuation.definition.unicode-escape.end.bracket.curly.js' + ] + delimsByScope = + "string.quoted.double.js": '"' + "string.quoted.single.js": "'" - it "tokenizes `#{keyword}` in the middle of ternary expressions", -> - {tokens} = grammar.tokenizeLine("a ? #{keyword} : b") - expect(tokens[2]).toEqual value: ' ', scopes: ['source.js'] - expect(tokens[3]).toEqual value: keyword, scopes: ['source.js', scope] + it "tokenises 2-digit sequences", -> + for scope, quote of delimsByScope + {tokens} = grammar.tokenizeLine(quote + '\\x2011' + quote) + expect(tokens[0]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: '\\x20', scopes: ['source.js', scope, 'constant.character.escape.js'] + expect(tokens[2]).toEqual value: '11', scopes: ['source.js', scope] + expect(tokens[3]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.end.js'] + + it "tokenises 4-digit sequences", -> + for scope, quote of delimsByScope + {tokens} = grammar.tokenizeLine(quote + '\\u2011' + quote) + expect(tokens[0]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: '\\u2011', scopes: ['source.js', scope, 'constant.character.escape.js'] + expect(tokens[2]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.end.js'] + + it "tokenises variable-length sequences", -> + for scope, quote of delimsByScope + {tokens} = grammar.tokenizeLine(quote + '\\u{2000}' + quote) + expect(tokens[0]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: '\\u', scopes: ['source.js', scope, 'constant.character.escape.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', scope, 'constant.character.escape.js', bracketScopes[0]] + expect(tokens[3]).toEqual value: '2000', scopes: ['source.js', scope, 'constant.character.escape.js'] + expect(tokens[4]).toEqual value: '}', scopes: ['source.js', scope, 'constant.character.escape.js', bracketScopes[1]] + expect(tokens[5]).toEqual value: quote, scopes: ['source.js', scope, 'punctuation.definition.string.end.js'] + + it "highlights sequences with invalid syntax", -> + for invalid in ['\\u', '\\u{2000', '\\u{G}'] + {tokens} = grammar.tokenizeLine('"' + invalid + '"') + expect(tokens[1]).toEqual value: invalid, scopes: ['source.js', 'string.quoted.double.js', 'invalid.illegal.unicode-escape.js'] + + it "highlights sequences with invalid codepoints", -> + maxCodepoint = 0x10FFFF + for codepoint in [0x5000, 0x11FFFF, 0x1000000, maxCodepoint] + pointStr = codepoint.toString(16).toUpperCase().replace(/^0x/, "") + {tokens} = grammar.tokenizeLine('"\\u{' + pointStr + '}"') + pointScopes = ['source.js', 'string.quoted.double.js', 'constant.character.escape.js'] + if codepoint > maxCodepoint then pointScopes.push 'invalid.illegal.unicode-escape.js' + expect(tokens[0]).toEqual value: '"', scopes: ['source.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: '\\u', scopes: ['source.js', 'string.quoted.double.js', 'constant.character.escape.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'string.quoted.double.js', 'constant.character.escape.js', bracketScopes[0]] + expect(tokens[3]).toEqual value: pointStr, scopes: pointScopes + expect(tokens[4]).toEqual value: '}', scopes: ['source.js', 'string.quoted.double.js', 'constant.character.escape.js', bracketScopes[1]] + expect(tokens[5]).toEqual value: '"', scopes: ['source.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] - it "tokenizes `#{keyword}` at the end of ternary expressions", -> - {tokens} = grammar.tokenizeLine("a ? b : #{keyword}") - expect(tokens[4]).toEqual value: ' ', scopes: ['source.js'] - expect(tokens[5]).toEqual value: keyword, scopes: ['source.js', scope] + describe "keywords", -> + keywords = ['await', 'break', 'catch', 'continue', 'do'] + + for keyword in keywords + it "tokenizes the #{keyword} keyword", -> + {tokens} = grammar.tokenizeLine(keyword) + expect(tokens[0]).toEqual value: keyword, scopes: ['source.js', 'keyword.control.js'] + + it "tokenizes the debugger statement", -> + {tokens} = grammar.tokenizeLine("debugger;") + expect(tokens[0]).toEqual value: "debugger", scopes: ['source.js', 'keyword.other.debugger.js'] + expect(tokens[1]).toEqual value: ";", scopes: ['source.js', 'punctuation.terminator.statement.js'] + + it "tokenises an `await` keyword after a spread operator", -> + {tokens} = grammar.tokenizeLine("...await stuff()") + expect(tokens[0]).toEqual value: '...', scopes: ['source.js', 'keyword.operator.spread.js'] + expect(tokens[1]).toEqual value: 'await', scopes: ['source.js', 'keyword.control.js'] + expect(tokens[3]).toEqual value: 'stuff', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(tokens[4]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[5]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] describe "built-in globals", -> it "tokenizes built-in classes", -> {tokens} = grammar.tokenizeLine('window') - expect(tokens[0]).toEqual value: 'window', scopes: ['source.js', 'support.class.js'] + expect(tokens[0]).toEqual value: 'window', scopes: ['source.js', 'support.variable.dom.js'] {tokens} = grammar.tokenizeLine('window.name') - expect(tokens[0]).toEqual value: 'window', scopes: ['source.js', 'support.class.js'] + expect(tokens[0]).toEqual value: 'window', scopes: ['source.js', 'support.variable.dom.js'] {tokens} = grammar.tokenizeLine('$window') expect(tokens[0]).toEqual value: '$window', scopes: ['source.js'] @@ -107,19 +150,35 @@ describe "Javascript grammar", -> describe "instantiation", -> it "tokenizes the new keyword and instance entities", -> {tokens} = grammar.tokenizeLine('new something') - expect(tokens[0]).toEqual value: 'new', scopes: ['source.js', 'meta.class.instance.constructor', 'keyword.operator.new.js'] - expect(tokens[1]).toEqual value: ' ', scopes: ['source.js', 'meta.class.instance.constructor'] - expect(tokens[2]).toEqual value: 'something', scopes: ['source.js', 'meta.class.instance.constructor', 'entity.name.type.instance.js'] + expect(tokens[0]).toEqual value: 'new', scopes: ['source.js', 'meta.class.instance.constructor.js', 'keyword.operator.new.js'] + expect(tokens[1]).toEqual value: ' ', scopes: ['source.js', 'meta.class.instance.constructor.js'] + expect(tokens[2]).toEqual value: 'something', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js'] {tokens} = grammar.tokenizeLine('new Something') - expect(tokens[0]).toEqual value: 'new', scopes: ['source.js', 'meta.class.instance.constructor', 'keyword.operator.new.js'] - expect(tokens[1]).toEqual value: ' ', scopes: ['source.js', 'meta.class.instance.constructor'] - expect(tokens[2]).toEqual value: 'Something', scopes: ['source.js', 'meta.class.instance.constructor', 'entity.name.type.instance.js'] + expect(tokens[0]).toEqual value: 'new', scopes: ['source.js', 'meta.class.instance.constructor.js', 'keyword.operator.new.js'] + expect(tokens[1]).toEqual value: ' ', scopes: ['source.js', 'meta.class.instance.constructor.js'] + expect(tokens[2]).toEqual value: 'Something', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js'] {tokens} = grammar.tokenizeLine('new $something') - expect(tokens[0]).toEqual value: 'new', scopes: ['source.js', 'meta.class.instance.constructor', 'keyword.operator.new.js'] - expect(tokens[1]).toEqual value: ' ', scopes: ['source.js', 'meta.class.instance.constructor'] - expect(tokens[2]).toEqual value: '$something', scopes: ['source.js', 'meta.class.instance.constructor', 'entity.name.type.instance.js'] + expect(tokens[0]).toEqual value: 'new', scopes: ['source.js', 'meta.class.instance.constructor.js', 'keyword.operator.new.js'] + expect(tokens[1]).toEqual value: ' ', scopes: ['source.js', 'meta.class.instance.constructor.js'] + expect(tokens[2]).toEqual value: '$something', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js'] + + {tokens} = grammar.tokenizeLine('var instance = new obj.ct.Cla$s();') + expect(tokens[0]).toEqual value: 'var', scopes: ['source.js', 'storage.type.var.js'] + expect(tokens[1]).toEqual value: ' instance ', scopes: ['source.js'] + expect(tokens[2]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js'] + expect(tokens[3]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[4]).toEqual value: 'new', scopes: ['source.js', 'meta.class.instance.constructor.js', 'keyword.operator.new.js'] + expect(tokens[5]).toEqual value: ' ', scopes: ['source.js', 'meta.class.instance.constructor.js'] + expect(tokens[6]).toEqual value: 'obj', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js'] + expect(tokens[7]).toEqual value: '.', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js', 'meta.delimiter.property.period.js'] + expect(tokens[8]).toEqual value: 'ct', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js'] + expect(tokens[9]).toEqual value: '.', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js', 'meta.delimiter.property.period.js'] + expect(tokens[10]).toEqual value: 'Cla$s', scopes: ['source.js', 'meta.class.instance.constructor.js', 'entity.name.type.instance.js'] + expect(tokens[11]).toEqual value: '(', scopes: ['source.js', 'meta.brace.round.js'] + expect(tokens[12]).toEqual value: ')', scopes: ['source.js', 'meta.brace.round.js'] + expect(tokens[13]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] describe "regular expressions", -> it "tokenizes regular expressions", -> @@ -128,6 +187,78 @@ describe "Javascript grammar", -> expect(tokens[1]).toEqual value: 'test', scopes: ['source.js', 'string.regexp.js'] expect(tokens[2]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + {tokens} = grammar.tokenizeLine('/random/g') + expect(tokens[0]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: 'random', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[2]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + expect(tokens[3]).toEqual value: 'g', scopes: ['source.js', 'string.regexp.js', 'meta.flag.regexp'] + + {tokens} = grammar.tokenizeLine('/rock(et)?/is') + expect(tokens[0]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: 'rock', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[2]).toEqual value: '(', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[3]).toEqual value: 'et', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp'] + expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[5]).toEqual value: '?', scopes: ['source.js', 'string.regexp.js', 'keyword.operator.quantifier.regexp'] + expect(tokens[6]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + expect(tokens[7]).toEqual value: 'is', scopes: ['source.js', 'string.regexp.js', 'meta.flag.regexp'] + + {tokens} = grammar.tokenizeLine('/(foo)bar\\1/') + expect(tokens[0]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[2]).toEqual value: 'foo', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp'] + expect(tokens[3]).toEqual value: ')', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[4]).toEqual value: 'bar', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[5]).toEqual value: '\\1', scopes: ['source.js', 'string.regexp.js', 'keyword.other.back-reference.regexp'] + expect(tokens[6]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + + {tokens} = grammar.tokenizeLine('/(?foo)bar\\k/') + expect(tokens[0]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: '(?', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[2]).toEqual value: 'foo', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp'] + expect(tokens[3]).toEqual value: ')', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[4]).toEqual value: 'bar', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[5]).toEqual value: '\\k', scopes: ['source.js', 'string.regexp.js', 'keyword.other.back-reference.regexp'] + expect(tokens[6]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + + {tokens} = grammar.tokenizeLine('/(?:foo)bar/') + expect(tokens[0]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: '(?:', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[2]).toEqual value: 'foo', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp'] + expect(tokens[3]).toEqual value: ')', scopes: ['source.js', 'string.regexp.js', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[4]).toEqual value: 'bar', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[5]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + + {tokens} = grammar.tokenizeLine('/(?<=foo)test(?=bar)/') + expect(tokens[0]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'string.regexp.js', 'meta.group.assertion.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[2]).toEqual value: '?<=', scopes: ['source.js', 'string.regexp.js', 'meta.group.assertion.regexp', 'meta.assertion.look-behind.regexp'] + expect(tokens[3]).toEqual value: 'foo', scopes: ['source.js', 'string.regexp.js', 'meta.group.assertion.regexp'] + expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'string.regexp.js', 'meta.group.assertion.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[5]).toEqual value: 'test', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[6]).toEqual value: '(', scopes: ['source.js', 'string.regexp.js', 'meta.group.assertion.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[7]).toEqual value: '?=', scopes: ['source.js', 'string.regexp.js', 'meta.group.assertion.regexp', 'meta.assertion.look-ahead.regexp'] + expect(tokens[8]).toEqual value: 'bar', scopes: ['source.js', 'string.regexp.js', 'meta.group.assertion.regexp'] + expect(tokens[9]).toEqual value: ')', scopes: ['source.js', 'string.regexp.js', 'meta.group.assertion.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[10]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + + {tokens} = grammar.tokenizeLine('/(? expect(tokens[6]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] expect(tokens[7]).toEqual value: ']', scopes: ['source.js', 'meta.brace.square.js'] - it "tokenizes regular expressions inside ternary expressions", -> - {tokens} = grammar.tokenizeLine('a ? /b/ : /c/') - expect(tokens[ 0]).toEqual value: 'a ', scopes: ['source.js'] - expect(tokens[ 1]).toEqual value: '?', scopes: ['source.js', 'keyword.operator.js'] - expect(tokens[ 2]).toEqual value: ' ', scopes: ['source.js', 'string.regexp.js'] - expect(tokens[ 3]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] - expect(tokens[ 4]).toEqual value: 'b', scopes: ['source.js', 'string.regexp.js'] - expect(tokens[ 5]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] - expect(tokens[ 6]).toEqual value: ' ', scopes: ['source.js'] - expect(tokens[ 7]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.js'] - expect(tokens[ 8]).toEqual value: ' ', scopes: ['source.js', 'string.regexp.js'] - expect(tokens[ 9]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] - expect(tokens[10]).toEqual value: 'c', scopes: ['source.js', 'string.regexp.js'] - expect(tokens[11]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + it "tokenizes regular expressions inside curly brackets", -> + {tokens} = grammar.tokenizeLine('{/test/}') + expect(tokens[0]).toEqual value: '{', scopes: ['source.js', 'meta.brace.curly.js'] + expect(tokens[1]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[2]).toEqual value: 'test', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[3]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + expect(tokens[4]).toEqual value: '}', scopes: ['source.js', 'meta.brace.curly.js'] it "tokenizes regular expressions inside arrow function expressions", -> {tokens} = grammar.tokenizeLine('getRegex = () => /^helloworld$/;') @@ -193,6 +317,17 @@ describe "Javascript grammar", -> {tokens} = grammar.tokenizeLine('0X1D306') expect(tokens[0]).toEqual value: '0X1D306', scopes: ['source.js', 'constant.numeric.hex.js'] + {tokens} = grammar.tokenizeLine('0x1D306n') + expect(tokens[0]).toEqual value: '0x1D306n', scopes: ['source.js', 'constant.numeric.hex.js'] + + {tokens} = grammar.tokenizeLine('0X1D306n') + expect(tokens[0]).toEqual value: '0X1D306n', scopes: ['source.js', 'constant.numeric.hex.js'] + + {tokens} = grammar.tokenizeLine('0X1D30_69A3') + expect(tokens[0]).toEqual value: '0X1D30', scopes: ['source.js', 'constant.numeric.hex.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.hex.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '69A3', scopes: ['source.js', 'constant.numeric.hex.js'] + it "tokenizes binary literals", -> {tokens} = grammar.tokenizeLine('0b011101110111010001100110') expect(tokens[0]).toEqual value: '0b011101110111010001100110', scopes: ['source.js', 'constant.numeric.binary.js'] @@ -200,6 +335,25 @@ describe "Javascript grammar", -> {tokens} = grammar.tokenizeLine('0B011101110111010001100110') expect(tokens[0]).toEqual value: '0B011101110111010001100110', scopes: ['source.js', 'constant.numeric.binary.js'] + {tokens} = grammar.tokenizeLine('0b011101110111010001100110n') + expect(tokens[0]).toEqual value: '0b011101110111010001100110n', scopes: ['source.js', 'constant.numeric.binary.js'] + + {tokens} = grammar.tokenizeLine('0B011101110111010001100110n') + expect(tokens[0]).toEqual value: '0B011101110111010001100110n', scopes: ['source.js', 'constant.numeric.binary.js'] + + {tokens} = grammar.tokenizeLine('0B0111_0111_0111_0100_0110_0110') + expect(tokens[0]).toEqual value: '0B0111', scopes: ['source.js', 'constant.numeric.binary.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.binary.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '0111', scopes: ['source.js', 'constant.numeric.binary.js'] + expect(tokens[3]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.binary.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[4]).toEqual value: '0111', scopes: ['source.js', 'constant.numeric.binary.js'] + expect(tokens[5]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.binary.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[6]).toEqual value: '0100', scopes: ['source.js', 'constant.numeric.binary.js'] + expect(tokens[7]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.binary.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[8]).toEqual value: '0110', scopes: ['source.js', 'constant.numeric.binary.js'] + expect(tokens[9]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.binary.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[10]).toEqual value: '0110', scopes: ['source.js', 'constant.numeric.binary.js'] + it "tokenizes octal literals", -> {tokens} = grammar.tokenizeLine('0o1411') expect(tokens[0]).toEqual value: '0o1411', scopes: ['source.js', 'constant.numeric.octal.js'] @@ -207,13 +361,41 @@ describe "Javascript grammar", -> {tokens} = grammar.tokenizeLine('0O1411') expect(tokens[0]).toEqual value: '0O1411', scopes: ['source.js', 'constant.numeric.octal.js'] + {tokens} = grammar.tokenizeLine('0o1411n') + expect(tokens[0]).toEqual value: '0o1411n', scopes: ['source.js', 'constant.numeric.octal.js'] + + {tokens} = grammar.tokenizeLine('0O1411n') + expect(tokens[0]).toEqual value: '0O1411n', scopes: ['source.js', 'constant.numeric.octal.js'] + {tokens} = grammar.tokenizeLine('0010') expect(tokens[0]).toEqual value: '0010', scopes: ['source.js', 'constant.numeric.octal.js'] + {tokens} = grammar.tokenizeLine('0010_7201_5112') + expect(tokens[0]).toEqual value: '0010', scopes: ['source.js', 'constant.numeric.octal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.octal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '7201', scopes: ['source.js', 'constant.numeric.octal.js'] + expect(tokens[3]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.octal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[4]).toEqual value: '5112', scopes: ['source.js', 'constant.numeric.octal.js'] + + {tokens} = grammar.tokenizeLine('0O1411_1236') + expect(tokens[0]).toEqual value: '0O1411', scopes: ['source.js', 'constant.numeric.octal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.octal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '1236', scopes: ['source.js', 'constant.numeric.octal.js'] + it "tokenizes decimals", -> {tokens} = grammar.tokenizeLine('1234') expect(tokens[0]).toEqual value: '1234', scopes: ['source.js', 'constant.numeric.decimal.js'] + {tokens} = grammar.tokenizeLine('123456789n') + expect(tokens[0]).toEqual value: '123456789n', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('123_456_789n') + expect(tokens[0]).toEqual value: '123', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '456', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[3]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[4]).toEqual value: '789n', scopes: ['source.js', 'constant.numeric.decimal.js'] + {tokens} = grammar.tokenizeLine('5e-10') expect(tokens[0]).toEqual value: '5e-10', scopes: ['source.js', 'constant.numeric.decimal.js'] @@ -224,24 +406,92 @@ describe "Javascript grammar", -> expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] + {tokens} = grammar.tokenizeLine('9_9.') + expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[3]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] + {tokens} = grammar.tokenizeLine('.9') expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] expect(tokens[1]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + {tokens} = grammar.tokenizeLine('.9_9') + expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] + expect(tokens[1]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[2]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[3]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + {tokens} = grammar.tokenizeLine('9.9') expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] expect(tokens[2]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + {tokens} = grammar.tokenizeLine('9_9.9_9') + expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[3]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] + expect(tokens[4]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[5]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[6]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + {tokens} = grammar.tokenizeLine('.1e-23') expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] expect(tokens[1]).toEqual value: '1e-23', scopes: ['source.js', 'constant.numeric.decimal.js'] + {tokens} = grammar.tokenizeLine('.1_1E+1_1') + expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] + expect(tokens[1]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[2]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[3]).toEqual value: '1E+1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[4]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[5]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + {tokens} = grammar.tokenizeLine('1.E3') expect(tokens[0]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] expect(tokens[2]).toEqual value: 'E3', scopes: ['source.js', 'constant.numeric.decimal.js'] + {tokens} = grammar.tokenizeLine('1_1.E-1_1') + expect(tokens[0]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[3]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] + expect(tokens[4]).toEqual value: 'E-1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[5]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[6]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('1_1.1_1E1_1') + expect(tokens[0]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[3]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] + expect(tokens[4]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[5]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[6]).toEqual value: '1E1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[7]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[8]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('9_9') + expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('9_9_9') + expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[3]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[4]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('999_999_999') + expect(tokens[0]).toEqual value: '999', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[2]).toEqual value: '999', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[3]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js'] + expect(tokens[4]).toEqual value: '999', scopes: ['source.js', 'constant.numeric.decimal.js'] + it "does not tokenize numbers that are part of a variable", -> {tokens} = grammar.tokenizeLine('hi$1') expect(tokens[0]).toEqual value: 'hi$1', scopes: ['source.js'] @@ -249,6 +499,20 @@ describe "Javascript grammar", -> {tokens} = grammar.tokenizeLine('hi_1') expect(tokens[0]).toEqual value: 'hi_1', scopes: ['source.js'] + {tokens} = grammar.tokenizeLine('_1') + expect(tokens[0]).toEqual value: '_1', scopes: ['source.js', 'constant.other.js'] + + {tokens} = grammar.tokenizeLine('1_') + expect(tokens[0]).toEqual value: '1_', scopes: ['source.js', 'invalid.illegal.identifier.js'] + + {tokens} = grammar.tokenizeLine('1_._1') + expect(tokens[0]).toEqual value: '1_', scopes: ['source.js', 'invalid.illegal.identifier.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.delimiter.property.period.js'] + expect(tokens[2]).toEqual value: '_1', scopes: ['source.js', 'variable.other.property.js'] + + {tokens} = grammar.tokenizeLine('1__1') + expect(tokens[0]).toEqual value: '1__1', scopes: ['source.js', 'invalid.illegal.identifier.js'] + describe "operators", -> it "tokenizes them", -> operators = ["delete", "in", "of", "instanceof", "new", "typeof", "void"] @@ -266,6 +530,18 @@ describe "Javascript grammar", -> expect(tokens[1]).toEqual value: '...', scopes: ['source.js', 'keyword.operator.spread.js'] expect(tokens[2]).toEqual value: 'iterableObj', scopes: ['source.js'] + {tokens} = grammar.tokenizeLine('...arguments') + expect(tokens[0]).toEqual value: '...', scopes: ['source.js', 'keyword.operator.spread.js'] + expect(tokens[1]).toEqual value: 'arguments', scopes: ['source.js', 'variable.language.js'] + + {tokens} = grammar.tokenizeLine('...super') + expect(tokens[0]).toEqual value: '...', scopes: ['source.js', 'keyword.operator.spread.js'] + expect(tokens[1]).toEqual value: 'super', scopes: ['source.js', 'variable.language.js'] + + {tokens} = grammar.tokenizeLine('...this') + expect(tokens[0]).toEqual value: '...', scopes: ['source.js', 'keyword.operator.spread.js'] + expect(tokens[1]).toEqual value: 'this', scopes: ['source.js', 'variable.language.js'] + describe "increment, decrement", -> it "tokenizes increment", -> {tokens} = grammar.tokenizeLine('i++') @@ -277,15 +553,6 @@ describe "Javascript grammar", -> expect(tokens[0]).toEqual value: 'i', scopes: ['source.js'] expect(tokens[1]).toEqual value: '--', scopes: ['source.js', 'keyword.operator.decrement.js'] - describe "conditional ternary", -> - it "tokenizes them", -> - {tokens} = grammar.tokenizeLine('test ? expr1 : expr2') - expect(tokens[0]).toEqual value: 'test ', scopes: ['source.js'] - expect(tokens[1]).toEqual value: '?', scopes: ['source.js', 'keyword.operator.js'] - expect(tokens[2]).toEqual value: ' expr1 ', scopes: ['source.js'] - expect(tokens[3]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.js'] - expect(tokens[4]).toEqual value: ' expr2', scopes: ['source.js'] - describe "logical", -> operators = ["&&", "||", "!"] @@ -312,6 +579,15 @@ describe "Javascript grammar", -> expect(tokens[0]).toEqual value: '~', scopes: ['source.js', 'keyword.operator.bitwise.js'] expect(tokens[1]).toEqual value: 'a', scopes: ['source.js'] + it "tokenizes bitwise shift operators", -> + operators = ["<<", ">>", ">>>"] + + for operator in operators + {tokens} = grammar.tokenizeLine('a ' + operator + ' b') + expect(tokens[0]).toEqual value: 'a ', scopes: ['source.js'] + expect(tokens[1]).toEqual value: operator, scopes: ['source.js', 'keyword.operator.bitwise.shift.js'] + expect(tokens[2]).toEqual value: ' b', scopes: ['source.js'] + it "tokenizes them", -> operators = ["|", "^", "&"] @@ -389,9 +665,20 @@ describe "Javascript grammar", -> {tokens} = grammar.tokenizeLine('OBJ.prop') expect(tokens[0]).toEqual value: 'OBJ', scopes: ['source.js', 'constant.other.object.js'] + it "tokenises constants with dollar signs", -> + constants = ['CON$TANT', 'ABC$', 'ABC$D', '$_ALL', 'ALL_$', 'ANGULAR$$$$$$$$$$$'] + for constant in constants + {tokens} = grammar.tokenizeLine(constant) + expect(tokens[0]).toEqual value: constant, scopes: ['source.js', 'constant.other.js'] + + it "doesn't tokenise constants without alphabetic characters", -> + for name in ['$_', '$', '_', '$_$_$_$___$___$____$'] + {tokens} = grammar.tokenizeLine(name) + expect(tokens[0]).toEqual value: name, scopes: ['source.js'] + it "tokenizes variables declared using `const` as constants", -> {tokens} = grammar.tokenizeLine('const myCoolVar = 42;') - expect(tokens[0]).toEqual value: 'const', scopes: ['source.js', 'storage.modifier.js'] + expect(tokens[0]).toEqual value: 'const', scopes: ['source.js', 'storage.type.const.js'] expect(tokens[1]).toEqual value: ' ', scopes: ['source.js'] expect(tokens[2]).toEqual value: 'myCoolVar', scopes: ['source.js', 'constant.other.js'] expect(tokens[3]).toEqual value: ' ', scopes: ['source.js'] @@ -406,7 +693,7 @@ describe "Javascript grammar", -> c if(a) """ - expect(lines[0][0]).toEqual value: 'const', scopes: ['source.js', 'storage.modifier.js'] + expect(lines[0][0]).toEqual value: 'const', scopes: ['source.js', 'storage.type.const.js'] expect(lines[0][1]).toEqual value: ' ', scopes: ['source.js'] expect(lines[0][2]).toEqual value: 'a', scopes: ['source.js', 'constant.other.js'] expect(lines[0][3]).toEqual value: ',', scopes: ['source.js', 'meta.delimiter.object.comma.js'] @@ -425,7 +712,7 @@ describe "Javascript grammar", -> c, } = foo """ - expect(lines[0][0]).toEqual value: 'const', scopes: ['source.js', 'storage.modifier.js'] + expect(lines[0][0]).toEqual value: 'const', scopes: ['source.js', 'storage.type.const.js'] expect(lines[0][1]).toEqual value: ' ', scopes: ['source.js'] expect(lines[0][2]).toEqual value: '{', scopes: ['source.js', 'meta.brace.curly.js'] expect(lines[1][0]).toEqual value: ' ', scopes: ['source.js'] @@ -443,14 +730,14 @@ describe "Javascript grammar", -> {tokens} = grammar.tokenizeLine('(const hi);') expect(tokens[0]).toEqual value: '(', scopes: ['source.js', 'meta.brace.round.js'] - expect(tokens[1]).toEqual value: 'const', scopes: ['source.js', 'storage.modifier.js'] + expect(tokens[1]).toEqual value: 'const', scopes: ['source.js', 'storage.type.const.js'] expect(tokens[2]).toEqual value: ' ', scopes: ['source.js'] expect(tokens[3]).toEqual value: 'hi', scopes: ['source.js', 'constant.other.js'] expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'meta.brace.round.js'] expect(tokens[5]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] {tokens} = grammar.tokenizeLine('const {first:f,second,...rest} = obj;') - expect(tokens[0]).toEqual value: 'const', scopes: ['source.js', 'storage.modifier.js'] + expect(tokens[0]).toEqual value: 'const', scopes: ['source.js', 'storage.type.const.js'] expect(tokens[1]).toEqual value: ' ', scopes: ['source.js'] expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'meta.brace.curly.js'] expect(tokens[3]).toEqual value: 'first', scopes: ['source.js'] @@ -468,7 +755,7 @@ describe "Javascript grammar", -> expect(tokens[15]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] {tokens} = grammar.tokenizeLine('const c = /regex/;') - expect(tokens[0]).toEqual value: 'const', scopes: ['source.js', 'storage.modifier.js'] + expect(tokens[0]).toEqual value: 'const', scopes: ['source.js', 'storage.type.const.js'] expect(tokens[1]).toEqual value: ' ', scopes: ['source.js'] expect(tokens[2]).toEqual value: 'c', scopes: ['source.js', 'constant.other.js'] expect(tokens[3]).toEqual value: ' ', scopes: ['source.js'] @@ -484,7 +771,7 @@ describe "Javascript grammar", -> expect(tokens[0]).toEqual value: 'for', scopes: ['source.js', 'keyword.control.js'] expect(tokens[1]).toEqual value: ' ', scopes: ['source.js'] expect(tokens[2]).toEqual value: '(', scopes: ['source.js', 'meta.brace.round.js'] - expect(tokens[3]).toEqual value: 'const', scopes: ['source.js', 'storage.modifier.js'] + expect(tokens[3]).toEqual value: 'const', scopes: ['source.js', 'storage.type.const.js'] expect(tokens[4]).toEqual value: ' ', scopes: ['source.js'] expect(tokens[5]).toEqual value: 'elem', scopes: ['source.js', 'constant.other.js'] expect(tokens[6]).toEqual value: ' ', scopes: ['source.js'] @@ -499,36 +786,31 @@ describe "Javascript grammar", -> expect(tokens[8]).toEqual value: ' object', scopes: ['source.js'] {tokens} = grammar.tokenizeLine 'const index = 0;' - expect(tokens[0]).toEqual value: 'const', scopes: ['source.js', 'storage.modifier.js'] + expect(tokens[0]).toEqual value: 'const', scopes: ['source.js', 'storage.type.const.js'] expect(tokens[2]).toEqual value: 'index', scopes: ['source.js', 'constant.other.js'] expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js'] {tokens} = grammar.tokenizeLine 'const offset = 0;' - expect(tokens[0]).toEqual value: 'const', scopes: ['source.js', 'storage.modifier.js'] + expect(tokens[0]).toEqual value: 'const', scopes: ['source.js', 'storage.type.const.js'] expect(tokens[2]).toEqual value: 'offset', scopes: ['source.js', 'constant.other.js'] expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js'] it "tokenizes support constants", -> - {tokens} = grammar.tokenizeLine('awesome = cool.systemLanguage;') + {tokens} = grammar.tokenizeLine('awesome = cool.EPSILON;') expect(tokens[0]).toEqual value: 'awesome ', scopes: ['source.js'] expect(tokens[1]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js'] expect(tokens[3]).toEqual value: 'cool', scopes: ['source.js', 'variable.other.object.js'] expect(tokens[4]).toEqual value: '.', scopes: ['source.js', 'meta.delimiter.property.period.js'] - expect(tokens[5]).toEqual value: 'systemLanguage', scopes: ['source.js', 'support.constant.js'] + expect(tokens[5]).toEqual value: 'EPSILON', scopes: ['source.js', 'support.constant.js'] expect(tokens[6]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] - it "does not tokenize constants when they are object keys", -> - {tokens} = grammar.tokenizeLine('FOO: 1') - expect(tokens[0]).toEqual value: 'FOO', scopes: ['source.js'] - expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.js'] - it "tokenizes constants in the middle of ternary expressions", -> {tokens} = grammar.tokenizeLine('a ? FOO : b') expect(tokens[3]).toEqual value: 'FOO', scopes: ['source.js', 'constant.other.js'] it "tokenizes constants at the end of ternary expressions", -> {tokens} = grammar.tokenizeLine('a ? b : FOO') - expect(tokens[5]).toEqual value: 'FOO', scopes: ['source.js', 'constant.other.js'] + expect(tokens[7]).toEqual value: 'FOO', scopes: ['source.js', 'constant.other.js'] describe "ES6 string templates", -> it "tokenizes them as strings", -> @@ -555,56 +837,103 @@ describe "Javascript grammar", -> expect(tokens[13]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] expect(tokens[14]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.js', 'punctuation.definition.string.end.js'] - describe "ES6 tagged HTML string templates", -> - it "tokenizes them as strings", -> + describe "HTML template strings", -> + # TODO: Remove after Atom 1.21 is released + [tagScope, entityScope] = [] + if parseFloat(atom.getVersion()) <= 1.21 + tagScope = 'meta.tag.inline.any.html' + entityScope = 'entity.name.tag.inline.any.html' + else + tagScope = 'meta.tag.inline.b.html' + entityScope = 'entity.name.tag.inline.b.html' + + beforeEach -> + waitsForPromise -> + atom.packages.activatePackage("language-html") + + it "tokenizes ES6 tagged HTML string templates", -> {tokens} = grammar.tokenizeLine('html`hey ${name}`') - expect(tokens[0]).toEqual value: 'html', scopes: [ 'source.js', 'string.quoted.template.html.js', 'entity.name.function.js' ] - expect(tokens[1]).toEqual value: '`', scopes: [ 'source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.begin.js' ] - expect(tokens[2]).toEqual value: 'hey ', scopes: ['source.js', 'string.quoted.template.html.js'] - expect(tokens[3]).toEqual value: '${', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] - expect(tokens[4]).toEqual value: 'name', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source'] - expect(tokens[5]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] - expect(tokens[6]).toEqual value: '', scopes: ['source.js', 'string.quoted.template.html.js'] - expect(tokens[7]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.end.js'] - - describe "ES6 tagged HTML string templates with expanded function name", -> - it "tokenizes them as strings", -> + expect(tokens[0]).toEqual value: 'html', scopes: ['source.js', 'string.quoted.template.html.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.begin.js'] + expect(tokens[2]).toEqual value: 'hey ', scopes: ['source.js', 'string.quoted.template.html.js'] + expect(tokens[3]).toEqual value: '<', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.begin.html'] + expect(tokens[4]).toEqual value: 'b', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, entityScope] + expect(tokens[5]).toEqual value: '>', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.end.html'] + expect(tokens[6]).toEqual value: '${', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[7]).toEqual value: 'name', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source'] + expect(tokens[8]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[9]).toEqual value: '', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.end.html'] + expect(tokens[12]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.end.js'] + + it "tokenizes innerHTML attribute declarations with string template tags", -> + {tokens} = grammar.tokenizeLine('text.innerHTML = `hey ${name}`') + expect(tokens[0]).toEqual value: 'text', scopes: ['source.js', 'variable.other.object.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.delimiter.property.period.js'] + expect(tokens[2]).toEqual value: 'innerHTML', scopes: ['source.js', 'variable.other.property.js'] + expect(tokens[3]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js'] + expect(tokens[5]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[6]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.begin.js'] + expect(tokens[7]).toEqual value: 'hey ', scopes: ['source.js', 'string.quoted.template.html.js'] + expect(tokens[8]).toEqual value: '<', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.begin.html'] + expect(tokens[9]).toEqual value: 'b', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, entityScope] + expect(tokens[10]).toEqual value: '>', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.end.html'] + expect(tokens[11]).toEqual value: '${', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[12]).toEqual value: 'name', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source'] + expect(tokens[13]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[14]).toEqual value: '', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.end.html'] + expect(tokens[17]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.end.js'] + + it "tokenizes ES6 tagged HTML string templates with expanded function name", -> {tokens} = grammar.tokenizeLine('escapeHTML`hey ${name}`') - expect(tokens[0]).toEqual value: 'escapeHTML', scopes: [ 'source.js', 'string.quoted.template.html.js', 'entity.name.function.js' ] - expect(tokens[1]).toEqual value: '`', scopes: [ 'source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.begin.js' ] - expect(tokens[2]).toEqual value: 'hey ', scopes: ['source.js', 'string.quoted.template.html.js'] - expect(tokens[3]).toEqual value: '${', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] - expect(tokens[4]).toEqual value: 'name', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source'] - expect(tokens[5]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] - expect(tokens[6]).toEqual value: '', scopes: ['source.js', 'string.quoted.template.html.js'] - expect(tokens[7]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.end.js'] - - describe "ES6 tagged HTML string templates with expanded function name and white space", -> - it "tokenizes them as strings", -> + expect(tokens[0]).toEqual value: 'escapeHTML', scopes: ['source.js', 'string.quoted.template.html.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.begin.js'] + expect(tokens[2]).toEqual value: 'hey ', scopes: ['source.js', 'string.quoted.template.html.js'] + expect(tokens[3]).toEqual value: '<', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.begin.html'] + expect(tokens[4]).toEqual value: 'b', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, entityScope] + expect(tokens[5]).toEqual value: '>', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.end.html'] + expect(tokens[6]).toEqual value: '${', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[7]).toEqual value: 'name', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source'] + expect(tokens[8]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[9]).toEqual value: '', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.end.html'] + expect(tokens[12]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.end.js'] + + it "tokenizes ES6 tagged HTML string templates with expanded function name and white space", -> {tokens} = grammar.tokenizeLine('escapeHTML `hey ${name}`') - expect(tokens[0]).toEqual value: 'escapeHTML', scopes: [ 'source.js', 'string.quoted.template.html.js', 'entity.name.function.js' ] - expect(tokens[1]).toEqual value: ' ', scopes: [ 'source.js', 'string.quoted.template.html.js' ] - expect(tokens[2]).toEqual value: '`', scopes: [ 'source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.begin.js' ] - expect(tokens[3]).toEqual value: 'hey ', scopes: ['source.js', 'string.quoted.template.html.js'] - expect(tokens[4]).toEqual value: '${', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] - expect(tokens[5]).toEqual value: 'name', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source'] - expect(tokens[6]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] - expect(tokens[7]).toEqual value: '', scopes: ['source.js', 'string.quoted.template.html.js'] - expect(tokens[8]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.end.js'] + expect(tokens[0]).toEqual value: 'escapeHTML', scopes: ['source.js', 'string.quoted.template.html.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: ' ', scopes: ['source.js', 'string.quoted.template.html.js'] + expect(tokens[2]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.begin.js'] + expect(tokens[3]).toEqual value: 'hey ', scopes: ['source.js', 'string.quoted.template.html.js'] + expect(tokens[4]).toEqual value: '<', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.begin.html'] + expect(tokens[5]).toEqual value: 'b', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, entityScope] + expect(tokens[6]).toEqual value: '>', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.end.html'] + expect(tokens[7]).toEqual value: '${', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[8]).toEqual value: 'name', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source'] + expect(tokens[9]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.html.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[10]).toEqual value: '', scopes: ['source.js', 'string.quoted.template.html.js', tagScope, 'punctuation.definition.tag.end.html'] + expect(tokens[13]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.html.js', 'punctuation.definition.string.end.js'] describe "ES6 tagged Relay.QL string templates", -> it "tokenizes them as strings", -> {tokens} = grammar.tokenizeLine('Relay.QL`fragment on Foo { id }`') - expect(tokens[0]).toEqual value: 'Relay.QL', scopes: [ 'source.js', 'string.quoted.template.graphql.js', 'entity.name.function.js' ] - expect(tokens[1]).toEqual value: '`', scopes: [ 'source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.begin.js' ] + expect(tokens[0]).toEqual value: 'Relay.QL', scopes: ['source.js', 'string.quoted.template.graphql.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.begin.js'] expect(tokens[2]).toEqual value: 'fragment on Foo { id }', scopes: ['source.js', 'string.quoted.template.graphql.js'] expect(tokens[3]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.end.js'] describe "ES6 tagged Relay.QL string templates with interpolation", -> it "tokenizes them as strings", -> {tokens} = grammar.tokenizeLine('Relay.QL`fragment on Foo { ${myFragment} }`') - expect(tokens[0]).toEqual value: 'Relay.QL', scopes: [ 'source.js', 'string.quoted.template.graphql.js', 'entity.name.function.js' ] - expect(tokens[1]).toEqual value: '`', scopes: [ 'source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.begin.js' ] + expect(tokens[0]).toEqual value: 'Relay.QL', scopes: ['source.js', 'string.quoted.template.graphql.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.begin.js'] expect(tokens[2]).toEqual value: 'fragment on Foo { ', scopes: ['source.js', 'string.quoted.template.graphql.js'] expect(tokens[3]).toEqual value: '${', scopes: ['source.js', 'string.quoted.template.graphql.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] expect(tokens[4]).toEqual value: 'myFragment', scopes: ['source.js', 'string.quoted.template.graphql.js', 'source.js.embedded.source'] @@ -612,6 +941,33 @@ describe "Javascript grammar", -> expect(tokens[6]).toEqual value: ' }', scopes: ['source.js', 'string.quoted.template.graphql.js'] expect(tokens[7]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.end.js'] + describe "ES6 tagged gql string templates", -> + it "tokenizes them as strings", -> + {tokens} = grammar.tokenizeLine('gql`fragment on Foo { id }`') + expect(tokens[0]).toEqual value: 'gql', scopes: ['source.js', 'string.quoted.template.graphql.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.begin.js'] + expect(tokens[2]).toEqual value: 'fragment on Foo { id }', scopes: ['source.js', 'string.quoted.template.graphql.js'] + expect(tokens[3]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.graphql.js', 'punctuation.definition.string.end.js'] + + describe "ES6 tagged SQL string templates", -> + it "tokenizes them as strings", -> + {tokens} = grammar.tokenizeLine('SQL`SELECT foo FROM bar WHERE id = :id`') + expect(tokens[0]).toEqual value: 'SQL', scopes: ['source.js', 'string.quoted.template.sql.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.sql.js', 'punctuation.definition.string.begin.js'] + expect(tokens[2]).toEqual value: 'SELECT foo FROM bar WHERE id = :id', scopes: ['source.js', 'string.quoted.template.sql.js'] + expect(tokens[3]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.sql.js', 'punctuation.definition.string.end.js'] + + describe "ES6 tagged SQL string templates with interpolation", -> + it "tokenizes them as strings", -> + {tokens} = grammar.tokenizeLine('SQL`SELECT foo FROM bar WHERE id = ${id}`') + expect(tokens[0]).toEqual value: 'SQL', scopes: ['source.js', 'string.quoted.template.sql.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.sql.js', 'punctuation.definition.string.begin.js'] + expect(tokens[2]).toEqual value: 'SELECT foo FROM bar WHERE id = ', scopes: ['source.js', 'string.quoted.template.sql.js'] + expect(tokens[3]).toEqual value: '${', scopes: ['source.js', 'string.quoted.template.sql.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[4]).toEqual value: 'id', scopes: ['source.js', 'string.quoted.template.sql.js', 'source.js.embedded.source'] + expect(tokens[5]).toEqual value: '}', scopes: ['source.js', 'string.quoted.template.sql.js', 'source.js.embedded.source', 'punctuation.section.embedded.js'] + expect(tokens[6]).toEqual value: '`', scopes: ['source.js', 'string.quoted.template.sql.js', 'punctuation.definition.string.end.js'] + describe "ES6 class", -> it "tokenizes class", -> {tokens} = grammar.tokenizeLine('class MyClass') @@ -729,14 +1085,22 @@ describe "Javascript grammar", -> member2 } from "module-name"; ''' - expect(lines[0][2]).toEqual value: '/*', scopes: ['source.js', 'meta.import.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(lines[0][2]).toEqual value: '/*', scopes: ['source.js', 'meta.import.js', 'comment.block.js', 'punctuation.definition.comment.begin.js'] expect(lines[0][3]).toEqual value: ' comment ', scopes: ['source.js', 'meta.import.js', 'comment.block.js'] - expect(lines[0][4]).toEqual value: '*/', scopes: ['source.js', 'meta.import.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(lines[0][4]).toEqual value: '*/', scopes: ['source.js', 'meta.import.js', 'comment.block.js', 'punctuation.definition.comment.end.js'] expect(lines[1][4]).toEqual value: '//', scopes: ['source.js', 'meta.import.js', 'comment.line.double-slash.js', 'punctuation.definition.comment.js'] expect(lines[1][5]).toEqual value: ' comment', scopes: ['source.js', 'meta.import.js', 'comment.line.double-slash.js'] - expect(lines[2][1]).toEqual value: '/*', scopes: ['source.js', 'meta.import.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(lines[2][1]).toEqual value: '/*', scopes: ['source.js', 'meta.import.js', 'comment.block.js', 'punctuation.definition.comment.begin.js'] expect(lines[2][2]).toEqual value: ' comment ', scopes: ['source.js', 'meta.import.js', 'comment.block.js'] - expect(lines[2][3]).toEqual value: '*/', scopes: ['source.js', 'meta.import.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(lines[2][3]).toEqual value: '*/', scopes: ['source.js', 'meta.import.js', 'comment.block.js', 'punctuation.definition.comment.end.js'] + + # https://github.com/atom/language-javascript/issues/485 + lines = grammar.tokenizeLines ''' + import a from 'a' // + import b from 'b' + ''' + expect(lines[0][10]).toEqual value: '//', scopes: ['source.js', 'meta.import.js', 'comment.line.double-slash.js', 'punctuation.definition.comment.js'] + expect(lines[1][0]).toEqual value: 'import', scopes: ['source.js', 'meta.import.js', 'keyword.control.js'] describe "ES6 export", -> it "tokenizes named export", -> @@ -754,7 +1118,7 @@ describe "Javascript grammar", -> {tokens} = grammar.tokenizeLine('export const CONSTANT = 0;') expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] - expect(tokens[2]).toEqual value: 'const', scopes: ['source.js', 'storage.modifier.js'] + expect(tokens[2]).toEqual value: 'const', scopes: ['source.js', 'storage.type.const.js'] expect(tokens[4]).toEqual value: 'CONSTANT', scopes: ['source.js', 'constant.other.js'] expect(tokens[6]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js'] @@ -817,7 +1181,7 @@ describe "Javascript grammar", -> expect(tokens[6]).toEqual value: "'", scopes: ['source.js', 'string.quoted.single.js', 'punctuation.definition.string.begin.js'] expect(tokens[7]).toEqual value: "prop", scopes: ['source.js', 'string.quoted.single.js'] expect(tokens[8]).toEqual value: "'", scopes: ['source.js', 'string.quoted.single.js', 'punctuation.definition.string.end.js'] - expect(tokens[9]).toEqual value: ":", scopes: ['source.js', 'keyword.operator.js'] + expect(tokens[9]).toEqual value: ":", scopes: ['source.js', 'keyword.operator.assignment.js'] expect(tokens[11]).toEqual value: "'", scopes: ['source.js', 'string.quoted.single.js', 'punctuation.definition.string.begin.js'] expect(tokens[12]).toEqual value: "value", scopes: ['source.js', 'string.quoted.single.js'] expect(tokens[13]).toEqual value: "'", scopes: ['source.js', 'string.quoted.single.js', 'punctuation.definition.string.end.js'] @@ -844,17 +1208,17 @@ describe "Javascript grammar", -> ''' expect(lines[1][4]).toEqual value: '//', scopes: ['source.js', 'meta.export.js', 'comment.line.double-slash.js', 'punctuation.definition.comment.js'] expect(lines[1][5]).toEqual value: ' comment', scopes: ['source.js', 'meta.export.js', 'comment.line.double-slash.js'] - expect(lines[2][1]).toEqual value: '/*', scopes: ['source.js', 'meta.export.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(lines[2][1]).toEqual value: '/*', scopes: ['source.js', 'meta.export.js', 'comment.block.js', 'punctuation.definition.comment.begin.js'] expect(lines[2][2]).toEqual value: ' comment ', scopes: ['source.js', 'meta.export.js', 'comment.block.js'] - expect(lines[2][3]).toEqual value: '*/', scopes: ['source.js', 'meta.export.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(lines[2][3]).toEqual value: '*/', scopes: ['source.js', 'meta.export.js', 'comment.block.js', 'punctuation.definition.comment.end.js'] {tokens} = grammar.tokenizeLine('export {member1, /* comment */ member2} /* comment */ from "module";') - expect(tokens[6]).toEqual value: '/*', scopes: ['source.js', 'meta.export.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokens[6]).toEqual value: '/*', scopes: ['source.js', 'meta.export.js', 'comment.block.js', 'punctuation.definition.comment.begin.js'] expect(tokens[7]).toEqual value: ' comment ', scopes: ['source.js', 'meta.export.js', 'comment.block.js'] - expect(tokens[8]).toEqual value: '*/', scopes: ['source.js', 'meta.export.js', 'comment.block.js', 'punctuation.definition.comment.js'] - expect(tokens[13]).toEqual value: '/*', scopes: ['source.js', 'meta.export.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokens[8]).toEqual value: '*/', scopes: ['source.js', 'meta.export.js', 'comment.block.js', 'punctuation.definition.comment.end.js'] + expect(tokens[13]).toEqual value: '/*', scopes: ['source.js', 'meta.export.js', 'comment.block.js', 'punctuation.definition.comment.begin.js'] expect(tokens[14]).toEqual value: ' comment ', scopes: ['source.js', 'meta.export.js', 'comment.block.js'] - expect(tokens[15]).toEqual value: '*/', scopes: ['source.js', 'meta.export.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokens[15]).toEqual value: '*/', scopes: ['source.js', 'meta.export.js', 'comment.block.js', 'punctuation.definition.comment.end.js'] it "tokenizes default class export", -> {tokens} = grammar.tokenizeLine('export default class {}') @@ -892,6 +1256,29 @@ describe "Javascript grammar", -> expect(tokens[10]).toEqual value: '}', scopes: ['source.js', 'meta.export.js', 'punctuation.definition.modules.end.js'] expect(tokens[12]).toEqual value: 'from', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + it "tokenizes multiline re-export", -> + lines = grammar.tokenizeLines ''' + export { + default as alias, + member1 as alias1, + member2, + } from "module-name"; + ''' + expect(lines[0][0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(lines[0][2]).toEqual value: '{', scopes: ['source.js', 'meta.export.js', 'punctuation.definition.modules.begin.js'] + expect(lines[1][1]).toEqual value: 'default', scopes: ['source.js', 'meta.export.js', 'variable.language.default.js'] + expect(lines[1][3]).toEqual value: 'as', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(lines[1][5]).toEqual value: 'alias', scopes: ['source.js', 'meta.export.js', 'variable.other.module-alias.js'] + expect(lines[1][6]).toEqual value: ',', scopes: ['source.js', 'meta.export.js', 'meta.delimiter.object.comma.js'] + expect(lines[2][1]).toEqual value: 'member1', scopes: ['source.js', 'meta.export.js', 'variable.other.module.js'] + expect(lines[2][3]).toEqual value: 'as', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + expect(lines[2][5]).toEqual value: 'alias1', scopes: ['source.js', 'meta.export.js', 'variable.other.module-alias.js'] + expect(lines[2][6]).toEqual value: ',', scopes: ['source.js', 'meta.export.js', 'meta.delimiter.object.comma.js'] + expect(lines[3][1]).toEqual value: 'member2', scopes: ['source.js', 'meta.export.js', 'variable.other.module.js'] + expect(lines[3][2]).toEqual value: ',', scopes: ['source.js', 'meta.export.js', 'meta.delimiter.object.comma.js'] + expect(lines[4][0]).toEqual value: '}', scopes: ['source.js', 'meta.export.js', 'punctuation.definition.modules.end.js'] + expect(lines[4][2]).toEqual value: 'from', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] + describe "ES6 yield", -> it "tokenizes yield", -> {tokens} = grammar.tokenizeLine('yield next') @@ -902,24 +1289,6 @@ describe "Javascript grammar", -> expect(tokens[0]).toEqual value: 'yield', scopes: ['source.js', 'meta.control.yield.js', 'keyword.control.js'] expect(tokens[2]).toEqual value: '*', scopes: ['source.js', 'meta.control.yield.js', 'storage.modifier.js'] - it "does not tokenize yield when it is an object key", -> - {tokens} = grammar.tokenizeLine('yield: 1') - expect(tokens[0]).toEqual value: 'yield', scopes: ['source.js'] - expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.js'] - - it "tokenizes yield in the middle of ternary expressions", -> - {tokens} = grammar.tokenizeLine('a ? yield : b') - expect(tokens[3]).toEqual value: 'yield', scopes: ['source.js', 'meta.control.yield.js', 'keyword.control.js'] - - it "tokenizes yield at the end of ternary expressions", -> - {tokens} = grammar.tokenizeLine('a ? b : yield') - expect(tokens[5]).toEqual value: 'yield', scopes: ['source.js', 'meta.control.yield.js', 'keyword.control.js'] - - describe "default: in a switch statement", -> - it "tokenizes it as a keyword", -> - {tokens} = grammar.tokenizeLine('default: ') - expect(tokens[0]).toEqual value: 'default', scopes: ['source.js', 'keyword.control.js'] - describe "functions", -> it "tokenizes regular function declarations", -> {tokens} = grammar.tokenizeLine('function foo(){}') @@ -986,6 +1355,15 @@ describe "Javascript grammar", -> expect(tokens[9]).toEqual value: 'onclick', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] expect(tokens[13]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] + it "tokenises getter/setter keywords", -> + {tokens} = grammar.tokenizeLine('get name(){ }') + expect(tokens[0]).toEqual value: 'get', scopes: ['source.js', 'meta.function.method.definition.js', 'keyword.operator.getter.js'] + expect(tokens[2]).toEqual value: 'name', scopes: ['source.js', 'meta.function.method.definition.js', 'entity.name.function.js'] + expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'punctuation.definition.function.body.end.bracket.curly.js'] + it "tokenizes ES6 method definitions", -> {tokens} = grammar.tokenizeLine('f(a, b) {}') expect(tokens[0]).toEqual value: 'f', scopes: ['source.js', 'meta.function.method.definition.js', 'entity.name.function.js'] @@ -1033,6 +1411,48 @@ describe "Javascript grammar", -> expect(tokens[5]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] expect(tokens[6]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + {tokens} = grammar.tokenizeLine('import (x) { return "Yeah"; }') + expect(tokens[0]).toEqual value: 'import', scopes: ['source.js', 'meta.function.method.definition.js', 'entity.name.function.js'] + expect(tokens[11]).toEqual value: 'Yeah', scopes: ['source.js', 'string.quoted.double.js'] + + {tokens} = grammar.tokenizeLine('export (x) { return "Nah"; }') + expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.function.method.definition.js', 'entity.name.function.js'] + expect(tokens[11]).toEqual value: 'Nah', scopes: ['source.js', 'string.quoted.double.js'] + + it "tokenises ES6 methods with computed names", -> + {tokens} = grammar.tokenizeLine('[ foo ] () { }') + expect(tokens[0]).toEqual value: '[', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.computed-key.js', 'punctuation.definition.computed-key.begin.bracket.square.js'] + expect(tokens[2]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.computed-key.js', 'variable.parameter.property.js'] + expect(tokens[4]).toEqual value: ']', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.computed-key.js', 'punctuation.definition.computed-key.end.bracket.square.js'] + expect(tokens[6]).toEqual value: '(', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[7]).toEqual value: ')', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(tokens[9]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + expect(tokens[11]).toEqual value: '}', scopes: ['source.js', 'punctuation.definition.function.body.end.bracket.curly.js'] + + {tokens} = grammar.tokenizeLine('[ "delet" + this ] (gun) { }') + expect(tokens[0]).toEqual value: '[', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.computed-key.js', 'punctuation.definition.computed-key.begin.bracket.square.js'] + expect(tokens[2]).toEqual value: '"', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.computed-key.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[3]).toEqual value: 'delet', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.computed-key.js', 'string.quoted.double.js'] + expect(tokens[4]).toEqual value: '"', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.computed-key.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[6]).toEqual value: '+', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.computed-key.js', 'keyword.operator.js'] + expect(tokens[8]).toEqual value: 'this', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.computed-key.js', 'variable.language.js'] + expect(tokens[10]).toEqual value: ']', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.computed-key.js', 'punctuation.definition.computed-key.end.bracket.square.js'] + expect(tokens[12]).toEqual value: '(', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[13]).toEqual value: 'gun', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[14]).toEqual value: ')', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(tokens[16]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + expect(tokens[18]).toEqual value: '}', scopes: ['source.js', 'punctuation.definition.function.body.end.bracket.curly.js'] + + {tokens} = grammar.tokenizeLine('get [ foo ] () { }') + expect(tokens[0]).toEqual value: 'get', scopes: ['source.js', 'meta.function.method.definition.js', 'keyword.operator.getter.js'] + expect(tokens[2]).toEqual value: '[', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.computed-key.js', 'punctuation.definition.computed-key.begin.bracket.square.js'] + expect(tokens[4]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.computed-key.js', 'variable.parameter.property.js'] + expect(tokens[6]).toEqual value: ']', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.computed-key.js', 'punctuation.definition.computed-key.end.bracket.square.js'] + expect(tokens[8]).toEqual value: '(', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + expect(tokens[9]).toEqual value: ')', scopes: ['source.js', 'meta.function.method.definition.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] + expect(tokens[11]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] + expect(tokens[13]).toEqual value: '}', scopes: ['source.js', 'punctuation.definition.function.body.end.bracket.curly.js'] + it "tokenizes constructors", -> {tokens} = grammar.tokenizeLine('constructor(p1, p2) { this.p1 = p1; }') expect(tokens[0]).toEqual value: 'constructor', scopes: ['source.js', 'meta.function.js', 'entity.name.function.constructor.js'] @@ -1043,6 +1463,9 @@ describe "Javascript grammar", -> expect(tokens[6]).toEqual value: ')', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'punctuation.definition.parameters.end.bracket.round.js'] expect(tokens[10]).toEqual value: 'this', scopes: ['source.js', 'variable.language.js'] + {tokens} = grammar.tokenizeLine('constructorABC: {}') + expect(tokens[0]).not.toEqual value: 'constructor', scopes: ['source.js', 'meta.function.js', 'entity.name.function.constructor.js'] + it "tokenizes named function expressions", -> {tokens} = grammar.tokenizeLine('var func = function foo(){}') expect(tokens[0]).toEqual value: 'var', scopes: ['source.js', 'storage.type.var.js'] @@ -1075,6 +1498,23 @@ describe "Javascript grammar", -> expect(tokens[7]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.json.js', 'entity.name.function.js'] expect(tokens[8]).toEqual value: '(', scopes: ['source.js', 'meta.function.json.js', 'meta.parameters.js', 'punctuation.definition.parameters.begin.bracket.round.js'] + {tokens} = grammar.tokenizeLine('function : a => a') + expect(tokens[0]).toEqual value: 'function', scopes: ['source.js', 'meta.function.arrow.json.js', 'entity.name.function.js'] + expect(tokens[2]).toEqual value: ':', scopes: ['source.js', 'meta.function.arrow.json.js', 'keyword.operator.assignment.js'] + expect(tokens[4]).toEqual value: 'a', scopes: ['source.js', 'meta.function.arrow.json.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[6]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.json.js', 'storage.type.function.arrow.js'] + expect(tokens[7]).toEqual value: ' a', scopes: ['source.js'] + + {tokens} = grammar.tokenizeLine('"func": a => a') + expect(tokens[1]).toEqual value: 'func', scopes: ['source.js', 'meta.function.arrow.json.js', 'string.quoted.double.js', 'entity.name.function.js'] + expect(tokens[3]).toEqual value: ':', scopes: ['source.js', 'meta.function.arrow.json.js', 'keyword.operator.assignment.js'] + expect(tokens[5]).toEqual value: 'a', scopes: ['source.js', 'meta.function.arrow.json.js', 'meta.parameters.js', 'variable.parameter.function.js'] + expect(tokens[7]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.json.js', 'storage.type.function.arrow.js'] + expect(tokens[8]).toEqual value: ' a', scopes: ['source.js'] + + {tokens} = grammar.tokenizeLine('"func" : a => a') + expect(tokens[8]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.json.js', 'storage.type.function.arrow.js'] + it "tokenizes generator functions", -> {tokens} = grammar.tokenizeLine('function* foo(){}') expect(tokens[0]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] @@ -1184,6 +1624,12 @@ describe "Javascript grammar", -> {tokens} = grammar.tokenizeLine('this.obj.prototype = new El()') expect(tokens[0]).toEqual value: 'this', scopes: ['source.js', 'variable.language.js'] + {tokens} = grammar.tokenizeLine('$this') + expect(tokens[0]).toEqual value: '$this', scopes: ['source.js'] + + {tokens} = grammar.tokenizeLine('this$') + expect(tokens[0]).toEqual value: 'this$', scopes: ['source.js'] + it "tokenizes 'super'", -> {tokens} = grammar.tokenizeLine('super') expect(tokens[0]).toEqual value: 'super', scopes: ['source.js', 'variable.language.js'] @@ -1247,7 +1693,7 @@ describe "Javascript grammar", -> expect(tokens[8]).toEqual value: ',', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.delimiter.object.comma.js'] expect(tokens[10]).toEqual value: '{', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.brace.curly.js'] expect(tokens[11]).toEqual value: 'a', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js'] - expect(tokens[12]).toEqual value: ':', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'keyword.operator.js'] + expect(tokens[12]).toEqual value: ':', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'keyword.operator.assignment.js'] expect(tokens[14]).toEqual value: '123', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'constant.numeric.decimal.js'] expect(tokens[15]).toEqual value: '}', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.brace.curly.js'] expect(tokens[16]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] @@ -1469,7 +1915,7 @@ describe "Javascript grammar", -> expect(tokens[5]).toEqual value: 'x', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', scope] expect(tokens[6]).toEqual value: delim, scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', scope, 'punctuation.definition.string.end.js'] expect(tokens[8]).toEqual value: '+', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'keyword.operator.js'] - expect(tokens[9]).toEqual value: ' y ', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js' ] + expect(tokens[9]).toEqual value: ' y ', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js'] expect(tokens[10]).toEqual value: '+', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'keyword.operator.js'] expect(tokens[12]).toEqual value: delim, scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', scope, 'punctuation.definition.string.begin.js'] expect(tokens[13]).toEqual value: ':function()', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', scope] @@ -1477,37 +1923,33 @@ describe "Javascript grammar", -> expect(tokens[15]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] describe "comments", -> + it "tokenizes // comments", -> + {tokens} = grammar.tokenizeLine '//' + expect(tokens[0]).toEqual value: '//', scopes: ['source.js', 'comment.line.double-slash.js', 'punctuation.definition.comment.js'] + + {tokens} = grammar.tokenizeLine '// stuff' + expect(tokens[0]).toEqual value: '//', scopes: ['source.js', 'comment.line.double-slash.js', 'punctuation.definition.comment.js'] + expect(tokens[1]).toEqual value: ' stuff', scopes: ['source.js', 'comment.line.double-slash.js'] + it "tokenizes /* */ comments", -> {tokens} = grammar.tokenizeLine('/**/') - expect(tokens[0]).toEqual value: '/*', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] - expect(tokens[1]).toEqual value: '*/', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokens[0]).toEqual value: '/*', scopes: ['source.js', 'comment.block.empty.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[1]).toEqual value: '*/', scopes: ['source.js', 'comment.block.empty.js', 'punctuation.definition.comment.end.js'] {tokens} = grammar.tokenizeLine('/* foo */') - expect(tokens[0]).toEqual value: '/*', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokens[0]).toEqual value: '/*', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.begin.js'] expect(tokens[1]).toEqual value: ' foo ', scopes: ['source.js', 'comment.block.js'] - expect(tokens[2]).toEqual value: '*/', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokens[2]).toEqual value: '*/', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.end.js'] it "tokenizes /** */ comments", -> {tokens} = grammar.tokenizeLine('/***/') - expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.js'] - expect(tokens[1]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.js'] + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[1]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] {tokens} = grammar.tokenizeLine('/** foo */') - expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.js'] + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] expect(tokens[1]).toEqual value: ' foo ', scopes: ['source.js', 'comment.block.documentation.js'] - expect(tokens[2]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.js'] - - {tokens} = grammar.tokenizeLine('/** @mixins */') - expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.js'] - expect(tokens[2]).toEqual value: '@mixins', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] - expect(tokens[3]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js'] - expect(tokens[4]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.js'] - - it "tokenizes JSDoc comment documentation", -> - {tokens} = grammar.tokenizeLine('/** @param {object} variable this is the description */') - expect(tokens[4]).toEqual value: '{object}', scopes: ['source.js', 'comment.block.documentation.js', 'other.meta.jsdoc', 'entity.name.type.instance.jsdoc'] - expect(tokens[6]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'other.meta.jsdoc', 'variable.other.jsdoc'] - expect(tokens[8]).toEqual value: 'this is the description ', scopes: ['source.js', 'comment.block.documentation.js', 'other.meta.jsdoc', 'other.description.jsdoc'] + expect(tokens[2]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] it "tokenizes // comments", -> {tokens} = grammar.tokenizeLine('// comment') @@ -1516,7 +1958,7 @@ describe "Javascript grammar", -> it "tokenizes comments inside constant definitions", -> {tokens} = grammar.tokenizeLine('const a, // comment') - expect(tokens[0]).toEqual value: 'const', scopes: ['source.js', 'storage.modifier.js'] + expect(tokens[0]).toEqual value: 'const', scopes: ['source.js', 'storage.type.const.js'] expect(tokens[2]).toEqual value: 'a', scopes: ['source.js', 'constant.other.js'] expect(tokens[3]).toEqual value: ',', scopes: ['source.js', 'meta.delimiter.object.comma.js'] expect(tokens[5]).toEqual value: '//', scopes: ['source.js', 'comment.line.double-slash.js', 'punctuation.definition.comment.js'] @@ -1525,25 +1967,25 @@ describe "Javascript grammar", -> it "tokenizes comments inside function declarations", -> {tokens} = grammar.tokenizeLine('function /* */ foo() /* */ {}') expect(tokens[0]).toEqual value: 'function', scopes: ['source.js', 'meta.function.js', 'storage.type.function.js'] - expect(tokens[2]).toEqual value: '/*', scopes: ['source.js', 'meta.function.js', 'comment.block.js', 'punctuation.definition.comment.js'] - expect(tokens[4]).toEqual value: '*/', scopes: ['source.js', 'meta.function.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokens[2]).toEqual value: '/*', scopes: ['source.js', 'meta.function.js', 'comment.block.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[4]).toEqual value: '*/', scopes: ['source.js', 'meta.function.js', 'comment.block.js', 'punctuation.definition.comment.end.js'] expect(tokens[6]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.js', 'entity.name.function.js'] - expect(tokens[10]).toEqual value: '/*', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] - expect(tokens[12]).toEqual value: '*/', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokens[10]).toEqual value: '/*', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[12]).toEqual value: '*/', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.end.js'] {tokens} = grammar.tokenizeLine('x => /* */ {}') expect(tokens[0]).toEqual value: 'x', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.function.js'] expect(tokens[2]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] - expect(tokens[4]).toEqual value: '/*', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] - expect(tokens[6]).toEqual value: '*/', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokens[4]).toEqual value: '/*', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[6]).toEqual value: '*/', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.end.js'] expect(tokens[8]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] {tokens} = grammar.tokenizeLine('.foo = x => /* */ {}') expect(tokens[1]).toEqual value: 'foo', scopes: ['source.js', 'meta.function.arrow.js', 'entity.name.function.js'] expect(tokens[5]).toEqual value: 'x', scopes: ['source.js', 'meta.function.arrow.js', 'meta.parameters.js', 'variable.parameter.function.js'] expect(tokens[7]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] - expect(tokens[9]).toEqual value: '/*', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] - expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokens[9]).toEqual value: '/*', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.end.js'] expect(tokens[13]).toEqual value: '{', scopes: ['source.js', 'punctuation.definition.function.body.begin.bracket.curly.js'] lines = grammar.tokenizeLines ''' @@ -1569,9 +2011,9 @@ describe "Javascript grammar", -> it "tokenizes comments inside function parameters correctly", -> {tokens} = grammar.tokenizeLine('function test(p1 /*, p2 */) {}') - expect(tokens[6]).toEqual value: '/*', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokens[6]).toEqual value: '/*', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'comment.block.js', 'punctuation.definition.comment.begin.js'] expect(tokens[7]).toEqual value: ', p2 ', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'comment.block.js'] - expect(tokens[8]).toEqual value: '*/', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokens[8]).toEqual value: '*/', scopes: ['source.js', 'meta.function.js', 'meta.parameters.js', 'comment.block.js', 'punctuation.definition.comment.end.js'] describe "console", -> it "tokenizes the console keyword", -> @@ -1596,8 +2038,8 @@ describe "Javascript grammar", -> {tokens} = grammar.tokenizeLine('console/**/.log()') expect(tokens[0]).toEqual value: 'console', scopes: ['source.js', 'entity.name.type.object.console.js'] - expect(tokens[1]).toEqual value: '/*', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] - expect(tokens[2]).toEqual value: '*/', scopes: ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokens[1]).toEqual value: '/*', scopes: ['source.js', 'comment.block.empty.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '*/', scopes: ['source.js', 'comment.block.empty.js', 'punctuation.definition.comment.end.js'] expect(tokens[3]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] expect(tokens[4]).toEqual value: 'log', scopes: ['source.js', 'meta.method-call.js', 'support.function.console.js'] expect(tokens[5]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] @@ -1675,6 +2117,185 @@ describe "Javascript grammar", -> expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] expect(tokens[5]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + describe "promise", -> + it "tokenizes the promise object", -> + {tokens} = grammar.tokenizeLine('Promise;') + expect(tokens[0]).toEqual value: 'Promise', scopes: ['source.js', 'support.class.promise.js'] + expect(tokens[1]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + it "tokenizes promise support functions", -> + {tokens} = grammar.tokenizeLine('Promise.race();') + expect(tokens[0]).toEqual value: 'Promise', scopes: ['source.js', 'support.class.promise.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(tokens[2]).toEqual value: 'race', scopes: ['source.js', 'meta.method-call.js', 'support.function.promise.js'] + expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(tokens[5]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + lines = grammar.tokenizeLines ''' + Promise + .resolve(); + ''' + expect(lines[0][0]).toEqual value: 'Promise', scopes: ['source.js', 'support.class.promise.js'] + expect(lines[1][0]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(lines[1][1]).toEqual value: 'resolve', scopes: ['source.js', 'meta.method-call.js', 'support.function.promise.js'] + expect(lines[1][2]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(lines[1][3]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(lines[1][4]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + it "tokenizes promise custom functions", -> + {tokens} = grammar.tokenizeLine('Promise.anExtraFunction();') + expect(tokens[0]).toEqual value: 'Promise', scopes: ['source.js', 'support.class.promise.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] + expect(tokens[2]).toEqual value: 'anExtraFunction', scopes: ['source.js', 'meta.method-call.js', 'entity.name.function.js'] + expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(tokens[5]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + describe "object literals", -> + keywords = ['super', 'this', 'null', 'true', 'false', 'debugger', 'exports', '__filename'] + + for keyword in keywords + it "tokenizes the #{keyword} keyword when it is an object key", -> + {tokens} = grammar.tokenizeLine("#{keyword}: 1") + expect(tokens[0]).toEqual value: keyword, scopes: ['source.js'] + expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.assignment.js'] + + it "tokenizes object keys", -> + {tokens} = grammar.tokenizeLine('foo: 1') + expect(tokens[0]).toEqual value: 'foo', scopes: ['source.js'] + expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.assignment.js'] + + {tokens} = grammar.tokenizeLine('$abc$: 1') + expect(tokens[0]).toEqual value: '$abc$', scopes: ['source.js'] + expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.assignment.js'] + + {tokens} = grammar.tokenizeLine('0abc: 1') + expect(tokens[0]).toEqual value: '0abc', scopes: ['source.js', 'invalid.illegal.identifier.js'] + expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.assignment.js'] + + {tokens} = grammar.tokenizeLine('"key": 1') + expect(tokens[0]).toEqual value: '"', scopes: ['source.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[1]).toEqual value: 'key', scopes: ['source.js', 'string.quoted.double.js'] + expect(tokens[2]).toEqual value: '"', scopes: ['source.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[3]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.assignment.js'] + + it "tokenizes numbers when they are object keys", -> + {tokens} = grammar.tokenizeLine('123: 1') + expect(tokens[0]).toEqual value: '123', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.assignment.js'] + + it "tokenizes constants when they are object keys", -> + {tokens} = grammar.tokenizeLine('FOO: 1') + expect(tokens[0]).toEqual value: 'FOO', scopes: ['source.js', 'constant.other.js'] + expect(tokens[1]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.assignment.js'] + + describe "ternary expressions", -> + map = + FOO: 'constant.other.js' + super: 'variable.language.js' + this: 'variable.language.js' + null: 'constant.language.null.js' + true: 'constant.language.boolean.true.js' + false: 'constant.language.boolean.false.js' + exports: 'support.variable.js' + __filename: 'support.variable.js' + + for keyword, scope of map + do (keyword, scope) -> + it "tokenizes `#{keyword}` in the middle of ternary expressions", -> + {tokens} = grammar.tokenizeLine("a ? #{keyword} : b") + expect(tokens[3]).toEqual value: keyword, scopes: ['source.js', scope] + + it "tokenizes `#{keyword}` at the end of ternary expressions", -> + {tokens} = grammar.tokenizeLine("a ? b : #{keyword}") + expect(tokens[7]).toEqual value: keyword, scopes: ['source.js', scope] + + it "tokenizes yield at the end of ternary expressions", -> + {tokens} = grammar.tokenizeLine('a ? b : yield') + expect(tokens[7]).toEqual value: 'yield', scopes: ['source.js', 'meta.control.yield.js', 'keyword.control.js'] + + it "tokenizes yield in the middle of ternary expressions", -> + {tokens} = grammar.tokenizeLine('a ? yield : b') + expect(tokens[3]).toEqual value: 'yield', scopes: ['source.js', 'meta.control.yield.js', 'keyword.control.js'] + + it "tokenizes regular expressions inside ternary expressions", -> + {tokens} = grammar.tokenizeLine('a ? /b/ : /c/') + expect(tokens[0]).toEqual value: 'a ', scopes: ['source.js'] + expect(tokens[1]).toEqual value: '?', scopes: ['source.js', 'keyword.operator.ternary.js'] + expect(tokens[2]).toEqual value: ' ', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[3]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[4]).toEqual value: 'b', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[5]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + expect(tokens[6]).toEqual value: ' ', scopes: ['source.js'] + expect(tokens[7]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.ternary.js'] + expect(tokens[8]).toEqual value: ' ', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[9]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] + expect(tokens[10]).toEqual value: 'c', scopes: ['source.js', 'string.regexp.js'] + expect(tokens[11]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.end.js'] + + it "tokenizes object literals in the middle of ternary expressions", -> + {tokens} = grammar.tokenizeLine('a ? {key: value} : b') + expect(tokens[1]).toEqual value: '?', scopes: ['source.js', 'keyword.operator.ternary.js'] + expect(tokens[9]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.ternary.js'] + + it "tokenizes arrow functions inside ternary expressions", -> + {tokens} = grammar.tokenizeLine('result = condition ? something : (a, b) => a + b') + expect(tokens[3]).toEqual value: '?', scopes: ['source.js', 'keyword.operator.ternary.js'] + expect(tokens[7]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.ternary.js'] + expect(tokens[16]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] + + {tokens} = grammar.tokenizeLine('result = condition ? (a, b) => a + b : something') + expect(tokens[3]).toEqual value: '?', scopes: ['source.js', 'keyword.operator.ternary.js'] + expect(tokens[12]).toEqual value: '=>', scopes: ['source.js', 'meta.function.arrow.js', 'storage.type.function.arrow.js'] + expect(tokens[18]).toEqual value: ':', scopes: ['source.js', 'keyword.operator.ternary.js'] + + describe "switch statements", -> + it "tokenizes the switch keyword", -> + {tokens} = grammar.tokenizeLine('switch(){}') + expect(tokens[0]).toEqual value: 'switch', scopes: ['source.js', 'meta.switch-statement.js', 'keyword.control.switch.js'] + + it "tokenizes switch expression", -> + {tokens} = grammar.tokenizeLine('switch(foo + bar){}') + expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.switch-statement.js', 'punctuation.definition.switch-expression.begin.bracket.round.js'] + expect(tokens[2]).toEqual value: 'foo ', scopes: ['source.js', 'meta.switch-statement.js'] + expect(tokens[3]).toEqual value: '+', scopes: ['source.js', 'meta.switch-statement.js', 'keyword.operator.js'] + expect(tokens[4]).toEqual value: ' bar', scopes: ['source.js', 'meta.switch-statement.js'] + expect(tokens[5]).toEqual value: ')', scopes: ['source.js', 'meta.switch-statement.js', 'punctuation.definition.switch-expression.end.bracket.round.js'] + + it "tokenizes switch block", -> + lines = grammar.tokenizeLines ''' + switch (foo()) + { + case abc: + case 1+1: + 2+2 + break; + case null: + default: + } + ''' + expect(lines[1][0]).toEqual value: '{', scopes: ['source.js', 'meta.switch-statement.js', 'punctuation.definition.section.switch-block.begin.bracket.curly.js'] + expect(lines[2][1]).toEqual value: 'case', scopes: ['source.js', 'meta.switch-statement.js', 'keyword.control.case.js'] + expect(lines[2][3]).toEqual value: 'abc', scopes: ['source.js', 'meta.switch-statement.js'] + expect(lines[2][4]).toEqual value: ':', scopes: ['source.js', 'meta.switch-statement.js', 'punctuation.definition.section.case-statement.js'] + expect(lines[3][1]).toEqual value: 'case', scopes: ['source.js', 'meta.switch-statement.js', 'keyword.control.case.js'] + expect(lines[3][3]).toEqual value: '1', scopes: ['source.js', 'meta.switch-statement.js', 'constant.numeric.decimal.js'] + expect(lines[3][4]).toEqual value: '+', scopes: ['source.js', 'meta.switch-statement.js', 'keyword.operator.js'] + expect(lines[3][5]).toEqual value: '1', scopes: ['source.js', 'meta.switch-statement.js', 'constant.numeric.decimal.js'] + expect(lines[3][6]).toEqual value: ':', scopes: ['source.js', 'meta.switch-statement.js', 'punctuation.definition.section.case-statement.js'] + expect(lines[4][1]).toEqual value: '2', scopes: ['source.js', 'meta.switch-statement.js', 'constant.numeric.decimal.js'] + expect(lines[4][2]).toEqual value: '+', scopes: ['source.js', 'meta.switch-statement.js', 'keyword.operator.js'] + expect(lines[4][3]).toEqual value: '2', scopes: ['source.js', 'meta.switch-statement.js', 'constant.numeric.decimal.js'] + expect(lines[5][1]).toEqual value: 'break', scopes: ['source.js', 'meta.switch-statement.js', 'keyword.control.js'] + expect(lines[5][2]).toEqual value: ';', scopes: ['source.js', 'meta.switch-statement.js', 'punctuation.terminator.statement.js'] + expect(lines[6][1]).toEqual value: 'case', scopes: ['source.js', 'meta.switch-statement.js', 'keyword.control.case.js'] + expect(lines[6][3]).toEqual value: 'null', scopes: ['source.js', 'meta.switch-statement.js', 'constant.language.null.js'] + expect(lines[6][4]).toEqual value: ':', scopes: ['source.js', 'meta.switch-statement.js', 'punctuation.definition.section.case-statement.js'] + expect(lines[7][1]).toEqual value: 'default', scopes: ['source.js', 'meta.switch-statement.js', 'keyword.control.default.js'] + expect(lines[7][2]).toEqual value: ':', scopes: ['source.js', 'meta.switch-statement.js', 'punctuation.definition.section.case-statement.js'] + expect(lines[8][0]).toEqual value: '}', scopes: ['source.js', 'meta.switch-statement.js', 'punctuation.definition.section.switch-block.end.bracket.curly.js'] + describe "indentation", -> editor = null @@ -1721,7 +2342,7 @@ describe "Javascript grammar", -> it "indents non-allman-style curly braces", -> expectPreservedIndentation """ if (true) { - for (;;) { + for (;;) { // " while (true) { x(); } @@ -1745,7 +2366,7 @@ describe "Javascript grammar", -> it "indents collection literals", -> expectPreservedIndentation """ - [ + [ // " { a: b, c: d @@ -1758,10 +2379,123 @@ describe "Javascript grammar", -> it "indents function arguments", -> expectPreservedIndentation """ f( - g( + g( // " h, i ), j ); """ + + describe "firstLineMatch", -> + it "recognises interpreter directives", -> + valid = """ + #!/usr/sbin/node foo + #!/usr/bin/iojs foo=bar/ + #!/usr/sbin/node + #!/usr/sbin/node foo bar baz + #!/usr/bin/node perl + #!/usr/bin/node bin/perl + #!/usr/bin/node + #!/bin/node + #!/usr/bin/node --script=usr/bin + #! /usr/bin/env A=003 B=149 C=150 D=xzd E=base64 F=tar G=gz H=head I=tail node + #!\t/usr/bin/env --foo=bar node --quu=quux + #! /usr/bin/node + #!/usr/bin/env node + """ + for line in valid.split /\n/ + expect(grammar.firstLineRegex.scanner.findNextMatchSync(line)).not.toBeNull() + + invalid = """ + \x20#!/usr/sbin/node + \t#!/usr/sbin/node + #!/usr/bin/env-node/node-env/ + #!/usr/bin/env-node + #! /usr/binnode + #!\t/usr/bin/env --node=bar + """ + for line in invalid.split /\n/ + expect(grammar.firstLineRegex.scanner.findNextMatchSync(line)).toBeNull() + + it "recognises Emacs modelines", -> + valid = """ + #-*-js-*- + #-*-mode:js-*- + /* -*-js-*- */ + // -*- JavaScript -*- + /* -*- mode:js -*- */ + // -*- font:bar;mode:JS -*- + // -*- font:bar;mode:JavaScript;foo:bar; -*- + // -*-font:mode;mode:JS-*- + // -*- foo:bar mode: js bar:baz -*- + " -*-foo:bar;mode:JS;bar:foo-*- "; + " -*-font-mode:foo;mode:JavaScript;foo-bar:quux-*-" + "-*-font:x;foo:bar; mode : js;bar:foo;foooooo:baaaaar;fo:ba;-*-"; + "-*- font:x;foo : bar ; mode : jS ; bar : foo ; foooooo:baaaaar;fo:ba-*-"; + """ + for line in valid.split /\n/ + expect(grammar.firstLineRegex.scanner.findNextMatchSync(line)).not.toBeNull() + + invalid = """ + /* --*js-*- */ + /* -*-- js -*- + /* -*- -- js -*- + /* -*- javascripts -;- -*- + // -*- iJS -*- + // -*- JS; -*- + // -*- js-stuff -*- + /* -*- model:js -*- + /* -*- indent-mode:js -*- + // -*- font:mode;JS -*- + // -*- mode: -*- JS + // -*- mode: grok-with-js -*- + // -*-font:mode;mode:js--*- + """ + for line in invalid.split /\n/ + expect(grammar.firstLineRegex.scanner.findNextMatchSync(line)).toBeNull() + + it "recognises Vim modelines", -> + valid = """ + vim: se filetype=javascript: + # vim: se ft=javascript: + # vim: set ft=javascript: + # vim: set filetype=JavaScript: + # vim: ft=javascript + # vim: syntax=jAvaScRIPT + # vim: se syntax=JAVASCRIPT: + # ex: syntax=javascript + # vim:ft=javascript + # vim600: ft=javascript + # vim>600: set ft=javascript: + # vi:noai:sw=3 ts=6 ft=javascript + # vi::::::::::noai:::::::::::: ft=javascript + # vim:ts=4:sts=4:sw=4:noexpandtab:ft=javascript + # vi:: noai : : : : sw =3 ts =6 ft =javascript + # vim: ts=4: pi sts=4: ft=javascript: noexpandtab: sw=4: + # vim: ts=4 sts=4: ft=javascript noexpandtab: + # vim:noexpandtab sts=4 ft=javascript ts=4 + # vim:noexpandtab:ft=javascript + # vim:ts=4:sts=4 ft=javascript:noexpandtab:\x20 + # vim:noexpandtab titlestring=hi\|there\\\\ ft=javascript ts=4 + """ + for line in valid.split /\n/ + expect(grammar.firstLineRegex.scanner.findNextMatchSync(line)).not.toBeNull() + + invalid = """ + ex: se filetype=javascript: + _vi: se filetype=javascript: + vi: se filetype=javascript + # vim set ft=javascripts + # vim: soft=javascript + # vim: hairy-syntax=javascript: + # vim set ft=javascript: + # vim: setft=javascript: + # vim: se ft=javascript backupdir=tmp + # vim: set ft=javascript set cmdheight=1 + # vim:noexpandtab sts:4 ft:javascript ts:4 + # vim:noexpandtab titlestring=hi\\|there\\ ft=javascript ts=4 + # vim:noexpandtab titlestring=hi\\|there\\\\\\ ft=javascript ts=4 + """ + for line in invalid.split /\n/ + expect(grammar.firstLineRegex.scanner.findNextMatchSync(line)).toBeNull() diff --git a/spec/jsdoc-spec.coffee b/spec/jsdoc-spec.coffee new file mode 100644 index 00000000..8baf4e28 --- /dev/null +++ b/spec/jsdoc-spec.coffee @@ -0,0 +1,1537 @@ +describe "JSDoc grammar", -> + grammar = null + + beforeEach -> + atom.config.set('core.useTreeSitterParsers', false) + + waitsForPromise -> + atom.packages.activatePackage("language-javascript") + + runs -> + grammar = atom.grammars.grammarForScopeName("source.js") + + describe "inline tags", -> + it "tokenises tags without descriptions", -> + {tokens} = grammar.tokenizeLine('/** Text {@link target} text */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[1]).toEqual value: ' Text ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[3]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.inline.tag.jsdoc'] + expect(tokens[4]).toEqual value: 'link', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc'] + expect(tokens[6]).toEqual value: 'target', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'variable.other.description.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[8]).toEqual value: ' text ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[9]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises tags with an embedded trailing description", -> + {tokens} = grammar.tokenizeLine('/** Text {@linkplain target|Description text} */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[1]).toEqual value: ' Text ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[3]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.inline.tag.jsdoc'] + expect(tokens[4]).toEqual value: 'linkplain', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc'] + expect(tokens[6]).toEqual value: 'target', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'variable.other.description.jsdoc'] + expect(tokens[7]).toEqual value: '|', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.separator.pipe.jsdoc'] + expect(tokens[8]).toEqual value: 'Description text', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[9]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** Text {@linkcode target Description text} */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[1]).toEqual value: ' Text ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[2]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[3]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.inline.tag.jsdoc'] + expect(tokens[4]).toEqual value: 'linkcode', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc'] + expect(tokens[6]).toEqual value: 'target', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'variable.other.description.jsdoc'] + expect(tokens[7]).toEqual value: ' Description text', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[8]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[10]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises tags with a preceding description", -> + {tokens} = grammar.tokenizeLine('/** Text [Description text]{@link target} */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[1]).toEqual value: ' Text ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[2]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'constant.other.description.jsdoc', 'punctuation.definition.bracket.square.begin.jsdoc'] + expect(tokens[3]).toEqual value: 'Description text', scopes: ['source.js', 'comment.block.documentation.js', 'constant.other.description.jsdoc'] + expect(tokens[4]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'constant.other.description.jsdoc', 'punctuation.definition.bracket.square.end.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.inline.tag.jsdoc'] + expect(tokens[7]).toEqual value: 'link', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc'] + expect(tokens[9]).toEqual value: 'target', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'variable.other.description.jsdoc'] + expect(tokens[10]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[12]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** Text [Description text]{@tutorial target|Description} */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[1]).toEqual value: ' Text ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[2]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'constant.other.description.jsdoc', 'punctuation.definition.bracket.square.begin.jsdoc'] + expect(tokens[3]).toEqual value: 'Description text', scopes: ['source.js', 'comment.block.documentation.js', 'constant.other.description.jsdoc'] + expect(tokens[4]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'constant.other.description.jsdoc', 'punctuation.definition.bracket.square.end.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.inline.tag.jsdoc'] + expect(tokens[7]).toEqual value: 'tutorial', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc'] + expect(tokens[9]).toEqual value: 'target', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'variable.other.description.jsdoc'] + expect(tokens[10]).toEqual value: '|', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.separator.pipe.jsdoc'] + expect(tokens[11]).toEqual value: 'Description', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[12]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[14]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises inline tags which follow block tags", -> + {tokens} = grammar.tokenizeLine('/** @param {object} variable - this is a {@link linked} description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' - this is a ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[12]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.inline.tag.jsdoc'] + expect(tokens[13]).toEqual value: 'link', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc'] + expect(tokens[15]).toEqual value: 'linked', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'variable.other.description.jsdoc'] + expect(tokens[16]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[17]).toEqual value: ' description ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[18]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} variable - this is a {@link linked#description}. */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' - this is a ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[12]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.inline.tag.jsdoc'] + expect(tokens[13]).toEqual value: 'link', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc'] + expect(tokens[15]).toEqual value: 'linked#description', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'variable.other.description.jsdoc'] + expect(tokens[16]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[17]).toEqual value: '. ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[18]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} variable - this is a [description with a]{@link example}. */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' - this is a ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'constant.other.description.jsdoc', 'punctuation.definition.bracket.square.begin.jsdoc'] + expect(tokens[12]).toEqual value: 'description with a', scopes: ['source.js', 'comment.block.documentation.js', 'constant.other.description.jsdoc'] + expect(tokens[13]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'constant.other.description.jsdoc', 'punctuation.definition.bracket.square.end.jsdoc'] + expect(tokens[14]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[15]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.inline.tag.jsdoc'] + expect(tokens[16]).toEqual value: 'link', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc'] + expect(tokens[18]).toEqual value: 'example', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'variable.other.description.jsdoc'] + expect(tokens[19]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[20]).toEqual value: '. ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[21]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises inline tags within default @param values", -> + {tokens} = grammar.tokenizeLine('/** @param {EntityType} [typeHint={@link EntityType.FILE}] */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'EntityType', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'typeHint', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[13]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.inline.tag.jsdoc'] + expect(tokens[14]).toEqual value: 'link', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc'] + expect(tokens[16]).toEqual value: 'EntityType.FILE', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'entity.name.type.instance.jsdoc', 'variable.other.description.jsdoc'] + expect(tokens[17]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[18]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[20]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + describe "block tags", -> + it "tokenises simple tags", -> + {tokens} = grammar.tokenizeLine('/** @mixins */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'mixins', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[4]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[5]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @global @static */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'global', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[4]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[5]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[6]).toEqual value: 'static', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[7]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[8]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises @see tags with basic links", -> + {tokens} = grammar.tokenizeLine('/** @see name#path */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'see', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[4]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[5]).toEqual value: 'name#path', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[6]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[7]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @see http://atom.io/ */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'see', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[4]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[5]).toEqual value: 'http://atom.io/', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.link.underline.jsdoc'] + expect(tokens[6]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[7]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises @see tags with {@link} tags", -> + {tokens} = grammar.tokenizeLine('/** @see {@link text|Description} */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'see', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.inline.tag.jsdoc'] + expect(tokens[7]).toEqual value: 'link', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc'] + expect(tokens[9]).toEqual value: 'text', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'variable.other.description.jsdoc'] + expect(tokens[10]).toEqual value: '|', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.separator.pipe.jsdoc'] + expect(tokens[11]).toEqual value: 'Description', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[12]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[14]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @see [Description]{@link name#path} */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'see', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '[Description]', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[6]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[7]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.inline.tag.jsdoc'] + expect(tokens[8]).toEqual value: 'link', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'storage.type.class.jsdoc'] + expect(tokens[10]).toEqual value: 'name#path', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'variable.other.description.jsdoc'] + expect(tokens[11]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[13]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises tags with type expressions", -> + {tokens} = grammar.tokenizeLine('/** @const {object} */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'const', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[4]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[8]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[9]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @define {object} */') + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'define', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + it "tokenises unnamed @param tags", -> + {tokens} = grammar.tokenizeLine('/** @param {object} */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[1]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[8]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[9]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises @param tags", -> + {tokens} = grammar.tokenizeLine('/** @param {object} variable */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises @param tags with a description", -> + {tokens} = grammar.tokenizeLine('/** @param {object} variable this is the description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @arg {object} variable this is the description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'arg', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @argument {object} variable this is the description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'argument', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} variable - this is the description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' - this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} $variable this is the description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '$variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises @param tags marked optional", -> + {tokens} = grammar.tokenizeLine('/** @param {object} [variable] this is the description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[12]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[13]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} [ variable ] this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: ' variable ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[12]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + describe "default @param values", -> + it "tokenises unquoted values", -> + {tokens} = grammar.tokenizeLine('/** @param {object} [variable=default value] this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: 'default value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[13]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[14]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} [variable = default value] this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'variable ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[13]).toEqual value: 'default value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[14]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[15]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} [ variable = default value ] this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: ' variable ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[13]).toEqual value: 'default value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[14]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[15]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[16]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} [variable=default.value] this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: 'default', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'variable.other.object.js'] + expect(tokens[13]).toEqual value: '.', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.delimiter.property.period.js'] + expect(tokens[14]).toEqual value: 'value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'support.variable.property.dom.js'] + expect(tokens[15]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[16]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + it "tokenises quoted values", -> + {tokens} = grammar.tokenizeLine('/** @param {object} [variable="default value"] this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[13]).toEqual value: 'default value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js'] + expect(tokens[14]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[15]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[16]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} [variable = "default value"] this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'variable ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[13]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[14]).toEqual value: 'default value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js'] + expect(tokens[15]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[16]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[17]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} [ variable = " default value " ] this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: ' variable ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[13]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[14]).toEqual value: ' default value ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js'] + expect(tokens[15]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[17]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[18]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine("/** @param {object} [variable='default value'] this is the description */") + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '\'', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js', 'punctuation.definition.string.begin.js'] + expect(tokens[13]).toEqual value: 'default value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js'] + expect(tokens[14]).toEqual value: '\'', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js', 'punctuation.definition.string.end.js'] + expect(tokens[15]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[16]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine("/** @param {object} [variable = 'default value'] this is the description */") + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'variable ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[13]).toEqual value: '\'', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js', 'punctuation.definition.string.begin.js'] + expect(tokens[14]).toEqual value: 'default value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js'] + expect(tokens[15]).toEqual value: '\'', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js', 'punctuation.definition.string.end.js'] + expect(tokens[16]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[17]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine("/** @param {object} [ variable = ' default value ' ] this is the description */") + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: ' variable ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[13]).toEqual value: '\'', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js', 'punctuation.definition.string.begin.js'] + expect(tokens[14]).toEqual value: ' default value ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js'] + expect(tokens[15]).toEqual value: '\'', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js', 'punctuation.definition.string.end.js'] + expect(tokens[17]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[18]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + it "tokenises object literals", -> + {tokens} = grammar.tokenizeLine('/** @param {Object} [variable={a: "b"}] - An object */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[13]).toEqual value: 'a', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[14]).toEqual value: ':', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'keyword.operator.assignment.js'] + expect(tokens[16]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[17]).toEqual value: 'b', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js'] + expect(tokens[18]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[19]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[20]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[21]).toEqual value: ' - An object ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Object} [ variable = { a : "b" } ] - An object */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: ' variable ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[13]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[14]).toEqual value: ' a ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[15]).toEqual value: ':', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'keyword.operator.assignment.js'] + expect(tokens[17]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[18]).toEqual value: 'b', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js'] + expect(tokens[19]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[21]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[23]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[24]).toEqual value: ' - An object ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Object} [variable={}] - Empty object */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'punctuation.section.scope.begin.js'] + expect(tokens[13]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'punctuation.section.scope.end.js'] + expect(tokens[14]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[15]).toEqual value: ' - Empty object ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Object} [ variable = { } ] - Empty object */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: ' variable ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[13]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[15]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[17]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[18]).toEqual value: ' - Empty object ', scopes: ['source.js', 'comment.block.documentation.js'] + + it "tokenises arrays", -> + {tokens} = grammar.tokenizeLine('/** @param {Array} [variable=[1,2,3]] - An array */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Array', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[13]).toEqual value: '1', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'constant.numeric.decimal.js'] + expect(tokens[14]).toEqual value: ',', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.delimiter.object.comma.js'] + expect(tokens[15]).toEqual value: '2', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'constant.numeric.decimal.js'] + expect(tokens[16]).toEqual value: ',', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.delimiter.object.comma.js'] + expect(tokens[17]).toEqual value: '3', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'constant.numeric.decimal.js'] + expect(tokens[18]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[19]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[20]).toEqual value: ' - An array ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Array} [ variable = [ 1 , 2 , 3 ] ] - An array */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Array', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: ' variable ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[13]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[15]).toEqual value: '1', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'constant.numeric.decimal.js'] + expect(tokens[17]).toEqual value: ',', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.delimiter.object.comma.js'] + expect(tokens[19]).toEqual value: '2', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'constant.numeric.decimal.js'] + expect(tokens[21]).toEqual value: ',', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.delimiter.object.comma.js'] + expect(tokens[23]).toEqual value: '3', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'constant.numeric.decimal.js'] + expect(tokens[25]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[27]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[28]).toEqual value: ' - An array ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Array} [variable=[]] - Empty array */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Array', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[13]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[14]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[15]).toEqual value: ' - Empty array ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Array} [ variable = [ ] ] - Empty array */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Array', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: ' variable ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[13]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[15]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[17]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[18]).toEqual value: ' - Empty array ', scopes: ['source.js', 'comment.block.documentation.js'] + + it "tokenizes arrays inside object literals", -> + {tokens} = grammar.tokenizeLine('/** @param {Object} [thing={a: [], b: [0, 2], c: "String"}] [Not Highlighted] [] [] [] */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'thing', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[13]).toEqual value: 'a', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[14]).toEqual value: ':', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'keyword.operator.assignment.js'] + expect(tokens[16]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[17]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[18]).toEqual value: ',', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.delimiter.object.comma.js'] + expect(tokens[19]).toEqual value: ' b', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[20]).toEqual value: ':', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'keyword.operator.assignment.js'] + expect(tokens[22]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[23]).toEqual value: '0', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'constant.numeric.decimal.js'] + expect(tokens[24]).toEqual value: ',', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.delimiter.object.comma.js'] + expect(tokens[26]).toEqual value: '2', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'constant.numeric.decimal.js'] + expect(tokens[27]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[28]).toEqual value: ',', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.delimiter.object.comma.js'] + expect(tokens[29]).toEqual value: ' c', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[30]).toEqual value: ':', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'keyword.operator.assignment.js'] + expect(tokens[32]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[33]).toEqual value: 'String', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js'] + expect(tokens[34]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[35]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[36]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[37]).toEqual value: ' [Not Highlighted] [] [] [] ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[38]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "flags any description text touching the closing bracket", -> + {tokens} = grammar.tokenizeLine('/** @param {Object} [thing={a: [], b: [0, 2], c: "String"}][Bad Description] [] [] [] */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'thing', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[13]).toEqual value: 'a', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[14]).toEqual value: ':', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'keyword.operator.assignment.js'] + expect(tokens[16]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[17]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[18]).toEqual value: ',', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.delimiter.object.comma.js'] + expect(tokens[19]).toEqual value: ' b', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[20]).toEqual value: ':', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'keyword.operator.assignment.js'] + expect(tokens[22]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[23]).toEqual value: '0', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'constant.numeric.decimal.js'] + expect(tokens[24]).toEqual value: ',', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.delimiter.object.comma.js'] + expect(tokens[26]).toEqual value: '2', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'constant.numeric.decimal.js'] + expect(tokens[27]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[28]).toEqual value: ',', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.delimiter.object.comma.js'] + expect(tokens[29]).toEqual value: ' c', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[30]).toEqual value: ':', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'keyword.operator.assignment.js'] + expect(tokens[32]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[33]).toEqual value: 'String', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js'] + expect(tokens[34]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[35]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[36]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[37]).toEqual value: '[Bad', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'invalid.illegal.syntax.jsdoc'] + expect(tokens[38]).toEqual value: ' Description] [] [] [] ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[39]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "does not tokenise arrays inside strings", -> + {tokens} = grammar.tokenizeLine('/** @param {Object} [thing=\'{a: [], b: [0, 2], c: "String"}][Quoted Description] [\'] [Unquoted Description] */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'thing', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: "'", scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js', 'punctuation.definition.string.begin.js'] + expect(tokens[13]).toEqual value: '{a: [], b: [0, 2], c: "String"}][Quoted Description] [', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js'] + expect(tokens[14]).toEqual value: "'", scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js', 'punctuation.definition.string.end.js'] + expect(tokens[15]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[16]).toEqual value: ' [Unquoted Description] ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[17]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Object} [thing="{a: [], b: [0, 2], c: \'String\'}][Quoted Description] ["] [] [] */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'thing', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[13]).toEqual value: '{a: [], b: [0, 2], c: \'String\'}][Quoted Description] [', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js'] + expect(tokens[14]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[15]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[16]).toEqual value: ' [] [] ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[17]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Object} [thing="{a: [], b: [0, 2], c: \'String\'}][Quoted Description] ["][Bad Description] [] */') + expect(tokens[14]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[15]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[16]).toEqual value: '[Bad', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'invalid.illegal.syntax.jsdoc'] + expect(tokens[17]).toEqual value: ' Description] [] ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Object} [thing=\'{a: [], b: [0, 2], c: "String"}][Quoted Description] [\'][Bad_Unquoted Description] */') + expect(tokens[14]).toEqual value: "'", scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js', 'punctuation.definition.string.end.js'] + expect(tokens[15]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[16]).toEqual value: '[Bad_Unquoted', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'invalid.illegal.syntax.jsdoc'] + expect(tokens[17]).toEqual value: ' Description] ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[18]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises escape sequences inside strings", -> + {tokens} = grammar.tokenizeLine('/** @param {String} [key="a[\\"]z"] */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'String', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'key', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[13]).toEqual value: 'a[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js'] + expect(tokens[14]).toEqual value: '\\"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'constant.character.escape.js'] + expect(tokens[15]).toEqual value: ']z', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js'] + expect(tokens[16]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[17]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[19]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine("/** @param {String} [key='a[\\']z'] */") + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'String', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'key', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '\'', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js', 'punctuation.definition.string.begin.js'] + expect(tokens[13]).toEqual value: 'a[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js'] + expect(tokens[14]).toEqual value: '\\\'', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js', 'constant.character.escape.js'] + expect(tokens[15]).toEqual value: ']z', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js'] + expect(tokens[16]).toEqual value: '\'', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.single.js', 'punctuation.definition.string.end.js'] + expect(tokens[17]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[19]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises @param tags with accessor-style names", -> + {tokens} = grammar.tokenizeLine('/** @param {object} parameter.property this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'parameter.property', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} [parameter.property] this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'parameter.property', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[12]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} [ parameter.property ] this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: ' parameter.property ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[12]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} [parameter.property=default value] this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'parameter.property', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: 'default value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[13]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[14]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} [parameter.property = default value] this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'parameter.property ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[13]).toEqual value: 'default value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[14]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[15]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {object} [ parameter.property = default value ] this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: ' parameter.property ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[13]).toEqual value: 'default value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[14]).toEqual value: ' ', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[15]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.end.bracket.square.jsdoc'] + expect(tokens[16]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + it "tokenises @param tags with wildcard types", -> + {tokens} = grammar.tokenizeLine('/** @param {*} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '*', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + {tokens} = grammar.tokenizeLine('/** @param {?} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '?', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + it "tokenises @param tags with qualified types", -> + {tokens} = grammar.tokenizeLine('/** @param {myNamespace.MyClass} variable this is the description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'myNamespace.MyClass', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Foo~cb} variable this is the description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Foo~cb', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises @param tags with multiple types", -> + {tokens} = grammar.tokenizeLine('/** @param {function|string} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function|string', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {string[]|number} variable this is the description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'string', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[8]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[9]).toEqual value: '|number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[10]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[12]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[13]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {string|number[]} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'string|number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[8]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[9]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[11]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {(number|function)} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '(number|function)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {(string[]|number)} variable this is the description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '(string', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[8]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[9]).toEqual value: '|number)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[10]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[12]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[13]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[14]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @param {(string|number[])} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '(string|number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[8]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[9]).toEqual value: ')', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[10]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[12]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[13]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + it "tokenises @param tags marked nullable or non-nullable", -> + {tokens} = grammar.tokenizeLine('/** @param {?number} variable this is the description */') + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '?number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {!number} variable this is the description */') + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '!number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + it "tokenises @param tags marked as variable-length", -> + {tokens} = grammar.tokenizeLine('/** @param {...number} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '...number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {...*} remainder */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '...*', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'remainder', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @param {...?} remainder */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '...?', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'remainder', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises @param tags using Google Closure Compiler syntax", -> + {tokens} = grammar.tokenizeLine('/** @param {number=} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'number=', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {number[]} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[8]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[9]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[11]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {Foo[].bar} variable this is the description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Foo', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[8]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[9]).toEqual value: '.bar', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[10]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[12]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[13]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[14]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Array} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Array', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {Array.} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Array.', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Array|Array} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Array|Array', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Array.|Array.} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Array.|Array.', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {(Array|Array)} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '(Array|Array)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {(Array.|Array.)} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '(Array.|Array.)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Object} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Object.} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Object.', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Object|Array} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Object|Array', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Object.|Array.} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Object.|Array.', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[10]).toEqual value: ' this is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + + {tokens} = grammar.tokenizeLine('/** @param {(Array|Object)} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '(Array|Object)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {(Array.|Object.)} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '(Array.|Object.)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function()} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function()', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function ()} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function ()', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function ( )} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function ( )', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function(string)} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function(string, number)} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string, number)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function(...string)} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(...string)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function(string, ...number)} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string, ...number)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function(string, number, ...number)} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string, number, ...number)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function(!string)} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(!string)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function(?string, !number)} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(?string, !number)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function(string[], number=)} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[8]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[9]).toEqual value: ', number=)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[10]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[12]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function():number} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function():number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function(string): number} variable this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string): number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @param {function(string) : number} variable this is the description */') + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string) : number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'variable', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + + it "tokenises @return tags without descriptions", -> + {tokens} = grammar.tokenizeLine('/** @return {object} */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'return', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @returns {object} */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'returns', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises @return tags with trailing descriptions", -> + {tokens} = grammar.tokenizeLine('/** @returns {object} this is the description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'returns', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'this', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[10]).toEqual value: ' is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @return {object} this is the description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'return', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'object', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: 'this', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[10]).toEqual value: ' is the description ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + {tokens} = grammar.tokenizeLine('/** @returns {(Something)} */') + expect(tokens[3]).toEqual value: 'returns', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '(Something)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + it "tokenises @return tags with multiple types", -> + {tokens} = grammar.tokenizeLine('/** @return {Some|Thing} Something to return */') + expect(tokens[3]).toEqual value: 'return', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Some|Thing', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {(String[]|Number[])} Description */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'return', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '(String', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[8]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[9]).toEqual value: '|Number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[10]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[11]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[12]).toEqual value: ')', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[13]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[15]).toEqual value: 'Description', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[17]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises function-type @return tags", -> + {tokens} = grammar.tokenizeLine('/** @return {function()} this is the description */') + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'return', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function()', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {function ()} this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function ()', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {function ( )} this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function ( )', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {function(string)} this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {function(string, number)} this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string, number)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {function(...string)} this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(...string)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {function(string, ...number)} this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string, ...number)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {function(string, number, ...number)} this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string, number, ...number)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {function(!string)} this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(!string)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {function(?string, !number)} this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(?string, !number)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {function(string[], number=)} this is the description */') + expect(tokens[3]).toEqual value: 'return', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[8]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[9]).toEqual value: ', number=)', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[10]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {function():number} this is the description */') + expect(tokens[3]).toEqual value: 'return', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function():number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {function(string): number} this is the description */') + expect(tokens[3]).toEqual value: 'return', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string): number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + {tokens} = grammar.tokenizeLine('/** @return {function(string) : number} this is the description */') + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'function(string) : number', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + + describe "highlighted JavaScript examples", -> + it "highlights JavaScript after an @example tag", -> + lines = grammar.tokenizeLines """ + /** + * @example foo("bar"); + */ + """ + expect(lines[1][0]).toEqual value: ' * ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(lines[1][1]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(lines[1][2]).toEqual value: 'example', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'storage.type.class.jsdoc'] + expect(lines[1][4]).toEqual value: 'foo', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(lines[1][5]).toEqual value: '(', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(lines[1][6]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(lines[1][7]).toEqual value: 'bar', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js'] + expect(lines[1][8]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(lines[1][9]).toEqual value: ')', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(lines[1][10]).toEqual value: ';', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'punctuation.terminator.statement.js'] + expect(lines[2][1]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + lines = grammar.tokenizeLines """ + /** + * @example + * foo("bar"); + */ + """ + expect(lines[1][0]).toEqual value: ' * ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(lines[1][1]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(lines[1][2]).toEqual value: 'example', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'storage.type.class.jsdoc'] + expect(lines[1][3]).toEqual value: '', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc'] + expect(lines[2][0]).toEqual value: ' * ', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc'] + expect(lines[2][1]).toEqual value: 'foo', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(lines[2][2]).toEqual value: '(', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(lines[2][3]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(lines[2][4]).toEqual value: 'bar', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js'] + expect(lines[2][5]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(lines[2][6]).toEqual value: ')', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(lines[2][7]).toEqual value: ';', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'punctuation.terminator.statement.js'] + expect(lines[3][1]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + lines = grammar.tokenizeLines """ + /** + * @example foo("bar"); + * foo(foo("bar")); // Comment + * 4 + 50; + */ + """ + expect(lines[1][0]).toEqual value: ' * ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(lines[1][1]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(lines[1][2]).toEqual value: 'example', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'storage.type.class.jsdoc'] + expect(lines[1][4]).toEqual value: 'foo', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(lines[1][5]).toEqual value: '(', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(lines[1][6]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(lines[1][7]).toEqual value: 'bar', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js'] + expect(lines[1][8]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(lines[1][9]).toEqual value: ')', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(lines[1][10]).toEqual value: ';', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'punctuation.terminator.statement.js'] + expect(lines[2][0]).toEqual value: ' * ', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc'] + expect(lines[2][1]).toEqual value: 'foo', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(lines[2][2]).toEqual value: '(', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(lines[2][3]).toEqual value: 'foo', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(lines[2][4]).toEqual value: '(', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(lines[2][5]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(lines[2][6]).toEqual value: 'bar', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js'] + expect(lines[2][7]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(lines[2][8]).toEqual value: ')', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(lines[2][9]).toEqual value: ')', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(lines[2][10]).toEqual value: ';', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'punctuation.terminator.statement.js'] + expect(lines[2][12]).toEqual value: '//', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'comment.line.double-slash.js', 'punctuation.definition.comment.js'] + expect(lines[2][13]).toEqual value: ' Comment', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'comment.line.double-slash.js'] + expect(lines[3][0]).toEqual value: ' * ', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc'] + expect(lines[3][1]).toEqual value: '4', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'constant.numeric.decimal.js'] + expect(lines[3][3]).toEqual value: '+', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'keyword.operator.js'] + expect(lines[3][5]).toEqual value: '50', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'constant.numeric.decimal.js'] + expect(lines[3][6]).toEqual value: ';', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'punctuation.terminator.statement.js'] + expect(lines[4][1]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + it "tokenises tags at the start of an example block", -> + {tokens} = grammar.tokenizeLine('/** @example Text */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'example', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '<', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'entity.name.tag.inline.jsdoc', 'punctuation.definition.bracket.angle.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'caption', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'entity.name.tag.inline.jsdoc'] + expect(tokens[7]).toEqual value: '>', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'entity.name.tag.inline.jsdoc', 'punctuation.definition.bracket.angle.end.jsdoc'] + expect(tokens[8]).toEqual value: 'Text', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'constant.other.description.jsdoc'] + expect(tokens[9]).toEqual value: '', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'entity.name.tag.inline.jsdoc', 'punctuation.definition.bracket.angle.end.jsdoc'] + expect(tokens[13]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + lines = grammar.tokenizeLines """ + /** + * @example Text + * foo("bar"); + * @return {String} + */ + """ + expect(lines[0][0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(lines[1][0]).toEqual value: ' * ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(lines[1][1]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(lines[1][2]).toEqual value: 'example', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'storage.type.class.jsdoc'] + expect(lines[1][4]).toEqual value: '<', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'entity.name.tag.inline.jsdoc', 'punctuation.definition.bracket.angle.begin.jsdoc'] + expect(lines[1][5]).toEqual value: 'caption', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'entity.name.tag.inline.jsdoc'] + expect(lines[1][6]).toEqual value: '>', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'entity.name.tag.inline.jsdoc', 'punctuation.definition.bracket.angle.end.jsdoc'] + expect(lines[1][7]).toEqual value: 'Text', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'constant.other.description.jsdoc'] + expect(lines[1][8]).toEqual value: '', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'entity.name.tag.inline.jsdoc', 'punctuation.definition.bracket.angle.end.jsdoc'] + expect(lines[2][0]).toEqual value: ' * ', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc'] + expect(lines[2][1]).toEqual value: 'foo', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(lines[2][2]).toEqual value: '(', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(lines[2][3]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(lines[2][4]).toEqual value: 'bar', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js'] + expect(lines[2][5]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(lines[2][6]).toEqual value: ')', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(lines[2][7]).toEqual value: ';', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'punctuation.terminator.statement.js'] + expect(lines[3][0]).toEqual value: ' * ', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc'] + expect(lines[3][1]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(lines[3][2]).toEqual value: 'return', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(lines[3][4]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(lines[3][5]).toEqual value: 'String', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(lines[3][6]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(lines[3][7]).toEqual value: '', scopes: ['source.js', 'comment.block.documentation.js'] + expect(lines[4][1]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + + + describe "when the containing comment ends unexpectedly", -> + it "terminates any unclosed tags", -> + {tokens} = grammar.tokenizeLine('/** @param {String */ aa') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'String ', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + expect(tokens[8]).toEqual value: ' aa', scopes: ['source.js'] + + {tokens} = grammar.tokenizeLine('/** @param {*} [name={value: {a:[{*/}}]}] */') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: '*', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'name', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[13]).toEqual value: 'value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[14]).toEqual value: ':', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'keyword.operator.assignment.js'] + expect(tokens[16]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[17]).toEqual value: 'a', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[18]).toEqual value: ':', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'keyword.operator.assignment.js'] + expect(tokens[19]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[20]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[21]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + expect(tokens[22]).toEqual value: '}}', scopes: ['source.js'] + expect(tokens[23]).toEqual value: ']', scopes: ['source.js', 'meta.brace.square.js'] + expect(tokens[24]).toEqual value: '}', scopes: ['source.js'] + expect(tokens[25]).toEqual value: ']', scopes: ['source.js', 'meta.brace.square.js'] + expect(tokens[27]).toEqual value: '*', scopes: ['source.js', 'keyword.operator.js'] + expect(tokens[28]).toEqual value: '/', scopes: ['source.js', 'keyword.operator.js'] + + {tokens} = grammar.tokenizeLine('/** @param {string="Foo*/oo"} bar') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'string="Foo', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + expect(tokens[8]).toEqual value: 'oo', scopes: ['source.js'] + expect(tokens[9]).toEqual value: '"', scopes: ['source.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[10]).toEqual value: '} bar', scopes: ['source.js', 'string.quoted.double.js'] + + it "terminates any embedded JavaScript code", -> + lines = grammar.tokenizeLines """ + /** + * @example + * foo("bar"); /* Comment */ + * @return {String} + */ + """ + expect(lines[0][0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(lines[1][0]).toEqual value: ' * ', scopes: ['source.js', 'comment.block.documentation.js'] + expect(lines[1][1]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(lines[1][2]).toEqual value: 'example', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'storage.type.class.jsdoc'] + expect(lines[1][3]).toEqual value: '', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc'] + expect(lines[2][0]).toEqual value: ' * ', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc'] + expect(lines[2][1]).toEqual value: 'foo', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(lines[2][2]).toEqual value: '(', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js'] + expect(lines[2][3]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(lines[2][4]).toEqual value: 'bar', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js'] + expect(lines[2][5]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(lines[2][6]).toEqual value: ')', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.end.bracket.round.js'] + expect(lines[2][7]).toEqual value: ';', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'punctuation.terminator.statement.js'] + expect(lines[2][9]).toEqual value: '/*', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'comment.block.js', 'punctuation.definition.comment.begin.js'] + expect(lines[2][10]).toEqual value: ' Comment ', scopes: ['source.js', 'comment.block.documentation.js', 'meta.example.jsdoc', 'source.embedded.js', 'comment.block.js'] + expect(lines[2][11]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + expect(lines[3][1]).toEqual value: '*', scopes: ['source.js', 'keyword.operator.js'] + expect(lines[3][2]).toEqual value: ' @', scopes: ['source.js'] + expect(lines[3][3]).toEqual value: 'return', scopes: ['source.js', 'keyword.control.js'] + expect(lines[3][5]).toEqual value: '{', scopes: ['source.js', 'meta.brace.curly.js'] + expect(lines[3][6]).toEqual value: 'String', scopes: ['source.js', 'support.class.js'] + expect(lines[3][7]).toEqual value: '}', scopes: ['source.js', 'meta.brace.curly.js'] + expect(lines[4][1]).toEqual value: '*', scopes: ['source.js', 'keyword.operator.js'] + expect(lines[4][2]).toEqual value: '/', scopes: ['source.js', 'keyword.operator.js'] + + {tokens} = grammar.tokenizeLine('/** @param {Something} [value={key: [value, ["22"}] */ 20;') + expect(tokens[0]).toEqual value: '/**', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.begin.js'] + expect(tokens[2]).toEqual value: '@', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc', 'punctuation.definition.block.tag.jsdoc'] + expect(tokens[3]).toEqual value: 'param', scopes: ['source.js', 'comment.block.documentation.js', 'storage.type.class.jsdoc'] + expect(tokens[5]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.begin.jsdoc'] + expect(tokens[6]).toEqual value: 'Something', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc'] + expect(tokens[7]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'entity.name.type.instance.jsdoc', 'punctuation.definition.bracket.curly.end.jsdoc'] + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'punctuation.definition.optional-value.begin.bracket.square.jsdoc'] + expect(tokens[10]).toEqual value: 'value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc'] + expect(tokens[11]).toEqual value: '=', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'keyword.operator.assignment.jsdoc'] + expect(tokens[12]).toEqual value: '{', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[13]).toEqual value: 'key', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[14]).toEqual value: ':', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'keyword.operator.assignment.js'] + expect(tokens[16]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[17]).toEqual value: 'value', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js'] + expect(tokens[18]).toEqual value: ',', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.delimiter.object.comma.js'] + expect(tokens[20]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[21]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.begin.js'] + expect(tokens[22]).toEqual value: '22', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js'] + expect(tokens[23]).toEqual value: '"', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'string.quoted.double.js', 'punctuation.definition.string.end.js'] + expect(tokens[24]).toEqual value: '}', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.curly.js'] + expect(tokens[25]).toEqual value: ']', scopes: ['source.js', 'comment.block.documentation.js', 'variable.other.jsdoc', 'source.embedded.js', 'meta.brace.square.js'] + expect(tokens[27]).toEqual value: '*/', scopes: ['source.js', 'comment.block.documentation.js', 'punctuation.definition.comment.end.js'] + expect(tokens[29]).toEqual value: '20', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[30]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] + + describe "when the line ends without a closing bracket", -> + it "does not attempt to match the optional value (regression)", -> + {tokens} = grammar.tokenizeLine('/** @param {array} [bar = "x" REMOVE THE CLOSE BRACKET HERE.') + expect(tokens[9]).toEqual value: '[', scopes: ['source.js', 'comment.block.documentation.js'] + expect(tokens[11]).toEqual value: ' = "x" REMOVE THE CLOSE BRACKET HERE.', scopes: ['source.js', 'comment.block.documentation.js'] diff --git a/spec/regular-expression-replacement-spec.coffee b/spec/regular-expression-replacement-spec.coffee index 5f2c682b..735aa823 100644 --- a/spec/regular-expression-replacement-spec.coffee +++ b/spec/regular-expression-replacement-spec.coffee @@ -2,6 +2,8 @@ describe "Regular Expression Replacement grammar", -> grammar = null beforeEach -> + atom.config.set('core.useTreeSitterParsers', false) + waitsForPromise -> atom.packages.activatePackage("language-javascript")