Skip to content

Commit 3fee144

Browse files
mike-lischkeGuilherme Saraiva
mike-lischke
authored and
Guilherme Saraiva
committed
Main patch for the WL #15837: Replace rule based Monaco (Monarch) tokenizer with an own one
This involves actually a number of things: - The standard tokenizer for JS/TS and the MSG monarch tokenizer have been replaced by a new semantic tokenizer. It's not a syntactic tokenizer as this does not provide information to effectively generate the highlighter tokens. - To avoid unnecessary tokenization for models that are used for individual execution blocks, they no longer get the execution blocks container assigned that exists in the main model and the tokenizer tests for that. Making the container optional required a number of changes in other parts of the code, though. - Introduced a better of creating the editor models, for more type safety. - With this change we no longer need language switches in notebooks. Therefore they have been removed from the code completion list and they are only handled in shell sessions now. - Also pasting of text with language switches has been updated. - The language of an execution block (context) can now be switched using a hotkey (ctrl/cmd + F1/F2/F3). This is likely to be configurable later. - Added some new tests for the new functionality, but because we mock the language services, it's not possible to do a deep check. To stay within our required code coverage I added the first set of the dynamic symbol table tests (which found a few bugs). Change-Id: I3a568d93e8a1fd40162588eb38b50ca83dd95231
1 parent 0fb839d commit 3fee144

File tree

74 files changed

+2092
-1505
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

74 files changed

+2092
-1505
lines changed

‎gui/extension/tests/e2e/lib/db.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -327,7 +327,7 @@ export class Database {
327327

328328
await driver.wait(async () => {
329329
try {
330-
const prompts = await driver.findElements(locator.notebook.codeEditor.prompt);
330+
const prompts = await driver.findElements(locator.notebook.codeEditor.prompt.exists);
331331
const sentences = await driver.findElements(locator.notebook.codeEditor.editor.sentence);
332332
let index = -1;
333333

@@ -448,7 +448,7 @@ export class Database {
448448
};
449449

450450
public static getPrompts = async (): Promise<number> => {
451-
return (await driver.findElements(locator.notebook.codeEditor.prompt)).length;
451+
return (await driver.findElements(locator.notebook.codeEditor.prompt.exists)).length;
452452
};
453453

454454
public static setRestService = async (restService: interfaces.IRestService): Promise<void> => {

‎gui/extension/tests/e2e/lib/locators.ts

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,10 @@ export const notebook = {
127127
},
128128
},
129129
},
130-
prompt: By.css(".codeEditor .margin-view-overlays > div"),
130+
prompt: {
131+
exists: By.css(".codeEditor .margin-view-overlays > div"),
132+
current: By.className("editorPromptFirst"),
133+
},
131134
},
132135
toolbar: {
133136
exists: By.id("dbEditorToolbar"),

‎gui/extension/tests/e2e/lib/misc.ts

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -464,6 +464,7 @@ export class Misc {
464464
timeout?: number,
465465
slowWriting?: boolean): Promise<Array<string | WebElement | boolean | undefined>> => {
466466

467+
await driver.sleep(500);
467468
cmd = cmd.replace(/(\r\n|\n|\r)/gm, "");
468469
const count = (cmd.match(/;|select|SELECT/g) || []).length;
469470
const hasMultipleQueries = count >= 3 && cmd.toLowerCase().startsWith("select");
@@ -1709,7 +1710,21 @@ export class Misc {
17091710
const toReturn: Array<string | WebElement | boolean | undefined> = [];
17101711
let zoneHost: WebElement;
17111712

1712-
if (!cmd.includes("disconnect")) {
1713+
if (cmd.match(/(\\js|\\javascript|\\ts|\\typescript|\\sql|\\py)/) !== null) {
1714+
await driver.wait(async () => {
1715+
try {
1716+
const prompts = await driver.findElements(locator.notebook.codeEditor.prompt.current);
1717+
const promptClasses = (await prompts[prompts.length - 1].getAttribute("class")).split(" ");
1718+
toReturn.push(promptClasses[2]);
1719+
1720+
return true;
1721+
} catch (e) {
1722+
if (!(e instanceof error.StaleElementReferenceError)) {
1723+
throw e;
1724+
}
1725+
}
1726+
}, constants.wait5seconds, "Could not get the result for changing notebook language");
1727+
} else if (!cmd.includes("disconnect")) {
17131728
if (blockId === "0") {
17141729
const zoneHosts = await driver.wait(until
17151730
.elementsLocated(locator.notebook.codeEditor.editor.result.exists), timeout,

‎gui/extension/tests/e2e/lib/shell.ts

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -62,13 +62,7 @@ export class Shell {
6262

6363
public static isValueOnDataSet = (resultHost: WebElement, value: String): Condition<boolean> => {
6464
return new Condition("Value is not on data set", async () => {
65-
const cells = await driver.wait(async () => {
66-
const cells = await resultHost.findElements(locator.notebook.codeEditor.editor.result.tableCell);
67-
if (cells.length > 0) {
68-
return cells;
69-
}
70-
}, constants.wait5seconds, "No cells were found");
71-
65+
const cells = await resultHost.findElements(locator.notebook.codeEditor.editor.result.tableCell);
7266
for (const cell of cells) {
7367
const text = await cell.getText();
7468
if (text === value) {

‎gui/extension/tests/e2e/tests/ui-notebook.ts

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ describe("NOTEBOOKS", () => {
107107
}
108108
});
109109

110-
describe("DB Editor", () => {
110+
describe.only("DB Editor", () => {
111111

112112
let clean = false;
113113

@@ -254,6 +254,8 @@ describe("NOTEBOOKS", () => {
254254
const result = await Misc.execCmd("SELECT * FROM sakila.actor;", constants.execFullBlockSql);
255255
expect(result[0]).to.match(/(\d+) record/);
256256
await driver.wait(async () => {
257+
console.log(await Database.getPrompts());
258+
257259
return (await Database.getPrompts()) > prompts;
258260
}, constants.wait5seconds, "A new prompt line should exist");
259261
});
@@ -421,10 +423,10 @@ describe("NOTEBOOKS", () => {
421423
try {
422424
const textArea = await driver.findElement(locator.notebook.codeEditor.textArea);
423425
const result = await Misc.execCmd("\\js ");
424-
expect(result[0]).to.include("Switched to JavaScript mode");
426+
expect(result[0]).to.equals("javascript");
425427
const result2 = await Misc.execCmd("Math.random();");
426428
expect(result2[0]).to.match(/(\d+).(\d+)/);
427-
expect((await Misc.execCmd("\\typescript "))[0]).equals("Switched to TypeScript mode");
429+
expect((await Misc.execCmd("\\typescript "))[0]).equals("typescript");
428430
const result4 = await Misc.execCmd("Math.random();");
429431
expect(result4[0]).to.match(/(\d+).(\d+)/);
430432
await textArea.sendKeys(Key.ARROW_UP);
@@ -437,8 +439,6 @@ describe("NOTEBOOKS", () => {
437439
const otherResult = await Misc.getCmdResultMsg();
438440
expect(otherResult).to.match(/(\d+).(\d+)/);
439441
expect(otherResult !== result2[0]).equals(true);
440-
await textArea.sendKeys(Key.ARROW_DOWN);
441-
await driver.sleep(500);
442442
await Misc.execOnEditor();
443443
await driver.wait(async () => {
444444
try {
@@ -460,7 +460,7 @@ describe("NOTEBOOKS", () => {
460460
it("Multi-line comments", async () => {
461461

462462
let result = await Misc.execCmd("\\sql ");
463-
expect(result[0]).to.include("Switched to MySQL mode");
463+
expect(result[0]).to.equals("mysql");
464464
result = await Misc.execCmd("select version();");
465465
expect(result[0]).to.include("1 record retrieved");
466466
const txt = await (result[1] as WebElement)
@@ -523,7 +523,7 @@ describe("NOTEBOOKS", () => {
523523
it("Pie Graph based on DB table", async () => {
524524

525525
let result = await Misc.execCmd("\\ts ");
526-
expect(result[0]).to.include("Switched to TypeScript mode");
526+
expect(result[0]).to.include("typescript");
527527
result = await Misc.execCmd(`
528528
const res = await runSql("SELECT Name, Capital FROM world_x_cst.country limit 10");
529529
const options: IGraphOptions = {
@@ -551,7 +551,7 @@ Graph.render(options);
551551
it("Schema autocomplete context menu", async () => {
552552

553553
const result = await Misc.execCmd("\\sql ");
554-
expect(result[0]).to.include("Switched to MySQL mode");
554+
expect(result[0]).to.equals("mysql");
555555
await Misc.writeCmd("select * from ");
556556
const textArea = await driver.findElement(locator.notebook.codeEditor.textArea);
557557
await textArea.sendKeys(Key.chord(Key.CONTROL, Key.SPACE));

‎gui/extension/tests/e2e/tests/ui-shell.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -369,11 +369,11 @@ describe("MYSQL SHELL CONSOLES", () => {
369369

370370
const editor = await driver.wait(until.elementLocated(locator.shellConsole.editor),
371371
10000, "Console was not loaded");
372-
let result = await Misc.execCmd("\\py ");
373-
expect(result[0]).to.equals("Switching to Python mode...");
372+
let result = await Misc.execCmd("\\py ", undefined, undefined, true);
373+
expect(result[0]).to.equals("python");
374374
expect(await Shell.getTech(editor)).to.equals("python");
375-
result = await Misc.execCmd("\\js ");
376-
expect(result[0]).to.equals("Switching to JavaScript mode...");
375+
result = await Misc.execCmd("\\js ", undefined, undefined, true);
376+
expect(result[0]).to.equals("javascript");
377377
expect(await Shell.getTech(editor)).to.equals("javascript");
378378

379379
});

‎gui/frontend/src/components/CommunicationDebugger/CommunicationDebugger.tsx

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -389,8 +389,14 @@ export class CommunicationDebugger
389389
};
390390

391391
private createEditorState(content: string): IEditorPersistentState {
392+
const model: ICodeEditorModel = Object.assign(Monaco.createModel(content, "javascript"), {
393+
executionContexts: new ExecutionContexts(undefined, 80024, "", ""),
394+
editorMode: CodeEditorMode.Standard,
395+
appEmbedded: false,
396+
});
397+
392398
const state: IEditorPersistentState = {
393-
model: Monaco.createModel(content, "javascript") as ICodeEditorModel,
399+
model,
394400
viewState: null,
395401
options: defaultEditorOptions,
396402
};
@@ -591,8 +597,8 @@ export class CommunicationDebugger
591597
}
592598

593599
if (lineDecorationClass.length > 0) {
594-
const editor = this.messageOutputRef.current.backend!;
595-
/*const decorations =*/ editor.deltaDecorations([], [
600+
const editor = this.messageOutputRef.current.backend;
601+
/*const decorations =*/ editor.getModel()?.deltaDecorations([], [
596602
{
597603
range: {
598604
startLineNumber: lastLineCount,

‎gui/frontend/src/components/Theming/Preview/ThemePreview.tsx

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -120,15 +120,19 @@ export class ThemePreview extends ComponentBase<{}, IThemePreviewState> {
120120
content += `\nprint("javascript");\n\\sql\n`;
121121
content += `\nselect "(my)sql" from dual;\n\\py\n`;
122122
content += `\nprint("python");\n`;
123-
const model = Monaco.createModel("", "msg") as ICodeEditorModel;
123+
124+
const model: ICodeEditorModel = Object.assign(Monaco.createModel(content, "javascript"), {
125+
executionContexts: new ExecutionContexts(undefined, 80024, "", ""),
126+
editorMode: CodeEditorMode.Standard,
127+
appEmbedded: false,
128+
});
129+
124130
if (model.getEndOfLineSequence() !== Monaco.EndOfLineSequence.LF) {
125131
model.setEOL(Monaco.EndOfLineSequence.LF);
126132
} else {
127133
model.setValue(content);
128134
}
129135

130-
model.executionContexts = new ExecutionContexts(undefined, 80024, "", "");
131-
model.editorMode = CodeEditorMode.Standard;
132136
model.setValue(content);
133137

134138
const editorState: IEditorPersistentState = {

‎gui/frontend/src/components/Theming/assets/default-dark-color-theme.json

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -917,7 +917,7 @@
917917
"name": "Generic variable",
918918
"scope": "variable",
919919
"settings": {
920-
"foreground": "#63bf8d"
920+
"foreground": "#d7b37c"
921921
}
922922
},
923923
{
@@ -931,7 +931,7 @@
931931
"name": "Language variable",
932932
"scope": "variable.language",
933933
"settings": {
934-
"foreground": "#45aa73"
934+
"foreground": "#d7b37c"
935935
}
936936
},
937937
{
@@ -1098,7 +1098,7 @@
10981098
{
10991099
"scope": "variable.predefined",
11001100
"settings": {
1101-
"foreground": "#4864AA"
1101+
"foreground": "#c8dfbd"
11021102
}
11031103
},
11041104
{
@@ -1280,6 +1280,13 @@
12801280
"settings": {
12811281
"foreground": "#909090"
12821282
}
1283+
},
1284+
{
1285+
"scope": "command",
1286+
"settings": {
1287+
"foreground": "#dadfbd",
1288+
"fontStyle": "bold underline"
1289+
}
12831290
}
12841291
]
12851292
}

‎gui/frontend/src/components/Theming/assets/default-light-color-theme.json

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1211,6 +1211,13 @@
12111211
"settings": {
12121212
"foreground": "#666666"
12131213
}
1214+
},
1215+
{
1216+
"scope": "command",
1217+
"settings": {
1218+
"foreground": "#7f8464",
1219+
"fontStyle": "bold underline"
1220+
}
12141221
}
12151222
]
12161223
}

‎gui/frontend/src/components/ui/CodeEditor/CodeCompletionProvider.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ export class CodeCompletionProvider implements languages.CompletionItemProvider
3333
public provideCompletionItems(model: IProviderEditorModel, position: Position): ProviderResult<CompletionList> {
3434

3535
const services = ScriptingLanguageServices.instance;
36-
const sourceContext = model.executionContexts.contextFromPosition(position);
36+
const sourceContext = model.executionContexts?.contextFromPosition(position);
3737
if (sourceContext) {
3838
if (sourceContext.isInternal) {
3939
const info = model.getWordUntilPosition(position);
@@ -290,7 +290,7 @@ export class CodeCompletionProvider implements languages.CompletionItemProvider
290290
kind: languages.CompletionItemKind.Keyword,
291291
range,
292292
insertText: "\\reconnect",
293-
detail: "Reconnects the current MySQL connection",
293+
detail: "Reconnects the current MySQL connection.",
294294
},
295295
];
296296
}

0 commit comments

Comments
 (0)