Fixed code completion + semantic highlightingt:wq

- Both invocation on keypress and after a dot char are now working again.
- Semantic highlighting now works properly also when more than one query starts or ends on the same line.
- Additionally it was optimized to avoid creating many promises for large scripts.

Change-Id: Id06cfd25bfdd3235103d12b66bb77b31e54613f9

Author: mike-lischke

gui/frontend/src/components/ui/CodeEditor/CodeEditor.tsx

@@ -433,8 +433,8 @@ export class CodeEditor extends ComponentBase<ICodeEditorProperties> {
             "links": detectLinks,
             "colorDecorators": true,
             "contextmenu": model?.editorMode !== CodeEditorMode.Terminal,
-            "suggest": suggest ?? {preview: true},
-            "inlineSuggest": {enabled: true},
+            "suggest": suggest ?? { preview: true },
+            "inlineSuggest": { enabled: true },
             "quickSuggestions": true,
             "emptySelectionClipboard": false,
             "copyWithSyntaxHighlighting": true,

gui/frontend/src/components/ui/CodeEditor/index.ts

@@ -117,23 +117,6 @@ export interface IExecutionContextState {
     statements: IStatementSpan[];
 }
 
-/** A statement consists of its text and its relative position within its block/context. */
-export interface IStatement {
-    /** The index of the statement in the statement list. */
-    index: number;
-
-    text: string;
-
-    /** The character offset of the statement in the containing model. */
-    offset: number;
-
-    /** The line number of the statement in the containing model. */
-    line: number;
-
-    /** Ditto for the column. */
-    column: number;
-}
-
 /** Options that control how SQL statements are executed. */
 export interface IScriptExecutionOptions {
     /** Any additional named parameters for placeholders in the query. */

gui/frontend/src/components/ui/CodeEditor/languages/msg/msg.ts

@@ -34,8 +34,9 @@ export const language: languages.IMonarchLanguage = {
     start: "msg",
     tokenizer: {
         msg: [
-            // By default all text is formatted as string. The semantic highlighter will update the formatting.
-            [/.*/, { token: "string.quoted.double.sql" }],
+            // Override highlighting of braces and make them appear as strings.
+            // The semantic highlighter will update the formatting.
+            [/[[\](){}]/, { token: "string" }],
         ],
     },
 

gui/frontend/src/modules/db-editor/DBConnectionTab.tsx

@@ -931,9 +931,9 @@ Execute \\help or \\? for help;`;
 
         context.clearResult();
         if (options.source) {
-            const sql = context.getStatementAtPosition(options.source)?.text ?? "";
+            const sql = (await context.getStatementAtPosition(options.source))?.text;
 
-            await this.executeQuery(context, 0, 0, pageSize, options, sql);
+            await this.executeQuery(context, 0, 0, pageSize, options, sql ?? "");
         } else {
             const statements = await context.getExecutableStatements();
             while (true) {

gui/frontend/src/parsing/SQLite/SQLiteParsingServices.ts

@@ -29,8 +29,8 @@ import {
 } from "antlr4ng";
 
 import {
-    ICompletionData, IParserErrorInfo, IStatementSpan, ISymbolInfo, ITokenInfo, QueryType, StatementFinishState,
-    tokenFromPosition,
+    ICompletionData, IParserErrorInfo, IStatement, IStatementSpan, ISymbolInfo, ITokenInfo, QueryType,
+    StatementFinishState, tokenFromPosition,
 } from "../parser-common.js";
 
 import { SQLiteErrorListener } from "./SQLiteErrorListener.js";
@@ -174,27 +174,33 @@ export class SQLiteParsingServices {
     }
 
     /**
-     * Creates a list of token info items for the given text.
+     * Creates a list of token info items for the given statements.
      *
-     * @param text The text to handle.
+     * @param statements A list of statements to tokenize.
      *
      * @returns The information about the symbol at the given offset, if there's one.
      */
-    public async tokenize(text: string): Promise<ITokenInfo[]> {
+    public async tokenize(statements: IStatement[]): Promise<ITokenInfo[]> {
         this.errors = [];
-        this.lexer.inputStream = CharStreams.fromString(text);
-        this.tokenStream.setTokenSource(this.lexer);
-        this.tokenStream.fill();
 
-        return Promise.resolve(this.tokenStream.getTokens().map((token: Token) => {
-            return {
-                type: this.lexerTypeToScope(token),
-                offset: token.start,
-                line: token.line,
-                column: token.column,
-                length: token.stop - token.start + 1,
-            };
-        }));
+        const result: ITokenInfo[] = [];
+        for (const statement of statements) {
+            this.lexer.inputStream = CharStreams.fromString(statement.text);
+            this.tokenStream.setTokenSource(this.lexer);
+            this.tokenStream.fill();
+
+            const tokens = this.tokenStream.getTokens();
+            for (const token of tokens) {
+                result.push({
+                    type: this.lexerTypeToScope(token),
+                    offset: token.start + statement.offset,
+                    length: token.stop - token.start + 1,
+                });
+            }
+        }
+
+        return Promise.resolve(result);
+
     }
 
     /**

gui/frontend/src/parsing/mysql/MySQLParsingServices.ts

@@ -38,9 +38,8 @@ import {
 import { MySQLErrorListener } from "./MySQLErrorListener.js";
 import { MySQLParseUnit } from "./MySQLServiceTypes.js";
 import {
-    ICompletionData, IParserErrorInfo, IStatementSpan, ISymbolInfo, ITokenInfo, QueryType, StatementFinishState,
-    SymbolKind,
-    TextSpan, tokenFromPosition,
+    ICompletionData, IParserErrorInfo, IStatement, IStatementSpan, ISymbolInfo, ITokenInfo, QueryType,
+    StatementFinishState, SymbolKind, TextSpan, tokenFromPosition,
 } from "../parser-common.js";
 
 import { SystemVariableSymbol, SystemFunctionSymbol, DBSymbolTable } from "../DBSymbolTable.js";
@@ -255,87 +254,64 @@ export class MySQLParsingServices {
     /**
      * Creates a list of token info items for the given text.
      *
-     * @param text The text to handle.
+     * @param statements A list of statements to tokenize.
      * @param serverVersion The version of MySQL to control the parsing process.
      * @param sqlMode The current SQL mode in the server.
      *
      * @returns The information about the symbol at the given offset, if there's one.
      */
-    public async tokenize(text: string, serverVersion: number, sqlMode: string): Promise<ITokenInfo[]> {
+    public async tokenize(statements: IStatement[], serverVersion: number,
+        sqlMode: string): Promise<ITokenInfo[]> {
         const result: ITokenInfo[] = [];
 
-        const splitMultiLineStatements = (text: string, type: string, firstIndex: number,
-            firstColumn: number): void => {
-            const lines = text.split("\n");
-            lines.forEach((line: string, lineIndex: number) => {
-                if (line.length > 0) {
-                    result.push({
-                        type,
-                        line: firstIndex + lineIndex,
-                        column: lineIndex === 0 ? firstColumn : 0,
-                        length: line.length,
-                    });
-                }
-            });
-
-        };
-
-        // Special treatment for delimiters.
-        if (text.match(/^\s*delimiter /i)) {
-            splitMultiLineStatements(text, "delimiter.sql", 1, 0);
-
-            return result;
-        }
-
-        this.applyServerDetails(serverVersion, sqlMode);
-        this.errors = [];
-        this.lexer.inputStream = CharStreams.fromString(text);
-        this.tokenStream.setTokenSource(this.lexer);
-        this.tokenStream.fill();
+        for (const statement of statements) {
+            // Special treatment for delimiters.
+            if (statement.text.match(/^\s*delimiter /i)) {
+                // Line and column values are one based.
+                result.push({
+                    type: "delimiter.sql",
+                    offset: statement.offset,
+                    length: statement.text.length,
+                });
 
-        const tokens = this.tokenStream.getTokens();
+                continue;
+            }
 
-        let variablePending = false;
-        const version = numberToVersion(serverVersion);
-        for (const token of tokens) {
-            switch (token.type) {
-                case MySQLMRSLexer.WHITESPACE:
-                case MySQLMRSLexer.EOF: {
-                    break;
-                }
+            this.applyServerDetails(serverVersion, sqlMode);
+            this.errors = [];
+            this.lexer.inputStream = CharStreams.fromString(statement.text);
+            this.tokenStream.setTokenSource(this.lexer);
+            this.tokenStream.fill();
 
-                case MySQLMRSLexer.BLOCK_COMMENT: {
-                    // Block comments can span multiple lines. Split them up into individual lines.
-                    splitMultiLineStatements(token.text, "comment.block.sql", token.line, token.column);
+            const tokens = this.tokenStream.getTokens();
 
-                    break;
-                }
+            let variablePending = false;
+            const version = numberToVersion(serverVersion);
+            for (const token of tokens) {
+                switch (token.type) {
+                    case MySQLMRSLexer.WHITESPACE:
+                    case MySQLMRSLexer.EOF: {
+                        break;
+                    }
 
-                case MySQLMRSLexer.DOLLAR_QUOTED_STRING_TEXT: {
-                    // Ditto for dollar quoted strings.
-                    splitMultiLineStatements(token.text, "string.sql", token.line, token.column);
+                    default: {
+                        let type = await this.lexerTypeToScope(token, version);
 
-                    break;
-                }
+                        // System variables appear as two tokens, the first one being the @@ sign.
+                        // Make both of them appear as variable.predefined.
+                        if (type === "variable.predefined") {
+                            variablePending = true;
+                        } else if (variablePending) {
+                            variablePending = false;
+                            type = "variable.predefined";
+                        }
 
-                default: {
-                    let type = await this.lexerTypeToScope(token, version);
-
-                    // System variables appear as two tokens, the first one being the @@ sign.
-                    // Make both of them appear as variable.predefined.
-                    if (type === "variable.predefined") {
-                        variablePending = true;
-                    } else if (variablePending) {
-                        variablePending = false;
-                        type = "variable.predefined";
+                        result.push({
+                            type: type + ".sql",
+                            offset: statement.offset + token.start,
+                            length: token.stop - token.start + 1,
+                        });
                     }
-
-                    result.push({
-                        type: type + ".sql",
-                        line: token.line,
-                        column: token.column,
-                        length: token.stop - token.start + 1,
-                    });
                 }
             }
         }
@@ -1031,6 +1007,7 @@ export class MySQLParsingServices {
                 return "string.quoted.single";
             }
 
+            case MySQLMRSLexer.DOLLAR_QUOTED_STRING_TEXT:
             case MySQLMRSLexer.DOUBLE_QUOTED_TEXT: {
                 return "string.quoted.double";
             }

gui/frontend/src/parsing/parser-common.ts

@@ -241,9 +241,10 @@ export interface IStatementSpan {
      */
     delimiter?: string;
 
-    /** Start and length of the entire statement, including leading whitespaces. */
+    /** Start and length of the entire statement, including leading whitespaces/comments. */
     span: TextSpan;
 
+    /** An alternative start offset, for the first non-whitespace/non-comment character. */
     contentStart: number;
 
     /** The offset where non-whitespace content starts. */
@@ -313,11 +314,8 @@ export interface ITokenInfo {
     /** The token type (highlighter scope). */
     type: string;
 
-    /** The (one-based) token line. */
-    line: number;
-
-    /** The (zero-based) offset in that line. */
-    column: number;
+    /** The position of the first token character relative to its containing statement. */
+    offset: number;
 
     /** The length of the token text. */
     length: number;
@@ -448,10 +446,32 @@ interface ILanguageWorkerCleanupData {
     language: ServiceLanguage;
 }
 
+/** A statement consists of its text and its relative position within its block/context. */
+export interface IStatement {
+    /** The index of the statement in the statement list. */
+    index: number;
+
+    /** The text of the statement without delimiter. */
+    text: string;
+
+    /** The character offset of the statement in the containing model. */
+    offset: number;
+
+    /** The line number of the statement in the containing model. */
+    line: number;
+
+    /** Ditto for the column. */
+    column: number;
+
+
+    /** The length of the delimiter given for this statement. */
+    delimiterLength?: number;
+}
+
 export interface ILanguageWorkerTokenizeData {
     api: "tokenize";
     language: ServiceLanguage;
-    sql: string;
+    statements: IStatement[];
     version: number;
     sqlMode: string;
 }

gui/frontend/src/parsing/worker/RdbmsLanguageService.ts

@@ -123,14 +123,12 @@ export class RdbmsLanguageService {
             this.localSymbols.addDependencies(model.symbols);
         }
 
-        return new Promise((resolve, reject) => {
-            const statement = context.getStatementAtPosition(position);
-            if (!statement) {
-                resolve(undefined);
-
-                return;
-            }
+        const statement = await context.getStatementAtPosition(position);
+        if (!statement) {
+            return;
+        }
 
+        return new Promise((resolve, reject) => {
             const infoData: ILanguageWorkerSuggestionData = {
                 api: "suggestion",
                 language: this.language,
@@ -148,7 +146,7 @@ export class RdbmsLanguageService {
                     // a whitespace character or the start of the line.
                     const line = model.getLineContent(position.lineNumber);
                     let index = position.column - 1;
-                    while (index > 0 && !/\s/.test(line[index - 1])) {
+                    while (index > 0 && !/[\s.]/.test(line[index - 1])) {
                         --index;
                     }
 

gui/frontend/src/parsing/worker/sql.worker.ts

@@ -147,7 +147,7 @@ ctx.addEventListener("message", (event: MessageEvent) => {
         }
 
         case "tokenize": {
-            void services.tokenize(data.sql, data.version, data.sqlMode).then((result) => {
+            void services.tokenize(data.statements, data.version, data.sqlMode).then((result) => {
                 postResultMessage(taskId, {
                     tokens: result,
                     final: true,