fix(common): #424 allTokens slice when caretTokenIndex use tokenIndexOffset

src/parser/common/basicSQL.ts

@@ -414,7 +414,7 @@ export abstract class BasicSQL<
 
             // A boundary consisting of the index of the input.
             const startIndex = startStatement?.start?.start ?? 0;
-            const stopIndex = stopStatement?.stop?.stop ?? inputSlice.length - 1;
+            const stopIndex = stopStatement?.stop?.stop ?? inputSlice.length;
 
             /**
              * Save offset of the tokenIndex in the range of input
@@ -518,6 +518,7 @@ export abstract class BasicSQL<
         } else {
             if (statementCount > 1) {
                 caretTokenIndex = caretTokenIndex - tokenIndexOffset;
+                allTokens = allTokens.slice(tokenIndexOffset);
             }
         }
 

test/parser/flink/suggestion/completeAfterSyntaxError.test.ts

@@ -7,6 +7,7 @@ describe('FlinkSQL Complete After Syntax Error', () => {
     const sql1 = `SELECT  FROM tb2;\nINSERT INTO `;
     const sql2 = `SELECT  FROM tb3;\nCREATE TABLE `;
     const sql3 = `SELECT FROM t1;\nSL`;
+    const sql4 = `SELECT FROM t1;\n-- the comment 1\n-- the comment 2\nSELECT * FROM db.`;
 
     test('Syntax error but end with semi, should suggest tableName', () => {
         const pos: CaretPosition = {
@@ -62,4 +63,19 @@ describe('FlinkSQL Complete After Syntax Error', () => {
         );
         expect(filterKeywords).toMatchUnorderedArray(['SELECT']);
     });
+
+    test('Syntax suggestion after error and comments', () => {
+        const pos: CaretPosition = {
+            lineNumber: 4,
+            column: 18,
+        };
+        const syntaxes = flink.getSuggestionAtCaretPosition(sql4, pos)?.syntax;
+        const suggestion = syntaxes?.find(
+            (syn) => syn.syntaxContextType === EntityContextType.TABLE
+        );
+
+        // syntax
+        expect(suggestion).not.toBeUndefined();
+        expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.']);
+    });
 });

test/parser/flink/suggestion/syntaxSuggestion.test.ts

@@ -497,4 +497,52 @@ describe('Flink SQL Syntax Suggestion', () => {
             expect(suggestion[0].syntaxContextType).toBe(scenario.entityContextType);
         });
     });
+
+    test('Syntax suggestion after a comment', () => {
+        const sql = `-- the comment\nSELECT * FROM db.`;
+        const pos: CaretPosition = {
+            lineNumber: 2,
+            column: 18,
+        };
+
+        const syntaxes = flink.getSuggestionAtCaretPosition(sql, pos)?.syntax;
+        const suggestion = syntaxes?.find(
+            (syn) => syn.syntaxContextType === EntityContextType.TABLE
+        );
+
+        expect(suggestion).not.toBeUndefined();
+        expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.']);
+    });
+
+    test('Syntax suggestion after comments', () => {
+        const sql = `-- the comment 1\n-- the comment 2\nSELECT * FROM db.`;
+        const pos: CaretPosition = {
+            lineNumber: 3,
+            column: 18,
+        };
+
+        const syntaxes = flink.getSuggestionAtCaretPosition(sql, pos)?.syntax;
+        const suggestion = syntaxes?.find(
+            (syn) => syn.syntaxContextType === EntityContextType.TABLE
+        );
+
+        expect(suggestion).not.toBeUndefined();
+        expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.']);
+    });
+
+    test('Syntax suggestion after comments', () => {
+        const sql = `-- SELECT FROM t1;\n-- the comment 1\n-- the comment 2\nSELECT * FROM db.`;
+        const pos: CaretPosition = {
+            lineNumber: 4,
+            column: 18,
+        };
+
+        const syntaxes = flink.getSuggestionAtCaretPosition(sql, pos)?.syntax;
+        const suggestion = syntaxes?.find(
+            (syn) => syn.syntaxContextType === EntityContextType.TABLE
+        );
+
+        expect(suggestion).not.toBeUndefined();
+        expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.']);
+    });
 });

test/parser/hive/suggestion/completeAfterSyntaxError.test.ts

@@ -7,6 +7,7 @@ describe('HiveSQL Complete After Syntax Error', () => {
     const sql1 = `SELECT  FROM tb2;\nINSERT INTO `;
     const sql2 = `SELECT  FROM tb3;\nCREATE TABLE `;
     const sql3 = `SELECT FROM t1;\nSL`;
+    const sql4 = `SELECT FROM t1;\n-- the comment 1\n-- the comment 2\nSELECT * FROM db.`;
 
     test('Syntax error but end with semi, should suggest tableName', () => {
         const pos: CaretPosition = {
@@ -63,4 +64,19 @@ describe('HiveSQL Complete After Syntax Error', () => {
         );
         expect(filterKeywords).toMatchUnorderedArray(['SELECT']);
     });
+
+    test('Syntax suggestion after error and comments', () => {
+        const pos: CaretPosition = {
+            lineNumber: 4,
+            column: 18,
+        };
+        const syntaxes = hive.getSuggestionAtCaretPosition(sql4, pos)?.syntax;
+        const suggestion = syntaxes?.find(
+            (syn) => syn.syntaxContextType === EntityContextType.TABLE
+        );
+
+        // syntax
+        expect(suggestion).not.toBeUndefined();
+        expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.']);
+    });
 });

test/parser/hive/suggestion/syntaxSuggestion.test.ts

@@ -591,4 +591,52 @@ describe('Hive SQL Syntax Suggestion', () => {
         expect(suggestion).not.toBeUndefined();
         expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['month']);
     });
+
+    test('Syntax suggestion after a comment', () => {
+        const sql = `-- the comment\nSELECT * FROM db.`;
+        const pos: CaretPosition = {
+            lineNumber: 2,
+            column: 18,
+        };
+
+        const syntaxes = hive.getSuggestionAtCaretPosition(sql, pos)?.syntax;
+        const suggestion = syntaxes?.find(
+            (syn) => syn.syntaxContextType === EntityContextType.TABLE
+        );
+
+        expect(suggestion).not.toBeUndefined();
+        expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.']);
+    });
+
+    test('Syntax suggestion after comments', () => {
+        const sql = `-- the comment 1\n-- the comment 2\nSELECT * FROM db.`;
+        const pos: CaretPosition = {
+            lineNumber: 3,
+            column: 18,
+        };
+
+        const syntaxes = hive.getSuggestionAtCaretPosition(sql, pos)?.syntax;
+        const suggestion = syntaxes?.find(
+            (syn) => syn.syntaxContextType === EntityContextType.TABLE
+        );
+
+        expect(suggestion).not.toBeUndefined();
+        expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.']);
+    });
+
+    test('Syntax suggestion after comments', () => {
+        const sql = `-- SELECT FROM t1;\n-- the comment 1\n-- the comment 2\nSELECT * FROM db.`;
+        const pos: CaretPosition = {
+            lineNumber: 4,
+            column: 18,
+        };
+
+        const syntaxes = hive.getSuggestionAtCaretPosition(sql, pos)?.syntax;
+        const suggestion = syntaxes?.find(
+            (syn) => syn.syntaxContextType === EntityContextType.TABLE
+        );
+
+        expect(suggestion).not.toBeUndefined();
+        expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.']);
+    });
 });

test/parser/impala/suggestion/completeAfterSyntaxError.test.ts

@@ -7,6 +7,7 @@ describe('ImpalaSQL Complete After Syntax Error', () => {
     const sql1 = `SELECT  FROM tb2;\nINSERT INTO `;
     const sql2 = `SELECT  FROM tb3;\nCREATE TABLE `;
     const sql3 = `SELECT FROM t1;\nSL`;
+    const sql4 = `SELECT FROM t1;\n-- the comment 1\n-- the comment 2\nSELECT * FROM db.`;
 
     test('Syntax error but end with semi, should suggest tableName', () => {
         const pos: CaretPosition = {
@@ -63,4 +64,19 @@ describe('ImpalaSQL Complete After Syntax Error', () => {
         );
         expect(filterKeywords).toMatchUnorderedArray(['SELECT']);
     });
+
+    test('Syntax suggestion after error and comments', () => {
+        const pos: CaretPosition = {
+            lineNumber: 4,
+            column: 18,
+        };
+        const syntaxes = impala.getSuggestionAtCaretPosition(sql4, pos)?.syntax;
+        const suggestion = syntaxes?.find(
+            (syn) => syn.syntaxContextType === EntityContextType.TABLE
+        );
+
+        // syntax
+        expect(suggestion).not.toBeUndefined();
+        expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.']);
+    });
 });

test/parser/impala/suggestion/syntaxSuggestion.test.ts

@@ -462,4 +462,52 @@ describe('Impala SQL Syntax Suggestion', () => {
         expect(suggestion).not.toBeUndefined();
         expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['YEAR']);
     });
+
+    test('Syntax suggestion after a comment', () => {
+        const sql = `-- the comment\nSELECT * FROM db.`;
+        const pos: CaretPosition = {
+            lineNumber: 2,
+            column: 18,
+        };
+
+        const syntaxes = impala.getSuggestionAtCaretPosition(sql, pos)?.syntax;
+        const suggestion = syntaxes?.find(
+            (syn) => syn.syntaxContextType === EntityContextType.TABLE
+        );
+
+        expect(suggestion).not.toBeUndefined();
+        expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.']);
+    });
+
+    test('Syntax suggestion after comments', () => {
+        const sql = `-- the comment 1\n-- the comment 2\nSELECT * FROM db.`;
+        const pos: CaretPosition = {
+            lineNumber: 3,
+            column: 18,
+        };
+
+        const syntaxes = impala.getSuggestionAtCaretPosition(sql, pos)?.syntax;
+        const suggestion = syntaxes?.find(
+            (syn) => syn.syntaxContextType === EntityContextType.TABLE
+        );
+
+        expect(suggestion).not.toBeUndefined();
+        expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.']);
+    });
+
+    test('Syntax suggestion after comments', () => {
+        const sql = `-- SELECT FROM t1;\n-- the comment 1\n-- the comment 2\nSELECT * FROM db.`;
+        const pos: CaretPosition = {
+            lineNumber: 4,
+            column: 18,
+        };
+
+        const syntaxes = impala.getSuggestionAtCaretPosition(sql, pos)?.syntax;
+        const suggestion = syntaxes?.find(
+            (syn) => syn.syntaxContextType === EntityContextType.TABLE
+        );
+
+        expect(suggestion).not.toBeUndefined();
+        expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.']);
+    });
 });

test/parser/mysql/suggestion/completeAfterSyntaxError.test.ts

@@ -7,6 +7,7 @@ describe('MySQL Complete After Syntax Error', () => {
     const sql1 = `SELECT  FROM tb2;\nINSERT INTO `;
     const sql2 = `SELECT  FROM tb3;\nCREATE TABLE `;
     const sql3 = `SELECT FROM t1;\nSL`;
+    const sql4 = `SELECT FROM t1;\n-- the comment 1\n-- the comment 2\nSELECT * FROM db.`;
 
     test('Syntax error but end with semi, should suggest tableName', () => {
         const pos: CaretPosition = {
@@ -62,4 +63,19 @@ describe('MySQL Complete After Syntax Error', () => {
         );
         expect(filterKeywords).toMatchUnorderedArray(['SELECT', 'SIGNAL']);
     });
+
+    test('Syntax suggestion after error and comments', () => {
+        const pos: CaretPosition = {
+            lineNumber: 4,
+            column: 18,
+        };
+        const syntaxes = mysql.getSuggestionAtCaretPosition(sql4, pos)?.syntax;
+        const suggestion = syntaxes?.find(
+            (syn) => syn.syntaxContextType === EntityContextType.TABLE
+        );
+
+        // syntax
+        expect(suggestion).not.toBeUndefined();
+        expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.']);
+    });
 });

test/parser/mysql/suggestion/syntaxSuggestion.test.ts

@@ -640,4 +640,52 @@ describe('MySQL Syntax Suggestion', () => {
         expect(suggestion).not.toBeUndefined();
         expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['score']);
     });
+
+    test('Syntax suggestion after a comment', () => {
+        const sql = `-- the comment\nSELECT * FROM db.`;
+        const pos: CaretPosition = {
+            lineNumber: 2,
+            column: 18,
+        };
+
+        const syntaxes = mysql.getSuggestionAtCaretPosition(sql, pos)?.syntax;
+        const suggestion = syntaxes?.find(
+            (syn) => syn.syntaxContextType === EntityContextType.TABLE
+        );
+
+        expect(suggestion).not.toBeUndefined();
+        expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.']);
+    });
+
+    test('Syntax suggestion after comments', () => {
+        const sql = `-- the comment 1\n-- the comment 2\nSELECT * FROM db.`;
+        const pos: CaretPosition = {
+            lineNumber: 3,
+            column: 18,
+        };
+
+        const syntaxes = mysql.getSuggestionAtCaretPosition(sql, pos)?.syntax;
+        const suggestion = syntaxes?.find(
+            (syn) => syn.syntaxContextType === EntityContextType.TABLE
+        );
+
+        expect(suggestion).not.toBeUndefined();
+        expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.']);
+    });
+
+    test('Syntax suggestion after comments', () => {
+        const sql = `-- SELECT FROM t1;\n-- the comment 1\n-- the comment 2\nSELECT * FROM db.`;
+        const pos: CaretPosition = {
+            lineNumber: 4,
+            column: 18,
+        };
+
+        const syntaxes = mysql.getSuggestionAtCaretPosition(sql, pos)?.syntax;
+        const suggestion = syntaxes?.find(
+            (syn) => syn.syntaxContextType === EntityContextType.TABLE
+        );
+
+        expect(suggestion).not.toBeUndefined();
+        expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.']);
+    });
 });

test/parser/postgresql/suggestion/completeAfterSyntaxError.test.ts

@@ -7,6 +7,7 @@ describe('PostgreSQL Complete After Syntax Error', () => {
     const sql1 = `SELECT  FROM tb2;\nINSERT INTO `;
     const sql2 = `SELECT  FROM tb3;\nCREATE TABLE `;
     const sql3 = `SELECT FROM t1;\nSL`;
+    const sql4 = `SELECT FROM t1;\n-- the comment 1\n-- the comment 2\nSELECT * FROM db.`;
 
     test('Syntax error but end with semi, should suggest tableName', () => {
         const pos: CaretPosition = {
@@ -62,4 +63,19 @@ describe('PostgreSQL Complete After Syntax Error', () => {
         );
         expect(filterKeywords).toMatchUnorderedArray(['SELECT']);
     });
+
+    test('Syntax suggestion after error and comments', () => {
+        const pos: CaretPosition = {
+            lineNumber: 4,
+            column: 18,
+        };
+        const syntaxes = postgresql.getSuggestionAtCaretPosition(sql4, pos)?.syntax;
+        const suggestion = syntaxes?.find(
+            (syn) => syn.syntaxContextType === EntityContextType.TABLE
+        );
+
+        // syntax
+        expect(suggestion).not.toBeUndefined();
+        expect(suggestion?.wordRanges.map((token) => token.text)).toEqual(['db', '.']);
+    });
 });