fix: split whiteSpace into multiple tokens to fix findCaretTokenIndex may undefined

Author: JackWang032

src/grammar/flink/FlinkSqlLexer.g4

@@ -11,7 +11,7 @@ options {
 
 // SKIP
 
-SPACE         : [ \t\r\n]+                                                         -> channel(HIDDEN);
+SPACE         : (' ' | '\t' | '\r' | '\n')                                         -> channel(HIDDEN);
 COMMENT_INPUT : '/*' .*? '*/'                                                      -> channel(HIDDEN);
 LINE_COMMENT: (('--' | '#') ~[\r\n]* ('\r'? '\n' | EOF) | '--' ('\r'? '\n' | EOF)) -> channel(HIDDEN);
 

src/grammar/impala/ImpalaSqlLexer.g4

@@ -350,4 +350,4 @@ SIMPLE_COMMENT: '--' ~[\r\n]* '\r'? '\n'? -> channel(HIDDEN);
 
 BRACKETED_COMMENT: '/*' .*? '*/' -> channel(HIDDEN);
 
-WS: [ \r\n\t]+ -> channel(HIDDEN);
+WS: (' ' | '\t' | '\r' | '\n') -> channel(HIDDEN);

src/grammar/mysql/MySqlLexer.g4

@@ -41,7 +41,7 @@ channels {
     ERRORCHANNEL
 }
 
-SPACE              : [ \t\r\n]+                                                           -> channel(HIDDEN);
+SPACE              : (' ' | '\t' | '\r' | '\n')                                           -> channel(HIDDEN);
 SPEC_MYSQL_COMMENT : '/*!' .+? '*/'                                                       -> channel(MYSQLCOMMENT);
 COMMENT_INPUT      : '/*' .*? '*/'                                                        -> channel(HIDDEN);
 LINE_COMMENT: (('--' [ \t]* | '#') ~[\r\n]* ('\r'? '\n' | EOF) | '--' ('\r'? '\n' | EOF)) -> channel(HIDDEN);

src/grammar/spark/SparkSqlLexer.g4

@@ -481,7 +481,7 @@ SIMPLE_COMMENT: '--' ('\\\n' | ~[\r\n])* '\r'? '\n'? -> channel(HIDDEN);
 BRACKETED_COMMENT:
     '/*' (BRACKETED_COMMENT | .)*? ('*/' | {this.markUnclosedComment();} EOF) -> channel(HIDDEN);
 
-WS: [ \r\n\t]+ -> channel(HIDDEN);
+WS: (' ' | '\t' | '\r' | '\n') -> channel(HIDDEN);
 
 // Catch-all for anything we can't recognize.
 // We use this to be able to ignore and recover all the text

src/grammar/trino/TrinoSql.g4

@@ -1648,7 +1648,7 @@ SIMPLE_COMMENT: '--' ~[\r\n]* '\r'? '\n'? -> channel(HIDDEN);
 
 BRACKETED_COMMENT: '/*' .*? '*/' -> channel(HIDDEN);
 
-WS: [ \r\n\t]+ -> channel(HIDDEN);
+WS: (' ' | '\t' | '\r' | '\n') -> channel(HIDDEN);
 
 // Catch-all for anything we can't recognize.
 // We use this to be able to ignore and recover all the text

src/lib/flink/FlinkSqlLexer.interp

@@ -32,9 +32,15 @@ export const readSQL = (dirname: string, fileName: string) => {
     return result;
 };
 
-export function commentOtherLine(sqlContent: string, line: number) {
+export function commentOtherLine(
+    sqlContent: string,
+    line: number | [startLine: number, endLine: number]
+) {
     const slices = sqlContent.split('\n').map((item, index) => {
-        if (index !== line - 1) {
+        if (
+            (Array.isArray(line) && (index + 1 < line[0] || index + 1 > line[1])) ||
+            (typeof line === 'number' && index + 1 !== line)
+        ) {
             return '-- ' + item;
         } else {
             return item;

src/lib/flink/FlinkSqlLexer.ts

@@ -4,4 +4,8 @@ USE
 ;
 CREATE 
 ;
-SHOW 
+SHOW 
+
+CREATE TABLE tb (id 
+ 
+);

src/lib/impala/ImpalaSqlLexer.interp

@@ -67,4 +67,16 @@ describe('Flink SQL Token Suggestion', () => {
             'JARS',
         ]);
     });
+
+    test('Suggestion in new line', () => {
+        const pos: CaretPosition = {
+            lineNumber: 10,
+            column: 2,
+        };
+        const suggestion = flink.getSuggestionAtCaretPosition(
+            commentOtherLine(tokenSql, [9, 11]),
+            pos
+        )?.keywords;
+        expect(suggestion.length).not.toBe(0);
+    });
 });

src/lib/impala/ImpalaSqlLexer.ts

@@ -18,3 +18,7 @@ LOAD
 ;
 SHOW 
 ;
+
+CREATE TABLE tb (id 
+ 
+);

src/lib/mysql/MySqlLexer.interp

@@ -219,4 +219,16 @@ describe('Hive SQL Token Suggestion', () => {
             'SCHEMAS',
         ]);
     });
+
+    test('Suggestion in new line', () => {
+        const pos: CaretPosition = {
+            lineNumber: 23,
+            column: 2,
+        };
+        const suggestion = hive.getSuggestionAtCaretPosition(
+            commentOtherLine(tokenSql, [21, 23]),
+            pos
+        )?.keywords;
+        expect(suggestion.length).not.toBe(0);
+    });
 });

src/lib/mysql/MySqlLexer.ts

@@ -9,3 +9,7 @@ INSERT ;
 SHOW ;
 
 CREATE TABLE t1 (id );
+
+CREATE TABLE tb (id 
+ 
+);

src/lib/spark/SparkSqlLexer.interp

@@ -143,4 +143,16 @@ describe('Impala SQL Token Suggestion', () => {
 
         expect(dataTypes.every((dataType) => suggestion.includes(dataType))).toBe(true);
     });
+
+    test('Suggestion in new line', () => {
+        const pos: CaretPosition = {
+            lineNumber: 14,
+            column: 2,
+        };
+        const suggestion = impala.getSuggestionAtCaretPosition(
+            commentOtherLine(tokenSql, [13, 15]),
+            pos
+        )?.keywords;
+        expect(suggestion.length).not.toBe(0);
+    });
 });

src/lib/spark/SparkSqlLexer.ts

@@ -14,3 +14,7 @@ LOAD
 ;
 SHOW 
 ;
+
+CREATE TABLE tb (id 
+ 
+);

src/lib/trino/TrinoSqlLexer.interp

@@ -242,4 +242,16 @@ describe('MySQL Token Suggestion', () => {
             'BINARY',
         ]);
     });
+
+    test('Suggestion in new line', () => {
+        const pos: CaretPosition = {
+            lineNumber: 19,
+            column: 2,
+        };
+        const suggestion = mysql.getSuggestionAtCaretPosition(
+            commentOtherLine(tokenSql, [18, 20]),
+            pos
+        )?.keywords;
+        expect(suggestion.length).not.toBe(0);
+    });
 });

src/lib/trino/TrinoSqlLexer.ts

@@ -8,5 +8,6 @@ DELETE ;
 
 CREATE ;
 
-
-
+CREATE TABLE tb (id 
+ 
+);

test/helper.ts

@@ -190,4 +190,16 @@ describe('Postgres SQL Token Suggestion', () => {
         )?.keywords;
         expect(suggestion).toMatchUnorderedArray(['INTO']);
     });
+
+    test('Suggestion in new line', () => {
+        const pos: CaretPosition = {
+            lineNumber: 12,
+            column: 2,
+        };
+        const suggestion = postgresql.getSuggestionAtCaretPosition(
+            commentOtherLine(tokenSql, [11, 13]),
+            pos
+        )?.keywords;
+        expect(suggestion.length).not.toBe(0);
+    });
 });

test/parser/flink/suggestion/fixtures/tokenSuggestion.sql

@@ -16,3 +16,7 @@ SHOW
 ;
 EXPORT 
 ;
+
+CREATE TABLE tb (id 
+ 
+);

test/parser/flink/suggestion/tokenSuggestion.test.ts

@@ -197,4 +197,16 @@ describe('Spark SQL Token Suggestion', () => {
 
         expect(suggestion).toMatchUnorderedArray(['TABLE']);
     });
+
+    test('Suggestion in new line', () => {
+        const pos: CaretPosition = {
+            lineNumber: 21,
+            column: 2,
+        };
+        const suggestion = spark.getSuggestionAtCaretPosition(
+            commentOtherLine(tokenSql, [20, 22]),
+            pos
+        )?.keywords;
+        expect(suggestion.length).not.toBe(0);
+    });
 });

test/parser/hive/suggestion/fixtures/tokenSuggestion.sql

@@ -11,3 +11,7 @@ DESCRIBE ;
 DROP ;
 
 INSERT ;
+
+CREATE TABLE tb (id 
+ 
+);

test/parser/hive/suggestion/tokenSuggestion.test.ts

@@ -124,4 +124,16 @@ describe('Trino SQL Token Suggestion', () => {
 
         expect(suggestion).toMatchUnorderedArray(['INTO']);
     });
+
+    test('Suggestion in new line', () => {
+        const pos: CaretPosition = {
+            lineNumber: 16,
+            column: 2,
+        };
+        const suggestion = trino.getSuggestionAtCaretPosition(
+            commentOtherLine(tokenSql, [15, 17]),
+            pos
+        )?.keywords;
+        expect(suggestion.length).not.toBe(0);
+    });
 });