Treat codepoints U+00 up to U+20 as whitespace

- See https://docwiki.embarcadero.com/RADStudio/en/Special:Search/Fundamental%20Syntactic%20Elements%20(Delphi)#The_Delphi_Character_Set
integrated-application-development · Oct 6, 2023 · 598c585 · 598c585
1 parent ae9c752
commit 598c585
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 1 deletion.
diff --git a/delphi-frontend/src/main/antlr3/au/com/integradev/delphi/antlr/Delphi.g b/delphi-frontend/src/main/antlr3/au/com/integradev/delphi/antlr/Delphi.g
@@ -1276,7 +1276,7 @@ COMMENT                 :  '//' ~('\n'|'\r')*                          {$channel
                           }
                         }
                         ;
-WHITESPACE              : (' '|'\t'|'\r'|'\n'|'\f')+ {$channel=HIDDEN;}
+WHITESPACE              : ('\u0000'..'\u0020' | '\u3000')+ {$channel=HIDDEN;}
                         ;
 UnicodeBOM              : '\uFEFF' {$channel=HIDDEN;}
                         ;

diff --git a/delphi-frontend/src/test/java/au/com/integradev/delphi/antlr/GrammarTest.java b/delphi-frontend/src/test/java/au/com/integradev/delphi/antlr/GrammarTest.java
@@ -237,4 +237,9 @@ void testEmptyFileShouldThrow() {
                     DelphiFileUtils.mockConfig()))
         .isInstanceOf(DelphiFileConstructionException.class);
   }
+
+  @Test
+  void testUnusualWhitespace() {
+    parseFile("UnusualWhitespace.pas");
+  }
 }
diff --git a/delphi-frontend/src/test/resources/au/com/integradev/delphi/grammar/UnusualWhitespace.pas b/delphi-frontend/src/test/resources/au/com/integradev/delphi/grammar/UnusualWhitespace.pas
-Original file line number
+Diff line change
@@ Expand Up / @@ -1276,7 +1276,7 @@ COMMENT : '//' ~('\n'|'\r')* {$channel @@
                               }
                             }
                             ;
-    WHITESPACE              : (' '|'\t'|'\r'|'\n'|'\f')+ {$channel=HIDDEN;}
+    WHITESPACE              : ('\u0000'..'\u0020' | '\u3000')+ {$channel=HIDDEN;}
                             ;
     UnicodeBOM              : '\uFEFF' {$channel=HIDDEN;}
                             ;
@@ Expand Down @@