From 0887ef47eda9ad0f5234159ed4e2f8fc64dd5dd3 Mon Sep 17 00:00:00 2001
From: Zane Duffield <zduffield@integradev.com.au>
Date: Fri, 6 Oct 2023 15:36:57 +1100
Subject: [PATCH] Treat codepoints U+00 up to U+20 as whitespace

- See https://docwiki.embarcadero.com/RADStudio/en/Special:Search/Fundamental%20Syntactic%20Elements%20(Delphi)#The_Delphi_Character_Set
---
 CHANGELOG.md                                     |   1 +
 .../au/com/integradev/delphi/antlr/Delphi.g      |   2 +-
 .../com/integradev/delphi/antlr/GrammarTest.java |   5 +++++
 .../delphi/grammar/UnusualWhitespace.pas         | Bin 0 -> 1105 bytes
 4 files changed, 7 insertions(+), 1 deletion(-)
 create mode 100644 delphi-frontend/src/test/resources/au/com/integradev/delphi/grammar/UnusualWhitespace.pas

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 39edf2d9b..52b984f1c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -60,6 +60,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `add` is no longer colorized as a keyword in the SonarQube web interface.
 - `remove` is no longer colorized as a keyword in the SonarQube web interface.
 - `variant` is no longer colorized as a keyword in the SonarQube web interface.
+- Parsing errors on unusual whitespace characters.
 
 ## [0.40.0] - 2023-01-10
 
diff --git a/delphi-frontend/src/main/antlr3/au/com/integradev/delphi/antlr/Delphi.g b/delphi-frontend/src/main/antlr3/au/com/integradev/delphi/antlr/Delphi.g
index 731cb86f8..0615b8a8d 100644
--- a/delphi-frontend/src/main/antlr3/au/com/integradev/delphi/antlr/Delphi.g
+++ b/delphi-frontend/src/main/antlr3/au/com/integradev/delphi/antlr/Delphi.g
@@ -1276,7 +1276,7 @@ COMMENT                 :  '//' ~('\n'|'\r')*                          {$channel
                           }
                         }
                         ;
-WHITESPACE              : (' '|'\t'|'\r'|'\n'|'\f')+ {$channel=HIDDEN;}
+WHITESPACE              : ('\u0000'..'\u0020' | '\u3000')+ {$channel=HIDDEN;}
                         ;
 UnicodeBOM              : '\uFEFF' {$channel=HIDDEN;}
                         ;
diff --git a/delphi-frontend/src/test/java/au/com/integradev/delphi/antlr/GrammarTest.java b/delphi-frontend/src/test/java/au/com/integradev/delphi/antlr/GrammarTest.java
index b263814ac..b984f1927 100644
--- a/delphi-frontend/src/test/java/au/com/integradev/delphi/antlr/GrammarTest.java
+++ b/delphi-frontend/src/test/java/au/com/integradev/delphi/antlr/GrammarTest.java
@@ -237,4 +237,9 @@ void testEmptyFileShouldThrow() {
                     DelphiFileUtils.mockConfig()))
         .isInstanceOf(DelphiFileConstructionException.class);
   }
+
+  @Test
+  void testUnusualWhitespace() {
+    parseFile("UnusualWhitespace.pas");
+  }
 }
diff --git a/delphi-frontend/src/test/resources/au/com/integradev/delphi/grammar/UnusualWhitespace.pas b/delphi-frontend/src/test/resources/au/com/integradev/delphi/grammar/UnusualWhitespace.pas
new file mode 100644
index 0000000000000000000000000000000000000000..3b4bad238909d49cf8941f095501fc7e2014afbb
GIT binary patch
literal 1105
zcmZ{j$xg#C7zSAd1j4?rzV+6M<wQ@dlhBHiin?JrhU9N+DRrX65!b#eFM=r&0;KN}
z#~F`h$(d3~%>q>xWtcpVq^2SZBdXh$ZAqmmADG#6GR_hjlhUD<srveo<!MB5nbV9P
zbn5JlhEg$7v2lOUtmw7?niKsOfaYC@7Tj(hSc|Uz2+&f4-y_s?(czby0!S+@bO_t(
zctBcxLeiRt9NXeOu-&@o0<t06fNVM)7oaVNUjnr4@GF3J9DWVZuETEt+VdOeI`+k@
z53B>X-vsE8H;{GYJ^<-h1Zck#S8!mRdg!M(yF=u>hsZ^M2ouu||72xbwHotNO}DJb
zD3W1fM$@NRq~D1Y(<5RsHj&DKq@0B+W))Lph3V9kS(sMMl29{~yew43JXMsGF~#QP
gpIrS_@yBG+jwv1HVK$P{pM!a6YUp?Mwq?fZCyoPb4FCWD

literal 0
HcmV?d00001