feat(lexer): add peek_char, support tokens ColonDash, QuestionDash

pluveto · Dec 14, 2023 · d1fd5a6 · d1fd5a6
1 parent a1061b1
commit d1fd5a6
Showing 1 changed file with 48 additions and 2 deletions.
diff --git a/rulog-core/src/lexer.rs b/rulog-core/src/lexer.rs
@@ -31,6 +31,14 @@ impl<'a> Lexer<'a> {
         self.next_position += 1;
     }
 
+    fn peek_char(&self) -> char {
+        if self.next_position >= self.input.len() {
+            '\0'
+        } else {
+            self.input.chars().nth(self.next_position).unwrap()
+        }
+    }
+
     pub fn next_token(&mut self) -> Result<Token, LexerError> {
         self.skip_whitespace();
 
@@ -47,15 +55,29 @@ impl<'a> Lexer<'a> {
             '|' => Ok(Token::Bar),
             '.' => Ok(Token::Period),
             ';' => Ok(Token::Semicolon),
-            ':' => Ok(Token::Colon),
-            '?' => Ok(Token::QuestionMark),
             '(' => Ok(Token::LeftParenthesis),
             ')' => Ok(Token::RightParenthesis),
             '[' => Ok(Token::LeftBracket),
             ']' => Ok(Token::RightBracket),
             '{' => Ok(Token::LeftCurlyBracket),
             '}' => Ok(Token::RightCurlyBracket),
             '\0' => Ok(Token::EndOfFile),
+            ':' => {
+                if self.peek_char() == '-' {
+                    self.read_char();
+                    Ok(Token::ColonDash)
+                } else {
+                    Ok(Token::Colon)
+                }
+            }
+            '?' => {
+                self.read_char();
+                if self.ch == '-' {
+                    Ok(Token::QuestionDash)
+                } else {
+                    Err(LexerError::UnexpectedCharacter(self.ch))
+                }
+            }
             '\"' => {
                 let start_position = self.position + 1;
                 self.read_char();
@@ -65,6 +87,21 @@ impl<'a> Lexer<'a> {
                 let text = &self.input[start_position..self.position];
                 Ok(Token::String(text.to_string()))
             }
+            '\'' => {
+                let start_position = self.position + 1;
+                self.read_char();
+                while self.ch != '\'' {
+                    self.read_char();
+                }
+                let text = &self.input[start_position..self.position];
+                Ok(Token::Atom(text.to_string()))
+            }
+            '%' => {
+                while self.ch != '\n' && self.ch != '\0' {
+                    self.read_char();
+                }
+                return self.next_token();
+            }
             _ => {
                 let ret = if self.ch.is_alphabetic() || self.ch == '_' {
                     Ok(self.read_identifier_or_variable())
@@ -200,4 +237,13 @@ mod tests {
         assert_eq!(lexer.next_token().unwrap(), Token::RightBracket);
         assert_eq!(lexer.next_token().unwrap(), Token::EndOfFile);
     }
+
+    #[test]
+    fn test_comment() {
+        setup_logger();
+        let mut lexer = Lexer::new("% comment\natom.");
+        assert_eq!(lexer.next_token().unwrap(), Token::Atom("atom".to_string()));
+        assert_eq!(lexer.next_token().unwrap(), Token::Period);
+        assert_eq!(lexer.next_token().unwrap(), Token::EndOfFile);
+    }
 }