Merge pull request #36 from doxygen/master

Various fixes to better support doxygen's lexer files

src/languages/lexLanguageTypes.ts

@@ -24,6 +24,7 @@ export enum TokenType {
     Unknown,
     Divider,
     Escape,
+    CharRange,
     EOL,
     EOS
 }

src/languages/parser/lexParser.ts

@@ -1,5 +1,5 @@
 import { createScanner } from "./lexScanner";
-import { TokenType } from "../lexLanguageTypes";
+import { ScannerState, TokenType } from "../lexLanguageTypes";
 import { binarySearch } from "./utils";
 import { Problem, ProblemType, ProblemRelated } from "../common";
 
@@ -114,9 +114,10 @@ export function parse(text: string, state: ParserState = ParserState.WaitingDecl
     let codeOffset = 0;
     let tokenText = '';
     let acceptingStates = false;
-    let lastToken = token;
     let isConditionScope = false;
+    let lastToken = TokenType.Invalid;
     while (end < 0 && token !== TokenType.EOS) {
+        //console.log('parse(type='+TokenType[token]+',token='+scanner.getTokenText()+',state='+ParserState[state]+')');
         offset = scanner.getTokenOffset();
         switch (token) {
             case TokenType.StartCode:
@@ -203,6 +204,7 @@ export function parse(text: string, state: ParserState = ParserState.WaitingDecl
             case ParserState.WaitingRule:
                 switch (token) {
                     case TokenType.Literal:
+                    case TokenType.CharRange:
                     case TokenType.Word:
                         break;
                     case TokenType.Predefined:
@@ -294,6 +296,19 @@ export function parse(text: string, state: ParserState = ParserState.WaitingDecl
                             end: scanner.getTokenEnd()
                         });
                         break;
+                    case TokenType.Action:
+                        tokenText = scanner.getTokenText();
+                        if (/^[a-zA-Z]\w*$/.test(tokenText))
+                            document.components.push({
+                                name: tokenText,
+                                offset: offset,
+                                length: scanner.getTokenLength(),
+                                end: scanner.getTokenEnd(),
+                                used: true,
+                                definition: [-1, -1],
+                                references: [[offset, scanner.getTokenEnd()]]
+                            });
+                        break;
                 }
                 break;
         }
@@ -346,4 +361,4 @@ export function parse(text: string, state: ParserState = ParserState.WaitingDecl
     });
 
     return document;
-}
+}

src/languages/parser/lexScanner.ts

@@ -1,5 +1,5 @@
 import { TokenType, ScannerState, Scanner } from '../lexLanguageTypes'
-import { MultiLineStream, _FSL, _AST, _NWL, _PCS, _BOP, _LAN, _BAR, _WSP, _DQO, _SQO, _BCL, _RAN, _BSL } from './utils';
+import { MultiLineStream, _FSL, _AST, _NWL, _PCS, _BOP, _LAN, _BAR, _WSP, _DQO, _SQO, _BCL, _RAN, _BSL, _SBO } from './utils';
 
 export function createScanner(input: string, initialOffset = 0, initialState: ScannerState = ScannerState.WithinContent): Scanner {
     const stream = new MultiLineStream(input, initialOffset);
@@ -17,6 +17,10 @@ export function createScanner(input: string, initialOffset = 0, initialState: Sc
         return stream.advanceIfRegExp(/^("(?:[^"\\\n]|\\.)*"|'(?:[^'\\\n]|\\.)*')/);
     }
 
+    function nextCharacterRange(): string {
+        return stream.advanceIfRegExp(/^((?<!\\\\)\[(.*?)(?<!\\\\)\])/);
+    }
+
     function finishToken(offset: number, type: TokenType, errorMessage?: string): TokenType {
         tokenType = type;
         tokenOffset = offset;
@@ -41,6 +45,8 @@ export function createScanner(input: string, initialOffset = 0, initialState: Sc
             stream.advance(1);
             return finishToken(offset, TokenType.Unknown);
         }
+        //console.log('scan('+stream.getSource().substring(tokenOffset, stream.pos())+
+        //            ',state='+ScannerState[state]+',oldState='+ScannerState[oldState]+')');
         return token;
     }
 
@@ -62,7 +68,19 @@ export function createScanner(input: string, initialOffset = 0, initialState: Sc
         }
 
         if (white) {
-            return finishToken(offset, TokenType.Divider);
+            // only allow comments to start after white space at the start of a line
+            if (stream.peekChar(0)==_FSL && (stream.peekChar(1)==_AST || stream.peekChar(1)==_FSL)) {
+                if (stream.peekChar(1)==_AST) { // /*
+                    state = ScannerState.WithinComment;
+                    return finishToken(offset, TokenType.StartComment);
+                } else { // //
+                    stream.advanceUntilChar(_NWL);
+                    stream.advance(1);
+                    return finishToken(offset, TokenType.Comment);
+                }
+            } else {
+                return finishToken(offset, TokenType.Divider);
+            }
         }
 
         switch (state) {
@@ -105,6 +123,9 @@ export function createScanner(input: string, initialOffset = 0, initialState: Sc
                     case _BOP: // {
                         state = ScannerState.WithinAction;
                         return finishToken(offset, TokenType.StartAction);
+                    case _BCL: // }
+                        state = ScannerState.WithinContent;
+                        break;
                     case _WSP: // ' '
                         stream.advanceUntilChar(_NWL);
                         return finishToken(offset, TokenType.Invalid);
@@ -122,6 +143,14 @@ export function createScanner(input: string, initialOffset = 0, initialState: Sc
                     case _BSL: // \
                         stream.advance(1); // include the next escaped character
                         return finishToken(offset, TokenType.Escape);
+                    case _SBO: // [
+                        stream.goBack(1);
+                        const charRange = nextCharacterRange()
+                        if (charRange.length > 0) {
+                            return finishToken(offset, TokenType.CharRange);
+                        }
+                        stream.advance(1);
+                        return finishToken(offset, TokenType.Unknown);
                 }
 
                 stream.goBack(1);

src/runner.ts

@@ -66,5 +66,5 @@ export function runSafe<T>(func: () => T, errorVal: T, errorMessage: string, tok
 
 function cancelValue<E>() {
     console.log("Request cancelled...");
-    return undefined;
+    return undefined as unknown as E;
 }

syntaxes/lex.tmLanguage.json

@@ -33,6 +33,9 @@
 				{
 					"include": "#comments"
 				},
+				{
+					"include": "#comment"
+				},
 				{
 					"include": "#strings"
 				}
@@ -58,6 +61,9 @@
 						{
 							"include": "#comments"
 						},
+						{
+							"include": "#comment"
+						},
 						{
 							"include": "#brackets"
 						},
@@ -178,7 +184,7 @@
 		},
 		"startcondition_block": {
 				"name": "entity.startcondition.lex",
-				"begin": "<[a-zA-Z0-9_,]+>[ ]*{(?=$)",
+				"begin": "<[a-zA-Z0-9_,]+>\\s*{\\s*(?=($|//|/\\*))",
 				"end": "^}",
 				"beginCaptures": {
 					"0": {
@@ -195,6 +201,9 @@
 						"include": "#comments"
 					},
 					{
+						"include": "#comment"
+					},
+ 					{
 						"include": "#strings"
 					},
 					{
@@ -247,6 +256,9 @@
 				{
 					"include": "#comments"
 				},
+				{
+					"include": "#comment"
+				},
 				{
 					"name": "constant.numeric.lex",
 					"match": "."
@@ -265,6 +277,9 @@
 				{
 					"include": "#comments"
 				},
+				{
+					"include": "#comment"
+				},
 				{
 					"include": "#user-define"
 				},
@@ -321,6 +336,11 @@
 			"name": "comment.block.yacc",
 			"begin": "/\\*",
 			"end": "\\*/"
+		},
+		"comment": {
+			"name": "comment.single.yacc",
+			"begin": "//",
+			"end": "$"
 		}
 	},
 	"scopeName": "source.l"