Skip to content

Commit

Permalink
Update lexer to handle recent syntax changes
Browse files Browse the repository at this point in the history
  • Loading branch information
@andymcn
andymcn committed Dec 14, 2015
1 parent a4ed786 commit 0466d5b
Show file tree
Hide file tree
Showing 5 changed files with 91 additions and 14 deletions.
Binary file modified PonyLanguage.v12.suo
View file
Binary file not shown.
2 changes: 1 addition & 1 deletion PonyLanguage/LexTagger.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ private void TextBufferChanged(object sender, TextContentChangedEventArgs e)
{
// Remove lines from line state list
int line = snapshot.GetLineFromPosition(change.NewPosition).LineNumber;
_lineStartState.RemoveRange(line, -change.LineCountDelta);
_lineStartState.RemoveRange(line + 1, -change.LineCountDelta);
}
}

Expand Down
70 changes: 66 additions & 4 deletions PonyLanguage/Lexer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ class Lexer
private string _buffer;
private int _state;
private readonly IDictionary<string, TokenId> _keywords = new Dictionary<string, TokenId>();
private readonly IDictionary<string, TokenId> _hash_keywords = new Dictionary<string, TokenId>();
private readonly IList<Tuple<string, TokenId>> _symbols = new List<Tuple<string, TokenId>>();

public const int newState = -2;
Expand All @@ -20,20 +21,24 @@ public Lexer()
_keywords["_"] = TokenId.DontCare;
_keywords["true"] = TokenId.TrueFalse;
_keywords["false"] = TokenId.TrueFalse;
_keywords["compiler_intrinsic"] = TokenId.Intrinsic;
_keywords["compile_intrinsic"] = TokenId.Intrinsic;
_keywords["use"] = TokenId.Use;
_keywords["type"] = TokenId.Type;
_keywords["interface"] = TokenId.Interface;
_keywords["trait"] = TokenId.Trait;
_keywords["primitive"] = TokenId.Primitive;
_keywords["struct"] = TokenId.Struct;
_keywords["class"] = TokenId.Class;
_keywords["actor"] = TokenId.Actor;
_keywords["object"] = TokenId.Object;
_keywords["lambda"] = TokenId.Lambda;
_keywords["delegate"] = TokenId.Delegate;
_keywords["as"] = TokenId.As;
_keywords["is"] = TokenId.Is;
_keywords["isnt"] = TokenId.Isnt;
_keywords["var"] = TokenId.Var;
_keywords["let"] = TokenId.Let;
_keywords["embed"] = TokenId.Embed;
_keywords["new"] = TokenId.New;
_keywords["fun"] = TokenId.Fun;
_keywords["be"] = TokenId.Be;
Expand All @@ -48,9 +53,11 @@ public Lexer()
_keywords["break"] = TokenId.Break;
_keywords["continue"] = TokenId.Continue;
_keywords["error"] = TokenId.Error;
_keywords["compile_error"] = TokenId.CompileError;
_keywords["consume"] = TokenId.Continue;
_keywords["recover"] = TokenId.Recover;
_keywords["if"] = TokenId.If;
_keywords["ifdef"] = TokenId.Ifdef;
_keywords["then"] = TokenId.Then;
_keywords["else"] = TokenId.Else;
_keywords["elseif"] = TokenId.ElseIf;
Expand All @@ -65,6 +72,17 @@ public Lexer()
_keywords["where"] = TokenId.Where;
_keywords["try"] = TokenId.Try;
_keywords["with"] = TokenId.With;
_keywords["not"] = TokenId.Not;
_keywords["and"] = TokenId.And;
_keywords["or"] = TokenId.Or;
_keywords["xor"] = TokenId.Xor;
_keywords["identityof"] = TokenId.IdentityOf;
_keywords["addressof"] = TokenId.AddressOf;

_hash_keywords["#any"] = TokenId.Capability;
_hash_keywords["#read"] = TokenId.Capability;
_hash_keywords["#send"] = TokenId.Capability;
_hash_keywords["#share"] = TokenId.Capability;

// Longer symbols must appear before shorter prefixes
_symbols.Add(new Tuple<string, TokenId>("...", TokenId.Ellipsis));
Expand Down Expand Up @@ -98,6 +116,7 @@ public Lexer()
_symbols.Add(new Tuple<string, TokenId>("|", TokenId.InfixOp));
_symbols.Add(new Tuple<string, TokenId>("&", TokenId.Ampersand));
_symbols.Add(new Tuple<string, TokenId>("^", TokenId.Ephemeral));
_symbols.Add(new Tuple<string, TokenId>("!", TokenId.Borrowed));
_symbols.Add(new Tuple<string, TokenId>("?", TokenId.Question));
}

Expand Down Expand Up @@ -154,12 +173,18 @@ private TokenId NextToken(out int length)
if(c == '"')
return ProcessString(out length);

if(c == '\'')
return ProcessCharLiteral(out length);

if(c >= '0' && c <= '9')
return ProcessNumber(out length);

if((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')
return ProcessId(out length);

if(c == '#')
return ProcessHash(out length);

if(IsSymbolChar(c))
return ProcessSymbol(out length);

Expand Down Expand Up @@ -245,14 +270,33 @@ private int NumberLength(int start_offset, int int_base)
else if((c >= 'A') && (c <= 'Z'))
digit = c - 'A' + 10;

if(digit >= int_base)
if(c != '_' && digit >= int_base)
return length;
}

return _buffer.Length;
}


private TokenId ProcessCharLiteral(out int length)
{
for(int len = 1; len < _buffer.Length; len++)
{
if(_buffer[len] == '\'')
{
length = len + 1;
return TokenId.Number;
}

if(_buffer[len] == '\\') // Escape, skip next character
len++;
}

length = _buffer.Length;
return TokenId.Number;
}


// Strings

private TokenId ProcessString(out int length)
Expand Down Expand Up @@ -285,10 +329,14 @@ private TokenId ProcessString(out int length)

private TokenId ProcessTripleString(out int length)
{
for(int len = 3; len < _buffer.Length; len++)
for(int len = 3; len <= _buffer.Length; len++)
{
if(_buffer[len - 2] == '"' && _buffer[len - 1] == '"' && _buffer[len] == '"')
if(_buffer[len - 3] == '"' && _buffer[len - 2] == '"' && _buffer[len - 1] == '"')
{
// End of triple string found. Check for trailing "s.
while(len < _buffer.Length && _buffer[len] == '"')
len++;

length = len;
_state = normalState;
return TokenId.String;
Expand Down Expand Up @@ -318,6 +366,20 @@ private TokenId ProcessId(out int length)
}


private TokenId ProcessHash(out int length)
{
length = IdLength();
string id = _buffer.Substring(0, length);

// Check for keywords
if(_hash_keywords.ContainsKey(id))
return _hash_keywords[id];

length = 1;
return TokenId.Ignore;
}


private int IdLength()
{
for(int length = 1; length < _buffer.Length; length++)
Expand Down
15 changes: 14 additions & 1 deletion PonyLanguage/SyntaxHighlighter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,14 +69,18 @@ public SyntaxHighlighter(ITextBuffer buffer, ITagAggregator<LexTag> lexTagAggreg
_colourMap[TokenId.Interface] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.Trait] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.Primitive] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.Struct] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.Class] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.Actor] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.Object] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.Lambda] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.Delegate] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.As] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.Is] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.Isnt] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.Var] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.Let] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.Embed] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.New] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.Fun] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.Be] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
Expand All @@ -86,9 +90,11 @@ public SyntaxHighlighter(ITextBuffer buffer, ITagAggregator<LexTag> lexTagAggreg
_colourMap[TokenId.Break] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.Continue] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.Error] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.Continue] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.CompileError] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.Consume] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.Recover] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.If] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.Ifdef] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.Then] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.Else] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.ElseIf] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
Expand All @@ -103,6 +109,12 @@ public SyntaxHighlighter(ITextBuffer buffer, ITagAggregator<LexTag> lexTagAggreg
_colourMap[TokenId.Where] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.Try] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.With] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Keyword);
_colourMap[TokenId.Not] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Operator);
_colourMap[TokenId.And] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Operator);
_colourMap[TokenId.Or] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Operator);
_colourMap[TokenId.Xor] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Operator);
_colourMap[TokenId.IdentityOf] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Operator);
_colourMap[TokenId.AddressOf] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Operator);

_colourMap[TokenId.Ellipsis] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Operator);
_colourMap[TokenId.Arrow] = typeService.GetClassificationType(ClassificationType);
Expand All @@ -122,6 +134,7 @@ public SyntaxHighlighter(ITextBuffer buffer, ITagAggregator<LexTag> lexTagAggreg
_colourMap[TokenId.At] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Operator);
_colourMap[TokenId.Ampersand] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Operator);
_colourMap[TokenId.Ephemeral] = typeService.GetClassificationType(ClassificationType);
_colourMap[TokenId.Borrowed] = typeService.GetClassificationType(ClassificationType);
_colourMap[TokenId.Question] = typeService.GetClassificationType(PredefinedClassificationTypeNames.Other);

_lexTags.TagsChanged += LexTagChanged;
Expand Down
18 changes: 10 additions & 8 deletions PonyLanguage/TokenId.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,21 +11,23 @@ public enum TokenId
LSquare, RSquare,
Comma, Dot, Tilde, Colon, Semi,
Arrow, DoubleArrow,
Assign, InfixOp, Not,
Ampersand, Ephemeral,
Assign, InfixOp,
Ampersand, Ephemeral, Borrowed,
At, Question, DontCare, Ellipsis,
Intrinsic,
Use,
Type, Interface, Trait, Primitive, Class, Actor,
Object,
Type, Interface, Trait, Primitive, Struct, Class, Actor,
Object, Lambda, Delegate,
As, Is, Isnt,
Var, Let, New, Fun, Be,
Var, Let, Embed, New, Fun, Be,
Capability,
This,
Return, Break, Continue, Error,
Return, Break, Continue, Error, CompileError,
Consume, Recover,
If, Then, Else, ElseIf, End,
If, Ifdef, Then, Else, ElseIf, End,
While, Do, Repeat, Until, For, In,
Match, Where, Try, With
Match, Where, Try, With,
Not, And, Or, Xor,
IdentityOf, AddressOf
}
}

0 comments on commit 0466d5b

Please # to comment.