Implement methods and data structures for Semantic Tokens

Project.toml

@@ -3,36 +3,36 @@ uuid = "2b0e0bc5-e4fd-59b4-8912-456d1b03d8d7"
 version = "4.1.1-DEV"
 
 [deps]
-Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
-UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
-Tokenize = "0796e94c-ce3b-5d07-9a54-7f471281c624"
-REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
+CSTParser = "00ebfdb7-1f24-5e51-bd34-a7502290713f"
 DocumentFormat = "ffa9a821-9c82-50df-894e-fbcef3ed31cd"
-StaticLint = "b3cc710f-9c33-5bdb-a03d-a94903873e97"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 JSONRPC = "b9b8584e-8fd3-41f9-ad0c-7255d428e418"
-CSTParser = "00ebfdb7-1f24-5e51-bd34-a7502290713f"
 Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
+StaticLint = "b3cc710f-9c33-5bdb-a03d-a94903873e97"
 SymbolServer = "cf896787-08d5-524d-9de7-132aaa0cb996"
+Tokenize = "0796e94c-ce3b-5d07-9a54-7f471281c624"
 URIParser = "30578b45-9adc-5946-b283-645ec420af67"
-
-[extras]
-Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
-SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
-LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433"
-Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
+UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [compat]
-JSON = "0.20, 0.21"
-julia = "1"
 CSTParser = "3.1"
 DocumentFormat = "3.2.2"
-StaticLint = "8.0"
-Tokenize = "0.5.10"
+JSON = "0.20, 0.21"
 JSONRPC = "1.1"
+StaticLint = "8.0"
 SymbolServer = "6, 7.0"
+Tokenize = "0.5.10"
 URIParser = "0.4.1"
+julia = "1"
+
+[extras]
+LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433"
+SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
+Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
 test = ["Test", "Sockets", "LibGit2", "Serialization", "SHA"]

src/LanguageServer.jl

@@ -2,7 +2,7 @@ module LanguageServer
 import URIParser
 using JSON, REPL, CSTParser, DocumentFormat, SymbolServer, StaticLint
 using CSTParser: EXPR, Tokenize.Tokens, Tokenize.Tokens.kind, headof, parentof, valof
-using StaticLint: refof, scopeof, bindingof
+using StaticLint: refof, scopeof, bindingof, CSTParser
 using UUIDs
 using Base.Docs, Markdown
 import JSONRPC
@@ -30,6 +30,7 @@ include("requests/actions.jl")
 include("requests/init.jl")
 include("requests/signatures.jl")
 include("requests/highlight.jl")
+include("requests/semantic.jl")
 include("utilities.jl")
 
 end

src/languageserverinstance.jl

@@ -324,6 +324,8 @@ function Base.run(server::LanguageServerInstance)
     msg_dispatcher[textDocument_prepareRename_request_type] = request_wrapper(textDocument_prepareRename_request, server)
     msg_dispatcher[textDocument_documentSymbol_request_type] = request_wrapper(textDocument_documentSymbol_request, server)
     msg_dispatcher[textDocument_documentHighlight_request_type] = request_wrapper(textDocument_documentHighlight_request, server)
+    msg_dispatcher[textDocument_semanticTokens_request_type] = request_wrapper(textDocument_semanticTokens_request, server)
+    msg_dispatcher[textDocument_semanticTokens_full_request_type] = request_wrapper(textDocument_semanticTokens_full_request, server)
     msg_dispatcher[julia_getModuleAt_request_type] = request_wrapper(julia_getModuleAt_request, server)
     msg_dispatcher[julia_getDocAt_request_type] = request_wrapper(julia_getDocAt_request, server)
     msg_dispatcher[textDocument_hover_request_type] = request_wrapper(textDocument_hover_request, server)

src/protocol/initialize.jl

@@ -77,6 +77,7 @@ end
     publishDiagnostics::Union{PublishDiagnosticsClientCapabilities,Missing}
     foldingRange::Union{FoldingRangeClientCapabilities,Missing}
     selectionRange::Union{SelectionRangeClientCapabilities,Missing}
+	semanticTokens::Union{SemanticTokensClientCapabilities,Missing}
 end
 
 @dict_readable struct WindowClientCapabilities <: Outbound
@@ -181,6 +182,7 @@ struct ServerCapabilities <: Outbound
     foldingRangeProvider::Union{Bool,FoldingRangeOptions,FoldingRangeRegistrationOptions,Missing}
     executeCommandProvider::Union{ExecuteCommandOptions,Missing}
     selectionRangeProvider::Union{Bool,SelectionRangeOptions,SelectionRangeRegistrationOptions,Missing}
+	semanticTokensProvider::Union{Bool,SemanticTokensOptions, SemanticTokensRegistrationOptions}
     workspaceSymbolProvider::Union{Bool,Missing}
     workspace::Union{WorkspaceOptions,Missing}
     experimental::Union{Any,Missing}

src/protocol/messagedefs.jl

@@ -8,6 +8,8 @@ const textDocument_rename_request_type = JSONRPC.RequestType("textDocument/renam
 const textDocument_prepareRename_request_type = JSONRPC.RequestType("textDocument/prepareRename", PrepareRenameParams, Range)
 const textDocument_documentSymbol_request_type = JSONRPC.RequestType("textDocument/documentSymbol", DocumentSymbolParams, Union{Vector{DocumentSymbol}, Vector{SymbolInformation}, Nothing})
 const textDocument_documentHighlight_request_type = JSONRPC.RequestType("textDocument/documentHighlight", DocumentHighlightParams, Union{Vector{DocumentHighlight}, Nothing})
+const textDocument_semanticTokens_request_type = JSONRPC.RequestType("textDocument/semanticTokens", SemanticTokensParams, Union{SemanticTokens, Nothing})
+const textDocument_semanticTokens_full_request_type = JSONRPC.RequestType("textDocument/semanticTokens/full", SemanticTokensParams, Union{SemanticTokens, Nothing})
 const textDocument_hover_request_type = JSONRPC.RequestType("textDocument/hover", TextDocumentPositionParams, Union{Hover, Nothing})
 const textDocument_didOpen_notification_type = JSONRPC.NotificationType("textDocument/didOpen", DidOpenTextDocumentParams)
 const textDocument_didClose_notification_type = JSONRPC.NotificationType("textDocument/didClose", DidCloseTextDocumentParams)

src/protocol/protocol.jl

@@ -5,6 +5,7 @@ include("formatting.jl")
 include("hover.jl")
 include("goto.jl")
 include("highlight.jl")
+include("semantic.jl")
 include("signature.jl")
 include("symbols.jl")
 include("features.jl")

src/protocol/semantic.jl

@@ -0,0 +1,152 @@
+# https://microsoft.github.io/language-server-protocol/specifications/specification-3-17/#textDocument_semanticTokens
+const SemanticTokenKind = String
+const SemanticTokenKinds = (
+    # Namespace = "namespace",
+    # Type = "type",
+    # Class = "class",
+    # Enum = "enum",
+    # Interface = "interface",
+	Struct = "struct",
+	TypeParameter = "typeParameter",
+	Parameter = "parameter",
+	Variable = "variable",
+	Property = "property",
+	# EnumMember = "enumMember",
+	# Event = "event",
+	Function = "function",
+	# Method = "method",
+	Macro = "macro",
+	Keyword = "keyword",
+	# Modifier = "modifier",
+	Comment = "comment",
+	String = "string",
+	Number = "number",
+	Regexp = "regexp",
+	Operator = "operator"
+)
+
+const SemanticTokenModifiersKind = String
+const SemanticTokenModifiersKinds = (
+    Declaration = "declaration",
+	Definition = "definition",
+	# Readonly = "readonly",
+	# Static = "static",
+	# Deprecated = "deprecated",
+	# Abstract = "abstract",
+	# Async = "async",
+	Modification = "modification",
+	Documentation = "documentation",
+	DefaultLibrary = "defaultLibrary"
+
+)
+
+
+struct SemanticTokensLegend <: Outbound
+	# /**
+	#  * The token types a server uses.
+	#  */
+	tokenTypes::Vector{String}
+
+	# /**
+	#  * The token modifiers a server uses.
+	#  */
+	tokenModifiers::Vector{String}
+end
+
+const JuliaSemanticTokensLegend = SemanticTokensLegend(
+		collect(values(SemanticTokenKinds)),
+		collect(values(SemanticTokenModifiersKinds))
+)
+
+function semantic_token_encoding(token :: String) :: UInt32
+	for (i, type) in enumerate(JuliaSemanticTokensLegend.tokenTypes)
+		if token == type
+			return i - 1 # -1 to shift to 0-based indexing
+		end
+	end
+end
+# function SemanticTokensLegend() :: SemanticTokensLegend
+# 	SemanticTokensLegend(
+# 	)
+# end
+# const Tok
+
+@dict_readable struct SemanticTokensFullDelta <: Outbound
+    delta::Union{Bool,Missing}
+end
+
+@dict_readable struct SemanticTokensClientCapabilitiesRequests <: Outbound
+    range::Union{Bool,Missing}
+    full::Union{Bool,Missing,SemanticTokensFullDelta}
+
+end
+@dict_readable struct SemanticTokensClientCapabilities <: Outbound
+    dynamicRegistration::Union{Bool,Missing}
+    tokenTypes::Vector{String}
+    tokenModifiers::Vector{String}
+    formats::Vector{String}
+    overlappingTokenSupport::Union{Bool,Missing}
+    multilineTokenSupport::Union{Bool,Missing}
+end
+
+struct SemanticTokensOptions <: Outbound
+	legend::SemanticTokensLegend
+	range::Union{Bool,Missing}
+	full::Union{Bool,SemanticTokensFullDelta,Missing}
+end
+
+struct SemanticTokensRegistrationOptions <: Outbound
+    documentSelector::Union{DocumentSelector,Nothing}
+    # workDoneProgress::Union{Bool,Missing}
+end
+
+@dict_readable struct SemanticTokensParams <: Outbound
+    textDocument::TextDocumentIdentifier
+    # position::Position
+    workDoneToken::Union{Int,String,Missing} # ProgressToken
+    partialResultToken::Union{Int,String,Missing} # ProgressToken
+end
+
+struct SemanticTokens <: Outbound
+    resultId::Union{String,Missing}
+    data::Vector{UInt32}
+end
+
+SemanticTokens(data::Vector{UInt32}) = SemanticTokens(missing, data)
+
+
+
+struct SemanticTokensPartialResult <: Outbound
+    data::Vector{UInt32}
+end
+
+struct SemanticTokensDeltaParams <: Outbound
+	workDoneToken::Union{Int,String,Missing}
+    partialResultToken::Union{Int,String,Missing} # ProgressToken
+    textDocument::TextDocumentIdentifier
+	previousResultId::String
+end
+struct SemanticTokensEdit <: Outbound
+	start::UInt32
+	deleteCount::Int
+	data::Union{Vector{Int},Missing}
+end
+struct SemanticTokensDelta <: Outbound
+	resultId::Union{String,Missing}
+	edits::Vector{SemanticTokensEdit}
+end
+
+struct SemanticTokensDeltaPartialResult <: Outbound
+	edits::Vector{SemanticTokensEdit}
+end
+
+struct SemanticTokensRangeParams <: Outbound
+	workDoneToken::Union{Int,String,Missing}
+    partialResultToken::Union{Int,String,Missing} # ProgressToken
+    textDocument::TextDocumentIdentifier
+	range::Range
+end
+
+struct SemanticTokensWorkspaceClientCapabilities <: Outbound
+	refreshSupport::Union{Bool,Missing}
+end	

src/requests/init.jl

@@ -28,6 +28,7 @@ function ServerCapabilities(client::ClientCapabilities)
     false,
     ExecuteCommandOptions(missing, collect(keys(LSActions))),
     true,
+    SemanticTokensOptions(JuliaSemanticTokensLegend, missing, true),
     true,
     WorkspaceOptions(WorkspaceFoldersOptions(true, true)),
     missing)