From c428b27cd28edf135e7098c7d1f036e2e518c46c Mon Sep 17 00:00:00 2001 From: Caleb Allen Date: Mon, 5 Jul 2021 13:46:55 -0700 Subject: [PATCH] add methods for semantic tokens --- Project.toml | 34 ++--- src/LanguageServer.jl | 3 +- src/Project.toml | 0 src/languageserverinstance.jl | 2 + src/protocol/initialize.jl | 2 + src/protocol/messagedefs.jl | 2 + src/protocol/protocol.jl | 1 + src/protocol/semantic.jl | 152 ++++++++++++++++++++ src/requests/init.jl | 1 + src/requests/semantic.jl | 233 +++++++++++++++++++++++++++++++ src/utilities.jl | 18 +++ test/requests/semantic_tokens.jl | 10 ++ test/runtests.jl | 3 + test/test_communication.jl | 1 + 14 files changed, 444 insertions(+), 18 deletions(-) create mode 100644 src/Project.toml create mode 100644 src/protocol/semantic.jl create mode 100644 src/requests/semantic.jl create mode 100644 test/requests/semantic_tokens.jl diff --git a/Project.toml b/Project.toml index 9c17763f..2e01e153 100644 --- a/Project.toml +++ b/Project.toml @@ -3,36 +3,36 @@ uuid = "2b0e0bc5-e4fd-59b4-8912-456d1b03d8d7" version = "4.0.1-DEV" [deps] -Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" -Tokenize = "0796e94c-ce3b-5d07-9a54-7f471281c624" -REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +CSTParser = "00ebfdb7-1f24-5e51-bd34-a7502290713f" DocumentFormat = "ffa9a821-9c82-50df-894e-fbcef3ed31cd" -StaticLint = "b3cc710f-9c33-5bdb-a03d-a94903873e97" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" JSONRPC = "b9b8584e-8fd3-41f9-ad0c-7255d428e418" -CSTParser = "00ebfdb7-1f24-5e51-bd34-a7502290713f" Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +StaticLint = "b3cc710f-9c33-5bdb-a03d-a94903873e97" SymbolServer = "cf896787-08d5-524d-9de7-132aaa0cb996" +Tokenize = "0796e94c-ce3b-5d07-9a54-7f471281c624" URIParser = "30578b45-9adc-5946-b283-645ec420af67" - -[extras] -Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" -SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" -LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433" -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" +UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] -JSON = "0.20, 0.21" -julia = "1" CSTParser = "3.1" DocumentFormat = "3.2.2" -StaticLint = "8.0" -Tokenize = "0.5.10" +JSON = "0.20, 0.21" JSONRPC = "1.1" +StaticLint = "8.0" SymbolServer = "6, 7.0" +Tokenize = "0.5.10" URIParser = "0.4.1" +julia = "1" + +[extras] +LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433" +SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" +Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] test = ["Test", "Sockets", "LibGit2", "Serialization", "SHA"] diff --git a/src/LanguageServer.jl b/src/LanguageServer.jl index 018bed60..267adeb1 100644 --- a/src/LanguageServer.jl +++ b/src/LanguageServer.jl @@ -2,7 +2,7 @@ module LanguageServer import URIParser using JSON, REPL, CSTParser, DocumentFormat, SymbolServer, StaticLint using CSTParser: EXPR, Tokenize.Tokens, Tokenize.Tokens.kind, headof, parentof, valof -using StaticLint: refof, scopeof, bindingof +using StaticLint: refof, scopeof, bindingof, CSTParser using UUIDs using Base.Docs, Markdown import JSONRPC @@ -30,6 +30,7 @@ include("requests/actions.jl") include("requests/init.jl") include("requests/signatures.jl") include("requests/highlight.jl") +include("requests/semantic.jl") include("utilities.jl") end diff --git a/src/Project.toml b/src/Project.toml new file mode 100644 index 00000000..e69de29b diff --git a/src/languageserverinstance.jl b/src/languageserverinstance.jl index 7cb1525c..4d8da364 100644 --- a/src/languageserverinstance.jl +++ b/src/languageserverinstance.jl @@ -316,6 +316,8 @@ function Base.run(server::LanguageServerInstance) msg_dispatcher[textDocument_rename_request_type] = request_wrapper(textDocument_rename_request, server) msg_dispatcher[textDocument_documentSymbol_request_type] = request_wrapper(textDocument_documentSymbol_request, server) msg_dispatcher[textDocument_documentHighlight_request_type] = request_wrapper(textDocument_documentHighlight_request, server) + msg_dispatcher[textDocument_semanticTokens_request_type] = request_wrapper(textDocument_semanticTokens_request, server) + msg_dispatcher[textDocument_semanticTokens_full_request_type] = request_wrapper(textDocument_semanticTokens_full_request, server) msg_dispatcher[julia_getModuleAt_request_type] = request_wrapper(julia_getModuleAt_request, server) msg_dispatcher[julia_getDocAt_request_type] = request_wrapper(julia_getDocAt_request, server) msg_dispatcher[textDocument_hover_request_type] = request_wrapper(textDocument_hover_request, server) diff --git a/src/protocol/initialize.jl b/src/protocol/initialize.jl index ff00b7ec..b669c79d 100644 --- a/src/protocol/initialize.jl +++ b/src/protocol/initialize.jl @@ -77,6 +77,7 @@ end publishDiagnostics::Union{PublishDiagnosticsClientCapabilities,Missing} foldingRange::Union{FoldingRangeClientCapabilities,Missing} selectionRange::Union{SelectionRangeClientCapabilities,Missing} + semanticTokens::Union{SemanticTokensClientCapabilities,Missing} end @dict_readable struct WindowClientCapabilities <: Outbound @@ -181,6 +182,7 @@ struct ServerCapabilities <: Outbound foldingRangeProvider::Union{Bool,FoldingRangeOptions,FoldingRangeRegistrationOptions,Missing} executeCommandProvider::Union{ExecuteCommandOptions,Missing} selectionRangeProvider::Union{Bool,SelectionRangeOptions,SelectionRangeRegistrationOptions,Missing} + semanticTokensProvider::Union{Bool,SemanticTokensOptions, SemanticTokensRegistrationOptions} workspaceSymbolProvider::Union{Bool,Missing} workspace::Union{WorkspaceOptions,Missing} experimental::Union{Any,Missing} diff --git a/src/protocol/messagedefs.jl b/src/protocol/messagedefs.jl index 12dc8ab6..9c3587e4 100644 --- a/src/protocol/messagedefs.jl +++ b/src/protocol/messagedefs.jl @@ -7,6 +7,8 @@ const textDocument_references_request_type = JSONRPC.RequestType("textDocument/r const textDocument_rename_request_type = JSONRPC.RequestType("textDocument/rename", RenameParams, Union{WorkspaceEdit, Nothing}) const textDocument_documentSymbol_request_type = JSONRPC.RequestType("textDocument/documentSymbol", DocumentSymbolParams, Union{Vector{DocumentSymbol}, Vector{SymbolInformation}, Nothing}) const textDocument_documentHighlight_request_type = JSONRPC.RequestType("textDocument/documentHighlight", DocumentHighlightParams, Union{Vector{DocumentHighlight}, Nothing}) +const textDocument_semanticTokens_request_type = JSONRPC.RequestType("textDocument/semanticTokens", SemanticTokensParams, Union{SemanticTokens, Nothing}) +const textDocument_semanticTokens_full_request_type = JSONRPC.RequestType("textDocument/semanticTokens/full", SemanticTokensParams, Union{SemanticTokens, Nothing}) const textDocument_hover_request_type = JSONRPC.RequestType("textDocument/hover", TextDocumentPositionParams, Union{Hover, Nothing}) const textDocument_didOpen_notification_type = JSONRPC.NotificationType("textDocument/didOpen", DidOpenTextDocumentParams) const textDocument_didClose_notification_type = JSONRPC.NotificationType("textDocument/didClose", DidCloseTextDocumentParams) diff --git a/src/protocol/protocol.jl b/src/protocol/protocol.jl index 5ce30e3c..bbdf3f76 100644 --- a/src/protocol/protocol.jl +++ b/src/protocol/protocol.jl @@ -5,6 +5,7 @@ include("formatting.jl") include("hover.jl") include("goto.jl") include("highlight.jl") +include("semantic.jl") include("signature.jl") include("symbols.jl") include("features.jl") diff --git a/src/protocol/semantic.jl b/src/protocol/semantic.jl new file mode 100644 index 00000000..214fd336 --- /dev/null +++ b/src/protocol/semantic.jl @@ -0,0 +1,152 @@ +# https://microsoft.github.io/language-server-protocol/specifications/specification-3-17/#textDocument_semanticTokens +const SemanticTokenKind = String +const SemanticTokenKinds = ( + # Namespace = "namespace", + # Type = "type", + # Class = "class", + # Enum = "enum", + # Interface = "interface", + Struct = "struct", + TypeParameter = "typeParameter", + Parameter = "parameter", + Variable = "variable", + Property = "property", + # EnumMember = "enumMember", + # Event = "event", + Function = "function", + # Method = "method", + Macro = "macro", + Keyword = "keyword", + # Modifier = "modifier", + Comment = "comment", + String = "string", + Number = "number", + Regexp = "regexp", + Operator = "operator" +) + +const SemanticTokenModifiersKind = String +const SemanticTokenModifiersKinds = ( + Declaration = "declaration", + Definition = "definition", + # Readonly = "readonly", + # Static = "static", + # Deprecated = "deprecated", + # Abstract = "abstract", + # Async = "async", + Modification = "modification", + Documentation = "documentation", + DefaultLibrary = "defaultLibrary" + +) + + +struct SemanticTokensLegend <: Outbound + # /** + # * The token types a server uses. + # */ + tokenTypes::Vector{String} + + # /** + # * The token modifiers a server uses. + # */ + tokenModifiers::Vector{String} +end + +const JuliaSemanticTokensLegend = SemanticTokensLegend( + collect(values(SemanticTokenKinds)), + collect(values(SemanticTokenModifiersKinds)) +) + +function semantic_token_encoding(token :: String) :: UInt32 + for (i, type) in enumerate(JuliaSemanticTokensLegend.tokenTypes) + if token == type + return i - 1 # -1 to shift to 0-based indexing + end + end +end +# function SemanticTokensLegend() :: SemanticTokensLegend +# SemanticTokensLegend( +# ) +# end +# const Tok + +@dict_readable struct SemanticTokensFullDelta <: Outbound + delta::Union{Bool,Missing} +end + +@dict_readable struct SemanticTokensClientCapabilitiesRequests <: Outbound + range::Union{Bool,Missing} + full::Union{Bool,Missing,SemanticTokensFullDelta} + +end +@dict_readable struct SemanticTokensClientCapabilities <: Outbound + dynamicRegistration::Union{Bool,Missing} + tokenTypes::Vector{String} + tokenModifiers::Vector{String} + formats::Vector{String} + overlappingTokenSupport::Union{Bool,Missing} + multilineTokenSupport::Union{Bool,Missing} +end + +struct SemanticTokensOptions <: Outbound + legend::SemanticTokensLegend + range::Union{Bool,Missing} + full::Union{Bool,SemanticTokensFullDelta,Missing} +end + +struct SemanticTokensRegistrationOptions <: Outbound + documentSelector::Union{DocumentSelector,Nothing} + # workDoneProgress::Union{Bool,Missing} +end + +@dict_readable struct SemanticTokensParams <: Outbound + textDocument::TextDocumentIdentifier + # position::Position + workDoneToken::Union{Int,String,Missing} # ProgressToken + partialResultToken::Union{Int,String,Missing} # ProgressToken +end + +struct SemanticTokens <: Outbound + resultId::Union{String,Missing} + data::Vector{UInt32} +end + +SemanticTokens(data::Vector{UInt32}) = SemanticTokens(missing, data) + + + +struct SemanticTokensPartialResult <: Outbound + data::Vector{UInt32} +end + +struct SemanticTokensDeltaParams <: Outbound + workDoneToken::Union{Int,String,Missing} + partialResultToken::Union{Int,String,Missing} # ProgressToken + textDocument::TextDocumentIdentifier + previousResultId::String +end +struct SemanticTokensEdit <: Outbound + start::UInt32 + deleteCount::Int + data::Union{Vector{Int},Missing} +end +struct SemanticTokensDelta <: Outbound + resultId::Union{String,Missing} + edits::Vector{SemanticTokensEdit} +end + +struct SemanticTokensDeltaPartialResult <: Outbound + edits::Vector{SemanticTokensEdit} +end + +struct SemanticTokensRangeParams <: Outbound + workDoneToken::Union{Int,String,Missing} + partialResultToken::Union{Int,String,Missing} # ProgressToken + textDocument::TextDocumentIdentifier + range::Range +end + +struct SemanticTokensWorkspaceClientCapabilities <: Outbound + refreshSupport::Union{Bool,Missing} +end \ No newline at end of file diff --git a/src/requests/init.jl b/src/requests/init.jl index 77b9698f..0113626f 100644 --- a/src/requests/init.jl +++ b/src/requests/init.jl @@ -25,6 +25,7 @@ const serverCapabilities = ServerCapabilities( false, ExecuteCommandOptions(missing, collect(keys(LSActions))), true, + SemanticTokensOptions(JuliaSemanticTokensLegend, missing, true), true, WorkspaceOptions(WorkspaceFoldersOptions(true, true)), missing) diff --git a/src/requests/semantic.jl b/src/requests/semantic.jl new file mode 100644 index 00000000..e1bfe616 --- /dev/null +++ b/src/requests/semantic.jl @@ -0,0 +1,233 @@ +# import Iterators.flatten +function textDocument_semanticTokens_request(params::SemanticTokensParams, server::LanguageServerInstance, conn) + doc = getdocument(server, URI2(params.textDocument.uri)) + offset = get_offset(doc, params.position) + identifier = get_identifier(getcst(doc), offset) + identifier !== nothing || return nothing + highlights = DocumentHighlight[] + for_each_ref(identifier) do ref, doc1, o + if doc1._uri == doc._uri + kind = StaticLint.hasbinding(ref) ? DocumentHighlightKinds.Write : DocumentHighlightKinds.Read + push!(highlights, DocumentHighlight(Range(doc, o .+ (0:ref.span)), kind)) + end + end + return isempty(highlights) ? nothing : highlights +end + +struct SemanticToken + # token line number, relative to the previous token + deltaLine::UInt32 + # token start character, relative to the previous token + # (relative to 0 or the previous token’s start if they are on the same line) + deltaStart::UInt32 + # the length of the token. + length::UInt32 + # will be looked up in SemanticTokensLegend.tokenTypes + tokenType::UInt32 + # each set bit will be looked up in SemanticTokensLegend.tokenModifiers + tokenModifiers::UInt32 +end + +function SemanticToken(deltaLine::UInt32, + deltaStart::UInt32, + length::UInt32, + tokenType::String, + tokenModifiers::String) + # TODO look up int encodings for tokenType and tokenModifiers + SemanticToken( + deltaLine, + deltaStart, + length, + semantic_token_encoding(tokenType), + 0 # TODO + ) +end + +# function SemanticToken(ex::EXPR) + +# end + + +function SemanticTokens(tokens::Vector{SemanticToken}) :: SemanticTokens + token_vectors = map(tokens) do token + # token_index = i - 1 + [ + token.deltaLine, + token.deltaStart, + token.length, + token.tokenType, + token.tokenModifiers + ] + end + SemanticTokens(Iterators.flatten(token_vectors) |> collect) +end + +function textDocument_semanticTokens_full_request(params::SemanticTokensParams, + server::LanguageServerInstance, conn) :: Union{SemanticTokens,Nothing} + uri = params.textDocument.uri + doc = getdocument(server, URI2(uri)) + # return nothing + ts = collect_semantic_tokens(getcst(doc), doc) + return SemanticTokens(ts) + # return collect_document_symbols(getcst(doc), server, doc) + + # doc = getdocument(server, URI2(params.textDocument.uri)) + # offset = get_offset(doc, params.position) + # identifier = get_identifier(getcst(doc), offset) + # identifier !== nothing || return nothing + # highlights = DocumentHighlight[] + # for_each_ref(identifier) do ref, doc1, o + # if doc1._uri == doc._uri + # kind = StaticLint.hasbinding(ref) ? DocumentHighlightKinds.Write : DocumentHighlightKinds.Read + # push!(highlights, DocumentHighlight(Range(doc, o .+ (0:ref.span)), kind)) + # end + # end + # return isempty(highlights) ? nothing : highlights +end +# import CSTParser: +""" + +parse applies these types + Parser.SyntaxNode + Parser.EXPR + Parser.INSTANCE + Parser.HEAD{K} + Parser.IDENTIFIER + Parser.KEYWORD{K} + Parser.LITERAL{K} + Parser.OPERATOR{P,K,dot} + Parser.PUNCTUATION + Parser.QUOTENODE + +┌ Info: 1:60 file( new scope lint ) +│ 1:60 function( Binding(main:: (1 refs)) new scope) +│ 1:8 call( ) +│ 1:4 main * +│ 9:48 block( ) +│ 9:28 1:2 OP: =( ) +│ 9:10 s Binding(s:: (3 refs)) * +│ 11:26 STRING: hello world! +│ 29:40 call( ) +│ 29:35 println * +│ 36:36 s * +│ 41:48 macrocall( ) +│ 41:46 @show * +│ 47:46 NOTHING: nothing +└ 47:48 s * +""" +function collect_semantic_tokens(expr::EXPR, doc :: Document, pos=0):: Vector{SemanticToken} + tokens = SemanticToken[] + + offset = pos + for ex in expr + # if isempty(ex) + # leaf of parse tree + + # C.isidentifier(ex) + kind = semantic_token_kind(ex) + if kind !== nothing + @info "Expression" ex kind + # identifier, identifier_pos = get_identifier_pos(ex, 0, 0) + name = C.get_name(ex) + name_offset = 0 + # get the offset of the name expr + if name !== nothing + found = false + for x in ex + if x == name + found = true + break + end + name_offset += x.fullspan + end + if !found + name_offset = -1 + end + end + @info "name" name name_offset + line, char = get_position_at(doc, offset) + # add this token + token = SemanticToken( + line, + char, + ex.span, + semantic_token_encoding(kind), + 0 + ) + push!(tokens, token) + end + if !isempty(ex) + # there are more nodes on this tree + sub_tokens = collect_semantic_tokens(ex, doc, offset) + push!(tokens, sub_tokens...) + end + offset += ex.fullspan + end + return tokens + # return collect_semantic_tokens(ex) + # @info get_toks(doc, 0) +end + +""" +Get the semantic token kind for `expr`, which is assumed to be an identifier +""" +function semantic_token_kind(expr::EXPR) :: Union{String, Nothing} + # C.isidentifier(expr) || return nothing + return if C.defines_function(expr)# || C.is_func_call(expr) + SemanticTokenKinds.Function + elseif C.defines_struct(expr) + SemanticTokenKinds.Struct + elseif C.defines_macro(expr) + SemanticTokenKinds.Macro + # elseif C.isoperator(expr) + # SemanticTokenKinds.Operator + end +end +const C = CSTParser +# function collect_semantic_tokens(ex::EXPR, pos=0) :: Vector{SemanticToken} +# tokens = SemanticToken[] + +# @info "expr.head = $(ex.head)" +# token_kind = if C.defines_function(ex) || C.is_func_call(ex) +# SemanticTokenKinds.Function +# elseif C.defines_struct(ex) +# SemanticTokenKinds.Struct +# elseif C.defines_macro(ex) +# SemanticTokenKinds.Macro +# elseif C.isoperator(ex) +# SemanticTokenKinds.Operator +# end +# @info token_kind + +# if token_kind !== nothing +# token = SemanticToken( +# 0, +# pos, +# ex.span, +# semantic_token_encoding(token_kind), +# 0 +# ) +# push!(tokens, token) +# end +# pos = pos + span(tokens) + +# if ex.args !== nothing + # sub_tokens = map(ex.args) do arg + # collect_semantic_tokens(arg, pos) + # end |> Iterators.flatten |> collect + # pos = pos + span(sub_tokens) + # push!(tokens, sub_tokens...) +# end + +# return tokens +# end + + +span(token :: SemanticToken) = token.length +function span(tokens :: Vector{SemanticToken}) + lengths = map(tokens) do token + token.length + end + + reduce(+, lengths) +end diff --git a/src/utilities.jl b/src/utilities.jl index 410d6f71..8d6a92bd 100644 --- a/src/utilities.jl +++ b/src/utilities.jl @@ -322,6 +322,24 @@ function get_identifier(x, offset, pos=0) end end +function get_identifier_pos(x, offset, pos=0) + if pos > offset + return nothing + end + if length(x) > 0 + for a in x + if pos <= offset <= (pos + a.span) + return get_identifier_pos(a, offset, pos) + end + pos += a.fullspan + end + elseif headof(x) === :IDENTIFIER && (pos <= offset <= (pos + x.span)) || pos == 0 + return (x, pos) + end +end + + + if VERSION < v"1.1" || Sys.iswindows() && VERSION < v"1.3" _splitdir_nodrive(path::String) = _splitdir_nodrive("", path) diff --git a/test/requests/semantic_tokens.jl b/test/requests/semantic_tokens.jl new file mode 100644 index 00000000..50dbaa64 --- /dev/null +++ b/test/requests/semantic_tokens.jl @@ -0,0 +1,10 @@ +semantic_token_test() = LanguageServer.textDocument_semanticTokens_full_request(LanguageServer.SemanticTokensParams(LanguageServer.TextDocumentIdentifier("testdoc"), missing, missing), server, server.jr_endpoint) + +@testset "function calls" begin + settestdoc(""" + function hello() + println("hello world") + end + """) + @test semantic_token_test() == SemanticToken(0, 9, 5, SemanticTokenKinds.Function) +end \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index fc38356a..6ebef66a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -58,6 +58,9 @@ end @testset "textdocument" begin include("requests/textdocument.jl") end + @testset "semantic tokens" begin + include("requests/semantic_tokens.jl") + end @testset "misc" begin include("requests/misc.jl") end diff --git a/test/test_communication.jl b/test/test_communication.jl index c39cea27..d37ed1e2 100644 --- a/test/test_communication.jl +++ b/test/test_communication.jl @@ -38,6 +38,7 @@ init_request = LanguageServer.InitializeParams( missing, # PublishDiagnosticsClientCapabilities(), missing, # FoldingRangeClientCapabilities(), missing, # SelectionRangeClientCapabilities() + missing, ), missing, missing