implement semantic highlighting

JasperDeSutter · JasperDeSutter · commit ead603bd1233 · 2021-11-21T16:45:56.000+01:00
diff --git a/ra-wasm/src/lib.rs b/ra-wasm/src/lib.rs
@@ -16,14 +16,13 @@ use ide_db::{
         SnippetCap,
     },
 };
+use return_types::*;
 use wasm_bindgen::prelude::*;
-
-mod to_proto;
+pub use wasm_bindgen_rayon::init_thread_pool;
 
 mod return_types;
-use return_types::*;
-
-pub use wasm_bindgen_rayon::init_thread_pool;
+mod semantic_tokens;
+mod to_proto;
 
 #[wasm_bindgen(start)]
 pub fn start() {
@@ -139,17 +138,6 @@ impl WorldState {
 
         let line_index = self.analysis().file_line_index(self.file_id).unwrap();
 
-        let highlights: Vec<_> = self
-            .analysis()
-            .highlight(file_id)
-            .unwrap()
-            .into_iter()
-            .map(|hl| Highlight {
-                tag: Some(hl.highlight.tag.to_string()),
-                range: to_proto::text_range(hl.range, &line_index),
-            })
-            .collect();
-
         let config = DiagnosticsConfig::default();
 
         let diagnostics: Vec<_> = self
@@ -171,7 +159,19 @@ impl WorldState {
             })
             .collect();
 
-        serde_wasm_bindgen::to_value(&UpdateResult { diagnostics, highlights }).unwrap()
+        serde_wasm_bindgen::to_value(&UpdateResult { diagnostics }).unwrap()
+    }
+
+    pub fn semantic_tokens(&self) -> Vec<u32> {
+        log::warn!("semantic_tokens");
+        // let mut builder = SemanticTokensBuilder::new();
+        let line_index = self.analysis().file_line_index(self.file_id).unwrap();
+        let file_text = self.analysis().file_text(self.file_id).unwrap();
+        to_proto::semantic_tokens(
+            &file_text,
+            &line_index,
+            self.analysis().highlight(self.file_id).unwrap(),
+        )
     }
 
     pub fn inlay_hints(&self) -> JsValue {
diff --git a/ra-wasm/src/return_types.rs b/ra-wasm/src/return_types.rs
@@ -62,7 +62,6 @@ pub struct TextEdit {
 #[derive(Serialize)]
 pub struct UpdateResult {
     pub diagnostics: Vec<Diagnostic>,
-    pub highlights: Vec<Highlight>,
 }
 
 #[derive(Serialize)]
diff --git a/ra-wasm/src/semantic_tokens.rs b/ra-wasm/src/semantic_tokens.rs
@@ -0,0 +1,133 @@
+//! Semantic Tokens helpers
+#![allow(non_camel_case_types)]
+
+use std::ops;
+
+use crate::return_types;
+
+#[repr(u8)]
+#[allow(dead_code)]
+pub(crate) enum SemanticTokenType {
+    COMMENT,
+    STRING,
+    KEYWORD,
+    NUMBER,
+    REGEXP,
+    OPERATOR,
+    NAMESPACE,
+    TYPE,
+    STRUCT,
+    CLASS,
+    INTERFACE,
+    ENUM,
+    TYPE_PARAMETER,
+    FUNCTION,
+    MEMBER,
+    MACRO,
+    VARIABLE,
+    PARAMETER,
+    PROPERTY,
+    LABEL,
+    UNSUPPORTED,
+}
+
+macro_rules! define_semantic_token_modifiers {
+    ($($ident:ident),*$(,)?) => {
+        #[derive(PartialEq)]
+        pub(crate) enum SemanticTokenModifier {
+            $($ident),*
+        }
+
+        pub(crate) const SUPPORTED_MODIFIERS: &[SemanticTokenModifier] = &[
+            $(SemanticTokenModifier::$ident),*
+        ];
+    };
+}
+
+define_semantic_token_modifiers![
+    DOCUMENTATION,
+    DECLARATION,
+    DEFINITION,
+    STATIC,
+    ABSTRACT,
+    DEPRECATED,
+    READONLY,
+    DEFAULT_LIBRARY,
+    // custom
+    ASYNC,
+    ATTRIBUTE_MODIFIER,
+    CALLABLE,
+    CONSTANT,
+    CONSUMING,
+    CONTROL_FLOW,
+    CRATE_ROOT,
+    INJECTED,
+    INTRA_DOC_LINK,
+    LIBRARY,
+    MUTABLE,
+    PUBLIC,
+    REFERENCE,
+    TRAIT_MODIFIER,
+    UNSAFE,
+];
+
+#[derive(Default)]
+pub(crate) struct ModifierSet(pub(crate) u32);
+
+impl ops::BitOrAssign<SemanticTokenModifier> for ModifierSet {
+    fn bitor_assign(&mut self, rhs: SemanticTokenModifier) {
+        let idx = SUPPORTED_MODIFIERS.iter().position(|it| it == &rhs).unwrap();
+        self.0 |= 1 << idx;
+    }
+}
+
+/// Tokens are encoded relative to each other.
+///
+/// This is a direct port of <https://github.com/microsoft/vscode-languageserver-node/blob/f425af9de46a0187adb78ec8a46b9b2ce80c5412/server/src/sematicTokens.proposed.ts#L45>
+pub(crate) struct SemanticTokensBuilder {
+    prev_line: u32,
+    prev_char: u32,
+    data: Vec<u32>,
+}
+
+impl SemanticTokensBuilder {
+    pub(crate) fn new() -> Self {
+        SemanticTokensBuilder { prev_line: 0, prev_char: 0, data: Vec::new() }
+    }
+
+    /// Push a new token onto the builder
+    pub(crate) fn push(
+        &mut self,
+        range: return_types::Range,
+        token_index: u32,
+        modifier_bitset: u32,
+    ) {
+        let mut push_line = range.startLineNumber - 1;
+        let mut push_char = range.startColumn - 1;
+
+        if !self.data.is_empty() {
+            push_line -= self.prev_line;
+            if push_line == 0 {
+                push_char -= self.prev_char;
+            }
+        }
+
+        // A token cannot be multiline
+        let token_len = range.endColumn - range.startColumn;
+
+        let token = [push_line, push_char, token_len, token_index, modifier_bitset];
+
+        self.data.extend_from_slice(&token);
+
+        self.prev_line = range.startLineNumber - 1;
+        self.prev_char = range.startColumn - 1;
+    }
+
+    pub(crate) fn build(self) -> Vec<u32> {
+        self.data
+    }
+}
+
+pub(crate) fn type_index(ty: SemanticTokenType) -> u32 {
+    ty as u32
+}
diff --git a/ra-wasm/src/to_proto.rs b/ra-wasm/src/to_proto.rs
@@ -1,5 +1,5 @@
 //! Conversion of rust-analyzer specific types to return_types equivalents.
-use crate::return_types;
+use crate::{return_types, semantic_tokens};
 
 pub(crate) fn text_range(
     range: ide::TextRange,
@@ -229,3 +229,123 @@ fn markdown_string(s: &str) -> return_types::MarkdownString {
 
     return_types::MarkdownString { value: processed_lines.join("\n") }
 }
+
+pub(crate) type SemanticTokens = Vec<u32>;
+
+pub(crate) fn semantic_tokens(
+    text: &str,
+    line_index: &ide::LineIndex,
+    highlights: Vec<ide::HlRange>,
+) -> SemanticTokens {
+    let mut builder = semantic_tokens::SemanticTokensBuilder::new();
+
+    for highlight_range in highlights {
+        if highlight_range.highlight.is_empty() {
+            continue;
+        }
+        let (ty, mods) = semantic_token_type_and_modifiers(highlight_range.highlight);
+        let token_index = semantic_tokens::type_index(ty);
+        let modifier_bitset = mods.0;
+
+        for mut text_range in line_index.lines(highlight_range.range) {
+            if text[text_range].ends_with('\n') {
+                text_range = ide::TextRange::new(
+                    text_range.start(),
+                    text_range.end() - ide::TextSize::of('\n'),
+                );
+            }
+            let range = self::text_range(text_range, line_index);
+
+            builder.push(range, token_index, modifier_bitset);
+        }
+    }
+
+    builder.build()
+}
+
+fn semantic_token_type_and_modifiers(
+    highlight: ide::Highlight,
+) -> (semantic_tokens::SemanticTokenType, semantic_tokens::ModifierSet) {
+    use ide::{HlMod, HlTag, SymbolKind};
+    use semantic_tokens::*;
+    let mut mods = ModifierSet::default();
+    let type_ = match highlight.tag {
+        HlTag::Symbol(symbol) => match symbol {
+            SymbolKind::Module => SemanticTokenType::NAMESPACE,
+            SymbolKind::Impl => SemanticTokenType::TYPE,
+            SymbolKind::Field => SemanticTokenType::PROPERTY,
+            SymbolKind::TypeParam => SemanticTokenType::TYPE_PARAMETER,
+            SymbolKind::ConstParam => SemanticTokenType::PARAMETER,
+            SymbolKind::LifetimeParam => SemanticTokenType::TYPE_PARAMETER,
+            SymbolKind::Label => SemanticTokenType::LABEL,
+            SymbolKind::ValueParam => SemanticTokenType::PARAMETER,
+            SymbolKind::SelfParam => SemanticTokenType::KEYWORD,
+            SymbolKind::Local => SemanticTokenType::VARIABLE,
+            SymbolKind::Function => {
+                if highlight.mods.contains(HlMod::Associated) {
+                    SemanticTokenType::MEMBER
+                } else {
+                    SemanticTokenType::FUNCTION
+                }
+            }
+            SymbolKind::Const => {
+                mods |= SemanticTokenModifier::CONSTANT;
+                mods |= SemanticTokenModifier::STATIC;
+                SemanticTokenType::VARIABLE
+            }
+            SymbolKind::Static => {
+                mods |= SemanticTokenModifier::STATIC;
+                SemanticTokenType::VARIABLE
+            }
+            SymbolKind::Struct => SemanticTokenType::TYPE,
+            SymbolKind::Enum => SemanticTokenType::TYPE,
+            SymbolKind::Variant => SemanticTokenType::MEMBER,
+            SymbolKind::Union => SemanticTokenType::TYPE,
+            SymbolKind::TypeAlias => SemanticTokenType::TYPE,
+            SymbolKind::Trait => SemanticTokenType::INTERFACE,
+            SymbolKind::Macro => SemanticTokenType::MACRO,
+        },
+        HlTag::Attribute => SemanticTokenType::UNSUPPORTED,
+        HlTag::BoolLiteral => SemanticTokenType::NUMBER,
+        HlTag::BuiltinAttr => SemanticTokenType::UNSUPPORTED,
+        HlTag::BuiltinType => SemanticTokenType::TYPE,
+        HlTag::ByteLiteral | HlTag::NumericLiteral => SemanticTokenType::NUMBER,
+        HlTag::CharLiteral => SemanticTokenType::STRING,
+        HlTag::Comment => SemanticTokenType::COMMENT,
+        HlTag::EscapeSequence => SemanticTokenType::NUMBER,
+        HlTag::FormatSpecifier => SemanticTokenType::MACRO,
+        HlTag::Keyword => SemanticTokenType::KEYWORD,
+        HlTag::None => SemanticTokenType::UNSUPPORTED,
+        HlTag::Operator(_op) => SemanticTokenType::OPERATOR,
+        HlTag::StringLiteral => SemanticTokenType::STRING,
+        HlTag::UnresolvedReference => SemanticTokenType::UNSUPPORTED,
+        HlTag::Punctuation(_punct) => SemanticTokenType::OPERATOR,
+    };
+
+    for modifier in highlight.mods.iter() {
+        let modifier = match modifier {
+            HlMod::Associated => continue,
+            HlMod::Async => SemanticTokenModifier::ASYNC,
+            HlMod::Attribute => SemanticTokenModifier::ATTRIBUTE_MODIFIER,
+            HlMod::Callable => SemanticTokenModifier::CALLABLE,
+            HlMod::Consuming => SemanticTokenModifier::CONSUMING,
+            HlMod::ControlFlow => SemanticTokenModifier::CONTROL_FLOW,
+            HlMod::CrateRoot => SemanticTokenModifier::CRATE_ROOT,
+            HlMod::DefaultLibrary => SemanticTokenModifier::DEFAULT_LIBRARY,
+            HlMod::Definition => SemanticTokenModifier::DECLARATION,
+            HlMod::Documentation => SemanticTokenModifier::DOCUMENTATION,
+            HlMod::Injected => SemanticTokenModifier::INJECTED,
+            HlMod::IntraDocLink => SemanticTokenModifier::INTRA_DOC_LINK,
+            HlMod::Library => SemanticTokenModifier::LIBRARY,
+            HlMod::Mutable => SemanticTokenModifier::MUTABLE,
+            HlMod::Public => SemanticTokenModifier::PUBLIC,
+            HlMod::Reference => SemanticTokenModifier::REFERENCE,
+            HlMod::Static => SemanticTokenModifier::STATIC,
+            HlMod::Trait => SemanticTokenModifier::TRAIT_MODIFIER,
+            HlMod::Unsafe => SemanticTokenModifier::UNSAFE,
+        };
+        mods |= modifier;
+    }
+
+    (type_, mods)
+}
diff --git a/www/example-code.rs b/www/example-code.rs
@@ -1,26 +1,47 @@
 use std::ops::Range;
 
-fn gav(x: i32, y: i32) -> i64 {
+unsafe fn gav(x: i32, y: i32) -> i64 {
     (x - y) * (x + y)
 }
 
+#[derive(Debug)]
+struct Gen<G, 'a> {
+    g: &'a G
+}
+
+impl<G, 'a> Gen<G, 'a> {
+    /// Create a new `Gen`
+    /// ```
+    /// let mut gen = Gen::new(&mut something);
+    /// ```
+    fn new(g: &mut G) -> Self {
+        Gen { g }
+    }
+
+    fn do(&mut self) -> () { }
+}
+
 fn main() {
     let num = 5;
     let a = vec![1, 2, 3];
     let b = Some(2);
     let c = None;
     let d = Range { start: 1, end: num };
     let e = 1..num;
-    let f = "sssss".to_string();
+    let mut f = "sssss".to_string();
+    let x = &mut f;
     for a in d {
         for b in e {
-            let c = gav(gav(a, b), a);
+            let c = unsafe { gav(gav(a, b), a) };
             assert_eq!(gav(a, b), a * a - b * b);
         }
     }
+
+    let mut gen = Gen::new(&mut f);
     let f = d
         .reduce(|a, b| {
-            println!("{}", a);
+            gen.do();
+            println!("value: {}", a);
             a * b
         })
         .unwrap();
diff --git a/www/index.js b/www/index.js
diff --git a/www/rust-grammar.js b/www/rust-grammar.js

Original file line number	Diff line number	Diff line change
`@@ -62,7 +62,6 @@ pub struct TextEdit {`
`62`	`62`	`#[derive(Serialize)]`
`63`	`63`	`pub struct UpdateResult {`
`64`	`64`	`pub diagnostics: Vec<Diagnostic>,`
`65`		`- pub highlights: Vec<Highlight>,`
`66`	`65`	`}`
`67`	`66`
`68`	`67`	`#[derive(Serialize)]`