From 9f0cfb7ad2120514ca8ffd21e08e3ddd0bfb34e9 Mon Sep 17 00:00:00 2001 From: kjeremy Date: Fri, 14 Feb 2020 17:56:28 -0500 Subject: Teach the server about Semantic Tokens proposed LSP --- crates/ra_ide/src/lib.rs | 2 +- crates/ra_ide/src/syntax_highlighting.rs | 52 ++++++++-------- crates/rust-analyzer/src/caps.rs | 24 +++++-- crates/rust-analyzer/src/conv.rs | 79 +++++++++++++++++++++-- crates/rust-analyzer/src/lib.rs | 1 + crates/rust-analyzer/src/main_loop.rs | 1 + crates/rust-analyzer/src/main_loop/handlers.rs | 28 ++++++++- crates/rust-analyzer/src/req.rs | 6 +- crates/rust-analyzer/src/semantic_tokens.rs | 86 ++++++++++++++++++++++++++ 9 files changed, 239 insertions(+), 40 deletions(-) create mode 100644 crates/rust-analyzer/src/semantic_tokens.rs diff --git a/crates/ra_ide/src/lib.rs b/crates/ra_ide/src/lib.rs index f86f98be7..82e10bc7e 100644 --- a/crates/ra_ide/src/lib.rs +++ b/crates/ra_ide/src/lib.rs @@ -75,7 +75,7 @@ pub use crate::{ runnables::{Runnable, RunnableKind, TestId}, source_change::{FileSystemEdit, SourceChange, SourceFileEdit}, ssr::SsrError, - syntax_highlighting::HighlightedRange, + syntax_highlighting::{tags, HighlightedRange}, }; pub use hir::Documentation; diff --git a/crates/ra_ide/src/syntax_highlighting.rs b/crates/ra_ide/src/syntax_highlighting.rs index d873f153e..812229b4e 100644 --- a/crates/ra_ide/src/syntax_highlighting.rs +++ b/crates/ra_ide/src/syntax_highlighting.rs @@ -17,32 +17,32 @@ use crate::{ }; pub mod tags { - pub(crate) const FIELD: &str = "field"; - pub(crate) const FUNCTION: &str = "function"; - pub(crate) const MODULE: &str = "module"; - pub(crate) const CONSTANT: &str = "constant"; - pub(crate) const MACRO: &str = "macro"; - - pub(crate) const VARIABLE: &str = "variable"; - pub(crate) const VARIABLE_MUT: &str = "variable.mut"; - - pub(crate) const TYPE: &str = "type"; - pub(crate) const TYPE_BUILTIN: &str = "type.builtin"; - pub(crate) const TYPE_SELF: &str = "type.self"; - pub(crate) const TYPE_PARAM: &str = "type.param"; - pub(crate) const TYPE_LIFETIME: &str = "type.lifetime"; - - pub(crate) const LITERAL_BYTE: &str = "literal.byte"; - pub(crate) const LITERAL_NUMERIC: &str = "literal.numeric"; - pub(crate) const LITERAL_CHAR: &str = "literal.char"; - - pub(crate) const LITERAL_COMMENT: &str = "comment"; - pub(crate) const LITERAL_STRING: &str = "string"; - pub(crate) const LITERAL_ATTRIBUTE: &str = "attribute"; - - pub(crate) const KEYWORD: &str = "keyword"; - pub(crate) const KEYWORD_UNSAFE: &str = "keyword.unsafe"; - pub(crate) const KEYWORD_CONTROL: &str = "keyword.control"; + pub const FIELD: &str = "field"; + pub const FUNCTION: &str = "function"; + pub const MODULE: &str = "module"; + pub const CONSTANT: &str = "constant"; + pub const MACRO: &str = "macro"; + + pub const VARIABLE: &str = "variable"; + pub const VARIABLE_MUT: &str = "variable.mut"; + + pub const TYPE: &str = "type"; + pub const TYPE_BUILTIN: &str = "type.builtin"; + pub const TYPE_SELF: &str = "type.self"; + pub const TYPE_PARAM: &str = "type.param"; + pub const TYPE_LIFETIME: &str = "type.lifetime"; + + pub const LITERAL_BYTE: &str = "literal.byte"; + pub const LITERAL_NUMERIC: &str = "literal.numeric"; + pub const LITERAL_CHAR: &str = "literal.char"; + + pub const LITERAL_COMMENT: &str = "comment"; + pub const LITERAL_STRING: &str = "string"; + pub const LITERAL_ATTRIBUTE: &str = "attribute"; + + pub const KEYWORD: &str = "keyword"; + pub const KEYWORD_UNSAFE: &str = "keyword.unsafe"; + pub const KEYWORD_CONTROL: &str = "keyword.control"; } #[derive(Debug)] diff --git a/crates/rust-analyzer/src/caps.rs b/crates/rust-analyzer/src/caps.rs index c9fd645f1..638987ee8 100644 --- a/crates/rust-analyzer/src/caps.rs +++ b/crates/rust-analyzer/src/caps.rs @@ -1,12 +1,15 @@ //! Advertizes the capabilities of the LSP Server. +use crate::semantic_tokens; + use lsp_types::{ CallHierarchyServerCapability, CodeActionProviderCapability, CodeLensOptions, CompletionOptions, DocumentOnTypeFormattingOptions, FoldingRangeProviderCapability, ImplementationProviderCapability, RenameOptions, RenameProviderCapability, SaveOptions, - SelectionRangeProviderCapability, ServerCapabilities, SignatureHelpOptions, - TextDocumentSyncCapability, TextDocumentSyncKind, TextDocumentSyncOptions, - TypeDefinitionProviderCapability, WorkDoneProgressOptions, + SelectionRangeProviderCapability, SemanticTokensDocumentProvider, SemanticTokensLegend, + SemanticTokensOptions, SemanticTokensServerCapabilities, ServerCapabilities, + SignatureHelpOptions, TextDocumentSyncCapability, TextDocumentSyncKind, + TextDocumentSyncOptions, TypeDefinitionProviderCapability, WorkDoneProgressOptions, }; pub fn server_capabilities() -> ServerCapabilities { @@ -57,7 +60,20 @@ pub fn server_capabilities() -> ServerCapabilities { execute_command_provider: None, workspace: None, call_hierarchy_provider: Some(CallHierarchyServerCapability::Simple(true)), - semantic_tokens_provider: None, + semantic_tokens_provider: Some(SemanticTokensServerCapabilities::SemanticTokensOptions( + SemanticTokensOptions { + legend: SemanticTokensLegend { + token_types: semantic_tokens::supported_token_types().iter().cloned().collect(), + token_modifiers: semantic_tokens::supported_token_modifiers() + .iter() + .cloned() + .collect(), + }, + + document_provider: Some(SemanticTokensDocumentProvider::Bool(true)), + ..SemanticTokensOptions::default() + }, + )), experimental: Default::default(), } } diff --git a/crates/rust-analyzer/src/conv.rs b/crates/rust-analyzer/src/conv.rs index 90ef74056..5fcb46b61 100644 --- a/crates/rust-analyzer/src/conv.rs +++ b/crates/rust-analyzer/src/conv.rs @@ -4,11 +4,12 @@ use lsp_types::{ self, CreateFile, DiagnosticSeverity, DocumentChangeOperation, DocumentChanges, Documentation, Location, LocationLink, MarkupContent, MarkupKind, Position, Range, RenameFile, ResourceOp, - SymbolKind, TextDocumentEdit, TextDocumentIdentifier, TextDocumentItem, - TextDocumentPositionParams, Url, VersionedTextDocumentIdentifier, WorkspaceEdit, + SemanticTokenModifier, SemanticTokenType, SymbolKind, TextDocumentEdit, TextDocumentIdentifier, + TextDocumentItem, TextDocumentPositionParams, Url, VersionedTextDocumentIdentifier, + WorkspaceEdit, }; use ra_ide::{ - translate_offset_with_edit, CompletionItem, CompletionItemKind, FileId, FilePosition, + tags, translate_offset_with_edit, CompletionItem, CompletionItemKind, FileId, FilePosition, FileRange, FileSystemEdit, Fold, FoldKind, InsertTextFormat, LineCol, LineIndex, NavigationTarget, RangeInfo, ReferenceAccess, Severity, SourceChange, SourceFileEdit, }; @@ -16,7 +17,7 @@ use ra_syntax::{SyntaxKind, TextRange, TextUnit}; use ra_text_edit::{AtomTextEdit, TextEdit}; use ra_vfs::LineEndings; -use crate::{req, world::WorldSnapshot, Result}; +use crate::{req, semantic_tokens, world::WorldSnapshot, Result}; pub trait Conv { type Output; @@ -302,6 +303,76 @@ impl ConvWith<&FoldConvCtx<'_>> for Fold { } } +impl Conv for &'static str { + type Output = (SemanticTokenType, Vec); + + fn conv(self) -> (SemanticTokenType, Vec) { + let token_type: SemanticTokenType = match self { + tags::FIELD => SemanticTokenType::MEMBER, + tags::FUNCTION => SemanticTokenType::FUNCTION, + tags::MODULE => SemanticTokenType::NAMESPACE, + tags::CONSTANT => { + return ( + SemanticTokenType::VARIABLE, + vec![SemanticTokenModifier::STATIC, SemanticTokenModifier::READONLY], + ) + } + tags::MACRO => SemanticTokenType::MACRO, + + tags::VARIABLE => { + return (SemanticTokenType::VARIABLE, vec![SemanticTokenModifier::READONLY]) + } + tags::VARIABLE_MUT => SemanticTokenType::VARIABLE, + + tags::TYPE => SemanticTokenType::TYPE, + tags::TYPE_BUILTIN => SemanticTokenType::TYPE, + tags::TYPE_SELF => { + return (SemanticTokenType::TYPE, vec![SemanticTokenModifier::REFERENCE]) + } + tags::TYPE_PARAM => SemanticTokenType::TYPE_PARAMETER, + tags::TYPE_LIFETIME => { + return (SemanticTokenType::LABEL, vec![SemanticTokenModifier::REFERENCE]) + } + + tags::LITERAL_BYTE => SemanticTokenType::NUMBER, + tags::LITERAL_NUMERIC => SemanticTokenType::NUMBER, + tags::LITERAL_CHAR => SemanticTokenType::NUMBER, + + tags::LITERAL_COMMENT => { + return (SemanticTokenType::COMMENT, vec![SemanticTokenModifier::DOCUMENTATION]) + } + + tags::LITERAL_STRING => SemanticTokenType::STRING, + tags::LITERAL_ATTRIBUTE => SemanticTokenType::KEYWORD, + + tags::KEYWORD => SemanticTokenType::KEYWORD, + tags::KEYWORD_UNSAFE => SemanticTokenType::KEYWORD, + tags::KEYWORD_CONTROL => SemanticTokenType::KEYWORD, + unknown => panic!("Unknown semantic token: {}", unknown), + }; + + (token_type, vec![]) + } +} + +impl Conv for (SemanticTokenType, Vec) { + type Output = (u32, u32); + + fn conv(self) -> Self::Output { + let token_index = + semantic_tokens::supported_token_types().iter().position(|it| *it == self.0).unwrap(); + let mut token_modifier_bitset = 0; + for modifier in self.1.iter() { + token_modifier_bitset |= semantic_tokens::supported_token_modifiers() + .iter() + .position(|it| it == modifier) + .unwrap(); + } + + (token_index as u32, token_modifier_bitset as u32) + } +} + impl, CTX> ConvWith for Option { type Output = Option; diff --git a/crates/rust-analyzer/src/lib.rs b/crates/rust-analyzer/src/lib.rs index 0dae30e46..a0f968823 100644 --- a/crates/rust-analyzer/src/lib.rs +++ b/crates/rust-analyzer/src/lib.rs @@ -36,6 +36,7 @@ pub mod req; mod config; mod world; mod diagnostics; +mod semantic_tokens; use serde::de::DeserializeOwned; diff --git a/crates/rust-analyzer/src/main_loop.rs b/crates/rust-analyzer/src/main_loop.rs index 98306986b..6e9e604a6 100644 --- a/crates/rust-analyzer/src/main_loop.rs +++ b/crates/rust-analyzer/src/main_loop.rs @@ -528,6 +528,7 @@ fn on_request( .on::(handlers::handle_call_hierarchy_incoming)? .on::(handlers::handle_call_hierarchy_outgoing)? .on::(handlers::handle_ssr)? + .on::(handlers::handle_semantic_tokens)? .finish(); Ok(()) } diff --git a/crates/rust-analyzer/src/main_loop/handlers.rs b/crates/rust-analyzer/src/main_loop/handlers.rs index bb7bab372..e13e7c95a 100644 --- a/crates/rust-analyzer/src/main_loop/handlers.rs +++ b/crates/rust-analyzer/src/main_loop/handlers.rs @@ -16,8 +16,9 @@ use lsp_types::{ CodeAction, CodeActionOrCommand, CodeActionResponse, CodeLens, Command, CompletionItem, Diagnostic, DocumentFormattingParams, DocumentHighlight, DocumentSymbol, FoldingRange, FoldingRangeParams, Hover, HoverContents, Location, MarkupContent, MarkupKind, Position, - PrepareRenameResponse, Range, RenameParams, SymbolInformation, TextDocumentIdentifier, - TextEdit, WorkspaceEdit, + PrepareRenameResponse, Range, RenameParams, SemanticTokenModifier, SemanticTokenType, + SemanticTokens, SemanticTokensParams, SemanticTokensResult, SymbolInformation, + TextDocumentIdentifier, TextEdit, WorkspaceEdit, }; use ra_ide::{ AssistId, FileId, FilePosition, FileRange, Query, RangeInfo, Runnable, RunnableKind, @@ -38,6 +39,7 @@ use crate::{ diagnostics::DiagnosticTask, from_json, req::{self, Decoration, InlayHint, InlayHintsParams, InlayKind}, + semantic_tokens::SemanticTokensBuilder, world::WorldSnapshot, LspError, Result, }; @@ -1068,3 +1070,25 @@ pub fn handle_call_hierarchy_outgoing( Ok(Some(res)) } + +pub fn handle_semantic_tokens( + world: WorldSnapshot, + params: SemanticTokensParams, +) -> Result> { + let _p = profile("handle_semantic_tokens"); + + let file_id = params.text_document.try_conv_with(&world)?; + let line_index = world.analysis().file_line_index(file_id)?; + + let mut builder = SemanticTokensBuilder::default(); + + for h in world.analysis().highlight(file_id)?.into_iter() { + let type_and_modifiers: (SemanticTokenType, Vec) = h.tag.conv(); + let (token_type, token_modifiers) = type_and_modifiers.conv(); + builder.push(h.range.conv_with(&line_index), token_type, token_modifiers); + } + + let tokens = SemanticTokens { data: builder.build(), ..Default::default() }; + + Ok(Some(tokens.into())) +} diff --git a/crates/rust-analyzer/src/req.rs b/crates/rust-analyzer/src/req.rs index 7ff7f60b3..3734899bc 100644 --- a/crates/rust-analyzer/src/req.rs +++ b/crates/rust-analyzer/src/req.rs @@ -12,9 +12,9 @@ pub use lsp_types::{ DocumentSymbolResponse, FileSystemWatcher, Hover, InitializeResult, MessageType, PartialResultParams, ProgressParams, ProgressParamsValue, ProgressToken, PublishDiagnosticsParams, ReferenceParams, Registration, RegistrationParams, SelectionRange, - SelectionRangeParams, ServerCapabilities, ShowMessageParams, SignatureHelp, SymbolKind, - TextDocumentEdit, TextDocumentPositionParams, TextEdit, WorkDoneProgressParams, WorkspaceEdit, - WorkspaceSymbolParams, + SelectionRangeParams, SemanticTokensParams, SemanticTokensResult, ServerCapabilities, + ShowMessageParams, SignatureHelp, SymbolKind, TextDocumentEdit, TextDocumentPositionParams, + TextEdit, WorkDoneProgressParams, WorkspaceEdit, WorkspaceSymbolParams, }; pub enum AnalyzerStatus {} diff --git a/crates/rust-analyzer/src/semantic_tokens.rs b/crates/rust-analyzer/src/semantic_tokens.rs new file mode 100644 index 000000000..f76605aaa --- /dev/null +++ b/crates/rust-analyzer/src/semantic_tokens.rs @@ -0,0 +1,86 @@ +use lsp_types::{Range, SemanticToken, SemanticTokenModifier, SemanticTokenType}; + +const SUPPORTED_TYPES: &[SemanticTokenType] = &[ + SemanticTokenType::COMMENT, + SemanticTokenType::KEYWORD, + SemanticTokenType::STRING, + SemanticTokenType::NUMBER, + SemanticTokenType::REGEXP, + SemanticTokenType::OPERATOR, + SemanticTokenType::NAMESPACE, + SemanticTokenType::TYPE, + SemanticTokenType::STRUCT, + SemanticTokenType::CLASS, + SemanticTokenType::INTERFACE, + SemanticTokenType::ENUM, + SemanticTokenType::TYPE_PARAMETER, + SemanticTokenType::FUNCTION, + SemanticTokenType::MEMBER, + SemanticTokenType::PROPERTY, + SemanticTokenType::MACRO, + SemanticTokenType::VARIABLE, + SemanticTokenType::PARAMETER, + SemanticTokenType::LABEL, +]; + +const SUPPORTED_MODIFIERS: &[SemanticTokenModifier] = &[ + SemanticTokenModifier::DOCUMENTATION, + SemanticTokenModifier::DECLARATION, + SemanticTokenModifier::DEFINITION, + SemanticTokenModifier::REFERENCE, + SemanticTokenModifier::STATIC, + SemanticTokenModifier::ABSTRACT, + SemanticTokenModifier::DEPRECATED, + SemanticTokenModifier::ASYNC, + SemanticTokenModifier::VOLATILE, + SemanticTokenModifier::READONLY, +]; + +pub(crate) fn supported_token_types() -> &'static [SemanticTokenType] { + SUPPORTED_TYPES +} + +pub(crate) fn supported_token_modifiers() -> &'static [SemanticTokenModifier] { + SUPPORTED_MODIFIERS +} + +#[derive(Default)] +pub(crate) struct SemanticTokensBuilder { + prev_line: u32, + prev_char: u32, + data: Vec, +} + +impl SemanticTokensBuilder { + pub fn push(&mut self, range: Range, token_index: u32, modifier_bitset: u32) { + let mut push_line = range.start.line as u32; + let mut push_char = range.start.character as u32; + + if !self.data.is_empty() { + push_line -= self.prev_line; + if push_line == 0 { + push_char -= self.prev_char; + } + } + + // A token cannot be multiline + let token_len = range.end.character - range.start.character; + + let token = SemanticToken { + delta_line: push_line, + delta_start: push_char, + length: token_len as u32, + token_type: token_index, + token_modifiers_bitset: modifier_bitset, + }; + + self.data.push(token); + + self.prev_line = range.start.line as u32; + self.prev_char = range.start.character as u32; + } + + pub fn build(self) -> Vec { + self.data + } +} -- cgit v1.2.3 From 17ffdf9c27a6bb5cf5d30ad6a677390609fcf861 Mon Sep 17 00:00:00 2001 From: Jeremy Kolb Date: Mon, 24 Feb 2020 21:17:20 -0500 Subject: Add docs --- crates/rust-analyzer/src/semantic_tokens.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/rust-analyzer/src/semantic_tokens.rs b/crates/rust-analyzer/src/semantic_tokens.rs index f76605aaa..ad000a3ce 100644 --- a/crates/rust-analyzer/src/semantic_tokens.rs +++ b/crates/rust-analyzer/src/semantic_tokens.rs @@ -1,3 +1,5 @@ +//! Semantic Tokens helpers + use lsp_types::{Range, SemanticToken, SemanticTokenModifier, SemanticTokenType}; const SUPPORTED_TYPES: &[SemanticTokenType] = &[ @@ -36,14 +38,19 @@ const SUPPORTED_MODIFIERS: &[SemanticTokenModifier] = &[ SemanticTokenModifier::READONLY, ]; +/// Token types that the server supports pub(crate) fn supported_token_types() -> &'static [SemanticTokenType] { SUPPORTED_TYPES } +/// Token modifiers that the server supports pub(crate) fn supported_token_modifiers() -> &'static [SemanticTokenModifier] { SUPPORTED_MODIFIERS } +/// Tokens are encoded relative to each other. +/// +/// This is a direct port of https://github.com/microsoft/vscode-languageserver-node/blob/f425af9de46a0187adb78ec8a46b9b2ce80c5412/server/src/sematicTokens.proposed.ts#L45 #[derive(Default)] pub(crate) struct SemanticTokensBuilder { prev_line: u32, @@ -52,6 +59,7 @@ pub(crate) struct SemanticTokensBuilder { } impl SemanticTokensBuilder { + /// Push a new token onto the builder pub fn push(&mut self, range: Range, token_index: u32, modifier_bitset: u32) { let mut push_line = range.start.line as u32; let mut push_char = range.start.character as u32; -- cgit v1.2.3