aboutsummaryrefslogtreecommitdiff
path: root/crates
diff options
context:
space:
mode:
authorbors[bot] <26634292+bors[bot]@users.noreply.github.com>2020-02-25 10:02:21 +0000
committerGitHub <[email protected]>2020-02-25 10:02:21 +0000
commit1fe48a0115c24240f5a3e1b329e642f18e2715d6 (patch)
tree88c3290bced7671039b4765a5f599a019aa3c6e6 /crates
parent5e6f4ca6905e02a5fa34ef292fa2abefcd982222 (diff)
parent17ffdf9c27a6bb5cf5d30ad6a677390609fcf861 (diff)
Merge #3159
3159: Server-side Semantic Tokens r=matklad a=kjeremy Takes the output of `syntax_highlighting` and converts it to the proposed semantic tokens API (so far only `textDocument/semanticTokens`. There's a lot of cool stuff we could do with this and the "Inspect Editor Tokens and Scopes" vscode command (pic below) is a cool way to see what has tokens and what doesn't. Incredibly hacky and could panic due to unwraps, `panic!` etc. To use: run with `code-insiders --enable-proposed-api matklad.rust-analyzer`. If you try to run this without `--enable-proposed-api` it will crash. ![image](https://user-images.githubusercontent.com/4325700/74595603-7c66cf00-5011-11ea-9593-312663f04fc1.png) @matklad I'm mostly looking for design feedback. Co-authored-by: kjeremy <[email protected]> Co-authored-by: Jeremy Kolb <[email protected]>
Diffstat (limited to 'crates')
-rw-r--r--crates/ra_ide/src/lib.rs2
-rw-r--r--crates/ra_ide/src/syntax_highlighting.rs52
-rw-r--r--crates/rust-analyzer/src/caps.rs24
-rw-r--r--crates/rust-analyzer/src/conv.rs79
-rw-r--r--crates/rust-analyzer/src/lib.rs1
-rw-r--r--crates/rust-analyzer/src/main_loop.rs1
-rw-r--r--crates/rust-analyzer/src/main_loop/handlers.rs28
-rw-r--r--crates/rust-analyzer/src/req.rs6
-rw-r--r--crates/rust-analyzer/src/semantic_tokens.rs94
9 files changed, 247 insertions, 40 deletions
diff --git a/crates/ra_ide/src/lib.rs b/crates/ra_ide/src/lib.rs
index f86f98be7..82e10bc7e 100644
--- a/crates/ra_ide/src/lib.rs
+++ b/crates/ra_ide/src/lib.rs
@@ -75,7 +75,7 @@ pub use crate::{
75 runnables::{Runnable, RunnableKind, TestId}, 75 runnables::{Runnable, RunnableKind, TestId},
76 source_change::{FileSystemEdit, SourceChange, SourceFileEdit}, 76 source_change::{FileSystemEdit, SourceChange, SourceFileEdit},
77 ssr::SsrError, 77 ssr::SsrError,
78 syntax_highlighting::HighlightedRange, 78 syntax_highlighting::{tags, HighlightedRange},
79}; 79};
80 80
81pub use hir::Documentation; 81pub use hir::Documentation;
diff --git a/crates/ra_ide/src/syntax_highlighting.rs b/crates/ra_ide/src/syntax_highlighting.rs
index d873f153e..812229b4e 100644
--- a/crates/ra_ide/src/syntax_highlighting.rs
+++ b/crates/ra_ide/src/syntax_highlighting.rs
@@ -17,32 +17,32 @@ use crate::{
17}; 17};
18 18
19pub mod tags { 19pub mod tags {
20 pub(crate) const FIELD: &str = "field"; 20 pub const FIELD: &str = "field";
21 pub(crate) const FUNCTION: &str = "function"; 21 pub const FUNCTION: &str = "function";
22 pub(crate) const MODULE: &str = "module"; 22 pub const MODULE: &str = "module";
23 pub(crate) const CONSTANT: &str = "constant"; 23 pub const CONSTANT: &str = "constant";
24 pub(crate) const MACRO: &str = "macro"; 24 pub const MACRO: &str = "macro";
25 25
26 pub(crate) const VARIABLE: &str = "variable"; 26 pub const VARIABLE: &str = "variable";
27 pub(crate) const VARIABLE_MUT: &str = "variable.mut"; 27 pub const VARIABLE_MUT: &str = "variable.mut";
28 28
29 pub(crate) const TYPE: &str = "type"; 29 pub const TYPE: &str = "type";
30 pub(crate) const TYPE_BUILTIN: &str = "type.builtin"; 30 pub const TYPE_BUILTIN: &str = "type.builtin";
31 pub(crate) const TYPE_SELF: &str = "type.self"; 31 pub const TYPE_SELF: &str = "type.self";
32 pub(crate) const TYPE_PARAM: &str = "type.param"; 32 pub const TYPE_PARAM: &str = "type.param";
33 pub(crate) const TYPE_LIFETIME: &str = "type.lifetime"; 33 pub const TYPE_LIFETIME: &str = "type.lifetime";
34 34
35 pub(crate) const LITERAL_BYTE: &str = "literal.byte"; 35 pub const LITERAL_BYTE: &str = "literal.byte";
36 pub(crate) const LITERAL_NUMERIC: &str = "literal.numeric"; 36 pub const LITERAL_NUMERIC: &str = "literal.numeric";
37 pub(crate) const LITERAL_CHAR: &str = "literal.char"; 37 pub const LITERAL_CHAR: &str = "literal.char";
38 38
39 pub(crate) const LITERAL_COMMENT: &str = "comment"; 39 pub const LITERAL_COMMENT: &str = "comment";
40 pub(crate) const LITERAL_STRING: &str = "string"; 40 pub const LITERAL_STRING: &str = "string";
41 pub(crate) const LITERAL_ATTRIBUTE: &str = "attribute"; 41 pub const LITERAL_ATTRIBUTE: &str = "attribute";
42 42
43 pub(crate) const KEYWORD: &str = "keyword"; 43 pub const KEYWORD: &str = "keyword";
44 pub(crate) const KEYWORD_UNSAFE: &str = "keyword.unsafe"; 44 pub const KEYWORD_UNSAFE: &str = "keyword.unsafe";
45 pub(crate) const KEYWORD_CONTROL: &str = "keyword.control"; 45 pub const KEYWORD_CONTROL: &str = "keyword.control";
46} 46}
47 47
48#[derive(Debug)] 48#[derive(Debug)]
diff --git a/crates/rust-analyzer/src/caps.rs b/crates/rust-analyzer/src/caps.rs
index c9fd645f1..638987ee8 100644
--- a/crates/rust-analyzer/src/caps.rs
+++ b/crates/rust-analyzer/src/caps.rs
@@ -1,12 +1,15 @@
1//! Advertizes the capabilities of the LSP Server. 1//! Advertizes the capabilities of the LSP Server.
2 2
3use crate::semantic_tokens;
4
3use lsp_types::{ 5use lsp_types::{
4 CallHierarchyServerCapability, CodeActionProviderCapability, CodeLensOptions, 6 CallHierarchyServerCapability, CodeActionProviderCapability, CodeLensOptions,
5 CompletionOptions, DocumentOnTypeFormattingOptions, FoldingRangeProviderCapability, 7 CompletionOptions, DocumentOnTypeFormattingOptions, FoldingRangeProviderCapability,
6 ImplementationProviderCapability, RenameOptions, RenameProviderCapability, SaveOptions, 8 ImplementationProviderCapability, RenameOptions, RenameProviderCapability, SaveOptions,
7 SelectionRangeProviderCapability, ServerCapabilities, SignatureHelpOptions, 9 SelectionRangeProviderCapability, SemanticTokensDocumentProvider, SemanticTokensLegend,
8 TextDocumentSyncCapability, TextDocumentSyncKind, TextDocumentSyncOptions, 10 SemanticTokensOptions, SemanticTokensServerCapabilities, ServerCapabilities,
9 TypeDefinitionProviderCapability, WorkDoneProgressOptions, 11 SignatureHelpOptions, TextDocumentSyncCapability, TextDocumentSyncKind,
12 TextDocumentSyncOptions, TypeDefinitionProviderCapability, WorkDoneProgressOptions,
10}; 13};
11 14
12pub fn server_capabilities() -> ServerCapabilities { 15pub fn server_capabilities() -> ServerCapabilities {
@@ -57,7 +60,20 @@ pub fn server_capabilities() -> ServerCapabilities {
57 execute_command_provider: None, 60 execute_command_provider: None,
58 workspace: None, 61 workspace: None,
59 call_hierarchy_provider: Some(CallHierarchyServerCapability::Simple(true)), 62 call_hierarchy_provider: Some(CallHierarchyServerCapability::Simple(true)),
60 semantic_tokens_provider: None, 63 semantic_tokens_provider: Some(SemanticTokensServerCapabilities::SemanticTokensOptions(
64 SemanticTokensOptions {
65 legend: SemanticTokensLegend {
66 token_types: semantic_tokens::supported_token_types().iter().cloned().collect(),
67 token_modifiers: semantic_tokens::supported_token_modifiers()
68 .iter()
69 .cloned()
70 .collect(),
71 },
72
73 document_provider: Some(SemanticTokensDocumentProvider::Bool(true)),
74 ..SemanticTokensOptions::default()
75 },
76 )),
61 experimental: Default::default(), 77 experimental: Default::default(),
62 } 78 }
63} 79}
diff --git a/crates/rust-analyzer/src/conv.rs b/crates/rust-analyzer/src/conv.rs
index 90ef74056..5fcb46b61 100644
--- a/crates/rust-analyzer/src/conv.rs
+++ b/crates/rust-analyzer/src/conv.rs
@@ -4,11 +4,12 @@
4use lsp_types::{ 4use lsp_types::{
5 self, CreateFile, DiagnosticSeverity, DocumentChangeOperation, DocumentChanges, Documentation, 5 self, CreateFile, DiagnosticSeverity, DocumentChangeOperation, DocumentChanges, Documentation,
6 Location, LocationLink, MarkupContent, MarkupKind, Position, Range, RenameFile, ResourceOp, 6 Location, LocationLink, MarkupContent, MarkupKind, Position, Range, RenameFile, ResourceOp,
7 SymbolKind, TextDocumentEdit, TextDocumentIdentifier, TextDocumentItem, 7 SemanticTokenModifier, SemanticTokenType, SymbolKind, TextDocumentEdit, TextDocumentIdentifier,
8 TextDocumentPositionParams, Url, VersionedTextDocumentIdentifier, WorkspaceEdit, 8 TextDocumentItem, TextDocumentPositionParams, Url, VersionedTextDocumentIdentifier,
9 WorkspaceEdit,
9}; 10};
10use ra_ide::{ 11use ra_ide::{
11 translate_offset_with_edit, CompletionItem, CompletionItemKind, FileId, FilePosition, 12 tags, translate_offset_with_edit, CompletionItem, CompletionItemKind, FileId, FilePosition,
12 FileRange, FileSystemEdit, Fold, FoldKind, InsertTextFormat, LineCol, LineIndex, 13 FileRange, FileSystemEdit, Fold, FoldKind, InsertTextFormat, LineCol, LineIndex,
13 NavigationTarget, RangeInfo, ReferenceAccess, Severity, SourceChange, SourceFileEdit, 14 NavigationTarget, RangeInfo, ReferenceAccess, Severity, SourceChange, SourceFileEdit,
14}; 15};
@@ -16,7 +17,7 @@ use ra_syntax::{SyntaxKind, TextRange, TextUnit};
16use ra_text_edit::{AtomTextEdit, TextEdit}; 17use ra_text_edit::{AtomTextEdit, TextEdit};
17use ra_vfs::LineEndings; 18use ra_vfs::LineEndings;
18 19
19use crate::{req, world::WorldSnapshot, Result}; 20use crate::{req, semantic_tokens, world::WorldSnapshot, Result};
20 21
21pub trait Conv { 22pub trait Conv {
22 type Output; 23 type Output;
@@ -302,6 +303,76 @@ impl ConvWith<&FoldConvCtx<'_>> for Fold {
302 } 303 }
303} 304}
304 305
306impl Conv for &'static str {
307 type Output = (SemanticTokenType, Vec<SemanticTokenModifier>);
308
309 fn conv(self) -> (SemanticTokenType, Vec<SemanticTokenModifier>) {
310 let token_type: SemanticTokenType = match self {
311 tags::FIELD => SemanticTokenType::MEMBER,
312 tags::FUNCTION => SemanticTokenType::FUNCTION,
313 tags::MODULE => SemanticTokenType::NAMESPACE,
314 tags::CONSTANT => {
315 return (
316 SemanticTokenType::VARIABLE,
317 vec![SemanticTokenModifier::STATIC, SemanticTokenModifier::READONLY],
318 )
319 }
320 tags::MACRO => SemanticTokenType::MACRO,
321
322 tags::VARIABLE => {
323 return (SemanticTokenType::VARIABLE, vec![SemanticTokenModifier::READONLY])
324 }
325 tags::VARIABLE_MUT => SemanticTokenType::VARIABLE,
326
327 tags::TYPE => SemanticTokenType::TYPE,
328 tags::TYPE_BUILTIN => SemanticTokenType::TYPE,
329 tags::TYPE_SELF => {
330 return (SemanticTokenType::TYPE, vec![SemanticTokenModifier::REFERENCE])
331 }
332 tags::TYPE_PARAM => SemanticTokenType::TYPE_PARAMETER,
333 tags::TYPE_LIFETIME => {
334 return (SemanticTokenType::LABEL, vec![SemanticTokenModifier::REFERENCE])
335 }
336
337 tags::LITERAL_BYTE => SemanticTokenType::NUMBER,
338 tags::LITERAL_NUMERIC => SemanticTokenType::NUMBER,
339 tags::LITERAL_CHAR => SemanticTokenType::NUMBER,
340
341 tags::LITERAL_COMMENT => {
342 return (SemanticTokenType::COMMENT, vec![SemanticTokenModifier::DOCUMENTATION])
343 }
344
345 tags::LITERAL_STRING => SemanticTokenType::STRING,
346 tags::LITERAL_ATTRIBUTE => SemanticTokenType::KEYWORD,
347
348 tags::KEYWORD => SemanticTokenType::KEYWORD,
349 tags::KEYWORD_UNSAFE => SemanticTokenType::KEYWORD,
350 tags::KEYWORD_CONTROL => SemanticTokenType::KEYWORD,
351 unknown => panic!("Unknown semantic token: {}", unknown),
352 };
353
354 (token_type, vec![])
355 }
356}
357
358impl Conv for (SemanticTokenType, Vec<SemanticTokenModifier>) {
359 type Output = (u32, u32);
360
361 fn conv(self) -> Self::Output {
362 let token_index =
363 semantic_tokens::supported_token_types().iter().position(|it| *it == self.0).unwrap();
364 let mut token_modifier_bitset = 0;
365 for modifier in self.1.iter() {
366 token_modifier_bitset |= semantic_tokens::supported_token_modifiers()
367 .iter()
368 .position(|it| it == modifier)
369 .unwrap();
370 }
371
372 (token_index as u32, token_modifier_bitset as u32)
373 }
374}
375
305impl<T: ConvWith<CTX>, CTX> ConvWith<CTX> for Option<T> { 376impl<T: ConvWith<CTX>, CTX> ConvWith<CTX> for Option<T> {
306 type Output = Option<T::Output>; 377 type Output = Option<T::Output>;
307 378
diff --git a/crates/rust-analyzer/src/lib.rs b/crates/rust-analyzer/src/lib.rs
index 0dae30e46..a0f968823 100644
--- a/crates/rust-analyzer/src/lib.rs
+++ b/crates/rust-analyzer/src/lib.rs
@@ -36,6 +36,7 @@ pub mod req;
36mod config; 36mod config;
37mod world; 37mod world;
38mod diagnostics; 38mod diagnostics;
39mod semantic_tokens;
39 40
40use serde::de::DeserializeOwned; 41use serde::de::DeserializeOwned;
41 42
diff --git a/crates/rust-analyzer/src/main_loop.rs b/crates/rust-analyzer/src/main_loop.rs
index 98306986b..6e9e604a6 100644
--- a/crates/rust-analyzer/src/main_loop.rs
+++ b/crates/rust-analyzer/src/main_loop.rs
@@ -528,6 +528,7 @@ fn on_request(
528 .on::<req::CallHierarchyIncomingCalls>(handlers::handle_call_hierarchy_incoming)? 528 .on::<req::CallHierarchyIncomingCalls>(handlers::handle_call_hierarchy_incoming)?
529 .on::<req::CallHierarchyOutgoingCalls>(handlers::handle_call_hierarchy_outgoing)? 529 .on::<req::CallHierarchyOutgoingCalls>(handlers::handle_call_hierarchy_outgoing)?
530 .on::<req::Ssr>(handlers::handle_ssr)? 530 .on::<req::Ssr>(handlers::handle_ssr)?
531 .on::<req::SemanticTokensRequest>(handlers::handle_semantic_tokens)?
531 .finish(); 532 .finish();
532 Ok(()) 533 Ok(())
533} 534}
diff --git a/crates/rust-analyzer/src/main_loop/handlers.rs b/crates/rust-analyzer/src/main_loop/handlers.rs
index bb7bab372..e13e7c95a 100644
--- a/crates/rust-analyzer/src/main_loop/handlers.rs
+++ b/crates/rust-analyzer/src/main_loop/handlers.rs
@@ -16,8 +16,9 @@ use lsp_types::{
16 CodeAction, CodeActionOrCommand, CodeActionResponse, CodeLens, Command, CompletionItem, 16 CodeAction, CodeActionOrCommand, CodeActionResponse, CodeLens, Command, CompletionItem,
17 Diagnostic, DocumentFormattingParams, DocumentHighlight, DocumentSymbol, FoldingRange, 17 Diagnostic, DocumentFormattingParams, DocumentHighlight, DocumentSymbol, FoldingRange,
18 FoldingRangeParams, Hover, HoverContents, Location, MarkupContent, MarkupKind, Position, 18 FoldingRangeParams, Hover, HoverContents, Location, MarkupContent, MarkupKind, Position,
19 PrepareRenameResponse, Range, RenameParams, SymbolInformation, TextDocumentIdentifier, 19 PrepareRenameResponse, Range, RenameParams, SemanticTokenModifier, SemanticTokenType,
20 TextEdit, WorkspaceEdit, 20 SemanticTokens, SemanticTokensParams, SemanticTokensResult, SymbolInformation,
21 TextDocumentIdentifier, TextEdit, WorkspaceEdit,
21}; 22};
22use ra_ide::{ 23use ra_ide::{
23 AssistId, FileId, FilePosition, FileRange, Query, RangeInfo, Runnable, RunnableKind, 24 AssistId, FileId, FilePosition, FileRange, Query, RangeInfo, Runnable, RunnableKind,
@@ -38,6 +39,7 @@ use crate::{
38 diagnostics::DiagnosticTask, 39 diagnostics::DiagnosticTask,
39 from_json, 40 from_json,
40 req::{self, Decoration, InlayHint, InlayHintsParams, InlayKind}, 41 req::{self, Decoration, InlayHint, InlayHintsParams, InlayKind},
42 semantic_tokens::SemanticTokensBuilder,
41 world::WorldSnapshot, 43 world::WorldSnapshot,
42 LspError, Result, 44 LspError, Result,
43}; 45};
@@ -1068,3 +1070,25 @@ pub fn handle_call_hierarchy_outgoing(
1068 1070
1069 Ok(Some(res)) 1071 Ok(Some(res))
1070} 1072}
1073
1074pub fn handle_semantic_tokens(
1075 world: WorldSnapshot,
1076 params: SemanticTokensParams,
1077) -> Result<Option<SemanticTokensResult>> {
1078 let _p = profile("handle_semantic_tokens");
1079
1080 let file_id = params.text_document.try_conv_with(&world)?;
1081 let line_index = world.analysis().file_line_index(file_id)?;
1082
1083 let mut builder = SemanticTokensBuilder::default();
1084
1085 for h in world.analysis().highlight(file_id)?.into_iter() {
1086 let type_and_modifiers: (SemanticTokenType, Vec<SemanticTokenModifier>) = h.tag.conv();
1087 let (token_type, token_modifiers) = type_and_modifiers.conv();
1088 builder.push(h.range.conv_with(&line_index), token_type, token_modifiers);
1089 }
1090
1091 let tokens = SemanticTokens { data: builder.build(), ..Default::default() };
1092
1093 Ok(Some(tokens.into()))
1094}
diff --git a/crates/rust-analyzer/src/req.rs b/crates/rust-analyzer/src/req.rs
index 7ff7f60b3..3734899bc 100644
--- a/crates/rust-analyzer/src/req.rs
+++ b/crates/rust-analyzer/src/req.rs
@@ -12,9 +12,9 @@ pub use lsp_types::{
12 DocumentSymbolResponse, FileSystemWatcher, Hover, InitializeResult, MessageType, 12 DocumentSymbolResponse, FileSystemWatcher, Hover, InitializeResult, MessageType,
13 PartialResultParams, ProgressParams, ProgressParamsValue, ProgressToken, 13 PartialResultParams, ProgressParams, ProgressParamsValue, ProgressToken,
14 PublishDiagnosticsParams, ReferenceParams, Registration, RegistrationParams, SelectionRange, 14 PublishDiagnosticsParams, ReferenceParams, Registration, RegistrationParams, SelectionRange,
15 SelectionRangeParams, ServerCapabilities, ShowMessageParams, SignatureHelp, SymbolKind, 15 SelectionRangeParams, SemanticTokensParams, SemanticTokensResult, ServerCapabilities,
16 TextDocumentEdit, TextDocumentPositionParams, TextEdit, WorkDoneProgressParams, WorkspaceEdit, 16 ShowMessageParams, SignatureHelp, SymbolKind, TextDocumentEdit, TextDocumentPositionParams,
17 WorkspaceSymbolParams, 17 TextEdit, WorkDoneProgressParams, WorkspaceEdit, WorkspaceSymbolParams,
18}; 18};
19 19
20pub enum AnalyzerStatus {} 20pub enum AnalyzerStatus {}
diff --git a/crates/rust-analyzer/src/semantic_tokens.rs b/crates/rust-analyzer/src/semantic_tokens.rs
new file mode 100644
index 000000000..ad000a3ce
--- /dev/null
+++ b/crates/rust-analyzer/src/semantic_tokens.rs
@@ -0,0 +1,94 @@
1//! Semantic Tokens helpers
2
3use lsp_types::{Range, SemanticToken, SemanticTokenModifier, SemanticTokenType};
4
5const SUPPORTED_TYPES: &[SemanticTokenType] = &[
6 SemanticTokenType::COMMENT,
7 SemanticTokenType::KEYWORD,
8 SemanticTokenType::STRING,
9 SemanticTokenType::NUMBER,
10 SemanticTokenType::REGEXP,
11 SemanticTokenType::OPERATOR,
12 SemanticTokenType::NAMESPACE,
13 SemanticTokenType::TYPE,
14 SemanticTokenType::STRUCT,
15 SemanticTokenType::CLASS,
16 SemanticTokenType::INTERFACE,
17 SemanticTokenType::ENUM,
18 SemanticTokenType::TYPE_PARAMETER,
19 SemanticTokenType::FUNCTION,
20 SemanticTokenType::MEMBER,
21 SemanticTokenType::PROPERTY,
22 SemanticTokenType::MACRO,
23 SemanticTokenType::VARIABLE,
24 SemanticTokenType::PARAMETER,
25 SemanticTokenType::LABEL,
26];
27
28const SUPPORTED_MODIFIERS: &[SemanticTokenModifier] = &[
29 SemanticTokenModifier::DOCUMENTATION,
30 SemanticTokenModifier::DECLARATION,
31 SemanticTokenModifier::DEFINITION,
32 SemanticTokenModifier::REFERENCE,
33 SemanticTokenModifier::STATIC,
34 SemanticTokenModifier::ABSTRACT,
35 SemanticTokenModifier::DEPRECATED,
36 SemanticTokenModifier::ASYNC,
37 SemanticTokenModifier::VOLATILE,
38 SemanticTokenModifier::READONLY,
39];
40
41/// Token types that the server supports
42pub(crate) fn supported_token_types() -> &'static [SemanticTokenType] {
43 SUPPORTED_TYPES
44}
45
46/// Token modifiers that the server supports
47pub(crate) fn supported_token_modifiers() -> &'static [SemanticTokenModifier] {
48 SUPPORTED_MODIFIERS
49}
50
51/// Tokens are encoded relative to each other.
52///
53/// This is a direct port of https://github.com/microsoft/vscode-languageserver-node/blob/f425af9de46a0187adb78ec8a46b9b2ce80c5412/server/src/sematicTokens.proposed.ts#L45
54#[derive(Default)]
55pub(crate) struct SemanticTokensBuilder {
56 prev_line: u32,
57 prev_char: u32,
58 data: Vec<SemanticToken>,
59}
60
61impl SemanticTokensBuilder {
62 /// Push a new token onto the builder
63 pub fn push(&mut self, range: Range, token_index: u32, modifier_bitset: u32) {
64 let mut push_line = range.start.line as u32;
65 let mut push_char = range.start.character as u32;
66
67 if !self.data.is_empty() {
68 push_line -= self.prev_line;
69 if push_line == 0 {
70 push_char -= self.prev_char;
71 }
72 }
73
74 // A token cannot be multiline
75 let token_len = range.end.character - range.start.character;
76
77 let token = SemanticToken {
78 delta_line: push_line,
79 delta_start: push_char,
80 length: token_len as u32,
81 token_type: token_index,
82 token_modifiers_bitset: modifier_bitset,
83 };
84
85 self.data.push(token);
86
87 self.prev_line = range.start.line as u32;
88 self.prev_char = range.start.character as u32;
89 }
90
91 pub fn build(self) -> Vec<SemanticToken> {
92 self.data
93 }
94}