From 58897dd8ddfd08ef494b7bc05ac15f5b1e3a4e1a Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Mon, 11 Feb 2019 19:28:39 +0300 Subject: assign ids to tokens --- crates/ra_mbe/src/syntax_bridge.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'crates/ra_mbe/src/syntax_bridge.rs') diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 9a2eceaba..d734f5597 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -37,7 +37,7 @@ fn convert_tt(tt: &SyntaxNode) -> Option { convert_tt(child)?.into() } else if child.kind().is_keyword() || child.kind() == IDENT { let text = child.leaf_text().unwrap().clone(); - tt::Leaf::from(tt::Ident { text }).into() + tt::Leaf::from(tt::Ident { text, id: tt::TokenId::unspecified() }).into() } else if child.kind().is_literal() { tt::Leaf::from(tt::Literal { text: child.leaf_text().unwrap().clone() }).into() } else { -- cgit v1.2.3 From 0d34a256de5d33565e9a62d53bf149cf59510937 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Mon, 11 Feb 2019 21:12:06 +0300 Subject: assign ids when converting tt --- crates/ra_mbe/src/syntax_bridge.rs | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) (limited to 'crates/ra_mbe/src/syntax_bridge.rs') diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index d734f5597..798f9d8fa 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -1,10 +1,34 @@ -use ra_syntax::{ast, AstNode, SyntaxNode, SyntaxKind::*}; +use ra_syntax::{ + AstNode, SyntaxNode, TextRange, + ast, SyntaxKind::*, TextUnit +}; -pub fn ast_to_token_tree(ast: &ast::TokenTree) -> Option { - convert_tt(ast.syntax()) +#[derive(Default)] +pub struct TokenMap { + /// Maps `tt::TokenId` to the *relative* source range. + toknes: Vec, } -fn convert_tt(tt: &SyntaxNode) -> Option { +pub fn ast_to_token_tree(ast: &ast::TokenTree) -> Option<(tt::Subtree, TokenMap)> { + let mut token_map = TokenMap::default(); + let node = ast.syntax(); + let tt = convert_tt(&mut token_map, node.range().start(), node)?; + Some((tt, token_map)) +} + +impl TokenMap { + fn alloc(&mut self, relative_range: TextRange) -> tt::TokenId { + let id = self.toknes.len(); + self.toknes.push(relative_range); + tt::TokenId(id as u32) + } +} + +fn convert_tt( + token_map: &mut TokenMap, + global_offset: TextUnit, + tt: &SyntaxNode, +) -> Option { let first_child = tt.first_child()?; let last_child = tt.last_child()?; let delimiter = match (first_child.kind(), last_child.kind()) { @@ -34,10 +58,12 @@ fn convert_tt(tt: &SyntaxNode) -> Option { } } else { let child: tt::TokenTree = if child.kind() == TOKEN_TREE { - convert_tt(child)?.into() + convert_tt(token_map, global_offset, child)?.into() } else if child.kind().is_keyword() || child.kind() == IDENT { + let relative_range = child.range() - global_offset; + let id = token_map.alloc(relative_range); let text = child.leaf_text().unwrap().clone(); - tt::Leaf::from(tt::Ident { text, id: tt::TokenId::unspecified() }).into() + tt::Leaf::from(tt::Ident { text, id }).into() } else if child.kind().is_literal() { tt::Leaf::from(tt::Literal { text: child.leaf_text().unwrap().clone() }).into() } else { -- cgit v1.2.3 From ae312680d6d7bb0cc00d2b8d9799249d36e0136e Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Mon, 11 Feb 2019 21:31:54 +0300 Subject: docs --- crates/ra_mbe/src/syntax_bridge.rs | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'crates/ra_mbe/src/syntax_bridge.rs') diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 798f9d8fa..848c785f8 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -3,12 +3,15 @@ use ra_syntax::{ ast, SyntaxKind::*, TextUnit }; +/// Maps `tt::TokenId` to the relative range of the original token. #[derive(Default)] pub struct TokenMap { /// Maps `tt::TokenId` to the *relative* source range. toknes: Vec, } +/// Convert the syntax tree (what user has written) to a `TokenTree` (what macro +/// will consume). pub fn ast_to_token_tree(ast: &ast::TokenTree) -> Option<(tt::Subtree, TokenMap)> { let mut token_map = TokenMap::default(); let node = ast.syntax(); @@ -17,6 +20,11 @@ pub fn ast_to_token_tree(ast: &ast::TokenTree) -> Option<(tt::Subtree, TokenMap) } impl TokenMap { + pub fn relative_range_of(&self, tt: tt::TokenId) -> Option { + let idx = tt.0 as usize; + self.toknes.get(idx).map(|&it| it) + } + fn alloc(&mut self, relative_range: TextRange) -> tt::TokenId { let id = self.toknes.len(); self.toknes.push(relative_range); -- cgit v1.2.3