From 58897dd8ddfd08ef494b7bc05ac15f5b1e3a4e1a Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Mon, 11 Feb 2019 19:28:39 +0300 Subject: assign ids to tokens --- crates/ra_mbe/src/mbe_expander.rs | 4 +++- crates/ra_mbe/src/mbe_parser.rs | 2 +- crates/ra_mbe/src/syntax_bridge.rs | 2 +- crates/ra_tt/src/lib.rs | 10 ++++++++++ 4 files changed, 15 insertions(+), 3 deletions(-) (limited to 'crates') diff --git a/crates/ra_mbe/src/mbe_expander.rs b/crates/ra_mbe/src/mbe_expander.rs index eec713d9c..f6177f078 100644 --- a/crates/ra_mbe/src/mbe_expander.rs +++ b/crates/ra_mbe/src/mbe_expander.rs @@ -3,6 +3,7 @@ /// `tt::TokenTree` for the result of the expansion. use rustc_hash::FxHashMap; use ra_syntax::SmolStr; +use tt::TokenId; use crate::tt_cursor::TtCursor; @@ -185,7 +186,8 @@ fn expand_tt( } crate::TokenTree::Leaf(leaf) => match leaf { crate::Leaf::Ident(ident) => { - tt::Leaf::from(tt::Ident { text: ident.text.clone() }).into() + tt::Leaf::from(tt::Ident { text: ident.text.clone(), id: TokenId::unspecified() }) + .into() } crate::Leaf::Punct(punct) => tt::Leaf::from(punct.clone()).into(), crate::Leaf::Var(v) => bindings.get(&v.text, nesting)?.clone(), diff --git a/crates/ra_mbe/src/mbe_parser.rs b/crates/ra_mbe/src/mbe_parser.rs index 60e566ed2..58e2533f1 100644 --- a/crates/ra_mbe/src/mbe_parser.rs +++ b/crates/ra_mbe/src/mbe_parser.rs @@ -41,7 +41,7 @@ fn parse_subtree(tt: &tt::Subtree) -> Option { } } tt::Leaf::Punct(punct) => crate::Leaf::from(*punct).into(), - tt::Leaf::Ident(tt::Ident { text }) => { + tt::Leaf::Ident(tt::Ident { text, id: _ }) => { crate::Leaf::from(crate::Ident { text: text.clone() }).into() } tt::Leaf::Literal(tt::Literal { text }) => { diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 9a2eceaba..d734f5597 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -37,7 +37,7 @@ fn convert_tt(tt: &SyntaxNode) -> Option { convert_tt(child)?.into() } else if child.kind().is_keyword() || child.kind() == IDENT { let text = child.leaf_text().unwrap().clone(); - tt::Leaf::from(tt::Ident { text }).into() + tt::Leaf::from(tt::Ident { text, id: tt::TokenId::unspecified() }).into() } else if child.kind().is_literal() { tt::Leaf::from(tt::Literal { text: child.leaf_text().unwrap().clone() }).into() } else { diff --git a/crates/ra_tt/src/lib.rs b/crates/ra_tt/src/lib.rs index df31f72f3..e0a4cdb8b 100644 --- a/crates/ra_tt/src/lib.rs +++ b/crates/ra_tt/src/lib.rs @@ -18,6 +18,15 @@ use std::fmt; use smol_str::SmolStr; +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct TokenId(pub u32); + +impl TokenId { + pub const fn unspecified() -> TokenId { + TokenId(!0) + } +} + #[derive(Debug, Clone)] pub enum TokenTree { Leaf(Leaf), @@ -67,6 +76,7 @@ pub enum Spacing { #[derive(Debug, Clone)] pub struct Ident { pub text: SmolStr, + pub id: TokenId, } impl fmt::Display for TokenTree { -- cgit v1.2.3 From 0d34a256de5d33565e9a62d53bf149cf59510937 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Mon, 11 Feb 2019 21:12:06 +0300 Subject: assign ids when converting tt --- crates/ra_mbe/src/lib.rs | 8 ++++---- crates/ra_mbe/src/syntax_bridge.rs | 38 ++++++++++++++++++++++++++++++++------ 2 files changed, 36 insertions(+), 10 deletions(-) (limited to 'crates') diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index b9f555ef6..ec2fd1eb5 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -144,8 +144,8 @@ impl_froms!(TokenTree: Leaf, Subtree); let macro_invocation = source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); - let definition_tt = ast_to_token_tree(macro_definition.token_tree().unwrap()).unwrap(); - let invocation_tt = ast_to_token_tree(macro_invocation.token_tree().unwrap()).unwrap(); + let (definition_tt, _) = ast_to_token_tree(macro_definition.token_tree().unwrap()).unwrap(); + let (invocation_tt, _) = ast_to_token_tree(macro_invocation.token_tree().unwrap()).unwrap(); let rules = crate::MacroRules::parse(&definition_tt).unwrap(); let expansion = rules.expand(&invocation_tt).unwrap(); assert_eq!( @@ -160,7 +160,7 @@ impl_froms!(TokenTree: Leaf, Subtree); let macro_definition = source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); - let definition_tt = ast_to_token_tree(macro_definition.token_tree().unwrap()).unwrap(); + let (definition_tt, _) = ast_to_token_tree(macro_definition.token_tree().unwrap()).unwrap(); crate::MacroRules::parse(&definition_tt).unwrap() } @@ -169,7 +169,7 @@ impl_froms!(TokenTree: Leaf, Subtree); let macro_invocation = source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); - let invocation_tt = ast_to_token_tree(macro_invocation.token_tree().unwrap()).unwrap(); + let (invocation_tt, _) = ast_to_token_tree(macro_invocation.token_tree().unwrap()).unwrap(); let expaned = rules.expand(&invocation_tt).unwrap(); assert_eq!(expaned.to_string(), expansion); diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index d734f5597..798f9d8fa 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -1,10 +1,34 @@ -use ra_syntax::{ast, AstNode, SyntaxNode, SyntaxKind::*}; +use ra_syntax::{ + AstNode, SyntaxNode, TextRange, + ast, SyntaxKind::*, TextUnit +}; -pub fn ast_to_token_tree(ast: &ast::TokenTree) -> Option { - convert_tt(ast.syntax()) +#[derive(Default)] +pub struct TokenMap { + /// Maps `tt::TokenId` to the *relative* source range. + toknes: Vec, } -fn convert_tt(tt: &SyntaxNode) -> Option { +pub fn ast_to_token_tree(ast: &ast::TokenTree) -> Option<(tt::Subtree, TokenMap)> { + let mut token_map = TokenMap::default(); + let node = ast.syntax(); + let tt = convert_tt(&mut token_map, node.range().start(), node)?; + Some((tt, token_map)) +} + +impl TokenMap { + fn alloc(&mut self, relative_range: TextRange) -> tt::TokenId { + let id = self.toknes.len(); + self.toknes.push(relative_range); + tt::TokenId(id as u32) + } +} + +fn convert_tt( + token_map: &mut TokenMap, + global_offset: TextUnit, + tt: &SyntaxNode, +) -> Option { let first_child = tt.first_child()?; let last_child = tt.last_child()?; let delimiter = match (first_child.kind(), last_child.kind()) { @@ -34,10 +58,12 @@ fn convert_tt(tt: &SyntaxNode) -> Option { } } else { let child: tt::TokenTree = if child.kind() == TOKEN_TREE { - convert_tt(child)?.into() + convert_tt(token_map, global_offset, child)?.into() } else if child.kind().is_keyword() || child.kind() == IDENT { + let relative_range = child.range() - global_offset; + let id = token_map.alloc(relative_range); let text = child.leaf_text().unwrap().clone(); - tt::Leaf::from(tt::Ident { text, id: tt::TokenId::unspecified() }).into() + tt::Leaf::from(tt::Ident { text, id }).into() } else if child.kind().is_literal() { tt::Leaf::from(tt::Literal { text: child.leaf_text().unwrap().clone() }).into() } else { -- cgit v1.2.3 From ae312680d6d7bb0cc00d2b8d9799249d36e0136e Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Mon, 11 Feb 2019 21:31:54 +0300 Subject: docs --- crates/ra_mbe/src/syntax_bridge.rs | 8 ++++++++ crates/ra_tt/src/lib.rs | 8 +++++++- 2 files changed, 15 insertions(+), 1 deletion(-) (limited to 'crates') diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 798f9d8fa..848c785f8 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -3,12 +3,15 @@ use ra_syntax::{ ast, SyntaxKind::*, TextUnit }; +/// Maps `tt::TokenId` to the relative range of the original token. #[derive(Default)] pub struct TokenMap { /// Maps `tt::TokenId` to the *relative* source range. toknes: Vec, } +/// Convert the syntax tree (what user has written) to a `TokenTree` (what macro +/// will consume). pub fn ast_to_token_tree(ast: &ast::TokenTree) -> Option<(tt::Subtree, TokenMap)> { let mut token_map = TokenMap::default(); let node = ast.syntax(); @@ -17,6 +20,11 @@ pub fn ast_to_token_tree(ast: &ast::TokenTree) -> Option<(tt::Subtree, TokenMap) } impl TokenMap { + pub fn relative_range_of(&self, tt: tt::TokenId) -> Option { + let idx = tt.0 as usize; + self.toknes.get(idx).map(|&it| it) + } + fn alloc(&mut self, relative_range: TextRange) -> tt::TokenId { let id = self.toknes.len(); self.toknes.push(relative_range); diff --git a/crates/ra_tt/src/lib.rs b/crates/ra_tt/src/lib.rs index e0a4cdb8b..c1f37b889 100644 --- a/crates/ra_tt/src/lib.rs +++ b/crates/ra_tt/src/lib.rs @@ -1,4 +1,4 @@ -/// `tt` crate defines a `TokenTree` datastructure: this is the interface (both +/// `tt` crate defines a `TokenTree` data structure: this is the interface (both /// input and output) of macros. It closely mirrors `proc_macro` crate's /// `TokenTree`. @@ -18,6 +18,12 @@ use std::fmt; use smol_str::SmolStr; +/// Represents identity of the token. +/// +/// For hygiene purposes, we need to track which expanded tokens originated from +/// which source tokens. We do it by assigning an distinct identity to each +/// source token and making sure that identities are preserved during macro +/// expansion. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct TokenId(pub u32); -- cgit v1.2.3