From 9fd546bec23ac817a45da28889e76118969db91e Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Sun, 3 Nov 2019 13:19:50 +0800 Subject: Add map_id to TokenId --- crates/ra_mbe/src/syntax_bridge.rs | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) (limited to 'crates/ra_mbe/src') diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 592fcf527..8d9217518 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -12,12 +12,26 @@ use tt::buffer::{Cursor, TokenBuffer}; use crate::subtree_source::SubtreeTokenSource; use crate::ExpandError; +use std::sync::atomic::{AtomicU32, Ordering}; /// Maps `tt::TokenId` to the relative range of the original token. -#[derive(Default)] pub struct TokenMap { /// Maps `tt::TokenId` to the *relative* source range. tokens: Vec, + map_id: u32, +} + +static TOKEN_MAP_COUNTER: AtomicU32 = AtomicU32::new(0); + +/// Generate an unique token map id for each instance +fn make_uniq_token_map_id() -> u32 { + TOKEN_MAP_COUNTER.fetch_add(1, Ordering::SeqCst) +} + +impl std::default::Default for TokenMap { + fn default() -> TokenMap { + TokenMap { tokens: Default::default(), map_id: make_uniq_token_map_id() } + } } /// Convert the syntax tree (what user has written) to a `TokenTree` (what macro @@ -105,14 +119,17 @@ pub fn token_tree_to_items(tt: &tt::Subtree) -> Result, E impl TokenMap { pub fn relative_range_of(&self, tt: tt::TokenId) -> Option { - let idx = tt.0 as usize; + if self.map_id != tt.map_id() { + return None; + } + let idx = tt.token_id() as usize; self.tokens.get(idx).copied() } fn alloc(&mut self, relative_range: TextRange) -> tt::TokenId { let id = self.tokens.len(); self.tokens.push(relative_range); - tt::TokenId(id as u32) + tt::TokenId::new(id as u32, self.map_id) } } -- cgit v1.2.3 From 159da285e9d8594a2cc4436b5cee7874b07806c9 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Sun, 3 Nov 2019 22:46:12 +0800 Subject: Add macro_expansion_info in hir_expand --- crates/ra_mbe/src/lib.rs | 2 +- crates/ra_mbe/src/syntax_bridge.rs | 113 ++++++++++++++++++++++++++++++------- crates/ra_mbe/src/tests.rs | 16 +++--- 3 files changed, 103 insertions(+), 28 deletions(-) (limited to 'crates/ra_mbe/src') diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index 15f000175..2926b29fd 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -32,7 +32,7 @@ pub enum ExpandError { pub use crate::syntax_bridge::{ ast_to_token_tree, syntax_node_to_token_tree, token_tree_to_expr, token_tree_to_items, - token_tree_to_macro_stmts, token_tree_to_pat, token_tree_to_ty, + token_tree_to_macro_stmts, token_tree_to_pat, token_tree_to_ty, TokenMap, }; /// This struct contains AST for a single `macro_rules` definition. What might diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 8d9217518..5db6647e3 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -15,6 +15,7 @@ use crate::ExpandError; use std::sync::atomic::{AtomicU32, Ordering}; /// Maps `tt::TokenId` to the relative range of the original token. +#[derive(Debug, PartialEq, Eq)] pub struct TokenMap { /// Maps `tt::TokenId` to the *relative* source range. tokens: Vec, @@ -34,6 +35,13 @@ impl std::default::Default for TokenMap { } } +/// Maps Relative range of the expanded syntax node to `tt::TokenId` +#[derive(Debug, PartialEq, Eq, Default)] +pub struct ExpandedRangeMap { + /// Maps `tt::TokenId` to the *relative* source range. + ranges: Vec<(TextRange, tt::TokenId)>, +} + /// Convert the syntax tree (what user has written) to a `TokenTree` (what macro /// will consume). pub fn ast_to_token_tree(ast: &ast::TokenTree) -> Option<(tt::Subtree, TokenMap)> { @@ -66,7 +74,7 @@ pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, Toke fn fragment_to_syntax_node( tt: &tt::Subtree, fragment_kind: FragmentKind, -) -> Result, ExpandError> { +) -> Result<(Parse, ExpandedRangeMap), ExpandError> { let tmp; let tokens = match tt { tt::Subtree { delimiter: tt::Delimiter::None, token_trees } => token_trees.as_slice(), @@ -77,44 +85,55 @@ fn fragment_to_syntax_node( }; let buffer = TokenBuffer::new(&tokens); let mut token_source = SubtreeTokenSource::new(&buffer); - let mut tree_sink = TtTreeSink::new(buffer.begin()); + let mut range_map = ExpandedRangeMap::default(); + let mut tree_sink = TtTreeSink::new(buffer.begin(), &mut range_map); ra_parser::parse_fragment(&mut token_source, &mut tree_sink, fragment_kind); if tree_sink.roots.len() != 1 { return Err(ExpandError::ConversionError); } //FIXME: would be cool to report errors let parse = tree_sink.inner.finish(); - Ok(parse) + Ok((parse, range_map)) } /// Parses the token tree (result of macro expansion) to an expression -pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result, ExpandError> { - let parse = fragment_to_syntax_node(tt, Expr)?; - parse.cast().ok_or_else(|| crate::ExpandError::ConversionError) +pub fn token_tree_to_expr( + tt: &tt::Subtree, +) -> Result<(Parse, ExpandedRangeMap), ExpandError> { + let (parse, map) = fragment_to_syntax_node(tt, Expr)?; + parse.cast().ok_or_else(|| crate::ExpandError::ConversionError).map(|p| (p, map)) } /// Parses the token tree (result of macro expansion) to a Pattern -pub fn token_tree_to_pat(tt: &tt::Subtree) -> Result, ExpandError> { - let parse = fragment_to_syntax_node(tt, Pattern)?; - parse.cast().ok_or_else(|| crate::ExpandError::ConversionError) +pub fn token_tree_to_pat( + tt: &tt::Subtree, +) -> Result<(Parse, ExpandedRangeMap), ExpandError> { + let (parse, map) = fragment_to_syntax_node(tt, Pattern)?; + parse.cast().ok_or_else(|| crate::ExpandError::ConversionError).map(|p| (p, map)) } /// Parses the token tree (result of macro expansion) to a Type -pub fn token_tree_to_ty(tt: &tt::Subtree) -> Result, ExpandError> { - let parse = fragment_to_syntax_node(tt, Type)?; - parse.cast().ok_or_else(|| crate::ExpandError::ConversionError) +pub fn token_tree_to_ty( + tt: &tt::Subtree, +) -> Result<(Parse, ExpandedRangeMap), ExpandError> { + let (parse, map) = fragment_to_syntax_node(tt, Type)?; + parse.cast().ok_or_else(|| crate::ExpandError::ConversionError).map(|p| (p, map)) } /// Parses the token tree (result of macro expansion) as a sequence of stmts -pub fn token_tree_to_macro_stmts(tt: &tt::Subtree) -> Result, ExpandError> { - let parse = fragment_to_syntax_node(tt, Statements)?; - parse.cast().ok_or_else(|| crate::ExpandError::ConversionError) +pub fn token_tree_to_macro_stmts( + tt: &tt::Subtree, +) -> Result<(Parse, ExpandedRangeMap), ExpandError> { + let (parse, map) = fragment_to_syntax_node(tt, Statements)?; + parse.cast().ok_or_else(|| crate::ExpandError::ConversionError).map(|p| (p, map)) } /// Parses the token tree (result of macro expansion) as a sequence of items -pub fn token_tree_to_items(tt: &tt::Subtree) -> Result, ExpandError> { - let parse = fragment_to_syntax_node(tt, Items)?; - parse.cast().ok_or_else(|| crate::ExpandError::ConversionError) +pub fn token_tree_to_items( + tt: &tt::Subtree, +) -> Result<(Parse, ExpandedRangeMap), ExpandError> { + let (parse, map) = fragment_to_syntax_node(tt, Items)?; + parse.cast().ok_or_else(|| crate::ExpandError::ConversionError).map(|p| (p, map)) } impl TokenMap { @@ -133,6 +152,28 @@ impl TokenMap { } } +impl ExpandedRangeMap { + fn set(&mut self, relative_range: TextRange, token_id: &tt::TokenId) { + self.ranges.push((relative_range, token_id.clone())) + } + + pub fn ranges(&self, to: &TokenMap) -> Vec<(TextRange, TextRange)> { + self.ranges + .iter() + .filter_map(|(r, tid)| { + if to.map_id == tid.map_id() { + return None; + } + if let Some(to_range) = to.relative_range_of(*tid) { + Some((*r, to_range)) + } else { + None + } + }) + .collect() + } +} + /// Returns the textual content of a doc comment block as a quoted string /// That is, strips leading `///` (or `/**`, etc) /// and strips the ending `*/` @@ -279,6 +320,8 @@ struct TtTreeSink<'a> { cursor: Cursor<'a>, text_pos: TextUnit, inner: SyntaxTreeBuilder, + range_marker: Option<(TextRange, tt::TokenId)>, + range_map: &'a mut ExpandedRangeMap, // Number of roots // Use for detect ill-form tree which is not single root @@ -286,13 +329,15 @@ struct TtTreeSink<'a> { } impl<'a> TtTreeSink<'a> { - fn new(cursor: Cursor<'a>) -> Self { + fn new(cursor: Cursor<'a>, range_map: &'a mut ExpandedRangeMap) -> Self { TtTreeSink { buf: String::new(), cursor, text_pos: 0.into(), inner: SyntaxTreeBuilder::default(), roots: smallvec::SmallVec::new(), + range_map, + range_marker: None, } } } @@ -317,6 +362,8 @@ impl<'a> TreeSink for TtTreeSink<'a> { return; } + let mut last_ident = None; + for _ in 0..n_tokens { if self.cursor.eof() { break; @@ -326,6 +373,10 @@ impl<'a> TreeSink for TtTreeSink<'a> { Some(tt::TokenTree::Leaf(leaf)) => { self.cursor = self.cursor.bump(); self.buf += &format!("{}", leaf); + + if let tt::Leaf::Ident(ident) = leaf { + last_ident = Some(ident); + } } Some(tt::TokenTree::Subtree(subtree)) => { self.cursor = self.cursor.subtree().unwrap(); @@ -345,6 +396,14 @@ impl<'a> TreeSink for TtTreeSink<'a> { self.buf.clear(); self.inner.token(kind, text); + // Mark the range if needed + if let Some((range, token_id)) = self.range_marker.as_mut() { + if let Some(ident) = last_ident { + *range = TextRange::offset_len(range.start(), TextUnit::of_str(&ident.text)); + *token_id = ident.id; + } + } + // Add whitespace between adjoint puncts let next = self.cursor.bump(); if let ( @@ -354,6 +413,7 @@ impl<'a> TreeSink for TtTreeSink<'a> { { if curr.spacing == tt::Spacing::Alone { self.inner.token(WHITESPACE, " ".into()); + self.text_pos += TextUnit::of_char(' '); } } } @@ -361,6 +421,15 @@ impl<'a> TreeSink for TtTreeSink<'a> { fn start_node(&mut self, kind: SyntaxKind) { self.inner.start_node(kind); + self.range_marker = if kind == IDENT { + Some(( + TextRange::offset_len(self.text_pos, TextUnit::from_usize(0)), + tt::TokenId::unspecified(), + )) + } else { + None + }; + match self.roots.last_mut() { None | Some(0) => self.roots.push(1), Some(ref mut n) => **n += 1, @@ -370,6 +439,12 @@ impl<'a> TreeSink for TtTreeSink<'a> { fn finish_node(&mut self) { self.inner.finish_node(); *self.roots.last_mut().unwrap() -= 1; + + if let Some(range) = self.range_marker { + if range.1 != tt::TokenId::unspecified() { + self.range_map.set(range.0, &range.1) + } + } } fn error(&mut self, error: ParseError) { diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs index a23e3afe3..a848ea334 100644 --- a/crates/ra_mbe/src/tests.rs +++ b/crates/ra_mbe/src/tests.rs @@ -126,7 +126,7 @@ fn test_expr_order() { "#, ); let expanded = expand(&rules, "foo! { 1 + 1}"); - let tree = token_tree_to_items(&expanded).unwrap().tree(); + let tree = token_tree_to_items(&expanded).unwrap().0.tree(); let dump = format!("{:#?}", tree.syntax()); assert_eq_text!( @@ -383,7 +383,7 @@ fn test_expand_to_item_list() { ", ); let expansion = expand(&rules, "structs!(Foo, Bar);"); - let tree = token_tree_to_items(&expansion).unwrap().tree(); + let tree = token_tree_to_items(&expansion).unwrap().0.tree(); assert_eq!( format!("{:#?}", tree.syntax()).trim(), r#" @@ -501,7 +501,7 @@ fn test_tt_to_stmts() { ); let expanded = expand(&rules, "foo!{}"); - let stmts = token_tree_to_macro_stmts(&expanded).unwrap().tree(); + let stmts = token_tree_to_macro_stmts(&expanded).unwrap().0.tree(); assert_eq!( format!("{:#?}", stmts.syntax()).trim(), @@ -946,7 +946,7 @@ fn test_vec() { ); let expansion = expand(&rules, r#"vec![1u32,2];"#); - let tree = token_tree_to_expr(&expansion).unwrap().tree(); + let tree = token_tree_to_expr(&expansion).unwrap().0.tree(); assert_eq!( format!("{:#?}", tree.syntax()).trim(), @@ -1436,8 +1436,8 @@ pub(crate) fn assert_expansion( }; let (expanded_tree, expected_tree) = match kind { MacroKind::Items => { - let expanded_tree = token_tree_to_items(&expanded).unwrap().tree(); - let expected_tree = token_tree_to_items(&expected).unwrap().tree(); + let expanded_tree = token_tree_to_items(&expanded).unwrap().0.tree(); + let expected_tree = token_tree_to_items(&expected).unwrap().0.tree(); ( debug_dump_ignore_spaces(expanded_tree.syntax()).trim().to_string(), @@ -1446,8 +1446,8 @@ pub(crate) fn assert_expansion( } MacroKind::Stmts => { - let expanded_tree = token_tree_to_macro_stmts(&expanded).unwrap().tree(); - let expected_tree = token_tree_to_macro_stmts(&expected).unwrap().tree(); + let expanded_tree = token_tree_to_macro_stmts(&expanded).unwrap().0.tree(); + let expected_tree = token_tree_to_macro_stmts(&expected).unwrap().0.tree(); ( debug_dump_ignore_spaces(expanded_tree.syntax()).trim().to_string(), -- cgit v1.2.3 From d560219d06780ea4bd3cca20e1d3883f2d68d59a Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Mon, 4 Nov 2019 01:45:58 +0800 Subject: Fixed bug in ExpandedRangeMap --- crates/ra_mbe/src/syntax_bridge.rs | 50 ++++++++++++-------------------------- 1 file changed, 16 insertions(+), 34 deletions(-) (limited to 'crates/ra_mbe/src') diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 5db6647e3..4965f9c33 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -157,15 +157,16 @@ impl ExpandedRangeMap { self.ranges.push((relative_range, token_id.clone())) } - pub fn ranges(&self, to: &TokenMap) -> Vec<(TextRange, TextRange)> { + pub fn ranges(&self, to: &TokenMap, start: TextUnit) -> Vec<(TextRange, TextRange)> { + dbg!(&self.ranges); self.ranges .iter() .filter_map(|(r, tid)| { - if to.map_id == tid.map_id() { + if to.map_id != tid.map_id() { return None; } if let Some(to_range) = to.relative_range_of(*tid) { - Some((*r, to_range)) + Some((*r, TextRange::from_to(to_range.start() + start, to_range.end() + start))) } else { None } @@ -320,7 +321,6 @@ struct TtTreeSink<'a> { cursor: Cursor<'a>, text_pos: TextUnit, inner: SyntaxTreeBuilder, - range_marker: Option<(TextRange, tt::TokenId)>, range_map: &'a mut ExpandedRangeMap, // Number of roots @@ -337,7 +337,6 @@ impl<'a> TtTreeSink<'a> { inner: SyntaxTreeBuilder::default(), roots: smallvec::SmallVec::new(), range_map, - range_marker: None, } } } @@ -362,8 +361,6 @@ impl<'a> TreeSink for TtTreeSink<'a> { return; } - let mut last_ident = None; - for _ in 0..n_tokens { if self.cursor.eof() { break; @@ -371,12 +368,20 @@ impl<'a> TreeSink for TtTreeSink<'a> { match self.cursor.token_tree() { Some(tt::TokenTree::Leaf(leaf)) => { - self.cursor = self.cursor.bump(); - self.buf += &format!("{}", leaf); - + // Mark the range if needed if let tt::Leaf::Ident(ident) = leaf { - last_ident = Some(ident); + if kind == IDENT { + let range = TextRange::offset_len( + self.text_pos + TextUnit::of_str(&self.buf), + TextUnit::of_str(&ident.text), + ); + let token_id = ident.id; + self.range_map.set(range, &token_id); + } } + + self.cursor = self.cursor.bump(); + self.buf += &format!("{}", leaf); } Some(tt::TokenTree::Subtree(subtree)) => { self.cursor = self.cursor.subtree().unwrap(); @@ -396,14 +401,6 @@ impl<'a> TreeSink for TtTreeSink<'a> { self.buf.clear(); self.inner.token(kind, text); - // Mark the range if needed - if let Some((range, token_id)) = self.range_marker.as_mut() { - if let Some(ident) = last_ident { - *range = TextRange::offset_len(range.start(), TextUnit::of_str(&ident.text)); - *token_id = ident.id; - } - } - // Add whitespace between adjoint puncts let next = self.cursor.bump(); if let ( @@ -421,15 +418,6 @@ impl<'a> TreeSink for TtTreeSink<'a> { fn start_node(&mut self, kind: SyntaxKind) { self.inner.start_node(kind); - self.range_marker = if kind == IDENT { - Some(( - TextRange::offset_len(self.text_pos, TextUnit::from_usize(0)), - tt::TokenId::unspecified(), - )) - } else { - None - }; - match self.roots.last_mut() { None | Some(0) => self.roots.push(1), Some(ref mut n) => **n += 1, @@ -439,12 +427,6 @@ impl<'a> TreeSink for TtTreeSink<'a> { fn finish_node(&mut self) { self.inner.finish_node(); *self.roots.last_mut().unwrap() -= 1; - - if let Some(range) = self.range_marker { - if range.1 != tt::TokenId::unspecified() { - self.range_map.set(range.0, &range.1) - } - } } fn error(&mut self, error: ParseError) { -- cgit v1.2.3 From 67226ebc8288b9489867ea2454f9d976b65ff15e Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Mon, 4 Nov 2019 01:47:59 +0800 Subject: Remove dbg! --- crates/ra_mbe/src/syntax_bridge.rs | 1 - 1 file changed, 1 deletion(-) (limited to 'crates/ra_mbe/src') diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 4965f9c33..8e86742d0 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -158,7 +158,6 @@ impl ExpandedRangeMap { } pub fn ranges(&self, to: &TokenMap, start: TextUnit) -> Vec<(TextRange, TextRange)> { - dbg!(&self.ranges); self.ranges .iter() .filter_map(|(r, tid)| { -- cgit v1.2.3 From e6709f64af1836460aee41aca34eb19ed3a337dd Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Mon, 4 Nov 2019 03:12:19 +0800 Subject: Rename and fix typos --- crates/ra_mbe/src/syntax_bridge.rs | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) (limited to 'crates/ra_mbe/src') diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 8e86742d0..2f21ad6e0 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -26,7 +26,11 @@ static TOKEN_MAP_COUNTER: AtomicU32 = AtomicU32::new(0); /// Generate an unique token map id for each instance fn make_uniq_token_map_id() -> u32 { - TOKEN_MAP_COUNTER.fetch_add(1, Ordering::SeqCst) + let res = TOKEN_MAP_COUNTER.fetch_add(1, Ordering::SeqCst); + if res == std::u32::MAX { + panic!("TOKEN_MAP_COUNTER is overflowed"); + } + res } impl std::default::Default for TokenMap { @@ -35,10 +39,9 @@ impl std::default::Default for TokenMap { } } -/// Maps Relative range of the expanded syntax node to `tt::TokenId` +/// Maps relative range of the expanded syntax node to `tt::TokenId` #[derive(Debug, PartialEq, Eq, Default)] pub struct ExpandedRangeMap { - /// Maps `tt::TokenId` to the *relative* source range. ranges: Vec<(TextRange, tt::TokenId)>, } @@ -85,14 +88,13 @@ fn fragment_to_syntax_node( }; let buffer = TokenBuffer::new(&tokens); let mut token_source = SubtreeTokenSource::new(&buffer); - let mut range_map = ExpandedRangeMap::default(); - let mut tree_sink = TtTreeSink::new(buffer.begin(), &mut range_map); + let mut tree_sink = TtTreeSink::new(buffer.begin()); ra_parser::parse_fragment(&mut token_source, &mut tree_sink, fragment_kind); if tree_sink.roots.len() != 1 { return Err(ExpandError::ConversionError); } //FIXME: would be cool to report errors - let parse = tree_sink.inner.finish(); + let (parse, range_map) = tree_sink.finish(); Ok((parse, range_map)) } @@ -320,7 +322,7 @@ struct TtTreeSink<'a> { cursor: Cursor<'a>, text_pos: TextUnit, inner: SyntaxTreeBuilder, - range_map: &'a mut ExpandedRangeMap, + range_map: ExpandedRangeMap, // Number of roots // Use for detect ill-form tree which is not single root @@ -328,16 +330,20 @@ struct TtTreeSink<'a> { } impl<'a> TtTreeSink<'a> { - fn new(cursor: Cursor<'a>, range_map: &'a mut ExpandedRangeMap) -> Self { + fn new(cursor: Cursor<'a>) -> Self { TtTreeSink { buf: String::new(), cursor, text_pos: 0.into(), inner: SyntaxTreeBuilder::default(), roots: smallvec::SmallVec::new(), - range_map, + range_map: ExpandedRangeMap::default(), } } + + fn finish(self) -> (Parse, ExpandedRangeMap) { + (self.inner.finish(), self.range_map) + } } fn delim_to_str(d: tt::Delimiter, closing: bool) -> SmolStr { -- cgit v1.2.3 From 604bdc6ffeea69dffed6d03f10b2fd4e05d04cda Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Tue, 5 Nov 2019 02:09:16 +0800 Subject: Use macro_rules shift to map text ranges --- crates/ra_mbe/src/lib.rs | 4 ++++ crates/ra_mbe/src/syntax_bridge.rs | 41 ++++++++++---------------------------- 2 files changed, 15 insertions(+), 30 deletions(-) (limited to 'crates/ra_mbe/src') diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index 2926b29fd..a0f1a6881 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -118,6 +118,10 @@ impl MacroRules { shift_subtree(&mut tt, self.shift); mbe_expander::expand(self, &tt) } + + pub fn shift(&self) -> u32 { + self.shift + } } impl Rule { diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 2f21ad6e0..0c499cd03 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -12,31 +12,12 @@ use tt::buffer::{Cursor, TokenBuffer}; use crate::subtree_source::SubtreeTokenSource; use crate::ExpandError; -use std::sync::atomic::{AtomicU32, Ordering}; /// Maps `tt::TokenId` to the relative range of the original token. -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Default)] pub struct TokenMap { /// Maps `tt::TokenId` to the *relative* source range. tokens: Vec, - map_id: u32, -} - -static TOKEN_MAP_COUNTER: AtomicU32 = AtomicU32::new(0); - -/// Generate an unique token map id for each instance -fn make_uniq_token_map_id() -> u32 { - let res = TOKEN_MAP_COUNTER.fetch_add(1, Ordering::SeqCst); - if res == std::u32::MAX { - panic!("TOKEN_MAP_COUNTER is overflowed"); - } - res -} - -impl std::default::Default for TokenMap { - fn default() -> TokenMap { - TokenMap { tokens: Default::default(), map_id: make_uniq_token_map_id() } - } } /// Maps relative range of the expanded syntax node to `tt::TokenId` @@ -140,17 +121,14 @@ pub fn token_tree_to_items( impl TokenMap { pub fn relative_range_of(&self, tt: tt::TokenId) -> Option { - if self.map_id != tt.map_id() { - return None; - } - let idx = tt.token_id() as usize; + let idx = tt.0 as usize; self.tokens.get(idx).copied() } fn alloc(&mut self, relative_range: TextRange) -> tt::TokenId { let id = self.tokens.len(); self.tokens.push(relative_range); - tt::TokenId::new(id as u32, self.map_id) + tt::TokenId(id as u32) } } @@ -159,14 +137,17 @@ impl ExpandedRangeMap { self.ranges.push((relative_range, token_id.clone())) } - pub fn ranges(&self, to: &TokenMap, start: TextUnit) -> Vec<(TextRange, TextRange)> { + pub fn map_ranges( + &self, + to: &TokenMap, + start: TextUnit, + shift: u32, + ) -> Vec<(TextRange, TextRange)> { self.ranges .iter() + .filter_map(|(r, tid)| if shift <= tid.0 { Some((r, tid.0 - shift)) } else { None }) .filter_map(|(r, tid)| { - if to.map_id != tid.map_id() { - return None; - } - if let Some(to_range) = to.relative_range_of(*tid) { + if let Some(to_range) = to.relative_range_of(tt::TokenId(tid)) { Some((*r, TextRange::from_to(to_range.start() + start, to_range.end() + start))) } else { None -- cgit v1.2.3 From 2a76bb5a0b70ae83824aa2e094ae2f592dcee694 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Tue, 5 Nov 2019 03:20:54 +0800 Subject: Refactor a bit --- crates/ra_mbe/src/syntax_bridge.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) (limited to 'crates/ra_mbe/src') diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 0c499cd03..02ddd25e7 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -133,7 +133,7 @@ impl TokenMap { } impl ExpandedRangeMap { - fn set(&mut self, relative_range: TextRange, token_id: &tt::TokenId) { + fn add(&mut self, relative_range: TextRange, token_id: &tt::TokenId) { self.ranges.push((relative_range, token_id.clone())) } @@ -145,13 +145,11 @@ impl ExpandedRangeMap { ) -> Vec<(TextRange, TextRange)> { self.ranges .iter() - .filter_map(|(r, tid)| if shift <= tid.0 { Some((r, tid.0 - shift)) } else { None }) .filter_map(|(r, tid)| { - if let Some(to_range) = to.relative_range_of(tt::TokenId(tid)) { - Some((*r, TextRange::from_to(to_range.start() + start, to_range.end() + start))) - } else { - None - } + let adjusted_id = tt::TokenId(tid.0.checked_sub(shift)?); + let to_range = to.relative_range_of(adjusted_id)?; + + Some((*r, TextRange::offset_len(to_range.start() + start, to_range.len()))) }) .collect() } @@ -361,8 +359,7 @@ impl<'a> TreeSink for TtTreeSink<'a> { self.text_pos + TextUnit::of_str(&self.buf), TextUnit::of_str(&ident.text), ); - let token_id = ident.id; - self.range_map.set(range, &token_id); + self.range_map.add(range, &ident.id); } } -- cgit v1.2.3 From 568f3cff41fd4ef49468b60f4343755a5b7b05cb Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Tue, 5 Nov 2019 09:43:59 +0800 Subject: Fix formatting --- crates/ra_mbe/src/syntax_bridge.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'crates/ra_mbe/src') diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 02ddd25e7..d897ac5de 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -148,7 +148,7 @@ impl ExpandedRangeMap { .filter_map(|(r, tid)| { let adjusted_id = tt::TokenId(tid.0.checked_sub(shift)?); let to_range = to.relative_range_of(adjusted_id)?; - + Some((*r, TextRange::offset_len(to_range.start() + start, to_range.len()))) }) .collect() -- cgit v1.2.3 From 62ed93db13877888379ea38c843729419bdf99d0 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Fri, 8 Nov 2019 09:57:01 +0800 Subject: Refactor and add more comments --- crates/ra_mbe/src/syntax_bridge.rs | 96 ++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 50 deletions(-) (limited to 'crates/ra_mbe/src') diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index d897ac5de..a49e63ace 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -22,7 +22,7 @@ pub struct TokenMap { /// Maps relative range of the expanded syntax node to `tt::TokenId` #[derive(Debug, PartialEq, Eq, Default)] -pub struct ExpandedRangeMap { +pub struct RevTokenMap { ranges: Vec<(TextRange, tt::TokenId)>, } @@ -58,7 +58,7 @@ pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, Toke fn fragment_to_syntax_node( tt: &tt::Subtree, fragment_kind: FragmentKind, -) -> Result<(Parse, ExpandedRangeMap), ExpandError> { +) -> Result<(Parse, RevTokenMap), ExpandError> { let tmp; let tokens = match tt { tt::Subtree { delimiter: tt::Delimiter::None, token_trees } => token_trees.as_slice(), @@ -79,44 +79,29 @@ fn fragment_to_syntax_node( Ok((parse, range_map)) } -/// Parses the token tree (result of macro expansion) to an expression -pub fn token_tree_to_expr( - tt: &tt::Subtree, -) -> Result<(Parse, ExpandedRangeMap), ExpandError> { - let (parse, map) = fragment_to_syntax_node(tt, Expr)?; - parse.cast().ok_or_else(|| crate::ExpandError::ConversionError).map(|p| (p, map)) -} - -/// Parses the token tree (result of macro expansion) to a Pattern -pub fn token_tree_to_pat( - tt: &tt::Subtree, -) -> Result<(Parse, ExpandedRangeMap), ExpandError> { - let (parse, map) = fragment_to_syntax_node(tt, Pattern)?; - parse.cast().ok_or_else(|| crate::ExpandError::ConversionError).map(|p| (p, map)) -} - -/// Parses the token tree (result of macro expansion) to a Type -pub fn token_tree_to_ty( - tt: &tt::Subtree, -) -> Result<(Parse, ExpandedRangeMap), ExpandError> { - let (parse, map) = fragment_to_syntax_node(tt, Type)?; - parse.cast().ok_or_else(|| crate::ExpandError::ConversionError).map(|p| (p, map)) -} - -/// Parses the token tree (result of macro expansion) as a sequence of stmts -pub fn token_tree_to_macro_stmts( - tt: &tt::Subtree, -) -> Result<(Parse, ExpandedRangeMap), ExpandError> { - let (parse, map) = fragment_to_syntax_node(tt, Statements)?; - parse.cast().ok_or_else(|| crate::ExpandError::ConversionError).map(|p| (p, map)) +macro_rules! impl_token_tree_conversions { + ($($(#[$attr:meta])* $name:ident => ($kind:ident, $t:ty) ),*) => { + $( + $(#[$attr])* + pub fn $name(tt: &tt::Subtree) -> Result<(Parse<$t>, RevTokenMap), ExpandError> { + let (parse, map) = fragment_to_syntax_node(tt, $kind)?; + parse.cast().ok_or_else(|| crate::ExpandError::ConversionError).map(|p| (p, map)) + } + )* + } } -/// Parses the token tree (result of macro expansion) as a sequence of items -pub fn token_tree_to_items( - tt: &tt::Subtree, -) -> Result<(Parse, ExpandedRangeMap), ExpandError> { - let (parse, map) = fragment_to_syntax_node(tt, Items)?; - parse.cast().ok_or_else(|| crate::ExpandError::ConversionError).map(|p| (p, map)) +impl_token_tree_conversions! { + /// Parses the token tree (result of macro expansion) to an expression + token_tree_to_expr => (Expr, ast::Expr), + /// Parses the token tree (result of macro expansion) to a Pattern + token_tree_to_pat => (Pattern, ast::Pat), + /// Parses the token tree (result of macro expansion) to a Type + token_tree_to_ty => (Type, ast::TypeRef), + /// Parses the token tree (result of macro expansion) as a sequence of stmts + token_tree_to_macro_stmts => (Statements, ast::MacroStmts), + /// Parses the token tree (result of macro expansion) as a sequence of items + token_tree_to_items => (Items, ast::MacroItems) } impl TokenMap { @@ -132,15 +117,28 @@ impl TokenMap { } } -impl ExpandedRangeMap { - fn add(&mut self, relative_range: TextRange, token_id: &tt::TokenId) { +impl RevTokenMap { + fn add(&mut self, relative_range: TextRange, token_id: tt::TokenId) { self.ranges.push((relative_range, token_id.clone())) } + /// Map a given token map to (Expanded syntax node, Input tokens) text-ranges pair + /// + /// This function do the following things: + /// + /// 1. Undo the increment of token-id `shift`: + /// When we output a token from from macro argument, we increased its id + /// by `shift` (so it's guaranteed to not to collide with anything from the definition) + /// We undo the increment here to rollback to its original token id. + /// 2. Offset the input tokens (`to`) by `parent` text-range: + /// We transforms the input tokens text-ranges from relative to original first token + /// to parent text-range + /// 3. Maps expanded tokens text-ranges to parent text-ranges + /// pub fn map_ranges( &self, to: &TokenMap, - start: TextUnit, + parent: TextRange, shift: u32, ) -> Vec<(TextRange, TextRange)> { self.ranges @@ -149,7 +147,7 @@ impl ExpandedRangeMap { let adjusted_id = tt::TokenId(tid.0.checked_sub(shift)?); let to_range = to.relative_range_of(adjusted_id)?; - Some((*r, TextRange::offset_len(to_range.start() + start, to_range.len()))) + Some((*r, TextRange::offset_len(to_range.start() + parent.start(), to_range.len()))) }) .collect() } @@ -301,7 +299,7 @@ struct TtTreeSink<'a> { cursor: Cursor<'a>, text_pos: TextUnit, inner: SyntaxTreeBuilder, - range_map: ExpandedRangeMap, + range_map: RevTokenMap, // Number of roots // Use for detect ill-form tree which is not single root @@ -316,11 +314,11 @@ impl<'a> TtTreeSink<'a> { text_pos: 0.into(), inner: SyntaxTreeBuilder::default(), roots: smallvec::SmallVec::new(), - range_map: ExpandedRangeMap::default(), + range_map: RevTokenMap::default(), } } - fn finish(self) -> (Parse, ExpandedRangeMap) { + fn finish(self) -> (Parse, RevTokenMap) { (self.inner.finish(), self.range_map) } } @@ -355,11 +353,9 @@ impl<'a> TreeSink for TtTreeSink<'a> { // Mark the range if needed if let tt::Leaf::Ident(ident) = leaf { if kind == IDENT { - let range = TextRange::offset_len( - self.text_pos + TextUnit::of_str(&self.buf), - TextUnit::of_str(&ident.text), - ); - self.range_map.add(range, &ident.id); + let range = + TextRange::offset_len(self.text_pos, TextUnit::of_str(&ident.text)); + self.range_map.add(range, ident.id); } } -- cgit v1.2.3 From d01e0abdb57c5be340934cb51e2f6b1a2e6c0373 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Sat, 9 Nov 2019 04:00:27 +0800 Subject: Refactor and simpfily --- crates/ra_mbe/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'crates/ra_mbe/src') diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index a0f1a6881..70a289f09 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -32,7 +32,7 @@ pub enum ExpandError { pub use crate::syntax_bridge::{ ast_to_token_tree, syntax_node_to_token_tree, token_tree_to_expr, token_tree_to_items, - token_tree_to_macro_stmts, token_tree_to_pat, token_tree_to_ty, TokenMap, + token_tree_to_macro_stmts, token_tree_to_pat, token_tree_to_ty, RevTokenMap, TokenMap, }; /// This struct contains AST for a single `macro_rules` definition. What might -- cgit v1.2.3 From 0a5ec69404a2556dd82e5bb00b295aebaa291f04 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Sat, 9 Nov 2019 12:00:46 +0800 Subject: Remove map_ranges in RevTokenMap --- crates/ra_mbe/src/syntax_bridge.rs | 32 +------------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) (limited to 'crates/ra_mbe/src') diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index a49e63ace..9653f7fef 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -23,7 +23,7 @@ pub struct TokenMap { /// Maps relative range of the expanded syntax node to `tt::TokenId` #[derive(Debug, PartialEq, Eq, Default)] pub struct RevTokenMap { - ranges: Vec<(TextRange, tt::TokenId)>, + pub ranges: Vec<(TextRange, tt::TokenId)>, } /// Convert the syntax tree (what user has written) to a `TokenTree` (what macro @@ -121,36 +121,6 @@ impl RevTokenMap { fn add(&mut self, relative_range: TextRange, token_id: tt::TokenId) { self.ranges.push((relative_range, token_id.clone())) } - - /// Map a given token map to (Expanded syntax node, Input tokens) text-ranges pair - /// - /// This function do the following things: - /// - /// 1. Undo the increment of token-id `shift`: - /// When we output a token from from macro argument, we increased its id - /// by `shift` (so it's guaranteed to not to collide with anything from the definition) - /// We undo the increment here to rollback to its original token id. - /// 2. Offset the input tokens (`to`) by `parent` text-range: - /// We transforms the input tokens text-ranges from relative to original first token - /// to parent text-range - /// 3. Maps expanded tokens text-ranges to parent text-ranges - /// - pub fn map_ranges( - &self, - to: &TokenMap, - parent: TextRange, - shift: u32, - ) -> Vec<(TextRange, TextRange)> { - self.ranges - .iter() - .filter_map(|(r, tid)| { - let adjusted_id = tt::TokenId(tid.0.checked_sub(shift)?); - let to_range = to.relative_range_of(adjusted_id)?; - - Some((*r, TextRange::offset_len(to_range.start() + parent.start(), to_range.len()))) - }) - .collect() - } } /// Returns the textual content of a doc comment block as a quoted string -- cgit v1.2.3 From 70f2a21b55c1b09e575798a75807e13991f2cfec Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 9 Nov 2019 13:14:10 +0300 Subject: Remove typed macro parsing API We do type-erasure on every path anyway, so it doesn't make much sense to duplicate this function for every type --- crates/ra_mbe/src/lib.rs | 3 +- crates/ra_mbe/src/syntax_bridge.rs | 34 ++----------- crates/ra_mbe/src/tests.rs | 102 ++++++++++++++++++++++++++----------- 3 files changed, 75 insertions(+), 64 deletions(-) (limited to 'crates/ra_mbe/src') diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index 70a289f09..8a31d1c36 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -31,8 +31,7 @@ pub enum ExpandError { } pub use crate::syntax_bridge::{ - ast_to_token_tree, syntax_node_to_token_tree, token_tree_to_expr, token_tree_to_items, - token_tree_to_macro_stmts, token_tree_to_pat, token_tree_to_ty, RevTokenMap, TokenMap, + ast_to_token_tree, syntax_node_to_token_tree, token_tree_to_syntax_node, RevTokenMap, TokenMap, }; /// This struct contains AST for a single `macro_rules` definition. What might diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 9653f7fef..3f57ce3b5 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -1,9 +1,6 @@ //! FIXME: write short doc here -use ra_parser::{ - FragmentKind::{self, *}, - ParseError, TreeSink, -}; +use ra_parser::{FragmentKind, ParseError, TreeSink}; use ra_syntax::{ ast, AstNode, AstToken, NodeOrToken, Parse, SmolStr, SyntaxKind, SyntaxKind::*, SyntaxNode, SyntaxTreeBuilder, TextRange, TextUnit, T, @@ -55,7 +52,7 @@ pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, Toke // * ImplItems(SmallVec<[ast::ImplItem; 1]>) // * ForeignItems(SmallVec<[ast::ForeignItem; 1]> -fn fragment_to_syntax_node( +pub fn token_tree_to_syntax_node( tt: &tt::Subtree, fragment_kind: FragmentKind, ) -> Result<(Parse, RevTokenMap), ExpandError> { @@ -79,31 +76,6 @@ fn fragment_to_syntax_node( Ok((parse, range_map)) } -macro_rules! impl_token_tree_conversions { - ($($(#[$attr:meta])* $name:ident => ($kind:ident, $t:ty) ),*) => { - $( - $(#[$attr])* - pub fn $name(tt: &tt::Subtree) -> Result<(Parse<$t>, RevTokenMap), ExpandError> { - let (parse, map) = fragment_to_syntax_node(tt, $kind)?; - parse.cast().ok_or_else(|| crate::ExpandError::ConversionError).map(|p| (p, map)) - } - )* - } -} - -impl_token_tree_conversions! { - /// Parses the token tree (result of macro expansion) to an expression - token_tree_to_expr => (Expr, ast::Expr), - /// Parses the token tree (result of macro expansion) to a Pattern - token_tree_to_pat => (Pattern, ast::Pat), - /// Parses the token tree (result of macro expansion) to a Type - token_tree_to_ty => (Type, ast::TypeRef), - /// Parses the token tree (result of macro expansion) as a sequence of stmts - token_tree_to_macro_stmts => (Statements, ast::MacroStmts), - /// Parses the token tree (result of macro expansion) as a sequence of items - token_tree_to_items => (Items, ast::MacroItems) -} - impl TokenMap { pub fn relative_range_of(&self, tt: tt::TokenId) -> Option { let idx = tt.0 as usize; @@ -446,6 +418,6 @@ mod tests { "#, ); let expansion = expand(&rules, "stmts!();"); - assert!(token_tree_to_expr(&expansion).is_err()); + assert!(token_tree_to_syntax_node(&expansion, FragmentKind::Expr).is_err()); } } diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs index a848ea334..0109a4d98 100644 --- a/crates/ra_mbe/src/tests.rs +++ b/crates/ra_mbe/src/tests.rs @@ -1,3 +1,4 @@ +use ra_parser::FragmentKind; use ra_syntax::{ast, AstNode, NodeOrToken, WalkEvent}; use test_utils::assert_eq_text; @@ -126,9 +127,9 @@ fn test_expr_order() { "#, ); let expanded = expand(&rules, "foo! { 1 + 1}"); - let tree = token_tree_to_items(&expanded).unwrap().0.tree(); + let tree = token_tree_to_syntax_node(&expanded, FragmentKind::Items).unwrap().0.syntax_node(); - let dump = format!("{:#?}", tree.syntax()); + let dump = format!("{:#?}", tree); assert_eq_text!( dump.trim(), r#"MACRO_ITEMS@[0; 15) @@ -383,9 +384,9 @@ fn test_expand_to_item_list() { ", ); let expansion = expand(&rules, "structs!(Foo, Bar);"); - let tree = token_tree_to_items(&expansion).unwrap().0.tree(); + let tree = token_tree_to_syntax_node(&expansion, FragmentKind::Items).unwrap().0.syntax_node(); assert_eq!( - format!("{:#?}", tree.syntax()).trim(), + format!("{:#?}", tree).trim(), r#" MACRO_ITEMS@[0; 40) STRUCT_DEF@[0; 20) @@ -501,10 +502,11 @@ fn test_tt_to_stmts() { ); let expanded = expand(&rules, "foo!{}"); - let stmts = token_tree_to_macro_stmts(&expanded).unwrap().0.tree(); + let stmts = + token_tree_to_syntax_node(&expanded, FragmentKind::Statements).unwrap().0.syntax_node(); assert_eq!( - format!("{:#?}", stmts.syntax()).trim(), + format!("{:#?}", stmts).trim(), r#"MACRO_STMTS@[0; 15) LET_STMT@[0; 7) LET_KW@[0; 3) "let" @@ -754,7 +756,10 @@ fn test_all_items() { } "#, ); - assert_expansion(MacroKind::Items, &rules, r#" + assert_expansion( + MacroKind::Items, + &rules, + r#" foo! { extern crate a; mod b; @@ -770,7 +775,9 @@ fn test_all_items() { extern {} type T = u8; } -"#, r#"extern crate a ; mod b ; mod c {} use d ; const E : i32 = 0 ; static F : i32 = 0 ; impl G {} struct H ; enum I {Foo} trait J {} fn h () {} extern {} type T = u8 ;"#); +"#, + r#"extern crate a ; mod b ; mod c {} use d ; const E : i32 = 0 ; static F : i32 = 0 ; impl G {} struct H ; enum I {Foo} trait J {} fn h () {} extern {} type T = u8 ;"#, + ); } #[test] @@ -946,10 +953,10 @@ fn test_vec() { ); let expansion = expand(&rules, r#"vec![1u32,2];"#); - let tree = token_tree_to_expr(&expansion).unwrap().0.tree(); + let tree = token_tree_to_syntax_node(&expansion, FragmentKind::Expr).unwrap().0.syntax_node(); assert_eq!( - format!("{:#?}", tree.syntax()).trim(), + format!("{:#?}", tree).trim(), r#"BLOCK_EXPR@[0; 45) BLOCK@[0; 45) L_CURLY@[0; 1) "{" @@ -1088,8 +1095,12 @@ macro_rules! generate_pattern_iterators { "#, ); - assert_expansion(MacroKind::Items, &rules, r#"generate_pattern_iterators ! ( double ended ; with # [ stable ( feature = "rust1" , since = "1.0.0" ) ] , Split , RSplit , & 'a str );"#, - "fn foo () {}"); + assert_expansion( + MacroKind::Items, + &rules, + r#"generate_pattern_iterators ! ( double ended ; with # [ stable ( feature = "rust1" , since = "1.0.0" ) ] , Split , RSplit , & 'a str );"#, + "fn foo () {}", + ); } #[test] @@ -1171,8 +1182,12 @@ fn test_impl_nonzero_fmt() { "#, ); - assert_expansion(MacroKind::Items, &rules, r#"impl_nonzero_fmt! { # [stable(feature= "nonzero",since="1.28.0")] (Debug,Display,Binary,Octal,LowerHex,UpperHex) for NonZeroU8}"#, - "fn foo () {}"); + assert_expansion( + MacroKind::Items, + &rules, + r#"impl_nonzero_fmt! { # [stable(feature= "nonzero",since="1.28.0")] (Debug,Display,Binary,Octal,LowerHex,UpperHex) for NonZeroU8}"#, + "fn foo () {}", + ); } #[test] @@ -1189,8 +1204,12 @@ fn test_cfg_if_items() { "#, ); - assert_expansion(MacroKind::Items, &rules, r#"__cfg_if_items ! { ( rustdoc , ) ; ( ( ) ( # [ cfg ( any ( target_os = "redox" , unix ) ) ] # [ stable ( feature = "rust1" , since = "1.0.0" ) ] pub use sys :: ext as unix ; # [ cfg ( windows ) ] # [ stable ( feature = "rust1" , since = "1.0.0" ) ] pub use sys :: ext as windows ; # [ cfg ( any ( target_os = "linux" , target_os = "l4re" ) ) ] pub mod linux ; ) ) , }"#, - "__cfg_if_items ! {(rustdoc ,) ;}"); + assert_expansion( + MacroKind::Items, + &rules, + r#"__cfg_if_items ! { ( rustdoc , ) ; ( ( ) ( # [ cfg ( any ( target_os = "redox" , unix ) ) ] # [ stable ( feature = "rust1" , since = "1.0.0" ) ] pub use sys :: ext as unix ; # [ cfg ( windows ) ] # [ stable ( feature = "rust1" , since = "1.0.0" ) ] pub use sys :: ext as windows ; # [ cfg ( any ( target_os = "linux" , target_os = "l4re" ) ) ] pub mod linux ; ) ) , }"#, + "__cfg_if_items ! {(rustdoc ,) ;}", + ); } #[test] @@ -1233,10 +1252,13 @@ cfg_if ! { "#, "__cfg_if_items ! {() ; ((target_env = \"msvc\") ()) , ((all (target_arch = \"wasm32\" , not (target_os = \"emscripten\"))) ()) , (() (mod libunwind ; pub use libunwind :: * ;)) ,}"); - assert_expansion(MacroKind::Items, &rules, r#" + assert_expansion( + MacroKind::Items, + &rules, + r#" cfg_if ! { @ __apply cfg ( all ( not ( any ( not ( any ( target_os = "solaris" , target_os = "illumos" ) ) ) ) ) ) , } "#, - "" + "", ); } @@ -1291,10 +1313,13 @@ macro_rules! RIDL { }"#, ); - let expanded = expand(&rules, r#" + let expanded = expand( + &rules, + r#" RIDL!{interface ID3D11Asynchronous(ID3D11AsynchronousVtbl): ID3D11DeviceChild(ID3D11DeviceChildVtbl) { fn GetDataSize(&mut self) -> UINT -}}"#); +}}"#, + ); assert_eq!(expanded.to_string(), "impl ID3D11Asynchronous {pub unsafe fn GetDataSize (& mut self) -> UINT {((* self . lpVtbl) .GetDataSize) (self)}}"); } @@ -1340,7 +1365,8 @@ quick_error ! (SORT [enum Wrapped # [derive (Debug)]] items [ #[test] fn test_empty_repeat_vars_in_empty_repeat_vars() { - let rules = create_rules(r#" + let rules = create_rules( + r#" macro_rules! delegate_impl { ([$self_type:ident, $self_wrap:ty, $self_map:ident] pub trait $name:ident $(: $sup:ident)* $(+ $more_sup:ident)* { @@ -1385,9 +1411,15 @@ macro_rules! delegate_impl { } } } -"#); +"#, + ); - assert_expansion(MacroKind::Items, &rules, r#"delegate_impl ! {[G , & 'a mut G , deref] pub trait Data : GraphBase {@ section type type NodeWeight ;}}"#, "impl <> Data for & \'a mut G where G : Data {}"); + assert_expansion( + MacroKind::Items, + &rules, + r#"delegate_impl ! {[G , & 'a mut G , deref] pub trait Data : GraphBase {@ section type type NodeWeight ;}}"#, + "impl <> Data for & \'a mut G where G : Data {}", + ); } pub(crate) fn create_rules(macro_definition: &str) -> MacroRules { @@ -1436,22 +1468,30 @@ pub(crate) fn assert_expansion( }; let (expanded_tree, expected_tree) = match kind { MacroKind::Items => { - let expanded_tree = token_tree_to_items(&expanded).unwrap().0.tree(); - let expected_tree = token_tree_to_items(&expected).unwrap().0.tree(); + let expanded_tree = + token_tree_to_syntax_node(&expanded, FragmentKind::Items).unwrap().0.syntax_node(); + let expected_tree = + token_tree_to_syntax_node(&expected, FragmentKind::Items).unwrap().0.syntax_node(); ( - debug_dump_ignore_spaces(expanded_tree.syntax()).trim().to_string(), - debug_dump_ignore_spaces(expected_tree.syntax()).trim().to_string(), + debug_dump_ignore_spaces(&expanded_tree).trim().to_string(), + debug_dump_ignore_spaces(&expected_tree).trim().to_string(), ) } MacroKind::Stmts => { - let expanded_tree = token_tree_to_macro_stmts(&expanded).unwrap().0.tree(); - let expected_tree = token_tree_to_macro_stmts(&expected).unwrap().0.tree(); + let expanded_tree = token_tree_to_syntax_node(&expanded, FragmentKind::Statements) + .unwrap() + .0 + .syntax_node(); + let expected_tree = token_tree_to_syntax_node(&expected, FragmentKind::Statements) + .unwrap() + .0 + .syntax_node(); ( - debug_dump_ignore_spaces(expanded_tree.syntax()).trim().to_string(), - debug_dump_ignore_spaces(expected_tree.syntax()).trim().to_string(), + debug_dump_ignore_spaces(&expanded_tree).trim().to_string(), + debug_dump_ignore_spaces(&expected_tree).trim().to_string(), ) } }; -- cgit v1.2.3