From c55a2dbc1de8ba42df57b70f652eb6a0c0bbc9f6 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Thu, 25 Apr 2019 23:12:57 +0800 Subject: Fix more bugs --- crates/ra_mbe/Cargo.toml | 1 + crates/ra_mbe/src/lib.rs | 112 +++++++++++++++++++++++++++++++++++- crates/ra_mbe/src/mbe_expander.rs | 69 +++++++++++++++++----- crates/ra_mbe/src/subtree_source.rs | 9 +-- crates/ra_mbe/src/syntax_bridge.rs | 16 +++++- crates/ra_mbe/src/tt_cursor.rs | 2 +- crates/ra_parser/src/grammar.rs | 17 +++++- 7 files changed, 202 insertions(+), 24 deletions(-) (limited to 'crates') diff --git a/crates/ra_mbe/Cargo.toml b/crates/ra_mbe/Cargo.toml index 1e5ed6907..68f559295 100644 --- a/crates/ra_mbe/Cargo.toml +++ b/crates/ra_mbe/Cargo.toml @@ -11,3 +11,4 @@ tt = { path = "../ra_tt", package = "ra_tt" } itertools = "0.8.0" rustc-hash = "1.0.0" smallvec = "0.6.9" +log = "0.4.5" diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index 7ebba807c..7817232d6 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -436,6 +436,30 @@ impl_froms!(TokenTree: Leaf, Subtree); assert_expansion(&rules, "foo! (fn baz {true true} )", "fn baz () -> bool {true &&true}"); } + #[test] + fn test_match_group_zero_match() { + let rules = create_rules( + r#" + macro_rules! foo { + ( $($i:ident)* ) => (); + }"#, + ); + + assert_expansion(&rules, "foo! ()", ""); + } + + #[test] + fn test_match_group_in_group() { + let rules = create_rules( + r#" + macro_rules! foo { + { $( ( $($i:ident)* ) )* } => ( $( ( $($i)* ) )* ); + }"#, + ); + + assert_expansion(&rules, "foo! ( (a b) )", "(a b)"); + } + #[test] fn test_expand_to_item_list() { let rules = create_rules( @@ -1118,7 +1142,37 @@ macro_rules! impl_fn_for_zst { |$( $arg: ident: $ArgTy: ty ),*| -> $ReturnTy: ty $body: block; )+ } => { - fn foo(){} + $( + $( #[$attr] )* + struct $Name; + + impl $( <$( $lifetime ),+> )? Fn<($( $ArgTy, )*)> for $Name { + #[inline] + extern "rust-call" fn call(&self, ($( $arg, )*): ($( $ArgTy, )*)) -> $ReturnTy { + $body + } + } + + impl $( <$( $lifetime ),+> )? FnMut<($( $ArgTy, )*)> for $Name { + #[inline] + extern "rust-call" fn call_mut( + &mut self, + ($( $arg, )*): ($( $ArgTy, )*) + ) -> $ReturnTy { + Fn::call(&*self, ($( $arg, )*)) + } + } + + impl $( <$( $lifetime ),+> )? FnOnce<($( $ArgTy, )*)> for $Name { + type Output = $ReturnTy; + + #[inline] + extern "rust-call" fn call_once(self, ($( $arg, )*): ($( $ArgTy, )*)) -> $ReturnTy { + Fn::call(&self, ($( $arg, )*)) + } + } + )+ +} } } "#, @@ -1141,7 +1195,7 @@ impl_fn_for_zst ! { } ; } "#, - "fn foo () {}"); + "# [derive (Clone)] struct CharEscapeDebugContinue ; impl Fn < (char ,) > for CharEscapeDebugContinue {# [inline] extern \"rust-call\" fn call (& self , (c ,) : (char ,)) -> char :: EscapeDebug {{c . escape_debug_ext (false)}}} impl FnMut < (char ,) > for CharEscapeDebugContinue {# [inline] extern \"rust-call\" fn call_mut (& mut self , (c ,) : (char ,)) -> char :: EscapeDebug {Fn :: call (&* self , (c ,))}} impl FnOnce < (char ,) > for CharEscapeDebugContinue {type Output = char :: EscapeDebug ; # [inline] extern \"rust-call\" fn call_once (self , (c ,) : (char ,)) -> char :: EscapeDebug {Fn :: call (& self , (c ,))}} # [derive (Clone)] struct CharEscapeUnicode ; impl Fn < (char ,) > for CharEscapeUnicode {# [inline] extern \"rust-call\" fn call (& self , (c ,) : (char ,)) -> char :: EscapeUnicode {{c . escape_unicode ()}}} impl FnMut < (char ,) > for CharEscapeUnicode {# [inline] extern \"rust-call\" fn call_mut (& mut self , (c ,) : (char ,)) -> char :: EscapeUnicode {Fn :: call (&* self , (c ,))}} impl FnOnce < (char ,) > for CharEscapeUnicode {type Output = char :: EscapeUnicode ; # [inline] extern \"rust-call\" fn call_once (self , (c ,) : (char ,)) -> char :: EscapeUnicode {Fn :: call (& self , (c ,))}} # [derive (Clone)] struct CharEscapeDefault ; impl Fn < (char ,) > for CharEscapeDefault {# [inline] extern \"rust-call\" fn call (& self , (c ,) : (char ,)) -> char :: EscapeDefault {{c . escape_default ()}}} impl FnMut < (char ,) > for CharEscapeDefault {# [inline] extern \"rust-call\" fn call_mut (& mut self , (c ,) : (char ,)) -> char :: EscapeDefault {Fn :: call (&* self , (c ,))}} impl FnOnce < (char ,) > for CharEscapeDefault {type Output = char :: EscapeDefault ; # [inline] extern \"rust-call\" fn call_once (self , (c ,) : (char ,)) -> char :: EscapeDefault {Fn :: call (& self , (c ,))}}"); } #[test] @@ -1160,4 +1214,58 @@ impl_fn_for_zst ! { assert_expansion(&rules, r#"impl_nonzero_fmt ! { # [ stable ( feature = "nonzero" , since = "1.28.0" ) ] ( Debug , Display , Binary , Octal , LowerHex , UpperHex ) for NonZeroU8 }"#, "fn foo () {}"); } + + #[test] + fn test_cfg_if_items() { + // from https://github.com/rust-lang/rust/blob/33fe1131cadba69d317156847be9a402b89f11bb/src/libstd/macros.rs#L986 + let rules = create_rules( + r#" + macro_rules! __cfg_if_items { + (($($not:meta,)*) ; ) => {}; + (($($not:meta,)*) ; ( ($($m:meta),*) ($($it:item)*) ), $($rest:tt)*) => { + __cfg_if_items! { ($($not,)* $($m,)*) ; $($rest)* } + } + } +"#, + ); + + assert_expansion(&rules, r#"__cfg_if_items ! { ( rustdoc , ) ; ( ( ) ( # [ cfg ( any ( target_os = "redox" , unix ) ) ] # [ stable ( feature = "rust1" , since = "1.0.0" ) ] pub use sys :: ext as unix ; # [ cfg ( windows ) ] # [ stable ( feature = "rust1" , since = "1.0.0" ) ] pub use sys :: ext as windows ; # [ cfg ( any ( target_os = "linux" , target_os = "l4re" ) ) ] pub mod linux ; ) ) , }"#, + "__cfg_if_items ! {(rustdoc , ) ; }"); + } + + #[test] + fn test_cfg_if_main() { + // from https://github.com/rust-lang/rust/blob/3d211248393686e0f73851fc7548f6605220fbe1/src/libpanic_unwind/macros.rs#L9 + let rules = create_rules( + r#" + macro_rules! cfg_if { + ($( + if #[cfg($($meta:meta),*)] { $($it:item)* } + ) else * else { + $($it2:item)* + }) => { + __cfg_if_items! { + () ; + $( ( ($($meta),*) ($($it)*) ), )* + ( () ($($it2)*) ), + } + } + } +"#, + ); + + assert_expansion(&rules, r#" +cfg_if ! { + if # [ cfg ( target_env = "msvc" ) ] { + // no extra unwinder support needed + } else if # [ cfg ( all ( target_arch = "wasm32" , not ( target_os = "emscripten" ) ) ) ] { + // no unwinder on the system! + } else { + mod libunwind ; + pub use libunwind :: * ; + } + } +"#, + "__cfg_if_items ! {() ; (() (mod libunwind ; pub use libunwind :: * ;)) ,}"); + } } diff --git a/crates/ra_mbe/src/mbe_expander.rs b/crates/ra_mbe/src/mbe_expander.rs index 7411dd8b1..d5189b537 100644 --- a/crates/ra_mbe/src/mbe_expander.rs +++ b/crates/ra_mbe/src/mbe_expander.rs @@ -21,7 +21,10 @@ fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> Result Result { // Dirty hack to make macro-expansion terminate. // This should be replaced by a propper macro-by-example implementation - let mut limit = 128; + let mut limit = 65536; let mut counter = 0; let mut memento = input.save(); @@ -236,6 +239,7 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result {:#?}\n{:#?}\n{:#?}\n{:#?}", subtree, input, kind, separator); break; } @@ -303,15 +307,21 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result { + bindings: &'a Bindings, + nesting: Vec, + var_expanded: bool, +} + fn expand_subtree( template: &crate::Subtree, - bindings: &Bindings, - nesting: &mut Vec, + ctx: &mut ExpandCtx, ) -> Result { let token_trees = template .token_trees .iter() - .map(|it| expand_tt(it, bindings, nesting)) + .map(|it| expand_tt(it, ctx)) .collect::, ExpandError>>()?; Ok(tt::Subtree { token_trees, delimiter: template.delimiter }) @@ -333,26 +343,43 @@ fn reduce_single_token(mut subtree: tt::Subtree) -> tt::TokenTree { fn expand_tt( template: &crate::TokenTree, - bindings: &Bindings, - nesting: &mut Vec, + ctx: &mut ExpandCtx, ) -> Result { let res: tt::TokenTree = match template { - crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, bindings, nesting)?.into(), + crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, ctx)?.into(), crate::TokenTree::Repeat(repeat) => { let mut token_trees: Vec = Vec::new(); - nesting.push(0); + ctx.nesting.push(0); // Dirty hack to make macro-expansion terminate. // This should be replaced by a propper macro-by-example implementation - let mut limit = 128; + let mut limit = 65536; let mut has_seps = 0; + let mut counter = 0; + + let mut some_var_expanded = false; + ctx.var_expanded = false; + + while let Ok(t) = expand_subtree(&repeat.subtree, ctx) { + // if no var expaned in the child, we count it as a fail + if !ctx.var_expanded { + break; + } + some_var_expanded = true; + ctx.var_expanded = false; - while let Ok(t) = expand_subtree(&repeat.subtree, bindings, nesting) { + counter += 1; limit -= 1; if limit == 0 { + log::warn!( + "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}", + template, + ctx + ); break; } - let idx = nesting.pop().unwrap(); - nesting.push(idx + 1); + + let idx = ctx.nesting.pop().unwrap(); + ctx.nesting.push(idx + 1); token_trees.push(reduce_single_token(t).into()); if let Some(ref sep) = repeat.separator { @@ -374,12 +401,23 @@ fn expand_tt( } } } + + if let crate::RepeatKind::ZeroOrOne = repeat.kind { + break; + } } - nesting.pop().unwrap(); + + ctx.var_expanded = some_var_expanded; + + ctx.nesting.pop().unwrap(); for _ in 0..has_seps { token_trees.pop(); } + if crate::RepeatKind::OneOrMore == repeat.kind && counter == 0 { + return Err(ExpandError::UnexpectedToken); + } + // Check if it is a singel token subtree without any delimiter // e.g {Delimiter:None> ['>'] /Delimiter:None>} reduce_single_token(tt::Subtree { token_trees, delimiter: tt::Delimiter::None }) @@ -396,7 +434,8 @@ fn expand_tt( tt::Leaf::from(tt::Ident { text: "$crate".into(), id: TokenId::unspecified() }) .into() } else { - let tkn = bindings.get(&v.text, nesting)?.clone(); + let tkn = ctx.bindings.get(&v.text, &ctx.nesting)?.clone(); + ctx.var_expanded = true; if let tt::TokenTree::Subtree(subtree) = tkn { reduce_single_token(subtree) diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs index e979777fe..845020f4e 100644 --- a/crates/ra_mbe/src/subtree_source.rs +++ b/crates/ra_mbe/src/subtree_source.rs @@ -212,7 +212,7 @@ impl<'a> SubTreeWalker<'a> { } pub(crate) trait Querier { - fn token(&self, uidx: usize) -> (SyntaxKind, SmolStr); + fn token(&self, uidx: usize) -> (SyntaxKind, SmolStr, bool); } // A wrapper class for ref cell @@ -292,9 +292,10 @@ impl<'a> WalkerOwner<'a> { } impl<'a> Querier for WalkerOwner<'a> { - fn token(&self, uidx: usize) -> (SyntaxKind, SmolStr) { - let tkn = self.get(uidx).unwrap(); - (tkn.kind, tkn.text) + fn token(&self, uidx: usize) -> (SyntaxKind, SmolStr, bool) { + self.get(uidx) + .map(|tkn| (tkn.kind, tkn.text, tkn.is_joint_to_next)) + .unwrap_or_else(|| (SyntaxKind::EOF, "".into(), false)) } } diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 9cca19dbb..e0f228ce9 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -123,6 +123,11 @@ fn convert_tt( global_offset: TextUnit, tt: &SyntaxNode, ) -> Option { + // This tree is empty + if tt.first_child_or_token().is_none() { + return Some(tt::Subtree { token_trees: vec![], delimiter: tt::Delimiter::None }); + } + let first_child = tt.first_child_or_token()?; let last_child = tt.last_child_or_token()?; let (delimiter, skip_first) = match (first_child.kind(), last_child.kind()) { @@ -233,7 +238,16 @@ impl<'a, Q: Querier> TreeSink for TtTreeSink<'a, Q> { self.text_pos += TextUnit::of_str(&self.buf); let text = SmolStr::new(self.buf.as_str()); self.buf.clear(); - self.inner.token(kind, text) + self.inner.token(kind, text); + + // // Add a white space to token + // let (last_kind, _, last_joint_to_next ) = self.src_querier.token(self.token_pos-n_tokens as usize); + // if !last_joint_to_next && last_kind.is_punct() { + // let (cur_kind, _, _ ) = self.src_querier.token(self.token_pos); + // if cur_kind.is_punct() { + // self.inner.token(WHITESPACE, " ".into()); + // } + // } } fn start_node(&mut self, kind: SyntaxKind) { diff --git a/crates/ra_mbe/src/tt_cursor.rs b/crates/ra_mbe/src/tt_cursor.rs index 343416421..eef642a9c 100644 --- a/crates/ra_mbe/src/tt_cursor.rs +++ b/crates/ra_mbe/src/tt_cursor.rs @@ -3,7 +3,7 @@ use crate::subtree_parser::Parser; use crate::subtree_source::TokenPeek; use smallvec::{SmallVec, smallvec}; -#[derive(Clone)] +#[derive(Debug, Clone)] pub(crate) struct TtCursor<'a> { subtree: &'a tt::Subtree, pos: usize, diff --git a/crates/ra_parser/src/grammar.rs b/crates/ra_parser/src/grammar.rs index 67eae749d..a538ec081 100644 --- a/crates/ra_parser/src/grammar.rs +++ b/crates/ra_parser/src/grammar.rs @@ -119,7 +119,22 @@ pub(crate) fn meta_item(p: &mut Parser) { items::token_tree(p); break; } else { - p.bump(); + // https://doc.rust-lang.org/reference/attributes.html + // https://doc.rust-lang.org/reference/paths.html#simple-paths + // The start of an meta must be a simple path + match p.current() { + IDENT | COLONCOLON | SUPER_KW | SELF_KW | CRATE_KW => p.bump(), + EQ => { + p.bump(); + match p.current() { + c if c.is_literal() => p.bump(), + TRUE_KW | FALSE_KW => p.bump(), + _ => {} + } + break; + } + _ => break, + } } } -- cgit v1.2.3