From da18f1130756c4fdd611d95e4f98e553b4b65995 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Sat, 18 Apr 2020 19:28:07 +0800 Subject: Split LIFETIME to two tokens in mbe --- crates/ra_mbe/src/mbe_expander/matcher.rs | 34 +++++++++++++++++++++------- crates/ra_mbe/src/subtree_source.rs | 32 +++++++++++++++++++++++++- crates/ra_mbe/src/syntax_bridge.rs | 37 ++++++++++++++++++++++++++----- 3 files changed, 89 insertions(+), 14 deletions(-) (limited to 'crates/ra_mbe') diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs index 2579382da..9485c62b8 100644 --- a/crates/ra_mbe/src/mbe_expander/matcher.rs +++ b/crates/ra_mbe/src/mbe_expander/matcher.rs @@ -202,6 +202,13 @@ impl<'a> TtIter<'a> { } pub(crate) fn expect_tt(&mut self) -> Result { + match self.peek_n(0) { + Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '\'' => { + return self.expect_lifetime(); + } + _ => (), + } + let tt = self.next().ok_or_else(|| ())?.clone(); let punct = match tt { tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if punct.spacing == tt::Spacing::Joint => { @@ -255,13 +262,21 @@ impl<'a> TtIter<'a> { } } - pub(crate) fn expect_lifetime(&mut self) -> Result<&tt::Ident, ()> { - let ident = self.expect_ident()?; - // check if it start from "`" - if !ident.text.starts_with('\'') { + pub(crate) fn expect_lifetime(&mut self) -> Result { + let punct = self.expect_punct()?; + if punct.char != '\'' { return Err(()); } - Ok(ident) + let ident = self.expect_ident()?; + + Ok(tt::Subtree { + delimiter: None, + token_trees: vec![ + tt::Leaf::Punct(punct.clone()).into(), + tt::Leaf::Ident(ident.clone()).into(), + ], + } + .into()) } pub(crate) fn expect_fragment( @@ -274,7 +289,10 @@ impl<'a> TtIter<'a> { } impl<'a> TreeSink for OffsetTokenSink<'a> { - fn token(&mut self, _kind: SyntaxKind, n_tokens: u8) { + fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) { + if kind == SyntaxKind::LIFETIME { + n_tokens = 2; + } for _ in 0..n_tokens { self.cursor = self.cursor.bump_subtree(); } @@ -286,7 +304,7 @@ impl<'a> TtIter<'a> { } } - let buffer = TokenBuffer::new(self.inner.as_slice()); + let buffer = TokenBuffer::new(&self.inner.as_slice()); let mut src = SubtreeTokenSource::new(&buffer); let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false }; @@ -422,7 +440,7 @@ fn match_meta_var(kind: &str, input: &mut TtIter) -> ExpandResult input.expect_tt().map(Some).map_err(|()| err!()), "lifetime" => input .expect_lifetime() - .map(|ident| Some(tt::Leaf::Ident(ident.clone()).into())) + .map(|tt| Some(tt)) .map_err(|()| err!("expected lifetime")), "literal" => input .expect_literal() diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs index 91e324db9..46791efaa 100644 --- a/crates/ra_mbe/src/subtree_source.rs +++ b/crates/ra_mbe/src/subtree_source.rs @@ -50,6 +50,26 @@ impl<'a> SubtreeTokenSource<'a> { } fn get(&self, pos: usize) -> Ref> { + fn is_lifetime(c: Cursor) -> Option<(Cursor, SmolStr)> { + let tkn = c.token_tree(); + + if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = tkn { + if punct.char == '\'' { + let next = c.bump(); + if let Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) = next.token_tree() { + let res_cursor = next.bump(); + let text = SmolStr::new("'".to_string() + &ident.to_string()); + + return Some((res_cursor, text)); + } else { + panic!("Next token must be ident : {:#?}", next.token_tree()); + } + } + } + + None + } + if pos < self.cached.borrow().len() { return Ref::map(self.cached.borrow(), |c| &c[pos]); } @@ -63,6 +83,12 @@ impl<'a> SubtreeTokenSource<'a> { continue; } + if let Some((curr, text)) = is_lifetime(cursor) { + cached.push(Some(TtToken { kind: LIFETIME, is_joint_to_next: false, text })); + self.cached_cursor.set(curr); + continue; + } + match cursor.token_tree() { Some(tt::TokenTree::Leaf(leaf)) => { cached.push(Some(convert_leaf(&leaf))); @@ -152,7 +178,11 @@ fn convert_ident(ident: &tt::Ident) -> TtToken { } fn convert_punct(p: tt::Punct) -> TtToken { - let kind = SyntaxKind::from_char(p.char).unwrap(); + let kind = match SyntaxKind::from_char(p.char) { + None => panic!("{:#?} is not a valid punct", p), + Some(kind) => kind, + }; + let text = { let mut buf = [0u8; 4]; let s: &str = p.char.encode_utf8(&mut buf); diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 31e9b22e7..70899bc5d 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -271,7 +271,7 @@ struct RawConvertor<'a> { inner: std::slice::Iter<'a, RawToken>, } -trait SrcToken { +trait SrcToken: std::fmt::Debug { fn kind(&self) -> SyntaxKind; fn to_char(&self) -> Option; @@ -361,8 +361,12 @@ trait TokenConvertor { Some(next) if next.kind().is_punct() => tt::Spacing::Joint, _ => tt::Spacing::Alone, }; - let char = token.to_char().expect("Token from lexer must be single char"); - + let char = match token.to_char() { + Some(c) => c, + None => { + panic!("Token from lexer must be single char: token = {:#?}", token); + } + }; tt::Leaf::from(tt::Punct { char, spacing, id: self.id_alloc().alloc(range) }).into() } } else { @@ -373,9 +377,28 @@ trait TokenConvertor { } let leaf: tt::Leaf = match k { T![true] | T![false] => make_leaf!(Literal), - IDENT | LIFETIME => make_leaf!(Ident), + IDENT => make_leaf!(Ident), k if k.is_keyword() => make_leaf!(Ident), k if k.is_literal() => make_leaf!(Literal), + LIFETIME => { + let char_unit = TextUnit::from_usize(1); + let r = TextRange::offset_len(range.start(), char_unit); + let apostrophe = tt::Leaf::from(tt::Punct { + char: '\'', + spacing: tt::Spacing::Joint, + id: self.id_alloc().alloc(r), + }); + result.push(apostrophe.into()); + + let r = + TextRange::offset_len(range.start() + char_unit, range.len() - char_unit); + let ident = tt::Leaf::from(tt::Ident { + text: SmolStr::new(&token.to_text()[1..]), + id: self.id_alloc().alloc(r), + }); + result.push(ident.into()); + return; + } _ => return, }; @@ -455,6 +478,7 @@ impl Convertor { } } +#[derive(Debug)] enum SynToken { Ordiniary(SyntaxToken), Punch(SyntaxToken, TextUnit), @@ -592,11 +616,14 @@ fn delim_to_str(d: Option, closing: bool) -> SmolStr { } impl<'a> TreeSink for TtTreeSink<'a> { - fn token(&mut self, kind: SyntaxKind, n_tokens: u8) { + fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) { if kind == L_DOLLAR || kind == R_DOLLAR { self.cursor = self.cursor.bump_subtree(); return; } + if kind == LIFETIME { + n_tokens = 2; + } let mut last = self.cursor; for _ in 0..n_tokens { -- cgit v1.2.3 From ce674be2176e14e0d57c3a3e9edc315304aa2fde Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Sun, 19 Apr 2020 02:45:17 +0800 Subject: Add mbe lifetime split test --- crates/ra_mbe/src/tests.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'crates/ra_mbe') diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs index 5d1274d21..f2a726538 100644 --- a/crates/ra_mbe/src/tests.rs +++ b/crates/ra_mbe/src/tests.rs @@ -214,6 +214,33 @@ SUBTREE $ ); } +#[test] +fn test_lifetime_split() { + parse_macro( + r#" +macro_rules! foo { + ($($t:tt)*) => { $($t)*} +} +"#, + ) + .assert_expand( + r#"foo!(static bar: &'static str = "hello";);"#, + r#" +SUBTREE $ + IDENT static 17 + IDENT bar 18 + PUNCH : [alone] 19 + PUNCH & [alone] 20 + PUNCH ' [joint] 21 + IDENT static 22 + IDENT str 23 + PUNCH = [alone] 24 + LITERAL "hello" 25 + PUNCH ; [joint] 26 +"#, + ); +} + #[test] fn test_expr_order() { let expanded = parse_macro( -- cgit v1.2.3 From a1b5cf81ebcac15299cc612b49023bb418507027 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Sun, 19 Apr 2020 03:24:17 +0800 Subject: Convert bool to ident instead of literal in mbe --- crates/ra_mbe/src/mbe_expander/matcher.rs | 6 +++++- crates/ra_mbe/src/subtree_source.rs | 15 ++++++--------- crates/ra_mbe/src/syntax_bridge.rs | 2 +- crates/ra_mbe/src/tests.rs | 30 ++++++++++++++++++++++++++++++ crates/ra_mbe/src/tt_iter.rs | 8 +++++--- 5 files changed, 47 insertions(+), 14 deletions(-) (limited to 'crates/ra_mbe') diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs index 9485c62b8..78f9efa1b 100644 --- a/crates/ra_mbe/src/mbe_expander/matcher.rs +++ b/crates/ra_mbe/src/mbe_expander/matcher.rs @@ -187,7 +187,11 @@ impl<'a> TtIter<'a> { _ => false, }, Separator::Literal(lhs) => match fork.expect_literal() { - Ok(rhs) => rhs.text == lhs.text, + Ok(rhs) => match rhs { + tt::Leaf::Literal(rhs) => rhs.text == lhs.text, + tt::Leaf::Ident(rhs) => rhs.text == lhs.text, + tt::Leaf::Punct(_) => false, + }, _ => false, }, Separator::Puncts(lhss) => lhss.iter().all(|lhs| match fork.expect_punct() { diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs index 46791efaa..d7866452d 100644 --- a/crates/ra_mbe/src/subtree_source.rs +++ b/crates/ra_mbe/src/subtree_source.rs @@ -158,20 +158,17 @@ fn convert_literal(l: &tt::Literal) -> TtToken { let kind = lex_single_syntax_kind(&l.text) .map(|(kind, _error)| kind) .filter(|kind| kind.is_literal()) - .unwrap_or_else(|| match l.text.as_ref() { - "true" => T![true], - "false" => T![false], - _ => panic!("Fail to convert given literal {:#?}", &l), - }); + .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &l)); TtToken { kind, is_joint_to_next: false, text: l.text.clone() } } fn convert_ident(ident: &tt::Ident) -> TtToken { - let kind = if ident.text.starts_with('\'') { - LIFETIME - } else { - SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT) + let kind = match ident.text.as_ref() { + "true" => T![true], + "false" => T![false], + i if i.starts_with('\'') => LIFETIME, + _ => SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT), }; TtToken { kind, is_joint_to_next: false, text: ident.text.clone() } diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 70899bc5d..2b4390eb2 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -376,7 +376,7 @@ trait TokenConvertor { }; } let leaf: tt::Leaf = match k { - T![true] | T![false] => make_leaf!(Literal), + T![true] | T![false] => make_leaf!(Ident), IDENT => make_leaf!(Ident), k if k.is_keyword() => make_leaf!(Ident), k if k.is_literal() => make_leaf!(Literal), diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs index f2a726538..100ed41f2 100644 --- a/crates/ra_mbe/src/tests.rs +++ b/crates/ra_mbe/src/tests.rs @@ -1015,6 +1015,36 @@ fn test_literal() { .assert_expand_items(r#"foo!(u8 0);"#, r#"const VALUE : u8 = 0 ;"#); } +#[test] +fn test_boolean_is_ident() { + parse_macro( + r#" + macro_rules! foo { + ($lit0:literal, $lit1:literal) => { const VALUE: (bool,bool) = ($lit0,$lit1); }; + } +"#, + ) + .assert_expand( + r#"foo!(true,false);"#, + r#" +SUBTREE $ + IDENT const 14 + IDENT VALUE 15 + PUNCH : [alone] 16 + SUBTREE () 17 + IDENT bool 18 + PUNCH , [alone] 19 + IDENT bool 20 + PUNCH = [alone] 21 + SUBTREE () 22 + IDENT true 29 + PUNCH , [joint] 25 + IDENT false 31 + PUNCH ; [alone] 28 +"#, + ); +} + #[test] fn test_vis() { parse_macro( diff --git a/crates/ra_mbe/src/tt_iter.rs b/crates/ra_mbe/src/tt_iter.rs index 100184e66..46c420718 100644 --- a/crates/ra_mbe/src/tt_iter.rs +++ b/crates/ra_mbe/src/tt_iter.rs @@ -40,9 +40,11 @@ impl<'a> TtIter<'a> { } } - pub(crate) fn expect_literal(&mut self) -> Result<&'a tt::Literal, ()> { - match self.expect_leaf()? { - tt::Leaf::Literal(it) => Ok(it), + pub(crate) fn expect_literal(&mut self) -> Result<&'a tt::Leaf, ()> { + let it = self.expect_leaf()?; + match it { + tt::Leaf::Literal(_) => Ok(it), + tt::Leaf::Ident(ident) if ident.text == "true" || ident.text == "false" => Ok(it), _ => Err(()), } } -- cgit v1.2.3