aboutsummaryrefslogtreecommitdiff
path: root/crates/ra_mbe
diff options
context:
space:
mode:
authorEdwin Cheng <[email protected]>2020-04-18 12:28:07 +0100
committerEdwin Cheng <[email protected]>2020-04-18 12:28:07 +0100
commitda18f1130756c4fdd611d95e4f98e553b4b65995 (patch)
treea3bbf8863a69a5a27e75e63f5ce0855759509024 /crates/ra_mbe
parentf78de3bb95acb996102a74b5b12d33054ba6d4c4 (diff)
Split LIFETIME to two tokens in mbe
Diffstat (limited to 'crates/ra_mbe')
-rw-r--r--crates/ra_mbe/src/mbe_expander/matcher.rs34
-rw-r--r--crates/ra_mbe/src/subtree_source.rs32
-rw-r--r--crates/ra_mbe/src/syntax_bridge.rs37
3 files changed, 89 insertions, 14 deletions
diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs
index 2579382da..9485c62b8 100644
--- a/crates/ra_mbe/src/mbe_expander/matcher.rs
+++ b/crates/ra_mbe/src/mbe_expander/matcher.rs
@@ -202,6 +202,13 @@ impl<'a> TtIter<'a> {
202 } 202 }
203 203
204 pub(crate) fn expect_tt(&mut self) -> Result<tt::TokenTree, ()> { 204 pub(crate) fn expect_tt(&mut self) -> Result<tt::TokenTree, ()> {
205 match self.peek_n(0) {
206 Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '\'' => {
207 return self.expect_lifetime();
208 }
209 _ => (),
210 }
211
205 let tt = self.next().ok_or_else(|| ())?.clone(); 212 let tt = self.next().ok_or_else(|| ())?.clone();
206 let punct = match tt { 213 let punct = match tt {
207 tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if punct.spacing == tt::Spacing::Joint => { 214 tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if punct.spacing == tt::Spacing::Joint => {
@@ -255,13 +262,21 @@ impl<'a> TtIter<'a> {
255 } 262 }
256 } 263 }
257 264
258 pub(crate) fn expect_lifetime(&mut self) -> Result<&tt::Ident, ()> { 265 pub(crate) fn expect_lifetime(&mut self) -> Result<tt::TokenTree, ()> {
259 let ident = self.expect_ident()?; 266 let punct = self.expect_punct()?;
260 // check if it start from "`" 267 if punct.char != '\'' {
261 if !ident.text.starts_with('\'') {
262 return Err(()); 268 return Err(());
263 } 269 }
264 Ok(ident) 270 let ident = self.expect_ident()?;
271
272 Ok(tt::Subtree {
273 delimiter: None,
274 token_trees: vec![
275 tt::Leaf::Punct(punct.clone()).into(),
276 tt::Leaf::Ident(ident.clone()).into(),
277 ],
278 }
279 .into())
265 } 280 }
266 281
267 pub(crate) fn expect_fragment( 282 pub(crate) fn expect_fragment(
@@ -274,7 +289,10 @@ impl<'a> TtIter<'a> {
274 } 289 }
275 290
276 impl<'a> TreeSink for OffsetTokenSink<'a> { 291 impl<'a> TreeSink for OffsetTokenSink<'a> {
277 fn token(&mut self, _kind: SyntaxKind, n_tokens: u8) { 292 fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) {
293 if kind == SyntaxKind::LIFETIME {
294 n_tokens = 2;
295 }
278 for _ in 0..n_tokens { 296 for _ in 0..n_tokens {
279 self.cursor = self.cursor.bump_subtree(); 297 self.cursor = self.cursor.bump_subtree();
280 } 298 }
@@ -286,7 +304,7 @@ impl<'a> TtIter<'a> {
286 } 304 }
287 } 305 }
288 306
289 let buffer = TokenBuffer::new(self.inner.as_slice()); 307 let buffer = TokenBuffer::new(&self.inner.as_slice());
290 let mut src = SubtreeTokenSource::new(&buffer); 308 let mut src = SubtreeTokenSource::new(&buffer);
291 let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false }; 309 let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false };
292 310
@@ -422,7 +440,7 @@ fn match_meta_var(kind: &str, input: &mut TtIter) -> ExpandResult<Option<Fragmen
422 "tt" => input.expect_tt().map(Some).map_err(|()| err!()), 440 "tt" => input.expect_tt().map(Some).map_err(|()| err!()),
423 "lifetime" => input 441 "lifetime" => input
424 .expect_lifetime() 442 .expect_lifetime()
425 .map(|ident| Some(tt::Leaf::Ident(ident.clone()).into())) 443 .map(|tt| Some(tt))
426 .map_err(|()| err!("expected lifetime")), 444 .map_err(|()| err!("expected lifetime")),
427 "literal" => input 445 "literal" => input
428 .expect_literal() 446 .expect_literal()
diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs
index 91e324db9..46791efaa 100644
--- a/crates/ra_mbe/src/subtree_source.rs
+++ b/crates/ra_mbe/src/subtree_source.rs
@@ -50,6 +50,26 @@ impl<'a> SubtreeTokenSource<'a> {
50 } 50 }
51 51
52 fn get(&self, pos: usize) -> Ref<Option<TtToken>> { 52 fn get(&self, pos: usize) -> Ref<Option<TtToken>> {
53 fn is_lifetime(c: Cursor) -> Option<(Cursor, SmolStr)> {
54 let tkn = c.token_tree();
55
56 if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = tkn {
57 if punct.char == '\'' {
58 let next = c.bump();
59 if let Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) = next.token_tree() {
60 let res_cursor = next.bump();
61 let text = SmolStr::new("'".to_string() + &ident.to_string());
62
63 return Some((res_cursor, text));
64 } else {
65 panic!("Next token must be ident : {:#?}", next.token_tree());
66 }
67 }
68 }
69
70 None
71 }
72
53 if pos < self.cached.borrow().len() { 73 if pos < self.cached.borrow().len() {
54 return Ref::map(self.cached.borrow(), |c| &c[pos]); 74 return Ref::map(self.cached.borrow(), |c| &c[pos]);
55 } 75 }
@@ -63,6 +83,12 @@ impl<'a> SubtreeTokenSource<'a> {
63 continue; 83 continue;
64 } 84 }
65 85
86 if let Some((curr, text)) = is_lifetime(cursor) {
87 cached.push(Some(TtToken { kind: LIFETIME, is_joint_to_next: false, text }));
88 self.cached_cursor.set(curr);
89 continue;
90 }
91
66 match cursor.token_tree() { 92 match cursor.token_tree() {
67 Some(tt::TokenTree::Leaf(leaf)) => { 93 Some(tt::TokenTree::Leaf(leaf)) => {
68 cached.push(Some(convert_leaf(&leaf))); 94 cached.push(Some(convert_leaf(&leaf)));
@@ -152,7 +178,11 @@ fn convert_ident(ident: &tt::Ident) -> TtToken {
152} 178}
153 179
154fn convert_punct(p: tt::Punct) -> TtToken { 180fn convert_punct(p: tt::Punct) -> TtToken {
155 let kind = SyntaxKind::from_char(p.char).unwrap(); 181 let kind = match SyntaxKind::from_char(p.char) {
182 None => panic!("{:#?} is not a valid punct", p),
183 Some(kind) => kind,
184 };
185
156 let text = { 186 let text = {
157 let mut buf = [0u8; 4]; 187 let mut buf = [0u8; 4];
158 let s: &str = p.char.encode_utf8(&mut buf); 188 let s: &str = p.char.encode_utf8(&mut buf);
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index 31e9b22e7..70899bc5d 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -271,7 +271,7 @@ struct RawConvertor<'a> {
271 inner: std::slice::Iter<'a, RawToken>, 271 inner: std::slice::Iter<'a, RawToken>,
272} 272}
273 273
274trait SrcToken { 274trait SrcToken: std::fmt::Debug {
275 fn kind(&self) -> SyntaxKind; 275 fn kind(&self) -> SyntaxKind;
276 276
277 fn to_char(&self) -> Option<char>; 277 fn to_char(&self) -> Option<char>;
@@ -361,8 +361,12 @@ trait TokenConvertor {
361 Some(next) if next.kind().is_punct() => tt::Spacing::Joint, 361 Some(next) if next.kind().is_punct() => tt::Spacing::Joint,
362 _ => tt::Spacing::Alone, 362 _ => tt::Spacing::Alone,
363 }; 363 };
364 let char = token.to_char().expect("Token from lexer must be single char"); 364 let char = match token.to_char() {
365 365 Some(c) => c,
366 None => {
367 panic!("Token from lexer must be single char: token = {:#?}", token);
368 }
369 };
366 tt::Leaf::from(tt::Punct { char, spacing, id: self.id_alloc().alloc(range) }).into() 370 tt::Leaf::from(tt::Punct { char, spacing, id: self.id_alloc().alloc(range) }).into()
367 } 371 }
368 } else { 372 } else {
@@ -373,9 +377,28 @@ trait TokenConvertor {
373 } 377 }
374 let leaf: tt::Leaf = match k { 378 let leaf: tt::Leaf = match k {
375 T![true] | T![false] => make_leaf!(Literal), 379 T![true] | T![false] => make_leaf!(Literal),
376 IDENT | LIFETIME => make_leaf!(Ident), 380 IDENT => make_leaf!(Ident),
377 k if k.is_keyword() => make_leaf!(Ident), 381 k if k.is_keyword() => make_leaf!(Ident),
378 k if k.is_literal() => make_leaf!(Literal), 382 k if k.is_literal() => make_leaf!(Literal),
383 LIFETIME => {
384 let char_unit = TextUnit::from_usize(1);
385 let r = TextRange::offset_len(range.start(), char_unit);
386 let apostrophe = tt::Leaf::from(tt::Punct {
387 char: '\'',
388 spacing: tt::Spacing::Joint,
389 id: self.id_alloc().alloc(r),
390 });
391 result.push(apostrophe.into());
392
393 let r =
394 TextRange::offset_len(range.start() + char_unit, range.len() - char_unit);
395 let ident = tt::Leaf::from(tt::Ident {
396 text: SmolStr::new(&token.to_text()[1..]),
397 id: self.id_alloc().alloc(r),
398 });
399 result.push(ident.into());
400 return;
401 }
379 _ => return, 402 _ => return,
380 }; 403 };
381 404
@@ -455,6 +478,7 @@ impl Convertor {
455 } 478 }
456} 479}
457 480
481#[derive(Debug)]
458enum SynToken { 482enum SynToken {
459 Ordiniary(SyntaxToken), 483 Ordiniary(SyntaxToken),
460 Punch(SyntaxToken, TextUnit), 484 Punch(SyntaxToken, TextUnit),
@@ -592,11 +616,14 @@ fn delim_to_str(d: Option<tt::DelimiterKind>, closing: bool) -> SmolStr {
592} 616}
593 617
594impl<'a> TreeSink for TtTreeSink<'a> { 618impl<'a> TreeSink for TtTreeSink<'a> {
595 fn token(&mut self, kind: SyntaxKind, n_tokens: u8) { 619 fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) {
596 if kind == L_DOLLAR || kind == R_DOLLAR { 620 if kind == L_DOLLAR || kind == R_DOLLAR {
597 self.cursor = self.cursor.bump_subtree(); 621 self.cursor = self.cursor.bump_subtree();
598 return; 622 return;
599 } 623 }
624 if kind == LIFETIME {
625 n_tokens = 2;
626 }
600 627
601 let mut last = self.cursor; 628 let mut last = self.cursor;
602 for _ in 0..n_tokens { 629 for _ in 0..n_tokens {