diff options
-rw-r--r-- | crates/ra_mbe/src/lib.rs | 154 | ||||
-rw-r--r-- | crates/ra_mbe/src/mbe_expander.rs | 13 | ||||
-rw-r--r-- | crates/ra_mbe/src/subtree_parser.rs | 15 | ||||
-rw-r--r-- | crates/ra_mbe/src/subtree_source.rs | 366 | ||||
-rw-r--r-- | crates/ra_mbe/src/syntax_bridge.rs | 47 | ||||
-rw-r--r-- | crates/ra_mbe/src/tt_cursor.rs | 15 | ||||
-rw-r--r-- | crates/ra_parser/src/grammar.rs | 12 | ||||
-rw-r--r-- | crates/ra_parser/src/grammar/expressions.rs | 50 | ||||
-rw-r--r-- | crates/ra_parser/src/grammar/patterns.rs | 2 | ||||
-rw-r--r-- | crates/ra_parser/src/lib.rs | 33 | ||||
-rw-r--r-- | crates/ra_parser/src/parser.rs | 90 | ||||
-rw-r--r-- | crates/ra_parser/src/syntax_kind/generated.rs | 4 | ||||
-rw-r--r-- | crates/ra_syntax/src/grammar.ron | 2 |
13 files changed, 560 insertions, 243 deletions
diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index a21ea4dbc..a530f3b03 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs | |||
@@ -39,7 +39,7 @@ pub enum ExpandError { | |||
39 | BindingError(String), | 39 | BindingError(String), |
40 | } | 40 | } |
41 | 41 | ||
42 | pub use crate::syntax_bridge::{ast_to_token_tree, token_tree_to_ast_item_list}; | 42 | pub use crate::syntax_bridge::{ast_to_token_tree, token_tree_to_ast_item_list, syntax_node_to_token_tree}; |
43 | 43 | ||
44 | /// This struct contains AST for a single `macro_rules` definition. What might | 44 | /// This struct contains AST for a single `macro_rules` definition. What might |
45 | /// be very confusing is that AST has almost exactly the same shape as | 45 | /// be very confusing is that AST has almost exactly the same shape as |
@@ -189,9 +189,26 @@ impl_froms!(TokenTree: Leaf, Subtree); | |||
189 | rules.expand(&invocation_tt).unwrap() | 189 | rules.expand(&invocation_tt).unwrap() |
190 | } | 190 | } |
191 | 191 | ||
192 | pub(crate) fn expand_to_syntax( | ||
193 | rules: &MacroRules, | ||
194 | invocation: &str, | ||
195 | ) -> ra_syntax::TreeArc<ast::SourceFile> { | ||
196 | let expanded = expand(rules, invocation); | ||
197 | token_tree_to_ast_item_list(&expanded) | ||
198 | } | ||
199 | |||
192 | pub(crate) fn assert_expansion(rules: &MacroRules, invocation: &str, expansion: &str) { | 200 | pub(crate) fn assert_expansion(rules: &MacroRules, invocation: &str, expansion: &str) { |
193 | let expanded = expand(rules, invocation); | 201 | let expanded = expand(rules, invocation); |
194 | assert_eq!(expanded.to_string(), expansion); | 202 | assert_eq!(expanded.to_string(), expansion); |
203 | |||
204 | let tree = token_tree_to_ast_item_list(&expanded); | ||
205 | |||
206 | // Eat all white space by parse it back and forth | ||
207 | let expansion = ast::SourceFile::parse(expansion); | ||
208 | let expansion = syntax_node_to_token_tree(expansion.syntax()).unwrap().0; | ||
209 | let file = token_tree_to_ast_item_list(&expansion); | ||
210 | |||
211 | assert_eq!(tree.syntax().debug_dump().trim(), file.syntax().debug_dump().trim()); | ||
195 | } | 212 | } |
196 | 213 | ||
197 | #[test] | 214 | #[test] |
@@ -288,6 +305,36 @@ impl_froms!(TokenTree: Leaf, Subtree); | |||
288 | } | 305 | } |
289 | 306 | ||
290 | #[test] | 307 | #[test] |
308 | fn test_match_group_pattern_with_multiple_defs() { | ||
309 | let rules = create_rules( | ||
310 | r#" | ||
311 | macro_rules! foo { | ||
312 | ($ ($ i:ident),*) => ( struct Bar { $ ( | ||
313 | fn $ i {} | ||
314 | )*} ); | ||
315 | } | ||
316 | "#, | ||
317 | ); | ||
318 | |||
319 | assert_expansion(&rules, "foo! { foo, bar }", "struct Bar {fn foo {} fn bar {}}"); | ||
320 | } | ||
321 | |||
322 | #[test] | ||
323 | fn test_match_group_pattern_with_multiple_statement() { | ||
324 | let rules = create_rules( | ||
325 | r#" | ||
326 | macro_rules! foo { | ||
327 | ($ ($ i:ident),*) => ( fn baz { $ ( | ||
328 | $ i (); | ||
329 | )*} ); | ||
330 | } | ||
331 | "#, | ||
332 | ); | ||
333 | |||
334 | assert_expansion(&rules, "foo! { foo, bar }", "fn baz {foo () ; bar () ;}"); | ||
335 | } | ||
336 | |||
337 | #[test] | ||
291 | fn expand_to_item_list() { | 338 | fn expand_to_item_list() { |
292 | let rules = create_rules( | 339 | let rules = create_rules( |
293 | " | 340 | " |
@@ -415,7 +462,7 @@ SOURCE_FILE@[0; 40) | |||
415 | assert_expansion( | 462 | assert_expansion( |
416 | &rules, | 463 | &rules, |
417 | "foo! { bar::<u8>::baz::<u8> }", | 464 | "foo! { bar::<u8>::baz::<u8> }", |
418 | "fn foo () {let a = bar ::< u8 > ::baz ::< u8 > ;}", | 465 | "fn foo () {let a = bar :: < u8 > :: baz :: < u8 > ;}", |
419 | ); | 466 | ); |
420 | } | 467 | } |
421 | 468 | ||
@@ -432,4 +479,107 @@ SOURCE_FILE@[0; 40) | |||
432 | ); | 479 | ); |
433 | assert_expansion(&rules, "foo! { foo, bar }", "fn foo () {let a = foo ; let b = bar ;}"); | 480 | assert_expansion(&rules, "foo! { foo, bar }", "fn foo () {let a = foo ; let b = bar ;}"); |
434 | } | 481 | } |
482 | |||
483 | #[test] | ||
484 | fn test_path_with_path() { | ||
485 | let rules = create_rules( | ||
486 | r#" | ||
487 | macro_rules! foo { | ||
488 | ($ i:path) => { | ||
489 | fn foo() { let a = $ i :: bar; } | ||
490 | } | ||
491 | } | ||
492 | "#, | ||
493 | ); | ||
494 | assert_expansion(&rules, "foo! { foo }", "fn foo () {let a = foo :: bar ;}"); | ||
495 | } | ||
496 | |||
497 | #[test] | ||
498 | fn test_expr() { | ||
499 | let rules = create_rules( | ||
500 | r#" | ||
501 | macro_rules! foo { | ||
502 | ($ i:expr) => { | ||
503 | fn bar() { $ i; } | ||
504 | } | ||
505 | } | ||
506 | "#, | ||
507 | ); | ||
508 | |||
509 | assert_expansion( | ||
510 | &rules, | ||
511 | "foo! { 2 + 2 * baz(3).quux() }", | ||
512 | "fn bar () {2 + 2 * baz (3) . quux () ;}", | ||
513 | ); | ||
514 | } | ||
515 | |||
516 | #[test] | ||
517 | fn test_expr_order() { | ||
518 | let rules = create_rules( | ||
519 | r#" | ||
520 | macro_rules! foo { | ||
521 | ($ i:expr) => { | ||
522 | fn bar() { $ i * 2; } | ||
523 | } | ||
524 | } | ||
525 | "#, | ||
526 | ); | ||
527 | |||
528 | assert_eq!( | ||
529 | expand_to_syntax(&rules, "foo! { 1 + 1 }").syntax().debug_dump().trim(), | ||
530 | r#"SOURCE_FILE@[0; 15) | ||
531 | FN_DEF@[0; 15) | ||
532 | FN_KW@[0; 2) "fn" | ||
533 | NAME@[2; 5) | ||
534 | IDENT@[2; 5) "bar" | ||
535 | PARAM_LIST@[5; 7) | ||
536 | L_PAREN@[5; 6) "(" | ||
537 | R_PAREN@[6; 7) ")" | ||
538 | BLOCK@[7; 15) | ||
539 | L_CURLY@[7; 8) "{" | ||
540 | EXPR_STMT@[8; 14) | ||
541 | BIN_EXPR@[8; 13) | ||
542 | BIN_EXPR@[8; 11) | ||
543 | LITERAL@[8; 9) | ||
544 | INT_NUMBER@[8; 9) "1" | ||
545 | PLUS@[9; 10) "+" | ||
546 | LITERAL@[10; 11) | ||
547 | INT_NUMBER@[10; 11) "1" | ||
548 | STAR@[11; 12) "*" | ||
549 | LITERAL@[12; 13) | ||
550 | INT_NUMBER@[12; 13) "2" | ||
551 | SEMI@[13; 14) ";" | ||
552 | R_CURLY@[14; 15) "}""#, | ||
553 | ); | ||
554 | } | ||
555 | |||
556 | #[test] | ||
557 | fn test_ty() { | ||
558 | let rules = create_rules( | ||
559 | r#" | ||
560 | macro_rules! foo { | ||
561 | ($ i:ty) => ( | ||
562 | fn bar() -> $ i { unimplemented!() } | ||
563 | ) | ||
564 | } | ||
565 | "#, | ||
566 | ); | ||
567 | assert_expansion( | ||
568 | &rules, | ||
569 | "foo! { Baz<u8> }", | ||
570 | "fn bar () -> Baz < u8 > {unimplemented ! ()}", | ||
571 | ); | ||
572 | } | ||
573 | |||
574 | #[test] | ||
575 | fn test_pat_() { | ||
576 | let rules = create_rules( | ||
577 | r#" | ||
578 | macro_rules! foo { | ||
579 | ($ i:pat) => { fn foo() { let $ i; } } | ||
580 | } | ||
581 | "#, | ||
582 | ); | ||
583 | assert_expansion(&rules, "foo! { (a, b) }", "fn foo () {let (a , b) ;}"); | ||
584 | } | ||
435 | } | 585 | } |
diff --git a/crates/ra_mbe/src/mbe_expander.rs b/crates/ra_mbe/src/mbe_expander.rs index ce41d7225..7a259f338 100644 --- a/crates/ra_mbe/src/mbe_expander.rs +++ b/crates/ra_mbe/src/mbe_expander.rs | |||
@@ -144,6 +144,19 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings, | |||
144 | input.eat_path().ok_or(ExpandError::UnexpectedToken)?.clone(); | 144 | input.eat_path().ok_or(ExpandError::UnexpectedToken)?.clone(); |
145 | res.inner.insert(text.clone(), Binding::Simple(path.into())); | 145 | res.inner.insert(text.clone(), Binding::Simple(path.into())); |
146 | } | 146 | } |
147 | "expr" => { | ||
148 | let expr = | ||
149 | input.eat_expr().ok_or(ExpandError::UnexpectedToken)?.clone(); | ||
150 | res.inner.insert(text.clone(), Binding::Simple(expr.into())); | ||
151 | } | ||
152 | "ty" => { | ||
153 | let ty = input.eat_ty().ok_or(ExpandError::UnexpectedToken)?.clone(); | ||
154 | res.inner.insert(text.clone(), Binding::Simple(ty.into())); | ||
155 | } | ||
156 | "pat" => { | ||
157 | let pat = input.eat_pat().ok_or(ExpandError::UnexpectedToken)?.clone(); | ||
158 | res.inner.insert(text.clone(), Binding::Simple(pat.into())); | ||
159 | } | ||
147 | _ => return Err(ExpandError::UnexpectedToken), | 160 | _ => return Err(ExpandError::UnexpectedToken), |
148 | } | 161 | } |
149 | } | 162 | } |
diff --git a/crates/ra_mbe/src/subtree_parser.rs b/crates/ra_mbe/src/subtree_parser.rs index ce39a40bb..13d5d2169 100644 --- a/crates/ra_mbe/src/subtree_parser.rs +++ b/crates/ra_mbe/src/subtree_parser.rs | |||
@@ -30,12 +30,23 @@ impl<'a> Parser<'a> { | |||
30 | self.parse(ra_parser::parse_path) | 30 | self.parse(ra_parser::parse_path) |
31 | } | 31 | } |
32 | 32 | ||
33 | pub fn parse_expr(self) -> Option<tt::TokenTree> { | ||
34 | self.parse(ra_parser::parse_expr) | ||
35 | } | ||
36 | |||
37 | pub fn parse_ty(self) -> Option<tt::TokenTree> { | ||
38 | self.parse(ra_parser::parse_ty) | ||
39 | } | ||
40 | |||
41 | pub fn parse_pat(self) -> Option<tt::TokenTree> { | ||
42 | self.parse(ra_parser::parse_pat) | ||
43 | } | ||
44 | |||
33 | fn parse<F>(self, f: F) -> Option<tt::TokenTree> | 45 | fn parse<F>(self, f: F) -> Option<tt::TokenTree> |
34 | where | 46 | where |
35 | F: FnOnce(&dyn TokenSource, &mut dyn TreeSink), | 47 | F: FnOnce(&dyn TokenSource, &mut dyn TreeSink), |
36 | { | 48 | { |
37 | let mut src = SubtreeTokenSource::new(self.subtree); | 49 | let mut src = SubtreeTokenSource::new(&self.subtree.token_trees[*self.cur_pos..]); |
38 | src.start_from_nth(*self.cur_pos); | ||
39 | let mut sink = OffsetTokenSink { token_pos: 0 }; | 50 | let mut sink = OffsetTokenSink { token_pos: 0 }; |
40 | 51 | ||
41 | f(&src, &mut sink); | 52 | f(&src, &mut sink); |
diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs index 4b37c2bda..0a070b46a 100644 --- a/crates/ra_mbe/src/subtree_source.rs +++ b/crates/ra_mbe/src/subtree_source.rs | |||
@@ -2,6 +2,64 @@ use ra_parser::{TokenSource}; | |||
2 | use ra_syntax::{classify_literal, SmolStr, SyntaxKind, SyntaxKind::*}; | 2 | use ra_syntax::{classify_literal, SmolStr, SyntaxKind, SyntaxKind::*}; |
3 | use std::cell::{RefCell}; | 3 | use std::cell::{RefCell}; |
4 | 4 | ||
5 | // A Sequece of Token, | ||
6 | #[derive(Debug, Clone, Eq, PartialEq)] | ||
7 | pub(super) enum TokenSeq<'a> { | ||
8 | Subtree(&'a tt::Subtree), | ||
9 | Seq(&'a [tt::TokenTree]), | ||
10 | } | ||
11 | |||
12 | impl<'a> From<&'a tt::Subtree> for TokenSeq<'a> { | ||
13 | fn from(s: &'a tt::Subtree) -> TokenSeq<'a> { | ||
14 | TokenSeq::Subtree(s) | ||
15 | } | ||
16 | } | ||
17 | |||
18 | impl<'a> From<&'a [tt::TokenTree]> for TokenSeq<'a> { | ||
19 | fn from(s: &'a [tt::TokenTree]) -> TokenSeq<'a> { | ||
20 | TokenSeq::Seq(s) | ||
21 | } | ||
22 | } | ||
23 | |||
24 | enum DelimToken<'a> { | ||
25 | Delim(&'a tt::Delimiter, bool), | ||
26 | Token(&'a tt::TokenTree), | ||
27 | End, | ||
28 | } | ||
29 | |||
30 | impl<'a> TokenSeq<'a> { | ||
31 | fn get(&self, pos: usize) -> DelimToken<'a> { | ||
32 | match self { | ||
33 | TokenSeq::Subtree(subtree) => { | ||
34 | let len = subtree.token_trees.len() + 2; | ||
35 | match pos { | ||
36 | p if p >= len => DelimToken::End, | ||
37 | p if p == len - 1 => DelimToken::Delim(&subtree.delimiter, true), | ||
38 | 0 => DelimToken::Delim(&subtree.delimiter, false), | ||
39 | p => DelimToken::Token(&subtree.token_trees[p - 1]), | ||
40 | } | ||
41 | } | ||
42 | TokenSeq::Seq(tokens) => { | ||
43 | tokens.get(pos).map(DelimToken::Token).unwrap_or(DelimToken::End) | ||
44 | } | ||
45 | } | ||
46 | } | ||
47 | |||
48 | fn len(&self) -> usize { | ||
49 | match self { | ||
50 | TokenSeq::Subtree(subtree) => subtree.token_trees.len() + 2, | ||
51 | TokenSeq::Seq(tokens) => tokens.len(), | ||
52 | } | ||
53 | } | ||
54 | |||
55 | fn child_slice(&self) -> &[tt::TokenTree] { | ||
56 | match self { | ||
57 | TokenSeq::Subtree(subtree) => &subtree.token_trees, | ||
58 | TokenSeq::Seq(tokens) => &tokens, | ||
59 | } | ||
60 | } | ||
61 | } | ||
62 | |||
5 | #[derive(Debug, Clone, Eq, PartialEq)] | 63 | #[derive(Debug, Clone, Eq, PartialEq)] |
6 | struct TtToken { | 64 | struct TtToken { |
7 | pub kind: SyntaxKind, | 65 | pub kind: SyntaxKind, |
@@ -12,29 +70,27 @@ struct TtToken { | |||
12 | 70 | ||
13 | #[derive(Debug, Clone, Eq, PartialEq)] | 71 | #[derive(Debug, Clone, Eq, PartialEq)] |
14 | enum WalkCursor { | 72 | enum WalkCursor { |
15 | DelimiterBegin(Option<TtToken>), | 73 | Token(usize, TtToken), |
16 | Token(usize, Option<TtToken>), | ||
17 | DelimiterEnd(Option<TtToken>), | ||
18 | Eof, | 74 | Eof, |
19 | } | 75 | } |
20 | 76 | ||
21 | #[derive(Debug)] | 77 | #[derive(Debug)] |
22 | struct SubTreeWalker<'a> { | 78 | struct SubTreeWalker<'a> { |
23 | pos: usize, | 79 | pos: usize, |
24 | stack: Vec<(&'a tt::Subtree, Option<usize>)>, | 80 | stack: Vec<(TokenSeq<'a>, usize)>, |
25 | cursor: WalkCursor, | 81 | cursor: WalkCursor, |
26 | last_steps: Vec<usize>, | 82 | last_steps: Vec<usize>, |
27 | subtree: &'a tt::Subtree, | 83 | ts: TokenSeq<'a>, |
28 | } | 84 | } |
29 | 85 | ||
30 | impl<'a> SubTreeWalker<'a> { | 86 | impl<'a> SubTreeWalker<'a> { |
31 | fn new(subtree: &tt::Subtree) -> SubTreeWalker { | 87 | fn new(ts: TokenSeq<'a>) -> SubTreeWalker { |
32 | let mut res = SubTreeWalker { | 88 | let mut res = SubTreeWalker { |
33 | pos: 0, | 89 | pos: 0, |
34 | stack: vec![], | 90 | stack: vec![], |
35 | cursor: WalkCursor::Eof, | 91 | cursor: WalkCursor::Eof, |
36 | last_steps: vec![], | 92 | last_steps: vec![], |
37 | subtree, | 93 | ts, |
38 | }; | 94 | }; |
39 | 95 | ||
40 | res.reset(); | 96 | res.reset(); |
@@ -47,209 +103,108 @@ impl<'a> SubTreeWalker<'a> { | |||
47 | 103 | ||
48 | fn reset(&mut self) { | 104 | fn reset(&mut self) { |
49 | self.pos = 0; | 105 | self.pos = 0; |
50 | self.stack = vec![(self.subtree, None)]; | 106 | self.stack = vec![]; |
51 | self.cursor = WalkCursor::DelimiterBegin(convert_delim(self.subtree.delimiter, false)); | ||
52 | self.last_steps = vec![]; | 107 | self.last_steps = vec![]; |
53 | 108 | ||
54 | while self.is_empty_delimiter() { | 109 | self.cursor = match self.ts.get(0) { |
55 | self.forward_unchecked(); | 110 | DelimToken::Token(token) => match token { |
56 | } | 111 | tt::TokenTree::Subtree(subtree) => { |
57 | } | 112 | let ts = TokenSeq::from(subtree); |
58 | 113 | self.stack.push((ts, 0)); | |
59 | // This funciton will fast forward the cursor, | 114 | WalkCursor::Token(0, convert_delim(subtree.delimiter, false)) |
60 | // Such that backward will stop at `start_pos` point | 115 | } |
61 | fn start_from_nth(&mut self, start_pos: usize) { | 116 | tt::TokenTree::Leaf(leaf) => { |
62 | self.reset(); | 117 | let next_tokens = self.ts.child_slice(); |
63 | self.pos = start_pos; | 118 | WalkCursor::Token(0, convert_leaf(&next_tokens, leaf)) |
64 | self.cursor = self.walk_token(start_pos, 0, false); | 119 | } |
65 | 120 | }, | |
66 | while self.is_empty_delimiter() { | 121 | DelimToken::Delim(delim, is_end) => { |
67 | self.forward_unchecked(); | 122 | assert!(!is_end); |
123 | WalkCursor::Token(0, convert_delim(*delim, false)) | ||
124 | } | ||
125 | DelimToken::End => WalkCursor::Eof, | ||
68 | } | 126 | } |
69 | } | 127 | } |
70 | 128 | ||
71 | fn current(&self) -> Option<&TtToken> { | 129 | fn current(&self) -> Option<&TtToken> { |
72 | match &self.cursor { | 130 | match &self.cursor { |
73 | WalkCursor::DelimiterBegin(t) => t.as_ref(), | 131 | WalkCursor::Token(_, t) => Some(t), |
74 | WalkCursor::Token(_, t) => t.as_ref(), | ||
75 | WalkCursor::DelimiterEnd(t) => t.as_ref(), | ||
76 | WalkCursor::Eof => None, | 132 | WalkCursor::Eof => None, |
77 | } | 133 | } |
78 | } | 134 | } |
79 | 135 | ||
80 | fn is_empty_delimiter(&self) -> bool { | 136 | fn top(&self) -> &TokenSeq { |
81 | match &self.cursor { | 137 | self.stack.last().map(|(t, _)| t).unwrap_or(&self.ts) |
82 | WalkCursor::DelimiterBegin(None) => true, | ||
83 | WalkCursor::DelimiterEnd(None) => true, | ||
84 | _ => false, | ||
85 | } | ||
86 | } | 138 | } |
87 | 139 | ||
88 | /// Move cursor backward by 1 step with empty checking | 140 | /// Move cursor backward by 1 step |
89 | fn backward(&mut self) { | 141 | fn backward(&mut self) { |
90 | if self.last_steps.is_empty() { | 142 | if self.last_steps.is_empty() { |
91 | return; | 143 | return; |
92 | } | 144 | } |
145 | |||
93 | self.pos -= 1; | 146 | self.pos -= 1; |
94 | loop { | 147 | let last_step = self.last_steps.pop().unwrap(); |
95 | self.backward_unchecked(); | ||
96 | // Skip Empty delimiter | ||
97 | if self.last_steps.is_empty() || !self.is_empty_delimiter() { | ||
98 | break; | ||
99 | } | ||
100 | } | ||
101 | 148 | ||
102 | // Move forward if it is empty delimiter | 149 | self.cursor = match self.cursor { |
103 | if self.last_steps.is_empty() { | 150 | WalkCursor::Token(idx, _) => self.walk_token(idx, last_step, true), |
104 | while self.is_empty_delimiter() { | 151 | WalkCursor::Eof => { |
105 | self.forward_unchecked(); | 152 | let len = self.top().len(); |
153 | self.walk_token(len, last_step, true) | ||
106 | } | 154 | } |
107 | } | 155 | } |
108 | } | 156 | } |
109 | 157 | ||
110 | /// Move cursor backward by 1 step without empty check | 158 | /// Move cursor forward by 1 step |
111 | /// | ||
112 | /// Depends on the current state of cursor: | ||
113 | /// | ||
114 | /// * Delimiter Begin => Pop the stack, goto last walking token (`walk_token`) | ||
115 | /// * Token => Goto prev token (`walk_token`) | ||
116 | /// * Delimiter End => Goto the last child token (`walk_token`) | ||
117 | /// * Eof => push the root subtree, and set it as Delimiter End | ||
118 | fn backward_unchecked(&mut self) { | ||
119 | if self.last_steps.is_empty() { | ||
120 | return; | ||
121 | } | ||
122 | |||
123 | let last_step = self.last_steps.pop().unwrap(); | ||
124 | let do_walk_token = match self.cursor { | ||
125 | WalkCursor::DelimiterBegin(_) => None, | ||
126 | WalkCursor::Token(u, _) => Some(u), | ||
127 | WalkCursor::DelimiterEnd(_) => { | ||
128 | let (top, _) = self.stack.last().unwrap(); | ||
129 | Some(top.token_trees.len()) | ||
130 | } | ||
131 | WalkCursor::Eof => None, | ||
132 | }; | ||
133 | |||
134 | self.cursor = match do_walk_token { | ||
135 | Some(u) => self.walk_token(u, last_step, true), | ||
136 | None => match self.cursor { | ||
137 | WalkCursor::Eof => { | ||
138 | self.stack.push((self.subtree, None)); | ||
139 | WalkCursor::DelimiterEnd(convert_delim( | ||
140 | self.stack.last().unwrap().0.delimiter, | ||
141 | true, | ||
142 | )) | ||
143 | } | ||
144 | _ => { | ||
145 | let (_, last_top_cursor) = self.stack.pop().unwrap(); | ||
146 | assert!(!self.stack.is_empty()); | ||
147 | |||
148 | self.walk_token(last_top_cursor.unwrap(), last_step, true) | ||
149 | } | ||
150 | }, | ||
151 | }; | ||
152 | } | ||
153 | |||
154 | /// Move cursor forward by 1 step with empty checking | ||
155 | fn forward(&mut self) { | 159 | fn forward(&mut self) { |
156 | if self.is_eof() { | 160 | if self.is_eof() { |
157 | return; | 161 | return; |
158 | } | 162 | } |
159 | |||
160 | self.pos += 1; | 163 | self.pos += 1; |
161 | loop { | ||
162 | self.forward_unchecked(); | ||
163 | if !self.is_empty_delimiter() { | ||
164 | break; | ||
165 | } | ||
166 | } | ||
167 | } | ||
168 | |||
169 | /// Move cursor forward by 1 step without empty checking | ||
170 | /// | ||
171 | /// Depends on the current state of cursor: | ||
172 | /// | ||
173 | /// * Delimiter Begin => Goto the first child token (`walk_token`) | ||
174 | /// * Token => Goto next token (`walk_token`) | ||
175 | /// * Delimiter End => Pop the stack, goto last walking token (`walk_token`) | ||
176 | /// | ||
177 | fn forward_unchecked(&mut self) { | ||
178 | if self.is_eof() { | ||
179 | return; | ||
180 | } | ||
181 | 164 | ||
182 | let step = self.current().map(|x| x.n_tokens).unwrap_or(1); | 165 | let step = self.current().map(|x| x.n_tokens).unwrap_or(1); |
183 | self.last_steps.push(step); | 166 | self.last_steps.push(step); |
184 | 167 | ||
185 | let do_walk_token = match self.cursor { | 168 | if let WalkCursor::Token(u, _) = self.cursor { |
186 | WalkCursor::DelimiterBegin(_) => Some((0, 0)), | 169 | self.cursor = self.walk_token(u, step, false) |
187 | WalkCursor::Token(u, _) => Some((u, step)), | 170 | } |
188 | WalkCursor::DelimiterEnd(_) => None, | ||
189 | _ => unreachable!(), | ||
190 | }; | ||
191 | |||
192 | self.cursor = match do_walk_token { | ||
193 | Some((u, step)) => self.walk_token(u, step, false), | ||
194 | None => { | ||
195 | let (_, last_top_idx) = self.stack.pop().unwrap(); | ||
196 | match self.stack.last() { | ||
197 | Some(_) => self.walk_token(last_top_idx.unwrap(), 1, false), | ||
198 | None => WalkCursor::Eof, | ||
199 | } | ||
200 | } | ||
201 | }; | ||
202 | } | 171 | } |
203 | 172 | ||
204 | /// Traversal child token | 173 | /// Traversal child token |
205 | /// Depends on the new position, it returns: | ||
206 | /// | ||
207 | /// * new position < 0 => DelimiterBegin | ||
208 | /// * new position > token_tree.len() => DelimiterEnd | ||
209 | /// * if new position is a subtree, depends on traversal direction: | ||
210 | /// ** backward => DelimiterEnd | ||
211 | /// ** forward => DelimiterBegin | ||
212 | /// * if new psoition is a leaf, return walk_leaf() | ||
213 | fn walk_token(&mut self, pos: usize, offset: usize, backward: bool) -> WalkCursor { | 174 | fn walk_token(&mut self, pos: usize, offset: usize, backward: bool) -> WalkCursor { |
214 | let (top, _) = self.stack.last().unwrap(); | 175 | let top = self.stack.last().map(|(t, _)| t).unwrap_or(&self.ts); |
215 | 176 | ||
216 | if backward && pos < offset { | 177 | if backward && pos < offset { |
217 | return WalkCursor::DelimiterBegin(convert_delim( | 178 | let (_, last_idx) = self.stack.pop().unwrap(); |
218 | self.stack.last().unwrap().0.delimiter, | 179 | return self.walk_token(last_idx, offset, backward); |
219 | false, | ||
220 | )); | ||
221 | } | ||
222 | |||
223 | if !backward && pos + offset >= top.token_trees.len() { | ||
224 | return WalkCursor::DelimiterEnd(convert_delim( | ||
225 | self.stack.last().unwrap().0.delimiter, | ||
226 | true, | ||
227 | )); | ||
228 | } | 180 | } |
229 | 181 | ||
230 | let pos = if backward { pos - offset } else { pos + offset }; | 182 | let pos = if backward { pos - offset } else { pos + offset }; |
231 | 183 | ||
232 | match &top.token_trees[pos] { | 184 | match top.get(pos) { |
233 | tt::TokenTree::Subtree(subtree) => { | 185 | DelimToken::Token(token) => match token { |
234 | self.stack.push((subtree, Some(pos))); | 186 | tt::TokenTree::Subtree(subtree) => { |
235 | let delim = convert_delim(self.stack.last().unwrap().0.delimiter, backward); | 187 | let ts = TokenSeq::from(subtree); |
236 | if backward { | 188 | let new_idx = if backward { ts.len() - 1 } else { 0 }; |
237 | WalkCursor::DelimiterEnd(delim) | 189 | self.stack.push((ts, pos)); |
238 | } else { | 190 | WalkCursor::Token(new_idx, convert_delim(subtree.delimiter, backward)) |
239 | WalkCursor::DelimiterBegin(delim) | 191 | } |
192 | tt::TokenTree::Leaf(leaf) => { | ||
193 | let next_tokens = top.child_slice(); | ||
194 | WalkCursor::Token(pos, convert_leaf(&next_tokens[pos..], leaf)) | ||
240 | } | 195 | } |
196 | }, | ||
197 | DelimToken::Delim(delim, is_end) => { | ||
198 | WalkCursor::Token(pos, convert_delim(*delim, is_end)) | ||
241 | } | 199 | } |
242 | tt::TokenTree::Leaf(leaf) => WalkCursor::Token(pos, Some(self.walk_leaf(leaf, pos))), | 200 | DelimToken::End => { |
243 | } | 201 | // it is the top level |
244 | } | 202 | if let Some((_, last_idx)) = self.stack.pop() { |
245 | 203 | assert!(!backward); | |
246 | fn walk_leaf(&mut self, leaf: &tt::Leaf, pos: usize) -> TtToken { | 204 | self.walk_token(last_idx, offset, backward) |
247 | match leaf { | 205 | } else { |
248 | tt::Leaf::Literal(l) => convert_literal(l), | 206 | WalkCursor::Eof |
249 | tt::Leaf::Ident(ident) => convert_ident(ident), | 207 | } |
250 | tt::Leaf::Punct(punct) => { | ||
251 | let (top, _) = self.stack.last().unwrap(); | ||
252 | convert_punct(punct, top, pos) | ||
253 | } | 208 | } |
254 | } | 209 | } |
255 | } | 210 | } |
@@ -263,27 +218,20 @@ pub(crate) trait Querier { | |||
263 | #[derive(Debug)] | 218 | #[derive(Debug)] |
264 | pub(crate) struct WalkerOwner<'a> { | 219 | pub(crate) struct WalkerOwner<'a> { |
265 | walker: RefCell<SubTreeWalker<'a>>, | 220 | walker: RefCell<SubTreeWalker<'a>>, |
266 | offset: usize, | ||
267 | } | 221 | } |
268 | 222 | ||
269 | impl<'a> WalkerOwner<'a> { | 223 | impl<'a> WalkerOwner<'a> { |
270 | fn new(subtree: &'a tt::Subtree) -> Self { | 224 | fn new<I: Into<TokenSeq<'a>>>(ts: I) -> Self { |
271 | WalkerOwner { walker: RefCell::new(SubTreeWalker::new(subtree)), offset: 0 } | 225 | WalkerOwner { walker: RefCell::new(SubTreeWalker::new(ts.into())) } |
272 | } | 226 | } |
273 | 227 | ||
274 | fn get<'b>(&self, pos: usize) -> Option<TtToken> { | 228 | fn get<'b>(&self, pos: usize) -> Option<TtToken> { |
275 | self.set_walker_pos(pos); | 229 | self.set_pos(pos); |
276 | let walker = self.walker.borrow(); | 230 | let walker = self.walker.borrow(); |
277 | walker.current().cloned() | 231 | walker.current().cloned() |
278 | } | 232 | } |
279 | 233 | ||
280 | fn start_from_nth(&mut self, pos: usize) { | 234 | fn set_pos(&self, pos: usize) { |
281 | self.offset = pos; | ||
282 | self.walker.borrow_mut().start_from_nth(pos); | ||
283 | } | ||
284 | |||
285 | fn set_walker_pos(&self, mut pos: usize) { | ||
286 | pos += self.offset; | ||
287 | let mut walker = self.walker.borrow_mut(); | 235 | let mut walker = self.walker.borrow_mut(); |
288 | while pos > walker.pos && !walker.is_eof() { | 236 | while pos > walker.pos && !walker.is_eof() { |
289 | walker.forward(); | 237 | walker.forward(); |
@@ -294,19 +242,26 @@ impl<'a> WalkerOwner<'a> { | |||
294 | } | 242 | } |
295 | 243 | ||
296 | fn collect_token_trees(&mut self, n: usize) -> Vec<&tt::TokenTree> { | 244 | fn collect_token_trees(&mut self, n: usize) -> Vec<&tt::TokenTree> { |
297 | self.start_from_nth(self.offset); | ||
298 | |||
299 | let mut res = vec![]; | 245 | let mut res = vec![]; |
300 | let mut walker = self.walker.borrow_mut(); | 246 | let mut walker = self.walker.borrow_mut(); |
247 | walker.reset(); | ||
301 | 248 | ||
302 | while walker.pos - self.offset < n { | 249 | while walker.pos < n { |
303 | if let WalkCursor::Token(u, tt) = &walker.cursor { | 250 | if let WalkCursor::Token(u, tt) = &walker.cursor { |
304 | if walker.stack.len() == 1 { | 251 | // We only collect the topmost child |
305 | // We only collect the topmost child | 252 | if walker.stack.len() == 0 { |
306 | res.push(&walker.stack[0].0.token_trees[*u]); | 253 | for i in 0..tt.n_tokens { |
307 | if let Some(tt) = tt { | 254 | if let DelimToken::Token(token) = walker.ts.get(u + i) { |
308 | for i in 0..tt.n_tokens - 1 { | 255 | res.push(token); |
309 | res.push(&walker.stack[0].0.token_trees[u + i]); | 256 | } |
257 | } | ||
258 | } else if walker.stack.len() == 1 { | ||
259 | if let DelimToken::Delim(_, is_end) = walker.top().get(*u) { | ||
260 | if !is_end { | ||
261 | let (_, last_idx) = &walker.stack[0]; | ||
262 | if let DelimToken::Token(token) = walker.ts.get(*last_idx) { | ||
263 | res.push(token); | ||
264 | } | ||
310 | } | 265 | } |
311 | } | 266 | } |
312 | } | 267 | } |
@@ -331,12 +286,8 @@ pub(crate) struct SubtreeTokenSource<'a> { | |||
331 | } | 286 | } |
332 | 287 | ||
333 | impl<'a> SubtreeTokenSource<'a> { | 288 | impl<'a> SubtreeTokenSource<'a> { |
334 | pub fn new(subtree: &tt::Subtree) -> SubtreeTokenSource { | 289 | pub fn new<I: Into<TokenSeq<'a>>>(ts: I) -> SubtreeTokenSource<'a> { |
335 | SubtreeTokenSource { walker: WalkerOwner::new(subtree) } | 290 | SubtreeTokenSource { walker: WalkerOwner::new(ts) } |
336 | } | ||
337 | |||
338 | pub fn start_from_nth(&mut self, n: usize) { | ||
339 | self.walker.start_from_nth(n); | ||
340 | } | 291 | } |
341 | 292 | ||
342 | pub fn querier<'b>(&'a self) -> &'b WalkerOwner<'a> | 293 | pub fn querier<'b>(&'a self) -> &'b WalkerOwner<'a> |
@@ -361,10 +312,16 @@ impl<'a> TokenSource for SubtreeTokenSource<'a> { | |||
361 | } | 312 | } |
362 | } | 313 | } |
363 | fn is_token_joint_to_next(&self, pos: usize) -> bool { | 314 | fn is_token_joint_to_next(&self, pos: usize) -> bool { |
364 | self.walker.get(pos).unwrap().is_joint_to_next | 315 | match self.walker.get(pos) { |
316 | Some(t) => t.is_joint_to_next, | ||
317 | _ => false, | ||
318 | } | ||
365 | } | 319 | } |
366 | fn is_keyword(&self, pos: usize, kw: &str) -> bool { | 320 | fn is_keyword(&self, pos: usize, kw: &str) -> bool { |
367 | self.walker.get(pos).unwrap().text == *kw | 321 | match self.walker.get(pos) { |
322 | Some(t) => t.text == *kw, | ||
323 | _ => false, | ||
324 | } | ||
368 | } | 325 | } |
369 | } | 326 | } |
370 | 327 | ||
@@ -467,18 +424,18 @@ where | |||
467 | None | 424 | None |
468 | } | 425 | } |
469 | 426 | ||
470 | fn convert_delim(d: tt::Delimiter, closing: bool) -> Option<TtToken> { | 427 | fn convert_delim(d: tt::Delimiter, closing: bool) -> TtToken { |
471 | let (kinds, texts) = match d { | 428 | let (kinds, texts) = match d { |
472 | tt::Delimiter::Parenthesis => ([L_PAREN, R_PAREN], "()"), | 429 | tt::Delimiter::Parenthesis => ([L_PAREN, R_PAREN], "()"), |
473 | tt::Delimiter::Brace => ([L_CURLY, R_CURLY], "{}"), | 430 | tt::Delimiter::Brace => ([L_CURLY, R_CURLY], "{}"), |
474 | tt::Delimiter::Bracket => ([L_BRACK, R_BRACK], "[]"), | 431 | tt::Delimiter::Bracket => ([L_BRACK, R_BRACK], "[]"), |
475 | tt::Delimiter::None => return None, | 432 | tt::Delimiter::None => ([L_DOLLAR, R_DOLLAR], ""), |
476 | }; | 433 | }; |
477 | 434 | ||
478 | let idx = closing as usize; | 435 | let idx = closing as usize; |
479 | let kind = kinds[idx]; | 436 | let kind = kinds[idx]; |
480 | let text = &texts[idx..texts.len() - (1 - idx)]; | 437 | let text = if texts.len() > 0 { &texts[idx..texts.len() - (1 - idx)] } else { "" }; |
481 | Some(TtToken { kind, is_joint_to_next: false, text: SmolStr::new(text), n_tokens: 1 }) | 438 | TtToken { kind, is_joint_to_next: false, text: SmolStr::new(text), n_tokens: 1 } |
482 | } | 439 | } |
483 | 440 | ||
484 | fn convert_literal(l: &tt::Literal) -> TtToken { | 441 | fn convert_literal(l: &tt::Literal) -> TtToken { |
@@ -495,8 +452,9 @@ fn convert_ident(ident: &tt::Ident) -> TtToken { | |||
495 | TtToken { kind, is_joint_to_next: false, text: ident.text.clone(), n_tokens: 1 } | 452 | TtToken { kind, is_joint_to_next: false, text: ident.text.clone(), n_tokens: 1 } |
496 | } | 453 | } |
497 | 454 | ||
498 | fn convert_punct(p: &tt::Punct, parent: &tt::Subtree, next: usize) -> TtToken { | 455 | fn convert_punct(p: &tt::Punct, next_tokens: &[tt::TokenTree]) -> TtToken { |
499 | let iter = parent.token_trees[next + 1..].iter(); | 456 | let mut iter = next_tokens.iter(); |
457 | iter.next(); | ||
500 | let mut peek = TokenPeek::new(iter); | 458 | let mut peek = TokenPeek::new(iter); |
501 | 459 | ||
502 | if let Some((kind, is_joint_to_next, text, size)) = convert_multi_char_punct(p, &mut peek) { | 460 | if let Some((kind, is_joint_to_next, text, size)) = convert_multi_char_punct(p, &mut peek) { |
@@ -519,3 +477,11 @@ fn convert_punct(p: &tt::Punct, parent: &tt::Subtree, next: usize) -> TtToken { | |||
519 | TtToken { kind, is_joint_to_next: p.spacing == tt::Spacing::Joint, text, n_tokens: 1 } | 477 | TtToken { kind, is_joint_to_next: p.spacing == tt::Spacing::Joint, text, n_tokens: 1 } |
520 | } | 478 | } |
521 | } | 479 | } |
480 | |||
481 | fn convert_leaf(tokens: &[tt::TokenTree], leaf: &tt::Leaf) -> TtToken { | ||
482 | match leaf { | ||
483 | tt::Leaf::Literal(l) => convert_literal(l), | ||
484 | tt::Leaf::Ident(ident) => convert_ident(ident), | ||
485 | tt::Leaf::Punct(punct) => convert_punct(punct, tokens), | ||
486 | } | ||
487 | } | ||
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 19c17bd55..28ded7870 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs | |||
@@ -22,6 +22,14 @@ pub fn ast_to_token_tree(ast: &ast::TokenTree) -> Option<(tt::Subtree, TokenMap) | |||
22 | Some((tt, token_map)) | 22 | Some((tt, token_map)) |
23 | } | 23 | } |
24 | 24 | ||
25 | /// Convert the syntax node to a `TokenTree` (what macro | ||
26 | /// will consume). | ||
27 | pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, TokenMap)> { | ||
28 | let mut token_map = TokenMap::default(); | ||
29 | let tt = convert_tt(&mut token_map, node.range().start(), node)?; | ||
30 | Some((tt, token_map)) | ||
31 | } | ||
32 | |||
25 | /// Parses the token tree (result of macro expansion) as a sequence of items | 33 | /// Parses the token tree (result of macro expansion) as a sequence of items |
26 | pub fn token_tree_to_ast_item_list(tt: &tt::Subtree) -> TreeArc<ast::SourceFile> { | 34 | pub fn token_tree_to_ast_item_list(tt: &tt::Subtree) -> TreeArc<ast::SourceFile> { |
27 | let token_source = SubtreeTokenSource::new(tt); | 35 | let token_source = SubtreeTokenSource::new(tt); |
@@ -51,15 +59,17 @@ fn convert_tt( | |||
51 | ) -> Option<tt::Subtree> { | 59 | ) -> Option<tt::Subtree> { |
52 | let first_child = tt.first_child_or_token()?; | 60 | let first_child = tt.first_child_or_token()?; |
53 | let last_child = tt.last_child_or_token()?; | 61 | let last_child = tt.last_child_or_token()?; |
54 | let delimiter = match (first_child.kind(), last_child.kind()) { | 62 | let (delimiter, skip_first) = match (first_child.kind(), last_child.kind()) { |
55 | (L_PAREN, R_PAREN) => tt::Delimiter::Parenthesis, | 63 | (L_PAREN, R_PAREN) => (tt::Delimiter::Parenthesis, true), |
56 | (L_CURLY, R_CURLY) => tt::Delimiter::Brace, | 64 | (L_CURLY, R_CURLY) => (tt::Delimiter::Brace, true), |
57 | (L_BRACK, R_BRACK) => tt::Delimiter::Bracket, | 65 | (L_BRACK, R_BRACK) => (tt::Delimiter::Bracket, true), |
58 | _ => return None, | 66 | _ => (tt::Delimiter::None, false), |
59 | }; | 67 | }; |
68 | |||
60 | let mut token_trees = Vec::new(); | 69 | let mut token_trees = Vec::new(); |
61 | for child in tt.children_with_tokens().skip(1) { | 70 | for child in tt.children_with_tokens().skip(skip_first as usize) { |
62 | if child == first_child || child == last_child || child.kind().is_trivia() { | 71 | if (skip_first && (child == first_child || child == last_child)) || child.kind().is_trivia() |
72 | { | ||
63 | continue; | 73 | continue; |
64 | } | 74 | } |
65 | match child { | 75 | match child { |
@@ -127,6 +137,11 @@ impl<'a, Q: Querier> TtTreeSink<'a, Q> { | |||
127 | 137 | ||
128 | impl<'a, Q: Querier> TreeSink for TtTreeSink<'a, Q> { | 138 | impl<'a, Q: Querier> TreeSink for TtTreeSink<'a, Q> { |
129 | fn token(&mut self, kind: SyntaxKind, n_tokens: u8) { | 139 | fn token(&mut self, kind: SyntaxKind, n_tokens: u8) { |
140 | if kind == L_DOLLAR || kind == R_DOLLAR { | ||
141 | self.token_pos += n_tokens as usize; | ||
142 | return; | ||
143 | } | ||
144 | |||
130 | for _ in 0..n_tokens { | 145 | for _ in 0..n_tokens { |
131 | self.buf += &self.src_querier.token(self.token_pos).1; | 146 | self.buf += &self.src_querier.token(self.token_pos).1; |
132 | self.token_pos += 1; | 147 | self.token_pos += 1; |
@@ -176,19 +191,19 @@ mod tests { | |||
176 | 191 | ||
177 | let query = tt_src.querier(); | 192 | let query = tt_src.querier(); |
178 | 193 | ||
179 | // [{] | 194 | // [${] |
180 | // [let] [a] [=] ['c'] [;] | 195 | // [let] [a] [=] ['c'] [;] |
181 | assert_eq!(query.token(1 + 3).1, "'c'"); | 196 | assert_eq!(query.token(2 + 3).1, "'c'"); |
182 | assert_eq!(query.token(1 + 3).0, CHAR); | 197 | assert_eq!(query.token(2 + 3).0, CHAR); |
183 | // [let] [c] [=] [1000] [;] | 198 | // [let] [c] [=] [1000] [;] |
184 | assert_eq!(query.token(1 + 5 + 3).1, "1000"); | 199 | assert_eq!(query.token(2 + 5 + 3).1, "1000"); |
185 | assert_eq!(query.token(1 + 5 + 3).0, INT_NUMBER); | 200 | assert_eq!(query.token(2 + 5 + 3).0, INT_NUMBER); |
186 | // [let] [f] [=] [12E+99_f64] [;] | 201 | // [let] [f] [=] [12E+99_f64] [;] |
187 | assert_eq!(query.token(1 + 10 + 3).1, "12E+99_f64"); | 202 | assert_eq!(query.token(2 + 10 + 3).1, "12E+99_f64"); |
188 | assert_eq!(query.token(1 + 10 + 3).0, FLOAT_NUMBER); | 203 | assert_eq!(query.token(2 + 10 + 3).0, FLOAT_NUMBER); |
189 | 204 | ||
190 | // [let] [s] [=] ["rust1"] [;] | 205 | // [let] [s] [=] ["rust1"] [;] |
191 | assert_eq!(query.token(1 + 15 + 3).1, "\"rust1\""); | 206 | assert_eq!(query.token(2 + 15 + 3).1, "\"rust1\""); |
192 | assert_eq!(query.token(1 + 15 + 3).0, STRING); | 207 | assert_eq!(query.token(2 + 15 + 3).0, STRING); |
193 | } | 208 | } |
194 | } | 209 | } |
diff --git a/crates/ra_mbe/src/tt_cursor.rs b/crates/ra_mbe/src/tt_cursor.rs index d29faa77c..f6cefe087 100644 --- a/crates/ra_mbe/src/tt_cursor.rs +++ b/crates/ra_mbe/src/tt_cursor.rs | |||
@@ -84,6 +84,21 @@ impl<'a> TtCursor<'a> { | |||
84 | parser.parse_path() | 84 | parser.parse_path() |
85 | } | 85 | } |
86 | 86 | ||
87 | pub(crate) fn eat_expr(&mut self) -> Option<tt::TokenTree> { | ||
88 | let parser = Parser::new(&mut self.pos, self.subtree); | ||
89 | parser.parse_expr() | ||
90 | } | ||
91 | |||
92 | pub(crate) fn eat_ty(&mut self) -> Option<tt::TokenTree> { | ||
93 | let parser = Parser::new(&mut self.pos, self.subtree); | ||
94 | parser.parse_ty() | ||
95 | } | ||
96 | |||
97 | pub(crate) fn eat_pat(&mut self) -> Option<tt::TokenTree> { | ||
98 | let parser = Parser::new(&mut self.pos, self.subtree); | ||
99 | parser.parse_pat() | ||
100 | } | ||
101 | |||
87 | pub(crate) fn expect_char(&mut self, char: char) -> Result<(), ParseError> { | 102 | pub(crate) fn expect_char(&mut self, char: char) -> Result<(), ParseError> { |
88 | if self.at_char(char) { | 103 | if self.at_char(char) { |
89 | self.bump(); | 104 | self.bump(); |
diff --git a/crates/ra_parser/src/grammar.rs b/crates/ra_parser/src/grammar.rs index c5f510e6b..5a7a55141 100644 --- a/crates/ra_parser/src/grammar.rs +++ b/crates/ra_parser/src/grammar.rs | |||
@@ -53,6 +53,18 @@ pub(crate) fn path(p: &mut Parser) { | |||
53 | paths::type_path(p); | 53 | paths::type_path(p); |
54 | } | 54 | } |
55 | 55 | ||
56 | pub(crate) fn expr(p: &mut Parser) { | ||
57 | expressions::expr(p); | ||
58 | } | ||
59 | |||
60 | pub(crate) fn type_(p: &mut Parser) { | ||
61 | types::type_(p) | ||
62 | } | ||
63 | |||
64 | pub(crate) fn pattern(p: &mut Parser) { | ||
65 | patterns::pattern(p) | ||
66 | } | ||
67 | |||
56 | pub(crate) fn reparser( | 68 | pub(crate) fn reparser( |
57 | node: SyntaxKind, | 69 | node: SyntaxKind, |
58 | first_child: Option<SyntaxKind>, | 70 | first_child: Option<SyntaxKind>, |
diff --git a/crates/ra_parser/src/grammar/expressions.rs b/crates/ra_parser/src/grammar/expressions.rs index 9b38b0a31..295577325 100644 --- a/crates/ra_parser/src/grammar/expressions.rs +++ b/crates/ra_parser/src/grammar/expressions.rs | |||
@@ -8,17 +8,20 @@ const EXPR_FIRST: TokenSet = LHS_FIRST; | |||
8 | 8 | ||
9 | pub(super) fn expr(p: &mut Parser) -> BlockLike { | 9 | pub(super) fn expr(p: &mut Parser) -> BlockLike { |
10 | let r = Restrictions { forbid_structs: false, prefer_stmt: false }; | 10 | let r = Restrictions { forbid_structs: false, prefer_stmt: false }; |
11 | expr_bp(p, r, 1).1 | 11 | let mut dollar_lvl = 0; |
12 | expr_bp(p, r, 1, &mut dollar_lvl).1 | ||
12 | } | 13 | } |
13 | 14 | ||
14 | pub(super) fn expr_stmt(p: &mut Parser) -> (Option<CompletedMarker>, BlockLike) { | 15 | pub(super) fn expr_stmt(p: &mut Parser) -> (Option<CompletedMarker>, BlockLike) { |
15 | let r = Restrictions { forbid_structs: false, prefer_stmt: true }; | 16 | let r = Restrictions { forbid_structs: false, prefer_stmt: true }; |
16 | expr_bp(p, r, 1) | 17 | let mut dollar_lvl = 0; |
18 | expr_bp(p, r, 1, &mut dollar_lvl) | ||
17 | } | 19 | } |
18 | 20 | ||
19 | fn expr_no_struct(p: &mut Parser) { | 21 | fn expr_no_struct(p: &mut Parser) { |
20 | let r = Restrictions { forbid_structs: true, prefer_stmt: false }; | 22 | let r = Restrictions { forbid_structs: true, prefer_stmt: false }; |
21 | expr_bp(p, r, 1); | 23 | let mut dollar_lvl = 0; |
24 | expr_bp(p, r, 1, &mut dollar_lvl); | ||
22 | } | 25 | } |
23 | 26 | ||
24 | // test block | 27 | // test block |
@@ -206,8 +209,23 @@ fn current_op(p: &Parser) -> (u8, Op) { | |||
206 | } | 209 | } |
207 | 210 | ||
208 | // Parses expression with binding power of at least bp. | 211 | // Parses expression with binding power of at least bp. |
209 | fn expr_bp(p: &mut Parser, r: Restrictions, bp: u8) -> (Option<CompletedMarker>, BlockLike) { | 212 | fn expr_bp( |
210 | let mut lhs = match lhs(p, r) { | 213 | p: &mut Parser, |
214 | r: Restrictions, | ||
215 | mut bp: u8, | ||
216 | dollar_lvl: &mut usize, | ||
217 | ) -> (Option<CompletedMarker>, BlockLike) { | ||
218 | // `newly_dollar_open` is a flag indicated that dollar is just closed after lhs, e.g. | ||
219 | // `$1$ + a` | ||
220 | // We use this flag to skip handling it. | ||
221 | let mut newly_dollar_open = false; | ||
222 | |||
223 | if p.at_l_dollar() { | ||
224 | *dollar_lvl += p.eat_l_dollars(); | ||
225 | newly_dollar_open = true; | ||
226 | } | ||
227 | |||
228 | let mut lhs = match lhs(p, r, dollar_lvl) { | ||
211 | Some((lhs, blocklike)) => { | 229 | Some((lhs, blocklike)) => { |
212 | // test stmt_bin_expr_ambiguity | 230 | // test stmt_bin_expr_ambiguity |
213 | // fn foo() { | 231 | // fn foo() { |
@@ -223,6 +241,15 @@ fn expr_bp(p: &mut Parser, r: Restrictions, bp: u8) -> (Option<CompletedMarker>, | |||
223 | }; | 241 | }; |
224 | 242 | ||
225 | loop { | 243 | loop { |
244 | if *dollar_lvl > 0 && p.at_r_dollar() { | ||
245 | *dollar_lvl -= p.eat_r_dollars(*dollar_lvl); | ||
246 | if !newly_dollar_open { | ||
247 | // We "pump" bp for make it highest priority | ||
248 | bp = 255; | ||
249 | } | ||
250 | newly_dollar_open = false; | ||
251 | } | ||
252 | |||
226 | let is_range = p.current() == DOTDOT || p.current() == DOTDOTEQ; | 253 | let is_range = p.current() == DOTDOT || p.current() == DOTDOTEQ; |
227 | let (op_bp, op) = current_op(p); | 254 | let (op_bp, op) = current_op(p); |
228 | if op_bp < bp { | 255 | if op_bp < bp { |
@@ -235,7 +262,8 @@ fn expr_bp(p: &mut Parser, r: Restrictions, bp: u8) -> (Option<CompletedMarker>, | |||
235 | p.bump_compound(kind, n); | 262 | p.bump_compound(kind, n); |
236 | } | 263 | } |
237 | } | 264 | } |
238 | expr_bp(p, r, op_bp + 1); | 265 | |
266 | expr_bp(p, r, op_bp + 1, dollar_lvl); | ||
239 | lhs = m.complete(p, if is_range { RANGE_EXPR } else { BIN_EXPR }); | 267 | lhs = m.complete(p, if is_range { RANGE_EXPR } else { BIN_EXPR }); |
240 | } | 268 | } |
241 | (Some(lhs), BlockLike::NotBlock) | 269 | (Some(lhs), BlockLike::NotBlock) |
@@ -244,7 +272,11 @@ fn expr_bp(p: &mut Parser, r: Restrictions, bp: u8) -> (Option<CompletedMarker>, | |||
244 | const LHS_FIRST: TokenSet = | 272 | const LHS_FIRST: TokenSet = |
245 | atom::ATOM_EXPR_FIRST.union(token_set![AMP, STAR, EXCL, DOTDOT, DOTDOTEQ, MINUS]); | 273 | atom::ATOM_EXPR_FIRST.union(token_set![AMP, STAR, EXCL, DOTDOT, DOTDOTEQ, MINUS]); |
246 | 274 | ||
247 | fn lhs(p: &mut Parser, r: Restrictions) -> Option<(CompletedMarker, BlockLike)> { | 275 | fn lhs( |
276 | p: &mut Parser, | ||
277 | r: Restrictions, | ||
278 | dollar_lvl: &mut usize, | ||
279 | ) -> Option<(CompletedMarker, BlockLike)> { | ||
248 | let m; | 280 | let m; |
249 | let kind = match p.current() { | 281 | let kind = match p.current() { |
250 | // test ref_expr | 282 | // test ref_expr |
@@ -275,7 +307,7 @@ fn lhs(p: &mut Parser, r: Restrictions) -> Option<(CompletedMarker, BlockLike)> | |||
275 | m = p.start(); | 307 | m = p.start(); |
276 | p.bump(); | 308 | p.bump(); |
277 | if p.at_ts(EXPR_FIRST) { | 309 | if p.at_ts(EXPR_FIRST) { |
278 | expr_bp(p, r, 2); | 310 | expr_bp(p, r, 2, dollar_lvl); |
279 | } | 311 | } |
280 | return Some((m.complete(p, RANGE_EXPR), BlockLike::NotBlock)); | 312 | return Some((m.complete(p, RANGE_EXPR), BlockLike::NotBlock)); |
281 | } | 313 | } |
@@ -287,7 +319,7 @@ fn lhs(p: &mut Parser, r: Restrictions) -> Option<(CompletedMarker, BlockLike)> | |||
287 | )); | 319 | )); |
288 | } | 320 | } |
289 | }; | 321 | }; |
290 | expr_bp(p, r, 255); | 322 | expr_bp(p, r, 255, dollar_lvl); |
291 | Some((m.complete(p, kind), BlockLike::NotBlock)) | 323 | Some((m.complete(p, kind), BlockLike::NotBlock)) |
292 | } | 324 | } |
293 | 325 | ||
diff --git a/crates/ra_parser/src/grammar/patterns.rs b/crates/ra_parser/src/grammar/patterns.rs index 9a307559b..03fa9b71e 100644 --- a/crates/ra_parser/src/grammar/patterns.rs +++ b/crates/ra_parser/src/grammar/patterns.rs | |||
@@ -5,7 +5,7 @@ pub(super) const PATTERN_FIRST: TokenSet = expressions::LITERAL_FIRST | |||
5 | .union(token_set![REF_KW, MUT_KW, L_PAREN, L_BRACK, AMP, UNDERSCORE, MINUS]); | 5 | .union(token_set![REF_KW, MUT_KW, L_PAREN, L_BRACK, AMP, UNDERSCORE, MINUS]); |
6 | 6 | ||
7 | pub(super) fn pattern(p: &mut Parser) { | 7 | pub(super) fn pattern(p: &mut Parser) { |
8 | pattern_r(p, PAT_RECOVERY_SET) | 8 | pattern_r(p, PAT_RECOVERY_SET); |
9 | } | 9 | } |
10 | 10 | ||
11 | /// Parses a pattern list separated by pipes `|` | 11 | /// Parses a pattern list separated by pipes `|` |
diff --git a/crates/ra_parser/src/lib.rs b/crates/ra_parser/src/lib.rs index 3ceeeebd7..56755c394 100644 --- a/crates/ra_parser/src/lib.rs +++ b/crates/ra_parser/src/lib.rs | |||
@@ -53,20 +53,39 @@ pub trait TreeSink { | |||
53 | fn error(&mut self, error: ParseError); | 53 | fn error(&mut self, error: ParseError); |
54 | } | 54 | } |
55 | 55 | ||
56 | /// Parse given tokens into the given sink as a rust file. | 56 | fn parse_from_tokens<F>(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink, f: F) |
57 | pub fn parse(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { | 57 | where |
58 | F: FnOnce(&mut parser::Parser), | ||
59 | { | ||
58 | let mut p = parser::Parser::new(token_source); | 60 | let mut p = parser::Parser::new(token_source); |
59 | grammar::root(&mut p); | 61 | f(&mut p); |
60 | let events = p.finish(); | 62 | let events = p.finish(); |
61 | event::process(tree_sink, events); | 63 | event::process(tree_sink, events); |
62 | } | 64 | } |
63 | 65 | ||
66 | /// Parse given tokens into the given sink as a rust file. | ||
67 | pub fn parse(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { | ||
68 | parse_from_tokens(token_source, tree_sink, grammar::root); | ||
69 | } | ||
70 | |||
64 | /// Parse given tokens into the given sink as a path | 71 | /// Parse given tokens into the given sink as a path |
65 | pub fn parse_path(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { | 72 | pub fn parse_path(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { |
66 | let mut p = parser::Parser::new(token_source); | 73 | parse_from_tokens(token_source, tree_sink, grammar::path); |
67 | grammar::path(&mut p); | 74 | } |
68 | let events = p.finish(); | 75 | |
69 | event::process(tree_sink, events); | 76 | /// Parse given tokens into the given sink as a expression |
77 | pub fn parse_expr(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { | ||
78 | parse_from_tokens(token_source, tree_sink, grammar::expr); | ||
79 | } | ||
80 | |||
81 | /// Parse given tokens into the given sink as a ty | ||
82 | pub fn parse_ty(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { | ||
83 | parse_from_tokens(token_source, tree_sink, grammar::type_); | ||
84 | } | ||
85 | |||
86 | /// Parse given tokens into the given sink as a pattern | ||
87 | pub fn parse_pat(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { | ||
88 | parse_from_tokens(token_source, tree_sink, grammar::pattern); | ||
70 | } | 89 | } |
71 | 90 | ||
72 | /// A parsing function for a specific braced-block. | 91 | /// A parsing function for a specific braced-block. |
diff --git a/crates/ra_parser/src/parser.rs b/crates/ra_parser/src/parser.rs index 56f8b7126..71f1f8b30 100644 --- a/crates/ra_parser/src/parser.rs +++ b/crates/ra_parser/src/parser.rs | |||
@@ -45,8 +45,9 @@ impl<'t> Parser<'t> { | |||
45 | /// | 45 | /// |
46 | /// Useful for parsing things like `>>`. | 46 | /// Useful for parsing things like `>>`. |
47 | pub(crate) fn current2(&self) -> Option<(SyntaxKind, SyntaxKind)> { | 47 | pub(crate) fn current2(&self) -> Option<(SyntaxKind, SyntaxKind)> { |
48 | let c1 = self.token_source.token_kind(self.token_pos); | 48 | let c1 = self.nth(0); |
49 | let c2 = self.token_source.token_kind(self.token_pos + 1); | 49 | let c2 = self.nth(1); |
50 | |||
50 | if self.token_source.is_token_joint_to_next(self.token_pos) { | 51 | if self.token_source.is_token_joint_to_next(self.token_pos) { |
51 | Some((c1, c2)) | 52 | Some((c1, c2)) |
52 | } else { | 53 | } else { |
@@ -59,9 +60,9 @@ impl<'t> Parser<'t> { | |||
59 | /// | 60 | /// |
60 | /// Useful for parsing things like `=>>`. | 61 | /// Useful for parsing things like `=>>`. |
61 | pub(crate) fn current3(&self) -> Option<(SyntaxKind, SyntaxKind, SyntaxKind)> { | 62 | pub(crate) fn current3(&self) -> Option<(SyntaxKind, SyntaxKind, SyntaxKind)> { |
62 | let c1 = self.token_source.token_kind(self.token_pos); | 63 | let c1 = self.nth(0); |
63 | let c2 = self.token_source.token_kind(self.token_pos + 1); | 64 | let c2 = self.nth(1); |
64 | let c3 = self.token_source.token_kind(self.token_pos + 2); | 65 | let c3 = self.nth(2); |
65 | if self.token_source.is_token_joint_to_next(self.token_pos) | 66 | if self.token_source.is_token_joint_to_next(self.token_pos) |
66 | && self.token_source.is_token_joint_to_next(self.token_pos + 1) | 67 | && self.token_source.is_token_joint_to_next(self.token_pos + 1) |
67 | { | 68 | { |
@@ -77,7 +78,23 @@ impl<'t> Parser<'t> { | |||
77 | let steps = self.steps.get(); | 78 | let steps = self.steps.get(); |
78 | assert!(steps <= 10_000_000, "the parser seems stuck"); | 79 | assert!(steps <= 10_000_000, "the parser seems stuck"); |
79 | self.steps.set(steps + 1); | 80 | self.steps.set(steps + 1); |
80 | self.token_source.token_kind(self.token_pos + n) | 81 | |
82 | // It is beecause the Dollar will appear between nth | ||
83 | // Following code skips through it | ||
84 | let mut non_dollars_count = 0; | ||
85 | let mut i = 0; | ||
86 | |||
87 | loop { | ||
88 | let kind = self.token_source.token_kind(self.token_pos + i); | ||
89 | i += 1; | ||
90 | |||
91 | match kind { | ||
92 | EOF => return EOF, | ||
93 | SyntaxKind::L_DOLLAR | SyntaxKind::R_DOLLAR => {} | ||
94 | _ if non_dollars_count == n => return kind, | ||
95 | _ => non_dollars_count += 1, | ||
96 | } | ||
97 | } | ||
81 | } | 98 | } |
82 | 99 | ||
83 | /// Checks if the current token is `kind`. | 100 | /// Checks if the current token is `kind`. |
@@ -180,6 +197,7 @@ impl<'t> Parser<'t> { | |||
180 | } | 197 | } |
181 | 198 | ||
182 | fn do_bump(&mut self, kind: SyntaxKind, n_raw_tokens: u8) { | 199 | fn do_bump(&mut self, kind: SyntaxKind, n_raw_tokens: u8) { |
200 | self.eat_dollars(); | ||
183 | self.token_pos += usize::from(n_raw_tokens); | 201 | self.token_pos += usize::from(n_raw_tokens); |
184 | self.push_event(Event::Token { kind, n_raw_tokens }); | 202 | self.push_event(Event::Token { kind, n_raw_tokens }); |
185 | } | 203 | } |
@@ -187,6 +205,66 @@ impl<'t> Parser<'t> { | |||
187 | fn push_event(&mut self, event: Event) { | 205 | fn push_event(&mut self, event: Event) { |
188 | self.events.push(event) | 206 | self.events.push(event) |
189 | } | 207 | } |
208 | |||
209 | fn eat_dollars(&mut self) { | ||
210 | loop { | ||
211 | match self.token_source.token_kind(self.token_pos) { | ||
212 | k @ SyntaxKind::L_DOLLAR | k @ SyntaxKind::R_DOLLAR => { | ||
213 | self.token_pos += 1; | ||
214 | self.push_event(Event::Token { kind: k, n_raw_tokens: 1 }); | ||
215 | } | ||
216 | _ => { | ||
217 | return; | ||
218 | } | ||
219 | } | ||
220 | } | ||
221 | } | ||
222 | |||
223 | pub(crate) fn eat_l_dollars(&mut self) -> usize { | ||
224 | let mut ate_count = 0; | ||
225 | loop { | ||
226 | match self.token_source.token_kind(self.token_pos) { | ||
227 | k @ SyntaxKind::L_DOLLAR => { | ||
228 | self.token_pos += 1; | ||
229 | self.push_event(Event::Token { kind: k, n_raw_tokens: 1 }); | ||
230 | ate_count += 1; | ||
231 | } | ||
232 | _ => { | ||
233 | return ate_count; | ||
234 | } | ||
235 | } | ||
236 | } | ||
237 | } | ||
238 | |||
239 | pub(crate) fn eat_r_dollars(&mut self, max_count: usize) -> usize { | ||
240 | let mut ate_count = 0; | ||
241 | loop { | ||
242 | match self.token_source.token_kind(self.token_pos) { | ||
243 | k @ SyntaxKind::R_DOLLAR => { | ||
244 | self.token_pos += 1; | ||
245 | self.push_event(Event::Token { kind: k, n_raw_tokens: 1 }); | ||
246 | ate_count += 1; | ||
247 | |||
248 | if max_count >= ate_count { | ||
249 | return ate_count; | ||
250 | } | ||
251 | } | ||
252 | _ => { | ||
253 | return ate_count; | ||
254 | } | ||
255 | } | ||
256 | } | ||
257 | } | ||
258 | |||
259 | pub(crate) fn at_l_dollar(&self) -> bool { | ||
260 | let kind = self.token_source.token_kind(self.token_pos); | ||
261 | (kind == SyntaxKind::L_DOLLAR) | ||
262 | } | ||
263 | |||
264 | pub(crate) fn at_r_dollar(&self) -> bool { | ||
265 | let kind = self.token_source.token_kind(self.token_pos); | ||
266 | (kind == SyntaxKind::R_DOLLAR) | ||
267 | } | ||
190 | } | 268 | } |
191 | 269 | ||
192 | /// See `Parser::start`. | 270 | /// See `Parser::start`. |
diff --git a/crates/ra_parser/src/syntax_kind/generated.rs b/crates/ra_parser/src/syntax_kind/generated.rs index b8350266f..498b0e164 100644 --- a/crates/ra_parser/src/syntax_kind/generated.rs +++ b/crates/ra_parser/src/syntax_kind/generated.rs | |||
@@ -120,6 +120,8 @@ pub enum SyntaxKind { | |||
120 | LIFETIME, | 120 | LIFETIME, |
121 | COMMENT, | 121 | COMMENT, |
122 | SHEBANG, | 122 | SHEBANG, |
123 | L_DOLLAR, | ||
124 | R_DOLLAR, | ||
123 | SOURCE_FILE, | 125 | SOURCE_FILE, |
124 | STRUCT_DEF, | 126 | STRUCT_DEF, |
125 | ENUM_DEF, | 127 | ENUM_DEF, |
@@ -477,6 +479,8 @@ impl SyntaxKind { | |||
477 | LIFETIME => &SyntaxInfo { name: "LIFETIME" }, | 479 | LIFETIME => &SyntaxInfo { name: "LIFETIME" }, |
478 | COMMENT => &SyntaxInfo { name: "COMMENT" }, | 480 | COMMENT => &SyntaxInfo { name: "COMMENT" }, |
479 | SHEBANG => &SyntaxInfo { name: "SHEBANG" }, | 481 | SHEBANG => &SyntaxInfo { name: "SHEBANG" }, |
482 | L_DOLLAR => &SyntaxInfo { name: "L_DOLLAR" }, | ||
483 | R_DOLLAR => &SyntaxInfo { name: "R_DOLLAR" }, | ||
480 | SOURCE_FILE => &SyntaxInfo { name: "SOURCE_FILE" }, | 484 | SOURCE_FILE => &SyntaxInfo { name: "SOURCE_FILE" }, |
481 | STRUCT_DEF => &SyntaxInfo { name: "STRUCT_DEF" }, | 485 | STRUCT_DEF => &SyntaxInfo { name: "STRUCT_DEF" }, |
482 | ENUM_DEF => &SyntaxInfo { name: "ENUM_DEF" }, | 486 | ENUM_DEF => &SyntaxInfo { name: "ENUM_DEF" }, |
diff --git a/crates/ra_syntax/src/grammar.ron b/crates/ra_syntax/src/grammar.ron index 0a35e25d5..b41241287 100644 --- a/crates/ra_syntax/src/grammar.ron +++ b/crates/ra_syntax/src/grammar.ron | |||
@@ -118,6 +118,8 @@ Grammar( | |||
118 | "LIFETIME", | 118 | "LIFETIME", |
119 | "COMMENT", | 119 | "COMMENT", |
120 | "SHEBANG", | 120 | "SHEBANG", |
121 | "L_DOLLAR", | ||
122 | "R_DOLLAR", | ||
121 | ], | 123 | ], |
122 | nodes: [ | 124 | nodes: [ |
123 | "SOURCE_FILE", | 125 | "SOURCE_FILE", |