aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--crates/ra_mbe/src/lib.rs34
-rw-r--r--crates/ra_mbe/src/subtree_parser.rs10
-rw-r--r--crates/ra_mbe/src/subtree_source.rs623
-rw-r--r--crates/ra_mbe/src/syntax_bridge.rs12
4 files changed, 421 insertions, 258 deletions
diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs
index 84ce2b783..a21ea4dbc 100644
--- a/crates/ra_mbe/src/lib.rs
+++ b/crates/ra_mbe/src/lib.rs
@@ -383,8 +383,22 @@ SOURCE_FILE@[0; 40)
383 assert_eq!(to_literal(&stm_tokens[15 + 3]).text, "\"rust1\""); 383 assert_eq!(to_literal(&stm_tokens[15 + 3]).text, "\"rust1\"");
384 } 384 }
385 385
386 /// The following tests are port from intellij-rust directly 386 #[test]
387 /// https://github.com/intellij-rust/intellij-rust/blob/c4e9feee4ad46e7953b1948c112533360b6087bb/src/test/kotlin/org/rust/lang/core/macros/RsMacroExpansionTest.kt 387 fn test_two_idents() {
388 let rules = create_rules(
389 r#"
390 macro_rules! foo {
391 ($ i:ident, $ j:ident) => {
392 fn foo() { let a = $ i; let b = $j; }
393 }
394 }
395"#,
396 );
397 assert_expansion(&rules, "foo! { foo, bar }", "fn foo () {let a = foo ; let b = bar ;}");
398 }
399
400 // The following tests are port from intellij-rust directly
401 // https://github.com/intellij-rust/intellij-rust/blob/c4e9feee4ad46e7953b1948c112533360b6087bb/src/test/kotlin/org/rust/lang/core/macros/RsMacroExpansionTest.kt
388 402
389 #[test] 403 #[test]
390 fn test_path() { 404 fn test_path() {
@@ -401,7 +415,21 @@ SOURCE_FILE@[0; 40)
401 assert_expansion( 415 assert_expansion(
402 &rules, 416 &rules,
403 "foo! { bar::<u8>::baz::<u8> }", 417 "foo! { bar::<u8>::baz::<u8> }",
404 "fn foo () {let a = bar :: < u8 > :: baz :: < u8 > ;}", 418 "fn foo () {let a = bar ::< u8 > ::baz ::< u8 > ;}",
419 );
420 }
421
422 #[test]
423 fn test_two_paths() {
424 let rules = create_rules(
425 r#"
426 macro_rules! foo {
427 ($ i:path, $ j:path) => {
428 fn foo() { let a = $ i; let b = $j; }
429 }
430 }
431"#,
405 ); 432 );
433 assert_expansion(&rules, "foo! { foo, bar }", "fn foo () {let a = foo ; let b = bar ;}");
406 } 434 }
407} 435}
diff --git a/crates/ra_mbe/src/subtree_parser.rs b/crates/ra_mbe/src/subtree_parser.rs
index 48eee6fa7..f198c8224 100644
--- a/crates/ra_mbe/src/subtree_parser.rs
+++ b/crates/ra_mbe/src/subtree_parser.rs
@@ -18,12 +18,12 @@ impl TreeSink for OffsetTokenSink {
18 18
19pub(crate) struct Parser<'a> { 19pub(crate) struct Parser<'a> {
20 subtree: &'a tt::Subtree, 20 subtree: &'a tt::Subtree,
21 pos: &'a mut usize, 21 cur_pos: &'a mut usize,
22} 22}
23 23
24impl<'a> Parser<'a> { 24impl<'a> Parser<'a> {
25 pub fn new(pos: &'a mut usize, subtree: &'a tt::Subtree) -> Parser<'a> { 25 pub fn new(cur_pos: &'a mut usize, subtree: &'a tt::Subtree) -> Parser<'a> {
26 Parser { pos, subtree } 26 Parser { cur_pos, subtree }
27 } 27 }
28 28
29 pub fn parse_path(self) -> Option<tt::TokenTree> { 29 pub fn parse_path(self) -> Option<tt::TokenTree> {
@@ -35,7 +35,7 @@ impl<'a> Parser<'a> {
35 F: FnOnce(&dyn TokenSource, &mut dyn TreeSink), 35 F: FnOnce(&dyn TokenSource, &mut dyn TreeSink),
36 { 36 {
37 let mut src = SubtreeTokenSource::new(self.subtree); 37 let mut src = SubtreeTokenSource::new(self.subtree);
38 src.advance(*self.pos, true); 38 src.start_from_nth(*self.cur_pos);
39 let mut sink = OffsetTokenSink { token_pos: 0 }; 39 let mut sink = OffsetTokenSink { token_pos: 0 };
40 40
41 f(&src, &mut sink); 41 f(&src, &mut sink);
@@ -44,7 +44,7 @@ impl<'a> Parser<'a> {
44 } 44 }
45 45
46 fn finish(self, parsed_token: usize, src: &mut SubtreeTokenSource) -> Option<tt::TokenTree> { 46 fn finish(self, parsed_token: usize, src: &mut SubtreeTokenSource) -> Option<tt::TokenTree> {
47 let res = src.bump_n(parsed_token, self.pos); 47 let res = src.bump_n(parsed_token, self.cur_pos);
48 let res: Vec<_> = res.into_iter().cloned().collect(); 48 let res: Vec<_> = res.into_iter().cloned().collect();
49 49
50 match res.len() { 50 match res.len() {
diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs
index d9ba5d3d0..9dd475f2c 100644
--- a/crates/ra_mbe/src/subtree_source.rs
+++ b/crates/ra_mbe/src/subtree_source.rs
@@ -1,7 +1,8 @@
1use ra_parser::{TokenSource}; 1use ra_parser::{TokenSource};
2use ra_syntax::{classify_literal, SmolStr, SyntaxKind, SyntaxKind::*}; 2use ra_syntax::{classify_literal, SmolStr, SyntaxKind, SyntaxKind::*};
3use std::cell::{RefCell};
3 4
4#[derive(Debug)] 5#[derive(Debug, Clone, Eq, PartialEq)]
5struct TtToken { 6struct TtToken {
6 pub kind: SyntaxKind, 7 pub kind: SyntaxKind,
7 pub is_joint_to_next: bool, 8 pub is_joint_to_next: bool,
@@ -9,107 +10,319 @@ struct TtToken {
9 pub n_tokens: usize, 10 pub n_tokens: usize,
10} 11}
11 12
12/// Querier let outside to query internal tokens as string 13#[derive(Debug, Clone, Eq, PartialEq)]
13pub(crate) struct Querier<'a> { 14enum WalkIndex {
14 src: &'a SubtreeTokenSource<'a>, 15 DelimiterBegin(Option<TtToken>),
16 Token(usize, Option<TtToken>),
17 DelimiterEnd(Option<TtToken>),
18 Eof,
15} 19}
16 20
17impl<'a> Querier<'a> { 21impl<'a> SubTreeWalker<'a> {
18 pub(crate) fn token(&self, uidx: usize) -> (SyntaxKind, &SmolStr) { 22 fn new(subtree: &tt::Subtree) -> SubTreeWalker {
19 let tkn = &self.src.tokens[uidx]; 23 let mut res = SubTreeWalker {
20 (tkn.kind, &tkn.text) 24 pos: 0,
25 stack: vec![],
26 idx: WalkIndex::Eof,
27 last_steps: vec![],
28 subtree,
29 };
30
31 res.reset();
32 res
33 }
34
35 fn reset(&mut self) {
36 self.pos = 0;
37 self.stack = vec![(self.subtree, None)];
38 self.idx = WalkIndex::DelimiterBegin(convert_delim(self.subtree.delimiter, false));
39 self.last_steps = vec![];
40
41 while self.is_empty_delimiter() {
42 self.forward_unchecked();
43 }
44 }
45
46 // This funciton will fast forward the pos cursor,
47 // Such that backward will stop at `start_pos` point
48 fn start_from_nth(&mut self, start_pos: usize) {
49 self.reset();
50 self.pos = start_pos;
51 self.idx = self.walk_token(start_pos, false);
52
53 while self.is_empty_delimiter() {
54 self.forward_unchecked();
55 }
56 }
57
58 fn current(&self) -> Option<&TtToken> {
59 match &self.idx {
60 WalkIndex::DelimiterBegin(t) => t.as_ref(),
61 WalkIndex::Token(_, t) => t.as_ref(),
62 WalkIndex::DelimiterEnd(t) => t.as_ref(),
63 WalkIndex::Eof => None,
64 }
65 }
66
67 fn is_empty_delimiter(&self) -> bool {
68 match &self.idx {
69 WalkIndex::DelimiterBegin(None) => true,
70 WalkIndex::DelimiterEnd(None) => true,
71 _ => false,
72 }
73 }
74
75 fn backward(&mut self) {
76 if self.last_steps.is_empty() {
77 return;
78 }
79 self.pos -= 1;
80 loop {
81 self.backward_unchecked();
82 // Skip Empty delimiter
83 if self.last_steps.is_empty() || !self.is_empty_delimiter() {
84 break;
85 }
86 }
87 }
88
89 fn backward_unchecked(&mut self) {
90 if self.last_steps.is_empty() {
91 return;
92 }
93
94 let last_step = self.last_steps.pop().unwrap();
95 let do_walk_token = match self.idx {
96 WalkIndex::DelimiterBegin(_) => None,
97 WalkIndex::Token(u, _) => Some(u),
98 WalkIndex::DelimiterEnd(_) => {
99 let (top, _) = self.stack.last().unwrap();
100 Some(top.token_trees.len())
101 }
102 WalkIndex::Eof => None,
103 };
104
105 self.idx = match do_walk_token {
106 Some(u) if last_step > u => WalkIndex::DelimiterBegin(convert_delim(
107 self.stack.last().unwrap().0.delimiter,
108 false,
109 )),
110 Some(u) => self.walk_token(u - last_step, true),
111 None => match self.idx {
112 WalkIndex::Eof => {
113 self.stack.push((self.subtree, None));
114 WalkIndex::DelimiterEnd(convert_delim(
115 self.stack.last().unwrap().0.delimiter,
116 true,
117 ))
118 }
119 _ => {
120 let (_, last_top_idx) = self.stack.pop().unwrap();
121 assert!(!self.stack.is_empty());
122
123 match last_top_idx.unwrap() {
124 0 => WalkIndex::DelimiterBegin(convert_delim(
125 self.stack.last().unwrap().0.delimiter,
126 false,
127 )),
128 c => self.walk_token(c - 1, true),
129 }
130 }
131 },
132 };
133 }
134
135 fn forward(&mut self) {
136 self.pos += 1;
137 loop {
138 self.forward_unchecked();
139 if !self.is_empty_delimiter() {
140 break;
141 }
142 }
143 }
144
145 fn forward_unchecked(&mut self) {
146 if self.idx == WalkIndex::Eof {
147 return;
148 }
149
150 let step = self.current().map(|x| x.n_tokens).unwrap_or(1);
151 self.last_steps.push(step);
152
153 let do_walk_token = match self.idx {
154 WalkIndex::DelimiterBegin(_) => Some(0),
155 WalkIndex::Token(u, _) => Some(u + step),
156 WalkIndex::DelimiterEnd(_) => None,
157 _ => unreachable!(),
158 };
159
160 let (top, _) = self.stack.last().unwrap();
161
162 self.idx = match do_walk_token {
163 Some(u) if u >= top.token_trees.len() => {
164 WalkIndex::DelimiterEnd(convert_delim(self.stack.last().unwrap().0.delimiter, true))
165 }
166 Some(u) => self.walk_token(u, false),
167 None => {
168 let (_, last_top_idx) = self.stack.pop().unwrap();
169 match self.stack.last() {
170 Some(top) => match last_top_idx.unwrap() {
171 idx if idx + 1 >= top.0.token_trees.len() => {
172 WalkIndex::DelimiterEnd(convert_delim(top.0.delimiter, true))
173 }
174 idx => self.walk_token(idx + 1, false),
175 },
176
177 None => WalkIndex::Eof,
178 }
179 }
180 };
181 }
182
183 fn walk_token(&mut self, pos: usize, backward: bool) -> WalkIndex {
184 let (top, _) = self.stack.last().unwrap();
185 match &top.token_trees[pos] {
186 tt::TokenTree::Subtree(subtree) => {
187 self.stack.push((subtree, Some(pos)));
188 let delim = convert_delim(self.stack.last().unwrap().0.delimiter, backward);
189 if backward {
190 WalkIndex::DelimiterEnd(delim)
191 } else {
192 WalkIndex::DelimiterBegin(delim)
193 }
194 }
195 tt::TokenTree::Leaf(leaf) => WalkIndex::Token(pos, Some(self.walk_leaf(leaf, pos))),
196 }
197 }
198
199 fn walk_leaf(&mut self, leaf: &tt::Leaf, pos: usize) -> TtToken {
200 match leaf {
201 tt::Leaf::Literal(l) => convert_literal(l),
202 tt::Leaf::Ident(ident) => convert_ident(ident),
203 tt::Leaf::Punct(punct) => {
204 let (top, _) = self.stack.last().unwrap();
205 convert_punct(punct, top, pos)
206 }
207 }
21 } 208 }
22} 209}
23 210
24pub(crate) struct SubtreeTokenSource<'a> { 211pub(crate) trait Querier {
25 tt_pos: usize, 212 fn token(&self, uidx: usize) -> (SyntaxKind, SmolStr);
26 tokens: Vec<TtToken>,
27 subtree: &'a tt::Subtree,
28} 213}
29 214
30impl<'a> SubtreeTokenSource<'a> { 215// A wrapper class for ref cell
31 pub fn new(subtree: &tt::Subtree) -> SubtreeTokenSource { 216pub(crate) struct WalkerOwner<'a> {
32 SubtreeTokenSource { tokens: TtTokenBuilder::build(subtree), tt_pos: 0, subtree } 217 walker: RefCell<SubTreeWalker<'a>>,
218 offset: usize,
219}
220
221impl<'a> WalkerOwner<'a> {
222 fn token_idx<'b>(&self, pos: usize) -> Option<TtToken> {
223 self.set_walker_pos(pos);
224 self.walker.borrow().current().cloned()
33 } 225 }
34 226
35 // Advance token source and skip the first delimiter 227 fn start_from_nth(&mut self, pos: usize) {
36 pub fn advance(&mut self, n_token: usize, skip_first_delimiter: bool) { 228 self.offset = pos;
37 if skip_first_delimiter { 229 self.walker.borrow_mut().start_from_nth(pos);
38 self.tt_pos += 1; 230 }
39 }
40 231
41 // Matching `TtToken` cursor to `tt::TokenTree` cursor 232 fn set_walker_pos(&self, mut pos: usize) {
42 // It is because TtToken is not One to One mapping to tt::Token 233 pos += self.offset;
43 // There are 3 case (`TtToken` <=> `tt::TokenTree`) : 234 let mut walker = self.walker.borrow_mut();
44 // * One to One => ident, single char punch 235 while pos > walker.pos {
45 // * Many to One => `tt::TokenTree::SubTree` 236 walker.forward();
46 // * One to Many => multibyte punct 237 }
47 // 238 while pos < walker.pos {
48 // Such that we cannot simpliy advance the cursor 239 walker.backward();
49 // We have to bump it one by one
50 let mut pos = 0;
51 while pos < n_token {
52 pos += self.bump(&self.subtree.token_trees[pos]);
53 } 240 }
241 assert!(pos == walker.pos);
54 } 242 }
55 243
56 pub fn querier(&self) -> Querier { 244 fn new(subtree: &'a tt::Subtree) -> Self {
57 Querier { src: self } 245 WalkerOwner { walker: RefCell::new(SubTreeWalker::new(subtree)), offset: 0 }
58 } 246 }
59 247
60 pub(crate) fn bump_n( 248 fn collect_token_tree(&mut self, n: usize) -> Vec<&tt::TokenTree> {
61 &mut self, 249 self.start_from_nth(self.offset);
62 n_tt_tokens: usize, 250
63 token_pos: &mut usize,
64 ) -> Vec<&tt::TokenTree> {
65 let mut res = vec![]; 251 let mut res = vec![];
66 // Matching `TtToken` cursor to `tt::TokenTree` cursor 252 let mut walker = self.walker.borrow_mut();
67 // It is because TtToken is not One to One mapping to tt::Token 253
68 // There are 3 case (`TtToken` <=> `tt::TokenTree`) : 254 while walker.pos - self.offset < n {
69 // * One to One => ident, single char punch 255 if let WalkIndex::Token(u, tt) = &walker.idx {
70 // * Many to One => `tt::TokenTree::SubTree` 256 if walker.stack.len() == 1 {
71 // * One to Many => multibyte punct 257 // We only collect the topmost child
72 // 258 res.push(&walker.stack[0].0.token_trees[*u]);
73 // Such that we cannot simpliy advance the cursor 259 if let Some(tt) = tt {
74 // We have to bump it one by one 260 for i in 0..tt.n_tokens - 1 {
75 let next_pos = self.tt_pos + n_tt_tokens; 261 res.push(&walker.stack[0].0.token_trees[u + i]);
76 262 }
77 while self.tt_pos < next_pos { 263 }
78 let current = &self.subtree.token_trees[*token_pos]; 264 }
79 let n = self.bump(current); 265 }
80 res.extend((0..n).map(|i| &self.subtree.token_trees[*token_pos + i])); 266
81 *token_pos += n; 267 walker.forward();
82 } 268 }
83 269
84 res 270 res
85 } 271 }
272}
273
274impl<'a> Querier for WalkerOwner<'a> {
275 fn token(&self, uidx: usize) -> (SyntaxKind, SmolStr) {
276 let tkn = self.token_idx(uidx).unwrap();
277 (tkn.kind, tkn.text)
278 }
279}
280
281pub(crate) struct SubtreeTokenSource<'a> {
282 walker: WalkerOwner<'a>,
283}
284
285impl<'a> SubtreeTokenSource<'a> {
286 pub fn new(subtree: &tt::Subtree) -> SubtreeTokenSource {
287 SubtreeTokenSource { walker: WalkerOwner::new(subtree) }
288 }
289
290 pub fn start_from_nth(&mut self, n: usize) {
291 self.walker.start_from_nth(n);
292 }
86 293
87 fn count(&self, tt: &tt::TokenTree) -> usize { 294 pub fn querier<'b>(&'a self) -> &'b WalkerOwner<'a>
88 assert!(!self.tokens.is_empty()); 295 where
89 TtTokenBuilder::count_tt_tokens(tt, None) 296 'a: 'b,
297 {
298 &self.walker
90 } 299 }
91 300
92 fn bump(&mut self, tt: &tt::TokenTree) -> usize { 301 pub(crate) fn bump_n(
93 let cur = &self.tokens[self.tt_pos]; 302 &mut self,
94 let n_tokens = cur.n_tokens; 303 parsed_tokens: usize,
95 self.tt_pos += self.count(tt); 304 cursor_pos: &mut usize,
96 n_tokens 305 ) -> Vec<&tt::TokenTree> {
306 let res = self.walker.collect_token_tree(parsed_tokens);
307 *cursor_pos += res.len();
308
309 res
97 } 310 }
98} 311}
99 312
100impl<'a> TokenSource for SubtreeTokenSource<'a> { 313impl<'a> TokenSource for SubtreeTokenSource<'a> {
101 fn token_kind(&self, pos: usize) -> SyntaxKind { 314 fn token_kind(&self, pos: usize) -> SyntaxKind {
102 if let Some(tok) = self.tokens.get(self.tt_pos + pos) { 315 if let Some(tok) = self.walker.token_idx(pos) {
103 tok.kind 316 tok.kind
104 } else { 317 } else {
105 SyntaxKind::EOF 318 SyntaxKind::EOF
106 } 319 }
107 } 320 }
108 fn is_token_joint_to_next(&self, pos: usize) -> bool { 321 fn is_token_joint_to_next(&self, pos: usize) -> bool {
109 self.tokens[self.tt_pos + pos].is_joint_to_next 322 self.walker.token_idx(pos).unwrap().is_joint_to_next
110 } 323 }
111 fn is_keyword(&self, pos: usize, kw: &str) -> bool { 324 fn is_keyword(&self, pos: usize, kw: &str) -> bool {
112 self.tokens[self.tt_pos + pos].text == *kw 325 self.walker.token_idx(pos).unwrap().text == *kw
113 } 326 }
114} 327}
115 328
@@ -136,10 +349,6 @@ where
136 TokenPeek { iter: itertools::multipeek(iter) } 349 TokenPeek { iter: itertools::multipeek(iter) }
137 } 350 }
138 351
139 pub fn next(&mut self) -> Option<&tt::TokenTree> {
140 self.iter.next()
141 }
142
143 fn current_punct2(&mut self, p: &tt::Punct) -> Option<((char, char), bool)> { 352 fn current_punct2(&mut self, p: &tt::Punct) -> Option<((char, char), bool)> {
144 if p.spacing != tt::Spacing::Joint { 353 if p.spacing != tt::Spacing::Joint {
145 return None; 354 return None;
@@ -162,191 +371,117 @@ where
162 } 371 }
163} 372}
164 373
165struct TtTokenBuilder { 374fn convert_multi_char_punct<'b, I>(
166 tokens: Vec<TtToken>, 375 p: &tt::Punct,
167} 376 iter: &mut TokenPeek<'b, I>,
168 377) -> Option<(SyntaxKind, bool, &'static str, usize)>
169impl TtTokenBuilder { 378where
170 fn build(sub: &tt::Subtree) -> Vec<TtToken> { 379 I: Iterator<Item = &'b tt::TokenTree>,
171 let mut res = TtTokenBuilder { tokens: vec![] }; 380{
172 res.convert_subtree(sub); 381 if let Some((m, is_joint_to_next)) = iter.current_punct3(p) {
173 res.tokens 382 if let Some((kind, text)) = match m {
174 } 383 ('<', '<', '=') => Some((SHLEQ, "<<=")),
175 384 ('>', '>', '=') => Some((SHREQ, ">>=")),
176 fn convert_subtree(&mut self, sub: &tt::Subtree) { 385 ('.', '.', '.') => Some((DOTDOTDOT, "...")),
177 self.push_delim(sub.delimiter, false); 386 ('.', '.', '=') => Some((DOTDOTEQ, "..=")),
178 let mut peek = TokenPeek::new(sub.token_trees.iter()); 387 _ => None,
179 while let Some(tt) = peek.iter.next() { 388 } {
180 self.convert_tt(tt, &mut peek); 389 return Some((kind, is_joint_to_next, text, 3));
181 } 390 }
182 self.push_delim(sub.delimiter, true)
183 } 391 }
184 392
185 fn convert_tt<'b, I>(&mut self, tt: &tt::TokenTree, iter: &mut TokenPeek<'b, I>) 393 if let Some((m, is_joint_to_next)) = iter.current_punct2(p) {
186 where 394 if let Some((kind, text)) = match m {
187 I: Iterator<Item = &'b tt::TokenTree>, 395 ('<', '<') => Some((SHL, "<<")),
188 { 396 ('>', '>') => Some((SHR, ">>")),
189 match tt { 397
190 tt::TokenTree::Leaf(token) => self.convert_token(token, iter), 398 ('|', '|') => Some((PIPEPIPE, "||")),
191 tt::TokenTree::Subtree(sub) => self.convert_subtree(sub), 399 ('&', '&') => Some((AMPAMP, "&&")),
400 ('%', '=') => Some((PERCENTEQ, "%=")),
401 ('*', '=') => Some((STAREQ, "*=")),
402 ('/', '=') => Some((SLASHEQ, "/=")),
403 ('^', '=') => Some((CARETEQ, "^=")),
404
405 ('&', '=') => Some((AMPEQ, "&=")),
406 ('|', '=') => Some((PIPEEQ, "|=")),
407 ('-', '=') => Some((MINUSEQ, "-=")),
408 ('+', '=') => Some((PLUSEQ, "+=")),
409 ('>', '=') => Some((GTEQ, ">=")),
410 ('<', '=') => Some((LTEQ, "<=")),
411
412 ('-', '>') => Some((THIN_ARROW, "->")),
413 ('!', '=') => Some((NEQ, "!=")),
414 ('=', '>') => Some((FAT_ARROW, "=>")),
415 ('=', '=') => Some((EQEQ, "==")),
416 ('.', '.') => Some((DOTDOT, "..")),
417 (':', ':') => Some((COLONCOLON, "::")),
418
419 _ => None,
420 } {
421 return Some((kind, is_joint_to_next, text, 2));
192 } 422 }
193 } 423 }
194 424
195 fn convert_token<'b, I>(&mut self, token: &tt::Leaf, iter: &mut TokenPeek<'b, I>) 425 None
196 where 426}
197 I: Iterator<Item = &'b tt::TokenTree>,
198 {
199 let tok = match token {
200 tt::Leaf::Literal(l) => TtToken {
201 kind: classify_literal(&l.text).unwrap().kind,
202 is_joint_to_next: false,
203 text: l.text.clone(),
204 n_tokens: 1,
205 },
206 tt::Leaf::Punct(p) => {
207 if let Some((kind, is_joint_to_next, text, size)) =
208 Self::convert_multi_char_punct(p, iter)
209 {
210 for _ in 0..size - 1 {
211 iter.next();
212 }
213
214 TtToken { kind, is_joint_to_next, text: text.into(), n_tokens: size }
215 } else {
216 let kind = match p.char {
217 // lexer may produce combpund tokens for these ones
218 '.' => DOT,
219 ':' => COLON,
220 '=' => EQ,
221 '!' => EXCL,
222 '-' => MINUS,
223 c => SyntaxKind::from_char(c).unwrap(),
224 };
225 let text = {
226 let mut buf = [0u8; 4];
227 let s: &str = p.char.encode_utf8(&mut buf);
228 SmolStr::new(s)
229 };
230 TtToken {
231 kind,
232 is_joint_to_next: p.spacing == tt::Spacing::Joint,
233 text,
234 n_tokens: 1,
235 }
236 }
237 }
238 tt::Leaf::Ident(ident) => {
239 let kind = SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT);
240 TtToken { kind, is_joint_to_next: false, text: ident.text.clone(), n_tokens: 1 }
241 }
242 };
243 self.tokens.push(tok)
244 }
245
246 fn convert_multi_char_punct<'b, I>(
247 p: &tt::Punct,
248 iter: &mut TokenPeek<'b, I>,
249 ) -> Option<(SyntaxKind, bool, &'static str, usize)>
250 where
251 I: Iterator<Item = &'b tt::TokenTree>,
252 {
253 if let Some((m, is_joint_to_next)) = iter.current_punct3(p) {
254 if let Some((kind, text)) = match m {
255 ('<', '<', '=') => Some((SHLEQ, "<<=")),
256 ('>', '>', '=') => Some((SHREQ, ">>=")),
257 ('.', '.', '.') => Some((DOTDOTDOT, "...")),
258 ('.', '.', '=') => Some((DOTDOTEQ, "..=")),
259 _ => None,
260 } {
261 return Some((kind, is_joint_to_next, text, 3));
262 }
263 }
264 427
265 if let Some((m, is_joint_to_next)) = iter.current_punct2(p) { 428struct SubTreeWalker<'a> {
266 if let Some((kind, text)) = match m { 429 pos: usize,
267 ('<', '<') => Some((SHL, "<<")), 430 stack: Vec<(&'a tt::Subtree, Option<usize>)>,
268 ('>', '>') => Some((SHR, ">>")), 431 idx: WalkIndex,
269 432 last_steps: Vec<usize>,
270 ('|', '|') => Some((PIPEPIPE, "||")), 433 subtree: &'a tt::Subtree,
271 ('&', '&') => Some((AMPAMP, "&&")), 434}
272 ('%', '=') => Some((PERCENTEQ, "%=")),
273 ('*', '=') => Some((STAREQ, "*=")),
274 ('/', '=') => Some((SLASHEQ, "/=")),
275 ('^', '=') => Some((CARETEQ, "^=")),
276
277 ('&', '=') => Some((AMPEQ, "&=")),
278 ('|', '=') => Some((PIPEEQ, "|=")),
279 ('-', '=') => Some((MINUSEQ, "-=")),
280 ('+', '=') => Some((PLUSEQ, "+=")),
281 ('>', '=') => Some((GTEQ, ">=")),
282 ('<', '=') => Some((LTEQ, "<=")),
283
284 ('-', '>') => Some((THIN_ARROW, "->")),
285 ('!', '=') => Some((NEQ, "!=")),
286 ('=', '>') => Some((FAT_ARROW, "=>")),
287 ('=', '=') => Some((EQEQ, "==")),
288 ('.', '.') => Some((DOTDOT, "..")),
289 (':', ':') => Some((COLONCOLON, "::")),
290
291 _ => None,
292 } {
293 return Some((kind, is_joint_to_next, text, 2));
294 }
295 }
296 435
297 None 436fn convert_delim(d: tt::Delimiter, closing: bool) -> Option<TtToken> {
298 } 437 let (kinds, texts) = match d {
438 tt::Delimiter::Parenthesis => ([L_PAREN, R_PAREN], "()"),
439 tt::Delimiter::Brace => ([L_CURLY, R_CURLY], "{}"),
440 tt::Delimiter::Bracket => ([L_BRACK, R_BRACK], "[]"),
441 tt::Delimiter::None => return None,
442 };
443
444 let idx = closing as usize;
445 let kind = kinds[idx];
446 let text = &texts[idx..texts.len() - (1 - idx)];
447 Some(TtToken { kind, is_joint_to_next: false, text: SmolStr::new(text), n_tokens: 1 })
448}
299 449
300 fn push_delim(&mut self, d: tt::Delimiter, closing: bool) { 450fn convert_literal(l: &tt::Literal) -> TtToken {
301 let (kinds, texts) = match d { 451 TtToken {
302 tt::Delimiter::Parenthesis => ([L_PAREN, R_PAREN], "()"), 452 kind: classify_literal(&l.text).unwrap().kind,
303 tt::Delimiter::Brace => ([L_CURLY, R_CURLY], "{}"), 453 is_joint_to_next: false,
304 tt::Delimiter::Bracket => ([L_BRACK, R_BRACK], "[]"), 454 text: l.text.clone(),
305 tt::Delimiter::None => return, 455 n_tokens: 1,
306 };
307 let idx = closing as usize;
308 let kind = kinds[idx];
309 let text = &texts[idx..texts.len() - (1 - idx)];
310 let tok = TtToken { kind, is_joint_to_next: false, text: SmolStr::new(text), n_tokens: 1 };
311 self.tokens.push(tok)
312 }
313
314 fn skip_sibling_leaf(leaf: &tt::Leaf, iter: &mut std::slice::Iter<tt::TokenTree>) {
315 if let tt::Leaf::Punct(p) = leaf {
316 let mut peek = TokenPeek::new(iter);
317 if let Some((_, _, _, size)) = TtTokenBuilder::convert_multi_char_punct(p, &mut peek) {
318 for _ in 0..size - 1 {
319 peek.next();
320 }
321 }
322 }
323 } 456 }
457}
324 458
325 fn count_tt_tokens( 459fn convert_ident(ident: &tt::Ident) -> TtToken {
326 tt: &tt::TokenTree, 460 let kind = SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT);
327 iter: Option<&mut std::slice::Iter<tt::TokenTree>>, 461 TtToken { kind, is_joint_to_next: false, text: ident.text.clone(), n_tokens: 1 }
328 ) -> usize { 462}
329 match tt {
330 tt::TokenTree::Subtree(sub_tree) => {
331 let mut iter = sub_tree.token_trees.iter();
332 let mut count = match sub_tree.delimiter {
333 tt::Delimiter::None => 0,
334 _ => 2,
335 };
336
337 while let Some(tt) = iter.next() {
338 count += Self::count_tt_tokens(&tt, Some(&mut iter));
339 }
340 count
341 }
342
343 tt::TokenTree::Leaf(leaf) => {
344 iter.map(|iter| {
345 Self::skip_sibling_leaf(leaf, iter);
346 });
347 463
348 1 464fn convert_punct(p: &tt::Punct, parent: &tt::Subtree, next: usize) -> TtToken {
349 } 465 let iter = parent.token_trees[next..].iter();
350 } 466 let mut peek = TokenPeek::new(iter);
467
468 if let Some((kind, is_joint_to_next, text, size)) = convert_multi_char_punct(p, &mut peek) {
469 TtToken { kind, is_joint_to_next, text: text.into(), n_tokens: size }
470 } else {
471 let kind = match p.char {
472 // lexer may produce combpund tokens for these ones
473 '.' => DOT,
474 ':' => COLON,
475 '=' => EQ,
476 '!' => EXCL,
477 '-' => MINUS,
478 c => SyntaxKind::from_char(c).unwrap(),
479 };
480 let text = {
481 let mut buf = [0u8; 4];
482 let s: &str = p.char.encode_utf8(&mut buf);
483 SmolStr::new(s)
484 };
485 TtToken { kind, is_joint_to_next: p.spacing == tt::Spacing::Joint, text, n_tokens: 1 }
351 } 486 }
352} 487}
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index b0fb91a63..19c17bd55 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -105,16 +105,16 @@ fn convert_tt(
105 Some(res) 105 Some(res)
106} 106}
107 107
108struct TtTreeSink<'a> { 108struct TtTreeSink<'a, Q: Querier> {
109 buf: String, 109 buf: String,
110 src_querier: Querier<'a>, 110 src_querier: &'a Q,
111 text_pos: TextUnit, 111 text_pos: TextUnit,
112 token_pos: usize, 112 token_pos: usize,
113 inner: SyntaxTreeBuilder, 113 inner: SyntaxTreeBuilder,
114} 114}
115 115
116impl<'a> TtTreeSink<'a> { 116impl<'a, Q: Querier> TtTreeSink<'a, Q> {
117 fn new(src_querier: Querier<'a>) -> TtTreeSink { 117 fn new(src_querier: &'a Q) -> Self {
118 TtTreeSink { 118 TtTreeSink {
119 buf: String::new(), 119 buf: String::new(),
120 src_querier, 120 src_querier,
@@ -125,10 +125,10 @@ impl<'a> TtTreeSink<'a> {
125 } 125 }
126} 126}
127 127
128impl<'a> TreeSink for TtTreeSink<'a> { 128impl<'a, Q: Querier> TreeSink for TtTreeSink<'a, Q> {
129 fn token(&mut self, kind: SyntaxKind, n_tokens: u8) { 129 fn token(&mut self, kind: SyntaxKind, n_tokens: u8) {
130 for _ in 0..n_tokens { 130 for _ in 0..n_tokens {
131 self.buf += self.src_querier.token(self.token_pos).1; 131 self.buf += &self.src_querier.token(self.token_pos).1;
132 self.token_pos += 1; 132 self.token_pos += 1;
133 } 133 }
134 self.text_pos += TextUnit::of_str(&self.buf); 134 self.text_pos += TextUnit::of_str(&self.buf);