diff options
author | Aleksey Kladov <[email protected]> | 2020-08-12 17:26:51 +0100 |
---|---|---|
committer | Aleksey Kladov <[email protected]> | 2020-08-12 17:30:53 +0100 |
commit | a1c187eef3ba08076aedb5154929f7eda8d1b424 (patch) | |
tree | 9d898eb9600b0c36a74e4f95238f679c683fa566 /crates/ra_syntax/src/parsing/text_token_source.rs | |
parent | 3d6889cba72a9d02199f7adaa2ecc69bc30af834 (diff) |
Rename ra_syntax -> syntax
Diffstat (limited to 'crates/ra_syntax/src/parsing/text_token_source.rs')
-rw-r--r-- | crates/ra_syntax/src/parsing/text_token_source.rs | 84 |
1 files changed, 0 insertions, 84 deletions
diff --git a/crates/ra_syntax/src/parsing/text_token_source.rs b/crates/ra_syntax/src/parsing/text_token_source.rs deleted file mode 100644 index df866dc2b..000000000 --- a/crates/ra_syntax/src/parsing/text_token_source.rs +++ /dev/null | |||
@@ -1,84 +0,0 @@ | |||
1 | //! See `TextTokenSource` docs. | ||
2 | |||
3 | use parser::TokenSource; | ||
4 | |||
5 | use crate::{parsing::lexer::Token, SyntaxKind::EOF, TextRange, TextSize}; | ||
6 | |||
7 | /// Implementation of `parser::TokenSource` that takes tokens from source code text. | ||
8 | pub(crate) struct TextTokenSource<'t> { | ||
9 | text: &'t str, | ||
10 | /// token and its start position (non-whitespace/comment tokens) | ||
11 | /// ```non-rust | ||
12 | /// struct Foo; | ||
13 | /// ^------^--^- | ||
14 | /// | | \________ | ||
15 | /// | \____ \ | ||
16 | /// | \ | | ||
17 | /// (struct, 0) (Foo, 7) (;, 10) | ||
18 | /// ``` | ||
19 | /// `[(struct, 0), (Foo, 7), (;, 10)]` | ||
20 | token_offset_pairs: Vec<(Token, TextSize)>, | ||
21 | |||
22 | /// Current token and position | ||
23 | curr: (parser::Token, usize), | ||
24 | } | ||
25 | |||
26 | impl<'t> TokenSource for TextTokenSource<'t> { | ||
27 | fn current(&self) -> parser::Token { | ||
28 | self.curr.0 | ||
29 | } | ||
30 | |||
31 | fn lookahead_nth(&self, n: usize) -> parser::Token { | ||
32 | mk_token(self.curr.1 + n, &self.token_offset_pairs) | ||
33 | } | ||
34 | |||
35 | fn bump(&mut self) { | ||
36 | if self.curr.0.kind == EOF { | ||
37 | return; | ||
38 | } | ||
39 | |||
40 | let pos = self.curr.1 + 1; | ||
41 | self.curr = (mk_token(pos, &self.token_offset_pairs), pos); | ||
42 | } | ||
43 | |||
44 | fn is_keyword(&self, kw: &str) -> bool { | ||
45 | self.token_offset_pairs | ||
46 | .get(self.curr.1) | ||
47 | .map(|(token, offset)| &self.text[TextRange::at(*offset, token.len)] == kw) | ||
48 | .unwrap_or(false) | ||
49 | } | ||
50 | } | ||
51 | |||
52 | fn mk_token(pos: usize, token_offset_pairs: &[(Token, TextSize)]) -> parser::Token { | ||
53 | let (kind, is_jointed_to_next) = match token_offset_pairs.get(pos) { | ||
54 | Some((token, offset)) => ( | ||
55 | token.kind, | ||
56 | token_offset_pairs | ||
57 | .get(pos + 1) | ||
58 | .map(|(_, next_offset)| offset + token.len == *next_offset) | ||
59 | .unwrap_or(false), | ||
60 | ), | ||
61 | None => (EOF, false), | ||
62 | }; | ||
63 | parser::Token { kind, is_jointed_to_next } | ||
64 | } | ||
65 | |||
66 | impl<'t> TextTokenSource<'t> { | ||
67 | /// Generate input from tokens(expect comment and whitespace). | ||
68 | pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> TextTokenSource<'t> { | ||
69 | let token_offset_pairs: Vec<_> = raw_tokens | ||
70 | .iter() | ||
71 | .filter_map({ | ||
72 | let mut len = 0.into(); | ||
73 | move |token| { | ||
74 | let pair = if token.kind.is_trivia() { None } else { Some((*token, len)) }; | ||
75 | len += token.len; | ||
76 | pair | ||
77 | } | ||
78 | }) | ||
79 | .collect(); | ||
80 | |||
81 | let first = mk_token(0, &token_offset_pairs); | ||
82 | TextTokenSource { text, token_offset_pairs, curr: (first, 0) } | ||
83 | } | ||
84 | } | ||