aboutsummaryrefslogtreecommitdiff
path: root/crates/syntax/src/parsing/text_token_source.rs
diff options
context:
space:
mode:
Diffstat (limited to 'crates/syntax/src/parsing/text_token_source.rs')
-rw-r--r--crates/syntax/src/parsing/text_token_source.rs84
1 files changed, 84 insertions, 0 deletions
diff --git a/crates/syntax/src/parsing/text_token_source.rs b/crates/syntax/src/parsing/text_token_source.rs
new file mode 100644
index 000000000..df866dc2b
--- /dev/null
+++ b/crates/syntax/src/parsing/text_token_source.rs
@@ -0,0 +1,84 @@
1//! See `TextTokenSource` docs.
2
3use parser::TokenSource;
4
5use crate::{parsing::lexer::Token, SyntaxKind::EOF, TextRange, TextSize};
6
7/// Implementation of `parser::TokenSource` that takes tokens from source code text.
8pub(crate) struct TextTokenSource<'t> {
9 text: &'t str,
10 /// token and its start position (non-whitespace/comment tokens)
11 /// ```non-rust
12 /// struct Foo;
13 /// ^------^--^-
14 /// | | \________
15 /// | \____ \
16 /// | \ |
17 /// (struct, 0) (Foo, 7) (;, 10)
18 /// ```
19 /// `[(struct, 0), (Foo, 7), (;, 10)]`
20 token_offset_pairs: Vec<(Token, TextSize)>,
21
22 /// Current token and position
23 curr: (parser::Token, usize),
24}
25
26impl<'t> TokenSource for TextTokenSource<'t> {
27 fn current(&self) -> parser::Token {
28 self.curr.0
29 }
30
31 fn lookahead_nth(&self, n: usize) -> parser::Token {
32 mk_token(self.curr.1 + n, &self.token_offset_pairs)
33 }
34
35 fn bump(&mut self) {
36 if self.curr.0.kind == EOF {
37 return;
38 }
39
40 let pos = self.curr.1 + 1;
41 self.curr = (mk_token(pos, &self.token_offset_pairs), pos);
42 }
43
44 fn is_keyword(&self, kw: &str) -> bool {
45 self.token_offset_pairs
46 .get(self.curr.1)
47 .map(|(token, offset)| &self.text[TextRange::at(*offset, token.len)] == kw)
48 .unwrap_or(false)
49 }
50}
51
52fn mk_token(pos: usize, token_offset_pairs: &[(Token, TextSize)]) -> parser::Token {
53 let (kind, is_jointed_to_next) = match token_offset_pairs.get(pos) {
54 Some((token, offset)) => (
55 token.kind,
56 token_offset_pairs
57 .get(pos + 1)
58 .map(|(_, next_offset)| offset + token.len == *next_offset)
59 .unwrap_or(false),
60 ),
61 None => (EOF, false),
62 };
63 parser::Token { kind, is_jointed_to_next }
64}
65
66impl<'t> TextTokenSource<'t> {
67 /// Generate input from tokens(expect comment and whitespace).
68 pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> TextTokenSource<'t> {
69 let token_offset_pairs: Vec<_> = raw_tokens
70 .iter()
71 .filter_map({
72 let mut len = 0.into();
73 move |token| {
74 let pair = if token.kind.is_trivia() { None } else { Some((*token, len)) };
75 len += token.len;
76 pair
77 }
78 })
79 .collect();
80
81 let first = mk_token(0, &token_offset_pairs);
82 TextTokenSource { text, token_offset_pairs, curr: (first, 0) }
83 }
84}