1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
|
//! See `TextTokenSource` docs.
use parser::TokenSource;
use crate::{parsing::lexer::Token, SyntaxKind::EOF, TextRange, TextSize};
/// Implementation of `parser::TokenSource` that takes tokens from source code text.
pub(crate) struct TextTokenSource<'t> {
text: &'t str,
/// token and its start position (non-whitespace/comment tokens)
/// ```non-rust
/// struct Foo;
/// ^------^--^-
/// | | \________
/// | \____ \
/// | \ |
/// (struct, 0) (Foo, 7) (;, 10)
/// ```
/// `[(struct, 0), (Foo, 7), (;, 10)]`
token_offset_pairs: Vec<(Token, TextSize)>,
/// Current token and position
curr: (parser::Token, usize),
}
impl<'t> TokenSource for TextTokenSource<'t> {
fn current(&self) -> parser::Token {
self.curr.0
}
fn lookahead_nth(&self, n: usize) -> parser::Token {
mk_token(self.curr.1 + n, &self.token_offset_pairs)
}
fn bump(&mut self) {
if self.curr.0.kind == EOF {
return;
}
let pos = self.curr.1 + 1;
self.curr = (mk_token(pos, &self.token_offset_pairs), pos);
}
fn is_keyword(&self, kw: &str) -> bool {
self.token_offset_pairs
.get(self.curr.1)
.map(|(token, offset)| &self.text[TextRange::at(*offset, token.len)] == kw)
.unwrap_or(false)
}
}
fn mk_token(pos: usize, token_offset_pairs: &[(Token, TextSize)]) -> parser::Token {
let (kind, is_jointed_to_next) = match token_offset_pairs.get(pos) {
Some((token, offset)) => (
token.kind,
token_offset_pairs
.get(pos + 1)
.map(|(_, next_offset)| offset + token.len == *next_offset)
.unwrap_or(false),
),
None => (EOF, false),
};
parser::Token { kind, is_jointed_to_next }
}
impl<'t> TextTokenSource<'t> {
/// Generate input from tokens(expect comment and whitespace).
pub(crate) fn new(text: &'t str, raw_tokens: &'t [Token]) -> TextTokenSource<'t> {
let token_offset_pairs: Vec<_> = raw_tokens
.iter()
.filter_map({
let mut len = 0.into();
move |token| {
let pair = if token.kind.is_trivia() { None } else { Some((*token, len)) };
len += token.len;
pair
}
})
.collect();
let first = mk_token(0, &token_offset_pairs);
TextTokenSource { text, token_offset_pairs, curr: (first, 0) }
}
}
|
