diff options
author | Aleksey Kladov <[email protected]> | 2019-02-23 13:07:29 +0000 |
---|---|---|
committer | Aleksey Kladov <[email protected]> | 2019-02-23 13:07:29 +0000 |
commit | 0c1cb981820c55127c3c09d93868814a1df98246 (patch) | |
tree | edb870567bf099c5fa5edc5b89bcab410a34e5e0 /crates/ra_syntax/src/parsing/text_token_source.rs | |
parent | e5fb33a94618d45051192d99e859bdd44c3daa36 (diff) |
rename
Diffstat (limited to 'crates/ra_syntax/src/parsing/text_token_source.rs')
-rw-r--r-- | crates/ra_syntax/src/parsing/text_token_source.rs | 67 |
1 files changed, 67 insertions, 0 deletions
diff --git a/crates/ra_syntax/src/parsing/text_token_source.rs b/crates/ra_syntax/src/parsing/text_token_source.rs new file mode 100644 index 000000000..a6277f66f --- /dev/null +++ b/crates/ra_syntax/src/parsing/text_token_source.rs | |||
@@ -0,0 +1,67 @@ | |||
1 | use ra_parser::TokenSource; | ||
2 | |||
3 | use crate::{ | ||
4 | SyntaxKind, SyntaxKind::EOF, TextRange, TextUnit, | ||
5 | parsing::lexer::Token, | ||
6 | }; | ||
7 | |||
8 | pub(crate) struct TextTokenSource<'t> { | ||
9 | text: &'t str, | ||
10 | /// start position of each token(expect whitespace and comment) | ||
11 | /// ```non-rust | ||
12 | /// struct Foo; | ||
13 | /// ^------^--- | ||
14 | /// | | ^- | ||
15 | /// 0 7 10 | ||
16 | /// ``` | ||
17 | /// (token, start_offset): `[(struct, 0), (Foo, 7), (;, 10)]` | ||
18 | start_offsets: Vec<TextUnit>, | ||
19 | /// non-whitespace/comment tokens | ||
20 | /// ```non-rust | ||
21 | /// struct Foo {} | ||
22 | /// ^^^^^^ ^^^ ^^ | ||
23 | /// ``` | ||
24 | /// tokens: `[struct, Foo, {, }]` | ||
25 | tokens: Vec<Token>, | ||
26 | } | ||
27 | |||
28 | impl<'t> TokenSource for TextTokenSource<'t> { | ||
29 | fn token_kind(&self, pos: usize) -> SyntaxKind { | ||
30 | if !(pos < self.tokens.len()) { | ||
31 | return EOF; | ||
32 | } | ||
33 | self.tokens[pos].kind | ||
34 | } | ||
35 | fn is_token_joint_to_next(&self, pos: usize) -> bool { | ||
36 | if !(pos + 1 < self.tokens.len()) { | ||
37 | return true; | ||
38 | } | ||
39 | self.start_offsets[pos] + self.tokens[pos].len == self.start_offsets[pos + 1] | ||
40 | } | ||
41 | fn is_keyword(&self, pos: usize, kw: &str) -> bool { | ||
42 | if !(pos < self.tokens.len()) { | ||
43 | return false; | ||
44 | } | ||
45 | let range = TextRange::offset_len(self.start_offsets[pos], self.tokens[pos].len); | ||
46 | |||
47 | self.text[range] == *kw | ||
48 | } | ||
49 | } | ||
50 | |||
51 | impl<'t> TextTokenSource<'t> { | ||
52 | /// Generate input from tokens(expect comment and whitespace). | ||
53 | pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> TextTokenSource<'t> { | ||
54 | let mut tokens = Vec::new(); | ||
55 | let mut start_offsets = Vec::new(); | ||
56 | let mut len = 0.into(); | ||
57 | for &token in raw_tokens.iter() { | ||
58 | if !token.kind.is_trivia() { | ||
59 | tokens.push(token); | ||
60 | start_offsets.push(len); | ||
61 | } | ||
62 | len += token.len; | ||
63 | } | ||
64 | |||
65 | TextTokenSource { text, start_offsets, tokens } | ||
66 | } | ||
67 | } | ||