diff options
author | bors[bot] <bors[bot]@users.noreply.github.com> | 2019-02-21 09:25:20 +0000 |
---|---|---|
committer | bors[bot] <bors[bot]@users.noreply.github.com> | 2019-02-21 09:25:20 +0000 |
commit | 18b0c509f77a8e06141fee6668532cced1ebf5d8 (patch) | |
tree | e6b1718e4557a33921bf6637e76bf74c0d3cfa96 /crates/ra_syntax/src/parsing/builder.rs | |
parent | c84561bb624280b84eb2fe6c6b2a6b9fe3f1dbf7 (diff) | |
parent | cd0d2866fc61e03aeca7bd7d1bc652e0443d2b3d (diff) |
Merge #865
865: Parser decoupling r=matklad a=matklad
More work on making parser independent from text
Co-authored-by: Aleksey Kladov <[email protected]>
Diffstat (limited to 'crates/ra_syntax/src/parsing/builder.rs')
-rw-r--r-- | crates/ra_syntax/src/parsing/builder.rs | 127 |
1 files changed, 107 insertions, 20 deletions
diff --git a/crates/ra_syntax/src/parsing/builder.rs b/crates/ra_syntax/src/parsing/builder.rs index ee0e2cce7..1041c6a7b 100644 --- a/crates/ra_syntax/src/parsing/builder.rs +++ b/crates/ra_syntax/src/parsing/builder.rs | |||
@@ -1,40 +1,63 @@ | |||
1 | use crate::{ | 1 | use crate::{ |
2 | SmolStr, SyntaxKind, SyntaxError, SyntaxErrorKind, TextUnit, | 2 | SmolStr, SyntaxError, SyntaxErrorKind, TextUnit, TextRange, |
3 | parsing::{TreeSink, ParseError}, | 3 | SyntaxKind::{self, *}, |
4 | parsing::{TreeSink, ParseError, Token}, | ||
4 | syntax_node::{GreenNode, RaTypes}, | 5 | syntax_node::{GreenNode, RaTypes}, |
5 | }; | 6 | }; |
6 | 7 | ||
7 | use rowan::GreenNodeBuilder; | 8 | use rowan::GreenNodeBuilder; |
8 | 9 | ||
9 | pub(crate) struct GreenBuilder { | 10 | pub(crate) struct TreeBuilder<'a> { |
11 | text: &'a str, | ||
12 | tokens: &'a [Token], | ||
10 | text_pos: TextUnit, | 13 | text_pos: TextUnit, |
14 | token_pos: usize, | ||
11 | errors: Vec<SyntaxError>, | 15 | errors: Vec<SyntaxError>, |
12 | inner: GreenNodeBuilder<RaTypes>, | 16 | inner: GreenNodeBuilder<RaTypes>, |
13 | } | 17 | } |
14 | 18 | ||
15 | impl Default for GreenBuilder { | 19 | impl<'a> TreeSink for TreeBuilder<'a> { |
16 | fn default() -> GreenBuilder { | ||
17 | GreenBuilder { | ||
18 | text_pos: TextUnit::default(), | ||
19 | errors: Vec::new(), | ||
20 | inner: GreenNodeBuilder::new(), | ||
21 | } | ||
22 | } | ||
23 | } | ||
24 | |||
25 | impl TreeSink for GreenBuilder { | ||
26 | type Tree = (GreenNode, Vec<SyntaxError>); | 20 | type Tree = (GreenNode, Vec<SyntaxError>); |
27 | 21 | ||
28 | fn leaf(&mut self, kind: SyntaxKind, text: SmolStr) { | 22 | fn leaf(&mut self, kind: SyntaxKind, n_tokens: u8) { |
29 | self.text_pos += TextUnit::of_str(text.as_str()); | 23 | self.eat_trivias(); |
30 | self.inner.leaf(kind, text); | 24 | let n_tokens = n_tokens as usize; |
25 | let len = self.tokens[self.token_pos..self.token_pos + n_tokens] | ||
26 | .iter() | ||
27 | .map(|it| it.len) | ||
28 | .sum::<TextUnit>(); | ||
29 | self.do_leaf(kind, len, n_tokens); | ||
31 | } | 30 | } |
32 | 31 | ||
33 | fn start_branch(&mut self, kind: SyntaxKind) { | 32 | fn start_branch(&mut self, kind: SyntaxKind, root: bool) { |
34 | self.inner.start_internal(kind) | 33 | if root { |
34 | self.inner.start_internal(kind); | ||
35 | return; | ||
36 | } | ||
37 | let n_trivias = | ||
38 | self.tokens[self.token_pos..].iter().take_while(|it| it.kind.is_trivia()).count(); | ||
39 | let leading_trivias = &self.tokens[self.token_pos..self.token_pos + n_trivias]; | ||
40 | let mut trivia_end = | ||
41 | self.text_pos + leading_trivias.iter().map(|it| it.len).sum::<TextUnit>(); | ||
42 | |||
43 | let n_attached_trivias = { | ||
44 | let leading_trivias = leading_trivias.iter().rev().map(|it| { | ||
45 | let next_end = trivia_end - it.len; | ||
46 | let range = TextRange::from_to(next_end, trivia_end); | ||
47 | trivia_end = next_end; | ||
48 | (it.kind, &self.text[range]) | ||
49 | }); | ||
50 | n_attached_trivias(kind, leading_trivias) | ||
51 | }; | ||
52 | self.eat_n_trivias(n_trivias - n_attached_trivias); | ||
53 | self.inner.start_internal(kind); | ||
54 | self.eat_n_trivias(n_attached_trivias); | ||
35 | } | 55 | } |
36 | 56 | ||
37 | fn finish_branch(&mut self) { | 57 | fn finish_branch(&mut self, root: bool) { |
58 | if root { | ||
59 | self.eat_trivias() | ||
60 | } | ||
38 | self.inner.finish_internal(); | 61 | self.inner.finish_internal(); |
39 | } | 62 | } |
40 | 63 | ||
@@ -47,3 +70,67 @@ impl TreeSink for GreenBuilder { | |||
47 | (self.inner.finish(), self.errors) | 70 | (self.inner.finish(), self.errors) |
48 | } | 71 | } |
49 | } | 72 | } |
73 | |||
74 | impl<'a> TreeBuilder<'a> { | ||
75 | pub(super) fn new(text: &'a str, tokens: &'a [Token]) -> TreeBuilder<'a> { | ||
76 | TreeBuilder { | ||
77 | text, | ||
78 | tokens, | ||
79 | text_pos: 0.into(), | ||
80 | token_pos: 0, | ||
81 | errors: Vec::new(), | ||
82 | inner: GreenNodeBuilder::new(), | ||
83 | } | ||
84 | } | ||
85 | fn eat_trivias(&mut self) { | ||
86 | while let Some(&token) = self.tokens.get(self.token_pos) { | ||
87 | if !token.kind.is_trivia() { | ||
88 | break; | ||
89 | } | ||
90 | self.do_leaf(token.kind, token.len, 1); | ||
91 | } | ||
92 | } | ||
93 | |||
94 | fn eat_n_trivias(&mut self, n: usize) { | ||
95 | for _ in 0..n { | ||
96 | let token = self.tokens[self.token_pos]; | ||
97 | assert!(token.kind.is_trivia()); | ||
98 | self.do_leaf(token.kind, token.len, 1); | ||
99 | } | ||
100 | } | ||
101 | |||
102 | fn do_leaf(&mut self, kind: SyntaxKind, len: TextUnit, n_tokens: usize) { | ||
103 | let range = TextRange::offset_len(self.text_pos, len); | ||
104 | let text: SmolStr = self.text[range].into(); | ||
105 | self.text_pos += len; | ||
106 | self.token_pos += n_tokens; | ||
107 | self.inner.leaf(kind, text); | ||
108 | } | ||
109 | } | ||
110 | |||
111 | fn n_attached_trivias<'a>( | ||
112 | kind: SyntaxKind, | ||
113 | trivias: impl Iterator<Item = (SyntaxKind, &'a str)>, | ||
114 | ) -> usize { | ||
115 | match kind { | ||
116 | CONST_DEF | TYPE_DEF | STRUCT_DEF | ENUM_DEF | ENUM_VARIANT | FN_DEF | TRAIT_DEF | ||
117 | | MODULE | NAMED_FIELD_DEF => { | ||
118 | let mut res = 0; | ||
119 | for (i, (kind, text)) in trivias.enumerate() { | ||
120 | match kind { | ||
121 | WHITESPACE => { | ||
122 | if text.contains("\n\n") { | ||
123 | break; | ||
124 | } | ||
125 | } | ||
126 | COMMENT => { | ||
127 | res = i + 1; | ||
128 | } | ||
129 | _ => (), | ||
130 | } | ||
131 | } | ||
132 | res | ||
133 | } | ||
134 | _ => 0, | ||
135 | } | ||
136 | } | ||