aboutsummaryrefslogtreecommitdiff
path: root/crates/ra_syntax/src/parsing
diff options
context:
space:
mode:
Diffstat (limited to 'crates/ra_syntax/src/parsing')
-rw-r--r--crates/ra_syntax/src/parsing/builder.rs127
-rw-r--r--crates/ra_syntax/src/parsing/event.rs193
-rw-r--r--crates/ra_syntax/src/parsing/grammar.rs26
-rw-r--r--crates/ra_syntax/src/parsing/reparsing.rs11
-rw-r--r--crates/ra_syntax/src/parsing/token_set.rs2
5 files changed, 166 insertions, 193 deletions
diff --git a/crates/ra_syntax/src/parsing/builder.rs b/crates/ra_syntax/src/parsing/builder.rs
index ee0e2cce7..1041c6a7b 100644
--- a/crates/ra_syntax/src/parsing/builder.rs
+++ b/crates/ra_syntax/src/parsing/builder.rs
@@ -1,40 +1,63 @@
1use crate::{ 1use crate::{
2 SmolStr, SyntaxKind, SyntaxError, SyntaxErrorKind, TextUnit, 2 SmolStr, SyntaxError, SyntaxErrorKind, TextUnit, TextRange,
3 parsing::{TreeSink, ParseError}, 3 SyntaxKind::{self, *},
4 parsing::{TreeSink, ParseError, Token},
4 syntax_node::{GreenNode, RaTypes}, 5 syntax_node::{GreenNode, RaTypes},
5}; 6};
6 7
7use rowan::GreenNodeBuilder; 8use rowan::GreenNodeBuilder;
8 9
9pub(crate) struct GreenBuilder { 10pub(crate) struct TreeBuilder<'a> {
11 text: &'a str,
12 tokens: &'a [Token],
10 text_pos: TextUnit, 13 text_pos: TextUnit,
14 token_pos: usize,
11 errors: Vec<SyntaxError>, 15 errors: Vec<SyntaxError>,
12 inner: GreenNodeBuilder<RaTypes>, 16 inner: GreenNodeBuilder<RaTypes>,
13} 17}
14 18
15impl Default for GreenBuilder { 19impl<'a> TreeSink for TreeBuilder<'a> {
16 fn default() -> GreenBuilder {
17 GreenBuilder {
18 text_pos: TextUnit::default(),
19 errors: Vec::new(),
20 inner: GreenNodeBuilder::new(),
21 }
22 }
23}
24
25impl TreeSink for GreenBuilder {
26 type Tree = (GreenNode, Vec<SyntaxError>); 20 type Tree = (GreenNode, Vec<SyntaxError>);
27 21
28 fn leaf(&mut self, kind: SyntaxKind, text: SmolStr) { 22 fn leaf(&mut self, kind: SyntaxKind, n_tokens: u8) {
29 self.text_pos += TextUnit::of_str(text.as_str()); 23 self.eat_trivias();
30 self.inner.leaf(kind, text); 24 let n_tokens = n_tokens as usize;
25 let len = self.tokens[self.token_pos..self.token_pos + n_tokens]
26 .iter()
27 .map(|it| it.len)
28 .sum::<TextUnit>();
29 self.do_leaf(kind, len, n_tokens);
31 } 30 }
32 31
33 fn start_branch(&mut self, kind: SyntaxKind) { 32 fn start_branch(&mut self, kind: SyntaxKind, root: bool) {
34 self.inner.start_internal(kind) 33 if root {
34 self.inner.start_internal(kind);
35 return;
36 }
37 let n_trivias =
38 self.tokens[self.token_pos..].iter().take_while(|it| it.kind.is_trivia()).count();
39 let leading_trivias = &self.tokens[self.token_pos..self.token_pos + n_trivias];
40 let mut trivia_end =
41 self.text_pos + leading_trivias.iter().map(|it| it.len).sum::<TextUnit>();
42
43 let n_attached_trivias = {
44 let leading_trivias = leading_trivias.iter().rev().map(|it| {
45 let next_end = trivia_end - it.len;
46 let range = TextRange::from_to(next_end, trivia_end);
47 trivia_end = next_end;
48 (it.kind, &self.text[range])
49 });
50 n_attached_trivias(kind, leading_trivias)
51 };
52 self.eat_n_trivias(n_trivias - n_attached_trivias);
53 self.inner.start_internal(kind);
54 self.eat_n_trivias(n_attached_trivias);
35 } 55 }
36 56
37 fn finish_branch(&mut self) { 57 fn finish_branch(&mut self, root: bool) {
58 if root {
59 self.eat_trivias()
60 }
38 self.inner.finish_internal(); 61 self.inner.finish_internal();
39 } 62 }
40 63
@@ -47,3 +70,67 @@ impl TreeSink for GreenBuilder {
47 (self.inner.finish(), self.errors) 70 (self.inner.finish(), self.errors)
48 } 71 }
49} 72}
73
74impl<'a> TreeBuilder<'a> {
75 pub(super) fn new(text: &'a str, tokens: &'a [Token]) -> TreeBuilder<'a> {
76 TreeBuilder {
77 text,
78 tokens,
79 text_pos: 0.into(),
80 token_pos: 0,
81 errors: Vec::new(),
82 inner: GreenNodeBuilder::new(),
83 }
84 }
85 fn eat_trivias(&mut self) {
86 while let Some(&token) = self.tokens.get(self.token_pos) {
87 if !token.kind.is_trivia() {
88 break;
89 }
90 self.do_leaf(token.kind, token.len, 1);
91 }
92 }
93
94 fn eat_n_trivias(&mut self, n: usize) {
95 for _ in 0..n {
96 let token = self.tokens[self.token_pos];
97 assert!(token.kind.is_trivia());
98 self.do_leaf(token.kind, token.len, 1);
99 }
100 }
101
102 fn do_leaf(&mut self, kind: SyntaxKind, len: TextUnit, n_tokens: usize) {
103 let range = TextRange::offset_len(self.text_pos, len);
104 let text: SmolStr = self.text[range].into();
105 self.text_pos += len;
106 self.token_pos += n_tokens;
107 self.inner.leaf(kind, text);
108 }
109}
110
111fn n_attached_trivias<'a>(
112 kind: SyntaxKind,
113 trivias: impl Iterator<Item = (SyntaxKind, &'a str)>,
114) -> usize {
115 match kind {
116 CONST_DEF | TYPE_DEF | STRUCT_DEF | ENUM_DEF | ENUM_VARIANT | FN_DEF | TRAIT_DEF
117 | MODULE | NAMED_FIELD_DEF => {
118 let mut res = 0;
119 for (i, (kind, text)) in trivias.enumerate() {
120 match kind {
121 WHITESPACE => {
122 if text.contains("\n\n") {
123 break;
124 }
125 }
126 COMMENT => {
127 res = i + 1;
128 }
129 _ => (),
130 }
131 }
132 res
133 }
134 _ => 0,
135 }
136}
diff --git a/crates/ra_syntax/src/parsing/event.rs b/crates/ra_syntax/src/parsing/event.rs
index f6f020eab..d6cbdffe0 100644
--- a/crates/ra_syntax/src/parsing/event.rs
+++ b/crates/ra_syntax/src/parsing/event.rs
@@ -10,13 +10,8 @@
10use std::mem; 10use std::mem;
11 11
12use crate::{ 12use crate::{
13 SmolStr,
14 SyntaxKind::{self, *}, 13 SyntaxKind::{self, *},
15 TextRange, TextUnit, 14 parsing::{ParseError, TreeSink},
16 parsing::{
17 ParseError, TreeSink,
18 lexer::Token,
19 },
20}; 15};
21 16
22/// `Parser` produces a flat list of `Event`s. 17/// `Parser` produces a flat list of `Event`s.
@@ -88,160 +83,46 @@ impl Event {
88 } 83 }
89} 84}
90 85
91pub(super) struct EventProcessor<'a, S: TreeSink> { 86/// Generate the syntax tree with the control of events.
92 sink: S, 87pub(super) fn process(sink: &mut impl TreeSink, mut events: Vec<Event>) {
93 text_pos: TextUnit, 88 let mut forward_parents = Vec::new();
94 text: &'a str, 89
95 token_pos: usize, 90 for i in 0..events.len() {
96 tokens: &'a [Token], 91 match mem::replace(&mut events[i], Event::tombstone()) {
97 events: &'a mut [Event], 92 Event::Start { kind: TOMBSTONE, .. } => (),
98} 93
99 94 Event::Start { kind, forward_parent } => {
100impl<'a, S: TreeSink> EventProcessor<'a, S> { 95 // For events[A, B, C], B is A's forward_parent, C is B's forward_parent,
101 pub(super) fn new( 96 // in the normal control flow, the parent-child relation: `A -> B -> C`,
102 sink: S, 97 // while with the magic forward_parent, it writes: `C <- B <- A`.
103 text: &'a str, 98
104 tokens: &'a [Token], 99 // append `A` into parents.
105 events: &'a mut [Event], 100 forward_parents.push(kind);
106 ) -> EventProcessor<'a, S> { 101 let mut idx = i;
107 EventProcessor { sink, text_pos: 0.into(), text, token_pos: 0, tokens, events } 102 let mut fp = forward_parent;
108 } 103 while let Some(fwd) = fp {
109 104 idx += fwd as usize;
110 /// Generate the syntax tree with the control of events. 105 // append `A`'s forward_parent `B`
111 pub(crate) fn process(mut self) -> S { 106 fp = match mem::replace(&mut events[idx], Event::tombstone()) {
112 let mut forward_parents = Vec::new(); 107 Event::Start { kind, forward_parent } => {
113 108 forward_parents.push(kind);
114 for i in 0..self.events.len() { 109 forward_parent
115 match mem::replace(&mut self.events[i], Event::tombstone()) { 110 }
116 Event::Start { kind: TOMBSTONE, .. } => (), 111 _ => unreachable!(),
117 112 };
118 Event::Start { kind, forward_parent } => { 113 // append `B`'s forward_parent `C` in the next stage.
119 // For events[A, B, C], B is A's forward_parent, C is B's forward_parent,
120 // in the normal control flow, the parent-child relation: `A -> B -> C`,
121 // while with the magic forward_parent, it writes: `C <- B <- A`.
122
123 // append `A` into parents.
124 forward_parents.push(kind);
125 let mut idx = i;
126 let mut fp = forward_parent;
127 while let Some(fwd) = fp {
128 idx += fwd as usize;
129 // append `A`'s forward_parent `B`
130 fp = match mem::replace(&mut self.events[idx], Event::tombstone()) {
131 Event::Start { kind, forward_parent } => {
132 forward_parents.push(kind);
133 forward_parent
134 }
135 _ => unreachable!(),
136 };
137 // append `B`'s forward_parent `C` in the next stage.
138 }
139
140 for kind in forward_parents.drain(..).rev() {
141 self.start(kind);
142 }
143 }
144 Event::Finish => {
145 let is_last = i == self.events.len() - 1;
146 self.finish(is_last);
147 }
148 Event::Token { kind, n_raw_tokens } => {
149 self.eat_trivias();
150 let n_raw_tokens = n_raw_tokens as usize;
151 let len = self.tokens[self.token_pos..self.token_pos + n_raw_tokens]
152 .iter()
153 .map(|it| it.len)
154 .sum::<TextUnit>();
155 self.leaf(kind, len, n_raw_tokens);
156 } 114 }
157 Event::Error { msg } => self.sink.error(msg),
158 }
159 }
160 self.sink
161 }
162
163 /// Add the node into syntax tree but discard the comments/whitespaces.
164 fn start(&mut self, kind: SyntaxKind) {
165 if kind == SOURCE_FILE {
166 self.sink.start_branch(kind);
167 return;
168 }
169 let n_trivias =
170 self.tokens[self.token_pos..].iter().take_while(|it| it.kind.is_trivia()).count();
171 let leading_trivias = &self.tokens[self.token_pos..self.token_pos + n_trivias];
172 let mut trivia_end =
173 self.text_pos + leading_trivias.iter().map(|it| it.len).sum::<TextUnit>();
174 115
175 let n_attached_trivias = { 116 for (j, kind) in forward_parents.drain(..).rev().enumerate() {
176 let leading_trivias = leading_trivias.iter().rev().map(|it| { 117 let is_root_node = i == 0 && j == 0;
177 let next_end = trivia_end - it.len; 118 sink.start_branch(kind, is_root_node);
178 let range = TextRange::from_to(next_end, trivia_end);
179 trivia_end = next_end;
180 (it.kind, &self.text[range])
181 });
182 n_attached_trivias(kind, leading_trivias)
183 };
184 self.eat_n_trivias(n_trivias - n_attached_trivias);
185 self.sink.start_branch(kind);
186 self.eat_n_trivias(n_attached_trivias);
187 }
188
189 fn finish(&mut self, is_last: bool) {
190 if is_last {
191 self.eat_trivias()
192 }
193 self.sink.finish_branch();
194 }
195
196 fn eat_trivias(&mut self) {
197 while let Some(&token) = self.tokens.get(self.token_pos) {
198 if !token.kind.is_trivia() {
199 break;
200 }
201 self.leaf(token.kind, token.len, 1);
202 }
203 }
204
205 fn eat_n_trivias(&mut self, n: usize) {
206 for _ in 0..n {
207 let token = self.tokens[self.token_pos];
208 assert!(token.kind.is_trivia());
209 self.leaf(token.kind, token.len, 1);
210 }
211 }
212
213 fn leaf(&mut self, kind: SyntaxKind, len: TextUnit, n_tokens: usize) {
214 let range = TextRange::offset_len(self.text_pos, len);
215 let text: SmolStr = self.text[range].into();
216 self.text_pos += len;
217 self.token_pos += n_tokens;
218 self.sink.leaf(kind, text);
219 }
220}
221
222fn n_attached_trivias<'a>(
223 kind: SyntaxKind,
224 trivias: impl Iterator<Item = (SyntaxKind, &'a str)>,
225) -> usize {
226 match kind {
227 CONST_DEF | TYPE_DEF | STRUCT_DEF | ENUM_DEF | ENUM_VARIANT | FN_DEF | TRAIT_DEF
228 | MODULE | NAMED_FIELD_DEF => {
229 let mut res = 0;
230 for (i, (kind, text)) in trivias.enumerate() {
231 match kind {
232 WHITESPACE => {
233 if text.contains("\n\n") {
234 break;
235 }
236 }
237 COMMENT => {
238 res = i + 1;
239 }
240 _ => (),
241 } 119 }
242 } 120 }
243 res 121 Event::Finish => sink.finish_branch(i == events.len() - 1),
122 Event::Token { kind, n_raw_tokens } => {
123 sink.leaf(kind, n_raw_tokens);
124 }
125 Event::Error { msg } => sink.error(msg),
244 } 126 }
245 _ => 0,
246 } 127 }
247} 128}
diff --git a/crates/ra_syntax/src/parsing/grammar.rs b/crates/ra_syntax/src/parsing/grammar.rs
index 7ca9c223c..800d5a4a2 100644
--- a/crates/ra_syntax/src/parsing/grammar.rs
+++ b/crates/ra_syntax/src/parsing/grammar.rs
@@ -37,7 +37,6 @@ mod type_params;
37mod types; 37mod types;
38 38
39use crate::{ 39use crate::{
40 SyntaxNode,
41 SyntaxKind::{self, *}, 40 SyntaxKind::{self, *},
42 parsing::{ 41 parsing::{
43 token_set::TokenSet, 42 token_set::TokenSet,
@@ -52,8 +51,12 @@ pub(super) fn root(p: &mut Parser) {
52 m.complete(p, SOURCE_FILE); 51 m.complete(p, SOURCE_FILE);
53} 52}
54 53
55pub(super) fn reparser(node: &SyntaxNode) -> Option<fn(&mut Parser)> { 54pub(super) fn reparser(
56 let res = match node.kind() { 55 node: SyntaxKind,
56 first_child: Option<SyntaxKind>,
57 parent: Option<SyntaxKind>,
58) -> Option<fn(&mut Parser)> {
59 let res = match node {
57 BLOCK => expressions::block, 60 BLOCK => expressions::block,
58 NAMED_FIELD_DEF_LIST => items::named_field_def_list, 61 NAMED_FIELD_DEF_LIST => items::named_field_def_list,
59 NAMED_FIELD_LIST => items::named_field_list, 62 NAMED_FIELD_LIST => items::named_field_list,
@@ -61,16 +64,13 @@ pub(super) fn reparser(node: &SyntaxNode) -> Option<fn(&mut Parser)> {
61 MATCH_ARM_LIST => items::match_arm_list, 64 MATCH_ARM_LIST => items::match_arm_list,
62 USE_TREE_LIST => items::use_tree_list, 65 USE_TREE_LIST => items::use_tree_list,
63 EXTERN_ITEM_LIST => items::extern_item_list, 66 EXTERN_ITEM_LIST => items::extern_item_list,
64 TOKEN_TREE if node.first_child().unwrap().kind() == L_CURLY => items::token_tree, 67 TOKEN_TREE if first_child? == L_CURLY => items::token_tree,
65 ITEM_LIST => { 68 ITEM_LIST => match parent? {
66 let parent = node.parent().unwrap(); 69 IMPL_BLOCK => items::impl_item_list,
67 match parent.kind() { 70 TRAIT_DEF => items::trait_item_list,
68 IMPL_BLOCK => items::impl_item_list, 71 MODULE => items::mod_item_list,
69 TRAIT_DEF => items::trait_item_list, 72 _ => return None,
70 MODULE => items::mod_item_list, 73 },
71 _ => return None,
72 }
73 }
74 _ => return None, 74 _ => return None,
75 }; 75 };
76 Some(res) 76 Some(res)
diff --git a/crates/ra_syntax/src/parsing/reparsing.rs b/crates/ra_syntax/src/parsing/reparsing.rs
index f2d218ab9..2c860b3df 100644
--- a/crates/ra_syntax/src/parsing/reparsing.rs
+++ b/crates/ra_syntax/src/parsing/reparsing.rs
@@ -5,7 +5,7 @@ use crate::{
5 syntax_error::SyntaxError, 5 syntax_error::SyntaxError,
6 parsing::{ 6 parsing::{
7 grammar, parse_with, 7 grammar, parse_with,
8 builder::GreenBuilder, 8 builder::TreeBuilder,
9 parser::Parser, 9 parser::Parser,
10 lexer::{tokenize, Token}, 10 lexer::{tokenize, Token},
11 } 11 }
@@ -61,7 +61,8 @@ fn reparse_block<'node>(
61 if !is_balanced(&tokens) { 61 if !is_balanced(&tokens) {
62 return None; 62 return None;
63 } 63 }
64 let (green, new_errors) = parse_with(GreenBuilder::default(), &text, &tokens, reparser); 64 let tree_sink = TreeBuilder::new(&text, &tokens);
65 let (green, new_errors) = parse_with(tree_sink, &text, &tokens, reparser);
65 Some((node, green, new_errors)) 66 Some((node, green, new_errors))
66} 67}
67 68
@@ -82,7 +83,11 @@ fn find_reparsable_node(
82 range: TextRange, 83 range: TextRange,
83) -> Option<(&SyntaxNode, fn(&mut Parser))> { 84) -> Option<(&SyntaxNode, fn(&mut Parser))> {
84 let node = algo::find_covering_node(node, range); 85 let node = algo::find_covering_node(node, range);
85 node.ancestors().find_map(|node| grammar::reparser(node).map(|r| (node, r))) 86 node.ancestors().find_map(|node| {
87 let first_child = node.first_child().map(|it| it.kind());
88 let parent = node.parent().map(|it| it.kind());
89 grammar::reparser(node.kind(), first_child, parent).map(|r| (node, r))
90 })
86} 91}
87 92
88fn is_balanced(tokens: &[Token]) -> bool { 93fn is_balanced(tokens: &[Token]) -> bool {
diff --git a/crates/ra_syntax/src/parsing/token_set.rs b/crates/ra_syntax/src/parsing/token_set.rs
index 5719fe5a2..24152a38a 100644
--- a/crates/ra_syntax/src/parsing/token_set.rs
+++ b/crates/ra_syntax/src/parsing/token_set.rs
@@ -34,7 +34,7 @@ macro_rules! token_set {
34#[test] 34#[test]
35fn token_set_works_for_tokens() { 35fn token_set_works_for_tokens() {
36 use crate::SyntaxKind::*; 36 use crate::SyntaxKind::*;
37 let ts = token_set! { EOF, SHEBANG }; 37 let ts = token_set![EOF, SHEBANG];
38 assert!(ts.contains(EOF)); 38 assert!(ts.contains(EOF));
39 assert!(ts.contains(SHEBANG)); 39 assert!(ts.contains(SHEBANG));
40 assert!(!ts.contains(PLUS)); 40 assert!(!ts.contains(PLUS));