aboutsummaryrefslogtreecommitdiff
path: root/crates/ra_syntax/src
diff options
context:
space:
mode:
authorAleksey Kladov <[email protected]>2018-10-08 13:44:00 +0100
committerAleksey Kladov <[email protected]>2018-10-08 13:44:00 +0100
commit86a7ac2d31e97c42a9da8d8fd539b5a0de0fa795 (patch)
tree17d16374b28e3221cfb793340b32aabe2f4fd1dd /crates/ra_syntax/src
parent3c12d38a32f5c87e87d9dbced774453216086bab (diff)
Simplify event processing
Diffstat (limited to 'crates/ra_syntax/src')
-rw-r--r--crates/ra_syntax/src/lib.rs3
-rw-r--r--crates/ra_syntax/src/parser_impl/event.rs148
-rw-r--r--crates/ra_syntax/src/parser_impl/mod.rs25
-rw-r--r--crates/ra_syntax/src/reparsing.rs3
-rw-r--r--crates/ra_syntax/src/yellow/builder.rs32
5 files changed, 117 insertions, 94 deletions
diff --git a/crates/ra_syntax/src/lib.rs b/crates/ra_syntax/src/lib.rs
index 738664afd..703469629 100644
--- a/crates/ra_syntax/src/lib.rs
+++ b/crates/ra_syntax/src/lib.rs
@@ -74,7 +74,8 @@ impl File {
74 } 74 }
75 pub fn parse(text: &str) -> File { 75 pub fn parse(text: &str) -> File {
76 let tokens = tokenize(&text); 76 let tokens = tokenize(&text);
77 let (green, errors) = parser_impl::parse_with::<yellow::GreenBuilder>( 77 let (green, errors) = parser_impl::parse_with(
78 yellow::GreenBuilder::new(),
78 text, &tokens, grammar::root, 79 text, &tokens, grammar::root,
79 ); 80 );
80 File::new(green, errors) 81 File::new(green, errors)
diff --git a/crates/ra_syntax/src/parser_impl/event.rs b/crates/ra_syntax/src/parser_impl/event.rs
index 9fd56b996..48f37c57c 100644
--- a/crates/ra_syntax/src/parser_impl/event.rs
+++ b/crates/ra_syntax/src/parser_impl/event.rs
@@ -9,6 +9,7 @@
9//! this stream to a real tree. 9//! this stream to a real tree.
10use std::mem; 10use std::mem;
11use { 11use {
12 TextUnit, TextRange, SmolStr,
12 lexer::Token, 13 lexer::Token,
13 parser_impl::Sink, 14 parser_impl::Sink,
14 SyntaxKind::{self, TOMBSTONE}, 15 SyntaxKind::{self, TOMBSTONE},
@@ -78,77 +79,104 @@ pub(crate) enum Event {
78 }, 79 },
79} 80}
80 81
82pub(super) struct EventProcessor<'a, S: Sink> {
83 sink: S,
84 text_pos: TextUnit,
85 text: &'a str,
86 token_pos: usize,
87 tokens: &'a [Token],
88 events: &'a mut [Event],
89}
81 90
82pub(super) fn process<'a, S: Sink<'a>>(builder: &mut S, tokens: &[Token], mut events: Vec<Event>) { 91impl<'a, S: Sink> EventProcessor<'a, S> {
83 fn tombstone() -> Event { 92 pub(super) fn new(sink: S, text: &'a str, tokens: &'a[Token], events: &'a mut [Event]) -> EventProcessor<'a, S> {
84 Event::Start { kind: TOMBSTONE, forward_parent: None } 93 EventProcessor {
94 sink,
95 text_pos: 0.into(),
96 text,
97 token_pos: 0,
98 tokens,
99 events
100 }
85 } 101 }
86 let eat_ws = |idx: &mut usize, builder: &mut S| { 102
87 while let Some(token) = tokens.get(*idx) { 103 pub(super) fn process(mut self) -> S {
88 if !token.kind.is_trivia() { 104 fn tombstone() -> Event {
89 break; 105 Event::Start { kind: TOMBSTONE, forward_parent: None }
90 }
91 builder.leaf(token.kind, token.len);
92 *idx += 1
93 } 106 }
94 }; 107 let mut depth = 0;
108 let mut forward_parents = Vec::new();
95 109
96 let events: &mut [Event] = &mut events; 110 for i in 0..self.events.len() {
97 let mut depth = 0; 111 match mem::replace(&mut self.events[i], tombstone()) {
98 let mut forward_parents = Vec::new(); 112 Event::Start {
99 let mut next_tok_idx = 0; 113 kind: TOMBSTONE, ..
100 for i in 0..events.len() { 114 } => (),
101 match mem::replace(&mut events[i], tombstone()) {
102 Event::Start {
103 kind: TOMBSTONE, ..
104 } => (),
105 115
106 Event::Start { kind, forward_parent } => { 116 Event::Start { kind, forward_parent } => {
107 forward_parents.push(kind); 117 forward_parents.push(kind);
108 let mut idx = i; 118 let mut idx = i;
109 let mut fp = forward_parent; 119 let mut fp = forward_parent;
110 while let Some(fwd) = fp { 120 while let Some(fwd) = fp {
111 idx += fwd as usize; 121 idx += fwd as usize;
112 fp = match mem::replace(&mut events[idx], tombstone()) { 122 fp = match mem::replace(&mut self.events[idx], tombstone()) {
113 Event::Start { 123 Event::Start {
114 kind, 124 kind,
115 forward_parent, 125 forward_parent,
116 } => { 126 } => {
117 forward_parents.push(kind); 127 forward_parents.push(kind);
118 forward_parent 128 forward_parent
119 }, 129 },
120 _ => unreachable!(), 130 _ => unreachable!(),
121 }; 131 };
132 }
133 for kind in forward_parents.drain(..).rev() {
134 if depth > 0 {
135 self.eat_ws();
136 }
137 depth += 1;
138 self.sink.start_internal(kind);
139 }
122 } 140 }
123 for kind in forward_parents.drain(..).rev() { 141 Event::Finish => {
124 if depth > 0 { 142 depth -= 1;
125 eat_ws(&mut next_tok_idx, builder); 143 if depth == 0 {
144 self.eat_ws();
126 } 145 }
127 depth += 1; 146
128 builder.start_internal(kind); 147 self.sink.finish_internal();
129 } 148 }
130 } 149 Event::Token {
131 Event::Finish => { 150 kind,
132 depth -= 1; 151 mut n_raw_tokens,
133 if depth == 0 { 152 } => {
134 eat_ws(&mut next_tok_idx, builder); 153 self.eat_ws();
154 let mut len = 0.into();
155 for _ in 0..n_raw_tokens {
156 len += self.tokens[self.token_pos].len;
157 }
158 self.leaf(kind, len, n_raw_tokens as usize);
135 } 159 }
136 160 Event::Error { msg } => self.sink.error(msg, self.text_pos),
137 builder.finish_internal();
138 } 161 }
139 Event::Token { 162 }
140 kind, 163 self.sink
141 mut n_raw_tokens, 164 }
142 } => { 165
143 eat_ws(&mut next_tok_idx, builder); 166 fn eat_ws(&mut self) {
144 let mut len = 0.into(); 167 while let Some(&token) = self.tokens.get(self.token_pos) {
145 for _ in 0..n_raw_tokens { 168 if !token.kind.is_trivia() {
146 len += tokens[next_tok_idx].len; 169 break;
147 next_tok_idx += 1;
148 }
149 builder.leaf(kind, len);
150 } 170 }
151 Event::Error { msg } => builder.error(msg), 171 self.leaf(token.kind, token.len, 1);
152 } 172 }
153 } 173 }
174
175 fn leaf(&mut self, kind: SyntaxKind, len: TextUnit, n_tokens: usize) {
176 let range = TextRange::offset_len(self.text_pos, len);
177 let text: SmolStr = self.text[range].into();
178 self.text_pos += len;
179 self.token_pos += n_tokens;
180 self.sink.leaf(kind, text);
181 }
154} 182}
diff --git a/crates/ra_syntax/src/parser_impl/mod.rs b/crates/ra_syntax/src/parser_impl/mod.rs
index b343b404f..8d74cef0e 100644
--- a/crates/ra_syntax/src/parser_impl/mod.rs
+++ b/crates/ra_syntax/src/parser_impl/mod.rs
@@ -4,45 +4,44 @@ mod input;
4use std::cell::Cell; 4use std::cell::Cell;
5 5
6use { 6use {
7 TextUnit, SmolStr,
7 lexer::Token, 8 lexer::Token,
8 parser_api::Parser, 9 parser_api::Parser,
9 parser_impl::{ 10 parser_impl::{
10 event::{process, Event}, 11 event::{EventProcessor, Event},
11 input::{InputPosition, ParserInput}, 12 input::{InputPosition, ParserInput},
12 }, 13 },
13 TextUnit,
14}; 14};
15 15
16use SyntaxKind::{self, EOF, TOMBSTONE}; 16use SyntaxKind::{self, EOF, TOMBSTONE};
17 17
18pub(crate) trait Sink<'a> { 18pub(crate) trait Sink {
19 type Tree; 19 type Tree;
20 20
21 fn new(text: &'a str) -> Self; 21 fn leaf(&mut self, kind: SyntaxKind, text: SmolStr);
22
23 fn leaf(&mut self, kind: SyntaxKind, len: TextUnit);
24 fn start_internal(&mut self, kind: SyntaxKind); 22 fn start_internal(&mut self, kind: SyntaxKind);
25 fn finish_internal(&mut self); 23 fn finish_internal(&mut self);
26 fn error(&mut self, err: String); 24 fn error(&mut self, message: String, offset: TextUnit);
27 fn finish(self) -> Self::Tree; 25 fn finish(self) -> Self::Tree;
28} 26}
29 27
30/// Parse a sequence of tokens into the representative node tree 28/// Parse a sequence of tokens into the representative node tree
31pub(crate) fn parse_with<'a, S: Sink<'a>>( 29pub(crate) fn parse_with<S: Sink>(
32 text: &'a str, 30 sink: S,
31 text: &str,
33 tokens: &[Token], 32 tokens: &[Token],
34 parser: fn(&mut Parser), 33 parser: fn(&mut Parser),
35) -> S::Tree { 34) -> S::Tree {
36 let events = { 35 let mut events = {
37 let input = input::ParserInput::new(text, tokens); 36 let input = input::ParserInput::new(text, tokens);
38 let parser_impl = ParserImpl::new(&input); 37 let parser_impl = ParserImpl::new(&input);
39 let mut parser_api = Parser(parser_impl); 38 let mut parser_api = Parser(parser_impl);
40 parser(&mut parser_api); 39 parser(&mut parser_api);
41 parser_api.0.into_events() 40 parser_api.0.into_events()
42 }; 41 };
43 let mut sink = S::new(text); 42 EventProcessor::new(sink, text, tokens, &mut events)
44 process(&mut sink, tokens, events); 43 .process()
45 sink.finish() 44 .finish()
46} 45}
47 46
48/// Implementation details of `Parser`, extracted 47/// Implementation details of `Parser`, extracted
diff --git a/crates/ra_syntax/src/reparsing.rs b/crates/ra_syntax/src/reparsing.rs
index dcafd2c40..d8b6a6a10 100644
--- a/crates/ra_syntax/src/reparsing.rs
+++ b/crates/ra_syntax/src/reparsing.rs
@@ -84,7 +84,8 @@ fn reparse_block<'node>(
84 return None; 84 return None;
85 } 85 }
86 let (green, new_errors) = 86 let (green, new_errors) =
87 parser_impl::parse_with::<yellow::GreenBuilder>( 87 parser_impl::parse_with(
88 yellow::GreenBuilder::new(),
88 &text, &tokens, reparser, 89 &text, &tokens, reparser,
89 ); 90 );
90 Some((node, green, new_errors)) 91 Some((node, green, new_errors))
diff --git a/crates/ra_syntax/src/yellow/builder.rs b/crates/ra_syntax/src/yellow/builder.rs
index 35dbaec05..c307b2bd0 100644
--- a/crates/ra_syntax/src/yellow/builder.rs
+++ b/crates/ra_syntax/src/yellow/builder.rs
@@ -1,33 +1,29 @@
1use rowan::GreenNodeBuilder; 1use rowan::GreenNodeBuilder;
2use { 2use {
3 TextUnit, SmolStr,
3 parser_impl::Sink, 4 parser_impl::Sink,
4 yellow::{GreenNode, SyntaxError, RaTypes}, 5 yellow::{GreenNode, SyntaxError, RaTypes},
5 SyntaxKind, TextRange, TextUnit, 6 SyntaxKind,
6}; 7};
7 8
8pub(crate) struct GreenBuilder<'a> { 9pub(crate) struct GreenBuilder {
9 text: &'a str,
10 pos: TextUnit,
11 errors: Vec<SyntaxError>, 10 errors: Vec<SyntaxError>,
12 inner: GreenNodeBuilder<RaTypes>, 11 inner: GreenNodeBuilder<RaTypes>,
13} 12}
14 13
15impl<'a> Sink<'a> for GreenBuilder<'a> { 14impl GreenBuilder {
16 type Tree = (GreenNode, Vec<SyntaxError>); 15 pub(crate) fn new() -> GreenBuilder {
17
18 fn new(text: &'a str) -> Self {
19 GreenBuilder { 16 GreenBuilder {
20 text,
21 pos: 0.into(),
22 errors: Vec::new(), 17 errors: Vec::new(),
23 inner: GreenNodeBuilder::new(), 18 inner: GreenNodeBuilder::new(),
24 } 19 }
25 } 20 }
21}
22
23impl Sink for GreenBuilder {
24 type Tree = (GreenNode, Vec<SyntaxError>);
26 25
27 fn leaf(&mut self, kind: SyntaxKind, len: TextUnit) { 26 fn leaf(&mut self, kind: SyntaxKind, text: SmolStr) {
28 let range = TextRange::offset_len(self.pos, len);
29 self.pos += len;
30 let text = self.text[range].into();
31 self.inner.leaf(kind, text); 27 self.inner.leaf(kind, text);
32 } 28 }
33 29
@@ -39,11 +35,9 @@ impl<'a> Sink<'a> for GreenBuilder<'a> {
39 self.inner.finish_internal(); 35 self.inner.finish_internal();
40 } 36 }
41 37
42 fn error(&mut self, message: String) { 38 fn error(&mut self, message: String, offset: TextUnit) {
43 self.errors.push(SyntaxError { 39 let error = SyntaxError { msg: message, offset };
44 msg: message, 40 self.errors.push(error)
45 offset: self.pos,
46 })
47 } 41 }
48 42
49 fn finish(self) -> (GreenNode, Vec<SyntaxError>) { 43 fn finish(self) -> (GreenNode, Vec<SyntaxError>) {