aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--src/parser/event.rs140
-rw-r--r--src/parser/event_parser/mod.rs74
-rw-r--r--src/parser/grammar/attributes.rs (renamed from src/parser/event_parser/grammar/attributes.rs)0
-rw-r--r--src/parser/grammar/expressions.rs (renamed from src/parser/event_parser/grammar/expressions.rs)0
-rw-r--r--src/parser/grammar/items/consts.rs (renamed from src/parser/event_parser/grammar/items/consts.rs)0
-rw-r--r--src/parser/grammar/items/mod.rs (renamed from src/parser/event_parser/grammar/items/mod.rs)0
-rw-r--r--src/parser/grammar/items/structs.rs (renamed from src/parser/event_parser/grammar/items/structs.rs)0
-rw-r--r--src/parser/grammar/items/traits.rs (renamed from src/parser/event_parser/grammar/items/traits.rs)0
-rw-r--r--src/parser/grammar/items/use_item.rs (renamed from src/parser/event_parser/grammar/items/use_item.rs)0
-rw-r--r--src/parser/grammar/mod.rs (renamed from src/parser/event_parser/grammar/mod.rs)0
-rw-r--r--src/parser/grammar/paths.rs (renamed from src/parser/event_parser/grammar/paths.rs)0
-rw-r--r--src/parser/grammar/type_params.rs (renamed from src/parser/event_parser/grammar/type_params.rs)0
-rw-r--r--src/parser/grammar/types.rs (renamed from src/parser/event_parser/grammar/types.rs)0
-rw-r--r--src/parser/mod.rs89
-rw-r--r--src/parser/parser.rs (renamed from src/parser/event_parser/parser.rs)2
15 files changed, 153 insertions, 152 deletions
diff --git a/src/parser/event.rs b/src/parser/event.rs
new file mode 100644
index 000000000..fd6bdc086
--- /dev/null
+++ b/src/parser/event.rs
@@ -0,0 +1,140 @@
1use {File, FileBuilder, Sink, SyntaxKind, Token};
2use syntax_kinds::TOMBSTONE;
3use super::is_insignificant;
4
5/// `Parser` produces a flat list of `Event`s.
6/// They are converted to a tree-structure in
7/// a separate pass, via `TreeBuilder`.
8#[derive(Debug)]
9pub(crate) enum Event {
10 /// This event signifies the start of the node.
11 /// It should be either abandoned (in which case the
12 /// `kind` is `TOMBSTONE`, and the event is ignored),
13 /// or completed via a `Finish` event.
14 ///
15 /// All tokens between a `Start` and a `Finish` would
16 /// become the children of the respective node.
17 ///
18 /// For left-recursive syntactic constructs, the parser produces
19 /// a child node before it sees a parent. `forward_parent`
20 /// exists to allow to tweak parent-child relationships.
21 ///
22 /// Consider this path
23 ///
24 /// foo::bar
25 ///
26 /// The events for it would look like this:
27 ///
28 ///
29 /// START(PATH) IDENT('foo') FINISH START(PATH) COLONCOLON IDENT('bar') FINISH
30 /// | /\
31 /// | |
32 /// +------forward-parent------+
33 ///
34 /// And the tree would look like this
35 ///
36 /// +--PATH---------+
37 /// | | |
38 /// | | |
39 /// | '::' 'bar'
40 /// |
41 /// PATH
42 /// |
43 /// 'foo'
44 ///
45 /// See also `CompleteMarker::precede`.
46 Start {
47 kind: SyntaxKind,
48 forward_parent: Option<u32>,
49 },
50
51 /// Complete the previous `Start` event
52 Finish,
53
54 /// Produce a single leaf-element.
55 /// `n_raw_tokens` is used to glue complex contextual tokens.
56 /// For example, lexer tokenizes `>>` as `>`, `>`, and
57 /// `n_raw_tokens = 2` is used to produced a single `>>`.
58 Token {
59 kind: SyntaxKind,
60 n_raw_tokens: u8,
61 },
62
63 Error {
64 message: String,
65 },
66}
67
68pub(super) fn to_file(text: String, tokens: &[Token], events: Vec<Event>) -> File {
69 let mut builder = FileBuilder::new(text);
70 let mut idx = 0;
71
72 let mut holes = Vec::new();
73 let mut forward_parents = Vec::new();
74
75 for (i, event) in events.iter().enumerate() {
76 if holes.last() == Some(&i) {
77 holes.pop();
78 continue;
79 }
80
81 match event {
82 &Event::Start {
83 kind: TOMBSTONE, ..
84 } => (),
85
86 &Event::Start { .. } => {
87 forward_parents.clear();
88 let mut idx = i;
89 loop {
90 let (kind, fwd) = match events[idx] {
91 Event::Start {
92 kind,
93 forward_parent,
94 } => (kind, forward_parent),
95 _ => unreachable!(),
96 };
97 forward_parents.push((idx, kind));
98 if let Some(fwd) = fwd {
99 idx += fwd as usize;
100 } else {
101 break;
102 }
103 }
104 for &(idx, kind) in forward_parents.iter().into_iter().rev() {
105 builder.start_internal(kind);
106 holes.push(idx);
107 }
108 holes.pop();
109 }
110 &Event::Finish => {
111 while idx < tokens.len() {
112 let token = tokens[idx];
113 if is_insignificant(token.kind) {
114 idx += 1;
115 builder.leaf(token.kind, token.len);
116 } else {
117 break;
118 }
119 }
120 builder.finish_internal()
121 }
122 &Event::Token {
123 kind: _,
124 mut n_raw_tokens,
125 } => loop {
126 let token = tokens[idx];
127 if !is_insignificant(token.kind) {
128 n_raw_tokens -= 1;
129 }
130 idx += 1;
131 builder.leaf(token.kind, token.len);
132 if n_raw_tokens == 0 {
133 break;
134 }
135 },
136 &Event::Error { ref message } => builder.error().message(message.clone()).emit(),
137 }
138 }
139 builder.finish()
140}
diff --git a/src/parser/event_parser/mod.rs b/src/parser/event_parser/mod.rs
deleted file mode 100644
index 7823c476c..000000000
--- a/src/parser/event_parser/mod.rs
+++ /dev/null
@@ -1,74 +0,0 @@
1use {SyntaxKind, Token};
2
3#[macro_use]
4mod parser;
5mod grammar;
6
7/// `Parser` produces a flat list of `Event`s.
8/// They are converted to a tree-structure in
9/// a separate pass, via `TreeBuilder`.
10#[derive(Debug)]
11pub(crate) enum Event {
12 /// This event signifies the start of the node.
13 /// It should be either abandoned (in which case the
14 /// `kind` is `TOMBSTONE`, and the event is ignored),
15 /// or completed via a `Finish` event.
16 ///
17 /// All tokens between a `Start` and a `Finish` would
18 /// become the children of the respective node.
19 ///
20 /// For left-recursive syntactic constructs, the parser produces
21 /// a child node before it sees a parent. `forward_parent`
22 /// exists to allow to tweak parent-child relationships.
23 ///
24 /// Consider this path
25 ///
26 /// foo::bar
27 ///
28 /// The events for it would look like this:
29 ///
30 ///
31 /// START(PATH) IDENT('foo') FINISH START(PATH) COLONCOLON IDENT('bar') FINISH
32 /// | /\
33 /// | |
34 /// +------forward-parent------+
35 ///
36 /// And the tree would look like this
37 ///
38 /// +--PATH---------+
39 /// | | |
40 /// | | |
41 /// | '::' 'bar'
42 /// |
43 /// PATH
44 /// |
45 /// 'foo'
46 ///
47 /// See also `CompleteMarker::precede`.
48 Start {
49 kind: SyntaxKind,
50 forward_parent: Option<u32>,
51 },
52
53 /// Complete the previous `Start` event
54 Finish,
55
56 /// Produce a single leaf-element.
57 /// `n_raw_tokens` is used to glue complex contextual tokens.
58 /// For example, lexer tokenizes `>>` as `>`, `>`, and
59 /// `n_raw_tokens = 2` is used to produced a single `>>`.
60 Token {
61 kind: SyntaxKind,
62 n_raw_tokens: u8,
63 },
64
65 Error {
66 message: String,
67 },
68}
69
70pub(crate) fn parse<'t>(text: &'t str, raw_tokens: &'t [Token]) -> Vec<Event> {
71 let mut parser = parser::Parser::new(text, raw_tokens);
72 grammar::file(&mut parser);
73 parser.into_events()
74}
diff --git a/src/parser/event_parser/grammar/attributes.rs b/src/parser/grammar/attributes.rs
index 8bf04afce..8bf04afce 100644
--- a/src/parser/event_parser/grammar/attributes.rs
+++ b/src/parser/grammar/attributes.rs
diff --git a/src/parser/event_parser/grammar/expressions.rs b/src/parser/grammar/expressions.rs
index 8caaf3553..8caaf3553 100644
--- a/src/parser/event_parser/grammar/expressions.rs
+++ b/src/parser/grammar/expressions.rs
diff --git a/src/parser/event_parser/grammar/items/consts.rs b/src/parser/grammar/items/consts.rs
index c9881d681..c9881d681 100644
--- a/src/parser/event_parser/grammar/items/consts.rs
+++ b/src/parser/grammar/items/consts.rs
diff --git a/src/parser/event_parser/grammar/items/mod.rs b/src/parser/grammar/items/mod.rs
index 3612802e1..3612802e1 100644
--- a/src/parser/event_parser/grammar/items/mod.rs
+++ b/src/parser/grammar/items/mod.rs
diff --git a/src/parser/event_parser/grammar/items/structs.rs b/src/parser/grammar/items/structs.rs
index 69d95c698..69d95c698 100644
--- a/src/parser/event_parser/grammar/items/structs.rs
+++ b/src/parser/grammar/items/structs.rs
diff --git a/src/parser/event_parser/grammar/items/traits.rs b/src/parser/grammar/items/traits.rs
index 3bef9639f..3bef9639f 100644
--- a/src/parser/event_parser/grammar/items/traits.rs
+++ b/src/parser/grammar/items/traits.rs
diff --git a/src/parser/event_parser/grammar/items/use_item.rs b/src/parser/grammar/items/use_item.rs
index 38e7b3f8a..38e7b3f8a 100644
--- a/src/parser/event_parser/grammar/items/use_item.rs
+++ b/src/parser/grammar/items/use_item.rs
diff --git a/src/parser/event_parser/grammar/mod.rs b/src/parser/grammar/mod.rs
index afce308d0..afce308d0 100644
--- a/src/parser/event_parser/grammar/mod.rs
+++ b/src/parser/grammar/mod.rs
diff --git a/src/parser/event_parser/grammar/paths.rs b/src/parser/grammar/paths.rs
index 6efac2610..6efac2610 100644
--- a/src/parser/event_parser/grammar/paths.rs
+++ b/src/parser/grammar/paths.rs
diff --git a/src/parser/event_parser/grammar/type_params.rs b/src/parser/grammar/type_params.rs
index 12c9a5362..12c9a5362 100644
--- a/src/parser/event_parser/grammar/type_params.rs
+++ b/src/parser/grammar/type_params.rs
diff --git a/src/parser/event_parser/grammar/types.rs b/src/parser/grammar/types.rs
index 1a3d44a0a..1a3d44a0a 100644
--- a/src/parser/event_parser/grammar/types.rs
+++ b/src/parser/grammar/types.rs
diff --git a/src/parser/mod.rs b/src/parser/mod.rs
index c5525ff9c..22ccb4921 100644
--- a/src/parser/mod.rs
+++ b/src/parser/mod.rs
@@ -1,88 +1,23 @@
1use {File, FileBuilder, Sink, SyntaxKind, Token}; 1use {File, SyntaxKind, Token};
2 2
3use syntax_kinds::*; 3use syntax_kinds::*;
4 4
5mod event_parser; 5#[macro_use]
6use self::event_parser::Event; 6mod parser;
7mod event;
8mod grammar;
9use self::event::Event;
7 10
8/// Parse a sequence of tokens into the representative node tree 11/// Parse a sequence of tokens into the representative node tree
9pub fn parse(text: String, tokens: &[Token]) -> File { 12pub fn parse(text: String, tokens: &[Token]) -> File {
10 let events = event_parser::parse(&text, tokens); 13 let events = parse_into_events(&text, tokens);
11 from_events_to_file(text, tokens, events) 14 event::to_file(text, tokens, events)
12} 15}
13 16
14fn from_events_to_file(text: String, tokens: &[Token], events: Vec<Event>) -> File { 17pub(crate) fn parse_into_events<'t>(text: &'t str, raw_tokens: &'t [Token]) -> Vec<Event> {
15 let mut builder = FileBuilder::new(text); 18 let mut parser = parser::Parser::new(text, raw_tokens);
16 let mut idx = 0; 19 grammar::file(&mut parser);
17 20 parser.into_events()
18 let mut holes = Vec::new();
19 let mut forward_parents = Vec::new();
20
21 for (i, event) in events.iter().enumerate() {
22 if holes.last() == Some(&i) {
23 holes.pop();
24 continue;
25 }
26
27 match event {
28 &Event::Start {
29 kind: TOMBSTONE, ..
30 } => (),
31
32 &Event::Start { .. } => {
33 forward_parents.clear();
34 let mut idx = i;
35 loop {
36 let (kind, fwd) = match events[idx] {
37 Event::Start {
38 kind,
39 forward_parent,
40 } => (kind, forward_parent),
41 _ => unreachable!(),
42 };
43 forward_parents.push((idx, kind));
44 if let Some(fwd) = fwd {
45 idx += fwd as usize;
46 } else {
47 break;
48 }
49 }
50 for &(idx, kind) in forward_parents.iter().into_iter().rev() {
51 builder.start_internal(kind);
52 holes.push(idx);
53 }
54 holes.pop();
55 }
56 &Event::Finish => {
57 while idx < tokens.len() {
58 let token = tokens[idx];
59 if is_insignificant(token.kind) {
60 idx += 1;
61 builder.leaf(token.kind, token.len);
62 } else {
63 break;
64 }
65 }
66 builder.finish_internal()
67 }
68 &Event::Token {
69 kind: _,
70 mut n_raw_tokens,
71 } => loop {
72 let token = tokens[idx];
73 if !is_insignificant(token.kind) {
74 n_raw_tokens -= 1;
75 }
76 idx += 1;
77 builder.leaf(token.kind, token.len);
78 if n_raw_tokens == 0 {
79 break;
80 }
81 },
82 &Event::Error { ref message } => builder.error().message(message.clone()).emit(),
83 }
84 }
85 builder.finish()
86} 21}
87 22
88fn is_insignificant(kind: SyntaxKind) -> bool { 23fn is_insignificant(kind: SyntaxKind) -> bool {
diff --git a/src/parser/event_parser/parser.rs b/src/parser/parser.rs
index 6cf6ac9b9..3f4c8a07d 100644
--- a/src/parser/event_parser/parser.rs
+++ b/src/parser/parser.rs
@@ -1,6 +1,6 @@
1use {SyntaxKind, TextUnit, Token}; 1use {SyntaxKind, TextUnit, Token};
2use super::Event; 2use super::Event;
3use super::super::is_insignificant; 3use super::is_insignificant;
4use SyntaxKind::{EOF, TOMBSTONE}; 4use SyntaxKind::{EOF, TOMBSTONE};
5 5
6pub(crate) struct Marker { 6pub(crate) struct Marker {