aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/parser/event.rs140
-rw-r--r--src/parser/event_parser/mod.rs74
-rw-r--r--src/parser/grammar/attributes.rs (renamed from src/parser/event_parser/grammar/attributes.rs)0
-rw-r--r--src/parser/grammar/expressions.rs (renamed from src/parser/event_parser/grammar/expressions.rs)0
-rw-r--r--src/parser/grammar/items/consts.rs (renamed from src/parser/event_parser/grammar/items/consts.rs)0
-rw-r--r--src/parser/grammar/items/mod.rs (renamed from src/parser/event_parser/grammar/items/mod.rs)18
-rw-r--r--src/parser/grammar/items/structs.rs (renamed from src/parser/event_parser/grammar/items/structs.rs)0
-rw-r--r--src/parser/grammar/items/traits.rs17
-rw-r--r--src/parser/grammar/items/use_item.rs (renamed from src/parser/event_parser/grammar/items/use_item.rs)0
-rw-r--r--src/parser/grammar/mod.rs (renamed from src/parser/event_parser/grammar/mod.rs)0
-rw-r--r--src/parser/grammar/paths.rs (renamed from src/parser/event_parser/grammar/paths.rs)0
-rw-r--r--src/parser/grammar/type_params.rs (renamed from src/parser/event_parser/grammar/type_params.rs)0
-rw-r--r--src/parser/grammar/types.rs (renamed from src/parser/event_parser/grammar/types.rs)0
-rw-r--r--src/parser/mod.rs91
-rw-r--r--src/parser/parser.rs (renamed from src/parser/event_parser/parser.rs)2
-rw-r--r--src/syntax_kinds.rs4
16 files changed, 192 insertions, 154 deletions
diff --git a/src/parser/event.rs b/src/parser/event.rs
new file mode 100644
index 000000000..fd6bdc086
--- /dev/null
+++ b/src/parser/event.rs
@@ -0,0 +1,140 @@
1use {File, FileBuilder, Sink, SyntaxKind, Token};
2use syntax_kinds::TOMBSTONE;
3use super::is_insignificant;
4
5/// `Parser` produces a flat list of `Event`s.
6/// They are converted to a tree-structure in
7/// a separate pass, via `TreeBuilder`.
8#[derive(Debug)]
9pub(crate) enum Event {
10 /// This event signifies the start of the node.
11 /// It should be either abandoned (in which case the
12 /// `kind` is `TOMBSTONE`, and the event is ignored),
13 /// or completed via a `Finish` event.
14 ///
15 /// All tokens between a `Start` and a `Finish` would
16 /// become the children of the respective node.
17 ///
18 /// For left-recursive syntactic constructs, the parser produces
19 /// a child node before it sees a parent. `forward_parent`
20 /// exists to allow to tweak parent-child relationships.
21 ///
22 /// Consider this path
23 ///
24 /// foo::bar
25 ///
26 /// The events for it would look like this:
27 ///
28 ///
29 /// START(PATH) IDENT('foo') FINISH START(PATH) COLONCOLON IDENT('bar') FINISH
30 /// | /\
31 /// | |
32 /// +------forward-parent------+
33 ///
34 /// And the tree would look like this
35 ///
36 /// +--PATH---------+
37 /// | | |
38 /// | | |
39 /// | '::' 'bar'
40 /// |
41 /// PATH
42 /// |
43 /// 'foo'
44 ///
45 /// See also `CompleteMarker::precede`.
46 Start {
47 kind: SyntaxKind,
48 forward_parent: Option<u32>,
49 },
50
51 /// Complete the previous `Start` event
52 Finish,
53
54 /// Produce a single leaf-element.
55 /// `n_raw_tokens` is used to glue complex contextual tokens.
56 /// For example, lexer tokenizes `>>` as `>`, `>`, and
57 /// `n_raw_tokens = 2` is used to produced a single `>>`.
58 Token {
59 kind: SyntaxKind,
60 n_raw_tokens: u8,
61 },
62
63 Error {
64 message: String,
65 },
66}
67
68pub(super) fn to_file(text: String, tokens: &[Token], events: Vec<Event>) -> File {
69 let mut builder = FileBuilder::new(text);
70 let mut idx = 0;
71
72 let mut holes = Vec::new();
73 let mut forward_parents = Vec::new();
74
75 for (i, event) in events.iter().enumerate() {
76 if holes.last() == Some(&i) {
77 holes.pop();
78 continue;
79 }
80
81 match event {
82 &Event::Start {
83 kind: TOMBSTONE, ..
84 } => (),
85
86 &Event::Start { .. } => {
87 forward_parents.clear();
88 let mut idx = i;
89 loop {
90 let (kind, fwd) = match events[idx] {
91 Event::Start {
92 kind,
93 forward_parent,
94 } => (kind, forward_parent),
95 _ => unreachable!(),
96 };
97 forward_parents.push((idx, kind));
98 if let Some(fwd) = fwd {
99 idx += fwd as usize;
100 } else {
101 break;
102 }
103 }
104 for &(idx, kind) in forward_parents.iter().into_iter().rev() {
105 builder.start_internal(kind);
106 holes.push(idx);
107 }
108 holes.pop();
109 }
110 &Event::Finish => {
111 while idx < tokens.len() {
112 let token = tokens[idx];
113 if is_insignificant(token.kind) {
114 idx += 1;
115 builder.leaf(token.kind, token.len);
116 } else {
117 break;
118 }
119 }
120 builder.finish_internal()
121 }
122 &Event::Token {
123 kind: _,
124 mut n_raw_tokens,
125 } => loop {
126 let token = tokens[idx];
127 if !is_insignificant(token.kind) {
128 n_raw_tokens -= 1;
129 }
130 idx += 1;
131 builder.leaf(token.kind, token.len);
132 if n_raw_tokens == 0 {
133 break;
134 }
135 },
136 &Event::Error { ref message } => builder.error().message(message.clone()).emit(),
137 }
138 }
139 builder.finish()
140}
diff --git a/src/parser/event_parser/mod.rs b/src/parser/event_parser/mod.rs
deleted file mode 100644
index 7823c476c..000000000
--- a/src/parser/event_parser/mod.rs
+++ /dev/null
@@ -1,74 +0,0 @@
1use {SyntaxKind, Token};
2
3#[macro_use]
4mod parser;
5mod grammar;
6
7/// `Parser` produces a flat list of `Event`s.
8/// They are converted to a tree-structure in
9/// a separate pass, via `TreeBuilder`.
10#[derive(Debug)]
11pub(crate) enum Event {
12 /// This event signifies the start of the node.
13 /// It should be either abandoned (in which case the
14 /// `kind` is `TOMBSTONE`, and the event is ignored),
15 /// or completed via a `Finish` event.
16 ///
17 /// All tokens between a `Start` and a `Finish` would
18 /// become the children of the respective node.
19 ///
20 /// For left-recursive syntactic constructs, the parser produces
21 /// a child node before it sees a parent. `forward_parent`
22 /// exists to allow to tweak parent-child relationships.
23 ///
24 /// Consider this path
25 ///
26 /// foo::bar
27 ///
28 /// The events for it would look like this:
29 ///
30 ///
31 /// START(PATH) IDENT('foo') FINISH START(PATH) COLONCOLON IDENT('bar') FINISH
32 /// | /\
33 /// | |
34 /// +------forward-parent------+
35 ///
36 /// And the tree would look like this
37 ///
38 /// +--PATH---------+
39 /// | | |
40 /// | | |
41 /// | '::' 'bar'
42 /// |
43 /// PATH
44 /// |
45 /// 'foo'
46 ///
47 /// See also `CompleteMarker::precede`.
48 Start {
49 kind: SyntaxKind,
50 forward_parent: Option<u32>,
51 },
52
53 /// Complete the previous `Start` event
54 Finish,
55
56 /// Produce a single leaf-element.
57 /// `n_raw_tokens` is used to glue complex contextual tokens.
58 /// For example, lexer tokenizes `>>` as `>`, `>`, and
59 /// `n_raw_tokens = 2` is used to produced a single `>>`.
60 Token {
61 kind: SyntaxKind,
62 n_raw_tokens: u8,
63 },
64
65 Error {
66 message: String,
67 },
68}
69
70pub(crate) fn parse<'t>(text: &'t str, raw_tokens: &'t [Token]) -> Vec<Event> {
71 let mut parser = parser::Parser::new(text, raw_tokens);
72 grammar::file(&mut parser);
73 parser.into_events()
74}
diff --git a/src/parser/event_parser/grammar/attributes.rs b/src/parser/grammar/attributes.rs
index 8bf04afce..8bf04afce 100644
--- a/src/parser/event_parser/grammar/attributes.rs
+++ b/src/parser/grammar/attributes.rs
diff --git a/src/parser/event_parser/grammar/expressions.rs b/src/parser/grammar/expressions.rs
index 8caaf3553..8caaf3553 100644
--- a/src/parser/event_parser/grammar/expressions.rs
+++ b/src/parser/grammar/expressions.rs
diff --git a/src/parser/event_parser/grammar/items/consts.rs b/src/parser/grammar/items/consts.rs
index c9881d681..c9881d681 100644
--- a/src/parser/event_parser/grammar/items/consts.rs
+++ b/src/parser/grammar/items/consts.rs
diff --git a/src/parser/event_parser/grammar/items/mod.rs b/src/parser/grammar/items/mod.rs
index 6d6fabbd7..3612802e1 100644
--- a/src/parser/event_parser/grammar/items/mod.rs
+++ b/src/parser/grammar/items/mod.rs
@@ -3,6 +3,7 @@ use super::*;
3mod structs; 3mod structs;
4mod use_item; 4mod use_item;
5mod consts; 5mod consts;
6mod traits;
6 7
7pub(super) fn mod_contents(p: &mut Parser, stop_on_r_curly: bool) { 8pub(super) fn mod_contents(p: &mut Parser, stop_on_r_curly: bool) {
8 attributes::inner_attributes(p); 9 attributes::inner_attributes(p);
@@ -80,6 +81,22 @@ fn item(p: &mut Parser) {
80 CONST_ITEM 81 CONST_ITEM
81 } 82 }
82 }, 83 },
84 // TODO: auto trait
85 // test unsafe_trait
86 // unsafe trait T {}
87 UNSAFE_KW if la == TRAIT_KW => {
88 p.bump();
89 traits::trait_item(p);
90 TRAIT_ITEM
91 }
92 // TODO: default impl
93 // test unsafe_impl
94 // unsafe impl Foo {}
95 UNSAFE_KW if la == IMPL_KW => {
96 p.bump();
97 traits::impl_item(p);
98 IMPL_ITEM
99 }
83 MOD_KW => { 100 MOD_KW => {
84 mod_item(p); 101 mod_item(p);
85 MOD_ITEM 102 MOD_ITEM
@@ -131,6 +148,7 @@ fn extern_block(p: &mut Parser) {
131 p.bump(); 148 p.bump();
132 p.expect(R_CURLY); 149 p.expect(R_CURLY);
133} 150}
151
134fn mod_item(p: &mut Parser) { 152fn mod_item(p: &mut Parser) {
135 assert!(p.at(MOD_KW)); 153 assert!(p.at(MOD_KW));
136 p.bump(); 154 p.bump();
diff --git a/src/parser/event_parser/grammar/items/structs.rs b/src/parser/grammar/items/structs.rs
index 69d95c698..69d95c698 100644
--- a/src/parser/event_parser/grammar/items/structs.rs
+++ b/src/parser/grammar/items/structs.rs
diff --git a/src/parser/grammar/items/traits.rs b/src/parser/grammar/items/traits.rs
new file mode 100644
index 000000000..3bef9639f
--- /dev/null
+++ b/src/parser/grammar/items/traits.rs
@@ -0,0 +1,17 @@
1use super::*;
2
3pub(super) fn trait_item(p: &mut Parser) {
4 assert!(p.at(TRAIT_KW));
5 p.bump();
6 p.expect(IDENT);
7 p.expect(L_CURLY);
8 p.expect(R_CURLY);
9}
10
11pub(super) fn impl_item(p: &mut Parser) {
12 assert!(p.at(IMPL_KW));
13 p.bump();
14 p.expect(IDENT);
15 p.expect(L_CURLY);
16 p.expect(R_CURLY);
17}
diff --git a/src/parser/event_parser/grammar/items/use_item.rs b/src/parser/grammar/items/use_item.rs
index 38e7b3f8a..38e7b3f8a 100644
--- a/src/parser/event_parser/grammar/items/use_item.rs
+++ b/src/parser/grammar/items/use_item.rs
diff --git a/src/parser/event_parser/grammar/mod.rs b/src/parser/grammar/mod.rs
index afce308d0..afce308d0 100644
--- a/src/parser/event_parser/grammar/mod.rs
+++ b/src/parser/grammar/mod.rs
diff --git a/src/parser/event_parser/grammar/paths.rs b/src/parser/grammar/paths.rs
index 6efac2610..6efac2610 100644
--- a/src/parser/event_parser/grammar/paths.rs
+++ b/src/parser/grammar/paths.rs
diff --git a/src/parser/event_parser/grammar/type_params.rs b/src/parser/grammar/type_params.rs
index 12c9a5362..12c9a5362 100644
--- a/src/parser/event_parser/grammar/type_params.rs
+++ b/src/parser/grammar/type_params.rs
diff --git a/src/parser/event_parser/grammar/types.rs b/src/parser/grammar/types.rs
index 1a3d44a0a..1a3d44a0a 100644
--- a/src/parser/event_parser/grammar/types.rs
+++ b/src/parser/grammar/types.rs
diff --git a/src/parser/mod.rs b/src/parser/mod.rs
index c5525ff9c..f17ffbf3a 100644
--- a/src/parser/mod.rs
+++ b/src/parser/mod.rs
@@ -1,88 +1,21 @@
1use {File, FileBuilder, Sink, SyntaxKind, Token}; 1use {File, SyntaxKind, Token};
2 2
3use syntax_kinds::*; 3use syntax_kinds::*;
4 4
5mod event_parser; 5#[macro_use]
6use self::event_parser::Event; 6mod parser;
7mod event;
8mod grammar;
9use self::event::Event;
7 10
8/// Parse a sequence of tokens into the representative node tree 11/// Parse a sequence of tokens into the representative node tree
9pub fn parse(text: String, tokens: &[Token]) -> File { 12pub fn parse(text: String, tokens: &[Token]) -> File {
10 let events = event_parser::parse(&text, tokens); 13 let events = {
11 from_events_to_file(text, tokens, events) 14 let mut parser = parser::Parser::new(&text, tokens);
12} 15 grammar::file(&mut parser);
13 16 parser.into_events()
14fn from_events_to_file(text: String, tokens: &[Token], events: Vec<Event>) -> File { 17 };
15 let mut builder = FileBuilder::new(text); 18 event::to_file(text, tokens, events)
16 let mut idx = 0;
17
18 let mut holes = Vec::new();
19 let mut forward_parents = Vec::new();
20
21 for (i, event) in events.iter().enumerate() {
22 if holes.last() == Some(&i) {
23 holes.pop();
24 continue;
25 }
26
27 match event {
28 &Event::Start {
29 kind: TOMBSTONE, ..
30 } => (),
31
32 &Event::Start { .. } => {
33 forward_parents.clear();
34 let mut idx = i;
35 loop {
36 let (kind, fwd) = match events[idx] {
37 Event::Start {
38 kind,
39 forward_parent,
40 } => (kind, forward_parent),
41 _ => unreachable!(),
42 };
43 forward_parents.push((idx, kind));
44 if let Some(fwd) = fwd {
45 idx += fwd as usize;
46 } else {
47 break;
48 }
49 }
50 for &(idx, kind) in forward_parents.iter().into_iter().rev() {
51 builder.start_internal(kind);
52 holes.push(idx);
53 }
54 holes.pop();
55 }
56 &Event::Finish => {
57 while idx < tokens.len() {
58 let token = tokens[idx];
59 if is_insignificant(token.kind) {
60 idx += 1;
61 builder.leaf(token.kind, token.len);
62 } else {
63 break;
64 }
65 }
66 builder.finish_internal()
67 }
68 &Event::Token {
69 kind: _,
70 mut n_raw_tokens,
71 } => loop {
72 let token = tokens[idx];
73 if !is_insignificant(token.kind) {
74 n_raw_tokens -= 1;
75 }
76 idx += 1;
77 builder.leaf(token.kind, token.len);
78 if n_raw_tokens == 0 {
79 break;
80 }
81 },
82 &Event::Error { ref message } => builder.error().message(message.clone()).emit(),
83 }
84 }
85 builder.finish()
86} 19}
87 20
88fn is_insignificant(kind: SyntaxKind) -> bool { 21fn is_insignificant(kind: SyntaxKind) -> bool {
diff --git a/src/parser/event_parser/parser.rs b/src/parser/parser.rs
index 6cf6ac9b9..3f4c8a07d 100644
--- a/src/parser/event_parser/parser.rs
+++ b/src/parser/parser.rs
@@ -1,6 +1,6 @@
1use {SyntaxKind, TextUnit, Token}; 1use {SyntaxKind, TextUnit, Token};
2use super::Event; 2use super::Event;
3use super::super::is_insignificant; 3use super::is_insignificant;
4use SyntaxKind::{EOF, TOMBSTONE}; 4use SyntaxKind::{EOF, TOMBSTONE};
5 5
6pub(crate) struct Marker { 6pub(crate) struct Marker {
diff --git a/src/syntax_kinds.rs b/src/syntax_kinds.rs
index c182aea78..22c615831 100644
--- a/src/syntax_kinds.rs
+++ b/src/syntax_kinds.rs
@@ -92,6 +92,8 @@ pub enum SyntaxKind {
92 USE_ITEM, 92 USE_ITEM,
93 STATIC_ITEM, 93 STATIC_ITEM,
94 CONST_ITEM, 94 CONST_ITEM,
95 TRAIT_ITEM,
96 IMPL_ITEM,
95 EXTERN_BLOCK, 97 EXTERN_BLOCK,
96 ENUM_VARIANT, 98 ENUM_VARIANT,
97 NAMED_FIELD, 99 NAMED_FIELD,
@@ -207,6 +209,8 @@ impl SyntaxKind {
207 USE_ITEM => &SyntaxInfo { name: "USE_ITEM" }, 209 USE_ITEM => &SyntaxInfo { name: "USE_ITEM" },
208 STATIC_ITEM => &SyntaxInfo { name: "STATIC_ITEM" }, 210 STATIC_ITEM => &SyntaxInfo { name: "STATIC_ITEM" },
209 CONST_ITEM => &SyntaxInfo { name: "CONST_ITEM" }, 211 CONST_ITEM => &SyntaxInfo { name: "CONST_ITEM" },
212 TRAIT_ITEM => &SyntaxInfo { name: "TRAIT_ITEM" },
213 IMPL_ITEM => &SyntaxInfo { name: "IMPL_ITEM" },
210 EXTERN_BLOCK => &SyntaxInfo { name: "EXTERN_BLOCK" }, 214 EXTERN_BLOCK => &SyntaxInfo { name: "EXTERN_BLOCK" },
211 ENUM_VARIANT => &SyntaxInfo { name: "ENUM_VARIANT" }, 215 ENUM_VARIANT => &SyntaxInfo { name: "ENUM_VARIANT" },
212 NAMED_FIELD => &SyntaxInfo { name: "NAMED_FIELD" }, 216 NAMED_FIELD => &SyntaxInfo { name: "NAMED_FIELD" },