aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--grammar.ron2
-rw-r--r--src/parser/event.rs140
-rw-r--r--src/parser/event_parser/mod.rs74
-rw-r--r--src/parser/grammar/attributes.rs (renamed from src/parser/event_parser/grammar/attributes.rs)0
-rw-r--r--src/parser/grammar/expressions.rs (renamed from src/parser/event_parser/grammar/expressions.rs)0
-rw-r--r--src/parser/grammar/items/consts.rs (renamed from src/parser/event_parser/grammar/items/consts.rs)0
-rw-r--r--src/parser/grammar/items/mod.rs (renamed from src/parser/event_parser/grammar/items/mod.rs)18
-rw-r--r--src/parser/grammar/items/structs.rs (renamed from src/parser/event_parser/grammar/items/structs.rs)0
-rw-r--r--src/parser/grammar/items/traits.rs17
-rw-r--r--src/parser/grammar/items/use_item.rs (renamed from src/parser/event_parser/grammar/items/use_item.rs)0
-rw-r--r--src/parser/grammar/mod.rs (renamed from src/parser/event_parser/grammar/mod.rs)0
-rw-r--r--src/parser/grammar/paths.rs (renamed from src/parser/event_parser/grammar/paths.rs)0
-rw-r--r--src/parser/grammar/type_params.rs (renamed from src/parser/event_parser/grammar/type_params.rs)0
-rw-r--r--src/parser/grammar/types.rs (renamed from src/parser/event_parser/grammar/types.rs)0
-rw-r--r--src/parser/mod.rs91
-rw-r--r--src/parser/parser.rs (renamed from src/parser/event_parser/parser.rs)2
-rw-r--r--src/syntax_kinds.rs4
-rw-r--r--tests/data/parser/inline/0007_unsafe_trait.rs1
-rw-r--r--tests/data/parser/inline/0007_unsafe_trait.txt11
-rw-r--r--tests/data/parser/inline/0008_unsafe_impl.rs1
-rw-r--r--tests/data/parser/inline/0008_unsafe_impl.txt11
-rw-r--r--tools/src/bin/collect-tests.rs17
22 files changed, 228 insertions, 161 deletions
diff --git a/grammar.ron b/grammar.ron
index 0717b7a76..c2fcc44f5 100644
--- a/grammar.ron
+++ b/grammar.ron
@@ -91,6 +91,8 @@ Grammar(
91 "USE_ITEM", 91 "USE_ITEM",
92 "STATIC_ITEM", 92 "STATIC_ITEM",
93 "CONST_ITEM", 93 "CONST_ITEM",
94 "TRAIT_ITEM",
95 "IMPL_ITEM",
94 96
95 "EXTERN_BLOCK", 97 "EXTERN_BLOCK",
96 "ENUM_VARIANT", 98 "ENUM_VARIANT",
diff --git a/src/parser/event.rs b/src/parser/event.rs
new file mode 100644
index 000000000..fd6bdc086
--- /dev/null
+++ b/src/parser/event.rs
@@ -0,0 +1,140 @@
1use {File, FileBuilder, Sink, SyntaxKind, Token};
2use syntax_kinds::TOMBSTONE;
3use super::is_insignificant;
4
5/// `Parser` produces a flat list of `Event`s.
6/// They are converted to a tree-structure in
7/// a separate pass, via `TreeBuilder`.
8#[derive(Debug)]
9pub(crate) enum Event {
10 /// This event signifies the start of the node.
11 /// It should be either abandoned (in which case the
12 /// `kind` is `TOMBSTONE`, and the event is ignored),
13 /// or completed via a `Finish` event.
14 ///
15 /// All tokens between a `Start` and a `Finish` would
16 /// become the children of the respective node.
17 ///
18 /// For left-recursive syntactic constructs, the parser produces
19 /// a child node before it sees a parent. `forward_parent`
20 /// exists to allow to tweak parent-child relationships.
21 ///
22 /// Consider this path
23 ///
24 /// foo::bar
25 ///
26 /// The events for it would look like this:
27 ///
28 ///
29 /// START(PATH) IDENT('foo') FINISH START(PATH) COLONCOLON IDENT('bar') FINISH
30 /// | /\
31 /// | |
32 /// +------forward-parent------+
33 ///
34 /// And the tree would look like this
35 ///
36 /// +--PATH---------+
37 /// | | |
38 /// | | |
39 /// | '::' 'bar'
40 /// |
41 /// PATH
42 /// |
43 /// 'foo'
44 ///
45 /// See also `CompleteMarker::precede`.
46 Start {
47 kind: SyntaxKind,
48 forward_parent: Option<u32>,
49 },
50
51 /// Complete the previous `Start` event
52 Finish,
53
54 /// Produce a single leaf-element.
55 /// `n_raw_tokens` is used to glue complex contextual tokens.
56 /// For example, lexer tokenizes `>>` as `>`, `>`, and
57 /// `n_raw_tokens = 2` is used to produced a single `>>`.
58 Token {
59 kind: SyntaxKind,
60 n_raw_tokens: u8,
61 },
62
63 Error {
64 message: String,
65 },
66}
67
68pub(super) fn to_file(text: String, tokens: &[Token], events: Vec<Event>) -> File {
69 let mut builder = FileBuilder::new(text);
70 let mut idx = 0;
71
72 let mut holes = Vec::new();
73 let mut forward_parents = Vec::new();
74
75 for (i, event) in events.iter().enumerate() {
76 if holes.last() == Some(&i) {
77 holes.pop();
78 continue;
79 }
80
81 match event {
82 &Event::Start {
83 kind: TOMBSTONE, ..
84 } => (),
85
86 &Event::Start { .. } => {
87 forward_parents.clear();
88 let mut idx = i;
89 loop {
90 let (kind, fwd) = match events[idx] {
91 Event::Start {
92 kind,
93 forward_parent,
94 } => (kind, forward_parent),
95 _ => unreachable!(),
96 };
97 forward_parents.push((idx, kind));
98 if let Some(fwd) = fwd {
99 idx += fwd as usize;
100 } else {
101 break;
102 }
103 }
104 for &(idx, kind) in forward_parents.iter().into_iter().rev() {
105 builder.start_internal(kind);
106 holes.push(idx);
107 }
108 holes.pop();
109 }
110 &Event::Finish => {
111 while idx < tokens.len() {
112 let token = tokens[idx];
113 if is_insignificant(token.kind) {
114 idx += 1;
115 builder.leaf(token.kind, token.len);
116 } else {
117 break;
118 }
119 }
120 builder.finish_internal()
121 }
122 &Event::Token {
123 kind: _,
124 mut n_raw_tokens,
125 } => loop {
126 let token = tokens[idx];
127 if !is_insignificant(token.kind) {
128 n_raw_tokens -= 1;
129 }
130 idx += 1;
131 builder.leaf(token.kind, token.len);
132 if n_raw_tokens == 0 {
133 break;
134 }
135 },
136 &Event::Error { ref message } => builder.error().message(message.clone()).emit(),
137 }
138 }
139 builder.finish()
140}
diff --git a/src/parser/event_parser/mod.rs b/src/parser/event_parser/mod.rs
deleted file mode 100644
index 7823c476c..000000000
--- a/src/parser/event_parser/mod.rs
+++ /dev/null
@@ -1,74 +0,0 @@
1use {SyntaxKind, Token};
2
3#[macro_use]
4mod parser;
5mod grammar;
6
7/// `Parser` produces a flat list of `Event`s.
8/// They are converted to a tree-structure in
9/// a separate pass, via `TreeBuilder`.
10#[derive(Debug)]
11pub(crate) enum Event {
12 /// This event signifies the start of the node.
13 /// It should be either abandoned (in which case the
14 /// `kind` is `TOMBSTONE`, and the event is ignored),
15 /// or completed via a `Finish` event.
16 ///
17 /// All tokens between a `Start` and a `Finish` would
18 /// become the children of the respective node.
19 ///
20 /// For left-recursive syntactic constructs, the parser produces
21 /// a child node before it sees a parent. `forward_parent`
22 /// exists to allow to tweak parent-child relationships.
23 ///
24 /// Consider this path
25 ///
26 /// foo::bar
27 ///
28 /// The events for it would look like this:
29 ///
30 ///
31 /// START(PATH) IDENT('foo') FINISH START(PATH) COLONCOLON IDENT('bar') FINISH
32 /// | /\
33 /// | |
34 /// +------forward-parent------+
35 ///
36 /// And the tree would look like this
37 ///
38 /// +--PATH---------+
39 /// | | |
40 /// | | |
41 /// | '::' 'bar'
42 /// |
43 /// PATH
44 /// |
45 /// 'foo'
46 ///
47 /// See also `CompleteMarker::precede`.
48 Start {
49 kind: SyntaxKind,
50 forward_parent: Option<u32>,
51 },
52
53 /// Complete the previous `Start` event
54 Finish,
55
56 /// Produce a single leaf-element.
57 /// `n_raw_tokens` is used to glue complex contextual tokens.
58 /// For example, lexer tokenizes `>>` as `>`, `>`, and
59 /// `n_raw_tokens = 2` is used to produced a single `>>`.
60 Token {
61 kind: SyntaxKind,
62 n_raw_tokens: u8,
63 },
64
65 Error {
66 message: String,
67 },
68}
69
70pub(crate) fn parse<'t>(text: &'t str, raw_tokens: &'t [Token]) -> Vec<Event> {
71 let mut parser = parser::Parser::new(text, raw_tokens);
72 grammar::file(&mut parser);
73 parser.into_events()
74}
diff --git a/src/parser/event_parser/grammar/attributes.rs b/src/parser/grammar/attributes.rs
index 8bf04afce..8bf04afce 100644
--- a/src/parser/event_parser/grammar/attributes.rs
+++ b/src/parser/grammar/attributes.rs
diff --git a/src/parser/event_parser/grammar/expressions.rs b/src/parser/grammar/expressions.rs
index 8caaf3553..8caaf3553 100644
--- a/src/parser/event_parser/grammar/expressions.rs
+++ b/src/parser/grammar/expressions.rs
diff --git a/src/parser/event_parser/grammar/items/consts.rs b/src/parser/grammar/items/consts.rs
index c9881d681..c9881d681 100644
--- a/src/parser/event_parser/grammar/items/consts.rs
+++ b/src/parser/grammar/items/consts.rs
diff --git a/src/parser/event_parser/grammar/items/mod.rs b/src/parser/grammar/items/mod.rs
index 6d6fabbd7..3612802e1 100644
--- a/src/parser/event_parser/grammar/items/mod.rs
+++ b/src/parser/grammar/items/mod.rs
@@ -3,6 +3,7 @@ use super::*;
3mod structs; 3mod structs;
4mod use_item; 4mod use_item;
5mod consts; 5mod consts;
6mod traits;
6 7
7pub(super) fn mod_contents(p: &mut Parser, stop_on_r_curly: bool) { 8pub(super) fn mod_contents(p: &mut Parser, stop_on_r_curly: bool) {
8 attributes::inner_attributes(p); 9 attributes::inner_attributes(p);
@@ -80,6 +81,22 @@ fn item(p: &mut Parser) {
80 CONST_ITEM 81 CONST_ITEM
81 } 82 }
82 }, 83 },
84 // TODO: auto trait
85 // test unsafe_trait
86 // unsafe trait T {}
87 UNSAFE_KW if la == TRAIT_KW => {
88 p.bump();
89 traits::trait_item(p);
90 TRAIT_ITEM
91 }
92 // TODO: default impl
93 // test unsafe_impl
94 // unsafe impl Foo {}
95 UNSAFE_KW if la == IMPL_KW => {
96 p.bump();
97 traits::impl_item(p);
98 IMPL_ITEM
99 }
83 MOD_KW => { 100 MOD_KW => {
84 mod_item(p); 101 mod_item(p);
85 MOD_ITEM 102 MOD_ITEM
@@ -131,6 +148,7 @@ fn extern_block(p: &mut Parser) {
131 p.bump(); 148 p.bump();
132 p.expect(R_CURLY); 149 p.expect(R_CURLY);
133} 150}
151
134fn mod_item(p: &mut Parser) { 152fn mod_item(p: &mut Parser) {
135 assert!(p.at(MOD_KW)); 153 assert!(p.at(MOD_KW));
136 p.bump(); 154 p.bump();
diff --git a/src/parser/event_parser/grammar/items/structs.rs b/src/parser/grammar/items/structs.rs
index 69d95c698..69d95c698 100644
--- a/src/parser/event_parser/grammar/items/structs.rs
+++ b/src/parser/grammar/items/structs.rs
diff --git a/src/parser/grammar/items/traits.rs b/src/parser/grammar/items/traits.rs
new file mode 100644
index 000000000..3bef9639f
--- /dev/null
+++ b/src/parser/grammar/items/traits.rs
@@ -0,0 +1,17 @@
1use super::*;
2
3pub(super) fn trait_item(p: &mut Parser) {
4 assert!(p.at(TRAIT_KW));
5 p.bump();
6 p.expect(IDENT);
7 p.expect(L_CURLY);
8 p.expect(R_CURLY);
9}
10
11pub(super) fn impl_item(p: &mut Parser) {
12 assert!(p.at(IMPL_KW));
13 p.bump();
14 p.expect(IDENT);
15 p.expect(L_CURLY);
16 p.expect(R_CURLY);
17}
diff --git a/src/parser/event_parser/grammar/items/use_item.rs b/src/parser/grammar/items/use_item.rs
index 38e7b3f8a..38e7b3f8a 100644
--- a/src/parser/event_parser/grammar/items/use_item.rs
+++ b/src/parser/grammar/items/use_item.rs
diff --git a/src/parser/event_parser/grammar/mod.rs b/src/parser/grammar/mod.rs
index afce308d0..afce308d0 100644
--- a/src/parser/event_parser/grammar/mod.rs
+++ b/src/parser/grammar/mod.rs
diff --git a/src/parser/event_parser/grammar/paths.rs b/src/parser/grammar/paths.rs
index 6efac2610..6efac2610 100644
--- a/src/parser/event_parser/grammar/paths.rs
+++ b/src/parser/grammar/paths.rs
diff --git a/src/parser/event_parser/grammar/type_params.rs b/src/parser/grammar/type_params.rs
index 12c9a5362..12c9a5362 100644
--- a/src/parser/event_parser/grammar/type_params.rs
+++ b/src/parser/grammar/type_params.rs
diff --git a/src/parser/event_parser/grammar/types.rs b/src/parser/grammar/types.rs
index 1a3d44a0a..1a3d44a0a 100644
--- a/src/parser/event_parser/grammar/types.rs
+++ b/src/parser/grammar/types.rs
diff --git a/src/parser/mod.rs b/src/parser/mod.rs
index c5525ff9c..f17ffbf3a 100644
--- a/src/parser/mod.rs
+++ b/src/parser/mod.rs
@@ -1,88 +1,21 @@
1use {File, FileBuilder, Sink, SyntaxKind, Token}; 1use {File, SyntaxKind, Token};
2 2
3use syntax_kinds::*; 3use syntax_kinds::*;
4 4
5mod event_parser; 5#[macro_use]
6use self::event_parser::Event; 6mod parser;
7mod event;
8mod grammar;
9use self::event::Event;
7 10
8/// Parse a sequence of tokens into the representative node tree 11/// Parse a sequence of tokens into the representative node tree
9pub fn parse(text: String, tokens: &[Token]) -> File { 12pub fn parse(text: String, tokens: &[Token]) -> File {
10 let events = event_parser::parse(&text, tokens); 13 let events = {
11 from_events_to_file(text, tokens, events) 14 let mut parser = parser::Parser::new(&text, tokens);
12} 15 grammar::file(&mut parser);
13 16 parser.into_events()
14fn from_events_to_file(text: String, tokens: &[Token], events: Vec<Event>) -> File { 17 };
15 let mut builder = FileBuilder::new(text); 18 event::to_file(text, tokens, events)
16 let mut idx = 0;
17
18 let mut holes = Vec::new();
19 let mut forward_parents = Vec::new();
20
21 for (i, event) in events.iter().enumerate() {
22 if holes.last() == Some(&i) {
23 holes.pop();
24 continue;
25 }
26
27 match event {
28 &Event::Start {
29 kind: TOMBSTONE, ..
30 } => (),
31
32 &Event::Start { .. } => {
33 forward_parents.clear();
34 let mut idx = i;
35 loop {
36 let (kind, fwd) = match events[idx] {
37 Event::Start {
38 kind,
39 forward_parent,
40 } => (kind, forward_parent),
41 _ => unreachable!(),
42 };
43 forward_parents.push((idx, kind));
44 if let Some(fwd) = fwd {
45 idx += fwd as usize;
46 } else {
47 break;
48 }
49 }
50 for &(idx, kind) in forward_parents.iter().into_iter().rev() {
51 builder.start_internal(kind);
52 holes.push(idx);
53 }
54 holes.pop();
55 }
56 &Event::Finish => {
57 while idx < tokens.len() {
58 let token = tokens[idx];
59 if is_insignificant(token.kind) {
60 idx += 1;
61 builder.leaf(token.kind, token.len);
62 } else {
63 break;
64 }
65 }
66 builder.finish_internal()
67 }
68 &Event::Token {
69 kind: _,
70 mut n_raw_tokens,
71 } => loop {
72 let token = tokens[idx];
73 if !is_insignificant(token.kind) {
74 n_raw_tokens -= 1;
75 }
76 idx += 1;
77 builder.leaf(token.kind, token.len);
78 if n_raw_tokens == 0 {
79 break;
80 }
81 },
82 &Event::Error { ref message } => builder.error().message(message.clone()).emit(),
83 }
84 }
85 builder.finish()
86} 19}
87 20
88fn is_insignificant(kind: SyntaxKind) -> bool { 21fn is_insignificant(kind: SyntaxKind) -> bool {
diff --git a/src/parser/event_parser/parser.rs b/src/parser/parser.rs
index 6cf6ac9b9..3f4c8a07d 100644
--- a/src/parser/event_parser/parser.rs
+++ b/src/parser/parser.rs
@@ -1,6 +1,6 @@
1use {SyntaxKind, TextUnit, Token}; 1use {SyntaxKind, TextUnit, Token};
2use super::Event; 2use super::Event;
3use super::super::is_insignificant; 3use super::is_insignificant;
4use SyntaxKind::{EOF, TOMBSTONE}; 4use SyntaxKind::{EOF, TOMBSTONE};
5 5
6pub(crate) struct Marker { 6pub(crate) struct Marker {
diff --git a/src/syntax_kinds.rs b/src/syntax_kinds.rs
index c182aea78..22c615831 100644
--- a/src/syntax_kinds.rs
+++ b/src/syntax_kinds.rs
@@ -92,6 +92,8 @@ pub enum SyntaxKind {
92 USE_ITEM, 92 USE_ITEM,
93 STATIC_ITEM, 93 STATIC_ITEM,
94 CONST_ITEM, 94 CONST_ITEM,
95 TRAIT_ITEM,
96 IMPL_ITEM,
95 EXTERN_BLOCK, 97 EXTERN_BLOCK,
96 ENUM_VARIANT, 98 ENUM_VARIANT,
97 NAMED_FIELD, 99 NAMED_FIELD,
@@ -207,6 +209,8 @@ impl SyntaxKind {
207 USE_ITEM => &SyntaxInfo { name: "USE_ITEM" }, 209 USE_ITEM => &SyntaxInfo { name: "USE_ITEM" },
208 STATIC_ITEM => &SyntaxInfo { name: "STATIC_ITEM" }, 210 STATIC_ITEM => &SyntaxInfo { name: "STATIC_ITEM" },
209 CONST_ITEM => &SyntaxInfo { name: "CONST_ITEM" }, 211 CONST_ITEM => &SyntaxInfo { name: "CONST_ITEM" },
212 TRAIT_ITEM => &SyntaxInfo { name: "TRAIT_ITEM" },
213 IMPL_ITEM => &SyntaxInfo { name: "IMPL_ITEM" },
210 EXTERN_BLOCK => &SyntaxInfo { name: "EXTERN_BLOCK" }, 214 EXTERN_BLOCK => &SyntaxInfo { name: "EXTERN_BLOCK" },
211 ENUM_VARIANT => &SyntaxInfo { name: "ENUM_VARIANT" }, 215 ENUM_VARIANT => &SyntaxInfo { name: "ENUM_VARIANT" },
212 NAMED_FIELD => &SyntaxInfo { name: "NAMED_FIELD" }, 216 NAMED_FIELD => &SyntaxInfo { name: "NAMED_FIELD" },
diff --git a/tests/data/parser/inline/0007_unsafe_trait.rs b/tests/data/parser/inline/0007_unsafe_trait.rs
new file mode 100644
index 000000000..04e021550
--- /dev/null
+++ b/tests/data/parser/inline/0007_unsafe_trait.rs
@@ -0,0 +1 @@
unsafe trait T {}
diff --git a/tests/data/parser/inline/0007_unsafe_trait.txt b/tests/data/parser/inline/0007_unsafe_trait.txt
new file mode 100644
index 000000000..d6f6a4cfa
--- /dev/null
+++ b/tests/data/parser/inline/0007_unsafe_trait.txt
@@ -0,0 +1,11 @@
1FILE@[0; 18)
2 TRAIT_ITEM@[0; 18)
3 UNSAFE_KW@[0; 6)
4 WHITESPACE@[6; 7)
5 TRAIT_KW@[7; 12)
6 WHITESPACE@[12; 13)
7 IDENT@[13; 14) "T"
8 WHITESPACE@[14; 15)
9 L_CURLY@[15; 16)
10 R_CURLY@[16; 17)
11 WHITESPACE@[17; 18)
diff --git a/tests/data/parser/inline/0008_unsafe_impl.rs b/tests/data/parser/inline/0008_unsafe_impl.rs
new file mode 100644
index 000000000..41055f41d
--- /dev/null
+++ b/tests/data/parser/inline/0008_unsafe_impl.rs
@@ -0,0 +1 @@
unsafe impl Foo {}
diff --git a/tests/data/parser/inline/0008_unsafe_impl.txt b/tests/data/parser/inline/0008_unsafe_impl.txt
new file mode 100644
index 000000000..a88a447cb
--- /dev/null
+++ b/tests/data/parser/inline/0008_unsafe_impl.txt
@@ -0,0 +1,11 @@
1FILE@[0; 19)
2 IMPL_ITEM@[0; 19)
3 UNSAFE_KW@[0; 6)
4 WHITESPACE@[6; 7)
5 IMPL_KW@[7; 11)
6 WHITESPACE@[11; 12)
7 IDENT@[12; 15) "Foo"
8 WHITESPACE@[15; 16)
9 L_CURLY@[16; 17)
10 R_CURLY@[17; 18)
11 WHITESPACE@[18; 19)
diff --git a/tools/src/bin/collect-tests.rs b/tools/src/bin/collect-tests.rs
index df9d2db81..a52e7b119 100644
--- a/tools/src/bin/collect-tests.rs
+++ b/tools/src/bin/collect-tests.rs
@@ -79,16 +79,19 @@ fn collect_tests(s: &str) -> Vec<Test> {
79 .map(str::trim_left) 79 .map(str::trim_left)
80 .group_by(|line| line.starts_with(prefix)); 80 .group_by(|line| line.starts_with(prefix));
81 81
82 for (is_comment, block) in comment_blocks.into_iter() { 82 'outer: for (is_comment, block) in comment_blocks.into_iter() {
83 if !is_comment { 83 if !is_comment {
84 continue; 84 continue;
85 } 85 }
86 let mut block = block.map(|line| &line[prefix.len()..]); 86 let mut block = block.map(|line| &line[prefix.len()..]);
87 let first = block.next().unwrap(); 87
88 if !first.starts_with("test ") { 88 let name = loop {
89 continue; 89 match block.next() {
90 } 90 Some(line) if line.starts_with("test ") => break line["test ".len()..].to_string(),
91 let name = first["test ".len()..].to_string(); 91 Some(_) => (),
92 None => continue 'outer,
93 }
94 };
92 let text: String = itertools::join(block.chain(::std::iter::once("")), "\n"); 95 let text: String = itertools::join(block.chain(::std::iter::once("")), "\n");
93 assert!(!text.trim().is_empty() && text.ends_with("\n")); 96 assert!(!text.trim().is_empty() && text.ends_with("\n"));
94 res.push(Test { name, text }) 97 res.push(Test { name, text })
@@ -121,7 +124,7 @@ fn inline_tests_dir() -> PathBuf {
121} 124}
122 125
123fn grammar_dir() -> PathBuf { 126fn grammar_dir() -> PathBuf {
124 base_dir().join("src/parser/event_parser/grammar") 127 base_dir().join("src/parser/grammar")
125} 128}
126 129
127fn base_dir() -> PathBuf { 130fn base_dir() -> PathBuf {