aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--grammar.ron1
-rw-r--r--src/parser/event_parser/grammar.rs6
-rw-r--r--src/parser/event_parser/mod.rs1
-rw-r--r--src/parser/event_parser/parser.rs22
-rw-r--r--src/parser/mod.rs41
-rw-r--r--src/syntax_kinds.rs4
-rw-r--r--src/tree/file_builder.rs24
-rw-r--r--tests/data/parser/0001_struct_item.rs3
-rw-r--r--tests/data/parser/0001_struct_item.txt12
9 files changed, 97 insertions, 17 deletions
diff --git a/grammar.ron b/grammar.ron
index fb2c6d90e..c4f6283e6 100644
--- a/grammar.ron
+++ b/grammar.ron
@@ -62,5 +62,6 @@ Grammar(
62 ], 62 ],
63 nodes: [ 63 nodes: [
64 "FILE", 64 "FILE",
65 "STRUCT_ITEM"
65 ] 66 ]
66) \ No newline at end of file 67) \ No newline at end of file
diff --git a/src/parser/event_parser/grammar.rs b/src/parser/event_parser/grammar.rs
index 5219ed535..0896506fb 100644
--- a/src/parser/event_parser/grammar.rs
+++ b/src/parser/event_parser/grammar.rs
@@ -40,6 +40,12 @@ fn mod_items(p: &mut Parser) {
40fn item(p: &mut Parser) -> Result { 40fn item(p: &mut Parser) -> Result {
41 outer_attributes(p)?; 41 outer_attributes(p)?;
42 visibility(p)?; 42 visibility(p)?;
43 if p.current_is(STRUCT_KW) {
44 p.start(STRUCT_ITEM);
45 p.bump();
46 p.finish();
47 return OK;
48 }
43 ERR 49 ERR
44} 50}
45 51
diff --git a/src/parser/event_parser/mod.rs b/src/parser/event_parser/mod.rs
index bdfd23974..1228236a9 100644
--- a/src/parser/event_parser/mod.rs
+++ b/src/parser/event_parser/mod.rs
@@ -4,6 +4,7 @@ use syntax_kinds::*;
4mod grammar; 4mod grammar;
5mod parser; 5mod parser;
6 6
7#[derive(Debug)]
7pub(crate) enum Event { 8pub(crate) enum Event {
8 Start { kind: SyntaxKind }, 9 Start { kind: SyntaxKind },
9 Finish, 10 Finish,
diff --git a/src/parser/event_parser/parser.rs b/src/parser/event_parser/parser.rs
index 0e4d44b79..2d5418a29 100644
--- a/src/parser/event_parser/parser.rs
+++ b/src/parser/event_parser/parser.rs
@@ -1,5 +1,6 @@
1use {Token, SyntaxKind, TextUnit}; 1use {Token, SyntaxKind, TextUnit};
2use super::Event; 2use super::{Event};
3use super::super::is_insignificant;
3use syntax_kinds::{WHITESPACE, COMMENT}; 4use syntax_kinds::{WHITESPACE, COMMENT};
4 5
5pub struct Parser<'t> { 6pub struct Parser<'t> {
@@ -16,9 +17,8 @@ impl<'t> Parser<'t> {
16 let mut non_ws_tokens = Vec::new(); 17 let mut non_ws_tokens = Vec::new();
17 let mut len = TextUnit::new(0); 18 let mut len = TextUnit::new(0);
18 for (idx, &token) in raw_tokens.iter().enumerate() { 19 for (idx, &token) in raw_tokens.iter().enumerate() {
19 match token.kind { 20 if !is_insignificant(token.kind) {
20 WHITESPACE | COMMENT => (), 21 non_ws_tokens.push((idx, len))
21 _ => non_ws_tokens.push((idx, len)),
22 } 22 }
23 len += token.len; 23 len += token.len;
24 } 24 }
@@ -50,15 +50,25 @@ impl<'t> Parser<'t> {
50 self.event(Event::Finish); 50 self.event(Event::Finish);
51 } 51 }
52 52
53 pub(crate) fn bump(&mut self) -> Option<SyntaxKind> { 53 pub(crate) fn current(&self) -> Option<SyntaxKind> {
54 if self.is_eof() { 54 if self.is_eof() {
55 return None; 55 return None;
56 } 56 }
57 let idx = self.non_ws_tokens[self.pos].0; 57 let idx = self.non_ws_tokens[self.pos].0;
58 self.pos += 1;
59 Some(self.raw_tokens[idx].kind) 58 Some(self.raw_tokens[idx].kind)
60 } 59 }
61 60
61 pub(crate) fn current_is(&self, kind: SyntaxKind) -> bool {
62 self.current() == Some(kind)
63 }
64
65 pub(crate) fn bump(&mut self) -> Option<SyntaxKind> {
66 let kind = self.current()?;
67 self.pos += 1;
68 self.event(Event::Token { kind, n_raw_tokens: 1 });
69 Some(kind)
70 }
71
62 fn event(&mut self, event: Event) { 72 fn event(&mut self, event: Event) {
63 self.events.push(event) 73 self.events.push(event)
64 } 74 }
diff --git a/src/parser/mod.rs b/src/parser/mod.rs
index ccccd78f9..a632fbc01 100644
--- a/src/parser/mod.rs
+++ b/src/parser/mod.rs
@@ -8,17 +8,50 @@ use self::event_parser::Event;
8 8
9pub fn parse(text: String, tokens: &[Token]) -> File { 9pub fn parse(text: String, tokens: &[Token]) -> File {
10 let events = event_parser::parse(&text, tokens); 10 let events = event_parser::parse(&text, tokens);
11 from_events_to_file(text, events) 11 from_events_to_file(text, tokens, events)
12} 12}
13 13
14fn from_events_to_file(text: String, events: Vec<Event>) -> File { 14fn from_events_to_file(
15 text: String,
16 tokens: &[Token],
17 events: Vec<Event>,
18) -> File {
15 let mut builder = FileBuilder::new(text); 19 let mut builder = FileBuilder::new(text);
20 let mut idx = 0;
16 for event in events { 21 for event in events {
17 match event { 22 match event {
18 Event::Start { kind } => builder.start_internal(kind), 23 Event::Start { kind } => builder.start_internal(kind),
19 Event::Finish => builder.finish_internal(), 24 Event::Finish => {
20 Event::Token { .. } => unimplemented!(), 25 while idx < tokens.len() {
26 let token = tokens[idx];
27 if is_insignificant(token.kind) {
28 idx += 1;
29 builder.leaf(token.kind, token.len);
30 } else {
31 break;
32 }
33 }
34 builder.finish_internal()
35 },
36 Event::Token { kind, mut n_raw_tokens } => loop {
37 let token = tokens[idx];
38 if !is_insignificant(token.kind) {
39 n_raw_tokens -= 1;
40 }
41 idx += 1;
42 builder.leaf(token.kind, token.len);
43 if n_raw_tokens == 0 {
44 break;
45 }
46 },
21 } 47 }
22 } 48 }
23 builder.finish() 49 builder.finish()
24} 50}
51
52fn is_insignificant(kind: SyntaxKind) -> bool {
53 match kind {
54 WHITESPACE | COMMENT => true,
55 _ => false,
56 }
57} \ No newline at end of file
diff --git a/src/syntax_kinds.rs b/src/syntax_kinds.rs
index a1bcad062..d9b1462a7 100644
--- a/src/syntax_kinds.rs
+++ b/src/syntax_kinds.rs
@@ -59,8 +59,9 @@ pub const COMMENT: SyntaxKind = SyntaxKind(54);
59pub const DOC_COMMENT: SyntaxKind = SyntaxKind(55); 59pub const DOC_COMMENT: SyntaxKind = SyntaxKind(55);
60pub const SHEBANG: SyntaxKind = SyntaxKind(56); 60pub const SHEBANG: SyntaxKind = SyntaxKind(56);
61pub const FILE: SyntaxKind = SyntaxKind(57); 61pub const FILE: SyntaxKind = SyntaxKind(57);
62pub const STRUCT_ITEM: SyntaxKind = SyntaxKind(58);
62 63
63static INFOS: [SyntaxInfo; 58] = [ 64static INFOS: [SyntaxInfo; 59] = [
64 SyntaxInfo { name: "USE_KW" }, 65 SyntaxInfo { name: "USE_KW" },
65 SyntaxInfo { name: "FN_KW" }, 66 SyntaxInfo { name: "FN_KW" },
66 SyntaxInfo { name: "STRUCT_KW" }, 67 SyntaxInfo { name: "STRUCT_KW" },
@@ -119,6 +120,7 @@ static INFOS: [SyntaxInfo; 58] = [
119 SyntaxInfo { name: "DOC_COMMENT" }, 120 SyntaxInfo { name: "DOC_COMMENT" },
120 SyntaxInfo { name: "SHEBANG" }, 121 SyntaxInfo { name: "SHEBANG" },
121 SyntaxInfo { name: "FILE" }, 122 SyntaxInfo { name: "FILE" },
123 SyntaxInfo { name: "STRUCT_ITEM" },
122]; 124];
123 125
124pub(crate) fn syntax_info(kind: SyntaxKind) -> &'static SyntaxInfo { 126pub(crate) fn syntax_info(kind: SyntaxKind) -> &'static SyntaxInfo {
diff --git a/src/tree/file_builder.rs b/src/tree/file_builder.rs
index eba850fef..da8b8f824 100644
--- a/src/tree/file_builder.rs
+++ b/src/tree/file_builder.rs
@@ -64,8 +64,14 @@ impl FileBuilder {
64 } 64 }
65 65
66 pub fn finish(self) -> File { 66 pub fn finish(self) -> File {
67 assert!(self.in_progress.is_empty()); 67 assert!(
68 assert!(self.pos == (self.text.len() as u32).into()); 68 self.in_progress.is_empty(),
69 "some nodes in FileBuilder are unfinished"
70 );
71 assert!(
72 self.pos == (self.text.len() as u32).into(),
73 "nodes in FileBuilder do not cover the whole file"
74 );
69 File { 75 File {
70 text: self.text, 76 text: self.text,
71 nodes: self.nodes, 77 nodes: self.nodes,
@@ -81,11 +87,17 @@ impl FileBuilder {
81 fn push_child(&mut self, mut child: NodeData) -> NodeIdx { 87 fn push_child(&mut self, mut child: NodeData) -> NodeIdx {
82 child.parent = Some(self.current_id()); 88 child.parent = Some(self.current_id());
83 let id = self.new_node(child); 89 let id = self.new_node(child);
84 if let Some(sibling) = self.current_sibling() { 90 {
85 fill(&mut sibling.next_sibling, id); 91
86 return id 92 let (parent, sibling) = *self.in_progress.last().unwrap();
93 let slot = if let Some(idx) = sibling {
94 &mut self.nodes[idx].next_sibling
95 } else {
96 &mut self.nodes[parent].first_child
97 };
98 fill(slot, id);
87 } 99 }
88 fill(&mut self.current_parent().first_child, id); 100 self.in_progress.last_mut().unwrap().1 = Some(id);
89 id 101 id
90 } 102 }
91 103
diff --git a/tests/data/parser/0001_struct_item.rs b/tests/data/parser/0001_struct_item.rs
new file mode 100644
index 000000000..d3a8c1d23
--- /dev/null
+++ b/tests/data/parser/0001_struct_item.rs
@@ -0,0 +1,3 @@
1struct S {
2
3} \ No newline at end of file
diff --git a/tests/data/parser/0001_struct_item.txt b/tests/data/parser/0001_struct_item.txt
new file mode 100644
index 000000000..5ef544282
--- /dev/null
+++ b/tests/data/parser/0001_struct_item.txt
@@ -0,0 +1,12 @@
1FILE@[0; 13)
2 STRUCT_ITEM@[0; 7)
3 STRUCT_KW@[0; 6)
4 WHITESPACE@[6; 7)
5 ERROR@[7; 9)
6 IDENT@[7; 8)
7 WHITESPACE@[8; 9)
8 ERROR@[9; 12)
9 L_CURLY@[9; 10)
10 WHITESPACE@[10; 12)
11 ERROR@[12; 13)
12 R_CURLY@[12; 13)