diff options
-rw-r--r-- | grammar.ron | 1 | ||||
-rw-r--r-- | src/parser/event_parser/grammar.rs | 6 | ||||
-rw-r--r-- | src/parser/event_parser/mod.rs | 1 | ||||
-rw-r--r-- | src/parser/event_parser/parser.rs | 22 | ||||
-rw-r--r-- | src/parser/mod.rs | 41 | ||||
-rw-r--r-- | src/syntax_kinds.rs | 4 | ||||
-rw-r--r-- | src/tree/file_builder.rs | 24 | ||||
-rw-r--r-- | tests/data/parser/0001_struct_item.rs | 3 | ||||
-rw-r--r-- | tests/data/parser/0001_struct_item.txt | 12 |
9 files changed, 97 insertions, 17 deletions
diff --git a/grammar.ron b/grammar.ron index fb2c6d90e..c4f6283e6 100644 --- a/grammar.ron +++ b/grammar.ron | |||
@@ -62,5 +62,6 @@ Grammar( | |||
62 | ], | 62 | ], |
63 | nodes: [ | 63 | nodes: [ |
64 | "FILE", | 64 | "FILE", |
65 | "STRUCT_ITEM" | ||
65 | ] | 66 | ] |
66 | ) \ No newline at end of file | 67 | ) \ No newline at end of file |
diff --git a/src/parser/event_parser/grammar.rs b/src/parser/event_parser/grammar.rs index 5219ed535..0896506fb 100644 --- a/src/parser/event_parser/grammar.rs +++ b/src/parser/event_parser/grammar.rs | |||
@@ -40,6 +40,12 @@ fn mod_items(p: &mut Parser) { | |||
40 | fn item(p: &mut Parser) -> Result { | 40 | fn item(p: &mut Parser) -> Result { |
41 | outer_attributes(p)?; | 41 | outer_attributes(p)?; |
42 | visibility(p)?; | 42 | visibility(p)?; |
43 | if p.current_is(STRUCT_KW) { | ||
44 | p.start(STRUCT_ITEM); | ||
45 | p.bump(); | ||
46 | p.finish(); | ||
47 | return OK; | ||
48 | } | ||
43 | ERR | 49 | ERR |
44 | } | 50 | } |
45 | 51 | ||
diff --git a/src/parser/event_parser/mod.rs b/src/parser/event_parser/mod.rs index bdfd23974..1228236a9 100644 --- a/src/parser/event_parser/mod.rs +++ b/src/parser/event_parser/mod.rs | |||
@@ -4,6 +4,7 @@ use syntax_kinds::*; | |||
4 | mod grammar; | 4 | mod grammar; |
5 | mod parser; | 5 | mod parser; |
6 | 6 | ||
7 | #[derive(Debug)] | ||
7 | pub(crate) enum Event { | 8 | pub(crate) enum Event { |
8 | Start { kind: SyntaxKind }, | 9 | Start { kind: SyntaxKind }, |
9 | Finish, | 10 | Finish, |
diff --git a/src/parser/event_parser/parser.rs b/src/parser/event_parser/parser.rs index 0e4d44b79..2d5418a29 100644 --- a/src/parser/event_parser/parser.rs +++ b/src/parser/event_parser/parser.rs | |||
@@ -1,5 +1,6 @@ | |||
1 | use {Token, SyntaxKind, TextUnit}; | 1 | use {Token, SyntaxKind, TextUnit}; |
2 | use super::Event; | 2 | use super::{Event}; |
3 | use super::super::is_insignificant; | ||
3 | use syntax_kinds::{WHITESPACE, COMMENT}; | 4 | use syntax_kinds::{WHITESPACE, COMMENT}; |
4 | 5 | ||
5 | pub struct Parser<'t> { | 6 | pub struct Parser<'t> { |
@@ -16,9 +17,8 @@ impl<'t> Parser<'t> { | |||
16 | let mut non_ws_tokens = Vec::new(); | 17 | let mut non_ws_tokens = Vec::new(); |
17 | let mut len = TextUnit::new(0); | 18 | let mut len = TextUnit::new(0); |
18 | for (idx, &token) in raw_tokens.iter().enumerate() { | 19 | for (idx, &token) in raw_tokens.iter().enumerate() { |
19 | match token.kind { | 20 | if !is_insignificant(token.kind) { |
20 | WHITESPACE | COMMENT => (), | 21 | non_ws_tokens.push((idx, len)) |
21 | _ => non_ws_tokens.push((idx, len)), | ||
22 | } | 22 | } |
23 | len += token.len; | 23 | len += token.len; |
24 | } | 24 | } |
@@ -50,15 +50,25 @@ impl<'t> Parser<'t> { | |||
50 | self.event(Event::Finish); | 50 | self.event(Event::Finish); |
51 | } | 51 | } |
52 | 52 | ||
53 | pub(crate) fn bump(&mut self) -> Option<SyntaxKind> { | 53 | pub(crate) fn current(&self) -> Option<SyntaxKind> { |
54 | if self.is_eof() { | 54 | if self.is_eof() { |
55 | return None; | 55 | return None; |
56 | } | 56 | } |
57 | let idx = self.non_ws_tokens[self.pos].0; | 57 | let idx = self.non_ws_tokens[self.pos].0; |
58 | self.pos += 1; | ||
59 | Some(self.raw_tokens[idx].kind) | 58 | Some(self.raw_tokens[idx].kind) |
60 | } | 59 | } |
61 | 60 | ||
61 | pub(crate) fn current_is(&self, kind: SyntaxKind) -> bool { | ||
62 | self.current() == Some(kind) | ||
63 | } | ||
64 | |||
65 | pub(crate) fn bump(&mut self) -> Option<SyntaxKind> { | ||
66 | let kind = self.current()?; | ||
67 | self.pos += 1; | ||
68 | self.event(Event::Token { kind, n_raw_tokens: 1 }); | ||
69 | Some(kind) | ||
70 | } | ||
71 | |||
62 | fn event(&mut self, event: Event) { | 72 | fn event(&mut self, event: Event) { |
63 | self.events.push(event) | 73 | self.events.push(event) |
64 | } | 74 | } |
diff --git a/src/parser/mod.rs b/src/parser/mod.rs index ccccd78f9..a632fbc01 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs | |||
@@ -8,17 +8,50 @@ use self::event_parser::Event; | |||
8 | 8 | ||
9 | pub fn parse(text: String, tokens: &[Token]) -> File { | 9 | pub fn parse(text: String, tokens: &[Token]) -> File { |
10 | let events = event_parser::parse(&text, tokens); | 10 | let events = event_parser::parse(&text, tokens); |
11 | from_events_to_file(text, events) | 11 | from_events_to_file(text, tokens, events) |
12 | } | 12 | } |
13 | 13 | ||
14 | fn from_events_to_file(text: String, events: Vec<Event>) -> File { | 14 | fn from_events_to_file( |
15 | text: String, | ||
16 | tokens: &[Token], | ||
17 | events: Vec<Event>, | ||
18 | ) -> File { | ||
15 | let mut builder = FileBuilder::new(text); | 19 | let mut builder = FileBuilder::new(text); |
20 | let mut idx = 0; | ||
16 | for event in events { | 21 | for event in events { |
17 | match event { | 22 | match event { |
18 | Event::Start { kind } => builder.start_internal(kind), | 23 | Event::Start { kind } => builder.start_internal(kind), |
19 | Event::Finish => builder.finish_internal(), | 24 | Event::Finish => { |
20 | Event::Token { .. } => unimplemented!(), | 25 | while idx < tokens.len() { |
26 | let token = tokens[idx]; | ||
27 | if is_insignificant(token.kind) { | ||
28 | idx += 1; | ||
29 | builder.leaf(token.kind, token.len); | ||
30 | } else { | ||
31 | break; | ||
32 | } | ||
33 | } | ||
34 | builder.finish_internal() | ||
35 | }, | ||
36 | Event::Token { kind, mut n_raw_tokens } => loop { | ||
37 | let token = tokens[idx]; | ||
38 | if !is_insignificant(token.kind) { | ||
39 | n_raw_tokens -= 1; | ||
40 | } | ||
41 | idx += 1; | ||
42 | builder.leaf(token.kind, token.len); | ||
43 | if n_raw_tokens == 0 { | ||
44 | break; | ||
45 | } | ||
46 | }, | ||
21 | } | 47 | } |
22 | } | 48 | } |
23 | builder.finish() | 49 | builder.finish() |
24 | } | 50 | } |
51 | |||
52 | fn is_insignificant(kind: SyntaxKind) -> bool { | ||
53 | match kind { | ||
54 | WHITESPACE | COMMENT => true, | ||
55 | _ => false, | ||
56 | } | ||
57 | } \ No newline at end of file | ||
diff --git a/src/syntax_kinds.rs b/src/syntax_kinds.rs index a1bcad062..d9b1462a7 100644 --- a/src/syntax_kinds.rs +++ b/src/syntax_kinds.rs | |||
@@ -59,8 +59,9 @@ pub const COMMENT: SyntaxKind = SyntaxKind(54); | |||
59 | pub const DOC_COMMENT: SyntaxKind = SyntaxKind(55); | 59 | pub const DOC_COMMENT: SyntaxKind = SyntaxKind(55); |
60 | pub const SHEBANG: SyntaxKind = SyntaxKind(56); | 60 | pub const SHEBANG: SyntaxKind = SyntaxKind(56); |
61 | pub const FILE: SyntaxKind = SyntaxKind(57); | 61 | pub const FILE: SyntaxKind = SyntaxKind(57); |
62 | pub const STRUCT_ITEM: SyntaxKind = SyntaxKind(58); | ||
62 | 63 | ||
63 | static INFOS: [SyntaxInfo; 58] = [ | 64 | static INFOS: [SyntaxInfo; 59] = [ |
64 | SyntaxInfo { name: "USE_KW" }, | 65 | SyntaxInfo { name: "USE_KW" }, |
65 | SyntaxInfo { name: "FN_KW" }, | 66 | SyntaxInfo { name: "FN_KW" }, |
66 | SyntaxInfo { name: "STRUCT_KW" }, | 67 | SyntaxInfo { name: "STRUCT_KW" }, |
@@ -119,6 +120,7 @@ static INFOS: [SyntaxInfo; 58] = [ | |||
119 | SyntaxInfo { name: "DOC_COMMENT" }, | 120 | SyntaxInfo { name: "DOC_COMMENT" }, |
120 | SyntaxInfo { name: "SHEBANG" }, | 121 | SyntaxInfo { name: "SHEBANG" }, |
121 | SyntaxInfo { name: "FILE" }, | 122 | SyntaxInfo { name: "FILE" }, |
123 | SyntaxInfo { name: "STRUCT_ITEM" }, | ||
122 | ]; | 124 | ]; |
123 | 125 | ||
124 | pub(crate) fn syntax_info(kind: SyntaxKind) -> &'static SyntaxInfo { | 126 | pub(crate) fn syntax_info(kind: SyntaxKind) -> &'static SyntaxInfo { |
diff --git a/src/tree/file_builder.rs b/src/tree/file_builder.rs index eba850fef..da8b8f824 100644 --- a/src/tree/file_builder.rs +++ b/src/tree/file_builder.rs | |||
@@ -64,8 +64,14 @@ impl FileBuilder { | |||
64 | } | 64 | } |
65 | 65 | ||
66 | pub fn finish(self) -> File { | 66 | pub fn finish(self) -> File { |
67 | assert!(self.in_progress.is_empty()); | 67 | assert!( |
68 | assert!(self.pos == (self.text.len() as u32).into()); | 68 | self.in_progress.is_empty(), |
69 | "some nodes in FileBuilder are unfinished" | ||
70 | ); | ||
71 | assert!( | ||
72 | self.pos == (self.text.len() as u32).into(), | ||
73 | "nodes in FileBuilder do not cover the whole file" | ||
74 | ); | ||
69 | File { | 75 | File { |
70 | text: self.text, | 76 | text: self.text, |
71 | nodes: self.nodes, | 77 | nodes: self.nodes, |
@@ -81,11 +87,17 @@ impl FileBuilder { | |||
81 | fn push_child(&mut self, mut child: NodeData) -> NodeIdx { | 87 | fn push_child(&mut self, mut child: NodeData) -> NodeIdx { |
82 | child.parent = Some(self.current_id()); | 88 | child.parent = Some(self.current_id()); |
83 | let id = self.new_node(child); | 89 | let id = self.new_node(child); |
84 | if let Some(sibling) = self.current_sibling() { | 90 | { |
85 | fill(&mut sibling.next_sibling, id); | 91 | |
86 | return id | 92 | let (parent, sibling) = *self.in_progress.last().unwrap(); |
93 | let slot = if let Some(idx) = sibling { | ||
94 | &mut self.nodes[idx].next_sibling | ||
95 | } else { | ||
96 | &mut self.nodes[parent].first_child | ||
97 | }; | ||
98 | fill(slot, id); | ||
87 | } | 99 | } |
88 | fill(&mut self.current_parent().first_child, id); | 100 | self.in_progress.last_mut().unwrap().1 = Some(id); |
89 | id | 101 | id |
90 | } | 102 | } |
91 | 103 | ||
diff --git a/tests/data/parser/0001_struct_item.rs b/tests/data/parser/0001_struct_item.rs new file mode 100644 index 000000000..d3a8c1d23 --- /dev/null +++ b/tests/data/parser/0001_struct_item.rs | |||
@@ -0,0 +1,3 @@ | |||
1 | struct S { | ||
2 | |||
3 | } \ No newline at end of file | ||
diff --git a/tests/data/parser/0001_struct_item.txt b/tests/data/parser/0001_struct_item.txt new file mode 100644 index 000000000..5ef544282 --- /dev/null +++ b/tests/data/parser/0001_struct_item.txt | |||
@@ -0,0 +1,12 @@ | |||
1 | FILE@[0; 13) | ||
2 | STRUCT_ITEM@[0; 7) | ||
3 | STRUCT_KW@[0; 6) | ||
4 | WHITESPACE@[6; 7) | ||
5 | ERROR@[7; 9) | ||
6 | IDENT@[7; 8) | ||
7 | WHITESPACE@[8; 9) | ||
8 | ERROR@[9; 12) | ||
9 | L_CURLY@[9; 10) | ||
10 | WHITESPACE@[10; 12) | ||
11 | ERROR@[12; 13) | ||
12 | R_CURLY@[12; 13) | ||