aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorAleksey Kladov <[email protected]>2018-01-07 18:09:05 +0000
committerAleksey Kladov <[email protected]>2018-01-07 18:09:05 +0000
commit5562931e4f3c6c57e9122c3ce34d941fb1ff6e5b (patch)
tree0e87cba8e91d4bf96d752ada65b4d4a73e4e0ac9 /src
parentfc4d6cc298fe901d2bf92a30a3efd67233c67a3a (diff)
Introduce EOF token
Diffstat (limited to 'src')
-rw-r--r--src/parser/event_parser/grammar.rs23
-rw-r--r--src/parser/event_parser/parser.rs35
-rw-r--r--src/tree/mod.rs218
3 files changed, 136 insertions, 140 deletions
diff --git a/src/parser/event_parser/grammar.rs b/src/parser/event_parser/grammar.rs
index f676a183c..64c6718cb 100644
--- a/src/parser/event_parser/grammar.rs
+++ b/src/parser/event_parser/grammar.rs
@@ -1,5 +1,6 @@
1use super::parser::Parser; 1use super::parser::Parser;
2use {SyntaxKind}; 2use {SyntaxKind};
3use tree::EOF;
3use syntax_kinds::*; 4use syntax_kinds::*;
4 5
5// Items // 6// Items //
@@ -18,11 +19,7 @@ pub(crate) fn file(p: &mut Parser) {
18} 19}
19 20
20fn item_first(p: &Parser) -> bool { 21fn item_first(p: &Parser) -> bool {
21 let current = match p.current() { 22 match p.current() {
22 Some(c) => c,
23 None => return false,
24 };
25 match current {
26 STRUCT_KW | FN_KW => true, 23 STRUCT_KW | FN_KW => true,
27 _ => false, 24 _ => false,
28 } 25 }
@@ -79,7 +76,7 @@ fn visibility(_: &mut Parser) {
79// Error recovery and high-order utils // 76// Error recovery and high-order utils //
80 77
81fn node_if<F: FnOnce(&mut Parser)>(p: &mut Parser, first: SyntaxKind, node_kind: SyntaxKind, rest: F) -> bool { 78fn node_if<F: FnOnce(&mut Parser)>(p: &mut Parser, first: SyntaxKind, node_kind: SyntaxKind, rest: F) -> bool {
82 p.current_is(first) && { node(p, node_kind, |p| { p.bump(); rest(p); }); true } 79 p.current() == first && { node(p, node_kind, |p| { p.bump(); rest(p); }); true }
83} 80}
84 81
85fn node<F: FnOnce(&mut Parser)>(p: &mut Parser, node_kind: SyntaxKind, rest: F) { 82fn node<F: FnOnce(&mut Parser)>(p: &mut Parser, node_kind: SyntaxKind, rest: F) {
@@ -95,7 +92,7 @@ fn many<F: Fn(&mut Parser) -> bool>(p: &mut Parser, f: F) {
95fn comma_list<F: Fn(&mut Parser) -> bool>(p: &mut Parser, f: F) { 92fn comma_list<F: Fn(&mut Parser) -> bool>(p: &mut Parser, f: F) {
96 many(p, |p| { 93 many(p, |p| {
97 f(p); 94 f(p);
98 if p.is_eof() { 95 if p.current() == EOF {
99 false 96 false
100 } else { 97 } else {
101 p.expect(COMMA); 98 p.expect(COMMA);
@@ -119,7 +116,7 @@ where
119 f(p); 116 f(p);
120 return true; 117 return true;
121 } 118 }
122 if p.is_eof() { 119 if p.current() == EOF {
123 if skipped { 120 if skipped {
124 p.finish(); 121 p.finish();
125 } 122 }
@@ -131,18 +128,14 @@ where
131 .message(message) 128 .message(message)
132 .emit(); 129 .emit();
133 } 130 }
134 p.bump().unwrap(); 131 p.bump();
135 skipped = true; 132 skipped = true;
136 } 133 }
137} 134}
138 135
139impl<'p> Parser<'p> { 136impl<'p> Parser<'p> {
140 fn current_is(&self, kind: SyntaxKind) -> bool {
141 self.current() == Some(kind)
142 }
143
144 pub(crate) fn expect(&mut self, kind: SyntaxKind) -> bool { 137 pub(crate) fn expect(&mut self, kind: SyntaxKind) -> bool {
145 if self.current_is(kind) { 138 if self.current() == kind {
146 self.bump(); 139 self.bump();
147 true 140 true
148 } else { 141 } else {
@@ -154,7 +147,7 @@ impl<'p> Parser<'p> {
154 } 147 }
155 148
156 fn optional(&mut self, kind: SyntaxKind) { 149 fn optional(&mut self, kind: SyntaxKind) {
157 if self.current_is(kind) { 150 if self.current() == kind {
158 self.bump(); 151 self.bump();
159 } 152 }
160 } 153 }
diff --git a/src/parser/event_parser/parser.rs b/src/parser/event_parser/parser.rs
index f0d1d358b..bec9dbab4 100644
--- a/src/parser/event_parser/parser.rs
+++ b/src/parser/event_parser/parser.rs
@@ -2,8 +2,7 @@ use {Token, SyntaxKind, TextUnit};
2use super::{Event}; 2use super::{Event};
3use super::super::is_insignificant; 3use super::super::is_insignificant;
4use syntax_kinds::{L_CURLY, R_CURLY, ERROR}; 4use syntax_kinds::{L_CURLY, R_CURLY, ERROR};
5 5use tree::EOF;
6pub(crate) const EOF: SyntaxKind = SyntaxKind(10000);
7 6
8 7
9pub(crate) struct Parser<'t> { 8pub(crate) struct Parser<'t> {
@@ -46,19 +45,22 @@ impl<'t> Parser<'t> {
46 } 45 }
47 46
48 pub(crate) fn into_events(self) -> Vec<Event> { 47 pub(crate) fn into_events(self) -> Vec<Event> {
49 assert!(self.is_eof()); 48 assert!(self.curly_limit.is_none());
49 assert!(self.current() == EOF);
50 self.events 50 self.events
51 } 51 }
52 52
53 pub(crate) fn is_eof(&self) -> bool { 53 pub(crate) fn current(&self) -> SyntaxKind {
54 if self.pos == self.tokens.len() { 54 if self.pos == self.tokens.len() {
55 return true 55 return EOF;
56 } 56 }
57 let token = self.tokens[self.pos];
57 if let Some(limit) = self.curly_limit { 58 if let Some(limit) = self.curly_limit {
58 let token = self.tokens[self.pos]; 59 if limit == self.curly_level && token.kind == R_CURLY {
59 return limit == self.curly_level && token.kind == R_CURLY; 60 return EOF
61 }
60 } 62 }
61 false 63 token.kind
62 } 64 }
63 65
64 pub(crate) fn start(&mut self, kind: SyntaxKind) { 66 pub(crate) fn start(&mut self, kind: SyntaxKind) {
@@ -73,24 +75,17 @@ impl<'t> Parser<'t> {
73 ErrorBuilder::new(self) 75 ErrorBuilder::new(self)
74 } 76 }
75 77
76 pub(crate) fn current(&self) -> Option<SyntaxKind> { 78 pub(crate) fn bump(&mut self) -> SyntaxKind {
77 if self.is_eof() { 79 let kind = self.current();
78 return None;
79 }
80 let token = self.tokens[self.pos];
81 Some(token.kind)
82 }
83
84 pub(crate) fn bump(&mut self) -> Option<SyntaxKind> {
85 let kind = self.current()?;
86 match kind { 80 match kind {
87 L_CURLY => self.curly_level += 1, 81 L_CURLY => self.curly_level += 1,
88 R_CURLY => self.curly_level -= 1, 82 R_CURLY => self.curly_level -= 1,
83 EOF => return EOF,
89 _ => (), 84 _ => (),
90 } 85 }
91 self.pos += 1; 86 self.pos += 1;
92 self.event(Event::Token { kind, n_raw_tokens: 1 }); 87 self.event(Event::Token { kind, n_raw_tokens: 1 });
93 Some(kind) 88 kind
94 } 89 }
95 90
96 pub(crate) fn lookahead(&self, kinds: &[SyntaxKind]) -> bool { 91 pub(crate) fn lookahead(&self, kinds: &[SyntaxKind]) -> bool {
@@ -114,7 +109,7 @@ impl<'t> Parser<'t> {
114 if !self.expect(R_CURLY) { 109 if !self.expect(R_CURLY) {
115 self.start(ERROR); 110 self.start(ERROR);
116 while self.curly_level > old_level { 111 while self.curly_level > old_level {
117 if self.bump().is_none() { 112 if self.bump() == EOF {
118 break; 113 break;
119 } 114 }
120 } 115 }
diff --git a/src/tree/mod.rs b/src/tree/mod.rs
index 7f4d427ba..d8f843737 100644
--- a/src/tree/mod.rs
+++ b/src/tree/mod.rs
@@ -10,8 +10,16 @@ pub use self::file_builder::{FileBuilder, Sink};
10#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] 10#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
11pub struct SyntaxKind(pub(crate) u32); 11pub struct SyntaxKind(pub(crate) u32);
12 12
13pub(crate) const EOF: SyntaxKind = SyntaxKind(10000);
14pub(crate) const EOF_INFO: SyntaxInfo = SyntaxInfo {
15 name: "EOF"
16};
17
13impl SyntaxKind { 18impl SyntaxKind {
14 fn info(self) -> &'static SyntaxInfo { 19 fn info(self) -> &'static SyntaxInfo {
20 if self == EOF {
21 return &EOF_INFO;
22 }
15 syntax_info(self) 23 syntax_info(self)
16 } 24 }
17} 25}
@@ -35,72 +43,72 @@ pub struct Token {
35} 43}
36 44
37pub struct File { 45pub struct File {
38 text: String, 46 text: String,
39 nodes: Vec<NodeData>, 47 nodes: Vec<NodeData>,
40 errors: Vec<SyntaxErrorData>, 48 errors: Vec<SyntaxErrorData>,
41} 49}
42 50
43impl File { 51impl File {
44 pub fn root<'f>(&'f self) -> Node<'f> { 52 pub fn root<'f>(&'f self) -> Node<'f> {
45 assert!(!self.nodes.is_empty()); 53 assert!(!self.nodes.is_empty());
46 Node { file: self, idx: NodeIdx(0) } 54 Node { file: self, idx: NodeIdx(0) }
47 } 55 }
48} 56}
49 57
50#[derive(Clone, Copy)] 58#[derive(Clone, Copy)]
51pub struct Node<'f> { 59pub struct Node<'f> {
52 file: &'f File, 60 file: &'f File,
53 idx: NodeIdx, 61 idx: NodeIdx,
54} 62}
55 63
56impl<'f> Node<'f> { 64impl<'f> Node<'f> {
57 pub fn kind(&self) -> SyntaxKind { 65 pub fn kind(&self) -> SyntaxKind {
58 self.data().kind 66 self.data().kind
59 } 67 }
60 68
61 pub fn range(&self) -> TextRange { 69 pub fn range(&self) -> TextRange {
62 self.data().range 70 self.data().range
63 } 71 }
64 72
65 pub fn text(&self) -> &'f str { 73 pub fn text(&self) -> &'f str {
66 &self.file.text.as_str()[self.range()] 74 &self.file.text.as_str()[self.range()]
67 } 75 }
68 76
69 pub fn parent(&self) -> Option<Node<'f>> { 77 pub fn parent(&self) -> Option<Node<'f>> {
70 self.as_node(self.data().parent) 78 self.as_node(self.data().parent)
71 } 79 }
72 80
73 pub fn children(&self) -> Children<'f> { 81 pub fn children(&self) -> Children<'f> {
74 Children { next: self.as_node(self.data().first_child) } 82 Children { next: self.as_node(self.data().first_child) }
75 } 83 }
76 84
77 pub fn errors(&self) -> SyntaxErrors<'f> { 85 pub fn errors(&self) -> SyntaxErrors<'f> {
78 let pos = self.file.errors.iter().position(|e| e.node == self.idx); 86 let pos = self.file.errors.iter().position(|e| e.node == self.idx);
79 let next = pos 87 let next = pos
80 .map(|i| ErrorIdx(i as u32)) 88 .map(|i| ErrorIdx(i as u32))
81 .map(|idx| SyntaxError { file: self.file, idx }); 89 .map(|idx| SyntaxError { file: self.file, idx });
82 SyntaxErrors { next } 90 SyntaxErrors { next }
83 } 91 }
84 92
85 fn data(&self) -> &'f NodeData { 93 fn data(&self) -> &'f NodeData {
86 &self.file.nodes[self.idx] 94 &self.file.nodes[self.idx]
87 } 95 }
88 96
89 fn as_node(&self, idx: Option<NodeIdx>) -> Option<Node<'f>> { 97 fn as_node(&self, idx: Option<NodeIdx>) -> Option<Node<'f>> {
90 idx.map(|idx| Node { file: self.file, idx }) 98 idx.map(|idx| Node { file: self.file, idx })
91 } 99 }
92} 100}
93 101
94impl<'f> fmt::Debug for Node<'f> { 102impl<'f> fmt::Debug for Node<'f> {
95 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { 103 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
96 write!(fmt, "{:?}@{:?}", self.kind(), self.range()) 104 write!(fmt, "{:?}@{:?}", self.kind(), self.range())
97 } 105 }
98} 106}
99 107
100impl<'f> cmp::PartialEq<Node<'f>> for Node<'f> { 108impl<'f> cmp::PartialEq<Node<'f>> for Node<'f> {
101 fn eq(&self, other: &Node<'f>) -> bool { 109 fn eq(&self, other: &Node<'f>) -> bool {
102 self.idx == other.idx && ::std::ptr::eq(self.file, other.file) 110 self.idx == other.idx && ::std::ptr::eq(self.file, other.file)
103 } 111 }
104} 112}
105 113
106impl<'f> cmp::Eq for Node<'f> { 114impl<'f> cmp::Eq for Node<'f> {
@@ -108,66 +116,66 @@ impl<'f> cmp::Eq for Node<'f> {
108 116
109#[derive(Clone, Copy)] 117#[derive(Clone, Copy)]
110pub struct SyntaxError<'f> { 118pub struct SyntaxError<'f> {
111 file: &'f File, 119 file: &'f File,
112 idx: ErrorIdx, 120 idx: ErrorIdx,
113} 121}
114 122
115impl<'f> SyntaxError<'f> { 123impl<'f> SyntaxError<'f> {
116 pub fn message(&self) -> &'f str { 124 pub fn message(&self) -> &'f str {
117 self.data().message.as_str() 125 self.data().message.as_str()
118 } 126 }
119 127
120 pub fn after_child(&self) -> Option<Node<'f>> { 128 pub fn after_child(&self) -> Option<Node<'f>> {
121 let idx = self.data().after_child?; 129 let idx = self.data().after_child?;
122 Some(Node { file: self.file, idx }) 130 Some(Node { file: self.file, idx })
123 } 131 }
124 132
125 fn data(&self) -> &'f SyntaxErrorData { 133 fn data(&self) -> &'f SyntaxErrorData {
126 &self.file.errors[self.idx] 134 &self.file.errors[self.idx]
127 } 135 }
128 136
129 fn next(&self) -> Option<SyntaxError<'f>> { 137 fn next(&self) -> Option<SyntaxError<'f>> {
130 let next_idx = self.idx.0 + 1; 138 let next_idx = self.idx.0 + 1;
131 if !((next_idx as usize) < self.file.errors.len()) { 139 if !((next_idx as usize) < self.file.errors.len()) {
132 return None; 140 return None;
133 } 141 }
134 let result = SyntaxError { 142 let result = SyntaxError {
135 file: self.file, 143 file: self.file,
136 idx: ErrorIdx(next_idx) 144 idx: ErrorIdx(next_idx)
137 }; 145 };
138 if result.data().node != self.data().node { 146 if result.data().node != self.data().node {
139 return None; 147 return None;
140 } 148 }
141 Some(result) 149 Some(result)
142 } 150 }
143} 151}
144 152
145pub struct Children<'f> { 153pub struct Children<'f> {
146 next: Option<Node<'f>>, 154 next: Option<Node<'f>>,
147} 155}
148 156
149impl<'f> Iterator for Children<'f> { 157impl<'f> Iterator for Children<'f> {
150 type Item = Node<'f>; 158 type Item = Node<'f>;
151 159
152 fn next(&mut self) -> Option<Node<'f>> { 160 fn next(&mut self) -> Option<Node<'f>> {
153 let next = self.next; 161 let next = self.next;
154 self.next = next.and_then(|node| node.as_node(node.data().next_sibling)); 162 self.next = next.and_then(|node| node.as_node(node.data().next_sibling));
155 next 163 next
156 } 164 }
157} 165}
158 166
159pub struct SyntaxErrors<'f> { 167pub struct SyntaxErrors<'f> {
160 next: Option<SyntaxError<'f>>, 168 next: Option<SyntaxError<'f>>,
161} 169}
162 170
163impl<'f> Iterator for SyntaxErrors<'f> { 171impl<'f> Iterator for SyntaxErrors<'f> {
164 type Item = SyntaxError<'f>; 172 type Item = SyntaxError<'f>;
165 173
166 fn next(&mut self) -> Option<SyntaxError<'f>> { 174 fn next(&mut self) -> Option<SyntaxError<'f>> {
167 let next = self.next; 175 let next = self.next;
168 self.next = next.as_ref().and_then(SyntaxError::next); 176 self.next = next.as_ref().and_then(SyntaxError::next);
169 next 177 next
170 } 178 }
171} 179}
172 180
173 181
@@ -175,40 +183,40 @@ impl<'f> Iterator for SyntaxErrors<'f> {
175struct NodeIdx(u32); 183struct NodeIdx(u32);
176 184
177struct NodeData { 185struct NodeData {
178 kind: SyntaxKind, 186 kind: SyntaxKind,
179 range: TextRange, 187 range: TextRange,
180 parent: Option<NodeIdx>, 188 parent: Option<NodeIdx>,
181 first_child: Option<NodeIdx>, 189 first_child: Option<NodeIdx>,
182 next_sibling: Option<NodeIdx>, 190 next_sibling: Option<NodeIdx>,
183} 191}
184 192
185impl ::std::ops::Index<NodeIdx> for Vec<NodeData> { 193impl ::std::ops::Index<NodeIdx> for Vec<NodeData> {
186 type Output = NodeData; 194 type Output = NodeData;
187 195
188 fn index(&self, NodeIdx(idx): NodeIdx) -> &NodeData { 196 fn index(&self, NodeIdx(idx): NodeIdx) -> &NodeData {
189 &self[idx as usize] 197 &self[idx as usize]
190 } 198 }
191} 199}
192 200
193impl ::std::ops::IndexMut<NodeIdx> for Vec<NodeData> { 201impl ::std::ops::IndexMut<NodeIdx> for Vec<NodeData> {
194 fn index_mut(&mut self, NodeIdx(idx): NodeIdx) -> &mut NodeData { 202 fn index_mut(&mut self, NodeIdx(idx): NodeIdx) -> &mut NodeData {
195 &mut self[idx as usize] 203 &mut self[idx as usize]
196 } 204 }
197} 205}
198 206
199#[derive(Clone, Copy)] 207#[derive(Clone, Copy)]
200struct ErrorIdx(u32); 208struct ErrorIdx(u32);
201 209
202struct SyntaxErrorData { 210struct SyntaxErrorData {
203 node: NodeIdx, 211 node: NodeIdx,
204 message: String, 212 message: String,
205 after_child: Option<NodeIdx>, 213 after_child: Option<NodeIdx>,
206} 214}
207 215
208impl ::std::ops::Index<ErrorIdx> for Vec<SyntaxErrorData> { 216impl ::std::ops::Index<ErrorIdx> for Vec<SyntaxErrorData> {
209 type Output = SyntaxErrorData; 217 type Output = SyntaxErrorData;
210 218
211 fn index(&self, ErrorIdx(idx): ErrorIdx) -> &SyntaxErrorData { 219 fn index(&self, ErrorIdx(idx): ErrorIdx) -> &SyntaxErrorData {
212 &self[idx as usize] 220 &self[idx as usize]
213 } 221 }
214} 222}