aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/lib.rs45
-rw-r--r--src/parser/event.rs10
-rw-r--r--src/parser/mod.rs14
-rw-r--r--src/tree/file_builder.rs150
-rw-r--r--src/tree/mod.rs221
-rw-r--r--src/yellow/green.rs7
-rw-r--r--src/yellow/mod.rs1
-rw-r--r--src/yellow/red.rs4
-rw-r--r--src/yellow/syntax.rs1
9 files changed, 15 insertions, 438 deletions
diff --git a/src/lib.rs b/src/lib.rs
index cf2e97024..619ad62e5 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -26,53 +26,22 @@ mod yellow;
26 26
27pub mod syntax_kinds; 27pub mod syntax_kinds;
28pub use text_unit::{TextRange, TextUnit}; 28pub use text_unit::{TextRange, TextUnit};
29pub use tree::{File, Node, SyntaxKind, Token}; 29pub use tree::{SyntaxKind, Token};
30pub(crate) use tree::{ErrorMsg, FileBuilder, Sink, GreenBuilder}; 30pub(crate) use tree::{Sink, GreenBuilder};
31pub use lexer::{next_token, tokenize}; 31pub use lexer::{next_token, tokenize};
32pub use yellow::SyntaxNode; 32pub use yellow::SyntaxNode;
33pub(crate) use yellow::SError; 33pub(crate) use yellow::SError;
34pub use parser::{parse, parse_green}; 34pub use parser::{parse_green};
35 35
36/// Utilities for simple uses of the parser. 36/// Utilities for simple uses of the parser.
37pub mod utils { 37pub mod utils {
38 use std::fmt::Write; 38 use std::fmt::Write;
39 39
40 use {File, Node, SyntaxNode}; 40 use {SyntaxNode};
41 use std::collections::BTreeSet; 41 use std::collections::BTreeSet;
42 use SError; 42 use SError;
43 43
44 /// Parse a file and create a string representation of the resulting parse tree. 44 /// Parse a file and create a string representation of the resulting parse tree.
45 pub fn dump_tree(file: &File) -> String {
46 let mut result = String::new();
47 go(file.root(), &mut result, 0);
48 return result;
49
50 fn go(node: Node, buff: &mut String, level: usize) {
51 buff.push_str(&String::from(" ").repeat(level));
52 write!(buff, "{:?}\n", node).unwrap();
53 let my_errors = node.errors().filter(|e| e.after_child().is_none());
54 let parent_errors = node.parent()
55 .into_iter()
56 .flat_map(|n| n.errors())
57 .filter(|e| e.after_child() == Some(node));
58
59 for err in my_errors {
60 buff.push_str(&String::from(" ").repeat(level));
61 write!(buff, "err: `{}`\n", err.message()).unwrap();
62 }
63
64 for child in node.children() {
65 go(child, buff, level + 1)
66 }
67
68 for err in parent_errors {
69 buff.push_str(&String::from(" ").repeat(level));
70 write!(buff, "err: `{}`\n", err.message()).unwrap();
71 }
72 }
73 }
74
75 /// Parse a file and create a string representation of the resulting parse tree.
76 pub fn dump_tree_green(syntax: &SyntaxNode) -> String { 45 pub fn dump_tree_green(syntax: &SyntaxNode) -> String {
77 let mut errors: BTreeSet<_> = syntax.root.errors.iter().cloned().collect(); 46 let mut errors: BTreeSet<_> = syntax.root.errors.iter().cloned().collect();
78 let mut result = String::new(); 47 let mut result = String::new();
@@ -82,12 +51,6 @@ pub mod utils {
82 fn go(node: &SyntaxNode, buff: &mut String, level: usize, errors: &mut BTreeSet<SError>) { 51 fn go(node: &SyntaxNode, buff: &mut String, level: usize, errors: &mut BTreeSet<SError>) {
83 buff.push_str(&String::from(" ").repeat(level)); 52 buff.push_str(&String::from(" ").repeat(level));
84 write!(buff, "{:?}\n", node).unwrap(); 53 write!(buff, "{:?}\n", node).unwrap();
85// let my_errors = node.errors().filter(|e| e.after_child().is_none());
86// let parent_errors = node.parent()
87// .into_iter()
88// .flat_map(|n| n.errors())
89// .filter(|e| e.after_child() == Some(node));
90//
91 let my_errors: Vec<_> = errors.iter().filter(|e| e.offset == node.range().start()) 54 let my_errors: Vec<_> = errors.iter().filter(|e| e.offset == node.range().start())
92 .cloned().collect(); 55 .cloned().collect();
93 for err in my_errors { 56 for err in my_errors {
diff --git a/src/parser/event.rs b/src/parser/event.rs
index 0fbfaeb9f..83039c664 100644
--- a/src/parser/event.rs
+++ b/src/parser/event.rs
@@ -1,5 +1,5 @@
1use { 1use {
2 ErrorMsg, File, FileBuilder, Sink, SyntaxKind, Token, GreenBuilder, 2 Sink, SyntaxKind, Token,
3 syntax_kinds::TOMBSTONE, 3 syntax_kinds::TOMBSTONE,
4}; 4};
5use super::is_insignificant; 5use super::is_insignificant;
@@ -67,12 +67,6 @@ pub(crate) enum Event {
67 }, 67 },
68} 68}
69 69
70pub(super) fn to_file(text: String, tokens: &[Token], events: Vec<Event>) -> File {
71 let mut builder = FileBuilder::new(text);
72 process(&mut builder, tokens, events);
73 builder.finish()
74}
75
76pub(super) fn process(builder: &mut Sink, tokens: &[Token], events: Vec<Event>) { 70pub(super) fn process(builder: &mut Sink, tokens: &[Token], events: Vec<Event>) {
77 let mut idx = 0; 71 let mut idx = 0;
78 72
@@ -147,7 +141,7 @@ pub(super) fn process(builder: &mut Sink, tokens: &[Token], events: Vec<Event>)
147 } 141 }
148 builder.leaf(kind, len); 142 builder.leaf(kind, len);
149 } 143 }
150 &Event::Error { ref msg } => builder.error(ErrorMsg { msg: msg.clone() }), 144 &Event::Error { ref msg } => builder.error(msg.clone()),
151 } 145 }
152 } 146 }
153} 147}
diff --git a/src/parser/mod.rs b/src/parser/mod.rs
index 26fbb6e3d..b7d5e5832 100644
--- a/src/parser/mod.rs
+++ b/src/parser/mod.rs
@@ -7,7 +7,7 @@ mod grammar;
7 7
8use std::sync::Arc; 8use std::sync::Arc;
9use { 9use {
10 File, SyntaxKind, Token, 10 Token,
11 yellow::SyntaxNode, 11 yellow::SyntaxNode,
12 syntax_kinds::* 12 syntax_kinds::*
13}; 13};
@@ -16,18 +16,6 @@ use parser::event::process;
16 16
17 17
18/// Parse a sequence of tokens into the representative node tree 18/// Parse a sequence of tokens into the representative node tree
19pub fn parse(text: String, tokens: &[Token]) -> File {
20 let events = {
21 let input = input::ParserInput::new(&text, tokens);
22 let parser_impl = parser::imp::ParserImpl::new(&input);
23 let mut parser = parser::Parser(parser_impl);
24 grammar::file(&mut parser);
25 parser.0.into_events()
26 };
27 event::to_file(text, tokens, events)
28}
29
30/// Parse a sequence of tokens into the representative node tree
31pub fn parse_green(text: String, tokens: &[Token]) -> SyntaxNode { 19pub fn parse_green(text: String, tokens: &[Token]) -> SyntaxNode {
32 let events = { 20 let events = {
33 let input = input::ParserInput::new(&text, tokens); 21 let input = input::ParserInput::new(&text, tokens);
diff --git a/src/tree/file_builder.rs b/src/tree/file_builder.rs
index 4983006cd..f5d1751f9 100644
--- a/src/tree/file_builder.rs
+++ b/src/tree/file_builder.rs
@@ -12,157 +12,13 @@ use {
12 SyntaxKind, TextRange, TextUnit, 12 SyntaxKind, TextRange, TextUnit,
13 yellow::GreenNode 13 yellow::GreenNode
14}; 14};
15use super::{File, NodeData, NodeIdx, SyntaxErrorData};
16use SError; 15use SError;
17 16
18pub(crate) trait Sink { 17pub(crate) trait Sink {
19 fn leaf(&mut self, kind: SyntaxKind, len: TextUnit); 18 fn leaf(&mut self, kind: SyntaxKind, len: TextUnit);
20 fn start_internal(&mut self, kind: SyntaxKind); 19 fn start_internal(&mut self, kind: SyntaxKind);
21 fn finish_internal(&mut self); 20 fn finish_internal(&mut self);
22 fn error(&mut self, err: ErrorMsg); 21 fn error(&mut self, err: String);
23}
24
25#[derive(Debug)]
26pub(crate) struct FileBuilder {
27 text: String,
28 nodes: Vec<NodeData>,
29 errors: Vec<SyntaxErrorData>,
30 in_progress: Vec<(NodeIdx, Option<NodeIdx>)>,
31 // (parent, last_child)
32 pos: TextUnit,
33}
34
35impl Sink for FileBuilder {
36 fn leaf(&mut self, kind: SyntaxKind, len: TextUnit) {
37 let leaf = NodeData {
38 kind,
39 range: TextRange::offset_len(self.pos, len),
40 parent: None,
41 first_child: None,
42 next_sibling: None,
43 };
44 self.pos += len;
45 let id = self.push_child(leaf);
46 self.add_len(id);
47 }
48
49 fn start_internal(&mut self, kind: SyntaxKind) {
50 let node = NodeData {
51 kind,
52 range: TextRange::offset_len(self.pos, 0.into()),
53 parent: None,
54 first_child: None,
55 next_sibling: None,
56 };
57 let id = if self.in_progress.is_empty() {
58 self.new_node(node)
59 } else {
60 self.push_child(node)
61 };
62 self.in_progress.push((id, None))
63 }
64
65 fn finish_internal(&mut self) {
66 let (id, _) = self.in_progress
67 .pop()
68 .expect("trying to complete a node, but there are no in-progress nodes");
69 if !self.in_progress.is_empty() {
70 self.add_len(id);
71 }
72 }
73
74 fn error(&mut self, err: ErrorMsg) {
75 let &(node, after_child) = self.in_progress.last().unwrap();
76 self.errors.push(SyntaxErrorData {
77 node,
78 message: err.msg,
79 after_child,
80 })
81 }
82}
83
84impl FileBuilder {
85 pub fn new(text: String) -> FileBuilder {
86 FileBuilder {
87 text,
88 nodes: Vec::new(),
89 errors: Vec::new(),
90 in_progress: Vec::new(),
91 pos: 0.into(),
92 }
93 }
94
95 pub fn finish(self) -> File {
96 assert!(
97 self.in_progress.is_empty(),
98 "some nodes in FileBuilder are unfinished: {:?}",
99 self.in_progress
100 .iter()
101 .map(|&(idx, _)| self.nodes[idx].kind)
102 .collect::<Vec<_>>()
103 );
104 assert_eq!(
105 self.pos,
106 (self.text.len() as u32).into(),
107 "nodes in FileBuilder do not cover the whole file"
108 );
109 File {
110 text: self.text,
111 nodes: self.nodes,
112 errors: self.errors,
113 }
114 }
115
116 fn new_node(&mut self, data: NodeData) -> NodeIdx {
117 let id = NodeIdx(self.nodes.len() as u32);
118 self.nodes.push(data);
119 id
120 }
121
122 fn push_child(&mut self, mut child: NodeData) -> NodeIdx {
123 child.parent = Some(self.current_id());
124 let id = self.new_node(child);
125 {
126 let (parent, sibling) = *self.in_progress.last().unwrap();
127 let slot = if let Some(idx) = sibling {
128 &mut self.nodes[idx].next_sibling
129 } else {
130 &mut self.nodes[parent].first_child
131 };
132 fill(slot, id);
133 }
134 self.in_progress.last_mut().unwrap().1 = Some(id);
135 id
136 }
137
138 fn add_len(&mut self, child: NodeIdx) {
139 let range = self.nodes[child].range;
140 grow(&mut self.current_parent().range, range);
141 }
142
143 fn current_id(&self) -> NodeIdx {
144 self.in_progress.last().unwrap().0
145 }
146
147 fn current_parent(&mut self) -> &mut NodeData {
148 let idx = self.current_id();
149 &mut self.nodes[idx]
150 }
151}
152
153fn fill<T>(slot: &mut Option<T>, value: T) {
154 assert!(slot.is_none());
155 *slot = Some(value);
156}
157
158fn grow(left: &mut TextRange, right: TextRange) {
159 assert_eq!(left.end(), right.start());
160 *left = TextRange::from_to(left.start(), right.end())
161}
162
163#[derive(Default)]
164pub(crate) struct ErrorMsg {
165 pub(crate) msg: String,
166} 22}
167 23
168pub(crate) struct GreenBuilder { 24pub(crate) struct GreenBuilder {
@@ -216,8 +72,8 @@ impl Sink for GreenBuilder {
216 } 72 }
217 } 73 }
218 74
219 fn error(&mut self, err: ErrorMsg) { 75 fn error(&mut self, message: String) {
220 self.errors.push(SError { message: err.msg, offset: self.pos }) 76 self.errors.push(SError { message, offset: self.pos })
221 } 77 }
222} 78}
223impl SyntaxKind { 79impl SyntaxKind {
diff --git a/src/tree/mod.rs b/src/tree/mod.rs
index 7abe17592..efba82825 100644
--- a/src/tree/mod.rs
+++ b/src/tree/mod.rs
@@ -1,8 +1,8 @@
1mod file_builder; 1mod file_builder;
2 2
3use ::{TextRange, TextUnit}; 3use ::{TextUnit};
4use std::{fmt, cmp}; 4use std::{fmt};
5pub(crate) use self::file_builder::{ErrorMsg, FileBuilder, Sink, GreenBuilder}; 5pub(crate) use self::file_builder::{Sink, GreenBuilder};
6 6
7pub use syntax_kinds::SyntaxKind; 7pub use syntax_kinds::SyntaxKind;
8 8
@@ -25,218 +25,3 @@ pub struct Token {
25 /// The length of the token. 25 /// The length of the token.
26 pub len: TextUnit, 26 pub len: TextUnit,
27} 27}
28
29/// The contents of a Rust source file.
30#[derive(Debug)]
31pub struct File {
32 text: String,
33 nodes: Vec<NodeData>,
34 errors: Vec<SyntaxErrorData>,
35}
36
37impl File {
38 /// The root node of this source file.
39 pub fn root<'f>(&'f self) -> Node<'f> {
40 assert!(!self.nodes.is_empty());
41 Node {
42 file: self,
43 idx: NodeIdx(0),
44 }
45 }
46}
47
48/// A reference to a token in a Rust source file.
49#[derive(Clone, Copy)]
50pub struct Node<'f> {
51 file: &'f File,
52 idx: NodeIdx,
53}
54
55impl<'f> Node<'f> {
56 /// The kind of the token at this node.
57 pub fn kind(&self) -> SyntaxKind {
58 self.data().kind
59 }
60
61 /// The text range covered by the token at this node.
62 pub fn range(&self) -> TextRange {
63 self.data().range
64 }
65
66 /// The text at this node.
67 pub fn text(&self) -> &'f str {
68 &self.file.text.as_str()[self.range()]
69 }
70
71 /// The parent node to this node.
72 pub fn parent(&self) -> Option<Node<'f>> {
73 self.as_node(self.data().parent)
74 }
75
76 /// The children nodes of this node.
77 pub fn children(&self) -> Children<'f> {
78 Children {
79 next: self.as_node(self.data().first_child),
80 }
81 }
82
83 /// Any errors contained in this node.
84 pub fn errors(&self) -> SyntaxErrors<'f> {
85 let pos = self.file.errors.iter().position(|e| e.node == self.idx);
86 let next = pos.map(|i| ErrorIdx(i as u32)).map(|idx| SyntaxError {
87 file: self.file,
88 idx,
89 });
90 SyntaxErrors { next }
91 }
92
93 fn data(&self) -> &'f NodeData {
94 &self.file.nodes[self.idx]
95 }
96
97 fn as_node(&self, idx: Option<NodeIdx>) -> Option<Node<'f>> {
98 idx.map(|idx| Node {
99 file: self.file,
100 idx,
101 })
102 }
103}
104
105impl<'f> fmt::Debug for Node<'f> {
106 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
107 write!(fmt, "{:?}@{:?}", self.kind(), self.range())?;
108 if has_short_text(self.kind()) {
109 write!(fmt, " \"{}\"", self.text())?;
110 }
111 Ok(())
112 }
113}
114
115fn has_short_text(kind: SyntaxKind) -> bool {
116 use syntax_kinds::*;
117 match kind {
118 IDENT | LIFETIME => true,
119 _ => false,
120 }
121}
122
123impl<'f> cmp::PartialEq<Node<'f>> for Node<'f> {
124 fn eq(&self, other: &Node<'f>) -> bool {
125 self.idx == other.idx && ::std::ptr::eq(self.file, other.file)
126 }
127}
128
129impl<'f> cmp::Eq for Node<'f> {}
130
131#[derive(Clone, Copy, Debug)]
132pub struct SyntaxError<'f> {
133 file: &'f File,
134 idx: ErrorIdx,
135}
136
137impl<'f> SyntaxError<'f> {
138 pub fn message(&self) -> &'f str {
139 self.data().message.as_str()
140 }
141
142 pub fn after_child(&self) -> Option<Node<'f>> {
143 let idx = self.data().after_child?;
144 Some(Node {
145 file: self.file,
146 idx,
147 })
148 }
149
150 fn data(&self) -> &'f SyntaxErrorData {
151 &self.file.errors[self.idx]
152 }
153
154 fn next(&self) -> Option<SyntaxError<'f>> {
155 let next_idx = self.idx.0 + 1;
156 if !((next_idx as usize) < self.file.errors.len()) {
157 return None;
158 }
159 let result = SyntaxError {
160 file: self.file,
161 idx: ErrorIdx(next_idx),
162 };
163 if result.data().node != self.data().node {
164 return None;
165 }
166 Some(result)
167 }
168}
169
170#[derive(Debug)]
171pub struct Children<'f> {
172 next: Option<Node<'f>>,
173}
174
175impl<'f> Iterator for Children<'f> {
176 type Item = Node<'f>;
177
178 fn next(&mut self) -> Option<Node<'f>> {
179 let next = self.next;
180 self.next = next.and_then(|node| node.as_node(node.data().next_sibling));
181 next
182 }
183}
184
185#[derive(Debug)]
186pub struct SyntaxErrors<'f> {
187 next: Option<SyntaxError<'f>>,
188}
189
190impl<'f> Iterator for SyntaxErrors<'f> {
191 type Item = SyntaxError<'f>;
192
193 fn next(&mut self) -> Option<SyntaxError<'f>> {
194 let next = self.next;
195 self.next = next.as_ref().and_then(SyntaxError::next);
196 next
197 }
198}
199
200#[derive(Clone, Copy, Debug, PartialEq, Eq)]
201struct NodeIdx(u32);
202
203#[derive(Debug)]
204struct NodeData {
205 kind: SyntaxKind,
206 range: TextRange,
207 parent: Option<NodeIdx>,
208 first_child: Option<NodeIdx>,
209 next_sibling: Option<NodeIdx>,
210}
211
212impl ::std::ops::Index<NodeIdx> for Vec<NodeData> {
213 type Output = NodeData;
214
215 fn index(&self, NodeIdx(idx): NodeIdx) -> &NodeData {
216 &self[idx as usize]
217 }
218}
219
220impl ::std::ops::IndexMut<NodeIdx> for Vec<NodeData> {
221 fn index_mut(&mut self, NodeIdx(idx): NodeIdx) -> &mut NodeData {
222 &mut self[idx as usize]
223 }
224}
225
226#[derive(Clone, Copy, Debug)]
227struct ErrorIdx(u32);
228
229#[derive(Debug)]
230struct SyntaxErrorData {
231 node: NodeIdx,
232 message: String,
233 after_child: Option<NodeIdx>,
234}
235
236impl ::std::ops::Index<ErrorIdx> for Vec<SyntaxErrorData> {
237 type Output = SyntaxErrorData;
238
239 fn index(&self, ErrorIdx(idx): ErrorIdx) -> &SyntaxErrorData {
240 &self[idx as usize]
241 }
242}
diff --git a/src/yellow/green.rs b/src/yellow/green.rs
index ede23b719..dafe1bb22 100644
--- a/src/yellow/green.rs
+++ b/src/yellow/green.rs
@@ -116,13 +116,6 @@ impl GreenNode {
116 GreenNodeData::Branch(_) => false 116 GreenNodeData::Branch(_) => false
117 } 117 }
118 } 118 }
119
120 pub(crate) fn leaf_text(&self) -> &str {
121 match &self.data {
122 GreenNodeData::Leaf(l) => l.text.as_str(),
123 GreenNodeData::Branch(_) => panic!("not a leaf")
124 }
125 }
126} 119}
127 120
128#[derive(Debug)] 121#[derive(Debug)]
diff --git a/src/yellow/mod.rs b/src/yellow/mod.rs
index 236328a7f..88d88e226 100644
--- a/src/yellow/mod.rs
+++ b/src/yellow/mod.rs
@@ -4,7 +4,6 @@ mod syntax;
4 4
5use std::{ 5use std::{
6 sync::{Arc, Weak}, 6 sync::{Arc, Weak},
7 ops::Deref,
8 mem 7 mem
9}; 8};
10pub(crate) use self::{ 9pub(crate) use self::{
diff --git a/src/yellow/red.rs b/src/yellow/red.rs
index feba99faa..71212a081 100644
--- a/src/yellow/red.rs
+++ b/src/yellow/red.rs
@@ -1,6 +1,6 @@
1use std::sync::{Arc, Weak, RwLock}; 1use std::sync::{Arc, RwLock};
2use { 2use {
3 TextUnit, SyntaxKind, TextRange, 3 TextUnit,
4 yellow::{Ptr, GreenNode, TextLen} 4 yellow::{Ptr, GreenNode, TextLen}
5}; 5};
6 6
diff --git a/src/yellow/syntax.rs b/src/yellow/syntax.rs
index 0c9ffeb14..7b1a05cd9 100644
--- a/src/yellow/syntax.rs
+++ b/src/yellow/syntax.rs
@@ -7,7 +7,6 @@ use {
7 TextRange, TextUnit, SyntaxKind, 7 TextRange, TextUnit, SyntaxKind,
8 yellow::{Ptr, RedNode, GreenNode, TextLen}, 8 yellow::{Ptr, RedNode, GreenNode, TextLen},
9}; 9};
10use yellow::green::GreenTrivia;
11 10
12#[derive(Clone)] 11#[derive(Clone)]
13pub struct SyntaxNode { 12pub struct SyntaxNode {