aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--src/parser/input.rs77
-rw-r--r--src/parser/mod.rs4
-rw-r--r--src/parser/parser.rs36
3 files changed, 89 insertions, 28 deletions
diff --git a/src/parser/input.rs b/src/parser/input.rs
new file mode 100644
index 000000000..162b9ef5f
--- /dev/null
+++ b/src/parser/input.rs
@@ -0,0 +1,77 @@
1use {SyntaxKind, TextRange, TextUnit, Token};
2use syntax_kinds::EOF;
3use super::is_insignificant;
4
5use std::ops::{Add, AddAssign};
6
7pub(crate) struct ParserInput<'t> {
8 #[allow(unused)]
9 text: &'t str,
10 #[allow(unused)]
11 start_offsets: Vec<TextUnit>,
12 tokens: Vec<Token>, // non-whitespace tokens
13}
14
15impl<'t> ParserInput<'t> {
16 pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> ParserInput<'t> {
17 let mut tokens = Vec::new();
18 let mut start_offsets = Vec::new();
19 let mut len = TextUnit::new(0);
20 for &token in raw_tokens.iter() {
21 if !is_insignificant(token.kind) {
22 tokens.push(token);
23 start_offsets.push(len);
24 }
25 len += token.len;
26 }
27
28 ParserInput {
29 text,
30 start_offsets,
31 tokens,
32 }
33 }
34
35 pub fn kind(&self, pos: InputPosition) -> SyntaxKind {
36 let idx = pos.0 as usize;
37 if !(idx < self.tokens.len()) {
38 return EOF;
39 }
40 self.tokens[idx].kind
41 }
42
43 #[allow(unused)]
44 pub fn text(&self, pos: InputPosition) -> &'t str {
45 let idx = pos.0 as usize;
46 if !(idx < self.tokens.len()) {
47 return "";
48 }
49 let start_offset = self.start_offsets[idx];
50 let end_offset = self.tokens[idx].len;
51 let range = TextRange::from_to(start_offset, end_offset);
52 &self.text[range]
53 }
54}
55
56#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)]
57pub(crate) struct InputPosition(u32);
58
59impl InputPosition {
60 pub fn new() -> Self {
61 InputPosition(0)
62 }
63}
64
65impl Add<u32> for InputPosition {
66 type Output = InputPosition;
67
68 fn add(self, rhs: u32) -> InputPosition {
69 InputPosition(self.0 + rhs)
70 }
71}
72
73impl AddAssign<u32> for InputPosition {
74 fn add_assign(&mut self, rhs: u32) {
75 self.0 += rhs
76 }
77}
diff --git a/src/parser/mod.rs b/src/parser/mod.rs
index f17ffbf3a..49a69900f 100644
--- a/src/parser/mod.rs
+++ b/src/parser/mod.rs
@@ -4,6 +4,7 @@ use syntax_kinds::*;
4 4
5#[macro_use] 5#[macro_use]
6mod parser; 6mod parser;
7mod input;
7mod event; 8mod event;
8mod grammar; 9mod grammar;
9use self::event::Event; 10use self::event::Event;
@@ -11,7 +12,8 @@ use self::event::Event;
11/// Parse a sequence of tokens into the representative node tree 12/// Parse a sequence of tokens into the representative node tree
12pub fn parse(text: String, tokens: &[Token]) -> File { 13pub fn parse(text: String, tokens: &[Token]) -> File {
13 let events = { 14 let events = {
14 let mut parser = parser::Parser::new(&text, tokens); 15 let input = input::ParserInput::new(&text, tokens);
16 let mut parser = parser::Parser::new(&input);
15 grammar::file(&mut parser); 17 grammar::file(&mut parser);
16 parser.into_events() 18 parser.into_events()
17 }; 19 };
diff --git a/src/parser/parser.rs b/src/parser/parser.rs
index 3f4c8a07d..bb775c4a5 100644
--- a/src/parser/parser.rs
+++ b/src/parser/parser.rs
@@ -1,7 +1,6 @@
1use {SyntaxKind, TextUnit, Token};
2use super::Event; 1use super::Event;
3use super::is_insignificant; 2use super::input::{InputPosition, ParserInput};
4use SyntaxKind::{EOF, TOMBSTONE}; 3use SyntaxKind::{self, EOF, TOMBSTONE};
5 4
6pub(crate) struct Marker { 5pub(crate) struct Marker {
7 pos: u32, 6 pos: u32,
@@ -98,35 +97,18 @@ macro_rules! token_set {
98} 97}
99 98
100pub(crate) struct Parser<'t> { 99pub(crate) struct Parser<'t> {
101 #[allow(unused)] 100 inp: &'t ParserInput<'t>,
102 text: &'t str,
103 #[allow(unused)]
104 start_offsets: Vec<TextUnit>,
105 tokens: Vec<Token>, // non-whitespace tokens
106 101
107 pos: usize, 102 pos: InputPosition,
108 events: Vec<Event>, 103 events: Vec<Event>,
109} 104}
110 105
111impl<'t> Parser<'t> { 106impl<'t> Parser<'t> {
112 pub(crate) fn new(text: &'t str, raw_tokens: &'t [Token]) -> Parser<'t> { 107 pub(crate) fn new(inp: &'t ParserInput<'t>) -> Parser<'t> {
113 let mut tokens = Vec::new();
114 let mut start_offsets = Vec::new();
115 let mut len = TextUnit::new(0);
116 for &token in raw_tokens.iter() {
117 if !is_insignificant(token.kind) {
118 tokens.push(token);
119 start_offsets.push(len);
120 }
121 len += token.len;
122 }
123
124 Parser { 108 Parser {
125 text, 109 inp,
126 start_offsets,
127 tokens,
128 110
129 pos: 0, 111 pos: InputPosition::new(),
130 events: Vec::new(), 112 events: Vec::new(),
131 } 113 }
132 } 114 }
@@ -163,8 +145,8 @@ impl<'t> Parser<'t> {
163 }); 145 });
164 } 146 }
165 147
166 pub(crate) fn nth(&self, n: usize) -> SyntaxKind { 148 pub(crate) fn nth(&self, n: u32) -> SyntaxKind {
167 self.tokens.get(self.pos + n).map(|t| t.kind).unwrap_or(EOF) 149 self.inp.kind(self.pos + n)
168 } 150 }
169 151
170 pub(crate) fn current(&self) -> SyntaxKind { 152 pub(crate) fn current(&self) -> SyntaxKind {