diff options
author | bors[bot] <bors[bot]@users.noreply.github.com> | 2018-02-04 12:40:24 +0000 |
---|---|---|
committer | bors[bot] <bors[bot]@users.noreply.github.com> | 2018-02-04 12:40:24 +0000 |
commit | b86e87c8885895e5b3fcec021ee65003d64c4cbe (patch) | |
tree | 0495db2ec299fd7d617f84febffa5ea954f2d513 /src/parser | |
parent | aa36ad008eae28d1251a4bf276b1d13398fcf89f (diff) | |
parent | 852543212ba5c68b3428a80187087cc641de612c (diff) |
Merge #42
42: Extract parser input into a separate struct r=matklad a=matklad
bors r+
Diffstat (limited to 'src/parser')
-rw-r--r-- | src/parser/input.rs | 77 | ||||
-rw-r--r-- | src/parser/mod.rs | 4 | ||||
-rw-r--r-- | src/parser/parser.rs | 36 |
3 files changed, 89 insertions, 28 deletions
diff --git a/src/parser/input.rs b/src/parser/input.rs new file mode 100644 index 000000000..162b9ef5f --- /dev/null +++ b/src/parser/input.rs | |||
@@ -0,0 +1,77 @@ | |||
1 | use {SyntaxKind, TextRange, TextUnit, Token}; | ||
2 | use syntax_kinds::EOF; | ||
3 | use super::is_insignificant; | ||
4 | |||
5 | use std::ops::{Add, AddAssign}; | ||
6 | |||
7 | pub(crate) struct ParserInput<'t> { | ||
8 | #[allow(unused)] | ||
9 | text: &'t str, | ||
10 | #[allow(unused)] | ||
11 | start_offsets: Vec<TextUnit>, | ||
12 | tokens: Vec<Token>, // non-whitespace tokens | ||
13 | } | ||
14 | |||
15 | impl<'t> ParserInput<'t> { | ||
16 | pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> ParserInput<'t> { | ||
17 | let mut tokens = Vec::new(); | ||
18 | let mut start_offsets = Vec::new(); | ||
19 | let mut len = TextUnit::new(0); | ||
20 | for &token in raw_tokens.iter() { | ||
21 | if !is_insignificant(token.kind) { | ||
22 | tokens.push(token); | ||
23 | start_offsets.push(len); | ||
24 | } | ||
25 | len += token.len; | ||
26 | } | ||
27 | |||
28 | ParserInput { | ||
29 | text, | ||
30 | start_offsets, | ||
31 | tokens, | ||
32 | } | ||
33 | } | ||
34 | |||
35 | pub fn kind(&self, pos: InputPosition) -> SyntaxKind { | ||
36 | let idx = pos.0 as usize; | ||
37 | if !(idx < self.tokens.len()) { | ||
38 | return EOF; | ||
39 | } | ||
40 | self.tokens[idx].kind | ||
41 | } | ||
42 | |||
43 | #[allow(unused)] | ||
44 | pub fn text(&self, pos: InputPosition) -> &'t str { | ||
45 | let idx = pos.0 as usize; | ||
46 | if !(idx < self.tokens.len()) { | ||
47 | return ""; | ||
48 | } | ||
49 | let start_offset = self.start_offsets[idx]; | ||
50 | let end_offset = self.tokens[idx].len; | ||
51 | let range = TextRange::from_to(start_offset, end_offset); | ||
52 | &self.text[range] | ||
53 | } | ||
54 | } | ||
55 | |||
56 | #[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)] | ||
57 | pub(crate) struct InputPosition(u32); | ||
58 | |||
59 | impl InputPosition { | ||
60 | pub fn new() -> Self { | ||
61 | InputPosition(0) | ||
62 | } | ||
63 | } | ||
64 | |||
65 | impl Add<u32> for InputPosition { | ||
66 | type Output = InputPosition; | ||
67 | |||
68 | fn add(self, rhs: u32) -> InputPosition { | ||
69 | InputPosition(self.0 + rhs) | ||
70 | } | ||
71 | } | ||
72 | |||
73 | impl AddAssign<u32> for InputPosition { | ||
74 | fn add_assign(&mut self, rhs: u32) { | ||
75 | self.0 += rhs | ||
76 | } | ||
77 | } | ||
diff --git a/src/parser/mod.rs b/src/parser/mod.rs index f17ffbf3a..49a69900f 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs | |||
@@ -4,6 +4,7 @@ use syntax_kinds::*; | |||
4 | 4 | ||
5 | #[macro_use] | 5 | #[macro_use] |
6 | mod parser; | 6 | mod parser; |
7 | mod input; | ||
7 | mod event; | 8 | mod event; |
8 | mod grammar; | 9 | mod grammar; |
9 | use self::event::Event; | 10 | use self::event::Event; |
@@ -11,7 +12,8 @@ use self::event::Event; | |||
11 | /// Parse a sequence of tokens into the representative node tree | 12 | /// Parse a sequence of tokens into the representative node tree |
12 | pub fn parse(text: String, tokens: &[Token]) -> File { | 13 | pub fn parse(text: String, tokens: &[Token]) -> File { |
13 | let events = { | 14 | let events = { |
14 | let mut parser = parser::Parser::new(&text, tokens); | 15 | let input = input::ParserInput::new(&text, tokens); |
16 | let mut parser = parser::Parser::new(&input); | ||
15 | grammar::file(&mut parser); | 17 | grammar::file(&mut parser); |
16 | parser.into_events() | 18 | parser.into_events() |
17 | }; | 19 | }; |
diff --git a/src/parser/parser.rs b/src/parser/parser.rs index 3f4c8a07d..bb775c4a5 100644 --- a/src/parser/parser.rs +++ b/src/parser/parser.rs | |||
@@ -1,7 +1,6 @@ | |||
1 | use {SyntaxKind, TextUnit, Token}; | ||
2 | use super::Event; | 1 | use super::Event; |
3 | use super::is_insignificant; | 2 | use super::input::{InputPosition, ParserInput}; |
4 | use SyntaxKind::{EOF, TOMBSTONE}; | 3 | use SyntaxKind::{self, EOF, TOMBSTONE}; |
5 | 4 | ||
6 | pub(crate) struct Marker { | 5 | pub(crate) struct Marker { |
7 | pos: u32, | 6 | pos: u32, |
@@ -98,35 +97,18 @@ macro_rules! token_set { | |||
98 | } | 97 | } |
99 | 98 | ||
100 | pub(crate) struct Parser<'t> { | 99 | pub(crate) struct Parser<'t> { |
101 | #[allow(unused)] | 100 | inp: &'t ParserInput<'t>, |
102 | text: &'t str, | ||
103 | #[allow(unused)] | ||
104 | start_offsets: Vec<TextUnit>, | ||
105 | tokens: Vec<Token>, // non-whitespace tokens | ||
106 | 101 | ||
107 | pos: usize, | 102 | pos: InputPosition, |
108 | events: Vec<Event>, | 103 | events: Vec<Event>, |
109 | } | 104 | } |
110 | 105 | ||
111 | impl<'t> Parser<'t> { | 106 | impl<'t> Parser<'t> { |
112 | pub(crate) fn new(text: &'t str, raw_tokens: &'t [Token]) -> Parser<'t> { | 107 | pub(crate) fn new(inp: &'t ParserInput<'t>) -> Parser<'t> { |
113 | let mut tokens = Vec::new(); | ||
114 | let mut start_offsets = Vec::new(); | ||
115 | let mut len = TextUnit::new(0); | ||
116 | for &token in raw_tokens.iter() { | ||
117 | if !is_insignificant(token.kind) { | ||
118 | tokens.push(token); | ||
119 | start_offsets.push(len); | ||
120 | } | ||
121 | len += token.len; | ||
122 | } | ||
123 | |||
124 | Parser { | 108 | Parser { |
125 | text, | 109 | inp, |
126 | start_offsets, | ||
127 | tokens, | ||
128 | 110 | ||
129 | pos: 0, | 111 | pos: InputPosition::new(), |
130 | events: Vec::new(), | 112 | events: Vec::new(), |
131 | } | 113 | } |
132 | } | 114 | } |
@@ -163,8 +145,8 @@ impl<'t> Parser<'t> { | |||
163 | }); | 145 | }); |
164 | } | 146 | } |
165 | 147 | ||
166 | pub(crate) fn nth(&self, n: usize) -> SyntaxKind { | 148 | pub(crate) fn nth(&self, n: u32) -> SyntaxKind { |
167 | self.tokens.get(self.pos + n).map(|t| t.kind).unwrap_or(EOF) | 149 | self.inp.kind(self.pos + n) |
168 | } | 150 | } |
169 | 151 | ||
170 | pub(crate) fn current(&self) -> SyntaxKind { | 152 | pub(crate) fn current(&self) -> SyntaxKind { |