aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorAleksey Kladov <[email protected]>2018-02-04 13:46:26 +0000
committerAleksey Kladov <[email protected]>2018-02-04 13:46:26 +0000
commit85c42fba1291f1cc41fb7bfec63117895b394fc5 (patch)
treecda942a62d244f6352dd870cf4c4e68f739909b5 /src
parent852543212ba5c68b3428a80187087cc641de612c (diff)
Support contextual tokens
Diffstat (limited to 'src')
-rw-r--r--src/parser/event.rs27
-rw-r--r--src/parser/grammar/items/mod.rs22
-rw-r--r--src/parser/input.rs7
-rw-r--r--src/parser/parser.rs19
-rw-r--r--src/syntax_kinds.rs110
5 files changed, 117 insertions, 68 deletions
diff --git a/src/parser/event.rs b/src/parser/event.rs
index fd6bdc086..64d751d63 100644
--- a/src/parser/event.rs
+++ b/src/parser/event.rs
@@ -1,4 +1,4 @@
1use {File, FileBuilder, Sink, SyntaxKind, Token}; 1use {File, FileBuilder, Sink, SyntaxKind, Token, TextUnit};
2use syntax_kinds::TOMBSTONE; 2use syntax_kinds::TOMBSTONE;
3use super::is_insignificant; 3use super::is_insignificant;
4 4
@@ -120,18 +120,25 @@ pub(super) fn to_file(text: String, tokens: &[Token], events: Vec<Event>) -> Fil
120 builder.finish_internal() 120 builder.finish_internal()
121 } 121 }
122 &Event::Token { 122 &Event::Token {
123 kind: _, 123 kind,
124 mut n_raw_tokens, 124 mut n_raw_tokens,
125 } => loop { 125 } => {
126 let token = tokens[idx]; 126 // FIXME: currently, we attach whitespace to some random node
127 if !is_insignificant(token.kind) { 127 // this should be done in a sensible manner instead
128 n_raw_tokens -= 1; 128 loop {
129 let token = tokens[idx];
130 if !is_insignificant(token.kind) {
131 break;
132 }
133 builder.leaf(token.kind, token.len);
134 idx += 1
129 } 135 }
130 idx += 1; 136 let mut len = TextUnit::new(0);
131 builder.leaf(token.kind, token.len); 137 for _ in 0..n_raw_tokens {
132 if n_raw_tokens == 0 { 138 len += tokens[idx].len;
133 break; 139 idx += 1;
134 } 140 }
141 builder.leaf(kind, len);
135 }, 142 },
136 &Event::Error { ref message } => builder.error().message(message.clone()).emit(), 143 &Event::Error { ref message } => builder.error().message(message.clone()).emit(),
137 } 144 }
diff --git a/src/parser/grammar/items/mod.rs b/src/parser/grammar/items/mod.rs
index 3612802e1..4afe2e418 100644
--- a/src/parser/grammar/items/mod.rs
+++ b/src/parser/grammar/items/mod.rs
@@ -81,7 +81,6 @@ fn item(p: &mut Parser) {
81 CONST_ITEM 81 CONST_ITEM
82 } 82 }
83 }, 83 },
84 // TODO: auto trait
85 // test unsafe_trait 84 // test unsafe_trait
86 // unsafe trait T {} 85 // unsafe trait T {}
87 UNSAFE_KW if la == TRAIT_KW => { 86 UNSAFE_KW if la == TRAIT_KW => {
@@ -89,7 +88,16 @@ fn item(p: &mut Parser) {
89 traits::trait_item(p); 88 traits::trait_item(p);
90 TRAIT_ITEM 89 TRAIT_ITEM
91 } 90 }
92 // TODO: default impl 91
92 // test unsafe_auto_trait
93 // unsafe auto trait T {}
94 UNSAFE_KW if p.at_kw(1, "auto") && p.nth(2) == TRAIT_KW => {
95 p.bump();
96 p.bump_remap(AUTO_KW);
97 traits::trait_item(p);
98 TRAIT_ITEM
99 }
100
93 // test unsafe_impl 101 // test unsafe_impl
94 // unsafe impl Foo {} 102 // unsafe impl Foo {}
95 UNSAFE_KW if la == IMPL_KW => { 103 UNSAFE_KW if la == IMPL_KW => {
@@ -97,6 +105,16 @@ fn item(p: &mut Parser) {
97 traits::impl_item(p); 105 traits::impl_item(p);
98 IMPL_ITEM 106 IMPL_ITEM
99 } 107 }
108
109 // test unsafe_default_impl
110 // unsafe default impl Foo {}
111 UNSAFE_KW if p.at_kw(1, "default") && p.nth(2) == IMPL_KW => {
112 p.bump();
113 p.bump_remap(DEFAULT_KW);
114 traits::impl_item(p);
115 IMPL_ITEM
116 }
117
100 MOD_KW => { 118 MOD_KW => {
101 mod_item(p); 119 mod_item(p);
102 MOD_ITEM 120 MOD_ITEM
diff --git a/src/parser/input.rs b/src/parser/input.rs
index 162b9ef5f..2ad621166 100644
--- a/src/parser/input.rs
+++ b/src/parser/input.rs
@@ -46,9 +46,10 @@ impl<'t> ParserInput<'t> {
46 if !(idx < self.tokens.len()) { 46 if !(idx < self.tokens.len()) {
47 return ""; 47 return "";
48 } 48 }
49 let start_offset = self.start_offsets[idx]; 49 let range = TextRange::from_len(
50 let end_offset = self.tokens[idx].len; 50 self.start_offsets[idx],
51 let range = TextRange::from_to(start_offset, end_offset); 51 self.tokens[idx].len
52 );
52 &self.text[range] 53 &self.text[range]
53 } 54 }
54} 55}
diff --git a/src/parser/parser.rs b/src/parser/parser.rs
index bb775c4a5..7e1b22ee5 100644
--- a/src/parser/parser.rs
+++ b/src/parser/parser.rs
@@ -1,6 +1,6 @@
1use super::Event; 1use super::Event;
2use super::input::{InputPosition, ParserInput}; 2use super::input::{InputPosition, ParserInput};
3use SyntaxKind::{self, EOF, TOMBSTONE}; 3use SyntaxKind::{self, EOF, TOMBSTONE, IDENT};
4 4
5pub(crate) struct Marker { 5pub(crate) struct Marker {
6 pos: u32, 6 pos: u32,
@@ -145,14 +145,31 @@ impl<'t> Parser<'t> {
145 }); 145 });
146 } 146 }
147 147
148 pub(crate) fn bump_remap(&mut self, kind: SyntaxKind) {
149 if self.current() == EOF {
150 // TODO: panic!?
151 return;
152 }
153 self.pos += 1;
154 self.event(Event::Token {
155 kind,
156 n_raw_tokens: 1,
157 });
158 }
159
148 pub(crate) fn nth(&self, n: u32) -> SyntaxKind { 160 pub(crate) fn nth(&self, n: u32) -> SyntaxKind {
149 self.inp.kind(self.pos + n) 161 self.inp.kind(self.pos + n)
150 } 162 }
151 163
164 pub(crate) fn at_kw(&self, n: u32, t: &str) -> bool {
165 self.nth(n) == IDENT && self.inp.text(self.pos + n) == t
166 }
167
152 pub(crate) fn current(&self) -> SyntaxKind { 168 pub(crate) fn current(&self) -> SyntaxKind {
153 self.nth(0) 169 self.nth(0)
154 } 170 }
155 171
172
156 fn event(&mut self, event: Event) { 173 fn event(&mut self, event: Event) {
157 self.events.push(event) 174 self.events.push(event)
158 } 175 }
diff --git a/src/syntax_kinds.rs b/src/syntax_kinds.rs
index 22c615831..27bc1cafa 100644
--- a/src/syntax_kinds.rs
+++ b/src/syntax_kinds.rs
@@ -6,32 +6,6 @@ use tree::SyntaxInfo;
6/// The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT_DEF`. 6/// The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT_DEF`.
7#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] 7#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
8pub enum SyntaxKind { 8pub enum SyntaxKind {
9 USE_KW,
10 FN_KW,
11 STRUCT_KW,
12 ENUM_KW,
13 TRAIT_KW,
14 IMPL_KW,
15 TRUE_KW,
16 FALSE_KW,
17 AS_KW,
18 EXTERN_KW,
19 CRATE_KW,
20 MOD_KW,
21 PUB_KW,
22 SELF_KW,
23 SUPER_KW,
24 IN_KW,
25 WHERE_KW,
26 FOR_KW,
27 LOOP_KW,
28 WHILE_KW,
29 IF_KW,
30 MATCH_KW,
31 CONST_KW,
32 STATIC_KW,
33 MUT_KW,
34 UNSAFE_KW,
35 ERROR, 9 ERROR,
36 IDENT, 10 IDENT,
37 UNDERSCORE, 11 UNDERSCORE,
@@ -83,6 +57,35 @@ pub enum SyntaxKind {
83 COMMENT, 57 COMMENT,
84 DOC_COMMENT, 58 DOC_COMMENT,
85 SHEBANG, 59 SHEBANG,
60 USE_KW,
61 FN_KW,
62 STRUCT_KW,
63 ENUM_KW,
64 TRAIT_KW,
65 IMPL_KW,
66 TRUE_KW,
67 FALSE_KW,
68 AS_KW,
69 EXTERN_KW,
70 CRATE_KW,
71 MOD_KW,
72 PUB_KW,
73 SELF_KW,
74 SUPER_KW,
75 IN_KW,
76 WHERE_KW,
77 FOR_KW,
78 LOOP_KW,
79 WHILE_KW,
80 IF_KW,
81 MATCH_KW,
82 CONST_KW,
83 STATIC_KW,
84 MUT_KW,
85 UNSAFE_KW,
86 AUTO_KW,
87 DEFAULT_KW,
88 UNION_KW,
86 FILE, 89 FILE,
87 STRUCT_ITEM, 90 STRUCT_ITEM,
88 ENUM_ITEM, 91 ENUM_ITEM,
@@ -123,32 +126,6 @@ pub(crate) use self::SyntaxKind::*;
123impl SyntaxKind { 126impl SyntaxKind {
124 pub(crate) fn info(self) -> &'static SyntaxInfo { 127 pub(crate) fn info(self) -> &'static SyntaxInfo {
125 match self { 128 match self {
126 USE_KW => &SyntaxInfo { name: "USE_KW" },
127 FN_KW => &SyntaxInfo { name: "FN_KW" },
128 STRUCT_KW => &SyntaxInfo { name: "STRUCT_KW" },
129 ENUM_KW => &SyntaxInfo { name: "ENUM_KW" },
130 TRAIT_KW => &SyntaxInfo { name: "TRAIT_KW" },
131 IMPL_KW => &SyntaxInfo { name: "IMPL_KW" },
132 TRUE_KW => &SyntaxInfo { name: "TRUE_KW" },
133 FALSE_KW => &SyntaxInfo { name: "FALSE_KW" },
134 AS_KW => &SyntaxInfo { name: "AS_KW" },
135 EXTERN_KW => &SyntaxInfo { name: "EXTERN_KW" },
136 CRATE_KW => &SyntaxInfo { name: "CRATE_KW" },
137 MOD_KW => &SyntaxInfo { name: "MOD_KW" },
138 PUB_KW => &SyntaxInfo { name: "PUB_KW" },
139 SELF_KW => &SyntaxInfo { name: "SELF_KW" },
140 SUPER_KW => &SyntaxInfo { name: "SUPER_KW" },
141 IN_KW => &SyntaxInfo { name: "IN_KW" },
142 WHERE_KW => &SyntaxInfo { name: "WHERE_KW" },
143 FOR_KW => &SyntaxInfo { name: "FOR_KW" },
144 LOOP_KW => &SyntaxInfo { name: "LOOP_KW" },
145 WHILE_KW => &SyntaxInfo { name: "WHILE_KW" },
146 IF_KW => &SyntaxInfo { name: "IF_KW" },
147 MATCH_KW => &SyntaxInfo { name: "MATCH_KW" },
148 CONST_KW => &SyntaxInfo { name: "CONST_KW" },
149 STATIC_KW => &SyntaxInfo { name: "STATIC_KW" },
150 MUT_KW => &SyntaxInfo { name: "MUT_KW" },
151 UNSAFE_KW => &SyntaxInfo { name: "UNSAFE_KW" },
152 ERROR => &SyntaxInfo { name: "ERROR" }, 129 ERROR => &SyntaxInfo { name: "ERROR" },
153 IDENT => &SyntaxInfo { name: "IDENT" }, 130 IDENT => &SyntaxInfo { name: "IDENT" },
154 UNDERSCORE => &SyntaxInfo { name: "UNDERSCORE" }, 131 UNDERSCORE => &SyntaxInfo { name: "UNDERSCORE" },
@@ -200,6 +177,35 @@ impl SyntaxKind {
200 COMMENT => &SyntaxInfo { name: "COMMENT" }, 177 COMMENT => &SyntaxInfo { name: "COMMENT" },
201 DOC_COMMENT => &SyntaxInfo { name: "DOC_COMMENT" }, 178 DOC_COMMENT => &SyntaxInfo { name: "DOC_COMMENT" },
202 SHEBANG => &SyntaxInfo { name: "SHEBANG" }, 179 SHEBANG => &SyntaxInfo { name: "SHEBANG" },
180 USE_KW => &SyntaxInfo { name: "USE_KW" },
181 FN_KW => &SyntaxInfo { name: "FN_KW" },
182 STRUCT_KW => &SyntaxInfo { name: "STRUCT_KW" },
183 ENUM_KW => &SyntaxInfo { name: "ENUM_KW" },
184 TRAIT_KW => &SyntaxInfo { name: "TRAIT_KW" },
185 IMPL_KW => &SyntaxInfo { name: "IMPL_KW" },
186 TRUE_KW => &SyntaxInfo { name: "TRUE_KW" },
187 FALSE_KW => &SyntaxInfo { name: "FALSE_KW" },
188 AS_KW => &SyntaxInfo { name: "AS_KW" },
189 EXTERN_KW => &SyntaxInfo { name: "EXTERN_KW" },
190 CRATE_KW => &SyntaxInfo { name: "CRATE_KW" },
191 MOD_KW => &SyntaxInfo { name: "MOD_KW" },
192 PUB_KW => &SyntaxInfo { name: "PUB_KW" },
193 SELF_KW => &SyntaxInfo { name: "SELF_KW" },
194 SUPER_KW => &SyntaxInfo { name: "SUPER_KW" },
195 IN_KW => &SyntaxInfo { name: "IN_KW" },
196 WHERE_KW => &SyntaxInfo { name: "WHERE_KW" },
197 FOR_KW => &SyntaxInfo { name: "FOR_KW" },
198 LOOP_KW => &SyntaxInfo { name: "LOOP_KW" },
199 WHILE_KW => &SyntaxInfo { name: "WHILE_KW" },
200 IF_KW => &SyntaxInfo { name: "IF_KW" },
201 MATCH_KW => &SyntaxInfo { name: "MATCH_KW" },
202 CONST_KW => &SyntaxInfo { name: "CONST_KW" },
203 STATIC_KW => &SyntaxInfo { name: "STATIC_KW" },
204 MUT_KW => &SyntaxInfo { name: "MUT_KW" },
205 UNSAFE_KW => &SyntaxInfo { name: "UNSAFE_KW" },
206 AUTO_KW => &SyntaxInfo { name: "AUTO_KW" },
207 DEFAULT_KW => &SyntaxInfo { name: "DEFAULT_KW" },
208 UNION_KW => &SyntaxInfo { name: "UNION_KW" },
203 FILE => &SyntaxInfo { name: "FILE" }, 209 FILE => &SyntaxInfo { name: "FILE" },
204 STRUCT_ITEM => &SyntaxInfo { name: "STRUCT_ITEM" }, 210 STRUCT_ITEM => &SyntaxInfo { name: "STRUCT_ITEM" },
205 ENUM_ITEM => &SyntaxInfo { name: "ENUM_ITEM" }, 211 ENUM_ITEM => &SyntaxInfo { name: "ENUM_ITEM" },
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-27 09:29:45 +0000