aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--grammar.ron5
-rw-r--r--src/parser/event.rs27
-rw-r--r--src/parser/grammar/items/mod.rs22
-rw-r--r--src/parser/input.rs7
-rw-r--r--src/parser/parser.rs19
-rw-r--r--src/syntax_kinds.rs110
-rw-r--r--tests/data/parser/inline/0009_unsafe_auto_trait.rs1
-rw-r--r--tests/data/parser/inline/0009_unsafe_auto_trait.txt13
-rw-r--r--tests/data/parser/inline/0010_unsafe_default_impl.rs1
-rw-r--r--tests/data/parser/inline/0010_unsafe_default_impl.txt13
-rw-r--r--tools/src/bin/gen.rs9
11 files changed, 155 insertions, 72 deletions
diff --git a/grammar.ron b/grammar.ron
index c2fcc44f5..e97ef0c2c 100644
--- a/grammar.ron
+++ b/grammar.ron
@@ -27,6 +27,11 @@ Grammar(
27 "mut", 27 "mut",
28 "unsafe", 28 "unsafe",
29 ], 29 ],
30 contextual_keywords: [
31 "auto",
32 "default",
33 "union",
34 ],
30 tokens: [ 35 tokens: [
31 "ERROR", 36 "ERROR",
32 "IDENT", 37 "IDENT",
diff --git a/src/parser/event.rs b/src/parser/event.rs
index fd6bdc086..64d751d63 100644
--- a/src/parser/event.rs
+++ b/src/parser/event.rs
@@ -1,4 +1,4 @@
1use {File, FileBuilder, Sink, SyntaxKind, Token}; 1use {File, FileBuilder, Sink, SyntaxKind, Token, TextUnit};
2use syntax_kinds::TOMBSTONE; 2use syntax_kinds::TOMBSTONE;
3use super::is_insignificant; 3use super::is_insignificant;
4 4
@@ -120,18 +120,25 @@ pub(super) fn to_file(text: String, tokens: &[Token], events: Vec<Event>) -> Fil
120 builder.finish_internal() 120 builder.finish_internal()
121 } 121 }
122 &Event::Token { 122 &Event::Token {
123 kind: _, 123 kind,
124 mut n_raw_tokens, 124 mut n_raw_tokens,
125 } => loop { 125 } => {
126 let token = tokens[idx]; 126 // FIXME: currently, we attach whitespace to some random node
127 if !is_insignificant(token.kind) { 127 // this should be done in a sensible manner instead
128 n_raw_tokens -= 1; 128 loop {
129 let token = tokens[idx];
130 if !is_insignificant(token.kind) {
131 break;
132 }
133 builder.leaf(token.kind, token.len);
134 idx += 1
129 } 135 }
130 idx += 1; 136 let mut len = TextUnit::new(0);
131 builder.leaf(token.kind, token.len); 137 for _ in 0..n_raw_tokens {
132 if n_raw_tokens == 0 { 138 len += tokens[idx].len;
133 break; 139 idx += 1;
134 } 140 }
141 builder.leaf(kind, len);
135 }, 142 },
136 &Event::Error { ref message } => builder.error().message(message.clone()).emit(), 143 &Event::Error { ref message } => builder.error().message(message.clone()).emit(),
137 } 144 }
diff --git a/src/parser/grammar/items/mod.rs b/src/parser/grammar/items/mod.rs
index 3612802e1..4afe2e418 100644
--- a/src/parser/grammar/items/mod.rs
+++ b/src/parser/grammar/items/mod.rs
@@ -81,7 +81,6 @@ fn item(p: &mut Parser) {
81 CONST_ITEM 81 CONST_ITEM
82 } 82 }
83 }, 83 },
84 // TODO: auto trait
85 // test unsafe_trait 84 // test unsafe_trait
86 // unsafe trait T {} 85 // unsafe trait T {}
87 UNSAFE_KW if la == TRAIT_KW => { 86 UNSAFE_KW if la == TRAIT_KW => {
@@ -89,7 +88,16 @@ fn item(p: &mut Parser) {
89 traits::trait_item(p); 88 traits::trait_item(p);
90 TRAIT_ITEM 89 TRAIT_ITEM
91 } 90 }
92 // TODO: default impl 91
92 // test unsafe_auto_trait
93 // unsafe auto trait T {}
94 UNSAFE_KW if p.at_kw(1, "auto") && p.nth(2) == TRAIT_KW => {
95 p.bump();
96 p.bump_remap(AUTO_KW);
97 traits::trait_item(p);
98 TRAIT_ITEM
99 }
100
93 // test unsafe_impl 101 // test unsafe_impl
94 // unsafe impl Foo {} 102 // unsafe impl Foo {}
95 UNSAFE_KW if la == IMPL_KW => { 103 UNSAFE_KW if la == IMPL_KW => {
@@ -97,6 +105,16 @@ fn item(p: &mut Parser) {
97 traits::impl_item(p); 105 traits::impl_item(p);
98 IMPL_ITEM 106 IMPL_ITEM
99 } 107 }
108
109 // test unsafe_default_impl
110 // unsafe default impl Foo {}
111 UNSAFE_KW if p.at_kw(1, "default") && p.nth(2) == IMPL_KW => {
112 p.bump();
113 p.bump_remap(DEFAULT_KW);
114 traits::impl_item(p);
115 IMPL_ITEM
116 }
117
100 MOD_KW => { 118 MOD_KW => {
101 mod_item(p); 119 mod_item(p);
102 MOD_ITEM 120 MOD_ITEM
diff --git a/src/parser/input.rs b/src/parser/input.rs
index 162b9ef5f..2ad621166 100644
--- a/src/parser/input.rs
+++ b/src/parser/input.rs
@@ -46,9 +46,10 @@ impl<'t> ParserInput<'t> {
46 if !(idx < self.tokens.len()) { 46 if !(idx < self.tokens.len()) {
47 return ""; 47 return "";
48 } 48 }
49 let start_offset = self.start_offsets[idx]; 49 let range = TextRange::from_len(
50 let end_offset = self.tokens[idx].len; 50 self.start_offsets[idx],
51 let range = TextRange::from_to(start_offset, end_offset); 51 self.tokens[idx].len
52 );
52 &self.text[range] 53 &self.text[range]
53 } 54 }
54} 55}
diff --git a/src/parser/parser.rs b/src/parser/parser.rs
index bb775c4a5..7e1b22ee5 100644
--- a/src/parser/parser.rs
+++ b/src/parser/parser.rs
@@ -1,6 +1,6 @@
1use super::Event; 1use super::Event;
2use super::input::{InputPosition, ParserInput}; 2use super::input::{InputPosition, ParserInput};
3use SyntaxKind::{self, EOF, TOMBSTONE}; 3use SyntaxKind::{self, EOF, TOMBSTONE, IDENT};
4 4
5pub(crate) struct Marker { 5pub(crate) struct Marker {
6 pos: u32, 6 pos: u32,
@@ -145,14 +145,31 @@ impl<'t> Parser<'t> {
145 }); 145 });
146 } 146 }
147 147
148 pub(crate) fn bump_remap(&mut self, kind: SyntaxKind) {
149 if self.current() == EOF {
150 // TODO: panic!?
151 return;
152 }
153 self.pos += 1;
154 self.event(Event::Token {
155 kind,
156 n_raw_tokens: 1,
157 });
158 }
159
148 pub(crate) fn nth(&self, n: u32) -> SyntaxKind { 160 pub(crate) fn nth(&self, n: u32) -> SyntaxKind {
149 self.inp.kind(self.pos + n) 161 self.inp.kind(self.pos + n)
150 } 162 }
151 163
164 pub(crate) fn at_kw(&self, n: u32, t: &str) -> bool {
165 self.nth(n) == IDENT && self.inp.text(self.pos + n) == t
166 }
167
152 pub(crate) fn current(&self) -> SyntaxKind { 168 pub(crate) fn current(&self) -> SyntaxKind {
153 self.nth(0) 169 self.nth(0)
154 } 170 }
155 171
172
156 fn event(&mut self, event: Event) { 173 fn event(&mut self, event: Event) {
157 self.events.push(event) 174 self.events.push(event)
158 } 175 }
diff --git a/src/syntax_kinds.rs b/src/syntax_kinds.rs
index 22c615831..27bc1cafa 100644
--- a/src/syntax_kinds.rs
+++ b/src/syntax_kinds.rs
@@ -6,32 +6,6 @@ use tree::SyntaxInfo;
6/// The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT_DEF`. 6/// The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT_DEF`.
7#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] 7#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
8pub enum SyntaxKind { 8pub enum SyntaxKind {
9 USE_KW,
10 FN_KW,
11 STRUCT_KW,
12 ENUM_KW,
13 TRAIT_KW,
14 IMPL_KW,
15 TRUE_KW,
16 FALSE_KW,
17 AS_KW,
18 EXTERN_KW,
19 CRATE_KW,
20 MOD_KW,
21 PUB_KW,
22 SELF_KW,
23 SUPER_KW,
24 IN_KW,
25 WHERE_KW,
26 FOR_KW,
27 LOOP_KW,
28 WHILE_KW,
29 IF_KW,
30 MATCH_KW,
31 CONST_KW,
32 STATIC_KW,
33 MUT_KW,
34 UNSAFE_KW,
35 ERROR, 9 ERROR,
36 IDENT, 10 IDENT,
37 UNDERSCORE, 11 UNDERSCORE,
@@ -83,6 +57,35 @@ pub enum SyntaxKind {
83 COMMENT, 57 COMMENT,
84 DOC_COMMENT, 58 DOC_COMMENT,
85 SHEBANG, 59 SHEBANG,
60 USE_KW,
61 FN_KW,
62 STRUCT_KW,
63 ENUM_KW,
64 TRAIT_KW,
65 IMPL_KW,
66 TRUE_KW,
67 FALSE_KW,
68 AS_KW,
69 EXTERN_KW,
70 CRATE_KW,
71 MOD_KW,
72 PUB_KW,
73 SELF_KW,
74 SUPER_KW,
75 IN_KW,
76 WHERE_KW,
77 FOR_KW,
78 LOOP_KW,
79 WHILE_KW,
80 IF_KW,
81 MATCH_KW,
82 CONST_KW,
83 STATIC_KW,
84 MUT_KW,
85 UNSAFE_KW,
86 AUTO_KW,
87 DEFAULT_KW,
88 UNION_KW,
86 FILE, 89 FILE,
87 STRUCT_ITEM, 90 STRUCT_ITEM,
88 ENUM_ITEM, 91 ENUM_ITEM,
@@ -123,32 +126,6 @@ pub(crate) use self::SyntaxKind::*;
123impl SyntaxKind { 126impl SyntaxKind {
124 pub(crate) fn info(self) -> &'static SyntaxInfo { 127 pub(crate) fn info(self) -> &'static SyntaxInfo {
125 match self { 128 match self {
126 USE_KW => &SyntaxInfo { name: "USE_KW" },
127 FN_KW => &SyntaxInfo { name: "FN_KW" },
128 STRUCT_KW => &SyntaxInfo { name: "STRUCT_KW" },
129 ENUM_KW => &SyntaxInfo { name: "ENUM_KW" },
130 TRAIT_KW => &SyntaxInfo { name: "TRAIT_KW" },
131 IMPL_KW => &SyntaxInfo { name: "IMPL_KW" },
132 TRUE_KW => &SyntaxInfo { name: "TRUE_KW" },
133 FALSE_KW => &SyntaxInfo { name: "FALSE_KW" },
134 AS_KW => &SyntaxInfo { name: "AS_KW" },
135 EXTERN_KW => &SyntaxInfo { name: "EXTERN_KW" },
136 CRATE_KW => &SyntaxInfo { name: "CRATE_KW" },
137 MOD_KW => &SyntaxInfo { name: "MOD_KW" },
138 PUB_KW => &SyntaxInfo { name: "PUB_KW" },
139 SELF_KW => &SyntaxInfo { name: "SELF_KW" },
140 SUPER_KW => &SyntaxInfo { name: "SUPER_KW" },
141 IN_KW => &SyntaxInfo { name: "IN_KW" },
142 WHERE_KW => &SyntaxInfo { name: "WHERE_KW" },
143 FOR_KW => &SyntaxInfo { name: "FOR_KW" },
144 LOOP_KW => &SyntaxInfo { name: "LOOP_KW" },
145 WHILE_KW => &SyntaxInfo { name: "WHILE_KW" },
146 IF_KW => &SyntaxInfo { name: "IF_KW" },
147 MATCH_KW => &SyntaxInfo { name: "MATCH_KW" },
148 CONST_KW => &SyntaxInfo { name: "CONST_KW" },
149 STATIC_KW => &SyntaxInfo { name: "STATIC_KW" },
150 MUT_KW => &SyntaxInfo { name: "MUT_KW" },
151 UNSAFE_KW => &SyntaxInfo { name: "UNSAFE_KW" },
152 ERROR => &SyntaxInfo { name: "ERROR" }, 129 ERROR => &SyntaxInfo { name: "ERROR" },
153 IDENT => &SyntaxInfo { name: "IDENT" }, 130 IDENT => &SyntaxInfo { name: "IDENT" },
154 UNDERSCORE => &SyntaxInfo { name: "UNDERSCORE" }, 131 UNDERSCORE => &SyntaxInfo { name: "UNDERSCORE" },
@@ -200,6 +177,35 @@ impl SyntaxKind {
200 COMMENT => &SyntaxInfo { name: "COMMENT" }, 177 COMMENT => &SyntaxInfo { name: "COMMENT" },
201 DOC_COMMENT => &SyntaxInfo { name: "DOC_COMMENT" }, 178 DOC_COMMENT => &SyntaxInfo { name: "DOC_COMMENT" },
202 SHEBANG => &SyntaxInfo { name: "SHEBANG" }, 179 SHEBANG => &SyntaxInfo { name: "SHEBANG" },
180 USE_KW => &SyntaxInfo { name: "USE_KW" },
181 FN_KW => &SyntaxInfo { name: "FN_KW" },
182 STRUCT_KW => &SyntaxInfo { name: "STRUCT_KW" },
183 ENUM_KW => &SyntaxInfo { name: "ENUM_KW" },
184 TRAIT_KW => &SyntaxInfo { name: "TRAIT_KW" },
185 IMPL_KW => &SyntaxInfo { name: "IMPL_KW" },
186 TRUE_KW => &SyntaxInfo { name: "TRUE_KW" },
187 FALSE_KW => &SyntaxInfo { name: "FALSE_KW" },
188 AS_KW => &SyntaxInfo { name: "AS_KW" },
189 EXTERN_KW => &SyntaxInfo { name: "EXTERN_KW" },
190 CRATE_KW => &SyntaxInfo { name: "CRATE_KW" },
191 MOD_KW => &SyntaxInfo { name: "MOD_KW" },
192 PUB_KW => &SyntaxInfo { name: "PUB_KW" },
193 SELF_KW => &SyntaxInfo { name: "SELF_KW" },
194 SUPER_KW => &SyntaxInfo { name: "SUPER_KW" },
195 IN_KW => &SyntaxInfo { name: "IN_KW" },
196 WHERE_KW => &SyntaxInfo { name: "WHERE_KW" },
197 FOR_KW => &SyntaxInfo { name: "FOR_KW" },
198 LOOP_KW => &SyntaxInfo { name: "LOOP_KW" },
199 WHILE_KW => &SyntaxInfo { name: "WHILE_KW" },
200 IF_KW => &SyntaxInfo { name: "IF_KW" },
201 MATCH_KW => &SyntaxInfo { name: "MATCH_KW" },
202 CONST_KW => &SyntaxInfo { name: "CONST_KW" },
203 STATIC_KW => &SyntaxInfo { name: "STATIC_KW" },
204 MUT_KW => &SyntaxInfo { name: "MUT_KW" },
205 UNSAFE_KW => &SyntaxInfo { name: "UNSAFE_KW" },
206 AUTO_KW => &SyntaxInfo { name: "AUTO_KW" },
207 DEFAULT_KW => &SyntaxInfo { name: "DEFAULT_KW" },
208 UNION_KW => &SyntaxInfo { name: "UNION_KW" },
203 FILE => &SyntaxInfo { name: "FILE" }, 209 FILE => &SyntaxInfo { name: "FILE" },
204 STRUCT_ITEM => &SyntaxInfo { name: "STRUCT_ITEM" }, 210 STRUCT_ITEM => &SyntaxInfo { name: "STRUCT_ITEM" },
205 ENUM_ITEM => &SyntaxInfo { name: "ENUM_ITEM" }, 211 ENUM_ITEM => &SyntaxInfo { name: "ENUM_ITEM" },
diff --git a/tests/data/parser/inline/0009_unsafe_auto_trait.rs b/tests/data/parser/inline/0009_unsafe_auto_trait.rs
new file mode 100644
index 000000000..03d29f324
--- /dev/null
+++ b/tests/data/parser/inline/0009_unsafe_auto_trait.rs
@@ -0,0 +1 @@
unsafe auto trait T {}
diff --git a/tests/data/parser/inline/0009_unsafe_auto_trait.txt b/tests/data/parser/inline/0009_unsafe_auto_trait.txt
new file mode 100644
index 000000000..0a9a1e117
--- /dev/null
+++ b/tests/data/parser/inline/0009_unsafe_auto_trait.txt
@@ -0,0 +1,13 @@
1FILE@[0; 23)
2 TRAIT_ITEM@[0; 23)
3 UNSAFE_KW@[0; 6)
4 WHITESPACE@[6; 7)
5 AUTO_KW@[7; 11)
6 WHITESPACE@[11; 12)
7 TRAIT_KW@[12; 17)
8 WHITESPACE@[17; 18)
9 IDENT@[18; 19) "T"
10 WHITESPACE@[19; 20)
11 L_CURLY@[20; 21)
12 R_CURLY@[21; 22)
13 WHITESPACE@[22; 23)
diff --git a/tests/data/parser/inline/0010_unsafe_default_impl.rs b/tests/data/parser/inline/0010_unsafe_default_impl.rs
new file mode 100644
index 000000000..9cd6c57bd
--- /dev/null
+++ b/tests/data/parser/inline/0010_unsafe_default_impl.rs
@@ -0,0 +1 @@
unsafe default impl Foo {}
diff --git a/tests/data/parser/inline/0010_unsafe_default_impl.txt b/tests/data/parser/inline/0010_unsafe_default_impl.txt
new file mode 100644
index 000000000..7450381cb
--- /dev/null
+++ b/tests/data/parser/inline/0010_unsafe_default_impl.txt
@@ -0,0 +1,13 @@
1FILE@[0; 27)
2 IMPL_ITEM@[0; 27)
3 UNSAFE_KW@[0; 6)
4 WHITESPACE@[6; 7)
5 DEFAULT_KW@[7; 14)
6 WHITESPACE@[14; 15)
7 IMPL_KW@[15; 19)
8 WHITESPACE@[19; 20)
9 IDENT@[20; 23) "Foo"
10 WHITESPACE@[23; 24)
11 L_CURLY@[24; 25)
12 R_CURLY@[25; 26)
13 WHITESPACE@[26; 27)
diff --git a/tools/src/bin/gen.rs b/tools/src/bin/gen.rs
index 17cdea7a1..c71e6da73 100644
--- a/tools/src/bin/gen.rs
+++ b/tools/src/bin/gen.rs
@@ -20,6 +20,7 @@ fn main() {
20#[derive(Deserialize)] 20#[derive(Deserialize)]
21struct Grammar { 21struct Grammar {
22 keywords: Vec<String>, 22 keywords: Vec<String>,
23 contextual_keywords: Vec<String>,
23 tokens: Vec<String>, 24 tokens: Vec<String>,
24 nodes: Vec<String>, 25 nodes: Vec<String>,
25} 26}
@@ -38,10 +39,9 @@ impl Grammar {
38 acc.push_str("use tree::SyntaxInfo;\n"); 39 acc.push_str("use tree::SyntaxInfo;\n");
39 acc.push_str("\n"); 40 acc.push_str("\n");
40 41
41 let syntax_kinds: Vec<String> = self.keywords 42 let syntax_kinds: Vec<String> =self.tokens.iter().cloned()
42 .iter() 43 .chain(self.keywords.iter().map(|kw| kw_token(kw)))
43 .map(|kw| kw_token(kw)) 44 .chain(self.contextual_keywords.iter().map(|kw| kw_token(kw)))
44 .chain(self.tokens.iter().cloned())
45 .chain(self.nodes.iter().cloned()) 45 .chain(self.nodes.iter().cloned())
46 .collect(); 46 .collect();
47 47
@@ -86,6 +86,7 @@ impl Grammar {
86 // fn ident_to_keyword 86 // fn ident_to_keyword
87 acc.push_str("pub(crate) fn ident_to_keyword(ident: &str) -> Option<SyntaxKind> {\n"); 87 acc.push_str("pub(crate) fn ident_to_keyword(ident: &str) -> Option<SyntaxKind> {\n");
88 acc.push_str(" match ident {\n"); 88 acc.push_str(" match ident {\n");
89 // NB: no contextual_keywords here!
89 for kw in self.keywords.iter() { 90 for kw in self.keywords.iter() {
90 write!(acc, " {:?} => Some({}),\n", kw, kw_token(kw)).unwrap(); 91 write!(acc, " {:?} => Some({}),\n", kw, kw_token(kw)).unwrap();
91 } 92 }
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 05:48:55 +0000