aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/grammar.ron74
-rw-r--r--src/lexer/mod.rs27
-rw-r--r--src/syntax_kinds/generated.rs113
-rw-r--r--src/syntax_kinds/generated.rs.tera26
-rw-r--r--tools/src/main.rs2
5 files changed, 139 insertions, 103 deletions
diff --git a/src/grammar.ron b/src/grammar.ron
index 8232ba1dc..38453b45c 100644
--- a/src/grammar.ron
+++ b/src/grammar.ron
@@ -1,4 +1,43 @@
1Grammar( 1Grammar(
2 single_byte_tokens: [
3 [";", "SEMI"],
4 [",", "COMMA"],
5 ["(", "L_PAREN"],
6 [")", "R_PAREN"],
7 ["{", "L_CURLY"],
8 ["}", "R_CURLY"],
9 ["[", "L_BRACK"],
10 ["]", "R_BRACK"],
11 ["<", "L_ANGLE"],
12 [">", "R_ANGLE"],
13 ["@", "AT"],
14 ["#", "POUND"],
15 ["~", "TILDE"],
16 ["?", "QUESTION"],
17 ["$", "DOLLAR"],
18 ["&", "AMPERSAND"],
19 ["|", "PIPE"],
20 ["+", "PLUS"],
21 ["*", "STAR"],
22 ["/", "SLASH"],
23 ["^", "CARET"],
24 ["%", "PERCENT"],
25 ],
26 multi_byte_tokens: [
27 [".", "DOT"],
28 ["..", "DOTDOT"],
29 ["...", "DOTDOTDOT"],
30 ["..=", "DOTDOTEQ"],
31 [":", "COLON"],
32 ["::", "COLONCOLON"],
33 ["=", "EQ"],
34 ["==", "EQEQ"],
35 ["=>", "FAT_ARROW"],
36 ["!", "EXCL"],
37 ["!=", "NEQ"],
38 ["-", "MINUS"],
39 ["->", "THIN_ARROW"],
40 ],
2 keywords: [ 41 keywords: [
3 "use", 42 "use",
4 "fn", 43 "fn",
@@ -42,32 +81,6 @@ Grammar(
42 "WHITESPACE", 81 "WHITESPACE",
43 "INT_NUMBER", 82 "INT_NUMBER",
44 "FLOAT_NUMBER", 83 "FLOAT_NUMBER",
45 "SEMI",
46 "COMMA",
47 "DOT",
48 "DOTDOT",
49 "DOTDOTDOT",
50 "DOTDOTEQ",
51 "L_PAREN",
52 "R_PAREN",
53 "L_CURLY",
54 "R_CURLY",
55 "L_BRACK",
56 "R_BRACK",
57 "L_ANGLE",
58 "R_ANGLE",
59 "AT",
60 "POUND",
61 "TILDE",
62 "QUESTION",
63 "COLON",
64 "COLONCOLON",
65 "DOLLAR",
66 "EQ",
67 "EQEQ",
68 "FAT_ARROW",
69 "NEQ",
70 "EXCL",
71 "LIFETIME", 84 "LIFETIME",
72 "CHAR", 85 "CHAR",
73 "BYTE", 86 "BYTE",
@@ -75,15 +88,6 @@ Grammar(
75 "RAW_STRING", 88 "RAW_STRING",
76 "BYTE_STRING", 89 "BYTE_STRING",
77 "RAW_BYTE_STRING", 90 "RAW_BYTE_STRING",
78 "PLUS",
79 "MINUS",
80 "STAR",
81 "SLASH",
82 "CARET",
83 "PERCENT",
84 "AMPERSAND",
85 "PIPE",
86 "THIN_ARROW",
87 "COMMENT", 91 "COMMENT",
88 "DOC_COMMENT", 92 "DOC_COMMENT",
89 "SHEBANG", 93 "SHEBANG",
diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs
index f647838ea..f8fdc41ac 100644
--- a/src/lexer/mod.rs
+++ b/src/lexer/mod.rs
@@ -79,30 +79,11 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
79 } 79 }
80 80
81 // One-byte tokens. 81 // One-byte tokens.
82 match c { 82 if let Some(kind) = SyntaxKind::from_char(c) {
83 ';' => return SEMI, 83 return kind;
84 ',' => return COMMA, 84 }
85 '(' => return L_PAREN,
86 ')' => return R_PAREN,
87 '{' => return L_CURLY,
88 '}' => return R_CURLY,
89 '[' => return L_BRACK,
90 ']' => return R_BRACK,
91 '<' => return L_ANGLE,
92 '>' => return R_ANGLE,
93 '@' => return AT,
94 '#' => return POUND,
95 '~' => return TILDE,
96 '?' => return QUESTION,
97 '$' => return DOLLAR,
98 '&' => return AMPERSAND,
99 '|' => return PIPE,
100 '+' => return PLUS,
101 '*' => return STAR,
102 '/' => return SLASH,
103 '^' => return CARET,
104 '%' => return PERCENT,
105 85
86 match c {
106 // Multi-byte tokens. 87 // Multi-byte tokens.
107 '.' => { 88 '.' => {
108 return match (ptr.next(), ptr.nnext()) { 89 return match (ptr.next(), ptr.nnext()) {
diff --git a/src/syntax_kinds/generated.rs b/src/syntax_kinds/generated.rs
index bc2a995f6..de0d1c51d 100644
--- a/src/syntax_kinds/generated.rs
+++ b/src/syntax_kinds/generated.rs
@@ -5,18 +5,8 @@ use super::SyntaxInfo;
5/// The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT_DEF`. 5/// The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT_DEF`.
6#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] 6#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
7pub enum SyntaxKind { 7pub enum SyntaxKind {
8 ERROR,
9 IDENT,
10 UNDERSCORE,
11 WHITESPACE,
12 INT_NUMBER,
13 FLOAT_NUMBER,
14 SEMI, 8 SEMI,
15 COMMA, 9 COMMA,
16 DOT,
17 DOTDOT,
18 DOTDOTDOT,
19 DOTDOTEQ,
20 L_PAREN, 10 L_PAREN,
21 R_PAREN, 11 R_PAREN,
22 L_CURLY, 12 L_CURLY,
@@ -29,14 +19,33 @@ pub enum SyntaxKind {
29 POUND, 19 POUND,
30 TILDE, 20 TILDE,
31 QUESTION, 21 QUESTION,
22 DOLLAR,
23 AMPERSAND,
24 PIPE,
25 PLUS,
26 STAR,
27 SLASH,
28 CARET,
29 PERCENT,
30 DOT,
31 DOTDOT,
32 DOTDOTDOT,
33 DOTDOTEQ,
32 COLON, 34 COLON,
33 COLONCOLON, 35 COLONCOLON,
34 DOLLAR,
35 EQ, 36 EQ,
36 EQEQ, 37 EQEQ,
37 FAT_ARROW, 38 FAT_ARROW,
38 NEQ,
39 EXCL, 39 EXCL,
40 NEQ,
41 MINUS,
42 THIN_ARROW,
43 ERROR,
44 IDENT,
45 UNDERSCORE,
46 WHITESPACE,
47 INT_NUMBER,
48 FLOAT_NUMBER,
40 LIFETIME, 49 LIFETIME,
41 CHAR, 50 CHAR,
42 BYTE, 51 BYTE,
@@ -44,15 +53,6 @@ pub enum SyntaxKind {
44 RAW_STRING, 53 RAW_STRING,
45 BYTE_STRING, 54 BYTE_STRING,
46 RAW_BYTE_STRING, 55 RAW_BYTE_STRING,
47 PLUS,
48 MINUS,
49 STAR,
50 SLASH,
51 CARET,
52 PERCENT,
53 AMPERSAND,
54 PIPE,
55 THIN_ARROW,
56 COMMENT, 56 COMMENT,
57 DOC_COMMENT, 57 DOC_COMMENT,
58 SHEBANG, 58 SHEBANG,
@@ -151,18 +151,8 @@ use self::SyntaxKind::*;
151impl SyntaxKind { 151impl SyntaxKind {
152 pub(crate) fn info(self) -> &'static SyntaxInfo { 152 pub(crate) fn info(self) -> &'static SyntaxInfo {
153 match self { 153 match self {
154 ERROR => &SyntaxInfo { name: "ERROR" },
155 IDENT => &SyntaxInfo { name: "IDENT" },
156 UNDERSCORE => &SyntaxInfo { name: "UNDERSCORE" },
157 WHITESPACE => &SyntaxInfo { name: "WHITESPACE" },
158 INT_NUMBER => &SyntaxInfo { name: "INT_NUMBER" },
159 FLOAT_NUMBER => &SyntaxInfo { name: "FLOAT_NUMBER" },
160 SEMI => &SyntaxInfo { name: "SEMI" }, 154 SEMI => &SyntaxInfo { name: "SEMI" },
161 COMMA => &SyntaxInfo { name: "COMMA" }, 155 COMMA => &SyntaxInfo { name: "COMMA" },
162 DOT => &SyntaxInfo { name: "DOT" },
163 DOTDOT => &SyntaxInfo { name: "DOTDOT" },
164 DOTDOTDOT => &SyntaxInfo { name: "DOTDOTDOT" },
165 DOTDOTEQ => &SyntaxInfo { name: "DOTDOTEQ" },
166 L_PAREN => &SyntaxInfo { name: "L_PAREN" }, 156 L_PAREN => &SyntaxInfo { name: "L_PAREN" },
167 R_PAREN => &SyntaxInfo { name: "R_PAREN" }, 157 R_PAREN => &SyntaxInfo { name: "R_PAREN" },
168 L_CURLY => &SyntaxInfo { name: "L_CURLY" }, 158 L_CURLY => &SyntaxInfo { name: "L_CURLY" },
@@ -175,14 +165,33 @@ impl SyntaxKind {
175 POUND => &SyntaxInfo { name: "POUND" }, 165 POUND => &SyntaxInfo { name: "POUND" },
176 TILDE => &SyntaxInfo { name: "TILDE" }, 166 TILDE => &SyntaxInfo { name: "TILDE" },
177 QUESTION => &SyntaxInfo { name: "QUESTION" }, 167 QUESTION => &SyntaxInfo { name: "QUESTION" },
168 DOLLAR => &SyntaxInfo { name: "DOLLAR" },
169 AMPERSAND => &SyntaxInfo { name: "AMPERSAND" },
170 PIPE => &SyntaxInfo { name: "PIPE" },
171 PLUS => &SyntaxInfo { name: "PLUS" },
172 STAR => &SyntaxInfo { name: "STAR" },
173 SLASH => &SyntaxInfo { name: "SLASH" },
174 CARET => &SyntaxInfo { name: "CARET" },
175 PERCENT => &SyntaxInfo { name: "PERCENT" },
176 DOT => &SyntaxInfo { name: "DOT" },
177 DOTDOT => &SyntaxInfo { name: "DOTDOT" },
178 DOTDOTDOT => &SyntaxInfo { name: "DOTDOTDOT" },
179 DOTDOTEQ => &SyntaxInfo { name: "DOTDOTEQ" },
178 COLON => &SyntaxInfo { name: "COLON" }, 180 COLON => &SyntaxInfo { name: "COLON" },
179 COLONCOLON => &SyntaxInfo { name: "COLONCOLON" }, 181 COLONCOLON => &SyntaxInfo { name: "COLONCOLON" },
180 DOLLAR => &SyntaxInfo { name: "DOLLAR" },
181 EQ => &SyntaxInfo { name: "EQ" }, 182 EQ => &SyntaxInfo { name: "EQ" },
182 EQEQ => &SyntaxInfo { name: "EQEQ" }, 183 EQEQ => &SyntaxInfo { name: "EQEQ" },
183 FAT_ARROW => &SyntaxInfo { name: "FAT_ARROW" }, 184 FAT_ARROW => &SyntaxInfo { name: "FAT_ARROW" },
184 NEQ => &SyntaxInfo { name: "NEQ" },
185 EXCL => &SyntaxInfo { name: "EXCL" }, 185 EXCL => &SyntaxInfo { name: "EXCL" },
186 NEQ => &SyntaxInfo { name: "NEQ" },
187 MINUS => &SyntaxInfo { name: "MINUS" },
188 THIN_ARROW => &SyntaxInfo { name: "THIN_ARROW" },
189 ERROR => &SyntaxInfo { name: "ERROR" },
190 IDENT => &SyntaxInfo { name: "IDENT" },
191 UNDERSCORE => &SyntaxInfo { name: "UNDERSCORE" },
192 WHITESPACE => &SyntaxInfo { name: "WHITESPACE" },
193 INT_NUMBER => &SyntaxInfo { name: "INT_NUMBER" },
194 FLOAT_NUMBER => &SyntaxInfo { name: "FLOAT_NUMBER" },
186 LIFETIME => &SyntaxInfo { name: "LIFETIME" }, 195 LIFETIME => &SyntaxInfo { name: "LIFETIME" },
187 CHAR => &SyntaxInfo { name: "CHAR" }, 196 CHAR => &SyntaxInfo { name: "CHAR" },
188 BYTE => &SyntaxInfo { name: "BYTE" }, 197 BYTE => &SyntaxInfo { name: "BYTE" },
@@ -190,15 +199,6 @@ impl SyntaxKind {
190 RAW_STRING => &SyntaxInfo { name: "RAW_STRING" }, 199 RAW_STRING => &SyntaxInfo { name: "RAW_STRING" },
191 BYTE_STRING => &SyntaxInfo { name: "BYTE_STRING" }, 200 BYTE_STRING => &SyntaxInfo { name: "BYTE_STRING" },
192 RAW_BYTE_STRING => &SyntaxInfo { name: "RAW_BYTE_STRING" }, 201 RAW_BYTE_STRING => &SyntaxInfo { name: "RAW_BYTE_STRING" },
193 PLUS => &SyntaxInfo { name: "PLUS" },
194 MINUS => &SyntaxInfo { name: "MINUS" },
195 STAR => &SyntaxInfo { name: "STAR" },
196 SLASH => &SyntaxInfo { name: "SLASH" },
197 CARET => &SyntaxInfo { name: "CARET" },
198 PERCENT => &SyntaxInfo { name: "PERCENT" },
199 AMPERSAND => &SyntaxInfo { name: "AMPERSAND" },
200 PIPE => &SyntaxInfo { name: "PIPE" },
201 THIN_ARROW => &SyntaxInfo { name: "THIN_ARROW" },
202 COMMENT => &SyntaxInfo { name: "COMMENT" }, 202 COMMENT => &SyntaxInfo { name: "COMMENT" },
203 DOC_COMMENT => &SyntaxInfo { name: "DOC_COMMENT" }, 203 DOC_COMMENT => &SyntaxInfo { name: "DOC_COMMENT" },
204 SHEBANG => &SyntaxInfo { name: "SHEBANG" }, 204 SHEBANG => &SyntaxInfo { name: "SHEBANG" },
@@ -325,5 +325,34 @@ impl SyntaxKind {
325 }; 325 };
326 Some(kw) 326 Some(kw)
327 } 327 }
328
329 pub(crate) fn from_char(c: char) -> Option<SyntaxKind> {
330 let tok = match c {
331 ';' => SEMI,
332 ',' => COMMA,
333 '(' => L_PAREN,
334 ')' => R_PAREN,
335 '{' => L_CURLY,
336 '}' => R_CURLY,
337 '[' => L_BRACK,
338 ']' => R_BRACK,
339 '<' => L_ANGLE,
340 '>' => R_ANGLE,
341 '@' => AT,
342 '#' => POUND,
343 '~' => TILDE,
344 '?' => QUESTION,
345 '$' => DOLLAR,
346 '&' => AMPERSAND,
347 '|' => PIPE,
348 '+' => PLUS,
349 '*' => STAR,
350 '/' => SLASH,
351 '^' => CARET,
352 '%' => PERCENT,
353 _ => return None,
354 };
355 Some(tok)
356 }
328} 357}
329 358
diff --git a/src/syntax_kinds/generated.rs.tera b/src/syntax_kinds/generated.rs.tera
index aa672d89a..21e471b71 100644
--- a/src/syntax_kinds/generated.rs.tera
+++ b/src/syntax_kinds/generated.rs.tera
@@ -5,7 +5,13 @@ use super::SyntaxInfo;
5/// The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT_DEF`. 5/// The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT_DEF`.
6#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] 6#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
7pub enum SyntaxKind { 7pub enum SyntaxKind {
8{%- for t in tokens %} 8{%- for t in single_byte_tokens %}
9 {{t.1}},
10{%- endfor -%}
11{% for t in multi_byte_tokens %}
12 {{t.1}},
13{%- endfor -%}
14{% for t in tokens %}
9 {{t}}, 15 {{t}},
10{%- endfor -%} 16{%- endfor -%}
11{% for kw in keywords %} 17{% for kw in keywords %}
@@ -29,7 +35,13 @@ use self::SyntaxKind::*;
29impl SyntaxKind { 35impl SyntaxKind {
30 pub(crate) fn info(self) -> &'static SyntaxInfo { 36 pub(crate) fn info(self) -> &'static SyntaxInfo {
31 match self { 37 match self {
32{%- for t in tokens %} 38{%- for t in single_byte_tokens %}
39 {{t.1}} => &SyntaxInfo { name: "{{t.1}}" },
40{%- endfor -%}
41{% for t in multi_byte_tokens %}
42 {{t.1}} => &SyntaxInfo { name: "{{t.1}}" },
43{%- endfor -%}
44{% for t in tokens %}
33 {{t}} => &SyntaxInfo { name: "{{t}}" }, 45 {{t}} => &SyntaxInfo { name: "{{t}}" },
34{%- endfor -%} 46{%- endfor -%}
35{% for kw in keywords %} 47{% for kw in keywords %}
@@ -55,5 +67,15 @@ impl SyntaxKind {
55 }; 67 };
56 Some(kw) 68 Some(kw)
57 } 69 }
70
71 pub(crate) fn from_char(c: char) -> Option<SyntaxKind> {
72 let tok = match c {
73{%- for t in single_byte_tokens %}
74 '{{t.0}}' => {{t.1}},
75{%- endfor %}
76 _ => return None,
77 };
78 Some(tok)
79 }
58} 80}
59 81
diff --git a/tools/src/main.rs b/tools/src/main.rs
index 6786b81ca..3290d1dce 100644
--- a/tools/src/main.rs
+++ b/tools/src/main.rs
@@ -64,7 +64,7 @@ fn get_kinds() -> Result<String> {
64 let grammar = grammar()?; 64 let grammar = grammar()?;
65 let template = fs::read_to_string(SYNTAX_KINDS_TEMPLATE)?; 65 let template = fs::read_to_string(SYNTAX_KINDS_TEMPLATE)?;
66 let ret = tera::Tera::one_off(&template, &grammar, false) 66 let ret = tera::Tera::one_off(&template, &grammar, false)
67 .map_err(|e| format_err!("template error: {}", e))?; 67 .map_err(|e| format_err!("template error: {:?}", e))?;
68 Ok(ret) 68 Ok(ret)
69} 69}
70 70
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 15:12:55 +0000