aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--grammar.ron9
-rw-r--r--src/lexer/mod.rs13
-rw-r--r--src/syntax_kinds.rs20
-rw-r--r--tests/data/lexer/0004_numbers.txt4
-rw-r--r--tests/data/lexer/0005_symbols.rs5
-rw-r--r--tests/data/lexer/0005_symbols.txt19
6 files changed, 65 insertions, 5 deletions
diff --git a/grammar.ron b/grammar.ron
index c0564e9cf..482e00f91 100644
--- a/grammar.ron
+++ b/grammar.ron
@@ -39,5 +39,14 @@ Grammar(
39 "RAW_STRING", 39 "RAW_STRING",
40 "BYTE_STRING", 40 "BYTE_STRING",
41 "RAW_BYTE_STRING", 41 "RAW_BYTE_STRING",
42 "PLUS",
43 "MINUS",
44 "STAR",
45 "SLASH",
46 "CARET",
47 "PERCENT",
48 "AMPERSAND",
49 "PIPE",
50 "THIN_ARROW",
42 ] 51 ]
43) \ No newline at end of file 52) \ No newline at end of file
diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs
index d66046ca1..be86416c4 100644
--- a/src/lexer/mod.rs
+++ b/src/lexer/mod.rs
@@ -59,6 +59,13 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
59 '~' => return TILDE, 59 '~' => return TILDE,
60 '?' => return QUESTION, 60 '?' => return QUESTION,
61 '$' => return DOLLAR, 61 '$' => return DOLLAR,
62 '&' => return AMPERSAND,
63 '|' => return PIPE,
64 '+' => return PLUS,
65 '*' => return STAR,
66 '/' => return SLASH,
67 '^' => return CARET,
68 '%' => return PERCENT,
62 69
63 // Multi-byte tokens. 70 // Multi-byte tokens.
64 '.' => return match (ptr.next(), ptr.nnext()) { 71 '.' => return match (ptr.next(), ptr.nnext()) {
@@ -103,6 +110,12 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
103 } 110 }
104 _ => NOT, 111 _ => NOT,
105 }, 112 },
113 '-' => return if ptr.next_is('>') {
114 ptr.bump();
115 THIN_ARROW
116 } else {
117 MINUS
118 },
106 119
107 // If the character is an ident start not followed by another single 120 // If the character is an ident start not followed by another single
108 // quote, then this is a lifetime name: 121 // quote, then this is a lifetime name:
diff --git a/src/syntax_kinds.rs b/src/syntax_kinds.rs
index 4a68acb31..83fabe403 100644
--- a/src/syntax_kinds.rs
+++ b/src/syntax_kinds.rs
@@ -40,8 +40,17 @@ pub const STRING: SyntaxKind = SyntaxKind(35);
40pub const RAW_STRING: SyntaxKind = SyntaxKind(36); 40pub const RAW_STRING: SyntaxKind = SyntaxKind(36);
41pub const BYTE_STRING: SyntaxKind = SyntaxKind(37); 41pub const BYTE_STRING: SyntaxKind = SyntaxKind(37);
42pub const RAW_BYTE_STRING: SyntaxKind = SyntaxKind(38); 42pub const RAW_BYTE_STRING: SyntaxKind = SyntaxKind(38);
43pub const PLUS: SyntaxKind = SyntaxKind(39);
44pub const MINUS: SyntaxKind = SyntaxKind(40);
45pub const STAR: SyntaxKind = SyntaxKind(41);
46pub const SLASH: SyntaxKind = SyntaxKind(42);
47pub const CARET: SyntaxKind = SyntaxKind(43);
48pub const PERCENT: SyntaxKind = SyntaxKind(44);
49pub const AMPERSAND: SyntaxKind = SyntaxKind(45);
50pub const PIPE: SyntaxKind = SyntaxKind(46);
51pub const THIN_ARROW: SyntaxKind = SyntaxKind(47);
43 52
44static INFOS: [SyntaxInfo; 39] = [ 53static INFOS: [SyntaxInfo; 48] = [
45 SyntaxInfo { name: "ERROR" }, 54 SyntaxInfo { name: "ERROR" },
46 SyntaxInfo { name: "IDENT" }, 55 SyntaxInfo { name: "IDENT" },
47 SyntaxInfo { name: "UNDERSCORE" }, 56 SyntaxInfo { name: "UNDERSCORE" },
@@ -81,6 +90,15 @@ static INFOS: [SyntaxInfo; 39] = [
81 SyntaxInfo { name: "RAW_STRING" }, 90 SyntaxInfo { name: "RAW_STRING" },
82 SyntaxInfo { name: "BYTE_STRING" }, 91 SyntaxInfo { name: "BYTE_STRING" },
83 SyntaxInfo { name: "RAW_BYTE_STRING" }, 92 SyntaxInfo { name: "RAW_BYTE_STRING" },
93 SyntaxInfo { name: "PLUS" },
94 SyntaxInfo { name: "MINUS" },
95 SyntaxInfo { name: "STAR" },
96 SyntaxInfo { name: "SLASH" },
97 SyntaxInfo { name: "CARET" },
98 SyntaxInfo { name: "PERCENT" },
99 SyntaxInfo { name: "AMPERSAND" },
100 SyntaxInfo { name: "PIPE" },
101 SyntaxInfo { name: "THIN_ARROW" },
84]; 102];
85 103
86pub(crate) fn syntax_info(kind: SyntaxKind) -> &'static SyntaxInfo { 104pub(crate) fn syntax_info(kind: SyntaxKind) -> &'static SyntaxInfo {
diff --git a/tests/data/lexer/0004_numbers.txt b/tests/data/lexer/0004_numbers.txt
index be8dee48b..4b5fd9f71 100644
--- a/tests/data/lexer/0004_numbers.txt
+++ b/tests/data/lexer/0004_numbers.txt
@@ -47,13 +47,13 @@ L_PAREN 1 "("
47R_PAREN 1 ")" 47R_PAREN 1 ")"
48WHITESPACE 1 "\n" 48WHITESPACE 1 "\n"
49INT_NUMBER 2 "0e" 49INT_NUMBER 2 "0e"
50ERROR 1 "+" 50PLUS 1 "+"
51INT_NUMBER 1 "1" 51INT_NUMBER 1 "1"
52WHITESPACE 1 "\n" 52WHITESPACE 1 "\n"
53INT_NUMBER 1 "0" 53INT_NUMBER 1 "0"
54DOT 1 "." 54DOT 1 "."
55IDENT 1 "e" 55IDENT 1 "e"
56ERROR 1 "+" 56PLUS 1 "+"
57INT_NUMBER 1 "1" 57INT_NUMBER 1 "1"
58WHITESPACE 1 "\n" 58WHITESPACE 1 "\n"
59FLOAT_NUMBER 6 "0.0E-2" 59FLOAT_NUMBER 6 "0.0E-2"
diff --git a/tests/data/lexer/0005_symbols.rs b/tests/data/lexer/0005_symbols.rs
index c41e985a1..487569b5a 100644
--- a/tests/data/lexer/0005_symbols.rs
+++ b/tests/data/lexer/0005_symbols.rs
@@ -1,5 +1,6 @@
1; , ( ) { } [ ] < > @ # ~ ? $ 1; , ( ) { } [ ] < > @ # ~ ? $ & | + * / ^ %
2. .. ... ..= 2. .. ... ..=
3: :: 3: ::
4= => 4= =>
5! != \ No newline at end of file 5! !=
6- ->
diff --git a/tests/data/lexer/0005_symbols.txt b/tests/data/lexer/0005_symbols.txt
index a1b777a92..ebb41accd 100644
--- a/tests/data/lexer/0005_symbols.txt
+++ b/tests/data/lexer/0005_symbols.txt
@@ -27,6 +27,20 @@ WHITESPACE 1 " "
27QUESTION 1 "?" 27QUESTION 1 "?"
28WHITESPACE 1 " " 28WHITESPACE 1 " "
29DOLLAR 1 "$" 29DOLLAR 1 "$"
30WHITESPACE 1 " "
31AMPERSAND 1 "&"
32WHITESPACE 1 " "
33PIPE 1 "|"
34WHITESPACE 1 " "
35PLUS 1 "+"
36WHITESPACE 1 " "
37STAR 1 "*"
38WHITESPACE 1 " "
39SLASH 1 "/"
40WHITESPACE 1 " "
41CARET 1 "^"
42WHITESPACE 1 " "
43PERCENT 1 "%"
30WHITESPACE 1 "\n" 44WHITESPACE 1 "\n"
31DOT 1 "." 45DOT 1 "."
32WHITESPACE 1 " " 46WHITESPACE 1 " "
@@ -47,3 +61,8 @@ WHITESPACE 1 "\n"
47NOT 1 "!" 61NOT 1 "!"
48WHITESPACE 1 " " 62WHITESPACE 1 " "
49NEQ 2 "!=" 63NEQ 2 "!="
64WHITESPACE 1 "\n"
65MINUS 1 "-"
66WHITESPACE 1 " "
67THIN_ARROW 2 "->"
68WHITESPACE 1 "\n"