diff options
-rw-r--r-- | grammar.ron | 9 | ||||
-rw-r--r-- | src/lexer/mod.rs | 13 | ||||
-rw-r--r-- | src/syntax_kinds.rs | 20 | ||||
-rw-r--r-- | tests/data/lexer/0004_numbers.txt | 4 | ||||
-rw-r--r-- | tests/data/lexer/0005_symbols.rs | 5 | ||||
-rw-r--r-- | tests/data/lexer/0005_symbols.txt | 19 |
6 files changed, 65 insertions, 5 deletions
diff --git a/grammar.ron b/grammar.ron index c0564e9cf..482e00f91 100644 --- a/grammar.ron +++ b/grammar.ron | |||
@@ -39,5 +39,14 @@ Grammar( | |||
39 | "RAW_STRING", | 39 | "RAW_STRING", |
40 | "BYTE_STRING", | 40 | "BYTE_STRING", |
41 | "RAW_BYTE_STRING", | 41 | "RAW_BYTE_STRING", |
42 | "PLUS", | ||
43 | "MINUS", | ||
44 | "STAR", | ||
45 | "SLASH", | ||
46 | "CARET", | ||
47 | "PERCENT", | ||
48 | "AMPERSAND", | ||
49 | "PIPE", | ||
50 | "THIN_ARROW", | ||
42 | ] | 51 | ] |
43 | ) \ No newline at end of file | 52 | ) \ No newline at end of file |
diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs index d66046ca1..be86416c4 100644 --- a/src/lexer/mod.rs +++ b/src/lexer/mod.rs | |||
@@ -59,6 +59,13 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind { | |||
59 | '~' => return TILDE, | 59 | '~' => return TILDE, |
60 | '?' => return QUESTION, | 60 | '?' => return QUESTION, |
61 | '$' => return DOLLAR, | 61 | '$' => return DOLLAR, |
62 | '&' => return AMPERSAND, | ||
63 | '|' => return PIPE, | ||
64 | '+' => return PLUS, | ||
65 | '*' => return STAR, | ||
66 | '/' => return SLASH, | ||
67 | '^' => return CARET, | ||
68 | '%' => return PERCENT, | ||
62 | 69 | ||
63 | // Multi-byte tokens. | 70 | // Multi-byte tokens. |
64 | '.' => return match (ptr.next(), ptr.nnext()) { | 71 | '.' => return match (ptr.next(), ptr.nnext()) { |
@@ -103,6 +110,12 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind { | |||
103 | } | 110 | } |
104 | _ => NOT, | 111 | _ => NOT, |
105 | }, | 112 | }, |
113 | '-' => return if ptr.next_is('>') { | ||
114 | ptr.bump(); | ||
115 | THIN_ARROW | ||
116 | } else { | ||
117 | MINUS | ||
118 | }, | ||
106 | 119 | ||
107 | // If the character is an ident start not followed by another single | 120 | // If the character is an ident start not followed by another single |
108 | // quote, then this is a lifetime name: | 121 | // quote, then this is a lifetime name: |
diff --git a/src/syntax_kinds.rs b/src/syntax_kinds.rs index 4a68acb31..83fabe403 100644 --- a/src/syntax_kinds.rs +++ b/src/syntax_kinds.rs | |||
@@ -40,8 +40,17 @@ pub const STRING: SyntaxKind = SyntaxKind(35); | |||
40 | pub const RAW_STRING: SyntaxKind = SyntaxKind(36); | 40 | pub const RAW_STRING: SyntaxKind = SyntaxKind(36); |
41 | pub const BYTE_STRING: SyntaxKind = SyntaxKind(37); | 41 | pub const BYTE_STRING: SyntaxKind = SyntaxKind(37); |
42 | pub const RAW_BYTE_STRING: SyntaxKind = SyntaxKind(38); | 42 | pub const RAW_BYTE_STRING: SyntaxKind = SyntaxKind(38); |
43 | pub const PLUS: SyntaxKind = SyntaxKind(39); | ||
44 | pub const MINUS: SyntaxKind = SyntaxKind(40); | ||
45 | pub const STAR: SyntaxKind = SyntaxKind(41); | ||
46 | pub const SLASH: SyntaxKind = SyntaxKind(42); | ||
47 | pub const CARET: SyntaxKind = SyntaxKind(43); | ||
48 | pub const PERCENT: SyntaxKind = SyntaxKind(44); | ||
49 | pub const AMPERSAND: SyntaxKind = SyntaxKind(45); | ||
50 | pub const PIPE: SyntaxKind = SyntaxKind(46); | ||
51 | pub const THIN_ARROW: SyntaxKind = SyntaxKind(47); | ||
43 | 52 | ||
44 | static INFOS: [SyntaxInfo; 39] = [ | 53 | static INFOS: [SyntaxInfo; 48] = [ |
45 | SyntaxInfo { name: "ERROR" }, | 54 | SyntaxInfo { name: "ERROR" }, |
46 | SyntaxInfo { name: "IDENT" }, | 55 | SyntaxInfo { name: "IDENT" }, |
47 | SyntaxInfo { name: "UNDERSCORE" }, | 56 | SyntaxInfo { name: "UNDERSCORE" }, |
@@ -81,6 +90,15 @@ static INFOS: [SyntaxInfo; 39] = [ | |||
81 | SyntaxInfo { name: "RAW_STRING" }, | 90 | SyntaxInfo { name: "RAW_STRING" }, |
82 | SyntaxInfo { name: "BYTE_STRING" }, | 91 | SyntaxInfo { name: "BYTE_STRING" }, |
83 | SyntaxInfo { name: "RAW_BYTE_STRING" }, | 92 | SyntaxInfo { name: "RAW_BYTE_STRING" }, |
93 | SyntaxInfo { name: "PLUS" }, | ||
94 | SyntaxInfo { name: "MINUS" }, | ||
95 | SyntaxInfo { name: "STAR" }, | ||
96 | SyntaxInfo { name: "SLASH" }, | ||
97 | SyntaxInfo { name: "CARET" }, | ||
98 | SyntaxInfo { name: "PERCENT" }, | ||
99 | SyntaxInfo { name: "AMPERSAND" }, | ||
100 | SyntaxInfo { name: "PIPE" }, | ||
101 | SyntaxInfo { name: "THIN_ARROW" }, | ||
84 | ]; | 102 | ]; |
85 | 103 | ||
86 | pub(crate) fn syntax_info(kind: SyntaxKind) -> &'static SyntaxInfo { | 104 | pub(crate) fn syntax_info(kind: SyntaxKind) -> &'static SyntaxInfo { |
diff --git a/tests/data/lexer/0004_numbers.txt b/tests/data/lexer/0004_numbers.txt index be8dee48b..4b5fd9f71 100644 --- a/tests/data/lexer/0004_numbers.txt +++ b/tests/data/lexer/0004_numbers.txt | |||
@@ -47,13 +47,13 @@ L_PAREN 1 "(" | |||
47 | R_PAREN 1 ")" | 47 | R_PAREN 1 ")" |
48 | WHITESPACE 1 "\n" | 48 | WHITESPACE 1 "\n" |
49 | INT_NUMBER 2 "0e" | 49 | INT_NUMBER 2 "0e" |
50 | ERROR 1 "+" | 50 | PLUS 1 "+" |
51 | INT_NUMBER 1 "1" | 51 | INT_NUMBER 1 "1" |
52 | WHITESPACE 1 "\n" | 52 | WHITESPACE 1 "\n" |
53 | INT_NUMBER 1 "0" | 53 | INT_NUMBER 1 "0" |
54 | DOT 1 "." | 54 | DOT 1 "." |
55 | IDENT 1 "e" | 55 | IDENT 1 "e" |
56 | ERROR 1 "+" | 56 | PLUS 1 "+" |
57 | INT_NUMBER 1 "1" | 57 | INT_NUMBER 1 "1" |
58 | WHITESPACE 1 "\n" | 58 | WHITESPACE 1 "\n" |
59 | FLOAT_NUMBER 6 "0.0E-2" | 59 | FLOAT_NUMBER 6 "0.0E-2" |
diff --git a/tests/data/lexer/0005_symbols.rs b/tests/data/lexer/0005_symbols.rs index c41e985a1..487569b5a 100644 --- a/tests/data/lexer/0005_symbols.rs +++ b/tests/data/lexer/0005_symbols.rs | |||
@@ -1,5 +1,6 @@ | |||
1 | ; , ( ) { } [ ] < > @ # ~ ? $ | 1 | ; , ( ) { } [ ] < > @ # ~ ? $ & | + * / ^ % |
2 | . .. ... ..= | 2 | . .. ... ..= |
3 | : :: | 3 | : :: |
4 | = => | 4 | = => |
5 | ! != \ No newline at end of file | 5 | ! != |
6 | - -> | ||
diff --git a/tests/data/lexer/0005_symbols.txt b/tests/data/lexer/0005_symbols.txt index a1b777a92..ebb41accd 100644 --- a/tests/data/lexer/0005_symbols.txt +++ b/tests/data/lexer/0005_symbols.txt | |||
@@ -27,6 +27,20 @@ WHITESPACE 1 " " | |||
27 | QUESTION 1 "?" | 27 | QUESTION 1 "?" |
28 | WHITESPACE 1 " " | 28 | WHITESPACE 1 " " |
29 | DOLLAR 1 "$" | 29 | DOLLAR 1 "$" |
30 | WHITESPACE 1 " " | ||
31 | AMPERSAND 1 "&" | ||
32 | WHITESPACE 1 " " | ||
33 | PIPE 1 "|" | ||
34 | WHITESPACE 1 " " | ||
35 | PLUS 1 "+" | ||
36 | WHITESPACE 1 " " | ||
37 | STAR 1 "*" | ||
38 | WHITESPACE 1 " " | ||
39 | SLASH 1 "/" | ||
40 | WHITESPACE 1 " " | ||
41 | CARET 1 "^" | ||
42 | WHITESPACE 1 " " | ||
43 | PERCENT 1 "%" | ||
30 | WHITESPACE 1 "\n" | 44 | WHITESPACE 1 "\n" |
31 | DOT 1 "." | 45 | DOT 1 "." |
32 | WHITESPACE 1 " " | 46 | WHITESPACE 1 " " |
@@ -47,3 +61,8 @@ WHITESPACE 1 "\n" | |||
47 | NOT 1 "!" | 61 | NOT 1 "!" |
48 | WHITESPACE 1 " " | 62 | WHITESPACE 1 " " |
49 | NEQ 2 "!=" | 63 | NEQ 2 "!=" |
64 | WHITESPACE 1 "\n" | ||
65 | MINUS 1 "-" | ||
66 | WHITESPACE 1 " " | ||
67 | THIN_ARROW 2 "->" | ||
68 | WHITESPACE 1 "\n" | ||