diff options
author | Aleksey Kladov <[email protected]> | 2018-07-30 15:46:50 +0100 |
---|---|---|
committer | Aleksey Kladov <[email protected]> | 2018-07-30 15:46:50 +0100 |
commit | 7fb7cfc2d1cd2781a20e4034b12384566d13c728 (patch) | |
tree | 86495030c1c1fdda29946165cdd750569dfe044f /src/lexer/mod.rs | |
parent | d4366940976a3d150a8b5bf92dc24f0162d4d62c (diff) |
generate single byte tokens
Diffstat (limited to 'src/lexer/mod.rs')
-rw-r--r-- | src/lexer/mod.rs | 27 |
1 files changed, 4 insertions, 23 deletions
diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs index f647838ea..f8fdc41ac 100644 --- a/src/lexer/mod.rs +++ b/src/lexer/mod.rs | |||
@@ -79,30 +79,11 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind { | |||
79 | } | 79 | } |
80 | 80 | ||
81 | // One-byte tokens. | 81 | // One-byte tokens. |
82 | match c { | 82 | if let Some(kind) = SyntaxKind::from_char(c) { |
83 | ';' => return SEMI, | 83 | return kind; |
84 | ',' => return COMMA, | 84 | } |
85 | '(' => return L_PAREN, | ||
86 | ')' => return R_PAREN, | ||
87 | '{' => return L_CURLY, | ||
88 | '}' => return R_CURLY, | ||
89 | '[' => return L_BRACK, | ||
90 | ']' => return R_BRACK, | ||
91 | '<' => return L_ANGLE, | ||
92 | '>' => return R_ANGLE, | ||
93 | '@' => return AT, | ||
94 | '#' => return POUND, | ||
95 | '~' => return TILDE, | ||
96 | '?' => return QUESTION, | ||
97 | '$' => return DOLLAR, | ||
98 | '&' => return AMPERSAND, | ||
99 | '|' => return PIPE, | ||
100 | '+' => return PLUS, | ||
101 | '*' => return STAR, | ||
102 | '/' => return SLASH, | ||
103 | '^' => return CARET, | ||
104 | '%' => return PERCENT, | ||
105 | 85 | ||
86 | match c { | ||
106 | // Multi-byte tokens. | 87 | // Multi-byte tokens. |
107 | '.' => { | 88 | '.' => { |
108 | return match (ptr.next(), ptr.nnext()) { | 89 | return match (ptr.next(), ptr.nnext()) { |