From fad3e50987311a3c42a45bd3d9dbcf7c7a77e544 Mon Sep 17 00:00:00 2001
From: Aleksey Kladov <aleksey.kladov@gmail.com>
Date: Sat, 30 Dec 2017 16:30:37 +0300
Subject: Lexer: symbols

---
 src/lexer/mod.rs    | 41 +++++++++++++++++++++++++++++++++++++++++
 src/syntax_kinds.rs | 40 +++++++++++++++++++++++++++++++++++++++-
 2 files changed, 80 insertions(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs
index e60dbbe8e..d2d4aaa22 100644
--- a/src/lexer/mod.rs
+++ b/src/lexer/mod.rs
@@ -37,6 +37,47 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
         return scan_number(c, ptr);
     }
 
+    // One-byte tokens.
+    match c {
+        ';' => return SEMI,
+        ',' => return COMMA,
+        '(' => return L_PAREN,
+        ')' => return R_PAREN,
+        '{' => return L_CURLY,
+        '}' => return R_CURLY,
+        '[' => return L_BRACK,
+        ']' => return R_BRACK,
+        '@' => return AT,
+        '#' => return POUND,
+        '~' => return TILDE,
+        '?' => return QUESTION,
+        '$' => return DOLLAR,
+        '.' => return match (ptr.next(), ptr.nnext()) {
+            (Some('.'), Some('.')) => {
+                ptr.bump();
+                ptr.bump();
+                DOTDOTDOT
+            },
+            (Some('.'), Some('=')) => {
+                ptr.bump();
+                ptr.bump();
+                DOTDOTEQ
+            },
+            (Some('.'), _) => {
+                ptr.bump();
+                DOTDOT
+            },
+            _ => DOT
+        },
+        ':' => return match ptr.next() {
+            Some(':') => {
+                ptr.bump();
+                COLONCOLON
+            }
+            _ => COLON
+        },
+        _ => (),
+    }
     ERROR
 }
 
diff --git a/src/syntax_kinds.rs b/src/syntax_kinds.rs
index bd1265bde..6982cba95 100644
--- a/src/syntax_kinds.rs
+++ b/src/syntax_kinds.rs
@@ -7,14 +7,52 @@ pub const UNDERSCORE: SyntaxKind = SyntaxKind(2);
 pub const WHITESPACE: SyntaxKind = SyntaxKind(3);
 pub const INT_NUMBER: SyntaxKind = SyntaxKind(4);
 pub const FLOAT_NUMBER: SyntaxKind = SyntaxKind(5);
+pub const SEMI: SyntaxKind = SyntaxKind(6);
+pub const COMMA: SyntaxKind = SyntaxKind(7);
+pub const DOT: SyntaxKind = SyntaxKind(8);
+pub const DOTDOT: SyntaxKind = SyntaxKind(9);
+pub const DOTDOTDOT: SyntaxKind = SyntaxKind(10);
+pub const DOTDOTEQ: SyntaxKind = SyntaxKind(11);
+pub const L_PAREN: SyntaxKind = SyntaxKind(12);
+pub const R_PAREN: SyntaxKind = SyntaxKind(13);
+pub const L_CURLY: SyntaxKind = SyntaxKind(14);
+pub const R_CURLY: SyntaxKind = SyntaxKind(15);
+pub const L_BRACK: SyntaxKind = SyntaxKind(16);
+pub const R_BRACK: SyntaxKind = SyntaxKind(17);
+pub const AT: SyntaxKind = SyntaxKind(18);
+pub const POUND: SyntaxKind = SyntaxKind(19);
+pub const TILDE: SyntaxKind = SyntaxKind(20);
+pub const QUESTION: SyntaxKind = SyntaxKind(21);
+pub const COLON: SyntaxKind = SyntaxKind(22);
+pub const COLONCOLON: SyntaxKind = SyntaxKind(23);
+pub const DOLLAR: SyntaxKind = SyntaxKind(24);
 
-static INFOS: [SyntaxInfo; 6] = [
+static INFOS: [SyntaxInfo; 25] = [
     SyntaxInfo { name: "ERROR" },
     SyntaxInfo { name: "IDENT" },
     SyntaxInfo { name: "UNDERSCORE" },
     SyntaxInfo { name: "WHITESPACE" },
     SyntaxInfo { name: "INT_NUMBER" },
     SyntaxInfo { name: "FLOAT_NUMBER" },
+    SyntaxInfo { name: "SEMI" },
+    SyntaxInfo { name: "COMMA" },
+    SyntaxInfo { name: "DOT" },
+    SyntaxInfo { name: "DOTDOT" },
+    SyntaxInfo { name: "DOTDOTDOT" },
+    SyntaxInfo { name: "DOTDOTEQ" },
+    SyntaxInfo { name: "L_PAREN" },
+    SyntaxInfo { name: "R_PAREN" },
+    SyntaxInfo { name: "L_CURLY" },
+    SyntaxInfo { name: "R_CURLY" },
+    SyntaxInfo { name: "L_BRACK" },
+    SyntaxInfo { name: "R_BRACK" },
+    SyntaxInfo { name: "AT" },
+    SyntaxInfo { name: "POUND" },
+    SyntaxInfo { name: "TILDE" },
+    SyntaxInfo { name: "QUESTION" },
+    SyntaxInfo { name: "COLON" },
+    SyntaxInfo { name: "COLONCOLON" },
+    SyntaxInfo { name: "DOLLAR" },
 ];
 
 pub(crate) fn syntax_info(kind: SyntaxKind) -> &'static SyntaxInfo {
-- 
cgit v1.2.3