Lexer: comments groundwork

2 files changed, 29 insertions, 5 deletions

diff --git a/src/lexer/comments.rs b/src/lexer/comments.rs
new file mode 100644
index 000000000..c61c85824
--- /dev/null
+++ b/src/lexer/comments.rs
@@ -0,0 +1,11 @@
+use lexer::ptr::Ptr;
+
+use {SyntaxKind};
+
+pub(crate) fn scan_shebang(ptr: &mut Ptr) -> bool {
+    false
+}
+
+pub(crate) fn scan_comment(ptr: &mut Ptr) -> Option<SyntaxKind> {
+    None
+}
\ No newline at end of file
diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs
index 3e49b1c2b..f46746bee 100644
--- a/src/lexer/mod.rs
+++ b/src/lexer/mod.rs
@@ -13,6 +13,9 @@ use self::numbers::scan_number;
 mod strings;
 use self::strings::{is_string_literal_start, scan_char, scan_byte_char_or_string, scan_string, scan_raw_string};
 
+mod comments;
+use self::comments::{scan_shebang, scan_comment};
+
 pub fn next_token(text: &str) -> Token {
     assert!(!text.is_empty());
     let mut ptr = Ptr::new(text);
@@ -23,16 +26,26 @@ pub fn next_token(text: &str) -> Token {
 }
 
 fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
-    let ident_start = is_ident_start(c) && !is_string_literal_start(c, ptr.next(), ptr.nnext());
-    if ident_start {
-        return scan_ident(c, ptr);
-    }
-
     if is_whitespace(c) {
         ptr.bump_while(is_whitespace);
         return WHITESPACE;
     }
 
+    match c {
+        '#' => if scan_shebang(ptr) {
+            return SHEBANG;
+        }
+        '/' => if let Some(kind) = scan_comment(ptr) {
+            return kind;
+        }
+        _ => (),
+    }
+
+    let ident_start = is_ident_start(c) && !is_string_literal_start(c, ptr.next(), ptr.nnext());
+    if ident_start {
+        return scan_ident(c, ptr);
+    }
+
     if is_dec_digit(c) {
         let kind = scan_number(c, ptr);
         scan_literal_suffix(ptr);