From a5eeef0eeed092cb663afc3b0cda2c0df0c7e793 Mon Sep 17 00:00:00 2001
From: Aleksey Kladov <aleksey.kladov@gmail.com>
Date: Sat, 18 Aug 2018 12:13:34 +0300
Subject: better char lexing
---
crates/libsyntax2/src/lexer/strings.rs | 28 +++++++++++++----------
crates/libsyntax2/tests/data/lexer/0006_chars.rs | 2 +-
crates/libsyntax2/tests/data/lexer/0006_chars.txt | 10 ++++++++
3 files changed, 27 insertions(+), 13 deletions(-)
(limited to 'crates/libsyntax2')
diff --git a/crates/libsyntax2/src/lexer/strings.rs b/crates/libsyntax2/src/lexer/strings.rs
index fbae767e5..795ea97b7 100644
--- a/crates/libsyntax2/src/lexer/strings.rs
+++ b/crates/libsyntax2/src/lexer/strings.rs
@@ -15,14 +15,23 @@ pub(crate) fn is_string_literal_start(c: char, c1: Option<char>, c2: Option<char
}
pub(crate) fn scan_char(ptr: &mut Ptr) {
- if ptr.bump().is_none() {
- return; // TODO: error reporting is upper in the stack
- }
- scan_char_or_byte(ptr);
- if !ptr.next_is('\'') {
- return; // TODO: error reporting
+ loop {
+ if ptr.next_is('\\') {
+ ptr.bump();
+ if ptr.next_is('\\') || ptr.next_is('\'') {
+ ptr.bump();
+ }
+ continue;
+ }
+ if ptr.next_is('\'') {
+ ptr.bump();
+ return;
+ }
+ if ptr.next_is('\n') {
+ break;
+ }
+ ptr.bump();
}
- ptr.bump();
}
pub(crate) fn scan_byte_char_or_string(ptr: &mut Ptr) -> SyntaxKind {
@@ -111,8 +120,3 @@ fn scan_raw_byte_string(ptr: &mut Ptr) {
}
}
}
-
-fn scan_char_or_byte(ptr: &mut Ptr) {
- //FIXME: deal with escape sequencies
- ptr.bump();
-}
diff --git a/crates/libsyntax2/tests/data/lexer/0006_chars.rs b/crates/libsyntax2/tests/data/lexer/0006_chars.rs
index 03598d908..454ee0a5f 100644
--- a/crates/libsyntax2/tests/data/lexer/0006_chars.rs
+++ b/crates/libsyntax2/tests/data/lexer/0006_chars.rs
@@ -1 +1 @@
-'x' ' ' '0'
+'x' ' ' '0' 'hello' '\x7f' '\n' '\\' '\''
diff --git a/crates/libsyntax2/tests/data/lexer/0006_chars.txt b/crates/libsyntax2/tests/data/lexer/0006_chars.txt
index ecaf22355..950954fbc 100644
--- a/crates/libsyntax2/tests/data/lexer/0006_chars.txt
+++ b/crates/libsyntax2/tests/data/lexer/0006_chars.txt
@@ -3,4 +3,14 @@ WHITESPACE 1 " "
CHAR 3 "\' \'"
WHITESPACE 1 " "
CHAR 3 "\'0\'"
+WHITESPACE 1 " "
+CHAR 7 "\'hello\'"
+WHITESPACE 1 " "
+CHAR 6 "\'\\x7f\'"
+WHITESPACE 1 " "
+CHAR 4 "\'\\n\'"
+WHITESPACE 1 " "
+CHAR 4 "\'\\\\\'"
+WHITESPACE 1 " "
+CHAR 4 "\'\\\'\'"
WHITESPACE 1 "\n"
--
cgit v1.2.3