aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--src/lexer/mod.rs12
-rw-r--r--src/lexer/strings.rs24
-rw-r--r--tests/data/lexer/0008_byte_strings.rs (renamed from tests/data/lexer/0008_strings.rs)0
-rw-r--r--tests/data/lexer/0008_byte_strings.txt (renamed from tests/data/lexer/0008_strings.txt)0
-rw-r--r--tests/data/lexer/0009_strings.rs1
-rw-r--r--tests/data/lexer/0009_strings.txt4
6 files changed, 39 insertions, 2 deletions
diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs
index d81ea0339..d66046ca1 100644
--- a/src/lexer/mod.rs
+++ b/src/lexer/mod.rs
@@ -11,7 +11,7 @@ mod numbers;
11use self::numbers::scan_number; 11use self::numbers::scan_number;
12 12
13mod strings; 13mod strings;
14use self::strings::{string_literal_start, scan_char, scan_byte_char_or_string}; 14use self::strings::{string_literal_start, scan_char, scan_byte_char_or_string, scan_string, scan_raw_string};
15 15
16pub fn next_token(text: &str) -> Token { 16pub fn next_token(text: &str) -> Token {
17 assert!(!text.is_empty()); 17 assert!(!text.is_empty());
@@ -128,6 +128,16 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
128 scan_literal_suffix(ptr); 128 scan_literal_suffix(ptr);
129 return kind 129 return kind
130 }, 130 },
131 '"' => {
132 scan_string(ptr);
133 scan_literal_suffix(ptr);
134 return STRING;
135 }
136 'r' => {
137 scan_raw_string(ptr);
138 scan_literal_suffix(ptr);
139 return RAW_STRING;
140 }
131 _ => (), 141 _ => (),
132 } 142 }
133 ERROR 143 ERROR
diff --git a/src/lexer/strings.rs b/src/lexer/strings.rs
index 283ce8feb..2c1d86374 100644
--- a/src/lexer/strings.rs
+++ b/src/lexer/strings.rs
@@ -47,6 +47,27 @@ pub(crate) fn scan_byte_char_or_string(ptr: &mut Ptr) -> SyntaxKind {
47 } 47 }
48} 48}
49 49
50pub(crate) fn scan_string(ptr: &mut Ptr) {
51 while let Some(c) = ptr.bump() {
52 if c == '"' {
53 return
54 }
55 }
56}
57
58pub(crate) fn scan_raw_string(ptr: &mut Ptr) {
59 if !ptr.next_is('"') {
60 return
61 }
62 ptr.bump();
63
64 while let Some(c) = ptr.bump() {
65 if c == '"' {
66 return
67 }
68 }
69}
70
50fn scan_byte(ptr: &mut Ptr) { 71fn scan_byte(ptr: &mut Ptr) {
51 if ptr.next_is('\'') { 72 if ptr.next_is('\'') {
52 ptr.bump(); 73 ptr.bump();
@@ -83,4 +104,5 @@ fn scan_raw_byte_string(ptr: &mut Ptr) {
83fn scan_char_or_byte(ptr: &mut Ptr) { 104fn scan_char_or_byte(ptr: &mut Ptr) {
84 //FIXME: deal with escape sequencies 105 //FIXME: deal with escape sequencies
85 ptr.bump(); 106 ptr.bump();
86} \ No newline at end of file 107}
108
diff --git a/tests/data/lexer/0008_strings.rs b/tests/data/lexer/0008_byte_strings.rs
index 9dd1570de..9dd1570de 100644
--- a/tests/data/lexer/0008_strings.rs
+++ b/tests/data/lexer/0008_byte_strings.rs
diff --git a/tests/data/lexer/0008_strings.txt b/tests/data/lexer/0008_byte_strings.txt
index ed8cd4bab..ed8cd4bab 100644
--- a/tests/data/lexer/0008_strings.txt
+++ b/tests/data/lexer/0008_byte_strings.txt
diff --git a/tests/data/lexer/0009_strings.rs b/tests/data/lexer/0009_strings.rs
new file mode 100644
index 000000000..7b7faa5d8
--- /dev/null
+++ b/tests/data/lexer/0009_strings.rs
@@ -0,0 +1 @@
"hello" r"world"
diff --git a/tests/data/lexer/0009_strings.txt b/tests/data/lexer/0009_strings.txt
new file mode 100644
index 000000000..7fb6b7b36
--- /dev/null
+++ b/tests/data/lexer/0009_strings.txt
@@ -0,0 +1,4 @@
1STRING 7 "\"hello\""
2WHITESPACE 1 " "
3RAW_STRING 8 "r\"world\""
4WHITESPACE 1 "\n"