aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--src/lexer/ptr.rs3
-rw-r--r--src/text.rs4
-rw-r--r--tests/data/lexer/0001_hello.txt6
-rw-r--r--tests/data/lexer/0002_whitespace.txt24
-rw-r--r--tests/data/lexer/0003_ident.txt28
-rw-r--r--tests/data/lexer/0004_number.txt124
-rw-r--r--tests/lexer.rs12
7 files changed, 103 insertions, 98 deletions
diff --git a/src/lexer/ptr.rs b/src/lexer/ptr.rs
index d441b826b..b380117e6 100644
--- a/src/lexer/ptr.rs
+++ b/src/lexer/ptr.rs
@@ -56,6 +56,7 @@ impl<'s> Ptr<'s> {
56 } 56 }
57 57
58 fn chars(&self) -> Chars { 58 fn chars(&self) -> Chars {
59 self.text[self.len.0 as usize ..].chars() 59 let len: u32 = self.len.into();
60 self.text[len as usize ..].chars()
60 } 61 }
61} 62}
diff --git a/src/text.rs b/src/text.rs
index 31e67b456..c3ef1ac8e 100644
--- a/src/text.rs
+++ b/src/text.rs
@@ -2,9 +2,7 @@ use std::fmt;
2use std::ops; 2use std::ops;
3 3
4#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] 4#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
5pub struct TextUnit( 5pub struct TextUnit(u32);
6 pub(crate) u32
7);
8 6
9impl TextUnit { 7impl TextUnit {
10 pub fn len_of_char(c: char) -> TextUnit { 8 pub fn len_of_char(c: char) -> TextUnit {
diff --git a/tests/data/lexer/0001_hello.txt b/tests/data/lexer/0001_hello.txt
index e0b6a1f10..27a5940a9 100644
--- a/tests/data/lexer/0001_hello.txt
+++ b/tests/data/lexer/0001_hello.txt
@@ -1,3 +1,3 @@
1IDENT 5 1IDENT 5 "hello"
2WHITESPACE 1 2WHITESPACE 1 " "
3IDENT 5 3IDENT 5 "world"
diff --git a/tests/data/lexer/0002_whitespace.txt b/tests/data/lexer/0002_whitespace.txt
index 4b9885e4a..01d260918 100644
--- a/tests/data/lexer/0002_whitespace.txt
+++ b/tests/data/lexer/0002_whitespace.txt
@@ -1,12 +1,12 @@
1IDENT 1 1IDENT 1 "a"
2WHITESPACE 1 2WHITESPACE 1 " "
3IDENT 1 3IDENT 1 "b"
4WHITESPACE 2 4WHITESPACE 2 " "
5IDENT 1 5IDENT 1 "c"
6WHITESPACE 1 6WHITESPACE 1 "\n"
7IDENT 1 7IDENT 1 "d"
8WHITESPACE 2 8WHITESPACE 2 "\n\n"
9IDENT 1 9IDENT 1 "e"
10WHITESPACE 1 10WHITESPACE 1 "\t"
11IDENT 1 11IDENT 1 "f"
12WHITESPACE 1 12WHITESPACE 1 "\n"
diff --git a/tests/data/lexer/0003_ident.txt b/tests/data/lexer/0003_ident.txt
index eec82fb91..4a0d5c053 100644
--- a/tests/data/lexer/0003_ident.txt
+++ b/tests/data/lexer/0003_ident.txt
@@ -1,14 +1,14 @@
1IDENT 3 1IDENT 3 "foo"
2WHITESPACE 1 2WHITESPACE 1 " "
3IDENT 4 3IDENT 4 "foo_"
4WHITESPACE 1 4WHITESPACE 1 " "
5IDENT 4 5IDENT 4 "_foo"
6WHITESPACE 1 6WHITESPACE 1 " "
7UNDERSCORE 1 7UNDERSCORE 1 "_"
8WHITESPACE 1 8WHITESPACE 1 " "
9IDENT 2 9IDENT 2 "__"
10WHITESPACE 1 10WHITESPACE 1 " "
11IDENT 1 11IDENT 1 "x"
12WHITESPACE 1 12WHITESPACE 1 " "
13IDENT 12 13IDENT 12 "привет"
14WHITESPACE 1 14WHITESPACE 1 "\n"
diff --git a/tests/data/lexer/0004_number.txt b/tests/data/lexer/0004_number.txt
index e9ad8410d..7dedd2cac 100644
--- a/tests/data/lexer/0004_number.txt
+++ b/tests/data/lexer/0004_number.txt
@@ -1,62 +1,62 @@
1INT_NUMBER 1 1INT_NUMBER 1 "0"
2WHITESPACE 1 2WHITESPACE 1 " "
3INT_NUMBER 2 3INT_NUMBER 2 "0b"
4WHITESPACE 1 4WHITESPACE 1 " "
5INT_NUMBER 2 5INT_NUMBER 2 "0o"
6WHITESPACE 1 6WHITESPACE 1 " "
7INT_NUMBER 2 7INT_NUMBER 2 "0x"
8WHITESPACE 1 8WHITESPACE 1 " "
9INT_NUMBER 2 9INT_NUMBER 2 "00"
10WHITESPACE 1 10WHITESPACE 1 " "
11INT_NUMBER 2 11INT_NUMBER 2 "0_"
12WHITESPACE 1 12WHITESPACE 1 " "
13FLOAT_NUMBER 2 13FLOAT_NUMBER 2 "0."
14WHITESPACE 1 14WHITESPACE 1 " "
15INT_NUMBER 2 15INT_NUMBER 2 "0e"
16WHITESPACE 1 16WHITESPACE 1 " "
17INT_NUMBER 2 17INT_NUMBER 2 "0E"
18WHITESPACE 1 18WHITESPACE 1 " "
19INT_NUMBER 1 19INT_NUMBER 1 "0"
20IDENT 1 20IDENT 1 "z"
21WHITESPACE 1 21WHITESPACE 1 "\n"
22INT_NUMBER 5 22INT_NUMBER 5 "01790"
23WHITESPACE 1 23WHITESPACE 1 " "
24INT_NUMBER 6 24INT_NUMBER 6 "0b1790"
25WHITESPACE 1 25WHITESPACE 1 " "
26INT_NUMBER 6 26INT_NUMBER 6 "0o1790"
27WHITESPACE 1 27WHITESPACE 1 " "
28INT_NUMBER 18 28INT_NUMBER 18 "0x1790aAbBcCdDeEfF"
29WHITESPACE 1 29WHITESPACE 1 " "
30INT_NUMBER 6 30INT_NUMBER 6 "001279"
31WHITESPACE 1 31WHITESPACE 1 " "
32INT_NUMBER 6 32INT_NUMBER 6 "0_1279"
33WHITESPACE 1 33WHITESPACE 1 " "
34FLOAT_NUMBER 6 34FLOAT_NUMBER 6 "0.1279"
35WHITESPACE 1 35WHITESPACE 1 " "
36INT_NUMBER 6 36INT_NUMBER 6 "0e1279"
37WHITESPACE 1 37WHITESPACE 1 " "
38INT_NUMBER 6 38INT_NUMBER 6 "0E1279"
39WHITESPACE 1 39WHITESPACE 1 "\n"
40INT_NUMBER 1 40INT_NUMBER 1 "0"
41ERROR 1 41ERROR 1 "."
42ERROR 1 42ERROR 1 "."
43INT_NUMBER 1 43INT_NUMBER 1 "2"
44WHITESPACE 1 44WHITESPACE 1 "\n"
45INT_NUMBER 1 45INT_NUMBER 1 "0"
46ERROR 1 46ERROR 1 "."
47IDENT 3 47IDENT 3 "foo"
48ERROR 1 48ERROR 1 "("
49ERROR 1 49ERROR 1 ")"
50WHITESPACE 1 50WHITESPACE 1 "\n"
51INT_NUMBER 2 51INT_NUMBER 2 "0e"
52ERROR 1 52ERROR 1 "+"
53INT_NUMBER 1 53INT_NUMBER 1 "1"
54WHITESPACE 1 54WHITESPACE 1 "\n"
55INT_NUMBER 1 55INT_NUMBER 1 "0"
56ERROR 1 56ERROR 1 "."
57IDENT 1 57IDENT 1 "e"
58ERROR 1 58ERROR 1 "+"
59INT_NUMBER 1 59INT_NUMBER 1 "1"
60WHITESPACE 1 60WHITESPACE 1 "\n"
61FLOAT_NUMBER 6 61FLOAT_NUMBER 6 "0.0E-2"
62WHITESPACE 1 62WHITESPACE 1 "\n"
diff --git a/tests/lexer.rs b/tests/lexer.rs
index a3c8916b1..6a9bab66b 100644
--- a/tests/lexer.rs
+++ b/tests/lexer.rs
@@ -31,6 +31,7 @@ fn lexer_test_cases() -> Vec<PathBuf> {
31 acc.push(path); 31 acc.push(path);
32 } 32 }
33 } 33 }
34 acc.sort();
34 acc 35 acc
35} 36}
36 37
@@ -38,7 +39,7 @@ fn lexer_test_case(path: &Path) {
38 let actual = { 39 let actual = {
39 let text = file::get_text(path).unwrap(); 40 let text = file::get_text(path).unwrap();
40 let tokens = tokenize(&text); 41 let tokens = tokenize(&text);
41 dump_tokens(&tokens) 42 dump_tokens(&tokens, &text)
42 }; 43 };
43 let expected = file::get_text(&path.with_extension("txt")).unwrap(); 44 let expected = file::get_text(&path.with_extension("txt")).unwrap();
44 let expected = expected.as_str(); 45 let expected = expected.as_str();
@@ -64,10 +65,15 @@ fn tokenize(text: &str) -> Vec<Token> {
64 acc 65 acc
65} 66}
66 67
67fn dump_tokens(tokens: &[Token]) -> String { 68fn dump_tokens(tokens: &[Token], text: &str) -> String {
68 let mut acc = String::new(); 69 let mut acc = String::new();
70 let mut offset = 0;
69 for token in tokens { 71 for token in tokens {
70 write!(acc, "{:?} {}\n", token.kind, token.len).unwrap() 72 let len: u32 = token.len.into();
73 let len = len as usize;
74 let token_text = &text[offset..offset + len];
75 offset += len;
76 write!(acc, "{:?} {} {:?}\n", token.kind, token.len, token_text).unwrap()
71 } 77 }
72 acc 78 acc
73} \ No newline at end of file 79} \ No newline at end of file