diff options
Diffstat (limited to 'crates/ra_syntax/src/parsing/lexer')
-rw-r--r-- | crates/ra_syntax/src/parsing/lexer/classes.rs | 26 | ||||
-rw-r--r-- | crates/ra_syntax/src/parsing/lexer/comments.rs | 57 | ||||
-rw-r--r-- | crates/ra_syntax/src/parsing/lexer/numbers.rs | 69 | ||||
-rw-r--r-- | crates/ra_syntax/src/parsing/lexer/ptr.rs | 162 | ||||
-rw-r--r-- | crates/ra_syntax/src/parsing/lexer/strings.rs | 112 |
5 files changed, 426 insertions, 0 deletions
diff --git a/crates/ra_syntax/src/parsing/lexer/classes.rs b/crates/ra_syntax/src/parsing/lexer/classes.rs new file mode 100644 index 000000000..4235d2648 --- /dev/null +++ b/crates/ra_syntax/src/parsing/lexer/classes.rs | |||
@@ -0,0 +1,26 @@ | |||
1 | use unicode_xid::UnicodeXID; | ||
2 | |||
3 | pub fn is_ident_start(c: char) -> bool { | ||
4 | (c >= 'a' && c <= 'z') | ||
5 | || (c >= 'A' && c <= 'Z') | ||
6 | || c == '_' | ||
7 | || (c > '\x7f' && UnicodeXID::is_xid_start(c)) | ||
8 | } | ||
9 | |||
10 | pub fn is_ident_continue(c: char) -> bool { | ||
11 | (c >= 'a' && c <= 'z') | ||
12 | || (c >= 'A' && c <= 'Z') | ||
13 | || (c >= '0' && c <= '9') | ||
14 | || c == '_' | ||
15 | || (c > '\x7f' && UnicodeXID::is_xid_continue(c)) | ||
16 | } | ||
17 | |||
18 | pub fn is_whitespace(c: char) -> bool { | ||
19 | //FIXME: use is_pattern_whitespace | ||
20 | //https://github.com/behnam/rust-unic/issues/192 | ||
21 | c.is_whitespace() | ||
22 | } | ||
23 | |||
24 | pub fn is_dec_digit(c: char) -> bool { | ||
25 | '0' <= c && c <= '9' | ||
26 | } | ||
diff --git a/crates/ra_syntax/src/parsing/lexer/comments.rs b/crates/ra_syntax/src/parsing/lexer/comments.rs new file mode 100644 index 000000000..8bbbe659b --- /dev/null +++ b/crates/ra_syntax/src/parsing/lexer/comments.rs | |||
@@ -0,0 +1,57 @@ | |||
1 | use crate::parsing::lexer::ptr::Ptr; | ||
2 | |||
3 | use crate::SyntaxKind::{self, *}; | ||
4 | |||
5 | pub(crate) fn scan_shebang(ptr: &mut Ptr) -> bool { | ||
6 | if ptr.at_str("!/") { | ||
7 | ptr.bump(); | ||
8 | ptr.bump(); | ||
9 | bump_until_eol(ptr); | ||
10 | true | ||
11 | } else { | ||
12 | false | ||
13 | } | ||
14 | } | ||
15 | |||
16 | fn scan_block_comment(ptr: &mut Ptr) -> Option<SyntaxKind> { | ||
17 | if ptr.at('*') { | ||
18 | ptr.bump(); | ||
19 | let mut depth: u32 = 1; | ||
20 | while depth > 0 { | ||
21 | if ptr.at_str("*/") { | ||
22 | depth -= 1; | ||
23 | ptr.bump(); | ||
24 | ptr.bump(); | ||
25 | } else if ptr.at_str("/*") { | ||
26 | depth += 1; | ||
27 | ptr.bump(); | ||
28 | ptr.bump(); | ||
29 | } else if ptr.bump().is_none() { | ||
30 | break; | ||
31 | } | ||
32 | } | ||
33 | Some(COMMENT) | ||
34 | } else { | ||
35 | None | ||
36 | } | ||
37 | } | ||
38 | |||
39 | pub(crate) fn scan_comment(ptr: &mut Ptr) -> Option<SyntaxKind> { | ||
40 | if ptr.at('/') { | ||
41 | bump_until_eol(ptr); | ||
42 | Some(COMMENT) | ||
43 | } else { | ||
44 | scan_block_comment(ptr) | ||
45 | } | ||
46 | } | ||
47 | |||
48 | fn bump_until_eol(ptr: &mut Ptr) { | ||
49 | loop { | ||
50 | if ptr.at('\n') || ptr.at_str("\r\n") { | ||
51 | return; | ||
52 | } | ||
53 | if ptr.bump().is_none() { | ||
54 | break; | ||
55 | } | ||
56 | } | ||
57 | } | ||
diff --git a/crates/ra_syntax/src/parsing/lexer/numbers.rs b/crates/ra_syntax/src/parsing/lexer/numbers.rs new file mode 100644 index 000000000..7f6abe1d5 --- /dev/null +++ b/crates/ra_syntax/src/parsing/lexer/numbers.rs | |||
@@ -0,0 +1,69 @@ | |||
1 | use crate::parsing::lexer::{ | ||
2 | ptr::Ptr, | ||
3 | classes::*, | ||
4 | }; | ||
5 | |||
6 | use crate::SyntaxKind::{self, *}; | ||
7 | |||
8 | pub(crate) fn scan_number(c: char, ptr: &mut Ptr) -> SyntaxKind { | ||
9 | if c == '0' { | ||
10 | match ptr.current().unwrap_or('\0') { | ||
11 | 'b' | 'o' => { | ||
12 | ptr.bump(); | ||
13 | scan_digits(ptr, false); | ||
14 | } | ||
15 | 'x' => { | ||
16 | ptr.bump(); | ||
17 | scan_digits(ptr, true); | ||
18 | } | ||
19 | '0'...'9' | '_' | '.' | 'e' | 'E' => { | ||
20 | scan_digits(ptr, true); | ||
21 | } | ||
22 | _ => return INT_NUMBER, | ||
23 | } | ||
24 | } else { | ||
25 | scan_digits(ptr, false); | ||
26 | } | ||
27 | |||
28 | // might be a float, but don't be greedy if this is actually an | ||
29 | // integer literal followed by field/method access or a range pattern | ||
30 | // (`0..2` and `12.foo()`) | ||
31 | if ptr.at('.') && !(ptr.at_str("..") || ptr.nth_is_p(1, is_ident_start)) { | ||
32 | // might have stuff after the ., and if it does, it needs to start | ||
33 | // with a number | ||
34 | ptr.bump(); | ||
35 | scan_digits(ptr, false); | ||
36 | scan_float_exponent(ptr); | ||
37 | return FLOAT_NUMBER; | ||
38 | } | ||
39 | // it might be a float if it has an exponent | ||
40 | if ptr.at('e') || ptr.at('E') { | ||
41 | scan_float_exponent(ptr); | ||
42 | return FLOAT_NUMBER; | ||
43 | } | ||
44 | INT_NUMBER | ||
45 | } | ||
46 | |||
47 | fn scan_digits(ptr: &mut Ptr, allow_hex: bool) { | ||
48 | while let Some(c) = ptr.current() { | ||
49 | match c { | ||
50 | '_' | '0'...'9' => { | ||
51 | ptr.bump(); | ||
52 | } | ||
53 | 'a'...'f' | 'A'...'F' if allow_hex => { | ||
54 | ptr.bump(); | ||
55 | } | ||
56 | _ => return, | ||
57 | } | ||
58 | } | ||
59 | } | ||
60 | |||
61 | fn scan_float_exponent(ptr: &mut Ptr) { | ||
62 | if ptr.at('e') || ptr.at('E') { | ||
63 | ptr.bump(); | ||
64 | if ptr.at('-') || ptr.at('+') { | ||
65 | ptr.bump(); | ||
66 | } | ||
67 | scan_digits(ptr, false); | ||
68 | } | ||
69 | } | ||
diff --git a/crates/ra_syntax/src/parsing/lexer/ptr.rs b/crates/ra_syntax/src/parsing/lexer/ptr.rs new file mode 100644 index 000000000..c341c4176 --- /dev/null +++ b/crates/ra_syntax/src/parsing/lexer/ptr.rs | |||
@@ -0,0 +1,162 @@ | |||
1 | use crate::TextUnit; | ||
2 | |||
3 | use std::str::Chars; | ||
4 | |||
5 | /// A simple view into the characters of a string. | ||
6 | pub(crate) struct Ptr<'s> { | ||
7 | text: &'s str, | ||
8 | len: TextUnit, | ||
9 | } | ||
10 | |||
11 | impl<'s> Ptr<'s> { | ||
12 | /// Creates a new `Ptr` from a string. | ||
13 | pub fn new(text: &'s str) -> Ptr<'s> { | ||
14 | Ptr { text, len: 0.into() } | ||
15 | } | ||
16 | |||
17 | /// Gets the length of the remaining string. | ||
18 | pub fn into_len(self) -> TextUnit { | ||
19 | self.len | ||
20 | } | ||
21 | |||
22 | /// Gets the current character, if one exists. | ||
23 | pub fn current(&self) -> Option<char> { | ||
24 | self.chars().next() | ||
25 | } | ||
26 | |||
27 | /// Gets the nth character from the current. | ||
28 | /// For example, 0 will return the current character, 1 will return the next, etc. | ||
29 | pub fn nth(&self, n: u32) -> Option<char> { | ||
30 | self.chars().nth(n as usize) | ||
31 | } | ||
32 | |||
33 | /// Checks whether the current character is `c`. | ||
34 | pub fn at(&self, c: char) -> bool { | ||
35 | self.current() == Some(c) | ||
36 | } | ||
37 | |||
38 | /// Checks whether the next characters match `s`. | ||
39 | pub fn at_str(&self, s: &str) -> bool { | ||
40 | let chars = self.chars(); | ||
41 | chars.as_str().starts_with(s) | ||
42 | } | ||
43 | |||
44 | /// Checks whether the current character satisfies the predicate `p`. | ||
45 | pub fn at_p<P: Fn(char) -> bool>(&self, p: P) -> bool { | ||
46 | self.current().map(p) == Some(true) | ||
47 | } | ||
48 | |||
49 | /// Checks whether the nth character satisfies the predicate `p`. | ||
50 | pub fn nth_is_p<P: Fn(char) -> bool>(&self, n: u32, p: P) -> bool { | ||
51 | self.nth(n).map(p) == Some(true) | ||
52 | } | ||
53 | |||
54 | /// Moves to the next character. | ||
55 | pub fn bump(&mut self) -> Option<char> { | ||
56 | let ch = self.chars().next()?; | ||
57 | self.len += TextUnit::of_char(ch); | ||
58 | Some(ch) | ||
59 | } | ||
60 | |||
61 | /// Moves to the next character as long as `pred` is satisfied. | ||
62 | pub fn bump_while<F: Fn(char) -> bool>(&mut self, pred: F) { | ||
63 | loop { | ||
64 | match self.current() { | ||
65 | Some(c) if pred(c) => { | ||
66 | self.bump(); | ||
67 | } | ||
68 | _ => return, | ||
69 | } | ||
70 | } | ||
71 | } | ||
72 | |||
73 | /// Returns the text up to the current point. | ||
74 | pub fn current_token_text(&self) -> &str { | ||
75 | let len: u32 = self.len.into(); | ||
76 | &self.text[..len as usize] | ||
77 | } | ||
78 | |||
79 | /// Returns an iterator over the remaining characters. | ||
80 | fn chars(&self) -> Chars { | ||
81 | let len: u32 = self.len.into(); | ||
82 | self.text[len as usize..].chars() | ||
83 | } | ||
84 | } | ||
85 | |||
86 | #[cfg(test)] | ||
87 | mod tests { | ||
88 | use super::*; | ||
89 | |||
90 | #[test] | ||
91 | fn test_current() { | ||
92 | let ptr = Ptr::new("test"); | ||
93 | assert_eq!(ptr.current(), Some('t')); | ||
94 | } | ||
95 | |||
96 | #[test] | ||
97 | fn test_nth() { | ||
98 | let ptr = Ptr::new("test"); | ||
99 | assert_eq!(ptr.nth(0), Some('t')); | ||
100 | assert_eq!(ptr.nth(1), Some('e')); | ||
101 | assert_eq!(ptr.nth(2), Some('s')); | ||
102 | assert_eq!(ptr.nth(3), Some('t')); | ||
103 | assert_eq!(ptr.nth(4), None); | ||
104 | } | ||
105 | |||
106 | #[test] | ||
107 | fn test_at() { | ||
108 | let ptr = Ptr::new("test"); | ||
109 | assert!(ptr.at('t')); | ||
110 | assert!(!ptr.at('a')); | ||
111 | } | ||
112 | |||
113 | #[test] | ||
114 | fn test_at_str() { | ||
115 | let ptr = Ptr::new("test"); | ||
116 | assert!(ptr.at_str("t")); | ||
117 | assert!(ptr.at_str("te")); | ||
118 | assert!(ptr.at_str("test")); | ||
119 | assert!(!ptr.at_str("tests")); | ||
120 | assert!(!ptr.at_str("rust")); | ||
121 | } | ||
122 | |||
123 | #[test] | ||
124 | fn test_at_p() { | ||
125 | let ptr = Ptr::new("test"); | ||
126 | assert!(ptr.at_p(|c| c == 't')); | ||
127 | assert!(!ptr.at_p(|c| c == 'e')); | ||
128 | } | ||
129 | |||
130 | #[test] | ||
131 | fn test_nth_is_p() { | ||
132 | let ptr = Ptr::new("test"); | ||
133 | assert!(ptr.nth_is_p(0, |c| c == 't')); | ||
134 | assert!(!ptr.nth_is_p(1, |c| c == 't')); | ||
135 | assert!(ptr.nth_is_p(3, |c| c == 't')); | ||
136 | assert!(!ptr.nth_is_p(150, |c| c == 't')); | ||
137 | } | ||
138 | |||
139 | #[test] | ||
140 | fn test_bump() { | ||
141 | let mut ptr = Ptr::new("test"); | ||
142 | assert_eq!(ptr.current(), Some('t')); | ||
143 | ptr.bump(); | ||
144 | assert_eq!(ptr.current(), Some('e')); | ||
145 | ptr.bump(); | ||
146 | assert_eq!(ptr.current(), Some('s')); | ||
147 | ptr.bump(); | ||
148 | assert_eq!(ptr.current(), Some('t')); | ||
149 | ptr.bump(); | ||
150 | assert_eq!(ptr.current(), None); | ||
151 | ptr.bump(); | ||
152 | assert_eq!(ptr.current(), None); | ||
153 | } | ||
154 | |||
155 | #[test] | ||
156 | fn test_bump_while() { | ||
157 | let mut ptr = Ptr::new("test"); | ||
158 | assert_eq!(ptr.current(), Some('t')); | ||
159 | ptr.bump_while(|c| c != 's'); | ||
160 | assert_eq!(ptr.current(), Some('s')); | ||
161 | } | ||
162 | } | ||
diff --git a/crates/ra_syntax/src/parsing/lexer/strings.rs b/crates/ra_syntax/src/parsing/lexer/strings.rs new file mode 100644 index 000000000..f74acff9e --- /dev/null +++ b/crates/ra_syntax/src/parsing/lexer/strings.rs | |||
@@ -0,0 +1,112 @@ | |||
1 | use crate::{ | ||
2 | parsing::lexer::ptr::Ptr, | ||
3 | SyntaxKind::{self, *}, | ||
4 | }; | ||
5 | |||
6 | pub(crate) fn is_string_literal_start(c: char, c1: Option<char>, c2: Option<char>) -> bool { | ||
7 | match (c, c1, c2) { | ||
8 | ('r', Some('"'), _) | ||
9 | | ('r', Some('#'), Some('"')) | ||
10 | | ('r', Some('#'), Some('#')) | ||
11 | | ('b', Some('"'), _) | ||
12 | | ('b', Some('\''), _) | ||
13 | | ('b', Some('r'), Some('"')) | ||
14 | | ('b', Some('r'), Some('#')) => true, | ||
15 | _ => false, | ||
16 | } | ||
17 | } | ||
18 | |||
19 | pub(crate) fn scan_char(ptr: &mut Ptr) { | ||
20 | while let Some(c) = ptr.current() { | ||
21 | match c { | ||
22 | '\\' => { | ||
23 | ptr.bump(); | ||
24 | if ptr.at('\\') || ptr.at('\'') { | ||
25 | ptr.bump(); | ||
26 | } | ||
27 | } | ||
28 | '\'' => { | ||
29 | ptr.bump(); | ||
30 | return; | ||
31 | } | ||
32 | '\n' => return, | ||
33 | _ => { | ||
34 | ptr.bump(); | ||
35 | } | ||
36 | } | ||
37 | } | ||
38 | } | ||
39 | |||
40 | pub(crate) fn scan_byte_char_or_string(ptr: &mut Ptr) -> SyntaxKind { | ||
41 | // unwrapping and not-exhaustive match are ok | ||
42 | // because of string_literal_start | ||
43 | let c = ptr.bump().unwrap(); | ||
44 | match c { | ||
45 | '\'' => { | ||
46 | scan_byte(ptr); | ||
47 | BYTE | ||
48 | } | ||
49 | '"' => { | ||
50 | scan_byte_string(ptr); | ||
51 | BYTE_STRING | ||
52 | } | ||
53 | 'r' => { | ||
54 | scan_raw_string(ptr); | ||
55 | RAW_BYTE_STRING | ||
56 | } | ||
57 | _ => unreachable!(), | ||
58 | } | ||
59 | } | ||
60 | |||
61 | pub(crate) fn scan_string(ptr: &mut Ptr) { | ||
62 | while let Some(c) = ptr.current() { | ||
63 | match c { | ||
64 | '\\' => { | ||
65 | ptr.bump(); | ||
66 | if ptr.at('\\') || ptr.at('"') { | ||
67 | ptr.bump(); | ||
68 | } | ||
69 | } | ||
70 | '"' => { | ||
71 | ptr.bump(); | ||
72 | return; | ||
73 | } | ||
74 | _ => { | ||
75 | ptr.bump(); | ||
76 | } | ||
77 | } | ||
78 | } | ||
79 | } | ||
80 | |||
81 | pub(crate) fn scan_raw_string(ptr: &mut Ptr) { | ||
82 | let mut hashes = 0; | ||
83 | while ptr.at('#') { | ||
84 | hashes += 1; | ||
85 | ptr.bump(); | ||
86 | } | ||
87 | if !ptr.at('"') { | ||
88 | return; | ||
89 | } | ||
90 | ptr.bump(); | ||
91 | |||
92 | while let Some(c) = ptr.bump() { | ||
93 | if c == '"' { | ||
94 | let mut hashes_left = hashes; | ||
95 | while ptr.at('#') && hashes_left > 0 { | ||
96 | hashes_left -= 1; | ||
97 | ptr.bump(); | ||
98 | } | ||
99 | if hashes_left == 0 { | ||
100 | return; | ||
101 | } | ||
102 | } | ||
103 | } | ||
104 | } | ||
105 | |||
106 | fn scan_byte(ptr: &mut Ptr) { | ||
107 | scan_char(ptr) | ||
108 | } | ||
109 | |||
110 | fn scan_byte_string(ptr: &mut Ptr) { | ||
111 | scan_string(ptr) | ||
112 | } | ||