aboutsummaryrefslogtreecommitdiff
path: root/crates/ra_syntax/src/string_lexing.rs
diff options
context:
space:
mode:
Diffstat (limited to 'crates/ra_syntax/src/string_lexing.rs')
-rw-r--r--crates/ra_syntax/src/string_lexing.rs513
1 files changed, 0 insertions, 513 deletions
diff --git a/crates/ra_syntax/src/string_lexing.rs b/crates/ra_syntax/src/string_lexing.rs
deleted file mode 100644
index d253c97e7..000000000
--- a/crates/ra_syntax/src/string_lexing.rs
+++ /dev/null
@@ -1,513 +0,0 @@
1use self::CharComponentKind::*;
2use rowan::{TextRange, TextUnit};
3
4pub fn parse_byte_string_literal(src: &str) -> ByteStringComponentIterator {
5 ByteStringComponentIterator {
6 parser: Parser::new(src),
7 has_closing_quote: false,
8 }
9}
10
11pub struct ByteStringComponentIterator<'a> {
12 parser: Parser<'a>,
13 pub has_closing_quote: bool,
14}
15
16impl<'a> Iterator for ByteStringComponentIterator<'a> {
17 type Item = StringComponent;
18 fn next(&mut self) -> Option<StringComponent> {
19 if self.parser.pos == 0 {
20 assert!(
21 self.parser.advance() == 'b',
22 "byte string literal should start with a `b`"
23 );
24
25 assert!(
26 self.parser.advance() == '"',
27 "byte string literal should start with a `b`, followed by double quotes"
28 );
29 }
30
31 if let Some(component) = self.parser.parse_string_component() {
32 return Some(component);
33 }
34
35 // We get here when there are no char components left to parse
36 if self.parser.peek() == Some('"') {
37 self.parser.advance();
38 self.has_closing_quote = true;
39 }
40
41 assert!(
42 self.parser.peek() == None,
43 "byte string literal should leave no unparsed input: src = {}, pos = {}, length = {}",
44 self.parser.src,
45 self.parser.pos,
46 self.parser.src.len()
47 );
48
49 None
50 }
51}
52
53pub fn parse_string_literal(src: &str) -> StringComponentIterator {
54 StringComponentIterator {
55 parser: Parser::new(src),
56 has_closing_quote: false,
57 }
58}
59
60#[derive(Debug, Eq, PartialEq, Clone)]
61pub struct StringComponent {
62 pub range: TextRange,
63 pub kind: StringComponentKind,
64}
65
66impl StringComponent {
67 fn new(range: TextRange, kind: StringComponentKind) -> StringComponent {
68 StringComponent { range, kind }
69 }
70}
71
72#[derive(Debug, Eq, PartialEq, Clone)]
73pub enum StringComponentKind {
74 IgnoreNewline,
75 Char(CharComponentKind),
76}
77
78pub struct StringComponentIterator<'a> {
79 parser: Parser<'a>,
80 pub has_closing_quote: bool,
81}
82
83impl<'a> Iterator for StringComponentIterator<'a> {
84 type Item = StringComponent;
85 fn next(&mut self) -> Option<StringComponent> {
86 if self.parser.pos == 0 {
87 assert!(
88 self.parser.advance() == '"',
89 "string literal should start with double quotes"
90 );
91 }
92
93 if let Some(component) = self.parser.parse_string_component() {
94 return Some(component);
95 }
96
97 // We get here when there are no char components left to parse
98 if self.parser.peek() == Some('"') {
99 self.parser.advance();
100 self.has_closing_quote = true;
101 }
102
103 assert!(
104 self.parser.peek() == None,
105 "string literal should leave no unparsed input: src = {}, pos = {}, length = {}",
106 self.parser.src,
107 self.parser.pos,
108 self.parser.src.len()
109 );
110
111 None
112 }
113}
114
115pub fn parse_byte_literal(src: &str) -> ByteComponentIterator {
116 ByteComponentIterator {
117 parser: Parser::new(src),
118 has_closing_quote: false,
119 }
120}
121
122pub struct ByteComponentIterator<'a> {
123 parser: Parser<'a>,
124 pub has_closing_quote: bool,
125}
126
127impl<'a> Iterator for ByteComponentIterator<'a> {
128 type Item = CharComponent;
129 fn next(&mut self) -> Option<CharComponent> {
130 if self.parser.pos == 0 {
131 assert!(
132 self.parser.advance() == 'b',
133 "Byte literal should start with a `b`"
134 );
135
136 assert!(
137 self.parser.advance() == '\'',
138 "Byte literal should start with a `b`, followed by a quote"
139 );
140 }
141
142
143 if let Some(component) = self.parser.parse_char_component() {
144 return Some(component);
145 }
146
147 // We get here when there are no char components left to parse
148 if self.parser.peek() == Some('\'') {
149 self.parser.advance();
150 self.has_closing_quote = true;
151 }
152
153 assert!(
154 self.parser.peek() == None,
155 "byte literal should leave no unparsed input: src = {}, pos = {}, length = {}",
156 self.parser.src,
157 self.parser.pos,
158 self.parser.src.len()
159 );
160
161 None
162 }
163}
164
165pub fn parse_char_literal(src: &str) -> CharComponentIterator {
166 CharComponentIterator {
167 parser: Parser::new(src),
168 has_closing_quote: false,
169 }
170}
171
172#[derive(Debug, Eq, PartialEq, Clone)]
173pub struct CharComponent {
174 pub range: TextRange,
175 pub kind: CharComponentKind,
176}
177
178impl CharComponent {
179 fn new(range: TextRange, kind: CharComponentKind) -> CharComponent {
180 CharComponent { range, kind }
181 }
182}
183
184#[derive(Debug, Eq, PartialEq, Clone)]
185pub enum CharComponentKind {
186 CodePoint,
187 AsciiEscape,
188 AsciiCodeEscape,
189 UnicodeEscape,
190}
191
192pub struct CharComponentIterator<'a> {
193 parser: Parser<'a>,
194 pub has_closing_quote: bool,
195}
196
197impl<'a> Iterator for CharComponentIterator<'a> {
198 type Item = CharComponent;
199 fn next(&mut self) -> Option<CharComponent> {
200 if self.parser.pos == 0 {
201 assert!(
202 self.parser.advance() == '\'',
203 "char literal should start with a quote"
204 );
205 }
206
207 if let Some(component) = self.parser.parse_char_component() {
208 return Some(component);
209 }
210
211 // We get here when there are no char components left to parse
212 if self.parser.peek() == Some('\'') {
213 self.parser.advance();
214 self.has_closing_quote = true;
215 }
216
217 assert!(
218 self.parser.peek() == None,
219 "char literal should leave no unparsed input: src = {}, pos = {}, length = {}",
220 self.parser.src,
221 self.parser.pos,
222 self.parser.src.len()
223 );
224
225 None
226 }
227}
228
229pub struct Parser<'a> {
230 src: &'a str,
231 pos: usize,
232}
233
234impl<'a> Parser<'a> {
235 pub fn new(src: &'a str) -> Parser<'a> {
236 Parser { src, pos: 0 }
237 }
238
239 // Utility methods
240
241 pub fn peek(&self) -> Option<char> {
242 if self.pos == self.src.len() {
243 return None;
244 }
245
246 self.src[self.pos..].chars().next()
247 }
248
249 pub fn advance(&mut self) -> char {
250 let next = self
251 .peek()
252 .expect("cannot advance if end of input is reached");
253 self.pos += next.len_utf8();
254 next
255 }
256
257 pub fn skip_whitespace(&mut self) {
258 while self.peek().map(|c| c.is_whitespace()) == Some(true) {
259 self.advance();
260 }
261 }
262
263 pub fn get_pos(&self) -> TextUnit {
264 (self.pos as u32).into()
265 }
266
267 // Char parsing methods
268
269 fn parse_unicode_escape(&mut self, start: TextUnit) -> CharComponent {
270 match self.peek() {
271 Some('{') => {
272 self.advance();
273
274 // Parse anything until we reach `}`
275 while let Some(next) = self.peek() {
276 self.advance();
277 if next == '}' {
278 break;
279 }
280 }
281
282 let end = self.get_pos();
283 CharComponent::new(TextRange::from_to(start, end), UnicodeEscape)
284 }
285 Some(_) | None => {
286 let end = self.get_pos();
287 CharComponent::new(TextRange::from_to(start, end), UnicodeEscape)
288 }
289 }
290 }
291
292 fn parse_ascii_code_escape(&mut self, start: TextUnit) -> CharComponent {
293 let code_start = self.get_pos();
294 while let Some(next) = self.peek() {
295 if next == '\'' || (self.get_pos() - code_start == 2.into()) {
296 break;
297 }
298
299 self.advance();
300 }
301
302 let end = self.get_pos();
303 CharComponent::new(TextRange::from_to(start, end), AsciiCodeEscape)
304 }
305
306 fn parse_escape(&mut self, start: TextUnit) -> CharComponent {
307 if self.peek().is_none() {
308 return CharComponent::new(TextRange::from_to(start, start), AsciiEscape);
309 }
310
311 let next = self.advance();
312 let end = self.get_pos();
313 let range = TextRange::from_to(start, end);
314 match next {
315 'x' => self.parse_ascii_code_escape(start),
316 'u' => self.parse_unicode_escape(start),
317 _ => CharComponent::new(range, AsciiEscape),
318 }
319 }
320
321 pub fn parse_char_component(&mut self) -> Option<CharComponent> {
322 let next = self.peek()?;
323
324 // Ignore character close
325 if next == '\'' {
326 return None;
327 }
328
329 let start = self.get_pos();
330 self.advance();
331
332 if next == '\\' {
333 Some(self.parse_escape(start))
334 } else {
335 let end = self.get_pos();
336 Some(CharComponent::new(
337 TextRange::from_to(start, end),
338 CodePoint,
339 ))
340 }
341 }
342
343 pub fn parse_ignore_newline(&mut self, start: TextUnit) -> Option<StringComponent> {
344 // In string literals, when a `\` occurs immediately before the newline, the `\`,
345 // the newline, and all whitespace at the beginning of the next line are ignored
346 match self.peek() {
347 Some('\n') | Some('\r') => {
348 self.skip_whitespace();
349 Some(StringComponent::new(
350 TextRange::from_to(start, self.get_pos()),
351 StringComponentKind::IgnoreNewline,
352 ))
353 }
354 _ => None,
355 }
356 }
357
358 pub fn parse_string_component(&mut self) -> Option<StringComponent> {
359 let next = self.peek()?;
360
361 // Ignore string close
362 if next == '"' {
363 return None;
364 }
365
366 let start = self.get_pos();
367 self.advance();
368
369 if next == '\\' {
370 // Strings can use `\` to ignore newlines, so we first try to parse one of those
371 // before falling back to parsing char escapes
372 self.parse_ignore_newline(start).or_else(|| {
373 let char_component = self.parse_escape(start);
374 Some(StringComponent::new(
375 char_component.range,
376 StringComponentKind::Char(char_component.kind),
377 ))
378 })
379 } else {
380 let end = self.get_pos();
381 Some(StringComponent::new(
382 TextRange::from_to(start, end),
383 StringComponentKind::Char(CodePoint),
384 ))
385 }
386 }
387}
388
389#[cfg(test)]
390mod tests {
391 use super::*;
392
393 fn parse(src: &str) -> (bool, Vec<CharComponent>) {
394 let component_iterator = &mut super::parse_char_literal(src);
395 let components: Vec<_> = component_iterator.collect();
396 (component_iterator.has_closing_quote, components)
397 }
398
399 fn unclosed_char_component(src: &str) -> CharComponent {
400 let (has_closing_quote, components) = parse(src);
401 assert!(!has_closing_quote, "char should not have closing quote");
402 assert!(components.len() == 1);
403 components[0].clone()
404 }
405
406 fn closed_char_component(src: &str) -> CharComponent {
407 let (has_closing_quote, components) = parse(src);
408 assert!(has_closing_quote, "char should have closing quote");
409 assert!(
410 components.len() == 1,
411 "Literal: {}\nComponents: {:#?}",
412 src,
413 components
414 );
415 components[0].clone()
416 }
417
418 fn closed_char_components(src: &str) -> Vec<CharComponent> {
419 let (has_closing_quote, components) = parse(src);
420 assert!(has_closing_quote, "char should have closing quote");
421 components
422 }
423
424 fn range_closed(src: &str) -> TextRange {
425 TextRange::from_to(1.into(), (src.len() as u32 - 1).into())
426 }
427
428 fn range_unclosed(src: &str) -> TextRange {
429 TextRange::from_to(1.into(), (src.len() as u32).into())
430 }
431
432 #[test]
433 fn test_unicode_escapes() {
434 let unicode_escapes = &[r"{DEAD}", "{BEEF}", "{FF}", "{}", ""];
435 for escape in unicode_escapes {
436 let escape_sequence = format!(r"'\u{}'", escape);
437 let component = closed_char_component(&escape_sequence);
438 let expected_range = range_closed(&escape_sequence);
439 assert_eq!(component.kind, CharComponentKind::UnicodeEscape);
440 assert_eq!(component.range, expected_range);
441 }
442 }
443
444 #[test]
445 fn test_unicode_escapes_unclosed() {
446 let unicode_escapes = &["{DEAD", "{BEEF", "{FF"];
447 for escape in unicode_escapes {
448 let escape_sequence = format!(r"'\u{}'", escape);
449 let component = unclosed_char_component(&escape_sequence);
450 let expected_range = range_unclosed(&escape_sequence);
451 assert_eq!(component.kind, CharComponentKind::UnicodeEscape);
452 assert_eq!(component.range, expected_range);
453 }
454 }
455
456 #[test]
457 fn test_empty_char() {
458 let (has_closing_quote, components) = parse("''");
459 assert!(has_closing_quote, "char should have closing quote");
460 assert!(components.len() == 0);
461 }
462
463 #[test]
464 fn test_unclosed_char() {
465 let component = unclosed_char_component("'a");
466 assert!(component.kind == CodePoint);
467 assert!(component.range == TextRange::from_to(1.into(), 2.into()));
468 }
469
470 #[test]
471 fn test_digit_escapes() {
472 let literals = &[r"", r"5", r"55"];
473
474 for literal in literals {
475 let lit_text = format!(r"'\x{}'", literal);
476 let component = closed_char_component(&lit_text);
477 assert!(component.kind == CharComponentKind::AsciiCodeEscape);
478 assert!(component.range == range_closed(&lit_text));
479 }
480
481 // More than 2 digits starts a new codepoint
482 let components = closed_char_components(r"'\x555'");
483 assert!(components.len() == 2);
484 assert!(components[1].kind == CharComponentKind::CodePoint);
485 }
486
487 #[test]
488 fn test_ascii_escapes() {
489 let literals = &[
490 r"\'", "\\\"", // equivalent to \"
491 r"\n", r"\r", r"\t", r"\\", r"\0",
492 ];
493
494 for literal in literals {
495 let lit_text = format!("'{}'", literal);
496 let component = closed_char_component(&lit_text);
497 assert!(component.kind == CharComponentKind::AsciiEscape);
498 assert!(component.range == range_closed(&lit_text));
499 }
500 }
501
502 #[test]
503 fn test_no_escapes() {
504 let literals = &['"', 'n', 'r', 't', '0', 'x', 'u'];
505
506 for &literal in literals {
507 let lit_text = format!("'{}'", literal);
508 let component = closed_char_component(&lit_text);
509 assert!(component.kind == CharComponentKind::CodePoint);
510 assert!(component.range == range_closed(&lit_text));
511 }
512 }
513}
generated by cgit v1.2.3 (git 2.25.1) at 2024-09-20 00:45:01 +0100