Merge #2911

2911: Implement collecting errors while tokenizing r=matklad a=Veetaha

Now we are collecting errors from `rustc_lexer` and returning them in `ParsedToken { token, error }` and `ParsedTokens { tokens, errors }` structures **([UPD]: this is now simplified, see updates bellow)**.

The main changes are introduced in `ra_syntax/parsing/lexer.rs`. It now exposes the following functions and types:

```rust
pub fn tokenize(text: &str) -> ParsedTokens;
pub fn tokenize_append(text: &str, parsed_tokens_to_append_to: &mut ParsedTokens);
pub fn first_token(text: &str) -> Option<ParsedToken>; // allows any number of tokens in text
pub fn single_token(text: &str) -> Option<ParsedToken>; // allows only a single token in text

pub struct ParsedToken  { pub token: Token,       pub error: Option<SyntaxError> }
pub struct ParsedTokens { pub tokens: Vec<Token>, pub errors: Vec<SyntaxError>   }

pub enum TokenizeError { /* Simple enum which reflects rustc_lexer tokenization errors */ }
```
In the first commit I implemented it with iterators, but then decided that since this crate is ad hoc for `rust-analyzer` and we clearly see the places of its usage it would be better to simplify it to vectors.

This is currently WIP, because I want to add tests for error messages generated by the lexer.
I'd like to listen to you thoughts how to define these tests in `ra_syntax/test-data` dir.

Related issues: #223 

**[UPD]**

After the PR review the API was simplified:
```rust
pub fn tokenize(text: &str) -> (Vec<Token>, Vec<SyntaxError>);
// Both lex functions do not check for unescape errors
pub fn lex_single_syntax_kind(text: &str) -> Option<(SyntaxKind, Option<SyntaxError>)>;
pub fn lex_single_valid_syntax_kind(text: &str) -> Option<SyntaxKind>;

// This will be removed in the next PR in favour of simlifying `SyntaxError` to `(String, TextRange)`
pub enum TokenizeError { /* Simple enum which reflects rustc_lexer tokenization errors */ }

// this is private, but may be made public if such demand would exist in future (least privilege principle)
fn lex_first_token(text: &str) -> Option<(Token, Option<SyntaxError>)>;
```

Co-authored-by: Veetaha <[email protected]>

114 files changed, 326 insertions, 0 deletions

diff --git a/crates/ra_syntax/test_data/lexer/err/0001_unclosed_char_at_eof.rs b/crates/ra_syntax/test_data/lexer/err/0001_unclosed_char_at_eof.rs
new file mode 100644
index 000000000..ad2823b48
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0001_unclosed_char_at_eof.rs
@@ -0,0 +1 @@
+'
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0001_unclosed_char_at_eof.txt b/crates/ra_syntax/test_data/lexer/err/0001_unclosed_char_at_eof.txt
new file mode 100644
index 000000000..f24e1fd32
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0001_unclosed_char_at_eof.txt
@@ -0,0 +1,2 @@
+CHAR 1 "\'"
+> error[0; 1) token("\'") msg(Missing trailing `'` symbol to terminate the character literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0002_unclosed_char_with_ferris.rs b/crates/ra_syntax/test_data/lexer/err/0002_unclosed_char_with_ferris.rs
new file mode 100644
index 000000000..e264a4152
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0002_unclosed_char_with_ferris.rs
@@ -0,0 +1 @@
+'🦀
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0002_unclosed_char_with_ferris.txt b/crates/ra_syntax/test_data/lexer/err/0002_unclosed_char_with_ferris.txt
new file mode 100644
index 000000000..bd08cfc44
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0002_unclosed_char_with_ferris.txt
@@ -0,0 +1,2 @@
+CHAR 5 "\'🦀"
+> error[0; 5) token("\'🦀") msg(Missing trailing `'` symbol to terminate the character literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0003_unclosed_char_with_ascii_escape.rs b/crates/ra_syntax/test_data/lexer/err/0003_unclosed_char_with_ascii_escape.rs
new file mode 100644
index 000000000..cf74b4dad
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0003_unclosed_char_with_ascii_escape.rs
@@ -0,0 +1 @@
+'\x7f
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0003_unclosed_char_with_ascii_escape.txt b/crates/ra_syntax/test_data/lexer/err/0003_unclosed_char_with_ascii_escape.txt
new file mode 100644
index 000000000..0ee22912d
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0003_unclosed_char_with_ascii_escape.txt
@@ -0,0 +1,2 @@
+CHAR 5 "\'\\x7f"
+> error[0; 5) token("\'\\x7f") msg(Missing trailing `'` symbol to terminate the character literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0004_unclosed_char_with_unicode_escape.rs b/crates/ra_syntax/test_data/lexer/err/0004_unclosed_char_with_unicode_escape.rs
new file mode 100644
index 000000000..50be91f68
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0004_unclosed_char_with_unicode_escape.rs
@@ -0,0 +1 @@
+'\u{20AA}
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0004_unclosed_char_with_unicode_escape.txt b/crates/ra_syntax/test_data/lexer/err/0004_unclosed_char_with_unicode_escape.txt
new file mode 100644
index 000000000..96fac42ce
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0004_unclosed_char_with_unicode_escape.txt
@@ -0,0 +1,2 @@
+CHAR 9 "\'\\u{20AA}"
+> error[0; 9) token("\'\\u{20AA}") msg(Missing trailing `'` symbol to terminate the character literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0005_unclosed_char_with_space.rs b/crates/ra_syntax/test_data/lexer/err/0005_unclosed_char_with_space.rs
new file mode 100644
index 000000000..309ecfe47
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0005_unclosed_char_with_space.rs
@@ -0,0 +1 @@
+' 
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0005_unclosed_char_with_space.txt b/crates/ra_syntax/test_data/lexer/err/0005_unclosed_char_with_space.txt
new file mode 100644
index 000000000..2059f3f81
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0005_unclosed_char_with_space.txt
@@ -0,0 +1,2 @@
+CHAR 2 "\' "
+> error[0; 2) token("\' ") msg(Missing trailing `'` symbol to terminate the character literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0006_unclosed_char_with_slash.rs b/crates/ra_syntax/test_data/lexer/err/0006_unclosed_char_with_slash.rs
new file mode 100644
index 000000000..6ba258b10
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0006_unclosed_char_with_slash.rs
@@ -0,0 +1 @@
+'\
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0006_unclosed_char_with_slash.txt b/crates/ra_syntax/test_data/lexer/err/0006_unclosed_char_with_slash.txt
new file mode 100644
index 000000000..7dd376e59
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0006_unclosed_char_with_slash.txt
@@ -0,0 +1,2 @@
+CHAR 2 "\'\\"
+> error[0; 2) token("\'\\") msg(Missing trailing `'` symbol to terminate the character literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0007_unclosed_char_with_slash_n.rs b/crates/ra_syntax/test_data/lexer/err/0007_unclosed_char_with_slash_n.rs
new file mode 100644
index 000000000..78bef7e3e
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0007_unclosed_char_with_slash_n.rs
@@ -0,0 +1 @@
+'\n
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0007_unclosed_char_with_slash_n.txt b/crates/ra_syntax/test_data/lexer/err/0007_unclosed_char_with_slash_n.txt
new file mode 100644
index 000000000..ef7a0a147
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0007_unclosed_char_with_slash_n.txt
@@ -0,0 +1,2 @@
+CHAR 3 "\'\\n"
+> error[0; 3) token("\'\\n") msg(Missing trailing `'` symbol to terminate the character literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0008_unclosed_char_with_slash_single_quote.rs b/crates/ra_syntax/test_data/lexer/err/0008_unclosed_char_with_slash_single_quote.rs
new file mode 100644
index 000000000..a0e722065
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0008_unclosed_char_with_slash_single_quote.rs
@@ -0,0 +1 @@
+'\'
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0008_unclosed_char_with_slash_single_quote.txt b/crates/ra_syntax/test_data/lexer/err/0008_unclosed_char_with_slash_single_quote.txt
new file mode 100644
index 000000000..13fc5ea9a
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0008_unclosed_char_with_slash_single_quote.txt
@@ -0,0 +1,2 @@
+CHAR 3 "\'\\\'"
+> error[0; 3) token("\'\\\'") msg(Missing trailing `'` symbol to terminate the character literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0009_unclosed_byte_at_eof.rs b/crates/ra_syntax/test_data/lexer/err/0009_unclosed_byte_at_eof.rs
new file mode 100644
index 000000000..795dc7e25
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0009_unclosed_byte_at_eof.rs
@@ -0,0 +1 @@
+b'
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0009_unclosed_byte_at_eof.txt b/crates/ra_syntax/test_data/lexer/err/0009_unclosed_byte_at_eof.txt
new file mode 100644
index 000000000..269d68c74
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0009_unclosed_byte_at_eof.txt
@@ -0,0 +1,2 @@
+BYTE 2 "b\'"
+> error[0; 2) token("b\'") msg(Missing trailing `'` symbol to terminate the byte literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0010_unclosed_byte_with_ferris.rs b/crates/ra_syntax/test_data/lexer/err/0010_unclosed_byte_with_ferris.rs
new file mode 100644
index 000000000..c9230dc24
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0010_unclosed_byte_with_ferris.rs
@@ -0,0 +1 @@
+b'🦀
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0010_unclosed_byte_with_ferris.txt b/crates/ra_syntax/test_data/lexer/err/0010_unclosed_byte_with_ferris.txt
new file mode 100644
index 000000000..91a76e479
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0010_unclosed_byte_with_ferris.txt
@@ -0,0 +1,2 @@
+BYTE 6 "b\'🦀"
+> error[0; 6) token("b\'🦀") msg(Missing trailing `'` symbol to terminate the byte literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0011_unclosed_byte_with_ascii_escape.rs b/crates/ra_syntax/test_data/lexer/err/0011_unclosed_byte_with_ascii_escape.rs
new file mode 100644
index 000000000..d146a8090
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0011_unclosed_byte_with_ascii_escape.rs
@@ -0,0 +1 @@
+b'\x7f
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0011_unclosed_byte_with_ascii_escape.txt b/crates/ra_syntax/test_data/lexer/err/0011_unclosed_byte_with_ascii_escape.txt
new file mode 100644
index 000000000..b8c804a18
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0011_unclosed_byte_with_ascii_escape.txt
@@ -0,0 +1,2 @@
+BYTE 6 "b\'\\x7f"
+> error[0; 6) token("b\'\\x7f") msg(Missing trailing `'` symbol to terminate the byte literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0012_unclosed_byte_with_unicode_escape.rs b/crates/ra_syntax/test_data/lexer/err/0012_unclosed_byte_with_unicode_escape.rs
new file mode 100644
index 000000000..a3dec7c25
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0012_unclosed_byte_with_unicode_escape.rs
@@ -0,0 +1 @@
+b'\u{20AA}
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0012_unclosed_byte_with_unicode_escape.txt b/crates/ra_syntax/test_data/lexer/err/0012_unclosed_byte_with_unicode_escape.txt
new file mode 100644
index 000000000..dfca22a59
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0012_unclosed_byte_with_unicode_escape.txt
@@ -0,0 +1,2 @@
+BYTE 10 "b\'\\u{20AA}"
+> error[0; 10) token("b\'\\u{20AA}") msg(Missing trailing `'` symbol to terminate the byte literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0013_unclosed_byte_with_space.rs b/crates/ra_syntax/test_data/lexer/err/0013_unclosed_byte_with_space.rs
new file mode 100644
index 000000000..93b7f9c87
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0013_unclosed_byte_with_space.rs
@@ -0,0 +1 @@
+b' 
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0013_unclosed_byte_with_space.txt b/crates/ra_syntax/test_data/lexer/err/0013_unclosed_byte_with_space.txt
new file mode 100644
index 000000000..51a1cceab
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0013_unclosed_byte_with_space.txt
@@ -0,0 +1,2 @@
+BYTE 3 "b\' "
+> error[0; 3) token("b\' ") msg(Missing trailing `'` symbol to terminate the byte literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0014_unclosed_byte_with_slash.rs b/crates/ra_syntax/test_data/lexer/err/0014_unclosed_byte_with_slash.rs
new file mode 100644
index 000000000..abffa5037
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0014_unclosed_byte_with_slash.rs
@@ -0,0 +1 @@
+b'\
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0014_unclosed_byte_with_slash.txt b/crates/ra_syntax/test_data/lexer/err/0014_unclosed_byte_with_slash.txt
new file mode 100644
index 000000000..24e835c27
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0014_unclosed_byte_with_slash.txt
@@ -0,0 +1,2 @@
+BYTE 3 "b\'\\"
+> error[0; 3) token("b\'\\") msg(Missing trailing `'` symbol to terminate the byte literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0015_unclosed_byte_with_slash_n.rs b/crates/ra_syntax/test_data/lexer/err/0015_unclosed_byte_with_slash_n.rs
new file mode 100644
index 000000000..4f46836a9
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0015_unclosed_byte_with_slash_n.rs
@@ -0,0 +1 @@
+b'\n
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0015_unclosed_byte_with_slash_n.txt b/crates/ra_syntax/test_data/lexer/err/0015_unclosed_byte_with_slash_n.txt
new file mode 100644
index 000000000..f1e39a41b
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0015_unclosed_byte_with_slash_n.txt
@@ -0,0 +1,2 @@
+BYTE 4 "b\'\\n"
+> error[0; 4) token("b\'\\n") msg(Missing trailing `'` symbol to terminate the byte literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0016_unclosed_byte_with_slash_single_quote.rs b/crates/ra_syntax/test_data/lexer/err/0016_unclosed_byte_with_slash_single_quote.rs
new file mode 100644
index 000000000..645b641ee
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0016_unclosed_byte_with_slash_single_quote.rs
@@ -0,0 +1 @@
+b'\'
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0016_unclosed_byte_with_slash_single_quote.txt b/crates/ra_syntax/test_data/lexer/err/0016_unclosed_byte_with_slash_single_quote.txt
new file mode 100644
index 000000000..f8ffe815d
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0016_unclosed_byte_with_slash_single_quote.txt
@@ -0,0 +1,2 @@
+BYTE 4 "b\'\\\'"
+> error[0; 4) token("b\'\\\'") msg(Missing trailing `'` symbol to terminate the byte literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0017_unclosed_string_at_eof.rs b/crates/ra_syntax/test_data/lexer/err/0017_unclosed_string_at_eof.rs
new file mode 100644
index 000000000..9d68933c4
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0017_unclosed_string_at_eof.rs
@@ -0,0 +1 @@
+"
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0017_unclosed_string_at_eof.txt b/crates/ra_syntax/test_data/lexer/err/0017_unclosed_string_at_eof.txt
new file mode 100644
index 000000000..823daaf6f
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0017_unclosed_string_at_eof.txt
@@ -0,0 +1,2 @@
+STRING 1 "\""
+> error[0; 1) token("\"") msg(Missing trailing `"` symbol to terminate the string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0018_unclosed_string_with_ferris.rs b/crates/ra_syntax/test_data/lexer/err/0018_unclosed_string_with_ferris.rs
new file mode 100644
index 000000000..d439b8d2a
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0018_unclosed_string_with_ferris.rs
@@ -0,0 +1 @@
+"🦀
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0018_unclosed_string_with_ferris.txt b/crates/ra_syntax/test_data/lexer/err/0018_unclosed_string_with_ferris.txt
new file mode 100644
index 000000000..164580eb3
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0018_unclosed_string_with_ferris.txt
@@ -0,0 +1,2 @@
+STRING 5 "\"🦀"
+> error[0; 5) token("\"🦀") msg(Missing trailing `"` symbol to terminate the string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0019_unclosed_string_with_ascii_escape.rs b/crates/ra_syntax/test_data/lexer/err/0019_unclosed_string_with_ascii_escape.rs
new file mode 100644
index 000000000..56186a344
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0019_unclosed_string_with_ascii_escape.rs
@@ -0,0 +1 @@
+"\x7f
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0019_unclosed_string_with_ascii_escape.txt b/crates/ra_syntax/test_data/lexer/err/0019_unclosed_string_with_ascii_escape.txt
new file mode 100644
index 000000000..4453827c3
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0019_unclosed_string_with_ascii_escape.txt
@@ -0,0 +1,2 @@
+STRING 5 "\"\\x7f"
+> error[0; 5) token("\"\\x7f") msg(Missing trailing `"` symbol to terminate the string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0020_unclosed_string_with_unicode_escape.rs b/crates/ra_syntax/test_data/lexer/err/0020_unclosed_string_with_unicode_escape.rs
new file mode 100644
index 000000000..ed24095c3
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0020_unclosed_string_with_unicode_escape.rs
@@ -0,0 +1 @@
+"\u{20AA}
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0020_unclosed_string_with_unicode_escape.txt b/crates/ra_syntax/test_data/lexer/err/0020_unclosed_string_with_unicode_escape.txt
new file mode 100644
index 000000000..aa614f304
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0020_unclosed_string_with_unicode_escape.txt
@@ -0,0 +1,2 @@
+STRING 9 "\"\\u{20AA}"
+> error[0; 9) token("\"\\u{20AA}") msg(Missing trailing `"` symbol to terminate the string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0021_unclosed_string_with_space.rs b/crates/ra_syntax/test_data/lexer/err/0021_unclosed_string_with_space.rs
new file mode 100644
index 000000000..72cdc841f
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0021_unclosed_string_with_space.rs
@@ -0,0 +1 @@
+" 
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0021_unclosed_string_with_space.txt b/crates/ra_syntax/test_data/lexer/err/0021_unclosed_string_with_space.txt
new file mode 100644
index 000000000..b7db1236f
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0021_unclosed_string_with_space.txt
@@ -0,0 +1,2 @@
+STRING 2 "\" "
+> error[0; 2) token("\" ") msg(Missing trailing `"` symbol to terminate the string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0022_unclosed_string_with_slash.rs b/crates/ra_syntax/test_data/lexer/err/0022_unclosed_string_with_slash.rs
new file mode 100644
index 000000000..00a258400
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0022_unclosed_string_with_slash.rs
@@ -0,0 +1 @@
+"\
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0022_unclosed_string_with_slash.txt b/crates/ra_syntax/test_data/lexer/err/0022_unclosed_string_with_slash.txt
new file mode 100644
index 000000000..9d3df3799
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0022_unclosed_string_with_slash.txt
@@ -0,0 +1,2 @@
+STRING 2 "\"\\"
+> error[0; 2) token("\"\\") msg(Missing trailing `"` symbol to terminate the string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0023_unclosed_string_with_slash_n.rs b/crates/ra_syntax/test_data/lexer/err/0023_unclosed_string_with_slash_n.rs
new file mode 100644
index 000000000..a0c29b8cf
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0023_unclosed_string_with_slash_n.rs
@@ -0,0 +1 @@
+"\n
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0023_unclosed_string_with_slash_n.txt b/crates/ra_syntax/test_data/lexer/err/0023_unclosed_string_with_slash_n.txt
new file mode 100644
index 000000000..e3eb672b6
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0023_unclosed_string_with_slash_n.txt
@@ -0,0 +1,2 @@
+STRING 3 "\"\\n"
+> error[0; 3) token("\"\\n") msg(Missing trailing `"` symbol to terminate the string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0024_unclosed_string_with_slash_double_quote.rs b/crates/ra_syntax/test_data/lexer/err/0024_unclosed_string_with_slash_double_quote.rs
new file mode 100644
index 000000000..403c2d6dd
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0024_unclosed_string_with_slash_double_quote.rs
@@ -0,0 +1 @@
+"\"
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0024_unclosed_string_with_slash_double_quote.txt b/crates/ra_syntax/test_data/lexer/err/0024_unclosed_string_with_slash_double_quote.txt
new file mode 100644
index 000000000..041d7fb6e
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0024_unclosed_string_with_slash_double_quote.txt
@@ -0,0 +1,2 @@
+STRING 3 "\"\\\""
+> error[0; 3) token("\"\\\"") msg(Missing trailing `"` symbol to terminate the string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0025_unclosed_byte_string_at_eof.rs b/crates/ra_syntax/test_data/lexer/err/0025_unclosed_byte_string_at_eof.rs
new file mode 100644
index 000000000..36f4f4321
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0025_unclosed_byte_string_at_eof.rs
@@ -0,0 +1 @@
+b"
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0025_unclosed_byte_string_at_eof.txt b/crates/ra_syntax/test_data/lexer/err/0025_unclosed_byte_string_at_eof.txt
new file mode 100644
index 000000000..be7970a83
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0025_unclosed_byte_string_at_eof.txt
@@ -0,0 +1,2 @@
+BYTE_STRING 2 "b\""
+> error[0; 2) token("b\"") msg(Missing trailing `"` symbol to terminate the byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0026_unclosed_byte_string_with_ferris.rs b/crates/ra_syntax/test_data/lexer/err/0026_unclosed_byte_string_with_ferris.rs
new file mode 100644
index 000000000..3c23a0372
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0026_unclosed_byte_string_with_ferris.rs
@@ -0,0 +1 @@
+b"🦀
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0026_unclosed_byte_string_with_ferris.txt b/crates/ra_syntax/test_data/lexer/err/0026_unclosed_byte_string_with_ferris.txt
new file mode 100644
index 000000000..bf9aab132
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0026_unclosed_byte_string_with_ferris.txt
@@ -0,0 +1,2 @@
+BYTE_STRING 6 "b\"🦀"
+> error[0; 6) token("b\"🦀") msg(Missing trailing `"` symbol to terminate the byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0027_unclosed_byte_string_with_ascii_escape.rs b/crates/ra_syntax/test_data/lexer/err/0027_unclosed_byte_string_with_ascii_escape.rs
new file mode 100644
index 000000000..836c112c1
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0027_unclosed_byte_string_with_ascii_escape.rs
@@ -0,0 +1 @@
+b"\x7f
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0027_unclosed_byte_string_with_ascii_escape.txt b/crates/ra_syntax/test_data/lexer/err/0027_unclosed_byte_string_with_ascii_escape.txt
new file mode 100644
index 000000000..76e16d7d3
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0027_unclosed_byte_string_with_ascii_escape.txt
@@ -0,0 +1,2 @@
+BYTE_STRING 6 "b\"\\x7f"
+> error[0; 6) token("b\"\\x7f") msg(Missing trailing `"` symbol to terminate the byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0028_unclosed_byte_string_with_unicode_escape.rs b/crates/ra_syntax/test_data/lexer/err/0028_unclosed_byte_string_with_unicode_escape.rs
new file mode 100644
index 000000000..1c6df1d00
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0028_unclosed_byte_string_with_unicode_escape.rs
@@ -0,0 +1 @@
+b"\u{20AA}
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0028_unclosed_byte_string_with_unicode_escape.txt b/crates/ra_syntax/test_data/lexer/err/0028_unclosed_byte_string_with_unicode_escape.txt
new file mode 100644
index 000000000..09adffa16
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0028_unclosed_byte_string_with_unicode_escape.txt
@@ -0,0 +1,2 @@
+BYTE_STRING 10 "b\"\\u{20AA}"
+> error[0; 10) token("b\"\\u{20AA}") msg(Missing trailing `"` symbol to terminate the byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0029_unclosed_byte_string_with_space.rs b/crates/ra_syntax/test_data/lexer/err/0029_unclosed_byte_string_with_space.rs
new file mode 100644
index 000000000..d6898541e
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0029_unclosed_byte_string_with_space.rs
@@ -0,0 +1 @@
+b" 
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0029_unclosed_byte_string_with_space.txt b/crates/ra_syntax/test_data/lexer/err/0029_unclosed_byte_string_with_space.txt
new file mode 100644
index 000000000..fcb7253c8
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0029_unclosed_byte_string_with_space.txt
@@ -0,0 +1,2 @@
+BYTE_STRING 3 "b\" "
+> error[0; 3) token("b\" ") msg(Missing trailing `"` symbol to terminate the byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0030_unclosed_byte_string_with_slash.rs b/crates/ra_syntax/test_data/lexer/err/0030_unclosed_byte_string_with_slash.rs
new file mode 100644
index 000000000..cce661538
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0030_unclosed_byte_string_with_slash.rs
@@ -0,0 +1 @@
+b"\
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0030_unclosed_byte_string_with_slash.txt b/crates/ra_syntax/test_data/lexer/err/0030_unclosed_byte_string_with_slash.txt
new file mode 100644
index 000000000..0a1b3e269
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0030_unclosed_byte_string_with_slash.txt
@@ -0,0 +1,2 @@
+BYTE_STRING 3 "b\"\\"
+> error[0; 3) token("b\"\\") msg(Missing trailing `"` symbol to terminate the byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0031_unclosed_byte_string_with_slash_n.rs b/crates/ra_syntax/test_data/lexer/err/0031_unclosed_byte_string_with_slash_n.rs
new file mode 100644
index 000000000..5e680aabb
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0031_unclosed_byte_string_with_slash_n.rs
@@ -0,0 +1 @@
+b"\n
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0031_unclosed_byte_string_with_slash_n.txt b/crates/ra_syntax/test_data/lexer/err/0031_unclosed_byte_string_with_slash_n.txt
new file mode 100644
index 000000000..1fb89d2b6
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0031_unclosed_byte_string_with_slash_n.txt
@@ -0,0 +1,2 @@
+BYTE_STRING 4 "b\"\\n"
+> error[0; 4) token("b\"\\n") msg(Missing trailing `"` symbol to terminate the byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0032_unclosed_byte_string_with_slash_double_quote.rs b/crates/ra_syntax/test_data/lexer/err/0032_unclosed_byte_string_with_slash_double_quote.rs
new file mode 100644
index 000000000..f2ff58ba9
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0032_unclosed_byte_string_with_slash_double_quote.rs
@@ -0,0 +1 @@
+b"\"
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0032_unclosed_byte_string_with_slash_double_quote.txt b/crates/ra_syntax/test_data/lexer/err/0032_unclosed_byte_string_with_slash_double_quote.txt
new file mode 100644
index 000000000..718d36992
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0032_unclosed_byte_string_with_slash_double_quote.txt
@@ -0,0 +1,2 @@
+BYTE_STRING 4 "b\"\\\""
+> error[0; 4) token("b\"\\\"") msg(Missing trailing `"` symbol to terminate the byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0033_unclosed_raw_string_at_eof.rs b/crates/ra_syntax/test_data/lexer/err/0033_unclosed_raw_string_at_eof.rs
new file mode 100644
index 000000000..557c59b62
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0033_unclosed_raw_string_at_eof.rs
@@ -0,0 +1 @@
+r##"
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0033_unclosed_raw_string_at_eof.txt b/crates/ra_syntax/test_data/lexer/err/0033_unclosed_raw_string_at_eof.txt
new file mode 100644
index 000000000..93348f548
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0033_unclosed_raw_string_at_eof.txt
@@ -0,0 +1,2 @@
+RAW_STRING 4 "r##\""
+> error[0; 4) token("r##\"") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0034_unclosed_raw_string_with_ferris.rs b/crates/ra_syntax/test_data/lexer/err/0034_unclosed_raw_string_with_ferris.rs
new file mode 100644
index 000000000..bd046e4bb
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0034_unclosed_raw_string_with_ferris.rs
@@ -0,0 +1 @@
+r##"🦀
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0034_unclosed_raw_string_with_ferris.txt b/crates/ra_syntax/test_data/lexer/err/0034_unclosed_raw_string_with_ferris.txt
new file mode 100644
index 000000000..42c70dfe8
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0034_unclosed_raw_string_with_ferris.txt
@@ -0,0 +1,2 @@
+RAW_STRING 8 "r##\"🦀"
+> error[0; 8) token("r##\"🦀") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0035_unclosed_raw_string_with_ascii_escape.rs b/crates/ra_syntax/test_data/lexer/err/0035_unclosed_raw_string_with_ascii_escape.rs
new file mode 100644
index 000000000..5bec883dc
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0035_unclosed_raw_string_with_ascii_escape.rs
@@ -0,0 +1 @@
+r##"\x7f
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0035_unclosed_raw_string_with_ascii_escape.txt b/crates/ra_syntax/test_data/lexer/err/0035_unclosed_raw_string_with_ascii_escape.txt
new file mode 100644
index 000000000..2bdeea0ff
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0035_unclosed_raw_string_with_ascii_escape.txt
@@ -0,0 +1,2 @@
+RAW_STRING 8 "r##\"\\x7f"
+> error[0; 8) token("r##\"\\x7f") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0036_unclosed_raw_string_with_unicode_escape.rs b/crates/ra_syntax/test_data/lexer/err/0036_unclosed_raw_string_with_unicode_escape.rs
new file mode 100644
index 000000000..bf05c3913
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0036_unclosed_raw_string_with_unicode_escape.rs
@@ -0,0 +1 @@
+r##"\u{20AA}
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0036_unclosed_raw_string_with_unicode_escape.txt b/crates/ra_syntax/test_data/lexer/err/0036_unclosed_raw_string_with_unicode_escape.txt
new file mode 100644
index 000000000..667d4d79f
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0036_unclosed_raw_string_with_unicode_escape.txt
@@ -0,0 +1,2 @@
+RAW_STRING 12 "r##\"\\u{20AA}"
+> error[0; 12) token("r##\"\\u{20AA}") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0037_unclosed_raw_string_with_space.rs b/crates/ra_syntax/test_data/lexer/err/0037_unclosed_raw_string_with_space.rs
new file mode 100644
index 000000000..f104bae4f
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0037_unclosed_raw_string_with_space.rs
@@ -0,0 +1 @@
+r##" 
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0037_unclosed_raw_string_with_space.txt b/crates/ra_syntax/test_data/lexer/err/0037_unclosed_raw_string_with_space.txt
new file mode 100644
index 000000000..dd9597a1a
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0037_unclosed_raw_string_with_space.txt
@@ -0,0 +1,2 @@
+RAW_STRING 5 "r##\" "
+> error[0; 5) token("r##\" ") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0038_unclosed_raw_string_with_slash.rs b/crates/ra_syntax/test_data/lexer/err/0038_unclosed_raw_string_with_slash.rs
new file mode 100644
index 000000000..9242077b8
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0038_unclosed_raw_string_with_slash.rs
@@ -0,0 +1 @@
+r##"\
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0038_unclosed_raw_string_with_slash.txt b/crates/ra_syntax/test_data/lexer/err/0038_unclosed_raw_string_with_slash.txt
new file mode 100644
index 000000000..6ac6e3d62
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0038_unclosed_raw_string_with_slash.txt
@@ -0,0 +1,2 @@
+RAW_STRING 5 "r##\"\\"
+> error[0; 5) token("r##\"\\") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0039_unclosed_raw_string_with_slash_n.rs b/crates/ra_syntax/test_data/lexer/err/0039_unclosed_raw_string_with_slash_n.rs
new file mode 100644
index 000000000..db1c16f2b
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0039_unclosed_raw_string_with_slash_n.rs
@@ -0,0 +1 @@
+r##"\n
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0039_unclosed_raw_string_with_slash_n.txt b/crates/ra_syntax/test_data/lexer/err/0039_unclosed_raw_string_with_slash_n.txt
new file mode 100644
index 000000000..9d35443f5
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0039_unclosed_raw_string_with_slash_n.txt
@@ -0,0 +1,2 @@
+RAW_STRING 6 "r##\"\\n"
+> error[0; 6) token("r##\"\\n") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0040_unclosed_raw_byte_string_at_eof.rs b/crates/ra_syntax/test_data/lexer/err/0040_unclosed_raw_byte_string_at_eof.rs
new file mode 100644
index 000000000..ae5bae622
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0040_unclosed_raw_byte_string_at_eof.rs
@@ -0,0 +1 @@
+br##"
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0040_unclosed_raw_byte_string_at_eof.txt b/crates/ra_syntax/test_data/lexer/err/0040_unclosed_raw_byte_string_at_eof.txt
new file mode 100644
index 000000000..81fa39ea5
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0040_unclosed_raw_byte_string_at_eof.txt
@@ -0,0 +1,2 @@
+RAW_BYTE_STRING 5 "br##\""
+> error[0; 5) token("br##\"") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0041_unclosed_raw_byte_string_with_ferris.rs b/crates/ra_syntax/test_data/lexer/err/0041_unclosed_raw_byte_string_with_ferris.rs
new file mode 100644
index 000000000..9ef01207a
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0041_unclosed_raw_byte_string_with_ferris.rs
@@ -0,0 +1 @@
+br##"🦀
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0041_unclosed_raw_byte_string_with_ferris.txt b/crates/ra_syntax/test_data/lexer/err/0041_unclosed_raw_byte_string_with_ferris.txt
new file mode 100644
index 000000000..c2503a4d0
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0041_unclosed_raw_byte_string_with_ferris.txt
@@ -0,0 +1,2 @@
+RAW_BYTE_STRING 9 "br##\"🦀"
+> error[0; 9) token("br##\"🦀") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0042_unclosed_raw_byte_string_with_ascii_escape.rs b/crates/ra_syntax/test_data/lexer/err/0042_unclosed_raw_byte_string_with_ascii_escape.rs
new file mode 100644
index 000000000..d50270afe
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0042_unclosed_raw_byte_string_with_ascii_escape.rs
@@ -0,0 +1 @@
+br##"\x7f
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0042_unclosed_raw_byte_string_with_ascii_escape.txt b/crates/ra_syntax/test_data/lexer/err/0042_unclosed_raw_byte_string_with_ascii_escape.txt
new file mode 100644
index 000000000..3bd3d8152
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0042_unclosed_raw_byte_string_with_ascii_escape.txt
@@ -0,0 +1,2 @@
+RAW_BYTE_STRING 9 "br##\"\\x7f"
+> error[0; 9) token("br##\"\\x7f") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0043_unclosed_raw_byte_string_with_unicode_escape.rs b/crates/ra_syntax/test_data/lexer/err/0043_unclosed_raw_byte_string_with_unicode_escape.rs
new file mode 100644
index 000000000..90e299a1a
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0043_unclosed_raw_byte_string_with_unicode_escape.rs
@@ -0,0 +1 @@
+br##"\u{20AA}
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0043_unclosed_raw_byte_string_with_unicode_escape.txt b/crates/ra_syntax/test_data/lexer/err/0043_unclosed_raw_byte_string_with_unicode_escape.txt
new file mode 100644
index 000000000..a512f0428
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0043_unclosed_raw_byte_string_with_unicode_escape.txt
@@ -0,0 +1,2 @@
+RAW_BYTE_STRING 13 "br##\"\\u{20AA}"
+> error[0; 13) token("br##\"\\u{20AA}") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0044_unclosed_raw_byte_string_with_space.rs b/crates/ra_syntax/test_data/lexer/err/0044_unclosed_raw_byte_string_with_space.rs
new file mode 100644
index 000000000..14c602fd2
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0044_unclosed_raw_byte_string_with_space.rs
@@ -0,0 +1 @@
+br##" 
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0044_unclosed_raw_byte_string_with_space.txt b/crates/ra_syntax/test_data/lexer/err/0044_unclosed_raw_byte_string_with_space.txt
new file mode 100644
index 000000000..dc616a623
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0044_unclosed_raw_byte_string_with_space.txt
@@ -0,0 +1,2 @@
+RAW_BYTE_STRING 6 "br##\" "
+> error[0; 6) token("br##\" ") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0045_unclosed_raw_byte_string_with_slash.rs b/crates/ra_syntax/test_data/lexer/err/0045_unclosed_raw_byte_string_with_slash.rs
new file mode 100644
index 000000000..0b3c015d7
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0045_unclosed_raw_byte_string_with_slash.rs
@@ -0,0 +1 @@
+br##"\
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0045_unclosed_raw_byte_string_with_slash.txt b/crates/ra_syntax/test_data/lexer/err/0045_unclosed_raw_byte_string_with_slash.txt
new file mode 100644
index 000000000..debafe380
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0045_unclosed_raw_byte_string_with_slash.txt
@@ -0,0 +1,2 @@
+RAW_BYTE_STRING 6 "br##\"\\"
+> error[0; 6) token("br##\"\\") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0046_unclosed_raw_byte_string_with_slash_n.rs b/crates/ra_syntax/test_data/lexer/err/0046_unclosed_raw_byte_string_with_slash_n.rs
new file mode 100644
index 000000000..0d8b0e7ab
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0046_unclosed_raw_byte_string_with_slash_n.rs
@@ -0,0 +1 @@
+br##"\n
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0046_unclosed_raw_byte_string_with_slash_n.txt b/crates/ra_syntax/test_data/lexer/err/0046_unclosed_raw_byte_string_with_slash_n.txt
new file mode 100644
index 000000000..524e617b7
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0046_unclosed_raw_byte_string_with_slash_n.txt
@@ -0,0 +1,2 @@
+RAW_BYTE_STRING 7 "br##\"\\n"
+> error[0; 7) token("br##\"\\n") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0047_unstarted_raw_string_at_eof.rs b/crates/ra_syntax/test_data/lexer/err/0047_unstarted_raw_string_at_eof.rs
new file mode 100644
index 000000000..eddf8d080
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0047_unstarted_raw_string_at_eof.rs
@@ -0,0 +1 @@
+r##
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0047_unstarted_raw_string_at_eof.txt b/crates/ra_syntax/test_data/lexer/err/0047_unstarted_raw_string_at_eof.txt
new file mode 100644
index 000000000..00b046840
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0047_unstarted_raw_string_at_eof.txt
@@ -0,0 +1,2 @@
+RAW_STRING 3 "r##"
+> error[0; 3) token("r##") msg(Missing `"` symbol after `#` symbols to begin the raw string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0048_unstarted_raw_byte_string_at_eof.rs b/crates/ra_syntax/test_data/lexer/err/0048_unstarted_raw_byte_string_at_eof.rs
new file mode 100644
index 000000000..7e8cadf4f
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0048_unstarted_raw_byte_string_at_eof.rs
@@ -0,0 +1 @@
+br##
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0048_unstarted_raw_byte_string_at_eof.txt b/crates/ra_syntax/test_data/lexer/err/0048_unstarted_raw_byte_string_at_eof.txt
new file mode 100644
index 000000000..33b25e60f
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0048_unstarted_raw_byte_string_at_eof.txt
@@ -0,0 +1,2 @@
+RAW_BYTE_STRING 4 "br##"
+> error[0; 4) token("br##") msg(Missing `"` symbol after `#` symbols to begin the raw byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0049_unstarted_raw_string_with_ascii.rs b/crates/ra_syntax/test_data/lexer/err/0049_unstarted_raw_string_with_ascii.rs
new file mode 100644
index 000000000..534668a9b
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0049_unstarted_raw_string_with_ascii.rs
@@ -0,0 +1 @@
+r## I lack a quote!
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0049_unstarted_raw_string_with_ascii.txt b/crates/ra_syntax/test_data/lexer/err/0049_unstarted_raw_string_with_ascii.txt
new file mode 100644
index 000000000..782dfd974
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0049_unstarted_raw_string_with_ascii.txt
@@ -0,0 +1,10 @@
+RAW_STRING 4 "r## "
+IDENT 1 "I"
+WHITESPACE 1 " "
+IDENT 4 "lack"
+WHITESPACE 1 " "
+IDENT 1 "a"
+WHITESPACE 1 " "
+IDENT 5 "quote"
+EXCL 1 "!"
+> error[0; 4) token("r## ") msg(Missing `"` symbol after `#` symbols to begin the raw string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0050_unstarted_raw_byte_string_with_ascii.rs b/crates/ra_syntax/test_data/lexer/err/0050_unstarted_raw_byte_string_with_ascii.rs
new file mode 100644
index 000000000..d9b55455a
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0050_unstarted_raw_byte_string_with_ascii.rs
@@ -0,0 +1 @@
+br## I lack a quote!
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0050_unstarted_raw_byte_string_with_ascii.txt b/crates/ra_syntax/test_data/lexer/err/0050_unstarted_raw_byte_string_with_ascii.txt
new file mode 100644
index 000000000..59c40cd65
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0050_unstarted_raw_byte_string_with_ascii.txt
@@ -0,0 +1,10 @@
+RAW_BYTE_STRING 5 "br## "
+IDENT 1 "I"
+WHITESPACE 1 " "
+IDENT 4 "lack"
+WHITESPACE 1 " "
+IDENT 1 "a"
+WHITESPACE 1 " "
+IDENT 5 "quote"
+EXCL 1 "!"
+> error[0; 5) token("br## ") msg(Missing `"` symbol after `#` symbols to begin the raw byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0051_unclosed_block_comment_at_eof.rs b/crates/ra_syntax/test_data/lexer/err/0051_unclosed_block_comment_at_eof.rs
new file mode 100644
index 000000000..22e83649f
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0051_unclosed_block_comment_at_eof.rs
@@ -0,0 +1 @@
+/*
\ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0051_unclosed_block_comment_at_eof.txt b/crates/ra_syntax/test_data/lexer/err/0051_unclosed_block_comment_at_eof.txt
new file mode 100644
index 000000000..5d04cdaa4
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0051_unclosed_block_comment_at_eof.txt
@@ -0,0 +1,2 @@
+COMMENT 2 "/*"
+> error[0; 2) token("/*") msg(Missing trailing `*/` symbols to terminate the block comment)
diff --git a/crates/ra_syntax/test_data/lexer/err/0052_unclosed_block_comment_with_content.rs b/crates/ra_syntax/test_data/lexer/err/0052_unclosed_block_comment_with_content.rs
new file mode 100644
index 000000000..c45c2844d
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0052_unclosed_block_comment_with_content.rs
@@ -0,0 +1 @@
+/* comment
diff --git a/crates/ra_syntax/test_data/lexer/err/0052_unclosed_block_comment_with_content.txt b/crates/ra_syntax/test_data/lexer/err/0052_unclosed_block_comment_with_content.txt
new file mode 100644
index 000000000..8c6b678e3
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0052_unclosed_block_comment_with_content.txt
@@ -0,0 +1,2 @@
+COMMENT 11 "/* comment\n"
+> error[0; 11) token("/* comment\n") msg(Missing trailing `*/` symbols to terminate the block comment)
diff --git a/crates/ra_syntax/test_data/lexer/err/0053_unclosed_nested_block_comment_entirely.rs b/crates/ra_syntax/test_data/lexer/err/0053_unclosed_nested_block_comment_entirely.rs
new file mode 100644
index 000000000..3fcfc9660
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0053_unclosed_nested_block_comment_entirely.rs
@@ -0,0 +1 @@
+/* /* /*
diff --git a/crates/ra_syntax/test_data/lexer/err/0053_unclosed_nested_block_comment_entirely.txt b/crates/ra_syntax/test_data/lexer/err/0053_unclosed_nested_block_comment_entirely.txt
new file mode 100644
index 000000000..250de34d9
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0053_unclosed_nested_block_comment_entirely.txt
@@ -0,0 +1,2 @@
+COMMENT 9 "/* /* /*\n"
+> error[0; 9) token("/* /* /*\n") msg(Missing trailing `*/` symbols to terminate the block comment)
diff --git a/crates/ra_syntax/test_data/lexer/err/0054_unclosed_nested_block_comment_partially.rs b/crates/ra_syntax/test_data/lexer/err/0054_unclosed_nested_block_comment_partially.rs
new file mode 100644
index 000000000..26c898f01
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0054_unclosed_nested_block_comment_partially.rs
@@ -0,0 +1 @@
+/** /*! /* comment */ */
diff --git a/crates/ra_syntax/test_data/lexer/err/0054_unclosed_nested_block_comment_partially.txt b/crates/ra_syntax/test_data/lexer/err/0054_unclosed_nested_block_comment_partially.txt
new file mode 100644
index 000000000..f97f2a8c7
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0054_unclosed_nested_block_comment_partially.txt
@@ -0,0 +1,2 @@
+COMMENT 25 "/** /*! /* comment */ */\n"
+> error[0; 25) token("/** /*! /* comment */ */\n") msg(Missing trailing `*/` symbols to terminate the block comment)
diff --git a/crates/ra_syntax/test_data/lexer/err/0055_empty_int.rs b/crates/ra_syntax/test_data/lexer/err/0055_empty_int.rs
new file mode 100644
index 000000000..aa2a9fdca
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0055_empty_int.rs
@@ -0,0 +1,17 @@
+0b
+0o
+0x
+
+0b_
+0o_
+0x_
+
+0bnoDigit
+0onoDigit
+0xnoDigit
+
+0xG
+0xg
+
+0x_g
+0x_G
diff --git a/crates/ra_syntax/test_data/lexer/err/0055_empty_int.txt b/crates/ra_syntax/test_data/lexer/err/0055_empty_int.txt
new file mode 100644
index 000000000..2fe5bd950
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0055_empty_int.txt
@@ -0,0 +1,39 @@
+INT_NUMBER 2 "0b"
+WHITESPACE 1 "\n"
+INT_NUMBER 2 "0o"
+WHITESPACE 1 "\n"
+INT_NUMBER 2 "0x"
+WHITESPACE 2 "\n\n"
+INT_NUMBER 3 "0b_"
+WHITESPACE 1 "\n"
+INT_NUMBER 3 "0o_"
+WHITESPACE 1 "\n"
+INT_NUMBER 3 "0x_"
+WHITESPACE 2 "\n\n"
+INT_NUMBER 9 "0bnoDigit"
+WHITESPACE 1 "\n"
+INT_NUMBER 9 "0onoDigit"
+WHITESPACE 1 "\n"
+INT_NUMBER 9 "0xnoDigit"
+WHITESPACE 2 "\n\n"
+INT_NUMBER 3 "0xG"
+WHITESPACE 1 "\n"
+INT_NUMBER 3 "0xg"
+WHITESPACE 2 "\n\n"
+INT_NUMBER 4 "0x_g"
+WHITESPACE 1 "\n"
+INT_NUMBER 4 "0x_G"
+WHITESPACE 1 "\n"
+> error[0; 2) token("0b") msg(Missing digits after the integer base prefix)
+> error[3; 5) token("0o") msg(Missing digits after the integer base prefix)
+> error[6; 8) token("0x") msg(Missing digits after the integer base prefix)
+> error[10; 13) token("0b_") msg(Missing digits after the integer base prefix)
+> error[14; 17) token("0o_") msg(Missing digits after the integer base prefix)
+> error[18; 21) token("0x_") msg(Missing digits after the integer base prefix)
+> error[23; 32) token("0bnoDigit") msg(Missing digits after the integer base prefix)
+> error[33; 42) token("0onoDigit") msg(Missing digits after the integer base prefix)
+> error[43; 52) token("0xnoDigit") msg(Missing digits after the integer base prefix)
+> error[54; 57) token("0xG") msg(Missing digits after the integer base prefix)
+> error[58; 61) token("0xg") msg(Missing digits after the integer base prefix)
+> error[63; 67) token("0x_g") msg(Missing digits after the integer base prefix)
+> error[68; 72) token("0x_G") msg(Missing digits after the integer base prefix)
diff --git a/crates/ra_syntax/test_data/lexer/err/0056_empty_exponent.rs b/crates/ra_syntax/test_data/lexer/err/0056_empty_exponent.rs
new file mode 100644
index 000000000..286584c88
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0056_empty_exponent.rs
@@ -0,0 +1,22 @@
+0e
+0E
+
+42e+
+42e-
+42E+
+42E-
+
+42.e+
+42.e-
+42.E+
+42.E-
+
+42.2e+
+42.2e-
+42.2E+
+42.2E-
+
+42.2e+f32
+42.2e-f32
+42.2E+f32
+42.2E-f32
diff --git a/crates/ra_syntax/test_data/lexer/err/0056_empty_exponent.txt b/crates/ra_syntax/test_data/lexer/err/0056_empty_exponent.txt
new file mode 100644
index 000000000..ab35e20a5
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0056_empty_exponent.txt
@@ -0,0 +1,62 @@
+FLOAT_NUMBER 2 "0e"
+WHITESPACE 1 "\n"
+FLOAT_NUMBER 2 "0E"
+WHITESPACE 2 "\n\n"
+FLOAT_NUMBER 4 "42e+"
+WHITESPACE 1 "\n"
+FLOAT_NUMBER 4 "42e-"
+WHITESPACE 1 "\n"
+FLOAT_NUMBER 4 "42E+"
+WHITESPACE 1 "\n"
+FLOAT_NUMBER 4 "42E-"
+WHITESPACE 2 "\n\n"
+INT_NUMBER 2 "42"
+DOT 1 "."
+IDENT 1 "e"
+PLUS 1 "+"
+WHITESPACE 1 "\n"
+INT_NUMBER 2 "42"
+DOT 1 "."
+IDENT 1 "e"
+MINUS 1 "-"
+WHITESPACE 1 "\n"
+INT_NUMBER 2 "42"
+DOT 1 "."
+IDENT 1 "E"
+PLUS 1 "+"
+WHITESPACE 1 "\n"
+INT_NUMBER 2 "42"
+DOT 1 "."
+IDENT 1 "E"
+MINUS 1 "-"
+WHITESPACE 2 "\n\n"
+FLOAT_NUMBER 6 "42.2e+"
+WHITESPACE 1 "\n"
+FLOAT_NUMBER 6 "42.2e-"
+WHITESPACE 1 "\n"
+FLOAT_NUMBER 6 "42.2E+"
+WHITESPACE 1 "\n"
+FLOAT_NUMBER 6 "42.2E-"
+WHITESPACE 2 "\n\n"
+FLOAT_NUMBER 9 "42.2e+f32"
+WHITESPACE 1 "\n"
+FLOAT_NUMBER 9 "42.2e-f32"
+WHITESPACE 1 "\n"
+FLOAT_NUMBER 9 "42.2E+f32"
+WHITESPACE 1 "\n"
+FLOAT_NUMBER 9 "42.2E-f32"
+WHITESPACE 1 "\n"
+> error[0; 2) token("0e") msg(Missing digits after the exponent symbol)
+> error[3; 5) token("0E") msg(Missing digits after the exponent symbol)
+> error[7; 11) token("42e+") msg(Missing digits after the exponent symbol)
+> error[12; 16) token("42e-") msg(Missing digits after the exponent symbol)
+> error[17; 21) token("42E+") msg(Missing digits after the exponent symbol)
+> error[22; 26) token("42E-") msg(Missing digits after the exponent symbol)
+> error[53; 59) token("42.2e+") msg(Missing digits after the exponent symbol)
+> error[60; 66) token("42.2e-") msg(Missing digits after the exponent symbol)
+> error[67; 73) token("42.2E+") msg(Missing digits after the exponent symbol)
+> error[74; 80) token("42.2E-") msg(Missing digits after the exponent symbol)
+> error[82; 91) token("42.2e+f32") msg(Missing digits after the exponent symbol)
+> error[92; 101) token("42.2e-f32") msg(Missing digits after the exponent symbol)
+> error[102; 111) token("42.2E+f32") msg(Missing digits after the exponent symbol)
+> error[112; 121) token("42.2E-f32") msg(Missing digits after the exponent symbol)
diff --git a/crates/ra_syntax/test_data/lexer/err/0057_lifetime_strarts_with_a_number.rs b/crates/ra_syntax/test_data/lexer/err/0057_lifetime_strarts_with_a_number.rs
new file mode 100644
index 000000000..a7698a404
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0057_lifetime_strarts_with_a_number.rs
@@ -0,0 +1,2 @@
+'1
+'1lifetime
diff --git a/crates/ra_syntax/test_data/lexer/err/0057_lifetime_strarts_with_a_number.txt b/crates/ra_syntax/test_data/lexer/err/0057_lifetime_strarts_with_a_number.txt
new file mode 100644
index 000000000..89b38bfac
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0057_lifetime_strarts_with_a_number.txt
@@ -0,0 +1,6 @@
+LIFETIME 2 "\'1"
+WHITESPACE 1 "\n"
+LIFETIME 10 "\'1lifetime"
+WHITESPACE 1 "\n"
+> error[0; 2) token("\'1") msg(Lifetime name cannot start with a number)
+> error[3; 13) token("\'1lifetime") msg(Lifetime name cannot start with a number)