aboutsummaryrefslogtreecommitdiff
path: root/crates/ra_syntax
diff options
context:
space:
mode:
Diffstat (limited to 'crates/ra_syntax')
-rw-r--r--crates/ra_syntax/src/algo.rs2
-rw-r--r--crates/ra_syntax/src/lib.rs4
-rw-r--r--crates/ra_syntax/src/parsing.rs18
-rw-r--r--crates/ra_syntax/src/parsing/lexer.rs324
-rw-r--r--crates/ra_syntax/src/parsing/reparsing.rs55
-rw-r--r--crates/ra_syntax/src/parsing/text_tree_sink.rs4
-rw-r--r--crates/ra_syntax/src/syntax_error.rs60
-rw-r--r--crates/ra_syntax/src/syntax_node.rs9
-rw-r--r--crates/ra_syntax/src/tests.rs79
-rw-r--r--crates/ra_syntax/src/validation.rs6
-rw-r--r--crates/ra_syntax/test_data/lexer/0010_comments.rs3
-rw-r--r--crates/ra_syntax/test_data/lexer/0010_comments.txt6
-rw-r--r--crates/ra_syntax/test_data/lexer/0014_unclosed_char.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/0014_unclosed_char.txt1
-rw-r--r--crates/ra_syntax/test_data/lexer/0015_unclosed_string.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/0015_unclosed_string.txt1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0001_unclosed_char_at_eof.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0001_unclosed_char_at_eof.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0002_unclosed_char_with_ferris.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0002_unclosed_char_with_ferris.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0003_unclosed_char_with_ascii_escape.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0003_unclosed_char_with_ascii_escape.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0004_unclosed_char_with_unicode_escape.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0004_unclosed_char_with_unicode_escape.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0005_unclosed_char_with_space.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0005_unclosed_char_with_space.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0006_unclosed_char_with_slash.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0006_unclosed_char_with_slash.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0007_unclosed_char_with_slash_n.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0007_unclosed_char_with_slash_n.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0008_unclosed_char_with_slash_single_quote.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0008_unclosed_char_with_slash_single_quote.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0009_unclosed_byte_at_eof.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0009_unclosed_byte_at_eof.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0010_unclosed_byte_with_ferris.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0010_unclosed_byte_with_ferris.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0011_unclosed_byte_with_ascii_escape.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0011_unclosed_byte_with_ascii_escape.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0012_unclosed_byte_with_unicode_escape.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0012_unclosed_byte_with_unicode_escape.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0013_unclosed_byte_with_space.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0013_unclosed_byte_with_space.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0014_unclosed_byte_with_slash.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0014_unclosed_byte_with_slash.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0015_unclosed_byte_with_slash_n.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0015_unclosed_byte_with_slash_n.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0016_unclosed_byte_with_slash_single_quote.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0016_unclosed_byte_with_slash_single_quote.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0017_unclosed_string_at_eof.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0017_unclosed_string_at_eof.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0018_unclosed_string_with_ferris.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0018_unclosed_string_with_ferris.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0019_unclosed_string_with_ascii_escape.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0019_unclosed_string_with_ascii_escape.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0020_unclosed_string_with_unicode_escape.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0020_unclosed_string_with_unicode_escape.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0021_unclosed_string_with_space.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0021_unclosed_string_with_space.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0022_unclosed_string_with_slash.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0022_unclosed_string_with_slash.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0023_unclosed_string_with_slash_n.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0023_unclosed_string_with_slash_n.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0024_unclosed_string_with_slash_double_quote.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0024_unclosed_string_with_slash_double_quote.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0025_unclosed_byte_string_at_eof.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0025_unclosed_byte_string_at_eof.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0026_unclosed_byte_string_with_ferris.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0026_unclosed_byte_string_with_ferris.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0027_unclosed_byte_string_with_ascii_escape.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0027_unclosed_byte_string_with_ascii_escape.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0028_unclosed_byte_string_with_unicode_escape.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0028_unclosed_byte_string_with_unicode_escape.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0029_unclosed_byte_string_with_space.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0029_unclosed_byte_string_with_space.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0030_unclosed_byte_string_with_slash.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0030_unclosed_byte_string_with_slash.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0031_unclosed_byte_string_with_slash_n.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0031_unclosed_byte_string_with_slash_n.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0032_unclosed_byte_string_with_slash_double_quote.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0032_unclosed_byte_string_with_slash_double_quote.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0033_unclosed_raw_string_at_eof.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0033_unclosed_raw_string_at_eof.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0034_unclosed_raw_string_with_ferris.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0034_unclosed_raw_string_with_ferris.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0035_unclosed_raw_string_with_ascii_escape.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0035_unclosed_raw_string_with_ascii_escape.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0036_unclosed_raw_string_with_unicode_escape.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0036_unclosed_raw_string_with_unicode_escape.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0037_unclosed_raw_string_with_space.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0037_unclosed_raw_string_with_space.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0038_unclosed_raw_string_with_slash.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0038_unclosed_raw_string_with_slash.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0039_unclosed_raw_string_with_slash_n.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0039_unclosed_raw_string_with_slash_n.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0040_unclosed_raw_byte_string_at_eof.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0040_unclosed_raw_byte_string_at_eof.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0041_unclosed_raw_byte_string_with_ferris.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0041_unclosed_raw_byte_string_with_ferris.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0042_unclosed_raw_byte_string_with_ascii_escape.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0042_unclosed_raw_byte_string_with_ascii_escape.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0043_unclosed_raw_byte_string_with_unicode_escape.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0043_unclosed_raw_byte_string_with_unicode_escape.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0044_unclosed_raw_byte_string_with_space.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0044_unclosed_raw_byte_string_with_space.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0045_unclosed_raw_byte_string_with_slash.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0045_unclosed_raw_byte_string_with_slash.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0046_unclosed_raw_byte_string_with_slash_n.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0046_unclosed_raw_byte_string_with_slash_n.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0047_unstarted_raw_string_at_eof.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0047_unstarted_raw_string_at_eof.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0048_unstarted_raw_byte_string_at_eof.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0048_unstarted_raw_byte_string_at_eof.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0049_unstarted_raw_string_with_ascii.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0049_unstarted_raw_string_with_ascii.txt10
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0050_unstarted_raw_byte_string_with_ascii.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0050_unstarted_raw_byte_string_with_ascii.txt10
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0051_unclosed_block_comment_at_eof.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0051_unclosed_block_comment_at_eof.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0052_unclosed_block_comment_with_content.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0052_unclosed_block_comment_with_content.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0053_unclosed_nested_block_comment_entirely.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0053_unclosed_nested_block_comment_entirely.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0054_unclosed_nested_block_comment_partially.rs1
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0054_unclosed_nested_block_comment_partially.txt2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0055_empty_int.rs17
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0055_empty_int.txt39
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0056_empty_exponent.rs22
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0056_empty_exponent.txt62
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0057_lifetime_strarts_with_a_number.rs2
-rw-r--r--crates/ra_syntax/test_data/lexer/err/0057_lifetime_strarts_with_a_number.txt6
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0001_hello.rs (renamed from crates/ra_syntax/test_data/lexer/0001_hello.rs)0
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0001_hello.txt (renamed from crates/ra_syntax/test_data/lexer/0001_hello.txt)0
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0002_whitespace.rs (renamed from crates/ra_syntax/test_data/lexer/0002_whitespace.rs)0
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0002_whitespace.txt (renamed from crates/ra_syntax/test_data/lexer/0002_whitespace.txt)0
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0003_ident.rs (renamed from crates/ra_syntax/test_data/lexer/0003_ident.rs)0
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0003_ident.txt (renamed from crates/ra_syntax/test_data/lexer/0003_ident.txt)0
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0004_numbers.rs (renamed from crates/ra_syntax/test_data/lexer/0004_numbers.rs)4
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0004_numbers.txt (renamed from crates/ra_syntax/test_data/lexer/0004_numbers.txt)11
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0005_symbols.rs (renamed from crates/ra_syntax/test_data/lexer/0005_symbols.rs)0
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0005_symbols.txt (renamed from crates/ra_syntax/test_data/lexer/0005_symbols.txt)0
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0006_chars.rs (renamed from crates/ra_syntax/test_data/lexer/0006_chars.rs)0
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0006_chars.txt (renamed from crates/ra_syntax/test_data/lexer/0006_chars.txt)0
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0007_lifetimes.rs (renamed from crates/ra_syntax/test_data/lexer/0007_lifetimes.rs)0
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0007_lifetimes.txt (renamed from crates/ra_syntax/test_data/lexer/0007_lifetimes.txt)0
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0008_byte_strings.rs (renamed from crates/ra_syntax/test_data/lexer/0008_byte_strings.rs)0
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0008_byte_strings.txt (renamed from crates/ra_syntax/test_data/lexer/0008_byte_strings.txt)0
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0009_strings.rs (renamed from crates/ra_syntax/test_data/lexer/0009_strings.rs)0
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0009_strings.txt (renamed from crates/ra_syntax/test_data/lexer/0009_strings.txt)0
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0010_single_line_comments.rs12
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0010_single_line_comments.txt22
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0011_keywords.rs (renamed from crates/ra_syntax/test_data/lexer/0011_keywords.rs)0
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0011_keywords.txt (renamed from crates/ra_syntax/test_data/lexer/0011_keywords.txt)0
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0012_block_comment.rs (renamed from crates/ra_syntax/test_data/lexer/00012_block_comment.rs)1
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0012_block_comment.txt (renamed from crates/ra_syntax/test_data/lexer/00012_block_comment.txt)1
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0013_raw_strings.rs (renamed from crates/ra_syntax/test_data/lexer/0013_raw_strings.rs)0
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0013_raw_strings.txt (renamed from crates/ra_syntax/test_data/lexer/0013_raw_strings.txt)0
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0014_raw_ident.rs (renamed from crates/ra_syntax/test_data/lexer/0016_raw_ident.rs)0
-rw-r--r--crates/ra_syntax/test_data/lexer/ok/0014_raw_ident.txt (renamed from crates/ra_syntax/test_data/lexer/0016_raw_ident.txt)0
158 files changed, 788 insertions, 163 deletions
diff --git a/crates/ra_syntax/src/algo.rs b/crates/ra_syntax/src/algo.rs
index 30a479f01..acf677e7d 100644
--- a/crates/ra_syntax/src/algo.rs
+++ b/crates/ra_syntax/src/algo.rs
@@ -81,7 +81,7 @@ impl TreeDiff {
81/// Specifically, returns a map whose keys are descendants of `from` and values 81/// Specifically, returns a map whose keys are descendants of `from` and values
82/// are descendants of `to`, such that `replace_descendants(from, map) == to`. 82/// are descendants of `to`, such that `replace_descendants(from, map) == to`.
83/// 83///
84/// A trivial solution is a singletom map `{ from: to }`, but this function 84/// A trivial solution is a singleton map `{ from: to }`, but this function
85/// tries to find a more fine-grained diff. 85/// tries to find a more fine-grained diff.
86pub fn diff(from: &SyntaxNode, to: &SyntaxNode) -> TreeDiff { 86pub fn diff(from: &SyntaxNode, to: &SyntaxNode) -> TreeDiff {
87 let mut buf = FxHashMap::default(); 87 let mut buf = FxHashMap::default();
diff --git a/crates/ra_syntax/src/lib.rs b/crates/ra_syntax/src/lib.rs
index 9931fec84..f8f4b64c1 100644
--- a/crates/ra_syntax/src/lib.rs
+++ b/crates/ra_syntax/src/lib.rs
@@ -41,7 +41,9 @@ use crate::syntax_node::GreenNode;
41pub use crate::{ 41pub use crate::{
42 algo::InsertPosition, 42 algo::InsertPosition,
43 ast::{AstNode, AstToken}, 43 ast::{AstNode, AstToken},
44 parsing::{classify_literal, tokenize, Token}, 44 parsing::{
45 lex_single_syntax_kind, lex_single_valid_syntax_kind, tokenize, Token, TokenizeError,
46 },
45 ptr::{AstPtr, SyntaxNodePtr}, 47 ptr::{AstPtr, SyntaxNodePtr},
46 syntax_error::{Location, SyntaxError, SyntaxErrorKind}, 48 syntax_error::{Location, SyntaxError, SyntaxErrorKind},
47 syntax_node::{ 49 syntax_node::{
diff --git a/crates/ra_syntax/src/parsing.rs b/crates/ra_syntax/src/parsing.rs
index 0387f0378..e5eb80850 100644
--- a/crates/ra_syntax/src/parsing.rs
+++ b/crates/ra_syntax/src/parsing.rs
@@ -7,15 +7,23 @@ mod text_tree_sink;
7mod reparsing; 7mod reparsing;
8 8
9use crate::{syntax_node::GreenNode, SyntaxError}; 9use crate::{syntax_node::GreenNode, SyntaxError};
10use text_token_source::TextTokenSource;
11use text_tree_sink::TextTreeSink;
10 12
11pub use self::lexer::{classify_literal, tokenize, Token}; 13pub use lexer::*;
12 14
13pub(crate) use self::reparsing::incremental_reparse; 15pub(crate) use self::reparsing::incremental_reparse;
14 16
15pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec<SyntaxError>) { 17pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec<SyntaxError>) {
16 let tokens = tokenize(&text); 18 let (tokens, lexer_errors) = tokenize(&text);
17 let mut token_source = text_token_source::TextTokenSource::new(text, &tokens); 19
18 let mut tree_sink = text_tree_sink::TextTreeSink::new(text, &tokens); 20 let mut token_source = TextTokenSource::new(text, &tokens);
21 let mut tree_sink = TextTreeSink::new(text, &tokens);
22
19 ra_parser::parse(&mut token_source, &mut tree_sink); 23 ra_parser::parse(&mut token_source, &mut tree_sink);
20 tree_sink.finish() 24
25 let (tree, mut parser_errors) = tree_sink.finish();
26 parser_errors.extend(lexer_errors);
27
28 (tree, parser_errors)
21} 29}
diff --git a/crates/ra_syntax/src/parsing/lexer.rs b/crates/ra_syntax/src/parsing/lexer.rs
index 6d839208d..f889e6a1d 100644
--- a/crates/ra_syntax/src/parsing/lexer.rs
+++ b/crates/ra_syntax/src/parsing/lexer.rs
@@ -1,8 +1,10 @@
1//! FIXME: write short doc here 1//! Lexer analyzes raw input string and produces lexemes (tokens).
2//! It is just a bridge to `rustc_lexer`.
2 3
3use crate::{ 4use crate::{
5 SyntaxError, SyntaxErrorKind,
4 SyntaxKind::{self, *}, 6 SyntaxKind::{self, *},
5 TextUnit, 7 TextRange, TextUnit,
6}; 8};
7 9
8/// A token of Rust source. 10/// A token of Rust source.
@@ -14,91 +16,261 @@ pub struct Token {
14 pub len: TextUnit, 16 pub len: TextUnit,
15} 17}
16 18
17fn match_literal_kind(kind: rustc_lexer::LiteralKind) -> SyntaxKind { 19/// Break a string up into its component tokens.
18 match kind { 20/// Beware that it checks for shebang first and its length contributes to resulting
19 rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER, 21/// tokens offsets.
20 rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER, 22pub fn tokenize(text: &str) -> (Vec<Token>, Vec<SyntaxError>) {
21 rustc_lexer::LiteralKind::Char { .. } => CHAR, 23 // non-empty string is a precondtion of `rustc_lexer::strip_shebang()`.
22 rustc_lexer::LiteralKind::Byte { .. } => BYTE, 24 if text.is_empty() {
23 rustc_lexer::LiteralKind::Str { .. } => STRING, 25 return Default::default();
24 rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING, 26 }
25 rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING, 27
26 rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING, 28 let mut tokens = Vec::new();
29 let mut errors = Vec::new();
30
31 let mut offset: usize = rustc_lexer::strip_shebang(text)
32 .map(|shebang_len| {
33 tokens.push(Token { kind: SHEBANG, len: TextUnit::from_usize(shebang_len) });
34 shebang_len
35 })
36 .unwrap_or(0);
37
38 let text_without_shebang = &text[offset..];
39
40 for rustc_token in rustc_lexer::tokenize(text_without_shebang) {
41 let token_len = TextUnit::from_usize(rustc_token.len);
42 let token_range = TextRange::offset_len(TextUnit::from_usize(offset), token_len);
43
44 let (syntax_kind, error) =
45 rustc_token_kind_to_syntax_kind(&rustc_token.kind, &text[token_range]);
46
47 tokens.push(Token { kind: syntax_kind, len: token_len });
48
49 if let Some(error) = error {
50 errors.push(SyntaxError::new(SyntaxErrorKind::TokenizeError(error), token_range));
51 }
52
53 offset += rustc_token.len;
27 } 54 }
55
56 (tokens, errors)
57}
58
59/// Returns `SyntaxKind` and `Option<SyntaxError>` of the first token
60/// encountered at the beginning of the string.
61///
62/// Returns `None` if the string contains zero *or two or more* tokens.
63/// The token is malformed if the returned error is not `None`.
64///
65/// Beware that unescape errors are not checked at tokenization time.
66pub fn lex_single_syntax_kind(text: &str) -> Option<(SyntaxKind, Option<SyntaxError>)> {
67 lex_first_token(text)
68 .filter(|(token, _)| token.len.to_usize() == text.len())
69 .map(|(token, error)| (token.kind, error))
70}
71
72/// The same as `lex_single_syntax_kind()` but returns only `SyntaxKind` and
73/// returns `None` if any tokenization error occured.
74///
75/// Beware that unescape errors are not checked at tokenization time.
76pub fn lex_single_valid_syntax_kind(text: &str) -> Option<SyntaxKind> {
77 lex_first_token(text)
78 .filter(|(token, error)| !error.is_some() && token.len.to_usize() == text.len())
79 .map(|(token, _error)| token.kind)
28} 80}
29 81
30/// Break a string up into its component tokens 82/// Returns `SyntaxKind` and `Option<SyntaxError>` of the first token
31pub fn tokenize(text: &str) -> Vec<Token> { 83/// encountered at the beginning of the string.
84///
85/// Returns `None` if the string contains zero tokens or if the token was parsed
86/// with an error.
87/// The token is malformed if the returned error is not `None`.
88///
89/// Beware that unescape errors are not checked at tokenization time.
90fn lex_first_token(text: &str) -> Option<(Token, Option<SyntaxError>)> {
91 // non-empty string is a precondtion of `rustc_lexer::first_token()`.
32 if text.is_empty() { 92 if text.is_empty() {
33 return vec![]; 93 return None;
34 }
35 let mut text = text;
36 let mut acc = Vec::new();
37 if let Some(len) = rustc_lexer::strip_shebang(text) {
38 acc.push(Token { kind: SHEBANG, len: TextUnit::from_usize(len) });
39 text = &text[len..];
40 } 94 }
41 while !text.is_empty() { 95
42 let rustc_token = rustc_lexer::first_token(text); 96 let rustc_token = rustc_lexer::first_token(text);
43 let kind = match rustc_token.kind { 97 let (syntax_kind, error) = rustc_token_kind_to_syntax_kind(&rustc_token.kind, text);
44 rustc_lexer::TokenKind::LineComment => COMMENT, 98
45 rustc_lexer::TokenKind::BlockComment { .. } => COMMENT, 99 let token = Token { kind: syntax_kind, len: TextUnit::from_usize(rustc_token.len) };
46 rustc_lexer::TokenKind::Whitespace => WHITESPACE, 100 let error = error.map(|error| {
47 rustc_lexer::TokenKind::Ident => { 101 SyntaxError::new(
48 let token_text = &text[..rustc_token.len]; 102 SyntaxErrorKind::TokenizeError(error),
103 TextRange::from_to(TextUnit::from(0), TextUnit::of_str(text)),
104 )
105 });
106
107 Some((token, error))
108}
109
110// FIXME: simplify TokenizeError to `SyntaxError(String, TextRange)` as per @matklad advice:
111// https://github.com/rust-analyzer/rust-analyzer/pull/2911/files#r371175067
112
113/// Describes the values of `SyntaxErrorKind::TokenizeError` enum variant.
114/// It describes all the types of errors that may happen during the tokenization
115/// of Rust source.
116#[derive(Debug, Clone, PartialEq, Eq, Hash)]
117pub enum TokenizeError {
118 /// Base prefix was provided, but there were no digits
119 /// after it, e.g. `0x`, `0b`.
120 EmptyInt,
121 /// Float exponent lacks digits e.g. `12.34e+`, `12.3E+`, `12e-`, `1_E-`,
122 EmptyExponent,
123
124 /// Block comment lacks trailing delimiter `*/`
125 UnterminatedBlockComment,
126 /// Character literal lacks trailing delimiter `'`
127 UnterminatedChar,
128 /// Characterish byte literal lacks trailing delimiter `'`
129 UnterminatedByte,
130 /// String literal lacks trailing delimiter `"`
131 UnterminatedString,
132 /// Byte string literal lacks trailing delimiter `"`
133 UnterminatedByteString,
134 /// Raw literal lacks trailing delimiter e.g. `"##`
135 UnterminatedRawString,
136 /// Raw byte string literal lacks trailing delimiter e.g. `"##`
137 UnterminatedRawByteString,
138
139 /// Raw string lacks a quote after the pound characters e.g. `r###`
140 UnstartedRawString,
141 /// Raw byte string lacks a quote after the pound characters e.g. `br###`
142 UnstartedRawByteString,
143
144 /// Lifetime starts with a number e.g. `'4ever`
145 LifetimeStartsWithNumber,
146}
147
148fn rustc_token_kind_to_syntax_kind(
149 rustc_token_kind: &rustc_lexer::TokenKind,
150 token_text: &str,
151) -> (SyntaxKind, Option<TokenizeError>) {
152 // A note on an intended tradeoff:
153 // We drop some useful infromation here (see patterns with double dots `..`)
154 // Storing that info in `SyntaxKind` is not possible due to its layout requirements of
155 // being `u16` that come from `rowan::SyntaxKind`.
156
157 let syntax_kind = {
158 use rustc_lexer::TokenKind as TK;
159 use TokenizeError as TE;
160
161 match rustc_token_kind {
162 TK::LineComment => COMMENT,
163
164 TK::BlockComment { terminated: true } => COMMENT,
165 TK::BlockComment { terminated: false } => {
166 return (COMMENT, Some(TE::UnterminatedBlockComment));
167 }
168
169 TK::Whitespace => WHITESPACE,
170
171 TK::Ident => {
49 if token_text == "_" { 172 if token_text == "_" {
50 UNDERSCORE 173 UNDERSCORE
51 } else { 174 } else {
52 SyntaxKind::from_keyword(&text[..rustc_token.len]).unwrap_or(IDENT) 175 SyntaxKind::from_keyword(token_text).unwrap_or(IDENT)
53 } 176 }
54 } 177 }
55 rustc_lexer::TokenKind::RawIdent => IDENT, 178
56 rustc_lexer::TokenKind::Literal { kind, .. } => match_literal_kind(kind), 179 TK::RawIdent => IDENT,
57 rustc_lexer::TokenKind::Lifetime { .. } => LIFETIME, 180 TK::Literal { kind, .. } => return match_literal_kind(&kind),
58 rustc_lexer::TokenKind::Semi => SEMI, 181
59 rustc_lexer::TokenKind::Comma => COMMA, 182 TK::Lifetime { starts_with_number: false } => LIFETIME,
60 rustc_lexer::TokenKind::Dot => DOT, 183 TK::Lifetime { starts_with_number: true } => {
61 rustc_lexer::TokenKind::OpenParen => L_PAREN, 184 return (LIFETIME, Some(TE::LifetimeStartsWithNumber))
62 rustc_lexer::TokenKind::CloseParen => R_PAREN, 185 }
63 rustc_lexer::TokenKind::OpenBrace => L_CURLY, 186
64 rustc_lexer::TokenKind::CloseBrace => R_CURLY, 187 TK::Semi => SEMI,
65 rustc_lexer::TokenKind::OpenBracket => L_BRACK, 188 TK::Comma => COMMA,
66 rustc_lexer::TokenKind::CloseBracket => R_BRACK, 189 TK::Dot => DOT,
67 rustc_lexer::TokenKind::At => AT, 190 TK::OpenParen => L_PAREN,
68 rustc_lexer::TokenKind::Pound => POUND, 191 TK::CloseParen => R_PAREN,
69 rustc_lexer::TokenKind::Tilde => TILDE, 192 TK::OpenBrace => L_CURLY,
70 rustc_lexer::TokenKind::Question => QUESTION, 193 TK::CloseBrace => R_CURLY,
71 rustc_lexer::TokenKind::Colon => COLON, 194 TK::OpenBracket => L_BRACK,
72 rustc_lexer::TokenKind::Dollar => DOLLAR, 195 TK::CloseBracket => R_BRACK,
73 rustc_lexer::TokenKind::Eq => EQ, 196 TK::At => AT,
74 rustc_lexer::TokenKind::Not => EXCL, 197 TK::Pound => POUND,
75 rustc_lexer::TokenKind::Lt => L_ANGLE, 198 TK::Tilde => TILDE,
76 rustc_lexer::TokenKind::Gt => R_ANGLE, 199 TK::Question => QUESTION,
77 rustc_lexer::TokenKind::Minus => MINUS, 200 TK::Colon => COLON,
78 rustc_lexer::TokenKind::And => AMP, 201 TK::Dollar => DOLLAR,
79 rustc_lexer::TokenKind::Or => PIPE, 202 TK::Eq => EQ,
80 rustc_lexer::TokenKind::Plus => PLUS, 203 TK::Not => EXCL,
81 rustc_lexer::TokenKind::Star => STAR, 204 TK::Lt => L_ANGLE,
82 rustc_lexer::TokenKind::Slash => SLASH, 205 TK::Gt => R_ANGLE,
83 rustc_lexer::TokenKind::Caret => CARET, 206 TK::Minus => MINUS,
84 rustc_lexer::TokenKind::Percent => PERCENT, 207 TK::And => AMP,
85 rustc_lexer::TokenKind::Unknown => ERROR, 208 TK::Or => PIPE,
209 TK::Plus => PLUS,
210 TK::Star => STAR,
211 TK::Slash => SLASH,
212 TK::Caret => CARET,
213 TK::Percent => PERCENT,
214 TK::Unknown => ERROR,
215 }
216 };
217
218 return (syntax_kind, None);
219
220 fn match_literal_kind(kind: &rustc_lexer::LiteralKind) -> (SyntaxKind, Option<TokenizeError>) {
221 use rustc_lexer::LiteralKind as LK;
222 use TokenizeError as TE;
223
224 #[rustfmt::skip]
225 let syntax_kind = match *kind {
226 LK::Int { empty_int: false, .. } => INT_NUMBER,
227 LK::Int { empty_int: true, .. } => {
228 return (INT_NUMBER, Some(TE::EmptyInt))
229 }
230
231 LK::Float { empty_exponent: false, .. } => FLOAT_NUMBER,
232 LK::Float { empty_exponent: true, .. } => {
233 return (FLOAT_NUMBER, Some(TE::EmptyExponent))
234 }
235
236 LK::Char { terminated: true } => CHAR,
237 LK::Char { terminated: false } => {
238 return (CHAR, Some(TE::UnterminatedChar))
239 }
240
241 LK::Byte { terminated: true } => BYTE,
242 LK::Byte { terminated: false } => {
243 return (BYTE, Some(TE::UnterminatedByte))
244 }
245
246 LK::Str { terminated: true } => STRING,
247 LK::Str { terminated: false } => {
248 return (STRING, Some(TE::UnterminatedString))
249 }
250
251
252 LK::ByteStr { terminated: true } => BYTE_STRING,
253 LK::ByteStr { terminated: false } => {
254 return (BYTE_STRING, Some(TE::UnterminatedByteString))
255 }
256
257 LK::RawStr { started: true, terminated: true, .. } => RAW_STRING,
258 LK::RawStr { started: true, terminated: false, .. } => {
259 return (RAW_STRING, Some(TE::UnterminatedRawString))
260 }
261 LK::RawStr { started: false, .. } => {
262 return (RAW_STRING, Some(TE::UnstartedRawString))
263 }
264
265 LK::RawByteStr { started: true, terminated: true, .. } => RAW_BYTE_STRING,
266 LK::RawByteStr { started: true, terminated: false, .. } => {
267 return (RAW_BYTE_STRING, Some(TE::UnterminatedRawByteString))
268 }
269 LK::RawByteStr { started: false, .. } => {
270 return (RAW_BYTE_STRING, Some(TE::UnstartedRawByteString))
271 }
86 }; 272 };
87 let token = Token { kind, len: TextUnit::from_usize(rustc_token.len) };
88 acc.push(token);
89 text = &text[rustc_token.len..];
90 }
91 acc
92}
93 273
94pub fn classify_literal(text: &str) -> Option<Token> { 274 (syntax_kind, None)
95 let t = rustc_lexer::first_token(text);
96 if t.len != text.len() {
97 return None;
98 } 275 }
99 let kind = match t.kind {
100 rustc_lexer::TokenKind::Literal { kind, .. } => match_literal_kind(kind),
101 _ => return None,
102 };
103 Some(Token { kind, len: TextUnit::from_usize(t.len) })
104} 276}
diff --git a/crates/ra_syntax/src/parsing/reparsing.rs b/crates/ra_syntax/src/parsing/reparsing.rs
index 06bdda11d..a86da0675 100644
--- a/crates/ra_syntax/src/parsing/reparsing.rs
+++ b/crates/ra_syntax/src/parsing/reparsing.rs
@@ -12,7 +12,7 @@ use ra_text_edit::AtomTextEdit;
12use crate::{ 12use crate::{
13 algo, 13 algo,
14 parsing::{ 14 parsing::{
15 lexer::{tokenize, Token}, 15 lexer::{lex_single_syntax_kind, tokenize, Token},
16 text_token_source::TextTokenSource, 16 text_token_source::TextTokenSource,
17 text_tree_sink::TextTreeSink, 17 text_tree_sink::TextTreeSink,
18 }, 18 },
@@ -41,37 +41,42 @@ fn reparse_token<'node>(
41 root: &'node SyntaxNode, 41 root: &'node SyntaxNode,
42 edit: &AtomTextEdit, 42 edit: &AtomTextEdit,
43) -> Option<(GreenNode, TextRange)> { 43) -> Option<(GreenNode, TextRange)> {
44 let token = algo::find_covering_element(root, edit.delete).as_token()?.clone(); 44 let prev_token = algo::find_covering_element(root, edit.delete).as_token()?.clone();
45 match token.kind() { 45 let prev_token_kind = prev_token.kind();
46 match prev_token_kind {
46 WHITESPACE | COMMENT | IDENT | STRING | RAW_STRING => { 47 WHITESPACE | COMMENT | IDENT | STRING | RAW_STRING => {
47 if token.kind() == WHITESPACE || token.kind() == COMMENT { 48 if prev_token_kind == WHITESPACE || prev_token_kind == COMMENT {
48 // removing a new line may extends previous token 49 // removing a new line may extends previous token
49 if token.text().to_string()[edit.delete - token.text_range().start()].contains('\n') 50 let deleted_range = edit.delete - prev_token.text_range().start();
50 { 51 if prev_token.text()[deleted_range].contains('\n') {
51 return None; 52 return None;
52 } 53 }
53 } 54 }
54 55
55 let text = get_text_after_edit(token.clone().into(), &edit); 56 let mut new_text = get_text_after_edit(prev_token.clone().into(), &edit);
56 let lex_tokens = tokenize(&text); 57 let (new_token_kind, _error) = lex_single_syntax_kind(&new_text)?;
57 let lex_token = match lex_tokens[..] {
58 [lex_token] if lex_token.kind == token.kind() => lex_token,
59 _ => return None,
60 };
61 58
62 if lex_token.kind == IDENT && is_contextual_kw(&text) { 59 if new_token_kind != prev_token_kind
60 || (new_token_kind == IDENT && is_contextual_kw(&new_text))
61 {
63 return None; 62 return None;
64 } 63 }
65 64
66 if let Some(next_char) = root.text().char_at(token.text_range().end()) { 65 // Check that edited token is not a part of the bigger token.
67 let tokens_with_next_char = tokenize(&format!("{}{}", text, next_char)); 66 // E.g. if for source code `bruh"str"` the user removed `ruh`, then
68 if tokens_with_next_char.len() == 1 { 67 // `b` no longer remains an identifier, but becomes a part of byte string literal
68 if let Some(next_char) = root.text().char_at(prev_token.text_range().end()) {
69 new_text.push(next_char);
70 let token_with_next_char = lex_single_syntax_kind(&new_text);
71 if let Some((_kind, _error)) = token_with_next_char {
69 return None; 72 return None;
70 } 73 }
74 new_text.pop();
71 } 75 }
72 76
73 let new_token = GreenToken::new(rowan::SyntaxKind(token.kind().into()), text.into()); 77 let new_token =
74 Some((token.replace_with(new_token), token.text_range())) 78 GreenToken::new(rowan::SyntaxKind(prev_token_kind.into()), new_text.into());
79 Some((prev_token.replace_with(new_token), prev_token.text_range()))
75 } 80 }
76 _ => None, 81 _ => None,
77 } 82 }
@@ -83,20 +88,26 @@ fn reparse_block<'node>(
83) -> Option<(GreenNode, Vec<SyntaxError>, TextRange)> { 88) -> Option<(GreenNode, Vec<SyntaxError>, TextRange)> {
84 let (node, reparser) = find_reparsable_node(root, edit.delete)?; 89 let (node, reparser) = find_reparsable_node(root, edit.delete)?;
85 let text = get_text_after_edit(node.clone().into(), &edit); 90 let text = get_text_after_edit(node.clone().into(), &edit);
86 let tokens = tokenize(&text); 91
92 let (tokens, new_lexer_errors) = tokenize(&text);
87 if !is_balanced(&tokens) { 93 if !is_balanced(&tokens) {
88 return None; 94 return None;
89 } 95 }
96
90 let mut token_source = TextTokenSource::new(&text, &tokens); 97 let mut token_source = TextTokenSource::new(&text, &tokens);
91 let mut tree_sink = TextTreeSink::new(&text, &tokens); 98 let mut tree_sink = TextTreeSink::new(&text, &tokens);
92 reparser.parse(&mut token_source, &mut tree_sink); 99 reparser.parse(&mut token_source, &mut tree_sink);
93 let (green, new_errors) = tree_sink.finish(); 100
94 Some((node.replace_with(green), new_errors, node.text_range())) 101 let (green, mut new_parser_errors) = tree_sink.finish();
102 new_parser_errors.extend(new_lexer_errors);
103
104 Some((node.replace_with(green), new_parser_errors, node.text_range()))
95} 105}
96 106
97fn get_text_after_edit(element: SyntaxElement, edit: &AtomTextEdit) -> String { 107fn get_text_after_edit(element: SyntaxElement, edit: &AtomTextEdit) -> String {
98 let edit = 108 let edit =
99 AtomTextEdit::replace(edit.delete - element.text_range().start(), edit.insert.clone()); 109 AtomTextEdit::replace(edit.delete - element.text_range().start(), edit.insert.clone());
110
100 let text = match element { 111 let text = match element {
101 NodeOrToken::Token(token) => token.text().to_string(), 112 NodeOrToken::Token(token) => token.text().to_string(),
102 NodeOrToken::Node(node) => node.text().to_string(), 113 NodeOrToken::Node(node) => node.text().to_string(),
@@ -113,6 +124,7 @@ fn is_contextual_kw(text: &str) -> bool {
113 124
114fn find_reparsable_node(node: &SyntaxNode, range: TextRange) -> Option<(SyntaxNode, Reparser)> { 125fn find_reparsable_node(node: &SyntaxNode, range: TextRange) -> Option<(SyntaxNode, Reparser)> {
115 let node = algo::find_covering_element(node, range); 126 let node = algo::find_covering_element(node, range);
127
116 let mut ancestors = match node { 128 let mut ancestors = match node {
117 NodeOrToken::Token(it) => it.parent().ancestors(), 129 NodeOrToken::Token(it) => it.parent().ancestors(),
118 NodeOrToken::Node(it) => it.ancestors(), 130 NodeOrToken::Node(it) => it.ancestors(),
@@ -182,7 +194,6 @@ mod tests {
182 let fully_reparsed = SourceFile::parse(&after); 194 let fully_reparsed = SourceFile::parse(&after);
183 let incrementally_reparsed: Parse<SourceFile> = { 195 let incrementally_reparsed: Parse<SourceFile> = {
184 let f = SourceFile::parse(&before); 196 let f = SourceFile::parse(&before);
185 let edit = AtomTextEdit { delete: range, insert: replace_with.to_string() };
186 let (green, new_errors, range) = 197 let (green, new_errors, range) =
187 incremental_reparse(f.tree().syntax(), &edit, f.errors.to_vec()).unwrap(); 198 incremental_reparse(f.tree().syntax(), &edit, f.errors.to_vec()).unwrap();
188 assert_eq!(range.len(), reparsed_len.into(), "reparsed fragment has wrong length"); 199 assert_eq!(range.len(), reparsed_len.into(), "reparsed fragment has wrong length");
diff --git a/crates/ra_syntax/src/parsing/text_tree_sink.rs b/crates/ra_syntax/src/parsing/text_tree_sink.rs
index c36756d6c..dd202601d 100644
--- a/crates/ra_syntax/src/parsing/text_tree_sink.rs
+++ b/crates/ra_syntax/src/parsing/text_tree_sink.rs
@@ -92,8 +92,8 @@ impl<'a> TreeSink for TextTreeSink<'a> {
92} 92}
93 93
94impl<'a> TextTreeSink<'a> { 94impl<'a> TextTreeSink<'a> {
95 pub(super) fn new(text: &'a str, tokens: &'a [Token]) -> TextTreeSink<'a> { 95 pub(super) fn new(text: &'a str, tokens: &'a [Token]) -> Self {
96 TextTreeSink { 96 Self {
97 text, 97 text,
98 tokens, 98 tokens,
99 text_pos: 0.into(), 99 text_pos: 0.into(),
diff --git a/crates/ra_syntax/src/syntax_error.rs b/crates/ra_syntax/src/syntax_error.rs
index 6c171df8d..7f9d36618 100644
--- a/crates/ra_syntax/src/syntax_error.rs
+++ b/crates/ra_syntax/src/syntax_error.rs
@@ -4,7 +4,7 @@ use std::fmt;
4 4
5use ra_parser::ParseError; 5use ra_parser::ParseError;
6 6
7use crate::{validation::EscapeError, TextRange, TextUnit}; 7use crate::{validation::EscapeError, TextRange, TextUnit, TokenizeError};
8 8
9#[derive(Debug, Clone, PartialEq, Eq, Hash)] 9#[derive(Debug, Clone, PartialEq, Eq, Hash)]
10pub struct SyntaxError { 10pub struct SyntaxError {
@@ -12,6 +12,10 @@ pub struct SyntaxError {
12 location: Location, 12 location: Location,
13} 13}
14 14
15// FIXME: Location should be just `Location(TextRange)`
16// TextUnit enum member just unnecessarily compicates things,
17// we should'n treat it specially, it just as a `TextRange { start: x, end: x + 1 }`
18// see `location_to_range()` in ra_ide/src/diagnostics
15#[derive(Clone, PartialEq, Eq, Hash)] 19#[derive(Clone, PartialEq, Eq, Hash)]
16pub enum Location { 20pub enum Location {
17 Offset(TextUnit), 21 Offset(TextUnit),
@@ -67,6 +71,10 @@ impl SyntaxError {
67 71
68 self 72 self
69 } 73 }
74
75 pub fn debug_dump(&self, acc: &mut impl fmt::Write) {
76 writeln!(acc, "error {:?}: {}", self.location(), self.kind()).unwrap();
77 }
70} 78}
71 79
72impl fmt::Display for SyntaxError { 80impl fmt::Display for SyntaxError {
@@ -79,6 +87,10 @@ impl fmt::Display for SyntaxError {
79pub enum SyntaxErrorKind { 87pub enum SyntaxErrorKind {
80 ParseError(ParseError), 88 ParseError(ParseError),
81 EscapeError(EscapeError), 89 EscapeError(EscapeError),
90 TokenizeError(TokenizeError),
91 // FIXME: the obvious pattern of this enum dictates that the following enum variants
92 // should be wrapped into something like `SemmanticError(SemmanticError)`
93 // or `ValidateError(ValidateError)` or `SemmanticValidateError(...)`
82 InvalidBlockAttr, 94 InvalidBlockAttr,
83 InvalidMatchInnerAttr, 95 InvalidMatchInnerAttr,
84 InvalidTupleIndexFormat, 96 InvalidTupleIndexFormat,
@@ -101,6 +113,7 @@ impl fmt::Display for SyntaxErrorKind {
101 } 113 }
102 ParseError(msg) => write!(f, "{}", msg.0), 114 ParseError(msg) => write!(f, "{}", msg.0),
103 EscapeError(err) => write!(f, "{}", err), 115 EscapeError(err) => write!(f, "{}", err),
116 TokenizeError(err) => write!(f, "{}", err),
104 VisibilityNotAllowed => { 117 VisibilityNotAllowed => {
105 write!(f, "unnecessary visibility qualifier") 118 write!(f, "unnecessary visibility qualifier")
106 } 119 }
@@ -111,6 +124,51 @@ impl fmt::Display for SyntaxErrorKind {
111 } 124 }
112} 125}
113 126
127impl fmt::Display for TokenizeError {
128 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
129 #[rustfmt::skip]
130 let msg = match self {
131 TokenizeError::EmptyInt => {
132 "Missing digits after the integer base prefix"
133 }
134 TokenizeError::EmptyExponent => {
135 "Missing digits after the exponent symbol"
136 }
137 TokenizeError::UnterminatedBlockComment => {
138 "Missing trailing `*/` symbols to terminate the block comment"
139 }
140 TokenizeError::UnterminatedChar => {
141 "Missing trailing `'` symbol to terminate the character literal"
142 }
143 TokenizeError::UnterminatedByte => {
144 "Missing trailing `'` symbol to terminate the byte literal"
145 }
146 TokenizeError::UnterminatedString => {
147 "Missing trailing `\"` symbol to terminate the string literal"
148 }
149 TokenizeError::UnterminatedByteString => {
150 "Missing trailing `\"` symbol to terminate the byte string literal"
151 }
152 TokenizeError::UnterminatedRawString => {
153 "Missing trailing `\"` with `#` symbols to terminate the raw string literal"
154 }
155 TokenizeError::UnterminatedRawByteString => {
156 "Missing trailing `\"` with `#` symbols to terminate the raw byte string literal"
157 }
158 TokenizeError::UnstartedRawString => {
159 "Missing `\"` symbol after `#` symbols to begin the raw string literal"
160 }
161 TokenizeError::UnstartedRawByteString => {
162 "Missing `\"` symbol after `#` symbols to begin the raw byte string literal"
163 }
164 TokenizeError::LifetimeStartsWithNumber => {
165 "Lifetime name cannot start with a number"
166 }
167 };
168 write!(f, "{}", msg)
169 }
170}
171
114impl fmt::Display for EscapeError { 172impl fmt::Display for EscapeError {
115 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 173 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
116 let msg = match self { 174 let msg = match self {
diff --git a/crates/ra_syntax/src/syntax_node.rs b/crates/ra_syntax/src/syntax_node.rs
index b3eb5da63..7c2b18af3 100644
--- a/crates/ra_syntax/src/syntax_node.rs
+++ b/crates/ra_syntax/src/syntax_node.rs
@@ -4,7 +4,7 @@
4//! `SyntaxNode`, and a basic traversal API (parent, children, siblings). 4//! `SyntaxNode`, and a basic traversal API (parent, children, siblings).
5//! 5//!
6//! The *real* implementation is in the (language-agnostic) `rowan` crate, this 6//! The *real* implementation is in the (language-agnostic) `rowan` crate, this
7//! modules just wraps its API. 7//! module just wraps its API.
8 8
9use ra_parser::ParseError; 9use ra_parser::ParseError;
10use rowan::{GreenNodeBuilder, Language}; 10use rowan::{GreenNodeBuilder, Language};
@@ -38,17 +38,12 @@ pub type SyntaxElementChildren = rowan::SyntaxElementChildren<RustLanguage>;
38 38
39pub use rowan::{Direction, NodeOrToken}; 39pub use rowan::{Direction, NodeOrToken};
40 40
41#[derive(Default)]
41pub struct SyntaxTreeBuilder { 42pub struct SyntaxTreeBuilder {
42 errors: Vec<SyntaxError>, 43 errors: Vec<SyntaxError>,
43 inner: GreenNodeBuilder<'static>, 44 inner: GreenNodeBuilder<'static>,
44} 45}
45 46
46impl Default for SyntaxTreeBuilder {
47 fn default() -> SyntaxTreeBuilder {
48 SyntaxTreeBuilder { errors: Vec::new(), inner: GreenNodeBuilder::new() }
49 }
50}
51
52impl SyntaxTreeBuilder { 47impl SyntaxTreeBuilder {
53 pub(crate) fn finish_raw(self) -> (GreenNode, Vec<SyntaxError>) { 48 pub(crate) fn finish_raw(self) -> (GreenNode, Vec<SyntaxError>) {
54 let green = self.inner.finish(); 49 let green = self.inner.finish();
diff --git a/crates/ra_syntax/src/tests.rs b/crates/ra_syntax/src/tests.rs
index 458920607..fb22b9e54 100644
--- a/crates/ra_syntax/src/tests.rs
+++ b/crates/ra_syntax/src/tests.rs
@@ -1,18 +1,28 @@
1use std::{ 1use std::{
2 fmt::Write, 2 fmt::Write,
3 path::{Component, PathBuf}, 3 path::{Component, Path, PathBuf},
4}; 4};
5 5
6use test_utils::{collect_tests, dir_tests, project_dir, read_text}; 6use test_utils::{collect_tests, dir_tests, project_dir, read_text};
7 7
8use crate::{fuzz, SourceFile}; 8use crate::{fuzz, tokenize, Location, SourceFile, SyntaxError, TextRange, Token};
9 9
10#[test] 10#[test]
11fn lexer_tests() { 11fn lexer_tests() {
12 dir_tests(&test_data_dir(), &["lexer"], |text, _| { 12 // FIXME:
13 let tokens = crate::tokenize(text); 13 // * Add tests for unicode escapes in byte-character and [raw]-byte-string literals
14 dump_tokens(&tokens, text) 14 // * Add tests for unescape errors
15 }) 15
16 dir_tests(&test_data_dir(), &["lexer/ok"], |text, path| {
17 let (tokens, errors) = tokenize(text);
18 assert_errors_are_absent(&errors, path);
19 dump_tokens_and_errors(&tokens, &errors, text)
20 });
21 dir_tests(&test_data_dir(), &["lexer/err"], |text, path| {
22 let (tokens, errors) = tokenize(text);
23 assert_errors_are_present(&errors, path);
24 dump_tokens_and_errors(&tokens, &errors, text)
25 });
16} 26}
17 27
18#[test] 28#[test]
@@ -32,18 +42,13 @@ fn parser_tests() {
32 dir_tests(&test_data_dir(), &["parser/inline/ok", "parser/ok"], |text, path| { 42 dir_tests(&test_data_dir(), &["parser/inline/ok", "parser/ok"], |text, path| {
33 let parse = SourceFile::parse(text); 43 let parse = SourceFile::parse(text);
34 let errors = parse.errors(); 44 let errors = parse.errors();
35 assert_eq!( 45 assert_errors_are_absent(&errors, path);
36 errors,
37 &[] as &[crate::SyntaxError],
38 "There should be no errors in the file {:?}",
39 path.display(),
40 );
41 parse.debug_dump() 46 parse.debug_dump()
42 }); 47 });
43 dir_tests(&test_data_dir(), &["parser/err", "parser/inline/err"], |text, path| { 48 dir_tests(&test_data_dir(), &["parser/err", "parser/inline/err"], |text, path| {
44 let parse = SourceFile::parse(text); 49 let parse = SourceFile::parse(text);
45 let errors = parse.errors(); 50 let errors = parse.errors();
46 assert!(!errors.is_empty(), "There should be errors in the file {:?}", path.display()); 51 assert_errors_are_present(&errors, path);
47 parse.debug_dump() 52 parse.debug_dump()
48 }); 53 });
49} 54}
@@ -75,7 +80,7 @@ fn self_hosting_parsing() {
75 .into_iter() 80 .into_iter()
76 .filter_entry(|entry| { 81 .filter_entry(|entry| {
77 !entry.path().components().any(|component| { 82 !entry.path().components().any(|component| {
78 // Get all files which are not in the crates/ra_syntax/tests/data folder 83 // Get all files which are not in the crates/ra_syntax/test_data folder
79 component == Component::Normal(OsStr::new("test_data")) 84 component == Component::Normal(OsStr::new("test_data"))
80 }) 85 })
81 }) 86 })
@@ -101,15 +106,47 @@ fn test_data_dir() -> PathBuf {
101 project_dir().join("crates/ra_syntax/test_data") 106 project_dir().join("crates/ra_syntax/test_data")
102} 107}
103 108
104fn dump_tokens(tokens: &[crate::Token], text: &str) -> String { 109fn assert_errors_are_present(errors: &[SyntaxError], path: &Path) {
110 assert!(!errors.is_empty(), "There should be errors in the file {:?}", path.display());
111}
112fn assert_errors_are_absent(errors: &[SyntaxError], path: &Path) {
113 assert_eq!(
114 errors,
115 &[] as &[SyntaxError],
116 "There should be no errors in the file {:?}",
117 path.display(),
118 );
119}
120
121fn dump_tokens_and_errors(tokens: &[Token], errors: &[SyntaxError], text: &str) -> String {
105 let mut acc = String::new(); 122 let mut acc = String::new();
106 let mut offset = 0; 123 let mut offset = 0;
107 for token in tokens { 124 for token in tokens {
108 let len: u32 = token.len.into(); 125 let token_len = token.len.to_usize();
109 let len = len as usize; 126 let token_text = &text[offset..offset + token_len];
110 let token_text = &text[offset..offset + len]; 127 offset += token_len;
111 offset += len; 128 writeln!(acc, "{:?} {} {:?}", token.kind, token_len, token_text).unwrap();
112 write!(acc, "{:?} {} {:?}\n", token.kind, token.len, token_text).unwrap() 129 }
130 for err in errors {
131 let err_range = location_to_range(err.location());
132 writeln!(
133 acc,
134 "> error{:?} token({:?}) msg({})",
135 err.location(),
136 &text[err_range],
137 err.kind()
138 )
139 .unwrap();
140 }
141 return acc;
142
143 // FIXME: copy-pasted this from `ra_ide/src/diagnostics.rs`
144 // `Location` will be refactored soon in new PR, see todos here:
145 // https://github.com/rust-analyzer/rust-analyzer/issues/223
146 fn location_to_range(location: Location) -> TextRange {
147 match location {
148 Location::Offset(offset) => TextRange::offset_len(offset, 1.into()),
149 Location::Range(range) => range,
150 }
113 } 151 }
114 acc
115} 152}
diff --git a/crates/ra_syntax/src/validation.rs b/crates/ra_syntax/src/validation.rs
index 445e3b3e4..8a5f0e4b7 100644
--- a/crates/ra_syntax/src/validation.rs
+++ b/crates/ra_syntax/src/validation.rs
@@ -94,6 +94,12 @@ impl From<rustc_lexer::unescape::EscapeError> for SyntaxErrorKind {
94} 94}
95 95
96pub(crate) fn validate(root: &SyntaxNode) -> Vec<SyntaxError> { 96pub(crate) fn validate(root: &SyntaxNode) -> Vec<SyntaxError> {
97 // FIXME:
98 // * Add validation of character literal containing only a single char
99 // * Add validation of `crate` keyword not appearing in the middle of the symbol path
100 // * Add validation of doc comments are being attached to nodes
101 // * Remove validation of unterminated literals (it is already implemented in `tokenize()`)
102
97 let mut errors = Vec::new(); 103 let mut errors = Vec::new();
98 for node in root.descendants() { 104 for node in root.descendants() {
99 match_ast! { 105 match_ast! {
diff --git a/crates/ra_syntax/test_data/lexer/0010_comments.rs b/crates/ra_syntax/test_data/lexer/0010_comments.rs
deleted file mode 100644
index 71bdd1f9c..000000000
--- a/crates/ra_syntax/test_data/lexer/0010_comments.rs
+++ /dev/null
@@ -1,3 +0,0 @@
1#!/usr/bin/env bash
2// hello
3//! World
diff --git a/crates/ra_syntax/test_data/lexer/0010_comments.txt b/crates/ra_syntax/test_data/lexer/0010_comments.txt
deleted file mode 100644
index 3c997de3f..000000000
--- a/crates/ra_syntax/test_data/lexer/0010_comments.txt
+++ /dev/null
@@ -1,6 +0,0 @@
1SHEBANG 19 "#!/usr/bin/env bash"
2WHITESPACE 1 "\n"
3COMMENT 8 "// hello"
4WHITESPACE 1 "\n"
5COMMENT 9 "//! World"
6WHITESPACE 1 "\n"
diff --git a/crates/ra_syntax/test_data/lexer/0014_unclosed_char.rs b/crates/ra_syntax/test_data/lexer/0014_unclosed_char.rs
deleted file mode 100644
index 9c0007077..000000000
--- a/crates/ra_syntax/test_data/lexer/0014_unclosed_char.rs
+++ /dev/null
@@ -1 +0,0 @@
1'1 \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/0014_unclosed_char.txt b/crates/ra_syntax/test_data/lexer/0014_unclosed_char.txt
deleted file mode 100644
index 737a300ee..000000000
--- a/crates/ra_syntax/test_data/lexer/0014_unclosed_char.txt
+++ /dev/null
@@ -1 +0,0 @@
1LIFETIME 2 "\'1"
diff --git a/crates/ra_syntax/test_data/lexer/0015_unclosed_string.rs b/crates/ra_syntax/test_data/lexer/0015_unclosed_string.rs
deleted file mode 100644
index d771a26d4..000000000
--- a/crates/ra_syntax/test_data/lexer/0015_unclosed_string.rs
+++ /dev/null
@@ -1 +0,0 @@
1"hello
diff --git a/crates/ra_syntax/test_data/lexer/0015_unclosed_string.txt b/crates/ra_syntax/test_data/lexer/0015_unclosed_string.txt
deleted file mode 100644
index 728c40b66..000000000
--- a/crates/ra_syntax/test_data/lexer/0015_unclosed_string.txt
+++ /dev/null
@@ -1 +0,0 @@
1STRING 7 "\"hello\n"
diff --git a/crates/ra_syntax/test_data/lexer/err/0001_unclosed_char_at_eof.rs b/crates/ra_syntax/test_data/lexer/err/0001_unclosed_char_at_eof.rs
new file mode 100644
index 000000000..ad2823b48
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0001_unclosed_char_at_eof.rs
@@ -0,0 +1 @@
' \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0001_unclosed_char_at_eof.txt b/crates/ra_syntax/test_data/lexer/err/0001_unclosed_char_at_eof.txt
new file mode 100644
index 000000000..f24e1fd32
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0001_unclosed_char_at_eof.txt
@@ -0,0 +1,2 @@
1CHAR 1 "\'"
2> error[0; 1) token("\'") msg(Missing trailing `'` symbol to terminate the character literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0002_unclosed_char_with_ferris.rs b/crates/ra_syntax/test_data/lexer/err/0002_unclosed_char_with_ferris.rs
new file mode 100644
index 000000000..e264a4152
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0002_unclosed_char_with_ferris.rs
@@ -0,0 +1 @@
'🦀 \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0002_unclosed_char_with_ferris.txt b/crates/ra_syntax/test_data/lexer/err/0002_unclosed_char_with_ferris.txt
new file mode 100644
index 000000000..bd08cfc44
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0002_unclosed_char_with_ferris.txt
@@ -0,0 +1,2 @@
1CHAR 5 "\'🦀"
2> error[0; 5) token("\'🦀") msg(Missing trailing `'` symbol to terminate the character literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0003_unclosed_char_with_ascii_escape.rs b/crates/ra_syntax/test_data/lexer/err/0003_unclosed_char_with_ascii_escape.rs
new file mode 100644
index 000000000..cf74b4dad
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0003_unclosed_char_with_ascii_escape.rs
@@ -0,0 +1 @@
'\x7f \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0003_unclosed_char_with_ascii_escape.txt b/crates/ra_syntax/test_data/lexer/err/0003_unclosed_char_with_ascii_escape.txt
new file mode 100644
index 000000000..0ee22912d
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0003_unclosed_char_with_ascii_escape.txt
@@ -0,0 +1,2 @@
1CHAR 5 "\'\\x7f"
2> error[0; 5) token("\'\\x7f") msg(Missing trailing `'` symbol to terminate the character literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0004_unclosed_char_with_unicode_escape.rs b/crates/ra_syntax/test_data/lexer/err/0004_unclosed_char_with_unicode_escape.rs
new file mode 100644
index 000000000..50be91f68
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0004_unclosed_char_with_unicode_escape.rs
@@ -0,0 +1 @@
'\u{20AA} \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0004_unclosed_char_with_unicode_escape.txt b/crates/ra_syntax/test_data/lexer/err/0004_unclosed_char_with_unicode_escape.txt
new file mode 100644
index 000000000..96fac42ce
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0004_unclosed_char_with_unicode_escape.txt
@@ -0,0 +1,2 @@
1CHAR 9 "\'\\u{20AA}"
2> error[0; 9) token("\'\\u{20AA}") msg(Missing trailing `'` symbol to terminate the character literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0005_unclosed_char_with_space.rs b/crates/ra_syntax/test_data/lexer/err/0005_unclosed_char_with_space.rs
new file mode 100644
index 000000000..309ecfe47
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0005_unclosed_char_with_space.rs
@@ -0,0 +1 @@
' \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0005_unclosed_char_with_space.txt b/crates/ra_syntax/test_data/lexer/err/0005_unclosed_char_with_space.txt
new file mode 100644
index 000000000..2059f3f81
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0005_unclosed_char_with_space.txt
@@ -0,0 +1,2 @@
1CHAR 2 "\' "
2> error[0; 2) token("\' ") msg(Missing trailing `'` symbol to terminate the character literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0006_unclosed_char_with_slash.rs b/crates/ra_syntax/test_data/lexer/err/0006_unclosed_char_with_slash.rs
new file mode 100644
index 000000000..6ba258b10
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0006_unclosed_char_with_slash.rs
@@ -0,0 +1 @@
'\ \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0006_unclosed_char_with_slash.txt b/crates/ra_syntax/test_data/lexer/err/0006_unclosed_char_with_slash.txt
new file mode 100644
index 000000000..7dd376e59
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0006_unclosed_char_with_slash.txt
@@ -0,0 +1,2 @@
1CHAR 2 "\'\\"
2> error[0; 2) token("\'\\") msg(Missing trailing `'` symbol to terminate the character literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0007_unclosed_char_with_slash_n.rs b/crates/ra_syntax/test_data/lexer/err/0007_unclosed_char_with_slash_n.rs
new file mode 100644
index 000000000..78bef7e3e
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0007_unclosed_char_with_slash_n.rs
@@ -0,0 +1 @@
'\n \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0007_unclosed_char_with_slash_n.txt b/crates/ra_syntax/test_data/lexer/err/0007_unclosed_char_with_slash_n.txt
new file mode 100644
index 000000000..ef7a0a147
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0007_unclosed_char_with_slash_n.txt
@@ -0,0 +1,2 @@
1CHAR 3 "\'\\n"
2> error[0; 3) token("\'\\n") msg(Missing trailing `'` symbol to terminate the character literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0008_unclosed_char_with_slash_single_quote.rs b/crates/ra_syntax/test_data/lexer/err/0008_unclosed_char_with_slash_single_quote.rs
new file mode 100644
index 000000000..a0e722065
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0008_unclosed_char_with_slash_single_quote.rs
@@ -0,0 +1 @@
'\' \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0008_unclosed_char_with_slash_single_quote.txt b/crates/ra_syntax/test_data/lexer/err/0008_unclosed_char_with_slash_single_quote.txt
new file mode 100644
index 000000000..13fc5ea9a
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0008_unclosed_char_with_slash_single_quote.txt
@@ -0,0 +1,2 @@
1CHAR 3 "\'\\\'"
2> error[0; 3) token("\'\\\'") msg(Missing trailing `'` symbol to terminate the character literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0009_unclosed_byte_at_eof.rs b/crates/ra_syntax/test_data/lexer/err/0009_unclosed_byte_at_eof.rs
new file mode 100644
index 000000000..795dc7e25
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0009_unclosed_byte_at_eof.rs
@@ -0,0 +1 @@
b' \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0009_unclosed_byte_at_eof.txt b/crates/ra_syntax/test_data/lexer/err/0009_unclosed_byte_at_eof.txt
new file mode 100644
index 000000000..269d68c74
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0009_unclosed_byte_at_eof.txt
@@ -0,0 +1,2 @@
1BYTE 2 "b\'"
2> error[0; 2) token("b\'") msg(Missing trailing `'` symbol to terminate the byte literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0010_unclosed_byte_with_ferris.rs b/crates/ra_syntax/test_data/lexer/err/0010_unclosed_byte_with_ferris.rs
new file mode 100644
index 000000000..c9230dc24
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0010_unclosed_byte_with_ferris.rs
@@ -0,0 +1 @@
b'🦀 \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0010_unclosed_byte_with_ferris.txt b/crates/ra_syntax/test_data/lexer/err/0010_unclosed_byte_with_ferris.txt
new file mode 100644
index 000000000..91a76e479
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0010_unclosed_byte_with_ferris.txt
@@ -0,0 +1,2 @@
1BYTE 6 "b\'🦀"
2> error[0; 6) token("b\'🦀") msg(Missing trailing `'` symbol to terminate the byte literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0011_unclosed_byte_with_ascii_escape.rs b/crates/ra_syntax/test_data/lexer/err/0011_unclosed_byte_with_ascii_escape.rs
new file mode 100644
index 000000000..d146a8090
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0011_unclosed_byte_with_ascii_escape.rs
@@ -0,0 +1 @@
b'\x7f \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0011_unclosed_byte_with_ascii_escape.txt b/crates/ra_syntax/test_data/lexer/err/0011_unclosed_byte_with_ascii_escape.txt
new file mode 100644
index 000000000..b8c804a18
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0011_unclosed_byte_with_ascii_escape.txt
@@ -0,0 +1,2 @@
1BYTE 6 "b\'\\x7f"
2> error[0; 6) token("b\'\\x7f") msg(Missing trailing `'` symbol to terminate the byte literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0012_unclosed_byte_with_unicode_escape.rs b/crates/ra_syntax/test_data/lexer/err/0012_unclosed_byte_with_unicode_escape.rs
new file mode 100644
index 000000000..a3dec7c25
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0012_unclosed_byte_with_unicode_escape.rs
@@ -0,0 +1 @@
b'\u{20AA} \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0012_unclosed_byte_with_unicode_escape.txt b/crates/ra_syntax/test_data/lexer/err/0012_unclosed_byte_with_unicode_escape.txt
new file mode 100644
index 000000000..dfca22a59
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0012_unclosed_byte_with_unicode_escape.txt
@@ -0,0 +1,2 @@
1BYTE 10 "b\'\\u{20AA}"
2> error[0; 10) token("b\'\\u{20AA}") msg(Missing trailing `'` symbol to terminate the byte literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0013_unclosed_byte_with_space.rs b/crates/ra_syntax/test_data/lexer/err/0013_unclosed_byte_with_space.rs
new file mode 100644
index 000000000..93b7f9c87
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0013_unclosed_byte_with_space.rs
@@ -0,0 +1 @@
b' \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0013_unclosed_byte_with_space.txt b/crates/ra_syntax/test_data/lexer/err/0013_unclosed_byte_with_space.txt
new file mode 100644
index 000000000..51a1cceab
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0013_unclosed_byte_with_space.txt
@@ -0,0 +1,2 @@
1BYTE 3 "b\' "
2> error[0; 3) token("b\' ") msg(Missing trailing `'` symbol to terminate the byte literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0014_unclosed_byte_with_slash.rs b/crates/ra_syntax/test_data/lexer/err/0014_unclosed_byte_with_slash.rs
new file mode 100644
index 000000000..abffa5037
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0014_unclosed_byte_with_slash.rs
@@ -0,0 +1 @@
b'\ \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0014_unclosed_byte_with_slash.txt b/crates/ra_syntax/test_data/lexer/err/0014_unclosed_byte_with_slash.txt
new file mode 100644
index 000000000..24e835c27
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0014_unclosed_byte_with_slash.txt
@@ -0,0 +1,2 @@
1BYTE 3 "b\'\\"
2> error[0; 3) token("b\'\\") msg(Missing trailing `'` symbol to terminate the byte literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0015_unclosed_byte_with_slash_n.rs b/crates/ra_syntax/test_data/lexer/err/0015_unclosed_byte_with_slash_n.rs
new file mode 100644
index 000000000..4f46836a9
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0015_unclosed_byte_with_slash_n.rs
@@ -0,0 +1 @@
b'\n \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0015_unclosed_byte_with_slash_n.txt b/crates/ra_syntax/test_data/lexer/err/0015_unclosed_byte_with_slash_n.txt
new file mode 100644
index 000000000..f1e39a41b
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0015_unclosed_byte_with_slash_n.txt
@@ -0,0 +1,2 @@
1BYTE 4 "b\'\\n"
2> error[0; 4) token("b\'\\n") msg(Missing trailing `'` symbol to terminate the byte literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0016_unclosed_byte_with_slash_single_quote.rs b/crates/ra_syntax/test_data/lexer/err/0016_unclosed_byte_with_slash_single_quote.rs
new file mode 100644
index 000000000..645b641ee
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0016_unclosed_byte_with_slash_single_quote.rs
@@ -0,0 +1 @@
b'\' \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0016_unclosed_byte_with_slash_single_quote.txt b/crates/ra_syntax/test_data/lexer/err/0016_unclosed_byte_with_slash_single_quote.txt
new file mode 100644
index 000000000..f8ffe815d
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0016_unclosed_byte_with_slash_single_quote.txt
@@ -0,0 +1,2 @@
1BYTE 4 "b\'\\\'"
2> error[0; 4) token("b\'\\\'") msg(Missing trailing `'` symbol to terminate the byte literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0017_unclosed_string_at_eof.rs b/crates/ra_syntax/test_data/lexer/err/0017_unclosed_string_at_eof.rs
new file mode 100644
index 000000000..9d68933c4
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0017_unclosed_string_at_eof.rs
@@ -0,0 +1 @@
" \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0017_unclosed_string_at_eof.txt b/crates/ra_syntax/test_data/lexer/err/0017_unclosed_string_at_eof.txt
new file mode 100644
index 000000000..823daaf6f
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0017_unclosed_string_at_eof.txt
@@ -0,0 +1,2 @@
1STRING 1 "\""
2> error[0; 1) token("\"") msg(Missing trailing `"` symbol to terminate the string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0018_unclosed_string_with_ferris.rs b/crates/ra_syntax/test_data/lexer/err/0018_unclosed_string_with_ferris.rs
new file mode 100644
index 000000000..d439b8d2a
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0018_unclosed_string_with_ferris.rs
@@ -0,0 +1 @@
"🦀 \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0018_unclosed_string_with_ferris.txt b/crates/ra_syntax/test_data/lexer/err/0018_unclosed_string_with_ferris.txt
new file mode 100644
index 000000000..164580eb3
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0018_unclosed_string_with_ferris.txt
@@ -0,0 +1,2 @@
1STRING 5 "\"🦀"
2> error[0; 5) token("\"🦀") msg(Missing trailing `"` symbol to terminate the string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0019_unclosed_string_with_ascii_escape.rs b/crates/ra_syntax/test_data/lexer/err/0019_unclosed_string_with_ascii_escape.rs
new file mode 100644
index 000000000..56186a344
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0019_unclosed_string_with_ascii_escape.rs
@@ -0,0 +1 @@
"\x7f \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0019_unclosed_string_with_ascii_escape.txt b/crates/ra_syntax/test_data/lexer/err/0019_unclosed_string_with_ascii_escape.txt
new file mode 100644
index 000000000..4453827c3
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0019_unclosed_string_with_ascii_escape.txt
@@ -0,0 +1,2 @@
1STRING 5 "\"\\x7f"
2> error[0; 5) token("\"\\x7f") msg(Missing trailing `"` symbol to terminate the string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0020_unclosed_string_with_unicode_escape.rs b/crates/ra_syntax/test_data/lexer/err/0020_unclosed_string_with_unicode_escape.rs
new file mode 100644
index 000000000..ed24095c3
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0020_unclosed_string_with_unicode_escape.rs
@@ -0,0 +1 @@
"\u{20AA} \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0020_unclosed_string_with_unicode_escape.txt b/crates/ra_syntax/test_data/lexer/err/0020_unclosed_string_with_unicode_escape.txt
new file mode 100644
index 000000000..aa614f304
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0020_unclosed_string_with_unicode_escape.txt
@@ -0,0 +1,2 @@
1STRING 9 "\"\\u{20AA}"
2> error[0; 9) token("\"\\u{20AA}") msg(Missing trailing `"` symbol to terminate the string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0021_unclosed_string_with_space.rs b/crates/ra_syntax/test_data/lexer/err/0021_unclosed_string_with_space.rs
new file mode 100644
index 000000000..72cdc841f
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0021_unclosed_string_with_space.rs
@@ -0,0 +1 @@
" \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0021_unclosed_string_with_space.txt b/crates/ra_syntax/test_data/lexer/err/0021_unclosed_string_with_space.txt
new file mode 100644
index 000000000..b7db1236f
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0021_unclosed_string_with_space.txt
@@ -0,0 +1,2 @@
1STRING 2 "\" "
2> error[0; 2) token("\" ") msg(Missing trailing `"` symbol to terminate the string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0022_unclosed_string_with_slash.rs b/crates/ra_syntax/test_data/lexer/err/0022_unclosed_string_with_slash.rs
new file mode 100644
index 000000000..00a258400
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0022_unclosed_string_with_slash.rs
@@ -0,0 +1 @@
"\ \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0022_unclosed_string_with_slash.txt b/crates/ra_syntax/test_data/lexer/err/0022_unclosed_string_with_slash.txt
new file mode 100644
index 000000000..9d3df3799
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0022_unclosed_string_with_slash.txt
@@ -0,0 +1,2 @@
1STRING 2 "\"\\"
2> error[0; 2) token("\"\\") msg(Missing trailing `"` symbol to terminate the string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0023_unclosed_string_with_slash_n.rs b/crates/ra_syntax/test_data/lexer/err/0023_unclosed_string_with_slash_n.rs
new file mode 100644
index 000000000..a0c29b8cf
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0023_unclosed_string_with_slash_n.rs
@@ -0,0 +1 @@
"\n \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0023_unclosed_string_with_slash_n.txt b/crates/ra_syntax/test_data/lexer/err/0023_unclosed_string_with_slash_n.txt
new file mode 100644
index 000000000..e3eb672b6
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0023_unclosed_string_with_slash_n.txt
@@ -0,0 +1,2 @@
1STRING 3 "\"\\n"
2> error[0; 3) token("\"\\n") msg(Missing trailing `"` symbol to terminate the string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0024_unclosed_string_with_slash_double_quote.rs b/crates/ra_syntax/test_data/lexer/err/0024_unclosed_string_with_slash_double_quote.rs
new file mode 100644
index 000000000..403c2d6dd
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0024_unclosed_string_with_slash_double_quote.rs
@@ -0,0 +1 @@
"\" \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0024_unclosed_string_with_slash_double_quote.txt b/crates/ra_syntax/test_data/lexer/err/0024_unclosed_string_with_slash_double_quote.txt
new file mode 100644
index 000000000..041d7fb6e
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0024_unclosed_string_with_slash_double_quote.txt
@@ -0,0 +1,2 @@
1STRING 3 "\"\\\""
2> error[0; 3) token("\"\\\"") msg(Missing trailing `"` symbol to terminate the string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0025_unclosed_byte_string_at_eof.rs b/crates/ra_syntax/test_data/lexer/err/0025_unclosed_byte_string_at_eof.rs
new file mode 100644
index 000000000..36f4f4321
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0025_unclosed_byte_string_at_eof.rs
@@ -0,0 +1 @@
b" \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0025_unclosed_byte_string_at_eof.txt b/crates/ra_syntax/test_data/lexer/err/0025_unclosed_byte_string_at_eof.txt
new file mode 100644
index 000000000..be7970a83
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0025_unclosed_byte_string_at_eof.txt
@@ -0,0 +1,2 @@
1BYTE_STRING 2 "b\""
2> error[0; 2) token("b\"") msg(Missing trailing `"` symbol to terminate the byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0026_unclosed_byte_string_with_ferris.rs b/crates/ra_syntax/test_data/lexer/err/0026_unclosed_byte_string_with_ferris.rs
new file mode 100644
index 000000000..3c23a0372
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0026_unclosed_byte_string_with_ferris.rs
@@ -0,0 +1 @@
b"🦀 \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0026_unclosed_byte_string_with_ferris.txt b/crates/ra_syntax/test_data/lexer/err/0026_unclosed_byte_string_with_ferris.txt
new file mode 100644
index 000000000..bf9aab132
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0026_unclosed_byte_string_with_ferris.txt
@@ -0,0 +1,2 @@
1BYTE_STRING 6 "b\"🦀"
2> error[0; 6) token("b\"🦀") msg(Missing trailing `"` symbol to terminate the byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0027_unclosed_byte_string_with_ascii_escape.rs b/crates/ra_syntax/test_data/lexer/err/0027_unclosed_byte_string_with_ascii_escape.rs
new file mode 100644
index 000000000..836c112c1
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0027_unclosed_byte_string_with_ascii_escape.rs
@@ -0,0 +1 @@
b"\x7f \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0027_unclosed_byte_string_with_ascii_escape.txt b/crates/ra_syntax/test_data/lexer/err/0027_unclosed_byte_string_with_ascii_escape.txt
new file mode 100644
index 000000000..76e16d7d3
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0027_unclosed_byte_string_with_ascii_escape.txt
@@ -0,0 +1,2 @@
1BYTE_STRING 6 "b\"\\x7f"
2> error[0; 6) token("b\"\\x7f") msg(Missing trailing `"` symbol to terminate the byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0028_unclosed_byte_string_with_unicode_escape.rs b/crates/ra_syntax/test_data/lexer/err/0028_unclosed_byte_string_with_unicode_escape.rs
new file mode 100644
index 000000000..1c6df1d00
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0028_unclosed_byte_string_with_unicode_escape.rs
@@ -0,0 +1 @@
b"\u{20AA} \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0028_unclosed_byte_string_with_unicode_escape.txt b/crates/ra_syntax/test_data/lexer/err/0028_unclosed_byte_string_with_unicode_escape.txt
new file mode 100644
index 000000000..09adffa16
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0028_unclosed_byte_string_with_unicode_escape.txt
@@ -0,0 +1,2 @@
1BYTE_STRING 10 "b\"\\u{20AA}"
2> error[0; 10) token("b\"\\u{20AA}") msg(Missing trailing `"` symbol to terminate the byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0029_unclosed_byte_string_with_space.rs b/crates/ra_syntax/test_data/lexer/err/0029_unclosed_byte_string_with_space.rs
new file mode 100644
index 000000000..d6898541e
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0029_unclosed_byte_string_with_space.rs
@@ -0,0 +1 @@
b" \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0029_unclosed_byte_string_with_space.txt b/crates/ra_syntax/test_data/lexer/err/0029_unclosed_byte_string_with_space.txt
new file mode 100644
index 000000000..fcb7253c8
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0029_unclosed_byte_string_with_space.txt
@@ -0,0 +1,2 @@
1BYTE_STRING 3 "b\" "
2> error[0; 3) token("b\" ") msg(Missing trailing `"` symbol to terminate the byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0030_unclosed_byte_string_with_slash.rs b/crates/ra_syntax/test_data/lexer/err/0030_unclosed_byte_string_with_slash.rs
new file mode 100644
index 000000000..cce661538
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0030_unclosed_byte_string_with_slash.rs
@@ -0,0 +1 @@
b"\ \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0030_unclosed_byte_string_with_slash.txt b/crates/ra_syntax/test_data/lexer/err/0030_unclosed_byte_string_with_slash.txt
new file mode 100644
index 000000000..0a1b3e269
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0030_unclosed_byte_string_with_slash.txt
@@ -0,0 +1,2 @@
1BYTE_STRING 3 "b\"\\"
2> error[0; 3) token("b\"\\") msg(Missing trailing `"` symbol to terminate the byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0031_unclosed_byte_string_with_slash_n.rs b/crates/ra_syntax/test_data/lexer/err/0031_unclosed_byte_string_with_slash_n.rs
new file mode 100644
index 000000000..5e680aabb
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0031_unclosed_byte_string_with_slash_n.rs
@@ -0,0 +1 @@
b"\n \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0031_unclosed_byte_string_with_slash_n.txt b/crates/ra_syntax/test_data/lexer/err/0031_unclosed_byte_string_with_slash_n.txt
new file mode 100644
index 000000000..1fb89d2b6
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0031_unclosed_byte_string_with_slash_n.txt
@@ -0,0 +1,2 @@
1BYTE_STRING 4 "b\"\\n"
2> error[0; 4) token("b\"\\n") msg(Missing trailing `"` symbol to terminate the byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0032_unclosed_byte_string_with_slash_double_quote.rs b/crates/ra_syntax/test_data/lexer/err/0032_unclosed_byte_string_with_slash_double_quote.rs
new file mode 100644
index 000000000..f2ff58ba9
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0032_unclosed_byte_string_with_slash_double_quote.rs
@@ -0,0 +1 @@
b"\" \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0032_unclosed_byte_string_with_slash_double_quote.txt b/crates/ra_syntax/test_data/lexer/err/0032_unclosed_byte_string_with_slash_double_quote.txt
new file mode 100644
index 000000000..718d36992
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0032_unclosed_byte_string_with_slash_double_quote.txt
@@ -0,0 +1,2 @@
1BYTE_STRING 4 "b\"\\\""
2> error[0; 4) token("b\"\\\"") msg(Missing trailing `"` symbol to terminate the byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0033_unclosed_raw_string_at_eof.rs b/crates/ra_syntax/test_data/lexer/err/0033_unclosed_raw_string_at_eof.rs
new file mode 100644
index 000000000..557c59b62
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0033_unclosed_raw_string_at_eof.rs
@@ -0,0 +1 @@
r##" \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0033_unclosed_raw_string_at_eof.txt b/crates/ra_syntax/test_data/lexer/err/0033_unclosed_raw_string_at_eof.txt
new file mode 100644
index 000000000..93348f548
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0033_unclosed_raw_string_at_eof.txt
@@ -0,0 +1,2 @@
1RAW_STRING 4 "r##\""
2> error[0; 4) token("r##\"") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0034_unclosed_raw_string_with_ferris.rs b/crates/ra_syntax/test_data/lexer/err/0034_unclosed_raw_string_with_ferris.rs
new file mode 100644
index 000000000..bd046e4bb
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0034_unclosed_raw_string_with_ferris.rs
@@ -0,0 +1 @@
r##"🦀 \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0034_unclosed_raw_string_with_ferris.txt b/crates/ra_syntax/test_data/lexer/err/0034_unclosed_raw_string_with_ferris.txt
new file mode 100644
index 000000000..42c70dfe8
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0034_unclosed_raw_string_with_ferris.txt
@@ -0,0 +1,2 @@
1RAW_STRING 8 "r##\"🦀"
2> error[0; 8) token("r##\"🦀") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0035_unclosed_raw_string_with_ascii_escape.rs b/crates/ra_syntax/test_data/lexer/err/0035_unclosed_raw_string_with_ascii_escape.rs
new file mode 100644
index 000000000..5bec883dc
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0035_unclosed_raw_string_with_ascii_escape.rs
@@ -0,0 +1 @@
r##"\x7f \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0035_unclosed_raw_string_with_ascii_escape.txt b/crates/ra_syntax/test_data/lexer/err/0035_unclosed_raw_string_with_ascii_escape.txt
new file mode 100644
index 000000000..2bdeea0ff
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0035_unclosed_raw_string_with_ascii_escape.txt
@@ -0,0 +1,2 @@
1RAW_STRING 8 "r##\"\\x7f"
2> error[0; 8) token("r##\"\\x7f") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0036_unclosed_raw_string_with_unicode_escape.rs b/crates/ra_syntax/test_data/lexer/err/0036_unclosed_raw_string_with_unicode_escape.rs
new file mode 100644
index 000000000..bf05c3913
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0036_unclosed_raw_string_with_unicode_escape.rs
@@ -0,0 +1 @@
r##"\u{20AA} \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0036_unclosed_raw_string_with_unicode_escape.txt b/crates/ra_syntax/test_data/lexer/err/0036_unclosed_raw_string_with_unicode_escape.txt
new file mode 100644
index 000000000..667d4d79f
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0036_unclosed_raw_string_with_unicode_escape.txt
@@ -0,0 +1,2 @@
1RAW_STRING 12 "r##\"\\u{20AA}"
2> error[0; 12) token("r##\"\\u{20AA}") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0037_unclosed_raw_string_with_space.rs b/crates/ra_syntax/test_data/lexer/err/0037_unclosed_raw_string_with_space.rs
new file mode 100644
index 000000000..f104bae4f
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0037_unclosed_raw_string_with_space.rs
@@ -0,0 +1 @@
r##" \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0037_unclosed_raw_string_with_space.txt b/crates/ra_syntax/test_data/lexer/err/0037_unclosed_raw_string_with_space.txt
new file mode 100644
index 000000000..dd9597a1a
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0037_unclosed_raw_string_with_space.txt
@@ -0,0 +1,2 @@
1RAW_STRING 5 "r##\" "
2> error[0; 5) token("r##\" ") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0038_unclosed_raw_string_with_slash.rs b/crates/ra_syntax/test_data/lexer/err/0038_unclosed_raw_string_with_slash.rs
new file mode 100644
index 000000000..9242077b8
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0038_unclosed_raw_string_with_slash.rs
@@ -0,0 +1 @@
r##"\ \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0038_unclosed_raw_string_with_slash.txt b/crates/ra_syntax/test_data/lexer/err/0038_unclosed_raw_string_with_slash.txt
new file mode 100644
index 000000000..6ac6e3d62
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0038_unclosed_raw_string_with_slash.txt
@@ -0,0 +1,2 @@
1RAW_STRING 5 "r##\"\\"
2> error[0; 5) token("r##\"\\") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0039_unclosed_raw_string_with_slash_n.rs b/crates/ra_syntax/test_data/lexer/err/0039_unclosed_raw_string_with_slash_n.rs
new file mode 100644
index 000000000..db1c16f2b
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0039_unclosed_raw_string_with_slash_n.rs
@@ -0,0 +1 @@
r##"\n \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0039_unclosed_raw_string_with_slash_n.txt b/crates/ra_syntax/test_data/lexer/err/0039_unclosed_raw_string_with_slash_n.txt
new file mode 100644
index 000000000..9d35443f5
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0039_unclosed_raw_string_with_slash_n.txt
@@ -0,0 +1,2 @@
1RAW_STRING 6 "r##\"\\n"
2> error[0; 6) token("r##\"\\n") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0040_unclosed_raw_byte_string_at_eof.rs b/crates/ra_syntax/test_data/lexer/err/0040_unclosed_raw_byte_string_at_eof.rs
new file mode 100644
index 000000000..ae5bae622
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0040_unclosed_raw_byte_string_at_eof.rs
@@ -0,0 +1 @@
br##" \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0040_unclosed_raw_byte_string_at_eof.txt b/crates/ra_syntax/test_data/lexer/err/0040_unclosed_raw_byte_string_at_eof.txt
new file mode 100644
index 000000000..81fa39ea5
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0040_unclosed_raw_byte_string_at_eof.txt
@@ -0,0 +1,2 @@
1RAW_BYTE_STRING 5 "br##\""
2> error[0; 5) token("br##\"") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0041_unclosed_raw_byte_string_with_ferris.rs b/crates/ra_syntax/test_data/lexer/err/0041_unclosed_raw_byte_string_with_ferris.rs
new file mode 100644
index 000000000..9ef01207a
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0041_unclosed_raw_byte_string_with_ferris.rs
@@ -0,0 +1 @@
br##"🦀 \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0041_unclosed_raw_byte_string_with_ferris.txt b/crates/ra_syntax/test_data/lexer/err/0041_unclosed_raw_byte_string_with_ferris.txt
new file mode 100644
index 000000000..c2503a4d0
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0041_unclosed_raw_byte_string_with_ferris.txt
@@ -0,0 +1,2 @@
1RAW_BYTE_STRING 9 "br##\"🦀"
2> error[0; 9) token("br##\"🦀") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0042_unclosed_raw_byte_string_with_ascii_escape.rs b/crates/ra_syntax/test_data/lexer/err/0042_unclosed_raw_byte_string_with_ascii_escape.rs
new file mode 100644
index 000000000..d50270afe
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0042_unclosed_raw_byte_string_with_ascii_escape.rs
@@ -0,0 +1 @@
br##"\x7f \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0042_unclosed_raw_byte_string_with_ascii_escape.txt b/crates/ra_syntax/test_data/lexer/err/0042_unclosed_raw_byte_string_with_ascii_escape.txt
new file mode 100644
index 000000000..3bd3d8152
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0042_unclosed_raw_byte_string_with_ascii_escape.txt
@@ -0,0 +1,2 @@
1RAW_BYTE_STRING 9 "br##\"\\x7f"
2> error[0; 9) token("br##\"\\x7f") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0043_unclosed_raw_byte_string_with_unicode_escape.rs b/crates/ra_syntax/test_data/lexer/err/0043_unclosed_raw_byte_string_with_unicode_escape.rs
new file mode 100644
index 000000000..90e299a1a
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0043_unclosed_raw_byte_string_with_unicode_escape.rs
@@ -0,0 +1 @@
br##"\u{20AA} \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0043_unclosed_raw_byte_string_with_unicode_escape.txt b/crates/ra_syntax/test_data/lexer/err/0043_unclosed_raw_byte_string_with_unicode_escape.txt
new file mode 100644
index 000000000..a512f0428
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0043_unclosed_raw_byte_string_with_unicode_escape.txt
@@ -0,0 +1,2 @@
1RAW_BYTE_STRING 13 "br##\"\\u{20AA}"
2> error[0; 13) token("br##\"\\u{20AA}") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0044_unclosed_raw_byte_string_with_space.rs b/crates/ra_syntax/test_data/lexer/err/0044_unclosed_raw_byte_string_with_space.rs
new file mode 100644
index 000000000..14c602fd2
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0044_unclosed_raw_byte_string_with_space.rs
@@ -0,0 +1 @@
br##" \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0044_unclosed_raw_byte_string_with_space.txt b/crates/ra_syntax/test_data/lexer/err/0044_unclosed_raw_byte_string_with_space.txt
new file mode 100644
index 000000000..dc616a623
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0044_unclosed_raw_byte_string_with_space.txt
@@ -0,0 +1,2 @@
1RAW_BYTE_STRING 6 "br##\" "
2> error[0; 6) token("br##\" ") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0045_unclosed_raw_byte_string_with_slash.rs b/crates/ra_syntax/test_data/lexer/err/0045_unclosed_raw_byte_string_with_slash.rs
new file mode 100644
index 000000000..0b3c015d7
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0045_unclosed_raw_byte_string_with_slash.rs
@@ -0,0 +1 @@
br##"\ \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0045_unclosed_raw_byte_string_with_slash.txt b/crates/ra_syntax/test_data/lexer/err/0045_unclosed_raw_byte_string_with_slash.txt
new file mode 100644
index 000000000..debafe380
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0045_unclosed_raw_byte_string_with_slash.txt
@@ -0,0 +1,2 @@
1RAW_BYTE_STRING 6 "br##\"\\"
2> error[0; 6) token("br##\"\\") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0046_unclosed_raw_byte_string_with_slash_n.rs b/crates/ra_syntax/test_data/lexer/err/0046_unclosed_raw_byte_string_with_slash_n.rs
new file mode 100644
index 000000000..0d8b0e7ab
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0046_unclosed_raw_byte_string_with_slash_n.rs
@@ -0,0 +1 @@
br##"\n \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0046_unclosed_raw_byte_string_with_slash_n.txt b/crates/ra_syntax/test_data/lexer/err/0046_unclosed_raw_byte_string_with_slash_n.txt
new file mode 100644
index 000000000..524e617b7
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0046_unclosed_raw_byte_string_with_slash_n.txt
@@ -0,0 +1,2 @@
1RAW_BYTE_STRING 7 "br##\"\\n"
2> error[0; 7) token("br##\"\\n") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0047_unstarted_raw_string_at_eof.rs b/crates/ra_syntax/test_data/lexer/err/0047_unstarted_raw_string_at_eof.rs
new file mode 100644
index 000000000..eddf8d080
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0047_unstarted_raw_string_at_eof.rs
@@ -0,0 +1 @@
r## \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0047_unstarted_raw_string_at_eof.txt b/crates/ra_syntax/test_data/lexer/err/0047_unstarted_raw_string_at_eof.txt
new file mode 100644
index 000000000..00b046840
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0047_unstarted_raw_string_at_eof.txt
@@ -0,0 +1,2 @@
1RAW_STRING 3 "r##"
2> error[0; 3) token("r##") msg(Missing `"` symbol after `#` symbols to begin the raw string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0048_unstarted_raw_byte_string_at_eof.rs b/crates/ra_syntax/test_data/lexer/err/0048_unstarted_raw_byte_string_at_eof.rs
new file mode 100644
index 000000000..7e8cadf4f
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0048_unstarted_raw_byte_string_at_eof.rs
@@ -0,0 +1 @@
br## \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0048_unstarted_raw_byte_string_at_eof.txt b/crates/ra_syntax/test_data/lexer/err/0048_unstarted_raw_byte_string_at_eof.txt
new file mode 100644
index 000000000..33b25e60f
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0048_unstarted_raw_byte_string_at_eof.txt
@@ -0,0 +1,2 @@
1RAW_BYTE_STRING 4 "br##"
2> error[0; 4) token("br##") msg(Missing `"` symbol after `#` symbols to begin the raw byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0049_unstarted_raw_string_with_ascii.rs b/crates/ra_syntax/test_data/lexer/err/0049_unstarted_raw_string_with_ascii.rs
new file mode 100644
index 000000000..534668a9b
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0049_unstarted_raw_string_with_ascii.rs
@@ -0,0 +1 @@
r## I lack a quote! \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0049_unstarted_raw_string_with_ascii.txt b/crates/ra_syntax/test_data/lexer/err/0049_unstarted_raw_string_with_ascii.txt
new file mode 100644
index 000000000..782dfd974
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0049_unstarted_raw_string_with_ascii.txt
@@ -0,0 +1,10 @@
1RAW_STRING 4 "r## "
2IDENT 1 "I"
3WHITESPACE 1 " "
4IDENT 4 "lack"
5WHITESPACE 1 " "
6IDENT 1 "a"
7WHITESPACE 1 " "
8IDENT 5 "quote"
9EXCL 1 "!"
10> error[0; 4) token("r## ") msg(Missing `"` symbol after `#` symbols to begin the raw string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0050_unstarted_raw_byte_string_with_ascii.rs b/crates/ra_syntax/test_data/lexer/err/0050_unstarted_raw_byte_string_with_ascii.rs
new file mode 100644
index 000000000..d9b55455a
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0050_unstarted_raw_byte_string_with_ascii.rs
@@ -0,0 +1 @@
br## I lack a quote! \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0050_unstarted_raw_byte_string_with_ascii.txt b/crates/ra_syntax/test_data/lexer/err/0050_unstarted_raw_byte_string_with_ascii.txt
new file mode 100644
index 000000000..59c40cd65
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0050_unstarted_raw_byte_string_with_ascii.txt
@@ -0,0 +1,10 @@
1RAW_BYTE_STRING 5 "br## "
2IDENT 1 "I"
3WHITESPACE 1 " "
4IDENT 4 "lack"
5WHITESPACE 1 " "
6IDENT 1 "a"
7WHITESPACE 1 " "
8IDENT 5 "quote"
9EXCL 1 "!"
10> error[0; 5) token("br## ") msg(Missing `"` symbol after `#` symbols to begin the raw byte string literal)
diff --git a/crates/ra_syntax/test_data/lexer/err/0051_unclosed_block_comment_at_eof.rs b/crates/ra_syntax/test_data/lexer/err/0051_unclosed_block_comment_at_eof.rs
new file mode 100644
index 000000000..22e83649f
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0051_unclosed_block_comment_at_eof.rs
@@ -0,0 +1 @@
/* \ No newline at end of file
diff --git a/crates/ra_syntax/test_data/lexer/err/0051_unclosed_block_comment_at_eof.txt b/crates/ra_syntax/test_data/lexer/err/0051_unclosed_block_comment_at_eof.txt
new file mode 100644
index 000000000..5d04cdaa4
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0051_unclosed_block_comment_at_eof.txt
@@ -0,0 +1,2 @@
1COMMENT 2 "/*"
2> error[0; 2) token("/*") msg(Missing trailing `*/` symbols to terminate the block comment)
diff --git a/crates/ra_syntax/test_data/lexer/err/0052_unclosed_block_comment_with_content.rs b/crates/ra_syntax/test_data/lexer/err/0052_unclosed_block_comment_with_content.rs
new file mode 100644
index 000000000..c45c2844d
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0052_unclosed_block_comment_with_content.rs
@@ -0,0 +1 @@
/* comment
diff --git a/crates/ra_syntax/test_data/lexer/err/0052_unclosed_block_comment_with_content.txt b/crates/ra_syntax/test_data/lexer/err/0052_unclosed_block_comment_with_content.txt
new file mode 100644
index 000000000..8c6b678e3
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0052_unclosed_block_comment_with_content.txt
@@ -0,0 +1,2 @@
1COMMENT 11 "/* comment\n"
2> error[0; 11) token("/* comment\n") msg(Missing trailing `*/` symbols to terminate the block comment)
diff --git a/crates/ra_syntax/test_data/lexer/err/0053_unclosed_nested_block_comment_entirely.rs b/crates/ra_syntax/test_data/lexer/err/0053_unclosed_nested_block_comment_entirely.rs
new file mode 100644
index 000000000..3fcfc9660
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0053_unclosed_nested_block_comment_entirely.rs
@@ -0,0 +1 @@
/* /* /*
diff --git a/crates/ra_syntax/test_data/lexer/err/0053_unclosed_nested_block_comment_entirely.txt b/crates/ra_syntax/test_data/lexer/err/0053_unclosed_nested_block_comment_entirely.txt
new file mode 100644
index 000000000..250de34d9
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0053_unclosed_nested_block_comment_entirely.txt
@@ -0,0 +1,2 @@
1COMMENT 9 "/* /* /*\n"
2> error[0; 9) token("/* /* /*\n") msg(Missing trailing `*/` symbols to terminate the block comment)
diff --git a/crates/ra_syntax/test_data/lexer/err/0054_unclosed_nested_block_comment_partially.rs b/crates/ra_syntax/test_data/lexer/err/0054_unclosed_nested_block_comment_partially.rs
new file mode 100644
index 000000000..26c898f01
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0054_unclosed_nested_block_comment_partially.rs
@@ -0,0 +1 @@
/** /*! /* comment */ */
diff --git a/crates/ra_syntax/test_data/lexer/err/0054_unclosed_nested_block_comment_partially.txt b/crates/ra_syntax/test_data/lexer/err/0054_unclosed_nested_block_comment_partially.txt
new file mode 100644
index 000000000..f97f2a8c7
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0054_unclosed_nested_block_comment_partially.txt
@@ -0,0 +1,2 @@
1COMMENT 25 "/** /*! /* comment */ */\n"
2> error[0; 25) token("/** /*! /* comment */ */\n") msg(Missing trailing `*/` symbols to terminate the block comment)
diff --git a/crates/ra_syntax/test_data/lexer/err/0055_empty_int.rs b/crates/ra_syntax/test_data/lexer/err/0055_empty_int.rs
new file mode 100644
index 000000000..aa2a9fdca
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0055_empty_int.rs
@@ -0,0 +1,17 @@
10b
20o
30x
4
50b_
60o_
70x_
8
90bnoDigit
100onoDigit
110xnoDigit
12
130xG
140xg
15
160x_g
170x_G
diff --git a/crates/ra_syntax/test_data/lexer/err/0055_empty_int.txt b/crates/ra_syntax/test_data/lexer/err/0055_empty_int.txt
new file mode 100644
index 000000000..2fe5bd950
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0055_empty_int.txt
@@ -0,0 +1,39 @@
1INT_NUMBER 2 "0b"
2WHITESPACE 1 "\n"
3INT_NUMBER 2 "0o"
4WHITESPACE 1 "\n"
5INT_NUMBER 2 "0x"
6WHITESPACE 2 "\n\n"
7INT_NUMBER 3 "0b_"
8WHITESPACE 1 "\n"
9INT_NUMBER 3 "0o_"
10WHITESPACE 1 "\n"
11INT_NUMBER 3 "0x_"
12WHITESPACE 2 "\n\n"
13INT_NUMBER 9 "0bnoDigit"
14WHITESPACE 1 "\n"
15INT_NUMBER 9 "0onoDigit"
16WHITESPACE 1 "\n"
17INT_NUMBER 9 "0xnoDigit"
18WHITESPACE 2 "\n\n"
19INT_NUMBER 3 "0xG"
20WHITESPACE 1 "\n"
21INT_NUMBER 3 "0xg"
22WHITESPACE 2 "\n\n"
23INT_NUMBER 4 "0x_g"
24WHITESPACE 1 "\n"
25INT_NUMBER 4 "0x_G"
26WHITESPACE 1 "\n"
27> error[0; 2) token("0b") msg(Missing digits after the integer base prefix)
28> error[3; 5) token("0o") msg(Missing digits after the integer base prefix)
29> error[6; 8) token("0x") msg(Missing digits after the integer base prefix)
30> error[10; 13) token("0b_") msg(Missing digits after the integer base prefix)
31> error[14; 17) token("0o_") msg(Missing digits after the integer base prefix)
32> error[18; 21) token("0x_") msg(Missing digits after the integer base prefix)
33> error[23; 32) token("0bnoDigit") msg(Missing digits after the integer base prefix)
34> error[33; 42) token("0onoDigit") msg(Missing digits after the integer base prefix)
35> error[43; 52) token("0xnoDigit") msg(Missing digits after the integer base prefix)
36> error[54; 57) token("0xG") msg(Missing digits after the integer base prefix)
37> error[58; 61) token("0xg") msg(Missing digits after the integer base prefix)
38> error[63; 67) token("0x_g") msg(Missing digits after the integer base prefix)
39> error[68; 72) token("0x_G") msg(Missing digits after the integer base prefix)
diff --git a/crates/ra_syntax/test_data/lexer/err/0056_empty_exponent.rs b/crates/ra_syntax/test_data/lexer/err/0056_empty_exponent.rs
new file mode 100644
index 000000000..286584c88
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0056_empty_exponent.rs
@@ -0,0 +1,22 @@
10e
20E
3
442e+
542e-
642E+
742E-
8
942.e+
1042.e-
1142.E+
1242.E-
13
1442.2e+
1542.2e-
1642.2E+
1742.2E-
18
1942.2e+f32
2042.2e-f32
2142.2E+f32
2242.2E-f32
diff --git a/crates/ra_syntax/test_data/lexer/err/0056_empty_exponent.txt b/crates/ra_syntax/test_data/lexer/err/0056_empty_exponent.txt
new file mode 100644
index 000000000..ab35e20a5
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0056_empty_exponent.txt
@@ -0,0 +1,62 @@
1FLOAT_NUMBER 2 "0e"
2WHITESPACE 1 "\n"
3FLOAT_NUMBER 2 "0E"
4WHITESPACE 2 "\n\n"
5FLOAT_NUMBER 4 "42e+"
6WHITESPACE 1 "\n"
7FLOAT_NUMBER 4 "42e-"
8WHITESPACE 1 "\n"
9FLOAT_NUMBER 4 "42E+"
10WHITESPACE 1 "\n"
11FLOAT_NUMBER 4 "42E-"
12WHITESPACE 2 "\n\n"
13INT_NUMBER 2 "42"
14DOT 1 "."
15IDENT 1 "e"
16PLUS 1 "+"
17WHITESPACE 1 "\n"
18INT_NUMBER 2 "42"
19DOT 1 "."
20IDENT 1 "e"
21MINUS 1 "-"
22WHITESPACE 1 "\n"
23INT_NUMBER 2 "42"
24DOT 1 "."
25IDENT 1 "E"
26PLUS 1 "+"
27WHITESPACE 1 "\n"
28INT_NUMBER 2 "42"
29DOT 1 "."
30IDENT 1 "E"
31MINUS 1 "-"
32WHITESPACE 2 "\n\n"
33FLOAT_NUMBER 6 "42.2e+"
34WHITESPACE 1 "\n"
35FLOAT_NUMBER 6 "42.2e-"
36WHITESPACE 1 "\n"
37FLOAT_NUMBER 6 "42.2E+"
38WHITESPACE 1 "\n"
39FLOAT_NUMBER 6 "42.2E-"
40WHITESPACE 2 "\n\n"
41FLOAT_NUMBER 9 "42.2e+f32"
42WHITESPACE 1 "\n"
43FLOAT_NUMBER 9 "42.2e-f32"
44WHITESPACE 1 "\n"
45FLOAT_NUMBER 9 "42.2E+f32"
46WHITESPACE 1 "\n"
47FLOAT_NUMBER 9 "42.2E-f32"
48WHITESPACE 1 "\n"
49> error[0; 2) token("0e") msg(Missing digits after the exponent symbol)
50> error[3; 5) token("0E") msg(Missing digits after the exponent symbol)
51> error[7; 11) token("42e+") msg(Missing digits after the exponent symbol)
52> error[12; 16) token("42e-") msg(Missing digits after the exponent symbol)
53> error[17; 21) token("42E+") msg(Missing digits after the exponent symbol)
54> error[22; 26) token("42E-") msg(Missing digits after the exponent symbol)
55> error[53; 59) token("42.2e+") msg(Missing digits after the exponent symbol)
56> error[60; 66) token("42.2e-") msg(Missing digits after the exponent symbol)
57> error[67; 73) token("42.2E+") msg(Missing digits after the exponent symbol)
58> error[74; 80) token("42.2E-") msg(Missing digits after the exponent symbol)
59> error[82; 91) token("42.2e+f32") msg(Missing digits after the exponent symbol)
60> error[92; 101) token("42.2e-f32") msg(Missing digits after the exponent symbol)
61> error[102; 111) token("42.2E+f32") msg(Missing digits after the exponent symbol)
62> error[112; 121) token("42.2E-f32") msg(Missing digits after the exponent symbol)
diff --git a/crates/ra_syntax/test_data/lexer/err/0057_lifetime_strarts_with_a_number.rs b/crates/ra_syntax/test_data/lexer/err/0057_lifetime_strarts_with_a_number.rs
new file mode 100644
index 000000000..a7698a404
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0057_lifetime_strarts_with_a_number.rs
@@ -0,0 +1,2 @@
1'1
2'1lifetime
diff --git a/crates/ra_syntax/test_data/lexer/err/0057_lifetime_strarts_with_a_number.txt b/crates/ra_syntax/test_data/lexer/err/0057_lifetime_strarts_with_a_number.txt
new file mode 100644
index 000000000..89b38bfac
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/err/0057_lifetime_strarts_with_a_number.txt
@@ -0,0 +1,6 @@
1LIFETIME 2 "\'1"
2WHITESPACE 1 "\n"
3LIFETIME 10 "\'1lifetime"
4WHITESPACE 1 "\n"
5> error[0; 2) token("\'1") msg(Lifetime name cannot start with a number)
6> error[3; 13) token("\'1lifetime") msg(Lifetime name cannot start with a number)
diff --git a/crates/ra_syntax/test_data/lexer/0001_hello.rs b/crates/ra_syntax/test_data/lexer/ok/0001_hello.rs
index 95d09f2b1..95d09f2b1 100644
--- a/crates/ra_syntax/test_data/lexer/0001_hello.rs
+++ b/crates/ra_syntax/test_data/lexer/ok/0001_hello.rs
diff --git a/crates/ra_syntax/test_data/lexer/0001_hello.txt b/crates/ra_syntax/test_data/lexer/ok/0001_hello.txt
index 27a5940a9..27a5940a9 100644
--- a/crates/ra_syntax/test_data/lexer/0001_hello.txt
+++ b/crates/ra_syntax/test_data/lexer/ok/0001_hello.txt
diff --git a/crates/ra_syntax/test_data/lexer/0002_whitespace.rs b/crates/ra_syntax/test_data/lexer/ok/0002_whitespace.rs
index 08fce1418..08fce1418 100644
--- a/crates/ra_syntax/test_data/lexer/0002_whitespace.rs
+++ b/crates/ra_syntax/test_data/lexer/ok/0002_whitespace.rs
diff --git a/crates/ra_syntax/test_data/lexer/0002_whitespace.txt b/crates/ra_syntax/test_data/lexer/ok/0002_whitespace.txt
index 01d260918..01d260918 100644
--- a/crates/ra_syntax/test_data/lexer/0002_whitespace.txt
+++ b/crates/ra_syntax/test_data/lexer/ok/0002_whitespace.txt
diff --git a/crates/ra_syntax/test_data/lexer/0003_ident.rs b/crates/ra_syntax/test_data/lexer/ok/0003_ident.rs
index c05c9c009..c05c9c009 100644
--- a/crates/ra_syntax/test_data/lexer/0003_ident.rs
+++ b/crates/ra_syntax/test_data/lexer/ok/0003_ident.rs
diff --git a/crates/ra_syntax/test_data/lexer/0003_ident.txt b/crates/ra_syntax/test_data/lexer/ok/0003_ident.txt
index 4a0d5c053..4a0d5c053 100644
--- a/crates/ra_syntax/test_data/lexer/0003_ident.txt
+++ b/crates/ra_syntax/test_data/lexer/ok/0003_ident.txt
diff --git a/crates/ra_syntax/test_data/lexer/0004_numbers.rs b/crates/ra_syntax/test_data/lexer/ok/0004_numbers.rs
index dc974b553..bc761c235 100644
--- a/crates/ra_syntax/test_data/lexer/0004_numbers.rs
+++ b/crates/ra_syntax/test_data/lexer/ok/0004_numbers.rs
@@ -1,4 +1,4 @@
10 0b 0o 0x 00 0_ 0. 0e 0E 0z 10 00 0_ 0. 0z
201790 0b1790 0o1790 0x1790aAbBcCdDeEfF 001279 0_1279 0.1279 0e1279 0E1279 201790 0b1790 0o1790 0x1790aAbBcCdDeEfF 001279 0_1279 0.1279 0e1279 0E1279
30..2 30..2
40.foo() 40.foo()
@@ -6,4 +6,4 @@
60.e+1 60.e+1
70.0E-2 70.0E-2
80___0.10000____0000e+111__ 80___0.10000____0000e+111__
91i64 92.0f32 11__s \ No newline at end of file 91i64 92.0f32 11__s
diff --git a/crates/ra_syntax/test_data/lexer/0004_numbers.txt b/crates/ra_syntax/test_data/lexer/ok/0004_numbers.txt
index 7bb89b8ae..e19fc5789 100644
--- a/crates/ra_syntax/test_data/lexer/0004_numbers.txt
+++ b/crates/ra_syntax/test_data/lexer/ok/0004_numbers.txt
@@ -1,21 +1,11 @@
1INT_NUMBER 1 "0" 1INT_NUMBER 1 "0"
2WHITESPACE 1 " " 2WHITESPACE 1 " "
3INT_NUMBER 2 "0b"
4WHITESPACE 1 " "
5INT_NUMBER 2 "0o"
6WHITESPACE 1 " "
7INT_NUMBER 2 "0x"
8WHITESPACE 1 " "
9INT_NUMBER 2 "00" 3INT_NUMBER 2 "00"
10WHITESPACE 1 " " 4WHITESPACE 1 " "
11INT_NUMBER 2 "0_" 5INT_NUMBER 2 "0_"
12WHITESPACE 1 " " 6WHITESPACE 1 " "
13FLOAT_NUMBER 2 "0." 7FLOAT_NUMBER 2 "0."
14WHITESPACE 1 " " 8WHITESPACE 1 " "
15FLOAT_NUMBER 2 "0e"
16WHITESPACE 1 " "
17FLOAT_NUMBER 2 "0E"
18WHITESPACE 1 " "
19INT_NUMBER 2 "0z" 9INT_NUMBER 2 "0z"
20WHITESPACE 1 "\n" 10WHITESPACE 1 "\n"
21INT_NUMBER 5 "01790" 11INT_NUMBER 5 "01790"
@@ -64,3 +54,4 @@ WHITESPACE 1 " "
64FLOAT_NUMBER 7 "92.0f32" 54FLOAT_NUMBER 7 "92.0f32"
65WHITESPACE 1 " " 55WHITESPACE 1 " "
66INT_NUMBER 5 "11__s" 56INT_NUMBER 5 "11__s"
57WHITESPACE 1 "\n"
diff --git a/crates/ra_syntax/test_data/lexer/0005_symbols.rs b/crates/ra_syntax/test_data/lexer/ok/0005_symbols.rs
index 487569b5a..487569b5a 100644
--- a/crates/ra_syntax/test_data/lexer/0005_symbols.rs
+++ b/crates/ra_syntax/test_data/lexer/ok/0005_symbols.rs
diff --git a/crates/ra_syntax/test_data/lexer/0005_symbols.txt b/crates/ra_syntax/test_data/lexer/ok/0005_symbols.txt
index 469a90e42..469a90e42 100644
--- a/crates/ra_syntax/test_data/lexer/0005_symbols.txt
+++ b/crates/ra_syntax/test_data/lexer/ok/0005_symbols.txt
diff --git a/crates/ra_syntax/test_data/lexer/0006_chars.rs b/crates/ra_syntax/test_data/lexer/ok/0006_chars.rs
index 454ee0a5f..454ee0a5f 100644
--- a/crates/ra_syntax/test_data/lexer/0006_chars.rs
+++ b/crates/ra_syntax/test_data/lexer/ok/0006_chars.rs
diff --git a/crates/ra_syntax/test_data/lexer/0006_chars.txt b/crates/ra_syntax/test_data/lexer/ok/0006_chars.txt
index 950954fbc..950954fbc 100644
--- a/crates/ra_syntax/test_data/lexer/0006_chars.txt
+++ b/crates/ra_syntax/test_data/lexer/ok/0006_chars.txt
diff --git a/crates/ra_syntax/test_data/lexer/0007_lifetimes.rs b/crates/ra_syntax/test_data/lexer/ok/0007_lifetimes.rs
index b764f1dce..b764f1dce 100644
--- a/crates/ra_syntax/test_data/lexer/0007_lifetimes.rs
+++ b/crates/ra_syntax/test_data/lexer/ok/0007_lifetimes.rs
diff --git a/crates/ra_syntax/test_data/lexer/0007_lifetimes.txt b/crates/ra_syntax/test_data/lexer/ok/0007_lifetimes.txt
index 005c29100..005c29100 100644
--- a/crates/ra_syntax/test_data/lexer/0007_lifetimes.txt
+++ b/crates/ra_syntax/test_data/lexer/ok/0007_lifetimes.txt
diff --git a/crates/ra_syntax/test_data/lexer/0008_byte_strings.rs b/crates/ra_syntax/test_data/lexer/ok/0008_byte_strings.rs
index b54930f5e..b54930f5e 100644
--- a/crates/ra_syntax/test_data/lexer/0008_byte_strings.rs
+++ b/crates/ra_syntax/test_data/lexer/ok/0008_byte_strings.rs
diff --git a/crates/ra_syntax/test_data/lexer/0008_byte_strings.txt b/crates/ra_syntax/test_data/lexer/ok/0008_byte_strings.txt
index bc03b51a8..bc03b51a8 100644
--- a/crates/ra_syntax/test_data/lexer/0008_byte_strings.txt
+++ b/crates/ra_syntax/test_data/lexer/ok/0008_byte_strings.txt
diff --git a/crates/ra_syntax/test_data/lexer/0009_strings.rs b/crates/ra_syntax/test_data/lexer/ok/0009_strings.rs
index 4ddb5bffc..4ddb5bffc 100644
--- a/crates/ra_syntax/test_data/lexer/0009_strings.rs
+++ b/crates/ra_syntax/test_data/lexer/ok/0009_strings.rs
diff --git a/crates/ra_syntax/test_data/lexer/0009_strings.txt b/crates/ra_syntax/test_data/lexer/ok/0009_strings.txt
index 4cb4d711d..4cb4d711d 100644
--- a/crates/ra_syntax/test_data/lexer/0009_strings.txt
+++ b/crates/ra_syntax/test_data/lexer/ok/0009_strings.txt
diff --git a/crates/ra_syntax/test_data/lexer/ok/0010_single_line_comments.rs b/crates/ra_syntax/test_data/lexer/ok/0010_single_line_comments.rs
new file mode 100644
index 000000000..4b6653f9c
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/ok/0010_single_line_comments.rs
@@ -0,0 +1,12 @@
1#!/usr/bin/env bash
2// hello
3//! World
4//!! Inner line doc
5/// Outer line doc
6//// Just a comment
7
8//
9//!
10//!!
11///
12////
diff --git a/crates/ra_syntax/test_data/lexer/ok/0010_single_line_comments.txt b/crates/ra_syntax/test_data/lexer/ok/0010_single_line_comments.txt
new file mode 100644
index 000000000..98a3818c0
--- /dev/null
+++ b/crates/ra_syntax/test_data/lexer/ok/0010_single_line_comments.txt
@@ -0,0 +1,22 @@
1SHEBANG 19 "#!/usr/bin/env bash"
2WHITESPACE 1 "\n"
3COMMENT 8 "// hello"
4WHITESPACE 1 "\n"
5COMMENT 9 "//! World"
6WHITESPACE 1 "\n"
7COMMENT 19 "//!! Inner line doc"
8WHITESPACE 1 "\n"
9COMMENT 18 "/// Outer line doc"
10WHITESPACE 1 "\n"
11COMMENT 19 "//// Just a comment"
12WHITESPACE 2 "\n\n"
13COMMENT 2 "//"
14WHITESPACE 1 "\n"
15COMMENT 3 "//!"
16WHITESPACE 1 "\n"
17COMMENT 4 "//!!"
18WHITESPACE 1 "\n"
19COMMENT 3 "///"
20WHITESPACE 1 "\n"
21COMMENT 4 "////"
22WHITESPACE 1 "\n"
diff --git a/crates/ra_syntax/test_data/lexer/0011_keywords.rs b/crates/ra_syntax/test_data/lexer/ok/0011_keywords.rs
index 1e91bff4e..1e91bff4e 100644
--- a/crates/ra_syntax/test_data/lexer/0011_keywords.rs
+++ b/crates/ra_syntax/test_data/lexer/ok/0011_keywords.rs
diff --git a/crates/ra_syntax/test_data/lexer/0011_keywords.txt b/crates/ra_syntax/test_data/lexer/ok/0011_keywords.txt
index 22c00eefb..22c00eefb 100644
--- a/crates/ra_syntax/test_data/lexer/0011_keywords.txt
+++ b/crates/ra_syntax/test_data/lexer/ok/0011_keywords.txt
diff --git a/crates/ra_syntax/test_data/lexer/00012_block_comment.rs b/crates/ra_syntax/test_data/lexer/ok/0012_block_comment.rs
index 708aac197..b880a59d9 100644
--- a/crates/ra_syntax/test_data/lexer/00012_block_comment.rs
+++ b/crates/ra_syntax/test_data/lexer/ok/0012_block_comment.rs
@@ -1,4 +1,3 @@
1/* */ 1/* */
2/**/ 2/**/
3/* /* */ */ 3/* /* */ */
4/*
diff --git a/crates/ra_syntax/test_data/lexer/00012_block_comment.txt b/crates/ra_syntax/test_data/lexer/ok/0012_block_comment.txt
index 9958b2518..2618e287e 100644
--- a/crates/ra_syntax/test_data/lexer/00012_block_comment.txt
+++ b/crates/ra_syntax/test_data/lexer/ok/0012_block_comment.txt
@@ -4,4 +4,3 @@ COMMENT 4 "/**/"
4WHITESPACE 1 "\n" 4WHITESPACE 1 "\n"
5COMMENT 11 "/* /* */ */" 5COMMENT 11 "/* /* */ */"
6WHITESPACE 1 "\n" 6WHITESPACE 1 "\n"
7COMMENT 3 "/*\n"
diff --git a/crates/ra_syntax/test_data/lexer/0013_raw_strings.rs b/crates/ra_syntax/test_data/lexer/ok/0013_raw_strings.rs
index e5ed0b693..e5ed0b693 100644
--- a/crates/ra_syntax/test_data/lexer/0013_raw_strings.rs
+++ b/crates/ra_syntax/test_data/lexer/ok/0013_raw_strings.rs
diff --git a/crates/ra_syntax/test_data/lexer/0013_raw_strings.txt b/crates/ra_syntax/test_data/lexer/ok/0013_raw_strings.txt
index 9cf0957d1..9cf0957d1 100644
--- a/crates/ra_syntax/test_data/lexer/0013_raw_strings.txt
+++ b/crates/ra_syntax/test_data/lexer/ok/0013_raw_strings.txt
diff --git a/crates/ra_syntax/test_data/lexer/0016_raw_ident.rs b/crates/ra_syntax/test_data/lexer/ok/0014_raw_ident.rs
index b40a1b6a2..b40a1b6a2 100644
--- a/crates/ra_syntax/test_data/lexer/0016_raw_ident.rs
+++ b/crates/ra_syntax/test_data/lexer/ok/0014_raw_ident.rs
diff --git a/crates/ra_syntax/test_data/lexer/0016_raw_ident.txt b/crates/ra_syntax/test_data/lexer/ok/0014_raw_ident.txt
index 484689693..484689693 100644
--- a/crates/ra_syntax/test_data/lexer/0016_raw_ident.txt
+++ b/crates/ra_syntax/test_data/lexer/ok/0014_raw_ident.txt