diff options
Diffstat (limited to 'crates/ra_syntax/src/parsing')
-rw-r--r-- | crates/ra_syntax/src/parsing/lexer.rs | 24 |
1 files changed, 13 insertions, 11 deletions
diff --git a/crates/ra_syntax/src/parsing/lexer.rs b/crates/ra_syntax/src/parsing/lexer.rs index 1fdc76d98..f450ef4a2 100644 --- a/crates/ra_syntax/src/parsing/lexer.rs +++ b/crates/ra_syntax/src/parsing/lexer.rs | |||
@@ -1,6 +1,8 @@ | |||
1 | //! Lexer analyzes raw input string and produces lexemes (tokens). | 1 | //! Lexer analyzes raw input string and produces lexemes (tokens). |
2 | //! It is just a bridge to `rustc_lexer`. | 2 | //! It is just a bridge to `rustc_lexer`. |
3 | 3 | ||
4 | use std::convert::TryInto; | ||
5 | |||
4 | use crate::{ | 6 | use crate::{ |
5 | SyntaxError, | 7 | SyntaxError, |
6 | SyntaxKind::{self, *}, | 8 | SyntaxKind::{self, *}, |
@@ -28,18 +30,19 @@ pub fn tokenize(text: &str) -> (Vec<Token>, Vec<SyntaxError>) { | |||
28 | let mut tokens = Vec::new(); | 30 | let mut tokens = Vec::new(); |
29 | let mut errors = Vec::new(); | 31 | let mut errors = Vec::new(); |
30 | 32 | ||
31 | let mut offset: usize = rustc_lexer::strip_shebang(text) | 33 | let mut offset = match rustc_lexer::strip_shebang(text) { |
32 | .map(|shebang_len| { | 34 | Some(shebang_len) => { |
33 | tokens.push(Token { kind: SHEBANG, len: TextSize::from_usize(shebang_len) }); | 35 | tokens.push(Token { kind: SHEBANG, len: shebang_len.try_into().unwrap() }); |
34 | shebang_len | 36 | shebang_len |
35 | }) | 37 | } |
36 | .unwrap_or(0); | 38 | None => 0, |
39 | }; | ||
37 | 40 | ||
38 | let text_without_shebang = &text[offset..]; | 41 | let text_without_shebang = &text[offset..]; |
39 | 42 | ||
40 | for rustc_token in rustc_lexer::tokenize(text_without_shebang) { | 43 | for rustc_token in rustc_lexer::tokenize(text_without_shebang) { |
41 | let token_len = TextSize::from_usize(rustc_token.len); | 44 | let token_len: TextSize = rustc_token.len.try_into().unwrap(); |
42 | let token_range = TextRange::at(TextSize::from_usize(offset), token_len); | 45 | let token_range = TextRange::at(offset.try_into().unwrap(), token_len); |
43 | 46 | ||
44 | let (syntax_kind, err_message) = | 47 | let (syntax_kind, err_message) = |
45 | rustc_token_kind_to_syntax_kind(&rustc_token.kind, &text[token_range]); | 48 | rustc_token_kind_to_syntax_kind(&rustc_token.kind, &text[token_range]); |
@@ -96,10 +99,9 @@ fn lex_first_token(text: &str) -> Option<(Token, Option<SyntaxError>)> { | |||
96 | let rustc_token = rustc_lexer::first_token(text); | 99 | let rustc_token = rustc_lexer::first_token(text); |
97 | let (syntax_kind, err_message) = rustc_token_kind_to_syntax_kind(&rustc_token.kind, text); | 100 | let (syntax_kind, err_message) = rustc_token_kind_to_syntax_kind(&rustc_token.kind, text); |
98 | 101 | ||
99 | let token = Token { kind: syntax_kind, len: TextSize::from_usize(rustc_token.len) }; | 102 | let token = Token { kind: syntax_kind, len: rustc_token.len.try_into().unwrap() }; |
100 | let optional_error = err_message.map(|err_message| { | 103 | let optional_error = err_message |
101 | SyntaxError::new(err_message, TextRange::new(0.into(), TextSize::of(text))) | 104 | .map(|err_message| SyntaxError::new(err_message, TextRange::up_to(TextSize::of(text)))); |
102 | }); | ||
103 | 105 | ||
104 | Some((token, optional_error)) | 106 | Some((token, optional_error)) |
105 | } | 107 | } |