aboutsummaryrefslogtreecommitdiff
path: root/crates/ra_syntax/src/parsing/lexer.rs
diff options
context:
space:
mode:
Diffstat (limited to 'crates/ra_syntax/src/parsing/lexer.rs')
-rw-r--r--crates/ra_syntax/src/parsing/lexer.rs18
1 files changed, 9 insertions, 9 deletions
diff --git a/crates/ra_syntax/src/parsing/lexer.rs b/crates/ra_syntax/src/parsing/lexer.rs
index 67c1f1b48..1fdc76d98 100644
--- a/crates/ra_syntax/src/parsing/lexer.rs
+++ b/crates/ra_syntax/src/parsing/lexer.rs
@@ -4,7 +4,7 @@
4use crate::{ 4use crate::{
5 SyntaxError, 5 SyntaxError,
6 SyntaxKind::{self, *}, 6 SyntaxKind::{self, *},
7 TextRange, TextUnit, T, 7 TextRange, TextSize, T,
8}; 8};
9 9
10/// A token of Rust source. 10/// A token of Rust source.
@@ -13,7 +13,7 @@ pub struct Token {
13 /// The kind of token. 13 /// The kind of token.
14 pub kind: SyntaxKind, 14 pub kind: SyntaxKind,
15 /// The length of the token. 15 /// The length of the token.
16 pub len: TextUnit, 16 pub len: TextSize,
17} 17}
18 18
19/// Break a string up into its component tokens. 19/// Break a string up into its component tokens.
@@ -30,7 +30,7 @@ pub fn tokenize(text: &str) -> (Vec<Token>, Vec<SyntaxError>) {
30 30
31 let mut offset: usize = rustc_lexer::strip_shebang(text) 31 let mut offset: usize = rustc_lexer::strip_shebang(text)
32 .map(|shebang_len| { 32 .map(|shebang_len| {
33 tokens.push(Token { kind: SHEBANG, len: TextUnit::from_usize(shebang_len) }); 33 tokens.push(Token { kind: SHEBANG, len: TextSize::from_usize(shebang_len) });
34 shebang_len 34 shebang_len
35 }) 35 })
36 .unwrap_or(0); 36 .unwrap_or(0);
@@ -38,8 +38,8 @@ pub fn tokenize(text: &str) -> (Vec<Token>, Vec<SyntaxError>) {
38 let text_without_shebang = &text[offset..]; 38 let text_without_shebang = &text[offset..];
39 39
40 for rustc_token in rustc_lexer::tokenize(text_without_shebang) { 40 for rustc_token in rustc_lexer::tokenize(text_without_shebang) {
41 let token_len = TextUnit::from_usize(rustc_token.len); 41 let token_len = TextSize::from_usize(rustc_token.len);
42 let token_range = TextRange::offset_len(TextUnit::from_usize(offset), token_len); 42 let token_range = TextRange::at(TextSize::from_usize(offset), token_len);
43 43
44 let (syntax_kind, err_message) = 44 let (syntax_kind, err_message) =
45 rustc_token_kind_to_syntax_kind(&rustc_token.kind, &text[token_range]); 45 rustc_token_kind_to_syntax_kind(&rustc_token.kind, &text[token_range]);
@@ -65,7 +65,7 @@ pub fn tokenize(text: &str) -> (Vec<Token>, Vec<SyntaxError>) {
65/// Beware that unescape errors are not checked at tokenization time. 65/// Beware that unescape errors are not checked at tokenization time.
66pub fn lex_single_syntax_kind(text: &str) -> Option<(SyntaxKind, Option<SyntaxError>)> { 66pub fn lex_single_syntax_kind(text: &str) -> Option<(SyntaxKind, Option<SyntaxError>)> {
67 lex_first_token(text) 67 lex_first_token(text)
68 .filter(|(token, _)| token.len == TextUnit::of_str(text)) 68 .filter(|(token, _)| token.len == TextSize::of(text))
69 .map(|(token, error)| (token.kind, error)) 69 .map(|(token, error)| (token.kind, error))
70} 70}
71 71
@@ -75,7 +75,7 @@ pub fn lex_single_syntax_kind(text: &str) -> Option<(SyntaxKind, Option<SyntaxEr
75/// Beware that unescape errors are not checked at tokenization time. 75/// Beware that unescape errors are not checked at tokenization time.
76pub fn lex_single_valid_syntax_kind(text: &str) -> Option<SyntaxKind> { 76pub fn lex_single_valid_syntax_kind(text: &str) -> Option<SyntaxKind> {
77 lex_first_token(text) 77 lex_first_token(text)
78 .filter(|(token, error)| !error.is_some() && token.len == TextUnit::of_str(text)) 78 .filter(|(token, error)| !error.is_some() && token.len == TextSize::of(text))
79 .map(|(token, _error)| token.kind) 79 .map(|(token, _error)| token.kind)
80} 80}
81 81
@@ -96,9 +96,9 @@ fn lex_first_token(text: &str) -> Option<(Token, Option<SyntaxError>)> {
96 let rustc_token = rustc_lexer::first_token(text); 96 let rustc_token = rustc_lexer::first_token(text);
97 let (syntax_kind, err_message) = rustc_token_kind_to_syntax_kind(&rustc_token.kind, text); 97 let (syntax_kind, err_message) = rustc_token_kind_to_syntax_kind(&rustc_token.kind, text);
98 98
99 let token = Token { kind: syntax_kind, len: TextUnit::from_usize(rustc_token.len) }; 99 let token = Token { kind: syntax_kind, len: TextSize::from_usize(rustc_token.len) };
100 let optional_error = err_message.map(|err_message| { 100 let optional_error = err_message.map(|err_message| {
101 SyntaxError::new(err_message, TextRange::from_to(0.into(), TextUnit::of_str(text))) 101 SyntaxError::new(err_message, TextRange::new(0.into(), TextSize::of(text)))
102 }); 102 });
103 103
104 Some((token, optional_error)) 104 Some((token, optional_error))