Merge #4131

4131: Switch to text-size r=matklad a=matklad



Co-authored-by: Aleksey Kladov <[email protected]>

1 files changed, 17 insertions, 15 deletions

diff --git a/crates/ra_syntax/src/parsing/lexer.rs b/crates/ra_syntax/src/parsing/lexer.rs
index 67c1f1b48..f450ef4a2 100644
--- a/crates/ra_syntax/src/parsing/lexer.rs
+++ b/crates/ra_syntax/src/parsing/lexer.rs
@@ -1,10 +1,12 @@
 //! Lexer analyzes raw input string and produces lexemes (tokens).
 //! It is just a bridge to `rustc_lexer`.
 
+use std::convert::TryInto;
+
 use crate::{
     SyntaxError,
     SyntaxKind::{self, *},
-    TextRange, TextUnit, T,
+    TextRange, TextSize, T,
 };
 
 /// A token of Rust source.
@@ -13,7 +15,7 @@ pub struct Token {
     /// The kind of token.
     pub kind: SyntaxKind,
     /// The length of the token.
-    pub len: TextUnit,
+    pub len: TextSize,
 }
 
 /// Break a string up into its component tokens.
@@ -28,18 +30,19 @@ pub fn tokenize(text: &str) -> (Vec<Token>, Vec<SyntaxError>) {
     let mut tokens = Vec::new();
     let mut errors = Vec::new();
 
-    let mut offset: usize = rustc_lexer::strip_shebang(text)
-        .map(|shebang_len| {
-            tokens.push(Token { kind: SHEBANG, len: TextUnit::from_usize(shebang_len) });
+    let mut offset = match rustc_lexer::strip_shebang(text) {
+        Some(shebang_len) => {
+            tokens.push(Token { kind: SHEBANG, len: shebang_len.try_into().unwrap() });
             shebang_len
-        })
-        .unwrap_or(0);
+        }
+        None => 0,
+    };
 
     let text_without_shebang = &text[offset..];
 
     for rustc_token in rustc_lexer::tokenize(text_without_shebang) {
-        let token_len = TextUnit::from_usize(rustc_token.len);
-        let token_range = TextRange::offset_len(TextUnit::from_usize(offset), token_len);
+        let token_len: TextSize = rustc_token.len.try_into().unwrap();
+        let token_range = TextRange::at(offset.try_into().unwrap(), token_len);
 
         let (syntax_kind, err_message) =
             rustc_token_kind_to_syntax_kind(&rustc_token.kind, &text[token_range]);
@@ -65,7 +68,7 @@ pub fn tokenize(text: &str) -> (Vec<Token>, Vec<SyntaxError>) {
 /// Beware that unescape errors are not checked at tokenization time.
 pub fn lex_single_syntax_kind(text: &str) -> Option<(SyntaxKind, Option<SyntaxError>)> {
     lex_first_token(text)
-        .filter(|(token, _)| token.len == TextUnit::of_str(text))
+        .filter(|(token, _)| token.len == TextSize::of(text))
         .map(|(token, error)| (token.kind, error))
 }
 
@@ -75,7 +78,7 @@ pub fn lex_single_syntax_kind(text: &str) -> Option<(SyntaxKind, Option<SyntaxEr
 /// Beware that unescape errors are not checked at tokenization time.
 pub fn lex_single_valid_syntax_kind(text: &str) -> Option<SyntaxKind> {
     lex_first_token(text)
-        .filter(|(token, error)| !error.is_some() && token.len == TextUnit::of_str(text))
+        .filter(|(token, error)| !error.is_some() && token.len == TextSize::of(text))
         .map(|(token, _error)| token.kind)
 }
 
@@ -96,10 +99,9 @@ fn lex_first_token(text: &str) -> Option<(Token, Option<SyntaxError>)> {
     let rustc_token = rustc_lexer::first_token(text);
     let (syntax_kind, err_message) = rustc_token_kind_to_syntax_kind(&rustc_token.kind, text);
 
-    let token = Token { kind: syntax_kind, len: TextUnit::from_usize(rustc_token.len) };
-    let optional_error = err_message.map(|err_message| {
-        SyntaxError::new(err_message, TextRange::from_to(0.into(), TextUnit::of_str(text)))
-    });
+    let token = Token { kind: syntax_kind, len: rustc_token.len.try_into().unwrap() };
+    let optional_error = err_message
+        .map(|err_message| SyntaxError::new(err_message, TextRange::up_to(TextSize::of(text))));
 
     Some((token, optional_error))
 }