diff options
Diffstat (limited to 'crates/ra_syntax/src/parsing')
-rw-r--r-- | crates/ra_syntax/src/parsing/lexer.rs | 32 | ||||
-rw-r--r-- | crates/ra_syntax/src/parsing/reparsing.rs | 4 | ||||
-rw-r--r-- | crates/ra_syntax/src/parsing/text_token_source.rs | 8 | ||||
-rw-r--r-- | crates/ra_syntax/src/parsing/text_tree_sink.rs | 14 |
4 files changed, 30 insertions, 28 deletions
diff --git a/crates/ra_syntax/src/parsing/lexer.rs b/crates/ra_syntax/src/parsing/lexer.rs index 67c1f1b48..f450ef4a2 100644 --- a/crates/ra_syntax/src/parsing/lexer.rs +++ b/crates/ra_syntax/src/parsing/lexer.rs | |||
@@ -1,10 +1,12 @@ | |||
1 | //! Lexer analyzes raw input string and produces lexemes (tokens). | 1 | //! Lexer analyzes raw input string and produces lexemes (tokens). |
2 | //! It is just a bridge to `rustc_lexer`. | 2 | //! It is just a bridge to `rustc_lexer`. |
3 | 3 | ||
4 | use std::convert::TryInto; | ||
5 | |||
4 | use crate::{ | 6 | use crate::{ |
5 | SyntaxError, | 7 | SyntaxError, |
6 | SyntaxKind::{self, *}, | 8 | SyntaxKind::{self, *}, |
7 | TextRange, TextUnit, T, | 9 | TextRange, TextSize, T, |
8 | }; | 10 | }; |
9 | 11 | ||
10 | /// A token of Rust source. | 12 | /// A token of Rust source. |
@@ -13,7 +15,7 @@ pub struct Token { | |||
13 | /// The kind of token. | 15 | /// The kind of token. |
14 | pub kind: SyntaxKind, | 16 | pub kind: SyntaxKind, |
15 | /// The length of the token. | 17 | /// The length of the token. |
16 | pub len: TextUnit, | 18 | pub len: TextSize, |
17 | } | 19 | } |
18 | 20 | ||
19 | /// Break a string up into its component tokens. | 21 | /// Break a string up into its component tokens. |
@@ -28,18 +30,19 @@ pub fn tokenize(text: &str) -> (Vec<Token>, Vec<SyntaxError>) { | |||
28 | let mut tokens = Vec::new(); | 30 | let mut tokens = Vec::new(); |
29 | let mut errors = Vec::new(); | 31 | let mut errors = Vec::new(); |
30 | 32 | ||
31 | let mut offset: usize = rustc_lexer::strip_shebang(text) | 33 | let mut offset = match rustc_lexer::strip_shebang(text) { |
32 | .map(|shebang_len| { | 34 | Some(shebang_len) => { |
33 | tokens.push(Token { kind: SHEBANG, len: TextUnit::from_usize(shebang_len) }); | 35 | tokens.push(Token { kind: SHEBANG, len: shebang_len.try_into().unwrap() }); |
34 | shebang_len | 36 | shebang_len |
35 | }) | 37 | } |
36 | .unwrap_or(0); | 38 | None => 0, |
39 | }; | ||
37 | 40 | ||
38 | let text_without_shebang = &text[offset..]; | 41 | let text_without_shebang = &text[offset..]; |
39 | 42 | ||
40 | for rustc_token in rustc_lexer::tokenize(text_without_shebang) { | 43 | for rustc_token in rustc_lexer::tokenize(text_without_shebang) { |
41 | let token_len = TextUnit::from_usize(rustc_token.len); | 44 | let token_len: TextSize = rustc_token.len.try_into().unwrap(); |
42 | let token_range = TextRange::offset_len(TextUnit::from_usize(offset), token_len); | 45 | let token_range = TextRange::at(offset.try_into().unwrap(), token_len); |
43 | 46 | ||
44 | let (syntax_kind, err_message) = | 47 | let (syntax_kind, err_message) = |
45 | rustc_token_kind_to_syntax_kind(&rustc_token.kind, &text[token_range]); | 48 | rustc_token_kind_to_syntax_kind(&rustc_token.kind, &text[token_range]); |
@@ -65,7 +68,7 @@ pub fn tokenize(text: &str) -> (Vec<Token>, Vec<SyntaxError>) { | |||
65 | /// Beware that unescape errors are not checked at tokenization time. | 68 | /// Beware that unescape errors are not checked at tokenization time. |
66 | pub fn lex_single_syntax_kind(text: &str) -> Option<(SyntaxKind, Option<SyntaxError>)> { | 69 | pub fn lex_single_syntax_kind(text: &str) -> Option<(SyntaxKind, Option<SyntaxError>)> { |
67 | lex_first_token(text) | 70 | lex_first_token(text) |
68 | .filter(|(token, _)| token.len == TextUnit::of_str(text)) | 71 | .filter(|(token, _)| token.len == TextSize::of(text)) |
69 | .map(|(token, error)| (token.kind, error)) | 72 | .map(|(token, error)| (token.kind, error)) |
70 | } | 73 | } |
71 | 74 | ||
@@ -75,7 +78,7 @@ pub fn lex_single_syntax_kind(text: &str) -> Option<(SyntaxKind, Option<SyntaxEr | |||
75 | /// Beware that unescape errors are not checked at tokenization time. | 78 | /// Beware that unescape errors are not checked at tokenization time. |
76 | pub fn lex_single_valid_syntax_kind(text: &str) -> Option<SyntaxKind> { | 79 | pub fn lex_single_valid_syntax_kind(text: &str) -> Option<SyntaxKind> { |
77 | lex_first_token(text) | 80 | lex_first_token(text) |
78 | .filter(|(token, error)| !error.is_some() && token.len == TextUnit::of_str(text)) | 81 | .filter(|(token, error)| !error.is_some() && token.len == TextSize::of(text)) |
79 | .map(|(token, _error)| token.kind) | 82 | .map(|(token, _error)| token.kind) |
80 | } | 83 | } |
81 | 84 | ||
@@ -96,10 +99,9 @@ fn lex_first_token(text: &str) -> Option<(Token, Option<SyntaxError>)> { | |||
96 | let rustc_token = rustc_lexer::first_token(text); | 99 | let rustc_token = rustc_lexer::first_token(text); |
97 | let (syntax_kind, err_message) = rustc_token_kind_to_syntax_kind(&rustc_token.kind, text); | 100 | let (syntax_kind, err_message) = rustc_token_kind_to_syntax_kind(&rustc_token.kind, text); |
98 | 101 | ||
99 | let token = Token { kind: syntax_kind, len: TextUnit::from_usize(rustc_token.len) }; | 102 | let token = Token { kind: syntax_kind, len: rustc_token.len.try_into().unwrap() }; |
100 | let optional_error = err_message.map(|err_message| { | 103 | let optional_error = err_message |
101 | SyntaxError::new(err_message, TextRange::from_to(0.into(), TextUnit::of_str(text))) | 104 | .map(|err_message| SyntaxError::new(err_message, TextRange::up_to(TextSize::of(text)))); |
102 | }); | ||
103 | 105 | ||
104 | Some((token, optional_error)) | 106 | Some((token, optional_error)) |
105 | } | 107 | } |
diff --git a/crates/ra_syntax/src/parsing/reparsing.rs b/crates/ra_syntax/src/parsing/reparsing.rs index 2d65b91f1..ffff0a7b2 100644 --- a/crates/ra_syntax/src/parsing/reparsing.rs +++ b/crates/ra_syntax/src/parsing/reparsing.rs | |||
@@ -19,7 +19,7 @@ use crate::{ | |||
19 | syntax_node::{GreenNode, GreenToken, NodeOrToken, SyntaxElement, SyntaxNode}, | 19 | syntax_node::{GreenNode, GreenToken, NodeOrToken, SyntaxElement, SyntaxNode}, |
20 | SyntaxError, | 20 | SyntaxError, |
21 | SyntaxKind::*, | 21 | SyntaxKind::*, |
22 | TextRange, TextUnit, T, | 22 | TextRange, TextSize, T, |
23 | }; | 23 | }; |
24 | 24 | ||
25 | pub(crate) fn incremental_reparse( | 25 | pub(crate) fn incremental_reparse( |
@@ -176,7 +176,7 @@ fn merge_errors( | |||
176 | if old_err_range.end() <= range_before_reparse.start() { | 176 | if old_err_range.end() <= range_before_reparse.start() { |
177 | res.push(old_err); | 177 | res.push(old_err); |
178 | } else if old_err_range.start() >= range_before_reparse.end() { | 178 | } else if old_err_range.start() >= range_before_reparse.end() { |
179 | let inserted_len = TextUnit::of_str(&edit.insert); | 179 | let inserted_len = TextSize::of(&edit.insert); |
180 | res.push(old_err.with_range((old_err_range + inserted_len) - edit.delete.len())); | 180 | res.push(old_err.with_range((old_err_range + inserted_len) - edit.delete.len())); |
181 | // Note: extra parens are intentional to prevent uint underflow, HWAB (here was a bug) | 181 | // Note: extra parens are intentional to prevent uint underflow, HWAB (here was a bug) |
182 | } | 182 | } |
diff --git a/crates/ra_syntax/src/parsing/text_token_source.rs b/crates/ra_syntax/src/parsing/text_token_source.rs index e2433913c..7ddc2c2c3 100644 --- a/crates/ra_syntax/src/parsing/text_token_source.rs +++ b/crates/ra_syntax/src/parsing/text_token_source.rs | |||
@@ -3,7 +3,7 @@ | |||
3 | use ra_parser::Token as PToken; | 3 | use ra_parser::Token as PToken; |
4 | use ra_parser::TokenSource; | 4 | use ra_parser::TokenSource; |
5 | 5 | ||
6 | use crate::{parsing::lexer::Token, SyntaxKind::EOF, TextRange, TextUnit}; | 6 | use crate::{parsing::lexer::Token, SyntaxKind::EOF, TextRange, TextSize}; |
7 | 7 | ||
8 | pub(crate) struct TextTokenSource<'t> { | 8 | pub(crate) struct TextTokenSource<'t> { |
9 | text: &'t str, | 9 | text: &'t str, |
@@ -15,7 +15,7 @@ pub(crate) struct TextTokenSource<'t> { | |||
15 | /// 0 7 10 | 15 | /// 0 7 10 |
16 | /// ``` | 16 | /// ``` |
17 | /// (token, start_offset): `[(struct, 0), (Foo, 7), (;, 10)]` | 17 | /// (token, start_offset): `[(struct, 0), (Foo, 7), (;, 10)]` |
18 | start_offsets: Vec<TextUnit>, | 18 | start_offsets: Vec<TextSize>, |
19 | /// non-whitespace/comment tokens | 19 | /// non-whitespace/comment tokens |
20 | /// ```non-rust | 20 | /// ```non-rust |
21 | /// struct Foo {} | 21 | /// struct Foo {} |
@@ -51,12 +51,12 @@ impl<'t> TokenSource for TextTokenSource<'t> { | |||
51 | if pos >= self.tokens.len() { | 51 | if pos >= self.tokens.len() { |
52 | return false; | 52 | return false; |
53 | } | 53 | } |
54 | let range = TextRange::offset_len(self.start_offsets[pos], self.tokens[pos].len); | 54 | let range = TextRange::at(self.start_offsets[pos], self.tokens[pos].len); |
55 | self.text[range] == *kw | 55 | self.text[range] == *kw |
56 | } | 56 | } |
57 | } | 57 | } |
58 | 58 | ||
59 | fn mk_token(pos: usize, start_offsets: &[TextUnit], tokens: &[Token]) -> PToken { | 59 | fn mk_token(pos: usize, start_offsets: &[TextSize], tokens: &[Token]) -> PToken { |
60 | let kind = tokens.get(pos).map(|t| t.kind).unwrap_or(EOF); | 60 | let kind = tokens.get(pos).map(|t| t.kind).unwrap_or(EOF); |
61 | let is_jointed_to_next = if pos + 1 < start_offsets.len() { | 61 | let is_jointed_to_next = if pos + 1 < start_offsets.len() { |
62 | start_offsets[pos] + tokens[pos].len == start_offsets[pos + 1] | 62 | start_offsets[pos] + tokens[pos].len == start_offsets[pos + 1] |
diff --git a/crates/ra_syntax/src/parsing/text_tree_sink.rs b/crates/ra_syntax/src/parsing/text_tree_sink.rs index 87bb21cd9..22aed1db1 100644 --- a/crates/ra_syntax/src/parsing/text_tree_sink.rs +++ b/crates/ra_syntax/src/parsing/text_tree_sink.rs | |||
@@ -9,7 +9,7 @@ use crate::{ | |||
9 | syntax_node::GreenNode, | 9 | syntax_node::GreenNode, |
10 | SmolStr, SyntaxError, | 10 | SmolStr, SyntaxError, |
11 | SyntaxKind::{self, *}, | 11 | SyntaxKind::{self, *}, |
12 | SyntaxTreeBuilder, TextRange, TextUnit, | 12 | SyntaxTreeBuilder, TextRange, TextSize, |
13 | }; | 13 | }; |
14 | 14 | ||
15 | /// Bridges the parser with our specific syntax tree representation. | 15 | /// Bridges the parser with our specific syntax tree representation. |
@@ -18,7 +18,7 @@ use crate::{ | |||
18 | pub(crate) struct TextTreeSink<'a> { | 18 | pub(crate) struct TextTreeSink<'a> { |
19 | text: &'a str, | 19 | text: &'a str, |
20 | tokens: &'a [Token], | 20 | tokens: &'a [Token], |
21 | text_pos: TextUnit, | 21 | text_pos: TextSize, |
22 | token_pos: usize, | 22 | token_pos: usize, |
23 | state: State, | 23 | state: State, |
24 | inner: SyntaxTreeBuilder, | 24 | inner: SyntaxTreeBuilder, |
@@ -42,7 +42,7 @@ impl<'a> TreeSink for TextTreeSink<'a> { | |||
42 | let len = self.tokens[self.token_pos..self.token_pos + n_tokens] | 42 | let len = self.tokens[self.token_pos..self.token_pos + n_tokens] |
43 | .iter() | 43 | .iter() |
44 | .map(|it| it.len) | 44 | .map(|it| it.len) |
45 | .sum::<TextUnit>(); | 45 | .sum::<TextSize>(); |
46 | self.do_token(kind, len, n_tokens); | 46 | self.do_token(kind, len, n_tokens); |
47 | } | 47 | } |
48 | 48 | ||
@@ -62,12 +62,12 @@ impl<'a> TreeSink for TextTreeSink<'a> { | |||
62 | self.tokens[self.token_pos..].iter().take_while(|it| it.kind.is_trivia()).count(); | 62 | self.tokens[self.token_pos..].iter().take_while(|it| it.kind.is_trivia()).count(); |
63 | let leading_trivias = &self.tokens[self.token_pos..self.token_pos + n_trivias]; | 63 | let leading_trivias = &self.tokens[self.token_pos..self.token_pos + n_trivias]; |
64 | let mut trivia_end = | 64 | let mut trivia_end = |
65 | self.text_pos + leading_trivias.iter().map(|it| it.len).sum::<TextUnit>(); | 65 | self.text_pos + leading_trivias.iter().map(|it| it.len).sum::<TextSize>(); |
66 | 66 | ||
67 | let n_attached_trivias = { | 67 | let n_attached_trivias = { |
68 | let leading_trivias = leading_trivias.iter().rev().map(|it| { | 68 | let leading_trivias = leading_trivias.iter().rev().map(|it| { |
69 | let next_end = trivia_end - it.len; | 69 | let next_end = trivia_end - it.len; |
70 | let range = TextRange::from_to(next_end, trivia_end); | 70 | let range = TextRange::new(next_end, trivia_end); |
71 | trivia_end = next_end; | 71 | trivia_end = next_end; |
72 | (it.kind, &self.text[range]) | 72 | (it.kind, &self.text[range]) |
73 | }); | 73 | }); |
@@ -132,8 +132,8 @@ impl<'a> TextTreeSink<'a> { | |||
132 | } | 132 | } |
133 | } | 133 | } |
134 | 134 | ||
135 | fn do_token(&mut self, kind: SyntaxKind, len: TextUnit, n_tokens: usize) { | 135 | fn do_token(&mut self, kind: SyntaxKind, len: TextSize, n_tokens: usize) { |
136 | let range = TextRange::offset_len(self.text_pos, len); | 136 | let range = TextRange::at(self.text_pos, len); |
137 | let text: SmolStr = self.text[range].into(); | 137 | let text: SmolStr = self.text[range].into(); |
138 | self.text_pos += len; | 138 | self.text_pos += len; |
139 | self.token_pos += n_tokens; | 139 | self.token_pos += n_tokens; |