diff options
author | bors[bot] <26634292+bors[bot]@users.noreply.github.com> | 2020-02-03 22:51:17 +0000 |
---|---|---|
committer | GitHub <[email protected]> | 2020-02-03 22:51:17 +0000 |
commit | 918547dbe9a2907401102eba491ac25cebe1404d (patch) | |
tree | e0aa3bdcec597e81f022ac1ce388d42724a92f51 /crates/ra_ide | |
parent | b090ee5a65f9630146c2842bc51fcfcc8da08da1 (diff) | |
parent | a3e5663ae0206270156fbeb926a174a40abbddb0 (diff) |
Merge #2911
2911: Implement collecting errors while tokenizing r=matklad a=Veetaha
Now we are collecting errors from `rustc_lexer` and returning them in `ParsedToken { token, error }` and `ParsedTokens { tokens, errors }` structures **([UPD]: this is now simplified, see updates bellow)**.
The main changes are introduced in `ra_syntax/parsing/lexer.rs`. It now exposes the following functions and types:
```rust
pub fn tokenize(text: &str) -> ParsedTokens;
pub fn tokenize_append(text: &str, parsed_tokens_to_append_to: &mut ParsedTokens);
pub fn first_token(text: &str) -> Option<ParsedToken>; // allows any number of tokens in text
pub fn single_token(text: &str) -> Option<ParsedToken>; // allows only a single token in text
pub struct ParsedToken { pub token: Token, pub error: Option<SyntaxError> }
pub struct ParsedTokens { pub tokens: Vec<Token>, pub errors: Vec<SyntaxError> }
pub enum TokenizeError { /* Simple enum which reflects rustc_lexer tokenization errors */ }
```
In the first commit I implemented it with iterators, but then decided that since this crate is ad hoc for `rust-analyzer` and we clearly see the places of its usage it would be better to simplify it to vectors.
This is currently WIP, because I want to add tests for error messages generated by the lexer.
I'd like to listen to you thoughts how to define these tests in `ra_syntax/test-data` dir.
Related issues: #223
**[UPD]**
After the PR review the API was simplified:
```rust
pub fn tokenize(text: &str) -> (Vec<Token>, Vec<SyntaxError>);
// Both lex functions do not check for unescape errors
pub fn lex_single_syntax_kind(text: &str) -> Option<(SyntaxKind, Option<SyntaxError>)>;
pub fn lex_single_valid_syntax_kind(text: &str) -> Option<SyntaxKind>;
// This will be removed in the next PR in favour of simlifying `SyntaxError` to `(String, TextRange)`
pub enum TokenizeError { /* Simple enum which reflects rustc_lexer tokenization errors */ }
// this is private, but may be made public if such demand would exist in future (least privilege principle)
fn lex_first_token(text: &str) -> Option<(Token, Option<SyntaxError>)>;
```
Co-authored-by: Veetaha <[email protected]>
Diffstat (limited to 'crates/ra_ide')
-rw-r--r-- | crates/ra_ide/src/references/rename.rs | 12 |
1 files changed, 6 insertions, 6 deletions
diff --git a/crates/ra_ide/src/references/rename.rs b/crates/ra_ide/src/references/rename.rs index 626efb603..9a84c1c88 100644 --- a/crates/ra_ide/src/references/rename.rs +++ b/crates/ra_ide/src/references/rename.rs | |||
@@ -2,7 +2,9 @@ | |||
2 | 2 | ||
3 | use hir::ModuleSource; | 3 | use hir::ModuleSource; |
4 | use ra_db::{RelativePath, RelativePathBuf, SourceDatabase, SourceDatabaseExt}; | 4 | use ra_db::{RelativePath, RelativePathBuf, SourceDatabase, SourceDatabaseExt}; |
5 | use ra_syntax::{algo::find_node_at_offset, ast, tokenize, AstNode, SyntaxKind, SyntaxNode}; | 5 | use ra_syntax::{ |
6 | algo::find_node_at_offset, ast, lex_single_valid_syntax_kind, AstNode, SyntaxKind, SyntaxNode, | ||
7 | }; | ||
6 | use ra_text_edit::TextEdit; | 8 | use ra_text_edit::TextEdit; |
7 | 9 | ||
8 | use crate::{ | 10 | use crate::{ |
@@ -17,11 +19,9 @@ pub(crate) fn rename( | |||
17 | position: FilePosition, | 19 | position: FilePosition, |
18 | new_name: &str, | 20 | new_name: &str, |
19 | ) -> Option<RangeInfo<SourceChange>> { | 21 | ) -> Option<RangeInfo<SourceChange>> { |
20 | let tokens = tokenize(new_name); | 22 | match lex_single_valid_syntax_kind(new_name)? { |
21 | if tokens.len() != 1 | 23 | SyntaxKind::IDENT | SyntaxKind::UNDERSCORE => (), |
22 | || (tokens[0].kind != SyntaxKind::IDENT && tokens[0].kind != SyntaxKind::UNDERSCORE) | 24 | _ => return None, |
23 | { | ||
24 | return None; | ||
25 | } | 25 | } |
26 | 26 | ||
27 | let parse = db.parse(position.file_id); | 27 | let parse = db.parse(position.file_id); |