From cd4ffc1945a3a1ca89776e9abdcd60b1896f356c Mon Sep 17 00:00:00 2001 From: Julian Wollersberger <24991778+Julian-Wollersberger@users.noreply.github.com> Date: Sun, 24 May 2020 13:12:16 +0200 Subject: Update to rustc_lexer version 660. Change `unescape_*()` to `unescape_literal()`. --- crates/ra_syntax/Cargo.toml | 2 +- crates/ra_syntax/src/ast/tokens.rs | 5 +++-- crates/ra_syntax/src/validation.rs | 16 ++++++++-------- 3 files changed, 12 insertions(+), 11 deletions(-) (limited to 'crates/ra_syntax') diff --git a/crates/ra_syntax/Cargo.toml b/crates/ra_syntax/Cargo.toml index c07ff488e..a9a5cc7bc 100644 --- a/crates/ra_syntax/Cargo.toml +++ b/crates/ra_syntax/Cargo.toml @@ -13,7 +13,7 @@ doctest = false [dependencies] itertools = "0.9.0" rowan = "0.10.0" -rustc_lexer = { version = "656.0.0", package = "rustc-ap-rustc_lexer" } +rustc_lexer = { version = "660.0.0", package = "rustc-ap-rustc_lexer" } rustc-hash = "1.1.0" arrayvec = "0.5.1" once_cell = "1.3.1" diff --git a/crates/ra_syntax/src/ast/tokens.rs b/crates/ra_syntax/src/ast/tokens.rs index 74906d8a6..3cd6d99c3 100644 --- a/crates/ra_syntax/src/ast/tokens.rs +++ b/crates/ra_syntax/src/ast/tokens.rs @@ -6,6 +6,7 @@ use crate::{ ast::{AstToken, Comment, RawString, String, Whitespace}, TextRange, TextSize, }; +use rustc_lexer::unescape::{unescape_literal, Mode}; impl Comment { pub fn kind(&self) -> CommentKind { @@ -147,7 +148,7 @@ impl HasStringValue for String { let mut buf = std::string::String::with_capacity(text.len()); let mut has_error = false; - rustc_lexer::unescape::unescape_str(text, &mut |_, unescaped_char| match unescaped_char { + unescape_literal(text, Mode::Str, &mut |_, unescaped_char| match unescaped_char { Ok(c) => buf.push(c), Err(_) => has_error = true, }); @@ -498,7 +499,7 @@ impl HasFormatSpecifier for String { let offset = self.text_range_between_quotes()?.start() - self.syntax().text_range().start(); let mut res = Vec::with_capacity(text.len()); - rustc_lexer::unescape::unescape_str(text, &mut |range, unescaped_char| { + unescape_literal(text, Mode::Str, &mut |range, unescaped_char| { res.push(( TextRange::new(range.start.try_into().unwrap(), range.end.try_into().unwrap()) + offset, diff --git a/crates/ra_syntax/src/validation.rs b/crates/ra_syntax/src/validation.rs index d68cf0a82..436ab033d 100644 --- a/crates/ra_syntax/src/validation.rs +++ b/crates/ra_syntax/src/validation.rs @@ -2,15 +2,15 @@ mod block; -use std::convert::TryFrom; - -use rustc_lexer::unescape; - use crate::{ ast, match_ast, AstNode, SyntaxError, SyntaxKind::{BYTE, BYTE_STRING, CHAR, CONST_DEF, FN_DEF, INT_NUMBER, STRING, TYPE_ALIAS_DEF}, SyntaxNode, SyntaxToken, TextSize, T, }; +use rustc_lexer::unescape::{ + self, unescape_byte, unescape_byte_literal, unescape_char, unescape_literal, Mode, +}; +use std::convert::TryFrom; fn rustc_unescape_error_to_string(err: unescape::EscapeError) -> &'static str { use unescape::EscapeError as EE; @@ -121,18 +121,18 @@ fn validate_literal(literal: ast::Literal, acc: &mut Vec) { match token.kind() { BYTE => { - if let Some(Err(e)) = unquote(text, 2, '\'').map(unescape::unescape_byte) { + if let Some(Err(e)) = unquote(text, 2, '\'').map(unescape_byte) { push_err(2, e); } } CHAR => { - if let Some(Err(e)) = unquote(text, 1, '\'').map(unescape::unescape_char) { + if let Some(Err(e)) = unquote(text, 1, '\'').map(unescape_char) { push_err(1, e); } } BYTE_STRING => { if let Some(without_quotes) = unquote(text, 2, '"') { - unescape::unescape_byte_str(without_quotes, &mut |range, char| { + unescape_byte_literal(without_quotes, Mode::ByteStr, &mut |range, char| { if let Err(err) = char { push_err(2, (range.start, err)); } @@ -141,7 +141,7 @@ fn validate_literal(literal: ast::Literal, acc: &mut Vec) { } STRING => { if let Some(without_quotes) = unquote(text, 1, '"') { - unescape::unescape_str(without_quotes, &mut |range, char| { + unescape_literal(without_quotes, Mode::Str, &mut |range, char| { if let Err(err) = char { push_err(1, (range.start, err)); } -- cgit v1.2.3 From ff9d553fe3af092e4ef5467f07cef62af27b868b Mon Sep 17 00:00:00 2001 From: Julian Wollersberger <24991778+Julian-Wollersberger@users.noreply.github.com> Date: Sun, 24 May 2020 13:17:16 +0200 Subject: Some FIXMEs were outdated: * Done at line 243: "Add validation of `crate` keyword not appearing in the middle of the symbol path" * Already happened: "Remove validation of unterminated literals (it is already implemented in `tokenize()`)" * Happens in `unescape()`: "Add validation of character literal containing only a single char" * Missing: "raw string literals and raw byte string literals" --- crates/ra_syntax/src/validation.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'crates/ra_syntax') diff --git a/crates/ra_syntax/src/validation.rs b/crates/ra_syntax/src/validation.rs index 436ab033d..fdec48fb0 100644 --- a/crates/ra_syntax/src/validation.rs +++ b/crates/ra_syntax/src/validation.rs @@ -81,10 +81,8 @@ fn rustc_unescape_error_to_string(err: unescape::EscapeError) -> &'static str { pub(crate) fn validate(root: &SyntaxNode) -> Vec { // FIXME: - // * Add validation of character literal containing only a single char - // * Add validation of `crate` keyword not appearing in the middle of the symbol path + // * Add unescape validation of raw string literals and raw byte string literals // * Add validation of doc comments are being attached to nodes - // * Remove validation of unterminated literals (it is already implemented in `tokenize()`) let mut errors = Vec::new(); for node in root.descendants() { -- cgit v1.2.3 From 9a0353a81e9288bfa9f16393391bf4e160851c36 Mon Sep 17 00:00:00 2001 From: kjeremy Date: Tue, 26 May 2020 14:35:09 -0400 Subject: Update lexer --- crates/ra_syntax/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'crates/ra_syntax') diff --git a/crates/ra_syntax/Cargo.toml b/crates/ra_syntax/Cargo.toml index a9a5cc7bc..a8ff2e74f 100644 --- a/crates/ra_syntax/Cargo.toml +++ b/crates/ra_syntax/Cargo.toml @@ -13,7 +13,7 @@ doctest = false [dependencies] itertools = "0.9.0" rowan = "0.10.0" -rustc_lexer = { version = "660.0.0", package = "rustc-ap-rustc_lexer" } +rustc_lexer = { version = "661.0.0", package = "rustc-ap-rustc_lexer" } rustc-hash = "1.1.0" arrayvec = "0.5.1" once_cell = "1.3.1" -- cgit v1.2.3 From 367487fe88dca78cffad5138673d5259f7f7ba6b Mon Sep 17 00:00:00 2001 From: robojumper Date: Thu, 28 May 2020 21:42:22 +0200 Subject: Support raw_ref_op's raw reference operator --- crates/ra_syntax/src/ast/generated/nodes.rs | 3 + .../test_data/parser/inline/ok/0082_ref_expr.rast | 172 +++++++++++++++------ .../test_data/parser/inline/ok/0082_ref_expr.rs | 6 + 3 files changed, 137 insertions(+), 44 deletions(-) (limited to 'crates/ra_syntax') diff --git a/crates/ra_syntax/src/ast/generated/nodes.rs b/crates/ra_syntax/src/ast/generated/nodes.rs index cf6067e57..255402fbc 100644 --- a/crates/ra_syntax/src/ast/generated/nodes.rs +++ b/crates/ra_syntax/src/ast/generated/nodes.rs @@ -1235,6 +1235,8 @@ impl CastExpr { /// ``` /// ❰ &foo ❱; /// ❰ &mut bar ❱; +/// ❰ &raw const bar ❱; +/// ❰ &raw mut bar ❱; /// ``` /// /// [Reference](https://doc.rust-lang.org/reference/expressions/operator-expr.html#borrow-operators) @@ -1247,6 +1249,7 @@ impl RefExpr { pub fn amp_token(&self) -> Option { support::token(&self.syntax, T![&]) } pub fn raw_token(&self) -> Option { support::token(&self.syntax, T![raw]) } pub fn mut_token(&self) -> Option { support::token(&self.syntax, T![mut]) } + pub fn const_token(&self) -> Option { support::token(&self.syntax, T![const]) } pub fn expr(&self) -> Option { support::child(&self.syntax) } } /// Prefix operator call. This is either `!` or `*` or `-`. diff --git a/crates/ra_syntax/test_data/parser/inline/ok/0082_ref_expr.rast b/crates/ra_syntax/test_data/parser/inline/ok/0082_ref_expr.rast index 7fe96e17d..58bdf7e34 100644 --- a/crates/ra_syntax/test_data/parser/inline/ok/0082_ref_expr.rast +++ b/crates/ra_syntax/test_data/parser/inline/ok/0082_ref_expr.rast @@ -1,5 +1,5 @@ -SOURCE_FILE@0..52 - FN_DEF@0..51 +SOURCE_FILE@0..200 + FN_DEF@0..199 FN_KW@0..2 "fn" WHITESPACE@2..3 " " NAME@3..6 @@ -8,47 +8,131 @@ SOURCE_FILE@0..52 L_PAREN@6..7 "(" R_PAREN@7..8 ")" WHITESPACE@8..9 " " - BLOCK_EXPR@9..51 + BLOCK_EXPR@9..199 L_CURLY@9..10 "{" WHITESPACE@10..15 "\n " - LET_STMT@15..26 - LET_KW@15..18 "let" - WHITESPACE@18..19 " " - PLACEHOLDER_PAT@19..20 - UNDERSCORE@19..20 "_" - WHITESPACE@20..21 " " - EQ@21..22 "=" - WHITESPACE@22..23 " " - REF_EXPR@23..25 - AMP@23..24 "&" - LITERAL@24..25 - INT_NUMBER@24..25 "1" - SEMICOLON@25..26 ";" - WHITESPACE@26..31 "\n " - LET_STMT@31..49 - LET_KW@31..34 "let" - WHITESPACE@34..35 " " - PLACEHOLDER_PAT@35..36 - UNDERSCORE@35..36 "_" - WHITESPACE@36..37 " " - EQ@37..38 "=" - WHITESPACE@38..39 " " - REF_EXPR@39..48 - AMP@39..40 "&" - MUT_KW@40..43 "mut" - WHITESPACE@43..44 " " - REF_EXPR@44..48 - AMP@44..45 "&" - CALL_EXPR@45..48 - PATH_EXPR@45..46 - PATH@45..46 - PATH_SEGMENT@45..46 - NAME_REF@45..46 - IDENT@45..46 "f" - ARG_LIST@46..48 - L_PAREN@46..47 "(" - R_PAREN@47..48 ")" - SEMICOLON@48..49 ";" - WHITESPACE@49..50 "\n" - R_CURLY@50..51 "}" - WHITESPACE@51..52 "\n" + COMMENT@15..36 "// reference operator" + WHITESPACE@36..41 "\n " + LET_STMT@41..52 + LET_KW@41..44 "let" + WHITESPACE@44..45 " " + PLACEHOLDER_PAT@45..46 + UNDERSCORE@45..46 "_" + WHITESPACE@46..47 " " + EQ@47..48 "=" + WHITESPACE@48..49 " " + REF_EXPR@49..51 + AMP@49..50 "&" + LITERAL@50..51 + INT_NUMBER@50..51 "1" + SEMICOLON@51..52 ";" + WHITESPACE@52..57 "\n " + LET_STMT@57..75 + LET_KW@57..60 "let" + WHITESPACE@60..61 " " + PLACEHOLDER_PAT@61..62 + UNDERSCORE@61..62 "_" + WHITESPACE@62..63 " " + EQ@63..64 "=" + WHITESPACE@64..65 " " + REF_EXPR@65..74 + AMP@65..66 "&" + MUT_KW@66..69 "mut" + WHITESPACE@69..70 " " + REF_EXPR@70..74 + AMP@70..71 "&" + CALL_EXPR@71..74 + PATH_EXPR@71..72 + PATH@71..72 + PATH_SEGMENT@71..72 + NAME_REF@71..72 + IDENT@71..72 "f" + ARG_LIST@72..74 + L_PAREN@72..73 "(" + R_PAREN@73..74 ")" + SEMICOLON@74..75 ";" + WHITESPACE@75..80 "\n " + LET_STMT@80..93 + LET_KW@80..83 "let" + WHITESPACE@83..84 " " + PLACEHOLDER_PAT@84..85 + UNDERSCORE@84..85 "_" + WHITESPACE@85..86 " " + EQ@86..87 "=" + WHITESPACE@87..88 " " + REF_EXPR@88..92 + AMP@88..89 "&" + PATH_EXPR@89..92 + PATH@89..92 + PATH_SEGMENT@89..92 + NAME_REF@89..92 + IDENT@89..92 "raw" + SEMICOLON@92..93 ";" + WHITESPACE@93..98 "\n " + LET_STMT@98..113 + LET_KW@98..101 "let" + WHITESPACE@101..102 " " + PLACEHOLDER_PAT@102..103 + UNDERSCORE@102..103 "_" + WHITESPACE@103..104 " " + EQ@104..105 "=" + WHITESPACE@105..106 " " + REF_EXPR@106..112 + AMP@106..107 "&" + FIELD_EXPR@107..112 + PATH_EXPR@107..110 + PATH@107..110 + PATH_SEGMENT@107..110 + NAME_REF@107..110 + IDENT@107..110 "raw" + DOT@110..111 "." + NAME_REF@111..112 + INT_NUMBER@111..112 "0" + SEMICOLON@112..113 ";" + WHITESPACE@113..118 "\n " + COMMENT@118..143 "// raw reference oper ..." + WHITESPACE@143..148 "\n " + LET_STMT@148..169 + LET_KW@148..151 "let" + WHITESPACE@151..152 " " + PLACEHOLDER_PAT@152..153 + UNDERSCORE@152..153 "_" + WHITESPACE@153..154 " " + EQ@154..155 "=" + WHITESPACE@155..156 " " + REF_EXPR@156..168 + AMP@156..157 "&" + RAW_KW@157..160 "raw" + WHITESPACE@160..161 " " + MUT_KW@161..164 "mut" + WHITESPACE@164..165 " " + PATH_EXPR@165..168 + PATH@165..168 + PATH_SEGMENT@165..168 + NAME_REF@165..168 + IDENT@165..168 "foo" + SEMICOLON@168..169 ";" + WHITESPACE@169..174 "\n " + LET_STMT@174..197 + LET_KW@174..177 "let" + WHITESPACE@177..178 " " + PLACEHOLDER_PAT@178..179 + UNDERSCORE@178..179 "_" + WHITESPACE@179..180 " " + EQ@180..181 "=" + WHITESPACE@181..182 " " + REF_EXPR@182..196 + AMP@182..183 "&" + RAW_KW@183..186 "raw" + WHITESPACE@186..187 " " + CONST_KW@187..192 "const" + WHITESPACE@192..193 " " + PATH_EXPR@193..196 + PATH@193..196 + PATH_SEGMENT@193..196 + NAME_REF@193..196 + IDENT@193..196 "foo" + SEMICOLON@196..197 ";" + WHITESPACE@197..198 "\n" + R_CURLY@198..199 "}" + WHITESPACE@199..200 "\n" diff --git a/crates/ra_syntax/test_data/parser/inline/ok/0082_ref_expr.rs b/crates/ra_syntax/test_data/parser/inline/ok/0082_ref_expr.rs index 2dac6be95..c5262f446 100644 --- a/crates/ra_syntax/test_data/parser/inline/ok/0082_ref_expr.rs +++ b/crates/ra_syntax/test_data/parser/inline/ok/0082_ref_expr.rs @@ -1,4 +1,10 @@ fn foo() { + // reference operator let _ = &1; let _ = &mut &f(); + let _ = &raw; + let _ = &raw.0; + // raw reference operator + let _ = &raw mut foo; + let _ = &raw const foo; } -- cgit v1.2.3 From a27ede88a3a9063a3a2265b668b1c41b589852f5 Mon Sep 17 00:00:00 2001 From: Roland Ruckerbauer Date: Sat, 30 May 2020 14:09:10 +0200 Subject: Fix problem with format string tokenization Fixed by just not handling closing curlybrace escaping. --- crates/ra_syntax/src/ast/tokens.rs | 5 ----- 1 file changed, 5 deletions(-) (limited to 'crates/ra_syntax') diff --git a/crates/ra_syntax/src/ast/tokens.rs b/crates/ra_syntax/src/ast/tokens.rs index 3cd6d99c3..04b0a4480 100644 --- a/crates/ra_syntax/src/ast/tokens.rs +++ b/crates/ra_syntax/src/ast/tokens.rs @@ -418,14 +418,9 @@ pub trait HasFormatSpecifier: AstToken { let mut cloned = chars.clone().take(2); let first = cloned.next().and_then(|next| next.1.as_ref().ok()).copied(); - let second = cloned.next().and_then(|next| next.1.as_ref().ok()).copied(); if first != Some('}') { continue; } - if second == Some('}') { - // Escaped format end specifier, `}}` - continue; - } skip_char_and_emit(&mut chars, FormatSpecifier::Close, &mut callback); } _ => { -- cgit v1.2.3 From 1cd78a3355ea70d3070cabb00c80a5d195499752 Mon Sep 17 00:00:00 2001 From: robojumper Date: Sun, 31 May 2020 10:59:40 +0200 Subject: correctly infer labelled breaks --- crates/ra_syntax/src/ast/generated/nodes.rs | 1 + 1 file changed, 1 insertion(+) (limited to 'crates/ra_syntax') diff --git a/crates/ra_syntax/src/ast/generated/nodes.rs b/crates/ra_syntax/src/ast/generated/nodes.rs index 255402fbc..cb430ca01 100644 --- a/crates/ra_syntax/src/ast/generated/nodes.rs +++ b/crates/ra_syntax/src/ast/generated/nodes.rs @@ -1081,6 +1081,7 @@ pub struct BlockExpr { impl ast::AttrsOwner for BlockExpr {} impl ast::ModuleItemOwner for BlockExpr {} impl BlockExpr { + pub fn label(&self) -> Option