From 5ba4f949c23dcf53f34995c90b7c01e6c641b1f0 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Fri, 6 Nov 2020 22:21:56 +0100 Subject: Kill RAW_ literals Syntactically, they are indistinguishable from non-raw versions, so it doesn't make sense to separate then *at the syntax* level. --- crates/syntax/src/ast/expr_ext.rs | 11 +++++-- crates/syntax/src/ast/generated/tokens.rs | 8 ++--- crates/syntax/src/ast/node_ext.rs | 8 +---- crates/syntax/src/ast/token_ext.rs | 52 +++++++++++++------------------ crates/syntax/src/parsing/lexer.rs | 4 +-- crates/syntax/src/parsing/reparsing.rs | 2 +- crates/syntax/src/validation.rs | 41 +++++++++++++----------- 7 files changed, 61 insertions(+), 65 deletions(-) (limited to 'crates/syntax/src') diff --git a/crates/syntax/src/ast/expr_ext.rs b/crates/syntax/src/ast/expr_ext.rs index 3d33cd1cf..eb44bb2ab 100644 --- a/crates/syntax/src/ast/expr_ext.rs +++ b/crates/syntax/src/ast/expr_ext.rs @@ -320,6 +320,13 @@ impl ast::Literal { ast::IntNumber::cast(self.token()) } + pub fn as_string(&self) -> Option { + ast::String::cast(self.token()) + } + pub fn as_byte_string(&self) -> Option { + ast::ByteString::cast(self.token()) + } + fn find_suffix(text: &str, possible_suffixes: &[&str]) -> Option { possible_suffixes .iter() @@ -351,10 +358,10 @@ impl ast::Literal { suffix: Self::find_suffix(&text, &ast::FloatNumber::SUFFIXES), } } - STRING | RAW_STRING => LiteralKind::String, + STRING => LiteralKind::String, T![true] => LiteralKind::Bool(true), T![false] => LiteralKind::Bool(false), - BYTE_STRING | RAW_BYTE_STRING => LiteralKind::ByteString, + BYTE_STRING => LiteralKind::ByteString, CHAR => LiteralKind::Char, BYTE => LiteralKind::Byte, _ => unreachable!(), diff --git a/crates/syntax/src/ast/generated/tokens.rs b/crates/syntax/src/ast/generated/tokens.rs index 1b8449221..728b72cd7 100644 --- a/crates/syntax/src/ast/generated/tokens.rs +++ b/crates/syntax/src/ast/generated/tokens.rs @@ -70,16 +70,16 @@ impl AstToken for String { } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct RawString { +pub struct ByteString { pub(crate) syntax: SyntaxToken, } -impl std::fmt::Display for RawString { +impl std::fmt::Display for ByteString { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { std::fmt::Display::fmt(&self.syntax, f) } } -impl AstToken for RawString { - fn can_cast(kind: SyntaxKind) -> bool { kind == RAW_STRING } +impl AstToken for ByteString { + fn can_cast(kind: SyntaxKind) -> bool { kind == BYTE_STRING } fn cast(syntax: SyntaxToken) -> Option { if Self::can_cast(syntax.kind()) { Some(Self { syntax }) diff --git a/crates/syntax/src/ast/node_ext.rs b/crates/syntax/src/ast/node_ext.rs index c5cd1c504..5579f72b9 100644 --- a/crates/syntax/src/ast/node_ext.rs +++ b/crates/syntax/src/ast/node_ext.rs @@ -55,13 +55,7 @@ impl ast::Attr { let key = self.simple_name()?; let value_token = lit.syntax().first_token()?; - let value: SmolStr = if let Some(s) = ast::String::cast(value_token.clone()) { - s.value()?.into() - } else if let Some(s) = ast::RawString::cast(value_token) { - s.value()?.into() - } else { - return None; - }; + let value: SmolStr = ast::String::cast(value_token.clone())?.value()?.into(); Some((key, value)) } diff --git a/crates/syntax/src/ast/token_ext.rs b/crates/syntax/src/ast/token_ext.rs index 8d3fad5a6..6cd20b6a6 100644 --- a/crates/syntax/src/ast/token_ext.rs +++ b/crates/syntax/src/ast/token_ext.rs @@ -139,14 +139,31 @@ pub trait HasQuotes: AstToken { } impl HasQuotes for ast::String {} -impl HasQuotes for ast::RawString {} pub trait HasStringValue: HasQuotes { fn value(&self) -> Option>; } +impl ast::String { + pub fn is_raw(&self) -> bool { + self.text().starts_with('r') + } + pub fn map_range_up(&self, range: TextRange) -> Option { + let contents_range = self.text_range_between_quotes()?; + assert!(TextRange::up_to(contents_range.len()).contains_range(range)); + Some(range + contents_range.start()) + } +} + impl HasStringValue for ast::String { fn value(&self) -> Option> { + if self.is_raw() { + let text = self.text().as_str(); + let text = + &text[self.text_range_between_quotes()? - self.syntax().text_range().start()]; + return Some(Cow::Borrowed(text)); + } + let text = self.text().as_str(); let text = &text[self.text_range_between_quotes()? - self.syntax().text_range().start()]; @@ -166,20 +183,9 @@ impl HasStringValue for ast::String { } } -// FIXME: merge `ast::RawString` and `ast::String`. -impl HasStringValue for ast::RawString { - fn value(&self) -> Option> { - let text = self.text().as_str(); - let text = &text[self.text_range_between_quotes()? - self.syntax().text_range().start()]; - Some(Cow::Borrowed(text)) - } -} - -impl ast::RawString { - pub fn map_range_up(&self, range: TextRange) -> Option { - let contents_range = self.text_range_between_quotes()?; - assert!(TextRange::up_to(contents_range.len()).contains_range(range)); - Some(range + contents_range.start()) +impl ast::ByteString { + pub fn is_raw(&self) -> bool { + self.text().starts_with("br") } } @@ -522,22 +528,6 @@ impl HasFormatSpecifier for ast::String { } } -impl HasFormatSpecifier for ast::RawString { - fn char_ranges( - &self, - ) -> Option)>> { - let text = self.text().as_str(); - let text = &text[self.text_range_between_quotes()? - self.syntax().text_range().start()]; - let offset = self.text_range_between_quotes()?.start() - self.syntax().text_range().start(); - - let mut res = Vec::with_capacity(text.len()); - for (idx, c) in text.char_indices() { - res.push((TextRange::at(idx.try_into().unwrap(), TextSize::of(c)) + offset, Ok(c))); - } - Some(res) - } -} - impl ast::IntNumber { #[rustfmt::skip] pub(crate) const SUFFIXES: &'static [&'static str] = &[ diff --git a/crates/syntax/src/parsing/lexer.rs b/crates/syntax/src/parsing/lexer.rs index 5674ecb84..8afd7e53b 100644 --- a/crates/syntax/src/parsing/lexer.rs +++ b/crates/syntax/src/parsing/lexer.rs @@ -235,7 +235,7 @@ fn rustc_token_kind_to_syntax_kind( RawStrError::TooManyDelimiters { .. } => "Too many `#` symbols: raw strings may be delimited by up to 65535 `#` symbols", }; }; - RAW_STRING + STRING } rustc_lexer::LiteralKind::RawByteStr { err: raw_str_err, .. } => { if let Some(raw_str_err) = raw_str_err { @@ -250,7 +250,7 @@ fn rustc_token_kind_to_syntax_kind( }; }; - RAW_BYTE_STRING + BYTE_STRING } }; diff --git a/crates/syntax/src/parsing/reparsing.rs b/crates/syntax/src/parsing/reparsing.rs index 4149f856a..190f5f67a 100644 --- a/crates/syntax/src/parsing/reparsing.rs +++ b/crates/syntax/src/parsing/reparsing.rs @@ -44,7 +44,7 @@ fn reparse_token<'node>( let prev_token = algo::find_covering_element(root, edit.delete).as_token()?.clone(); let prev_token_kind = prev_token.kind(); match prev_token_kind { - WHITESPACE | COMMENT | IDENT | STRING | RAW_STRING => { + WHITESPACE | COMMENT | IDENT | STRING => { if prev_token_kind == WHITESPACE || prev_token_kind == COMMENT { // removing a new line may extends previous token let deleted_range = edit.delete - prev_token.text_range().start(); diff --git a/crates/syntax/src/validation.rs b/crates/syntax/src/validation.rs index 0f9a5e8ae..62a37c50a 100644 --- a/crates/syntax/src/validation.rs +++ b/crates/syntax/src/validation.rs @@ -4,7 +4,7 @@ mod block; use crate::{ algo, ast, match_ast, AstNode, SyntaxError, - SyntaxKind::{BYTE, BYTE_STRING, CHAR, CONST, FN, INT_NUMBER, STRING, TYPE_ALIAS}, + SyntaxKind::{BYTE, CHAR, CONST, FN, INT_NUMBER, TYPE_ALIAS}, SyntaxNode, SyntaxToken, TextSize, T, }; use rowan::Direction; @@ -121,18 +121,19 @@ fn validate_literal(literal: ast::Literal, acc: &mut Vec) { acc.push(SyntaxError::new_at_offset(rustc_unescape_error_to_string(err), off)); }; - match token.kind() { - BYTE => { - if let Some(Err(e)) = unquote(text, 2, '\'').map(unescape_byte) { - push_err(2, e); - } - } - CHAR => { - if let Some(Err(e)) = unquote(text, 1, '\'').map(unescape_char) { - push_err(1, e); + if let Some(s) = literal.as_string() { + if !s.is_raw() { + if let Some(without_quotes) = unquote(text, 1, '"') { + unescape_literal(without_quotes, Mode::Str, &mut |range, char| { + if let Err(err) = char { + push_err(1, (range.start, err)); + } + }) } } - BYTE_STRING => { + } + if let Some(s) = literal.as_byte_string() { + if !s.is_raw() { if let Some(without_quotes) = unquote(text, 2, '"') { unescape_byte_literal(without_quotes, Mode::ByteStr, &mut |range, char| { if let Err(err) = char { @@ -141,13 +142,17 @@ fn validate_literal(literal: ast::Literal, acc: &mut Vec) { }) } } - STRING => { - if let Some(without_quotes) = unquote(text, 1, '"') { - unescape_literal(without_quotes, Mode::Str, &mut |range, char| { - if let Err(err) = char { - push_err(1, (range.start, err)); - } - }) + } + + match token.kind() { + BYTE => { + if let Some(Err(e)) = unquote(text, 2, '\'').map(unescape_byte) { + push_err(2, e); + } + } + CHAR => { + if let Some(Err(e)) = unquote(text, 1, '\'').map(unescape_char) { + push_err(1, e); } } _ => (), -- cgit v1.2.3