5 files changed, 213 insertions, 171 deletions

diff --git a/crates/syntax/src/ast/expr_ext.rs b/crates/syntax/src/ast/expr_ext.rs
index 3aff01e83..9253c97d0 100644
--- a/crates/syntax/src/ast/expr_ext.rs
+++ b/crates/syntax/src/ast/expr_ext.rs
@@ -2,7 +2,7 @@
 
 use crate::{
     ast::{self, support, AstChildren, AstNode},
-    SmolStr,
+    AstToken,
     SyntaxKind::*,
     SyntaxToken, T,
 };
@@ -298,12 +298,12 @@ impl ast::ArrayExpr {
 
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
 pub enum LiteralKind {
-    String,
-    ByteString,
+    String(ast::String),
+    ByteString(ast::ByteString),
+    IntNumber(ast::IntNumber),
+    FloatNumber(ast::FloatNumber),
     Char,
     Byte,
-    IntNumber { suffix: Option<SmolStr> },
-    FloatNumber { suffix: Option<SmolStr> },
     Bool(bool),
 }
 
@@ -315,114 +315,30 @@ impl ast::Literal {
             .and_then(|e| e.into_token())
             .unwrap()
     }
-
-    fn find_suffix(text: &str, possible_suffixes: &[&str]) -> Option<SmolStr> {
-        possible_suffixes
-            .iter()
-            .find(|&suffix| text.ends_with(suffix))
-            .map(|&suffix| SmolStr::new(suffix))
-    }
-
     pub fn kind(&self) -> LiteralKind {
-        const INT_SUFFIXES: [&str; 12] = [
-            "u64", "u32", "u16", "u8", "usize", "isize", "i64", "i32", "i16", "i8", "u128", "i128",
-        ];
-        const FLOAT_SUFFIXES: [&str; 2] = ["f32", "f64"];
-
         let token = self.token();
 
+        if let Some(t) = ast::IntNumber::cast(token.clone()) {
+            return LiteralKind::IntNumber(t);
+        }
+        if let Some(t) = ast::FloatNumber::cast(token.clone()) {
+            return LiteralKind::FloatNumber(t);
+        }
+        if let Some(t) = ast::String::cast(token.clone()) {
+            return LiteralKind::String(t);
+        }
+        if let Some(t) = ast::ByteString::cast(token.clone()) {
+            return LiteralKind::ByteString(t);
+        }
+
         match token.kind() {
-            INT_NUMBER => {
-                // FYI: there was a bug here previously, thus the if statement below is necessary.
-                // The lexer treats e.g. `1f64` as an integer literal. See
-                // https://github.com/rust-analyzer/rust-analyzer/issues/1592
-                // and the comments on the linked PR.
-
-                let text = token.text();
-                if let suffix @ Some(_) = Self::find_suffix(&text, &FLOAT_SUFFIXES) {
-                    LiteralKind::FloatNumber { suffix }
-                } else {
-                    LiteralKind::IntNumber { suffix: Self::find_suffix(&text, &INT_SUFFIXES) }
-                }
-            }
-            FLOAT_NUMBER => {
-                let text = token.text();
-                LiteralKind::FloatNumber { suffix: Self::find_suffix(&text, &FLOAT_SUFFIXES) }
-            }
-            STRING | RAW_STRING => LiteralKind::String,
             T![true] => LiteralKind::Bool(true),
             T![false] => LiteralKind::Bool(false),
-            BYTE_STRING | RAW_BYTE_STRING => LiteralKind::ByteString,
             CHAR => LiteralKind::Char,
             BYTE => LiteralKind::Byte,
             _ => unreachable!(),
         }
     }
-
-    // FIXME: should probably introduce string token type?
-    // https://github.com/rust-analyzer/rust-analyzer/issues/6308
-    pub fn int_value(&self) -> Option<(Radix, u128)> {
-        let suffix = match self.kind() {
-            LiteralKind::IntNumber { suffix } => suffix,
-            _ => return None,
-        };
-
-        let token = self.token();
-        let mut text = token.text().as_str();
-        text = &text[..text.len() - suffix.map_or(0, |it| it.len())];
-
-        let buf;
-        if text.contains("_") {
-            buf = text.replace('_', "");
-            text = buf.as_str();
-        };
-
-        let radix = Radix::identify(text)?;
-        let digits = &text[radix.prefix_len()..];
-        let value = u128::from_str_radix(digits, radix as u32).ok()?;
-        Some((radix, value))
-    }
-}
-
-#[derive(Debug, PartialEq, Eq, Copy, Clone)]
-pub enum Radix {
-    Binary = 2,
-    Octal = 8,
-    Decimal = 10,
-    Hexadecimal = 16,
-}
-
-impl Radix {
-    pub const ALL: &'static [Radix] =
-        &[Radix::Binary, Radix::Octal, Radix::Decimal, Radix::Hexadecimal];
-
-    fn identify(literal_text: &str) -> Option<Self> {
-        // We cannot express a literal in anything other than decimal in under 3 characters, so we return here if possible.
-        if literal_text.len() < 3 && literal_text.chars().all(|c| c.is_digit(10)) {
-            return Some(Self::Decimal);
-        }
-
-        let res = match &literal_text[..2] {
-            "0b" => Radix::Binary,
-            "0o" => Radix::Octal,
-            "0x" => Radix::Hexadecimal,
-            _ => Radix::Decimal,
-        };
-
-        // Checks that all characters after the base prefix are all valid digits for that base.
-        if literal_text[res.prefix_len()..].chars().all(|c| c.is_digit(res as u32)) {
-            Some(res)
-        } else {
-            None
-        }
-    }
-
-    const fn prefix_len(&self) -> usize {
-        match self {
-            Self::Decimal => 0,
-            _ => 2,
-        }
-    }
 }
 
 #[derive(Debug, Clone, PartialEq, Eq)]
diff --git a/crates/syntax/src/ast/generated/tokens.rs b/crates/syntax/src/ast/generated/tokens.rs
index abadd0b61..728b72cd7 100644
--- a/crates/syntax/src/ast/generated/tokens.rs
+++ b/crates/syntax/src/ast/generated/tokens.rs
@@ -70,16 +70,58 @@ impl AstToken for String {
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct RawString {
+pub struct ByteString {
     pub(crate) syntax: SyntaxToken,
 }
-impl std::fmt::Display for RawString {
+impl std::fmt::Display for ByteString {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         std::fmt::Display::fmt(&self.syntax, f)
     }
 }
-impl AstToken for RawString {
-    fn can_cast(kind: SyntaxKind) -> bool { kind == RAW_STRING }
+impl AstToken for ByteString {
+    fn can_cast(kind: SyntaxKind) -> bool { kind == BYTE_STRING }
+    fn cast(syntax: SyntaxToken) -> Option<Self> {
+        if Self::can_cast(syntax.kind()) {
+            Some(Self { syntax })
+        } else {
+            None
+        }
+    }
+    fn syntax(&self) -> &SyntaxToken { &self.syntax }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct IntNumber {
+    pub(crate) syntax: SyntaxToken,
+}
+impl std::fmt::Display for IntNumber {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        std::fmt::Display::fmt(&self.syntax, f)
+    }
+}
+impl AstToken for IntNumber {
+    fn can_cast(kind: SyntaxKind) -> bool { kind == INT_NUMBER }
+    fn cast(syntax: SyntaxToken) -> Option<Self> {
+        if Self::can_cast(syntax.kind()) {
+            Some(Self { syntax })
+        } else {
+            None
+        }
+    }
+    fn syntax(&self) -> &SyntaxToken { &self.syntax }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct FloatNumber {
+    pub(crate) syntax: SyntaxToken,
+}
+impl std::fmt::Display for FloatNumber {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        std::fmt::Display::fmt(&self.syntax, f)
+    }
+}
+impl AstToken for FloatNumber {
+    fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_NUMBER }
     fn cast(syntax: SyntaxToken) -> Option<Self> {
         if Self::can_cast(syntax.kind()) {
             Some(Self { syntax })
diff --git a/crates/syntax/src/ast/make.rs b/crates/syntax/src/ast/make.rs
index 2cf436e7a..876659a2b 100644
--- a/crates/syntax/src/ast/make.rs
+++ b/crates/syntax/src/ast/make.rs
@@ -25,6 +25,10 @@ pub fn assoc_item_list() -> ast::AssocItemList {
     ast_from_text("impl C for D {};")
 }
 
+pub fn impl_trait(trait_: ast::Path, ty: ast::Path) -> ast::Impl {
+    ast_from_text(&format!("impl {} for {} {{}}", trait_, ty))
+}
+
 pub fn path_segment(name_ref: ast::NameRef) -> ast::PathSegment {
     ast_from_text(&format!("use {};", name_ref))
 }
@@ -110,8 +114,16 @@ pub fn record_expr_field(name: ast::NameRef, expr: Option<ast::Expr>) -> ast::Re
     }
 }
 
-pub fn record_field(name: ast::NameRef, ty: ast::Type) -> ast::RecordField {
-    ast_from_text(&format!("struct S {{ {}: {}, }}", name, ty))
+pub fn record_field(
+    visibility: Option<ast::Visibility>,
+    name: ast::Name,
+    ty: ast::Type,
+) -> ast::RecordField {
+    let visibility = match visibility {
+        None => String::new(),
+        Some(it) => format!("{} ", it),
+    };
+    ast_from_text(&format!("struct S {{ {}{}: {}, }}", visibility, name, ty))
 }
 
 pub fn block_expr(
@@ -360,6 +372,13 @@ pub fn tuple_field_list(fields: impl IntoIterator<Item = ast::TupleField>) -> as
     ast_from_text(&format!("struct f({});", fields))
 }
 
+pub fn record_field_list(
+    fields: impl IntoIterator<Item = ast::RecordField>,
+) -> ast::RecordFieldList {
+    let fields = fields.into_iter().join(", ");
+    ast_from_text(&format!("struct f {{ {} }}", fields))
+}
+
 pub fn tuple_field(visibility: Option<ast::Visibility>, ty: ast::Type) -> ast::TupleField {
     let visibility = match visibility {
         None => String::new(),
@@ -368,6 +387,14 @@ pub fn tuple_field(visibility: Option<ast::Visibility>, ty: ast::Type) -> ast::T
     ast_from_text(&format!("struct f({}{});", visibility, ty))
 }
 
+pub fn variant(name: ast::Name, field_list: Option<ast::FieldList>) -> ast::Variant {
+    let field_list = match field_list {
+        None => String::new(),
+        Some(it) => format!("{}", it),
+    };
+    ast_from_text(&format!("enum f {{ {}{} }}", name, field_list))
+}
+
 pub fn fn_(
     visibility: Option<ast::Visibility>,
     fn_name: ast::Name,
diff --git a/crates/syntax/src/ast/node_ext.rs b/crates/syntax/src/ast/node_ext.rs
index c5cd1c504..ce35ac01a 100644
--- a/crates/syntax/src/ast/node_ext.rs
+++ b/crates/syntax/src/ast/node_ext.rs
@@ -7,7 +7,7 @@ use itertools::Itertools;
 use parser::SyntaxKind;
 
 use crate::{
-    ast::{self, support, token_ext::HasStringValue, AstNode, AstToken, NameOwner, SyntaxNode},
+    ast::{self, support, AstNode, AstToken, NameOwner, SyntaxNode},
     SmolStr, SyntaxElement, SyntaxToken, T,
 };
 
@@ -55,13 +55,7 @@ impl ast::Attr {
         let key = self.simple_name()?;
         let value_token = lit.syntax().first_token()?;
 
-        let value: SmolStr = if let Some(s) = ast::String::cast(value_token.clone()) {
-            s.value()?.into()
-        } else if let Some(s) = ast::RawString::cast(value_token) {
-            s.value()?.into()
-        } else {
-            return None;
-        };
+        let value: SmolStr = ast::String::cast(value_token.clone())?.value()?.into();
 
         Some((key, value))
     }
diff --git a/crates/syntax/src/ast/token_ext.rs b/crates/syntax/src/ast/token_ext.rs
index c5ef92733..e4e512f2e 100644
--- a/crates/syntax/src/ast/token_ext.rs
+++ b/crates/syntax/src/ast/token_ext.rs
@@ -8,11 +8,11 @@ use std::{
 use rustc_lexer::unescape::{unescape_literal, Mode};
 
 use crate::{
-    ast::{AstToken, Comment, RawString, String, Whitespace},
+    ast::{self, AstToken},
     TextRange, TextSize,
 };
 
-impl Comment {
+impl ast::Comment {
     pub fn kind(&self) -> CommentKind {
         kind_by_prefix(self.text())
     }
@@ -80,7 +80,7 @@ fn kind_by_prefix(text: &str) -> CommentKind {
     panic!("bad comment text: {:?}", text)
 }
 
-impl Whitespace {
+impl ast::Whitespace {
     pub fn spans_multiple_lines(&self) -> bool {
         let text = self.text();
         text.find('\n').map_or(false, |idx| text[idx + 1..].contains('\n'))
@@ -114,43 +114,28 @@ impl QuoteOffsets {
     }
 }
 
-pub trait HasQuotes: AstToken {
-    fn quote_offsets(&self) -> Option<QuoteOffsets> {
-        let text = self.text().as_str();
-        let offsets = QuoteOffsets::new(text)?;
-        let o = self.syntax().text_range().start();
-        let offsets = QuoteOffsets {
-            quotes: (offsets.quotes.0 + o, offsets.quotes.1 + o),
-            contents: offsets.contents + o,
-        };
-        Some(offsets)
-    }
-    fn open_quote_text_range(&self) -> Option<TextRange> {
-        self.quote_offsets().map(|it| it.quotes.0)
+impl ast::String {
+    pub fn is_raw(&self) -> bool {
+        self.text().starts_with('r')
     }
-
-    fn close_quote_text_range(&self) -> Option<TextRange> {
-        self.quote_offsets().map(|it| it.quotes.1)
+    pub fn map_range_up(&self, range: TextRange) -> Option<TextRange> {
+        let contents_range = self.text_range_between_quotes()?;
+        assert!(TextRange::up_to(contents_range.len()).contains_range(range));
+        Some(range + contents_range.start())
     }
 
-    fn text_range_between_quotes(&self) -> Option<TextRange> {
-        self.quote_offsets().map(|it| it.contents)
-    }
-}
-
-impl HasQuotes for String {}
-impl HasQuotes for RawString {}
-
-pub trait HasStringValue: HasQuotes {
-    fn value(&self) -> Option<Cow<'_, str>>;
-}
+    pub fn value(&self) -> Option<Cow<'_, str>> {
+        if self.is_raw() {
+            let text = self.text().as_str();
+            let text =
+                &text[self.text_range_between_quotes()? - self.syntax().text_range().start()];
+            return Some(Cow::Borrowed(text));
+        }
 
-impl HasStringValue for String {
-    fn value(&self) -> Option<Cow<'_, str>> {
         let text = self.text().as_str();
         let text = &text[self.text_range_between_quotes()? - self.syntax().text_range().start()];
 
-        let mut buf = std::string::String::with_capacity(text.len());
+        let mut buf = String::with_capacity(text.len());
         let mut has_error = false;
         unescape_literal(text, Mode::Str, &mut |_, unescaped_char| match unescaped_char {
             Ok(c) => buf.push(c),
@@ -164,21 +149,31 @@ impl HasStringValue for String {
         let res = if buf == text { Cow::Borrowed(text) } else { Cow::Owned(buf) };
         Some(res)
     }
-}
 
-impl HasStringValue for RawString {
-    fn value(&self) -> Option<Cow<'_, str>> {
+    pub fn quote_offsets(&self) -> Option<QuoteOffsets> {
         let text = self.text().as_str();
-        let text = &text[self.text_range_between_quotes()? - self.syntax().text_range().start()];
-        Some(Cow::Borrowed(text))
+        let offsets = QuoteOffsets::new(text)?;
+        let o = self.syntax().text_range().start();
+        let offsets = QuoteOffsets {
+            quotes: (offsets.quotes.0 + o, offsets.quotes.1 + o),
+            contents: offsets.contents + o,
+        };
+        Some(offsets)
+    }
+    pub fn text_range_between_quotes(&self) -> Option<TextRange> {
+        self.quote_offsets().map(|it| it.contents)
+    }
+    pub fn open_quote_text_range(&self) -> Option<TextRange> {
+        self.quote_offsets().map(|it| it.quotes.0)
+    }
+    pub fn close_quote_text_range(&self) -> Option<TextRange> {
+        self.quote_offsets().map(|it| it.quotes.1)
     }
 }
 
-impl RawString {
-    pub fn map_range_up(&self, range: TextRange) -> Option<TextRange> {
-        let contents_range = self.text_range_between_quotes()?;
-        assert!(TextRange::up_to(contents_range.len()).contains_range(range));
-        Some(range + contents_range.start())
+impl ast::ByteString {
+    pub fn is_raw(&self) -> bool {
+        self.text().starts_with("br")
     }
 }
 
@@ -500,7 +495,7 @@ pub trait HasFormatSpecifier: AstToken {
     }
 }
 
-impl HasFormatSpecifier for String {
+impl HasFormatSpecifier for ast::String {
     fn char_ranges(
         &self,
     ) -> Option<Vec<(TextRange, Result<char, rustc_lexer::unescape::EscapeError>)>> {
@@ -521,18 +516,86 @@ impl HasFormatSpecifier for String {
     }
 }
 
-impl HasFormatSpecifier for RawString {
-    fn char_ranges(
-        &self,
-    ) -> Option<Vec<(TextRange, Result<char, rustc_lexer::unescape::EscapeError>)>> {
-        let text = self.text().as_str();
-        let text = &text[self.text_range_between_quotes()? - self.syntax().text_range().start()];
-        let offset = self.text_range_between_quotes()?.start() - self.syntax().text_range().start();
+impl ast::IntNumber {
+    const SUFFIXES: &'static [&'static str] = &[
+        "u8", "u16", "u32", "u64", "u128", "usize", // Unsigned.
+        "i8", "i16", "i32", "i64", "i128", "isize", // Signed.
+    ];
+
+    pub fn radix(&self) -> Radix {
+        match self.text().get(..2).unwrap_or_default() {
+            "0b" => Radix::Binary,
+            "0o" => Radix::Octal,
+            "0x" => Radix::Hexadecimal,
+            _ => Radix::Decimal,
+        }
+    }
 
-        let mut res = Vec::with_capacity(text.len());
-        for (idx, c) in text.char_indices() {
-            res.push((TextRange::at(idx.try_into().unwrap(), TextSize::of(c)) + offset, Ok(c)));
+    pub fn value(&self) -> Option<u128> {
+        let token = self.syntax();
+
+        let mut text = token.text().as_str();
+        if let Some(suffix) = self.suffix() {
+            text = &text[..text.len() - suffix.len()]
+        }
+
+        let radix = self.radix();
+        text = &text[radix.prefix_len()..];
+
+        let buf;
+        if text.contains("_") {
+            buf = text.replace('_', "");
+            text = buf.as_str();
+        };
+
+        let value = u128::from_str_radix(text, radix as u32).ok()?;
+        Some(value)
+    }
+
+    pub fn suffix(&self) -> Option<&str> {
+        let text = self.text();
+        // FIXME: don't check a fixed set of suffixes, `1_0_1_l_o_l` is valid
+        // syntax, suffix is `l_o_l`.
+        ast::IntNumber::SUFFIXES.iter().chain(ast::FloatNumber::SUFFIXES.iter()).find_map(
+            |suffix| {
+                if text.ends_with(suffix) {
+                    return Some(&text[text.len() - suffix.len()..]);
+                }
+                None
+            },
+        )
+    }
+}
+
+impl ast::FloatNumber {
+    const SUFFIXES: &'static [&'static str] = &["f32", "f64"];
+    pub fn suffix(&self) -> Option<&str> {
+        let text = self.text();
+        ast::FloatNumber::SUFFIXES.iter().find_map(|suffix| {
+            if text.ends_with(suffix) {
+                return Some(&text[text.len() - suffix.len()..]);
+            }
+            None
+        })
+    }
+}
+
+#[derive(Debug, PartialEq, Eq, Copy, Clone)]
+pub enum Radix {
+    Binary = 2,
+    Octal = 8,
+    Decimal = 10,
+    Hexadecimal = 16,
+}
+
+impl Radix {
+    pub const ALL: &'static [Radix] =
+        &[Radix::Binary, Radix::Octal, Radix::Decimal, Radix::Hexadecimal];
+
+    const fn prefix_len(&self) -> usize {
+        match self {
+            Self::Decimal => 0,
+            _ => 2,
         }
-        Some(res)
     }
 }