Scale token generation back

3 files changed, 59 insertions, 238 deletions

diff --git a/xtask/src/ast_src.rs b/xtask/src/ast_src.rs
index f9cd12d0d..554afc76a 100644
--- a/xtask/src/ast_src.rs
+++ b/xtask/src/ast_src.rs
@@ -227,7 +227,6 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc {
 pub(crate) struct AstSrc<'a> {
     pub(crate) nodes: &'a [AstNodeSrc<'a>],
     pub(crate) enums: &'a [AstEnumSrc<'a>],
-    pub(crate) token_enums: &'a [AstEnumSrc<'a>],
 }
 
 pub(crate) struct AstNodeSrc<'a> {
@@ -415,11 +414,11 @@ pub(crate) const AST_SRC: AstSrc = AstSrc {
         struct TupleType { T!['('], fields: [TypeRef], T![')'] }
         struct NeverType { T![!] }
         struct PathType { Path }
-        struct PointerType { Star, T![const], T![mut], TypeRef }
+        struct PointerType { T![*], T![const], T![mut], TypeRef }
         struct ArrayType { T!['['], TypeRef, T![;], Expr, T![']'] }
         struct SliceType { T!['['], TypeRef, T![']'] }
-        struct ReferenceType { Amp, Lifetime, T![mut], TypeRef }
-        struct PlaceholderType { Underscore }
+        struct ReferenceType { T![&], T![lifetime], T![mut], TypeRef }
+        struct PlaceholderType { T![_] }
         struct FnPointerType { Abi, T![unsafe], T![fn], ParamList, RetType }
         struct ForType { T![for], TypeParamList, TypeRef }
         struct ImplTraitType: TypeBoundsOwner { T![impl] }
@@ -447,33 +446,33 @@ pub(crate) const AST_SRC: AstSrc = AstSrc {
             iterable: Expr,
         }
         struct WhileExpr: AttrsOwner, LoopBodyOwner { T![while], Condition }
-        struct ContinueExpr: AttrsOwner { T![continue], Lifetime }
-        struct BreakExpr: AttrsOwner { T![break], Lifetime, Expr }
-        struct Label { Lifetime }
+        struct ContinueExpr: AttrsOwner { T![continue], T![lifetime] }
+        struct BreakExpr: AttrsOwner { T![break], T![lifetime], Expr }
+        struct Label { T![lifetime] }
         struct BlockExpr: AttrsOwner { Label, T![unsafe], Block  }
         struct ReturnExpr: AttrsOwner { Expr }
         struct CallExpr: ArgListOwner { Expr }
         struct MethodCallExpr: AttrsOwner, ArgListOwner {
-            Expr, Dot, NameRef, TypeArgList,
+            Expr, T![.], NameRef, TypeArgList,
         }
         struct IndexExpr: AttrsOwner { T!['['], T![']'] }
-        struct FieldExpr: AttrsOwner { Expr, Dot, NameRef }
-        struct AwaitExpr: AttrsOwner { Expr, Dot, T![await] }
+        struct FieldExpr: AttrsOwner { Expr, T![.], NameRef }
+        struct AwaitExpr: AttrsOwner { Expr, T![.], T![await] }
         struct TryExpr: AttrsOwner { T![try], Expr }
         struct CastExpr: AttrsOwner { Expr, T![as], TypeRef }
-        struct RefExpr: AttrsOwner { Amp, T![raw], T![mut], Expr }
-        struct PrefixExpr: AttrsOwner { PrefixOp, Expr }
+        struct RefExpr: AttrsOwner { T![&], T![raw], T![mut], Expr }
+        struct PrefixExpr: AttrsOwner { /*PrefixOp,*/ Expr }
         struct BoxExpr: AttrsOwner { T![box], Expr }
-        struct RangeExpr: AttrsOwner { RangeOp }
+        struct RangeExpr: AttrsOwner { /*RangeOp*/ }
         struct BinExpr: AttrsOwner { /*BinOp*/ }
-        struct Literal { LiteralToken }
+        struct Literal { /*LiteralToken*/ }
 
         struct MatchExpr: AttrsOwner { T![match], Expr, MatchArmList }
         struct MatchArmList: AttrsOwner { T!['{'], arms: [MatchArm], T!['}'] }
         struct MatchArm: AttrsOwner {
             pat: Pat,
             guard: MatchGuard,
-            FatArrow,
+            T![=>],
             Expr,
         }
         struct MatchGuard { T![if], Expr }
@@ -482,22 +481,22 @@ pub(crate) const AST_SRC: AstSrc = AstSrc {
         struct RecordFieldList {
             T!['{'],
             fields: [RecordField],
-            Dotdot,
+            T![..],
             spread: Expr,
             T!['}']
         }
-        struct RecordField: AttrsOwner { NameRef, Colon, Expr }
+        struct RecordField: AttrsOwner { NameRef, T![:], Expr }
 
         struct OrPat { pats: [Pat] }
         struct ParenPat { T!['('], Pat, T![')'] }
-        struct RefPat { Amp, T![mut], Pat }
+        struct RefPat { T![&], T![mut], Pat }
         struct BoxPat { T![box], Pat }
-        struct BindPat: AttrsOwner, NameOwner { T![ref], T![mut], At, Pat }
-        struct PlaceholderPat { Underscore }
-        struct DotDotPat { Dotdot }
+        struct BindPat: AttrsOwner, NameOwner { T![ref], T![mut], T![@], Pat }
+        struct PlaceholderPat { T![_] }
+        struct DotDotPat { T![..] }
         struct PathPat { Path }
         struct SlicePat { T!['['], args: [Pat], T![']'] }
-        struct RangePat { RangeSeparator }
+        struct RangePat { /*RangeSeparator*/ }
         struct LiteralPat { Literal }
         struct MacroPat { MacroCall }
 
@@ -507,30 +506,30 @@ pub(crate) const AST_SRC: AstSrc = AstSrc {
             pats: [RecordInnerPat],
             record_field_pats: [RecordFieldPat],
             bind_pats: [BindPat],
-            Dotdot,
+            T![..],
             T!['}']
         }
-        struct RecordFieldPat: AttrsOwner, NameOwner { Colon, Pat }
+        struct RecordFieldPat: AttrsOwner, NameOwner { T![:], Pat }
 
         struct TupleStructPat { Path, T!['('], args: [Pat], T![')'] }
         struct TuplePat { T!['('], args: [Pat], T![')'] }
 
         struct Visibility { T![pub], T![super], T![self], T![crate] }
-        struct Name { Ident }
-        struct NameRef { NameRefToken }
+        struct Name { T![ident] }
+        struct NameRef { /*NameRefToken*/ }
 
         struct MacroCall: NameOwner, AttrsOwner,DocCommentsOwner {
             Path, T![!], TokenTree, T![;]
         }
-        struct Attr { Pound, T![!], T!['['], Path, T![=], input: AttrInput, T![']'] }
+        struct Attr { T![#], T![!], T!['['], Path, T![=], input: AttrInput, T![']'] }
         struct TokenTree {}
         struct TypeParamList {
-            LAngle,
+            T![<],
             generic_params: [GenericParam],
             type_params: [TypeParam],
             lifetime_params: [LifetimeParam],
             const_params: [ConstParam],
-            RAngle
+            T![>]
         }
         struct TypeParam: NameOwner, AttrsOwner, TypeBoundsOwner {
             T![=],
@@ -540,12 +539,12 @@ pub(crate) const AST_SRC: AstSrc = AstSrc {
             T![=],
             default_val: Expr,
         }
-        struct LifetimeParam: AttrsOwner { Lifetime}
-        struct TypeBound { Lifetime, /* Question,  */ T![const], /* Question,  */ TypeRef}
+        struct LifetimeParam: AttrsOwner { T![lifetime] }
+        struct TypeBound { T![lifetime], /* Question,  */ T![const], /* Question,  */ TypeRef}
         struct TypeBoundList { bounds: [TypeBound] }
-        struct WherePred: TypeBoundsOwner { Lifetime, TypeRef }
+        struct WherePred: TypeBoundsOwner { T![lifetime], TypeRef }
         struct WhereClause { T![where], predicates: [WherePred] }
-        struct Abi { String }
+        struct Abi { /*String*/ }
         struct ExprStmt: AttrsOwner { Expr, T![;] }
         struct LetStmt: AttrsOwner, TypeAscriptionOwner {
             T![let],
@@ -567,17 +566,17 @@ pub(crate) const AST_SRC: AstSrc = AstSrc {
             params: [Param],
             T![')']
         }
-        struct SelfParam: TypeAscriptionOwner, AttrsOwner { Amp, Lifetime, T![self] }
+        struct SelfParam: TypeAscriptionOwner, AttrsOwner { T![&], T![lifetime], T![self] }
         struct Param: TypeAscriptionOwner, AttrsOwner {
             Pat,
-            Dotdotdot
+            T![...]
         }
         struct UseItem: AttrsOwner, VisibilityOwner {
             T![use],
             UseTree,
         }
         struct UseTree {
-            Path, Star, UseTreeList, Alias
+            Path, T![*], UseTreeList, Alias
         }
         struct Alias: NameOwner { T![as] }
         struct UseTreeList { T!['{'], use_trees: [UseTree], T!['}'] }
@@ -594,21 +593,21 @@ pub(crate) const AST_SRC: AstSrc = AstSrc {
             qualifier: Path,
         }
         struct PathSegment {
-            Coloncolon, LAngle, NameRef, TypeArgList, ParamList, RetType, PathType, RAngle
+            T![::], T![<], NameRef, TypeArgList, ParamList, RetType, PathType, T![>]
         }
         struct TypeArgList {
-            Coloncolon,
-            LAngle,
+            T![::],
+            T![<],
             generic_args: [GenericArg],
             type_args: [TypeArg],
             lifetime_args: [LifetimeArg],
             assoc_type_args: [AssocTypeArg],
             const_args: [ConstArg],
-            RAngle
+            T![>]
         }
         struct TypeArg { TypeRef }
         struct AssocTypeArg : TypeBoundsOwner { NameRef, T![=], TypeRef }
-        struct LifetimeArg { Lifetime }
+        struct LifetimeArg { T![lifetime] }
         struct ConstArg { Literal, T![=], BlockExpr }
 
         struct MacroItems: ModuleItemOwner{ }
@@ -767,37 +766,4 @@ pub(crate) const AST_SRC: AstSrc = AstSrc {
             TupleFieldDefList,
         }
     },
-
-    token_enums: &ast_enums! {
-        enum RangeSeparator { Dotdot, Dotdotdot, Dotdoteq}
-
-        enum PrefixOp {
-            Minus,
-            T![!],
-            Star
-        }
-
-        enum RangeOp {
-            Dotdot,
-            Dotdoteq
-        }
-
-        enum LiteralToken {
-            IntNumber,
-            FloatNumber,
-            String,
-            RawString,
-            // TrueKw,
-            // FalseKw,
-            ByteString,
-            RawByteString,
-            Char,
-            Byte
-        }
-
-        enum NameRefToken {
-            Ident,
-            IntNumber
-        }
-    },
 };
diff --git a/xtask/src/codegen.rs b/xtask/src/codegen.rs
index 678b40133..e6ba2009c 100644
--- a/xtask/src/codegen.rs
+++ b/xtask/src/codegen.rs
@@ -24,7 +24,6 @@ const ERR_INLINE_TESTS_DIR: &str = "crates/ra_syntax/test_data/parser/inline/err
 
 const SYNTAX_KINDS: &str = "crates/ra_parser/src/syntax_kind/generated.rs";
 const AST_NODES: &str = "crates/ra_syntax/src/ast/generated/nodes.rs";
-const AST_TOKENS: &str = "crates/ra_syntax/src/ast/generated/tokens.rs";
 
 const ASSISTS_DIR: &str = "crates/ra_assists/src/handlers";
 const ASSISTS_TESTS: &str = "crates/ra_assists/src/doc_tests/generated.rs";
diff --git a/xtask/src/codegen/gen_syntax.rs b/xtask/src/codegen/gen_syntax.rs
index 4c9e447a3..ce18f2b8f 100644
--- a/xtask/src/codegen/gen_syntax.rs
+++ b/xtask/src/codegen/gen_syntax.rs
@@ -3,10 +3,7 @@
 //! Specifically, it generates the `SyntaxKind` enum and a number of newtype
 //! wrappers around `SyntaxNode` which implement `ra_syntax::AstNode`.
 
-use std::{
-    borrow::Cow,
-    collections::{BTreeSet, HashSet},
-};
+use std::collections::{BTreeSet, HashSet};
 
 use proc_macro2::{Punct, Spacing};
 use quote::{format_ident, quote};
@@ -26,10 +23,6 @@ pub fn generate_syntax(mode: Mode) -> Result<()> {
     let contents = generate_nodes(KINDS_SRC, AST_SRC)?;
     update(ast_nodes_file.as_path(), &contents, mode)?;
 
-    let ast_tokens_file = project_root().join(codegen::AST_TOKENS);
-    let contents = generate_tokens(KINDS_SRC, AST_SRC)?;
-    update(ast_tokens_file.as_path(), &contents, mode)?;
-
     Ok(())
 }
 
@@ -40,147 +33,7 @@ struct ElementKinds {
     has_tokens: bool,
 }
 
-fn generate_tokens(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result<String> {
-    let all_token_kinds: Vec<_> = kinds
-        .punct
-        .into_iter()
-        .map(|(_, kind)| kind)
-        .copied()
-        .map(|x| x.into())
-        .chain(
-            kinds
-                .keywords
-                .into_iter()
-                .chain(kinds.contextual_keywords.into_iter())
-                .map(|name| Cow::Owned(format!("{}_KW", to_upper_snake_case(&name)))),
-        )
-        .chain(kinds.literals.into_iter().copied().map(|x| x.into()))
-        .chain(kinds.tokens.into_iter().copied().map(|x| x.into()))
-        .collect();
-
-    let tokens = all_token_kinds.iter().filter_map(|kind_str| {
-        if kind_str.ends_with("_KW") {
-            return None;
-        }
-        let kind_str = &**kind_str;
-        let kind = format_ident!("{}", kind_str);
-        let name = format_ident!("{}", to_pascal_case(kind_str));
-        let res = quote! {
-            #[derive(Debug, Clone, PartialEq, Eq, Hash)]
-            pub struct #name {
-                pub(crate) syntax: SyntaxToken,
-            }
-
-            impl std::fmt::Display for #name {
-                fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-                    std::fmt::Display::fmt(&self.syntax, f)
-                }
-            }
-
-            impl AstToken for #name {
-                fn can_cast(kind: SyntaxKind) -> bool { kind == #kind }
-                fn cast(syntax: SyntaxToken) -> Option<Self> {
-                    if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
-                }
-                fn syntax(&self) -> &SyntaxToken { &self.syntax }
-            }
-        };
-        Some(res)
-    });
-
-    let enums = grammar.token_enums.iter().map(|en| {
-        let variants = en.variants.iter().map(|var| format_ident!("{}", var)).collect::<Vec<_>>();
-        let name = format_ident!("{}", en.name);
-        let kinds = variants
-            .iter()
-            .map(|name| format_ident!("{}", to_upper_snake_case(&name.to_string())))
-            .collect::<Vec<_>>();
-        assert!(en.traits.is_empty());
-
-        quote! {
-                #[derive(Debug, Clone, PartialEq, Eq, Hash)]
-                pub enum #name {
-                    #(#variants(#variants),)*
-                }
-
-                #(
-                impl From<#variants> for #name {
-                    fn from(node: #variants) -> #name {
-                        #name::#variants(node)
-                    }
-                }
-                )*
-
-                impl std::fmt::Display for #name {
-                    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-                        std::fmt::Display::fmt(self.syntax(), f)
-                    }
-                }
-
-                impl AstToken for #name {
-                    fn can_cast(kind: SyntaxKind) -> bool {
-                        match kind {
-                            #(#kinds)|* => true,
-                            _ => false,
-                        }
-                    }
-                    fn cast(syntax: SyntaxToken) -> Option<Self> {
-                        let res = match syntax.kind() {
-                            #(
-                            #kinds => #name::#variants(#variants { syntax }),
-                            )*
-                            _ => return None,
-                        };
-                        Some(res)
-                    }
-                    fn syntax(&self) -> &SyntaxToken {
-                        match self {
-                            #(
-                            #name::#variants(it) => &it.syntax,
-                            )*
-                        }
-                    }
-                }
-        }
-    });
-
-    crate::reformat(quote! {
-        use crate::{SyntaxToken, SyntaxKind::{self, *}, ast::AstToken};
-
-        #(#tokens)*
-        #(#enums)*
-    })
-}
-
 fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result<String> {
-    let all_token_kinds: Vec<_> = kinds
-        .punct
-        .into_iter()
-        .map(|(_, kind)| kind)
-        .copied()
-        .map(|x| x.into())
-        .chain(
-            kinds
-                .keywords
-                .into_iter()
-                .chain(kinds.contextual_keywords.into_iter())
-                .map(|name| Cow::Owned(format!("{}_KW", to_upper_snake_case(&name)))),
-        )
-        .chain(kinds.literals.into_iter().copied().map(|x| x.into()))
-        .chain(kinds.tokens.into_iter().copied().map(|x| x.into()))
-        .collect();
-
-    let mut token_kinds = HashSet::new();
-    for kind in &all_token_kinds {
-        let kind = &**kind;
-        let name = to_pascal_case(kind);
-        token_kinds.insert(name);
-    }
-
-    for en in grammar.token_enums {
-        token_kinds.insert(en.name.to_string());
-    }
-
     let nodes = grammar.nodes.iter().map(|node| {
         let name = format_ident!("{}", node.name);
         let kind = format_ident!("{}", to_upper_snake_case(&name.to_string()));
@@ -207,19 +60,9 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result<String> {
                         }
                     }
                 } else {
-                    let is_token = token_kinds.contains(&ty.to_string());
-                    if is_token {
-                        let method_name = format_ident!("{}_token", method_name);
-                        quote! {
-                            pub fn #method_name(&self) -> Option<#ty> {
-                                support::token(&self.syntax)
-                            }
-                        }
-                    } else {
-                        quote! {
-                            pub fn #method_name(&self) -> Option<#ty> {
-                                support::child(&self.syntax)
-                            }
+                    quote! {
+                        pub fn #method_name(&self) -> Option<#ty> {
+                            support::child(&self.syntax)
                         }
                     }
                 }
@@ -338,8 +181,6 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result<String> {
             T,
         };
 
-        use super::tokens::*;
-
         #(#nodes)*
         #(#enums)*
         #(#displays)*
@@ -456,6 +297,8 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> Result<String> {
         macro_rules! T {
             #((#punctuation_values) => { $crate::SyntaxKind::#punctuation };)*
             #((#all_keywords_idents) => { $crate::SyntaxKind::#all_keywords };)*
+            (lifetime) => { $crate::SyntaxKind::LIFETIME };
+            (ident) => { $crate::SyntaxKind::IDENT };
         }
     };
 
@@ -535,8 +378,21 @@ impl Field<'_> {
                     "')'" => "r_paren",
                     "'['" => "l_brack",
                     "']'" => "r_brack",
+                    "<" => "l_angle",
+                    ">" => "r_angle",
                     "=" => "eq",
                     "!" => "excl",
+                    "*" => "star",
+                    "&" => "amp",
+                    "_" => "underscore",
+                    "." => "dot",
+                    ".." => "dotdot",
+                    "..." => "dotdotdot",
+                    "=>" => "fat_arrow",
+                    "@" => "at",
+                    ":" => "colon",
+                    "::" => "coloncolon",
+                    "#" => "pound",
                     _ => name,
                 };
                 format_ident!("{}_token", name)