From 8d71a6bf0ca51ae099a5b470afdb957bca321441 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Fri, 10 Apr 2020 11:49:13 +0200 Subject: Scale token generation back --- xtask/src/ast_src.rs | 114 +++++++++---------------- xtask/src/codegen.rs | 1 - xtask/src/codegen/gen_syntax.rs | 182 +++++----------------------------------- 3 files changed, 59 insertions(+), 238 deletions(-) (limited to 'xtask') diff --git a/xtask/src/ast_src.rs b/xtask/src/ast_src.rs index f9cd12d0d..554afc76a 100644 --- a/xtask/src/ast_src.rs +++ b/xtask/src/ast_src.rs @@ -227,7 +227,6 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc { pub(crate) struct AstSrc<'a> { pub(crate) nodes: &'a [AstNodeSrc<'a>], pub(crate) enums: &'a [AstEnumSrc<'a>], - pub(crate) token_enums: &'a [AstEnumSrc<'a>], } pub(crate) struct AstNodeSrc<'a> { @@ -415,11 +414,11 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct TupleType { T!['('], fields: [TypeRef], T![')'] } struct NeverType { T![!] } struct PathType { Path } - struct PointerType { Star, T![const], T![mut], TypeRef } + struct PointerType { T![*], T![const], T![mut], TypeRef } struct ArrayType { T!['['], TypeRef, T![;], Expr, T![']'] } struct SliceType { T!['['], TypeRef, T![']'] } - struct ReferenceType { Amp, Lifetime, T![mut], TypeRef } - struct PlaceholderType { Underscore } + struct ReferenceType { T![&], T![lifetime], T![mut], TypeRef } + struct PlaceholderType { T![_] } struct FnPointerType { Abi, T![unsafe], T![fn], ParamList, RetType } struct ForType { T![for], TypeParamList, TypeRef } struct ImplTraitType: TypeBoundsOwner { T![impl] } @@ -447,33 +446,33 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { iterable: Expr, } struct WhileExpr: AttrsOwner, LoopBodyOwner { T![while], Condition } - struct ContinueExpr: AttrsOwner { T![continue], Lifetime } - struct BreakExpr: AttrsOwner { T![break], Lifetime, Expr } - struct Label { Lifetime } + struct ContinueExpr: AttrsOwner { T![continue], T![lifetime] } + struct BreakExpr: AttrsOwner { T![break], T![lifetime], Expr } + struct Label { T![lifetime] } struct BlockExpr: AttrsOwner { Label, T![unsafe], Block } struct ReturnExpr: AttrsOwner { Expr } struct CallExpr: ArgListOwner { Expr } struct MethodCallExpr: AttrsOwner, ArgListOwner { - Expr, Dot, NameRef, TypeArgList, + Expr, T![.], NameRef, TypeArgList, } struct IndexExpr: AttrsOwner { T!['['], T![']'] } - struct FieldExpr: AttrsOwner { Expr, Dot, NameRef } - struct AwaitExpr: AttrsOwner { Expr, Dot, T![await] } + struct FieldExpr: AttrsOwner { Expr, T![.], NameRef } + struct AwaitExpr: AttrsOwner { Expr, T![.], T![await] } struct TryExpr: AttrsOwner { T![try], Expr } struct CastExpr: AttrsOwner { Expr, T![as], TypeRef } - struct RefExpr: AttrsOwner { Amp, T![raw], T![mut], Expr } - struct PrefixExpr: AttrsOwner { PrefixOp, Expr } + struct RefExpr: AttrsOwner { T![&], T![raw], T![mut], Expr } + struct PrefixExpr: AttrsOwner { /*PrefixOp,*/ Expr } struct BoxExpr: AttrsOwner { T![box], Expr } - struct RangeExpr: AttrsOwner { RangeOp } + struct RangeExpr: AttrsOwner { /*RangeOp*/ } struct BinExpr: AttrsOwner { /*BinOp*/ } - struct Literal { LiteralToken } + struct Literal { /*LiteralToken*/ } struct MatchExpr: AttrsOwner { T![match], Expr, MatchArmList } struct MatchArmList: AttrsOwner { T!['{'], arms: [MatchArm], T!['}'] } struct MatchArm: AttrsOwner { pat: Pat, guard: MatchGuard, - FatArrow, + T![=>], Expr, } struct MatchGuard { T![if], Expr } @@ -482,22 +481,22 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct RecordFieldList { T!['{'], fields: [RecordField], - Dotdot, + T![..], spread: Expr, T!['}'] } - struct RecordField: AttrsOwner { NameRef, Colon, Expr } + struct RecordField: AttrsOwner { NameRef, T![:], Expr } struct OrPat { pats: [Pat] } struct ParenPat { T!['('], Pat, T![')'] } - struct RefPat { Amp, T![mut], Pat } + struct RefPat { T![&], T![mut], Pat } struct BoxPat { T![box], Pat } - struct BindPat: AttrsOwner, NameOwner { T![ref], T![mut], At, Pat } - struct PlaceholderPat { Underscore } - struct DotDotPat { Dotdot } + struct BindPat: AttrsOwner, NameOwner { T![ref], T![mut], T![@], Pat } + struct PlaceholderPat { T![_] } + struct DotDotPat { T![..] } struct PathPat { Path } struct SlicePat { T!['['], args: [Pat], T![']'] } - struct RangePat { RangeSeparator } + struct RangePat { /*RangeSeparator*/ } struct LiteralPat { Literal } struct MacroPat { MacroCall } @@ -507,30 +506,30 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { pats: [RecordInnerPat], record_field_pats: [RecordFieldPat], bind_pats: [BindPat], - Dotdot, + T![..], T!['}'] } - struct RecordFieldPat: AttrsOwner, NameOwner { Colon, Pat } + struct RecordFieldPat: AttrsOwner, NameOwner { T![:], Pat } struct TupleStructPat { Path, T!['('], args: [Pat], T![')'] } struct TuplePat { T!['('], args: [Pat], T![')'] } struct Visibility { T![pub], T![super], T![self], T![crate] } - struct Name { Ident } - struct NameRef { NameRefToken } + struct Name { T![ident] } + struct NameRef { /*NameRefToken*/ } struct MacroCall: NameOwner, AttrsOwner,DocCommentsOwner { Path, T![!], TokenTree, T![;] } - struct Attr { Pound, T![!], T!['['], Path, T![=], input: AttrInput, T![']'] } + struct Attr { T![#], T![!], T!['['], Path, T![=], input: AttrInput, T![']'] } struct TokenTree {} struct TypeParamList { - LAngle, + T![<], generic_params: [GenericParam], type_params: [TypeParam], lifetime_params: [LifetimeParam], const_params: [ConstParam], - RAngle + T![>] } struct TypeParam: NameOwner, AttrsOwner, TypeBoundsOwner { T![=], @@ -540,12 +539,12 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { T![=], default_val: Expr, } - struct LifetimeParam: AttrsOwner { Lifetime} - struct TypeBound { Lifetime, /* Question, */ T![const], /* Question, */ TypeRef} + struct LifetimeParam: AttrsOwner { T![lifetime] } + struct TypeBound { T![lifetime], /* Question, */ T![const], /* Question, */ TypeRef} struct TypeBoundList { bounds: [TypeBound] } - struct WherePred: TypeBoundsOwner { Lifetime, TypeRef } + struct WherePred: TypeBoundsOwner { T![lifetime], TypeRef } struct WhereClause { T![where], predicates: [WherePred] } - struct Abi { String } + struct Abi { /*String*/ } struct ExprStmt: AttrsOwner { Expr, T![;] } struct LetStmt: AttrsOwner, TypeAscriptionOwner { T![let], @@ -567,17 +566,17 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { params: [Param], T![')'] } - struct SelfParam: TypeAscriptionOwner, AttrsOwner { Amp, Lifetime, T![self] } + struct SelfParam: TypeAscriptionOwner, AttrsOwner { T![&], T![lifetime], T![self] } struct Param: TypeAscriptionOwner, AttrsOwner { Pat, - Dotdotdot + T![...] } struct UseItem: AttrsOwner, VisibilityOwner { T![use], UseTree, } struct UseTree { - Path, Star, UseTreeList, Alias + Path, T![*], UseTreeList, Alias } struct Alias: NameOwner { T![as] } struct UseTreeList { T!['{'], use_trees: [UseTree], T!['}'] } @@ -594,21 +593,21 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { qualifier: Path, } struct PathSegment { - Coloncolon, LAngle, NameRef, TypeArgList, ParamList, RetType, PathType, RAngle + T![::], T![<], NameRef, TypeArgList, ParamList, RetType, PathType, T![>] } struct TypeArgList { - Coloncolon, - LAngle, + T![::], + T![<], generic_args: [GenericArg], type_args: [TypeArg], lifetime_args: [LifetimeArg], assoc_type_args: [AssocTypeArg], const_args: [ConstArg], - RAngle + T![>] } struct TypeArg { TypeRef } struct AssocTypeArg : TypeBoundsOwner { NameRef, T![=], TypeRef } - struct LifetimeArg { Lifetime } + struct LifetimeArg { T![lifetime] } struct ConstArg { Literal, T![=], BlockExpr } struct MacroItems: ModuleItemOwner{ } @@ -767,37 +766,4 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { TupleFieldDefList, } }, - - token_enums: &ast_enums! { - enum RangeSeparator { Dotdot, Dotdotdot, Dotdoteq} - - enum PrefixOp { - Minus, - T![!], - Star - } - - enum RangeOp { - Dotdot, - Dotdoteq - } - - enum LiteralToken { - IntNumber, - FloatNumber, - String, - RawString, - // TrueKw, - // FalseKw, - ByteString, - RawByteString, - Char, - Byte - } - - enum NameRefToken { - Ident, - IntNumber - } - }, }; diff --git a/xtask/src/codegen.rs b/xtask/src/codegen.rs index 678b40133..e6ba2009c 100644 --- a/xtask/src/codegen.rs +++ b/xtask/src/codegen.rs @@ -24,7 +24,6 @@ const ERR_INLINE_TESTS_DIR: &str = "crates/ra_syntax/test_data/parser/inline/err const SYNTAX_KINDS: &str = "crates/ra_parser/src/syntax_kind/generated.rs"; const AST_NODES: &str = "crates/ra_syntax/src/ast/generated/nodes.rs"; -const AST_TOKENS: &str = "crates/ra_syntax/src/ast/generated/tokens.rs"; const ASSISTS_DIR: &str = "crates/ra_assists/src/handlers"; const ASSISTS_TESTS: &str = "crates/ra_assists/src/doc_tests/generated.rs"; diff --git a/xtask/src/codegen/gen_syntax.rs b/xtask/src/codegen/gen_syntax.rs index 4c9e447a3..ce18f2b8f 100644 --- a/xtask/src/codegen/gen_syntax.rs +++ b/xtask/src/codegen/gen_syntax.rs @@ -3,10 +3,7 @@ //! Specifically, it generates the `SyntaxKind` enum and a number of newtype //! wrappers around `SyntaxNode` which implement `ra_syntax::AstNode`. -use std::{ - borrow::Cow, - collections::{BTreeSet, HashSet}, -}; +use std::collections::{BTreeSet, HashSet}; use proc_macro2::{Punct, Spacing}; use quote::{format_ident, quote}; @@ -26,10 +23,6 @@ pub fn generate_syntax(mode: Mode) -> Result<()> { let contents = generate_nodes(KINDS_SRC, AST_SRC)?; update(ast_nodes_file.as_path(), &contents, mode)?; - let ast_tokens_file = project_root().join(codegen::AST_TOKENS); - let contents = generate_tokens(KINDS_SRC, AST_SRC)?; - update(ast_tokens_file.as_path(), &contents, mode)?; - Ok(()) } @@ -40,147 +33,7 @@ struct ElementKinds { has_tokens: bool, } -fn generate_tokens(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { - let all_token_kinds: Vec<_> = kinds - .punct - .into_iter() - .map(|(_, kind)| kind) - .copied() - .map(|x| x.into()) - .chain( - kinds - .keywords - .into_iter() - .chain(kinds.contextual_keywords.into_iter()) - .map(|name| Cow::Owned(format!("{}_KW", to_upper_snake_case(&name)))), - ) - .chain(kinds.literals.into_iter().copied().map(|x| x.into())) - .chain(kinds.tokens.into_iter().copied().map(|x| x.into())) - .collect(); - - let tokens = all_token_kinds.iter().filter_map(|kind_str| { - if kind_str.ends_with("_KW") { - return None; - } - let kind_str = &**kind_str; - let kind = format_ident!("{}", kind_str); - let name = format_ident!("{}", to_pascal_case(kind_str)); - let res = quote! { - #[derive(Debug, Clone, PartialEq, Eq, Hash)] - pub struct #name { - pub(crate) syntax: SyntaxToken, - } - - impl std::fmt::Display for #name { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - std::fmt::Display::fmt(&self.syntax, f) - } - } - - impl AstToken for #name { - fn can_cast(kind: SyntaxKind) -> bool { kind == #kind } - fn cast(syntax: SyntaxToken) -> Option { - if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None } - } - fn syntax(&self) -> &SyntaxToken { &self.syntax } - } - }; - Some(res) - }); - - let enums = grammar.token_enums.iter().map(|en| { - let variants = en.variants.iter().map(|var| format_ident!("{}", var)).collect::>(); - let name = format_ident!("{}", en.name); - let kinds = variants - .iter() - .map(|name| format_ident!("{}", to_upper_snake_case(&name.to_string()))) - .collect::>(); - assert!(en.traits.is_empty()); - - quote! { - #[derive(Debug, Clone, PartialEq, Eq, Hash)] - pub enum #name { - #(#variants(#variants),)* - } - - #( - impl From<#variants> for #name { - fn from(node: #variants) -> #name { - #name::#variants(node) - } - } - )* - - impl std::fmt::Display for #name { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - std::fmt::Display::fmt(self.syntax(), f) - } - } - - impl AstToken for #name { - fn can_cast(kind: SyntaxKind) -> bool { - match kind { - #(#kinds)|* => true, - _ => false, - } - } - fn cast(syntax: SyntaxToken) -> Option { - let res = match syntax.kind() { - #( - #kinds => #name::#variants(#variants { syntax }), - )* - _ => return None, - }; - Some(res) - } - fn syntax(&self) -> &SyntaxToken { - match self { - #( - #name::#variants(it) => &it.syntax, - )* - } - } - } - } - }); - - crate::reformat(quote! { - use crate::{SyntaxToken, SyntaxKind::{self, *}, ast::AstToken}; - - #(#tokens)* - #(#enums)* - }) -} - fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { - let all_token_kinds: Vec<_> = kinds - .punct - .into_iter() - .map(|(_, kind)| kind) - .copied() - .map(|x| x.into()) - .chain( - kinds - .keywords - .into_iter() - .chain(kinds.contextual_keywords.into_iter()) - .map(|name| Cow::Owned(format!("{}_KW", to_upper_snake_case(&name)))), - ) - .chain(kinds.literals.into_iter().copied().map(|x| x.into())) - .chain(kinds.tokens.into_iter().copied().map(|x| x.into())) - .collect(); - - let mut token_kinds = HashSet::new(); - for kind in &all_token_kinds { - let kind = &**kind; - let name = to_pascal_case(kind); - token_kinds.insert(name); - } - - for en in grammar.token_enums { - token_kinds.insert(en.name.to_string()); - } - let nodes = grammar.nodes.iter().map(|node| { let name = format_ident!("{}", node.name); let kind = format_ident!("{}", to_upper_snake_case(&name.to_string())); @@ -207,19 +60,9 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { } } } else { - let is_token = token_kinds.contains(&ty.to_string()); - if is_token { - let method_name = format_ident!("{}_token", method_name); - quote! { - pub fn #method_name(&self) -> Option<#ty> { - support::token(&self.syntax) - } - } - } else { - quote! { - pub fn #method_name(&self) -> Option<#ty> { - support::child(&self.syntax) - } + quote! { + pub fn #method_name(&self) -> Option<#ty> { + support::child(&self.syntax) } } } @@ -338,8 +181,6 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { T, }; - use super::tokens::*; - #(#nodes)* #(#enums)* #(#displays)* @@ -456,6 +297,8 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> Result { macro_rules! T { #((#punctuation_values) => { $crate::SyntaxKind::#punctuation };)* #((#all_keywords_idents) => { $crate::SyntaxKind::#all_keywords };)* + (lifetime) => { $crate::SyntaxKind::LIFETIME }; + (ident) => { $crate::SyntaxKind::IDENT }; } }; @@ -535,8 +378,21 @@ impl Field<'_> { "')'" => "r_paren", "'['" => "l_brack", "']'" => "r_brack", + "<" => "l_angle", + ">" => "r_angle", "=" => "eq", "!" => "excl", + "*" => "star", + "&" => "amp", + "_" => "underscore", + "." => "dot", + ".." => "dotdot", + "..." => "dotdotdot", + "=>" => "fat_arrow", + "@" => "at", + ":" => "colon", + "::" => "coloncolon", + "#" => "pound", _ => name, }; format_ident!("{}_token", name) -- cgit v1.2.3