From e6d22187a67e762bb950de244a6ca15f3a0b0731 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Thu, 9 Apr 2020 18:25:36 +0200 Subject: Add _token suffix to token accessors I think this makes is more clear which things are : AstNode and which are : AstToken --- xtask/src/codegen/gen_syntax.rs | 1 + 1 file changed, 1 insertion(+) (limited to 'xtask') diff --git a/xtask/src/codegen/gen_syntax.rs b/xtask/src/codegen/gen_syntax.rs index 6657c9fc5..b5594e3a9 100644 --- a/xtask/src/codegen/gen_syntax.rs +++ b/xtask/src/codegen/gen_syntax.rs @@ -208,6 +208,7 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { FieldSrc::Optional(_) | FieldSrc::Shorthand => { let is_token = token_kinds.contains(&ty.to_string()); if is_token { + let method_name = format_ident!("{}_token", method_name); quote! { pub fn #method_name(&self) -> Option<#ty> { support::token(&self.syntax) -- cgit v1.2.3 From 2bfb65db93e48d8f9e8ecac0b2ea837c081a4db5 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Thu, 9 Apr 2020 18:40:43 +0200 Subject: Be consistent about token accesors --- xtask/src/ast_src.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'xtask') diff --git a/xtask/src/ast_src.rs b/xtask/src/ast_src.rs index eba66ff4d..74a87e900 100644 --- a/xtask/src/ast_src.rs +++ b/xtask/src/ast_src.rs @@ -408,7 +408,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct TupleType { LParen, fields: [TypeRef], RParen } struct NeverType { Excl } struct PathType { Path } - struct PointerType { Star, ConstKw, TypeRef } + struct PointerType { Star, ConstKw, MutKw, TypeRef } struct ArrayType { LBrack, TypeRef, Semi, Expr, RBrack } struct SliceType { LBrack, TypeRef, RBrack } struct ReferenceType { Amp, Lifetime, MutKw, TypeRef } @@ -485,7 +485,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct ParenPat { LParen, Pat, RParen } struct RefPat { Amp, MutKw, Pat } struct BoxPat { BoxKw, Pat } - struct BindPat: AttrsOwner, NameOwner { RefKw, MutKw, Pat } + struct BindPat: AttrsOwner, NameOwner { RefKw, MutKw, At, Pat } struct PlaceholderPat { Underscore } struct DotDotPat { Dotdot } struct PathPat { Path } @@ -545,6 +545,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { Pat, Eq, initializer: Expr, + Semi, } struct Condition { LetKw, Pat, Eq, Expr } struct Block: AttrsOwner, ModuleItemOwner { -- cgit v1.2.3 From 0ed27c388adca887cda3d8141efaf974e90a5958 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Thu, 9 Apr 2020 23:02:10 +0200 Subject: Drop needless trait --- xtask/src/ast_src.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'xtask') diff --git a/xtask/src/ast_src.rs b/xtask/src/ast_src.rs index 74a87e900..ac4cc9717 100644 --- a/xtask/src/ast_src.rs +++ b/xtask/src/ast_src.rs @@ -298,7 +298,7 @@ macro_rules! ast_enums { pub(crate) const AST_SRC: AstSrc = AstSrc { nodes: &ast_nodes! { - struct SourceFile: ModuleItemOwner, FnDefOwner, AttrsOwner { + struct SourceFile: ModuleItemOwner, AttrsOwner { modules: [Module], } @@ -364,7 +364,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { Semi } - struct ItemList: FnDefOwner, ModuleItemOwner { + struct ItemList: ModuleItemOwner { LCurly, impl_items: [ImplItem], RCurly @@ -604,14 +604,14 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct LifetimeArg { Lifetime } struct ConstArg { Literal, Eq, BlockExpr } - struct MacroItems: ModuleItemOwner, FnDefOwner { } + struct MacroItems: ModuleItemOwner{ } struct MacroStmts { statements: [Stmt], Expr, } - struct ExternItemList: FnDefOwner, ModuleItemOwner { + struct ExternItemList: ModuleItemOwner { LCurly, extern_items: [ExternItem], RCurly -- cgit v1.2.3 From 30084a56a5731343bd4cec727646a6c55900234f Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Thu, 9 Apr 2020 23:35:05 +0200 Subject: Simpler acessors for keywords --- xtask/src/ast_src.rs | 4 ++-- xtask/src/codegen/gen_syntax.rs | 33 +++++++++++++++++++++++++-------- 2 files changed, 27 insertions(+), 10 deletions(-) (limited to 'xtask') diff --git a/xtask/src/ast_src.rs b/xtask/src/ast_src.rs index ac4cc9717..bb97b13fe 100644 --- a/xtask/src/ast_src.rs +++ b/xtask/src/ast_src.rs @@ -814,8 +814,8 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { FloatNumber, String, RawString, - TrueKw, - FalseKw, + // TrueKw, + // FalseKw, ByteString, RawByteString, Char, diff --git a/xtask/src/codegen/gen_syntax.rs b/xtask/src/codegen/gen_syntax.rs index b5594e3a9..cc98802f6 100644 --- a/xtask/src/codegen/gen_syntax.rs +++ b/xtask/src/codegen/gen_syntax.rs @@ -58,11 +58,14 @@ fn generate_tokens(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { .chain(kinds.tokens.into_iter().copied().map(|x| x.into())) .collect(); - let tokens = all_token_kinds.iter().map(|kind_str| { + let tokens = all_token_kinds.iter().filter_map(|kind_str| { + if kind_str.ends_with("_KW") { + return None; + } let kind_str = &**kind_str; let kind = format_ident!("{}", kind_str); let name = format_ident!("{}", to_pascal_case(kind_str)); - quote! { + let res = quote! { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct #name { pub(crate) syntax: SyntaxToken, @@ -81,7 +84,8 @@ fn generate_tokens(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { } fn syntax(&self) -> &SyntaxToken { &self.syntax } } - } + }; + Some(res) }); let enums = grammar.token_enums.iter().map(|en| { @@ -186,8 +190,12 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { }); let methods = node.fields.iter().map(|(name, field)| { + let is_kw = name.ends_with("Kw"); let method_name = match field { - FieldSrc::Shorthand => format_ident!("{}", to_lower_snake_case(&name)), + FieldSrc::Shorthand => { + let name = if is_kw { &name[..name.len() - 2] } else { &name }; + format_ident!("{}", to_lower_snake_case(name)) + } _ => format_ident!("{}", name), }; let ty = match field { @@ -209,9 +217,18 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { let is_token = token_kinds.contains(&ty.to_string()); if is_token { let method_name = format_ident!("{}_token", method_name); - quote! { - pub fn #method_name(&self) -> Option<#ty> { - support::token(&self.syntax) + if is_kw { + let token_kind = format_ident!("{}", to_upper_snake_case(name)); + quote! { + pub fn #method_name(&self) -> Option { + support::token2(&self.syntax, #token_kind) + } + } + } else { + quote! { + pub fn #method_name(&self) -> Option<#ty> { + support::token(&self.syntax) + } } } } else { @@ -332,7 +349,7 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { let ast = quote! { use crate::{ - SyntaxNode, SyntaxKind::{self, *}, + SyntaxNode, SyntaxToken, SyntaxKind::{self, *}, ast::{self, AstNode, AstChildren, support}, }; -- cgit v1.2.3 From ff5643c52445c516b8b32bc6062c66203ca13ee4 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Fri, 10 Apr 2020 09:14:14 +0200 Subject: Simplify --- xtask/src/ast_src.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'xtask') diff --git a/xtask/src/ast_src.rs b/xtask/src/ast_src.rs index bb97b13fe..15bd8a2e4 100644 --- a/xtask/src/ast_src.rs +++ b/xtask/src/ast_src.rs @@ -233,13 +233,13 @@ pub(crate) struct AstSrc<'a> { pub(crate) struct AstNodeSrc<'a> { pub(crate) name: &'a str, pub(crate) traits: &'a [&'a str], - pub(crate) fields: &'a [(&'a str, FieldSrc<&'a str>)], + pub(crate) fields: &'a [(&'a str, FieldSrc<'a>)], } -pub(crate) enum FieldSrc { +pub(crate) enum FieldSrc<'a> { Shorthand, - Optional(T), - Many(T), + Optional(&'a str), + Many(&'a str), } pub(crate) struct AstEnumSrc<'a> { -- cgit v1.2.3 From f89f2e38855f5b47f68758e98139aa962cb7a01d Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Fri, 10 Apr 2020 10:07:09 +0200 Subject: More readable ast_src for keywords --- xtask/src/ast_src.rs | 147 +++++++++++++++++++++------------------- xtask/src/codegen/gen_syntax.rs | 90 ++++++++++++++---------- 2 files changed, 133 insertions(+), 104 deletions(-) (limited to 'xtask') diff --git a/xtask/src/ast_src.rs b/xtask/src/ast_src.rs index 15bd8a2e4..3da280551 100644 --- a/xtask/src/ast_src.rs +++ b/xtask/src/ast_src.rs @@ -233,7 +233,12 @@ pub(crate) struct AstSrc<'a> { pub(crate) struct AstNodeSrc<'a> { pub(crate) name: &'a str, pub(crate) traits: &'a [&'a str], - pub(crate) fields: &'a [(&'a str, FieldSrc<'a>)], + pub(crate) fields: &'a [Field<'a>], +} + +pub(crate) enum Field<'a> { + Token(&'a str), + Node { name: &'a str, src: FieldSrc<'a> }, } pub(crate) enum FieldSrc<'a> { @@ -251,31 +256,34 @@ pub(crate) struct AstEnumSrc<'a> { macro_rules! ast_nodes { ($( struct $name:ident$(: $($trait:ident),*)? { - $($field_name:ident $(: $ty:tt)?),*$(,)? + $($field_name:ident $(![$token:tt])? $(: $ty:tt)?),*$(,)? } )*) => { [$( AstNodeSrc { name: stringify!($name), traits: &[$($(stringify!($trait)),*)?], - fields: &[$( - (stringify!($field_name), field_ty!($field_name $($ty)?)) - ),*], + fields: &[ + $(field!($(T![$token])? $field_name $($ty)?)),* + ], } ),*] }; } -macro_rules! field_ty { +macro_rules! field { + (T![$token:tt] T) => { + Field::Token(stringify!($token)) + }; ($field_name:ident) => { - FieldSrc::Shorthand + Field::Node { name: stringify!($field_name), src: FieldSrc::Shorthand } }; ($field_name:ident [$ty:ident]) => { - FieldSrc::Many(stringify!($ty)) + Field::Node { name: stringify!($field_name), src: FieldSrc::Many(stringify!($ty)) } }; ($field_name:ident $ty:ident) => { - FieldSrc::Optional(stringify!($ty)) + Field::Node { name: stringify!($field_name), src: FieldSrc::Optional(stringify!($ty)) } }; } @@ -290,7 +298,6 @@ macro_rules! ast_enums { name: stringify!($name), traits: &[$($(stringify!($trait)),*)?], variants: &[$(stringify!($variant)),*], - } ),*] }; @@ -304,11 +311,11 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct FnDef: VisibilityOwner, NameOwner, TypeParamsOwner, DocCommentsOwner, AttrsOwner { Abi, - ConstKw, - DefaultKw, - AsyncKw, - UnsafeKw, - FnKw, + T![const], + T![default], + T![async], + T![unsafe], + T![fn], ParamList, RetType, body: BlockExpr, @@ -318,13 +325,13 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct RetType { ThinArrow, TypeRef } struct StructDef: VisibilityOwner, NameOwner, TypeParamsOwner, AttrsOwner, DocCommentsOwner { - StructKw, + T![struct], FieldDefList, Semi } struct UnionDef: VisibilityOwner, NameOwner, TypeParamsOwner, AttrsOwner, DocCommentsOwner { - UnionKw, + T![union], RecordFieldDefList, } @@ -337,7 +344,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { } struct EnumDef: VisibilityOwner, NameOwner, TypeParamsOwner, AttrsOwner, DocCommentsOwner { - EnumKw, + T![enum], variant_list: EnumVariantList, } struct EnumVariantList { @@ -352,14 +359,14 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { } struct TraitDef: VisibilityOwner, NameOwner, AttrsOwner, DocCommentsOwner, TypeParamsOwner, TypeBoundsOwner { - UnsafeKw, - AutoKw, - TraitKw, + T![unsafe], + T![auto], + T![trait], ItemList, } struct Module: VisibilityOwner, NameOwner, AttrsOwner, DocCommentsOwner { - ModKw, + T![mod], ItemList, Semi } @@ -371,36 +378,36 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { } struct ConstDef: VisibilityOwner, NameOwner, TypeParamsOwner, AttrsOwner, DocCommentsOwner, TypeAscriptionOwner { - DefaultKw, - ConstKw, + T![default], + T![const], Eq, body: Expr, Semi } struct StaticDef: VisibilityOwner, NameOwner, TypeParamsOwner, AttrsOwner, DocCommentsOwner, TypeAscriptionOwner { - StaticKw, - MutKw, + T![static], + T![mut], Eq, body: Expr, Semi } struct TypeAliasDef: VisibilityOwner, NameOwner, TypeParamsOwner, AttrsOwner, DocCommentsOwner, TypeBoundsOwner { - DefaultKw, - TypeKw, + T![default], + T![type], Eq, TypeRef, Semi } struct ImplDef: TypeParamsOwner, AttrsOwner { - DefaultKw, - ConstKw, - UnsafeKw, - ImplKw, + T![default], + T![const], + T![unsafe], + T![impl], Excl, - ForKw, + T![for], ItemList, } @@ -408,42 +415,42 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct TupleType { LParen, fields: [TypeRef], RParen } struct NeverType { Excl } struct PathType { Path } - struct PointerType { Star, ConstKw, MutKw, TypeRef } + struct PointerType { Star, T![const], T![mut], TypeRef } struct ArrayType { LBrack, TypeRef, Semi, Expr, RBrack } struct SliceType { LBrack, TypeRef, RBrack } - struct ReferenceType { Amp, Lifetime, MutKw, TypeRef } + struct ReferenceType { Amp, Lifetime, T![mut], TypeRef } struct PlaceholderType { Underscore } - struct FnPointerType { Abi, UnsafeKw, FnKw, ParamList, RetType } - struct ForType { ForKw, TypeParamList, TypeRef } - struct ImplTraitType: TypeBoundsOwner { ImplKw } - struct DynTraitType: TypeBoundsOwner { DynKw } + struct FnPointerType { Abi, T![unsafe], T![fn], ParamList, RetType } + struct ForType { T![for], TypeParamList, TypeRef } + struct ImplTraitType: TypeBoundsOwner { T![impl] } + struct DynTraitType: TypeBoundsOwner { T![dyn] } struct TupleExpr: AttrsOwner { LParen, exprs: [Expr], RParen } struct ArrayExpr: AttrsOwner { LBrack, exprs: [Expr], Semi, RBrack } struct ParenExpr: AttrsOwner { LParen, Expr, RParen } struct PathExpr { Path } struct LambdaExpr: AttrsOwner { - StaticKw, - AsyncKw, - MoveKw, + T![static], + T![async], + T![move], ParamList, RetType, body: Expr, } - struct IfExpr: AttrsOwner { IfKw, Condition } - struct LoopExpr: AttrsOwner, LoopBodyOwner { LoopKw } - struct TryBlockExpr: AttrsOwner { TryKw, body: BlockExpr } + struct IfExpr: AttrsOwner { T![if], Condition } + struct LoopExpr: AttrsOwner, LoopBodyOwner { T![loop] } + struct TryBlockExpr: AttrsOwner { T![try], body: BlockExpr } struct ForExpr: AttrsOwner, LoopBodyOwner { - ForKw, + T![for], Pat, - InKw, + T![in], iterable: Expr, } - struct WhileExpr: AttrsOwner, LoopBodyOwner { WhileKw, Condition } - struct ContinueExpr: AttrsOwner { ContinueKw, Lifetime } - struct BreakExpr: AttrsOwner { BreakKw, Lifetime, Expr } + struct WhileExpr: AttrsOwner, LoopBodyOwner { T![while], Condition } + struct ContinueExpr: AttrsOwner { T![continue], Lifetime } + struct BreakExpr: AttrsOwner { T![break], Lifetime, Expr } struct Label { Lifetime } - struct BlockExpr: AttrsOwner { Label, UnsafeKw, Block } + struct BlockExpr: AttrsOwner { Label, T![unsafe], Block } struct ReturnExpr: AttrsOwner { Expr } struct CallExpr: ArgListOwner { Expr } struct MethodCallExpr: AttrsOwner, ArgListOwner { @@ -451,17 +458,17 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { } struct IndexExpr: AttrsOwner { LBrack, RBrack } struct FieldExpr: AttrsOwner { Expr, Dot, NameRef } - struct AwaitExpr: AttrsOwner { Expr, Dot, AwaitKw } - struct TryExpr: AttrsOwner { TryKw, Expr } - struct CastExpr: AttrsOwner { Expr, AsKw, TypeRef } - struct RefExpr: AttrsOwner { Amp, RawKw, MutKw, Expr } + struct AwaitExpr: AttrsOwner { Expr, Dot, T![await] } + struct TryExpr: AttrsOwner { T![try], Expr } + struct CastExpr: AttrsOwner { Expr, T![as], TypeRef } + struct RefExpr: AttrsOwner { Amp, T![raw], T![mut], Expr } struct PrefixExpr: AttrsOwner { PrefixOp, Expr } - struct BoxExpr: AttrsOwner { BoxKw, Expr } + struct BoxExpr: AttrsOwner { T![box], Expr } struct RangeExpr: AttrsOwner { RangeOp } struct BinExpr: AttrsOwner { BinOp } struct Literal { LiteralToken } - struct MatchExpr: AttrsOwner { MatchKw, Expr, MatchArmList } + struct MatchExpr: AttrsOwner { T![match], Expr, MatchArmList } struct MatchArmList: AttrsOwner { LCurly, arms: [MatchArm], RCurly } struct MatchArm: AttrsOwner { pat: Pat, @@ -469,7 +476,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { FatArrow, Expr, } - struct MatchGuard { IfKw, Expr } + struct MatchGuard { T![if], Expr } struct RecordLit { Path, RecordFieldList} struct RecordFieldList { @@ -483,9 +490,9 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct OrPat { pats: [Pat] } struct ParenPat { LParen, Pat, RParen } - struct RefPat { Amp, MutKw, Pat } - struct BoxPat { BoxKw, Pat } - struct BindPat: AttrsOwner, NameOwner { RefKw, MutKw, At, Pat } + struct RefPat { Amp, T![mut], Pat } + struct BoxPat { T![box], Pat } + struct BindPat: AttrsOwner, NameOwner { T![ref], T![mut], At, Pat } struct PlaceholderPat { Underscore } struct DotDotPat { Dotdot } struct PathPat { Path } @@ -508,7 +515,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct TupleStructPat { Path, LParen, args: [Pat], RParen } struct TuplePat { LParen, args: [Pat], RParen } - struct Visibility { PubKw, SuperKw, SelfKw, CrateKw } + struct Visibility { T![pub], T![super], T![self], T![crate] } struct Name { Ident } struct NameRef { NameRefToken } @@ -534,20 +541,20 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { default_val: Expr, } struct LifetimeParam: AttrsOwner { Lifetime} - struct TypeBound { Lifetime, /* Question, */ ConstKw, /* Question, */ TypeRef} + struct TypeBound { Lifetime, /* Question, */ T![const], /* Question, */ TypeRef} struct TypeBoundList { bounds: [TypeBound] } struct WherePred: TypeBoundsOwner { Lifetime, TypeRef } - struct WhereClause { WhereKw, predicates: [WherePred] } + struct WhereClause { T![where], predicates: [WherePred] } struct Abi { String } struct ExprStmt: AttrsOwner { Expr, Semi } struct LetStmt: AttrsOwner, TypeAscriptionOwner { - LetKw, + T![let], Pat, Eq, initializer: Expr, Semi, } - struct Condition { LetKw, Pat, Eq, Expr } + struct Condition { T![let], Pat, Eq, Expr } struct Block: AttrsOwner, ModuleItemOwner { LCurly, statements: [Stmt], @@ -560,22 +567,22 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { params: [Param], RParen } - struct SelfParam: TypeAscriptionOwner, AttrsOwner { Amp, Lifetime, SelfKw } + struct SelfParam: TypeAscriptionOwner, AttrsOwner { Amp, Lifetime, T![self] } struct Param: TypeAscriptionOwner, AttrsOwner { Pat, Dotdotdot } struct UseItem: AttrsOwner, VisibilityOwner { - UseKw, + T![use], UseTree, } struct UseTree { Path, Star, UseTreeList, Alias } - struct Alias: NameOwner { AsKw } + struct Alias: NameOwner { T![as] } struct UseTreeList { LCurly, use_trees: [UseTree], RCurly } struct ExternCrateItem: AttrsOwner, VisibilityOwner { - ExternKw, CrateKw, NameRef, Alias, + T![extern], T![crate], NameRef, Alias, } struct ArgList { LParen, diff --git a/xtask/src/codegen/gen_syntax.rs b/xtask/src/codegen/gen_syntax.rs index cc98802f6..c4fb29bbf 100644 --- a/xtask/src/codegen/gen_syntax.rs +++ b/xtask/src/codegen/gen_syntax.rs @@ -12,7 +12,7 @@ use proc_macro2::{Punct, Spacing}; use quote::{format_ident, quote}; use crate::{ - ast_src::{AstSrc, FieldSrc, KindsSrc, AST_SRC, KINDS_SRC}, + ast_src::{AstSrc, Field, FieldSrc, KindsSrc, AST_SRC, KINDS_SRC}, codegen::{self, update, Mode}, project_root, Result, }; @@ -189,46 +189,30 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { quote!(impl ast::#trait_name for #name {}) }); - let methods = node.fields.iter().map(|(name, field)| { - let is_kw = name.ends_with("Kw"); - let method_name = match field { - FieldSrc::Shorthand => { - let name = if is_kw { &name[..name.len() - 2] } else { &name }; - format_ident!("{}", to_lower_snake_case(name)) - } - _ => format_ident!("{}", name), - }; - let ty = match field { - FieldSrc::Optional(ty) | FieldSrc::Many(ty) => ty, - FieldSrc::Shorthand => name, - }; - - let ty = format_ident!("{}", ty); + let methods = node.fields.iter().map(|field| { + let method_name = field.method_name(); + let ty = field.ty(); - match field { - FieldSrc::Many(_) => { + if field.is_many() { + quote! { + pub fn #method_name(&self) -> AstChildren<#ty> { + support::children(&self.syntax) + } + } + } else { + if let Some(token_kind) = field.token_kind() { quote! { - pub fn #method_name(&self) -> AstChildren<#ty> { - support::children(&self.syntax) + pub fn #method_name(&self) -> Option<#ty> { + support::token2(&self.syntax, #token_kind) } } - } - FieldSrc::Optional(_) | FieldSrc::Shorthand => { + } else { let is_token = token_kinds.contains(&ty.to_string()); if is_token { let method_name = format_ident!("{}_token", method_name); - if is_kw { - let token_kind = format_ident!("{}", to_upper_snake_case(name)); - quote! { - pub fn #method_name(&self) -> Option { - support::token2(&self.syntax, #token_kind) - } - } - } else { - quote! { - pub fn #method_name(&self) -> Option<#ty> { - support::token(&self.syntax) - } + quote! { + pub fn #method_name(&self) -> Option<#ty> { + support::token(&self.syntax) } } } else { @@ -351,6 +335,7 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { use crate::{ SyntaxNode, SyntaxToken, SyntaxKind::{self, *}, ast::{self, AstNode, AstChildren, support}, + T, }; use super::tokens::*; @@ -519,3 +504,40 @@ fn to_pascal_case(s: &str) -> String { } buf } + +impl Field<'_> { + fn is_many(&self) -> bool { + match self { + Field::Node { src: FieldSrc::Many(_), .. } => true, + _ => false, + } + } + fn token_kind(&self) -> Option { + let res = match self { + Field::Token(token) => { + let token = format_ident!("{}", token); + quote! { T![#token] } + } + _ => return None, + }; + Some(res) + } + fn method_name(&self) -> proc_macro2::Ident { + match self { + Field::Token(name) => format_ident!("{}_token", name), + Field::Node { name, src } => match src { + FieldSrc::Shorthand => format_ident!("{}", to_lower_snake_case(name)), + _ => format_ident!("{}", name), + }, + } + } + fn ty(&self) -> proc_macro2::Ident { + match self { + Field::Token(_) => format_ident!("SyntaxToken"), + Field::Node { name, src } => match src { + FieldSrc::Optional(ty) | FieldSrc::Many(ty) => format_ident!("{}", ty), + FieldSrc::Shorthand => format_ident!("{}", name), + }, + } + } +} -- cgit v1.2.3 From c8b4c36f8161d34c8145a49965efee4514275989 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Fri, 10 Apr 2020 10:11:05 +0200 Subject: Semicolon token --- xtask/src/ast_src.rs | 22 +++++++++++----------- xtask/src/codegen/gen_syntax.rs | 10 ++++++++-- 2 files changed, 19 insertions(+), 13 deletions(-) (limited to 'xtask') diff --git a/xtask/src/ast_src.rs b/xtask/src/ast_src.rs index 3da280551..1a6ee935f 100644 --- a/xtask/src/ast_src.rs +++ b/xtask/src/ast_src.rs @@ -319,7 +319,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { ParamList, RetType, body: BlockExpr, - Semi + T![;] } struct RetType { ThinArrow, TypeRef } @@ -327,7 +327,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct StructDef: VisibilityOwner, NameOwner, TypeParamsOwner, AttrsOwner, DocCommentsOwner { T![struct], FieldDefList, - Semi + T![;] } struct UnionDef: VisibilityOwner, NameOwner, TypeParamsOwner, AttrsOwner, DocCommentsOwner { @@ -368,7 +368,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct Module: VisibilityOwner, NameOwner, AttrsOwner, DocCommentsOwner { T![mod], ItemList, - Semi + T![;] } struct ItemList: ModuleItemOwner { @@ -382,7 +382,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { T![const], Eq, body: Expr, - Semi + T![;] } struct StaticDef: VisibilityOwner, NameOwner, TypeParamsOwner, AttrsOwner, DocCommentsOwner, TypeAscriptionOwner { @@ -390,7 +390,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { T![mut], Eq, body: Expr, - Semi + T![;] } struct TypeAliasDef: VisibilityOwner, NameOwner, TypeParamsOwner, AttrsOwner, DocCommentsOwner, TypeBoundsOwner { @@ -398,7 +398,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { T![type], Eq, TypeRef, - Semi + T![;] } struct ImplDef: TypeParamsOwner, AttrsOwner { @@ -416,7 +416,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct NeverType { Excl } struct PathType { Path } struct PointerType { Star, T![const], T![mut], TypeRef } - struct ArrayType { LBrack, TypeRef, Semi, Expr, RBrack } + struct ArrayType { LBrack, TypeRef, T![;], Expr, RBrack } struct SliceType { LBrack, TypeRef, RBrack } struct ReferenceType { Amp, Lifetime, T![mut], TypeRef } struct PlaceholderType { Underscore } @@ -426,7 +426,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct DynTraitType: TypeBoundsOwner { T![dyn] } struct TupleExpr: AttrsOwner { LParen, exprs: [Expr], RParen } - struct ArrayExpr: AttrsOwner { LBrack, exprs: [Expr], Semi, RBrack } + struct ArrayExpr: AttrsOwner { LBrack, exprs: [Expr], T![;], RBrack } struct ParenExpr: AttrsOwner { LParen, Expr, RParen } struct PathExpr { Path } struct LambdaExpr: AttrsOwner { @@ -520,7 +520,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct NameRef { NameRefToken } struct MacroCall: NameOwner, AttrsOwner,DocCommentsOwner { - Path, Excl, TokenTree, Semi + Path, Excl, TokenTree, T![;] } struct Attr { Pound, Excl, LBrack, Path, Eq, input: AttrInput, RBrack } struct TokenTree {} @@ -546,13 +546,13 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct WherePred: TypeBoundsOwner { Lifetime, TypeRef } struct WhereClause { T![where], predicates: [WherePred] } struct Abi { String } - struct ExprStmt: AttrsOwner { Expr, Semi } + struct ExprStmt: AttrsOwner { Expr, T![;] } struct LetStmt: AttrsOwner, TypeAscriptionOwner { T![let], Pat, Eq, initializer: Expr, - Semi, + T![;], } struct Condition { T![let], Pat, Eq, Expr } struct Block: AttrsOwner, ModuleItemOwner { diff --git a/xtask/src/codegen/gen_syntax.rs b/xtask/src/codegen/gen_syntax.rs index c4fb29bbf..26f541da1 100644 --- a/xtask/src/codegen/gen_syntax.rs +++ b/xtask/src/codegen/gen_syntax.rs @@ -515,7 +515,7 @@ impl Field<'_> { fn token_kind(&self) -> Option { let res = match self { Field::Token(token) => { - let token = format_ident!("{}", token); + let token: proc_macro2::TokenStream = token.parse().unwrap(); quote! { T![#token] } } _ => return None, @@ -524,7 +524,13 @@ impl Field<'_> { } fn method_name(&self) -> proc_macro2::Ident { match self { - Field::Token(name) => format_ident!("{}_token", name), + Field::Token(name) => { + let name = match *name { + ";" => "semicolon", + _ => name, + }; + format_ident!("{}_token", name) + } Field::Node { name, src } => match src { FieldSrc::Shorthand => format_ident!("{}", to_lower_snake_case(name)), _ => format_ident!("{}", name), -- cgit v1.2.3 From 1c5d8591952cbb017fd6679a1ef29a86e1499025 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Fri, 10 Apr 2020 10:18:43 +0200 Subject: Start replacing tokens --- xtask/src/ast_src.rs | 2 +- xtask/src/codegen/gen_syntax.rs | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'xtask') diff --git a/xtask/src/ast_src.rs b/xtask/src/ast_src.rs index 1a6ee935f..8c83450e6 100644 --- a/xtask/src/ast_src.rs +++ b/xtask/src/ast_src.rs @@ -322,7 +322,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { T![;] } - struct RetType { ThinArrow, TypeRef } + struct RetType { T![->], TypeRef } struct StructDef: VisibilityOwner, NameOwner, TypeParamsOwner, AttrsOwner, DocCommentsOwner { T![struct], diff --git a/xtask/src/codegen/gen_syntax.rs b/xtask/src/codegen/gen_syntax.rs index 26f541da1..b7698ee6e 100644 --- a/xtask/src/codegen/gen_syntax.rs +++ b/xtask/src/codegen/gen_syntax.rs @@ -345,6 +345,7 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { #(#displays)* }; + let ast = ast.to_string().replace("T ! [ ", "T![").replace(" ] )", "])"); let pretty = crate::reformat(ast)?; Ok(pretty) } @@ -527,6 +528,9 @@ impl Field<'_> { Field::Token(name) => { let name = match *name { ";" => "semicolon", + "->" => "thin_arrow", + "'{'" => "l_curly", + "'}'" => "r_curly", _ => name, }; format_ident!("{}_token", name) -- cgit v1.2.3 From 460c8bbdec8561a581d035f9767b7dd1ba36cfd7 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Fri, 10 Apr 2020 10:27:23 +0200 Subject: Curley tokens --- xtask/src/ast_src.rs | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) (limited to 'xtask') diff --git a/xtask/src/ast_src.rs b/xtask/src/ast_src.rs index 8c83450e6..1483dc08e 100644 --- a/xtask/src/ast_src.rs +++ b/xtask/src/ast_src.rs @@ -335,7 +335,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { RecordFieldDefList, } - struct RecordFieldDefList { LCurly, fields: [RecordFieldDef], RCurly } + struct RecordFieldDefList { T!['{'], fields: [RecordFieldDef], T!['}'] } struct RecordFieldDef: VisibilityOwner, NameOwner, AttrsOwner, DocCommentsOwner, TypeAscriptionOwner { } struct TupleFieldDefList { LParen, fields: [TupleFieldDef], RParen } @@ -348,9 +348,9 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { variant_list: EnumVariantList, } struct EnumVariantList { - LCurly, + T!['{'], variants: [EnumVariant], - RCurly + T!['}'] } struct EnumVariant: VisibilityOwner, NameOwner, DocCommentsOwner, AttrsOwner { FieldDefList, @@ -372,9 +372,9 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { } struct ItemList: ModuleItemOwner { - LCurly, + T!['{'], impl_items: [ImplItem], - RCurly + T!['}'] } struct ConstDef: VisibilityOwner, NameOwner, TypeParamsOwner, AttrsOwner, DocCommentsOwner, TypeAscriptionOwner { @@ -469,7 +469,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct Literal { LiteralToken } struct MatchExpr: AttrsOwner { T![match], Expr, MatchArmList } - struct MatchArmList: AttrsOwner { LCurly, arms: [MatchArm], RCurly } + struct MatchArmList: AttrsOwner { T!['{'], arms: [MatchArm], T!['}'] } struct MatchArm: AttrsOwner { pat: Pat, guard: MatchGuard, @@ -480,11 +480,11 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct RecordLit { Path, RecordFieldList} struct RecordFieldList { - LCurly, + T!['{'], fields: [RecordField], Dotdot, spread: Expr, - RCurly + T!['}'] } struct RecordField: AttrsOwner { NameRef, Colon, Expr } @@ -503,12 +503,12 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct RecordPat { RecordFieldPatList, Path } struct RecordFieldPatList { - LCurly, + T!['{'], pats: [RecordInnerPat], record_field_pats: [RecordFieldPat], bind_pats: [BindPat], Dotdot, - RCurly + T!['}'] } struct RecordFieldPat: AttrsOwner, NameOwner { Colon, Pat } @@ -556,10 +556,10 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { } struct Condition { T![let], Pat, Eq, Expr } struct Block: AttrsOwner, ModuleItemOwner { - LCurly, + T!['{'], statements: [Stmt], Expr, - RCurly, + T!['}'], } struct ParamList { LParen, @@ -580,7 +580,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { Path, Star, UseTreeList, Alias } struct Alias: NameOwner { T![as] } - struct UseTreeList { LCurly, use_trees: [UseTree], RCurly } + struct UseTreeList { T!['{'], use_trees: [UseTree], T!['}'] } struct ExternCrateItem: AttrsOwner, VisibilityOwner { T![extern], T![crate], NameRef, Alias, } @@ -619,9 +619,9 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { } struct ExternItemList: ModuleItemOwner { - LCurly, + T!['{'], extern_items: [ExternItem], - RCurly + T!['}'] } struct ExternBlock { @@ -769,8 +769,6 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { }, token_enums: &ast_enums! { - enum LeftDelimiter { LParen, LBrack, LCurly } - enum RightDelimiter { RParen, RBrack, RCurly } enum RangeSeparator { Dotdot, Dotdotdot, Dotdoteq} enum BinOp { -- cgit v1.2.3 From 548f562ddad3bca515b304f82e110aeb319591e3 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Fri, 10 Apr 2020 10:29:59 +0200 Subject: Other delimiters --- xtask/src/ast_src.rs | 36 ++++++++++++++++++------------------ xtask/src/codegen/gen_syntax.rs | 4 ++++ 2 files changed, 22 insertions(+), 18 deletions(-) (limited to 'xtask') diff --git a/xtask/src/ast_src.rs b/xtask/src/ast_src.rs index 1483dc08e..943c91e23 100644 --- a/xtask/src/ast_src.rs +++ b/xtask/src/ast_src.rs @@ -338,7 +338,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct RecordFieldDefList { T!['{'], fields: [RecordFieldDef], T!['}'] } struct RecordFieldDef: VisibilityOwner, NameOwner, AttrsOwner, DocCommentsOwner, TypeAscriptionOwner { } - struct TupleFieldDefList { LParen, fields: [TupleFieldDef], RParen } + struct TupleFieldDefList { T!['('], fields: [TupleFieldDef], T![')'] } struct TupleFieldDef: VisibilityOwner, AttrsOwner { TypeRef, } @@ -411,13 +411,13 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { ItemList, } - struct ParenType { LParen, TypeRef, RParen } - struct TupleType { LParen, fields: [TypeRef], RParen } + struct ParenType { T!['('], TypeRef, T![')'] } + struct TupleType { T!['('], fields: [TypeRef], T![')'] } struct NeverType { Excl } struct PathType { Path } struct PointerType { Star, T![const], T![mut], TypeRef } - struct ArrayType { LBrack, TypeRef, T![;], Expr, RBrack } - struct SliceType { LBrack, TypeRef, RBrack } + struct ArrayType { T!['['], TypeRef, T![;], Expr, T![']'] } + struct SliceType { T!['['], TypeRef, T![']'] } struct ReferenceType { Amp, Lifetime, T![mut], TypeRef } struct PlaceholderType { Underscore } struct FnPointerType { Abi, T![unsafe], T![fn], ParamList, RetType } @@ -425,9 +425,9 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct ImplTraitType: TypeBoundsOwner { T![impl] } struct DynTraitType: TypeBoundsOwner { T![dyn] } - struct TupleExpr: AttrsOwner { LParen, exprs: [Expr], RParen } - struct ArrayExpr: AttrsOwner { LBrack, exprs: [Expr], T![;], RBrack } - struct ParenExpr: AttrsOwner { LParen, Expr, RParen } + struct TupleExpr: AttrsOwner { T!['('], exprs: [Expr], T![')'] } + struct ArrayExpr: AttrsOwner { T!['['], exprs: [Expr], T![;], T![']'] } + struct ParenExpr: AttrsOwner { T!['('], Expr, T![')'] } struct PathExpr { Path } struct LambdaExpr: AttrsOwner { T![static], @@ -456,7 +456,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct MethodCallExpr: AttrsOwner, ArgListOwner { Expr, Dot, NameRef, TypeArgList, } - struct IndexExpr: AttrsOwner { LBrack, RBrack } + struct IndexExpr: AttrsOwner { T!['['], T![']'] } struct FieldExpr: AttrsOwner { Expr, Dot, NameRef } struct AwaitExpr: AttrsOwner { Expr, Dot, T![await] } struct TryExpr: AttrsOwner { T![try], Expr } @@ -489,14 +489,14 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct RecordField: AttrsOwner { NameRef, Colon, Expr } struct OrPat { pats: [Pat] } - struct ParenPat { LParen, Pat, RParen } + struct ParenPat { T!['('], Pat, T![')'] } struct RefPat { Amp, T![mut], Pat } struct BoxPat { T![box], Pat } struct BindPat: AttrsOwner, NameOwner { T![ref], T![mut], At, Pat } struct PlaceholderPat { Underscore } struct DotDotPat { Dotdot } struct PathPat { Path } - struct SlicePat { LBrack, args: [Pat], RBrack } + struct SlicePat { T!['['], args: [Pat], T![']'] } struct RangePat { RangeSeparator } struct LiteralPat { Literal } struct MacroPat { MacroCall } @@ -512,8 +512,8 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { } struct RecordFieldPat: AttrsOwner, NameOwner { Colon, Pat } - struct TupleStructPat { Path, LParen, args: [Pat], RParen } - struct TuplePat { LParen, args: [Pat], RParen } + struct TupleStructPat { Path, T!['('], args: [Pat], T![')'] } + struct TuplePat { T!['('], args: [Pat], T![')'] } struct Visibility { T![pub], T![super], T![self], T![crate] } struct Name { Ident } @@ -522,7 +522,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct MacroCall: NameOwner, AttrsOwner,DocCommentsOwner { Path, Excl, TokenTree, T![;] } - struct Attr { Pound, Excl, LBrack, Path, Eq, input: AttrInput, RBrack } + struct Attr { Pound, Excl, T!['['], Path, Eq, input: AttrInput, T![']'] } struct TokenTree {} struct TypeParamList { LAngle, @@ -562,10 +562,10 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { T!['}'], } struct ParamList { - LParen, + T!['('], SelfParam, params: [Param], - RParen + T![')'] } struct SelfParam: TypeAscriptionOwner, AttrsOwner { Amp, Lifetime, T![self] } struct Param: TypeAscriptionOwner, AttrsOwner { @@ -585,9 +585,9 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { T![extern], T![crate], NameRef, Alias, } struct ArgList { - LParen, + T!['('], args: [Expr], - RParen + T![')'] } struct Path { segment: PathSegment, diff --git a/xtask/src/codegen/gen_syntax.rs b/xtask/src/codegen/gen_syntax.rs index b7698ee6e..a903d1da7 100644 --- a/xtask/src/codegen/gen_syntax.rs +++ b/xtask/src/codegen/gen_syntax.rs @@ -531,6 +531,10 @@ impl Field<'_> { "->" => "thin_arrow", "'{'" => "l_curly", "'}'" => "r_curly", + "'('" => "l_paren", + "')'" => "r_paren", + "'['" => "l_brack", + "']'" => "r_brack", _ => name, }; format_ident!("{}_token", name) -- cgit v1.2.3 From 779f06ed77e868b9409a1724f736a045415d4922 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Fri, 10 Apr 2020 10:35:39 +0200 Subject: Convert more tokens --- xtask/src/ast_src.rs | 66 +++++++++++------------------------------ xtask/src/codegen/gen_syntax.rs | 2 ++ 2 files changed, 19 insertions(+), 49 deletions(-) (limited to 'xtask') diff --git a/xtask/src/ast_src.rs b/xtask/src/ast_src.rs index 943c91e23..f9cd12d0d 100644 --- a/xtask/src/ast_src.rs +++ b/xtask/src/ast_src.rs @@ -354,7 +354,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { } struct EnumVariant: VisibilityOwner, NameOwner, DocCommentsOwner, AttrsOwner { FieldDefList, - Eq, + T![=], Expr } @@ -380,7 +380,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct ConstDef: VisibilityOwner, NameOwner, TypeParamsOwner, AttrsOwner, DocCommentsOwner, TypeAscriptionOwner { T![default], T![const], - Eq, + T![=], body: Expr, T![;] } @@ -388,7 +388,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct StaticDef: VisibilityOwner, NameOwner, TypeParamsOwner, AttrsOwner, DocCommentsOwner, TypeAscriptionOwner { T![static], T![mut], - Eq, + T![=], body: Expr, T![;] } @@ -396,7 +396,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct TypeAliasDef: VisibilityOwner, NameOwner, TypeParamsOwner, AttrsOwner, DocCommentsOwner, TypeBoundsOwner { T![default], T![type], - Eq, + T![=], TypeRef, T![;] } @@ -406,14 +406,14 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { T![const], T![unsafe], T![impl], - Excl, + T![!], T![for], ItemList, } struct ParenType { T!['('], TypeRef, T![')'] } struct TupleType { T!['('], fields: [TypeRef], T![')'] } - struct NeverType { Excl } + struct NeverType { T![!] } struct PathType { Path } struct PointerType { Star, T![const], T![mut], TypeRef } struct ArrayType { T!['['], TypeRef, T![;], Expr, T![']'] } @@ -465,7 +465,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct PrefixExpr: AttrsOwner { PrefixOp, Expr } struct BoxExpr: AttrsOwner { T![box], Expr } struct RangeExpr: AttrsOwner { RangeOp } - struct BinExpr: AttrsOwner { BinOp } + struct BinExpr: AttrsOwner { /*BinOp*/ } struct Literal { LiteralToken } struct MatchExpr: AttrsOwner { T![match], Expr, MatchArmList } @@ -520,9 +520,9 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct NameRef { NameRefToken } struct MacroCall: NameOwner, AttrsOwner,DocCommentsOwner { - Path, Excl, TokenTree, T![;] + Path, T![!], TokenTree, T![;] } - struct Attr { Pound, Excl, T!['['], Path, Eq, input: AttrInput, T![']'] } + struct Attr { Pound, T![!], T!['['], Path, T![=], input: AttrInput, T![']'] } struct TokenTree {} struct TypeParamList { LAngle, @@ -533,11 +533,11 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { RAngle } struct TypeParam: NameOwner, AttrsOwner, TypeBoundsOwner { - Eq, + T![=], default_type: TypeRef, } struct ConstParam: NameOwner, AttrsOwner, TypeAscriptionOwner { - Eq, + T![=], default_val: Expr, } struct LifetimeParam: AttrsOwner { Lifetime} @@ -550,11 +550,11 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct LetStmt: AttrsOwner, TypeAscriptionOwner { T![let], Pat, - Eq, + T![=], initializer: Expr, T![;], } - struct Condition { T![let], Pat, Eq, Expr } + struct Condition { T![let], Pat, T![=], Expr } struct Block: AttrsOwner, ModuleItemOwner { T!['{'], statements: [Stmt], @@ -607,9 +607,9 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { RAngle } struct TypeArg { TypeRef } - struct AssocTypeArg : TypeBoundsOwner { NameRef, Eq, TypeRef } + struct AssocTypeArg : TypeBoundsOwner { NameRef, T![=], TypeRef } struct LifetimeArg { Lifetime } - struct ConstArg { Literal, Eq, BlockExpr } + struct ConstArg { Literal, T![=], BlockExpr } struct MacroItems: ModuleItemOwner{ } @@ -630,7 +630,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { } struct MetaItem { - Path, Eq, AttrInput, nested_meta_items: [MetaItem] + Path, T![=], AttrInput, nested_meta_items: [MetaItem] } struct MacroDef { @@ -771,41 +771,9 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { token_enums: &ast_enums! { enum RangeSeparator { Dotdot, Dotdotdot, Dotdoteq} - enum BinOp { - Pipepipe, - Ampamp, - Eqeq, - Neq, - Lteq, - Gteq, - LAngle, - RAngle, - Plus, - Star, - Minus, - Slash, - Percent, - Shl, - Shr, - Caret, - Pipe, - Amp, - Eq, - Pluseq, - Slasheq, - Stareq, - Percenteq, - Shreq, - Shleq, - Minuseq, - Pipeeq, - Ampeq, - Careteq, - } - enum PrefixOp { Minus, - Excl, + T![!], Star } diff --git a/xtask/src/codegen/gen_syntax.rs b/xtask/src/codegen/gen_syntax.rs index a903d1da7..4c9e447a3 100644 --- a/xtask/src/codegen/gen_syntax.rs +++ b/xtask/src/codegen/gen_syntax.rs @@ -535,6 +535,8 @@ impl Field<'_> { "')'" => "r_paren", "'['" => "l_brack", "']'" => "r_brack", + "=" => "eq", + "!" => "excl", _ => name, }; format_ident!("{}_token", name) -- cgit v1.2.3 From 8d71a6bf0ca51ae099a5b470afdb957bca321441 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Fri, 10 Apr 2020 11:49:13 +0200 Subject: Scale token generation back --- xtask/src/ast_src.rs | 114 +++++++++---------------- xtask/src/codegen.rs | 1 - xtask/src/codegen/gen_syntax.rs | 182 +++++----------------------------------- 3 files changed, 59 insertions(+), 238 deletions(-) (limited to 'xtask') diff --git a/xtask/src/ast_src.rs b/xtask/src/ast_src.rs index f9cd12d0d..554afc76a 100644 --- a/xtask/src/ast_src.rs +++ b/xtask/src/ast_src.rs @@ -227,7 +227,6 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc { pub(crate) struct AstSrc<'a> { pub(crate) nodes: &'a [AstNodeSrc<'a>], pub(crate) enums: &'a [AstEnumSrc<'a>], - pub(crate) token_enums: &'a [AstEnumSrc<'a>], } pub(crate) struct AstNodeSrc<'a> { @@ -415,11 +414,11 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct TupleType { T!['('], fields: [TypeRef], T![')'] } struct NeverType { T![!] } struct PathType { Path } - struct PointerType { Star, T![const], T![mut], TypeRef } + struct PointerType { T![*], T![const], T![mut], TypeRef } struct ArrayType { T!['['], TypeRef, T![;], Expr, T![']'] } struct SliceType { T!['['], TypeRef, T![']'] } - struct ReferenceType { Amp, Lifetime, T![mut], TypeRef } - struct PlaceholderType { Underscore } + struct ReferenceType { T![&], T![lifetime], T![mut], TypeRef } + struct PlaceholderType { T![_] } struct FnPointerType { Abi, T![unsafe], T![fn], ParamList, RetType } struct ForType { T![for], TypeParamList, TypeRef } struct ImplTraitType: TypeBoundsOwner { T![impl] } @@ -447,33 +446,33 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { iterable: Expr, } struct WhileExpr: AttrsOwner, LoopBodyOwner { T![while], Condition } - struct ContinueExpr: AttrsOwner { T![continue], Lifetime } - struct BreakExpr: AttrsOwner { T![break], Lifetime, Expr } - struct Label { Lifetime } + struct ContinueExpr: AttrsOwner { T![continue], T![lifetime] } + struct BreakExpr: AttrsOwner { T![break], T![lifetime], Expr } + struct Label { T![lifetime] } struct BlockExpr: AttrsOwner { Label, T![unsafe], Block } struct ReturnExpr: AttrsOwner { Expr } struct CallExpr: ArgListOwner { Expr } struct MethodCallExpr: AttrsOwner, ArgListOwner { - Expr, Dot, NameRef, TypeArgList, + Expr, T![.], NameRef, TypeArgList, } struct IndexExpr: AttrsOwner { T!['['], T![']'] } - struct FieldExpr: AttrsOwner { Expr, Dot, NameRef } - struct AwaitExpr: AttrsOwner { Expr, Dot, T![await] } + struct FieldExpr: AttrsOwner { Expr, T![.], NameRef } + struct AwaitExpr: AttrsOwner { Expr, T![.], T![await] } struct TryExpr: AttrsOwner { T![try], Expr } struct CastExpr: AttrsOwner { Expr, T![as], TypeRef } - struct RefExpr: AttrsOwner { Amp, T![raw], T![mut], Expr } - struct PrefixExpr: AttrsOwner { PrefixOp, Expr } + struct RefExpr: AttrsOwner { T![&], T![raw], T![mut], Expr } + struct PrefixExpr: AttrsOwner { /*PrefixOp,*/ Expr } struct BoxExpr: AttrsOwner { T![box], Expr } - struct RangeExpr: AttrsOwner { RangeOp } + struct RangeExpr: AttrsOwner { /*RangeOp*/ } struct BinExpr: AttrsOwner { /*BinOp*/ } - struct Literal { LiteralToken } + struct Literal { /*LiteralToken*/ } struct MatchExpr: AttrsOwner { T![match], Expr, MatchArmList } struct MatchArmList: AttrsOwner { T!['{'], arms: [MatchArm], T!['}'] } struct MatchArm: AttrsOwner { pat: Pat, guard: MatchGuard, - FatArrow, + T![=>], Expr, } struct MatchGuard { T![if], Expr } @@ -482,22 +481,22 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { struct RecordFieldList { T!['{'], fields: [RecordField], - Dotdot, + T![..], spread: Expr, T!['}'] } - struct RecordField: AttrsOwner { NameRef, Colon, Expr } + struct RecordField: AttrsOwner { NameRef, T![:], Expr } struct OrPat { pats: [Pat] } struct ParenPat { T!['('], Pat, T![')'] } - struct RefPat { Amp, T![mut], Pat } + struct RefPat { T![&], T![mut], Pat } struct BoxPat { T![box], Pat } - struct BindPat: AttrsOwner, NameOwner { T![ref], T![mut], At, Pat } - struct PlaceholderPat { Underscore } - struct DotDotPat { Dotdot } + struct BindPat: AttrsOwner, NameOwner { T![ref], T![mut], T![@], Pat } + struct PlaceholderPat { T![_] } + struct DotDotPat { T![..] } struct PathPat { Path } struct SlicePat { T!['['], args: [Pat], T![']'] } - struct RangePat { RangeSeparator } + struct RangePat { /*RangeSeparator*/ } struct LiteralPat { Literal } struct MacroPat { MacroCall } @@ -507,30 +506,30 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { pats: [RecordInnerPat], record_field_pats: [RecordFieldPat], bind_pats: [BindPat], - Dotdot, + T![..], T!['}'] } - struct RecordFieldPat: AttrsOwner, NameOwner { Colon, Pat } + struct RecordFieldPat: AttrsOwner, NameOwner { T![:], Pat } struct TupleStructPat { Path, T!['('], args: [Pat], T![')'] } struct TuplePat { T!['('], args: [Pat], T![')'] } struct Visibility { T![pub], T![super], T![self], T![crate] } - struct Name { Ident } - struct NameRef { NameRefToken } + struct Name { T![ident] } + struct NameRef { /*NameRefToken*/ } struct MacroCall: NameOwner, AttrsOwner,DocCommentsOwner { Path, T![!], TokenTree, T![;] } - struct Attr { Pound, T![!], T!['['], Path, T![=], input: AttrInput, T![']'] } + struct Attr { T![#], T![!], T!['['], Path, T![=], input: AttrInput, T![']'] } struct TokenTree {} struct TypeParamList { - LAngle, + T![<], generic_params: [GenericParam], type_params: [TypeParam], lifetime_params: [LifetimeParam], const_params: [ConstParam], - RAngle + T![>] } struct TypeParam: NameOwner, AttrsOwner, TypeBoundsOwner { T![=], @@ -540,12 +539,12 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { T![=], default_val: Expr, } - struct LifetimeParam: AttrsOwner { Lifetime} - struct TypeBound { Lifetime, /* Question, */ T![const], /* Question, */ TypeRef} + struct LifetimeParam: AttrsOwner { T![lifetime] } + struct TypeBound { T![lifetime], /* Question, */ T![const], /* Question, */ TypeRef} struct TypeBoundList { bounds: [TypeBound] } - struct WherePred: TypeBoundsOwner { Lifetime, TypeRef } + struct WherePred: TypeBoundsOwner { T![lifetime], TypeRef } struct WhereClause { T![where], predicates: [WherePred] } - struct Abi { String } + struct Abi { /*String*/ } struct ExprStmt: AttrsOwner { Expr, T![;] } struct LetStmt: AttrsOwner, TypeAscriptionOwner { T![let], @@ -567,17 +566,17 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { params: [Param], T![')'] } - struct SelfParam: TypeAscriptionOwner, AttrsOwner { Amp, Lifetime, T![self] } + struct SelfParam: TypeAscriptionOwner, AttrsOwner { T![&], T![lifetime], T![self] } struct Param: TypeAscriptionOwner, AttrsOwner { Pat, - Dotdotdot + T![...] } struct UseItem: AttrsOwner, VisibilityOwner { T![use], UseTree, } struct UseTree { - Path, Star, UseTreeList, Alias + Path, T![*], UseTreeList, Alias } struct Alias: NameOwner { T![as] } struct UseTreeList { T!['{'], use_trees: [UseTree], T!['}'] } @@ -594,21 +593,21 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { qualifier: Path, } struct PathSegment { - Coloncolon, LAngle, NameRef, TypeArgList, ParamList, RetType, PathType, RAngle + T![::], T![<], NameRef, TypeArgList, ParamList, RetType, PathType, T![>] } struct TypeArgList { - Coloncolon, - LAngle, + T![::], + T![<], generic_args: [GenericArg], type_args: [TypeArg], lifetime_args: [LifetimeArg], assoc_type_args: [AssocTypeArg], const_args: [ConstArg], - RAngle + T![>] } struct TypeArg { TypeRef } struct AssocTypeArg : TypeBoundsOwner { NameRef, T![=], TypeRef } - struct LifetimeArg { Lifetime } + struct LifetimeArg { T![lifetime] } struct ConstArg { Literal, T![=], BlockExpr } struct MacroItems: ModuleItemOwner{ } @@ -767,37 +766,4 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { TupleFieldDefList, } }, - - token_enums: &ast_enums! { - enum RangeSeparator { Dotdot, Dotdotdot, Dotdoteq} - - enum PrefixOp { - Minus, - T![!], - Star - } - - enum RangeOp { - Dotdot, - Dotdoteq - } - - enum LiteralToken { - IntNumber, - FloatNumber, - String, - RawString, - // TrueKw, - // FalseKw, - ByteString, - RawByteString, - Char, - Byte - } - - enum NameRefToken { - Ident, - IntNumber - } - }, }; diff --git a/xtask/src/codegen.rs b/xtask/src/codegen.rs index 678b40133..e6ba2009c 100644 --- a/xtask/src/codegen.rs +++ b/xtask/src/codegen.rs @@ -24,7 +24,6 @@ const ERR_INLINE_TESTS_DIR: &str = "crates/ra_syntax/test_data/parser/inline/err const SYNTAX_KINDS: &str = "crates/ra_parser/src/syntax_kind/generated.rs"; const AST_NODES: &str = "crates/ra_syntax/src/ast/generated/nodes.rs"; -const AST_TOKENS: &str = "crates/ra_syntax/src/ast/generated/tokens.rs"; const ASSISTS_DIR: &str = "crates/ra_assists/src/handlers"; const ASSISTS_TESTS: &str = "crates/ra_assists/src/doc_tests/generated.rs"; diff --git a/xtask/src/codegen/gen_syntax.rs b/xtask/src/codegen/gen_syntax.rs index 4c9e447a3..ce18f2b8f 100644 --- a/xtask/src/codegen/gen_syntax.rs +++ b/xtask/src/codegen/gen_syntax.rs @@ -3,10 +3,7 @@ //! Specifically, it generates the `SyntaxKind` enum and a number of newtype //! wrappers around `SyntaxNode` which implement `ra_syntax::AstNode`. -use std::{ - borrow::Cow, - collections::{BTreeSet, HashSet}, -}; +use std::collections::{BTreeSet, HashSet}; use proc_macro2::{Punct, Spacing}; use quote::{format_ident, quote}; @@ -26,10 +23,6 @@ pub fn generate_syntax(mode: Mode) -> Result<()> { let contents = generate_nodes(KINDS_SRC, AST_SRC)?; update(ast_nodes_file.as_path(), &contents, mode)?; - let ast_tokens_file = project_root().join(codegen::AST_TOKENS); - let contents = generate_tokens(KINDS_SRC, AST_SRC)?; - update(ast_tokens_file.as_path(), &contents, mode)?; - Ok(()) } @@ -40,147 +33,7 @@ struct ElementKinds { has_tokens: bool, } -fn generate_tokens(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { - let all_token_kinds: Vec<_> = kinds - .punct - .into_iter() - .map(|(_, kind)| kind) - .copied() - .map(|x| x.into()) - .chain( - kinds - .keywords - .into_iter() - .chain(kinds.contextual_keywords.into_iter()) - .map(|name| Cow::Owned(format!("{}_KW", to_upper_snake_case(&name)))), - ) - .chain(kinds.literals.into_iter().copied().map(|x| x.into())) - .chain(kinds.tokens.into_iter().copied().map(|x| x.into())) - .collect(); - - let tokens = all_token_kinds.iter().filter_map(|kind_str| { - if kind_str.ends_with("_KW") { - return None; - } - let kind_str = &**kind_str; - let kind = format_ident!("{}", kind_str); - let name = format_ident!("{}", to_pascal_case(kind_str)); - let res = quote! { - #[derive(Debug, Clone, PartialEq, Eq, Hash)] - pub struct #name { - pub(crate) syntax: SyntaxToken, - } - - impl std::fmt::Display for #name { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - std::fmt::Display::fmt(&self.syntax, f) - } - } - - impl AstToken for #name { - fn can_cast(kind: SyntaxKind) -> bool { kind == #kind } - fn cast(syntax: SyntaxToken) -> Option { - if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None } - } - fn syntax(&self) -> &SyntaxToken { &self.syntax } - } - }; - Some(res) - }); - - let enums = grammar.token_enums.iter().map(|en| { - let variants = en.variants.iter().map(|var| format_ident!("{}", var)).collect::>(); - let name = format_ident!("{}", en.name); - let kinds = variants - .iter() - .map(|name| format_ident!("{}", to_upper_snake_case(&name.to_string()))) - .collect::>(); - assert!(en.traits.is_empty()); - - quote! { - #[derive(Debug, Clone, PartialEq, Eq, Hash)] - pub enum #name { - #(#variants(#variants),)* - } - - #( - impl From<#variants> for #name { - fn from(node: #variants) -> #name { - #name::#variants(node) - } - } - )* - - impl std::fmt::Display for #name { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - std::fmt::Display::fmt(self.syntax(), f) - } - } - - impl AstToken for #name { - fn can_cast(kind: SyntaxKind) -> bool { - match kind { - #(#kinds)|* => true, - _ => false, - } - } - fn cast(syntax: SyntaxToken) -> Option { - let res = match syntax.kind() { - #( - #kinds => #name::#variants(#variants { syntax }), - )* - _ => return None, - }; - Some(res) - } - fn syntax(&self) -> &SyntaxToken { - match self { - #( - #name::#variants(it) => &it.syntax, - )* - } - } - } - } - }); - - crate::reformat(quote! { - use crate::{SyntaxToken, SyntaxKind::{self, *}, ast::AstToken}; - - #(#tokens)* - #(#enums)* - }) -} - fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { - let all_token_kinds: Vec<_> = kinds - .punct - .into_iter() - .map(|(_, kind)| kind) - .copied() - .map(|x| x.into()) - .chain( - kinds - .keywords - .into_iter() - .chain(kinds.contextual_keywords.into_iter()) - .map(|name| Cow::Owned(format!("{}_KW", to_upper_snake_case(&name)))), - ) - .chain(kinds.literals.into_iter().copied().map(|x| x.into())) - .chain(kinds.tokens.into_iter().copied().map(|x| x.into())) - .collect(); - - let mut token_kinds = HashSet::new(); - for kind in &all_token_kinds { - let kind = &**kind; - let name = to_pascal_case(kind); - token_kinds.insert(name); - } - - for en in grammar.token_enums { - token_kinds.insert(en.name.to_string()); - } - let nodes = grammar.nodes.iter().map(|node| { let name = format_ident!("{}", node.name); let kind = format_ident!("{}", to_upper_snake_case(&name.to_string())); @@ -207,19 +60,9 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { } } } else { - let is_token = token_kinds.contains(&ty.to_string()); - if is_token { - let method_name = format_ident!("{}_token", method_name); - quote! { - pub fn #method_name(&self) -> Option<#ty> { - support::token(&self.syntax) - } - } - } else { - quote! { - pub fn #method_name(&self) -> Option<#ty> { - support::child(&self.syntax) - } + quote! { + pub fn #method_name(&self) -> Option<#ty> { + support::child(&self.syntax) } } } @@ -338,8 +181,6 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { T, }; - use super::tokens::*; - #(#nodes)* #(#enums)* #(#displays)* @@ -456,6 +297,8 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> Result { macro_rules! T { #((#punctuation_values) => { $crate::SyntaxKind::#punctuation };)* #((#all_keywords_idents) => { $crate::SyntaxKind::#all_keywords };)* + (lifetime) => { $crate::SyntaxKind::LIFETIME }; + (ident) => { $crate::SyntaxKind::IDENT }; } }; @@ -535,8 +378,21 @@ impl Field<'_> { "')'" => "r_paren", "'['" => "l_brack", "']'" => "r_brack", + "<" => "l_angle", + ">" => "r_angle", "=" => "eq", "!" => "excl", + "*" => "star", + "&" => "amp", + "_" => "underscore", + "." => "dot", + ".." => "dotdot", + "..." => "dotdotdot", + "=>" => "fat_arrow", + "@" => "at", + ":" => "colon", + "::" => "coloncolon", + "#" => "pound", _ => name, }; format_ident!("{}_token", name) -- cgit v1.2.3 From 4560fe2abffde05e6ceb084e6d42207e0ce84b68 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Fri, 10 Apr 2020 15:53:09 +0200 Subject: Generate only minimal set of ineresting tokens --- xtask/src/ast_src.rs | 2 ++ xtask/src/codegen.rs | 1 + xtask/src/codegen/gen_syntax.rs | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+) (limited to 'xtask') diff --git a/xtask/src/ast_src.rs b/xtask/src/ast_src.rs index 554afc76a..7a20d9991 100644 --- a/xtask/src/ast_src.rs +++ b/xtask/src/ast_src.rs @@ -225,6 +225,7 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc { }; pub(crate) struct AstSrc<'a> { + pub(crate) tokens: &'a [&'a str], pub(crate) nodes: &'a [AstNodeSrc<'a>], pub(crate) enums: &'a [AstEnumSrc<'a>], } @@ -303,6 +304,7 @@ macro_rules! ast_enums { } pub(crate) const AST_SRC: AstSrc = AstSrc { + tokens: &["Whitespace", "Comment", "String", "RawString"], nodes: &ast_nodes! { struct SourceFile: ModuleItemOwner, AttrsOwner { modules: [Module], diff --git a/xtask/src/codegen.rs b/xtask/src/codegen.rs index e6ba2009c..678b40133 100644 --- a/xtask/src/codegen.rs +++ b/xtask/src/codegen.rs @@ -24,6 +24,7 @@ const ERR_INLINE_TESTS_DIR: &str = "crates/ra_syntax/test_data/parser/inline/err const SYNTAX_KINDS: &str = "crates/ra_parser/src/syntax_kind/generated.rs"; const AST_NODES: &str = "crates/ra_syntax/src/ast/generated/nodes.rs"; +const AST_TOKENS: &str = "crates/ra_syntax/src/ast/generated/tokens.rs"; const ASSISTS_DIR: &str = "crates/ra_assists/src/handlers"; const ASSISTS_TESTS: &str = "crates/ra_assists/src/doc_tests/generated.rs"; diff --git a/xtask/src/codegen/gen_syntax.rs b/xtask/src/codegen/gen_syntax.rs index ce18f2b8f..fa48853d2 100644 --- a/xtask/src/codegen/gen_syntax.rs +++ b/xtask/src/codegen/gen_syntax.rs @@ -19,6 +19,10 @@ pub fn generate_syntax(mode: Mode) -> Result<()> { let syntax_kinds = generate_syntax_kinds(KINDS_SRC)?; update(syntax_kinds_file.as_path(), &syntax_kinds, mode)?; + let ast_tokens_file = project_root().join(codegen::AST_TOKENS); + let contents = generate_tokens(KINDS_SRC, AST_SRC)?; + update(ast_tokens_file.as_path(), &contents, mode)?; + let ast_nodes_file = project_root().join(codegen::AST_NODES); let contents = generate_nodes(KINDS_SRC, AST_SRC)?; update(ast_nodes_file.as_path(), &contents, mode)?; @@ -33,6 +37,37 @@ struct ElementKinds { has_tokens: bool, } +fn generate_tokens(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { + let tokens = grammar.tokens.iter().map(|token| { + let name = format_ident!("{}", token); + let kind = format_ident!("{}", to_upper_snake_case(token)); + quote! { + #[derive(Debug, Clone, PartialEq, Eq, Hash)] + pub struct #name { + pub(crate) syntax: SyntaxToken, + } + impl std::fmt::Display for #name { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + std::fmt::Display::fmt(&self.syntax, f) + } + } + impl AstToken for #name { + fn can_cast(kind: SyntaxKind) -> bool { kind == #kind } + fn cast(syntax: SyntaxToken) -> Option { + if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None } + } + fn syntax(&self) -> &SyntaxToken { &self.syntax } + } + } + }); + + let pretty = crate::reformat(quote! { + use crate::{SyntaxKind::{self, *}, SyntaxToken, ast::AstToken}; + #(#tokens)* + })?; + Ok(pretty) +} + fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { let nodes = grammar.nodes.iter().map(|node| { let name = format_ident!("{}", node.name); -- cgit v1.2.3 From e0f02d233fa3e26e4f10bffacbaef11b6bcb0ada Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Fri, 10 Apr 2020 15:54:05 +0200 Subject: Remove dead code --- xtask/src/codegen/gen_syntax.rs | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) (limited to 'xtask') diff --git a/xtask/src/codegen/gen_syntax.rs b/xtask/src/codegen/gen_syntax.rs index fa48853d2..7dc7f396c 100644 --- a/xtask/src/codegen/gen_syntax.rs +++ b/xtask/src/codegen/gen_syntax.rs @@ -3,7 +3,7 @@ //! Specifically, it generates the `SyntaxKind` enum and a number of newtype //! wrappers around `SyntaxNode` which implement `ra_syntax::AstNode`. -use std::collections::{BTreeSet, HashSet}; +use std::collections::HashSet; use proc_macro2::{Punct, Spacing}; use quote::{format_ident, quote}; @@ -20,7 +20,7 @@ pub fn generate_syntax(mode: Mode) -> Result<()> { update(syntax_kinds_file.as_path(), &syntax_kinds, mode)?; let ast_tokens_file = project_root().join(codegen::AST_TOKENS); - let contents = generate_tokens(KINDS_SRC, AST_SRC)?; + let contents = generate_tokens(AST_SRC)?; update(ast_tokens_file.as_path(), &contents, mode)?; let ast_nodes_file = project_root().join(codegen::AST_NODES); @@ -30,14 +30,7 @@ pub fn generate_syntax(mode: Mode) -> Result<()> { Ok(()) } -#[derive(Debug, Default, Clone)] -struct ElementKinds { - kinds: BTreeSet, - has_nodes: bool, - has_tokens: bool, -} - -fn generate_tokens(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { +fn generate_tokens(grammar: AstSrc<'_>) -> Result { let tokens = grammar.tokens.iter().map(|token| { let name = format_ident!("{}", token); let kind = format_ident!("{}", to_upper_snake_case(token)); @@ -91,7 +84,7 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { if let Some(token_kind) = field.token_kind() { quote! { pub fn #method_name(&self) -> Option<#ty> { - support::token2(&self.syntax, #token_kind) + support::token(&self.syntax, #token_kind) } } } else { -- cgit v1.2.3 From d4332760d81c5575dd04b39275c4979b59bb9fc4 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Fri, 10 Apr 2020 16:00:24 +0200 Subject: Better readability --- xtask/src/codegen/gen_syntax.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'xtask') diff --git a/xtask/src/codegen/gen_syntax.rs b/xtask/src/codegen/gen_syntax.rs index 7dc7f396c..ff290bb33 100644 --- a/xtask/src/codegen/gen_syntax.rs +++ b/xtask/src/codegen/gen_syntax.rs @@ -57,7 +57,8 @@ fn generate_tokens(grammar: AstSrc<'_>) -> Result { let pretty = crate::reformat(quote! { use crate::{SyntaxKind::{self, *}, SyntaxToken, ast::AstToken}; #(#tokens)* - })?; + })? + .replace("#[derive", "\n#[derive"); Ok(pretty) } @@ -215,7 +216,7 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { }; let ast = ast.to_string().replace("T ! [ ", "T![").replace(" ] )", "])"); - let pretty = crate::reformat(ast)?; + let pretty = crate::reformat(ast)?.replace("#[derive", "\n#[derive"); Ok(pretty) } -- cgit v1.2.3 From 5c5bde47fb759440d007c90fd83021de538120b8 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Fri, 10 Apr 2020 17:06:57 +0200 Subject: Rename some tokens --- xtask/src/ast_src.rs | 18 +++++++++--------- xtask/src/codegen/gen_syntax.rs | 8 ++++---- 2 files changed, 13 insertions(+), 13 deletions(-) (limited to 'xtask') diff --git a/xtask/src/ast_src.rs b/xtask/src/ast_src.rs index 7a20d9991..8c0b5f5a8 100644 --- a/xtask/src/ast_src.rs +++ b/xtask/src/ast_src.rs @@ -11,7 +11,7 @@ pub(crate) struct KindsSrc<'a> { pub(crate) const KINDS_SRC: KindsSrc = KindsSrc { punct: &[ - (";", "SEMI"), + (";", "SEMICOLON"), (",", "COMMA"), ("(", "L_PAREN"), (")", "R_PAREN"), @@ -35,15 +35,15 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc { ("%", "PERCENT"), ("_", "UNDERSCORE"), (".", "DOT"), - ("..", "DOTDOT"), - ("...", "DOTDOTDOT"), - ("..=", "DOTDOTEQ"), + ("..", "DOT2"), + ("...", "DOT3"), + ("..=", "DOT2EQ"), (":", "COLON"), - ("::", "COLONCOLON"), + ("::", "COLON2"), ("=", "EQ"), - ("==", "EQEQ"), + ("==", "EQ2"), ("=>", "FAT_ARROW"), - ("!", "EXCL"), + ("!", "BANG"), ("!=", "NEQ"), ("-", "MINUS"), ("->", "THIN_ARROW"), @@ -57,8 +57,8 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc { ("/=", "SLASHEQ"), ("*=", "STAREQ"), ("%=", "PERCENTEQ"), - ("&&", "AMPAMP"), - ("||", "PIPEPIPE"), + ("&&", "AMP2"), + ("||", "PIPE2"), ("<<", "SHL"), (">>", "SHR"), ("<<=", "SHLEQ"), diff --git a/xtask/src/codegen/gen_syntax.rs b/xtask/src/codegen/gen_syntax.rs index ff290bb33..ec1f6ad8a 100644 --- a/xtask/src/codegen/gen_syntax.rs +++ b/xtask/src/codegen/gen_syntax.rs @@ -324,10 +324,10 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> Result { #[macro_export] macro_rules! T { - #((#punctuation_values) => { $crate::SyntaxKind::#punctuation };)* - #((#all_keywords_idents) => { $crate::SyntaxKind::#all_keywords };)* - (lifetime) => { $crate::SyntaxKind::LIFETIME }; - (ident) => { $crate::SyntaxKind::IDENT }; + #([#punctuation_values] => { $crate::SyntaxKind::#punctuation };)* + #([#all_keywords_idents] => { $crate::SyntaxKind::#all_keywords };)* + [lifetime] => { $crate::SyntaxKind::LIFETIME }; + [ident] => { $crate::SyntaxKind::IDENT }; } }; -- cgit v1.2.3 From c476742f47dd905fa366e62f480e3f656c62e66c Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Fri, 10 Apr 2020 17:47:49 +0200 Subject: Simplify --- xtask/src/ast_src.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'xtask') diff --git a/xtask/src/ast_src.rs b/xtask/src/ast_src.rs index 8c0b5f5a8..69cba9168 100644 --- a/xtask/src/ast_src.rs +++ b/xtask/src/ast_src.rs @@ -568,7 +568,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc { params: [Param], T![')'] } - struct SelfParam: TypeAscriptionOwner, AttrsOwner { T![&], T![lifetime], T![self] } + struct SelfParam: TypeAscriptionOwner, AttrsOwner { T![&], T![mut], T![lifetime], T![self] } struct Param: TypeAscriptionOwner, AttrsOwner { Pat, T![...] -- cgit v1.2.3 From af04d45d32b5d9bdf949beb17152ef78dd4a0523 Mon Sep 17 00:00:00 2001 From: Chris Hopman Date: Fri, 10 Apr 2020 13:41:11 -0700 Subject: Change missing impl assist to use todo!() instead of unimplemented() todo!() "Indicates unfinished code" (https://doc.rust-lang.org/std/macro.todo.html) Rust documentation provides further clarification: > The difference between unimplemented! and todo! is that while todo! > conveys an intent of implementing the functionality later and the > message is "not yet implemented", unimplemented! makes no such claims. todo!() seems more appropriate for assists that insert missing impls. --- xtask/tests/tidy-tests/main.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'xtask') diff --git a/xtask/tests/tidy-tests/main.rs b/xtask/tests/tidy-tests/main.rs index 80911a68e..4f525fcd0 100644 --- a/xtask/tests/tidy-tests/main.rs +++ b/xtask/tests/tidy-tests/main.rs @@ -20,7 +20,16 @@ fn rust_files_are_tidy() { } fn check_todo(path: &Path, text: &str) { - if path.ends_with("tests/cli.rs") { + let whitelist = &[ + // This file itself is whitelisted since this test itself contains matches. + "tests/cli.rs", + // Some of our assists generate `todo!()` so those files are whitelisted. + "doc_tests/generated.rs", + "handlers/add_missing_impl_members.rs", + // To support generating `todo!()` in assists, we have `expr_todo()` in ast::make. + "ast/make.rs", + ]; + if whitelist.iter().any(|p| path.ends_with(p)) { return; } if text.contains("TODO") || text.contains("TOOD") || text.contains("todo!") { -- cgit v1.2.3