From 6379061e21dffd0faf6225484a1550a0ecc523cf Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Thu, 9 Apr 2020 15:47:48 +0200 Subject: Cleanup import --- xtask/src/codegen/gen_syntax.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'xtask/src/codegen') diff --git a/xtask/src/codegen/gen_syntax.rs b/xtask/src/codegen/gen_syntax.rs index 6dae93aa2..af7d4ffac 100644 --- a/xtask/src/codegen/gen_syntax.rs +++ b/xtask/src/codegen/gen_syntax.rs @@ -3,10 +3,13 @@ //! Specifically, it generates the `SyntaxKind` enum and a number of newtype //! wrappers around `SyntaxNode` which implement `ra_syntax::AstNode`. +use std::{ + borrow::Cow, + collections::{BTreeSet, HashMap, HashSet}, +}; + use proc_macro2::{Punct, Spacing}; use quote::{format_ident, quote}; -use std::borrow::Cow; -use std::collections::{BTreeSet, HashMap, HashSet}; use crate::{ ast_src::{AstSrc, FieldSrc, KindsSrc, AST_SRC, KINDS_SRC}, -- cgit v1.2.3 From d61932ef7e74f3d9f7863c33c6647c2c52538ed5 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Thu, 9 Apr 2020 16:17:18 +0200 Subject: Prepare for spliting generated into tokens and nodes --- xtask/src/codegen/gen_syntax.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'xtask/src/codegen') diff --git a/xtask/src/codegen/gen_syntax.rs b/xtask/src/codegen/gen_syntax.rs index af7d4ffac..c2734d638 100644 --- a/xtask/src/codegen/gen_syntax.rs +++ b/xtask/src/codegen/gen_syntax.rs @@ -22,7 +22,7 @@ pub fn generate_syntax(mode: Mode) -> Result<()> { let syntax_kinds = generate_syntax_kinds(KINDS_SRC)?; update(syntax_kinds_file.as_path(), &syntax_kinds, mode)?; - let ast_file = project_root().join(codegen::AST); + let ast_file = project_root().join(codegen::AST_NODES); let ast = generate_ast(KINDS_SRC, AST_SRC)?; update(ast_file.as_path(), &ast, mode)?; -- cgit v1.2.3 From 59b6b2278b7f7d2089b5caf6fcfd57528d9aafd0 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Thu, 9 Apr 2020 16:25:06 +0200 Subject: Start ast/generated/tokens --- xtask/src/codegen/gen_syntax.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'xtask/src/codegen') diff --git a/xtask/src/codegen/gen_syntax.rs b/xtask/src/codegen/gen_syntax.rs index c2734d638..75bcd1973 100644 --- a/xtask/src/codegen/gen_syntax.rs +++ b/xtask/src/codegen/gen_syntax.rs @@ -22,9 +22,13 @@ pub fn generate_syntax(mode: Mode) -> Result<()> { let syntax_kinds = generate_syntax_kinds(KINDS_SRC)?; update(syntax_kinds_file.as_path(), &syntax_kinds, mode)?; - let ast_file = project_root().join(codegen::AST_NODES); - let ast = generate_ast(KINDS_SRC, AST_SRC)?; - update(ast_file.as_path(), &ast, mode)?; + let ast_nodes_file = project_root().join(codegen::AST_NODES); + let contents = generate_ast(KINDS_SRC, AST_SRC)?; + update(ast_nodes_file.as_path(), &contents, mode)?; + + let ast_tokens_file = project_root().join(codegen::AST_TOKENS); + let contents = "//! Generated file, do not edit by hand, see `xtask/src/codegen`"; + update(ast_tokens_file.as_path(), &contents, mode)?; Ok(()) } -- cgit v1.2.3 From c80795e27491527487f97368be54b4467eb8b41e Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Thu, 9 Apr 2020 17:47:46 +0200 Subject: Move generated tokens to a separate file --- xtask/src/codegen/gen_syntax.rs | 98 +++++++++++++++++++++++++---------------- 1 file changed, 61 insertions(+), 37 deletions(-) (limited to 'xtask/src/codegen') diff --git a/xtask/src/codegen/gen_syntax.rs b/xtask/src/codegen/gen_syntax.rs index 75bcd1973..683901a0a 100644 --- a/xtask/src/codegen/gen_syntax.rs +++ b/xtask/src/codegen/gen_syntax.rs @@ -23,11 +23,11 @@ pub fn generate_syntax(mode: Mode) -> Result<()> { update(syntax_kinds_file.as_path(), &syntax_kinds, mode)?; let ast_nodes_file = project_root().join(codegen::AST_NODES); - let contents = generate_ast(KINDS_SRC, AST_SRC)?; + let contents = generate_nodes(KINDS_SRC, AST_SRC)?; update(ast_nodes_file.as_path(), &contents, mode)?; let ast_tokens_file = project_root().join(codegen::AST_TOKENS); - let contents = "//! Generated file, do not edit by hand, see `xtask/src/codegen`"; + let contents = generate_tokens(KINDS_SRC, AST_SRC)?; update(ast_tokens_file.as_path(), &contents, mode)?; Ok(()) @@ -40,7 +40,62 @@ struct ElementKinds { has_tokens: bool, } -fn generate_ast(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { +fn generate_tokens(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { + let all_token_kinds: Vec<_> = kinds + .punct + .into_iter() + .map(|(_, kind)| kind) + .copied() + .map(|x| x.into()) + .chain( + kinds + .keywords + .into_iter() + .chain(kinds.contextual_keywords.into_iter()) + .map(|name| Cow::Owned(format!("{}_KW", to_upper_snake_case(&name)))), + ) + .chain(kinds.literals.into_iter().copied().map(|x| x.into())) + .chain(kinds.tokens.into_iter().copied().map(|x| x.into())) + .collect(); + let tokens = all_token_kinds.iter().map(|kind_str| { + let kind_str = &**kind_str; + let kind = format_ident!("{}", kind_str); + let name = format_ident!("{}", to_pascal_case(kind_str)); + quote! { + #[derive(Debug, Clone, PartialEq, Eq, Hash)] + pub struct #name { + pub(crate) syntax: SyntaxToken, + } + + impl std::fmt::Display for #name { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + std::fmt::Display::fmt(&self.syntax, f) + } + } + + impl AstToken for #name { + fn can_cast(kind: SyntaxKind) -> bool { + match kind { + #kind => true, + _ => false, + } + } + fn cast(syntax: SyntaxToken) -> Option { + if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None } + } + fn syntax(&self) -> &SyntaxToken { &self.syntax } + } + } + }); + + crate::reformat(quote! { + use crate::{SyntaxToken, SyntaxKind::{self, *}, ast::AstToken}; + + #(#tokens)* + }) +} + +fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { let all_token_kinds: Vec<_> = kinds .punct .into_iter() @@ -98,37 +153,6 @@ fn generate_ast(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { element_kinds_map.insert(en.name.to_string(), element_kinds); } - let tokens = all_token_kinds.iter().map(|kind_str| { - let kind_str = &**kind_str; - let kind = format_ident!("{}", kind_str); - let name = format_ident!("{}", to_pascal_case(kind_str)); - quote! { - #[derive(Debug, Clone, PartialEq, Eq, Hash)] - pub struct #name { - pub(crate) syntax: SyntaxToken, - } - - impl std::fmt::Display for #name { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - std::fmt::Display::fmt(&self.syntax, f) - } - } - - impl AstToken for #name { - fn can_cast(kind: SyntaxKind) -> bool { - match kind { - #kind => true, - _ => false, - } - } - fn cast(syntax: SyntaxToken) -> Option { - if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None } - } - fn syntax(&self) -> &SyntaxToken { &self.syntax } - } - } - }); - let nodes = grammar.nodes.iter().map(|node| { let name = format_ident!("{}", node.name); let kind = format_ident!("{}", to_upper_snake_case(&name.to_string())); @@ -301,13 +325,13 @@ fn generate_ast(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { } let ast = quote! { - #[allow(unused_imports)] use crate::{ - SyntaxNode, SyntaxToken, SyntaxElement, NodeOrToken, SyntaxKind::{self, *}, + SyntaxNode, SyntaxToken, SyntaxKind::{self, *}, ast::{self, AstNode, AstToken, AstChildren, support}, }; - #(#tokens)* + use super::tokens::*; + #(#nodes)* #(#enums)* }; -- cgit v1.2.3 From 4a063e651f0ec0da852ce1f87589d5c0a893c59d Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Thu, 9 Apr 2020 17:58:15 +0200 Subject: Move the rest of the tokens to generated/tokens --- xtask/src/codegen/gen_syntax.rs | 170 +++++++++++++++++++++------------------- 1 file changed, 90 insertions(+), 80 deletions(-) (limited to 'xtask/src/codegen') diff --git a/xtask/src/codegen/gen_syntax.rs b/xtask/src/codegen/gen_syntax.rs index 683901a0a..9801a3dba 100644 --- a/xtask/src/codegen/gen_syntax.rs +++ b/xtask/src/codegen/gen_syntax.rs @@ -5,7 +5,7 @@ use std::{ borrow::Cow, - collections::{BTreeSet, HashMap, HashSet}, + collections::{BTreeSet, HashSet}, }; use proc_macro2::{Punct, Spacing}; @@ -57,6 +57,7 @@ fn generate_tokens(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { .chain(kinds.literals.into_iter().copied().map(|x| x.into())) .chain(kinds.tokens.into_iter().copied().map(|x| x.into())) .collect(); + let tokens = all_token_kinds.iter().map(|kind_str| { let kind_str = &**kind_str; let kind = format_ident!("{}", kind_str); @@ -88,10 +89,67 @@ fn generate_tokens(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { } }); + let enums = grammar.token_enums.iter().map(|en| { + let variants = en.variants.iter().map(|var| format_ident!("{}", var)).collect::>(); + let name = format_ident!("{}", en.name); + let kinds = variants + .iter() + .map(|name| format_ident!("{}", to_upper_snake_case(&name.to_string()))) + .collect::>(); + assert!(en.traits.is_empty()); + + quote! { + #[derive(Debug, Clone, PartialEq, Eq, Hash)] + pub enum #name { + #(#variants(#variants),)* + } + + #( + impl From<#variants> for #name { + fn from(node: #variants) -> #name { + #name::#variants(node) + } + } + )* + + impl std::fmt::Display for #name { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + std::fmt::Display::fmt(self.syntax(), f) + } + } + + impl AstToken for #name { + fn can_cast(kind: SyntaxKind) -> bool { + match kind { + #(#kinds)|* => true, + _ => false, + } + } + fn cast(syntax: SyntaxToken) -> Option { + let res = match syntax.kind() { + #( + #kinds => #name::#variants(#variants { syntax }), + )* + _ => return None, + }; + Some(res) + } + fn syntax(&self) -> &SyntaxToken { + match self { + #( + #name::#variants(it) => &it.syntax, + )* + } + } + } + } + }); + crate::reformat(quote! { use crate::{SyntaxToken, SyntaxKind::{self, *}, ast::AstToken}; #(#tokens)* + #(#enums)* }) } @@ -113,44 +171,15 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { .chain(kinds.tokens.into_iter().copied().map(|x| x.into())) .collect(); - let mut element_kinds_map = HashMap::new(); + let mut token_kinds = HashSet::new(); for kind in &all_token_kinds { let kind = &**kind; let name = to_pascal_case(kind); - element_kinds_map.insert( - name, - ElementKinds { - kinds: Some(format_ident!("{}", kind)).into_iter().collect(), - has_nodes: false, - has_tokens: true, - }, - ); + token_kinds.insert(name); } - for kind in kinds.nodes { - let name = to_pascal_case(kind); - element_kinds_map.insert( - name, - ElementKinds { - kinds: Some(format_ident!("{}", *kind)).into_iter().collect(), - has_nodes: true, - has_tokens: false, - }, - ); - } - - for en in grammar.enums { - let mut element_kinds: ElementKinds = Default::default(); - for variant in en.variants { - if let Some(variant_element_kinds) = element_kinds_map.get(*variant) { - element_kinds.kinds.extend(variant_element_kinds.kinds.iter().cloned()); - element_kinds.has_tokens |= variant_element_kinds.has_tokens; - element_kinds.has_nodes |= variant_element_kinds.has_nodes; - } else { - panic!("Enum variant has type that does not exist or was not declared before the enum: {}", *variant); - } - } - element_kinds_map.insert(en.name.to_string(), element_kinds); + for en in grammar.token_enums { + token_kinds.insert(en.name.to_string()); } let nodes = grammar.nodes.iter().map(|node| { @@ -182,7 +211,7 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { } } FieldSrc::Optional(_) | FieldSrc::Shorthand => { - let is_token = element_kinds_map[&ty.to_string()].has_tokens; + let is_token = token_kinds.contains(&ty.to_string()); if is_token { quote! { pub fn #method_name(&self) -> Option<#ty> { @@ -245,48 +274,6 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { quote!(impl ast::#trait_name for #name {}) }); - let element_kinds = &element_kinds_map[&en.name.to_string()]; - assert!( - element_kinds.has_nodes ^ element_kinds.has_tokens, - "{}: {:#?}", - name, - element_kinds - ); - let specific_ast_trait = { - let (ast_trait, syntax_type) = if element_kinds.has_tokens { - (quote!(AstToken), quote!(SyntaxToken)) - } else { - (quote!(AstNode), quote!(SyntaxNode)) - }; - - quote! { - impl #ast_trait for #name { - fn can_cast(kind: SyntaxKind) -> bool { - match kind { - #(#kinds)|* => true, - _ => false, - } - } - fn cast(syntax: #syntax_type) -> Option { - let res = match syntax.kind() { - #( - #kinds => #name::#variants(#variants { syntax }), - )* - _ => return None, - }; - Some(res) - } - fn syntax(&self) -> &#syntax_type { - match self { - #( - #name::#variants(it) => &it.syntax, - )* - } - } - } - } - }; - quote! { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum #name { @@ -307,7 +294,30 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { } } - #specific_ast_trait + impl AstNode for #name { + fn can_cast(kind: SyntaxKind) -> bool { + match kind { + #(#kinds)|* => true, + _ => false, + } + } + fn cast(syntax: SyntaxNode) -> Option { + let res = match syntax.kind() { + #( + #kinds => #name::#variants(#variants { syntax }), + )* + _ => return None, + }; + Some(res) + } + fn syntax(&self) -> &SyntaxNode { + match self { + #( + #name::#variants(it) => &it.syntax, + )* + } + } + } #(#traits)* } @@ -326,8 +336,8 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { let ast = quote! { use crate::{ - SyntaxNode, SyntaxToken, SyntaxKind::{self, *}, - ast::{self, AstNode, AstToken, AstChildren, support}, + SyntaxNode, SyntaxKind::{self, *}, + ast::{self, AstNode, AstChildren, support}, }; use super::tokens::*; -- cgit v1.2.3 From 9285cbffb66e1c497469fedc5c181b7f295742fd Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Thu, 9 Apr 2020 18:08:54 +0200 Subject: More compact --- xtask/src/codegen/gen_syntax.rs | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) (limited to 'xtask/src/codegen') diff --git a/xtask/src/codegen/gen_syntax.rs b/xtask/src/codegen/gen_syntax.rs index 9801a3dba..f9a50c5b9 100644 --- a/xtask/src/codegen/gen_syntax.rs +++ b/xtask/src/codegen/gen_syntax.rs @@ -75,12 +75,7 @@ fn generate_tokens(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { } impl AstToken for #name { - fn can_cast(kind: SyntaxKind) -> bool { - match kind { - #kind => true, - _ => false, - } - } + fn can_cast(kind: SyntaxKind) -> bool { kind == #kind } fn cast(syntax: SyntaxToken) -> Option { if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None } } @@ -243,10 +238,7 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { impl AstNode for #name { fn can_cast(kind: SyntaxKind) -> bool { - match kind { - #kind => true, - _ => false, - } + kind == #kind } fn cast(syntax: SyntaxNode) -> Option { if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None } -- cgit v1.2.3 From 56c8581b901427ee3e63052c531f3ba3b1ec112d Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Thu, 9 Apr 2020 18:11:16 +0200 Subject: Put displays at the end --- xtask/src/codegen/gen_syntax.rs | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) (limited to 'xtask/src/codegen') diff --git a/xtask/src/codegen/gen_syntax.rs b/xtask/src/codegen/gen_syntax.rs index f9a50c5b9..6657c9fc5 100644 --- a/xtask/src/codegen/gen_syntax.rs +++ b/xtask/src/codegen/gen_syntax.rs @@ -230,12 +230,6 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { pub(crate) syntax: SyntaxNode, } - impl std::fmt::Display for #name { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - std::fmt::Display::fmt(self.syntax(), f) - } - } - impl AstNode for #name { fn can_cast(kind: SyntaxKind) -> bool { kind == #kind @@ -280,12 +274,6 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { } )* - impl std::fmt::Display for #name { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - std::fmt::Display::fmt(self.syntax(), f) - } - } - impl AstNode for #name { fn can_cast(kind: SyntaxKind) -> bool { match kind { @@ -315,6 +303,21 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { } }); + let displays = grammar + .enums + .iter() + .map(|it| format_ident!("{}", it.name)) + .chain(grammar.nodes.iter().map(|it| format_ident!("{}", it.name))) + .map(|name| { + quote! { + impl std::fmt::Display for #name { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + std::fmt::Display::fmt(self.syntax(), f) + } + } + } + }); + let defined_nodes: HashSet<_> = grammar.nodes.iter().map(|node| node.name).collect(); for node in kinds @@ -336,6 +339,7 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result { #(#nodes)* #(#enums)* + #(#displays)* }; let pretty = crate::reformat(ast)?; -- cgit v1.2.3