3 files changed, 897 insertions, 45 deletions

diff --git a/xtask/src/codegen/gen_syntax.rs b/xtask/src/codegen/gen_syntax.rs
index 745a25862..cafad8070 100644
--- a/xtask/src/codegen/gen_syntax.rs
+++ b/xtask/src/codegen/gen_syntax.rs
@@ -3,34 +3,41 @@
 //! Specifically, it generates the `SyntaxKind` enum and a number of newtype
 //! wrappers around `SyntaxNode` which implement `ra_syntax::AstNode`.
 
-use std::{collections::HashSet, fmt::Write};
+use std::{
+    collections::{BTreeSet, HashSet},
+    fmt::Write,
+};
 
 use proc_macro2::{Punct, Spacing};
 use quote::{format_ident, quote};
+use ungrammar::{rust_grammar, Grammar, Rule};
 
 use crate::{
-    ast_src::{AstSrc, Field, FieldSrc, KindsSrc, AST_SRC, KINDS_SRC},
+    ast_src::{AstEnumSrc, AstNodeSrc, AstSrc, Cardinality, Field, KindsSrc, KINDS_SRC},
     codegen::{self, update, Mode},
     project_root, Result,
 };
 
 pub fn generate_syntax(mode: Mode) -> Result<()> {
+    let grammar = rust_grammar();
+    let ast = lower(&grammar);
+
     let syntax_kinds_file = project_root().join(codegen::SYNTAX_KINDS);
     let syntax_kinds = generate_syntax_kinds(KINDS_SRC)?;
     update(syntax_kinds_file.as_path(), &syntax_kinds, mode)?;
 
     let ast_tokens_file = project_root().join(codegen::AST_TOKENS);
-    let contents = generate_tokens(AST_SRC)?;
+    let contents = generate_tokens(&ast)?;
     update(ast_tokens_file.as_path(), &contents, mode)?;
 
     let ast_nodes_file = project_root().join(codegen::AST_NODES);
-    let contents = generate_nodes(KINDS_SRC, AST_SRC)?;
+    let contents = generate_nodes(KINDS_SRC, &ast)?;
     update(ast_nodes_file.as_path(), &contents, mode)?;
 
     Ok(())
 }
 
-fn generate_tokens(grammar: AstSrc<'_>) -> Result<String> {
+fn generate_tokens(grammar: &AstSrc) -> Result<String> {
     let tokens = grammar.tokens.iter().map(|token| {
         let name = format_ident!("{}", token);
         let kind = format_ident!("{}", to_upper_snake_case(token));
@@ -62,13 +69,13 @@ fn generate_tokens(grammar: AstSrc<'_>) -> Result<String> {
     Ok(pretty)
 }
 
-fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result<String> {
+fn generate_nodes(kinds: KindsSrc<'_>, grammar: &AstSrc) -> Result<String> {
     let (node_defs, node_boilerplate_impls): (Vec<_>, Vec<_>) = grammar
         .nodes
         .iter()
         .map(|node| {
             let name = format_ident!("{}", node.name);
-            let kind = format_ident!("{}", to_upper_snake_case(node.name));
+            let kind = format_ident!("{}", to_upper_snake_case(&node.name));
             let traits = node.traits.iter().map(|trait_name| {
                 let trait_name = format_ident!("{}", trait_name);
                 quote!(impl ast::#trait_name for #name {})
@@ -144,25 +151,10 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result<String> {
                 quote!(impl ast::#trait_name for #name {})
             });
 
-            (
-                quote! {
-                    #[pretty_doc_comment_placeholder_workaround]
-                    #[derive(Debug, Clone, PartialEq, Eq, Hash)]
-                    pub enum #name {
-                        #(#variants(#variants),)*
-                    }
-
-                    #(#traits)*
-                },
+            let ast_node = if en.name == "Stmt" {
+                quote! {}
+            } else {
                 quote! {
-                    #(
-                    impl From<#variants> for #name {
-                        fn from(node: #variants) -> #name {
-                            #name::#variants(node)
-                        }
-                    }
-                    )*
-
                     impl AstNode for #name {
                         fn can_cast(kind: SyntaxKind) -> bool {
                             match kind {
@@ -187,13 +179,35 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result<String> {
                             }
                         }
                     }
+                }
+            };
+
+            (
+                quote! {
+                    #[pretty_doc_comment_placeholder_workaround]
+                    #[derive(Debug, Clone, PartialEq, Eq, Hash)]
+                    pub enum #name {
+                        #(#variants(#variants),)*
+                    }
+
+                    #(#traits)*
+                },
+                quote! {
+                    #(
+                        impl From<#variants> for #name {
+                            fn from(node: #variants) -> #name {
+                                #name::#variants(node)
+                            }
+                        }
+                    )*
+                    #ast_node
                 },
             )
         })
         .unzip();
 
-    let enum_names = grammar.enums.iter().map(|it| it.name);
-    let node_names = grammar.nodes.iter().map(|it| it.name);
+    let enum_names = grammar.enums.iter().map(|it| &it.name);
+    let node_names = grammar.nodes.iter().map(|it| &it.name);
 
     let display_impls =
         enum_names.chain(node_names.clone()).map(|it| format_ident!("{}", it)).map(|name| {
@@ -212,9 +226,11 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result<String> {
         .nodes
         .iter()
         .map(|kind| to_pascal_case(kind))
-        .filter(|name| !defined_nodes.contains(name.as_str()))
+        .filter(|name| !defined_nodes.iter().any(|&it| it == name))
     {
-        eprintln!("Warning: node {} not defined in ast source", node);
+        drop(node)
+        // TODO: restore this
+        // eprintln!("Warning: node {} not defined in ast source", node);
     }
 
     let ast = quote! {
@@ -236,12 +252,12 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result<String> {
     let mut res = String::with_capacity(ast.len() * 2);
 
     let mut docs =
-        grammar.nodes.iter().map(|it| it.doc).chain(grammar.enums.iter().map(|it| it.doc));
+        grammar.nodes.iter().map(|it| &it.doc).chain(grammar.enums.iter().map(|it| &it.doc));
 
     for chunk in ast.split("# [ pretty_doc_comment_placeholder_workaround ]") {
         res.push_str(chunk);
         if let Some(doc) = docs.next() {
-            write_doc_comment(doc, &mut res);
+            write_doc_comment(&doc, &mut res);
         }
     }
 
@@ -249,7 +265,7 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result<String> {
     Ok(pretty)
 }
 
-fn write_doc_comment(contents: &[&str], dest: &mut String) {
+fn write_doc_comment(contents: &[String], dest: &mut String) {
     for line in contents {
         writeln!(dest, "///{}", line).unwrap();
     }
@@ -296,7 +312,7 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> Result<String> {
 
     let ast = quote! {
         #![allow(bad_style, missing_docs, unreachable_pub)]
-        /// The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT_DEF`.
+        /// The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT`.
         #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
         #[repr(u16)]
         pub enum SyntaxKind {
@@ -363,6 +379,7 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> Result<String> {
             #([#all_keywords_idents] => { $crate::SyntaxKind::#all_keywords };)*
             [lifetime] => { $crate::SyntaxKind::LIFETIME };
             [ident] => { $crate::SyntaxKind::IDENT };
+            [shebang] => { $crate::SyntaxKind::SHEBANG };
         }
     };
 
@@ -413,9 +430,13 @@ fn to_pascal_case(s: &str) -> String {
     buf
 }
 
-impl Field<'_> {
+fn pluralize(s: &str) -> String {
+    format!("{}s", s)
+}
+
+impl Field {
     fn is_many(&self) -> bool {
-        matches!(self, Field::Node { src: FieldSrc::Many(_), .. })
+        matches!(self, Field::Node { cardinality: Cardinality::Many, .. })
     }
     fn token_kind(&self) -> Option<proc_macro2::TokenStream> {
         match self {
@@ -429,7 +450,7 @@ impl Field<'_> {
     fn method_name(&self) -> proc_macro2::Ident {
         match self {
             Field::Token(name) => {
-                let name = match *name {
+                let name = match name.as_str() {
                     ";" => "semicolon",
                     "->" => "thin_arrow",
                     "'{'" => "l_curly",
@@ -448,29 +469,273 @@ impl Field<'_> {
                     "." => "dot",
                     ".." => "dotdot",
                     "..." => "dotdotdot",
+                    "..=" => "dotdoteq",
                     "=>" => "fat_arrow",
                     "@" => "at",
                     ":" => "colon",
                     "::" => "coloncolon",
                     "#" => "pound",
                     "?" => "question_mark",
+                    "," => "comma",
                     _ => name,
                 };
                 format_ident!("{}_token", name)
             }
-            Field::Node { name, src } => match src {
-                FieldSrc::Shorthand => format_ident!("{}", to_lower_snake_case(name)),
-                _ => format_ident!("{}", name),
-            },
+            Field::Node { name, .. } => {
+                if name == "type" {
+                    format_ident!("ty")
+                } else {
+                    format_ident!("{}", name)
+                }
+            }
         }
     }
     fn ty(&self) -> proc_macro2::Ident {
         match self {
             Field::Token(_) => format_ident!("SyntaxToken"),
-            Field::Node { name, src } => match src {
-                FieldSrc::Optional(ty) | FieldSrc::Many(ty) => format_ident!("{}", ty),
-                FieldSrc::Shorthand => format_ident!("{}", name),
-            },
+            Field::Node { ty, .. } => format_ident!("{}", ty),
         }
     }
 }
+
+fn lower(grammar: &Grammar) -> AstSrc {
+    let mut res = AstSrc::default();
+    res.tokens = vec!["Whitespace".into(), "Comment".into(), "String".into(), "RawString".into()];
+
+    let nodes = grammar.iter().collect::<Vec<_>>();
+
+    for &node in &nodes {
+        let name = grammar[node].name.clone();
+        let rule = &grammar[node].rule;
+        match lower_enum(grammar, rule) {
+            Some(variants) => {
+                let enum_src = AstEnumSrc { doc: Vec::new(), name, traits: Vec::new(), variants };
+                res.enums.push(enum_src);
+            }
+            None => {
+                let mut fields = Vec::new();
+                lower_rule(&mut fields, grammar, None, rule);
+                res.nodes.push(AstNodeSrc { doc: Vec::new(), name, traits: Vec::new(), fields });
+            }
+        }
+    }
+
+    deduplicate_fields(&mut res);
+    extract_enums(&mut res);
+    extract_struct_traits(&mut res);
+    extract_enum_traits(&mut res);
+    res
+}
+
+fn lower_enum(grammar: &Grammar, rule: &Rule) -> Option<Vec<String>> {
+    let alternatives = match rule {
+        Rule::Alt(it) => it,
+        _ => return None,
+    };
+    let mut variants = Vec::new();
+    for alternative in alternatives {
+        match alternative {
+            Rule::Node(it) => variants.push(grammar[*it].name.clone()),
+            Rule::Token(it) if grammar[*it].name == ";" => (),
+            _ => return None,
+        }
+    }
+    Some(variants)
+}
+
+fn lower_rule(acc: &mut Vec<Field>, grammar: &Grammar, label: Option<&String>, rule: &Rule) {
+    if lower_comma_list(acc, grammar, label, rule) {
+        return;
+    }
+
+    match rule {
+        Rule::Node(node) => {
+            let ty = grammar[*node].name.clone();
+            let name = label.cloned().unwrap_or_else(|| to_lower_snake_case(&ty));
+            let field = Field::Node { name, ty, cardinality: Cardinality::Optional };
+            acc.push(field);
+        }
+        Rule::Token(token) => {
+            assert!(label.is_none());
+            let mut name = grammar[*token].name.clone();
+            if name != "int_number" && name != "string" {
+                if "[]{}()".contains(&name) {
+                    name = format!("'{}'", name);
+                }
+                let field = Field::Token(name);
+                acc.push(field);
+            }
+        }
+        Rule::Rep(inner) => {
+            if let Rule::Node(node) = &**inner {
+                let ty = grammar[*node].name.clone();
+                let name = label.cloned().unwrap_or_else(|| pluralize(&to_lower_snake_case(&ty)));
+                let field = Field::Node { name, ty, cardinality: Cardinality::Many };
+                acc.push(field);
+                return;
+            }
+            todo!("{:?}", rule)
+        }
+        Rule::Labeled { label: l, rule } => {
+            assert!(label.is_none());
+            let manually_implemented = matches!(
+                l.as_str(),
+                "lhs"
+                    | "rhs"
+                    | "then_branch"
+                    | "else_branch"
+                    | "start"
+                    | "end"
+                    | "op"
+                    | "index"
+                    | "base"
+                    | "value"
+                    | "trait"
+                    | "self_ty"
+            );
+            if manually_implemented {
+                return;
+            }
+            lower_rule(acc, grammar, Some(l), rule);
+        }
+        Rule::Seq(rules) | Rule::Alt(rules) => {
+            for rule in rules {
+                lower_rule(acc, grammar, label, rule)
+            }
+        }
+        Rule::Opt(rule) => lower_rule(acc, grammar, label, rule),
+    }
+}
+
+// (T (',' T)* ','?)
+fn lower_comma_list(
+    acc: &mut Vec<Field>,
+    grammar: &Grammar,
+    label: Option<&String>,
+    rule: &Rule,
+) -> bool {
+    let rule = match rule {
+        Rule::Seq(it) => it,
+        _ => return false,
+    };
+    let (node, repeat, trailing_comma) = match rule.as_slice() {
+        [Rule::Node(node), Rule::Rep(repeat), Rule::Opt(trailing_comma)] => {
+            (node, repeat, trailing_comma)
+        }
+        _ => return false,
+    };
+    let repeat = match &**repeat {
+        Rule::Seq(it) => it,
+        _ => return false,
+    };
+    match repeat.as_slice() {
+        [comma, Rule::Node(n)] if comma == &**trailing_comma && n == node => (),
+        _ => return false,
+    }
+    let ty = grammar[*node].name.clone();
+    let name = label.cloned().unwrap_or_else(|| pluralize(&to_lower_snake_case(&ty)));
+    let field = Field::Node { name, ty, cardinality: Cardinality::Many };
+    acc.push(field);
+    true
+}
+
+fn deduplicate_fields(ast: &mut AstSrc) {
+    for node in &mut ast.nodes {
+        let mut i = 0;
+        'outer: while i < node.fields.len() {
+            for j in 0..i {
+                let f1 = &node.fields[i];
+                let f2 = &node.fields[j];
+                if f1 == f2 {
+                    node.fields.remove(i);
+                    continue 'outer;
+                }
+            }
+            i += 1;
+        }
+    }
+}
+
+fn extract_enums(ast: &mut AstSrc) {
+    for node in &mut ast.nodes {
+        for enm in &ast.enums {
+            let mut to_remove = Vec::new();
+            for (i, field) in node.fields.iter().enumerate() {
+                let ty = field.ty().to_string();
+                if enm.variants.iter().any(|it| it == &ty) {
+                    to_remove.push(i);
+                }
+            }
+            if to_remove.len() == enm.variants.len() {
+                node.remove_field(to_remove);
+                let ty = enm.name.clone();
+                let name = to_lower_snake_case(&ty);
+                node.fields.push(Field::Node { name, ty, cardinality: Cardinality::Optional });
+            }
+        }
+    }
+}
+
+fn extract_struct_traits(ast: &mut AstSrc) {
+    let traits: &[(&str, &[&str])] = &[
+        ("AttrsOwner", &["attrs"]),
+        ("NameOwner", &["name"]),
+        ("VisibilityOwner", &["visibility"]),
+        ("GenericParamsOwner", &["generic_param_list", "where_clause"]),
+        ("TypeBoundsOwner", &["type_bound_list", "colon_token"]),
+        ("ModuleItemOwner", &["items"]),
+        ("LoopBodyOwner", &["label", "loop_body"]),
+        ("ArgListOwner", &["arg_list"]),
+    ];
+
+    for node in &mut ast.nodes {
+        for (name, methods) in traits {
+            extract_struct_trait(node, name, methods);
+        }
+    }
+}
+
+fn extract_struct_trait(node: &mut AstNodeSrc, trait_name: &str, methods: &[&str]) {
+    let mut to_remove = Vec::new();
+    for (i, field) in node.fields.iter().enumerate() {
+        let method_name = field.method_name().to_string();
+        if methods.iter().any(|&it| it == &method_name) {
+            to_remove.push(i);
+        }
+    }
+    if to_remove.len() == methods.len() {
+        node.traits.push(trait_name.to_string());
+        node.remove_field(to_remove);
+    }
+}
+
+fn extract_enum_traits(ast: &mut AstSrc) {
+    for enm in &mut ast.enums {
+        if enm.name == "Stmt" {
+            continue;
+        }
+        let nodes = &ast.nodes;
+        let mut variant_traits = enm
+            .variants
+            .iter()
+            .map(|var| nodes.iter().find(|it| &it.name == var).unwrap())
+            .map(|node| node.traits.iter().cloned().collect::<BTreeSet<_>>());
+
+        let mut enum_traits = match variant_traits.next() {
+            Some(it) => it,
+            None => continue,
+        };
+        for traits in variant_traits {
+            enum_traits = enum_traits.intersection(&traits).cloned().collect();
+        }
+        enm.traits = enum_traits.into_iter().collect();
+    }
+}
+
+impl AstNodeSrc {
+    fn remove_field(&mut self, to_remove: Vec<usize>) {
+        to_remove.into_iter().rev().for_each(|idx| {
+            self.fields.remove(idx);
+        });
+    }
+}
diff --git a/xtask/src/codegen/gen_unstable_future_descriptor.rs b/xtask/src/codegen/gen_unstable_future_descriptor.rs
index 09c4f61f2..298696fbb 100644
--- a/xtask/src/codegen/gen_unstable_future_descriptor.rs
+++ b/xtask/src/codegen/gen_unstable_future_descriptor.rs
@@ -42,7 +42,7 @@ pub fn generate_unstable_future_descriptor(mode: Mode) -> Result<()> {
     let ts = quote! {
         use crate::completion::LintCompletion;
 
-        const UNSTABLE_FEATURE_DESCRIPTOR:  &[LintCompletion] = &[
+        pub const UNSTABLE_FEATURE_DESCRIPTOR:  &[LintCompletion] = &[
             #(#definitions),*
         ];
     };
diff --git a/xtask/src/codegen/rust.ungram b/xtask/src/codegen/rust.ungram
new file mode 100644
index 000000000..aca23890c
--- /dev/null
+++ b/xtask/src/codegen/rust.ungram
@@ -0,0 +1,587 @@
+//*************************//
+// Names, Paths and Macros //
+//*************************//
+
+Name =
+  'ident'
+
+NameRef =
+  'ident' | 'int_number'
+
+Path =
+  (qualifier:Path '::')? segment:PathSegment
+
+PathSegment =
+  'crate' | 'self' | 'super'
+| '::' NameRef
+| NameRef GenericArgList?
+| NameRef ParamList RetType?
+| '<' PathType ('as' PathType)? '>'
+
+GenericArgList =
+  '::'? '<' (GenericArg (',' GenericArg)* ','?)? '>'
+
+GenericArg =
+  TypeArg
+| AssocTypeArg
+| LifetimeArg
+| ConstArg
+
+TypeArg =
+  Type
+
+AssocTypeArg =
+  NameRef (':' TypeBoundList | '=' Type)
+
+LifetimeArg =
+  'lifetime'
+
+ConstArg =
+  Expr
+
+MacroCall =
+  Attr* Path '!' Name? TokenTree ';'?
+
+TokenTree =
+  '(' ')'
+| '{' '}'
+| '[' ']'
+
+MacroItems =
+  Item*
+
+MacroStmts =
+  statements:Stmt*
+  Expr?
+
+//*************************//
+//          Items          //
+//*************************//
+
+SourceFile =
+  'shebang'?
+  Attr*
+  Item*
+
+Item =
+  Const
+| Enum
+| ExternBlock
+| ExternCrate
+| Fn
+| Impl
+| MacroCall
+| Module
+| Static
+| Struct
+| Trait
+| TypeAlias
+| Union
+| Use
+
+Module =
+  Attr* Visibility? 'mod' Name
+  (ItemList | ';')
+
+ItemList =
+  '{' Attr* Item* '}'
+
+ExternCrate =
+  Attr* Visibility? 'extern' 'crate' (NameRef | 'self') Rename? ';'
+
+Rename =
+  'as' (Name | '_')
+
+Use =
+  Attr* Visibility? 'use' UseTree ';'
+
+UseTree =
+  (Path? '::')? ('*' | UseTreeList )
+| Path Rename?
+
+UseTreeList =
+  '{' (UseTree (',' UseTree)* ','?)? '}'
+
+Fn =
+ Attr* Visibility?
+ 'default'? ('async' | 'const')? 'unsafe'? Abi?
+ 'fn' Name GenericParamList? ParamList RetType?
+ WhereClause?
+ (body:BlockExpr | ';')
+
+Abi =
+  'extern' 'string'?
+
+ParamList =
+  '('(
+    SelfParam
+  | (SelfParam ',')? (Param (',' Param)* ','?)?
+  )')'
+
+SelfParam =
+  Attr* (
+    ('&' 'lifetime'?)? 'mut'? 'self'
+  | 'mut'? 'self' ':' Type
+  )
+
+Param =
+  Attr* (
+    Pat (':' Type)
+  | Type
+  | '...'
+  )
+
+RetType =
+  '->' Type
+
+TypeAlias =
+  Attr* Visibility? 'default'? 'type' Name GenericParamList? (':' TypeBoundList?)? WhereClause?
+  '=' Type ';'
+
+Struct =
+  Attr* Visibility? 'struct' Name GenericParamList? (
+    WhereClause?  (RecordFieldList | ';')
+  | TupleFieldList WhereClause? ';'
+  )
+
+RecordFieldList =
+ '{' fields:(RecordField (',' RecordField)* ','?)? '}'
+
+RecordField =
+  Attr* Visibility? Name ':' Type
+
+TupleFieldList =
+  '(' fields:(TupleField (',' TupleField)* ','?)? ')'
+
+TupleField =
+  Attr* Visibility? Type
+
+FieldList =
+  RecordFieldList
+| TupleFieldList
+
+Enum =
+  Attr* Visibility? 'enum' Name GenericParamList? WhereClause?
+  VariantList
+
+VariantList =
+ '{' (Variant (',' Variant)* ','?)? '}'
+
+Variant =
+  Attr* Visibility? Name FieldList ('=' Expr)?
+
+Union =
+  Attr* Visibility? 'union' Name GenericParamList? WhereClause?
+  RecordFieldList
+
+AdtDef =
+  Enum
+| Struct
+| Union
+
+Const =
+  Attr* Visibility? 'default'? 'const' (Name | '_') ':' Type
+  '=' body:Expr ';'
+
+Static =
+  Attr* Visibility? 'static'? 'mut'? Name ':' Type
+  '=' body:Expr ';'
+
+Trait =
+  Attr* Visibility? 'unsafe'? 'auto'? 'trait' Name GenericParamList
+  (':' TypeBoundList?)? WhereClause
+  AssocItemList
+
+AssocItemList =
+  '{' Attr* AssocItem* '}'
+
+AssocItem =
+  Const
+| Fn
+| MacroCall
+| TypeAlias
+
+Impl =
+  Attr* Visibility?
+  'default'? 'unsafe'? 'impl' 'const'? GenericParamList?
+  ('!'? target_trait:Type 'for')? target_type:Type
+  WhereClause?
+  AssocItemList
+
+ExternBlock =
+  Attr* Abi ExternItemList
+
+ExternItemList =
+  '{' Attr* ExternItem* '}'
+
+ExternItem =
+  Fn | Static | MacroCall
+
+GenericParamList =
+  '<' (GenericParam (',' GenericParam)* ','?)? '>'
+
+GenericParam =
+  ConstParam
+| LifetimeParam
+| TypeParam
+
+TypeParam =
+  Attr* Name (':' TypeBoundList?)?
+  ('=' default_type:Type)?
+
+ConstParam =
+  Attr* 'const' Name ':' Type
+  ('=' default_val:Expr)?
+
+LifetimeParam =
+  Attr* 'lifetime' (':' TypeBoundList?)?
+
+WhereClause =
+  'where' predicates:(WherePred (',' WherePred)* ','?)
+
+WherePred =
+  ('for' GenericParamList)?  ('lifetime' | Type) ':' TypeBoundList
+
+Visibility =
+  'pub' ('('
+    'super'
+  | 'self'
+  | 'crate'
+  | 'in' Path
+  ')')?
+
+Attr =
+  '#' '!'? '[' Path ('=' Literal | TokenTree)? ']'
+
+//****************************//
+// Statements and Expressions //
+//****************************//
+
+Stmt =
+  ExprStmt
+| Item
+| LetStmt
+
+LetStmt =
+  Attr* 'let' Pat (':' Type)?
+  '=' initializer:Expr ';'
+
+ExprStmt =
+  Attr* Expr ';'?
+
+Expr =
+  ArrayExpr
+| AwaitExpr
+| BinExpr
+| BlockExpr
+| BoxExpr
+| BreakExpr
+| CallExpr
+| CastExpr
+| ClosureExpr
+| ContinueExpr
+| EffectExpr
+| FieldExpr
+| ForExpr
+| IfExpr
+| IndexExpr
+| Literal
+| LoopExpr
+| MacroCall
+| MatchExpr
+| MethodCallExpr
+| ParenExpr
+| PathExpr
+| PrefixExpr
+| RangeExpr
+| RecordExpr
+| RefExpr
+| ReturnExpr
+| TryExpr
+| TupleExpr
+| WhileExpr
+
+Literal =
+  Attr* value:(
+    'int_number' | 'float_number'
+  | 'string' | 'raw_string'
+  | 'byte_string' | 'raw_byte_string'
+  | 'true' | 'false'
+  | 'char' | 'byte'
+  )
+
+PathExpr =
+  Attr* Path
+
+BlockExpr =
+  '{'
+    Attr*
+    statements:Stmt*
+    Expr?
+  '}'
+
+RefExpr =
+  Attr* '&' ('raw' |'mut' | 'const') Expr
+
+TryExpr =
+  Attr* Expr '?'
+
+EffectExpr =
+  Attr* Label? ('try' | 'unsafe' | 'async') BlockExpr
+
+PrefixExpr =
+  Attr* op:('-' | '!' | '*') Expr
+
+BinExpr =
+  Attr*
+  lhs:Expr
+  op:(
+    '||' | '&&'
+    | '==' | '!=' | '<=' | '>=' | '<' | '>'
+    | '+' | '*' | '-' | '/' | '%' | '<<' | '>>' | '^' | '|' | '&'
+    | '=' | '+=' | '/=' | '*=' | '%=' | '>>=' | '<<=' | '-=' | '|=' | '&=' | '^='
+  )
+  rhs:Expr
+
+CastExpr =
+  Attr* Expr 'as' Type
+
+ParenExpr =
+  Attr* '(' Attr* Expr ')'
+
+ArrayExpr =
+  Attr* '[' Attr* (
+    (Expr (',' Expr)* ','?)?
+  | Expr ';' Expr
+  ) ']'
+
+IndexExpr =
+  Attr* base:Expr '[' index:Expr ']'
+
+TupleExpr =
+  Attr* '(' Attr* fields:(Expr (',' Expr)* ','?)? ')'
+
+RecordExpr =
+  Path RecordExprFieldList
+
+RecordExprFieldList =
+  '{'
+    Attr*
+    fields:(RecordExprField (',' RecordExprField)* ','?)
+    ('..' spread:Expr)?
+  '}'
+
+RecordExprField =
+  Attr* NameRef (':' Expr)?
+
+CallExpr =
+  Attr* Expr ArgList
+
+ArgList =
+  '(' args:(Expr (',' Expr)* ','?)? ')'
+
+MethodCallExpr =
+  Attr* Expr '.' NameRef GenericArgList? ArgList
+
+FieldExpr =
+  Attr* Expr '.' NameRef
+
+ClosureExpr =
+  Attr* 'static'? 'async'? 'move'?  ParamList RetType?
+  body:Expr
+
+IfExpr =
+  Attr* 'if' Condition then_branch:BlockExpr
+  ('else' else_branch:(IfExpr | BlockExpr))?
+
+Condition =
+  'let' Pat '=' Expr
+| Expr
+
+LoopExpr =
+  Attr* Label? 'loop'
+  loop_body:BlockExpr
+
+ForExpr =
+  Attr* Label? 'for' Pat 'in' iterable:Expr
+  loop_body:BlockExpr
+
+WhileExpr =
+  Attr* Label? 'while' Condition
+  loop_body:BlockExpr
+
+Label =
+  'lifetime'
+
+BreakExpr =
+  Attr* 'break' 'lifetime'? Expr?
+
+ContinueExpr =
+  Attr* 'continue' 'lifetime'?
+
+RangeExpr =
+  Attr* start:Expr? op:('..' | '..=') end:Expr?
+
+MatchExpr =
+  Attr* 'match' Expr MatchArmList
+
+MatchArmList =
+  '{'
+    Attr*
+    arms:MatchArm*
+  '}'
+
+MatchArm =
+  Attr* Pat guard:MatchGuard? '=>' Expr ','?
+
+MatchGuard =
+  'if' Expr
+
+ReturnExpr =
+  Attr* 'return' Expr?
+
+AwaitExpr =
+  Attr* Expr '.' 'await'
+
+BoxExpr =
+  Attr* 'box' Expr
+
+//*************************//
+//          Types          //
+//*************************//
+
+Type =
+  ArrayType
+| DynTraitType
+| FnPointerType
+| ForType
+| ImplTraitType
+| InferType
+| NeverType
+| ParenType
+| PathType
+| PointerType
+| ReferenceType
+| SliceType
+| TupleType
+
+ParenType =
+  '(' Type ')'
+
+NeverType =
+  '!'
+
+PathType =
+  Path
+
+TupleType =
+  '(' fields:(Type (',' Type)* ','?)? ')'
+
+PointerType =
+  '*' ('const' | 'mut') Type
+
+ReferenceType =
+  '&' 'lifetime'? 'mut'? Type
+
+ArrayType =
+  '[' Type ';' Expr ']'
+
+SliceType =
+  '[' Type ']'
+
+InferType =
+  '_'
+
+FnPointerType =
+  'const'? 'async'? 'unsafe'? Abi? 'fn' ParamList RetType?
+
+ForType =
+  'for' GenericParamList Type
+
+ImplTraitType =
+  'impl' TypeBoundList
+
+DynTraitType =
+  'dyn' TypeBoundList
+
+TypeBoundList =
+  bounds:(TypeBound ('+' TypeBound)* '+'?)
+
+TypeBound =
+  'lifetime'
+| '?'? Type
+
+//************************//
+//        Patterns        //
+//************************//
+
+Pat =
+  IdentPat
+| BoxPat
+| RestPat
+| LiteralPat
+| MacroPat
+| OrPat
+| ParenPat
+| PathPat
+| WildcardPat
+| RangePat
+| RecordPat
+| RefPat
+| SlicePat
+| TuplePat
+| TupleStructPat
+
+LiteralPat =
+  Literal
+
+IdentPat =
+  Attr* 'ref'? 'mut'? Name ('@' Pat)?
+
+WildcardPat =
+  '_'
+
+RangePat =
+  start:Pat op:('..' | '..=') end:Pat
+
+RefPat =
+  '&' 'mut'? Pat
+
+RecordPat =
+  Path RecordPatFieldList
+
+RecordPatFieldList =
+  '{'
+    fields:(RecordPatField (',' RecordPatField)* ','?)
+    '..'?
+  '}'
+
+RecordPatField =
+  Attr* (NameRef ':')? Pat
+
+TupleStructPat =
+   Path '(' fields:(Pat (',' Pat)* ','?)? ')'
+
+TuplePat =
+   '(' fields:(Pat (',' Pat)* ','?)? ')'
+
+ParenPat =
+  '(' Pat ')'
+
+SlicePat =
+  '[' (Pat (',' Pat)* ','?)? ']'
+
+PathPat =
+  Path
+
+OrPat =
+  (Pat ('|' Pat)* '|'?)
+
+BoxPat =
+  'box' Pat
+
+RestPat =
+  '..'
+
+MacroPat =
+  MacroCall