3 files changed, 38 insertions, 0 deletions

diff --git a/xtask/src/ast_src.rs b/xtask/src/ast_src.rs
index 554afc76a..7a20d9991 100644
--- a/xtask/src/ast_src.rs
+++ b/xtask/src/ast_src.rs
@@ -225,6 +225,7 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc {
 };
 
 pub(crate) struct AstSrc<'a> {
+    pub(crate) tokens: &'a [&'a str],
     pub(crate) nodes: &'a [AstNodeSrc<'a>],
     pub(crate) enums: &'a [AstEnumSrc<'a>],
 }
@@ -303,6 +304,7 @@ macro_rules! ast_enums {
 }
 
 pub(crate) const AST_SRC: AstSrc = AstSrc {
+    tokens: &["Whitespace", "Comment", "String", "RawString"],
     nodes: &ast_nodes! {
         struct SourceFile: ModuleItemOwner, AttrsOwner {
             modules: [Module],
diff --git a/xtask/src/codegen.rs b/xtask/src/codegen.rs
index e6ba2009c..678b40133 100644
--- a/xtask/src/codegen.rs
+++ b/xtask/src/codegen.rs
@@ -24,6 +24,7 @@ const ERR_INLINE_TESTS_DIR: &str = "crates/ra_syntax/test_data/parser/inline/err
 
 const SYNTAX_KINDS: &str = "crates/ra_parser/src/syntax_kind/generated.rs";
 const AST_NODES: &str = "crates/ra_syntax/src/ast/generated/nodes.rs";
+const AST_TOKENS: &str = "crates/ra_syntax/src/ast/generated/tokens.rs";
 
 const ASSISTS_DIR: &str = "crates/ra_assists/src/handlers";
 const ASSISTS_TESTS: &str = "crates/ra_assists/src/doc_tests/generated.rs";
diff --git a/xtask/src/codegen/gen_syntax.rs b/xtask/src/codegen/gen_syntax.rs
index ce18f2b8f..fa48853d2 100644
--- a/xtask/src/codegen/gen_syntax.rs
+++ b/xtask/src/codegen/gen_syntax.rs
@@ -19,6 +19,10 @@ pub fn generate_syntax(mode: Mode) -> Result<()> {
     let syntax_kinds = generate_syntax_kinds(KINDS_SRC)?;
     update(syntax_kinds_file.as_path(), &syntax_kinds, mode)?;
 
+    let ast_tokens_file = project_root().join(codegen::AST_TOKENS);
+    let contents = generate_tokens(KINDS_SRC, AST_SRC)?;
+    update(ast_tokens_file.as_path(), &contents, mode)?;
+
     let ast_nodes_file = project_root().join(codegen::AST_NODES);
     let contents = generate_nodes(KINDS_SRC, AST_SRC)?;
     update(ast_nodes_file.as_path(), &contents, mode)?;
@@ -33,6 +37,37 @@ struct ElementKinds {
     has_tokens: bool,
 }
 
+fn generate_tokens(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result<String> {
+    let tokens = grammar.tokens.iter().map(|token| {
+        let name = format_ident!("{}", token);
+        let kind = format_ident!("{}", to_upper_snake_case(token));
+        quote! {
+            #[derive(Debug, Clone, PartialEq, Eq, Hash)]
+            pub struct #name {
+                pub(crate) syntax: SyntaxToken,
+            }
+            impl std::fmt::Display for #name {
+                fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+                    std::fmt::Display::fmt(&self.syntax, f)
+                }
+            }
+            impl AstToken for #name {
+                fn can_cast(kind: SyntaxKind) -> bool { kind == #kind }
+                fn cast(syntax: SyntaxToken) -> Option<Self> {
+                    if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
+                }
+                fn syntax(&self) -> &SyntaxToken { &self.syntax }
+            }
+        }
+    });
+
+    let pretty = crate::reformat(quote! {
+        use crate::{SyntaxKind::{self, *}, SyntaxToken, ast::AstToken};
+        #(#tokens)*
+    })?;
+    Ok(pretty)
+}
+
 fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result<String> {
     let nodes = grammar.nodes.iter().map(|node| {
         let name = format_ident!("{}", node.name);