9 files changed, 196 insertions, 89 deletions

diff --git a/xtask/src/codegen.rs b/xtask/src/codegen.rs
index 78a84f68d..45b17bb48 100644
--- a/xtask/src/codegen.rs
+++ b/xtask/src/codegen.rs
@@ -9,38 +9,51 @@ mod gen_syntax;
 mod gen_parser_tests;
 mod gen_assists_docs;
 mod gen_feature_docs;
+mod gen_features;
 
 use std::{
     fmt, mem,
     path::{Path, PathBuf},
 };
 
-use crate::{not_bash::fs2, project_root, Result};
+use crate::{
+    ensure_rustfmt,
+    not_bash::{fs2, pushenv, run},
+    project_root, Result,
+};
 
 pub use self::{
     gen_assists_docs::{generate_assists_docs, generate_assists_tests},
     gen_feature_docs::generate_feature_docs,
+    gen_features::generate_features,
     gen_parser_tests::generate_parser_tests,
     gen_syntax::generate_syntax,
 };
 
-const GRAMMAR_DIR: &str = "crates/parser/src/grammar";
-const OK_INLINE_TESTS_DIR: &str = "crates/syntax/test_data/parser/inline/ok";
-const ERR_INLINE_TESTS_DIR: &str = "crates/syntax/test_data/parser/inline/err";
-
-const SYNTAX_KINDS: &str = "crates/parser/src/syntax_kind/generated.rs";
-const AST_NODES: &str = "crates/syntax/src/ast/generated/nodes.rs";
-const AST_TOKENS: &str = "crates/syntax/src/ast/generated/tokens.rs";
-
-const ASSISTS_DIR: &str = "crates/assists/src/handlers";
-const ASSISTS_TESTS: &str = "crates/assists/src/tests/generated.rs";
-
 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
 pub enum Mode {
     Overwrite,
     Verify,
 }
 
+pub struct CodegenCmd {
+    pub features: bool,
+}
+
+impl CodegenCmd {
+    pub fn run(self) -> Result<()> {
+        if self.features {
+            generate_features(Mode::Overwrite)?;
+        }
+        generate_syntax(Mode::Overwrite)?;
+        generate_parser_tests(Mode::Overwrite)?;
+        generate_assists_tests(Mode::Overwrite)?;
+        generate_assists_docs(Mode::Overwrite)?;
+        generate_feature_docs(Mode::Overwrite)?;
+        Ok(())
+    }
+}
+
 /// A helper to update file on disk if it has changed.
 /// With verify = false,
 fn update(path: &Path, contents: &str, mode: Mode) -> Result<()> {
@@ -62,6 +75,18 @@ fn update(path: &Path, contents: &str, mode: Mode) -> Result<()> {
     }
 }
 
+const PREAMBLE: &str = "Generated file, do not edit by hand, see `xtask/src/codegen`";
+
+fn reformat(text: impl std::fmt::Display) -> Result<String> {
+    let _e = pushenv("RUSTUP_TOOLCHAIN", "stable");
+    ensure_rustfmt()?;
+    let stdout = run!(
+        "rustfmt --config-path {} --config fn_single_line=true", project_root().join("rustfmt.toml").display();
+        <text.to_string().as_bytes()
+    )?;
+    Ok(format!("//! {}\n\n{}\n", PREAMBLE, stdout))
+}
+
 fn extract_comment_blocks(text: &str) -> Vec<Vec<String>> {
     do_extract_comment_blocks(text, false).into_iter().map(|(_line, block)| block).collect()
 }
diff --git a/xtask/src/codegen/gen_assists_docs.rs b/xtask/src/codegen/gen_assists_docs.rs
index 526941f73..f0ded8b87 100644
--- a/xtask/src/codegen/gen_assists_docs.rs
+++ b/xtask/src/codegen/gen_assists_docs.rs
@@ -3,7 +3,7 @@
 use std::{fmt, fs, path::Path};
 
 use crate::{
-    codegen::{self, extract_comment_blocks_with_empty_lines, Location, Mode},
+    codegen::{self, extract_comment_blocks_with_empty_lines, reformat, Location, Mode, PREAMBLE},
     project_root, rust_files, Result,
 };
 
@@ -15,7 +15,7 @@ pub fn generate_assists_tests(mode: Mode) -> Result<()> {
 pub fn generate_assists_docs(mode: Mode) -> Result<()> {
     let assists = Assist::collect()?;
     let contents = assists.into_iter().map(|it| it.to_string()).collect::<Vec<_>>().join("\n\n");
-    let contents = contents.trim().to_string() + "\n";
+    let contents = format!("//{}\n{}\n", PREAMBLE, contents.trim());
     let dst = project_root().join("docs/user/generated_assists.adoc");
     codegen::update(&dst, &contents, mode)
 }
@@ -32,7 +32,7 @@ struct Assist {
 impl Assist {
     fn collect() -> Result<Vec<Assist>> {
         let mut res = Vec::new();
-        for path in rust_files(&project_root().join(codegen::ASSISTS_DIR)) {
+        for path in rust_files(&project_root().join("crates/assists/src/handlers")) {
             collect_file(&mut res, path.as_path())?;
         }
         res.sort_by(|lhs, rhs| lhs.id.cmp(&rhs.id));
@@ -134,8 +134,8 @@ r#####"
 
         buf.push_str(&test)
     }
-    let buf = crate::reformat(buf)?;
-    codegen::update(&project_root().join(codegen::ASSISTS_TESTS), &buf, mode)
+    let buf = reformat(buf)?;
+    codegen::update(&project_root().join("crates/assists/src/tests/generated.rs"), &buf, mode)
 }
 
 fn hide_hash_comments(text: &str) -> String {
diff --git a/xtask/src/codegen/gen_feature_docs.rs b/xtask/src/codegen/gen_feature_docs.rs
index 31bc3839d..3f0013e82 100644
--- a/xtask/src/codegen/gen_feature_docs.rs
+++ b/xtask/src/codegen/gen_feature_docs.rs
@@ -3,14 +3,14 @@
 use std::{fmt, fs, path::PathBuf};
 
 use crate::{
-    codegen::{self, extract_comment_blocks_with_empty_lines, Location, Mode},
+    codegen::{self, extract_comment_blocks_with_empty_lines, Location, Mode, PREAMBLE},
     project_root, rust_files, Result,
 };
 
 pub fn generate_feature_docs(mode: Mode) -> Result<()> {
     let features = Feature::collect()?;
     let contents = features.into_iter().map(|it| it.to_string()).collect::<Vec<_>>().join("\n\n");
-    let contents = contents.trim().to_string() + "\n";
+    let contents = format!("//{}\n{}\n", PREAMBLE, contents.trim());
     let dst = project_root().join("docs/user/generated_features.adoc");
     codegen::update(&dst, &contents, mode)?;
     Ok(())
diff --git a/xtask/src/codegen/gen_features.rs b/xtask/src/codegen/gen_features.rs
new file mode 100644
index 000000000..78268308b
--- /dev/null
+++ b/xtask/src/codegen/gen_features.rs
@@ -0,0 +1,50 @@
+//! Generates descriptors structure for unstable feature from Unstable Book
+use std::path::{Path, PathBuf};
+
+use quote::quote;
+use walkdir::WalkDir;
+
+use crate::{
+    codegen::{project_root, reformat, update, Mode, Result},
+    not_bash::{fs2, run},
+};
+
+pub fn generate_features(mode: Mode) -> Result<()> {
+    if !Path::new("./target/rust").exists() {
+        run!("git clone https://github.com/rust-lang/rust ./target/rust")?;
+    }
+
+    let contents = generate_descriptor("./target/rust/src/doc/unstable-book/src".into())?;
+
+    let destination = project_root().join("crates/ide/src/completion/generated_features.rs");
+    update(destination.as_path(), &contents, mode)?;
+
+    Ok(())
+}
+
+fn generate_descriptor(src_dir: PathBuf) -> Result<String> {
+    let definitions = ["language-features", "library-features"]
+        .iter()
+        .flat_map(|it| WalkDir::new(src_dir.join(it)))
+        .filter_map(|e| e.ok())
+        .filter(|entry| {
+            // Get all `.md ` files
+            entry.file_type().is_file() && entry.path().extension().unwrap_or_default() == "md"
+        })
+        .map(|entry| {
+            let path = entry.path();
+            let feature_ident = path.file_stem().unwrap().to_str().unwrap().replace("-", "_");
+            let doc = fs2::read_to_string(path).unwrap();
+
+            quote! { LintCompletion { label: #feature_ident, description: #doc } }
+        });
+
+    let ts = quote! {
+        use crate::completion::complete_attribute::LintCompletion;
+
+        pub(super) const FEATURES:  &[LintCompletion] = &[
+            #(#definitions),*
+        ];
+    };
+    reformat(ts)
+}
diff --git a/xtask/src/codegen/gen_parser_tests.rs b/xtask/src/codegen/gen_parser_tests.rs
index 2977da2fa..96fdd9216 100644
--- a/xtask/src/codegen/gen_parser_tests.rs
+++ b/xtask/src/codegen/gen_parser_tests.rs
@@ -8,12 +8,12 @@ use std::{
 };
 
 use crate::{
-    codegen::{self, extract_comment_blocks, update, Mode},
+    codegen::{extract_comment_blocks, update, Mode},
     project_root, Result,
 };
 
 pub fn generate_parser_tests(mode: Mode) -> Result<()> {
-    let tests = tests_from_dir(&project_root().join(Path::new(codegen::GRAMMAR_DIR)))?;
+    let tests = tests_from_dir(&project_root().join(Path::new("crates/parser/src/grammar")))?;
     fn install_tests(tests: &HashMap<String, Test>, into: &str, mode: Mode) -> Result<()> {
         let tests_dir = project_root().join(into);
         if !tests_dir.is_dir() {
@@ -39,8 +39,8 @@ pub fn generate_parser_tests(mode: Mode) -> Result<()> {
         }
         Ok(())
     }
-    install_tests(&tests.ok, codegen::OK_INLINE_TESTS_DIR, mode)?;
-    install_tests(&tests.err, codegen::ERR_INLINE_TESTS_DIR, mode)
+    install_tests(&tests.ok, "crates/syntax/test_data/parser/inline/ok", mode)?;
+    install_tests(&tests.err, "crates/syntax/test_data/parser/inline/err", mode)
 }
 
 #[derive(Debug)]
diff --git a/xtask/src/codegen/gen_syntax.rs b/xtask/src/codegen/gen_syntax.rs
index dd1f4d6a2..53ae9f11c 100644
--- a/xtask/src/codegen/gen_syntax.rs
+++ b/xtask/src/codegen/gen_syntax.rs
@@ -14,7 +14,7 @@ use ungrammar::{rust_grammar, Grammar, Rule};
 
 use crate::{
     ast_src::{AstEnumSrc, AstNodeSrc, AstSrc, Cardinality, Field, KindsSrc, KINDS_SRC},
-    codegen::{self, update, Mode},
+    codegen::{reformat, update, Mode},
     project_root, Result,
 };
 
@@ -22,15 +22,15 @@ pub fn generate_syntax(mode: Mode) -> Result<()> {
     let grammar = rust_grammar();
     let ast = lower(&grammar);
 
-    let syntax_kinds_file = project_root().join(codegen::SYNTAX_KINDS);
+    let syntax_kinds_file = project_root().join("crates/parser/src/syntax_kind/generated.rs");
     let syntax_kinds = generate_syntax_kinds(KINDS_SRC)?;
     update(syntax_kinds_file.as_path(), &syntax_kinds, mode)?;
 
-    let ast_tokens_file = project_root().join(codegen::AST_TOKENS);
+    let ast_tokens_file = project_root().join("crates/syntax/src/ast/generated/tokens.rs");
     let contents = generate_tokens(&ast)?;
     update(ast_tokens_file.as_path(), &contents, mode)?;
 
-    let ast_nodes_file = project_root().join(codegen::AST_NODES);
+    let ast_nodes_file = project_root().join("crates/syntax/src/ast/generated/nodes.rs");
     let contents = generate_nodes(KINDS_SRC, &ast)?;
     update(ast_nodes_file.as_path(), &contents, mode)?;
 
@@ -61,7 +61,7 @@ fn generate_tokens(grammar: &AstSrc) -> Result<String> {
         }
     });
 
-    let pretty = crate::reformat(quote! {
+    let pretty = reformat(quote! {
         use crate::{SyntaxKind::{self, *}, SyntaxToken, ast::AstToken};
         #(#tokens)*
     })?
@@ -261,7 +261,7 @@ fn generate_nodes(kinds: KindsSrc<'_>, grammar: &AstSrc) -> Result<String> {
         }
     }
 
-    let pretty = crate::reformat(res)?;
+    let pretty = reformat(res)?;
     Ok(pretty)
 }
 
@@ -383,7 +383,7 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> Result<String> {
         }
     };
 
-    crate::reformat(ast)
+    reformat(ast)
 }
 
 fn to_upper_snake_case(s: &str) -> String {
diff --git a/xtask/src/lib.rs b/xtask/src/lib.rs
index 807ef587c..e790d995f 100644
--- a/xtask/src/lib.rs
+++ b/xtask/src/lib.rs
@@ -3,14 +3,15 @@
 //! See https://github.com/matklad/cargo-xtask/
 
 pub mod not_bash;
+pub mod codegen;
+mod ast_src;
+
 pub mod install;
 pub mod release;
 pub mod dist;
 pub mod pre_commit;
 pub mod metrics;
-
-pub mod codegen;
-mod ast_src;
+pub mod pre_cache;
 
 use std::{
     env,
@@ -21,7 +22,7 @@ use walkdir::{DirEntry, WalkDir};
 
 use crate::{
     codegen::Mode,
-    not_bash::{fs2, pushd, pushenv, rm_rf},
+    not_bash::{pushd, pushenv},
 };
 
 pub use anyhow::{bail, Context as _, Result};
@@ -62,17 +63,6 @@ pub fn run_rustfmt(mode: Mode) -> Result<()> {
     Ok(())
 }
 
-fn reformat(text: impl std::fmt::Display) -> Result<String> {
-    let _e = pushenv("RUSTUP_TOOLCHAIN", "stable");
-    ensure_rustfmt()?;
-    let stdout = run!(
-        "rustfmt --config-path {} --config fn_single_line=true", project_root().join("rustfmt.toml").display();
-        <text.to_string().as_bytes()
-    )?;
-    let preamble = "Generated file, do not edit by hand, see `xtask/src/codegen`";
-    Ok(format!("//! {}\n\n{}\n", preamble, stdout))
-}
-
 fn ensure_rustfmt() -> Result<()> {
     let out = run!("rustfmt --version")?;
     if !out.contains("stable") {
@@ -119,42 +109,6 @@ pub fn run_fuzzer() -> Result<()> {
     Ok(())
 }
 
-/// Cleans the `./target` dir after the build such that only
-/// dependencies are cached on CI.
-pub fn run_pre_cache() -> Result<()> {
-    let slow_tests_cookie = Path::new("./target/.slow_tests_cookie");
-    if !slow_tests_cookie.exists() {
-        panic!("slow tests were skipped on CI!")
-    }
-    rm_rf(slow_tests_cookie)?;
-
-    for entry in Path::new("./target/debug").read_dir()? {
-        let entry = entry?;
-        if entry.file_type().map(|it| it.is_file()).ok() == Some(true) {
-            // Can't delete yourself on windows :-(
-            if !entry.path().ends_with("xtask.exe") {
-                rm_rf(&entry.path())?
-            }
-        }
-    }
-
-    fs2::remove_file("./target/.rustc_info.json")?;
-    let to_delete = ["hir", "heavy_test", "xtask", "ide", "rust-analyzer"];
-    for &dir in ["./target/debug/deps", "target/debug/.fingerprint"].iter() {
-        for entry in Path::new(dir).read_dir()? {
-            let entry = entry?;
-            if to_delete.iter().any(|&it| entry.path().display().to_string().contains(it)) {
-                // Can't delete yourself on windows :-(
-                if !entry.path().ends_with("xtask.exe") {
-                    rm_rf(&entry.path())?
-                }
-            }
-        }
-    }
-
-    Ok(())
-}
-
 fn is_release_tag(tag: &str) -> bool {
     tag.len() == "2020-02-24".len() && tag.starts_with(|c: char| c.is_ascii_digit())
 }
diff --git a/xtask/src/main.rs b/xtask/src/main.rs
index b69b884e5..3f4aa5497 100644
--- a/xtask/src/main.rs
+++ b/xtask/src/main.rs
@@ -10,6 +10,7 @@
 
 use std::env;
 
+use codegen::CodegenCmd;
 use pico_args::Arguments;
 use xtask::{
     codegen::{self, Mode},
@@ -17,9 +18,10 @@ use xtask::{
     install::{ClientOpt, InstallCmd, Malloc, ServerOpt},
     metrics::MetricsCmd,
     not_bash::pushd,
+    pre_cache::PreCacheCmd,
     pre_commit, project_root,
     release::{PromoteCmd, ReleaseCmd},
-    run_clippy, run_fuzzer, run_pre_cache, run_rustfmt, Result,
+    run_clippy, run_fuzzer, run_rustfmt, Result,
 };
 
 fn main() -> Result<()> {
@@ -74,13 +76,9 @@ FLAGS:
             .run()
         }
         "codegen" => {
+            let features = args.contains("--features");
             args.finish()?;
-            codegen::generate_syntax(Mode::Overwrite)?;
-            codegen::generate_parser_tests(Mode::Overwrite)?;
-            codegen::generate_assists_tests(Mode::Overwrite)?;
-            codegen::generate_assists_docs(Mode::Overwrite)?;
-            codegen::generate_feature_docs(Mode::Overwrite)?;
-            Ok(())
+            CodegenCmd { features }.run()
         }
         "format" => {
             args.finish()?;
@@ -100,7 +98,7 @@ FLAGS:
         }
         "pre-cache" => {
             args.finish()?;
-            run_pre_cache()
+            PreCacheCmd.run()
         }
         "release" => {
             let dry_run = args.contains("--dry-run");
diff --git a/xtask/src/pre_cache.rs b/xtask/src/pre_cache.rs
new file mode 100644
index 000000000..47ba6ba24
--- /dev/null
+++ b/xtask/src/pre_cache.rs
@@ -0,0 +1,80 @@
+use std::{
+    fs::FileType,
+    path::{Path, PathBuf},
+};
+
+use anyhow::Result;
+
+use crate::not_bash::{fs2, rm_rf};
+
+pub struct PreCacheCmd;
+
+impl PreCacheCmd {
+    /// Cleans the `./target` dir after the build such that only
+    /// dependencies are cached on CI.
+    pub fn run(self) -> Result<()> {
+        let slow_tests_cookie = Path::new("./target/.slow_tests_cookie");
+        if !slow_tests_cookie.exists() {
+            panic!("slow tests were skipped on CI!")
+        }
+        rm_rf(slow_tests_cookie)?;
+
+        for path in read_dir("./target/debug", FileType::is_file)? {
+            // Can't delete yourself on windows :-(
+            if !path.ends_with("xtask.exe") {
+                rm_rf(&path)?
+            }
+        }
+
+        fs2::remove_file("./target/.rustc_info.json")?;
+
+        let to_delete = read_dir("./crates", FileType::is_dir)?
+            .into_iter()
+            .map(|path| path.file_name().unwrap().to_string_lossy().replace('-', "_"))
+            .collect::<Vec<_>>();
+
+        for &dir in ["./target/debug/deps", "target/debug/.fingerprint"].iter() {
+            for path in read_dir(dir, |_file_type| true)? {
+                if path.ends_with("xtask.exe") {
+                    continue;
+                }
+                let file_name = path.file_name().unwrap().to_string_lossy();
+                let (stem, _) = match rsplit_once(&file_name, '-') {
+                    Some(it) => it,
+                    None => {
+                        rm_rf(path)?;
+                        continue;
+                    }
+                };
+                let stem = stem.replace('-', "_");
+                if to_delete.contains(&stem) {
+                    rm_rf(path)?;
+                }
+            }
+        }
+
+        Ok(())
+    }
+}
+fn read_dir(path: impl AsRef<Path>, cond: impl Fn(&FileType) -> bool) -> Result<Vec<PathBuf>> {
+    read_dir_impl(path.as_ref(), &cond)
+}
+
+fn read_dir_impl(path: &Path, cond: &dyn Fn(&FileType) -> bool) -> Result<Vec<PathBuf>> {
+    let mut res = Vec::new();
+    for entry in path.read_dir()? {
+        let entry = entry?;
+        let file_type = entry.file_type()?;
+        if cond(&file_type) {
+            res.push(entry.path())
+        }
+    }
+    Ok(res)
+}
+
+fn rsplit_once(haystack: &str, delim: char) -> Option<(&str, &str)> {
+    let mut split = haystack.rsplitn(2, delim);
+    let suffix = split.next()?;
+    let prefix = split.next()?;
+    Some((prefix, suffix))
+}