Merge branch 'master' into feature/themes

7 files changed, 267 insertions, 77 deletions

diff --git a/xtask/src/codegen.rs b/xtask/src/codegen.rs
index 948b86719..4ec8ab75a 100644
--- a/xtask/src/codegen.rs
+++ b/xtask/src/codegen.rs
@@ -7,12 +7,22 @@
 
 mod gen_syntax;
 mod gen_parser_tests;
+mod gen_assists_docs;
 
-use std::{fs, path::Path};
+use std::{
+    fs,
+    io::Write,
+    mem,
+    path::Path,
+    process::{Command, Stdio},
+};
 
-use crate::Result;
+use crate::{project_root, Result};
 
-pub use self::{gen_parser_tests::generate_parser_tests, gen_syntax::generate_syntax};
+pub use self::{
+    gen_assists_docs::generate_assists_docs, gen_parser_tests::generate_parser_tests,
+    gen_syntax::generate_syntax,
+};
 
 pub const GRAMMAR: &str = "crates/ra_syntax/src/grammar.ron";
 const GRAMMAR_DIR: &str = "crates/ra_parser/src/grammar";
@@ -22,6 +32,10 @@ const ERR_INLINE_TESTS_DIR: &str = "crates/ra_syntax/test_data/parser/inline/err
 pub const SYNTAX_KINDS: &str = "crates/ra_parser/src/syntax_kind/generated.rs";
 pub const AST: &str = "crates/ra_syntax/src/ast/generated.rs";
 
+const ASSISTS_DIR: &str = "crates/ra_assists/src/assists";
+const ASSISTS_TESTS: &str = "crates/ra_assists/src/doc_tests/generated.rs";
+const ASSISTS_DOCS: &str = "docs/user/assists.md";
+
 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
 pub enum Mode {
     Overwrite,
@@ -30,7 +44,7 @@ pub enum Mode {
 
 /// A helper to update file on disk if it has changed.
 /// With verify = false,
-pub fn update(path: &Path, contents: &str, mode: Mode) -> Result<()> {
+fn update(path: &Path, contents: &str, mode: Mode) -> Result<()> {
     match fs::read_to_string(path) {
         Ok(ref old_contents) if old_contents == contents => {
             return Ok(());
@@ -44,3 +58,53 @@ pub fn update(path: &Path, contents: &str, mode: Mode) -> Result<()> {
     fs::write(path, contents)?;
     Ok(())
 }
+
+fn reformat(text: impl std::fmt::Display) -> Result<String> {
+    let mut rustfmt = Command::new("rustfmt")
+        .arg("--config-path")
+        .arg(project_root().join("rustfmt.toml"))
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .spawn()?;
+    write!(rustfmt.stdin.take().unwrap(), "{}", text)?;
+    let output = rustfmt.wait_with_output()?;
+    let stdout = String::from_utf8(output.stdout)?;
+    let preamble = "Generated file, do not edit by hand, see `crate/ra_tools/src/codegen`";
+    Ok(format!("//! {}\n\n{}", preamble, stdout))
+}
+
+fn extract_comment_blocks(text: &str) -> Vec<Vec<String>> {
+    do_extract_comment_blocks(text, false)
+}
+
+fn extract_comment_blocks_with_empty_lines(text: &str) -> Vec<Vec<String>> {
+    do_extract_comment_blocks(text, true)
+}
+
+fn do_extract_comment_blocks(text: &str, allow_blocks_with_empty_lins: bool) -> Vec<Vec<String>> {
+    let mut res = Vec::new();
+
+    let prefix = "// ";
+    let lines = text.lines().map(str::trim_start);
+
+    let mut block = vec![];
+    for line in lines {
+        if line == "//" && allow_blocks_with_empty_lins {
+            block.push(String::new());
+            continue;
+        }
+
+        let is_comment = line.starts_with(prefix);
+        if is_comment {
+            block.push(line[prefix.len()..].to_string());
+        } else {
+            if !block.is_empty() {
+                res.push(mem::replace(&mut block, Vec::new()))
+            }
+        }
+    }
+    if !block.is_empty() {
+        res.push(mem::replace(&mut block, Vec::new()))
+    }
+    res
+}
diff --git a/xtask/src/codegen/gen_assists_docs.rs b/xtask/src/codegen/gen_assists_docs.rs
new file mode 100644
index 000000000..05afda8f1
--- /dev/null
+++ b/xtask/src/codegen/gen_assists_docs.rs
@@ -0,0 +1,137 @@
+use std::{fs, path::Path};
+
+use crate::{
+    codegen::{self, extract_comment_blocks_with_empty_lines, Mode},
+    project_root, Result,
+};
+
+pub fn generate_assists_docs(mode: Mode) -> Result<()> {
+    let assists = collect_assists()?;
+    generate_tests(&assists, mode)?;
+    generate_docs(&assists, mode)?;
+    Ok(())
+}
+
+#[derive(Debug)]
+struct Assist {
+    id: String,
+    doc: String,
+    before: String,
+    after: String,
+}
+
+fn collect_assists() -> Result<Vec<Assist>> {
+    let mut res = Vec::new();
+    for entry in fs::read_dir(project_root().join(codegen::ASSISTS_DIR))? {
+        let entry = entry?;
+        let path = entry.path();
+        if path.is_file() {
+            collect_file(&mut res, path.as_path())?;
+        }
+    }
+    res.sort_by(|lhs, rhs| lhs.id.cmp(&rhs.id));
+    return Ok(res);
+
+    fn collect_file(acc: &mut Vec<Assist>, path: &Path) -> Result<()> {
+        let text = fs::read_to_string(path)?;
+        let comment_blocks = extract_comment_blocks_with_empty_lines(&text);
+
+        for block in comment_blocks {
+            // FIXME: doesn't support blank lines yet, need to tweak
+            // `extract_comment_blocks` for that.
+            let mut lines = block.iter();
+            let first_line = lines.next().unwrap();
+            if !first_line.starts_with("Assist: ") {
+                continue;
+            }
+            let id = first_line["Assist: ".len()..].to_string();
+            assert!(
+                id.chars().all(|it| it.is_ascii_lowercase() || it == '_'),
+                "invalid assist id: {:?}",
+                id
+            );
+
+            let doc = take_until(lines.by_ref(), "```").trim().to_string();
+            assert!(
+                doc.chars().next().unwrap().is_ascii_uppercase() && doc.ends_with('.'),
+                "\n\n{}: assist docs should be proper sentences, with capitalization and a full stop at the end.\n\n{}\n\n",
+                id, doc,
+            );
+
+            let before = take_until(lines.by_ref(), "```");
+
+            assert_eq!(lines.next().unwrap().as_str(), "->");
+            assert_eq!(lines.next().unwrap().as_str(), "```");
+            let after = take_until(lines.by_ref(), "```");
+            acc.push(Assist { id, doc, before, after })
+        }
+
+        fn take_until<'a>(lines: impl Iterator<Item = &'a String>, marker: &str) -> String {
+            let mut buf = Vec::new();
+            for line in lines {
+                if line == marker {
+                    break;
+                }
+                buf.push(line.clone());
+            }
+            buf.join("\n")
+        }
+        Ok(())
+    }
+}
+
+fn generate_tests(assists: &[Assist], mode: Mode) -> Result<()> {
+    let mut buf = String::from("use super::check;\n");
+
+    for assist in assists.iter() {
+        let test = format!(
+            r######"
+#[test]
+fn doctest_{}() {{
+    check(
+        "{}",
+r#####"
+{}
+"#####, r#####"
+{}
+"#####)
+}}
+"######,
+            assist.id, assist.id, assist.before, assist.after
+        );
+
+        buf.push_str(&test)
+    }
+    let buf = codegen::reformat(buf)?;
+    codegen::update(&project_root().join(codegen::ASSISTS_TESTS), &buf, mode)
+}
+
+fn generate_docs(assists: &[Assist], mode: Mode) -> Result<()> {
+    let mut buf = String::from(
+        "# Assists\n\nCursor position or selection is signified by `┃` character.\n\n",
+    );
+
+    for assist in assists {
+        let before = assist.before.replace("<|>", "┃"); // Unicode pseudo-graphics bar
+        let after = assist.after.replace("<|>", "┃");
+        let docs = format!(
+            "
+## `{}`
+
+{}
+
+```rust
+// BEFORE
+{}
+
+// AFTER
+{}
+```
+",
+            assist.id, assist.doc, before, after
+        );
+        buf.push_str(&docs);
+    }
+
+    codegen::update(&project_root().join(codegen::ASSISTS_DOCS), &buf, mode)
+}
diff --git a/xtask/src/codegen/gen_parser_tests.rs b/xtask/src/codegen/gen_parser_tests.rs
index 0f550d948..d0f0f683b 100644
--- a/xtask/src/codegen/gen_parser_tests.rs
+++ b/xtask/src/codegen/gen_parser_tests.rs
@@ -3,12 +3,12 @@
 
 use std::{
     collections::HashMap,
-    fs,
+    fs, iter,
     path::{Path, PathBuf},
 };
 
 use crate::{
-    codegen::{self, update, Mode},
+    codegen::{self, extract_comment_blocks, update, Mode},
     project_root, Result,
 };
 
@@ -56,48 +56,29 @@ struct Tests {
     pub err: HashMap<String, Test>,
 }
 
-fn collect_tests(s: &str) -> Vec<(usize, Test)> {
-    let mut res = vec![];
-    let prefix = "// ";
-    let lines = s.lines().map(str::trim_start).enumerate();
-
-    let mut block = vec![];
-    for (line_idx, line) in lines {
-        let is_comment = line.starts_with(prefix);
-        if is_comment {
-            block.push((line_idx, &line[prefix.len()..]));
+fn collect_tests(s: &str) -> Vec<Test> {
+    let mut res = Vec::new();
+    for comment_block in extract_comment_blocks(s) {
+        let first_line = &comment_block[0];
+        let (name, ok) = if first_line.starts_with("test ") {
+            let name = first_line["test ".len()..].to_string();
+            (name, true)
+        } else if first_line.starts_with("test_err ") {
+            let name = first_line["test_err ".len()..].to_string();
+            (name, false)
         } else {
-            process_block(&mut res, &block);
-            block.clear();
-        }
-    }
-    process_block(&mut res, &block);
-    return res;
-
-    fn process_block(acc: &mut Vec<(usize, Test)>, block: &[(usize, &str)]) {
-        if block.is_empty() {
-            return;
-        }
-        let mut ok = true;
-        let mut block = block.iter();
-        let (start_line, name) = loop {
-            match block.next() {
-                Some(&(idx, line)) if line.starts_with("test ") => {
-                    break (idx, line["test ".len()..].to_string());
-                }
-                Some(&(idx, line)) if line.starts_with("test_err ") => {
-                    ok = false;
-                    break (idx, line["test_err ".len()..].to_string());
-                }
-                Some(_) => (),
-                None => return,
-            }
+            continue;
         };
-        let text: String =
-            block.map(|(_, line)| *line).chain(std::iter::once("")).collect::<Vec<_>>().join("\n");
+        let text: String = comment_block[1..]
+            .iter()
+            .cloned()
+            .chain(iter::once(String::new()))
+            .collect::<Vec<_>>()
+            .join("\n");
         assert!(!text.trim().is_empty() && text.ends_with('\n'));
-        acc.push((start_line, Test { name, text, ok }))
+        res.push(Test { name, text, ok })
     }
+    res
 }
 
 fn tests_from_dir(dir: &Path) -> Result<Tests> {
@@ -118,15 +99,13 @@ fn tests_from_dir(dir: &Path) -> Result<Tests> {
     fn process_file(res: &mut Tests, path: &Path) -> Result<()> {
         let text = fs::read_to_string(path)?;
 
-        for (_, test) in collect_tests(&text) {
+        for test in collect_tests(&text) {
             if test.ok {
                 if let Some(old_test) = res.ok.insert(test.name.clone(), test) {
-                    Err(format!("Duplicate test: {}", old_test.name))?
-                }
-            } else {
-                if let Some(old_test) = res.err.insert(test.name.clone(), test) {
-                    Err(format!("Duplicate test: {}", old_test.name))?
+                    return Err(format!("Duplicate test: {}", old_test.name).into());
                 }
+            } else if let Some(old_test) = res.err.insert(test.name.clone(), test) {
+                return Err(format!("Duplicate test: {}", old_test.name).into());
             }
         }
         Ok(())
diff --git a/xtask/src/codegen/gen_syntax.rs b/xtask/src/codegen/gen_syntax.rs
index 6a81c0e4d..88f2ac0e3 100644
--- a/xtask/src/codegen/gen_syntax.rs
+++ b/xtask/src/codegen/gen_syntax.rs
@@ -3,12 +3,7 @@
 //! Specifically, it generates the `SyntaxKind` enum and a number of newtype
 //! wrappers around `SyntaxNode` which implement `ra_syntax::AstNode`.
 
-use std::{
-    collections::BTreeMap,
-    fs,
-    io::Write,
-    process::{Command, Stdio},
-};
+use std::{collections::BTreeMap, fs};
 
 use proc_macro2::{Punct, Spacing};
 use quote::{format_ident, quote};
@@ -163,7 +158,7 @@ fn generate_ast(grammar: &Grammar) -> Result<String> {
         #(#nodes)*
     };
 
-    let pretty = reformat(ast)?;
+    let pretty = codegen::reformat(ast)?;
     Ok(pretty)
 }
 
@@ -276,21 +271,7 @@ fn generate_syntax_kinds(grammar: &Grammar) -> Result<String> {
         }
     };
 
-    reformat(ast)
-}
-
-fn reformat(text: impl std::fmt::Display) -> Result<String> {
-    let mut rustfmt = Command::new("rustfmt")
-        .arg("--config-path")
-        .arg(project_root().join("rustfmt.toml"))
-        .stdin(Stdio::piped())
-        .stdout(Stdio::piped())
-        .spawn()?;
-    write!(rustfmt.stdin.take().unwrap(), "{}", text)?;
-    let output = rustfmt.wait_with_output()?;
-    let stdout = String::from_utf8(output.stdout)?;
-    let preamble = "Generated file, do not edit by hand, see `crate/ra_tools/src/codegen`";
-    Ok(format!("//! {}\n\n{}", preamble, stdout))
+    codegen::reformat(ast)
 }
 
 #[derive(Deserialize, Debug)]
diff --git a/xtask/src/main.rs b/xtask/src/main.rs
index db901ced2..e04e45f15 100644
--- a/xtask/src/main.rs
+++ b/xtask/src/main.rs
@@ -1,5 +1,12 @@
-//! FIXME: write short doc here
-
+//! See https://github.com/matklad/cargo-xtask/.
+//!
+//! This binary defines various auxiliary build commands, which are not
+//! expressible with just `cargo`. Notably, it provides `cargo xtask codegen`
+//! for code generation and `cargo xtask install` for installation of
+//! rust-analyzer server and client.
+//!
+//! This binary is integrated into the `cargo` command line by using an alias in
+//! `.cargo/config`.
 mod help;
 
 use core::fmt::Write;
@@ -53,7 +60,7 @@ fn main() -> Result<()> {
             matches.finish().or_else(handle_extra_flags)?;
             let opts = InstallOpt {
                 client: if server { None } else { Some(ClientOpt::VsCode) },
-                server: if client_code { None } else { Some(ServerOpt { jemalloc: jemalloc }) },
+                server: if client_code { None } else { Some(ServerOpt { jemalloc }) },
             };
             install(opts)?
         }
@@ -64,6 +71,7 @@ fn main() -> Result<()> {
             }
             codegen::generate_syntax(Mode::Overwrite)?;
             codegen::generate_parser_tests(Mode::Overwrite)?;
+            codegen::generate_assists_docs(Mode::Overwrite)?;
         }
         "format" => {
             if matches.contains(["-h", "--help"]) {
@@ -158,6 +166,17 @@ fn fix_path_for_mac() -> Result<()> {
 }
 
 fn install_client(ClientOpt::VsCode: ClientOpt) -> Result<()> {
+    let npm_version = Cmd {
+        unix: r"npm --version",
+        windows: r"cmd.exe /c npm.cmd --version",
+        work_dir: "./editors/code",
+    }
+    .run();
+
+    if npm_version.is_err() {
+        eprintln!("\nERROR: `npm --version` failed, `npm` is required to build the VS Code plugin")
+    }
+
     Cmd { unix: r"npm ci", windows: r"cmd.exe /c npm.cmd ci", work_dir: "./editors/code" }.run()?;
     Cmd {
         unix: r"npm run package",
@@ -210,6 +229,7 @@ fn install_server(opts: ServerOpt) -> Result<()> {
     let mut old_rust = false;
     if let Ok(output) = run_with_output("cargo --version", ".") {
         if let Ok(stdout) = String::from_utf8(output.stdout) {
+            println!("{}", stdout);
             if !check_version(&stdout, REQUIRED_RUST_VERSION) {
                 old_rust = true;
             }
diff --git a/xtask/tests/tidy-tests/cli.rs b/xtask/tests/tidy-tests/cli.rs
index 543c7d7c4..573ffadbf 100644
--- a/xtask/tests/tidy-tests/cli.rs
+++ b/xtask/tests/tidy-tests/cli.rs
@@ -19,6 +19,13 @@ fn generated_tests_are_fresh() {
 }
 
 #[test]
+fn generated_assists_are_fresh() {
+    if let Err(error) = codegen::generate_assists_docs(Mode::Verify) {
+        panic!("{}. Please update assists by running `cargo xtask codegen`", error);
+    }
+}
+
+#[test]
 fn check_code_formatting() {
     if let Err(error) = run_rustfmt(Mode::Verify) {
         panic!("{}. Please format the code by running `cargo format`", error);
diff --git a/xtask/tests/tidy-tests/docs.rs b/xtask/tests/tidy-tests/docs.rs
index fe5852bc6..6a629ce63 100644
--- a/xtask/tests/tidy-tests/docs.rs
+++ b/xtask/tests/tidy-tests/docs.rs
@@ -8,7 +8,9 @@ use walkdir::{DirEntry, WalkDir};
 use xtask::project_root;
 
 fn is_exclude_dir(p: &Path) -> bool {
-    let exclude_dirs = ["tests", "test_data"];
+    // Test hopefully don't really need comments, and for assists we already
+    // have special comments which are source of doc tests and user docs.
+    let exclude_dirs = ["tests", "test_data", "assists"];
     let mut cur_path = p;
     while let Some(path) = cur_path.parent() {
         if exclude_dirs.iter().any(|dir| path.ends_with(dir)) {
@@ -27,7 +29,7 @@ fn is_exclude_file(d: &DirEntry) -> bool {
 }
 
 fn is_hidden(entry: &DirEntry) -> bool {
-    entry.file_name().to_str().map(|s| s.starts_with(".")).unwrap_or(false)
+    entry.file_name().to_str().map(|s| s.starts_with('.')).unwrap_or(false)
 }
 
 #[test]