Merge branch 'master' into assists_extract_enum

7 files changed, 279 insertions, 136 deletions

diff --git a/xtask/src/ast_src.rs b/xtask/src/ast_src.rs
index 394a7bc88..f60f0fb16 100644
--- a/xtask/src/ast_src.rs
+++ b/xtask/src/ast_src.rs
@@ -1058,7 +1058,7 @@ pub(crate) const AST_SRC: AstSrc = AstSrc {
         /// [Reference](https://doc.rust-lang.org/reference/expressions/block-expr.html)
         /// [Labels for blocks RFC](https://github.com/rust-lang/rfcs/blob/master/text/2046-label-break-value.md)
         struct BlockExpr: AttrsOwner, ModuleItemOwner {
-            T!['{'], statements: [Stmt], Expr, T!['}'],
+            Label, T!['{'], statements: [Stmt], Expr, T!['}'],
         }
 
         /// Return expression.
@@ -1153,10 +1153,12 @@ pub(crate) const AST_SRC: AstSrc = AstSrc {
         /// ```
         /// ❰ &foo ❱;
         /// ❰ &mut bar ❱;
+        /// ❰ &raw const bar ❱;
+        /// ❰ &raw mut bar ❱;
         /// ```
         ///
         /// [Reference](https://doc.rust-lang.org/reference/expressions/operator-expr.html#borrow-operators)
-        struct RefExpr: AttrsOwner { T![&], T![raw], T![mut], Expr }
+        struct RefExpr: AttrsOwner { T![&], T![raw], T![mut], T![const], Expr }
 
         /// Prefix operator call. This is either `!` or `*` or `-`.
         ///
diff --git a/xtask/src/codegen.rs b/xtask/src/codegen.rs
index b4907f4b2..f5f4b964a 100644
--- a/xtask/src/codegen.rs
+++ b/xtask/src/codegen.rs
@@ -8,13 +8,19 @@
 mod gen_syntax;
 mod gen_parser_tests;
 mod gen_assists_docs;
+mod gen_feature_docs;
 
-use std::{mem, path::Path};
+use std::{
+    fmt, mem,
+    path::{Path, PathBuf},
+};
 
-use crate::{not_bash::fs2, Result};
+use crate::{not_bash::fs2, project_root, Result};
 
 pub use self::{
-    gen_assists_docs::generate_assists_docs, gen_parser_tests::generate_parser_tests,
+    gen_assists_docs::{generate_assists_docs, generate_assists_tests},
+    gen_feature_docs::generate_feature_docs,
+    gen_parser_tests::generate_parser_tests,
     gen_syntax::generate_syntax,
 };
 
@@ -28,7 +34,6 @@ const AST_TOKENS: &str = "crates/ra_syntax/src/ast/generated/tokens.rs";
 
 const ASSISTS_DIR: &str = "crates/ra_assists/src/handlers";
 const ASSISTS_TESTS: &str = "crates/ra_assists/src/tests/generated.rs";
-const ASSISTS_DOCS: &str = "docs/user/assists.md";
 
 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
 pub enum Mode {
@@ -40,7 +45,7 @@ pub enum Mode {
 /// With verify = false,
 fn update(path: &Path, contents: &str, mode: Mode) -> Result<()> {
     match fs2::read_to_string(path) {
-        Ok(ref old_contents) if normalize(old_contents) == normalize(contents) => {
+        Ok(old_contents) if normalize(&old_contents) == normalize(contents) => {
             return Ok(());
         }
         _ => (),
@@ -58,35 +63,85 @@ fn update(path: &Path, contents: &str, mode: Mode) -> Result<()> {
 }
 
 fn extract_comment_blocks(text: &str) -> Vec<Vec<String>> {
-    do_extract_comment_blocks(text, false)
+    do_extract_comment_blocks(text, false).into_iter().map(|(_line, block)| block).collect()
+}
+
+fn extract_comment_blocks_with_empty_lines(tag: &str, text: &str) -> Vec<CommentBlock> {
+    assert!(tag.starts_with(char::is_uppercase));
+    let tag = format!("{}:", tag);
+    let mut res = Vec::new();
+    for (line, mut block) in do_extract_comment_blocks(text, true) {
+        let first = block.remove(0);
+        if first.starts_with(&tag) {
+            let id = first[tag.len()..].trim().to_string();
+            let block = CommentBlock { id, line, contents: block };
+            res.push(block);
+        }
+    }
+    res
 }
 
-fn extract_comment_blocks_with_empty_lines(text: &str) -> Vec<Vec<String>> {
-    do_extract_comment_blocks(text, true)
+struct CommentBlock {
+    id: String,
+    line: usize,
+    contents: Vec<String>,
 }
 
-fn do_extract_comment_blocks(text: &str, allow_blocks_with_empty_lines: bool) -> Vec<Vec<String>> {
+fn do_extract_comment_blocks(
+    text: &str,
+    allow_blocks_with_empty_lines: bool,
+) -> Vec<(usize, Vec<String>)> {
     let mut res = Vec::new();
 
     let prefix = "// ";
     let lines = text.lines().map(str::trim_start);
 
-    let mut block = vec![];
-    for line in lines {
+    let mut block = (0, vec![]);
+    for (line_num, line) in lines.enumerate() {
         if line == "//" && allow_blocks_with_empty_lines {
-            block.push(String::new());
+            block.1.push(String::new());
             continue;
         }
 
         let is_comment = line.starts_with(prefix);
         if is_comment {
-            block.push(line[prefix.len()..].to_string());
-        } else if !block.is_empty() {
-            res.push(mem::replace(&mut block, Vec::new()));
+            block.1.push(line[prefix.len()..].to_string());
+        } else {
+            if !block.1.is_empty() {
+                res.push(mem::take(&mut block));
+            }
+            block.0 = line_num + 2;
         }
     }
-    if !block.is_empty() {
-        res.push(mem::replace(&mut block, Vec::new()))
+    if !block.1.is_empty() {
+        res.push(block)
     }
     res
 }
+
+#[derive(Debug)]
+struct Location {
+    file: PathBuf,
+    line: usize,
+}
+
+impl Location {
+    fn new(file: PathBuf, line: usize) -> Self {
+        Self { file, line }
+    }
+}
+
+impl fmt::Display for Location {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let path = self.file.strip_prefix(&project_root()).unwrap().display().to_string();
+        let path = path.replace('\\', "/");
+        let name = self.file.file_name().unwrap();
+        write!(
+            f,
+            "https://github.com/rust-analyzer/rust-analyzer/blob/master/{}#L{}[{}]",
+            path,
+            self.line,
+            name.to_str().unwrap()
+        )
+    }
+}
diff --git a/xtask/src/codegen/gen_assists_docs.rs b/xtask/src/codegen/gen_assists_docs.rs
index 4bd6b5f0c..526941f73 100644
--- a/xtask/src/codegen/gen_assists_docs.rs
+++ b/xtask/src/codegen/gen_assists_docs.rs
@@ -1,102 +1,113 @@
 //! Generates `assists.md` documentation.
 
-use std::{fs, path::Path};
+use std::{fmt, fs, path::Path};
 
 use crate::{
-    codegen::{self, extract_comment_blocks_with_empty_lines, Mode},
+    codegen::{self, extract_comment_blocks_with_empty_lines, Location, Mode},
     project_root, rust_files, Result,
 };
 
+pub fn generate_assists_tests(mode: Mode) -> Result<()> {
+    let assists = Assist::collect()?;
+    generate_tests(&assists, mode)
+}
+
 pub fn generate_assists_docs(mode: Mode) -> Result<()> {
-    let assists = collect_assists()?;
-    generate_tests(&assists, mode)?;
-    generate_docs(&assists, mode)?;
-    Ok(())
+    let assists = Assist::collect()?;
+    let contents = assists.into_iter().map(|it| it.to_string()).collect::<Vec<_>>().join("\n\n");
+    let contents = contents.trim().to_string() + "\n";
+    let dst = project_root().join("docs/user/generated_assists.adoc");
+    codegen::update(&dst, &contents, mode)
 }
 
 #[derive(Debug)]
 struct Assist {
     id: String,
+    location: Location,
     doc: String,
     before: String,
     after: String,
 }
 
-fn hide_hash_comments(text: &str) -> String {
-    text.split('\n') // want final newline
-        .filter(|&it| !(it.starts_with("# ") || it == "#"))
-        .map(|it| format!("{}\n", it))
-        .collect()
-}
-
-fn reveal_hash_comments(text: &str) -> String {
-    text.split('\n') // want final newline
-        .map(|it| {
-            if it.starts_with("# ") {
-                &it[2..]
-            } else if it == "#" {
-                ""
-            } else {
-                it
-            }
-        })
-        .map(|it| format!("{}\n", it))
-        .collect()
-}
-
-fn collect_assists() -> Result<Vec<Assist>> {
-    let mut res = Vec::new();
-    for path in rust_files(&project_root().join(codegen::ASSISTS_DIR)) {
-        collect_file(&mut res, path.as_path())?;
-    }
-    res.sort_by(|lhs, rhs| lhs.id.cmp(&rhs.id));
-    return Ok(res);
-
-    fn collect_file(acc: &mut Vec<Assist>, path: &Path) -> Result<()> {
-        let text = fs::read_to_string(path)?;
-        let comment_blocks = extract_comment_blocks_with_empty_lines(&text);
-
-        for block in comment_blocks {
-            // FIXME: doesn't support blank lines yet, need to tweak
-            // `extract_comment_blocks` for that.
-            let mut lines = block.iter();
-            let first_line = lines.next().unwrap();
-            if !first_line.starts_with("Assist: ") {
-                continue;
-            }
-            let id = first_line["Assist: ".len()..].to_string();
-            assert!(
-                id.chars().all(|it| it.is_ascii_lowercase() || it == '_'),
-                "invalid assist id: {:?}",
-                id
-            );
-
-            let doc = take_until(lines.by_ref(), "```").trim().to_string();
-            assert!(
-                doc.chars().next().unwrap().is_ascii_uppercase() && doc.ends_with('.'),
-                "\n\n{}: assist docs should be proper sentences, with capitalization and a full stop at the end.\n\n{}\n\n",
-                id, doc,
-            );
-
-            let before = take_until(lines.by_ref(), "```");
-
-            assert_eq!(lines.next().unwrap().as_str(), "->");
-            assert_eq!(lines.next().unwrap().as_str(), "```");
-            let after = take_until(lines.by_ref(), "```");
-            acc.push(Assist { id, doc, before, after })
+impl Assist {
+    fn collect() -> Result<Vec<Assist>> {
+        let mut res = Vec::new();
+        for path in rust_files(&project_root().join(codegen::ASSISTS_DIR)) {
+            collect_file(&mut res, path.as_path())?;
         }
+        res.sort_by(|lhs, rhs| lhs.id.cmp(&rhs.id));
+        return Ok(res);
+
+        fn collect_file(acc: &mut Vec<Assist>, path: &Path) -> Result<()> {
+            let text = fs::read_to_string(path)?;
+            let comment_blocks = extract_comment_blocks_with_empty_lines("Assist", &text);
+
+            for block in comment_blocks {
+                // FIXME: doesn't support blank lines yet, need to tweak
+                // `extract_comment_blocks` for that.
+                let id = block.id;
+                assert!(
+                    id.chars().all(|it| it.is_ascii_lowercase() || it == '_'),
+                    "invalid assist id: {:?}",
+                    id
+                );
+                let mut lines = block.contents.iter();
+
+                let doc = take_until(lines.by_ref(), "```").trim().to_string();
+                assert!(
+                    doc.chars().next().unwrap().is_ascii_uppercase() && doc.ends_with('.'),
+                    "\n\n{}: assist docs should be proper sentences, with capitalization and a full stop at the end.\n\n{}\n\n",
+                    id, doc,
+                );
+
+                let before = take_until(lines.by_ref(), "```");
+
+                assert_eq!(lines.next().unwrap().as_str(), "->");
+                assert_eq!(lines.next().unwrap().as_str(), "```");
+                let after = take_until(lines.by_ref(), "```");
+                let location = Location::new(path.to_path_buf(), block.line);
+                acc.push(Assist { id, location, doc, before, after })
+            }
 
-        fn take_until<'a>(lines: impl Iterator<Item = &'a String>, marker: &str) -> String {
-            let mut buf = Vec::new();
-            for line in lines {
-                if line == marker {
-                    break;
+            fn take_until<'a>(lines: impl Iterator<Item = &'a String>, marker: &str) -> String {
+                let mut buf = Vec::new();
+                for line in lines {
+                    if line == marker {
+                        break;
+                    }
+                    buf.push(line.clone());
                 }
-                buf.push(line.clone());
+                buf.join("\n")
             }
-            buf.join("\n")
+            Ok(())
         }
-        Ok(())
+    }
+}
+
+impl fmt::Display for Assist {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let before = self.before.replace("<|>", "┃"); // Unicode pseudo-graphics bar
+        let after = self.after.replace("<|>", "┃");
+        writeln!(
+            f,
+            "[discrete]\n=== `{}`
+**Source:** {}
+
+{}
+
+.Before
+```rust
+{}```
+
+.After
+```rust
+{}```",
+            self.id,
+            self.location,
+            self.doc,
+            hide_hash_comments(&before),
+            hide_hash_comments(&after)
+        )
     }
 }
 
@@ -127,33 +138,24 @@ r#####"
     codegen::update(&project_root().join(codegen::ASSISTS_TESTS), &buf, mode)
 }
 
-fn generate_docs(assists: &[Assist], mode: Mode) -> Result<()> {
-    let mut buf = String::from(
-        "# Assists\n\nCursor position or selection is signified by `┃` character.\n\n",
-    );
-
-    for assist in assists {
-        let before = assist.before.replace("<|>", "┃"); // Unicode pseudo-graphics bar
-        let after = assist.after.replace("<|>", "┃");
-        let docs = format!(
-            "
-## `{}`
-
-{}
-
-```rust
-// BEFORE
-{}
-// AFTER
-{}```
-",
-            assist.id,
-            assist.doc,
-            hide_hash_comments(&before),
-            hide_hash_comments(&after)
-        );
-        buf.push_str(&docs);
-    }
+fn hide_hash_comments(text: &str) -> String {
+    text.split('\n') // want final newline
+        .filter(|&it| !(it.starts_with("# ") || it == "#"))
+        .map(|it| format!("{}\n", it))
+        .collect()
+}
 
-    codegen::update(&project_root().join(codegen::ASSISTS_DOCS), &buf, mode)
+fn reveal_hash_comments(text: &str) -> String {
+    text.split('\n') // want final newline
+        .map(|it| {
+            if it.starts_with("# ") {
+                &it[2..]
+            } else if it == "#" {
+                ""
+            } else {
+                it
+            }
+        })
+        .map(|it| format!("{}\n", it))
+        .collect()
 }
diff --git a/xtask/src/codegen/gen_feature_docs.rs b/xtask/src/codegen/gen_feature_docs.rs
new file mode 100644
index 000000000..31bc3839d
--- /dev/null
+++ b/xtask/src/codegen/gen_feature_docs.rs
@@ -0,0 +1,75 @@
+//! Generates `assists.md` documentation.
+
+use std::{fmt, fs, path::PathBuf};
+
+use crate::{
+    codegen::{self, extract_comment_blocks_with_empty_lines, Location, Mode},
+    project_root, rust_files, Result,
+};
+
+pub fn generate_feature_docs(mode: Mode) -> Result<()> {
+    let features = Feature::collect()?;
+    let contents = features.into_iter().map(|it| it.to_string()).collect::<Vec<_>>().join("\n\n");
+    let contents = contents.trim().to_string() + "\n";
+    let dst = project_root().join("docs/user/generated_features.adoc");
+    codegen::update(&dst, &contents, mode)?;
+    Ok(())
+}
+
+#[derive(Debug)]
+struct Feature {
+    id: String,
+    location: Location,
+    doc: String,
+}
+
+impl Feature {
+    fn collect() -> Result<Vec<Feature>> {
+        let mut res = Vec::new();
+        for path in rust_files(&project_root()) {
+            collect_file(&mut res, path)?;
+        }
+        res.sort_by(|lhs, rhs| lhs.id.cmp(&rhs.id));
+        return Ok(res);
+
+        fn collect_file(acc: &mut Vec<Feature>, path: PathBuf) -> Result<()> {
+            let text = fs::read_to_string(&path)?;
+            let comment_blocks = extract_comment_blocks_with_empty_lines("Feature", &text);
+
+            for block in comment_blocks {
+                let id = block.id;
+                assert!(is_valid_feature_name(&id), "invalid feature name: {:?}", id);
+                let doc = block.contents.join("\n");
+                let location = Location::new(path.clone(), block.line);
+                acc.push(Feature { id, location, doc })
+            }
+
+            Ok(())
+        }
+    }
+}
+
+fn is_valid_feature_name(feature: &str) -> bool {
+    'word: for word in feature.split_whitespace() {
+        for &short in ["to", "and"].iter() {
+            if word == short {
+                continue 'word;
+            }
+        }
+        for &short in ["To", "And"].iter() {
+            if word == short {
+                return false;
+            }
+        }
+        if !word.starts_with(char::is_uppercase) {
+            return false;
+        }
+    }
+    true
+}
+
+impl fmt::Display for Feature {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        writeln!(f, "=== {}\n**Source:** {}\n{}", self.id, self.location, self.doc)
+    }
+}
diff --git a/xtask/src/lib.rs b/xtask/src/lib.rs
index 2b7a461e5..739f49f7b 100644
--- a/xtask/src/lib.rs
+++ b/xtask/src/lib.rs
@@ -160,6 +160,8 @@ pub fn run_release(dry_run: bool) -> Result<()> {
         run!("git reset --hard tags/nightly")?;
         run!("git push")?;
     }
+    codegen::generate_assists_docs(Mode::Overwrite)?;
+    codegen::generate_feature_docs(Mode::Overwrite)?;
 
     let website_root = project_root().join("../rust-analyzer.github.io");
     let changelog_dir = website_root.join("./thisweek/_posts");
@@ -191,7 +193,11 @@ Release: release:{}[]
     let path = changelog_dir.join(format!("{}-changelog-{}.adoc", today, changelog_n));
     fs2::write(&path, &contents)?;
 
-    fs2::copy(project_root().join("./docs/user/readme.adoc"), website_root.join("manual.adoc"))?;
+    for &adoc in ["manual.adoc", "generated_features.adoc", "generated_assists.adoc"].iter() {
+        let src = project_root().join("./docs/user/").join(adoc);
+        let dst = website_root.join(adoc);
+        fs2::copy(src, dst)?;
+    }
 
     let tags = run!("git tag --list"; echo = false)?;
     let prev_tag = tags.lines().filter(|line| is_release_tag(line)).last().unwrap();
diff --git a/xtask/src/main.rs b/xtask/src/main.rs
index dff3ce4a1..81bb3a33f 100644
--- a/xtask/src/main.rs
+++ b/xtask/src/main.rs
@@ -74,7 +74,9 @@ FLAGS:
             args.finish()?;
             codegen::generate_syntax(Mode::Overwrite)?;
             codegen::generate_parser_tests(Mode::Overwrite)?;
+            codegen::generate_assists_tests(Mode::Overwrite)?;
             codegen::generate_assists_docs(Mode::Overwrite)?;
+            codegen::generate_feature_docs(Mode::Overwrite)?;
             Ok(())
         }
         "format" => {
diff --git a/xtask/tests/tidy.rs b/xtask/tests/tidy.rs
index 2e9fcf07c..d38ac7f17 100644
--- a/xtask/tests/tidy.rs
+++ b/xtask/tests/tidy.rs
@@ -25,7 +25,7 @@ fn generated_tests_are_fresh() {
 
 #[test]
 fn generated_assists_are_fresh() {
-    if let Err(error) = codegen::generate_assists_docs(Mode::Verify) {
+    if let Err(error) = codegen::generate_assists_tests(Mode::Verify) {
         panic!("{}. Please update assists by running `cargo xtask codegen`", error);
     }
 }
@@ -95,7 +95,7 @@ impl TidyDocs {
     fn visit(&mut self, path: &Path, text: &str) {
         // Test hopefully don't really need comments, and for assists we already
         // have special comments which are source of doc tests and user docs.
-        if is_exclude_dir(path, &["tests", "test_data", "handlers"]) {
+        if is_exclude_dir(path, &["tests", "test_data"]) {
             return;
         }
 
@@ -110,9 +110,12 @@ impl TidyDocs {
 
         if first_line.starts_with("//!") {
             if first_line.contains("FIXME") {
-                self.contains_fixme.push(path.to_path_buf())
+                self.contains_fixme.push(path.to_path_buf());
             }
         } else {
+            if text.contains("// Feature:") || text.contains("// Assist:") {
+                return;
+            }
             self.missing_docs.push(path.display().to_string());
         }
 
@@ -170,13 +173,11 @@ impl TidyDocs {
 }
 
 fn is_exclude_dir(p: &Path, dirs_to_exclude: &[&str]) -> bool {
-    let mut cur_path = p;
-    while let Some(path) = cur_path.parent() {
-        if dirs_to_exclude.iter().any(|dir| path.ends_with(dir)) {
-            return true;
-        }
-        cur_path = path;
-    }
-
-    false
+    p.strip_prefix(project_root())
+        .unwrap()
+        .components()
+        .rev()
+        .skip(1)
+        .filter_map(|it| it.as_os_str().to_str())
+        .any(|it| dirs_to_exclude.contains(&it))
 }