13 files changed, 259 insertions, 68 deletions

diff --git a/crates/ra_ide/src/snapshots/highlight_doctest.html b/crates/ra_ide/src/snapshots/highlight_doctest.html
index 63199cdbe..ac546806e 100644
--- a/crates/ra_ide/src/snapshots/highlight_doctest.html
+++ b/crates/ra_ide/src/snapshots/highlight_doctest.html
@@ -32,7 +32,10 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .keyword.unsafe     { color: #BC8383; font-weight: bold; }
 .control            { font-style: italic; }
 </style>
-<pre><code><span class="keyword">struct</span> <span class="struct declaration">Foo</span> {
+<pre><code><span class="comment documentation">/// ```</span>
+<span class="comment documentation">/// </span><span class="keyword">let</span> _ = <span class="string_literal">"early doctests should not go boom"</span>;
+<span class="comment documentation">/// ```</span>
+<span class="keyword">struct</span> <span class="struct declaration">Foo</span> {
     <span class="field declaration">bar</span>: <span class="builtin_type">bool</span>,
 }
 
diff --git a/crates/ra_ide/src/ssr.rs b/crates/ra_ide/src/ssr.rs
index 59c230f6c..03f18c617 100644
--- a/crates/ra_ide/src/ssr.rs
+++ b/crates/ra_ide/src/ssr.rs
@@ -9,6 +9,7 @@ use ra_ssr::{MatchFinder, SsrError, SsrRule};
 // Search and replace with named wildcards that will match any expression, type, path, pattern or item.
 // The syntax for a structural search replace command is `<search_pattern> ==>> <replace_pattern>`.
 // A `$<name>` placeholder in the search pattern will match any AST node and `$<name>` will reference it in the replacement.
+// Within a macro call, a placeholder will match up until whatever token follows the placeholder.
 // Available via the command `rust-analyzer.ssr`.
 //
 // ```rust
diff --git a/crates/ra_ide/src/syntax_highlighting/injection.rs b/crates/ra_ide/src/syntax_highlighting/injection.rs
index 415f24a6d..9d82b4009 100644
--- a/crates/ra_ide/src/syntax_highlighting/injection.rs
+++ b/crates/ra_ide/src/syntax_highlighting/injection.rs
@@ -155,17 +155,21 @@ pub(super) fn highlight_doc_comment(
         let mut start_offset = None;
         let mut end_offset = None;
         for (line_start, orig_line_start) in range_mapping.range(..h.range.end()).rev() {
+            // It's possible for orig_line_start - line_start to be negative. Add h.range.start()
+            // here and remove it from the end range after the loop below so that the values are
+            // always non-negative.
+            let offset = h.range.start() + orig_line_start - line_start;
             if line_start <= &h.range.start() {
-                start_offset.get_or_insert(orig_line_start - line_start);
+                start_offset.get_or_insert(offset);
                 break;
             } else {
-                end_offset.get_or_insert(orig_line_start - line_start);
+                end_offset.get_or_insert(offset);
             }
         }
         if let Some(start_offset) = start_offset {
             h.range = TextRange::new(
-                h.range.start() + start_offset,
-                h.range.end() + end_offset.unwrap_or(start_offset),
+                start_offset,
+                h.range.end() + end_offset.unwrap_or(start_offset) - h.range.start(),
             );
 
             stack.add(h);
diff --git a/crates/ra_ide/src/syntax_highlighting/tests.rs b/crates/ra_ide/src/syntax_highlighting/tests.rs
index 93a276ffe..b1f48f03b 100644
--- a/crates/ra_ide/src/syntax_highlighting/tests.rs
+++ b/crates/ra_ide/src/syntax_highlighting/tests.rs
@@ -291,6 +291,9 @@ fn main() {
 fn test_highlight_doctest() {
     check_highlighting(
         r#"
+/// ```
+/// let _ = "early doctests should not go boom";
+/// ```
 struct Foo {
     bar: bool,
 }
diff --git a/crates/ra_ssr/src/lib.rs b/crates/ra_ssr/src/lib.rs
index fc716ae82..da26ee669 100644
--- a/crates/ra_ssr/src/lib.rs
+++ b/crates/ra_ssr/src/lib.rs
@@ -91,14 +91,16 @@ impl<'db> MatchFinder<'db> {
             if let Ok(mut m) = matching::get_match(false, rule, &code, restrict_range, &self.sema) {
                 // Continue searching in each of our placeholders.
                 for placeholder_value in m.placeholder_values.values_mut() {
-                    // Don't search our placeholder if it's the entire matched node, otherwise we'd
-                    // find the same match over and over until we got a stack overflow.
-                    if placeholder_value.node != *code {
-                        self.find_matches(
-                            &placeholder_value.node,
-                            restrict_range,
-                            &mut placeholder_value.inner_matches,
-                        );
+                    if let Some(placeholder_node) = &placeholder_value.node {
+                        // Don't search our placeholder if it's the entire matched node, otherwise we'd
+                        // find the same match over and over until we got a stack overflow.
+                        if placeholder_node != code {
+                            self.find_matches(
+                                placeholder_node,
+                                restrict_range,
+                                &mut placeholder_value.inner_matches,
+                            );
+                        }
                     }
                 }
                 matches_out.matches.push(m);
diff --git a/crates/ra_ssr/src/matching.rs b/crates/ra_ssr/src/matching.rs
index 265b6d793..bdaba9f1b 100644
--- a/crates/ra_ssr/src/matching.rs
+++ b/crates/ra_ssr/src/matching.rs
@@ -61,8 +61,9 @@ pub(crate) struct Var(pub String);
 /// Information about a placeholder bound in a match.
 #[derive(Debug)]
 pub(crate) struct PlaceholderMatch {
-    /// The node that the placeholder matched to.
-    pub(crate) node: SyntaxNode,
+    /// The node that the placeholder matched to. If set, then we'll search for further matches
+    /// within this node. It isn't set when we match tokens within a macro call's token tree.
+    pub(crate) node: Option<SyntaxNode>,
     pub(crate) range: FileRange,
     /// More matches, found within `node`.
     pub(crate) inner_matches: SsrMatches,
@@ -195,6 +196,7 @@ impl<'db, 'sema> MatchState<'db, 'sema> {
             SyntaxKind::RECORD_FIELD_LIST => {
                 self.attempt_match_record_field_list(match_inputs, pattern, code)
             }
+            SyntaxKind::TOKEN_TREE => self.attempt_match_token_tree(match_inputs, pattern, code),
             _ => self.attempt_match_node_children(match_inputs, pattern, code),
         }
     }
@@ -340,6 +342,90 @@ impl<'db, 'sema> MatchState<'db, 'sema> {
         Ok(())
     }
 
+    /// Outside of token trees, a placeholder can only match a single AST node, whereas in a token
+    /// tree it can match a sequence of tokens.
+    fn attempt_match_token_tree(
+        &mut self,
+        match_inputs: &MatchInputs,
+        pattern: &SyntaxNode,
+        code: &ra_syntax::SyntaxNode,
+    ) -> Result<(), MatchFailed> {
+        let mut pattern = PatternIterator::new(pattern).peekable();
+        let mut children = code.children_with_tokens();
+        while let Some(child) = children.next() {
+            if let Some(placeholder) = pattern.peek().and_then(|p| match_inputs.get_placeholder(p))
+            {
+                pattern.next();
+                let next_pattern_token = pattern
+                    .peek()
+                    .and_then(|p| match p {
+                        SyntaxElement::Token(t) => Some(t.clone()),
+                        SyntaxElement::Node(n) => n.first_token(),
+                    })
+                    .map(|p| p.text().to_string());
+                let first_matched_token = child.clone();
+                let mut last_matched_token = child;
+                // Read code tokens util we reach one equal to the next token from our pattern
+                // or we reach the end of the token tree.
+                while let Some(next) = children.next() {
+                    match &next {
+                        SyntaxElement::Token(t) => {
+                            if Some(t.to_string()) == next_pattern_token {
+                                pattern.next();
+                                break;
+                            }
+                        }
+                        SyntaxElement::Node(n) => {
+                            if let Some(first_token) = n.first_token() {
+                                if Some(first_token.to_string()) == next_pattern_token {
+                                    if let Some(SyntaxElement::Node(p)) = pattern.next() {
+                                        // We have a subtree that starts with the next token in our pattern.
+                                        self.attempt_match_token_tree(match_inputs, &p, &n)?;
+                                        break;
+                                    }
+                                }
+                            }
+                        }
+                    };
+                    last_matched_token = next;
+                }
+                if let Some(match_out) = &mut self.match_out {
+                    match_out.placeholder_values.insert(
+                        Var(placeholder.ident.to_string()),
+                        PlaceholderMatch::from_range(FileRange {
+                            file_id: self.sema.original_range(code).file_id,
+                            range: first_matched_token
+                                .text_range()
+                                .cover(last_matched_token.text_range()),
+                        }),
+                    );
+                }
+                continue;
+            }
+            // Match literal (non-placeholder) tokens.
+            match child {
+                SyntaxElement::Token(token) => {
+                    self.attempt_match_token(&mut pattern, &token)?;
+                }
+                SyntaxElement::Node(node) => match pattern.next() {
+                    Some(SyntaxElement::Node(p)) => {
+                        self.attempt_match_token_tree(match_inputs, &p, &node)?;
+                    }
+                    Some(SyntaxElement::Token(p)) => fail_match!(
+                        "Pattern has token '{}', code has subtree '{}'",
+                        p.text(),
+                        node.text()
+                    ),
+                    None => fail_match!("Pattern has nothing, code has '{}'", node.text()),
+                },
+            }
+        }
+        if let Some(p) = pattern.next() {
+            fail_match!("Reached end of token tree in code, but pattern still has {:?}", p);
+        }
+        Ok(())
+    }
+
     fn next_non_trivial(&mut self, code_it: &mut SyntaxElementChildren) -> Option<SyntaxElement> {
         loop {
             let c = code_it.next();
@@ -399,7 +485,11 @@ fn recording_match_fail_reasons() -> bool {
 
 impl PlaceholderMatch {
     fn new(node: &SyntaxNode, range: FileRange) -> Self {
-        Self { node: node.clone(), range, inner_matches: SsrMatches::default() }
+        Self { node: Some(node.clone()), range, inner_matches: SsrMatches::default() }
+    }
+
+    fn from_range(range: FileRange) -> Self {
+        Self { node: None, range, inner_matches: SsrMatches::default() }
     }
 }
 
@@ -484,7 +574,14 @@ mod tests {
         assert_eq!(matches.matches.len(), 1);
         assert_eq!(matches.matches[0].matched_node.text(), "foo(1+2)");
         assert_eq!(matches.matches[0].placeholder_values.len(), 1);
-        assert_eq!(matches.matches[0].placeholder_values[&Var("x".to_string())].node.text(), "1+2");
+        assert_eq!(
+            matches.matches[0].placeholder_values[&Var("x".to_string())]
+                .node
+                .as_ref()
+                .unwrap()
+                .text(),
+            "1+2"
+        );
 
         let edit = crate::replacing::matches_to_edit(&matches);
         let mut after = input.to_string();
diff --git a/crates/ra_ssr/src/replacing.rs b/crates/ra_ssr/src/replacing.rs
index 81a5e06a9..5dcde82a2 100644
--- a/crates/ra_ssr/src/replacing.rs
+++ b/crates/ra_ssr/src/replacing.rs
@@ -24,6 +24,7 @@ fn matches_to_edit_at_offset(matches: &SsrMatches, relative_start: TextSize) ->
 
 fn render_replace(match_info: &Match) -> String {
     let mut out = String::new();
+    let match_start = match_info.matched_node.text_range().start();
     for r in &match_info.template.tokens {
         match r {
             PatternElement::Token(t) => out.push_str(t.text.as_str()),
@@ -32,7 +33,14 @@ fn render_replace(match_info: &Match) -> String {
                     match_info.placeholder_values.get(&Var(p.ident.to_string()))
                 {
                     let range = &placeholder_value.range.range;
-                    let mut matched_text = placeholder_value.node.text().to_string();
+                    let mut matched_text = if let Some(node) = &placeholder_value.node {
+                        node.text().to_string()
+                    } else {
+                        let relative_range = range.checked_sub(match_start).unwrap();
+                        match_info.matched_node.text().to_string()
+                            [usize::from(relative_range.start())..usize::from(relative_range.end())]
+                            .to_string()
+                    };
                     let edit =
                         matches_to_edit_at_offset(&placeholder_value.inner_matches, range.start());
                     edit.apply(&mut matched_text);
diff --git a/crates/ra_ssr/src/tests.rs b/crates/ra_ssr/src/tests.rs
index 4b747fe18..3ee1e74e9 100644
--- a/crates/ra_ssr/src/tests.rs
+++ b/crates/ra_ssr/src/tests.rs
@@ -427,6 +427,45 @@ fn match_reordered_struct_instantiation() {
 }
 
 #[test]
+fn match_macro_invocation() {
+    assert_matches("foo!($a)", "fn() {foo(foo!(foo()))}", &["foo!(foo())"]);
+    assert_matches("foo!(41, $a, 43)", "fn() {foo!(41, 42, 43)}", &["foo!(41, 42, 43)"]);
+    assert_no_match("foo!(50, $a, 43)", "fn() {foo!(41, 42, 43}");
+    assert_no_match("foo!(41, $a, 50)", "fn() {foo!(41, 42, 43}");
+    assert_matches("foo!($a())", "fn() {foo!(bar())}", &["foo!(bar())"]);
+}
+
+// When matching within a macro expansion, we only allow matches of nodes that originated from
+// the macro call, not from the macro definition.
+#[test]
+fn no_match_expression_from_macro() {
+    assert_no_match(
+        "$a.clone()",
+        r#"
+            macro_rules! m1 {
+                () => {42.clone()}
+            }
+            fn f1() {m1!()}
+            "#,
+    );
+}
+
+// We definitely don't want to allow matching of an expression that part originates from the
+// macro call `42` and part from the macro definition `.clone()`.
+#[test]
+fn no_match_split_expression() {
+    assert_no_match(
+        "$a.clone()",
+        r#"
+            macro_rules! m1 {
+                ($x:expr) => {$x.clone()}
+            }
+            fn f1() {m1!(42)}
+            "#,
+    );
+}
+
+#[test]
 fn replace_function_call() {
     assert_ssr_transform("foo() ==>> bar()", "fn f1() {foo(); foo();}", "fn f1() {bar(); bar();}");
 }
@@ -468,6 +507,20 @@ fn replace_struct_init() {
 }
 
 #[test]
+fn replace_macro_invocations() {
+    assert_ssr_transform(
+        "try!($a) ==>> $a?",
+        "fn f1() -> Result<(), E> {bar(try!(foo()));}",
+        "fn f1() -> Result<(), E> {bar(foo()?);}",
+    );
+    assert_ssr_transform(
+        "foo!($a($b)) ==>> foo($b, $a)",
+        "fn f1() {foo!(abc(def() + 2));}",
+        "fn f1() {foo(def() + 2, abc);}",
+    );
+}
+
+#[test]
 fn replace_binary_op() {
     assert_ssr_transform(
         "$a + $b ==>> $b + $a",
diff --git a/docs/user/manual.adoc b/docs/user/manual.adoc
index ea714f49a..f1b7ed7fc 100644
--- a/docs/user/manual.adoc
+++ b/docs/user/manual.adoc
@@ -269,6 +269,10 @@ Gnome Builder currently has support for RLS, and there's no way to configure the
 1. Rename, symlink or copy the `rust-analyzer` binary to `rls` and place it somewhere Builder can find (in `PATH`, or under `~/.cargo/bin`).
 2. Enable the Rust Builder plugin.
 
+==== GNOME Builder (Nightly)
+
+https://nightly.gnome.org/repo/appstream/org.gnome.Builder.flatpakref[GNOME Builder (Nightly)] has now native support for `rust-analyzer` out of the box. If the `rust-analyzer` binary is not available, GNOME Builder can install it when opening a Rust source file.
+
 == Non-Cargo Based Projects
 
 rust-analyzer does not require Cargo.
diff --git a/editors/code/package.json b/editors/code/package.json
index e6ceb235f..68484a370 100644
--- a/editors/code/package.json
+++ b/editors/code/package.json
@@ -426,7 +426,7 @@
                         "Full log"
                     ],
                     "default": "off",
-                    "description": "Trace requests to the rust-analyzer"
+                    "description": "Trace requests to the rust-analyzer (this is usually overly verbose and not recommended for regular users)"
                 },
                 "rust-analyzer.trace.extension": {
                     "description": "Enable logging of VS Code extensions itself",
diff --git a/editors/code/src/main.ts b/editors/code/src/main.ts
index 670f2ebfd..12b4d0510 100644
--- a/editors/code/src/main.ts
+++ b/editors/code/src/main.ts
@@ -43,12 +43,16 @@ export async function activate(context: vscode.ExtensionContext) {
     const config = new Config(context);
     const state = new PersistentState(context.globalState);
     const serverPath = await bootstrap(config, state).catch(err => {
-        let message = "Failed to bootstrap rust-analyzer.";
+        let message = "bootstrap error. ";
+
         if (err.code === "EBUSY" || err.code === "ETXTBSY") {
-            message += " Other vscode windows might be using rust-analyzer, " +
-                "you should close them and reload this window to retry.";
+            message += "Other vscode windows might be using rust-analyzer, ";
+            message += "you should close them and reload this window to retry. ";
         }
-        message += " Open \"Help > Toggle Developer Tools > Console\" to see the logs";
+
+        message += 'Open "Help > Toggle Developer Tools > Console" to see the logs ';
+        message += '(enable verbose logs with "rust-analyzer.trace.extension")';
+
         log.error("Bootstrap error", err);
         throw new Error(message);
     });
@@ -178,7 +182,11 @@ async function bootstrapExtension(config: Config, state: PersistentState): Promi
     assert(!!artifact, `Bad release: ${JSON.stringify(release)}`);
 
     const dest = path.join(config.globalStoragePath, "rust-analyzer.vsix");
-    await download(artifact.browser_download_url, dest, "Downloading rust-analyzer extension");
+    await download({
+        url: artifact.browser_download_url,
+        dest,
+        progressTitle: "Downloading rust-analyzer extension",
+    });
 
     await vscode.commands.executeCommand("workbench.extensions.installExtension", vscode.Uri.file(dest));
     await fs.unlink(dest);
@@ -299,7 +307,12 @@ async function getServer(config: Config, state: PersistentState): Promise<string
         if (err.code !== "ENOENT") throw err;
     });
 
-    await download(artifact.browser_download_url, dest, "Downloading rust-analyzer server", { mode: 0o755 });
+    await download({
+        url: artifact.browser_download_url,
+        dest,
+        progressTitle: "Downloading rust-analyzer server",
+        mode: 0o755
+    });
 
     // Patching executable if that's NixOS.
     if (await fs.stat("/etc/nixos").then(_ => true).catch(_ => false)) {
diff --git a/editors/code/src/net.ts b/editors/code/src/net.ts
index 9debdc57b..e02fd6d4f 100644
--- a/editors/code/src/net.ts
+++ b/editors/code/src/net.ts
@@ -60,32 +60,40 @@ export interface GithubRelease {
     }>;
 }
 
+interface DownloadOpts {
+    progressTitle: string;
+    url: string;
+    dest: string;
+    mode?: number;
+}
 
-export async function download(
-    downloadUrl: string,
-    destinationPath: string,
-    progressTitle: string,
-    { mode }: { mode?: number } = {},
-) {
-    await vscode.window.withProgress(
-        {
-            location: vscode.ProgressLocation.Notification,
-            cancellable: false,
-            title: progressTitle
-        },
-        async (progress, _cancellationToken) => {
-            let lastPercentage = 0;
-            await downloadFile(downloadUrl, destinationPath, mode, (readBytes, totalBytes) => {
-                const newPercentage = (readBytes / totalBytes) * 100;
-                progress.report({
-                    message: newPercentage.toFixed(0) + "%",
-                    increment: newPercentage - lastPercentage
+export async function download(opts: DownloadOpts) {
+    // Put the artifact into a temporary folder to prevent partially downloaded files when user kills vscode
+    await withTempDir(async tempDir => {
+        const tempFile = path.join(tempDir, path.basename(opts.dest));
+
+        await vscode.window.withProgress(
+            {
+                location: vscode.ProgressLocation.Notification,
+                cancellable: false,
+                title: opts.progressTitle
+            },
+            async (progress, _cancellationToken) => {
+                let lastPercentage = 0;
+                await downloadFile(opts.url, tempFile, opts.mode, (readBytes, totalBytes) => {
+                    const newPercentage = (readBytes / totalBytes) * 100;
+                    progress.report({
+                        message: newPercentage.toFixed(0) + "%",
+                        increment: newPercentage - lastPercentage
+                    });
+
+                    lastPercentage = newPercentage;
                 });
+            }
+        );
 
-                lastPercentage = newPercentage;
-            });
-        }
-    );
+        await moveFile(tempFile, opts.dest);
+    });
 }
 
 /**
@@ -114,28 +122,23 @@ async function downloadFile(
 
     log.debug("Downloading file of", totalBytes, "bytes size from", url, "to", destFilePath);
 
-    // Put the artifact into a temporary folder to prevent partially downloaded files when user kills vscode
-    await withTempFile(async tempFilePath => {
-        const destFileStream = fs.createWriteStream(tempFilePath, { mode });
-
-        let readBytes = 0;
-        res.body.on("data", (chunk: Buffer) => {
-            readBytes += chunk.length;
-            onProgress(readBytes, totalBytes);
-        });
+    let readBytes = 0;
+    res.body.on("data", (chunk: Buffer) => {
+        readBytes += chunk.length;
+        onProgress(readBytes, totalBytes);
+    });
 
-        await pipeline(res.body, destFileStream);
-        await new Promise<void>(resolve => {
-            destFileStream.on("close", resolve);
-            destFileStream.destroy();
-            // This workaround is awaiting to be removed when vscode moves to newer nodejs version:
-            // https://github.com/rust-analyzer/rust-analyzer/issues/3167
-        });
-        await moveFile(tempFilePath, destFilePath);
+    const destFileStream = fs.createWriteStream(destFilePath, { mode });
+    await pipeline(res.body, destFileStream);
+    await new Promise<void>(resolve => {
+        destFileStream.on("close", resolve);
+        destFileStream.destroy();
+        // This workaround is awaiting to be removed when vscode moves to newer nodejs version:
+        // https://github.com/rust-analyzer/rust-analyzer/issues/3167
     });
 }
 
-async function withTempFile(scope: (tempFilePath: string) => Promise<void>) {
+async function withTempDir(scope: (tempDirPath: string) => Promise<void>) {
     // Based on the great article: https://advancedweb.hu/secure-tempfiles-in-nodejs-without-dependencies/
 
     // `.realpath()` should handle the cases where os.tmpdir() contains symlinks
@@ -144,7 +147,7 @@ async function withTempFile(scope: (tempFilePath: string) => Promise<void>) {
     const tempDir = await fs.promises.mkdtemp(path.join(osTempDir, "rust-analyzer"));
 
     try {
-        return await scope(path.join(tempDir, "file"));
+        return await scope(tempDir);
     } finally {
         // We are good citizens :D
         void fs.promises.rmdir(tempDir, { recursive: true }).catch(log.error);
@@ -161,6 +164,7 @@ async function moveFile(src: fs.PathLike, dest: fs.PathLike) {
             await fs.promises.unlink(src);
         } else {
             log.error(`Failed to rename the file ${src} -> ${dest}`, err);
+            throw err;
         }
     }
 }
diff --git a/editors/code/src/util.ts b/editors/code/src/util.ts
index fe3fb71cd..fec4c3295 100644
--- a/editors/code/src/util.ts
+++ b/editors/code/src/util.ts
@@ -26,7 +26,6 @@ export const log = new class {
     }
 
     error(message?: any, ...optionalParams: any[]): void {
-        if (!log.enabled) return;
         debugger;
         // eslint-disable-next-line no-console
         console.error(message, ...optionalParams);