Merge #3053

3053: Feature: downloading lsp server from GitHub r=matklad a=Veetaha

This is currently very WIP, I may need to change this and that, add "download language server command", logging stuff (for future bug reports), etc., but it already works.
Also didn't test this on windows yet and mac (don't have the latter)

The quirks:
* Downloaded binary doesn't have executable permissions by default, that's why we ~~`chmod 111`~~ (**[UPD]** `chmod 755` as per @lnicola [suggestion](https://github.com/rust-analyzer/rust-analyzer/pull/3053#discussion_r376694456)) for it.
* To remove installed binary run `rm /${HOME}/.config/Code/User/globalStorage/matklad.rust-analyzer/ra_lsp_server-linux`, ~~note that `-f` flag is necessary, because of `111` permissions (I think this should be changed)~~ (**[UPD]** --force is no longer needed due to 755 permissions).

I also tried to keep things simple and not to use too many dependencies, all the ones added have 0 dependencies, (`ts-not-nil` is my personal npm package, that imitates `unwrap()` in TypeScript)

**[UPD]** I reduced throttle latency of progress indicator to 200ms for smoother UX


// TODO:
- [x] ~~Add `Rust Analyzer: Download latest language server` vscode command.~~ **[UPD]**: having reviewed the code and estimated available options I concluded that this feature requires too many code changes, I'd like to extract this into a separate PR after we merge this one.
- [x] Add some logging for future debugging
- [x] ~~Gracefully handle the case when language server is not available (e.g. no internet connection, user explicitly rejected the download, etc.)~~ **[UPD]** Decided to postpone better implementation of graceful degradation logic as per [conversation](https://rust-lang.zulipchat.com/#narrow/stream/185405-t-compiler.2Fwg-rls-2.2E0/topic/Deployment.20and.20installation/near/187758550).

Demo (**[UPD]** this is a bit outdated, but still mainly reflects the feature):
![ra-github-release-download-mvp](https://user-images.githubusercontent.com/36276403/74077961-4f248a80-4a2d-11ea-962f-27c650fd6c4c.gif)

Related issue: #2988 #3007 

Co-authored-by: Veetaha <[email protected]>
Co-authored-by: Veetaha <[email protected]>

10 files changed, 396 insertions, 29 deletions

diff --git a/editors/code/package-lock.json b/editors/code/package-lock.json
index 353af06bf..5c056463e 100644
--- a/editors/code/package-lock.json
+++ b/editors/code/package-lock.json
@@ -82,6 +82,15 @@
             "integrity": "sha512-nf1LMGZvgFX186geVZR1xMZKKblJiRfiASTHw85zED2kI1yDKHDwTKMdkaCbTlXoRKlGKaDfYywt+V0As30q3w==",
             "dev": true
         },
+        "@types/node-fetch": {
+            "version": "2.5.4",
+            "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.4.tgz",
+            "integrity": "sha512-Oz6id++2qAOFuOlE1j0ouk1dzl3mmI1+qINPNBhi9nt/gVOz0G+13Ao6qjhdF0Ys+eOkhu6JnFmt38bR3H0POQ==",
+            "dev": true,
+            "requires": {
+                "@types/node": "*"
+            }
+        },
         "@types/resolve": {
             "version": "0.0.8",
             "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-0.0.8.tgz",
@@ -91,6 +100,12 @@
                 "@types/node": "*"
             }
         },
+        "@types/throttle-debounce": {
+            "version": "2.1.0",
+            "resolved": "https://registry.npmjs.org/@types/throttle-debounce/-/throttle-debounce-2.1.0.tgz",
+            "integrity": "sha512-5eQEtSCoESnh2FsiLTxE121IiE60hnMqcb435fShf4bpLRjEu1Eoekht23y6zXS9Ts3l+Szu3TARnTsA0GkOkQ==",
+            "dev": true
+        },
         "@types/vscode": {
             "version": "1.41.0",
             "resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.41.0.tgz",
@@ -536,6 +551,11 @@
             "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==",
             "dev": true
         },
+        "node-fetch": {
+            "version": "2.6.0",
+            "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz",
+            "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA=="
+        },
         "nth-check": {
             "version": "1.0.2",
             "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz",
@@ -719,6 +739,11 @@
                 "has-flag": "^3.0.0"
             }
         },
+        "throttle-debounce": {
+            "version": "2.1.0",
+            "resolved": "https://registry.npmjs.org/throttle-debounce/-/throttle-debounce-2.1.0.tgz",
+            "integrity": "sha512-AOvyNahXQuU7NN+VVvOOX+uW6FPaWdAOdRP5HfwYxAfCzXTFKRMoIMk+n+po318+ktcChx+F1Dd91G3YHeMKyg=="
+        },
         "tmp": {
             "version": "0.0.29",
             "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.29.tgz",
diff --git a/editors/code/package.json b/editors/code/package.json
index 11d37053e..f687eb8d4 100644
--- a/editors/code/package.json
+++ b/editors/code/package.json
@@ -25,18 +25,22 @@
     },
     "dependencies": {
         "jsonc-parser": "^2.1.0",
+        "node-fetch": "^2.6.0",
+        "throttle-debounce": "^2.1.0",
         "vscode-languageclient": "^6.1.0"
     },
     "devDependencies": {
         "@rollup/plugin-commonjs": "^11.0.2",
         "@rollup/plugin-node-resolve": "^7.1.1",
         "@types/node": "^12.12.25",
+        "@types/node-fetch": "^2.5.4",
+        "@types/throttle-debounce": "^2.1.0",
         "@types/vscode": "^1.41.0",
         "rollup": "^1.31.0",
         "tslib": "^1.10.0",
         "tslint": "^5.20.1",
-        "typescript-formatter": "^7.2.2",
         "typescript": "^3.7.5",
+        "typescript-formatter": "^7.2.2",
         "vsce": "^1.71.0"
     },
     "activationEvents": [
@@ -169,10 +173,11 @@
                 },
                 "rust-analyzer.raLspServerPath": {
                     "type": [
+                        "null",
                         "string"
                     ],
-                    "default": "ra_lsp_server",
-                    "description": "Path to ra_lsp_server executable"
+                    "default": null,
+                    "description": "Path to ra_lsp_server executable (points to bundled binary by default)"
                 },
                 "rust-analyzer.excludeGlobs": {
                     "type": "array",
diff --git a/editors/code/src/client.ts b/editors/code/src/client.ts
index 7e7e909dd..2e3d4aba2 100644
--- a/editors/code/src/client.ts
+++ b/editors/code/src/client.ts
@@ -1,24 +1,18 @@
-import { homedir } from 'os';
 import * as lc from 'vscode-languageclient';
-import { spawnSync } from 'child_process';
 
 import { window, workspace } from 'vscode';
 import { Config } from './config';
+import { ensureLanguageServerBinary } from './installation/language_server';
 
-export function createClient(config: Config): lc.LanguageClient {
+export async function createClient(config: Config): Promise<null | lc.LanguageClient> {
     // '.' Is the fallback if no folder is open
     // TODO?: Workspace folders support Uri's (eg: file://test.txt).
     // It might be a good idea to test if the uri points to a file.
     const workspaceFolderPath = workspace.workspaceFolders?.[0]?.uri.fsPath ?? '.';
 
-    const raLspServerPath = expandPathResolving(config.raLspServerPath);
-    if (spawnSync(raLspServerPath, ["--version"]).status !== 0) {
-        window.showErrorMessage(
-            `Unable to execute '${raLspServerPath} --version'\n\n` +
-            `Perhaps it is not in $PATH?\n\n` +
-            `PATH=${process.env.PATH}\n`
-        );
-    }
+    const raLspServerPath = await ensureLanguageServerBinary(config.langServerSource);
+    if (!raLspServerPath) return null;
+
     const run: lc.Executable = {
         command: raLspServerPath,
         options: { cwd: workspaceFolderPath },
@@ -87,9 +81,3 @@ export function createClient(config: Config): lc.LanguageClient {
     res.registerProposedFeatures();
     return res;
 }
-function expandPathResolving(path: string) {
-    if (path.startsWith('~/')) {
-        return path.replace('~', homedir());
-    }
-    return path;
-}
diff --git a/editors/code/src/config.ts b/editors/code/src/config.ts
index 524620433..d5f3da2ed 100644
--- a/editors/code/src/config.ts
+++ b/editors/code/src/config.ts
@@ -1,4 +1,6 @@
+import * as os from "os";
 import * as vscode from 'vscode';
+import { BinarySource } from "./installation/interfaces";
 
 const RA_LSP_DEBUG = process.env.__RA_LSP_SERVER_DEBUG;
 
@@ -16,10 +18,11 @@ export interface CargoFeatures {
 }
 
 export class Config {
+    langServerSource!: null | BinarySource;
+
     highlightingOn = true;
     rainbowHighlightingOn = false;
     enableEnhancedTyping = true;
-    raLspServerPath = RA_LSP_DEBUG || 'ra_lsp_server';
     lruCapacity: null | number = null;
     displayInlayHints = true;
     maxInlayHintLength: null | number = null;
@@ -45,11 +48,72 @@ export class Config {
     private prevCargoWatchOptions: null | CargoWatchOptions = null;
 
     constructor(ctx: vscode.ExtensionContext) {
-        vscode.workspace.onDidChangeConfiguration(_ => this.refresh(), null, ctx.subscriptions);
-        this.refresh();
+        vscode.workspace.onDidChangeConfiguration(_ => this.refresh(ctx), null, ctx.subscriptions);
+        this.refresh(ctx);
+    }
+
+    private static expandPathResolving(path: string) {
+        if (path.startsWith('~/')) {
+            return path.replace('~', os.homedir());
+        }
+        return path;
+    }
+
+    /**
+     * Name of the binary artifact for `ra_lsp_server` that is published for
+     * `platform` on GitHub releases. (It is also stored under the same name when
+     * downloaded by the extension).
+     */
+    private static prebuiltLangServerFileName(platform: NodeJS.Platform): null | string {
+        switch (platform) {
+            case "linux":  return "ra_lsp_server-linux";
+            case "darwin": return "ra_lsp_server-mac";
+            case "win32":  return "ra_lsp_server-windows.exe";
+
+            // Users on these platforms yet need to manually build from sources
+            case "aix":
+            case "android":
+            case "freebsd":
+            case "openbsd":
+            case "sunos":
+            case "cygwin":
+            case "netbsd": return null;
+            // The list of platforms is exhaustive (see `NodeJS.Platform` type definition)
+        }
+    }
+
+    private static langServerBinarySource(
+        ctx: vscode.ExtensionContext,
+        config: vscode.WorkspaceConfiguration
+    ): null | BinarySource {
+        const langServerPath = RA_LSP_DEBUG ?? config.get<null | string>("raLspServerPath");
+
+        if (langServerPath) {
+            return {
+                type: BinarySource.Type.ExplicitPath,
+                path: Config.expandPathResolving(langServerPath)
+            };
+        }
+
+        const prebuiltBinaryName = Config.prebuiltLangServerFileName(process.platform);
+
+        if (!prebuiltBinaryName) return null;
+
+        return {
+            type: BinarySource.Type.GithubRelease,
+            dir: ctx.globalStoragePath,
+            file: prebuiltBinaryName,
+            repo: {
+                name: "rust-analyzer",
+                owner: "rust-analyzer",
+            }
+        };
     }
 
-    private refresh() {
+
+    // FIXME: revisit the logic for `if (.has(...)) config.get(...)` set default
+    // values only in one place (i.e. remove default values from non-readonly members declarations)
+    private refresh(ctx: vscode.ExtensionContext) {
         const config = vscode.workspace.getConfiguration('rust-analyzer');
 
         let requireReloadMessage = null;
@@ -82,10 +146,7 @@ export class Config {
             this.prevEnhancedTyping = this.enableEnhancedTyping;
         }
 
-        if (config.has('raLspServerPath')) {
-            this.raLspServerPath =
-                RA_LSP_DEBUG || (config.get('raLspServerPath') as string);
-        }
+        this.langServerSource = Config.langServerBinarySource(ctx, config);
 
         if (config.has('cargo-watch.enable')) {
             this.cargoWatchOptions.enable = config.get<boolean>(
diff --git a/editors/code/src/ctx.ts b/editors/code/src/ctx.ts
index aa75943bf..70042a479 100644
--- a/editors/code/src/ctx.ts
+++ b/editors/code/src/ctx.ts
@@ -11,6 +11,9 @@ export class Ctx {
     // deal with it.
     //
     // Ideally, this should be replaced with async getter though.
+    // FIXME: this actually needs syncronization of some kind (check how
+    // vscode deals with `deactivate()` call when extension has some work scheduled
+    // on the event loop to get a better picture of what we can do here)
     client: lc.LanguageClient | null = null;
     private extCtx: vscode.ExtensionContext;
     private onDidRestartHooks: Array<(client: lc.LanguageClient) => void> = [];
@@ -26,7 +29,14 @@ export class Ctx {
             await old.stop();
         }
         this.client = null;
-        const client = createClient(this.config);
+        const client = await createClient(this.config);
+        if (!client) {
+            throw new Error(
+                "Rust Analyzer Language Server is not available. " +
+                "Please, ensure its [proper installation](https://github.com/rust-analyzer/rust-analyzer/tree/master/docs/user#vs-code)."
+            );
+        }
+
         this.pushCleanup(client.start());
         await client.onReady();
 
diff --git a/editors/code/src/installation/download_file.ts b/editors/code/src/installation/download_file.ts
new file mode 100644
index 000000000..b51602ef9
--- /dev/null
+++ b/editors/code/src/installation/download_file.ts
@@ -0,0 +1,34 @@
+import fetch from "node-fetch";
+import * as fs from "fs";
+import { strict as assert } from "assert";
+
+/**
+ * Downloads file from `url` and stores it at `destFilePath`.
+ * `onProgress` callback is called on recieveing each chunk of bytes
+ * to track the progress of downloading, it gets the already read and total
+ * amount of bytes to read as its parameters.
+ */
+export async function downloadFile(
+    url: string,
+    destFilePath: fs.PathLike,
+    onProgress: (readBytes: number, totalBytes: number) => void
+): Promise<void> {
+    const response = await fetch(url);
+
+    const totalBytes = Number(response.headers.get('content-length'));
+    assert(!Number.isNaN(totalBytes), "Sanity check of content-length protocol");
+
+    let readBytes = 0;
+
+    console.log("Downloading file of", totalBytes, "bytes size from", url, "to", destFilePath);
+
+    return new Promise<void>((resolve, reject) => response.body
+        .on("data", (chunk: Buffer) => {
+            readBytes += chunk.length;
+            onProgress(readBytes, totalBytes);
+        })
+        .on("end", resolve)
+        .on("error", reject)
+        .pipe(fs.createWriteStream(destFilePath))
+    );
+}
diff --git a/editors/code/src/installation/fetch_latest_artifact_metadata.ts b/editors/code/src/installation/fetch_latest_artifact_metadata.ts
new file mode 100644
index 000000000..7e3700603
--- /dev/null
+++ b/editors/code/src/installation/fetch_latest_artifact_metadata.ts
@@ -0,0 +1,46 @@
+import fetch from "node-fetch";
+import { GithubRepo, ArtifactMetadata } from "./interfaces";
+
+const GITHUB_API_ENDPOINT_URL = "https://api.github.com";
+
+/**
+ * Fetches the latest release from GitHub `repo` and returns metadata about
+ * `artifactFileName` shipped with this release or `null` if no such artifact was published.
+ */
+export async function fetchLatestArtifactMetadata(
+    repo: GithubRepo, artifactFileName: string
+): Promise<null | ArtifactMetadata> {
+
+    const repoOwner = encodeURIComponent(repo.owner);
+    const repoName  = encodeURIComponent(repo.name);
+
+    const apiEndpointPath = `/repos/${repoOwner}/${repoName}/releases/latest`;
+    const requestUrl = GITHUB_API_ENDPOINT_URL + apiEndpointPath;
+
+    // We skip runtime type checks for simplicity (here we cast from `any` to `GithubRelease`)
+
+    console.log("Issuing request for released artifacts metadata to", requestUrl);
+
+    const response: GithubRelease = await fetch(requestUrl, {
+            headers: { Accept: "application/vnd.github.v3+json" }
+        })
+        .then(res => res.json());
+
+    const artifact = response.assets.find(artifact => artifact.name === artifactFileName);
+
+    if (!artifact) return null;
+
+    return {
+        releaseName: response.name,
+        downloadUrl: artifact.browser_download_url
+    };
+
+    // We omit declaration of tremendous amount of fields that we are not using here
+    interface GithubRelease {
+        name: string;
+        assets: Array<{
+            name: string;
+            browser_download_url: string;
+        }>;
+    }
+}
diff --git a/editors/code/src/installation/interfaces.ts b/editors/code/src/installation/interfaces.ts
new file mode 100644
index 000000000..8039d0b90
--- /dev/null
+++ b/editors/code/src/installation/interfaces.ts
@@ -0,0 +1,55 @@
+export interface GithubRepo {
+    name: string;
+    owner: string;
+}
+
+/**
+ * Metadata about particular artifact retrieved from GitHub releases.
+ */
+export interface ArtifactMetadata {
+    releaseName: string;
+    downloadUrl: string;
+}
+
+/**
+ * Represents the source of a binary artifact which is either specified by the user
+ * explicitly, or bundled by this extension from GitHub releases.
+ */
+export type BinarySource = BinarySource.ExplicitPath | BinarySource.GithubRelease;
+
+export namespace BinarySource {
+    /**
+     * Type tag for `BinarySource` discriminated union.
+     */
+    export const enum Type { ExplicitPath, GithubRelease }
+
+    export interface ExplicitPath {
+        type: Type.ExplicitPath;
+
+        /**
+         * Filesystem path to the binary specified by the user explicitly.
+         */
+        path: string;
+    }
+
+    export interface GithubRelease {
+        type: Type.GithubRelease;
+
+        /**
+         * Repository where the binary is stored.
+         */
+        repo: GithubRepo;
+
+        /**
+         * Directory on the filesystem where the bundled binary is stored.
+         */
+        dir: string;
+
+        /**
+         * Name of the binary file. It is stored under the same name on GitHub releases
+         * and in local `.dir`.
+         */
+        file: string;
+    }
+
+}
diff --git a/editors/code/src/installation/language_server.ts b/editors/code/src/installation/language_server.ts
new file mode 100644
index 000000000..1ce67b8b2
--- /dev/null
+++ b/editors/code/src/installation/language_server.ts
@@ -0,0 +1,141 @@
+import * as vscode from "vscode";
+import * as path from "path";
+import { strict as assert } from "assert";
+import { promises as fs } from "fs";
+import { promises as dns } from "dns";
+import { spawnSync } from "child_process";
+import { throttle } from "throttle-debounce";
+
+import { BinarySource } from "./interfaces";
+import { fetchLatestArtifactMetadata } from "./fetch_latest_artifact_metadata";
+import { downloadFile } from "./download_file";
+
+export async function downloadLatestLanguageServer(
+    {file: artifactFileName, dir: installationDir, repo}: BinarySource.GithubRelease
+) {
+    const { releaseName, downloadUrl } = (await fetchLatestArtifactMetadata(
+        repo, artifactFileName
+    ))!;
+
+    await fs.mkdir(installationDir).catch(err => assert.strictEqual(
+        err?.code,
+        "EEXIST",
+        `Couldn't create directory "${installationDir}" to download `+
+        `language server binary: ${err.message}`
+    ));
+
+    const installationPath = path.join(installationDir, artifactFileName);
+
+    console.time("Downloading ra_lsp_server");
+    await vscode.window.withProgress(
+        {
+            location: vscode.ProgressLocation.Notification,
+            cancellable: false, // FIXME: add support for canceling download?
+            title: `Downloading language server (${releaseName})`
+        },
+        async (progress, _cancellationToken) => {
+            let lastPrecentage = 0;
+            await downloadFile(downloadUrl, installationPath, throttle(
+                200,
+                /* noTrailing: */ true,
+                (readBytes, totalBytes) => {
+                    const newPercentage = (readBytes / totalBytes) * 100;
+                    progress.report({
+                        message: newPercentage.toFixed(0) + "%",
+                        increment: newPercentage - lastPrecentage
+                    });
+
+                    lastPrecentage = newPercentage;
+                })
+            );
+        }
+    );
+    console.timeEnd("Downloading ra_lsp_server");
+
+    await fs.chmod(installationPath, 0o755); // Set (rwx, r_x, r_x) permissions
+}
+export async function ensureLanguageServerBinary(
+    langServerSource: null | BinarySource
+): Promise<null | string> {
+
+    if (!langServerSource) {
+        vscode.window.showErrorMessage(
+            "Unfortunately we don't ship binaries for your platform yet. " +
+            "You need to manually clone rust-analyzer repository and " +
+            "run `cargo xtask install --server` to build the language server from sources. " +
+            "If you feel that your platform should be supported, please create an issue " +
+            "about that [here](https://github.com/rust-analyzer/rust-analyzer/issues) and we " +
+            "will consider it."
+        );
+        return null;
+    }
+
+    switch (langServerSource.type) {
+        case BinarySource.Type.ExplicitPath: {
+            if (isBinaryAvailable(langServerSource.path)) {
+                return langServerSource.path;
+            }
+
+            vscode.window.showErrorMessage(
+                `Unable to run ${langServerSource.path} binary. ` +
+                `To use the pre-built language server, set "rust-analyzer.raLspServerPath" ` +
+                "value to `null` or remove it from the settings to use it by default."
+            );
+            return null;
+        }
+        case BinarySource.Type.GithubRelease: {
+            const prebuiltBinaryPath = path.join(langServerSource.dir, langServerSource.file);
+
+            if (isBinaryAvailable(prebuiltBinaryPath)) {
+                return prebuiltBinaryPath;
+            }
+
+            const userResponse = await vscode.window.showInformationMessage(
+                "Language server binary for rust-analyzer was not found. " +
+                "Do you want to download it now?",
+                "Download now", "Cancel"
+            );
+            if (userResponse !== "Download now") return null;
+
+            try {
+                await downloadLatestLanguageServer(langServerSource);
+            } catch (err) {
+                await vscode.window.showErrorMessage(
+                    `Failed to download language server from ${langServerSource.repo.name} ` +
+                    `GitHub repository: ${err.message}`
+                );
+
+                await dns.resolve('www.google.com').catch(err => {
+                    console.error("DNS resolution failed, there might be an issue with Internet availability");
+                    console.error(err);
+                });
+
+                return null;
+            }
+
+            if (!isBinaryAvailable(prebuiltBinaryPath)) assert(false,
+                `Downloaded language server binary is not functional.` +
+                `Downloaded from: ${JSON.stringify(langServerSource)}`
+            );
+
+
+            vscode.window.showInformationMessage(
+                "Rust analyzer language server was successfully installed 🦀"
+            );
+
+            return prebuiltBinaryPath;
+        }
+    }
+
+    function isBinaryAvailable(binaryPath: string) {
+        const res = spawnSync(binaryPath, ["--version"]);
+
+        // ACHTUNG! `res` type declaration is inherently wrong, see
+        // https://github.com/DefinitelyTyped/DefinitelyTyped/issues/42221
+
+        console.log("Checked binary availablity via --version", res);
+        console.log(binaryPath, "--version output:", res.output?.map(String));
+
+        return res.status === 0;
+    }
+}
diff --git a/editors/code/tsconfig.json b/editors/code/tsconfig.json
index e60eb8e5e..0c7702974 100644
--- a/editors/code/tsconfig.json
+++ b/editors/code/tsconfig.json
@@ -6,6 +6,8 @@
         "lib": [
             "es2019"
         ],
+        "esModuleInterop": true,
+        "allowSyntheticDefaultImports": true,
         "sourceMap": true,
         "rootDir": "src",
         "strict": true,