internal: switch from CLI to internal benchmarking

We have a CLI for benchmarking, but no one actually uses it it seems.
Let's try switching to "internal" benchmarks, implemented as rust tests.
They should be easier to "script" to automate tracking of perf
regressions.

6 files changed, 76 insertions, 262 deletions

diff --git a/crates/rust-analyzer/src/benchmarks.rs b/crates/rust-analyzer/src/benchmarks.rs
new file mode 100644
index 000000000..a6f997af8
--- /dev/null
+++ b/crates/rust-analyzer/src/benchmarks.rs
@@ -0,0 +1,69 @@
+//! Fully integrated benchmarks for rust-analyzer, which load real cargo
+//! projects.
+//!
+//! The benchmark here is used to debug specific performance regressions. If you
+//! notice that, eg, completion is slow in some specific case, you can  modify
+//! code here exercise this specific completion, and thus have a fast
+//! edit/compile/test cycle.
+//!
+//! Note that "Rust Analyzer: Run" action does not allow running a single test
+//! in release mode in VS Code. There's however "Rust Analyzer: Copy Run Command Line"
+//! which you can use to paste the command in terminal and add `--release` manually.
+
+use std::sync::Arc;
+
+use ide::Change;
+use test_utils::project_root;
+use vfs::{AbsPathBuf, VfsPath};
+
+use crate::cli::load_cargo::{load_workspace_at, LoadCargoConfig};
+
+#[test]
+fn benchmark_integrated_highlighting() {
+    // Don't run slow benchmark by default
+    if true {
+        return;
+    }
+
+    // Load rust-analyzer itself.
+    let workspace_to_load = project_root();
+    let file = "./crates/ide_db/src/apply_change.rs";
+
+    let cargo_config = Default::default();
+    let load_cargo_config =
+        LoadCargoConfig { load_out_dirs_from_check: true, with_proc_macro: false };
+
+    let (mut host, vfs, _proc_macro) = {
+        let _it = stdx::timeit("workspace loading");
+        load_workspace_at(&workspace_to_load, &cargo_config, &load_cargo_config, &|_| {}).unwrap()
+    };
+
+    let file_id = {
+        let file = workspace_to_load.join(file);
+        let path = VfsPath::from(AbsPathBuf::assert(file));
+        vfs.file_id(&path).unwrap_or_else(|| panic!("can't find virtual file for {}", path))
+    };
+
+    {
+        let _it = stdx::timeit("initial");
+        let analysis = host.analysis();
+        analysis.highlight_as_html(file_id, false).unwrap();
+    }
+
+    profile::init_from("*>100");
+
+    {
+        let _it = stdx::timeit("change");
+        let mut text = host.analysis().file_text(file_id).unwrap().to_string();
+        text.push_str("\npub fn _dummy() {}\n");
+        let mut change = Change::new();
+        change.change_file(file_id, Some(Arc::new(text)));
+        host.apply_change(change);
+    }
+
+    {
+        let _it = stdx::timeit("after change");
+        let analysis = host.analysis();
+        analysis.highlight_as_html(file_id, false).unwrap();
+    }
+}
diff --git a/crates/rust-analyzer/src/bin/flags.rs b/crates/rust-analyzer/src/bin/flags.rs
index d8987633d..b05fc00b9 100644
--- a/crates/rust-analyzer/src/bin/flags.rs
+++ b/crates/rust-analyzer/src/bin/flags.rs
@@ -1,10 +1,9 @@
 //! Grammar for the command-line arguments.
 #![allow(unreachable_pub)]
-use std::{env, path::PathBuf};
+use std::path::PathBuf;
 
 use ide_ssr::{SsrPattern, SsrRule};
-use rust_analyzer::cli::{BenchWhat, Position, Verbosity};
-use vfs::AbsPathBuf;
+use rust_analyzer::cli::Verbosity;
 
 xflags::xflags! {
     src "./src/bin/flags.rs"
@@ -74,27 +73,6 @@ xflags::xflags! {
             optional --with-proc-macro
         }
 
-        /// Benchmark specific analysis operation
-        cmd analysis-bench
-            /// Directory with Cargo.toml.
-            required path: PathBuf
-        {
-            /// Collect memory usage statistics.
-            optional --memory-usage
-
-            /// Compute syntax highlighting for this file
-            optional --highlight path: PathBuf
-            /// Compute completions at file:line:column location.
-            optional --complete location: Position
-            /// Compute goto definition at file:line:column location.
-            optional --goto-def location: Position
-
-            /// Load OUT_DIR values by running `cargo check` before analysis.
-            optional --load-output-dirs
-            /// Use proc-macro-srv for proc-macro expanding.
-            optional --with-proc-macro
-        }
-
         cmd diagnostics
             /// Directory with Cargo.toml.
             required path: PathBuf
@@ -142,7 +120,6 @@ pub enum RustAnalyzerCmd {
     Symbols(Symbols),
     Highlight(Highlight),
     AnalysisStats(AnalysisStats),
-    AnalysisBench(AnalysisBench),
     Diagnostics(Diagnostics),
     Ssr(Ssr),
     Search(Search),
@@ -184,18 +161,6 @@ pub struct AnalysisStats {
 }
 
 #[derive(Debug)]
-pub struct AnalysisBench {
-    pub path: PathBuf,
-
-    pub memory_usage: bool,
-    pub highlight: Option<PathBuf>,
-    pub complete: Option<Position>,
-    pub goto_def: Option<Position>,
-    pub load_output_dirs: bool,
-    pub with_proc_macro: bool,
-}
-
-#[derive(Debug)]
 pub struct Diagnostics {
     pub path: PathBuf,
 
@@ -239,17 +204,3 @@ impl RustAnalyzer {
         }
     }
 }
-
-impl AnalysisBench {
-    pub(crate) fn what(&self) -> BenchWhat {
-        match (&self.highlight, &self.complete, &self.goto_def) {
-            (Some(path), None, None) => {
-                let path = env::current_dir().unwrap().join(path);
-                BenchWhat::Highlight { path: AbsPathBuf::assert(path) }
-            }
-            (None, Some(position), None) => BenchWhat::Complete(position.clone()),
-            (None, None, Some(position)) => BenchWhat::GotoDef(position.clone()),
-            _ => panic!("exactly one of  `--highlight`, `--complete` or `--goto-def` must be set"),
-        }
-    }
-}
diff --git a/crates/rust-analyzer/src/bin/main.rs b/crates/rust-analyzer/src/bin/main.rs
index a0b611bff..ae99eefe3 100644
--- a/crates/rust-analyzer/src/bin/main.rs
+++ b/crates/rust-analyzer/src/bin/main.rs
@@ -9,7 +9,7 @@ use std::{convert::TryFrom, env, fs, path::Path, process};
 use lsp_server::Connection;
 use project_model::ProjectManifest;
 use rust_analyzer::{
-    cli::{self, AnalysisStatsCmd, BenchCmd},
+    cli::{self, AnalysisStatsCmd},
     config::Config,
     from_json,
     lsp_ext::supports_utf8,
@@ -80,17 +80,6 @@ fn try_main() -> Result<()> {
             with_proc_macro: cmd.with_proc_macro,
         }
         .run(verbosity)?,
-        flags::RustAnalyzerCmd::AnalysisBench(cmd) => {
-            let what = cmd.what();
-            BenchCmd {
-                memory_usage: cmd.memory_usage,
-                path: cmd.path,
-                load_output_dirs: cmd.load_output_dirs,
-                with_proc_macro: cmd.with_proc_macro,
-                what,
-            }
-            .run(verbosity)?
-        }
 
         flags::RustAnalyzerCmd::Diagnostics(cmd) => {
             cli::diagnostics(&cmd.path, cmd.load_output_dirs, cmd.with_proc_macro)?
diff --git a/crates/rust-analyzer/src/cli.rs b/crates/rust-analyzer/src/cli.rs
index ed732eb38..76b666dc2 100644
--- a/crates/rust-analyzer/src/cli.rs
+++ b/crates/rust-analyzer/src/cli.rs
@@ -1,8 +1,7 @@
 //! Various batch processing tasks, intended primarily for debugging.
 
-mod load_cargo;
+pub(crate) mod load_cargo;
 mod analysis_stats;
-mod analysis_bench;
 mod diagnostics;
 mod progress_report;
 mod ssr;
@@ -15,7 +14,6 @@ use syntax::{AstNode, SourceFile};
 use vfs::Vfs;
 
 pub use self::{
-    analysis_bench::{BenchCmd, BenchWhat, Position},
     analysis_stats::AnalysisStatsCmd,
     diagnostics::diagnostics,
     load_cargo::{load_workspace, load_workspace_at, LoadCargoConfig},
diff --git a/crates/rust-analyzer/src/cli/analysis_bench.rs b/crates/rust-analyzer/src/cli/analysis_bench.rs
deleted file mode 100644
index 49994824f..000000000
--- a/crates/rust-analyzer/src/cli/analysis_bench.rs
+++ /dev/null
@@ -1,196 +0,0 @@
-//! Benchmark operations like highlighting or goto definition.
-
-use std::{env, path::PathBuf, str::FromStr, sync::Arc, time::Instant};
-
-use anyhow::{bail, format_err, Result};
-use hir::PrefixKind;
-use ide::{
-    Analysis, AnalysisHost, Change, CompletionConfig, DiagnosticsConfig, FilePosition, LineCol,
-};
-use ide_db::{
-    base_db::{
-        salsa::{Database, Durability},
-        FileId,
-    },
-    helpers::{insert_use::InsertUseConfig, SnippetCap},
-};
-use vfs::AbsPathBuf;
-
-use crate::cli::{
-    load_cargo::{load_workspace_at, LoadCargoConfig},
-    print_memory_usage, Verbosity,
-};
-
-pub struct BenchCmd {
-    pub path: PathBuf,
-    pub what: BenchWhat,
-    pub memory_usage: bool,
-    pub load_output_dirs: bool,
-    pub with_proc_macro: bool,
-}
-
-pub enum BenchWhat {
-    Highlight { path: AbsPathBuf },
-    Complete(Position),
-    GotoDef(Position),
-}
-
-#[derive(Debug, Clone)]
-pub struct Position {
-    pub path: AbsPathBuf,
-    pub line: u32,
-    pub column: u32,
-}
-
-impl FromStr for Position {
-    type Err = anyhow::Error;
-    fn from_str(s: &str) -> Result<Self> {
-        let mut split = s.rsplitn(3, ':');
-        match (split.next(), split.next(), split.next()) {
-            (Some(column), Some(line), Some(path)) => {
-                let path = env::current_dir().unwrap().join(path);
-                let path = AbsPathBuf::assert(path);
-                Ok(Position { path, line: line.parse()?, column: column.parse()? })
-            }
-            _ => bail!("position should be in file:line:column format: {:?}", s),
-        }
-    }
-}
-
-impl BenchCmd {
-    pub fn run(self, verbosity: Verbosity) -> Result<()> {
-        profile::init();
-
-        let start = Instant::now();
-        eprint!("loading: ");
-
-        let cargo_config = Default::default();
-        let load_cargo_config = LoadCargoConfig {
-            load_out_dirs_from_check: self.load_output_dirs,
-            with_proc_macro: self.with_proc_macro,
-        };
-        let (mut host, vfs, _proc_macro) =
-            load_workspace_at(&self.path, &cargo_config, &load_cargo_config, &|_| {})?;
-        eprintln!("{:?}\n", start.elapsed());
-
-        let file_id = {
-            let path = match &self.what {
-                BenchWhat::Highlight { path } => path,
-                BenchWhat::Complete(pos) | BenchWhat::GotoDef(pos) => &pos.path,
-            };
-            let path = path.clone().into();
-            vfs.file_id(&path).ok_or_else(|| format_err!("Can't find {}", path))?
-        };
-
-        match &self.what {
-            BenchWhat::Highlight { .. } => {
-                let res = do_work(&mut host, file_id, |analysis| {
-                    analysis.diagnostics(&DiagnosticsConfig::default(), file_id).unwrap();
-                    analysis.highlight_as_html(file_id, false).unwrap()
-                });
-                if verbosity.is_verbose() {
-                    println!("\n{}", res);
-                }
-            }
-            BenchWhat::Complete(pos) | BenchWhat::GotoDef(pos) => {
-                let is_completion = matches!(self.what, BenchWhat::Complete(..));
-
-                let offset = host
-                    .analysis()
-                    .file_line_index(file_id)?
-                    .offset(LineCol { line: pos.line - 1, col: pos.column });
-                let file_position = FilePosition { file_id, offset };
-
-                if is_completion {
-                    let options = CompletionConfig {
-                        enable_postfix_completions: true,
-                        enable_imports_on_the_fly: true,
-                        add_call_parenthesis: true,
-                        add_call_argument_snippets: true,
-                        snippet_cap: SnippetCap::new(true),
-                        insert_use: InsertUseConfig {
-                            merge: None,
-                            prefix_kind: PrefixKind::Plain,
-                            group: true,
-                        },
-                    };
-                    let res = do_work(&mut host, file_id, |analysis| {
-                        analysis.completions(&options, file_position)
-                    });
-                    if verbosity.is_verbose() {
-                        println!("\n{:#?}", res);
-                    }
-                } else {
-                    let res = do_work(&mut host, file_id, |analysis| {
-                        analysis.goto_definition(file_position)
-                    });
-                    if verbosity.is_verbose() {
-                        println!("\n{:#?}", res);
-                    }
-                }
-            }
-        }
-
-        if self.memory_usage {
-            print_memory_usage(host, vfs);
-        }
-
-        Ok(())
-    }
-}
-
-fn do_work<F: Fn(&Analysis) -> T, T>(host: &mut AnalysisHost, file_id: FileId, work: F) -> T {
-    {
-        let start = Instant::now();
-        eprint!("from scratch:   ");
-        work(&host.analysis());
-        eprintln!("{:?}", start.elapsed());
-    }
-    {
-        let start = Instant::now();
-        eprint!("no change:      ");
-        work(&host.analysis());
-        eprintln!("{:?}", start.elapsed());
-    }
-    {
-        let start = Instant::now();
-        eprint!("trivial change: ");
-        host.raw_database_mut().salsa_runtime_mut().synthetic_write(Durability::LOW);
-        work(&host.analysis());
-        eprintln!("{:?}", start.elapsed());
-    }
-    {
-        let start = Instant::now();
-        eprint!("comment change: ");
-        {
-            let mut text = host.analysis().file_text(file_id).unwrap().to_string();
-            text.push_str("\n/* Hello world */\n");
-            let mut change = Change::new();
-            change.change_file(file_id, Some(Arc::new(text)));
-            host.apply_change(change);
-        }
-        work(&host.analysis());
-        eprintln!("{:?}", start.elapsed());
-    }
-    {
-        let start = Instant::now();
-        eprint!("item change:    ");
-        {
-            let mut text = host.analysis().file_text(file_id).unwrap().to_string();
-            text.push_str("\npub fn _dummy() {}\n");
-            let mut change = Change::new();
-            change.change_file(file_id, Some(Arc::new(text)));
-            host.apply_change(change);
-        }
-        work(&host.analysis());
-        eprintln!("{:?}", start.elapsed());
-    }
-    {
-        let start = Instant::now();
-        eprint!("const change:   ");
-        host.raw_database_mut().salsa_runtime_mut().synthetic_write(Durability::HIGH);
-        let res = work(&host.analysis());
-        eprintln!("{:?}", start.elapsed());
-        res
-    }
-}
diff --git a/crates/rust-analyzer/src/lib.rs b/crates/rust-analyzer/src/lib.rs
index 8b874239c..d9a5030a0 100644
--- a/crates/rust-analyzer/src/lib.rs
+++ b/crates/rust-analyzer/src/lib.rs
@@ -39,6 +39,9 @@ mod op_queue;
 pub mod lsp_ext;
 pub mod config;
 
+#[cfg(test)]
+mod benchmarks;
+
 use serde::de::DeserializeOwned;
 use std::fmt;