Merge #3200

3200: Merge ra_vfs_glob and ra_lsp_server r=matklad a=matklad



Co-authored-by: Aleksey Kladov <[email protected]>

4 files changed, 690 insertions, 0 deletions

diff --git a/crates/ra_lsp_server/src/cli/analysis_bench.rs b/crates/ra_lsp_server/src/cli/analysis_bench.rs
new file mode 100644
index 000000000..e00f81073
--- /dev/null
+++ b/crates/ra_lsp_server/src/cli/analysis_bench.rs
@@ -0,0 +1,158 @@
+//! FIXME: write short doc here
+
+use std::{
+    path::{Path, PathBuf},
+    str::FromStr,
+    sync::Arc,
+    time::Instant,
+};
+
+use anyhow::{format_err, Result};
+use ra_db::{
+    salsa::{Database, Durability},
+    FileId, SourceDatabaseExt,
+};
+use ra_ide::{Analysis, AnalysisChange, AnalysisHost, FilePosition, LineCol};
+
+use crate::cli::{load_cargo::load_cargo, Verbosity};
+
+pub enum BenchWhat {
+    Highlight { path: PathBuf },
+    Complete(Position),
+    GotoDef(Position),
+}
+
+pub struct Position {
+    pub path: PathBuf,
+    pub line: u32,
+    pub column: u32,
+}
+
+impl FromStr for Position {
+    type Err = anyhow::Error;
+    fn from_str(s: &str) -> Result<Self> {
+        let (path_line, column) = rsplit_at_char(s, ':')?;
+        let (path, line) = rsplit_at_char(path_line, ':')?;
+        Ok(Position { path: path.into(), line: line.parse()?, column: column.parse()? })
+    }
+}
+
+fn rsplit_at_char(s: &str, c: char) -> Result<(&str, &str)> {
+    let idx = s.rfind(c).ok_or_else(|| format_err!("no `{}` in {}", c, s))?;
+    Ok((&s[..idx], &s[idx + 1..]))
+}
+
+pub fn analysis_bench(verbosity: Verbosity, path: &Path, what: BenchWhat) -> Result<()> {
+    ra_prof::init();
+
+    let start = Instant::now();
+    eprint!("loading: ");
+    let (mut host, roots) = load_cargo(path)?;
+    let db = host.raw_database();
+    eprintln!("{:?}\n", start.elapsed());
+
+    let file_id = {
+        let path = match &what {
+            BenchWhat::Highlight { path } => path,
+            BenchWhat::Complete(pos) | BenchWhat::GotoDef(pos) => &pos.path,
+        };
+        let path = std::env::current_dir()?.join(path).canonicalize()?;
+        roots
+            .iter()
+            .find_map(|(source_root_id, project_root)| {
+                if project_root.is_member() {
+                    for file_id in db.source_root(*source_root_id).walk() {
+                        let rel_path = db.file_relative_path(file_id);
+                        let abs_path = rel_path.to_path(project_root.path());
+                        if abs_path == path {
+                            return Some(file_id);
+                        }
+                    }
+                }
+                None
+            })
+            .ok_or_else(|| format_err!("Can't find {}", path.display()))?
+    };
+
+    match &what {
+        BenchWhat::Highlight { .. } => {
+            let res = do_work(&mut host, file_id, |analysis| {
+                analysis.diagnostics(file_id).unwrap();
+                analysis.highlight_as_html(file_id, false).unwrap()
+            });
+            if verbosity.is_verbose() {
+                println!("\n{}", res);
+            }
+        }
+        BenchWhat::Complete(pos) | BenchWhat::GotoDef(pos) => {
+            let is_completion = match what {
+                BenchWhat::Complete(..) => true,
+                _ => false,
+            };
+
+            let offset = host
+                .analysis()
+                .file_line_index(file_id)?
+                .offset(LineCol { line: pos.line - 1, col_utf16: pos.column });
+            let file_postion = FilePosition { file_id, offset };
+
+            if is_completion {
+                let res =
+                    do_work(&mut host, file_id, |analysis| analysis.completions(file_postion));
+                if verbosity.is_verbose() {
+                    println!("\n{:#?}", res);
+                }
+            } else {
+                let res =
+                    do_work(&mut host, file_id, |analysis| analysis.goto_definition(file_postion));
+                if verbosity.is_verbose() {
+                    println!("\n{:#?}", res);
+                }
+            }
+        }
+    }
+    Ok(())
+}
+
+fn do_work<F: Fn(&Analysis) -> T, T>(host: &mut AnalysisHost, file_id: FileId, work: F) -> T {
+    {
+        let start = Instant::now();
+        eprint!("from scratch:   ");
+        work(&host.analysis());
+        eprintln!("{:?}", start.elapsed());
+    }
+    {
+        let start = Instant::now();
+        eprint!("no change:      ");
+        work(&host.analysis());
+        eprintln!("{:?}", start.elapsed());
+    }
+    {
+        let start = Instant::now();
+        eprint!("trivial change: ");
+        host.raw_database_mut().salsa_runtime_mut().synthetic_write(Durability::LOW);
+        work(&host.analysis());
+        eprintln!("{:?}", start.elapsed());
+    }
+    {
+        let start = Instant::now();
+        eprint!("comment change: ");
+        {
+            let mut text = host.analysis().file_text(file_id).unwrap().to_string();
+            text.push_str("\n/* Hello world */\n");
+            let mut change = AnalysisChange::new();
+            change.change_file(file_id, Arc::new(text));
+            host.apply_change(change);
+        }
+        work(&host.analysis());
+        eprintln!("{:?}", start.elapsed());
+    }
+    {
+        let start = Instant::now();
+        eprint!("const change:   ");
+        host.raw_database_mut().salsa_runtime_mut().synthetic_write(Durability::HIGH);
+        let res = work(&host.analysis());
+        eprintln!("{:?}", start.elapsed());
+        res
+    }
+}
diff --git a/crates/ra_lsp_server/src/cli/analysis_stats.rs b/crates/ra_lsp_server/src/cli/analysis_stats.rs
new file mode 100644
index 000000000..c27fabe3c
--- /dev/null
+++ b/crates/ra_lsp_server/src/cli/analysis_stats.rs
@@ -0,0 +1,259 @@
+//! FIXME: write short doc here
+
+use std::{collections::HashSet, fmt::Write, path::Path, time::Instant};
+
+use hir::{
+    db::{DefDatabase, HirDatabase},
+    AssocItem, Crate, HasSource, HirDisplay, ModuleDef,
+};
+use hir_def::FunctionId;
+use hir_ty::{Ty, TypeWalk};
+use itertools::Itertools;
+use ra_db::SourceDatabaseExt;
+use ra_syntax::AstNode;
+use rand::{seq::SliceRandom, thread_rng};
+
+use crate::cli::{load_cargo::load_cargo, progress_report::ProgressReport, Result, Verbosity};
+
+pub fn analysis_stats(
+    verbosity: Verbosity,
+    memory_usage: bool,
+    path: &Path,
+    only: Option<&str>,
+    with_deps: bool,
+    randomize: bool,
+) -> Result<()> {
+    let db_load_time = Instant::now();
+    let (mut host, roots) = load_cargo(path)?;
+    let db = host.raw_database();
+    println!("Database loaded, {} roots, {:?}", roots.len(), db_load_time.elapsed());
+    let analysis_time = Instant::now();
+    let mut num_crates = 0;
+    let mut visited_modules = HashSet::new();
+    let mut visit_queue = Vec::new();
+
+    let members =
+        roots
+            .into_iter()
+            .filter_map(|(source_root_id, project_root)| {
+                if with_deps || project_root.is_member() {
+                    Some(source_root_id)
+                } else {
+                    None
+                }
+            })
+            .collect::<HashSet<_>>();
+
+    let mut krates = Crate::all(db);
+    if randomize {
+        krates.shuffle(&mut thread_rng());
+    }
+    for krate in krates {
+        let module = krate.root_module(db).expect("crate without root module");
+        let file_id = module.definition_source(db).file_id;
+        if members.contains(&db.file_source_root(file_id.original_file(db))) {
+            num_crates += 1;
+            visit_queue.push(module);
+        }
+    }
+
+    if randomize {
+        visit_queue.shuffle(&mut thread_rng());
+    }
+
+    println!("Crates in this dir: {}", num_crates);
+    let mut num_decls = 0;
+    let mut funcs = Vec::new();
+    while let Some(module) = visit_queue.pop() {
+        if visited_modules.insert(module) {
+            visit_queue.extend(module.children(db));
+
+            for decl in module.declarations(db) {
+                num_decls += 1;
+                if let ModuleDef::Function(f) = decl {
+                    funcs.push(f);
+                }
+            }
+
+            for impl_block in module.impl_blocks(db) {
+                for item in impl_block.items(db) {
+                    num_decls += 1;
+                    if let AssocItem::Function(f) = item {
+                        funcs.push(f);
+                    }
+                }
+            }
+        }
+    }
+    println!("Total modules found: {}", visited_modules.len());
+    println!("Total declarations: {}", num_decls);
+    println!("Total functions: {}", funcs.len());
+    println!("Item Collection: {:?}, {}", analysis_time.elapsed(), ra_prof::memory_usage());
+
+    if randomize {
+        funcs.shuffle(&mut thread_rng());
+    }
+
+    let inference_time = Instant::now();
+    let mut bar = match verbosity {
+        Verbosity::Quiet | Verbosity::Spammy => ProgressReport::hidden(),
+        _ => ProgressReport::new(funcs.len() as u64),
+    };
+
+    bar.tick();
+    let mut num_exprs = 0;
+    let mut num_exprs_unknown = 0;
+    let mut num_exprs_partially_unknown = 0;
+    let mut num_type_mismatches = 0;
+    for f in funcs {
+        let name = f.name(db);
+        let full_name = f
+            .module(db)
+            .path_to_root(db)
+            .into_iter()
+            .rev()
+            .filter_map(|it| it.name(db))
+            .chain(Some(f.name(db)))
+            .join("::");
+        if let Some(only_name) = only {
+            if name.to_string() != only_name && full_name != only_name {
+                continue;
+            }
+        }
+        let mut msg = format!("processing: {}", full_name);
+        if verbosity.is_verbose() {
+            let src = f.source(db);
+            let original_file = src.file_id.original_file(db);
+            let path = db.file_relative_path(original_file);
+            let syntax_range = src.value.syntax().text_range();
+            write!(msg, " ({:?} {})", path, syntax_range).unwrap();
+        }
+        if verbosity.is_spammy() {
+            bar.println(format!("{}", msg));
+        }
+        bar.set_message(&msg);
+        let f_id = FunctionId::from(f);
+        let body = db.body(f_id.into());
+        let inference_result = db.infer(f_id.into());
+        let (previous_exprs, previous_unknown, previous_partially_unknown) =
+            (num_exprs, num_exprs_unknown, num_exprs_partially_unknown);
+        for (expr_id, _) in body.exprs.iter() {
+            let ty = &inference_result[expr_id];
+            num_exprs += 1;
+            if let Ty::Unknown = ty {
+                num_exprs_unknown += 1;
+            } else {
+                let mut is_partially_unknown = false;
+                ty.walk(&mut |ty| {
+                    if let Ty::Unknown = ty {
+                        is_partially_unknown = true;
+                    }
+                });
+                if is_partially_unknown {
+                    num_exprs_partially_unknown += 1;
+                }
+            }
+            if only.is_some() && verbosity.is_spammy() {
+                // in super-verbose mode for just one function, we print every single expression
+                let (_, sm) = db.body_with_source_map(f_id.into());
+                let src = sm.expr_syntax(expr_id);
+                if let Some(src) = src {
+                    let original_file = src.file_id.original_file(db);
+                    let line_index = host.analysis().file_line_index(original_file).unwrap();
+                    let text_range = src.value.either(
+                        |it| it.syntax_node_ptr().range(),
+                        |it| it.syntax_node_ptr().range(),
+                    );
+                    let (start, end) = (
+                        line_index.line_col(text_range.start()),
+                        line_index.line_col(text_range.end()),
+                    );
+                    bar.println(format!(
+                        "{}:{}-{}:{}: {}",
+                        start.line + 1,
+                        start.col_utf16,
+                        end.line + 1,
+                        end.col_utf16,
+                        ty.display(db)
+                    ));
+                } else {
+                    bar.println(format!("unknown location: {}", ty.display(db)));
+                }
+            }
+            if let Some(mismatch) = inference_result.type_mismatch_for_expr(expr_id) {
+                num_type_mismatches += 1;
+                if verbosity.is_verbose() {
+                    let (_, sm) = db.body_with_source_map(f_id.into());
+                    let src = sm.expr_syntax(expr_id);
+                    if let Some(src) = src {
+                        // FIXME: it might be nice to have a function (on Analysis?) that goes from Source<T> -> (LineCol, LineCol) directly
+                        let original_file = src.file_id.original_file(db);
+                        let path = db.file_relative_path(original_file);
+                        let line_index = host.analysis().file_line_index(original_file).unwrap();
+                        let text_range = src.value.either(
+                            |it| it.syntax_node_ptr().range(),
+                            |it| it.syntax_node_ptr().range(),
+                        );
+                        let (start, end) = (
+                            line_index.line_col(text_range.start()),
+                            line_index.line_col(text_range.end()),
+                        );
+                        bar.println(format!(
+                            "{} {}:{}-{}:{}: Expected {}, got {}",
+                            path,
+                            start.line + 1,
+                            start.col_utf16,
+                            end.line + 1,
+                            end.col_utf16,
+                            mismatch.expected.display(db),
+                            mismatch.actual.display(db)
+                        ));
+                    } else {
+                        bar.println(format!(
+                            "{}: Expected {}, got {}",
+                            name,
+                            mismatch.expected.display(db),
+                            mismatch.actual.display(db)
+                        ));
+                    }
+                }
+            }
+        }
+        if verbosity.is_spammy() {
+            bar.println(format!(
+                "In {}: {} exprs, {} unknown, {} partial",
+                full_name,
+                num_exprs - previous_exprs,
+                num_exprs_unknown - previous_unknown,
+                num_exprs_partially_unknown - previous_partially_unknown
+            ));
+        }
+        bar.inc(1);
+    }
+    bar.finish_and_clear();
+    println!("Total expressions: {}", num_exprs);
+    println!(
+        "Expressions of unknown type: {} ({}%)",
+        num_exprs_unknown,
+        if num_exprs > 0 { num_exprs_unknown * 100 / num_exprs } else { 100 }
+    );
+    println!(
+        "Expressions of partially unknown type: {} ({}%)",
+        num_exprs_partially_unknown,
+        if num_exprs > 0 { num_exprs_partially_unknown * 100 / num_exprs } else { 100 }
+    );
+    println!("Type mismatches: {}", num_type_mismatches);
+    println!("Inference: {:?}, {}", inference_time.elapsed(), ra_prof::memory_usage());
+    println!("Total: {:?}, {}", analysis_time.elapsed(), ra_prof::memory_usage());
+
+    if memory_usage {
+        for (name, bytes) in host.per_query_memory_usage() {
+            println!("{:>8} {}", bytes, name)
+        }
+        let before = ra_prof::memory_usage();
+        drop(host);
+        println!("leftover: {}", before.allocated - ra_prof::memory_usage().allocated)
+    }
+
+    Ok(())
+}
diff --git a/crates/ra_lsp_server/src/cli/load_cargo.rs b/crates/ra_lsp_server/src/cli/load_cargo.rs
new file mode 100644
index 000000000..bb3e1513b
--- /dev/null
+++ b/crates/ra_lsp_server/src/cli/load_cargo.rs
@@ -0,0 +1,153 @@
+//! FIXME: write short doc here
+
+use std::{collections::HashSet, path::Path};
+
+use crossbeam_channel::{unbounded, Receiver};
+use ra_db::{CrateGraph, FileId, SourceRootId};
+use ra_ide::{AnalysisChange, AnalysisHost, FeatureFlags};
+use ra_project_model::{get_rustc_cfg_options, PackageRoot, ProjectWorkspace};
+use ra_vfs::{RootEntry, Vfs, VfsChange, VfsTask, Watch};
+use rustc_hash::FxHashMap;
+
+use crate::vfs_glob::RustPackageFilterBuilder;
+
+use anyhow::Result;
+
+fn vfs_file_to_id(f: ra_vfs::VfsFile) -> FileId {
+    FileId(f.0)
+}
+fn vfs_root_to_id(r: ra_vfs::VfsRoot) -> SourceRootId {
+    SourceRootId(r.0)
+}
+
+pub fn load_cargo(root: &Path) -> Result<(AnalysisHost, FxHashMap<SourceRootId, PackageRoot>)> {
+    let root = std::env::current_dir()?.join(root);
+    let ws = ProjectWorkspace::discover(root.as_ref(), &Default::default())?;
+    let project_roots = ws.to_roots();
+    let (sender, receiver) = unbounded();
+    let sender = Box::new(move |t| sender.send(t).unwrap());
+    let (mut vfs, roots) = Vfs::new(
+        project_roots
+            .iter()
+            .map(|pkg_root| {
+                RootEntry::new(
+                    pkg_root.path().clone(),
+                    RustPackageFilterBuilder::default()
+                        .set_member(pkg_root.is_member())
+                        .into_vfs_filter(),
+                )
+            })
+            .collect(),
+        sender,
+        Watch(false),
+    );
+
+    // FIXME: cfg options?
+    let default_cfg_options = {
+        let mut opts = get_rustc_cfg_options();
+        opts.insert_atom("test".into());
+        opts.insert_atom("debug_assertion".into());
+        opts
+    };
+
+    let (crate_graph, _crate_names) =
+        ws.to_crate_graph(&default_cfg_options, &mut |path: &Path| {
+            let vfs_file = vfs.load(path);
+            log::debug!("vfs file {:?} -> {:?}", path, vfs_file);
+            vfs_file.map(vfs_file_to_id)
+        });
+    log::debug!("crate graph: {:?}", crate_graph);
+
+    let source_roots = roots
+        .iter()
+        .map(|&vfs_root| {
+            let source_root_id = vfs_root_to_id(vfs_root);
+            let project_root = project_roots
+                .iter()
+                .find(|it| it.path() == &vfs.root2path(vfs_root))
+                .unwrap()
+                .clone();
+            (source_root_id, project_root)
+        })
+        .collect::<FxHashMap<_, _>>();
+    let host = load(&source_roots, crate_graph, &mut vfs, receiver);
+    Ok((host, source_roots))
+}
+
+pub fn load(
+    source_roots: &FxHashMap<SourceRootId, PackageRoot>,
+    crate_graph: CrateGraph,
+    vfs: &mut Vfs,
+    receiver: Receiver<VfsTask>,
+) -> AnalysisHost {
+    let lru_cap = std::env::var("RA_LRU_CAP").ok().and_then(|it| it.parse::<usize>().ok());
+    let mut host = AnalysisHost::new(lru_cap, FeatureFlags::default());
+    let mut analysis_change = AnalysisChange::new();
+    analysis_change.set_crate_graph(crate_graph);
+
+    // wait until Vfs has loaded all roots
+    let mut roots_loaded = HashSet::new();
+    for task in receiver {
+        vfs.handle_task(task);
+        let mut done = false;
+        for change in vfs.commit_changes() {
+            match change {
+                VfsChange::AddRoot { root, files } => {
+                    let source_root_id = vfs_root_to_id(root);
+                    let is_local = source_roots[&source_root_id].is_member();
+                    log::debug!(
+                        "loaded source root {:?} with path {:?}",
+                        source_root_id,
+                        vfs.root2path(root)
+                    );
+                    analysis_change.add_root(source_root_id, is_local);
+                    analysis_change.set_debug_root_path(
+                        source_root_id,
+                        source_roots[&source_root_id].path().display().to_string(),
+                    );
+
+                    let mut file_map = FxHashMap::default();
+                    for (vfs_file, path, text) in files {
+                        let file_id = vfs_file_to_id(vfs_file);
+                        analysis_change.add_file(source_root_id, file_id, path.clone(), text);
+                        file_map.insert(path, file_id);
+                    }
+                    roots_loaded.insert(source_root_id);
+                    if roots_loaded.len() == vfs.n_roots() {
+                        done = true;
+                    }
+                }
+                VfsChange::AddFile { root, file, path, text } => {
+                    let source_root_id = vfs_root_to_id(root);
+                    let file_id = vfs_file_to_id(file);
+                    analysis_change.add_file(source_root_id, file_id, path, text);
+                }
+                VfsChange::RemoveFile { .. } | VfsChange::ChangeFile { .. } => {
+                    // We just need the first scan, so just ignore these
+                }
+            }
+        }
+        if done {
+            break;
+        }
+    }
+
+    host.apply_change(analysis_change);
+    host
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use hir::Crate;
+
+    #[test]
+    fn test_loading_rust_analyzer() {
+        let path = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap().parent().unwrap();
+        let (host, _roots) = load_cargo(path).unwrap();
+        let n_crates = Crate::all(host.raw_database()).len();
+        // RA has quite a few crates, but the exact count doesn't matter
+        assert!(n_crates > 20);
+    }
+}
diff --git a/crates/ra_lsp_server/src/cli/progress_report.rs b/crates/ra_lsp_server/src/cli/progress_report.rs
new file mode 100644
index 000000000..31867a1e9
--- /dev/null
+++ b/crates/ra_lsp_server/src/cli/progress_report.rs
@@ -0,0 +1,120 @@
+//! A simple progress bar
+//!
+//! A single thread non-optimized progress bar
+use std::io::Write;
+
+/// A Simple ASCII Progress Bar
+pub struct ProgressReport {
+    curr: f32,
+    text: String,
+    hidden: bool,
+
+    len: u64,
+    pos: u64,
+    msg: String,
+}
+
+impl ProgressReport {
+    pub fn new(len: u64) -> ProgressReport {
+        ProgressReport {
+            curr: 0.0,
+            text: String::new(),
+            hidden: false,
+            len,
+            pos: 0,
+            msg: String::new(),
+        }
+    }
+
+    pub fn hidden() -> ProgressReport {
+        ProgressReport {
+            curr: 0.0,
+            text: String::new(),
+            hidden: true,
+            len: 0,
+            pos: 0,
+            msg: String::new(),
+        }
+    }
+
+    pub fn set_message(&mut self, msg: &str) {
+        self.msg = msg.to_string();
+        self.tick();
+    }
+
+    pub fn println<I: Into<String>>(&mut self, msg: I) {
+        self.clear();
+        println!("{}", msg.into());
+        self.tick();
+    }
+
+    pub fn inc(&mut self, delta: u64) {
+        self.pos += delta;
+        if self.len == 0 {
+            self.set_value(0.0)
+        } else {
+            self.set_value((self.pos as f32) / (self.len as f32))
+        }
+        self.tick();
+    }
+
+    pub fn finish_and_clear(&mut self) {
+        self.clear();
+    }
+
+    pub fn tick(&mut self) {
+        if self.hidden {
+            return;
+        }
+        let percent = (self.curr * 100.0) as u32;
+        let text = format!("{}/{} {:3>}% {}", self.pos, self.len, percent, self.msg);
+        self.update_text(&text);
+    }
+
+    fn update_text(&mut self, text: &str) {
+        // Get length of common portion
+        let mut common_prefix_length = 0;
+        let common_length = usize::min(self.text.len(), text.len());
+
+        while common_prefix_length < common_length
+            && text.chars().nth(common_prefix_length).unwrap()
+                == self.text.chars().nth(common_prefix_length).unwrap()
+        {
+            common_prefix_length += 1;
+        }
+
+        // Backtrack to the first differing character
+        let mut output = String::new();
+        output += &'\x08'.to_string().repeat(self.text.len() - common_prefix_length);
+        // Output new suffix
+        output += &text[common_prefix_length..text.len()];
+
+        // If the new text is shorter than the old one: delete overlapping characters
+        if let Some(overlap_count) = self.text.len().checked_sub(text.len()) {
+            if overlap_count > 0 {
+                output += &" ".repeat(overlap_count);
+                output += &"\x08".repeat(overlap_count);
+            }
+        }
+
+        let _ = std::io::stdout().write(output.as_bytes());
+        let _ = std::io::stdout().flush();
+        self.text = text.to_string();
+    }
+
+    fn set_value(&mut self, value: f32) {
+        self.curr = f32::max(0.0, f32::min(1.0, value));
+    }
+
+    fn clear(&mut self) {
+        if self.hidden {
+            return;
+        }
+
+        // Fill all last text to space and return the cursor
+        let spaces = " ".repeat(self.text.len());
+        let backspaces = "\x08".repeat(self.text.len());
+        print!("{}{}{}", backspaces, spaces, backspaces);
+        self.text = String::new();
+    }
+}