Merge #3200

3200: Merge ra_vfs_glob and ra_lsp_server r=matklad a=matklad



Co-authored-by: Aleksey Kladov <[email protected]>

11 files changed, 1135 insertions, 19 deletions

diff --git a/crates/ra_lsp_server/src/args.rs b/crates/ra_lsp_server/src/args.rs
new file mode 100644
index 000000000..89e2139ce
--- /dev/null
+++ b/crates/ra_lsp_server/src/args.rs
@@ -0,0 +1,242 @@
+//! Command like parsing for rust-analyzer.
+//!
+//! If run started args, we run the LSP server loop. With a subcommand, we do a
+//! one-time batch processing.
+
+use anyhow::{bail, Result};
+use pico_args::Arguments;
+use ra_lsp_server::cli::{BenchWhat, Position, Verbosity};
+
+use std::{fmt::Write, path::PathBuf};
+
+pub(crate) struct Args {
+    pub(crate) verbosity: Verbosity,
+    pub(crate) command: Command,
+}
+
+pub(crate) enum Command {
+    Parse {
+        no_dump: bool,
+    },
+    Symbols,
+    Highlight {
+        rainbow: bool,
+    },
+    Stats {
+        randomize: bool,
+        memory_usage: bool,
+        only: Option<String>,
+        with_deps: bool,
+        path: PathBuf,
+    },
+    Bench {
+        path: PathBuf,
+        what: BenchWhat,
+    },
+    RunServer,
+    Version,
+}
+
+impl Args {
+    pub(crate) fn parse() -> Result<Result<Args, HelpPrinted>> {
+        let mut matches = Arguments::from_env();
+
+        if matches.contains("--version") {
+            matches.finish().or_else(handle_extra_flags)?;
+            return Ok(Ok(Args { verbosity: Verbosity::Normal, command: Command::Version }));
+        }
+
+        let verbosity = match (
+            matches.contains(["-vv", "--spammy"]),
+            matches.contains(["-v", "--verbose"]),
+            matches.contains(["-q", "--quiet"]),
+        ) {
+            (true, _, true) => bail!("Invalid flags: -q conflicts with -vv"),
+            (true, _, false) => Verbosity::Spammy,
+            (false, false, false) => Verbosity::Normal,
+            (false, false, true) => Verbosity::Quiet,
+            (false, true, false) => Verbosity::Verbose,
+            (false, true, true) => bail!("Invalid flags: -q conflicts with -v"),
+        };
+
+        let subcommand = match matches.subcommand()? {
+            Some(it) => it,
+            None => {
+                matches.finish().or_else(handle_extra_flags)?;
+                return Ok(Ok(Args { verbosity, command: Command::RunServer }));
+            }
+        };
+        let command = match subcommand.as_str() {
+            "parse" => {
+                if matches.contains(["-h", "--help"]) {
+                    eprintln!(
+                        "\
+ra-cli-parse
+
+USAGE:
+    ra_cli parse [FLAGS]
+
+FLAGS:
+    -h, --help       Prints help inforamtion
+        --no-dump"
+                    );
+                    return Ok(Err(HelpPrinted));
+                }
+
+                let no_dump = matches.contains("--no-dump");
+                matches.finish().or_else(handle_extra_flags)?;
+                Command::Parse { no_dump }
+            }
+            "symbols" => {
+                if matches.contains(["-h", "--help"]) {
+                    eprintln!(
+                        "\
+ra-cli-symbols
+
+USAGE:
+    ra_cli highlight [FLAGS]
+
+FLAGS:
+    -h, --help    Prints help inforamtion"
+                    );
+                    return Ok(Err(HelpPrinted));
+                }
+
+                matches.finish().or_else(handle_extra_flags)?;
+
+                Command::Symbols
+            }
+            "highlight" => {
+                if matches.contains(["-h", "--help"]) {
+                    eprintln!(
+                        "\
+ra-cli-highlight
+
+USAGE:
+    ra_cli highlight [FLAGS]
+
+FLAGS:
+    -h, --help       Prints help information
+    -r, --rainbow"
+                    );
+                    return Ok(Err(HelpPrinted));
+                }
+
+                let rainbow = matches.contains(["-r", "--rainbow"]);
+                matches.finish().or_else(handle_extra_flags)?;
+                Command::Highlight { rainbow }
+            }
+            "analysis-stats" => {
+                if matches.contains(["-h", "--help"]) {
+                    eprintln!(
+                        "\
+ra-cli-analysis-stats
+
+USAGE:
+    ra_cli analysis-stats [FLAGS] [OPTIONS] [PATH]
+
+FLAGS:
+    -h, --help            Prints help information
+        --memory-usage
+    -v, --verbose
+    -q, --quiet
+
+OPTIONS:
+    -o <ONLY>
+
+ARGS:
+    <PATH>"
+                    );
+                    return Ok(Err(HelpPrinted));
+                }
+
+                let randomize = matches.contains("--randomize");
+                let memory_usage = matches.contains("--memory-usage");
+                let only: Option<String> = matches.opt_value_from_str(["-o", "--only"])?;
+                let with_deps: bool = matches.contains("--with-deps");
+                let path = {
+                    let mut trailing = matches.free()?;
+                    if trailing.len() != 1 {
+                        bail!("Invalid flags");
+                    }
+                    trailing.pop().unwrap().into()
+                };
+
+                Command::Stats { randomize, memory_usage, only, with_deps, path }
+            }
+            "analysis-bench" => {
+                if matches.contains(["-h", "--help"]) {
+                    eprintln!(
+                        "\
+ra_cli-analysis-bench
+
+USAGE:
+    ra_cli analysis-bench [FLAGS] [OPTIONS] [PATH]
+
+FLAGS:
+    -h, --help        Prints help information
+    -v, --verbose
+
+OPTIONS:
+    --complete <PATH:LINE:COLUMN>    Compute completions at this location
+    --highlight <PATH>               Hightlight this file
+
+ARGS:
+    <PATH>    Project to analyse"
+                    );
+                    return Ok(Err(HelpPrinted));
+                }
+
+                let path: PathBuf = matches.opt_value_from_str("--path")?.unwrap_or_default();
+                let highlight_path: Option<String> = matches.opt_value_from_str("--highlight")?;
+                let complete_path: Option<Position> = matches.opt_value_from_str("--complete")?;
+                let goto_def_path: Option<Position> = matches.opt_value_from_str("--goto-def")?;
+                let what = match (highlight_path, complete_path, goto_def_path) {
+                    (Some(path), None, None) => BenchWhat::Highlight { path: path.into() },
+                    (None, Some(position), None) => BenchWhat::Complete(position),
+                    (None, None, Some(position)) => BenchWhat::GotoDef(position),
+                    _ => panic!(
+                        "exactly one of  `--highlight`, `--complete` or `--goto-def` must be set"
+                    ),
+                };
+                Command::Bench { path, what }
+            }
+            _ => {
+                eprintln!(
+                    "\
+ra-cli
+
+USAGE:
+    ra_cli <SUBCOMMAND>
+
+FLAGS:
+    -h, --help        Prints help information
+
+SUBCOMMANDS:
+    analysis-bench
+    analysis-stats
+    highlight
+    parse
+    symbols"
+                );
+                return Ok(Err(HelpPrinted));
+            }
+        };
+        Ok(Ok(Args { verbosity, command }))
+    }
+}
+
+pub(crate) struct HelpPrinted;
+
+fn handle_extra_flags(e: pico_args::Error) -> Result<()> {
+    if let pico_args::Error::UnusedArgsLeft(flags) = e {
+        let mut invalid_flags = String::new();
+        for flag in flags {
+            write!(&mut invalid_flags, "{}, ", flag)?;
+        }
+        let (invalid_flags, _) = invalid_flags.split_at(invalid_flags.len() - 2);
+        bail!("Invalid flags: {}", invalid_flags);
+    } else {
+        bail!(e);
+    }
+}
diff --git a/crates/ra_lsp_server/src/cli.rs b/crates/ra_lsp_server/src/cli.rs
new file mode 100644
index 000000000..3c7b8e250
--- /dev/null
+++ b/crates/ra_lsp_server/src/cli.rs
@@ -0,0 +1,75 @@
+//! FIXME: write short doc here
+
+mod load_cargo;
+mod analysis_stats;
+mod analysis_bench;
+mod progress_report;
+
+use std::io::Read;
+
+use anyhow::Result;
+use ra_ide::{file_structure, Analysis};
+use ra_prof::profile;
+use ra_syntax::{AstNode, SourceFile};
+
+#[derive(Clone, Copy)]
+pub enum Verbosity {
+    Spammy,
+    Verbose,
+    Normal,
+    Quiet,
+}
+
+impl Verbosity {
+    pub fn is_verbose(self) -> bool {
+        match self {
+            Verbosity::Verbose | Verbosity::Spammy => true,
+            _ => false,
+        }
+    }
+    pub fn is_spammy(self) -> bool {
+        match self {
+            Verbosity::Spammy => true,
+            _ => false,
+        }
+    }
+}
+
+pub fn parse(no_dump: bool) -> Result<()> {
+    let _p = profile("parsing");
+    let file = file()?;
+    if !no_dump {
+        println!("{:#?}", file.syntax());
+    }
+    std::mem::forget(file);
+    Ok(())
+}
+
+pub fn symbols() -> Result<()> {
+    let file = file()?;
+    for s in file_structure(&file) {
+        println!("{:?}", s);
+    }
+    Ok(())
+}
+
+pub fn highlight(rainbow: bool) -> Result<()> {
+    let (analysis, file_id) = Analysis::from_single_file(read_stdin()?);
+    let html = analysis.highlight_as_html(file_id, rainbow).unwrap();
+    println!("{}", html);
+    Ok(())
+}
+
+pub use analysis_bench::{analysis_bench, BenchWhat, Position};
+pub use analysis_stats::analysis_stats;
+
+fn file() -> Result<SourceFile> {
+    let text = read_stdin()?;
+    Ok(SourceFile::parse(&text).tree())
+}
+
+fn read_stdin() -> Result<String> {
+    let mut buff = String::new();
+    std::io::stdin().read_to_string(&mut buff)?;
+    Ok(buff)
+}
diff --git a/crates/ra_lsp_server/src/cli/analysis_bench.rs b/crates/ra_lsp_server/src/cli/analysis_bench.rs
new file mode 100644
index 000000000..e00f81073
--- /dev/null
+++ b/crates/ra_lsp_server/src/cli/analysis_bench.rs
@@ -0,0 +1,158 @@
+//! FIXME: write short doc here
+
+use std::{
+    path::{Path, PathBuf},
+    str::FromStr,
+    sync::Arc,
+    time::Instant,
+};
+
+use anyhow::{format_err, Result};
+use ra_db::{
+    salsa::{Database, Durability},
+    FileId, SourceDatabaseExt,
+};
+use ra_ide::{Analysis, AnalysisChange, AnalysisHost, FilePosition, LineCol};
+
+use crate::cli::{load_cargo::load_cargo, Verbosity};
+
+pub enum BenchWhat {
+    Highlight { path: PathBuf },
+    Complete(Position),
+    GotoDef(Position),
+}
+
+pub struct Position {
+    pub path: PathBuf,
+    pub line: u32,
+    pub column: u32,
+}
+
+impl FromStr for Position {
+    type Err = anyhow::Error;
+    fn from_str(s: &str) -> Result<Self> {
+        let (path_line, column) = rsplit_at_char(s, ':')?;
+        let (path, line) = rsplit_at_char(path_line, ':')?;
+        Ok(Position { path: path.into(), line: line.parse()?, column: column.parse()? })
+    }
+}
+
+fn rsplit_at_char(s: &str, c: char) -> Result<(&str, &str)> {
+    let idx = s.rfind(c).ok_or_else(|| format_err!("no `{}` in {}", c, s))?;
+    Ok((&s[..idx], &s[idx + 1..]))
+}
+
+pub fn analysis_bench(verbosity: Verbosity, path: &Path, what: BenchWhat) -> Result<()> {
+    ra_prof::init();
+
+    let start = Instant::now();
+    eprint!("loading: ");
+    let (mut host, roots) = load_cargo(path)?;
+    let db = host.raw_database();
+    eprintln!("{:?}\n", start.elapsed());
+
+    let file_id = {
+        let path = match &what {
+            BenchWhat::Highlight { path } => path,
+            BenchWhat::Complete(pos) | BenchWhat::GotoDef(pos) => &pos.path,
+        };
+        let path = std::env::current_dir()?.join(path).canonicalize()?;
+        roots
+            .iter()
+            .find_map(|(source_root_id, project_root)| {
+                if project_root.is_member() {
+                    for file_id in db.source_root(*source_root_id).walk() {
+                        let rel_path = db.file_relative_path(file_id);
+                        let abs_path = rel_path.to_path(project_root.path());
+                        if abs_path == path {
+                            return Some(file_id);
+                        }
+                    }
+                }
+                None
+            })
+            .ok_or_else(|| format_err!("Can't find {}", path.display()))?
+    };
+
+    match &what {
+        BenchWhat::Highlight { .. } => {
+            let res = do_work(&mut host, file_id, |analysis| {
+                analysis.diagnostics(file_id).unwrap();
+                analysis.highlight_as_html(file_id, false).unwrap()
+            });
+            if verbosity.is_verbose() {
+                println!("\n{}", res);
+            }
+        }
+        BenchWhat::Complete(pos) | BenchWhat::GotoDef(pos) => {
+            let is_completion = match what {
+                BenchWhat::Complete(..) => true,
+                _ => false,
+            };
+
+            let offset = host
+                .analysis()
+                .file_line_index(file_id)?
+                .offset(LineCol { line: pos.line - 1, col_utf16: pos.column });
+            let file_postion = FilePosition { file_id, offset };
+
+            if is_completion {
+                let res =
+                    do_work(&mut host, file_id, |analysis| analysis.completions(file_postion));
+                if verbosity.is_verbose() {
+                    println!("\n{:#?}", res);
+                }
+            } else {
+                let res =
+                    do_work(&mut host, file_id, |analysis| analysis.goto_definition(file_postion));
+                if verbosity.is_verbose() {
+                    println!("\n{:#?}", res);
+                }
+            }
+        }
+    }
+    Ok(())
+}
+
+fn do_work<F: Fn(&Analysis) -> T, T>(host: &mut AnalysisHost, file_id: FileId, work: F) -> T {
+    {
+        let start = Instant::now();
+        eprint!("from scratch:   ");
+        work(&host.analysis());
+        eprintln!("{:?}", start.elapsed());
+    }
+    {
+        let start = Instant::now();
+        eprint!("no change:      ");
+        work(&host.analysis());
+        eprintln!("{:?}", start.elapsed());
+    }
+    {
+        let start = Instant::now();
+        eprint!("trivial change: ");
+        host.raw_database_mut().salsa_runtime_mut().synthetic_write(Durability::LOW);
+        work(&host.analysis());
+        eprintln!("{:?}", start.elapsed());
+    }
+    {
+        let start = Instant::now();
+        eprint!("comment change: ");
+        {
+            let mut text = host.analysis().file_text(file_id).unwrap().to_string();
+            text.push_str("\n/* Hello world */\n");
+            let mut change = AnalysisChange::new();
+            change.change_file(file_id, Arc::new(text));
+            host.apply_change(change);
+        }
+        work(&host.analysis());
+        eprintln!("{:?}", start.elapsed());
+    }
+    {
+        let start = Instant::now();
+        eprint!("const change:   ");
+        host.raw_database_mut().salsa_runtime_mut().synthetic_write(Durability::HIGH);
+        let res = work(&host.analysis());
+        eprintln!("{:?}", start.elapsed());
+        res
+    }
+}
diff --git a/crates/ra_lsp_server/src/cli/analysis_stats.rs b/crates/ra_lsp_server/src/cli/analysis_stats.rs
new file mode 100644
index 000000000..c27fabe3c
--- /dev/null
+++ b/crates/ra_lsp_server/src/cli/analysis_stats.rs
@@ -0,0 +1,259 @@
+//! FIXME: write short doc here
+
+use std::{collections::HashSet, fmt::Write, path::Path, time::Instant};
+
+use hir::{
+    db::{DefDatabase, HirDatabase},
+    AssocItem, Crate, HasSource, HirDisplay, ModuleDef,
+};
+use hir_def::FunctionId;
+use hir_ty::{Ty, TypeWalk};
+use itertools::Itertools;
+use ra_db::SourceDatabaseExt;
+use ra_syntax::AstNode;
+use rand::{seq::SliceRandom, thread_rng};
+
+use crate::cli::{load_cargo::load_cargo, progress_report::ProgressReport, Result, Verbosity};
+
+pub fn analysis_stats(
+    verbosity: Verbosity,
+    memory_usage: bool,
+    path: &Path,
+    only: Option<&str>,
+    with_deps: bool,
+    randomize: bool,
+) -> Result<()> {
+    let db_load_time = Instant::now();
+    let (mut host, roots) = load_cargo(path)?;
+    let db = host.raw_database();
+    println!("Database loaded, {} roots, {:?}", roots.len(), db_load_time.elapsed());
+    let analysis_time = Instant::now();
+    let mut num_crates = 0;
+    let mut visited_modules = HashSet::new();
+    let mut visit_queue = Vec::new();
+
+    let members =
+        roots
+            .into_iter()
+            .filter_map(|(source_root_id, project_root)| {
+                if with_deps || project_root.is_member() {
+                    Some(source_root_id)
+                } else {
+                    None
+                }
+            })
+            .collect::<HashSet<_>>();
+
+    let mut krates = Crate::all(db);
+    if randomize {
+        krates.shuffle(&mut thread_rng());
+    }
+    for krate in krates {
+        let module = krate.root_module(db).expect("crate without root module");
+        let file_id = module.definition_source(db).file_id;
+        if members.contains(&db.file_source_root(file_id.original_file(db))) {
+            num_crates += 1;
+            visit_queue.push(module);
+        }
+    }
+
+    if randomize {
+        visit_queue.shuffle(&mut thread_rng());
+    }
+
+    println!("Crates in this dir: {}", num_crates);
+    let mut num_decls = 0;
+    let mut funcs = Vec::new();
+    while let Some(module) = visit_queue.pop() {
+        if visited_modules.insert(module) {
+            visit_queue.extend(module.children(db));
+
+            for decl in module.declarations(db) {
+                num_decls += 1;
+                if let ModuleDef::Function(f) = decl {
+                    funcs.push(f);
+                }
+            }
+
+            for impl_block in module.impl_blocks(db) {
+                for item in impl_block.items(db) {
+                    num_decls += 1;
+                    if let AssocItem::Function(f) = item {
+                        funcs.push(f);
+                    }
+                }
+            }
+        }
+    }
+    println!("Total modules found: {}", visited_modules.len());
+    println!("Total declarations: {}", num_decls);
+    println!("Total functions: {}", funcs.len());
+    println!("Item Collection: {:?}, {}", analysis_time.elapsed(), ra_prof::memory_usage());
+
+    if randomize {
+        funcs.shuffle(&mut thread_rng());
+    }
+
+    let inference_time = Instant::now();
+    let mut bar = match verbosity {
+        Verbosity::Quiet | Verbosity::Spammy => ProgressReport::hidden(),
+        _ => ProgressReport::new(funcs.len() as u64),
+    };
+
+    bar.tick();
+    let mut num_exprs = 0;
+    let mut num_exprs_unknown = 0;
+    let mut num_exprs_partially_unknown = 0;
+    let mut num_type_mismatches = 0;
+    for f in funcs {
+        let name = f.name(db);
+        let full_name = f
+            .module(db)
+            .path_to_root(db)
+            .into_iter()
+            .rev()
+            .filter_map(|it| it.name(db))
+            .chain(Some(f.name(db)))
+            .join("::");
+        if let Some(only_name) = only {
+            if name.to_string() != only_name && full_name != only_name {
+                continue;
+            }
+        }
+        let mut msg = format!("processing: {}", full_name);
+        if verbosity.is_verbose() {
+            let src = f.source(db);
+            let original_file = src.file_id.original_file(db);
+            let path = db.file_relative_path(original_file);
+            let syntax_range = src.value.syntax().text_range();
+            write!(msg, " ({:?} {})", path, syntax_range).unwrap();
+        }
+        if verbosity.is_spammy() {
+            bar.println(format!("{}", msg));
+        }
+        bar.set_message(&msg);
+        let f_id = FunctionId::from(f);
+        let body = db.body(f_id.into());
+        let inference_result = db.infer(f_id.into());
+        let (previous_exprs, previous_unknown, previous_partially_unknown) =
+            (num_exprs, num_exprs_unknown, num_exprs_partially_unknown);
+        for (expr_id, _) in body.exprs.iter() {
+            let ty = &inference_result[expr_id];
+            num_exprs += 1;
+            if let Ty::Unknown = ty {
+                num_exprs_unknown += 1;
+            } else {
+                let mut is_partially_unknown = false;
+                ty.walk(&mut |ty| {
+                    if let Ty::Unknown = ty {
+                        is_partially_unknown = true;
+                    }
+                });
+                if is_partially_unknown {
+                    num_exprs_partially_unknown += 1;
+                }
+            }
+            if only.is_some() && verbosity.is_spammy() {
+                // in super-verbose mode for just one function, we print every single expression
+                let (_, sm) = db.body_with_source_map(f_id.into());
+                let src = sm.expr_syntax(expr_id);
+                if let Some(src) = src {
+                    let original_file = src.file_id.original_file(db);
+                    let line_index = host.analysis().file_line_index(original_file).unwrap();
+                    let text_range = src.value.either(
+                        |it| it.syntax_node_ptr().range(),
+                        |it| it.syntax_node_ptr().range(),
+                    );
+                    let (start, end) = (
+                        line_index.line_col(text_range.start()),
+                        line_index.line_col(text_range.end()),
+                    );
+                    bar.println(format!(
+                        "{}:{}-{}:{}: {}",
+                        start.line + 1,
+                        start.col_utf16,
+                        end.line + 1,
+                        end.col_utf16,
+                        ty.display(db)
+                    ));
+                } else {
+                    bar.println(format!("unknown location: {}", ty.display(db)));
+                }
+            }
+            if let Some(mismatch) = inference_result.type_mismatch_for_expr(expr_id) {
+                num_type_mismatches += 1;
+                if verbosity.is_verbose() {
+                    let (_, sm) = db.body_with_source_map(f_id.into());
+                    let src = sm.expr_syntax(expr_id);
+                    if let Some(src) = src {
+                        // FIXME: it might be nice to have a function (on Analysis?) that goes from Source<T> -> (LineCol, LineCol) directly
+                        let original_file = src.file_id.original_file(db);
+                        let path = db.file_relative_path(original_file);
+                        let line_index = host.analysis().file_line_index(original_file).unwrap();
+                        let text_range = src.value.either(
+                            |it| it.syntax_node_ptr().range(),
+                            |it| it.syntax_node_ptr().range(),
+                        );
+                        let (start, end) = (
+                            line_index.line_col(text_range.start()),
+                            line_index.line_col(text_range.end()),
+                        );
+                        bar.println(format!(
+                            "{} {}:{}-{}:{}: Expected {}, got {}",
+                            path,
+                            start.line + 1,
+                            start.col_utf16,
+                            end.line + 1,
+                            end.col_utf16,
+                            mismatch.expected.display(db),
+                            mismatch.actual.display(db)
+                        ));
+                    } else {
+                        bar.println(format!(
+                            "{}: Expected {}, got {}",
+                            name,
+                            mismatch.expected.display(db),
+                            mismatch.actual.display(db)
+                        ));
+                    }
+                }
+            }
+        }
+        if verbosity.is_spammy() {
+            bar.println(format!(
+                "In {}: {} exprs, {} unknown, {} partial",
+                full_name,
+                num_exprs - previous_exprs,
+                num_exprs_unknown - previous_unknown,
+                num_exprs_partially_unknown - previous_partially_unknown
+            ));
+        }
+        bar.inc(1);
+    }
+    bar.finish_and_clear();
+    println!("Total expressions: {}", num_exprs);
+    println!(
+        "Expressions of unknown type: {} ({}%)",
+        num_exprs_unknown,
+        if num_exprs > 0 { num_exprs_unknown * 100 / num_exprs } else { 100 }
+    );
+    println!(
+        "Expressions of partially unknown type: {} ({}%)",
+        num_exprs_partially_unknown,
+        if num_exprs > 0 { num_exprs_partially_unknown * 100 / num_exprs } else { 100 }
+    );
+    println!("Type mismatches: {}", num_type_mismatches);
+    println!("Inference: {:?}, {}", inference_time.elapsed(), ra_prof::memory_usage());
+    println!("Total: {:?}, {}", analysis_time.elapsed(), ra_prof::memory_usage());
+
+    if memory_usage {
+        for (name, bytes) in host.per_query_memory_usage() {
+            println!("{:>8} {}", bytes, name)
+        }
+        let before = ra_prof::memory_usage();
+        drop(host);
+        println!("leftover: {}", before.allocated - ra_prof::memory_usage().allocated)
+    }
+
+    Ok(())
+}
diff --git a/crates/ra_lsp_server/src/cli/load_cargo.rs b/crates/ra_lsp_server/src/cli/load_cargo.rs
new file mode 100644
index 000000000..bb3e1513b
--- /dev/null
+++ b/crates/ra_lsp_server/src/cli/load_cargo.rs
@@ -0,0 +1,153 @@
+//! FIXME: write short doc here
+
+use std::{collections::HashSet, path::Path};
+
+use crossbeam_channel::{unbounded, Receiver};
+use ra_db::{CrateGraph, FileId, SourceRootId};
+use ra_ide::{AnalysisChange, AnalysisHost, FeatureFlags};
+use ra_project_model::{get_rustc_cfg_options, PackageRoot, ProjectWorkspace};
+use ra_vfs::{RootEntry, Vfs, VfsChange, VfsTask, Watch};
+use rustc_hash::FxHashMap;
+
+use crate::vfs_glob::RustPackageFilterBuilder;
+
+use anyhow::Result;
+
+fn vfs_file_to_id(f: ra_vfs::VfsFile) -> FileId {
+    FileId(f.0)
+}
+fn vfs_root_to_id(r: ra_vfs::VfsRoot) -> SourceRootId {
+    SourceRootId(r.0)
+}
+
+pub fn load_cargo(root: &Path) -> Result<(AnalysisHost, FxHashMap<SourceRootId, PackageRoot>)> {
+    let root = std::env::current_dir()?.join(root);
+    let ws = ProjectWorkspace::discover(root.as_ref(), &Default::default())?;
+    let project_roots = ws.to_roots();
+    let (sender, receiver) = unbounded();
+    let sender = Box::new(move |t| sender.send(t).unwrap());
+    let (mut vfs, roots) = Vfs::new(
+        project_roots
+            .iter()
+            .map(|pkg_root| {
+                RootEntry::new(
+                    pkg_root.path().clone(),
+                    RustPackageFilterBuilder::default()
+                        .set_member(pkg_root.is_member())
+                        .into_vfs_filter(),
+                )
+            })
+            .collect(),
+        sender,
+        Watch(false),
+    );
+
+    // FIXME: cfg options?
+    let default_cfg_options = {
+        let mut opts = get_rustc_cfg_options();
+        opts.insert_atom("test".into());
+        opts.insert_atom("debug_assertion".into());
+        opts
+    };
+
+    let (crate_graph, _crate_names) =
+        ws.to_crate_graph(&default_cfg_options, &mut |path: &Path| {
+            let vfs_file = vfs.load(path);
+            log::debug!("vfs file {:?} -> {:?}", path, vfs_file);
+            vfs_file.map(vfs_file_to_id)
+        });
+    log::debug!("crate graph: {:?}", crate_graph);
+
+    let source_roots = roots
+        .iter()
+        .map(|&vfs_root| {
+            let source_root_id = vfs_root_to_id(vfs_root);
+            let project_root = project_roots
+                .iter()
+                .find(|it| it.path() == &vfs.root2path(vfs_root))
+                .unwrap()
+                .clone();
+            (source_root_id, project_root)
+        })
+        .collect::<FxHashMap<_, _>>();
+    let host = load(&source_roots, crate_graph, &mut vfs, receiver);
+    Ok((host, source_roots))
+}
+
+pub fn load(
+    source_roots: &FxHashMap<SourceRootId, PackageRoot>,
+    crate_graph: CrateGraph,
+    vfs: &mut Vfs,
+    receiver: Receiver<VfsTask>,
+) -> AnalysisHost {
+    let lru_cap = std::env::var("RA_LRU_CAP").ok().and_then(|it| it.parse::<usize>().ok());
+    let mut host = AnalysisHost::new(lru_cap, FeatureFlags::default());
+    let mut analysis_change = AnalysisChange::new();
+    analysis_change.set_crate_graph(crate_graph);
+
+    // wait until Vfs has loaded all roots
+    let mut roots_loaded = HashSet::new();
+    for task in receiver {
+        vfs.handle_task(task);
+        let mut done = false;
+        for change in vfs.commit_changes() {
+            match change {
+                VfsChange::AddRoot { root, files } => {
+                    let source_root_id = vfs_root_to_id(root);
+                    let is_local = source_roots[&source_root_id].is_member();
+                    log::debug!(
+                        "loaded source root {:?} with path {:?}",
+                        source_root_id,
+                        vfs.root2path(root)
+                    );
+                    analysis_change.add_root(source_root_id, is_local);
+                    analysis_change.set_debug_root_path(
+                        source_root_id,
+                        source_roots[&source_root_id].path().display().to_string(),
+                    );
+
+                    let mut file_map = FxHashMap::default();
+                    for (vfs_file, path, text) in files {
+                        let file_id = vfs_file_to_id(vfs_file);
+                        analysis_change.add_file(source_root_id, file_id, path.clone(), text);
+                        file_map.insert(path, file_id);
+                    }
+                    roots_loaded.insert(source_root_id);
+                    if roots_loaded.len() == vfs.n_roots() {
+                        done = true;
+                    }
+                }
+                VfsChange::AddFile { root, file, path, text } => {
+                    let source_root_id = vfs_root_to_id(root);
+                    let file_id = vfs_file_to_id(file);
+                    analysis_change.add_file(source_root_id, file_id, path, text);
+                }
+                VfsChange::RemoveFile { .. } | VfsChange::ChangeFile { .. } => {
+                    // We just need the first scan, so just ignore these
+                }
+            }
+        }
+        if done {
+            break;
+        }
+    }
+
+    host.apply_change(analysis_change);
+    host
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use hir::Crate;
+
+    #[test]
+    fn test_loading_rust_analyzer() {
+        let path = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap().parent().unwrap();
+        let (host, _roots) = load_cargo(path).unwrap();
+        let n_crates = Crate::all(host.raw_database()).len();
+        // RA has quite a few crates, but the exact count doesn't matter
+        assert!(n_crates > 20);
+    }
+}
diff --git a/crates/ra_lsp_server/src/cli/progress_report.rs b/crates/ra_lsp_server/src/cli/progress_report.rs
new file mode 100644
index 000000000..31867a1e9
--- /dev/null
+++ b/crates/ra_lsp_server/src/cli/progress_report.rs
@@ -0,0 +1,120 @@
+//! A simple progress bar
+//!
+//! A single thread non-optimized progress bar
+use std::io::Write;
+
+/// A Simple ASCII Progress Bar
+pub struct ProgressReport {
+    curr: f32,
+    text: String,
+    hidden: bool,
+
+    len: u64,
+    pos: u64,
+    msg: String,
+}
+
+impl ProgressReport {
+    pub fn new(len: u64) -> ProgressReport {
+        ProgressReport {
+            curr: 0.0,
+            text: String::new(),
+            hidden: false,
+            len,
+            pos: 0,
+            msg: String::new(),
+        }
+    }
+
+    pub fn hidden() -> ProgressReport {
+        ProgressReport {
+            curr: 0.0,
+            text: String::new(),
+            hidden: true,
+            len: 0,
+            pos: 0,
+            msg: String::new(),
+        }
+    }
+
+    pub fn set_message(&mut self, msg: &str) {
+        self.msg = msg.to_string();
+        self.tick();
+    }
+
+    pub fn println<I: Into<String>>(&mut self, msg: I) {
+        self.clear();
+        println!("{}", msg.into());
+        self.tick();
+    }
+
+    pub fn inc(&mut self, delta: u64) {
+        self.pos += delta;
+        if self.len == 0 {
+            self.set_value(0.0)
+        } else {
+            self.set_value((self.pos as f32) / (self.len as f32))
+        }
+        self.tick();
+    }
+
+    pub fn finish_and_clear(&mut self) {
+        self.clear();
+    }
+
+    pub fn tick(&mut self) {
+        if self.hidden {
+            return;
+        }
+        let percent = (self.curr * 100.0) as u32;
+        let text = format!("{}/{} {:3>}% {}", self.pos, self.len, percent, self.msg);
+        self.update_text(&text);
+    }
+
+    fn update_text(&mut self, text: &str) {
+        // Get length of common portion
+        let mut common_prefix_length = 0;
+        let common_length = usize::min(self.text.len(), text.len());
+
+        while common_prefix_length < common_length
+            && text.chars().nth(common_prefix_length).unwrap()
+                == self.text.chars().nth(common_prefix_length).unwrap()
+        {
+            common_prefix_length += 1;
+        }
+
+        // Backtrack to the first differing character
+        let mut output = String::new();
+        output += &'\x08'.to_string().repeat(self.text.len() - common_prefix_length);
+        // Output new suffix
+        output += &text[common_prefix_length..text.len()];
+
+        // If the new text is shorter than the old one: delete overlapping characters
+        if let Some(overlap_count) = self.text.len().checked_sub(text.len()) {
+            if overlap_count > 0 {
+                output += &" ".repeat(overlap_count);
+                output += &"\x08".repeat(overlap_count);
+            }
+        }
+
+        let _ = std::io::stdout().write(output.as_bytes());
+        let _ = std::io::stdout().flush();
+        self.text = text.to_string();
+    }
+
+    fn set_value(&mut self, value: f32) {
+        self.curr = f32::max(0.0, f32::min(1.0, value));
+    }
+
+    fn clear(&mut self) {
+        if self.hidden {
+            return;
+        }
+
+        // Fill all last text to space and return the cursor
+        let spaces = " ".repeat(self.text.len());
+        let backspaces = "\x08".repeat(self.text.len());
+        print!("{}{}{}", backspaces, spaces, backspaces);
+        self.text = String::new();
+    }
+}
diff --git a/crates/ra_lsp_server/src/lib.rs b/crates/ra_lsp_server/src/lib.rs
index a3464a5a3..958c70fe5 100644
--- a/crates/ra_lsp_server/src/lib.rs
+++ b/crates/ra_lsp_server/src/lib.rs
@@ -7,6 +7,8 @@
 //! state, and `main_loop` module defines the rules for modifying it.
 #![recursion_limit = "512"]
 
+pub mod cli;
+
 #[allow(unused)]
 macro_rules! println {
     ($($tt:tt)*) => {
@@ -21,6 +23,7 @@ macro_rules! print {
     };
 }
 
+mod vfs_glob;
 mod caps;
 mod cargo_target_spec;
 mod conv;
diff --git a/crates/ra_lsp_server/src/main.rs b/crates/ra_lsp_server/src/main.rs
index ed2eaabd4..a549e5ff1 100644
--- a/crates/ra_lsp_server/src/main.rs
+++ b/crates/ra_lsp_server/src/main.rs
@@ -1,14 +1,39 @@
 //! `ra_lsp_server` binary
+mod args;
 
 use lsp_server::Connection;
-use ra_lsp_server::{from_json, show_message, Result, ServerConfig};
+use ra_lsp_server::{cli, from_json, show_message, Result, ServerConfig};
 use ra_prof;
 
+use crate::args::HelpPrinted;
+
 fn main() -> Result<()> {
     setup_logging()?;
-    match Args::parse()? {
-        Args::Version => println!("rust-analyzer {}", env!("REV")),
-        Args::Run => run_server()?,
+    let args = match args::Args::parse()? {
+        Ok(it) => it,
+        Err(HelpPrinted) => return Ok(()),
+    };
+    match args.command {
+        args::Command::Parse { no_dump } => cli::parse(no_dump)?,
+        args::Command::Symbols => cli::symbols()?,
+        args::Command::Highlight { rainbow } => cli::highlight(rainbow)?,
+        args::Command::Stats { randomize, memory_usage, only, with_deps, path } => {
+            cli::analysis_stats(
+                args.verbosity,
+                memory_usage,
+                path.as_ref(),
+                only.as_ref().map(String::as_ref),
+                with_deps,
+                randomize,
+            )?
+        }
+
+        args::Command::Bench { path, what } => {
+            cli::analysis_bench(args.verbosity, path.as_ref(), what)?
+        }
+
+        args::Command::RunServer => run_server()?,
+        args::Command::Version => println!("rust-analyzer {}", env!("REV")),
     }
     Ok(())
 }
@@ -20,19 +45,6 @@ fn setup_logging() -> Result<()> {
     Ok(())
 }
 
-enum Args {
-    Version,
-    Run,
-}
-
-impl Args {
-    fn parse() -> Result<Args> {
-        let res =
-            if std::env::args().any(|it| it == "--version") { Args::Version } else { Args::Run };
-        Ok(res)
-    }
-}
-
 fn run_server() -> Result<()> {
     log::info!("lifecycle: server started");
 
diff --git a/crates/ra_lsp_server/src/main_loop.rs b/crates/ra_lsp_server/src/main_loop.rs
index 7ae2e1e6f..944074118 100644
--- a/crates/ra_lsp_server/src/main_loop.rs
+++ b/crates/ra_lsp_server/src/main_loop.rs
@@ -135,7 +135,7 @@ pub fn main_loop(
         let globs = config
             .exclude_globs
             .iter()
-            .map(|glob| ra_vfs_glob::Glob::new(glob))
+            .map(|glob| crate::vfs_glob::Glob::new(glob))
             .collect::<std::result::Result<Vec<_>, _>>()?;
 
         if config.use_client_watching {
diff --git a/crates/ra_lsp_server/src/vfs_glob.rs b/crates/ra_lsp_server/src/vfs_glob.rs
new file mode 100644
index 000000000..12401d75a
--- /dev/null
+++ b/crates/ra_lsp_server/src/vfs_glob.rs
@@ -0,0 +1,94 @@
+//! `ra_vfs_glob` crate implements exclusion rules for vfs.
+//!
+//! By default, we include only `.rs` files, and skip some know offenders like
+//! `/target` or `/node_modules` altogether.
+//!
+//! It's also possible to add custom exclusion globs.
+
+use globset::{GlobSet, GlobSetBuilder};
+use ra_vfs::{Filter, RelativePath};
+
+pub use globset::{Glob, GlobBuilder};
+
+const ALWAYS_IGNORED: &[&str] = &["target/**", "**/node_modules/**", "**/.git/**"];
+const IGNORED_FOR_NON_MEMBERS: &[&str] = &["examples/**", "tests/**", "benches/**"];
+
+pub struct RustPackageFilterBuilder {
+    is_member: bool,
+    exclude: GlobSetBuilder,
+}
+
+impl Default for RustPackageFilterBuilder {
+    fn default() -> RustPackageFilterBuilder {
+        RustPackageFilterBuilder { is_member: false, exclude: GlobSetBuilder::new() }
+    }
+}
+
+impl RustPackageFilterBuilder {
+    pub fn set_member(mut self, is_member: bool) -> RustPackageFilterBuilder {
+        self.is_member = is_member;
+        self
+    }
+    pub fn exclude(mut self, glob: Glob) -> RustPackageFilterBuilder {
+        self.exclude.add(glob);
+        self
+    }
+    pub fn into_vfs_filter(self) -> Box<dyn Filter> {
+        let RustPackageFilterBuilder { is_member, mut exclude } = self;
+        for &glob in ALWAYS_IGNORED {
+            exclude.add(Glob::new(glob).unwrap());
+        }
+        if !is_member {
+            for &glob in IGNORED_FOR_NON_MEMBERS {
+                exclude.add(Glob::new(glob).unwrap());
+            }
+        }
+        Box::new(RustPackageFilter { exclude: exclude.build().unwrap() })
+    }
+}
+
+struct RustPackageFilter {
+    exclude: GlobSet,
+}
+
+impl Filter for RustPackageFilter {
+    fn include_dir(&self, dir_path: &RelativePath) -> bool {
+        !self.exclude.is_match(dir_path.as_str())
+    }
+
+    fn include_file(&self, file_path: &RelativePath) -> bool {
+        file_path.extension() == Some("rs")
+    }
+}
+
+#[test]
+fn test_globs() {
+    let filter = RustPackageFilterBuilder::default().set_member(true).into_vfs_filter();
+
+    assert!(filter.include_dir(RelativePath::new("src/tests")));
+    assert!(filter.include_dir(RelativePath::new("src/target")));
+    assert!(filter.include_dir(RelativePath::new("tests")));
+    assert!(filter.include_dir(RelativePath::new("benches")));
+
+    assert!(!filter.include_dir(RelativePath::new("target")));
+    assert!(!filter.include_dir(RelativePath::new("src/foo/.git")));
+    assert!(!filter.include_dir(RelativePath::new("foo/node_modules")));
+
+    let filter = RustPackageFilterBuilder::default().set_member(false).into_vfs_filter();
+
+    assert!(filter.include_dir(RelativePath::new("src/tests")));
+    assert!(filter.include_dir(RelativePath::new("src/target")));
+
+    assert!(!filter.include_dir(RelativePath::new("target")));
+    assert!(!filter.include_dir(RelativePath::new("src/foo/.git")));
+    assert!(!filter.include_dir(RelativePath::new("foo/node_modules")));
+    assert!(!filter.include_dir(RelativePath::new("tests")));
+    assert!(!filter.include_dir(RelativePath::new("benches")));
+
+    let filter = RustPackageFilterBuilder::default()
+        .set_member(true)
+        .exclude(Glob::new("src/llvm-project/**").unwrap())
+        .into_vfs_filter();
+
+    assert!(!filter.include_dir(RelativePath::new("src/llvm-project/clang")));
+}
diff --git a/crates/ra_lsp_server/src/world.rs b/crates/ra_lsp_server/src/world.rs
index d993c5fc4..71c95d4af 100644
--- a/crates/ra_lsp_server/src/world.rs
+++ b/crates/ra_lsp_server/src/world.rs
@@ -19,12 +19,12 @@ use ra_ide::{
 };
 use ra_project_model::{get_rustc_cfg_options, ProjectWorkspace};
 use ra_vfs::{LineEndings, RootEntry, Vfs, VfsChange, VfsFile, VfsRoot, VfsTask, Watch};
-use ra_vfs_glob::{Glob, RustPackageFilterBuilder};
 use relative_path::RelativePathBuf;
 
 use crate::{
     diagnostics::{CheckFixes, DiagnosticCollection},
     main_loop::pending_requests::{CompletedRequest, LatestRequests},
+    vfs_glob::{Glob, RustPackageFilterBuilder},
     LspError, Result,
 };