Merge #678

678: WIP: automatically collect garbage r=matklad a=matklad



Co-authored-by: Aleksey Kladov <[email protected]>

8 files changed, 104 insertions, 22 deletions

diff --git a/crates/ra_db/Cargo.toml b/crates/ra_db/Cargo.toml
index 2d39b77ed..3bae22c74 100644
--- a/crates/ra_db/Cargo.toml
+++ b/crates/ra_db/Cargo.toml
@@ -5,8 +5,8 @@ version = "0.1.0"
 authors = ["Aleksey Kladov <[email protected]>"]
 
 [dependencies]
+salsa = "0.10.0"
 relative-path = "0.4.0"
-salsa = "0.10.0-alpha5"
 rustc-hash = "1.0"
 parking_lot = "0.7.0"
 
diff --git a/crates/ra_ide_api/Cargo.toml b/crates/ra_ide_api/Cargo.toml
index 79e473463..ad9dd2088 100644
--- a/crates/ra_ide_api/Cargo.toml
+++ b/crates/ra_ide_api/Cargo.toml
@@ -14,6 +14,8 @@ fst = "0.3.1"
 rustc-hash = "1.0"
 parking_lot = "0.7.0"
 unicase = "2.2.0"
+jemallocator = "0.1.9"
+jemalloc-ctl = "0.2.0"
 
 ra_syntax = { path = "../ra_syntax" }
 ra_ide_api_light = { path = "../ra_ide_api_light" }
diff --git a/crates/ra_ide_api/src/db.rs b/crates/ra_ide_api/src/db.rs
index 3da93ec35..6850811d7 100644
--- a/crates/ra_ide_api/src/db.rs
+++ b/crates/ra_ide_api/src/db.rs
@@ -1,4 +1,7 @@
-use std::sync::Arc;
+use std::{
+    sync::Arc,
+    time,
+};
 
 use ra_db::{
     CheckCanceled, FileId, Canceled, SourceDatabase,
@@ -17,6 +20,8 @@ use crate::{LineIndex, symbol_index::{self, SymbolsDatabase}};
 pub(crate) struct RootDatabase {
     runtime: salsa::Runtime<RootDatabase>,
     interner: Arc<hir::HirInterner>,
+    pub(crate) last_gc: time::Instant,
+    pub(crate) last_gc_check: time::Instant,
 }
 
 impl salsa::Database for RootDatabase {
@@ -33,6 +38,8 @@ impl Default for RootDatabase {
         let mut db = RootDatabase {
             runtime: salsa::Runtime::default(),
             interner: Default::default(),
+            last_gc: time::Instant::now(),
+            last_gc_check: time::Instant::now(),
         };
         db.set_crate_graph(Default::default());
         db.set_local_roots(Default::default());
@@ -46,6 +53,8 @@ impl salsa::ParallelDatabase for RootDatabase {
         salsa::Snapshot::new(RootDatabase {
             runtime: self.runtime.snapshot(self),
             interner: Arc::clone(&self.interner),
+            last_gc: self.last_gc.clone(),
+            last_gc_check: self.last_gc_check.clone(),
         })
     }
 }
diff --git a/crates/ra_ide_api/src/imp.rs b/crates/ra_ide_api/src/imp.rs
index 399433a01..31e0f5d6d 100644
--- a/crates/ra_ide_api/src/imp.rs
+++ b/crates/ra_ide_api/src/imp.rs
@@ -1,4 +1,7 @@
-use std::sync::Arc;
+use std::{
+    sync::Arc,
+    time,
+};
 
 use hir::{
     self, Problem, source_binder
@@ -19,12 +22,14 @@ use crate::{
     CrateId, db, Diagnostic, FileId, FilePosition, FileRange, FileSystemEdit,
     Query, RootChange, SourceChange, SourceFileEdit,
     symbol_index::{FileSymbol, SymbolsDatabase},
+    status::syntax_tree_stats
 };
 
+const GC_COOLDOWN: time::Duration = time::Duration::from_millis(100);
+
 impl db::RootDatabase {
     pub(crate) fn apply_change(&mut self, change: AnalysisChange) {
         log::info!("apply_change {:?}", change);
-        // self.gc_syntax_trees();
         if !change.new_roots.is_empty() {
             let mut local_roots = Vec::clone(&self.local_roots());
             for (root_id, is_local) in change.new_roots {
@@ -72,18 +77,36 @@ impl db::RootDatabase {
         self.set_source_root(root_id, Arc::new(source_root));
     }
 
-    /// Ideally, we should call this function from time to time to collect heavy
-    /// syntax trees. However, if we actually do that, everything is recomputed
-    /// for some reason. Needs investigation.
+    pub(crate) fn maybe_collect_garbage(&mut self) {
+        if self.last_gc_check.elapsed() > GC_COOLDOWN {
+            self.last_gc_check = time::Instant::now();
+            let retained_trees = syntax_tree_stats(self).retained;
+            if retained_trees > 100 {
+                log::info!(
+                    "automatic garbadge collection, {} retained trees",
+                    retained_trees
+                );
+                self.collect_garbage();
+            }
+        }
+    }
+
     pub(crate) fn collect_garbage(&mut self) {
-        self.query(ra_db::ParseQuery)
-            .sweep(SweepStrategy::default().discard_values());
-        self.query(hir::db::HirParseQuery)
-            .sweep(SweepStrategy::default().discard_values());
-        self.query(hir::db::FileItemsQuery)
-            .sweep(SweepStrategy::default().discard_values());
-        self.query(hir::db::FileItemQuery)
-            .sweep(SweepStrategy::default().discard_values());
+        self.last_gc = time::Instant::now();
+
+        let sweep = SweepStrategy::default()
+            .discard_values()
+            .sweep_all_revisions();
+
+        self.query(ra_db::ParseQuery).sweep(sweep);
+
+        self.query(hir::db::HirParseQuery).sweep(sweep);
+        self.query(hir::db::FileItemsQuery).sweep(sweep);
+        self.query(hir::db::FileItemQuery).sweep(sweep);
+
+        self.query(hir::db::LowerModuleQuery).sweep(sweep);
+        self.query(hir::db::LowerModuleSourceMapQuery).sweep(sweep);
+        self.query(hir::db::BodySyntaxMappingQuery).sweep(sweep);
     }
 }
 
diff --git a/crates/ra_ide_api/src/lib.rs b/crates/ra_ide_api/src/lib.rs
index 43c8bea71..dc531e068 100644
--- a/crates/ra_ide_api/src/lib.rs
+++ b/crates/ra_ide_api/src/lib.rs
@@ -59,6 +59,11 @@ pub use ra_db::{
     Canceled, CrateGraph, CrateId, FileId, FilePosition, FileRange, SourceRootId
 };
 
+// We use jemalloc mainly to get heap usage statistics, actual performance
+// differnece is not measures.
+#[global_allocator]
+static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
+
 pub type Cancelable<T> = Result<T, Canceled>;
 
 #[derive(Default)]
@@ -286,6 +291,10 @@ impl AnalysisHost {
         self.db.apply_change(change)
     }
 
+    pub fn maybe_collect_garbage(&mut self) {
+        self.db.maybe_collect_garbage();
+    }
+
     pub fn collect_garbage(&mut self) {
         self.db.collect_garbage();
     }
diff --git a/crates/ra_ide_api/src/status.rs b/crates/ra_ide_api/src/status.rs
index e11eed223..c3e5745d5 100644
--- a/crates/ra_ide_api/src/status.rs
+++ b/crates/ra_ide_api/src/status.rs
@@ -15,9 +15,13 @@ use crate::{
     symbol_index::{SymbolIndex, LibrarySymbolsQuery},
 };
 
+pub(crate) fn syntax_tree_stats(db: &RootDatabase) -> SyntaxTreeStats {
+    db.query(ParseQuery).entries::<SyntaxTreeStats>()
+}
+
 pub(crate) fn status(db: &RootDatabase) -> String {
     let files_stats = db.query(FileTextQuery).entries::<FilesStats>();
-    let syntax_tree_stats = db.query(ParseQuery).entries::<SyntaxTreeStats>();
+    let syntax_tree_stats = syntax_tree_stats(db);
     let symbols_stats = db
         .query(LibrarySymbolsQuery)
         .entries::<LibrarySymbolsStats>();
@@ -26,8 +30,13 @@ pub(crate) fn status(db: &RootDatabase) -> String {
         interner.len()
     };
     format!(
-        "{}\n{}\n{}\nn_defs {}\n",
-        files_stats, symbols_stats, syntax_tree_stats, n_defs
+        "{}\n{}\n{}\n{} defs\n\nmemory:\n{}\ngc {:?} seconds ago",
+        files_stats,
+        symbols_stats,
+        syntax_tree_stats,
+        n_defs,
+        MemoryStats::current(),
+        db.last_gc.elapsed().as_secs(),
     )
 }
 
@@ -58,9 +67,9 @@ impl FromIterator<TableEntry<FileId, Arc<String>>> for FilesStats {
 }
 
 #[derive(Default)]
-struct SyntaxTreeStats {
+pub(crate) struct SyntaxTreeStats {
     total: usize,
-    retained: usize,
+    pub(crate) retained: usize,
     retained_size: Bytes,
 }
 
@@ -118,6 +127,31 @@ impl FromIterator<TableEntry<SourceRootId, Arc<SymbolIndex>>> for LibrarySymbols
     }
 }
 
+struct MemoryStats {
+    allocated: Bytes,
+    resident: Bytes,
+}
+
+impl MemoryStats {
+    fn current() -> MemoryStats {
+        jemalloc_ctl::epoch().unwrap();
+        MemoryStats {
+            allocated: Bytes(jemalloc_ctl::stats::allocated().unwrap()),
+            resident: Bytes(jemalloc_ctl::stats::resident().unwrap()),
+        }
+    }
+}
+
+impl fmt::Display for MemoryStats {
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            fmt,
+            "{} allocated {} resident",
+            self.allocated, self.resident,
+        )
+    }
+}
+
 #[derive(Default)]
 struct Bytes(usize);
 
diff --git a/crates/ra_lsp_server/src/main_loop.rs b/crates/ra_lsp_server/src/main_loop.rs
index ddd20a41f..e430ac6de 100644
--- a/crates/ra_lsp_server/src/main_loop.rs
+++ b/crates/ra_lsp_server/src/main_loop.rs
@@ -172,6 +172,7 @@ fn main_loop_inner(
 
     let (libdata_sender, libdata_receiver) = unbounded();
     loop {
+        state.maybe_collect_garbage();
         log::trace!("selecting");
         let event = select! {
             recv(msg_receiver) -> msg => match msg {
@@ -207,7 +208,7 @@ fn main_loop_inner(
                     };
                     match req.cast::<req::CollectGarbage>() {
                         Ok((id, ())) => {
-                            state.collect_garbadge();
+                            state.collect_garbage();
                             let resp = RawResponse::ok::<req::CollectGarbage>(id, &());
                             msg_sender.send(RawMessage::Response(resp)).unwrap()
                         }
diff --git a/crates/ra_lsp_server/src/server_world.rs b/crates/ra_lsp_server/src/server_world.rs
index bf04f1125..c2167c5d8 100644
--- a/crates/ra_lsp_server/src/server_world.rs
+++ b/crates/ra_lsp_server/src/server_world.rs
@@ -232,7 +232,11 @@ impl ServerWorldState {
         }
     }
 
-    pub fn collect_garbadge(&mut self) {
+    pub fn maybe_collect_garbage(&mut self) {
+        self.analysis_host.maybe_collect_garbage()
+    }
+
+    pub fn collect_garbage(&mut self) {
         self.analysis_host.collect_garbage()
     }
 }