From 7801f6b50f69b33743145f61989bcb945c24dbaa Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 26 Jan 2019 20:33:33 +0300 Subject: automatically collect garbage --- crates/ra_db/Cargo.toml | 2 +- crates/ra_ide_api/src/db.rs | 11 +++++++- crates/ra_ide_api/src/imp.rs | 48 +++++++++++++++++++++++++------- crates/ra_ide_api/src/lib.rs | 4 +++ crates/ra_ide_api/src/status.rs | 18 ++++++++---- crates/ra_lsp_server/src/main_loop.rs | 3 +- crates/ra_lsp_server/src/server_world.rs | 6 +++- 7 files changed, 73 insertions(+), 19 deletions(-) (limited to 'crates') diff --git a/crates/ra_db/Cargo.toml b/crates/ra_db/Cargo.toml index 2d39b77ed..e479b38a0 100644 --- a/crates/ra_db/Cargo.toml +++ b/crates/ra_db/Cargo.toml @@ -5,8 +5,8 @@ version = "0.1.0" authors = ["Aleksey Kladov "] [dependencies] +salsa = { path = "/home/matklad/projects/salsa" } relative-path = "0.4.0" -salsa = "0.10.0-alpha5" rustc-hash = "1.0" parking_lot = "0.7.0" diff --git a/crates/ra_ide_api/src/db.rs b/crates/ra_ide_api/src/db.rs index 3da93ec35..6850811d7 100644 --- a/crates/ra_ide_api/src/db.rs +++ b/crates/ra_ide_api/src/db.rs @@ -1,4 +1,7 @@ -use std::sync::Arc; +use std::{ + sync::Arc, + time, +}; use ra_db::{ CheckCanceled, FileId, Canceled, SourceDatabase, @@ -17,6 +20,8 @@ use crate::{LineIndex, symbol_index::{self, SymbolsDatabase}}; pub(crate) struct RootDatabase { runtime: salsa::Runtime, interner: Arc, + pub(crate) last_gc: time::Instant, + pub(crate) last_gc_check: time::Instant, } impl salsa::Database for RootDatabase { @@ -33,6 +38,8 @@ impl Default for RootDatabase { let mut db = RootDatabase { runtime: salsa::Runtime::default(), interner: Default::default(), + last_gc: time::Instant::now(), + last_gc_check: time::Instant::now(), }; db.set_crate_graph(Default::default()); db.set_local_roots(Default::default()); @@ -46,6 +53,8 @@ impl salsa::ParallelDatabase for RootDatabase { salsa::Snapshot::new(RootDatabase { runtime: self.runtime.snapshot(self), interner: Arc::clone(&self.interner), + last_gc: self.last_gc.clone(), + last_gc_check: self.last_gc_check.clone(), }) } } diff --git a/crates/ra_ide_api/src/imp.rs b/crates/ra_ide_api/src/imp.rs index 399433a01..0dd3e8cb0 100644 --- a/crates/ra_ide_api/src/imp.rs +++ b/crates/ra_ide_api/src/imp.rs @@ -1,4 +1,7 @@ -use std::sync::Arc; +use std::{ + sync::Arc, + time, +}; use hir::{ self, Problem, source_binder @@ -19,12 +22,14 @@ use crate::{ CrateId, db, Diagnostic, FileId, FilePosition, FileRange, FileSystemEdit, Query, RootChange, SourceChange, SourceFileEdit, symbol_index::{FileSymbol, SymbolsDatabase}, + status::syntax_tree_stats }; +const GC_COOLDOWN: time::Duration = time::Duration::from_millis(100); + impl db::RootDatabase { pub(crate) fn apply_change(&mut self, change: AnalysisChange) { log::info!("apply_change {:?}", change); - // self.gc_syntax_trees(); if !change.new_roots.is_empty() { let mut local_roots = Vec::clone(&self.local_roots()); for (root_id, is_local) in change.new_roots { @@ -72,18 +77,41 @@ impl db::RootDatabase { self.set_source_root(root_id, Arc::new(source_root)); } + pub(crate) fn maybe_collect_garbage(&mut self) { + if self.last_gc_check.elapsed() > GC_COOLDOWN { + self.last_gc_check = time::Instant::now(); + let retained_trees = syntax_tree_stats(self).retained; + if retained_trees > 100 { + log::info!( + "automatic garbadge collection, {} retained trees", + retained_trees + ); + self.collect_garbage(); + } + } + } + /// Ideally, we should call this function from time to time to collect heavy /// syntax trees. However, if we actually do that, everything is recomputed /// for some reason. Needs investigation. pub(crate) fn collect_garbage(&mut self) { - self.query(ra_db::ParseQuery) - .sweep(SweepStrategy::default().discard_values()); - self.query(hir::db::HirParseQuery) - .sweep(SweepStrategy::default().discard_values()); - self.query(hir::db::FileItemsQuery) - .sweep(SweepStrategy::default().discard_values()); - self.query(hir::db::FileItemQuery) - .sweep(SweepStrategy::default().discard_values()); + self.last_gc = time::Instant::now(); + + let sweep = SweepStrategy::default() + .discard_values() + .discard_all_revisions(); + + self.query(ra_db::ParseQuery).sweep(sweep.clone()); + + self.query(hir::db::HirParseQuery).sweep(sweep.clone()); + self.query(hir::db::FileItemsQuery).sweep(sweep.clone()); + self.query(hir::db::FileItemQuery).sweep(sweep.clone()); + + self.query(hir::db::LowerModuleQuery).sweep(sweep.clone()); + self.query(hir::db::LowerModuleSourceMapQuery) + .sweep(sweep.clone()); + self.query(hir::db::BodySyntaxMappingQuery) + .sweep(sweep.clone()); } } diff --git a/crates/ra_ide_api/src/lib.rs b/crates/ra_ide_api/src/lib.rs index 43c8bea71..a7caac02d 100644 --- a/crates/ra_ide_api/src/lib.rs +++ b/crates/ra_ide_api/src/lib.rs @@ -286,6 +286,10 @@ impl AnalysisHost { self.db.apply_change(change) } + pub fn maybe_collect_garbage(&mut self) { + self.db.maybe_collect_garbage(); + } + pub fn collect_garbage(&mut self) { self.db.collect_garbage(); } diff --git a/crates/ra_ide_api/src/status.rs b/crates/ra_ide_api/src/status.rs index e11eed223..0dde30ae0 100644 --- a/crates/ra_ide_api/src/status.rs +++ b/crates/ra_ide_api/src/status.rs @@ -15,9 +15,13 @@ use crate::{ symbol_index::{SymbolIndex, LibrarySymbolsQuery}, }; +pub(crate) fn syntax_tree_stats(db: &RootDatabase) -> SyntaxTreeStats { + db.query(ParseQuery).entries::() +} + pub(crate) fn status(db: &RootDatabase) -> String { let files_stats = db.query(FileTextQuery).entries::(); - let syntax_tree_stats = db.query(ParseQuery).entries::(); + let syntax_tree_stats = syntax_tree_stats(db); let symbols_stats = db .query(LibrarySymbolsQuery) .entries::(); @@ -26,8 +30,12 @@ pub(crate) fn status(db: &RootDatabase) -> String { interner.len() }; format!( - "{}\n{}\n{}\nn_defs {}\n", - files_stats, symbols_stats, syntax_tree_stats, n_defs + "{}\n{}\n{}\nn_defs {}\nGC {:?} seconds ago", + files_stats, + symbols_stats, + syntax_tree_stats, + n_defs, + db.last_gc.elapsed().as_secs(), ) } @@ -58,9 +66,9 @@ impl FromIterator>> for FilesStats { } #[derive(Default)] -struct SyntaxTreeStats { +pub(crate) struct SyntaxTreeStats { total: usize, - retained: usize, + pub(crate) retained: usize, retained_size: Bytes, } diff --git a/crates/ra_lsp_server/src/main_loop.rs b/crates/ra_lsp_server/src/main_loop.rs index ddd20a41f..e430ac6de 100644 --- a/crates/ra_lsp_server/src/main_loop.rs +++ b/crates/ra_lsp_server/src/main_loop.rs @@ -172,6 +172,7 @@ fn main_loop_inner( let (libdata_sender, libdata_receiver) = unbounded(); loop { + state.maybe_collect_garbage(); log::trace!("selecting"); let event = select! { recv(msg_receiver) -> msg => match msg { @@ -207,7 +208,7 @@ fn main_loop_inner( }; match req.cast::() { Ok((id, ())) => { - state.collect_garbadge(); + state.collect_garbage(); let resp = RawResponse::ok::(id, &()); msg_sender.send(RawMessage::Response(resp)).unwrap() } diff --git a/crates/ra_lsp_server/src/server_world.rs b/crates/ra_lsp_server/src/server_world.rs index bf04f1125..c2167c5d8 100644 --- a/crates/ra_lsp_server/src/server_world.rs +++ b/crates/ra_lsp_server/src/server_world.rs @@ -232,7 +232,11 @@ impl ServerWorldState { } } - pub fn collect_garbadge(&mut self) { + pub fn maybe_collect_garbage(&mut self) { + self.analysis_host.maybe_collect_garbage() + } + + pub fn collect_garbage(&mut self) { self.analysis_host.collect_garbage() } } -- cgit v1.2.3 From 09b5dc8e02317c14cff80890f2d4591843322f47 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 26 Jan 2019 20:57:47 +0300 Subject: drop obsolete comment --- crates/ra_ide_api/src/imp.rs | 3 --- 1 file changed, 3 deletions(-) (limited to 'crates') diff --git a/crates/ra_ide_api/src/imp.rs b/crates/ra_ide_api/src/imp.rs index 0dd3e8cb0..4e347e973 100644 --- a/crates/ra_ide_api/src/imp.rs +++ b/crates/ra_ide_api/src/imp.rs @@ -91,9 +91,6 @@ impl db::RootDatabase { } } - /// Ideally, we should call this function from time to time to collect heavy - /// syntax trees. However, if we actually do that, everything is recomputed - /// for some reason. Needs investigation. pub(crate) fn collect_garbage(&mut self) { self.last_gc = time::Instant::now(); -- cgit v1.2.3 From c7f4e3a401ec1919e1a578abe5938df430f46fc9 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 26 Jan 2019 21:12:16 +0300 Subject: show jemalloc --- crates/ra_ide_api/Cargo.toml | 2 ++ crates/ra_ide_api/src/lib.rs | 5 +++++ crates/ra_ide_api/src/status.rs | 28 +++++++++++++++++++++++++++- 3 files changed, 34 insertions(+), 1 deletion(-) (limited to 'crates') diff --git a/crates/ra_ide_api/Cargo.toml b/crates/ra_ide_api/Cargo.toml index 79e473463..ad9dd2088 100644 --- a/crates/ra_ide_api/Cargo.toml +++ b/crates/ra_ide_api/Cargo.toml @@ -14,6 +14,8 @@ fst = "0.3.1" rustc-hash = "1.0" parking_lot = "0.7.0" unicase = "2.2.0" +jemallocator = "0.1.9" +jemalloc-ctl = "0.2.0" ra_syntax = { path = "../ra_syntax" } ra_ide_api_light = { path = "../ra_ide_api_light" } diff --git a/crates/ra_ide_api/src/lib.rs b/crates/ra_ide_api/src/lib.rs index a7caac02d..dc531e068 100644 --- a/crates/ra_ide_api/src/lib.rs +++ b/crates/ra_ide_api/src/lib.rs @@ -59,6 +59,11 @@ pub use ra_db::{ Canceled, CrateGraph, CrateId, FileId, FilePosition, FileRange, SourceRootId }; +// We use jemalloc mainly to get heap usage statistics, actual performance +// differnece is not measures. +#[global_allocator] +static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; + pub type Cancelable = Result; #[derive(Default)] diff --git a/crates/ra_ide_api/src/status.rs b/crates/ra_ide_api/src/status.rs index 0dde30ae0..686ab97d2 100644 --- a/crates/ra_ide_api/src/status.rs +++ b/crates/ra_ide_api/src/status.rs @@ -30,11 +30,12 @@ pub(crate) fn status(db: &RootDatabase) -> String { interner.len() }; format!( - "{}\n{}\n{}\nn_defs {}\nGC {:?} seconds ago", + "{}\n{}\n{}\nn_defs {}\n\njemalloc: {}\nGC {:?} seconds ago", files_stats, symbols_stats, syntax_tree_stats, n_defs, + MemoryStats::current(), db.last_gc.elapsed().as_secs(), ) } @@ -126,6 +127,31 @@ impl FromIterator>> for LibrarySymbols } } +struct MemoryStats { + allocated: Bytes, + resident: Bytes, +} + +impl MemoryStats { + fn current() -> MemoryStats { + jemalloc_ctl::epoch().unwrap(); + MemoryStats { + allocated: Bytes(jemalloc_ctl::stats::allocated().unwrap()), + resident: Bytes(jemalloc_ctl::stats::resident().unwrap()), + } + } +} + +impl fmt::Display for MemoryStats { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + write!( + fmt, + "{} allocated {} resident", + self.allocated, self.resident, + ) + } +} + #[derive(Default)] struct Bytes(usize); -- cgit v1.2.3 From b4c8baadb1d9490868ad5bcb2bd576634cdf6931 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 26 Jan 2019 21:28:32 +0300 Subject: tweak words --- crates/ra_ide_api/src/status.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'crates') diff --git a/crates/ra_ide_api/src/status.rs b/crates/ra_ide_api/src/status.rs index 686ab97d2..c3e5745d5 100644 --- a/crates/ra_ide_api/src/status.rs +++ b/crates/ra_ide_api/src/status.rs @@ -30,7 +30,7 @@ pub(crate) fn status(db: &RootDatabase) -> String { interner.len() }; format!( - "{}\n{}\n{}\nn_defs {}\n\njemalloc: {}\nGC {:?} seconds ago", + "{}\n{}\n{}\n{} defs\n\nmemory:\n{}\ngc {:?} seconds ago", files_stats, symbols_stats, syntax_tree_stats, -- cgit v1.2.3 From c4ceea3417f0d24ea732549eb481fd788248cd00 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 26 Jan 2019 21:51:06 +0300 Subject: use salsa from my repo --- crates/ra_db/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'crates') diff --git a/crates/ra_db/Cargo.toml b/crates/ra_db/Cargo.toml index e479b38a0..3bacd8660 100644 --- a/crates/ra_db/Cargo.toml +++ b/crates/ra_db/Cargo.toml @@ -5,7 +5,7 @@ version = "0.1.0" authors = ["Aleksey Kladov "] [dependencies] -salsa = { path = "/home/matklad/projects/salsa" } +salsa = "0.10.0-alpha5" relative-path = "0.4.0" rustc-hash = "1.0" parking_lot = "0.7.0" -- cgit v1.2.3 From 53e3e82d919e36cc2225d4f7f867a45f0cede53a Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sun, 27 Jan 2019 22:11:17 +0300 Subject: update salsa --- crates/ra_db/Cargo.toml | 2 +- crates/ra_ide_api/src/imp.rs | 18 ++++++++---------- 2 files changed, 9 insertions(+), 11 deletions(-) (limited to 'crates') diff --git a/crates/ra_db/Cargo.toml b/crates/ra_db/Cargo.toml index 3bacd8660..3bae22c74 100644 --- a/crates/ra_db/Cargo.toml +++ b/crates/ra_db/Cargo.toml @@ -5,7 +5,7 @@ version = "0.1.0" authors = ["Aleksey Kladov "] [dependencies] -salsa = "0.10.0-alpha5" +salsa = "0.10.0" relative-path = "0.4.0" rustc-hash = "1.0" parking_lot = "0.7.0" diff --git a/crates/ra_ide_api/src/imp.rs b/crates/ra_ide_api/src/imp.rs index 4e347e973..31e0f5d6d 100644 --- a/crates/ra_ide_api/src/imp.rs +++ b/crates/ra_ide_api/src/imp.rs @@ -96,19 +96,17 @@ impl db::RootDatabase { let sweep = SweepStrategy::default() .discard_values() - .discard_all_revisions(); + .sweep_all_revisions(); - self.query(ra_db::ParseQuery).sweep(sweep.clone()); + self.query(ra_db::ParseQuery).sweep(sweep); - self.query(hir::db::HirParseQuery).sweep(sweep.clone()); - self.query(hir::db::FileItemsQuery).sweep(sweep.clone()); - self.query(hir::db::FileItemQuery).sweep(sweep.clone()); + self.query(hir::db::HirParseQuery).sweep(sweep); + self.query(hir::db::FileItemsQuery).sweep(sweep); + self.query(hir::db::FileItemQuery).sweep(sweep); - self.query(hir::db::LowerModuleQuery).sweep(sweep.clone()); - self.query(hir::db::LowerModuleSourceMapQuery) - .sweep(sweep.clone()); - self.query(hir::db::BodySyntaxMappingQuery) - .sweep(sweep.clone()); + self.query(hir::db::LowerModuleQuery).sweep(sweep); + self.query(hir::db::LowerModuleSourceMapQuery).sweep(sweep); + self.query(hir::db::BodySyntaxMappingQuery).sweep(sweep); } } -- cgit v1.2.3