aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAleksey Kladov <[email protected]>2020-06-15 12:29:07 +0100
committerAleksey Kladov <[email protected]>2020-06-16 12:42:29 +0100
commitc002322bde06a73c8cfa02cd1cbe33cf225da47b (patch)
tree4eac72ea31ec4420f184d42592e616cee82563dd
parentdb6100dbaa4f09543c7e6f2ab9986daa55919ad5 (diff)
New VFS API
-rw-r--r--Cargo.lock12
-rw-r--r--crates/vfs/Cargo.toml14
-rw-r--r--crates/vfs/src/file_set.rs99
-rw-r--r--crates/vfs/src/lib.rs138
-rw-r--r--crates/vfs/src/loader.rs69
-rw-r--r--crates/vfs/src/path_interner.rs31
-rw-r--r--crates/vfs/src/vfs_path.rs49
-rw-r--r--crates/vfs/src/walkdir_loader.rs108
8 files changed, 520 insertions, 0 deletions
diff --git a/Cargo.lock b/Cargo.lock
index 5848e61c7..6d83c9276 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1730,6 +1730,18 @@ dependencies = [
1730] 1730]
1731 1731
1732[[package]] 1732[[package]]
1733name = "vfs"
1734version = "0.1.0"
1735dependencies = [
1736 "crossbeam-channel",
1737 "globset",
1738 "jod-thread",
1739 "paths",
1740 "rustc-hash",
1741 "walkdir",
1742]
1743
1744[[package]]
1733name = "walkdir" 1745name = "walkdir"
1734version = "2.3.1" 1746version = "2.3.1"
1735source = "registry+https://github.com/rust-lang/crates.io-index" 1747source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/crates/vfs/Cargo.toml b/crates/vfs/Cargo.toml
new file mode 100644
index 000000000..c03e6363b
--- /dev/null
+++ b/crates/vfs/Cargo.toml
@@ -0,0 +1,14 @@
1[package]
2name = "vfs"
3version = "0.1.0"
4authors = ["rust-analyzer developers"]
5edition = "2018"
6
7[dependencies]
8rustc-hash = "1.0"
9jod-thread = "0.1.0"
10walkdir = "2.3.1"
11globset = "0.4.5"
12crossbeam-channel = "0.4.0"
13
14paths = { path = "../paths" }
diff --git a/crates/vfs/src/file_set.rs b/crates/vfs/src/file_set.rs
new file mode 100644
index 000000000..7dc721f7e
--- /dev/null
+++ b/crates/vfs/src/file_set.rs
@@ -0,0 +1,99 @@
1//! Partitions a list of files into disjoint subsets.
2//!
3//! Files which do not belong to any explicitly configured `FileSet` belong to
4//! the default `FileSet`.
5use std::{cmp, fmt, iter};
6
7use paths::AbsPathBuf;
8use rustc_hash::FxHashMap;
9
10use crate::{FileId, Vfs, VfsPath};
11
12#[derive(Default, Clone, Eq, PartialEq)]
13pub struct FileSet {
14 files: FxHashMap<VfsPath, FileId>,
15 paths: FxHashMap<FileId, VfsPath>,
16}
17
18impl FileSet {
19 pub fn resolve_path(&self, anchor: FileId, path: &str) -> Option<FileId> {
20 let mut base = self.paths[&anchor].clone();
21 base.pop();
22 let path = base.join(path);
23 let res = self.files.get(&path).copied();
24 res
25 }
26 pub fn insert(&mut self, file_id: FileId, path: VfsPath) {
27 self.files.insert(path.clone(), file_id);
28 self.paths.insert(file_id, path);
29 }
30 pub fn iter(&self) -> impl Iterator<Item = FileId> + '_ {
31 self.paths.keys().copied()
32 }
33}
34
35impl fmt::Debug for FileSet {
36 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
37 f.debug_struct("FileSet").field("n_files", &self.files.len()).finish()
38 }
39}
40
41#[derive(Debug)]
42pub struct FileSetConfig {
43 n_file_sets: usize,
44 roots: Vec<(AbsPathBuf, usize)>,
45}
46
47impl FileSetConfig {
48 pub fn builder() -> FileSetConfigBuilder {
49 FileSetConfigBuilder::default()
50 }
51 pub fn partition(&self, vfs: &Vfs) -> Vec<FileSet> {
52 let mut res = vec![FileSet::default(); self.len()];
53 for (file_id, path) in vfs.iter() {
54 let root = self.classify(&path);
55 res[root].insert(file_id, path)
56 }
57 res
58 }
59 fn len(&self) -> usize {
60 self.n_file_sets
61 }
62 fn classify(&self, path: &VfsPath) -> usize {
63 for (root, idx) in self.roots.iter() {
64 if let Some(path) = path.as_path() {
65 if path.starts_with(root) {
66 return *idx;
67 }
68 }
69 }
70 self.len() - 1
71 }
72}
73
74pub struct FileSetConfigBuilder {
75 roots: Vec<Vec<AbsPathBuf>>,
76}
77
78impl Default for FileSetConfigBuilder {
79 fn default() -> Self {
80 FileSetConfigBuilder { roots: Vec::new() }
81 }
82}
83
84impl FileSetConfigBuilder {
85 pub fn add_file_set(&mut self, roots: Vec<AbsPathBuf>) {
86 self.roots.push(roots)
87 }
88 pub fn build(self) -> FileSetConfig {
89 let n_file_sets = self.roots.len() + 1;
90 let mut roots: Vec<(AbsPathBuf, usize)> = self
91 .roots
92 .into_iter()
93 .enumerate()
94 .flat_map(|(i, paths)| paths.into_iter().zip(iter::repeat(i)))
95 .collect();
96 roots.sort_by_key(|(path, _)| cmp::Reverse(path.to_string_lossy().len()));
97 FileSetConfig { n_file_sets, roots }
98 }
99}
diff --git a/crates/vfs/src/lib.rs b/crates/vfs/src/lib.rs
new file mode 100644
index 000000000..75ce61cf9
--- /dev/null
+++ b/crates/vfs/src/lib.rs
@@ -0,0 +1,138 @@
1//! # Virtual File System
2//!
3//! VFS stores all files read by rust-analyzer. Reading file contents from VFS
4//! always returns the same contents, unless VFS was explicitly modified with
5//! `set_file_contents`. All changes to VFS are logged, and can be retrieved via
6//! `take_changes` method. The pack of changes is then pushed to `salsa` and
7//! triggers incremental recomputation.
8//!
9//! Files in VFS are identified with `FileId`s -- interned paths. The notion of
10//! the path, `VfsPath` is somewhat abstract: at the moment, it is represented
11//! as an `std::path::PathBuf` internally, but this is an implementation detail.
12//!
13//! VFS doesn't do IO or file watching itself. For that, see the `loader`
14//! module. `loader::Handle` is an object-safe trait which abstracts both file
15//! loading and file watching. `Handle` is dynamically configured with a set of
16//! directory entries which should be scanned and watched. `Handle` then
17//! asynchronously pushes file changes. Directory entries are configured in
18//! free-form via list of globs, it's up to the `Handle` to interpret the globs
19//! in any specific way.
20//!
21//! A simple `WalkdirLoaderHandle` is provided, which doesn't implement watching
22//! and just scans the directory using walkdir.
23//!
24//! VFS stores a flat list of files. `FileSet` can partition this list of files
25//! into disjoint sets of files. Traversal-like operations (including getting
26//! the neighbor file by the relative path) are handled by the `FileSet`.
27//! `FileSet`s are also pushed to salsa and cause it to re-check `mod foo;`
28//! declarations when files are created or deleted.
29//!
30//! `file_set::FileSet` and `loader::Entry` play similar, but different roles.
31//! Both specify the "set of paths/files", one is geared towards file watching,
32//! the other towards salsa changes. In particular, single `file_set::FileSet`
33//! may correspond to several `loader::Entry`. For example, a crate from
34//! crates.io which uses code generation would have two `Entries` -- for sources
35//! in `~/.cargo`, and for generated code in `./target/debug/build`. It will
36//! have a single `FileSet` which unions the two sources.
37mod vfs_path;
38mod path_interner;
39pub mod file_set;
40pub mod loader;
41pub mod walkdir_loader;
42
43use std::{fmt, mem};
44
45use crate::path_interner::PathInterner;
46
47pub use crate::vfs_path::VfsPath;
48pub use paths::{AbsPath, AbsPathBuf};
49
50#[derive(Copy, Clone, Debug, Ord, PartialOrd, Eq, PartialEq, Hash)]
51pub struct FileId(pub u32);
52
53#[derive(Default)]
54pub struct Vfs {
55 interner: PathInterner,
56 data: Vec<Option<Vec<u8>>>,
57 changes: Vec<ChangedFile>,
58}
59
60pub struct ChangedFile {
61 pub file_id: FileId,
62 pub change_kind: ChangeKind,
63}
64
65impl ChangedFile {
66 pub fn exists(&self) -> bool {
67 self.change_kind != ChangeKind::Delete
68 }
69 pub fn is_created_or_deleted(&self) -> bool {
70 matches!(self.change_kind, ChangeKind::Create | ChangeKind::Delete)
71 }
72}
73
74#[derive(Eq, PartialEq)]
75pub enum ChangeKind {
76 Create,
77 Modify,
78 Delete,
79}
80
81impl Vfs {
82 pub fn file_id(&self, path: &VfsPath) -> Option<FileId> {
83 self.interner.get(path).filter(|&it| self.get(it).is_some())
84 }
85 pub fn file_path(&self, file_id: FileId) -> VfsPath {
86 self.interner.lookup(file_id).clone()
87 }
88 pub fn file_contents(&self, file_id: FileId) -> &[u8] {
89 self.get(file_id).as_deref().unwrap()
90 }
91 pub fn iter(&self) -> impl Iterator<Item = (FileId, VfsPath)> + '_ {
92 (0..self.data.len())
93 .map(|it| FileId(it as u32))
94 .filter(move |&file_id| self.get(file_id).is_some())
95 .map(move |file_id| {
96 let path = self.interner.lookup(file_id).clone();
97 (file_id, path)
98 })
99 }
100 pub fn set_file_contents(&mut self, path: VfsPath, contents: Option<Vec<u8>>) {
101 let file_id = self.alloc_file_id(path);
102 let change_kind = match (&self.get(file_id), &contents) {
103 (None, None) => return,
104 (None, Some(_)) => ChangeKind::Create,
105 (Some(_), None) => ChangeKind::Delete,
106 (Some(old), Some(new)) if old == new => return,
107 (Some(_), Some(_)) => ChangeKind::Modify,
108 };
109
110 *self.get_mut(file_id) = contents;
111 self.changes.push(ChangedFile { file_id, change_kind })
112 }
113 pub fn has_changes(&self) -> bool {
114 !self.changes.is_empty()
115 }
116 pub fn take_changes(&mut self) -> Vec<ChangedFile> {
117 mem::take(&mut self.changes)
118 }
119 fn alloc_file_id(&mut self, path: VfsPath) -> FileId {
120 let file_id = self.interner.intern(path);
121 let idx = file_id.0 as usize;
122 let len = self.data.len().max(idx + 1);
123 self.data.resize_with(len, || None);
124 file_id
125 }
126 fn get(&self, file_id: FileId) -> &Option<Vec<u8>> {
127 &self.data[file_id.0 as usize]
128 }
129 fn get_mut(&mut self, file_id: FileId) -> &mut Option<Vec<u8>> {
130 &mut self.data[file_id.0 as usize]
131 }
132}
133
134impl fmt::Debug for Vfs {
135 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
136 f.debug_struct("Vfs").field("n_files", &self.data.len()).finish()
137 }
138}
diff --git a/crates/vfs/src/loader.rs b/crates/vfs/src/loader.rs
new file mode 100644
index 000000000..5a0ca68f3
--- /dev/null
+++ b/crates/vfs/src/loader.rs
@@ -0,0 +1,69 @@
1//! Object safe interface for file watching and reading.
2use std::fmt;
3
4use paths::AbsPathBuf;
5
6pub enum Entry {
7 Files(Vec<AbsPathBuf>),
8 Directory { path: AbsPathBuf, globs: Vec<String> },
9}
10
11pub struct Config {
12 pub load: Vec<Entry>,
13 pub watch: Vec<usize>,
14}
15
16pub enum Message {
17 DidSwitchConfig { n_entries: usize },
18 DidLoadAllEntries,
19 Loaded { files: Vec<(AbsPathBuf, Option<Vec<u8>>)> },
20}
21
22pub type Sender = Box<dyn Fn(Message) + Send>;
23
24pub trait Handle: fmt::Debug {
25 fn spawn(sender: Sender) -> Self
26 where
27 Self: Sized;
28 fn set_config(&mut self, config: Config);
29 fn invalidate(&mut self, path: AbsPathBuf);
30 fn load_sync(&mut self, path: &AbsPathBuf) -> Option<Vec<u8>>;
31}
32
33impl Entry {
34 pub fn rs_files_recursively(base: AbsPathBuf) -> Entry {
35 Entry::Directory { path: base, globs: globs(&["*.rs"]) }
36 }
37 pub fn local_cargo_package(base: AbsPathBuf) -> Entry {
38 Entry::Directory { path: base, globs: globs(&["*.rs", "!/target/"]) }
39 }
40 pub fn cargo_package_dependency(base: AbsPathBuf) -> Entry {
41 Entry::Directory {
42 path: base,
43 globs: globs(&["*.rs", "!/tests/", "!/examples/", "!/benches/"]),
44 }
45 }
46}
47
48fn globs(globs: &[&str]) -> Vec<String> {
49 globs.iter().map(|it| it.to_string()).collect()
50}
51
52impl fmt::Debug for Message {
53 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
54 match self {
55 Message::Loaded { files } => {
56 f.debug_struct("Loaded").field("n_files", &files.len()).finish()
57 }
58 Message::DidSwitchConfig { n_entries } => {
59 f.debug_struct("DidSwitchConfig").field("n_entries", n_entries).finish()
60 }
61 Message::DidLoadAllEntries => f.debug_struct("DidLoadAllEntries").finish(),
62 }
63 }
64}
65
66#[test]
67fn handle_is_object_safe() {
68 fn _assert(_: &dyn Handle) {}
69}
diff --git a/crates/vfs/src/path_interner.rs b/crates/vfs/src/path_interner.rs
new file mode 100644
index 000000000..4f70d61e8
--- /dev/null
+++ b/crates/vfs/src/path_interner.rs
@@ -0,0 +1,31 @@
1//! Maps paths to compact integer ids. We don't care about clearings paths which
2//! no longer exist -- the assumption is total size of paths we ever look at is
3//! not too big.
4use rustc_hash::FxHashMap;
5
6use crate::{FileId, VfsPath};
7
8#[derive(Default)]
9pub(crate) struct PathInterner {
10 map: FxHashMap<VfsPath, FileId>,
11 vec: Vec<VfsPath>,
12}
13
14impl PathInterner {
15 pub(crate) fn get(&self, path: &VfsPath) -> Option<FileId> {
16 self.map.get(path).copied()
17 }
18 pub(crate) fn intern(&mut self, path: VfsPath) -> FileId {
19 if let Some(id) = self.get(&path) {
20 return id;
21 }
22 let id = FileId(self.vec.len() as u32);
23 self.map.insert(path.clone(), id);
24 self.vec.push(path);
25 id
26 }
27
28 pub(crate) fn lookup(&self, id: FileId) -> &VfsPath {
29 &self.vec[id.0 as usize]
30 }
31}
diff --git a/crates/vfs/src/vfs_path.rs b/crates/vfs/src/vfs_path.rs
new file mode 100644
index 000000000..de5dc0bf3
--- /dev/null
+++ b/crates/vfs/src/vfs_path.rs
@@ -0,0 +1,49 @@
1//! Abstract-ish representation of paths for VFS.
2use std::fmt;
3
4use paths::{AbsPath, AbsPathBuf};
5
6/// Long-term, we want to support files which do not reside in the file-system,
7/// so we treat VfsPaths as opaque identifiers.
8#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq, Hash)]
9pub struct VfsPath(VfsPathRepr);
10
11impl VfsPath {
12 pub fn as_path(&self) -> Option<&AbsPath> {
13 match &self.0 {
14 VfsPathRepr::PathBuf(it) => Some(it.as_path()),
15 }
16 }
17 pub fn join(&self, path: &str) -> VfsPath {
18 match &self.0 {
19 VfsPathRepr::PathBuf(it) => {
20 let res = it.join(path).normalize();
21 VfsPath(VfsPathRepr::PathBuf(res))
22 }
23 }
24 }
25 pub fn pop(&mut self) -> bool {
26 match &mut self.0 {
27 VfsPathRepr::PathBuf(it) => it.pop(),
28 }
29 }
30}
31
32#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq, Hash)]
33enum VfsPathRepr {
34 PathBuf(AbsPathBuf),
35}
36
37impl From<AbsPathBuf> for VfsPath {
38 fn from(v: AbsPathBuf) -> Self {
39 VfsPath(VfsPathRepr::PathBuf(v))
40 }
41}
42
43impl fmt::Display for VfsPath {
44 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
45 match &self.0 {
46 VfsPathRepr::PathBuf(it) => fmt::Display::fmt(&it.display(), f),
47 }
48 }
49}
diff --git a/crates/vfs/src/walkdir_loader.rs b/crates/vfs/src/walkdir_loader.rs
new file mode 100644
index 000000000..13e59e3f3
--- /dev/null
+++ b/crates/vfs/src/walkdir_loader.rs
@@ -0,0 +1,108 @@
1//! A walkdir-based implementation of `loader::Handle`, which doesn't try to
2//! watch files.
3use std::convert::TryFrom;
4
5use globset::{Glob, GlobSetBuilder};
6use paths::{AbsPath, AbsPathBuf};
7use walkdir::WalkDir;
8
9use crate::loader;
10
11#[derive(Debug)]
12pub struct WalkdirLoaderHandle {
13 // Relative order of fields below is significant.
14 sender: crossbeam_channel::Sender<Message>,
15 _thread: jod_thread::JoinHandle,
16}
17
18enum Message {
19 Config(loader::Config),
20 Invalidate(AbsPathBuf),
21}
22
23impl loader::Handle for WalkdirLoaderHandle {
24 fn spawn(sender: loader::Sender) -> WalkdirLoaderHandle {
25 let actor = WalkdirLoaderActor { sender };
26 let (sender, receiver) = crossbeam_channel::unbounded::<Message>();
27 let thread = jod_thread::spawn(move || actor.run(receiver));
28 WalkdirLoaderHandle { sender, _thread: thread }
29 }
30 fn set_config(&mut self, config: loader::Config) {
31 self.sender.send(Message::Config(config)).unwrap()
32 }
33 fn invalidate(&mut self, path: AbsPathBuf) {
34 self.sender.send(Message::Invalidate(path)).unwrap();
35 }
36 fn load_sync(&mut self, path: &AbsPathBuf) -> Option<Vec<u8>> {
37 read(path)
38 }
39}
40
41struct WalkdirLoaderActor {
42 sender: loader::Sender,
43}
44
45impl WalkdirLoaderActor {
46 fn run(mut self, receiver: crossbeam_channel::Receiver<Message>) {
47 for msg in receiver {
48 match msg {
49 Message::Config(config) => {
50 self.send(loader::Message::DidSwitchConfig { n_entries: config.load.len() });
51 for entry in config.load.into_iter() {
52 let files = self.load_entry(entry);
53 self.send(loader::Message::Loaded { files });
54 }
55 drop(config.watch);
56 self.send(loader::Message::DidLoadAllEntries);
57 }
58 Message::Invalidate(path) => {
59 let contents = read(path.as_path());
60 let files = vec![(path, contents)];
61 self.send(loader::Message::Loaded { files });
62 }
63 }
64 }
65 }
66 fn load_entry(&mut self, entry: loader::Entry) -> Vec<(AbsPathBuf, Option<Vec<u8>>)> {
67 match entry {
68 loader::Entry::Files(files) => files
69 .into_iter()
70 .map(|file| {
71 let contents = read(file.as_path());
72 (file, contents)
73 })
74 .collect::<Vec<_>>(),
75 loader::Entry::Directory { path, globs } => {
76 let globset = {
77 let mut builder = GlobSetBuilder::new();
78 for glob in &globs {
79 builder.add(Glob::new(glob).unwrap());
80 }
81 builder.build().unwrap()
82 };
83
84 let files = WalkDir::new(path)
85 .into_iter()
86 .filter_map(|it| it.ok())
87 .filter(|it| it.file_type().is_file())
88 .map(|it| it.into_path())
89 .map(|it| AbsPathBuf::try_from(it).unwrap())
90 .filter(|it| globset.is_match(&it));
91
92 files
93 .map(|file| {
94 let contents = read(file.as_path());
95 (file, contents)
96 })
97 .collect()
98 }
99 }
100 }
101 fn send(&mut self, msg: loader::Message) {
102 (self.sender)(msg)
103 }
104}
105
106fn read(path: &AbsPath) -> Option<Vec<u8>> {
107 std::fs::read(path).ok()
108}