aboutsummaryrefslogtreecommitdiff
path: root/crates/ra_vfs
diff options
context:
space:
mode:
Diffstat (limited to 'crates/ra_vfs')
-rw-r--r--crates/ra_vfs/Cargo.toml17
-rw-r--r--crates/ra_vfs/src/arena.rs53
-rw-r--r--crates/ra_vfs/src/io.rs76
-rw-r--r--crates/ra_vfs/src/lib.rs350
-rw-r--r--crates/ra_vfs/tests/vfs.rs101
5 files changed, 597 insertions, 0 deletions
diff --git a/crates/ra_vfs/Cargo.toml b/crates/ra_vfs/Cargo.toml
new file mode 100644
index 000000000..ccea8a866
--- /dev/null
+++ b/crates/ra_vfs/Cargo.toml
@@ -0,0 +1,17 @@
1[package]
2edition = "2018"
3name = "ra_vfs"
4version = "0.1.0"
5authors = ["Aleksey Kladov <[email protected]>"]
6
7[dependencies]
8walkdir = "2.2.7"
9relative-path = "0.4.0"
10rustc-hash = "1.0"
11crossbeam-channel = "0.2.4"
12log = "0.4.6"
13
14thread_worker = { path = "../thread_worker" }
15
16[dev-dependencies]
17tempfile = "3"
diff --git a/crates/ra_vfs/src/arena.rs b/crates/ra_vfs/src/arena.rs
new file mode 100644
index 000000000..6b42ae26d
--- /dev/null
+++ b/crates/ra_vfs/src/arena.rs
@@ -0,0 +1,53 @@
1use std::{
2 marker::PhantomData,
3 ops::{Index, IndexMut},
4};
5
6#[derive(Clone, Debug)]
7pub(crate) struct Arena<ID: ArenaId, T> {
8 data: Vec<T>,
9 _ty: PhantomData<ID>,
10}
11
12pub(crate) trait ArenaId {
13 fn from_u32(id: u32) -> Self;
14 fn to_u32(self) -> u32;
15}
16
17impl<ID: ArenaId, T> Arena<ID, T> {
18 pub fn alloc(&mut self, value: T) -> ID {
19 let id = self.data.len() as u32;
20 self.data.push(value);
21 ID::from_u32(id)
22 }
23 pub fn iter<'a>(&'a self) -> impl Iterator<Item = (ID, &'a T)> {
24 self.data
25 .iter()
26 .enumerate()
27 .map(|(idx, value)| (ID::from_u32(idx as u32), value))
28 }
29}
30
31impl<ID: ArenaId, T> Default for Arena<ID, T> {
32 fn default() -> Arena<ID, T> {
33 Arena {
34 data: Vec::new(),
35 _ty: PhantomData,
36 }
37 }
38}
39
40impl<ID: ArenaId, T> Index<ID> for Arena<ID, T> {
41 type Output = T;
42 fn index(&self, idx: ID) -> &T {
43 let idx = idx.to_u32() as usize;
44 &self.data[idx]
45 }
46}
47
48impl<ID: ArenaId, T> IndexMut<ID> for Arena<ID, T> {
49 fn index_mut(&mut self, idx: ID) -> &mut T {
50 let idx = idx.to_u32() as usize;
51 &mut self.data[idx]
52 }
53}
diff --git a/crates/ra_vfs/src/io.rs b/crates/ra_vfs/src/io.rs
new file mode 100644
index 000000000..be400bae9
--- /dev/null
+++ b/crates/ra_vfs/src/io.rs
@@ -0,0 +1,76 @@
1use std::{
2 fmt,
3 fs,
4 path::{Path, PathBuf},
5};
6
7use walkdir::{DirEntry, WalkDir};
8use thread_worker::{WorkerHandle};
9use relative_path::RelativePathBuf;
10
11use crate::VfsRoot;
12
13pub(crate) struct Task {
14 pub(crate) root: VfsRoot,
15 pub(crate) path: PathBuf,
16 pub(crate) filter: Box<Fn(&DirEntry) -> bool + Send>,
17}
18
19pub struct TaskResult {
20 pub(crate) root: VfsRoot,
21 pub(crate) files: Vec<(RelativePathBuf, String)>,
22}
23
24impl fmt::Debug for TaskResult {
25 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
26 f.write_str("TaskResult { ... }")
27 }
28}
29
30pub(crate) type Worker = thread_worker::Worker<Task, TaskResult>;
31
32pub(crate) fn start() -> (Worker, WorkerHandle) {
33 thread_worker::spawn("vfs", 128, |input_receiver, output_sender| {
34 input_receiver
35 .map(handle_task)
36 .for_each(|it| output_sender.send(it))
37 })
38}
39
40fn handle_task(task: Task) -> TaskResult {
41 let Task { root, path, filter } = task;
42 log::debug!("loading {} ...", path.as_path().display());
43 let files = load_root(path.as_path(), &*filter);
44 log::debug!("... loaded {}", path.as_path().display());
45 TaskResult { root, files }
46}
47
48fn load_root(root: &Path, filter: &dyn Fn(&DirEntry) -> bool) -> Vec<(RelativePathBuf, String)> {
49 let mut res = Vec::new();
50 for entry in WalkDir::new(root).into_iter().filter_entry(filter) {
51 let entry = match entry {
52 Ok(entry) => entry,
53 Err(e) => {
54 log::warn!("watcher error: {}", e);
55 continue;
56 }
57 };
58 if !entry.file_type().is_file() {
59 continue;
60 }
61 let path = entry.path();
62 if path.extension().and_then(|os| os.to_str()) != Some("rs") {
63 continue;
64 }
65 let text = match fs::read_to_string(path) {
66 Ok(text) => text,
67 Err(e) => {
68 log::warn!("watcher error: {}", e);
69 continue;
70 }
71 };
72 let path = RelativePathBuf::from_path(path.strip_prefix(root).unwrap()).unwrap();
73 res.push((path.to_owned(), text))
74 }
75 res
76}
diff --git a/crates/ra_vfs/src/lib.rs b/crates/ra_vfs/src/lib.rs
new file mode 100644
index 000000000..4de07b093
--- /dev/null
+++ b/crates/ra_vfs/src/lib.rs
@@ -0,0 +1,350 @@
1//! VFS stands for Virtual File System.
2//!
3//! When doing analysis, we don't want to do any IO, we want to keep all source
4//! code in memory. However, the actual source code is stored on disk, so you
5//! component which does this.
6//! need to get it into the memory in the first place somehow. VFS is the
7//!
8//! It also is responsible for watching the disk for changes, and for merging
9//! editor state (modified, unsaved files) with disk state.
10//!
11//! VFS is based on a concept of roots: a set of directories on the file system
12//! whihc are watched for changes. Typically, there will be a root for each
13//! Cargo package.
14mod arena;
15mod io;
16
17use std::{
18 fmt,
19 mem,
20 thread,
21 cmp::Reverse,
22 path::{Path, PathBuf},
23 ffi::OsStr,
24 sync::Arc,
25 fs,
26};
27
28use rustc_hash::{FxHashMap, FxHashSet};
29use relative_path::RelativePathBuf;
30use crossbeam_channel::Receiver;
31use walkdir::DirEntry;
32use thread_worker::{WorkerHandle};
33
34use crate::{
35 arena::{ArenaId, Arena},
36};
37
38pub use crate::io::TaskResult as VfsTask;
39
40/// `RootFilter` is a predicate that checks if a file can belong to a root. If
41/// several filters match a file (nested dirs), the most nested one wins.
42struct RootFilter {
43 root: PathBuf,
44 file_filter: fn(&Path) -> bool,
45}
46
47impl RootFilter {
48 fn new(root: PathBuf) -> RootFilter {
49 RootFilter {
50 root,
51 file_filter: has_rs_extension,
52 }
53 }
54 /// Check if this root can contain `path`. NB: even if this returns
55 /// true, the `path` might actually be conained in some nested root.
56 fn can_contain(&self, path: &Path) -> Option<RelativePathBuf> {
57 if !(self.file_filter)(path) {
58 return None;
59 }
60 if !(path.starts_with(&self.root)) {
61 return None;
62 }
63 let path = path.strip_prefix(&self.root).unwrap();
64 let path = RelativePathBuf::from_path(path).unwrap();
65 Some(path)
66 }
67}
68
69fn has_rs_extension(p: &Path) -> bool {
70 p.extension() == Some(OsStr::new("rs"))
71}
72
73#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)]
74pub struct VfsRoot(pub u32);
75
76impl ArenaId for VfsRoot {
77 fn from_u32(idx: u32) -> VfsRoot {
78 VfsRoot(idx)
79 }
80 fn to_u32(self) -> u32 {
81 self.0
82 }
83}
84
85#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)]
86pub struct VfsFile(pub u32);
87
88impl ArenaId for VfsFile {
89 fn from_u32(idx: u32) -> VfsFile {
90 VfsFile(idx)
91 }
92 fn to_u32(self) -> u32 {
93 self.0
94 }
95}
96
97struct VfsFileData {
98 root: VfsRoot,
99 path: RelativePathBuf,
100 text: Arc<String>,
101}
102
103pub struct Vfs {
104 roots: Arena<VfsRoot, RootFilter>,
105 files: Arena<VfsFile, VfsFileData>,
106 root2files: FxHashMap<VfsRoot, FxHashSet<VfsFile>>,
107 pending_changes: Vec<VfsChange>,
108 worker: io::Worker,
109 worker_handle: WorkerHandle,
110}
111
112impl fmt::Debug for Vfs {
113 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
114 f.write_str("Vfs { ... }")
115 }
116}
117
118impl Vfs {
119 pub fn new(mut roots: Vec<PathBuf>) -> (Vfs, Vec<VfsRoot>) {
120 let (worker, worker_handle) = io::start();
121
122 let mut res = Vfs {
123 roots: Arena::default(),
124 files: Arena::default(),
125 root2files: FxHashMap::default(),
126 worker,
127 worker_handle,
128 pending_changes: Vec::new(),
129 };
130
131 // A hack to make nesting work.
132 roots.sort_by_key(|it| Reverse(it.as_os_str().len()));
133 for (i, path) in roots.iter().enumerate() {
134 let root = res.roots.alloc(RootFilter::new(path.clone()));
135 res.root2files.insert(root, Default::default());
136 let nested = roots[..i]
137 .iter()
138 .filter(|it| it.starts_with(path))
139 .map(|it| it.clone())
140 .collect::<Vec<_>>();
141 let filter = move |entry: &DirEntry| {
142 if entry.file_type().is_file() {
143 has_rs_extension(entry.path())
144 } else {
145 nested.iter().all(|it| it != entry.path())
146 }
147 };
148 let task = io::Task {
149 root,
150 path: path.clone(),
151 filter: Box::new(filter),
152 };
153 res.worker.inp.send(task);
154 }
155 let roots = res.roots.iter().map(|(id, _)| id).collect();
156 (res, roots)
157 }
158
159 pub fn root2path(&self, root: VfsRoot) -> PathBuf {
160 self.roots[root].root.clone()
161 }
162
163 pub fn path2file(&self, path: &Path) -> Option<VfsFile> {
164 if let Some((_root, _path, Some(file))) = self.find_root(path) {
165 return Some(file);
166 }
167 None
168 }
169
170 pub fn file2path(&self, file: VfsFile) -> PathBuf {
171 let rel_path = &self.files[file].path;
172 let root_path = &self.roots[self.files[file].root].root;
173 rel_path.to_path(root_path)
174 }
175
176 pub fn file_for_path(&self, path: &Path) -> Option<VfsFile> {
177 if let Some((_root, _path, Some(file))) = self.find_root(path) {
178 return Some(file);
179 }
180 None
181 }
182
183 pub fn load(&mut self, path: &Path) -> Option<VfsFile> {
184 if let Some((root, rel_path, file)) = self.find_root(path) {
185 return if let Some(file) = file {
186 Some(file)
187 } else {
188 let text = fs::read_to_string(path).unwrap_or_default();
189 let text = Arc::new(text);
190 let file = self.add_file(root, rel_path.clone(), Arc::clone(&text));
191 let change = VfsChange::AddFile {
192 file,
193 text,
194 root,
195 path: rel_path,
196 };
197 self.pending_changes.push(change);
198 Some(file)
199 };
200 }
201 None
202 }
203
204 pub fn task_receiver(&self) -> &Receiver<io::TaskResult> {
205 &self.worker.out
206 }
207
208 pub fn handle_task(&mut self, task: io::TaskResult) {
209 let mut files = Vec::new();
210 // While we were scanning the root in the backgound, a file might have
211 // been open in the editor, so we need to account for that.
212 let exising = self.root2files[&task.root]
213 .iter()
214 .map(|&file| (self.files[file].path.clone(), file))
215 .collect::<FxHashMap<_, _>>();
216 for (path, text) in task.files {
217 if let Some(&file) = exising.get(&path) {
218 let text = Arc::clone(&self.files[file].text);
219 files.push((file, path, text));
220 continue;
221 }
222 let text = Arc::new(text);
223 let file = self.add_file(task.root, path.clone(), Arc::clone(&text));
224 files.push((file, path, text));
225 }
226
227 let change = VfsChange::AddRoot {
228 root: task.root,
229 files,
230 };
231 self.pending_changes.push(change);
232 }
233
234 pub fn add_file_overlay(&mut self, path: &Path, text: String) -> Option<VfsFile> {
235 let mut res = None;
236 if let Some((root, path, file)) = self.find_root(path) {
237 let text = Arc::new(text);
238 let change = if let Some(file) = file {
239 res = Some(file);
240 self.change_file(file, Arc::clone(&text));
241 VfsChange::ChangeFile { file, text }
242 } else {
243 let file = self.add_file(root, path.clone(), Arc::clone(&text));
244 res = Some(file);
245 VfsChange::AddFile {
246 file,
247 text,
248 root,
249 path,
250 }
251 };
252 self.pending_changes.push(change);
253 }
254 res
255 }
256
257 pub fn change_file_overlay(&mut self, path: &Path, new_text: String) {
258 if let Some((_root, _path, file)) = self.find_root(path) {
259 let file = file.expect("can't change a file which wasn't added");
260 let text = Arc::new(new_text);
261 self.change_file(file, Arc::clone(&text));
262 let change = VfsChange::ChangeFile { file, text };
263 self.pending_changes.push(change);
264 }
265 }
266
267 pub fn remove_file_overlay(&mut self, path: &Path) -> Option<VfsFile> {
268 let mut res = None;
269 if let Some((root, path, file)) = self.find_root(path) {
270 let file = file.expect("can't remove a file which wasn't added");
271 res = Some(file);
272 let full_path = path.to_path(&self.roots[root].root);
273 let change = if let Ok(text) = fs::read_to_string(&full_path) {
274 let text = Arc::new(text);
275 self.change_file(file, Arc::clone(&text));
276 VfsChange::ChangeFile { file, text }
277 } else {
278 self.remove_file(file);
279 VfsChange::RemoveFile { root, file, path }
280 };
281 self.pending_changes.push(change);
282 }
283 res
284 }
285
286 pub fn commit_changes(&mut self) -> Vec<VfsChange> {
287 mem::replace(&mut self.pending_changes, Vec::new())
288 }
289
290 /// Sutdown the VFS and terminate the background watching thread.
291 pub fn shutdown(self) -> thread::Result<()> {
292 let _ = self.worker.shutdown();
293 self.worker_handle.shutdown()
294 }
295
296 fn add_file(&mut self, root: VfsRoot, path: RelativePathBuf, text: Arc<String>) -> VfsFile {
297 let data = VfsFileData { root, path, text };
298 let file = self.files.alloc(data);
299 self.root2files.get_mut(&root).unwrap().insert(file);
300 file
301 }
302
303 fn change_file(&mut self, file: VfsFile, new_text: Arc<String>) {
304 self.files[file].text = new_text;
305 }
306
307 fn remove_file(&mut self, file: VfsFile) {
308 //FIXME: use arena with removal
309 self.files[file].text = Default::default();
310 self.files[file].path = Default::default();
311 let root = self.files[file].root;
312 let removed = self.root2files.get_mut(&root).unwrap().remove(&file);
313 assert!(removed);
314 }
315
316 fn find_root(&self, path: &Path) -> Option<(VfsRoot, RelativePathBuf, Option<VfsFile>)> {
317 let (root, path) = self
318 .roots
319 .iter()
320 .find_map(|(root, data)| data.can_contain(path).map(|it| (root, it)))?;
321 let file = self.root2files[&root]
322 .iter()
323 .map(|&it| it)
324 .find(|&file| self.files[file].path == path);
325 Some((root, path, file))
326 }
327}
328
329#[derive(Debug, Clone)]
330pub enum VfsChange {
331 AddRoot {
332 root: VfsRoot,
333 files: Vec<(VfsFile, RelativePathBuf, Arc<String>)>,
334 },
335 AddFile {
336 root: VfsRoot,
337 file: VfsFile,
338 path: RelativePathBuf,
339 text: Arc<String>,
340 },
341 RemoveFile {
342 root: VfsRoot,
343 file: VfsFile,
344 path: RelativePathBuf,
345 },
346 ChangeFile {
347 file: VfsFile,
348 text: Arc<String>,
349 },
350}
diff --git a/crates/ra_vfs/tests/vfs.rs b/crates/ra_vfs/tests/vfs.rs
new file mode 100644
index 000000000..f56fc4603
--- /dev/null
+++ b/crates/ra_vfs/tests/vfs.rs
@@ -0,0 +1,101 @@
1use std::{
2 fs,
3 collections::HashSet,
4};
5
6use tempfile::tempdir;
7
8use ra_vfs::{Vfs, VfsChange};
9
10#[test]
11fn test_vfs_works() -> std::io::Result<()> {
12 let files = [
13 ("a/foo.rs", "hello"),
14 ("a/bar.rs", "world"),
15 ("a/b/baz.rs", "nested hello"),
16 ];
17
18 let dir = tempdir()?;
19 for (path, text) in files.iter() {
20 let file_path = dir.path().join(path);
21 fs::create_dir_all(file_path.parent().unwrap())?;
22 fs::write(file_path, text)?
23 }
24
25 let a_root = dir.path().join("a");
26 let b_root = dir.path().join("a/b");
27
28 let (mut vfs, _) = Vfs::new(vec![a_root, b_root]);
29 for _ in 0..2 {
30 let task = vfs.task_receiver().recv().unwrap();
31 vfs.handle_task(task);
32 }
33 {
34 let files = vfs
35 .commit_changes()
36 .into_iter()
37 .flat_map(|change| {
38 let files = match change {
39 VfsChange::AddRoot { files, .. } => files,
40 _ => panic!("unexpected change"),
41 };
42 files.into_iter().map(|(_id, path, text)| {
43 let text: String = (&*text).clone();
44 (format!("{}", path.display()), text)
45 })
46 })
47 .collect::<HashSet<_>>();
48
49 let expected_files = [
50 ("foo.rs", "hello"),
51 ("bar.rs", "world"),
52 ("baz.rs", "nested hello"),
53 ]
54 .iter()
55 .map(|(path, text)| (path.to_string(), text.to_string()))
56 .collect::<HashSet<_>>();
57
58 assert_eq!(files, expected_files);
59 }
60
61 vfs.add_file_overlay(&dir.path().join("a/b/baz.rs"), "quux".to_string());
62 let change = vfs.commit_changes().pop().unwrap();
63 match change {
64 VfsChange::ChangeFile { text, .. } => assert_eq!(&*text, "quux"),
65 _ => panic!("unexpected change"),
66 }
67
68 vfs.change_file_overlay(&dir.path().join("a/b/baz.rs"), "m".to_string());
69 let change = vfs.commit_changes().pop().unwrap();
70 match change {
71 VfsChange::ChangeFile { text, .. } => assert_eq!(&*text, "m"),
72 _ => panic!("unexpected change"),
73 }
74
75 vfs.remove_file_overlay(&dir.path().join("a/b/baz.rs"));
76 let change = vfs.commit_changes().pop().unwrap();
77 match change {
78 VfsChange::ChangeFile { text, .. } => assert_eq!(&*text, "nested hello"),
79 _ => panic!("unexpected change"),
80 }
81
82 vfs.add_file_overlay(&dir.path().join("a/b/spam.rs"), "spam".to_string());
83 let change = vfs.commit_changes().pop().unwrap();
84 match change {
85 VfsChange::AddFile { text, path, .. } => {
86 assert_eq!(&*text, "spam");
87 assert_eq!(path, "spam.rs");
88 }
89 _ => panic!("unexpected change"),
90 }
91
92 vfs.remove_file_overlay(&dir.path().join("a/b/spam.rs"));
93 let change = vfs.commit_changes().pop().unwrap();
94 match change {
95 VfsChange::RemoveFile { .. } => (),
96 _ => panic!("unexpected change"),
97 }
98
99 vfs.shutdown().unwrap();
100 Ok(())
101}