Merge #4976

4976: New VFS r=matklad a=matklad

Still a draft, but mostly working already. 

Co-authored-by: Aleksey Kladov <[email protected]>

6 files changed, 67 insertions, 134 deletions

diff --git a/crates/vfs/Cargo.toml b/crates/vfs/Cargo.toml
index c03e6363b..263069002 100644
--- a/crates/vfs/Cargo.toml
+++ b/crates/vfs/Cargo.toml
@@ -6,9 +6,5 @@ edition = "2018"
 
 [dependencies]
 rustc-hash = "1.0"
-jod-thread = "0.1.0"
-walkdir = "2.3.1"
-globset = "0.4.5"
-crossbeam-channel = "0.4.0"
 
 paths = { path = "../paths" }
diff --git a/crates/vfs/src/file_set.rs b/crates/vfs/src/file_set.rs
index 724606a3d..0173f7464 100644
--- a/crates/vfs/src/file_set.rs
+++ b/crates/vfs/src/file_set.rs
@@ -4,7 +4,6 @@
 //! the default `FileSet`.
 use std::{fmt, iter};
 
-use paths::AbsPathBuf;
 use rustc_hash::FxHashMap;
 
 use crate::{FileId, Vfs, VfsPath};
@@ -41,7 +40,7 @@ impl fmt::Debug for FileSet {
 #[derive(Debug)]
 pub struct FileSetConfig {
     n_file_sets: usize,
-    roots: Vec<(AbsPathBuf, usize)>,
+    roots: Vec<(VfsPath, usize)>,
 }
 
 impl Default for FileSetConfig {
@@ -66,11 +65,7 @@ impl FileSetConfig {
         self.n_file_sets
     }
     fn classify(&self, path: &VfsPath) -> usize {
-        let path = match path.as_path() {
-            Some(it) => it,
-            None => return self.len() - 1,
-        };
-        let idx = match self.roots.binary_search_by(|(p, _)| p.as_path().cmp(path)) {
+        let idx = match self.roots.binary_search_by(|(p, _)| p.cmp(path)) {
             Ok(it) => it,
             Err(it) => it.saturating_sub(1),
         };
@@ -83,7 +78,7 @@ impl FileSetConfig {
 }
 
 pub struct FileSetConfigBuilder {
-    roots: Vec<Vec<AbsPathBuf>>,
+    roots: Vec<Vec<VfsPath>>,
 }
 
 impl Default for FileSetConfigBuilder {
@@ -96,12 +91,12 @@ impl FileSetConfigBuilder {
     pub fn len(&self) -> usize {
         self.roots.len()
     }
-    pub fn add_file_set(&mut self, roots: Vec<AbsPathBuf>) {
+    pub fn add_file_set(&mut self, roots: Vec<VfsPath>) {
         self.roots.push(roots)
     }
     pub fn build(self) -> FileSetConfig {
         let n_file_sets = self.roots.len() + 1;
-        let mut roots: Vec<(AbsPathBuf, usize)> = self
+        let mut roots: Vec<(VfsPath, usize)> = self
             .roots
             .into_iter()
             .enumerate()
diff --git a/crates/vfs/src/lib.rs b/crates/vfs/src/lib.rs
index 055219b0c..024e58018 100644
--- a/crates/vfs/src/lib.rs
+++ b/crates/vfs/src/lib.rs
@@ -38,7 +38,6 @@ mod vfs_path;
 mod path_interner;
 pub mod file_set;
 pub mod loader;
-pub mod walkdir_loader;
 
 use std::{fmt, mem};
 
diff --git a/crates/vfs/src/loader.rs b/crates/vfs/src/loader.rs
index 5a0ca68f3..a216b5f13 100644
--- a/crates/vfs/src/loader.rs
+++ b/crates/vfs/src/loader.rs
@@ -3,19 +3,20 @@ use std::fmt;
 
 use paths::AbsPathBuf;
 
+#[derive(Debug)]
 pub enum Entry {
     Files(Vec<AbsPathBuf>),
-    Directory { path: AbsPathBuf, globs: Vec<String> },
+    Directory { path: AbsPathBuf, include: Vec<String> },
 }
 
+#[derive(Debug)]
 pub struct Config {
     pub load: Vec<Entry>,
     pub watch: Vec<usize>,
 }
 
 pub enum Message {
-    DidSwitchConfig { n_entries: usize },
-    DidLoadAllEntries,
+    Progress { n_entries_total: usize, n_entries_done: usize },
     Loaded { files: Vec<(AbsPathBuf, Option<Vec<u8>>)> },
 }
 
@@ -32,15 +33,15 @@ pub trait Handle: fmt::Debug {
 
 impl Entry {
     pub fn rs_files_recursively(base: AbsPathBuf) -> Entry {
-        Entry::Directory { path: base, globs: globs(&["*.rs"]) }
+        Entry::Directory { path: base, include: globs(&["*.rs", "!/.git/"]) }
     }
     pub fn local_cargo_package(base: AbsPathBuf) -> Entry {
-        Entry::Directory { path: base, globs: globs(&["*.rs", "!/target/"]) }
+        Entry::Directory { path: base, include: globs(&["*.rs", "!/target/", "!/.git/"]) }
     }
     pub fn cargo_package_dependency(base: AbsPathBuf) -> Entry {
         Entry::Directory {
             path: base,
-            globs: globs(&["*.rs", "!/tests/", "!/examples/", "!/benches/"]),
+            include: globs(&["*.rs", "!/tests/", "!/examples/", "!/benches/", "!/.git/"]),
         }
     }
 }
@@ -55,10 +56,11 @@ impl fmt::Debug for Message {
             Message::Loaded { files } => {
                 f.debug_struct("Loaded").field("n_files", &files.len()).finish()
             }
-            Message::DidSwitchConfig { n_entries } => {
-                f.debug_struct("DidSwitchConfig").field("n_entries", n_entries).finish()
-            }
-            Message::DidLoadAllEntries => f.debug_struct("DidLoadAllEntries").finish(),
+            Message::Progress { n_entries_total, n_entries_done } => f
+                .debug_struct("Progress")
+                .field("n_entries_total", n_entries_total)
+                .field("n_entries_done", n_entries_done)
+                .finish(),
         }
     }
 }
diff --git a/crates/vfs/src/vfs_path.rs b/crates/vfs/src/vfs_path.rs
index de5dc0bf3..0a8a86c62 100644
--- a/crates/vfs/src/vfs_path.rs
+++ b/crates/vfs/src/vfs_path.rs
@@ -9,9 +9,17 @@ use paths::{AbsPath, AbsPathBuf};
 pub struct VfsPath(VfsPathRepr);
 
 impl VfsPath {
+    /// Creates an "in-memory" path from `/`-separates string.
+    /// This is most useful for testing, to avoid windows/linux differences
+    pub fn new_virtual_path(path: String) -> VfsPath {
+        assert!(path.starts_with('/'));
+        VfsPath(VfsPathRepr::VirtualPath(VirtualPath(path)))
+    }
+
     pub fn as_path(&self) -> Option<&AbsPath> {
         match &self.0 {
             VfsPathRepr::PathBuf(it) => Some(it.as_path()),
+            VfsPathRepr::VirtualPath(_) => None,
         }
     }
     pub fn join(&self, path: &str) -> VfsPath {
@@ -20,11 +28,24 @@ impl VfsPath {
                 let res = it.join(path).normalize();
                 VfsPath(VfsPathRepr::PathBuf(res))
             }
+            VfsPathRepr::VirtualPath(it) => {
+                let res = it.join(path);
+                VfsPath(VfsPathRepr::VirtualPath(res))
+            }
         }
     }
     pub fn pop(&mut self) -> bool {
         match &mut self.0 {
             VfsPathRepr::PathBuf(it) => it.pop(),
+            VfsPathRepr::VirtualPath(it) => it.pop(),
+        }
+    }
+    pub fn starts_with(&self, other: &VfsPath) -> bool {
+        match (&self.0, &other.0) {
+            (VfsPathRepr::PathBuf(lhs), VfsPathRepr::PathBuf(rhs)) => lhs.starts_with(rhs),
+            (VfsPathRepr::PathBuf(_), _) => false,
+            (VfsPathRepr::VirtualPath(lhs), VfsPathRepr::VirtualPath(rhs)) => lhs.starts_with(rhs),
+            (VfsPathRepr::VirtualPath(_), _) => false,
         }
     }
 }
@@ -32,11 +53,12 @@ impl VfsPath {
 #[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq, Hash)]
 enum VfsPathRepr {
     PathBuf(AbsPathBuf),
+    VirtualPath(VirtualPath),
 }
 
 impl From<AbsPathBuf> for VfsPath {
     fn from(v: AbsPathBuf) -> Self {
-        VfsPath(VfsPathRepr::PathBuf(v))
+        VfsPath(VfsPathRepr::PathBuf(v.normalize()))
     }
 }
 
@@ -44,6 +66,33 @@ impl fmt::Display for VfsPath {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match &self.0 {
             VfsPathRepr::PathBuf(it) => fmt::Display::fmt(&it.display(), f),
+            VfsPathRepr::VirtualPath(VirtualPath(it)) => fmt::Display::fmt(it, f),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq, Hash)]
+struct VirtualPath(String);
+
+impl VirtualPath {
+    fn starts_with(&self, other: &VirtualPath) -> bool {
+        self.0.starts_with(&other.0)
+    }
+    fn pop(&mut self) -> bool {
+        let pos = match self.0.rfind('/') {
+            Some(pos) => pos,
+            None => return false,
+        };
+        self.0 = self.0[..pos].to_string();
+        true
+    }
+    fn join(&self, mut path: &str) -> VirtualPath {
+        let mut res = self.clone();
+        while path.starts_with("../") {
+            assert!(res.pop());
+            path = &path["../".len()..]
         }
+        res.0 = format!("{}/{}", res.0, path);
+        res
     }
 }
diff --git a/crates/vfs/src/walkdir_loader.rs b/crates/vfs/src/walkdir_loader.rs
deleted file mode 100644
index 13e59e3f3..000000000
--- a/crates/vfs/src/walkdir_loader.rs
+++ /dev/null
@@ -1,108 +0,0 @@
-//! A walkdir-based implementation of `loader::Handle`, which doesn't try to
-//! watch files.
-use std::convert::TryFrom;
-
-use globset::{Glob, GlobSetBuilder};
-use paths::{AbsPath, AbsPathBuf};
-use walkdir::WalkDir;
-
-use crate::loader;
-
-#[derive(Debug)]
-pub struct WalkdirLoaderHandle {
-    // Relative order of fields below is significant.
-    sender: crossbeam_channel::Sender<Message>,
-    _thread: jod_thread::JoinHandle,
-}
-
-enum Message {
-    Config(loader::Config),
-    Invalidate(AbsPathBuf),
-}
-
-impl loader::Handle for WalkdirLoaderHandle {
-    fn spawn(sender: loader::Sender) -> WalkdirLoaderHandle {
-        let actor = WalkdirLoaderActor { sender };
-        let (sender, receiver) = crossbeam_channel::unbounded::<Message>();
-        let thread = jod_thread::spawn(move || actor.run(receiver));
-        WalkdirLoaderHandle { sender, _thread: thread }
-    }
-    fn set_config(&mut self, config: loader::Config) {
-        self.sender.send(Message::Config(config)).unwrap()
-    }
-    fn invalidate(&mut self, path: AbsPathBuf) {
-        self.sender.send(Message::Invalidate(path)).unwrap();
-    }
-    fn load_sync(&mut self, path: &AbsPathBuf) -> Option<Vec<u8>> {
-        read(path)
-    }
-}
-
-struct WalkdirLoaderActor {
-    sender: loader::Sender,
-}
-
-impl WalkdirLoaderActor {
-    fn run(mut self, receiver: crossbeam_channel::Receiver<Message>) {
-        for msg in receiver {
-            match msg {
-                Message::Config(config) => {
-                    self.send(loader::Message::DidSwitchConfig { n_entries: config.load.len() });
-                    for entry in config.load.into_iter() {
-                        let files = self.load_entry(entry);
-                        self.send(loader::Message::Loaded { files });
-                    }
-                    drop(config.watch);
-                    self.send(loader::Message::DidLoadAllEntries);
-                }
-                Message::Invalidate(path) => {
-                    let contents = read(path.as_path());
-                    let files = vec![(path, contents)];
-                    self.send(loader::Message::Loaded { files });
-                }
-            }
-        }
-    }
-    fn load_entry(&mut self, entry: loader::Entry) -> Vec<(AbsPathBuf, Option<Vec<u8>>)> {
-        match entry {
-            loader::Entry::Files(files) => files
-                .into_iter()
-                .map(|file| {
-                    let contents = read(file.as_path());
-                    (file, contents)
-                })
-                .collect::<Vec<_>>(),
-            loader::Entry::Directory { path, globs } => {
-                let globset = {
-                    let mut builder = GlobSetBuilder::new();
-                    for glob in &globs {
-                        builder.add(Glob::new(glob).unwrap());
-                    }
-                    builder.build().unwrap()
-                };
-
-                let files = WalkDir::new(path)
-                    .into_iter()
-                    .filter_map(|it| it.ok())
-                    .filter(|it| it.file_type().is_file())
-                    .map(|it| it.into_path())
-                    .map(|it| AbsPathBuf::try_from(it).unwrap())
-                    .filter(|it| globset.is_match(&it));
-
-                files
-                    .map(|file| {
-                        let contents = read(file.as_path());
-                        (file, contents)
-                    })
-                    .collect()
-            }
-        }
-    }
-    fn send(&mut self, msg: loader::Message) {
-        (self.sender)(msg)
-    }
-}
-
-fn read(path: &AbsPath) -> Option<Vec<u8>> {
-    std::fs::read(path).ok()
-}