8 files changed, 181 insertions, 87 deletions

diff --git a/crates/ra_hir_expand/src/ast_id_map.rs b/crates/ra_hir_expand/src/ast_id_map.rs
index d19569245..f4d31526a 100644
--- a/crates/ra_hir_expand/src/ast_id_map.rs
+++ b/crates/ra_hir_expand/src/ast_id_map.rs
@@ -6,6 +6,8 @@
 //! changes.
 
 use std::{
+    any::type_name,
+    fmt,
     hash::{Hash, Hasher},
     marker::PhantomData,
 };
@@ -14,7 +16,6 @@ use ra_arena::{Arena, Idx};
 use ra_syntax::{ast, AstNode, AstPtr, SyntaxNode, SyntaxNodePtr};
 
 /// `AstId` points to an AST node in a specific file.
-#[derive(Debug)]
 pub struct FileAstId<N: AstNode> {
     raw: ErasedFileAstId,
     _ty: PhantomData<fn() -> N>,
@@ -39,11 +40,17 @@ impl<N: AstNode> Hash for FileAstId<N> {
     }
 }
 
+impl<N: AstNode> fmt::Debug for FileAstId<N> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "FileAstId::<{}>({})", type_name::<N>(), self.raw.into_raw())
+    }
+}
+
 impl<N: AstNode> FileAstId<N> {
     // Can't make this a From implementation because of coherence
     pub fn upcast<M: AstNode>(self) -> FileAstId<M>
     where
-        M: From<N>,
+        N: Into<M>,
     {
         FileAstId { raw: self.raw, _ty: PhantomData }
     }
@@ -89,7 +96,7 @@ impl AstIdMap {
         }
     }
 
-    pub(crate) fn get<N: AstNode>(&self, id: FileAstId<N>) -> AstPtr<N> {
+    pub fn get<N: AstNode>(&self, id: FileAstId<N>) -> AstPtr<N> {
         self.arena[id.raw].clone().cast::<N>().unwrap()
     }
 
diff --git a/crates/ra_hir_expand/src/builtin_derive.rs b/crates/ra_hir_expand/src/builtin_derive.rs
index 1dc9cac66..f2d664863 100644
--- a/crates/ra_hir_expand/src/builtin_derive.rs
+++ b/crates/ra_hir_expand/src/builtin_derive.rs
@@ -8,8 +8,7 @@ use ra_syntax::{
     match_ast,
 };
 
-use crate::db::AstDatabase;
-use crate::{name, quote, LazyMacroId, MacroCallId, MacroDefId, MacroDefKind};
+use crate::{db::AstDatabase, name, quote, LazyMacroId, MacroDefId, MacroDefKind};
 
 macro_rules! register_builtin {
     ( $($trait:ident => $expand:ident),* ) => {
@@ -156,23 +155,13 @@ fn expand_simple_derive(
 fn find_builtin_crate(db: &dyn AstDatabase, id: LazyMacroId) -> tt::TokenTree {
     // FIXME: make hygiene works for builtin derive macro
     // such that $crate can be used here.
-
-    let m: MacroCallId = id.into();
-    let file_id = m.as_file().original_file(db);
     let cg = db.crate_graph();
-    let krates = db.relevant_crates(file_id);
-    let krate = match krates.get(0) {
-        Some(krate) => krate,
-        None => {
-            let tt = quote! { core };
-            return tt.token_trees[0].clone();
-        }
-    };
+    let krate = db.lookup_intern_macro(id).krate;
 
     // XXX
     //  All crates except core itself should have a dependency on core,
     //  We detect `core` by seeing whether it doesn't have such a dependency.
-    let tt = if cg[*krate].dependencies.iter().any(|dep| dep.name == "core") {
+    let tt = if cg[krate].dependencies.iter().any(|dep| &*dep.name == "core") {
         quote! { core }
     } else {
         quote! { crate }
@@ -264,10 +253,12 @@ fn partial_ord_expand(
 
 #[cfg(test)]
 mod tests {
-    use super::*;
-    use crate::{test_db::TestDB, AstId, MacroCallId, MacroCallKind, MacroCallLoc};
     use name::{known, Name};
-    use ra_db::{fixture::WithFixture, SourceDatabase};
+    use ra_db::{fixture::WithFixture, CrateId, SourceDatabase};
+
+    use crate::{test_db::TestDB, AstId, MacroCallId, MacroCallKind, MacroCallLoc};
+
+    use super::*;
 
     fn expand_builtin_derive(s: &str, name: Name) -> String {
         let def = find_builtin_derive(&name).unwrap();
@@ -291,7 +282,11 @@ mod tests {
 
         let attr_id = AstId::new(file_id.into(), ast_id_map.ast_id(&items[0]));
 
-        let loc = MacroCallLoc { def, kind: MacroCallKind::Attr(attr_id, name.to_string()) };
+        let loc = MacroCallLoc {
+            def,
+            krate: CrateId(0),
+            kind: MacroCallKind::Attr(attr_id, name.to_string()),
+        };
 
         let id: MacroCallId = db.intern_macro(loc).into();
         let parsed = db.parse_or_expand(id.as_file()).unwrap();
diff --git a/crates/ra_hir_expand/src/builtin_macro.rs b/crates/ra_hir_expand/src/builtin_macro.rs
index 3bce8f673..9f50569dc 100644
--- a/crates/ra_hir_expand/src/builtin_macro.rs
+++ b/crates/ra_hir_expand/src/builtin_macro.rs
@@ -1,15 +1,14 @@
 //! Builtin macro
-use crate::db::AstDatabase;
 use crate::{
-    ast::{self, AstToken, HasStringValue},
-    name, AstId, CrateId, MacroDefId, MacroDefKind, TextSize,
+    db::AstDatabase, name, quote, AstId, CrateId, EagerMacroId, LazyMacroId, MacroCallId,
+    MacroDefId, MacroDefKind, TextSize,
 };
 
-use crate::{quote, EagerMacroId, LazyMacroId, MacroCallId};
 use either::Either;
 use mbe::parse_to_token_tree;
-use ra_db::{FileId, RelativePath};
+use ra_db::FileId;
 use ra_parser::FragmentKind;
+use ra_syntax::ast::{self, AstToken, HasStringValue};
 
 macro_rules! register_builtin {
     ( LAZY: $(($name:ident, $kind: ident) => $expand:ident),* , EAGER: $(($e_name:ident, $e_kind: ident) => $e_expand:ident),*  ) => {
@@ -100,6 +99,8 @@ register_builtin! {
     EAGER:
     (concat, Concat) => concat_expand,
     (include, Include) => include_expand,
+    (include_bytes, IncludeBytes) => include_bytes_expand,
+    (include_str, IncludeStr) => include_str_expand,
     (env, Env) => env_expand,
     (option_env, OptionEnv) => option_env_expand
 }
@@ -271,7 +272,7 @@ fn format_args_expand(
 fn unquote_str(lit: &tt::Literal) -> Option<String> {
     let lit = ast::make::tokens::literal(&lit.to_string());
     let token = ast::String::cast(lit)?;
-    token.value()
+    token.value().map(|it| it.into_owned())
 }
 
 fn concat_expand(
@@ -293,21 +294,20 @@ fn concat_expand(
     Ok((quote!(#text), FragmentKind::Expr))
 }
 
-fn relative_file(db: &dyn AstDatabase, call_id: MacroCallId, path: &str) -> Option<FileId> {
+fn relative_file(
+    db: &dyn AstDatabase,
+    call_id: MacroCallId,
+    path: &str,
+    allow_recursion: bool,
+) -> Option<FileId> {
     let call_site = call_id.as_file().original_file(db);
-
-    // Handle trivial case
-    if let Some(res) = db.resolve_relative_path(call_site, &RelativePath::new(&path)) {
-        // Prevent include itself
-        return if res == call_site { None } else { Some(res) };
+    let res = db.resolve_path(call_site, path)?;
+    // Prevent include itself
+    if res == call_site && !allow_recursion {
+        None
+    } else {
+        Some(res)
     }
-
-    // Extern paths ?
-    let krate = *db.relevant_crates(call_site).get(0)?;
-    let (extern_source_id, relative_file) =
-        db.crate_graph()[krate].extern_source.extern_path(path)?;
-
-    db.resolve_extern_path(extern_source_id, &relative_file)
 }
 
 fn parse_string(tt: &tt::Subtree) -> Result<String, mbe::ExpandError> {
@@ -326,8 +326,8 @@ fn include_expand(
     tt: &tt::Subtree,
 ) -> Result<(tt::Subtree, FragmentKind), mbe::ExpandError> {
     let path = parse_string(tt)?;
-    let file_id =
-        relative_file(db, arg_id.into(), &path).ok_or_else(|| mbe::ExpandError::ConversionError)?;
+    let file_id = relative_file(db, arg_id.into(), &path, false)
+        .ok_or_else(|| mbe::ExpandError::ConversionError)?;
 
     // FIXME:
     // Handle include as expression
@@ -338,11 +338,50 @@ fn include_expand(
     Ok((res, FragmentKind::Items))
 }
 
-fn get_env_inner(db: &dyn AstDatabase, arg_id: EagerMacroId, key: &str) -> Option<String> {
-    let call_id: MacroCallId = arg_id.into();
-    let original_file = call_id.as_file().original_file(db);
+fn include_bytes_expand(
+    _db: &dyn AstDatabase,
+    _arg_id: EagerMacroId,
+    tt: &tt::Subtree,
+) -> Result<(tt::Subtree, FragmentKind), mbe::ExpandError> {
+    let _path = parse_string(tt)?;
+
+    // FIXME: actually read the file here if the user asked for macro expansion
+    let res = tt::Subtree {
+        delimiter: None,
+        token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
+            text: r#"b"""#.into(),
+            id: tt::TokenId::unspecified(),
+        }))],
+    };
+    Ok((res, FragmentKind::Expr))
+}
+
+fn include_str_expand(
+    db: &dyn AstDatabase,
+    arg_id: EagerMacroId,
+    tt: &tt::Subtree,
+) -> Result<(tt::Subtree, FragmentKind), mbe::ExpandError> {
+    let path = parse_string(tt)?;
+
+    // FIXME: we're not able to read excluded files (which is most of them because
+    // it's unusual to `include_str!` a Rust file), but we can return an empty string.
+    // Ideally, we'd be able to offer a precise expansion if the user asks for macro
+    // expansion.
+    let file_id = match relative_file(db, arg_id.into(), &path, true) {
+        Some(file_id) => file_id,
+        None => {
+            return Ok((quote!(""), FragmentKind::Expr));
+        }
+    };
+
+    let text = db.file_text(file_id);
+    let text = &*text;
 
-    let krate = *db.relevant_crates(original_file).get(0)?;
+    Ok((quote!(#text), FragmentKind::Expr))
+}
+
+fn get_env_inner(db: &dyn AstDatabase, arg_id: EagerMacroId, key: &str) -> Option<String> {
+    let krate = db.lookup_intern_eager_expansion(arg_id).krate;
     db.crate_graph()[krate].env.get(key)
 }
 
@@ -401,6 +440,7 @@ mod tests {
 
         let expander = find_by_name(&macro_calls[0].name().unwrap().as_name()).unwrap();
 
+        let krate = CrateId(0);
         let file_id = match expander {
             Either::Left(expander) => {
                 // the first one should be a macro_rules
@@ -413,6 +453,7 @@ mod tests {
 
                 let loc = MacroCallLoc {
                     def,
+                    krate,
                     kind: MacroCallKind::FnLike(AstId::new(
                         file_id.into(),
                         ast_id_map.ast_id(&macro_calls[1]),
@@ -425,7 +466,7 @@ mod tests {
             Either::Right(expander) => {
                 // the first one should be a macro_rules
                 let def = MacroDefId {
-                    krate: Some(CrateId(0)),
+                    krate: Some(krate),
                     ast_id: Some(AstId::new(file_id.into(), ast_id_map.ast_id(&macro_calls[0]))),
                     kind: MacroDefKind::BuiltInEager(expander),
                     local_inner: false,
@@ -439,6 +480,7 @@ mod tests {
                         def,
                         fragment: FragmentKind::Expr,
                         subtree: Arc::new(parsed_args.clone()),
+                        krate,
                         file_id: file_id.into(),
                     }
                 });
@@ -448,6 +490,7 @@ mod tests {
                     def,
                     fragment,
                     subtree: Arc::new(subtree),
+                    krate,
                     file_id: file_id.into(),
                 };
 
@@ -587,4 +630,20 @@ mod tests {
             r#"std::fmt::Arguments::new_v1(&[], &[std::fmt::ArgumentV1::new(&(arg1(a,b,c)),std::fmt::Display::fmt),std::fmt::ArgumentV1::new(&(arg2),std::fmt::Display::fmt),])"#
         );
     }
+
+    #[test]
+    fn test_include_bytes_expand() {
+        let expanded = expand_builtin_macro(
+            r#"
+            #[rustc_builtin_macro]
+            macro_rules! include_bytes {
+                ($file:expr) => {{ /* compiler built-in */ }};
+                ($file:expr,) => {{ /* compiler built-in */ }};
+            }
+            include_bytes("foo");
+            "#,
+        );
+
+        assert_eq!(expanded, r#"b"""#);
+    }
 }
diff --git a/crates/ra_hir_expand/src/diagnostics.rs b/crates/ra_hir_expand/src/diagnostics.rs
index 99209c6e8..545cff9bd 100644
--- a/crates/ra_hir_expand/src/diagnostics.rs
+++ b/crates/ra_hir_expand/src/diagnostics.rs
@@ -28,7 +28,7 @@ pub trait Diagnostic: Any + Send + Sync + fmt::Debug + 'static {
 
 pub trait AstDiagnostic {
     type AST;
-    fn ast(&self, db: &impl AstDatabase) -> Self::AST;
+    fn ast(&self, db: &dyn AstDatabase) -> Self::AST;
 }
 
 impl dyn Diagnostic {
diff --git a/crates/ra_hir_expand/src/eager.rs b/crates/ra_hir_expand/src/eager.rs
index 932f47c30..302d2b3e0 100644
--- a/crates/ra_hir_expand/src/eager.rs
+++ b/crates/ra_hir_expand/src/eager.rs
@@ -25,12 +25,14 @@ use crate::{
     EagerCallLoc, EagerMacroId, InFile, MacroCallId, MacroCallKind, MacroDefId, MacroDefKind,
 };
 
+use ra_db::CrateId;
 use ra_parser::FragmentKind;
 use ra_syntax::{algo::SyntaxRewriter, SyntaxNode};
 use std::sync::Arc;
 
 pub fn expand_eager_macro(
     db: &dyn AstDatabase,
+    krate: CrateId,
     macro_call: InFile<ast::MacroCall>,
     def: MacroDefId,
     resolver: &dyn Fn(ast::Path) -> Option<MacroDefId>,
@@ -47,6 +49,7 @@ pub fn expand_eager_macro(
             def,
             fragment: FragmentKind::Expr,
             subtree: Arc::new(parsed_args.clone()),
+            krate,
             file_id: macro_call.file_id,
         }
     });
@@ -56,14 +59,20 @@ pub fn expand_eager_macro(
     let result = eager_macro_recur(
         db,
         InFile::new(arg_file_id.as_file(), parsed_args.syntax_node()),
+        krate,
         resolver,
     )?;
     let subtree = to_subtree(&result)?;
 
     if let MacroDefKind::BuiltInEager(eager) = def.kind {
         let (subtree, fragment) = eager.expand(db, arg_id, &subtree).ok()?;
-        let eager =
-            EagerCallLoc { def, fragment, subtree: Arc::new(subtree), file_id: macro_call.file_id };
+        let eager = EagerCallLoc {
+            def,
+            fragment,
+            subtree: Arc::new(subtree),
+            krate,
+            file_id: macro_call.file_id,
+        };
 
         Some(db.intern_eager_expansion(eager))
     } else {
@@ -81,11 +90,12 @@ fn lazy_expand(
     db: &dyn AstDatabase,
     def: &MacroDefId,
     macro_call: InFile<ast::MacroCall>,
+    krate: CrateId,
 ) -> Option<InFile<SyntaxNode>> {
     let ast_id = db.ast_id_map(macro_call.file_id).ast_id(&macro_call.value);
 
     let id: MacroCallId =
-        def.as_lazy_macro(db, MacroCallKind::FnLike(macro_call.with_value(ast_id))).into();
+        def.as_lazy_macro(db, krate, MacroCallKind::FnLike(macro_call.with_value(ast_id))).into();
 
     db.parse_or_expand(id.as_file()).map(|node| InFile::new(id.as_file(), node))
 }
@@ -93,6 +103,7 @@ fn lazy_expand(
 fn eager_macro_recur(
     db: &dyn AstDatabase,
     curr: InFile<SyntaxNode>,
+    krate: CrateId,
     macro_resolver: &dyn Fn(ast::Path) -> Option<MacroDefId>,
 ) -> Option<SyntaxNode> {
     let original = curr.value.clone();
@@ -105,18 +116,23 @@ fn eager_macro_recur(
         let def: MacroDefId = macro_resolver(child.path()?)?;
         let insert = match def.kind {
             MacroDefKind::BuiltInEager(_) => {
-                let id: MacroCallId =
-                    expand_eager_macro(db, curr.with_value(child.clone()), def, macro_resolver)?
-                        .into();
+                let id: MacroCallId = expand_eager_macro(
+                    db,
+                    krate,
+                    curr.with_value(child.clone()),
+                    def,
+                    macro_resolver,
+                )?
+                .into();
                 db.parse_or_expand(id.as_file())?
             }
             MacroDefKind::Declarative
             | MacroDefKind::BuiltIn(_)
             | MacroDefKind::BuiltInDerive(_)
             | MacroDefKind::CustomDerive(_) => {
-                let expanded = lazy_expand(db, &def, curr.with_value(child.clone()))?;
+                let expanded = lazy_expand(db, &def, curr.with_value(child.clone()), krate)?;
                 // replace macro inside
-                eager_macro_recur(db, expanded, macro_resolver)?
+                eager_macro_recur(db, expanded, krate, macro_resolver)?
             }
         };
 
diff --git a/crates/ra_hir_expand/src/lib.rs b/crates/ra_hir_expand/src/lib.rs
index f440c073b..1cf6c1ba9 100644
--- a/crates/ra_hir_expand/src/lib.rs
+++ b/crates/ra_hir_expand/src/lib.rs
@@ -88,6 +88,25 @@ impl HirFileId {
         }
     }
 
+    pub fn expansion_level(self, db: &dyn db::AstDatabase) -> u32 {
+        let mut level = 0;
+        let mut curr = self;
+        while let HirFileIdRepr::MacroFile(macro_file) = curr.0 {
+            level += 1;
+            curr = match macro_file.macro_call_id {
+                MacroCallId::LazyMacro(id) => {
+                    let loc = db.lookup_intern_macro(id);
+                    loc.kind.file_id()
+                }
+                MacroCallId::EagerMacro(id) => {
+                    let loc = db.lookup_intern_eager_expansion(id);
+                    loc.file_id
+                }
+            };
+        }
+        level
+    }
+
     /// If this is a macro call, returns the syntax node of the call.
     pub fn call_node(self, db: &dyn db::AstDatabase) -> Option<InFile<SyntaxNode>> {
         match self.0 {
@@ -209,8 +228,13 @@ pub struct MacroDefId {
 }
 
 impl MacroDefId {
-    pub fn as_lazy_macro(self, db: &dyn db::AstDatabase, kind: MacroCallKind) -> LazyMacroId {
-        db.intern_macro(MacroCallLoc { def: self, kind })
+    pub fn as_lazy_macro(
+        self,
+        db: &dyn db::AstDatabase,
+        krate: CrateId,
+        kind: MacroCallKind,
+    ) -> LazyMacroId {
+        db.intern_macro(MacroCallLoc { def: self, krate, kind })
     }
 }
 
@@ -227,6 +251,7 @@ pub enum MacroDefKind {
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct MacroCallLoc {
     pub(crate) def: MacroDefId,
+    pub(crate) krate: CrateId,
     pub(crate) kind: MacroCallKind,
 }
 
@@ -274,6 +299,7 @@ pub struct EagerCallLoc {
     pub(crate) def: MacroDefId,
     pub(crate) fragment: FragmentKind,
     pub(crate) subtree: Arc<tt::Subtree>,
+    pub(crate) krate: CrateId,
     pub(crate) file_id: HirFileId,
 }
 
diff --git a/crates/ra_hir_expand/src/name.rs b/crates/ra_hir_expand/src/name.rs
index ea495cb11..969a2e5b8 100644
--- a/crates/ra_hir_expand/src/name.rs
+++ b/crates/ra_hir_expand/src/name.rs
@@ -117,7 +117,7 @@ impl AsName for ast::FieldKind {
 
 impl AsName for ra_db::Dependency {
     fn as_name(&self) -> Name {
-        Name::new_text(self.name.clone())
+        Name::new_text(SmolStr::new(&*self.name))
     }
 }
 
@@ -153,6 +153,7 @@ pub mod known {
         str,
         // Special names
         macro_rules,
+        doc,
         // Components of known path (value or mod name)
         std,
         core,
@@ -190,6 +191,8 @@ pub mod known {
         stringify,
         concat,
         include,
+        include_bytes,
+        include_str,
         format_args,
         format_args_nl,
         env,
diff --git a/crates/ra_hir_expand/src/test_db.rs b/crates/ra_hir_expand/src/test_db.rs
index c1fb762de..332fa556f 100644
--- a/crates/ra_hir_expand/src/test_db.rs
+++ b/crates/ra_hir_expand/src/test_db.rs
@@ -1,36 +1,35 @@
 //! Database used for testing `hir_expand`.
 
 use std::{
-    panic,
+    fmt, panic,
     sync::{Arc, Mutex},
 };
 
-use ra_db::{salsa, CrateId, ExternSourceId, FileId, FileLoader, FileLoaderDelegate, RelativePath};
+use ra_db::{salsa, CrateId, FileId, FileLoader, FileLoaderDelegate};
+use rustc_hash::FxHashSet;
 
 #[salsa::database(
     ra_db::SourceDatabaseExtStorage,
     ra_db::SourceDatabaseStorage,
     crate::db::AstDatabaseStorage
 )]
-#[derive(Debug, Default)]
+#[derive(Default)]
 pub struct TestDB {
-    runtime: salsa::Runtime<TestDB>,
-    events: Mutex<Option<Vec<salsa::Event<TestDB>>>>,
+    storage: salsa::Storage<TestDB>,
+    events: Mutex<Option<Vec<salsa::Event>>>,
 }
 
-impl salsa::Database for TestDB {
-    fn salsa_runtime(&self) -> &salsa::Runtime<Self> {
-        &self.runtime
-    }
-
-    fn salsa_runtime_mut(&mut self) -> &mut salsa::Runtime<Self> {
-        &mut self.runtime
+impl fmt::Debug for TestDB {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("TestDB").finish()
     }
+}
 
-    fn salsa_event(&self, event: impl Fn() -> salsa::Event<TestDB>) {
+impl salsa::Database for TestDB {
+    fn salsa_event(&self, event: salsa::Event) {
         let mut events = self.events.lock().unwrap();
         if let Some(events) = &mut *events {
-            events.push(event());
+            events.push(event);
         }
     }
 }
@@ -41,21 +40,10 @@ impl FileLoader for TestDB {
     fn file_text(&self, file_id: FileId) -> Arc<String> {
         FileLoaderDelegate(self).file_text(file_id)
     }
-    fn resolve_relative_path(
-        &self,
-        anchor: FileId,
-        relative_path: &RelativePath,
-    ) -> Option<FileId> {
-        FileLoaderDelegate(self).resolve_relative_path(anchor, relative_path)
+    fn resolve_path(&self, anchor: FileId, path: &str) -> Option<FileId> {
+        FileLoaderDelegate(self).resolve_path(anchor, path)
     }
-    fn relevant_crates(&self, file_id: FileId) -> Arc<Vec<CrateId>> {
+    fn relevant_crates(&self, file_id: FileId) -> Arc<FxHashSet<CrateId>> {
         FileLoaderDelegate(self).relevant_crates(file_id)
     }
-    fn resolve_extern_path(
-        &self,
-        anchor: ExternSourceId,
-        relative_path: &RelativePath,
-    ) -> Option<FileId> {
-        FileLoaderDelegate(self).resolve_extern_path(anchor, relative_path)
-    }
 }