move expansion-related code to a separate crate

4 files changed, 298 insertions, 0 deletions

diff --git a/crates/ra_hir_def/Cargo.toml b/crates/ra_hir_def/Cargo.toml
index 7c57d56bd..049f8a4fc 100644
--- a/crates/ra_hir_def/Cargo.toml
+++ b/crates/ra_hir_def/Cargo.toml
@@ -5,6 +5,11 @@ version = "0.1.0"
 authors = ["rust-analyzer developers"]
 
 [dependencies]
+log = "0.4.5"
+
 ra_arena = { path = "../ra_arena" }
 ra_db = { path = "../ra_db" }
 ra_syntax = { path = "../ra_syntax" }
+ra_prof = { path = "../ra_prof" }
+tt = { path = "../ra_tt", package = "ra_tt" }
+mbe = { path = "../ra_mbe", package = "ra_mbe" }
diff --git a/crates/ra_hir_def/src/db.rs b/crates/ra_hir_def/src/db.rs
new file mode 100644
index 000000000..7133b61db
--- /dev/null
+++ b/crates/ra_hir_def/src/db.rs
@@ -0,0 +1,46 @@
+use std::sync::Arc;
+
+use ra_db::{salsa, SourceDatabase};
+use ra_syntax::{Parse, SyntaxNode};
+
+use crate::{
+    ast_id_map::{AstIdMap, ErasedFileAstId},
+    expand::{HirFileId, MacroCallId, MacroCallLoc, MacroDefId, MacroFile},
+};
+
+#[salsa::query_group(AstDatabaseStorage)]
+pub trait AstDatabase: SourceDatabase {
+    fn ast_id_map(&self, file_id: HirFileId) -> Arc<AstIdMap>;
+    #[salsa::transparent]
+    fn ast_id_to_node(&self, file_id: HirFileId, ast_id: ErasedFileAstId) -> SyntaxNode;
+
+    #[salsa::transparent]
+    #[salsa::invoke(crate::expand::parse_or_expand_query)]
+    fn parse_or_expand(&self, file_id: HirFileId) -> Option<SyntaxNode>;
+
+    #[salsa::interned]
+    fn intern_macro(&self, macro_call: MacroCallLoc) -> MacroCallId;
+    #[salsa::invoke(crate::expand::macro_arg_query)]
+    fn macro_arg(&self, id: MacroCallId) -> Option<Arc<tt::Subtree>>;
+    #[salsa::invoke(crate::expand::macro_def_query)]
+    fn macro_def(&self, id: MacroDefId) -> Option<Arc<mbe::MacroRules>>;
+    #[salsa::invoke(crate::expand::parse_macro_query)]
+    fn parse_macro(&self, macro_file: MacroFile) -> Option<Parse<SyntaxNode>>;
+    #[salsa::invoke(crate::expand::macro_expand_query)]
+    fn macro_expand(&self, macro_call: MacroCallId) -> Result<Arc<tt::Subtree>, String>;
+}
+
+pub(crate) fn ast_id_map(db: &impl AstDatabase, file_id: HirFileId) -> Arc<AstIdMap> {
+    let map =
+        db.parse_or_expand(file_id).map_or_else(AstIdMap::default, |it| AstIdMap::from_source(&it));
+    Arc::new(map)
+}
+
+pub(crate) fn ast_id_to_node(
+    db: &impl AstDatabase,
+    file_id: HirFileId,
+    ast_id: ErasedFileAstId,
+) -> SyntaxNode {
+    let node = db.parse_or_expand(file_id).unwrap();
+    db.ast_id_map(file_id)[ast_id].to_node(&node)
+}
diff --git a/crates/ra_hir_def/src/expand.rs b/crates/ra_hir_def/src/expand.rs
new file mode 100644
index 000000000..6517ea84d
--- /dev/null
+++ b/crates/ra_hir_def/src/expand.rs
@@ -0,0 +1,243 @@
+use std::{
+    hash::{Hash, Hasher},
+    sync::Arc,
+};
+
+use mbe::MacroRules;
+use ra_db::{salsa, CrateId, FileId};
+use ra_prof::profile;
+use ra_syntax::{
+    ast::{self, AstNode},
+    Parse, SyntaxNode,
+};
+
+use crate::{ast_id_map::FileAstId, db::AstDatabase};
+
+macro_rules! impl_intern_key {
+    ($name:ident) => {
+        impl salsa::InternKey for $name {
+            fn from_intern_id(v: salsa::InternId) -> Self {
+                $name(v)
+            }
+            fn as_intern_id(&self) -> salsa::InternId {
+                self.0
+            }
+        }
+    };
+}
+
+/// Input to the analyzer is a set of files, where each file is identified by
+/// `FileId` and contains source code. However, another source of source code in
+/// Rust are macros: each macro can be thought of as producing a "temporary
+/// file". To assign an id to such a file, we use the id of the macro call that
+/// produced the file. So, a `HirFileId` is either a `FileId` (source code
+/// written by user), or a `MacroCallId` (source code produced by macro).
+///
+/// What is a `MacroCallId`? Simplifying, it's a `HirFileId` of a file
+/// containing the call plus the offset of the macro call in the file. Note that
+/// this is a recursive definition! However, the size_of of `HirFileId` is
+/// finite (because everything bottoms out at the real `FileId`) and small
+/// (`MacroCallId` uses the location interner).
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum HirFileId {
+    FileId(FileId),
+    MacroFile(MacroFile),
+}
+
+impl From<FileId> for HirFileId {
+    fn from(id: FileId) -> Self {
+        HirFileId::FileId(id)
+    }
+}
+
+impl From<MacroFile> for HirFileId {
+    fn from(id: MacroFile) -> Self {
+        HirFileId::MacroFile(id)
+    }
+}
+
+impl HirFileId {
+    /// For macro-expansion files, returns the file original source file the
+    /// expansion originated from.
+    pub fn original_file(self, db: &impl AstDatabase) -> FileId {
+        match self {
+            HirFileId::FileId(file_id) => file_id,
+            HirFileId::MacroFile(macro_file) => {
+                let loc = db.lookup_intern_macro(macro_file.macro_call_id);
+                loc.ast_id.file_id().original_file(db)
+            }
+        }
+    }
+
+    /// Get the crate which the macro lives in, if it is a macro file.
+    pub fn macro_crate(self, db: &impl AstDatabase) -> Option<CrateId> {
+        match self {
+            HirFileId::FileId(_) => None,
+            HirFileId::MacroFile(macro_file) => {
+                let loc = db.lookup_intern_macro(macro_file.macro_call_id);
+                Some(loc.def.krate)
+            }
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct MacroFile {
+    macro_call_id: MacroCallId,
+    macro_file_kind: MacroFileKind,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum MacroFileKind {
+    Items,
+    Expr,
+}
+
+/// `MacroCallId` identifies a particular macro invocation, like
+/// `println!("Hello, {}", world)`.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct MacroCallId(salsa::InternId);
+impl_intern_key!(MacroCallId);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct MacroDefId {
+    pub krate: CrateId,
+    pub ast_id: AstId<ast::MacroCall>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct MacroCallLoc {
+    pub def: MacroDefId,
+    pub ast_id: AstId<ast::MacroCall>,
+}
+
+impl MacroCallId {
+    pub fn loc(self, db: &impl AstDatabase) -> MacroCallLoc {
+        db.lookup_intern_macro(self)
+    }
+
+    pub fn as_file(self, kind: MacroFileKind) -> HirFileId {
+        let macro_file = MacroFile { macro_call_id: self, macro_file_kind: kind };
+        macro_file.into()
+    }
+}
+
+impl MacroCallLoc {
+    pub fn id(self, db: &impl AstDatabase) -> MacroCallId {
+        db.intern_macro(self)
+    }
+}
+
+/// `AstId` points to an AST node in any file.
+///
+/// It is stable across reparses, and can be used as salsa key/value.
+// FIXME: isn't this just a `Source<FileAstId<N>>` ?
+#[derive(Debug)]
+pub struct AstId<N: AstNode> {
+    file_id: HirFileId,
+    file_ast_id: FileAstId<N>,
+}
+
+impl<N: AstNode> Clone for AstId<N> {
+    fn clone(&self) -> AstId<N> {
+        *self
+    }
+}
+impl<N: AstNode> Copy for AstId<N> {}
+
+impl<N: AstNode> PartialEq for AstId<N> {
+    fn eq(&self, other: &Self) -> bool {
+        (self.file_id, self.file_ast_id) == (other.file_id, other.file_ast_id)
+    }
+}
+impl<N: AstNode> Eq for AstId<N> {}
+impl<N: AstNode> Hash for AstId<N> {
+    fn hash<H: Hasher>(&self, hasher: &mut H) {
+        (self.file_id, self.file_ast_id).hash(hasher);
+    }
+}
+
+impl<N: AstNode> AstId<N> {
+    pub fn new(file_id: HirFileId, file_ast_id: FileAstId<N>) -> AstId<N> {
+        AstId { file_id, file_ast_id }
+    }
+
+    pub fn file_id(&self) -> HirFileId {
+        self.file_id
+    }
+
+    pub fn to_node(&self, db: &impl AstDatabase) -> N {
+        let syntax_node = db.ast_id_to_node(self.file_id, self.file_ast_id.into());
+        N::cast(syntax_node).unwrap()
+    }
+}
+
+pub(crate) fn macro_def_query(db: &impl AstDatabase, id: MacroDefId) -> Option<Arc<MacroRules>> {
+    let macro_call = id.ast_id.to_node(db);
+    let arg = macro_call.token_tree()?;
+    let (tt, _) = mbe::ast_to_token_tree(&arg).or_else(|| {
+        log::warn!("fail on macro_def to token tree: {:#?}", arg);
+        None
+    })?;
+    let rules = MacroRules::parse(&tt).ok().or_else(|| {
+        log::warn!("fail on macro_def parse: {:#?}", tt);
+        None
+    })?;
+    Some(Arc::new(rules))
+}
+
+pub(crate) fn macro_arg_query(db: &impl AstDatabase, id: MacroCallId) -> Option<Arc<tt::Subtree>> {
+    let loc = db.lookup_intern_macro(id);
+    let macro_call = loc.ast_id.to_node(db);
+    let arg = macro_call.token_tree()?;
+    let (tt, _) = mbe::ast_to_token_tree(&arg)?;
+    Some(Arc::new(tt))
+}
+
+pub(crate) fn macro_expand_query(
+    db: &impl AstDatabase,
+    id: MacroCallId,
+) -> Result<Arc<tt::Subtree>, String> {
+    let loc = db.lookup_intern_macro(id);
+    let macro_arg = db.macro_arg(id).ok_or("Fail to args in to tt::TokenTree")?;
+
+    let macro_rules = db.macro_def(loc.def).ok_or("Fail to find macro definition")?;
+    let tt = macro_rules.expand(&macro_arg).map_err(|err| format!("{:?}", err))?;
+    // Set a hard limit for the expanded tt
+    let count = tt.count();
+    if count > 65536 {
+        return Err(format!("Total tokens count exceed limit : count = {}", count));
+    }
+    Ok(Arc::new(tt))
+}
+
+pub(crate) fn parse_or_expand_query(
+    db: &impl AstDatabase,
+    file_id: HirFileId,
+) -> Option<SyntaxNode> {
+    match file_id {
+        HirFileId::FileId(file_id) => Some(db.parse(file_id).tree().syntax().clone()),
+        HirFileId::MacroFile(macro_file) => db.parse_macro(macro_file).map(|it| it.syntax_node()),
+    }
+}
+
+pub(crate) fn parse_macro_query(
+    db: &impl AstDatabase,
+    macro_file: MacroFile,
+) -> Option<Parse<SyntaxNode>> {
+    let _p = profile("parse_macro_query");
+    let macro_call_id = macro_file.macro_call_id;
+    let tt = db
+        .macro_expand(macro_call_id)
+        .map_err(|err| {
+            // Note:
+            // The final goal we would like to make all parse_macro success,
+            // such that the following log will not call anyway.
+            log::warn!("fail on macro_parse: (reason: {})", err,);
+        })
+        .ok()?;
+    match macro_file.macro_file_kind {
+        MacroFileKind::Items => mbe::token_tree_to_items(&tt).ok().map(Parse::to_syntax),
+        MacroFileKind::Expr => mbe::token_tree_to_expr(&tt).ok().map(Parse::to_syntax),
+    }
+}
diff --git a/crates/ra_hir_def/src/lib.rs b/crates/ra_hir_def/src/lib.rs
index 4d4d2cb19..6ccb11068 100644
--- a/crates/ra_hir_def/src/lib.rs
+++ b/crates/ra_hir_def/src/lib.rs
@@ -4,4 +4,8 @@
 //! Note that we are in the process of moving parts of `ra_hir` into
 //! `ra_hir_def`, so this crates doesn't contain a lot at the moment.
 
+pub mod db;
+
 pub mod ast_id_map;
+
+pub mod expand;