From 5b803055b7b690a57f1c08da8f1640139c739e76 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Tue, 29 Oct 2019 14:59:55 +0300 Subject: rename hir_def -> hir_expand --- crates/ra_hir_expand/src/ast_id_map.rs | 114 ++++++++++++++++ crates/ra_hir_expand/src/db.rs | 46 +++++++ crates/ra_hir_expand/src/expand.rs | 243 +++++++++++++++++++++++++++++++++ crates/ra_hir_expand/src/lib.rs | 11 ++ 4 files changed, 414 insertions(+) create mode 100644 crates/ra_hir_expand/src/ast_id_map.rs create mode 100644 crates/ra_hir_expand/src/db.rs create mode 100644 crates/ra_hir_expand/src/expand.rs create mode 100644 crates/ra_hir_expand/src/lib.rs (limited to 'crates/ra_hir_expand/src') diff --git a/crates/ra_hir_expand/src/ast_id_map.rs b/crates/ra_hir_expand/src/ast_id_map.rs new file mode 100644 index 000000000..c3b389102 --- /dev/null +++ b/crates/ra_hir_expand/src/ast_id_map.rs @@ -0,0 +1,114 @@ +//! `AstIdMap` allows to create stable IDs for "large" syntax nodes like items +//! and macro calls. +//! +//! Specifically, it enumerates all items in a file and uses position of a an +//! item as an ID. That way, id's don't change unless the set of items itself +//! changes. + +use std::{ + hash::{Hash, Hasher}, + marker::PhantomData, + ops, +}; + +use ra_arena::{impl_arena_id, Arena, RawId}; +use ra_syntax::{ast, AstNode, SyntaxNode, SyntaxNodePtr}; + +/// `AstId` points to an AST node in a specific file. +#[derive(Debug)] +pub struct FileAstId { + raw: ErasedFileAstId, + _ty: PhantomData N>, +} + +impl Clone for FileAstId { + fn clone(&self) -> FileAstId { + *self + } +} +impl Copy for FileAstId {} + +impl PartialEq for FileAstId { + fn eq(&self, other: &Self) -> bool { + self.raw == other.raw + } +} +impl Eq for FileAstId {} +impl Hash for FileAstId { + fn hash(&self, hasher: &mut H) { + self.raw.hash(hasher); + } +} + +impl From> for ErasedFileAstId { + fn from(id: FileAstId) -> Self { + id.raw + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ErasedFileAstId(RawId); +impl_arena_id!(ErasedFileAstId); + +/// Maps items' `SyntaxNode`s to `ErasedFileAstId`s and back. +#[derive(Debug, PartialEq, Eq, Default)] +pub struct AstIdMap { + arena: Arena, +} + +impl AstIdMap { + pub fn from_source(node: &SyntaxNode) -> AstIdMap { + assert!(node.parent().is_none()); + let mut res = AstIdMap { arena: Arena::default() }; + // By walking the tree in bread-first order we make sure that parents + // get lower ids then children. That is, adding a new child does not + // change parent's id. This means that, say, adding a new function to a + // trait does not change ids of top-level items, which helps caching. + bfs(node, |it| { + if let Some(module_item) = ast::ModuleItem::cast(it.clone()) { + res.alloc(module_item.syntax()); + } else if let Some(macro_call) = ast::MacroCall::cast(it) { + res.alloc(macro_call.syntax()); + } + }); + res + } + + pub fn ast_id(&self, item: &N) -> FileAstId { + let ptr = SyntaxNodePtr::new(item.syntax()); + let raw = match self.arena.iter().find(|(_id, i)| **i == ptr) { + Some((it, _)) => it, + None => panic!( + "Can't find {:?} in AstIdMap:\n{:?}", + item.syntax(), + self.arena.iter().map(|(_id, i)| i).collect::>(), + ), + }; + + FileAstId { raw, _ty: PhantomData } + } + + fn alloc(&mut self, item: &SyntaxNode) -> ErasedFileAstId { + self.arena.alloc(SyntaxNodePtr::new(item)) + } +} + +impl ops::Index for AstIdMap { + type Output = SyntaxNodePtr; + fn index(&self, index: ErasedFileAstId) -> &SyntaxNodePtr { + &self.arena[index] + } +} + +/// Walks the subtree in bfs order, calling `f` for each node. +fn bfs(node: &SyntaxNode, mut f: impl FnMut(SyntaxNode)) { + let mut curr_layer = vec![node.clone()]; + let mut next_layer = vec![]; + while !curr_layer.is_empty() { + curr_layer.drain(..).for_each(|node| { + next_layer.extend(node.children()); + f(node); + }); + std::mem::swap(&mut curr_layer, &mut next_layer); + } +} diff --git a/crates/ra_hir_expand/src/db.rs b/crates/ra_hir_expand/src/db.rs new file mode 100644 index 000000000..7133b61db --- /dev/null +++ b/crates/ra_hir_expand/src/db.rs @@ -0,0 +1,46 @@ +use std::sync::Arc; + +use ra_db::{salsa, SourceDatabase}; +use ra_syntax::{Parse, SyntaxNode}; + +use crate::{ + ast_id_map::{AstIdMap, ErasedFileAstId}, + expand::{HirFileId, MacroCallId, MacroCallLoc, MacroDefId, MacroFile}, +}; + +#[salsa::query_group(AstDatabaseStorage)] +pub trait AstDatabase: SourceDatabase { + fn ast_id_map(&self, file_id: HirFileId) -> Arc; + #[salsa::transparent] + fn ast_id_to_node(&self, file_id: HirFileId, ast_id: ErasedFileAstId) -> SyntaxNode; + + #[salsa::transparent] + #[salsa::invoke(crate::expand::parse_or_expand_query)] + fn parse_or_expand(&self, file_id: HirFileId) -> Option; + + #[salsa::interned] + fn intern_macro(&self, macro_call: MacroCallLoc) -> MacroCallId; + #[salsa::invoke(crate::expand::macro_arg_query)] + fn macro_arg(&self, id: MacroCallId) -> Option>; + #[salsa::invoke(crate::expand::macro_def_query)] + fn macro_def(&self, id: MacroDefId) -> Option>; + #[salsa::invoke(crate::expand::parse_macro_query)] + fn parse_macro(&self, macro_file: MacroFile) -> Option>; + #[salsa::invoke(crate::expand::macro_expand_query)] + fn macro_expand(&self, macro_call: MacroCallId) -> Result, String>; +} + +pub(crate) fn ast_id_map(db: &impl AstDatabase, file_id: HirFileId) -> Arc { + let map = + db.parse_or_expand(file_id).map_or_else(AstIdMap::default, |it| AstIdMap::from_source(&it)); + Arc::new(map) +} + +pub(crate) fn ast_id_to_node( + db: &impl AstDatabase, + file_id: HirFileId, + ast_id: ErasedFileAstId, +) -> SyntaxNode { + let node = db.parse_or_expand(file_id).unwrap(); + db.ast_id_map(file_id)[ast_id].to_node(&node) +} diff --git a/crates/ra_hir_expand/src/expand.rs b/crates/ra_hir_expand/src/expand.rs new file mode 100644 index 000000000..6517ea84d --- /dev/null +++ b/crates/ra_hir_expand/src/expand.rs @@ -0,0 +1,243 @@ +use std::{ + hash::{Hash, Hasher}, + sync::Arc, +}; + +use mbe::MacroRules; +use ra_db::{salsa, CrateId, FileId}; +use ra_prof::profile; +use ra_syntax::{ + ast::{self, AstNode}, + Parse, SyntaxNode, +}; + +use crate::{ast_id_map::FileAstId, db::AstDatabase}; + +macro_rules! impl_intern_key { + ($name:ident) => { + impl salsa::InternKey for $name { + fn from_intern_id(v: salsa::InternId) -> Self { + $name(v) + } + fn as_intern_id(&self) -> salsa::InternId { + self.0 + } + } + }; +} + +/// Input to the analyzer is a set of files, where each file is identified by +/// `FileId` and contains source code. However, another source of source code in +/// Rust are macros: each macro can be thought of as producing a "temporary +/// file". To assign an id to such a file, we use the id of the macro call that +/// produced the file. So, a `HirFileId` is either a `FileId` (source code +/// written by user), or a `MacroCallId` (source code produced by macro). +/// +/// What is a `MacroCallId`? Simplifying, it's a `HirFileId` of a file +/// containing the call plus the offset of the macro call in the file. Note that +/// this is a recursive definition! However, the size_of of `HirFileId` is +/// finite (because everything bottoms out at the real `FileId`) and small +/// (`MacroCallId` uses the location interner). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum HirFileId { + FileId(FileId), + MacroFile(MacroFile), +} + +impl From for HirFileId { + fn from(id: FileId) -> Self { + HirFileId::FileId(id) + } +} + +impl From for HirFileId { + fn from(id: MacroFile) -> Self { + HirFileId::MacroFile(id) + } +} + +impl HirFileId { + /// For macro-expansion files, returns the file original source file the + /// expansion originated from. + pub fn original_file(self, db: &impl AstDatabase) -> FileId { + match self { + HirFileId::FileId(file_id) => file_id, + HirFileId::MacroFile(macro_file) => { + let loc = db.lookup_intern_macro(macro_file.macro_call_id); + loc.ast_id.file_id().original_file(db) + } + } + } + + /// Get the crate which the macro lives in, if it is a macro file. + pub fn macro_crate(self, db: &impl AstDatabase) -> Option { + match self { + HirFileId::FileId(_) => None, + HirFileId::MacroFile(macro_file) => { + let loc = db.lookup_intern_macro(macro_file.macro_call_id); + Some(loc.def.krate) + } + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct MacroFile { + macro_call_id: MacroCallId, + macro_file_kind: MacroFileKind, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum MacroFileKind { + Items, + Expr, +} + +/// `MacroCallId` identifies a particular macro invocation, like +/// `println!("Hello, {}", world)`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct MacroCallId(salsa::InternId); +impl_intern_key!(MacroCallId); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct MacroDefId { + pub krate: CrateId, + pub ast_id: AstId, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct MacroCallLoc { + pub def: MacroDefId, + pub ast_id: AstId, +} + +impl MacroCallId { + pub fn loc(self, db: &impl AstDatabase) -> MacroCallLoc { + db.lookup_intern_macro(self) + } + + pub fn as_file(self, kind: MacroFileKind) -> HirFileId { + let macro_file = MacroFile { macro_call_id: self, macro_file_kind: kind }; + macro_file.into() + } +} + +impl MacroCallLoc { + pub fn id(self, db: &impl AstDatabase) -> MacroCallId { + db.intern_macro(self) + } +} + +/// `AstId` points to an AST node in any file. +/// +/// It is stable across reparses, and can be used as salsa key/value. +// FIXME: isn't this just a `Source>` ? +#[derive(Debug)] +pub struct AstId { + file_id: HirFileId, + file_ast_id: FileAstId, +} + +impl Clone for AstId { + fn clone(&self) -> AstId { + *self + } +} +impl Copy for AstId {} + +impl PartialEq for AstId { + fn eq(&self, other: &Self) -> bool { + (self.file_id, self.file_ast_id) == (other.file_id, other.file_ast_id) + } +} +impl Eq for AstId {} +impl Hash for AstId { + fn hash(&self, hasher: &mut H) { + (self.file_id, self.file_ast_id).hash(hasher); + } +} + +impl AstId { + pub fn new(file_id: HirFileId, file_ast_id: FileAstId) -> AstId { + AstId { file_id, file_ast_id } + } + + pub fn file_id(&self) -> HirFileId { + self.file_id + } + + pub fn to_node(&self, db: &impl AstDatabase) -> N { + let syntax_node = db.ast_id_to_node(self.file_id, self.file_ast_id.into()); + N::cast(syntax_node).unwrap() + } +} + +pub(crate) fn macro_def_query(db: &impl AstDatabase, id: MacroDefId) -> Option> { + let macro_call = id.ast_id.to_node(db); + let arg = macro_call.token_tree()?; + let (tt, _) = mbe::ast_to_token_tree(&arg).or_else(|| { + log::warn!("fail on macro_def to token tree: {:#?}", arg); + None + })?; + let rules = MacroRules::parse(&tt).ok().or_else(|| { + log::warn!("fail on macro_def parse: {:#?}", tt); + None + })?; + Some(Arc::new(rules)) +} + +pub(crate) fn macro_arg_query(db: &impl AstDatabase, id: MacroCallId) -> Option> { + let loc = db.lookup_intern_macro(id); + let macro_call = loc.ast_id.to_node(db); + let arg = macro_call.token_tree()?; + let (tt, _) = mbe::ast_to_token_tree(&arg)?; + Some(Arc::new(tt)) +} + +pub(crate) fn macro_expand_query( + db: &impl AstDatabase, + id: MacroCallId, +) -> Result, String> { + let loc = db.lookup_intern_macro(id); + let macro_arg = db.macro_arg(id).ok_or("Fail to args in to tt::TokenTree")?; + + let macro_rules = db.macro_def(loc.def).ok_or("Fail to find macro definition")?; + let tt = macro_rules.expand(¯o_arg).map_err(|err| format!("{:?}", err))?; + // Set a hard limit for the expanded tt + let count = tt.count(); + if count > 65536 { + return Err(format!("Total tokens count exceed limit : count = {}", count)); + } + Ok(Arc::new(tt)) +} + +pub(crate) fn parse_or_expand_query( + db: &impl AstDatabase, + file_id: HirFileId, +) -> Option { + match file_id { + HirFileId::FileId(file_id) => Some(db.parse(file_id).tree().syntax().clone()), + HirFileId::MacroFile(macro_file) => db.parse_macro(macro_file).map(|it| it.syntax_node()), + } +} + +pub(crate) fn parse_macro_query( + db: &impl AstDatabase, + macro_file: MacroFile, +) -> Option> { + let _p = profile("parse_macro_query"); + let macro_call_id = macro_file.macro_call_id; + let tt = db + .macro_expand(macro_call_id) + .map_err(|err| { + // Note: + // The final goal we would like to make all parse_macro success, + // such that the following log will not call anyway. + log::warn!("fail on macro_parse: (reason: {})", err,); + }) + .ok()?; + match macro_file.macro_file_kind { + MacroFileKind::Items => mbe::token_tree_to_items(&tt).ok().map(Parse::to_syntax), + MacroFileKind::Expr => mbe::token_tree_to_expr(&tt).ok().map(Parse::to_syntax), + } +} diff --git a/crates/ra_hir_expand/src/lib.rs b/crates/ra_hir_expand/src/lib.rs new file mode 100644 index 000000000..6ccb11068 --- /dev/null +++ b/crates/ra_hir_expand/src/lib.rs @@ -0,0 +1,11 @@ +//! `ra_hir_def` contains initial "phases" of the compiler. Roughly, everything +//! before types. +//! +//! Note that we are in the process of moving parts of `ra_hir` into +//! `ra_hir_def`, so this crates doesn't contain a lot at the moment. + +pub mod db; + +pub mod ast_id_map; + +pub mod expand; -- cgit v1.2.3