aboutsummaryrefslogtreecommitdiff
path: root/crates/ra_hir_expand
diff options
context:
space:
mode:
authorbors[bot] <26634292+bors[bot]@users.noreply.github.com>2019-10-29 13:19:45 +0000
committerGitHub <[email protected]>2019-10-29 13:19:45 +0000
commite38cdf6e56d963525fcc656b80965e7114756496 (patch)
tree4ca01adaa4ade6e90d7bcf20748dd691cdb5151d /crates/ra_hir_expand
parente960414a4d2a654898e697d267a5d95eb93271a0 (diff)
parentbca708ba4c5eb474448ef2f2882a66ec935f2fee (diff)
Merge #2114
2114: Move macro expansion to a separate crate r=matklad a=matklad Co-authored-by: Aleksey Kladov <[email protected]>
Diffstat (limited to 'crates/ra_hir_expand')
-rw-r--r--crates/ra_hir_expand/Cargo.toml15
-rw-r--r--crates/ra_hir_expand/src/ast_id_map.rs106
-rw-r--r--crates/ra_hir_expand/src/db.rs104
-rw-r--r--crates/ra_hir_expand/src/lib.rs161
4 files changed, 386 insertions, 0 deletions
diff --git a/crates/ra_hir_expand/Cargo.toml b/crates/ra_hir_expand/Cargo.toml
new file mode 100644
index 000000000..9bf5b7918
--- /dev/null
+++ b/crates/ra_hir_expand/Cargo.toml
@@ -0,0 +1,15 @@
1[package]
2edition = "2018"
3name = "ra_hir_expand"
4version = "0.1.0"
5authors = ["rust-analyzer developers"]
6
7[dependencies]
8log = "0.4.5"
9
10ra_arena = { path = "../ra_arena" }
11ra_db = { path = "../ra_db" }
12ra_syntax = { path = "../ra_syntax" }
13ra_prof = { path = "../ra_prof" }
14tt = { path = "../ra_tt", package = "ra_tt" }
15mbe = { path = "../ra_mbe", package = "ra_mbe" }
diff --git a/crates/ra_hir_expand/src/ast_id_map.rs b/crates/ra_hir_expand/src/ast_id_map.rs
new file mode 100644
index 000000000..cb464c3ff
--- /dev/null
+++ b/crates/ra_hir_expand/src/ast_id_map.rs
@@ -0,0 +1,106 @@
1//! `AstIdMap` allows to create stable IDs for "large" syntax nodes like items
2//! and macro calls.
3//!
4//! Specifically, it enumerates all items in a file and uses position of a an
5//! item as an ID. That way, id's don't change unless the set of items itself
6//! changes.
7
8use std::{
9 hash::{Hash, Hasher},
10 marker::PhantomData,
11};
12
13use ra_arena::{impl_arena_id, Arena, RawId};
14use ra_syntax::{ast, AstNode, AstPtr, SyntaxNode, SyntaxNodePtr};
15
16/// `AstId` points to an AST node in a specific file.
17#[derive(Debug)]
18pub struct FileAstId<N: AstNode> {
19 raw: ErasedFileAstId,
20 _ty: PhantomData<fn() -> N>,
21}
22
23impl<N: AstNode> Clone for FileAstId<N> {
24 fn clone(&self) -> FileAstId<N> {
25 *self
26 }
27}
28impl<N: AstNode> Copy for FileAstId<N> {}
29
30impl<N: AstNode> PartialEq for FileAstId<N> {
31 fn eq(&self, other: &Self) -> bool {
32 self.raw == other.raw
33 }
34}
35impl<N: AstNode> Eq for FileAstId<N> {}
36impl<N: AstNode> Hash for FileAstId<N> {
37 fn hash<H: Hasher>(&self, hasher: &mut H) {
38 self.raw.hash(hasher);
39 }
40}
41
42#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
43struct ErasedFileAstId(RawId);
44impl_arena_id!(ErasedFileAstId);
45
46/// Maps items' `SyntaxNode`s to `ErasedFileAstId`s and back.
47#[derive(Debug, PartialEq, Eq, Default)]
48pub struct AstIdMap {
49 arena: Arena<ErasedFileAstId, SyntaxNodePtr>,
50}
51
52impl AstIdMap {
53 pub(crate) fn from_source(node: &SyntaxNode) -> AstIdMap {
54 assert!(node.parent().is_none());
55 let mut res = AstIdMap { arena: Arena::default() };
56 // By walking the tree in bread-first order we make sure that parents
57 // get lower ids then children. That is, adding a new child does not
58 // change parent's id. This means that, say, adding a new function to a
59 // trait does not change ids of top-level items, which helps caching.
60 bfs(node, |it| {
61 if let Some(module_item) = ast::ModuleItem::cast(it.clone()) {
62 res.alloc(module_item.syntax());
63 } else if let Some(macro_call) = ast::MacroCall::cast(it) {
64 res.alloc(macro_call.syntax());
65 }
66 });
67 res
68 }
69
70 pub fn ast_id<N: AstNode>(&self, item: &N) -> FileAstId<N> {
71 let raw = self.erased_ast_id(item.syntax());
72 FileAstId { raw, _ty: PhantomData }
73 }
74 fn erased_ast_id(&self, item: &SyntaxNode) -> ErasedFileAstId {
75 let ptr = SyntaxNodePtr::new(item);
76 match self.arena.iter().find(|(_id, i)| **i == ptr) {
77 Some((it, _)) => it,
78 None => panic!(
79 "Can't find {:?} in AstIdMap:\n{:?}",
80 item,
81 self.arena.iter().map(|(_id, i)| i).collect::<Vec<_>>(),
82 ),
83 }
84 }
85
86 pub(crate) fn get<N: AstNode>(&self, id: FileAstId<N>) -> AstPtr<N> {
87 self.arena[id.raw].cast::<N>().unwrap()
88 }
89
90 fn alloc(&mut self, item: &SyntaxNode) -> ErasedFileAstId {
91 self.arena.alloc(SyntaxNodePtr::new(item))
92 }
93}
94
95/// Walks the subtree in bfs order, calling `f` for each node.
96fn bfs(node: &SyntaxNode, mut f: impl FnMut(SyntaxNode)) {
97 let mut curr_layer = vec![node.clone()];
98 let mut next_layer = vec![];
99 while !curr_layer.is_empty() {
100 curr_layer.drain(..).for_each(|node| {
101 next_layer.extend(node.children());
102 f(node);
103 });
104 std::mem::swap(&mut curr_layer, &mut next_layer);
105 }
106}
diff --git a/crates/ra_hir_expand/src/db.rs b/crates/ra_hir_expand/src/db.rs
new file mode 100644
index 000000000..a4ee9a529
--- /dev/null
+++ b/crates/ra_hir_expand/src/db.rs
@@ -0,0 +1,104 @@
1//! Defines database & queries for macro expansion.
2
3use std::sync::Arc;
4
5use mbe::MacroRules;
6use ra_db::{salsa, SourceDatabase};
7use ra_prof::profile;
8use ra_syntax::{AstNode, Parse, SyntaxNode};
9
10use crate::{
11 ast_id_map::AstIdMap, HirFileId, HirFileIdRepr, MacroCallId, MacroCallLoc, MacroDefId,
12 MacroFile, MacroFileKind,
13};
14
15// FIXME: rename to ExpandDatabase
16#[salsa::query_group(AstDatabaseStorage)]
17pub trait AstDatabase: SourceDatabase {
18 fn ast_id_map(&self, file_id: HirFileId) -> Arc<AstIdMap>;
19
20 #[salsa::transparent]
21 fn parse_or_expand(&self, file_id: HirFileId) -> Option<SyntaxNode>;
22
23 #[salsa::interned]
24 fn intern_macro(&self, macro_call: MacroCallLoc) -> MacroCallId;
25 fn macro_arg(&self, id: MacroCallId) -> Option<Arc<tt::Subtree>>;
26 fn macro_def(&self, id: MacroDefId) -> Option<Arc<mbe::MacroRules>>;
27 fn parse_macro(&self, macro_file: MacroFile) -> Option<Parse<SyntaxNode>>;
28 fn macro_expand(&self, macro_call: MacroCallId) -> Result<Arc<tt::Subtree>, String>;
29}
30
31pub(crate) fn ast_id_map(db: &dyn AstDatabase, file_id: HirFileId) -> Arc<AstIdMap> {
32 let map =
33 db.parse_or_expand(file_id).map_or_else(AstIdMap::default, |it| AstIdMap::from_source(&it));
34 Arc::new(map)
35}
36
37pub(crate) fn macro_def(db: &dyn AstDatabase, id: MacroDefId) -> Option<Arc<MacroRules>> {
38 let macro_call = id.ast_id.to_node(db);
39 let arg = macro_call.token_tree()?;
40 let (tt, _) = mbe::ast_to_token_tree(&arg).or_else(|| {
41 log::warn!("fail on macro_def to token tree: {:#?}", arg);
42 None
43 })?;
44 let rules = MacroRules::parse(&tt).ok().or_else(|| {
45 log::warn!("fail on macro_def parse: {:#?}", tt);
46 None
47 })?;
48 Some(Arc::new(rules))
49}
50
51pub(crate) fn macro_arg(db: &dyn AstDatabase, id: MacroCallId) -> Option<Arc<tt::Subtree>> {
52 let loc = db.lookup_intern_macro(id);
53 let macro_call = loc.ast_id.to_node(db);
54 let arg = macro_call.token_tree()?;
55 let (tt, _) = mbe::ast_to_token_tree(&arg)?;
56 Some(Arc::new(tt))
57}
58
59pub(crate) fn macro_expand(
60 db: &dyn AstDatabase,
61 id: MacroCallId,
62) -> Result<Arc<tt::Subtree>, String> {
63 let loc = db.lookup_intern_macro(id);
64 let macro_arg = db.macro_arg(id).ok_or("Fail to args in to tt::TokenTree")?;
65
66 let macro_rules = db.macro_def(loc.def).ok_or("Fail to find macro definition")?;
67 let tt = macro_rules.expand(&macro_arg).map_err(|err| format!("{:?}", err))?;
68 // Set a hard limit for the expanded tt
69 let count = tt.count();
70 if count > 65536 {
71 return Err(format!("Total tokens count exceed limit : count = {}", count));
72 }
73 Ok(Arc::new(tt))
74}
75
76pub(crate) fn parse_or_expand(db: &dyn AstDatabase, file_id: HirFileId) -> Option<SyntaxNode> {
77 match file_id.0 {
78 HirFileIdRepr::FileId(file_id) => Some(db.parse(file_id).tree().syntax().clone()),
79 HirFileIdRepr::MacroFile(macro_file) => {
80 db.parse_macro(macro_file).map(|it| it.syntax_node())
81 }
82 }
83}
84
85pub(crate) fn parse_macro(
86 db: &dyn AstDatabase,
87 macro_file: MacroFile,
88) -> Option<Parse<SyntaxNode>> {
89 let _p = profile("parse_macro_query");
90 let macro_call_id = macro_file.macro_call_id;
91 let tt = db
92 .macro_expand(macro_call_id)
93 .map_err(|err| {
94 // Note:
95 // The final goal we would like to make all parse_macro success,
96 // such that the following log will not call anyway.
97 log::warn!("fail on macro_parse: (reason: {})", err,);
98 })
99 .ok()?;
100 match macro_file.macro_file_kind {
101 MacroFileKind::Items => mbe::token_tree_to_items(&tt).ok().map(Parse::to_syntax),
102 MacroFileKind::Expr => mbe::token_tree_to_expr(&tt).ok().map(Parse::to_syntax),
103 }
104}
diff --git a/crates/ra_hir_expand/src/lib.rs b/crates/ra_hir_expand/src/lib.rs
new file mode 100644
index 000000000..6b3538673
--- /dev/null
+++ b/crates/ra_hir_expand/src/lib.rs
@@ -0,0 +1,161 @@
1//! `ra_hir_expand` deals with macro expansion.
2//!
3//! Specifically, it implements a concept of `MacroFile` -- a file whose syntax
4//! tree originates not from the text of some `FileId`, but from some macro
5//! expansion.
6
7pub mod db;
8pub mod ast_id_map;
9
10use std::hash::{Hash, Hasher};
11
12use ra_db::{salsa, CrateId, FileId};
13use ra_syntax::ast::{self, AstNode};
14
15use crate::{ast_id_map::FileAstId, db::AstDatabase};
16
17/// Input to the analyzer is a set of files, where each file is identified by
18/// `FileId` and contains source code. However, another source of source code in
19/// Rust are macros: each macro can be thought of as producing a "temporary
20/// file". To assign an id to such a file, we use the id of the macro call that
21/// produced the file. So, a `HirFileId` is either a `FileId` (source code
22/// written by user), or a `MacroCallId` (source code produced by macro).
23///
24/// What is a `MacroCallId`? Simplifying, it's a `HirFileId` of a file
25/// containing the call plus the offset of the macro call in the file. Note that
26/// this is a recursive definition! However, the size_of of `HirFileId` is
27/// finite (because everything bottoms out at the real `FileId`) and small
28/// (`MacroCallId` uses the location interner).
29#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
30pub struct HirFileId(HirFileIdRepr);
31
32#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
33enum HirFileIdRepr {
34 FileId(FileId),
35 MacroFile(MacroFile),
36}
37
38impl From<FileId> for HirFileId {
39 fn from(id: FileId) -> Self {
40 HirFileId(HirFileIdRepr::FileId(id))
41 }
42}
43
44impl From<MacroFile> for HirFileId {
45 fn from(id: MacroFile) -> Self {
46 HirFileId(HirFileIdRepr::MacroFile(id))
47 }
48}
49
50impl HirFileId {
51 /// For macro-expansion files, returns the file original source file the
52 /// expansion originated from.
53 pub fn original_file(self, db: &dyn AstDatabase) -> FileId {
54 match self.0 {
55 HirFileIdRepr::FileId(file_id) => file_id,
56 HirFileIdRepr::MacroFile(macro_file) => {
57 let loc = db.lookup_intern_macro(macro_file.macro_call_id);
58 loc.ast_id.file_id().original_file(db)
59 }
60 }
61 }
62
63 /// Get the crate which the macro lives in, if it is a macro file.
64 pub fn macro_crate(self, db: &dyn AstDatabase) -> Option<CrateId> {
65 match self.0 {
66 HirFileIdRepr::FileId(_) => None,
67 HirFileIdRepr::MacroFile(macro_file) => {
68 let loc = db.lookup_intern_macro(macro_file.macro_call_id);
69 Some(loc.def.krate)
70 }
71 }
72 }
73}
74
75#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
76pub struct MacroFile {
77 macro_call_id: MacroCallId,
78 macro_file_kind: MacroFileKind,
79}
80
81#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
82pub enum MacroFileKind {
83 Items,
84 Expr,
85}
86
87/// `MacroCallId` identifies a particular macro invocation, like
88/// `println!("Hello, {}", world)`.
89#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
90pub struct MacroCallId(salsa::InternId);
91impl salsa::InternKey for MacroCallId {
92 fn from_intern_id(v: salsa::InternId) -> Self {
93 MacroCallId(v)
94 }
95 fn as_intern_id(&self) -> salsa::InternId {
96 self.0
97 }
98}
99
100#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
101pub struct MacroDefId {
102 pub krate: CrateId,
103 pub ast_id: AstId<ast::MacroCall>,
104}
105
106#[derive(Debug, Clone, PartialEq, Eq, Hash)]
107pub struct MacroCallLoc {
108 pub def: MacroDefId,
109 pub ast_id: AstId<ast::MacroCall>,
110}
111
112impl MacroCallId {
113 pub fn as_file(self, kind: MacroFileKind) -> HirFileId {
114 let macro_file = MacroFile { macro_call_id: self, macro_file_kind: kind };
115 macro_file.into()
116 }
117}
118
119/// `AstId` points to an AST node in any file.
120///
121/// It is stable across reparses, and can be used as salsa key/value.
122// FIXME: isn't this just a `Source<FileAstId<N>>` ?
123#[derive(Debug)]
124pub struct AstId<N: AstNode> {
125 file_id: HirFileId,
126 file_ast_id: FileAstId<N>,
127}
128
129impl<N: AstNode> Clone for AstId<N> {
130 fn clone(&self) -> AstId<N> {
131 *self
132 }
133}
134impl<N: AstNode> Copy for AstId<N> {}
135
136impl<N: AstNode> PartialEq for AstId<N> {
137 fn eq(&self, other: &Self) -> bool {
138 (self.file_id, self.file_ast_id) == (other.file_id, other.file_ast_id)
139 }
140}
141impl<N: AstNode> Eq for AstId<N> {}
142impl<N: AstNode> Hash for AstId<N> {
143 fn hash<H: Hasher>(&self, hasher: &mut H) {
144 (self.file_id, self.file_ast_id).hash(hasher);
145 }
146}
147
148impl<N: AstNode> AstId<N> {
149 pub fn new(file_id: HirFileId, file_ast_id: FileAstId<N>) -> AstId<N> {
150 AstId { file_id, file_ast_id }
151 }
152
153 pub fn file_id(&self) -> HirFileId {
154 self.file_id
155 }
156
157 pub fn to_node(&self, db: &dyn AstDatabase) -> N {
158 let root = db.parse_or_expand(self.file_id).unwrap();
159 db.ast_id_map(self.file_id).get(self.file_ast_id).to_node(&root)
160 }
161}