Merge #2205

2205: Implement bulitin line! macro r=matklad a=edwin0cheng

This PR implements bulitin macro `line!` and add basic infra-structure for other bulitin macros:

1. Extend `MacroDefId` to support builtin macros
2. Add a `quote!` macro for simple quasi quoting.

Note that for support others builtin macros, eager macro expansion have to be supported first, this PR not try to handle it. :)

Co-authored-by: Edwin Cheng <[email protected]>

6 files changed, 410 insertions, 19 deletions

diff --git a/crates/ra_hir_expand/src/builtin_macro.rs b/crates/ra_hir_expand/src/builtin_macro.rs
new file mode 100644
index 000000000..97fb0cb55
--- /dev/null
+++ b/crates/ra_hir_expand/src/builtin_macro.rs
@@ -0,0 +1,80 @@
+//! Builtin macro
+use crate::db::AstDatabase;
+use crate::{
+    ast::{self, AstNode},
+    name, AstId, CrateId, HirFileId, MacroCallId, MacroDefId, MacroDefKind, MacroFileKind,
+    TextUnit,
+};
+
+use crate::quote;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum BuiltinExpander {
+    Line,
+}
+
+impl BuiltinExpander {
+    pub fn expand(
+        &self,
+        db: &dyn AstDatabase,
+        id: MacroCallId,
+        tt: &tt::Subtree,
+    ) -> Result<tt::Subtree, mbe::ExpandError> {
+        match self {
+            BuiltinExpander::Line => line_expand(db, id, tt),
+        }
+    }
+}
+
+pub fn find_builtin_macro(
+    ident: &name::Name,
+    krate: CrateId,
+    ast_id: AstId<ast::MacroCall>,
+) -> Option<MacroDefId> {
+    // FIXME: Better registering method
+    if ident == &name::LINE_MACRO {
+        Some(MacroDefId { krate, ast_id, kind: MacroDefKind::BuiltIn(BuiltinExpander::Line) })
+    } else {
+        None
+    }
+}
+
+fn to_line_number(db: &dyn AstDatabase, file: HirFileId, pos: TextUnit) -> usize {
+    // FIXME: Use expansion info
+    let file_id = file.original_file(db);
+    let text = db.file_text(file_id);
+    let mut line_num = 1;
+
+    // Count line end
+    for (i, c) in text.chars().enumerate() {
+        if i == pos.to_usize() {
+            break;
+        }
+        if c == '\n' {
+            line_num += 1;
+        }
+    }
+
+    line_num
+}
+
+fn line_expand(
+    db: &dyn AstDatabase,
+    id: MacroCallId,
+    _tt: &tt::Subtree,
+) -> Result<tt::Subtree, mbe::ExpandError> {
+    let loc = db.lookup_intern_macro(id);
+    let macro_call = loc.ast_id.to_node(db);
+
+    let arg = macro_call.token_tree().ok_or_else(|| mbe::ExpandError::UnexpectedToken)?;
+    let arg_start = arg.syntax().text_range().start();
+
+    let file = id.as_file(MacroFileKind::Expr);
+    let line_num = to_line_number(db, file, arg_start);
+
+    let expanded = quote! {
+        #line_num
+    };
+
+    Ok(expanded)
+}
diff --git a/crates/ra_hir_expand/src/db.rs b/crates/ra_hir_expand/src/db.rs
index b4dafe1d8..5eadee9c2 100644
--- a/crates/ra_hir_expand/src/db.rs
+++ b/crates/ra_hir_expand/src/db.rs
@@ -9,10 +9,37 @@ use ra_prof::profile;
 use ra_syntax::{AstNode, Parse, SyntaxNode};
 
 use crate::{
-    ast_id_map::AstIdMap, HirFileId, HirFileIdRepr, MacroCallId, MacroCallLoc, MacroDefId,
-    MacroFile, MacroFileKind,
+    ast_id_map::AstIdMap, BuiltinExpander, HirFileId, HirFileIdRepr, MacroCallId, MacroCallLoc,
+    MacroDefId, MacroDefKind, MacroFile, MacroFileKind,
 };
 
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub enum TokenExpander {
+    MacroRules(mbe::MacroRules),
+    Builtin(BuiltinExpander),
+}
+
+impl TokenExpander {
+    pub fn expand(
+        &self,
+        db: &dyn AstDatabase,
+        id: MacroCallId,
+        tt: &tt::Subtree,
+    ) -> Result<tt::Subtree, mbe::ExpandError> {
+        match self {
+            TokenExpander::MacroRules(it) => it.expand(tt),
+            TokenExpander::Builtin(it) => it.expand(db, id, tt),
+        }
+    }
+
+    pub fn shift(&self) -> u32 {
+        match self {
+            TokenExpander::MacroRules(it) => it.shift(),
+            TokenExpander::Builtin(_) => 0,
+        }
+    }
+}
+
 // FIXME: rename to ExpandDatabase
 #[salsa::query_group(AstDatabaseStorage)]
 pub trait AstDatabase: SourceDatabase {
@@ -24,7 +51,7 @@ pub trait AstDatabase: SourceDatabase {
     #[salsa::interned]
     fn intern_macro(&self, macro_call: MacroCallLoc) -> MacroCallId;
     fn macro_arg(&self, id: MacroCallId) -> Option<Arc<(tt::Subtree, mbe::TokenMap)>>;
-    fn macro_def(&self, id: MacroDefId) -> Option<Arc<(mbe::MacroRules, mbe::TokenMap)>>;
+    fn macro_def(&self, id: MacroDefId) -> Option<Arc<(TokenExpander, mbe::TokenMap)>>;
     fn parse_macro(
         &self,
         macro_file: MacroFile,
@@ -41,18 +68,25 @@ pub(crate) fn ast_id_map(db: &dyn AstDatabase, file_id: HirFileId) -> Arc<AstIdM
 pub(crate) fn macro_def(
     db: &dyn AstDatabase,
     id: MacroDefId,
-) -> Option<Arc<(mbe::MacroRules, mbe::TokenMap)>> {
-    let macro_call = id.ast_id.to_node(db);
-    let arg = macro_call.token_tree()?;
-    let (tt, tmap) = mbe::ast_to_token_tree(&arg).or_else(|| {
-        log::warn!("fail on macro_def to token tree: {:#?}", arg);
-        None
-    })?;
-    let rules = MacroRules::parse(&tt).ok().or_else(|| {
-        log::warn!("fail on macro_def parse: {:#?}", tt);
-        None
-    })?;
-    Some(Arc::new((rules, tmap)))
+) -> Option<Arc<(TokenExpander, mbe::TokenMap)>> {
+    match id.kind {
+        MacroDefKind::Declarative => {
+            let macro_call = id.ast_id.to_node(db);
+            let arg = macro_call.token_tree()?;
+            let (tt, tmap) = mbe::ast_to_token_tree(&arg).or_else(|| {
+                log::warn!("fail on macro_def to token tree: {:#?}", arg);
+                None
+            })?;
+            let rules = MacroRules::parse(&tt).ok().or_else(|| {
+                log::warn!("fail on macro_def parse: {:#?}", tt);
+                None
+            })?;
+            Some(Arc::new((TokenExpander::MacroRules(rules), tmap)))
+        }
+        MacroDefKind::BuiltIn(expander) => {
+            Some(Arc::new((TokenExpander::Builtin(expander.clone()), mbe::TokenMap::default())))
+        }
+    }
 }
 
 pub(crate) fn macro_arg(
@@ -74,7 +108,7 @@ pub(crate) fn macro_expand(
     let macro_arg = db.macro_arg(id).ok_or("Fail to args in to tt::TokenTree")?;
 
     let macro_rules = db.macro_def(loc.def).ok_or("Fail to find macro definition")?;
-    let tt = macro_rules.0.expand(&macro_arg.0).map_err(|err| format!("{:?}", err))?;
+    let tt = macro_rules.0.expand(db, id, &macro_arg.0).map_err(|err| format!("{:?}", err))?;
     // Set a hard limit for the expanded tt
     let count = tt.count();
     if count > 65536 {
diff --git a/crates/ra_hir_expand/src/hygiene.rs b/crates/ra_hir_expand/src/hygiene.rs
index 77428ec99..379562a2c 100644
--- a/crates/ra_hir_expand/src/hygiene.rs
+++ b/crates/ra_hir_expand/src/hygiene.rs
@@ -9,7 +9,7 @@ use crate::{
     db::AstDatabase,
     either::Either,
     name::{AsName, Name},
-    HirFileId, HirFileIdRepr,
+    HirFileId, HirFileIdRepr, MacroDefKind,
 };
 
 #[derive(Debug)]
@@ -24,7 +24,10 @@ impl Hygiene {
             HirFileIdRepr::FileId(_) => None,
             HirFileIdRepr::MacroFile(macro_file) => {
                 let loc = db.lookup_intern_macro(macro_file.macro_call_id);
-                Some(loc.def.krate)
+                match loc.def.kind {
+                    MacroDefKind::Declarative => Some(loc.def.krate),
+                    MacroDefKind::BuiltIn(_) => None,
+                }
             }
         };
         Hygiene { def_crate }
diff --git a/crates/ra_hir_expand/src/lib.rs b/crates/ra_hir_expand/src/lib.rs
index 151d1d785..c6ffa2c6f 100644
--- a/crates/ra_hir_expand/src/lib.rs
+++ b/crates/ra_hir_expand/src/lib.rs
@@ -10,6 +10,8 @@ pub mod either;
 pub mod name;
 pub mod hygiene;
 pub mod diagnostics;
+pub mod builtin_macro;
+pub mod quote;
 
 use std::hash::{Hash, Hasher};
 use std::sync::Arc;
@@ -21,6 +23,7 @@ use ra_syntax::{
 };
 
 use crate::ast_id_map::FileAstId;
+use crate::builtin_macro::BuiltinExpander;
 
 /// Input to the analyzer is a set of files, where each file is identified by
 /// `FileId` and contains source code. However, another source of source code in
@@ -122,6 +125,13 @@ impl salsa::InternKey for MacroCallId {
 pub struct MacroDefId {
     pub krate: CrateId,
     pub ast_id: AstId<ast::MacroCall>,
+    pub kind: MacroDefKind,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum MacroDefKind {
+    Declarative,
+    BuiltIn(BuiltinExpander),
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
@@ -144,7 +154,7 @@ pub struct ExpansionInfo {
     pub(crate) def_start: (HirFileId, TextUnit),
     pub(crate) shift: u32,
 
-    pub(crate) macro_def: Arc<(mbe::MacroRules, mbe::TokenMap)>,
+    pub(crate) macro_def: Arc<(db::TokenExpander, mbe::TokenMap)>,
     pub(crate) macro_arg: Arc<(tt::Subtree, mbe::TokenMap)>,
     pub(crate) exp_map: Arc<mbe::RevTokenMap>,
 }
diff --git a/crates/ra_hir_expand/src/name.rs b/crates/ra_hir_expand/src/name.rs
index 720896ee8..1bf17d12b 100644
--- a/crates/ra_hir_expand/src/name.rs
+++ b/crates/ra_hir_expand/src/name.rs
@@ -140,3 +140,6 @@ pub const RESULT_TYPE: Name = Name::new_inline_ascii(6, b"Result");
 pub const OUTPUT_TYPE: Name = Name::new_inline_ascii(6, b"Output");
 pub const TARGET_TYPE: Name = Name::new_inline_ascii(6, b"Target");
 pub const BOX_TYPE: Name = Name::new_inline_ascii(3, b"Box");
+
+// Builtin Macros
+pub const LINE_MACRO: Name = Name::new_inline_ascii(4, b"line");
diff --git a/crates/ra_hir_expand/src/quote.rs b/crates/ra_hir_expand/src/quote.rs
new file mode 100644
index 000000000..9cd17f0e3
--- /dev/null
+++ b/crates/ra_hir_expand/src/quote.rs
@@ -0,0 +1,261 @@
+//! A simplified version of quote-crate like quasi quote macro
+
+// A helper macro quote macro
+// FIXME:
+// 1. Not all puncts are handled
+// 2. #()* pattern repetition not supported now
+//    * But we can do it manually, see `test_quote_derive_copy_hack`
+#[doc(hidden)]
+#[macro_export]
+macro_rules! __quote {
+    () => {
+        Vec::<tt::TokenTree>::new()
+    };
+
+    ( @SUBTREE $delim:ident $($tt:tt)* ) => {
+        {
+            let children = $crate::__quote!($($tt)*);
+            let subtree = tt::Subtree {
+                delimiter: tt::Delimiter::$delim,
+                token_trees: $crate::quote::IntoTt::to_tokens(children),
+            };
+            subtree
+        }
+    };
+
+    ( @PUNCT $first:literal ) => {
+        {
+            vec![
+                tt::Leaf::Punct(tt::Punct {
+                    char: $first,
+                    spacing: tt::Spacing::Alone,
+                }).into()
+            ]
+        }
+    };
+
+    ( @PUNCT $first:literal, $sec:literal ) => {
+        {
+            vec![
+                tt::Leaf::Punct(tt::Punct {
+                    char: $first,
+                    spacing: tt::Spacing::Joint,
+                }).into(),
+                tt::Leaf::Punct(tt::Punct {
+                    char: $sec,
+                    spacing: tt::Spacing::Alone,
+                }).into()
+            ]
+        }
+    };
+
+    // hash variable
+    ( # $first:ident $($tail:tt)* ) => {
+        {
+            let token = $crate::quote::ToTokenTree::to_token($first);
+            let mut tokens = vec![token.into()];
+            let mut tail_tokens = $crate::quote::IntoTt::to_tokens($crate::__quote!($($tail)*));
+            tokens.append(&mut tail_tokens);
+            tokens
+        }
+    };
+
+    // Brace
+    ( { $($tt:tt)* } ) => { $crate::__quote!(@SUBTREE Brace $($tt)*) };
+    // Bracket
+    ( [ $($tt:tt)* ] ) => { $crate::__quote!(@SUBTREE Bracket $($tt)*) };
+    // Parenthesis
+    ( ( $($tt:tt)* ) ) => { $crate::__quote!(@SUBTREE Parenthesis $($tt)*) };
+
+    // Literal
+    ( $tt:literal ) => { vec![$crate::quote::ToTokenTree::to_token($tt).into()] };
+    // Ident
+    ( $tt:ident ) => {
+        vec![ {
+            tt::Leaf::Ident(tt::Ident {
+                text: stringify!($tt).into(),
+                id: tt::TokenId::unspecified(),
+            }).into()
+        }]
+    };
+
+    // Puncts
+    // FIXME: Not all puncts are handled
+    ( -> ) => {$crate::__quote!(@PUNCT '-', '>')};
+    ( & ) => {$crate::__quote!(@PUNCT '&')};
+    ( , ) => {$crate::__quote!(@PUNCT ',')};
+    ( : ) => {$crate::__quote!(@PUNCT ':')};
+    ( . ) => {$crate::__quote!(@PUNCT '.')};
+
+    ( $first:tt $($tail:tt)+ ) => {
+        {
+            let mut tokens = $crate::quote::IntoTt::to_tokens($crate::__quote!($first));
+            let mut tail_tokens = $crate::quote::IntoTt::to_tokens($crate::__quote!($($tail)*));
+
+            tokens.append(&mut tail_tokens);
+            tokens
+        }
+    };
+}
+
+/// FIXME:
+/// It probably should implement in proc-macro
+#[macro_export]
+macro_rules! quote {
+    ( $($tt:tt)* ) => {
+        $crate::quote::IntoTt::to_subtree($crate::__quote!($($tt)*))
+    }
+}
+
+pub(crate) trait IntoTt {
+    fn to_subtree(self) -> tt::Subtree;
+    fn to_tokens(self) -> Vec<tt::TokenTree>;
+}
+
+impl IntoTt for Vec<tt::TokenTree> {
+    fn to_subtree(self) -> tt::Subtree {
+        tt::Subtree { delimiter: tt::Delimiter::None, token_trees: self }
+    }
+
+    fn to_tokens(self) -> Vec<tt::TokenTree> {
+        self
+    }
+}
+
+impl IntoTt for tt::Subtree {
+    fn to_subtree(self) -> tt::Subtree {
+        self
+    }
+
+    fn to_tokens(self) -> Vec<tt::TokenTree> {
+        vec![tt::TokenTree::Subtree(self)]
+    }
+}
+
+pub(crate) trait ToTokenTree {
+    fn to_token(self) -> tt::TokenTree;
+}
+
+impl ToTokenTree for tt::TokenTree {
+    fn to_token(self) -> tt::TokenTree {
+        self
+    }
+}
+
+impl ToTokenTree for tt::Subtree {
+    fn to_token(self) -> tt::TokenTree {
+        self.into()
+    }
+}
+
+macro_rules! impl_to_to_tokentrees {
+    ($($ty:ty => $this:ident $im:block);*) => {
+        $(
+            impl ToTokenTree for $ty {
+                fn to_token($this) -> tt::TokenTree {
+                    let leaf: tt::Leaf = $im.into();
+                    leaf.into()
+                }
+            }
+
+            impl ToTokenTree for &$ty {
+                fn to_token($this) -> tt::TokenTree {
+                    let leaf: tt::Leaf = $im.clone().into();
+                    leaf.into()
+                }
+            }
+        )*
+    }
+}
+
+impl_to_to_tokentrees! {
+    u32 => self { tt::Literal{text: self.to_string().into()} };
+    usize => self { tt::Literal{text: self.to_string().into()}};
+    i32 => self { tt::Literal{text: self.to_string().into()}};
+    &str => self { tt::Literal{text: self.to_string().into()}};
+    String => self { tt::Literal{text: self.into()}};
+    tt::Leaf => self { self };
+    tt::Literal => self { self };
+    tt::Ident => self { self };
+    tt::Punct => self { self }
+}
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn test_quote_delimiters() {
+        assert_eq!(quote!({}).to_string(), "{}");
+        assert_eq!(quote!(()).to_string(), "()");
+        assert_eq!(quote!([]).to_string(), "[]");
+    }
+
+    #[test]
+    fn test_quote_idents() {
+        assert_eq!(quote!(32).to_string(), "32");
+        assert_eq!(quote!(struct).to_string(), "struct");
+    }
+
+    #[test]
+    fn test_quote_hash_simple_literal() {
+        let a = 20;
+        assert_eq!(quote!(#a).to_string(), "20");
+        let s: String = "hello".into();
+        assert_eq!(quote!(#s).to_string(), "hello");
+    }
+
+    fn mk_ident(name: &str) -> tt::Ident {
+        tt::Ident { text: name.into(), id: tt::TokenId::unspecified() }
+    }
+
+    #[test]
+    fn test_quote_hash_token_tree() {
+        let a = mk_ident("hello");
+
+        let quoted = quote!(#a);
+        assert_eq!(quoted.to_string(), "hello");
+        let t = format!("{:?}", quoted);
+        assert_eq!(t, "Subtree { delimiter: None, token_trees: [Leaf(Ident(Ident { text: \"hello\", id: TokenId(4294967295) }))] }");
+    }
+
+    #[test]
+    fn test_quote_simple_derive_copy() {
+        let name = mk_ident("Foo");
+
+        let quoted = quote! {
+            impl Clone for #name {
+                fn clone(&self) -> Self {
+                    Self {}
+                }
+            }
+        };
+
+        assert_eq!(quoted.to_string(), "impl Clone for Foo {fn clone (& self) -> Self {Self {}}}");
+    }
+
+    #[test]
+    fn test_quote_derive_copy_hack() {
+        // Assume the given struct is:
+        // struct Foo {
+        //  name: String,
+        //  id: u32,
+        // }
+        let struct_name = mk_ident("Foo");
+        let fields = [mk_ident("name"), mk_ident("id")];
+        let fields = fields
+            .into_iter()
+            .map(|it| quote!(#it: self.#it.clone(), ).token_trees.clone())
+            .flatten();
+
+        let list = tt::Subtree { delimiter: tt::Delimiter::Brace, token_trees: fields.collect() };
+
+        let quoted = quote! {
+            impl Clone for #struct_name {
+                fn clone(&self) -> Self {
+                    Self #list
+                }
+            }
+        };
+
+        assert_eq!(quoted.to_string(), "impl Clone for Foo {fn clone (& self) -> Self {Self {name : self . name . clone () , id : self . id . clone () ,}}}");
+    }
+}