Merge #8796

8796: internal: rewrite `#[derive]` removal to be based on AST (take 2) r=jonas-schievink a=jonas-schievink

Second attempt of https://github.com/rust-analyzer/rust-analyzer/pull/8443, this uses syntactical attribute offsets in `hir_expand`, and changes `attr.rs` to make those easy to derive.

This will make it easy to add similar attribute removal for attribute macros, unblocking them.

Co-authored-by: Jonas Schievink <[email protected]>

5 files changed, 114 insertions, 112 deletions

diff --git a/crates/hir_expand/src/builtin_derive.rs b/crates/hir_expand/src/builtin_derive.rs
index 537c03028..b6a6d602f 100644
--- a/crates/hir_expand/src/builtin_derive.rs
+++ b/crates/hir_expand/src/builtin_derive.rs
@@ -269,7 +269,7 @@ mod tests {
     use expect_test::{expect, Expect};
     use name::AsName;
 
-    use crate::{test_db::TestDB, AstId, AttrId, MacroCallId, MacroCallKind, MacroCallLoc};
+    use crate::{test_db::TestDB, AstId, MacroCallId, MacroCallKind, MacroCallLoc};
 
     use super::*;
 
@@ -320,7 +320,7 @@ $0
             kind: MacroCallKind::Derive {
                 ast_id,
                 derive_name: name.to_string(),
-                derive_attr: AttrId(0),
+                derive_attr_index: 0,
             },
         };
 
diff --git a/crates/hir_expand/src/db.rs b/crates/hir_expand/src/db.rs
index 6647e57e7..9fa419fcf 100644
--- a/crates/hir_expand/src/db.rs
+++ b/crates/hir_expand/src/db.rs
@@ -12,9 +12,9 @@ use syntax::{
 };
 
 use crate::{
-    ast_id_map::AstIdMap, hygiene::HygieneFrame, BuiltinDeriveExpander, BuiltinFnLikeExpander,
-    EagerCallLoc, EagerMacroId, HirFileId, HirFileIdRepr, LazyMacroId, MacroCallId, MacroCallLoc,
-    MacroDefId, MacroDefKind, MacroFile, ProcMacroExpander,
+    ast_id_map::AstIdMap, hygiene::HygieneFrame, input::process_macro_input, BuiltinDeriveExpander,
+    BuiltinFnLikeExpander, EagerCallLoc, EagerMacroId, HirFileId, HirFileIdRepr, LazyMacroId,
+    MacroCallId, MacroCallLoc, MacroDefId, MacroDefKind, MacroFile, ProcMacroExpander,
 };
 
 /// Total limit on the number of tokens produced by any macro invocation.
@@ -281,6 +281,7 @@ fn macro_arg_text(db: &dyn AstDatabase, id: MacroCallId) -> Option<GreenNode> {
     };
     let loc = db.lookup_intern_macro(id);
     let arg = loc.kind.arg(db)?;
+    let arg = process_macro_input(db, arg, id);
     Some(arg.green().into())
 }
 
diff --git a/crates/hir_expand/src/input.rs b/crates/hir_expand/src/input.rs
new file mode 100644
index 000000000..112216859
--- /dev/null
+++ b/crates/hir_expand/src/input.rs
@@ -0,0 +1,94 @@
+//! Macro input conditioning.
+
+use syntax::{
+    ast::{self, AttrsOwner},
+    AstNode, SyntaxNode,
+};
+
+use crate::{
+    db::AstDatabase,
+    name::{name, AsName},
+    LazyMacroId, MacroCallKind, MacroCallLoc,
+};
+
+pub(crate) fn process_macro_input(
+    db: &dyn AstDatabase,
+    node: SyntaxNode,
+    id: LazyMacroId,
+) -> SyntaxNode {
+    let loc: MacroCallLoc = db.lookup_intern_macro(id);
+
+    match loc.kind {
+        MacroCallKind::FnLike { .. } => node,
+        MacroCallKind::Derive { derive_attr_index, .. } => {
+            let item = match ast::Item::cast(node.clone()) {
+                Some(item) => item,
+                None => return node,
+            };
+
+            remove_derives_up_to(item, derive_attr_index as usize).syntax().clone()
+        }
+    }
+}
+
+/// Removes `#[derive]` attributes from `item`, up to `attr_index`.
+fn remove_derives_up_to(item: ast::Item, attr_index: usize) -> ast::Item {
+    let item = item.clone_for_update();
+    for attr in item.attrs().take(attr_index + 1) {
+        if let Some(name) =
+            attr.path().and_then(|path| path.as_single_segment()).and_then(|seg| seg.name_ref())
+        {
+            if name.as_name() == name![derive] {
+                attr.syntax().detach();
+            }
+        }
+    }
+    item
+}
+
+#[cfg(test)]
+mod tests {
+    use base_db::fixture::WithFixture;
+    use base_db::SourceDatabase;
+    use expect_test::{expect, Expect};
+
+    use crate::test_db::TestDB;
+
+    use super::*;
+
+    fn test_remove_derives_up_to(attr: usize, ra_fixture: &str, expect: Expect) {
+        let (db, file_id) = TestDB::with_single_file(&ra_fixture);
+        let parsed = db.parse(file_id);
+
+        let mut items: Vec<_> =
+            parsed.syntax_node().descendants().filter_map(ast::Item::cast).collect();
+        assert_eq!(items.len(), 1);
+
+        let item = remove_derives_up_to(items.pop().unwrap(), attr);
+        expect.assert_eq(&item.to_string());
+    }
+
+    #[test]
+    fn remove_derive() {
+        test_remove_derives_up_to(
+            2,
+            r#"
+#[allow(unused)]
+#[derive(Copy)]
+#[derive(Hello)]
+#[derive(Clone)]
+struct A {
+    bar: u32
+}
+        "#,
+            expect![[r#"
+#[allow(unused)]
+
+
+#[derive(Clone)]
+struct A {
+    bar: u32
+}"#]],
+        );
+    }
+}
diff --git a/crates/hir_expand/src/lib.rs b/crates/hir_expand/src/lib.rs
index 80ab3aeee..5df11856e 100644
--- a/crates/hir_expand/src/lib.rs
+++ b/crates/hir_expand/src/lib.rs
@@ -14,6 +14,7 @@ pub mod builtin_macro;
 pub mod proc_macro;
 pub mod quote;
 pub mod eager;
+mod input;
 
 use either::Either;
 
@@ -292,13 +293,21 @@ pub struct MacroCallLoc {
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub enum MacroCallKind {
-    FnLike { ast_id: AstId<ast::MacroCall>, fragment: FragmentKind },
-    Derive { ast_id: AstId<ast::Item>, derive_name: String, derive_attr: AttrId },
+    FnLike {
+        ast_id: AstId<ast::MacroCall>,
+        fragment: FragmentKind,
+    },
+    Derive {
+        ast_id: AstId<ast::Item>,
+        derive_name: String,
+        /// Syntactical index of the invoking `#[derive]` attribute.
+        ///
+        /// Outer attributes are counted first, then inner attributes. This does not support
+        /// out-of-line modules, which may have attributes spread across 2 files!
+        derive_attr_index: u32,
+    },
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub struct AttrId(pub u32);
-
 impl MacroCallKind {
     fn file_id(&self) -> HirFileId {
         match self {
diff --git a/crates/hir_expand/src/proc_macro.rs b/crates/hir_expand/src/proc_macro.rs
index 75e950816..d5643393a 100644
--- a/crates/hir_expand/src/proc_macro.rs
+++ b/crates/hir_expand/src/proc_macro.rs
@@ -2,7 +2,6 @@
 
 use crate::db::AstDatabase;
 use base_db::{CrateId, ProcMacroId};
-use tt::buffer::{Cursor, TokenBuffer};
 
 #[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
 pub struct ProcMacroExpander {
@@ -44,9 +43,6 @@ impl ProcMacroExpander {
                     .clone()
                     .ok_or_else(|| err!("No derive macro found."))?;
 
-                let tt = remove_derive_attrs(tt)
-                    .ok_or_else(|| err!("Fail to remove derive for custom derive"))?;
-
                 // Proc macros have access to the environment variables of the invoking crate.
                 let env = &krate_graph[calling_crate].env;
 
@@ -56,101 +52,3 @@ impl ProcMacroExpander {
         }
     }
 }
-
-fn eat_punct(cursor: &mut Cursor, c: char) -> bool {
-    if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(punct), _)) = cursor.token_tree() {
-        if punct.char == c {
-            *cursor = cursor.bump();
-            return true;
-        }
-    }
-    false
-}
-
-fn eat_subtree(cursor: &mut Cursor, kind: tt::DelimiterKind) -> bool {
-    if let Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) = cursor.token_tree() {
-        if Some(kind) == subtree.delimiter_kind() {
-            *cursor = cursor.bump_subtree();
-            return true;
-        }
-    }
-    false
-}
-
-fn eat_ident(cursor: &mut Cursor, t: &str) -> bool {
-    if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Ident(ident), _)) = cursor.token_tree() {
-        if t == ident.text.as_str() {
-            *cursor = cursor.bump();
-            return true;
-        }
-    }
-    false
-}
-
-fn remove_derive_attrs(tt: &tt::Subtree) -> Option<tt::Subtree> {
-    let buffer = TokenBuffer::from_tokens(&tt.token_trees);
-    let mut p = buffer.begin();
-    let mut result = tt::Subtree::default();
-
-    while !p.eof() {
-        let curr = p;
-
-        if eat_punct(&mut p, '#') {
-            eat_punct(&mut p, '!');
-            let parent = p;
-            if eat_subtree(&mut p, tt::DelimiterKind::Bracket) {
-                if eat_ident(&mut p, "derive") {
-                    p = parent.bump();
-                    continue;
-                }
-            }
-        }
-
-        result.token_trees.push(curr.token_tree()?.cloned());
-        p = curr.bump();
-    }
-
-    Some(result)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use test_utils::assert_eq_text;
-
-    #[test]
-    fn test_remove_derive_attrs() {
-        let tt = mbe::parse_to_token_tree(
-            r#"
-    #[allow(unused)]
-    #[derive(Copy)]
-    #[derive(Hello)]
-    struct A {
-        bar: u32
-    }
-"#,
-        )
-        .unwrap()
-        .0;
-        let result = format!("{:#?}", remove_derive_attrs(&tt).unwrap());
-
-        assert_eq_text!(
-            r#"
-SUBTREE $
-  PUNCH   # [alone] 0
-  SUBTREE [] 1
-    IDENT   allow 2
-    SUBTREE () 3
-      IDENT   unused 4
-  IDENT   struct 15
-  IDENT   A 16
-  SUBTREE {} 17
-    IDENT   bar 18
-    PUNCH   : [alone] 19
-    IDENT   u32 20
-"#
-            .trim(),
-            &result
-        );
-    }
-}