Fix TokenStream::from_str for input consisting of a single Group

TokenStream holds a `tt::Subtree` but assumes its `delimiter` is always
`None`. In particular, the iterator implementation iterates over the
inner `token_trees` and ignores the `delimiter`.

However, `TokenStream::from_str` violated this assumption when the input
consists of a single Group by producing a Subtree with an outer
delimiter, which was ignored as seen by a procedural macro.

In this case, wrap an extra level of Subtree around it.

Fixes #7810
Fixes #7875

1 files changed, 34 insertions, 2 deletions

diff --git a/crates/proc_macro_srv/src/rustc_server.rs b/crates/proc_macro_srv/src/rustc_server.rs
index 14c853c77..f02b0af60 100644
--- a/crates/proc_macro_srv/src/rustc_server.rs
+++ b/crates/proc_macro_srv/src/rustc_server.rs
@@ -34,7 +34,16 @@ impl TokenStream {
     }
 
     pub fn with_subtree(subtree: tt::Subtree) -> Self {
-        TokenStream { subtree }
+        if subtree.delimiter.is_some() {
+            TokenStream {
+                subtree: tt::Subtree {
+                    token_trees: vec![TokenTree::Subtree(subtree)],
+                    delimiter: None,
+                },
+            }
+        } else {
+            TokenStream { subtree }
+        }
     }
 
     pub fn is_empty(&self) -> bool {
@@ -185,7 +194,7 @@ pub mod token_stream {
                 mbe::parse_to_token_tree(src).ok_or("Failed to parse from mbe")?;
 
             let subtree = subtree_replace_token_ids_with_unspecified(subtree);
-            Ok(TokenStream { subtree })
+            Ok(TokenStream::with_subtree(subtree))
         }
     }
 
@@ -779,4 +788,27 @@ mod tests {
 
         assert_eq!(s.to_string(), "struct T {}");
     }
+
+    #[test]
+    fn test_rustc_server_from_str() {
+        use std::str::FromStr;
+        let subtree_paren_a = tt::TokenTree::Subtree(tt::Subtree {
+            delimiter: Some(tt::Delimiter {
+                id: tt::TokenId::unspecified(),
+                kind: tt::DelimiterKind::Parenthesis,
+            }),
+            token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
+                text: "a".into(),
+                id: tt::TokenId::unspecified(),
+            }))],
+        });
+
+        let t1 = TokenStream::from_str("(a)").unwrap();
+        assert_eq!(t1.subtree.token_trees.len(), 1);
+        assert_eq!(t1.subtree.token_trees[0], subtree_paren_a);
+
+        let t2 = TokenStream::from_str("(a);").unwrap();
+        assert_eq!(t2.subtree.token_trees.len(), 2);
+        assert_eq!(t2.subtree.token_trees[0], subtree_paren_a);
+    }
 }