aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorbors[bot] <26634292+bors[bot]@users.noreply.github.com>2021-03-07 00:32:56 +0000
committerGitHub <[email protected]>2021-03-07 00:32:56 +0000
commitf0b7c02f16f717744e7edc79a405db14110393cf (patch)
treec0aaf8373aa42ce3b4037fad2f42efba07680f53
parent07a54f7ae451451292e3282f1e7defb4391b766f (diff)
parentaea974939064b0f7b83de371a93ee4190c80e544 (diff)
Merge #7892
7892: Fix TokenStream::from_str for input consisting of a single group with delimiter r=edwin0cheng a=kevinmehall TokenStream holds a `tt::Subtree` but assumes its `delimiter` is always `None`. In particular, the iterator implementation iterates over the inner `token_trees` and ignores the `delimiter`. However, `TokenStream::from_str` violated this assumption when the input consists of a single group by producing a Subtree with an outer delimiter, which was ignored as seen by a procedural macro. `tt::Subtree` is just `pub delimiter: Option<Delimiter>, pub token_trees: Vec<TokenTree>`, so a Subtree that is statically guaranteed not to have a delimiter is just `Vec<TokenTree>`. Fixes #7810 Fixes #7875 Co-authored-by: Kevin Mehall <[email protected]>
-rw-r--r--crates/proc_macro_srv/src/dylib.rs6
-rw-r--r--crates/proc_macro_srv/src/proc_macro/bridge/client.rs2
-rw-r--r--crates/proc_macro_srv/src/rustc_server.rs141
-rw-r--r--crates/proc_macro_srv/src/tests/utils.rs2
4 files changed, 89 insertions, 62 deletions
diff --git a/crates/proc_macro_srv/src/dylib.rs b/crates/proc_macro_srv/src/dylib.rs
index 28a6ee547..baf10fea9 100644
--- a/crates/proc_macro_srv/src/dylib.rs
+++ b/crates/proc_macro_srv/src/dylib.rs
@@ -138,7 +138,7 @@ impl Expander {
138 parsed_body, 138 parsed_body,
139 false, 139 false,
140 ); 140 );
141 return res.map(|it| it.subtree); 141 return res.map(|it| it.into_subtree());
142 } 142 }
143 bridge::client::ProcMacro::Bang { name, client } if *name == macro_name => { 143 bridge::client::ProcMacro::Bang { name, client } if *name == macro_name => {
144 let res = client.run( 144 let res = client.run(
@@ -147,7 +147,7 @@ impl Expander {
147 parsed_body, 147 parsed_body,
148 false, 148 false,
149 ); 149 );
150 return res.map(|it| it.subtree); 150 return res.map(|it| it.into_subtree());
151 } 151 }
152 bridge::client::ProcMacro::Attr { name, client } if *name == macro_name => { 152 bridge::client::ProcMacro::Attr { name, client } if *name == macro_name => {
153 let res = client.run( 153 let res = client.run(
@@ -157,7 +157,7 @@ impl Expander {
157 parsed_body, 157 parsed_body,
158 false, 158 false,
159 ); 159 );
160 return res.map(|it| it.subtree); 160 return res.map(|it| it.into_subtree());
161 } 161 }
162 _ => continue, 162 _ => continue,
163 } 163 }
diff --git a/crates/proc_macro_srv/src/proc_macro/bridge/client.rs b/crates/proc_macro_srv/src/proc_macro/bridge/client.rs
index ca6749b9b..b036d4e20 100644
--- a/crates/proc_macro_srv/src/proc_macro/bridge/client.rs
+++ b/crates/proc_macro_srv/src/proc_macro/bridge/client.rs
@@ -238,7 +238,7 @@ macro_rules! define_client_side {
238 $(impl $name { 238 $(impl $name {
239 #[allow(unused)] 239 #[allow(unused)]
240 $(pub(crate) fn $method($($arg: $arg_ty),*) $(-> $ret_ty)* { 240 $(pub(crate) fn $method($($arg: $arg_ty),*) $(-> $ret_ty)* {
241 panic!("hello"); 241 panic!("crates should be linked against the sysroot version of proc_macro, not this one from rust-analyzer");
242 // Bridge::with(|bridge| { 242 // Bridge::with(|bridge| {
243 // let mut b = bridge.cached_buffer.take(); 243 // let mut b = bridge.cached_buffer.take();
244 244
diff --git a/crates/proc_macro_srv/src/rustc_server.rs b/crates/proc_macro_srv/src/rustc_server.rs
index 14c853c77..ceefd187d 100644
--- a/crates/proc_macro_srv/src/rustc_server.rs
+++ b/crates/proc_macro_srv/src/rustc_server.rs
@@ -25,27 +25,35 @@ type Span = tt::TokenId;
25 25
26#[derive(Debug, Clone)] 26#[derive(Debug, Clone)]
27pub struct TokenStream { 27pub struct TokenStream {
28 pub subtree: tt::Subtree, 28 pub token_trees: Vec<TokenTree>,
29} 29}
30 30
31impl TokenStream { 31impl TokenStream {
32 pub fn new() -> Self { 32 pub fn new() -> Self {
33 TokenStream { subtree: Default::default() } 33 TokenStream { token_trees: Default::default() }
34 } 34 }
35 35
36 pub fn with_subtree(subtree: tt::Subtree) -> Self { 36 pub fn with_subtree(subtree: tt::Subtree) -> Self {
37 TokenStream { subtree } 37 if subtree.delimiter.is_some() {
38 TokenStream { token_trees: vec![TokenTree::Subtree(subtree)] }
39 } else {
40 TokenStream { token_trees: subtree.token_trees }
41 }
42 }
43
44 pub fn into_subtree(self) -> tt::Subtree {
45 tt::Subtree { delimiter: None, token_trees: self.token_trees }
38 } 46 }
39 47
40 pub fn is_empty(&self) -> bool { 48 pub fn is_empty(&self) -> bool {
41 self.subtree.token_trees.is_empty() 49 self.token_trees.is_empty()
42 } 50 }
43} 51}
44 52
45/// Creates a token stream containing a single token tree. 53/// Creates a token stream containing a single token tree.
46impl From<TokenTree> for TokenStream { 54impl From<TokenTree> for TokenStream {
47 fn from(tree: TokenTree) -> TokenStream { 55 fn from(tree: TokenTree) -> TokenStream {
48 TokenStream { subtree: tt::Subtree { delimiter: None, token_trees: vec![tree] } } 56 TokenStream { token_trees: vec![tree] }
49 } 57 }
50} 58}
51 59
@@ -78,10 +86,10 @@ impl Extend<TokenStream> for TokenStream {
78 for tkn in item { 86 for tkn in item {
79 match tkn { 87 match tkn {
80 tt::TokenTree::Subtree(subtree) if subtree.delimiter.is_none() => { 88 tt::TokenTree::Subtree(subtree) if subtree.delimiter.is_none() => {
81 self.subtree.token_trees.extend(subtree.token_trees); 89 self.token_trees.extend(subtree.token_trees);
82 } 90 }
83 _ => { 91 _ => {
84 self.subtree.token_trees.push(tkn); 92 self.token_trees.push(tkn);
85 } 93 }
86 } 94 }
87 } 95 }
@@ -164,7 +172,7 @@ pub mod token_stream {
164 type IntoIter = super::IntoIter<TokenTree>; 172 type IntoIter = super::IntoIter<TokenTree>;
165 173
166 fn into_iter(self) -> Self::IntoIter { 174 fn into_iter(self) -> Self::IntoIter {
167 self.subtree.token_trees.into_iter() 175 self.token_trees.into_iter()
168 } 176 }
169 } 177 }
170 178
@@ -185,28 +193,22 @@ pub mod token_stream {
185 mbe::parse_to_token_tree(src).ok_or("Failed to parse from mbe")?; 193 mbe::parse_to_token_tree(src).ok_or("Failed to parse from mbe")?;
186 194
187 let subtree = subtree_replace_token_ids_with_unspecified(subtree); 195 let subtree = subtree_replace_token_ids_with_unspecified(subtree);
188 Ok(TokenStream { subtree }) 196 Ok(TokenStream::with_subtree(subtree))
189 } 197 }
190 } 198 }
191 199
192 impl ToString for TokenStream { 200 impl ToString for TokenStream {
193 fn to_string(&self) -> String { 201 fn to_string(&self) -> String {
194 let tt = self.subtree.clone().into(); 202 return tokentrees_to_text(&self.token_trees[..]);
195 to_text(&tt)
196 }
197 }
198 203
199 fn to_text(tkn: &tt::TokenTree) -> String { 204 fn tokentrees_to_text(tkns: &[tt::TokenTree]) -> String {
200 match tkn { 205 tkns.iter()
201 tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => ident.text.clone().into(),
202 tt::TokenTree::Leaf(tt::Leaf::Literal(literal)) => literal.text.clone().into(),
203 tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => format!("{}", punct.char),
204 tt::TokenTree::Subtree(subtree) => {
205 let content = subtree
206 .token_trees
207 .iter()
208 .fold((String::new(), true), |(last, last_to_joint), tkn| { 206 .fold((String::new(), true), |(last, last_to_joint), tkn| {
209 let s = [last, to_text(tkn)].join(if last_to_joint { "" } else { " " }); 207 let s = [last, tokentree_to_text(tkn)].join(if last_to_joint {
208 ""
209 } else {
210 " "
211 });
210 let mut is_joint = false; 212 let mut is_joint = false;
211 if let tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) = tkn { 213 if let tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) = tkn {
212 if punct.spacing == tt::Spacing::Joint { 214 if punct.spacing == tt::Spacing::Joint {
@@ -215,15 +217,25 @@ pub mod token_stream {
215 } 217 }
216 (s, is_joint) 218 (s, is_joint)
217 }) 219 })
218 .0; 220 .0
219 221 }
220 let (open, close) = match subtree.delimiter.map(|it| it.kind) { 222
221 None => ("", ""), 223 fn tokentree_to_text(tkn: &tt::TokenTree) -> String {
222 Some(tt::DelimiterKind::Brace) => ("{", "}"), 224 match tkn {
223 Some(tt::DelimiterKind::Parenthesis) => ("(", ")"), 225 tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => ident.text.clone().into(),
224 Some(tt::DelimiterKind::Bracket) => ("[", "]"), 226 tt::TokenTree::Leaf(tt::Leaf::Literal(literal)) => literal.text.clone().into(),
225 }; 227 tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => format!("{}", punct.char),
226 format!("{}{}{}", open, content, close) 228 tt::TokenTree::Subtree(subtree) => {
229 let content = tokentrees_to_text(&subtree.token_trees);
230 let (open, close) = match subtree.delimiter.map(|it| it.kind) {
231 None => ("", ""),
232 Some(tt::DelimiterKind::Brace) => ("{", "}"),
233 Some(tt::DelimiterKind::Parenthesis) => ("(", ")"),
234 Some(tt::DelimiterKind::Bracket) => ("[", "]"),
235 };
236 format!("{}{}{}", open, content, close)
237 }
238 }
227 } 239 }
228 } 240 }
229 } 241 }
@@ -433,10 +445,7 @@ fn spacing_to_external(spacing: Spacing) -> bridge::Spacing {
433 445
434impl server::Group for Rustc { 446impl server::Group for Rustc {
435 fn new(&mut self, delimiter: bridge::Delimiter, stream: Self::TokenStream) -> Self::Group { 447 fn new(&mut self, delimiter: bridge::Delimiter, stream: Self::TokenStream) -> Self::Group {
436 Self::Group { 448 Self::Group { delimiter: delim_to_internal(delimiter), token_trees: stream.token_trees }
437 delimiter: delim_to_internal(delimiter),
438 token_trees: stream.subtree.token_trees,
439 }
440 } 449 }
441 fn delimiter(&mut self, group: &Self::Group) -> bridge::Delimiter { 450 fn delimiter(&mut self, group: &Self::Group) -> bridge::Delimiter {
442 delim_to_external(group.delimiter) 451 delim_to_external(group.delimiter)
@@ -444,9 +453,7 @@ impl server::Group for Rustc {
444 453
445 // NOTE: Return value of do not include delimiter 454 // NOTE: Return value of do not include delimiter
446 fn stream(&mut self, group: &Self::Group) -> Self::TokenStream { 455 fn stream(&mut self, group: &Self::Group) -> Self::TokenStream {
447 TokenStream { 456 TokenStream { token_trees: group.token_trees.clone() }
448 subtree: tt::Subtree { delimiter: None, token_trees: group.token_trees.clone() },
449 }
450 } 457 }
451 458
452 fn span(&mut self, group: &Self::Group) -> Self::Span { 459 fn span(&mut self, group: &Self::Group) -> Self::Span {
@@ -755,28 +762,48 @@ mod tests {
755 #[test] 762 #[test]
756 fn test_rustc_server_to_string() { 763 fn test_rustc_server_to_string() {
757 let s = TokenStream { 764 let s = TokenStream {
758 subtree: tt::Subtree { 765 token_trees: vec![
759 delimiter: None, 766 tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
760 token_trees: vec![ 767 text: "struct".into(),
761 tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { 768 id: tt::TokenId::unspecified(),
762 text: "struct".into(), 769 })),
763 id: tt::TokenId::unspecified(), 770 tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
764 })), 771 text: "T".into(),
765 tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { 772 id: tt::TokenId::unspecified(),
766 text: "T".into(), 773 })),
774 tt::TokenTree::Subtree(tt::Subtree {
775 delimiter: Some(tt::Delimiter {
767 id: tt::TokenId::unspecified(), 776 id: tt::TokenId::unspecified(),
768 })), 777 kind: tt::DelimiterKind::Brace,
769 tt::TokenTree::Subtree(tt::Subtree {
770 delimiter: Some(tt::Delimiter {
771 id: tt::TokenId::unspecified(),
772 kind: tt::DelimiterKind::Brace,
773 }),
774 token_trees: vec![],
775 }), 778 }),
776 ], 779 token_trees: vec![],
777 }, 780 }),
781 ],
778 }; 782 };
779 783
780 assert_eq!(s.to_string(), "struct T {}"); 784 assert_eq!(s.to_string(), "struct T {}");
781 } 785 }
786
787 #[test]
788 fn test_rustc_server_from_str() {
789 use std::str::FromStr;
790 let subtree_paren_a = tt::TokenTree::Subtree(tt::Subtree {
791 delimiter: Some(tt::Delimiter {
792 id: tt::TokenId::unspecified(),
793 kind: tt::DelimiterKind::Parenthesis,
794 }),
795 token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
796 text: "a".into(),
797 id: tt::TokenId::unspecified(),
798 }))],
799 });
800
801 let t1 = TokenStream::from_str("(a)").unwrap();
802 assert_eq!(t1.token_trees.len(), 1);
803 assert_eq!(t1.token_trees[0], subtree_paren_a);
804
805 let t2 = TokenStream::from_str("(a);").unwrap();
806 assert_eq!(t2.token_trees.len(), 2);
807 assert_eq!(t2.token_trees[0], subtree_paren_a);
808 }
782} 809}
diff --git a/crates/proc_macro_srv/src/tests/utils.rs b/crates/proc_macro_srv/src/tests/utils.rs
index 22813052d..0484c3af4 100644
--- a/crates/proc_macro_srv/src/tests/utils.rs
+++ b/crates/proc_macro_srv/src/tests/utils.rs
@@ -52,7 +52,7 @@ pub fn assert_expand(
52 let expander = dylib::Expander::new(&path).unwrap(); 52 let expander = dylib::Expander::new(&path).unwrap();
53 let fixture = parse_string(ra_fixture).unwrap(); 53 let fixture = parse_string(ra_fixture).unwrap();
54 54
55 let res = expander.expand(macro_name, &fixture.subtree, None).unwrap(); 55 let res = expander.expand(macro_name, &fixture.into_subtree(), None).unwrap();
56 assert_eq_text!(&expect.trim(), &format!("{:?}", res)); 56 assert_eq_text!(&expect.trim(), &format!("{:?}", res));
57} 57}
58 58