aboutsummaryrefslogtreecommitdiff
path: root/crates
diff options
context:
space:
mode:
authorbors[bot] <26634292+bors[bot]@users.noreply.github.com>2021-03-05 20:00:04 +0000
committerGitHub <[email protected]>2021-03-05 20:00:04 +0000
commit94012e24e038bc1b96e06638b6e2b38852a510eb (patch)
tree41a38fc808f26ecc9fa7b62c2466869d258e4fd1 /crates
parent7199d5b56d47378bb09aab1be718c29ae0284369 (diff)
parent83230b270455c82636147ee631c425fefdce26ff (diff)
Merge #7884
7884: Simplify TokenStream FromStr r=edwin0cheng a=edwin0cheng Make sure `FromStr` ignore all `TokenMap` in all cases. bors r+ Co-authored-by: Edwin Cheng <[email protected]>
Diffstat (limited to 'crates')
-rw-r--r--crates/proc_macro_srv/src/rustc_server.rs80
-rw-r--r--crates/proc_macro_srv/src/tests/fixtures/test_serialize_proc_macro.txt2
2 files changed, 43 insertions, 39 deletions
diff --git a/crates/proc_macro_srv/src/rustc_server.rs b/crates/proc_macro_srv/src/rustc_server.rs
index 952b4a97f..14c853c77 100644
--- a/crates/proc_macro_srv/src/rustc_server.rs
+++ b/crates/proc_macro_srv/src/rustc_server.rs
@@ -184,6 +184,7 @@ pub mod token_stream {
184 let (subtree, _token_map) = 184 let (subtree, _token_map) =
185 mbe::parse_to_token_tree(src).ok_or("Failed to parse from mbe")?; 185 mbe::parse_to_token_tree(src).ok_or("Failed to parse from mbe")?;
186 186
187 let subtree = subtree_replace_token_ids_with_unspecified(subtree);
187 Ok(TokenStream { subtree }) 188 Ok(TokenStream { subtree })
188 } 189 }
189 } 190 }
@@ -226,6 +227,44 @@ pub mod token_stream {
226 } 227 }
227 } 228 }
228 } 229 }
230
231 fn subtree_replace_token_ids_with_unspecified(subtree: tt::Subtree) -> tt::Subtree {
232 tt::Subtree {
233 delimiter: subtree
234 .delimiter
235 .map(|d| tt::Delimiter { id: tt::TokenId::unspecified(), ..d }),
236 token_trees: subtree
237 .token_trees
238 .into_iter()
239 .map(|t| token_tree_replace_token_ids_with_unspecified(t))
240 .collect(),
241 }
242 }
243
244 fn token_tree_replace_token_ids_with_unspecified(tt: tt::TokenTree) -> tt::TokenTree {
245 match tt {
246 tt::TokenTree::Leaf(leaf) => {
247 tt::TokenTree::Leaf(leaf_replace_token_ids_with_unspecified(leaf))
248 }
249 tt::TokenTree::Subtree(subtree) => {
250 tt::TokenTree::Subtree(subtree_replace_token_ids_with_unspecified(subtree))
251 }
252 }
253 }
254
255 fn leaf_replace_token_ids_with_unspecified(leaf: tt::Leaf) -> tt::Leaf {
256 match leaf {
257 tt::Leaf::Literal(lit) => {
258 tt::Leaf::Literal(tt::Literal { id: tt::TokenId::unspecified(), ..lit })
259 }
260 tt::Leaf::Punct(punct) => {
261 tt::Leaf::Punct(tt::Punct { id: tt::TokenId::unspecified(), ..punct })
262 }
263 tt::Leaf::Ident(ident) => {
264 tt::Leaf::Ident(tt::Ident { id: tt::TokenId::unspecified(), ..ident })
265 }
266 }
267 }
229} 268}
230 269
231impl TokenStreamBuilder { 270impl TokenStreamBuilder {
@@ -277,42 +316,6 @@ impl server::FreeFunctions for Rustc {
277 } 316 }
278} 317}
279 318
280fn subtree_replace_token_ids_with_unspecified(subtree: tt::Subtree) -> tt::Subtree {
281 tt::Subtree {
282 delimiter: subtree.delimiter.map(|d| tt::Delimiter { id: tt::TokenId::unspecified(), ..d }),
283 token_trees: subtree
284 .token_trees
285 .into_iter()
286 .map(|t| token_tree_replace_token_ids_with_unspecified(t))
287 .collect(),
288 }
289}
290
291fn token_tree_replace_token_ids_with_unspecified(tt: tt::TokenTree) -> tt::TokenTree {
292 match tt {
293 tt::TokenTree::Leaf(leaf) => {
294 tt::TokenTree::Leaf(leaf_replace_token_ids_with_unspecified(leaf))
295 }
296 tt::TokenTree::Subtree(subtree) => {
297 tt::TokenTree::Subtree(subtree_replace_token_ids_with_unspecified(subtree))
298 }
299 }
300}
301
302fn leaf_replace_token_ids_with_unspecified(leaf: tt::Leaf) -> tt::Leaf {
303 match leaf {
304 tt::Leaf::Literal(lit) => {
305 tt::Leaf::Literal(tt::Literal { id: tt::TokenId::unspecified(), ..lit })
306 }
307 tt::Leaf::Punct(punct) => {
308 tt::Leaf::Punct(tt::Punct { id: tt::TokenId::unspecified(), ..punct })
309 }
310 tt::Leaf::Ident(ident) => {
311 tt::Leaf::Ident(tt::Ident { id: tt::TokenId::unspecified(), ..ident })
312 }
313 }
314}
315
316impl server::TokenStream for Rustc { 319impl server::TokenStream for Rustc {
317 fn new(&mut self) -> Self::TokenStream { 320 fn new(&mut self) -> Self::TokenStream {
318 Self::TokenStream::new() 321 Self::TokenStream::new()
@@ -322,8 +325,9 @@ impl server::TokenStream for Rustc {
322 stream.is_empty() 325 stream.is_empty()
323 } 326 }
324 fn from_str(&mut self, src: &str) -> Self::TokenStream { 327 fn from_str(&mut self, src: &str) -> Self::TokenStream {
325 let (subtree, _) = mbe::parse_to_token_tree(src).expect("cannot parse string"); 328 use std::str::FromStr;
326 TokenStream::with_subtree(subtree_replace_token_ids_with_unspecified(subtree)) 329
330 Self::TokenStream::from_str(src).expect("cannot parse string")
327 } 331 }
328 fn to_string(&mut self, stream: &Self::TokenStream) -> String { 332 fn to_string(&mut self, stream: &Self::TokenStream) -> String {
329 stream.to_string() 333 stream.to_string()
diff --git a/crates/proc_macro_srv/src/tests/fixtures/test_serialize_proc_macro.txt b/crates/proc_macro_srv/src/tests/fixtures/test_serialize_proc_macro.txt
index ea34e688f..fa581f110 100644
--- a/crates/proc_macro_srv/src/tests/fixtures/test_serialize_proc_macro.txt
+++ b/crates/proc_macro_srv/src/tests/fixtures/test_serialize_proc_macro.txt
@@ -101,7 +101,7 @@ SUBTREE $
101 PUNCH : [alone] 4294967295 101 PUNCH : [alone] 4294967295
102 IDENT Serialize 4294967295 102 IDENT Serialize 4294967295
103 IDENT for 4294967295 103 IDENT for 4294967295
104 IDENT Foo 1 104 IDENT Foo 4294967295
105 SUBTREE {} 4294967295 105 SUBTREE {} 4294967295
106 IDENT fn 4294967295 106 IDENT fn 4294967295
107 IDENT serialize 4294967295 107 IDENT serialize 4294967295