From 18f6a995d0fc1f45099f3cc810a5d55d5401b41b Mon Sep 17 00:00:00 2001
From: Florian Diebold <flodiebold@gmail.com>
Date: Thu, 5 Dec 2019 15:10:33 +0100
Subject: Add expansion infrastructure for derive macros

---
 crates/ra_mbe/src/syntax_bridge.rs | 34 ++++++++++++++++++++++------------
 1 file changed, 22 insertions(+), 12 deletions(-)

(limited to 'crates/ra_mbe/src/syntax_bridge.rs')

diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index 1de399fee..0fbcb2f66 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -2,7 +2,7 @@
 
 use ra_parser::{FragmentKind, ParseError, TreeSink};
 use ra_syntax::{
-    ast, AstNode, AstToken, NodeOrToken, Parse, SmolStr, SyntaxKind, SyntaxKind::*, SyntaxNode,
+    ast, AstToken, NodeOrToken, Parse, SmolStr, SyntaxKind, SyntaxKind::*, SyntaxNode,
     SyntaxTreeBuilder, TextRange, TextUnit, T,
 };
 use std::iter::successors;
@@ -20,7 +20,7 @@ pub struct TokenMap {
 
 /// Convert the syntax tree (what user has written) to a `TokenTree` (what macro
 /// will consume).
-pub fn ast_to_token_tree(ast: &ast::TokenTree) -> Option<(tt::Subtree, TokenMap)> {
+pub fn ast_to_token_tree(ast: &impl ast::AstNode) -> Option<(tt::Subtree, TokenMap)> {
     syntax_node_to_token_tree(ast.syntax())
 }
 
@@ -208,13 +208,8 @@ impl Convertor {
                     } else if token.kind().is_trivia() {
                         continue;
                     } else if token.kind().is_punct() {
-                        assert!(
-                            token.text().len() == 1,
-                            "Input ast::token punct must be single char."
-                        );
-                        let char = token.text().chars().next().unwrap();
-
-                        let spacing = match child_iter.peek() {
+                        // we need to pull apart joined punctuation tokens
+                        let last_spacing = match child_iter.peek() {
                             Some(NodeOrToken::Token(token)) => {
                                 if token.kind().is_punct() {
                                     tt::Spacing::Joint
@@ -224,8 +219,12 @@ impl Convertor {
                             }
                             _ => tt::Spacing::Alone,
                         };
-
-                        token_trees.push(tt::Leaf::from(tt::Punct { char, spacing }).into());
+                        let spacing_iter = std::iter::repeat(tt::Spacing::Joint)
+                            .take(token.text().len() - 1)
+                            .chain(std::iter::once(last_spacing));
+                        for (char, spacing) in token.text().chars().zip(spacing_iter) {
+                            token_trees.push(tt::Leaf::from(tt::Punct { char, spacing }).into());
+                        }
                     } else {
                         let child: tt::TokenTree =
                             if token.kind() == T![true] || token.kind() == T![false] {
@@ -389,7 +388,10 @@ mod tests {
     use super::*;
     use crate::tests::{create_rules, expand};
     use ra_parser::TokenSource;
-    use ra_syntax::algo::{insert_children, InsertPosition};
+    use ra_syntax::{
+        algo::{insert_children, InsertPosition},
+        ast::AstNode,
+    };
 
     #[test]
     fn convert_tt_token_source() {
@@ -491,4 +493,12 @@ mod tests {
 
         assert_eq!(tt.delimiter, tt::Delimiter::Brace);
     }
+
+    #[test]
+    fn test_token_tree_multi_char_punct() {
+        let source_file = ast::SourceFile::parse("struct Foo { a: x::Y }").ok().unwrap();
+        let struct_def = source_file.syntax().descendants().find_map(ast::StructDef::cast).unwrap();
+        let tt = ast_to_token_tree(&struct_def).unwrap().0;
+        token_tree_to_syntax_node(&tt, FragmentKind::Item).unwrap();
+    }
 }
-- 
cgit v1.2.3


From ab4ecca210d8d280a4e216c2b6edfff303269144 Mon Sep 17 00:00:00 2001
From: Florian Diebold <flodiebold@gmail.com>
Date: Thu, 5 Dec 2019 19:27:39 +0100
Subject: Don't wrap most syntax trees in invisible delimiters when converting
 to token tree

Otherwise parsing them again doesn't work.
---
 crates/ra_mbe/src/syntax_bridge.rs | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

(limited to 'crates/ra_mbe/src/syntax_bridge.rs')

diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index 0fbcb2f66..66c1f0337 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -245,8 +245,14 @@ impl Convertor {
                     }
                 }
                 NodeOrToken::Node(node) => {
-                    let child = self.go(&node)?.into();
-                    token_trees.push(child);
+                    let child_subtree = self.go(&node)?;
+                    if child_subtree.delimiter == tt::Delimiter::None
+                        && node.kind() != SyntaxKind::TOKEN_TREE
+                    {
+                        token_trees.extend(child_subtree.token_trees);
+                    } else {
+                        token_trees.push(child_subtree.into());
+                    }
                 }
             };
         }
-- 
cgit v1.2.3