handle multibyte tokens

2 files changed, 31 insertions, 23 deletions

diff --git a/crates/ra_hir/src/macros.rs b/crates/ra_hir/src/macros.rs
index 4740e5337..0497d2168 100644
--- a/crates/ra_hir/src/macros.rs
+++ b/crates/ra_hir/src/macros.rs
@@ -221,27 +221,32 @@ fn convert_tt(tt: &SyntaxNode) -> Option<tt::Subtree> {
         if child == first_child || child == last_child || child.kind().is_trivia() {
             continue;
         }
-        let child: tt::TokenTree = if child.kind() == TOKEN_TREE {
-            convert_tt(child)?.into()
-        } else if child.kind().is_keyword() || child.kind() == IDENT {
-            let text = child.leaf_text().unwrap().clone();
-            tt::Leaf::from(tt::Ident { text }).into()
-        } else if child.kind().is_punct() {
-            // FIXME: multibyte tokens
-            tt::Leaf::from(tt::Punct {
-                char: child.text().char_at(0)?,
-            })
-            .into()
-        } else if child.kind().is_literal() {
-            tt::Leaf::from(tt::Literal {
-                text: child.leaf_text().unwrap().clone(),
-            })
-            .into()
+        if child.kind().is_punct() {
+            let leaves = child
+                .leaf_text()
+                .unwrap()
+                .chars()
+                .map(|char| tt::Punct { char })
+                .map(tt::Leaf::from)
+                .map(tt::TokenTree::from);
+            token_trees.extend(leaves);
         } else {
-            log::error!("unknown kind: {:?}", child);
-            return None;
-        };
-        token_trees.push(child)
+            let child: tt::TokenTree = if child.kind() == TOKEN_TREE {
+                convert_tt(child)?.into()
+            } else if child.kind().is_keyword() || child.kind() == IDENT {
+                let text = child.leaf_text().unwrap().clone();
+                tt::Leaf::from(tt::Ident { text }).into()
+            } else if child.kind().is_literal() {
+                tt::Leaf::from(tt::Literal {
+                    text: child.leaf_text().unwrap().clone(),
+                })
+                .into()
+            } else {
+                log::error!("unknown kind: {:?}", child);
+                return None;
+            };
+            token_trees.push(child)
+        }
     }
 
     let res = tt::Subtree {
diff --git a/crates/ra_hir/src/macros/mbe.rs b/crates/ra_hir/src/macros/mbe.rs
index 3d6dcefa8..62b7fa24c 100644
--- a/crates/ra_hir/src/macros/mbe.rs
+++ b/crates/ra_hir/src/macros/mbe.rs
@@ -86,14 +86,17 @@ pub(crate) fn parse(tt: &tt::Subtree) -> Option<MacroRules> {
 
 fn parse_rule(p: &mut RulesParser) -> Option<Rule> {
     let lhs = parse_subtree(p.eat_subtree()?)?;
-    p.eat_punct('=');
-    p.eat_punct('>');
+    p.eat_punct('=')?;
+    p.eat_punct('>')?;
     let rhs = parse_subtree(p.eat_subtree()?)?;
     Some(Rule { lhs, rhs })
 }
 
 fn parse_subtree(tt: &tt::Subtree) -> Option<Subtree> {
-    None
+    Some(Subtree {
+        token_trees: Vec::new(),
+        delimiter: Delimiter::None,
+    })
 }
 
 struct RulesParser<'a> {