3 files changed, 228 insertions, 25 deletions

diff --git a/crates/ra_mbe/Cargo.toml b/crates/ra_mbe/Cargo.toml
index 6e785f570..1d0c2a340 100644
--- a/crates/ra_mbe/Cargo.toml
+++ b/crates/ra_mbe/Cargo.toml
@@ -8,5 +8,5 @@ authors = ["rust-analyzer developers"]
 ra_syntax = { path = "../ra_syntax" }
 ra_parser = { path = "../ra_parser" }
 tt = { path = "../ra_tt", package = "ra_tt" }
-
+itertools = "0.8.0"
 rustc-hash = "1.0.0"
diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs
index 93246f54a..4203929d4 100644
--- a/crates/ra_mbe/src/lib.rs
+++ b/crates/ra_mbe/src/lib.rs
@@ -167,7 +167,7 @@ impl_froms!(TokenTree: Leaf, Subtree);
     )
     }
 
-    fn create_rules(macro_definition: &str) -> MacroRules {
+    pub(crate) fn create_rules(macro_definition: &str) -> MacroRules {
         let source_file = ast::SourceFile::parse(macro_definition);
         let macro_definition =
             source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
@@ -176,7 +176,7 @@ impl_froms!(TokenTree: Leaf, Subtree);
         crate::MacroRules::parse(&definition_tt).unwrap()
     }
 
-    fn expand(rules: &MacroRules, invocation: &str) -> tt::Subtree {
+    pub(crate) fn expand(rules: &MacroRules, invocation: &str) -> tt::Subtree {
         let source_file = ast::SourceFile::parse(invocation);
         let macro_invocation =
             source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
@@ -186,7 +186,7 @@ impl_froms!(TokenTree: Leaf, Subtree);
         rules.expand(&invocation_tt).unwrap()
     }
 
-    fn assert_expansion(rules: &MacroRules, invocation: &str, expansion: &str) {
+    pub(crate) fn assert_expansion(rules: &MacroRules, invocation: &str, expansion: &str) {
         let expanded = expand(rules, invocation);
         assert_eq!(expanded.to_string(), expansion);
     }
@@ -337,4 +337,46 @@ SOURCE_FILE@[0; 40)
         );
     }
 
+    #[test]
+    fn expand_literals_to_token_tree() {
+        fn to_subtree(tt: &tt::TokenTree) -> &tt::Subtree {
+            if let tt::TokenTree::Subtree(subtree) = tt {
+                return &subtree;
+            }
+            unreachable!("It is not a subtree");
+        }
+
+        fn to_literal(tt: &tt::TokenTree) -> &tt::Literal {
+            if let tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) = tt {
+                return lit;
+            }
+            unreachable!("It is not a literal");
+        }
+
+        let rules = create_rules(
+            r#"
+            macro_rules! literals {
+                ($i:ident) => {
+                    {
+                        let a = 'c';
+                        let c = 1000;
+                        let f = 12E+99_f64;
+                        let s = "rust1";
+                    }
+                }
+            }
+            "#,
+        );
+        let expansion = expand(&rules, "literals!(foo)");
+        let stm_tokens = &to_subtree(&expansion.token_trees[0]).token_trees;
+
+        // [let] [a] [=] ['c'] [;]
+        assert_eq!(to_literal(&stm_tokens[3]).text, "'c'");
+        // [let] [c] [=] [1000] [;]
+        assert_eq!(to_literal(&stm_tokens[5 + 3]).text, "1000");
+        // [let] [f] [=] [12E+99_f64] [;]
+        assert_eq!(to_literal(&stm_tokens[10 + 3]).text, "12E+99_f64");
+        // [let] [s] [=] ["rust1"] [;]
+        assert_eq!(to_literal(&stm_tokens[15 + 3]).text, "\"rust1\"");
+    }
 }
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index 05f9817da..139a0fd33 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -1,7 +1,7 @@
 use ra_parser::{TokenSource, TreeSink, ParseError};
 use ra_syntax::{
     AstNode, SyntaxNode, TextRange, SyntaxKind, SmolStr, SyntaxTreeBuilder, TreeArc, SyntaxElement,
-    ast, SyntaxKind::*, TextUnit
+    ast, SyntaxKind::*, TextUnit, classify_literal
 };
 
 /// Maps `tt::TokenId` to the relative range of the original token.
@@ -103,16 +103,63 @@ fn convert_tt(
     Some(res)
 }
 
+#[derive(Debug)]
 struct TtTokenSource {
     tokens: Vec<TtToken>,
 }
 
+#[derive(Debug)]
 struct TtToken {
     kind: SyntaxKind,
     is_joint_to_next: bool,
     text: SmolStr,
 }
 
+// Some helper functions
+fn to_punct(tt: &tt::TokenTree) -> Option<&tt::Punct> {
+    if let tt::TokenTree::Leaf(tt::Leaf::Punct(pp)) = tt {
+        return Some(pp);
+    }
+    None
+}
+
+struct TokenPeek<'a, I>
+where
+    I: Iterator<Item = &'a tt::TokenTree>,
+{
+    iter: itertools::MultiPeek<I>,
+}
+
+impl<'a, I> TokenPeek<'a, I>
+where
+    I: Iterator<Item = &'a tt::TokenTree>,
+{
+    fn next(&mut self) -> Option<&tt::TokenTree> {
+        self.iter.next()
+    }
+
+    fn current_punct2(&mut self, p: &tt::Punct) -> Option<((char, char), bool)> {
+        if p.spacing != tt::Spacing::Joint {
+            return None;
+        }
+
+        self.iter.reset_peek();
+        let p1 = to_punct(self.iter.peek()?)?;
+        Some(((p.char, p1.char), p1.spacing == tt::Spacing::Joint))
+    }
+
+    fn current_punct3(&mut self, p: &tt::Punct) -> Option<((char, char, char), bool)> {
+        self.current_punct2(p).and_then(|((p0, p1), last_joint)| {
+            if !last_joint {
+                None
+            } else {
+                let p2 = to_punct(*self.iter.peek()?)?;
+                Some(((p0, p1, p2.char), p2.spacing == tt::Spacing::Joint))
+            }
+        })
+    }
+}
+
 impl TtTokenSource {
     fn new(tt: &tt::Subtree) -> TtTokenSource {
         let mut res = TtTokenSource { tokens: Vec::new() };
@@ -121,38 +168,53 @@ impl TtTokenSource {
     }
     fn convert_subtree(&mut self, sub: &tt::Subtree) {
         self.push_delim(sub.delimiter, false);
-        sub.token_trees.iter().for_each(|tt| self.convert_tt(tt));
+        let mut peek = TokenPeek { iter: itertools::multipeek(sub.token_trees.iter()) };
+        while let Some(tt) = peek.iter.next() {
+            self.convert_tt(tt, &mut peek);
+        }
         self.push_delim(sub.delimiter, true)
     }
-    fn convert_tt(&mut self, tt: &tt::TokenTree) {
+
+    fn convert_tt<'a, I>(&mut self, tt: &tt::TokenTree, iter: &mut TokenPeek<'a, I>)
+    where
+        I: Iterator<Item = &'a tt::TokenTree>,
+    {
         match tt {
-            tt::TokenTree::Leaf(token) => self.convert_token(token),
+            tt::TokenTree::Leaf(token) => self.convert_token(token, iter),
             tt::TokenTree::Subtree(sub) => self.convert_subtree(sub),
         }
     }
-    fn convert_token(&mut self, token: &tt::Leaf) {
+
+    fn convert_token<'a, I>(&mut self, token: &tt::Leaf, iter: &mut TokenPeek<'a, I>)
+    where
+        I: Iterator<Item = &'a tt::TokenTree>,
+    {
         let tok = match token {
             tt::Leaf::Literal(l) => TtToken {
-                kind: SyntaxKind::INT_NUMBER, // FIXME
+                kind: classify_literal(&l.text).unwrap().kind,
                 is_joint_to_next: false,
                 text: l.text.clone(),
             },
             tt::Leaf::Punct(p) => {
-                let kind = match p.char {
-                    // lexer may produce combpund tokens for these ones
-                    '.' => DOT,
-                    ':' => COLON,
-                    '=' => EQ,
-                    '!' => EXCL,
-                    '-' => MINUS,
-                    c => SyntaxKind::from_char(c).unwrap(),
-                };
-                let text = {
-                    let mut buf = [0u8; 4];
-                    let s: &str = p.char.encode_utf8(&mut buf);
-                    SmolStr::new(s)
-                };
-                TtToken { kind, is_joint_to_next: p.spacing == tt::Spacing::Joint, text }
+                if let Some(tt) = Self::convert_multi_char_punct(p, iter) {
+                    tt
+                } else {
+                    let kind = match p.char {
+                        // lexer may produce combpund tokens for these ones
+                        '.' => DOT,
+                        ':' => COLON,
+                        '=' => EQ,
+                        '!' => EXCL,
+                        '-' => MINUS,
+                        c => SyntaxKind::from_char(c).unwrap(),
+                    };
+                    let text = {
+                        let mut buf = [0u8; 4];
+                        let s: &str = p.char.encode_utf8(&mut buf);
+                        SmolStr::new(s)
+                    };
+                    TtToken { kind, is_joint_to_next: p.spacing == tt::Spacing::Joint, text }
+                }
             }
             tt::Leaf::Ident(ident) => {
                 let kind = SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT);
@@ -161,6 +223,64 @@ impl TtTokenSource {
         };
         self.tokens.push(tok)
     }
+
+    fn convert_multi_char_punct<'a, I>(
+        p: &tt::Punct,
+        iter: &mut TokenPeek<'a, I>,
+    ) -> Option<TtToken>
+    where
+        I: Iterator<Item = &'a tt::TokenTree>,
+    {
+        if let Some((m, is_joint_to_next)) = iter.current_punct3(p) {
+            if let Some((kind, text)) = match m {
+                ('<', '<', '=') => Some((SHLEQ, "<<=")),
+                ('>', '>', '=') => Some((SHREQ, ">>=")),
+                ('.', '.', '.') => Some((DOTDOTDOT, "...")),
+                ('.', '.', '=') => Some((DOTDOTEQ, "..=")),
+                _ => None,
+            } {
+                iter.next();
+                iter.next();
+                return Some(TtToken { kind, is_joint_to_next, text: text.into() });
+            }
+        }
+
+        if let Some((m, is_joint_to_next)) = iter.current_punct2(p) {
+            if let Some((kind, text)) = match m {
+                ('<', '<') => Some((SHL, "<<")),
+                ('>', '>') => Some((SHR, ">>")),
+
+                ('|', '|') => Some((PIPEPIPE, "||")),
+                ('&', '&') => Some((AMPAMP, "&&")),
+                ('%', '=') => Some((PERCENTEQ, "%=")),
+                ('*', '=') => Some((STAREQ, "*=")),
+                ('/', '=') => Some((SLASHEQ, "/=")),
+                ('^', '=') => Some((CARETEQ, "^=")),
+
+                ('&', '=') => Some((AMPEQ, "&=")),
+                ('|', '=') => Some((PIPEEQ, "|=")),
+                ('-', '=') => Some((MINUSEQ, "-=")),
+                ('+', '=') => Some((PLUSEQ, "+=")),
+                ('>', '=') => Some((GTEQ, ">=")),
+                ('<', '=') => Some((LTEQ, "<=")),
+
+                ('-', '>') => Some((THIN_ARROW, "->")),
+                ('!', '=') => Some((NEQ, "!=")),
+                ('=', '>') => Some((FAT_ARROW, "=>")),
+                ('=', '=') => Some((EQEQ, "==")),
+                ('.', '.') => Some((DOTDOT, "..")),
+                (':', ':') => Some((COLONCOLON, "::")),
+
+                _ => None,
+            } {
+                iter.next();
+                return Some(TtToken { kind, is_joint_to_next, text: text.into() });
+            }
+        }
+
+        None
+    }
+
     fn push_delim(&mut self, d: tt::Delimiter, closing: bool) {
         let (kinds, texts) = match d {
             tt::Delimiter::Parenthesis => ([L_PAREN, R_PAREN], "()"),
@@ -237,3 +357,44 @@ impl<'a> TreeSink for TtTreeSink<'a> {
         self.inner.error(error, self.text_pos)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::tests::{expand, create_rules};
+
+    #[test]
+    fn convert_tt_token_source() {
+        let rules = create_rules(
+            r#"
+            macro_rules! literals {
+                ($i:ident) => {
+                    {
+                        let a = 'c';
+                        let c = 1000;
+                        let f = 12E+99_f64;
+                        let s = "rust1";
+                    }
+                }
+            }
+            "#,
+        );
+        let expansion = expand(&rules, "literals!(foo)");
+        let tt_src = TtTokenSource::new(&expansion);
+
+        // [{]
+        // [let] [a] [=] ['c'] [;]
+        assert_eq!(tt_src.tokens[1 + 3].text, "'c'");
+        assert_eq!(tt_src.tokens[1 + 3].kind, CHAR);
+        // [let] [c] [=] [1000] [;]
+        assert_eq!(tt_src.tokens[1 + 5 + 3].text, "1000");
+        assert_eq!(tt_src.tokens[1 + 5 + 3].kind, INT_NUMBER);
+        // [let] [f] [=] [12E+99_f64] [;]
+        assert_eq!(tt_src.tokens[1 + 10 + 3].text, "12E+99_f64");
+        assert_eq!(tt_src.tokens[1 + 10 + 3].kind, FLOAT_NUMBER);
+
+        // [let] [s] [=] ["rust1"] [;]
+        assert_eq!(tt_src.tokens[1 + 15 + 3].text, "\"rust1\"");
+        assert_eq!(tt_src.tokens[1 + 15 + 3].kind, STRING);
+    }
+}