Merge #1112

1112: Fix literal support in token tree to ast item list r=matklad a=edwin0cheng

This PR implements following things :

1. Expose `next_token` from `ra_parse`
2. Fix the literal conversion in `token_tree_to_ast_item_list`
3. Add test for the conversion

Co-authored-by: Edwin Cheng <[email protected]>

1 files changed, 45 insertions, 2 deletions

diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index 257503de8..139a0fd33 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -1,7 +1,7 @@
 use ra_parser::{TokenSource, TreeSink, ParseError};
 use ra_syntax::{
     AstNode, SyntaxNode, TextRange, SyntaxKind, SmolStr, SyntaxTreeBuilder, TreeArc, SyntaxElement,
-    ast, SyntaxKind::*, TextUnit
+    ast, SyntaxKind::*, TextUnit, classify_literal
 };
 
 /// Maps `tt::TokenId` to the relative range of the original token.
@@ -103,10 +103,12 @@ fn convert_tt(
     Some(res)
 }
 
+#[derive(Debug)]
 struct TtTokenSource {
     tokens: Vec<TtToken>,
 }
 
+#[derive(Debug)]
 struct TtToken {
     kind: SyntaxKind,
     is_joint_to_next: bool,
@@ -189,7 +191,7 @@ impl TtTokenSource {
     {
         let tok = match token {
             tt::Leaf::Literal(l) => TtToken {
-                kind: SyntaxKind::INT_NUMBER, // FIXME
+                kind: classify_literal(&l.text).unwrap().kind,
                 is_joint_to_next: false,
                 text: l.text.clone(),
             },
@@ -355,3 +357,44 @@ impl<'a> TreeSink for TtTreeSink<'a> {
         self.inner.error(error, self.text_pos)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::tests::{expand, create_rules};
+
+    #[test]
+    fn convert_tt_token_source() {
+        let rules = create_rules(
+            r#"
+            macro_rules! literals {
+                ($i:ident) => {
+                    {
+                        let a = 'c';
+                        let c = 1000;
+                        let f = 12E+99_f64;
+                        let s = "rust1";
+                    }
+                }
+            }
+            "#,
+        );
+        let expansion = expand(&rules, "literals!(foo)");
+        let tt_src = TtTokenSource::new(&expansion);
+
+        // [{]
+        // [let] [a] [=] ['c'] [;]
+        assert_eq!(tt_src.tokens[1 + 3].text, "'c'");
+        assert_eq!(tt_src.tokens[1 + 3].kind, CHAR);
+        // [let] [c] [=] [1000] [;]
+        assert_eq!(tt_src.tokens[1 + 5 + 3].text, "1000");
+        assert_eq!(tt_src.tokens[1 + 5 + 3].kind, INT_NUMBER);
+        // [let] [f] [=] [12E+99_f64] [;]
+        assert_eq!(tt_src.tokens[1 + 10 + 3].text, "12E+99_f64");
+        assert_eq!(tt_src.tokens[1 + 10 + 3].kind, FLOAT_NUMBER);
+
+        // [let] [s] [=] ["rust1"] [;]
+        assert_eq!(tt_src.tokens[1 + 15 + 3].text, "\"rust1\"");
+        assert_eq!(tt_src.tokens[1 + 15 + 3].kind, STRING);
+    }
+}