From 1ab78d60561ed701b29d3065061cbc3175e20c4a Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Fri, 5 Apr 2019 18:23:01 +0800 Subject: Fix literal support in token tree to ast item list --- crates/ra_mbe/src/syntax_bridge.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'crates/ra_mbe/src/syntax_bridge.rs') diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 257503de8..fbf3d2e71 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -1,7 +1,7 @@ use ra_parser::{TokenSource, TreeSink, ParseError}; use ra_syntax::{ AstNode, SyntaxNode, TextRange, SyntaxKind, SmolStr, SyntaxTreeBuilder, TreeArc, SyntaxElement, - ast, SyntaxKind::*, TextUnit + ast, SyntaxKind::*, TextUnit, next_token }; /// Maps `tt::TokenId` to the relative range of the original token. @@ -189,7 +189,7 @@ impl TtTokenSource { { let tok = match token { tt::Leaf::Literal(l) => TtToken { - kind: SyntaxKind::INT_NUMBER, // FIXME + kind: next_token(&l.text).kind, is_joint_to_next: false, text: l.text.clone(), }, -- cgit v1.2.3 From 1ea0238e538dc332b23698d54c02d8bd037f58bb Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Fri, 5 Apr 2019 18:45:19 +0800 Subject: Add classify_literal and undo expose next_token --- crates/ra_mbe/src/syntax_bridge.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'crates/ra_mbe/src/syntax_bridge.rs') diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index fbf3d2e71..9664280b5 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -1,7 +1,7 @@ use ra_parser::{TokenSource, TreeSink, ParseError}; use ra_syntax::{ AstNode, SyntaxNode, TextRange, SyntaxKind, SmolStr, SyntaxTreeBuilder, TreeArc, SyntaxElement, - ast, SyntaxKind::*, TextUnit, next_token + ast, SyntaxKind::*, TextUnit, classify_literal }; /// Maps `tt::TokenId` to the relative range of the original token. @@ -189,7 +189,7 @@ impl TtTokenSource { { let tok = match token { tt::Leaf::Literal(l) => TtToken { - kind: next_token(&l.text).kind, + kind: classify_literal(&l.text).unwrap().kind, is_joint_to_next: false, text: l.text.clone(), }, -- cgit v1.2.3 From 7abc06bd576264cb6b7c8becdbd1a8c0e914463d Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Fri, 5 Apr 2019 20:58:24 +0800 Subject: Add proper test for literals and fixed typo bug --- crates/ra_mbe/src/syntax_bridge.rs | 43 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) (limited to 'crates/ra_mbe/src/syntax_bridge.rs') diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 9664280b5..139a0fd33 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -103,10 +103,12 @@ fn convert_tt( Some(res) } +#[derive(Debug)] struct TtTokenSource { tokens: Vec, } +#[derive(Debug)] struct TtToken { kind: SyntaxKind, is_joint_to_next: bool, @@ -355,3 +357,44 @@ impl<'a> TreeSink for TtTreeSink<'a> { self.inner.error(error, self.text_pos) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{expand, create_rules}; + + #[test] + fn convert_tt_token_source() { + let rules = create_rules( + r#" + macro_rules! literals { + ($i:ident) => { + { + let a = 'c'; + let c = 1000; + let f = 12E+99_f64; + let s = "rust1"; + } + } + } + "#, + ); + let expansion = expand(&rules, "literals!(foo)"); + let tt_src = TtTokenSource::new(&expansion); + + // [{] + // [let] [a] [=] ['c'] [;] + assert_eq!(tt_src.tokens[1 + 3].text, "'c'"); + assert_eq!(tt_src.tokens[1 + 3].kind, CHAR); + // [let] [c] [=] [1000] [;] + assert_eq!(tt_src.tokens[1 + 5 + 3].text, "1000"); + assert_eq!(tt_src.tokens[1 + 5 + 3].kind, INT_NUMBER); + // [let] [f] [=] [12E+99_f64] [;] + assert_eq!(tt_src.tokens[1 + 10 + 3].text, "12E+99_f64"); + assert_eq!(tt_src.tokens[1 + 10 + 3].kind, FLOAT_NUMBER); + + // [let] [s] [=] ["rust1"] [;] + assert_eq!(tt_src.tokens[1 + 15 + 3].text, "\"rust1\""); + assert_eq!(tt_src.tokens[1 + 15 + 3].kind, STRING); + } +} -- cgit v1.2.3