From 1ab78d60561ed701b29d3065061cbc3175e20c4a Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Fri, 5 Apr 2019 18:23:01 +0800 Subject: Fix literal support in token tree to ast item list --- crates/ra_mbe/src/lib.rs | 41 ++++++++++++++++++++++++++++++++++++++ crates/ra_mbe/src/syntax_bridge.rs | 4 ++-- 2 files changed, 43 insertions(+), 2 deletions(-) (limited to 'crates/ra_mbe/src') diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index 93246f54a..3b127526d 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -337,4 +337,45 @@ SOURCE_FILE@[0; 40) ); } + #[test] + fn expand_literals_to_item_list() { + fn to_subtree(tt: &tt::TokenTree) -> &tt::Subtree { + if let tt::TokenTree::Subtree(subtree) = tt { + return &subtree; + } + assert!(false, "It is not a subtree"); + unreachable!(); + } + + fn to_literal(tt: &tt::TokenTree) -> &tt::Literal { + if let tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) = tt { + return lit; + } + assert!(false, "It is not a literal"); + unreachable!(); + } + + let rules = create_rules( + r#" + macro_rules! literals { + ($i:ident) => { + { + let a = 'c'; + let c = 1000; + let f = 12E+99_f64; + } + } + } + "#, + ); + let expansion = expand(&rules, "literals!(foo)"); + let stm_tokens = &to_subtree(&expansion.token_trees[0]).token_trees; + + // [let] [a] [=] ['c'] [;] + assert_eq!(to_literal(&stm_tokens[3]).text, "'c'"); + // [let] [c] [=] [1000] [;] + assert_eq!(to_literal(&stm_tokens[5 + 3]).text, "1000"); + // [let] [f] [=] [12E+99_f64] [;] + assert_eq!(to_literal(&stm_tokens[10 + 3]).text, "12E+99_f64"); + } } diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 257503de8..fbf3d2e71 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -1,7 +1,7 @@ use ra_parser::{TokenSource, TreeSink, ParseError}; use ra_syntax::{ AstNode, SyntaxNode, TextRange, SyntaxKind, SmolStr, SyntaxTreeBuilder, TreeArc, SyntaxElement, - ast, SyntaxKind::*, TextUnit + ast, SyntaxKind::*, TextUnit, next_token }; /// Maps `tt::TokenId` to the relative range of the original token. @@ -189,7 +189,7 @@ impl TtTokenSource { { let tok = match token { tt::Leaf::Literal(l) => TtToken { - kind: SyntaxKind::INT_NUMBER, // FIXME + kind: next_token(&l.text).kind, is_joint_to_next: false, text: l.text.clone(), }, -- cgit v1.2.3 From 1ea0238e538dc332b23698d54c02d8bd037f58bb Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Fri, 5 Apr 2019 18:45:19 +0800 Subject: Add classify_literal and undo expose next_token --- crates/ra_mbe/src/lib.rs | 6 ++---- crates/ra_mbe/src/syntax_bridge.rs | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) (limited to 'crates/ra_mbe/src') diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index 3b127526d..38f0049ed 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -343,16 +343,14 @@ SOURCE_FILE@[0; 40) if let tt::TokenTree::Subtree(subtree) = tt { return &subtree; } - assert!(false, "It is not a subtree"); - unreachable!(); + unreachable!("It is not a subtree"); } fn to_literal(tt: &tt::TokenTree) -> &tt::Literal { if let tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) = tt { return lit; } - assert!(false, "It is not a literal"); - unreachable!(); + unreachable!("It is not a literal"); } let rules = create_rules( diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index fbf3d2e71..9664280b5 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -1,7 +1,7 @@ use ra_parser::{TokenSource, TreeSink, ParseError}; use ra_syntax::{ AstNode, SyntaxNode, TextRange, SyntaxKind, SmolStr, SyntaxTreeBuilder, TreeArc, SyntaxElement, - ast, SyntaxKind::*, TextUnit, next_token + ast, SyntaxKind::*, TextUnit, classify_literal }; /// Maps `tt::TokenId` to the relative range of the original token. @@ -189,7 +189,7 @@ impl TtTokenSource { { let tok = match token { tt::Leaf::Literal(l) => TtToken { - kind: next_token(&l.text).kind, + kind: classify_literal(&l.text).unwrap().kind, is_joint_to_next: false, text: l.text.clone(), }, -- cgit v1.2.3 From 7abc06bd576264cb6b7c8becdbd1a8c0e914463d Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Fri, 5 Apr 2019 20:58:24 +0800 Subject: Add proper test for literals and fixed typo bug --- crates/ra_mbe/src/lib.rs | 11 ++++++---- crates/ra_mbe/src/syntax_bridge.rs | 43 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) (limited to 'crates/ra_mbe/src') diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index 38f0049ed..4203929d4 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -167,7 +167,7 @@ impl_froms!(TokenTree: Leaf, Subtree); ) } - fn create_rules(macro_definition: &str) -> MacroRules { + pub(crate) fn create_rules(macro_definition: &str) -> MacroRules { let source_file = ast::SourceFile::parse(macro_definition); let macro_definition = source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); @@ -176,7 +176,7 @@ impl_froms!(TokenTree: Leaf, Subtree); crate::MacroRules::parse(&definition_tt).unwrap() } - fn expand(rules: &MacroRules, invocation: &str) -> tt::Subtree { + pub(crate) fn expand(rules: &MacroRules, invocation: &str) -> tt::Subtree { let source_file = ast::SourceFile::parse(invocation); let macro_invocation = source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); @@ -186,7 +186,7 @@ impl_froms!(TokenTree: Leaf, Subtree); rules.expand(&invocation_tt).unwrap() } - fn assert_expansion(rules: &MacroRules, invocation: &str, expansion: &str) { + pub(crate) fn assert_expansion(rules: &MacroRules, invocation: &str, expansion: &str) { let expanded = expand(rules, invocation); assert_eq!(expanded.to_string(), expansion); } @@ -338,7 +338,7 @@ SOURCE_FILE@[0; 40) } #[test] - fn expand_literals_to_item_list() { + fn expand_literals_to_token_tree() { fn to_subtree(tt: &tt::TokenTree) -> &tt::Subtree { if let tt::TokenTree::Subtree(subtree) = tt { return &subtree; @@ -361,6 +361,7 @@ SOURCE_FILE@[0; 40) let a = 'c'; let c = 1000; let f = 12E+99_f64; + let s = "rust1"; } } } @@ -375,5 +376,7 @@ SOURCE_FILE@[0; 40) assert_eq!(to_literal(&stm_tokens[5 + 3]).text, "1000"); // [let] [f] [=] [12E+99_f64] [;] assert_eq!(to_literal(&stm_tokens[10 + 3]).text, "12E+99_f64"); + // [let] [s] [=] ["rust1"] [;] + assert_eq!(to_literal(&stm_tokens[15 + 3]).text, "\"rust1\""); } } diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 9664280b5..139a0fd33 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -103,10 +103,12 @@ fn convert_tt( Some(res) } +#[derive(Debug)] struct TtTokenSource { tokens: Vec, } +#[derive(Debug)] struct TtToken { kind: SyntaxKind, is_joint_to_next: bool, @@ -355,3 +357,44 @@ impl<'a> TreeSink for TtTreeSink<'a> { self.inner.error(error, self.text_pos) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{expand, create_rules}; + + #[test] + fn convert_tt_token_source() { + let rules = create_rules( + r#" + macro_rules! literals { + ($i:ident) => { + { + let a = 'c'; + let c = 1000; + let f = 12E+99_f64; + let s = "rust1"; + } + } + } + "#, + ); + let expansion = expand(&rules, "literals!(foo)"); + let tt_src = TtTokenSource::new(&expansion); + + // [{] + // [let] [a] [=] ['c'] [;] + assert_eq!(tt_src.tokens[1 + 3].text, "'c'"); + assert_eq!(tt_src.tokens[1 + 3].kind, CHAR); + // [let] [c] [=] [1000] [;] + assert_eq!(tt_src.tokens[1 + 5 + 3].text, "1000"); + assert_eq!(tt_src.tokens[1 + 5 + 3].kind, INT_NUMBER); + // [let] [f] [=] [12E+99_f64] [;] + assert_eq!(tt_src.tokens[1 + 10 + 3].text, "12E+99_f64"); + assert_eq!(tt_src.tokens[1 + 10 + 3].kind, FLOAT_NUMBER); + + // [let] [s] [=] ["rust1"] [;] + assert_eq!(tt_src.tokens[1 + 15 + 3].text, "\"rust1\""); + assert_eq!(tt_src.tokens[1 + 15 + 3].kind, STRING); + } +} -- cgit v1.2.3