From 1ab78d60561ed701b29d3065061cbc3175e20c4a Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Fri, 5 Apr 2019 18:23:01 +0800 Subject: Fix literal support in token tree to ast item list --- crates/ra_mbe/src/lib.rs | 41 ++++++++++++++++++++++++++++++++++++++ crates/ra_mbe/src/syntax_bridge.rs | 4 ++-- crates/ra_syntax/src/lib.rs | 2 +- crates/ra_syntax/src/parsing.rs | 2 +- 4 files changed, 45 insertions(+), 4 deletions(-) diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index 93246f54a..3b127526d 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -337,4 +337,45 @@ SOURCE_FILE@[0; 40) ); } + #[test] + fn expand_literals_to_item_list() { + fn to_subtree(tt: &tt::TokenTree) -> &tt::Subtree { + if let tt::TokenTree::Subtree(subtree) = tt { + return &subtree; + } + assert!(false, "It is not a subtree"); + unreachable!(); + } + + fn to_literal(tt: &tt::TokenTree) -> &tt::Literal { + if let tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) = tt { + return lit; + } + assert!(false, "It is not a literal"); + unreachable!(); + } + + let rules = create_rules( + r#" + macro_rules! literals { + ($i:ident) => { + { + let a = 'c'; + let c = 1000; + let f = 12E+99_f64; + } + } + } + "#, + ); + let expansion = expand(&rules, "literals!(foo)"); + let stm_tokens = &to_subtree(&expansion.token_trees[0]).token_trees; + + // [let] [a] [=] ['c'] [;] + assert_eq!(to_literal(&stm_tokens[3]).text, "'c'"); + // [let] [c] [=] [1000] [;] + assert_eq!(to_literal(&stm_tokens[5 + 3]).text, "1000"); + // [let] [f] [=] [12E+99_f64] [;] + assert_eq!(to_literal(&stm_tokens[10 + 3]).text, "12E+99_f64"); + } } diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 257503de8..fbf3d2e71 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -1,7 +1,7 @@ use ra_parser::{TokenSource, TreeSink, ParseError}; use ra_syntax::{ AstNode, SyntaxNode, TextRange, SyntaxKind, SmolStr, SyntaxTreeBuilder, TreeArc, SyntaxElement, - ast, SyntaxKind::*, TextUnit + ast, SyntaxKind::*, TextUnit, next_token }; /// Maps `tt::TokenId` to the relative range of the original token. @@ -189,7 +189,7 @@ impl TtTokenSource { { let tok = match token { tt::Leaf::Literal(l) => TtToken { - kind: SyntaxKind::INT_NUMBER, // FIXME + kind: next_token(&l.text).kind, is_joint_to_next: false, text: l.text.clone(), }, diff --git a/crates/ra_syntax/src/lib.rs b/crates/ra_syntax/src/lib.rs index e1088e296..1cbc3e9e1 100644 --- a/crates/ra_syntax/src/lib.rs +++ b/crates/ra_syntax/src/lib.rs @@ -40,7 +40,7 @@ pub use crate::{ syntax_text::SyntaxText, syntax_node::{Direction, SyntaxNode, WalkEvent, TreeArc, SyntaxTreeBuilder, SyntaxElement, SyntaxToken}, ptr::{SyntaxNodePtr, AstPtr}, - parsing::{tokenize, Token}, + parsing::{tokenize, next_token, Token}, }; use ra_text_edit::AtomTextEdit; diff --git a/crates/ra_syntax/src/parsing.rs b/crates/ra_syntax/src/parsing.rs index ad5668a65..f0750d737 100644 --- a/crates/ra_syntax/src/parsing.rs +++ b/crates/ra_syntax/src/parsing.rs @@ -11,7 +11,7 @@ use crate::{ syntax_node::GreenNode, }; -pub use self::lexer::{tokenize, Token}; +pub use self::lexer::{tokenize, next_token, Token}; pub(crate) use self::reparsing::incremental_reparse; -- cgit v1.2.3 From 1ea0238e538dc332b23698d54c02d8bd037f58bb Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Fri, 5 Apr 2019 18:45:19 +0800 Subject: Add classify_literal and undo expose next_token --- crates/ra_mbe/src/lib.rs | 6 ++---- crates/ra_mbe/src/syntax_bridge.rs | 4 ++-- crates/ra_syntax/src/lib.rs | 2 +- crates/ra_syntax/src/parsing.rs | 2 +- crates/ra_syntax/src/parsing/lexer.rs | 9 +++++++++ 5 files changed, 15 insertions(+), 8 deletions(-) diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index 3b127526d..38f0049ed 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -343,16 +343,14 @@ SOURCE_FILE@[0; 40) if let tt::TokenTree::Subtree(subtree) = tt { return &subtree; } - assert!(false, "It is not a subtree"); - unreachable!(); + unreachable!("It is not a subtree"); } fn to_literal(tt: &tt::TokenTree) -> &tt::Literal { if let tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) = tt { return lit; } - assert!(false, "It is not a literal"); - unreachable!(); + unreachable!("It is not a literal"); } let rules = create_rules( diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index fbf3d2e71..9664280b5 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -1,7 +1,7 @@ use ra_parser::{TokenSource, TreeSink, ParseError}; use ra_syntax::{ AstNode, SyntaxNode, TextRange, SyntaxKind, SmolStr, SyntaxTreeBuilder, TreeArc, SyntaxElement, - ast, SyntaxKind::*, TextUnit, next_token + ast, SyntaxKind::*, TextUnit, classify_literal }; /// Maps `tt::TokenId` to the relative range of the original token. @@ -189,7 +189,7 @@ impl TtTokenSource { { let tok = match token { tt::Leaf::Literal(l) => TtToken { - kind: next_token(&l.text).kind, + kind: classify_literal(&l.text).unwrap().kind, is_joint_to_next: false, text: l.text.clone(), }, diff --git a/crates/ra_syntax/src/lib.rs b/crates/ra_syntax/src/lib.rs index 1cbc3e9e1..c56bc9f16 100644 --- a/crates/ra_syntax/src/lib.rs +++ b/crates/ra_syntax/src/lib.rs @@ -40,7 +40,7 @@ pub use crate::{ syntax_text::SyntaxText, syntax_node::{Direction, SyntaxNode, WalkEvent, TreeArc, SyntaxTreeBuilder, SyntaxElement, SyntaxToken}, ptr::{SyntaxNodePtr, AstPtr}, - parsing::{tokenize, next_token, Token}, + parsing::{tokenize, classify_literal, Token}, }; use ra_text_edit::AtomTextEdit; diff --git a/crates/ra_syntax/src/parsing.rs b/crates/ra_syntax/src/parsing.rs index f0750d737..15d69c5ab 100644 --- a/crates/ra_syntax/src/parsing.rs +++ b/crates/ra_syntax/src/parsing.rs @@ -11,7 +11,7 @@ use crate::{ syntax_node::GreenNode, }; -pub use self::lexer::{tokenize, next_token, Token}; +pub use self::lexer::{tokenize, classify_literal, Token}; pub(crate) use self::reparsing::incremental_reparse; diff --git a/crates/ra_syntax/src/parsing/lexer.rs b/crates/ra_syntax/src/parsing/lexer.rs index 36e841609..e75f3aae0 100644 --- a/crates/ra_syntax/src/parsing/lexer.rs +++ b/crates/ra_syntax/src/parsing/lexer.rs @@ -214,3 +214,12 @@ fn scan_literal_suffix(ptr: &mut Ptr) { } ptr.bump_while(is_ident_continue); } + +pub fn classify_literal(text: &str) -> Option { + let tkn = next_token(text); + if tkn.kind.is_literal() || tkn.len.to_usize() != text.len() { + return None; + } + + Some(tkn) +} -- cgit v1.2.3 From 7abc06bd576264cb6b7c8becdbd1a8c0e914463d Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Fri, 5 Apr 2019 20:58:24 +0800 Subject: Add proper test for literals and fixed typo bug --- crates/ra_mbe/src/lib.rs | 11 +++++---- crates/ra_mbe/src/syntax_bridge.rs | 43 +++++++++++++++++++++++++++++++++++ crates/ra_syntax/src/parsing/lexer.rs | 2 +- 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index 38f0049ed..4203929d4 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -167,7 +167,7 @@ impl_froms!(TokenTree: Leaf, Subtree); ) } - fn create_rules(macro_definition: &str) -> MacroRules { + pub(crate) fn create_rules(macro_definition: &str) -> MacroRules { let source_file = ast::SourceFile::parse(macro_definition); let macro_definition = source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); @@ -176,7 +176,7 @@ impl_froms!(TokenTree: Leaf, Subtree); crate::MacroRules::parse(&definition_tt).unwrap() } - fn expand(rules: &MacroRules, invocation: &str) -> tt::Subtree { + pub(crate) fn expand(rules: &MacroRules, invocation: &str) -> tt::Subtree { let source_file = ast::SourceFile::parse(invocation); let macro_invocation = source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); @@ -186,7 +186,7 @@ impl_froms!(TokenTree: Leaf, Subtree); rules.expand(&invocation_tt).unwrap() } - fn assert_expansion(rules: &MacroRules, invocation: &str, expansion: &str) { + pub(crate) fn assert_expansion(rules: &MacroRules, invocation: &str, expansion: &str) { let expanded = expand(rules, invocation); assert_eq!(expanded.to_string(), expansion); } @@ -338,7 +338,7 @@ SOURCE_FILE@[0; 40) } #[test] - fn expand_literals_to_item_list() { + fn expand_literals_to_token_tree() { fn to_subtree(tt: &tt::TokenTree) -> &tt::Subtree { if let tt::TokenTree::Subtree(subtree) = tt { return &subtree; @@ -361,6 +361,7 @@ SOURCE_FILE@[0; 40) let a = 'c'; let c = 1000; let f = 12E+99_f64; + let s = "rust1"; } } } @@ -375,5 +376,7 @@ SOURCE_FILE@[0; 40) assert_eq!(to_literal(&stm_tokens[5 + 3]).text, "1000"); // [let] [f] [=] [12E+99_f64] [;] assert_eq!(to_literal(&stm_tokens[10 + 3]).text, "12E+99_f64"); + // [let] [s] [=] ["rust1"] [;] + assert_eq!(to_literal(&stm_tokens[15 + 3]).text, "\"rust1\""); } } diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 9664280b5..139a0fd33 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -103,10 +103,12 @@ fn convert_tt( Some(res) } +#[derive(Debug)] struct TtTokenSource { tokens: Vec, } +#[derive(Debug)] struct TtToken { kind: SyntaxKind, is_joint_to_next: bool, @@ -355,3 +357,44 @@ impl<'a> TreeSink for TtTreeSink<'a> { self.inner.error(error, self.text_pos) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::{expand, create_rules}; + + #[test] + fn convert_tt_token_source() { + let rules = create_rules( + r#" + macro_rules! literals { + ($i:ident) => { + { + let a = 'c'; + let c = 1000; + let f = 12E+99_f64; + let s = "rust1"; + } + } + } + "#, + ); + let expansion = expand(&rules, "literals!(foo)"); + let tt_src = TtTokenSource::new(&expansion); + + // [{] + // [let] [a] [=] ['c'] [;] + assert_eq!(tt_src.tokens[1 + 3].text, "'c'"); + assert_eq!(tt_src.tokens[1 + 3].kind, CHAR); + // [let] [c] [=] [1000] [;] + assert_eq!(tt_src.tokens[1 + 5 + 3].text, "1000"); + assert_eq!(tt_src.tokens[1 + 5 + 3].kind, INT_NUMBER); + // [let] [f] [=] [12E+99_f64] [;] + assert_eq!(tt_src.tokens[1 + 10 + 3].text, "12E+99_f64"); + assert_eq!(tt_src.tokens[1 + 10 + 3].kind, FLOAT_NUMBER); + + // [let] [s] [=] ["rust1"] [;] + assert_eq!(tt_src.tokens[1 + 15 + 3].text, "\"rust1\""); + assert_eq!(tt_src.tokens[1 + 15 + 3].kind, STRING); + } +} diff --git a/crates/ra_syntax/src/parsing/lexer.rs b/crates/ra_syntax/src/parsing/lexer.rs index e75f3aae0..3ae42912c 100644 --- a/crates/ra_syntax/src/parsing/lexer.rs +++ b/crates/ra_syntax/src/parsing/lexer.rs @@ -217,7 +217,7 @@ fn scan_literal_suffix(ptr: &mut Ptr) { pub fn classify_literal(text: &str) -> Option { let tkn = next_token(text); - if tkn.kind.is_literal() || tkn.len.to_usize() != text.len() { + if !tkn.kind.is_literal() || tkn.len.to_usize() != text.len() { return None; } -- cgit v1.2.3