From 0e6b94de78bb5b356d6fd14a6db91ceb8d76856d Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Thu, 13 Aug 2020 10:04:37 +0200 Subject: Minor --- Cargo.lock | 36 ++++++++++++++++++------------------ crates/proc_macro_srv/Cargo.toml | 8 +++++--- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index efc17c430..614548cca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -901,6 +901,23 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "proc_macro_srv" +version = "0.0.0" +dependencies = [ + "cargo_metadata", + "difference", + "goblin", + "libloading", + "memmap", + "ra_mbe", + "ra_proc_macro", + "serde_derive", + "test_utils", + "toolchain", + "tt", +] + [[package]] name = "profile" version = "0.0.0" @@ -1132,23 +1149,6 @@ dependencies = [ "tt", ] -[[package]] -name = "proc_macro_srv" -version = "0.0.0" -dependencies = [ - "cargo_metadata", - "difference", - "goblin", - "libloading", - "memmap", - "ra_mbe", - "ra_proc_macro", - "serde_derive", - "test_utils", - "toolchain", - "tt", -] - [[package]] name = "ra_project_model" version = "0.1.0" @@ -1271,6 +1271,7 @@ dependencies = [ "oorandom", "parking_lot", "pico-args", + "proc_macro_srv", "profile", "ra_cfg", "ra_db", @@ -1280,7 +1281,6 @@ dependencies = [ "ra_ide", "ra_ide_db", "ra_mbe", - "proc_macro_srv", "ra_project_model", "ra_ssr", "rayon", diff --git a/crates/proc_macro_srv/Cargo.toml b/crates/proc_macro_srv/Cargo.toml index 775af890e..f7d481cba 100644 --- a/crates/proc_macro_srv/Cargo.toml +++ b/crates/proc_macro_srv/Cargo.toml @@ -9,12 +9,13 @@ edition = "2018" doctest = false [dependencies] -tt = { path = "../tt" } -ra_mbe = { path = "../ra_mbe" } -ra_proc_macro = { path = "../ra_proc_macro" } goblin = "0.2.1" libloading = "0.6.0" memmap = "0.7" + +tt = { path = "../tt" } +ra_mbe = { path = "../ra_mbe" } +ra_proc_macro = { path = "../ra_proc_macro" } test_utils = { path = "../test_utils" } [dev-dependencies] @@ -22,4 +23,5 @@ cargo_metadata = "0.11.1" difference = "2.0.0" # used as proc macro test target serde_derive = "1.0.106" + toolchain = { path = "../toolchain" } -- cgit v1.2.3 From d42ba6397668fe28bd9cd92db829755905469a69 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Thu, 13 Aug 2020 10:04:55 +0200 Subject: :arrow_up: crates --- Cargo.lock | 55 +++++++++++++++++++------------------------------------ 1 file changed, 19 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 614548cca..6bace7bd2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15,15 +15,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" -[[package]] -name = "aho-corasick" -version = "0.7.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "043164d8ba5c4c3035fec9bbee8647c0261d788f3474306f93bb65901cae0e86" -dependencies = [ - "memchr", -] - [[package]] name = "ansi_term" version = "0.12.1" @@ -289,9 +280,9 @@ checksum = "9bda8e21c04aca2ae33ffc2fd8c23134f3cac46db123ba97bd9d3f3b8a4a85e1" [[package]] name = "either" -version = "1.5.3" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" +checksum = "cd56b59865bce947ac5958779cfa508f6c3b9497cc762b7e24a12d11ccde2c4f" [[package]] name = "ena" @@ -551,9 +542,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "lazycell" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b294d6fa9ee409a054354afc4352b0b9ef7ca222c69b8812cbea9e7d2bf3783f" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" @@ -644,12 +635,6 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" -[[package]] -name = "memchr" -version = "2.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" - [[package]] name = "memmap" version = "0.7.0" @@ -1219,10 +1204,7 @@ version = "1.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" dependencies = [ - "aho-corasick", - "memchr", "regex-syntax", - "thread_local", ] [[package]] @@ -1414,18 +1396,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.114" +version = "1.0.115" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5317f7588f0a5078ee60ef675ef96735a1442132dc645eb1d12c018620ed8cd3" +checksum = "e54c9a88f2da7238af84b5101443f0c0d0a3bbdc455e34a5c9497b1903ed55d5" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.114" +version = "1.0.115" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0be94b04690fbaed37cddffc5c134bf537c8e3329d53e982fe04c374978f8e" +checksum = "609feed1d0a73cc36a0182a840a9b37b4a82f0b1150369f0536a9e3f2a31dc48" dependencies = [ "proc-macro2", "quote", @@ -1471,9 +1453,9 @@ checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" [[package]] name = "smallvec" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3757cb9d89161a2f24e1cf78efa0c1fcff485d18e3f55e0aa3480824ddaa0f3f" +checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252" [[package]] name = "smol_str" @@ -1614,9 +1596,9 @@ dependencies = [ [[package]] name = "tracing" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0aae59226cf195d8e74d4b34beae1859257efb4e5fed3f147d2dc2c7d372178" +checksum = "6d79ca061b032d6ce30c660fded31189ca0b9922bf483cd70759f13a2d86786c" dependencies = [ "cfg-if", "tracing-attributes", @@ -1625,9 +1607,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0693bf8d6f2bf22c690fc61a9d21ac69efdbb894a17ed596b9af0f01e64b84b" +checksum = "1fe233f4227389ab7df5b32649239da7ebe0b281824b4e84b342d04d3fd8c25e" dependencies = [ "proc-macro2", "quote", @@ -1636,9 +1618,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d593f98af59ebc017c0648f0117525db358745a8894a8d684e185ba3f45954f9" +checksum = "db63662723c316b43ca36d833707cc93dff82a02ba3d7e354f342682cc8b3545" dependencies = [ "lazy_static", ] @@ -1666,9 +1648,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7b33f8b2ef2ab0c3778c12646d9c42a24f7772bee4cdafc72199644a9f58fdc" +checksum = "abd165311cc4d7a555ad11cc77a37756df836182db0d81aac908c8184c584f40" dependencies = [ "ansi_term", "chrono", @@ -1679,6 +1661,7 @@ dependencies = [ "serde_json", "sharded-slab", "smallvec", + "thread_local", "tracing-core", "tracing-log", "tracing-serde", -- cgit v1.2.3 From 2f45cfc415626cfae5cba89c88a25fb3225486f7 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Thu, 13 Aug 2020 10:08:11 +0200 Subject: Rename ra_mbe -> mbe --- Cargo.lock | 36 +- crates/mbe/Cargo.toml | 21 + crates/mbe/src/lib.rs | 278 ++++ crates/mbe/src/mbe_expander.rs | 180 +++ crates/mbe/src/mbe_expander/matcher.rs | 477 +++++++ crates/mbe/src/mbe_expander/transcriber.rs | 254 ++++ crates/mbe/src/parser.rs | 184 +++ crates/mbe/src/subtree_source.rs | 197 +++ crates/mbe/src/syntax_bridge.rs | 832 +++++++++++ crates/mbe/src/tests.rs | 1898 +++++++++++++++++++++++++ crates/mbe/src/tt_iter.rs | 75 + crates/proc_macro_srv/Cargo.toml | 2 +- crates/proc_macro_srv/src/rustc_server.rs | 2 +- crates/ra_cfg/Cargo.toml | 2 +- crates/ra_cfg/src/cfg_expr.rs | 2 +- crates/ra_hir_def/Cargo.toml | 2 +- crates/ra_hir_expand/Cargo.toml | 2 +- crates/ra_mbe/Cargo.toml | 20 - crates/ra_mbe/src/lib.rs | 278 ---- crates/ra_mbe/src/mbe_expander.rs | 180 --- crates/ra_mbe/src/mbe_expander/matcher.rs | 477 ------- crates/ra_mbe/src/mbe_expander/transcriber.rs | 254 ---- crates/ra_mbe/src/parser.rs | 184 --- crates/ra_mbe/src/subtree_source.rs | 197 --- crates/ra_mbe/src/syntax_bridge.rs | 832 ----------- crates/ra_mbe/src/tests.rs | 1898 ------------------------- crates/ra_mbe/src/tt_iter.rs | 75 - crates/rust-analyzer/Cargo.toml | 2 +- xtask/tests/tidy.rs | 2 +- 29 files changed, 4422 insertions(+), 4421 deletions(-) create mode 100644 crates/mbe/Cargo.toml create mode 100644 crates/mbe/src/lib.rs create mode 100644 crates/mbe/src/mbe_expander.rs create mode 100644 crates/mbe/src/mbe_expander/matcher.rs create mode 100644 crates/mbe/src/mbe_expander/transcriber.rs create mode 100644 crates/mbe/src/parser.rs create mode 100644 crates/mbe/src/subtree_source.rs create mode 100644 crates/mbe/src/syntax_bridge.rs create mode 100644 crates/mbe/src/tests.rs create mode 100644 crates/mbe/src/tt_iter.rs delete mode 100644 crates/ra_mbe/Cargo.toml delete mode 100644 crates/ra_mbe/src/lib.rs delete mode 100644 crates/ra_mbe/src/mbe_expander.rs delete mode 100644 crates/ra_mbe/src/mbe_expander/matcher.rs delete mode 100644 crates/ra_mbe/src/mbe_expander/transcriber.rs delete mode 100644 crates/ra_mbe/src/parser.rs delete mode 100644 crates/ra_mbe/src/subtree_source.rs delete mode 100644 crates/ra_mbe/src/syntax_bridge.rs delete mode 100644 crates/ra_mbe/src/tests.rs delete mode 100644 crates/ra_mbe/src/tt_iter.rs diff --git a/Cargo.lock b/Cargo.lock index 6bace7bd2..a63cd58f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -635,6 +635,19 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" +[[package]] +name = "mbe" +version = "0.0.0" +dependencies = [ + "log", + "parser", + "rustc-hash", + "smallvec", + "syntax", + "test_utils", + "tt", +] + [[package]] name = "memmap" version = "0.7.0" @@ -894,8 +907,8 @@ dependencies = [ "difference", "goblin", "libloading", + "mbe", "memmap", - "ra_mbe", "ra_proc_macro", "serde_derive", "test_utils", @@ -946,7 +959,7 @@ dependencies = [ name = "ra_cfg" version = "0.1.0" dependencies = [ - "ra_mbe", + "mbe", "rustc-hash", "syntax", "tt", @@ -1006,12 +1019,12 @@ dependencies = [ "indexmap", "itertools", "log", + "mbe", "once_cell", "profile", "ra_cfg", "ra_db", "ra_hir_expand", - "ra_mbe", "rustc-hash", "smallvec", "stdx", @@ -1027,10 +1040,10 @@ dependencies = [ "arena", "either", "log", + "mbe", "parser", "profile", "ra_db", - "ra_mbe", "rustc-hash", "syntax", "test_utils", @@ -1109,19 +1122,6 @@ dependencies = [ "text_edit", ] -[[package]] -name = "ra_mbe" -version = "0.1.0" -dependencies = [ - "log", - "parser", - "rustc-hash", - "smallvec", - "syntax", - "test_utils", - "tt", -] - [[package]] name = "ra_proc_macro" version = "0.1.0" @@ -1249,6 +1249,7 @@ dependencies = [ "log", "lsp-server", "lsp-types", + "mbe", "mimalloc", "oorandom", "parking_lot", @@ -1262,7 +1263,6 @@ dependencies = [ "ra_hir_ty", "ra_ide", "ra_ide_db", - "ra_mbe", "ra_project_model", "ra_ssr", "rayon", diff --git a/crates/mbe/Cargo.toml b/crates/mbe/Cargo.toml new file mode 100644 index 000000000..1aba8b7c4 --- /dev/null +++ b/crates/mbe/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "mbe" +version = "0.0.0" +license = "MIT OR Apache-2.0" +authors = ["rust-analyzer developers"] +edition = "2018" + +[lib] +doctest = false + +[dependencies] +rustc-hash = "1.1.0" +smallvec = "1.2.0" +log = "0.4.8" + +syntax = { path = "../syntax" } +parser = { path = "../parser" } +tt = { path = "../tt" } + +[dev-dependencies] +test_utils = { path = "../test_utils" } diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs new file mode 100644 index 000000000..f854ca09a --- /dev/null +++ b/crates/mbe/src/lib.rs @@ -0,0 +1,278 @@ +//! `mbe` (short for Macro By Example) crate contains code for handling +//! `macro_rules` macros. It uses `TokenTree` (from `tt` package) as the +//! interface, although it contains some code to bridge `SyntaxNode`s and +//! `TokenTree`s as well! + +mod parser; +mod mbe_expander; +mod syntax_bridge; +mod tt_iter; +mod subtree_source; + +#[cfg(test)] +mod tests; + +pub use tt::{Delimiter, Punct}; + +use crate::{ + parser::{parse_pattern, Op}, + tt_iter::TtIter, +}; + +#[derive(Debug, PartialEq, Eq)] +pub enum ParseError { + Expected(String), + RepetitionEmtpyTokenTree, +} + +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum ExpandError { + NoMatchingRule, + UnexpectedToken, + BindingError(String), + ConversionError, + InvalidRepeat, + ProcMacroError(tt::ExpansionError), +} + +impl From for ExpandError { + fn from(it: tt::ExpansionError) -> Self { + ExpandError::ProcMacroError(it) + } +} + +pub use crate::syntax_bridge::{ + ast_to_token_tree, parse_to_token_tree, syntax_node_to_token_tree, token_tree_to_syntax_node, + TokenMap, +}; + +/// This struct contains AST for a single `macro_rules` definition. What might +/// be very confusing is that AST has almost exactly the same shape as +/// `tt::TokenTree`, but there's a crucial difference: in macro rules, `$ident` +/// and `$()*` have special meaning (see `Var` and `Repeat` data structures) +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MacroRules { + rules: Vec, + /// Highest id of the token we have in TokenMap + shift: Shift, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct Rule { + lhs: tt::Subtree, + rhs: tt::Subtree, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Shift(u32); + +impl Shift { + fn new(tt: &tt::Subtree) -> Shift { + // Note that TokenId is started from zero, + // We have to add 1 to prevent duplication. + let value = max_id(tt).map_or(0, |it| it + 1); + return Shift(value); + + // Find the max token id inside a subtree + fn max_id(subtree: &tt::Subtree) -> Option { + subtree + .token_trees + .iter() + .filter_map(|tt| match tt { + tt::TokenTree::Subtree(subtree) => { + let tree_id = max_id(subtree); + match subtree.delimiter { + Some(it) if it.id != tt::TokenId::unspecified() => { + Some(tree_id.map_or(it.id.0, |t| t.max(it.id.0))) + } + _ => tree_id, + } + } + tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) + if ident.id != tt::TokenId::unspecified() => + { + Some(ident.id.0) + } + _ => None, + }) + .max() + } + } + + /// Shift given TokenTree token id + fn shift_all(self, tt: &mut tt::Subtree) { + for t in tt.token_trees.iter_mut() { + match t { + tt::TokenTree::Leaf(leaf) => match leaf { + tt::Leaf::Ident(ident) => ident.id = self.shift(ident.id), + tt::Leaf::Punct(punct) => punct.id = self.shift(punct.id), + tt::Leaf::Literal(lit) => lit.id = self.shift(lit.id), + }, + tt::TokenTree::Subtree(tt) => { + if let Some(it) = tt.delimiter.as_mut() { + it.id = self.shift(it.id); + }; + self.shift_all(tt) + } + } + } + } + + fn shift(self, id: tt::TokenId) -> tt::TokenId { + if id == tt::TokenId::unspecified() { + return id; + } + tt::TokenId(id.0 + self.0) + } + + fn unshift(self, id: tt::TokenId) -> Option { + id.0.checked_sub(self.0).map(tt::TokenId) + } +} + +#[derive(Debug, Eq, PartialEq)] +pub enum Origin { + Def, + Call, +} + +impl MacroRules { + pub fn parse(tt: &tt::Subtree) -> Result { + // Note: this parsing can be implemented using mbe machinery itself, by + // matching against `$($lhs:tt => $rhs:tt);*` pattern, but implementing + // manually seems easier. + let mut src = TtIter::new(tt); + let mut rules = Vec::new(); + while src.len() > 0 { + let rule = Rule::parse(&mut src)?; + rules.push(rule); + if let Err(()) = src.expect_char(';') { + if src.len() > 0 { + return Err(ParseError::Expected("expected `:`".to_string())); + } + break; + } + } + + for rule in rules.iter() { + validate(&rule.lhs)?; + } + + Ok(MacroRules { rules, shift: Shift::new(tt) }) + } + + pub fn expand(&self, tt: &tt::Subtree) -> ExpandResult { + // apply shift + let mut tt = tt.clone(); + self.shift.shift_all(&mut tt); + mbe_expander::expand(self, &tt) + } + + pub fn map_id_down(&self, id: tt::TokenId) -> tt::TokenId { + self.shift.shift(id) + } + + pub fn map_id_up(&self, id: tt::TokenId) -> (tt::TokenId, Origin) { + match self.shift.unshift(id) { + Some(id) => (id, Origin::Call), + None => (id, Origin::Def), + } + } +} + +impl Rule { + fn parse(src: &mut TtIter) -> Result { + let mut lhs = src + .expect_subtree() + .map_err(|()| ParseError::Expected("expected subtree".to_string()))? + .clone(); + lhs.delimiter = None; + src.expect_char('=').map_err(|()| ParseError::Expected("expected `=`".to_string()))?; + src.expect_char('>').map_err(|()| ParseError::Expected("expected `>`".to_string()))?; + let mut rhs = src + .expect_subtree() + .map_err(|()| ParseError::Expected("expected subtree".to_string()))? + .clone(); + rhs.delimiter = None; + Ok(crate::Rule { lhs, rhs }) + } +} + +fn to_parse_error(e: ExpandError) -> ParseError { + let msg = match e { + ExpandError::InvalidRepeat => "invalid repeat".to_string(), + _ => "invalid macro definition".to_string(), + }; + ParseError::Expected(msg) +} + +fn validate(pattern: &tt::Subtree) -> Result<(), ParseError> { + for op in parse_pattern(pattern) { + let op = op.map_err(to_parse_error)?; + + match op { + Op::TokenTree(tt::TokenTree::Subtree(subtree)) => validate(subtree)?, + Op::Repeat { subtree, separator, .. } => { + // Checks that no repetition which could match an empty token + // https://github.com/rust-lang/rust/blob/a58b1ed44f5e06976de2bdc4d7dc81c36a96934f/src/librustc_expand/mbe/macro_rules.rs#L558 + + if separator.is_none() { + if parse_pattern(subtree).all(|child_op| { + match child_op.map_err(to_parse_error) { + Ok(Op::Var { kind, .. }) => { + // vis is optional + if kind.map_or(false, |it| it == "vis") { + return true; + } + } + Ok(Op::Repeat { kind, .. }) => { + return matches!( + kind, + parser::RepeatKind::ZeroOrMore | parser::RepeatKind::ZeroOrOne + ) + } + _ => {} + } + false + }) { + return Err(ParseError::RepetitionEmtpyTokenTree); + } + } + validate(subtree)? + } + _ => (), + } + } + Ok(()) +} + +#[derive(Debug)] +pub struct ExpandResult(pub T, pub Option); + +impl ExpandResult { + pub fn ok(t: T) -> ExpandResult { + ExpandResult(t, None) + } + + pub fn only_err(err: ExpandError) -> ExpandResult + where + T: Default, + { + ExpandResult(Default::default(), Some(err)) + } + + pub fn map(self, f: impl FnOnce(T) -> U) -> ExpandResult { + ExpandResult(f(self.0), self.1) + } + + pub fn result(self) -> Result { + self.1.map(Err).unwrap_or(Ok(self.0)) + } +} + +impl From> for ExpandResult { + fn from(result: Result) -> ExpandResult { + result + .map_or_else(|e| ExpandResult(Default::default(), Some(e)), |it| ExpandResult(it, None)) + } +} diff --git a/crates/mbe/src/mbe_expander.rs b/crates/mbe/src/mbe_expander.rs new file mode 100644 index 000000000..1ad8b9f8a --- /dev/null +++ b/crates/mbe/src/mbe_expander.rs @@ -0,0 +1,180 @@ +//! This module takes a (parsed) definition of `macro_rules` invocation, a +//! `tt::TokenTree` representing an argument of macro invocation, and produces a +//! `tt::TokenTree` for the result of the expansion. + +mod matcher; +mod transcriber; + +use rustc_hash::FxHashMap; +use syntax::SmolStr; + +use crate::{ExpandError, ExpandResult}; + +pub(crate) fn expand(rules: &crate::MacroRules, input: &tt::Subtree) -> ExpandResult { + expand_rules(&rules.rules, input) +} + +fn expand_rules(rules: &[crate::Rule], input: &tt::Subtree) -> ExpandResult { + let mut match_: Option<(matcher::Match, &crate::Rule)> = None; + for rule in rules { + let new_match = match matcher::match_(&rule.lhs, input) { + Ok(m) => m, + Err(_e) => { + // error in pattern parsing + continue; + } + }; + if new_match.err.is_none() { + // If we find a rule that applies without errors, we're done. + // Unconditionally returning the transcription here makes the + // `test_repeat_bad_var` test fail. + let ExpandResult(res, transcribe_err) = + transcriber::transcribe(&rule.rhs, &new_match.bindings); + if transcribe_err.is_none() { + return ExpandResult::ok(res); + } + } + // Use the rule if we matched more tokens, or had fewer errors + if let Some((prev_match, _)) = &match_ { + if (new_match.unmatched_tts, new_match.err_count) + < (prev_match.unmatched_tts, prev_match.err_count) + { + match_ = Some((new_match, rule)); + } + } else { + match_ = Some((new_match, rule)); + } + } + if let Some((match_, rule)) = match_ { + // if we got here, there was no match without errors + let ExpandResult(result, transcribe_err) = + transcriber::transcribe(&rule.rhs, &match_.bindings); + ExpandResult(result, match_.err.or(transcribe_err)) + } else { + ExpandResult(tt::Subtree::default(), Some(ExpandError::NoMatchingRule)) + } +} + +/// The actual algorithm for expansion is not too hard, but is pretty tricky. +/// `Bindings` structure is the key to understanding what we are doing here. +/// +/// On the high level, it stores mapping from meta variables to the bits of +/// syntax it should be substituted with. For example, if `$e:expr` is matched +/// with `1 + 1` by macro_rules, the `Binding` will store `$e -> 1 + 1`. +/// +/// The tricky bit is dealing with repetitions (`$()*`). Consider this example: +/// +/// ```not_rust +/// macro_rules! foo { +/// ($($ i:ident $($ e:expr),*);*) => { +/// $(fn $ i() { $($ e);*; })* +/// } +/// } +/// foo! { foo 1,2,3; bar 4,5,6 } +/// ``` +/// +/// Here, the `$i` meta variable is matched first with `foo` and then with +/// `bar`, and `$e` is matched in turn with `1`, `2`, `3`, `4`, `5`, `6`. +/// +/// To represent such "multi-mappings", we use a recursive structures: we map +/// variables not to values, but to *lists* of values or other lists (that is, +/// to the trees). +/// +/// For the above example, the bindings would store +/// +/// ```not_rust +/// i -> [foo, bar] +/// e -> [[1, 2, 3], [4, 5, 6]] +/// ``` +/// +/// We construct `Bindings` in the `match_lhs`. The interesting case is +/// `TokenTree::Repeat`, where we use `push_nested` to create the desired +/// nesting structure. +/// +/// The other side of the puzzle is `expand_subtree`, where we use the bindings +/// to substitute meta variables in the output template. When expanding, we +/// maintain a `nesting` stack of indices which tells us which occurrence from +/// the `Bindings` we should take. We push to the stack when we enter a +/// repetition. +/// +/// In other words, `Bindings` is a *multi* mapping from `SmolStr` to +/// `tt::TokenTree`, where the index to select a particular `TokenTree` among +/// many is not a plain `usize`, but an `&[usize]`. +#[derive(Debug, Default)] +struct Bindings { + inner: FxHashMap, +} + +#[derive(Debug)] +enum Binding { + Fragment(Fragment), + Nested(Vec), + Empty, +} + +#[derive(Debug, Clone)] +enum Fragment { + /// token fragments are just copy-pasted into the output + Tokens(tt::TokenTree), + /// Ast fragments are inserted with fake delimiters, so as to make things + /// like `$i * 2` where `$i = 1 + 1` work as expectd. + Ast(tt::TokenTree), +} + +#[cfg(test)] +mod tests { + use syntax::{ast, AstNode}; + + use super::*; + use crate::ast_to_token_tree; + + #[test] + fn test_expand_rule() { + assert_err( + "($($i:ident);*) => ($i)", + "foo!{a}", + ExpandError::BindingError(String::from( + "expected simple binding, found nested binding `i`", + )), + ); + + // FIXME: + // Add an err test case for ($($i:ident)) => ($()) + } + + fn assert_err(macro_body: &str, invocation: &str, err: ExpandError) { + assert_eq!(expand_first(&create_rules(&format_macro(macro_body)), invocation).1, Some(err)); + } + + fn format_macro(macro_body: &str) -> String { + format!( + " + macro_rules! foo {{ + {} + }} +", + macro_body + ) + } + + fn create_rules(macro_definition: &str) -> crate::MacroRules { + let source_file = ast::SourceFile::parse(macro_definition).ok().unwrap(); + let macro_definition = + source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); + + let (definition_tt, _) = + ast_to_token_tree(¯o_definition.token_tree().unwrap()).unwrap(); + crate::MacroRules::parse(&definition_tt).unwrap() + } + + fn expand_first(rules: &crate::MacroRules, invocation: &str) -> ExpandResult { + let source_file = ast::SourceFile::parse(invocation).ok().unwrap(); + let macro_invocation = + source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); + + let (invocation_tt, _) = + ast_to_token_tree(¯o_invocation.token_tree().unwrap()).unwrap(); + + expand_rules(&rules.rules, &invocation_tt) + } +} diff --git a/crates/mbe/src/mbe_expander/matcher.rs b/crates/mbe/src/mbe_expander/matcher.rs new file mode 100644 index 000000000..b698b9832 --- /dev/null +++ b/crates/mbe/src/mbe_expander/matcher.rs @@ -0,0 +1,477 @@ +//! FIXME: write short doc here + +use crate::{ + mbe_expander::{Binding, Bindings, Fragment}, + parser::{parse_pattern, Op, RepeatKind, Separator}, + subtree_source::SubtreeTokenSource, + tt_iter::TtIter, + ExpandError, +}; + +use super::ExpandResult; +use parser::{FragmentKind::*, TreeSink}; +use syntax::{SmolStr, SyntaxKind}; +use tt::buffer::{Cursor, TokenBuffer}; + +impl Bindings { + fn push_optional(&mut self, name: &SmolStr) { + // FIXME: Do we have a better way to represent an empty token ? + // Insert an empty subtree for empty token + let tt = tt::Subtree::default().into(); + self.inner.insert(name.clone(), Binding::Fragment(Fragment::Tokens(tt))); + } + + fn push_empty(&mut self, name: &SmolStr) { + self.inner.insert(name.clone(), Binding::Empty); + } + + fn push_nested(&mut self, idx: usize, nested: Bindings) -> Result<(), ExpandError> { + for (key, value) in nested.inner { + if !self.inner.contains_key(&key) { + self.inner.insert(key.clone(), Binding::Nested(Vec::new())); + } + match self.inner.get_mut(&key) { + Some(Binding::Nested(it)) => { + // insert empty nested bindings before this one + while it.len() < idx { + it.push(Binding::Nested(vec![])); + } + it.push(value); + } + _ => { + return Err(ExpandError::BindingError(format!( + "could not find binding `{}`", + key + ))); + } + } + } + Ok(()) + } +} + +macro_rules! err { + () => { + ExpandError::BindingError(format!("")) + }; + ($($tt:tt)*) => { + ExpandError::BindingError(format!($($tt)*)) + }; +} + +#[derive(Debug, Default)] +pub(super) struct Match { + pub bindings: Bindings, + /// We currently just keep the first error and count the rest to compare matches. + pub err: Option, + pub err_count: usize, + /// How many top-level token trees were left to match. + pub unmatched_tts: usize, +} + +impl Match { + pub fn add_err(&mut self, err: ExpandError) { + let prev_err = self.err.take(); + self.err = prev_err.or(Some(err)); + self.err_count += 1; + } +} + +// General note: These functions have two channels to return errors, a `Result` +// return value and the `&mut Match`. The returned Result is for pattern parsing +// errors; if a branch of the macro definition doesn't parse, it doesn't make +// sense to try using it. Matching errors are added to the `Match`. It might +// make sense to make pattern parsing a separate step? + +pub(super) fn match_(pattern: &tt::Subtree, src: &tt::Subtree) -> Result { + assert!(pattern.delimiter == None); + + let mut res = Match::default(); + let mut src = TtIter::new(src); + + match_subtree(&mut res, pattern, &mut src)?; + + if src.len() > 0 { + res.unmatched_tts += src.len(); + res.add_err(err!("leftover tokens")); + } + + Ok(res) +} + +fn match_subtree( + res: &mut Match, + pattern: &tt::Subtree, + src: &mut TtIter, +) -> Result<(), ExpandError> { + for op in parse_pattern(pattern) { + match op? { + Op::TokenTree(tt::TokenTree::Leaf(lhs)) => { + let rhs = match src.expect_leaf() { + Ok(l) => l, + Err(()) => { + res.add_err(err!("expected leaf: `{}`", lhs)); + continue; + } + }; + match (lhs, rhs) { + ( + tt::Leaf::Punct(tt::Punct { char: lhs, .. }), + tt::Leaf::Punct(tt::Punct { char: rhs, .. }), + ) if lhs == rhs => (), + ( + tt::Leaf::Ident(tt::Ident { text: lhs, .. }), + tt::Leaf::Ident(tt::Ident { text: rhs, .. }), + ) if lhs == rhs => (), + ( + tt::Leaf::Literal(tt::Literal { text: lhs, .. }), + tt::Leaf::Literal(tt::Literal { text: rhs, .. }), + ) if lhs == rhs => (), + _ => { + res.add_err(ExpandError::UnexpectedToken); + } + } + } + Op::TokenTree(tt::TokenTree::Subtree(lhs)) => { + let rhs = match src.expect_subtree() { + Ok(s) => s, + Err(()) => { + res.add_err(err!("expected subtree")); + continue; + } + }; + if lhs.delimiter_kind() != rhs.delimiter_kind() { + res.add_err(err!("mismatched delimiter")); + continue; + } + let mut src = TtIter::new(rhs); + match_subtree(res, lhs, &mut src)?; + if src.len() > 0 { + res.add_err(err!("leftover tokens")); + } + } + Op::Var { name, kind } => { + let kind = match kind { + Some(k) => k, + None => { + res.add_err(ExpandError::UnexpectedToken); + continue; + } + }; + let ExpandResult(matched, match_err) = match_meta_var(kind.as_str(), src); + match matched { + Some(fragment) => { + res.bindings.inner.insert(name.clone(), Binding::Fragment(fragment)); + } + None if match_err.is_none() => res.bindings.push_optional(name), + _ => {} + } + if let Some(err) = match_err { + res.add_err(err); + } + } + Op::Repeat { subtree, kind, separator } => { + match_repeat(res, subtree, kind, separator, src)?; + } + } + } + Ok(()) +} + +impl<'a> TtIter<'a> { + fn eat_separator(&mut self, separator: &Separator) -> bool { + let mut fork = self.clone(); + let ok = match separator { + Separator::Ident(lhs) => match fork.expect_ident() { + Ok(rhs) => rhs.text == lhs.text, + _ => false, + }, + Separator::Literal(lhs) => match fork.expect_literal() { + Ok(rhs) => match rhs { + tt::Leaf::Literal(rhs) => rhs.text == lhs.text, + tt::Leaf::Ident(rhs) => rhs.text == lhs.text, + tt::Leaf::Punct(_) => false, + }, + _ => false, + }, + Separator::Puncts(lhss) => lhss.iter().all(|lhs| match fork.expect_punct() { + Ok(rhs) => rhs.char == lhs.char, + _ => false, + }), + }; + if ok { + *self = fork; + } + ok + } + + pub(crate) fn expect_tt(&mut self) -> Result { + match self.peek_n(0) { + Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '\'' => { + return self.expect_lifetime(); + } + _ => (), + } + + let tt = self.next().ok_or_else(|| ())?.clone(); + let punct = match tt { + tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if punct.spacing == tt::Spacing::Joint => { + punct + } + _ => return Ok(tt), + }; + + let (second, third) = match (self.peek_n(0), self.peek_n(1)) { + ( + Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p2))), + Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p3))), + ) if p2.spacing == tt::Spacing::Joint => (p2.char, Some(p3.char)), + (Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p2))), _) => (p2.char, None), + _ => return Ok(tt), + }; + + match (punct.char, second, third) { + ('.', '.', Some('.')) + | ('.', '.', Some('=')) + | ('<', '<', Some('=')) + | ('>', '>', Some('=')) => { + let tt2 = self.next().unwrap().clone(); + let tt3 = self.next().unwrap().clone(); + Ok(tt::Subtree { delimiter: None, token_trees: vec![tt, tt2, tt3] }.into()) + } + ('-', '=', None) + | ('-', '>', None) + | (':', ':', None) + | ('!', '=', None) + | ('.', '.', None) + | ('*', '=', None) + | ('/', '=', None) + | ('&', '&', None) + | ('&', '=', None) + | ('%', '=', None) + | ('^', '=', None) + | ('+', '=', None) + | ('<', '<', None) + | ('<', '=', None) + | ('=', '=', None) + | ('=', '>', None) + | ('>', '=', None) + | ('>', '>', None) + | ('|', '=', None) + | ('|', '|', None) => { + let tt2 = self.next().unwrap().clone(); + Ok(tt::Subtree { delimiter: None, token_trees: vec![tt, tt2] }.into()) + } + _ => Ok(tt), + } + } + + pub(crate) fn expect_lifetime(&mut self) -> Result { + let punct = self.expect_punct()?; + if punct.char != '\'' { + return Err(()); + } + let ident = self.expect_ident()?; + + Ok(tt::Subtree { + delimiter: None, + token_trees: vec![ + tt::Leaf::Punct(*punct).into(), + tt::Leaf::Ident(ident.clone()).into(), + ], + } + .into()) + } + + pub(crate) fn expect_fragment( + &mut self, + fragment_kind: parser::FragmentKind, + ) -> ExpandResult> { + pub(crate) struct OffsetTokenSink<'a> { + pub(crate) cursor: Cursor<'a>, + pub(crate) error: bool, + } + + impl<'a> TreeSink for OffsetTokenSink<'a> { + fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) { + if kind == SyntaxKind::LIFETIME { + n_tokens = 2; + } + for _ in 0..n_tokens { + self.cursor = self.cursor.bump_subtree(); + } + } + fn start_node(&mut self, _kind: SyntaxKind) {} + fn finish_node(&mut self) {} + fn error(&mut self, _error: parser::ParseError) { + self.error = true; + } + } + + let buffer = TokenBuffer::new(&self.inner.as_slice()); + let mut src = SubtreeTokenSource::new(&buffer); + let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false }; + + parser::parse_fragment(&mut src, &mut sink, fragment_kind); + + let mut err = None; + if !sink.cursor.is_root() || sink.error { + err = Some(err!("expected {:?}", fragment_kind)); + } + + let mut curr = buffer.begin(); + let mut res = vec![]; + + if sink.cursor.is_root() { + while curr != sink.cursor { + if let Some(token) = curr.token_tree() { + res.push(token); + } + curr = curr.bump(); + } + } + self.inner = self.inner.as_slice()[res.len()..].iter(); + if res.len() == 0 && err.is_none() { + err = Some(err!("no tokens consumed")); + } + let res = match res.len() { + 1 => Some(res[0].clone()), + 0 => None, + _ => Some(tt::TokenTree::Subtree(tt::Subtree { + delimiter: None, + token_trees: res.into_iter().cloned().collect(), + })), + }; + ExpandResult(res, err) + } + + pub(crate) fn eat_vis(&mut self) -> Option { + let mut fork = self.clone(); + match fork.expect_fragment(Visibility) { + ExpandResult(tt, None) => { + *self = fork; + tt + } + ExpandResult(_, Some(_)) => None, + } + } +} + +pub(super) fn match_repeat( + res: &mut Match, + pattern: &tt::Subtree, + kind: RepeatKind, + separator: Option, + src: &mut TtIter, +) -> Result<(), ExpandError> { + // Dirty hack to make macro-expansion terminate. + // This should be replaced by a propper macro-by-example implementation + let mut limit = 65536; + let mut counter = 0; + + for i in 0.. { + let mut fork = src.clone(); + + if let Some(separator) = &separator { + if i != 0 && !fork.eat_separator(separator) { + break; + } + } + + let mut nested = Match::default(); + match_subtree(&mut nested, pattern, &mut fork)?; + if nested.err.is_none() { + limit -= 1; + if limit == 0 { + log::warn!( + "match_lhs exceeded repeat pattern limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", + pattern, + src, + kind, + separator + ); + break; + } + *src = fork; + + if let Err(err) = res.bindings.push_nested(counter, nested.bindings) { + res.add_err(err); + } + counter += 1; + if counter == 1 { + if let RepeatKind::ZeroOrOne = kind { + break; + } + } + } else { + break; + } + } + + match (kind, counter) { + (RepeatKind::OneOrMore, 0) => { + res.add_err(ExpandError::UnexpectedToken); + } + (_, 0) => { + // Collect all empty variables in subtrees + let mut vars = Vec::new(); + collect_vars(&mut vars, pattern)?; + for var in vars { + res.bindings.push_empty(&var) + } + } + _ => (), + } + Ok(()) +} + +fn match_meta_var(kind: &str, input: &mut TtIter) -> ExpandResult> { + let fragment = match kind { + "path" => Path, + "expr" => Expr, + "ty" => Type, + "pat" => Pattern, + "stmt" => Statement, + "block" => Block, + "meta" => MetaItem, + "item" => Item, + _ => { + let tt_result = match kind { + "ident" => input + .expect_ident() + .map(|ident| Some(tt::Leaf::from(ident.clone()).into())) + .map_err(|()| err!("expected ident")), + "tt" => input.expect_tt().map(Some).map_err(|()| err!()), + "lifetime" => input + .expect_lifetime() + .map(|tt| Some(tt)) + .map_err(|()| err!("expected lifetime")), + "literal" => input + .expect_literal() + .map(|literal| Some(tt::Leaf::from(literal.clone()).into())) + .map_err(|()| err!()), + // `vis` is optional + "vis" => match input.eat_vis() { + Some(vis) => Ok(Some(vis)), + None => Ok(None), + }, + _ => Err(ExpandError::UnexpectedToken), + }; + return tt_result.map(|it| it.map(Fragment::Tokens)).into(); + } + }; + let result = input.expect_fragment(fragment); + result.map(|tt| if kind == "expr" { tt.map(Fragment::Ast) } else { tt.map(Fragment::Tokens) }) +} + +fn collect_vars(buf: &mut Vec, pattern: &tt::Subtree) -> Result<(), ExpandError> { + for op in parse_pattern(pattern) { + match op? { + Op::Var { name, .. } => buf.push(name.clone()), + Op::TokenTree(tt::TokenTree::Leaf(_)) => (), + Op::TokenTree(tt::TokenTree::Subtree(subtree)) => collect_vars(buf, subtree)?, + Op::Repeat { subtree, .. } => collect_vars(buf, subtree)?, + } + } + Ok(()) +} diff --git a/crates/mbe/src/mbe_expander/transcriber.rs b/crates/mbe/src/mbe_expander/transcriber.rs new file mode 100644 index 000000000..c9525c5bf --- /dev/null +++ b/crates/mbe/src/mbe_expander/transcriber.rs @@ -0,0 +1,254 @@ +//! Transcriber takes a template, like `fn $ident() {}`, a set of bindings like +//! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}` + +use syntax::SmolStr; + +use super::ExpandResult; +use crate::{ + mbe_expander::{Binding, Bindings, Fragment}, + parser::{parse_template, Op, RepeatKind, Separator}, + ExpandError, +}; + +impl Bindings { + fn contains(&self, name: &str) -> bool { + self.inner.contains_key(name) + } + + fn get(&self, name: &str, nesting: &mut [NestingState]) -> Result<&Fragment, ExpandError> { + let mut b = self.inner.get(name).ok_or_else(|| { + ExpandError::BindingError(format!("could not find binding `{}`", name)) + })?; + for nesting_state in nesting.iter_mut() { + nesting_state.hit = true; + b = match b { + Binding::Fragment(_) => break, + Binding::Nested(bs) => bs.get(nesting_state.idx).ok_or_else(|| { + nesting_state.at_end = true; + ExpandError::BindingError(format!("could not find nested binding `{}`", name)) + })?, + Binding::Empty => { + nesting_state.at_end = true; + return Err(ExpandError::BindingError(format!( + "could not find empty binding `{}`", + name + ))); + } + }; + } + match b { + Binding::Fragment(it) => Ok(it), + Binding::Nested(_) => Err(ExpandError::BindingError(format!( + "expected simple binding, found nested binding `{}`", + name + ))), + Binding::Empty => Err(ExpandError::BindingError(format!( + "expected simple binding, found empty binding `{}`", + name + ))), + } + } +} + +pub(super) fn transcribe(template: &tt::Subtree, bindings: &Bindings) -> ExpandResult { + assert!(template.delimiter == None); + let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new() }; + let mut arena: Vec = Vec::new(); + expand_subtree(&mut ctx, template, &mut arena) +} + +#[derive(Debug)] +struct NestingState { + idx: usize, + /// `hit` is currently necessary to tell `expand_repeat` if it should stop + /// because there is no variable in use by the current repetition + hit: bool, + /// `at_end` is currently necessary to tell `expand_repeat` if it should stop + /// because there is no more value avaible for the current repetition + at_end: bool, +} + +#[derive(Debug)] +struct ExpandCtx<'a> { + bindings: &'a Bindings, + nesting: Vec, +} + +fn expand_subtree( + ctx: &mut ExpandCtx, + template: &tt::Subtree, + arena: &mut Vec, +) -> ExpandResult { + // remember how many elements are in the arena now - when returning, we want to drain exactly how many elements we added. This way, the recursive uses of the arena get their own "view" of the arena, but will reuse the allocation + let start_elements = arena.len(); + let mut err = None; + for op in parse_template(template) { + let op = match op { + Ok(op) => op, + Err(e) => { + err = Some(e); + break; + } + }; + match op { + Op::TokenTree(tt @ tt::TokenTree::Leaf(..)) => arena.push(tt.clone()), + Op::TokenTree(tt::TokenTree::Subtree(tt)) => { + let ExpandResult(tt, e) = expand_subtree(ctx, tt, arena); + err = err.or(e); + arena.push(tt.into()); + } + Op::Var { name, kind: _ } => { + let ExpandResult(fragment, e) = expand_var(ctx, name); + err = err.or(e); + push_fragment(arena, fragment); + } + Op::Repeat { subtree, kind, separator } => { + let ExpandResult(fragment, e) = expand_repeat(ctx, subtree, kind, separator, arena); + err = err.or(e); + push_fragment(arena, fragment) + } + } + } + // drain the elements added in this instance of expand_subtree + let tts = arena.drain(start_elements..arena.len()).collect(); + ExpandResult(tt::Subtree { delimiter: template.delimiter, token_trees: tts }, err) +} + +fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> ExpandResult { + if v == "crate" { + // We simply produce identifier `$crate` here. And it will be resolved when lowering ast to Path. + let tt = + tt::Leaf::from(tt::Ident { text: "$crate".into(), id: tt::TokenId::unspecified() }) + .into(); + ExpandResult::ok(Fragment::Tokens(tt)) + } else if !ctx.bindings.contains(v) { + // Note that it is possible to have a `$var` inside a macro which is not bound. + // For example: + // ``` + // macro_rules! foo { + // ($a:ident, $b:ident, $c:tt) => { + // macro_rules! bar { + // ($bi:ident) => { + // fn $bi() -> u8 {$c} + // } + // } + // } + // ``` + // We just treat it a normal tokens + let tt = tt::Subtree { + delimiter: None, + token_trees: vec![ + tt::Leaf::from(tt::Punct { + char: '$', + spacing: tt::Spacing::Alone, + id: tt::TokenId::unspecified(), + }) + .into(), + tt::Leaf::from(tt::Ident { text: v.clone(), id: tt::TokenId::unspecified() }) + .into(), + ], + } + .into(); + ExpandResult::ok(Fragment::Tokens(tt)) + } else { + ctx.bindings.get(&v, &mut ctx.nesting).map_or_else( + |e| ExpandResult(Fragment::Tokens(tt::TokenTree::empty()), Some(e)), + |b| ExpandResult::ok(b.clone()), + ) + } +} + +fn expand_repeat( + ctx: &mut ExpandCtx, + template: &tt::Subtree, + kind: RepeatKind, + separator: Option, + arena: &mut Vec, +) -> ExpandResult { + let mut buf: Vec = Vec::new(); + ctx.nesting.push(NestingState { idx: 0, at_end: false, hit: false }); + // Dirty hack to make macro-expansion terminate. + // This should be replaced by a proper macro-by-example implementation + let limit = 65536; + let mut has_seps = 0; + let mut counter = 0; + + loop { + let ExpandResult(mut t, e) = expand_subtree(ctx, template, arena); + let nesting_state = ctx.nesting.last_mut().unwrap(); + if nesting_state.at_end || !nesting_state.hit { + break; + } + nesting_state.idx += 1; + nesting_state.hit = false; + + counter += 1; + if counter == limit { + log::warn!( + "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}", + template, + ctx + ); + break; + } + + if e.is_some() { + continue; + } + + t.delimiter = None; + push_subtree(&mut buf, t); + + if let Some(ref sep) = separator { + match sep { + Separator::Ident(ident) => { + has_seps = 1; + buf.push(tt::Leaf::from(ident.clone()).into()); + } + Separator::Literal(lit) => { + has_seps = 1; + buf.push(tt::Leaf::from(lit.clone()).into()); + } + + Separator::Puncts(puncts) => { + has_seps = puncts.len(); + for punct in puncts { + buf.push(tt::Leaf::from(*punct).into()); + } + } + } + } + + if RepeatKind::ZeroOrOne == kind { + break; + } + } + + ctx.nesting.pop().unwrap(); + for _ in 0..has_seps { + buf.pop(); + } + + // Check if it is a single token subtree without any delimiter + // e.g {Delimiter:None> ['>'] /Delimiter:None>} + let tt = tt::Subtree { delimiter: None, token_trees: buf }.into(); + + if RepeatKind::OneOrMore == kind && counter == 0 { + return ExpandResult(Fragment::Tokens(tt), Some(ExpandError::UnexpectedToken)); + } + ExpandResult::ok(Fragment::Tokens(tt)) +} + +fn push_fragment(buf: &mut Vec, fragment: Fragment) { + match fragment { + Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt), + Fragment::Tokens(tt) | Fragment::Ast(tt) => buf.push(tt), + } +} + +fn push_subtree(buf: &mut Vec, tt: tt::Subtree) { + match tt.delimiter { + None => buf.extend(tt.token_trees), + _ => buf.push(tt.into()), + } +} diff --git a/crates/mbe/src/parser.rs b/crates/mbe/src/parser.rs new file mode 100644 index 000000000..6b46a1673 --- /dev/null +++ b/crates/mbe/src/parser.rs @@ -0,0 +1,184 @@ +//! Parser recognizes special macro syntax, `$var` and `$(repeat)*`, in token +//! trees. + +use smallvec::SmallVec; +use syntax::SmolStr; + +use crate::{tt_iter::TtIter, ExpandError}; + +#[derive(Debug)] +pub(crate) enum Op<'a> { + Var { name: &'a SmolStr, kind: Option<&'a SmolStr> }, + Repeat { subtree: &'a tt::Subtree, kind: RepeatKind, separator: Option }, + TokenTree(&'a tt::TokenTree), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum RepeatKind { + ZeroOrMore, + OneOrMore, + ZeroOrOne, +} + +#[derive(Clone, Debug, Eq)] +pub(crate) enum Separator { + Literal(tt::Literal), + Ident(tt::Ident), + Puncts(SmallVec<[tt::Punct; 3]>), +} + +// Note that when we compare a Separator, we just care about its textual value. +impl PartialEq for Separator { + fn eq(&self, other: &Separator) -> bool { + use Separator::*; + + match (self, other) { + (Ident(ref a), Ident(ref b)) => a.text == b.text, + (Literal(ref a), Literal(ref b)) => a.text == b.text, + (Puncts(ref a), Puncts(ref b)) if a.len() == b.len() => { + let a_iter = a.iter().map(|a| a.char); + let b_iter = b.iter().map(|b| b.char); + a_iter.eq(b_iter) + } + _ => false, + } + } +} + +pub(crate) fn parse_template( + template: &tt::Subtree, +) -> impl Iterator, ExpandError>> { + parse_inner(template, Mode::Template) +} + +pub(crate) fn parse_pattern( + pattern: &tt::Subtree, +) -> impl Iterator, ExpandError>> { + parse_inner(pattern, Mode::Pattern) +} + +#[derive(Clone, Copy)] +enum Mode { + Pattern, + Template, +} + +fn parse_inner(src: &tt::Subtree, mode: Mode) -> impl Iterator, ExpandError>> { + let mut src = TtIter::new(src); + std::iter::from_fn(move || { + let first = src.next()?; + Some(next_op(first, &mut src, mode)) + }) +} + +macro_rules! err { + ($($tt:tt)*) => { + ExpandError::UnexpectedToken + }; +} + +macro_rules! bail { + ($($tt:tt)*) => { + return Err(err!($($tt)*)) + }; +} + +fn next_op<'a>( + first: &'a tt::TokenTree, + src: &mut TtIter<'a>, + mode: Mode, +) -> Result, ExpandError> { + let res = match first { + tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: '$', .. })) => { + // Note that the '$' itself is a valid token inside macro_rules. + let second = match src.next() { + None => return Ok(Op::TokenTree(first)), + Some(it) => it, + }; + match second { + tt::TokenTree::Subtree(subtree) => { + let (separator, kind) = parse_repeat(src)?; + Op::Repeat { subtree, separator, kind } + } + tt::TokenTree::Leaf(leaf) => match leaf { + tt::Leaf::Punct(..) => return Err(ExpandError::UnexpectedToken), + tt::Leaf::Ident(ident) => { + let name = &ident.text; + let kind = eat_fragment_kind(src, mode)?; + Op::Var { name, kind } + } + tt::Leaf::Literal(lit) => { + if is_boolean_literal(lit) { + let name = &lit.text; + let kind = eat_fragment_kind(src, mode)?; + Op::Var { name, kind } + } else { + bail!("bad var 2"); + } + } + }, + } + } + tt => Op::TokenTree(tt), + }; + Ok(res) +} + +fn eat_fragment_kind<'a>( + src: &mut TtIter<'a>, + mode: Mode, +) -> Result, ExpandError> { + if let Mode::Pattern = mode { + src.expect_char(':').map_err(|()| err!("bad fragment specifier 1"))?; + let ident = src.expect_ident().map_err(|()| err!("bad fragment specifier 1"))?; + return Ok(Some(&ident.text)); + }; + Ok(None) +} + +fn is_boolean_literal(lit: &tt::Literal) -> bool { + matches!(lit.text.as_str(), "true" | "false") +} + +fn parse_repeat(src: &mut TtIter) -> Result<(Option, RepeatKind), ExpandError> { + let mut separator = Separator::Puncts(SmallVec::new()); + for tt in src { + let tt = match tt { + tt::TokenTree::Leaf(leaf) => leaf, + tt::TokenTree::Subtree(_) => return Err(ExpandError::InvalidRepeat), + }; + let has_sep = match &separator { + Separator::Puncts(puncts) => !puncts.is_empty(), + _ => true, + }; + match tt { + tt::Leaf::Ident(_) | tt::Leaf::Literal(_) if has_sep => { + return Err(ExpandError::InvalidRepeat) + } + tt::Leaf::Ident(ident) => separator = Separator::Ident(ident.clone()), + tt::Leaf::Literal(lit) => separator = Separator::Literal(lit.clone()), + tt::Leaf::Punct(punct) => { + let repeat_kind = match punct.char { + '*' => RepeatKind::ZeroOrMore, + '+' => RepeatKind::OneOrMore, + '?' => RepeatKind::ZeroOrOne, + _ => { + match &mut separator { + Separator::Puncts(puncts) => { + if puncts.len() == 3 { + return Err(ExpandError::InvalidRepeat); + } + puncts.push(punct.clone()) + } + _ => return Err(ExpandError::InvalidRepeat), + } + continue; + } + }; + let separator = if has_sep { Some(separator) } else { None }; + return Ok((separator, repeat_kind)); + } + } + } + Err(ExpandError::InvalidRepeat) +} diff --git a/crates/mbe/src/subtree_source.rs b/crates/mbe/src/subtree_source.rs new file mode 100644 index 000000000..41461b315 --- /dev/null +++ b/crates/mbe/src/subtree_source.rs @@ -0,0 +1,197 @@ +//! FIXME: write short doc here + +use parser::{Token, TokenSource}; +use std::cell::{Cell, Ref, RefCell}; +use syntax::{lex_single_syntax_kind, SmolStr, SyntaxKind, SyntaxKind::*, T}; +use tt::buffer::{Cursor, TokenBuffer}; + +#[derive(Debug, Clone, Eq, PartialEq)] +struct TtToken { + pub kind: SyntaxKind, + pub is_joint_to_next: bool, + pub text: SmolStr, +} + +pub(crate) struct SubtreeTokenSource<'a> { + cached_cursor: Cell>, + cached: RefCell>>, + curr: (Token, usize), +} + +impl<'a> SubtreeTokenSource<'a> { + // Helper function used in test + #[cfg(test)] + pub fn text(&self) -> SmolStr { + match *self.get(self.curr.1) { + Some(ref tt) => tt.text.clone(), + _ => SmolStr::new(""), + } + } +} + +impl<'a> SubtreeTokenSource<'a> { + pub fn new(buffer: &'a TokenBuffer) -> SubtreeTokenSource<'a> { + let cursor = buffer.begin(); + + let mut res = SubtreeTokenSource { + curr: (Token { kind: EOF, is_jointed_to_next: false }, 0), + cached_cursor: Cell::new(cursor), + cached: RefCell::new(Vec::with_capacity(10)), + }; + res.curr = (res.mk_token(0), 0); + res + } + + fn mk_token(&self, pos: usize) -> Token { + match *self.get(pos) { + Some(ref tt) => Token { kind: tt.kind, is_jointed_to_next: tt.is_joint_to_next }, + None => Token { kind: EOF, is_jointed_to_next: false }, + } + } + + fn get(&self, pos: usize) -> Ref> { + fn is_lifetime(c: Cursor) -> Option<(Cursor, SmolStr)> { + let tkn = c.token_tree(); + + if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = tkn { + if punct.char == '\'' { + let next = c.bump(); + if let Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) = next.token_tree() { + let res_cursor = next.bump(); + let text = SmolStr::new("'".to_string() + &ident.to_string()); + + return Some((res_cursor, text)); + } else { + panic!("Next token must be ident : {:#?}", next.token_tree()); + } + } + } + + None + } + + if pos < self.cached.borrow().len() { + return Ref::map(self.cached.borrow(), |c| &c[pos]); + } + + { + let mut cached = self.cached.borrow_mut(); + while pos >= cached.len() { + let cursor = self.cached_cursor.get(); + if cursor.eof() { + cached.push(None); + continue; + } + + if let Some((curr, text)) = is_lifetime(cursor) { + cached.push(Some(TtToken { kind: LIFETIME, is_joint_to_next: false, text })); + self.cached_cursor.set(curr); + continue; + } + + match cursor.token_tree() { + Some(tt::TokenTree::Leaf(leaf)) => { + cached.push(Some(convert_leaf(&leaf))); + self.cached_cursor.set(cursor.bump()); + } + Some(tt::TokenTree::Subtree(subtree)) => { + self.cached_cursor.set(cursor.subtree().unwrap()); + cached.push(Some(convert_delim(subtree.delimiter_kind(), false))); + } + None => { + if let Some(subtree) = cursor.end() { + cached.push(Some(convert_delim(subtree.delimiter_kind(), true))); + self.cached_cursor.set(cursor.bump()); + } + } + } + } + } + + Ref::map(self.cached.borrow(), |c| &c[pos]) + } +} + +impl<'a> TokenSource for SubtreeTokenSource<'a> { + fn current(&self) -> Token { + self.curr.0 + } + + /// Lookahead n token + fn lookahead_nth(&self, n: usize) -> Token { + self.mk_token(self.curr.1 + n) + } + + /// bump cursor to next token + fn bump(&mut self) { + if self.current().kind == EOF { + return; + } + + self.curr = (self.mk_token(self.curr.1 + 1), self.curr.1 + 1); + } + + /// Is the current token a specified keyword? + fn is_keyword(&self, kw: &str) -> bool { + match *self.get(self.curr.1) { + Some(ref t) => t.text == *kw, + _ => false, + } + } +} + +fn convert_delim(d: Option, closing: bool) -> TtToken { + let (kinds, texts) = match d { + Some(tt::DelimiterKind::Parenthesis) => ([T!['('], T![')']], "()"), + Some(tt::DelimiterKind::Brace) => ([T!['{'], T!['}']], "{}"), + Some(tt::DelimiterKind::Bracket) => ([T!['['], T![']']], "[]"), + None => ([L_DOLLAR, R_DOLLAR], ""), + }; + + let idx = closing as usize; + let kind = kinds[idx]; + let text = if !texts.is_empty() { &texts[idx..texts.len() - (1 - idx)] } else { "" }; + TtToken { kind, is_joint_to_next: false, text: SmolStr::new(text) } +} + +fn convert_literal(l: &tt::Literal) -> TtToken { + let kind = lex_single_syntax_kind(&l.text) + .map(|(kind, _error)| kind) + .filter(|kind| kind.is_literal()) + .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &l)); + + TtToken { kind, is_joint_to_next: false, text: l.text.clone() } +} + +fn convert_ident(ident: &tt::Ident) -> TtToken { + let kind = match ident.text.as_ref() { + "true" => T![true], + "false" => T![false], + i if i.starts_with('\'') => LIFETIME, + _ => SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT), + }; + + TtToken { kind, is_joint_to_next: false, text: ident.text.clone() } +} + +fn convert_punct(p: tt::Punct) -> TtToken { + let kind = match SyntaxKind::from_char(p.char) { + None => panic!("{:#?} is not a valid punct", p), + Some(kind) => kind, + }; + + let text = { + let mut buf = [0u8; 4]; + let s: &str = p.char.encode_utf8(&mut buf); + SmolStr::new(s) + }; + TtToken { kind, is_joint_to_next: p.spacing == tt::Spacing::Joint, text } +} + +fn convert_leaf(leaf: &tt::Leaf) -> TtToken { + match leaf { + tt::Leaf::Literal(l) => convert_literal(l), + tt::Leaf::Ident(ident) => convert_ident(ident), + tt::Leaf::Punct(punct) => convert_punct(*punct), + } +} diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs new file mode 100644 index 000000000..a8ad917fb --- /dev/null +++ b/crates/mbe/src/syntax_bridge.rs @@ -0,0 +1,832 @@ +//! FIXME: write short doc here + +use parser::{FragmentKind, ParseError, TreeSink}; +use rustc_hash::FxHashMap; +use syntax::{ + ast::{self, make::tokens::doc_comment}, + tokenize, AstToken, Parse, SmolStr, SyntaxKind, + SyntaxKind::*, + SyntaxNode, SyntaxToken, SyntaxTreeBuilder, TextRange, TextSize, Token as RawToken, T, +}; +use tt::buffer::{Cursor, TokenBuffer}; + +use crate::subtree_source::SubtreeTokenSource; +use crate::ExpandError; + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum TokenTextRange { + Token(TextRange), + Delimiter(TextRange, TextRange), +} + +impl TokenTextRange { + pub fn by_kind(self, kind: SyntaxKind) -> Option { + match self { + TokenTextRange::Token(it) => Some(it), + TokenTextRange::Delimiter(open, close) => match kind { + T!['{'] | T!['('] | T!['['] => Some(open), + T!['}'] | T![')'] | T![']'] => Some(close), + _ => None, + }, + } + } +} + +/// Maps `tt::TokenId` to the relative range of the original token. +#[derive(Debug, PartialEq, Eq, Clone, Default)] +pub struct TokenMap { + /// Maps `tt::TokenId` to the *relative* source range. + entries: Vec<(tt::TokenId, TokenTextRange)>, +} + +/// Convert the syntax tree (what user has written) to a `TokenTree` (what macro +/// will consume). +pub fn ast_to_token_tree(ast: &impl ast::AstNode) -> Option<(tt::Subtree, TokenMap)> { + syntax_node_to_token_tree(ast.syntax()) +} + +/// Convert the syntax node to a `TokenTree` (what macro +/// will consume). +pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, TokenMap)> { + let global_offset = node.text_range().start(); + let mut c = Convertor::new(node, global_offset); + let subtree = c.go()?; + Some((subtree, c.id_alloc.map)) +} + +// The following items are what `rustc` macro can be parsed into : +// link: https://github.com/rust-lang/rust/blob/9ebf47851a357faa4cd97f4b1dc7835f6376e639/src/libsyntax/ext/expand.rs#L141 +// * Expr(P) -> token_tree_to_expr +// * Pat(P) -> token_tree_to_pat +// * Ty(P) -> token_tree_to_ty +// * Stmts(SmallVec<[ast::Stmt; 1]>) -> token_tree_to_stmts +// * Items(SmallVec<[P; 1]>) -> token_tree_to_items +// +// * TraitItems(SmallVec<[ast::TraitItem; 1]>) +// * AssocItems(SmallVec<[ast::AssocItem; 1]>) +// * ForeignItems(SmallVec<[ast::ForeignItem; 1]> + +pub fn token_tree_to_syntax_node( + tt: &tt::Subtree, + fragment_kind: FragmentKind, +) -> Result<(Parse, TokenMap), ExpandError> { + let tmp; + let tokens = match tt { + tt::Subtree { delimiter: None, token_trees } => token_trees.as_slice(), + _ => { + tmp = [tt.clone().into()]; + &tmp[..] + } + }; + let buffer = TokenBuffer::new(&tokens); + let mut token_source = SubtreeTokenSource::new(&buffer); + let mut tree_sink = TtTreeSink::new(buffer.begin()); + parser::parse_fragment(&mut token_source, &mut tree_sink, fragment_kind); + if tree_sink.roots.len() != 1 { + return Err(ExpandError::ConversionError); + } + //FIXME: would be cool to report errors + let (parse, range_map) = tree_sink.finish(); + Ok((parse, range_map)) +} + +/// Convert a string to a `TokenTree` +pub fn parse_to_token_tree(text: &str) -> Option<(tt::Subtree, TokenMap)> { + let (tokens, errors) = tokenize(text); + if !errors.is_empty() { + return None; + } + + let mut conv = RawConvertor { + text, + offset: TextSize::default(), + inner: tokens.iter(), + id_alloc: TokenIdAlloc { + map: Default::default(), + global_offset: TextSize::default(), + next_id: 0, + }, + }; + + let subtree = conv.go()?; + Some((subtree, conv.id_alloc.map)) +} + +impl TokenMap { + pub fn token_by_range(&self, relative_range: TextRange) -> Option { + let &(token_id, _) = self.entries.iter().find(|(_, range)| match range { + TokenTextRange::Token(it) => *it == relative_range, + TokenTextRange::Delimiter(open, close) => { + *open == relative_range || *close == relative_range + } + })?; + Some(token_id) + } + + pub fn range_by_token(&self, token_id: tt::TokenId) -> Option { + let &(_, range) = self.entries.iter().find(|(tid, _)| *tid == token_id)?; + Some(range) + } + + fn insert(&mut self, token_id: tt::TokenId, relative_range: TextRange) { + self.entries.push((token_id, TokenTextRange::Token(relative_range))); + } + + fn insert_delim( + &mut self, + token_id: tt::TokenId, + open_relative_range: TextRange, + close_relative_range: TextRange, + ) -> usize { + let res = self.entries.len(); + self.entries + .push((token_id, TokenTextRange::Delimiter(open_relative_range, close_relative_range))); + res + } + + fn update_close_delim(&mut self, idx: usize, close_relative_range: TextRange) { + let (_, token_text_range) = &mut self.entries[idx]; + if let TokenTextRange::Delimiter(dim, _) = token_text_range { + *token_text_range = TokenTextRange::Delimiter(*dim, close_relative_range); + } + } + + fn remove_delim(&mut self, idx: usize) { + // FIXME: This could be accidently quadratic + self.entries.remove(idx); + } +} + +/// Returns the textual content of a doc comment block as a quoted string +/// That is, strips leading `///` (or `/**`, etc) +/// and strips the ending `*/` +/// And then quote the string, which is needed to convert to `tt::Literal` +fn doc_comment_text(comment: &ast::Comment) -> SmolStr { + let prefix_len = comment.prefix().len(); + let mut text = &comment.text()[prefix_len..]; + + // Remove ending "*/" + if comment.kind().shape == ast::CommentShape::Block { + text = &text[0..text.len() - 2]; + } + + // Quote the string + // Note that `tt::Literal` expect an escaped string + let text = format!("{:?}", text.escape_default().to_string()); + text.into() +} + +fn convert_doc_comment(token: &syntax::SyntaxToken) -> Option> { + let comment = ast::Comment::cast(token.clone())?; + let doc = comment.kind().doc?; + + // Make `doc="\" Comments\"" + let mut meta_tkns = Vec::new(); + meta_tkns.push(mk_ident("doc")); + meta_tkns.push(mk_punct('=')); + meta_tkns.push(mk_doc_literal(&comment)); + + // Make `#![]` + let mut token_trees = Vec::new(); + token_trees.push(mk_punct('#')); + if let ast::CommentPlacement::Inner = doc { + token_trees.push(mk_punct('!')); + } + token_trees.push(tt::TokenTree::from(tt::Subtree { + delimiter: Some(tt::Delimiter { + kind: tt::DelimiterKind::Bracket, + id: tt::TokenId::unspecified(), + }), + token_trees: meta_tkns, + })); + + return Some(token_trees); + + // Helper functions + fn mk_ident(s: &str) -> tt::TokenTree { + tt::TokenTree::from(tt::Leaf::from(tt::Ident { + text: s.into(), + id: tt::TokenId::unspecified(), + })) + } + + fn mk_punct(c: char) -> tt::TokenTree { + tt::TokenTree::from(tt::Leaf::from(tt::Punct { + char: c, + spacing: tt::Spacing::Alone, + id: tt::TokenId::unspecified(), + })) + } + + fn mk_doc_literal(comment: &ast::Comment) -> tt::TokenTree { + let lit = tt::Literal { text: doc_comment_text(comment), id: tt::TokenId::unspecified() }; + + tt::TokenTree::from(tt::Leaf::from(lit)) + } +} + +struct TokenIdAlloc { + map: TokenMap, + global_offset: TextSize, + next_id: u32, +} + +impl TokenIdAlloc { + fn alloc(&mut self, absolute_range: TextRange) -> tt::TokenId { + let relative_range = absolute_range - self.global_offset; + let token_id = tt::TokenId(self.next_id); + self.next_id += 1; + self.map.insert(token_id, relative_range); + token_id + } + + fn open_delim(&mut self, open_abs_range: TextRange) -> (tt::TokenId, usize) { + let token_id = tt::TokenId(self.next_id); + self.next_id += 1; + let idx = self.map.insert_delim( + token_id, + open_abs_range - self.global_offset, + open_abs_range - self.global_offset, + ); + (token_id, idx) + } + + fn close_delim(&mut self, idx: usize, close_abs_range: Option) { + match close_abs_range { + None => { + self.map.remove_delim(idx); + } + Some(close) => { + self.map.update_close_delim(idx, close - self.global_offset); + } + } + } +} + +/// A Raw Token (straightly from lexer) convertor +struct RawConvertor<'a> { + text: &'a str, + offset: TextSize, + id_alloc: TokenIdAlloc, + inner: std::slice::Iter<'a, RawToken>, +} + +trait SrcToken: std::fmt::Debug { + fn kind(&self) -> SyntaxKind; + + fn to_char(&self) -> Option; + + fn to_text(&self) -> SmolStr; +} + +trait TokenConvertor { + type Token: SrcToken; + + fn go(&mut self) -> Option { + let mut subtree = tt::Subtree::default(); + subtree.delimiter = None; + while self.peek().is_some() { + self.collect_leaf(&mut subtree.token_trees); + } + if subtree.token_trees.is_empty() { + return None; + } + if subtree.token_trees.len() == 1 { + if let tt::TokenTree::Subtree(first) = &subtree.token_trees[0] { + return Some(first.clone()); + } + } + Some(subtree) + } + + fn collect_leaf(&mut self, result: &mut Vec) { + let (token, range) = match self.bump() { + None => return, + Some(it) => it, + }; + + let k: SyntaxKind = token.kind(); + if k == COMMENT { + if let Some(tokens) = self.convert_doc_comment(&token) { + result.extend(tokens); + } + return; + } + + result.push(if k.is_punct() { + assert_eq!(range.len(), TextSize::of('.')); + let delim = match k { + T!['('] => Some((tt::DelimiterKind::Parenthesis, T![')'])), + T!['{'] => Some((tt::DelimiterKind::Brace, T!['}'])), + T!['['] => Some((tt::DelimiterKind::Bracket, T![']'])), + _ => None, + }; + + if let Some((kind, closed)) = delim { + let mut subtree = tt::Subtree::default(); + let (id, idx) = self.id_alloc().open_delim(range); + subtree.delimiter = Some(tt::Delimiter { kind, id }); + + while self.peek().map(|it| it.kind() != closed).unwrap_or(false) { + self.collect_leaf(&mut subtree.token_trees); + } + let last_range = match self.bump() { + None => { + // For error resilience, we insert an char punct for the opening delim here + self.id_alloc().close_delim(idx, None); + let leaf: tt::Leaf = tt::Punct { + id: self.id_alloc().alloc(range), + char: token.to_char().unwrap(), + spacing: tt::Spacing::Alone, + } + .into(); + result.push(leaf.into()); + result.extend(subtree.token_trees); + return; + } + Some(it) => it.1, + }; + self.id_alloc().close_delim(idx, Some(last_range)); + subtree.into() + } else { + let spacing = match self.peek() { + Some(next) + if next.kind().is_trivia() + || next.kind() == T!['['] + || next.kind() == T!['{'] + || next.kind() == T!['('] => + { + tt::Spacing::Alone + } + Some(next) if next.kind().is_punct() => tt::Spacing::Joint, + _ => tt::Spacing::Alone, + }; + let char = match token.to_char() { + Some(c) => c, + None => { + panic!("Token from lexer must be single char: token = {:#?}", token); + } + }; + tt::Leaf::from(tt::Punct { char, spacing, id: self.id_alloc().alloc(range) }).into() + } + } else { + macro_rules! make_leaf { + ($i:ident) => { + tt::$i { id: self.id_alloc().alloc(range), text: token.to_text() }.into() + }; + } + let leaf: tt::Leaf = match k { + T![true] | T![false] => make_leaf!(Ident), + IDENT => make_leaf!(Ident), + k if k.is_keyword() => make_leaf!(Ident), + k if k.is_literal() => make_leaf!(Literal), + LIFETIME => { + let char_unit = TextSize::of('\''); + let r = TextRange::at(range.start(), char_unit); + let apostrophe = tt::Leaf::from(tt::Punct { + char: '\'', + spacing: tt::Spacing::Joint, + id: self.id_alloc().alloc(r), + }); + result.push(apostrophe.into()); + + let r = TextRange::at(range.start() + char_unit, range.len() - char_unit); + let ident = tt::Leaf::from(tt::Ident { + text: SmolStr::new(&token.to_text()[1..]), + id: self.id_alloc().alloc(r), + }); + result.push(ident.into()); + return; + } + _ => return, + }; + + leaf.into() + }); + } + + fn convert_doc_comment(&self, token: &Self::Token) -> Option>; + + fn bump(&mut self) -> Option<(Self::Token, TextRange)>; + + fn peek(&self) -> Option; + + fn id_alloc(&mut self) -> &mut TokenIdAlloc; +} + +impl<'a> SrcToken for (RawToken, &'a str) { + fn kind(&self) -> SyntaxKind { + self.0.kind + } + + fn to_char(&self) -> Option { + self.1.chars().next() + } + + fn to_text(&self) -> SmolStr { + self.1.into() + } +} + +impl RawConvertor<'_> {} + +impl<'a> TokenConvertor for RawConvertor<'a> { + type Token = (RawToken, &'a str); + + fn convert_doc_comment(&self, token: &Self::Token) -> Option> { + convert_doc_comment(&doc_comment(token.1)) + } + + fn bump(&mut self) -> Option<(Self::Token, TextRange)> { + let token = self.inner.next()?; + let range = TextRange::at(self.offset, token.len); + self.offset += token.len; + + Some(((*token, &self.text[range]), range)) + } + + fn peek(&self) -> Option { + let token = self.inner.as_slice().get(0).cloned(); + + token.map(|it| { + let range = TextRange::at(self.offset, it.len); + (it, &self.text[range]) + }) + } + + fn id_alloc(&mut self) -> &mut TokenIdAlloc { + &mut self.id_alloc + } +} + +struct Convertor { + id_alloc: TokenIdAlloc, + current: Option, + range: TextRange, + punct_offset: Option<(SyntaxToken, TextSize)>, +} + +impl Convertor { + fn new(node: &SyntaxNode, global_offset: TextSize) -> Convertor { + Convertor { + id_alloc: { TokenIdAlloc { map: TokenMap::default(), global_offset, next_id: 0 } }, + current: node.first_token(), + range: node.text_range(), + punct_offset: None, + } + } +} + +#[derive(Debug)] +enum SynToken { + Ordiniary(SyntaxToken), + Punch(SyntaxToken, TextSize), +} + +impl SynToken { + fn token(&self) -> &SyntaxToken { + match self { + SynToken::Ordiniary(it) => it, + SynToken::Punch(it, _) => it, + } + } +} + +impl SrcToken for SynToken { + fn kind(&self) -> SyntaxKind { + self.token().kind() + } + fn to_char(&self) -> Option { + match self { + SynToken::Ordiniary(_) => None, + SynToken::Punch(it, i) => it.text().chars().nth((*i).into()), + } + } + fn to_text(&self) -> SmolStr { + self.token().text().clone() + } +} + +impl TokenConvertor for Convertor { + type Token = SynToken; + fn convert_doc_comment(&self, token: &Self::Token) -> Option> { + convert_doc_comment(token.token()) + } + + fn bump(&mut self) -> Option<(Self::Token, TextRange)> { + if let Some((punct, offset)) = self.punct_offset.clone() { + if usize::from(offset) + 1 < punct.text().len() { + let offset = offset + TextSize::of('.'); + let range = punct.text_range(); + self.punct_offset = Some((punct.clone(), offset)); + let range = TextRange::at(range.start() + offset, TextSize::of('.')); + return Some((SynToken::Punch(punct, offset), range)); + } + } + + let curr = self.current.clone()?; + if !&self.range.contains_range(curr.text_range()) { + return None; + } + self.current = curr.next_token(); + + let token = if curr.kind().is_punct() { + let range = curr.text_range(); + let range = TextRange::at(range.start(), TextSize::of('.')); + self.punct_offset = Some((curr.clone(), 0.into())); + (SynToken::Punch(curr, 0.into()), range) + } else { + self.punct_offset = None; + let range = curr.text_range(); + (SynToken::Ordiniary(curr), range) + }; + + Some(token) + } + + fn peek(&self) -> Option { + if let Some((punct, mut offset)) = self.punct_offset.clone() { + offset = offset + TextSize::of('.'); + if usize::from(offset) < punct.text().len() { + return Some(SynToken::Punch(punct, offset)); + } + } + + let curr = self.current.clone()?; + if !self.range.contains_range(curr.text_range()) { + return None; + } + + let token = if curr.kind().is_punct() { + SynToken::Punch(curr, 0.into()) + } else { + SynToken::Ordiniary(curr) + }; + Some(token) + } + + fn id_alloc(&mut self) -> &mut TokenIdAlloc { + &mut self.id_alloc + } +} + +struct TtTreeSink<'a> { + buf: String, + cursor: Cursor<'a>, + open_delims: FxHashMap, + text_pos: TextSize, + inner: SyntaxTreeBuilder, + token_map: TokenMap, + + // Number of roots + // Use for detect ill-form tree which is not single root + roots: smallvec::SmallVec<[usize; 1]>, +} + +impl<'a> TtTreeSink<'a> { + fn new(cursor: Cursor<'a>) -> Self { + TtTreeSink { + buf: String::new(), + cursor, + open_delims: FxHashMap::default(), + text_pos: 0.into(), + inner: SyntaxTreeBuilder::default(), + roots: smallvec::SmallVec::new(), + token_map: TokenMap::default(), + } + } + + fn finish(self) -> (Parse, TokenMap) { + (self.inner.finish(), self.token_map) + } +} + +fn delim_to_str(d: Option, closing: bool) -> SmolStr { + let texts = match d { + Some(tt::DelimiterKind::Parenthesis) => "()", + Some(tt::DelimiterKind::Brace) => "{}", + Some(tt::DelimiterKind::Bracket) => "[]", + None => return "".into(), + }; + + let idx = closing as usize; + let text = &texts[idx..texts.len() - (1 - idx)]; + text.into() +} + +impl<'a> TreeSink for TtTreeSink<'a> { + fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) { + if kind == L_DOLLAR || kind == R_DOLLAR { + self.cursor = self.cursor.bump_subtree(); + return; + } + if kind == LIFETIME { + n_tokens = 2; + } + + let mut last = self.cursor; + for _ in 0..n_tokens { + if self.cursor.eof() { + break; + } + last = self.cursor; + let text: SmolStr = match self.cursor.token_tree() { + Some(tt::TokenTree::Leaf(leaf)) => { + // Mark the range if needed + let (text, id) = match leaf { + tt::Leaf::Ident(ident) => (ident.text.clone(), ident.id), + tt::Leaf::Punct(punct) => { + (SmolStr::new_inline_from_ascii(1, &[punct.char as u8]), punct.id) + } + tt::Leaf::Literal(lit) => (lit.text.clone(), lit.id), + }; + let range = TextRange::at(self.text_pos, TextSize::of(text.as_str())); + self.token_map.insert(id, range); + self.cursor = self.cursor.bump(); + text + } + Some(tt::TokenTree::Subtree(subtree)) => { + self.cursor = self.cursor.subtree().unwrap(); + if let Some(id) = subtree.delimiter.map(|it| it.id) { + self.open_delims.insert(id, self.text_pos); + } + delim_to_str(subtree.delimiter_kind(), false) + } + None => { + if let Some(parent) = self.cursor.end() { + self.cursor = self.cursor.bump(); + if let Some(id) = parent.delimiter.map(|it| it.id) { + if let Some(open_delim) = self.open_delims.get(&id) { + let open_range = TextRange::at(*open_delim, TextSize::of('(')); + let close_range = TextRange::at(self.text_pos, TextSize::of('(')); + self.token_map.insert_delim(id, open_range, close_range); + } + } + delim_to_str(parent.delimiter_kind(), true) + } else { + continue; + } + } + }; + self.buf += &text; + self.text_pos += TextSize::of(text.as_str()); + } + + let text = SmolStr::new(self.buf.as_str()); + self.buf.clear(); + self.inner.token(kind, text); + + // Add whitespace between adjoint puncts + let next = last.bump(); + if let ( + Some(tt::TokenTree::Leaf(tt::Leaf::Punct(curr))), + Some(tt::TokenTree::Leaf(tt::Leaf::Punct(_))), + ) = (last.token_tree(), next.token_tree()) + { + // Note: We always assume the semi-colon would be the last token in + // other parts of RA such that we don't add whitespace here. + if curr.spacing == tt::Spacing::Alone && curr.char != ';' { + self.inner.token(WHITESPACE, " ".into()); + self.text_pos += TextSize::of(' '); + } + } + } + + fn start_node(&mut self, kind: SyntaxKind) { + self.inner.start_node(kind); + + match self.roots.last_mut() { + None | Some(0) => self.roots.push(1), + Some(ref mut n) => **n += 1, + }; + } + + fn finish_node(&mut self) { + self.inner.finish_node(); + *self.roots.last_mut().unwrap() -= 1; + } + + fn error(&mut self, error: ParseError) { + self.inner.error(error, self.text_pos) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::parse_macro; + use parser::TokenSource; + use syntax::{ + algo::{insert_children, InsertPosition}, + ast::AstNode, + }; + + #[test] + fn convert_tt_token_source() { + let expansion = parse_macro( + r#" + macro_rules! literals { + ($i:ident) => { + { + let a = 'c'; + let c = 1000; + let f = 12E+99_f64; + let s = "rust1"; + } + } + } + "#, + ) + .expand_tt("literals!(foo);"); + let tts = &[expansion.into()]; + let buffer = tt::buffer::TokenBuffer::new(tts); + let mut tt_src = SubtreeTokenSource::new(&buffer); + let mut tokens = vec![]; + while tt_src.current().kind != EOF { + tokens.push((tt_src.current().kind, tt_src.text())); + tt_src.bump(); + } + + // [${] + // [let] [a] [=] ['c'] [;] + assert_eq!(tokens[2 + 3].1, "'c'"); + assert_eq!(tokens[2 + 3].0, CHAR); + // [let] [c] [=] [1000] [;] + assert_eq!(tokens[2 + 5 + 3].1, "1000"); + assert_eq!(tokens[2 + 5 + 3].0, INT_NUMBER); + // [let] [f] [=] [12E+99_f64] [;] + assert_eq!(tokens[2 + 10 + 3].1, "12E+99_f64"); + assert_eq!(tokens[2 + 10 + 3].0, FLOAT_NUMBER); + + // [let] [s] [=] ["rust1"] [;] + assert_eq!(tokens[2 + 15 + 3].1, "\"rust1\""); + assert_eq!(tokens[2 + 15 + 3].0, STRING); + } + + #[test] + fn stmts_token_trees_to_expr_is_err() { + let expansion = parse_macro( + r#" + macro_rules! stmts { + () => { + let a = 0; + let b = 0; + let c = 0; + let d = 0; + } + } + "#, + ) + .expand_tt("stmts!();"); + assert!(token_tree_to_syntax_node(&expansion, FragmentKind::Expr).is_err()); + } + + #[test] + fn test_token_tree_last_child_is_white_space() { + let source_file = ast::SourceFile::parse("f!({} );").ok().unwrap(); + let macro_call = source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); + let token_tree = macro_call.token_tree().unwrap(); + + // Token Tree now is : + // TokenTree + // - T!['('] + // - TokenTree + // - T!['{'] + // - T!['}'] + // - WHITE_SPACE + // - T![')'] + + let rbrace = + token_tree.syntax().descendants_with_tokens().find(|it| it.kind() == T!['}']).unwrap(); + let space = token_tree + .syntax() + .descendants_with_tokens() + .find(|it| it.kind() == SyntaxKind::WHITESPACE) + .unwrap(); + + // reorder th white space, such that the white is inside the inner token-tree. + let token_tree = insert_children( + &rbrace.parent().unwrap(), + InsertPosition::Last, + std::iter::once(space), + ); + + // Token Tree now is : + // TokenTree + // - T!['{'] + // - T!['}'] + // - WHITE_SPACE + let token_tree = ast::TokenTree::cast(token_tree).unwrap(); + let tt = ast_to_token_tree(&token_tree).unwrap().0; + + assert_eq!(tt.delimiter_kind(), Some(tt::DelimiterKind::Brace)); + } + + #[test] + fn test_token_tree_multi_char_punct() { + let source_file = ast::SourceFile::parse("struct Foo { a: x::Y }").ok().unwrap(); + let struct_def = source_file.syntax().descendants().find_map(ast::Struct::cast).unwrap(); + let tt = ast_to_token_tree(&struct_def).unwrap().0; + token_tree_to_syntax_node(&tt, FragmentKind::Item).unwrap(); + } +} diff --git a/crates/mbe/src/tests.rs b/crates/mbe/src/tests.rs new file mode 100644 index 000000000..0796ceee1 --- /dev/null +++ b/crates/mbe/src/tests.rs @@ -0,0 +1,1898 @@ +use std::fmt::Write; + +use ::parser::FragmentKind; +use syntax::{ast, AstNode, NodeOrToken, SyntaxKind::IDENT, SyntaxNode, WalkEvent, T}; +use test_utils::assert_eq_text; + +use super::*; + +mod rule_parsing { + use syntax::{ast, AstNode}; + + use crate::ast_to_token_tree; + + use super::*; + + #[test] + fn test_valid_arms() { + fn check(macro_body: &str) { + let m = parse_macro_arm(macro_body); + m.unwrap(); + } + + check("($i:ident) => ()"); + check("($($i:ident)*) => ($_)"); + check("($($true:ident)*) => ($true)"); + check("($($false:ident)*) => ($false)"); + check("($) => ($)"); + } + + #[test] + fn test_invalid_arms() { + fn check(macro_body: &str, err: &str) { + let m = parse_macro_arm(macro_body); + assert_eq!(m, Err(ParseError::Expected(String::from(err)))); + } + + check("invalid", "expected subtree"); + + check("$i:ident => ()", "expected subtree"); + check("($i:ident) ()", "expected `=`"); + check("($($i:ident)_) => ()", "invalid repeat"); + + check("($i) => ($i)", "invalid macro definition"); + check("($i:) => ($i)", "invalid macro definition"); + } + + fn parse_macro_arm(arm_definition: &str) -> Result { + let macro_definition = format!(" macro_rules! m {{ {} }} ", arm_definition); + let source_file = ast::SourceFile::parse(¯o_definition).ok().unwrap(); + let macro_definition = + source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); + + let (definition_tt, _) = + ast_to_token_tree(¯o_definition.token_tree().unwrap()).unwrap(); + crate::MacroRules::parse(&definition_tt) + } +} + +// Good first issue (although a slightly challenging one): +// +// * Pick a random test from here +// https://github.com/intellij-rust/intellij-rust/blob/c4e9feee4ad46e7953b1948c112533360b6087bb/src/test/kotlin/org/rust/lang/core/macros/RsMacroExpansionTest.kt +// * Port the test to rust and add it to this module +// * Make it pass :-) + +#[test] +fn test_token_id_shift() { + let expansion = parse_macro( + r#" +macro_rules! foobar { + ($e:ident) => { foo bar $e } +} +"#, + ) + .expand_tt("foobar!(baz);"); + + fn get_id(t: &tt::TokenTree) -> Option { + if let tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) = t { + return Some(ident.id.0); + } + None + } + + assert_eq!(expansion.token_trees.len(), 3); + // {($e:ident) => { foo bar $e }} + // 012345 67 8 9 T 12 + assert_eq!(get_id(&expansion.token_trees[0]), Some(9)); + assert_eq!(get_id(&expansion.token_trees[1]), Some(10)); + + // The input args of macro call include parentheses: + // (baz) + // So baz should be 12+1+1 + assert_eq!(get_id(&expansion.token_trees[2]), Some(14)); +} + +#[test] +fn test_token_map() { + let expanded = parse_macro( + r#" +macro_rules! foobar { + ($e:ident) => { fn $e() {} } +} +"#, + ) + .expand_tt("foobar!(baz);"); + + let (node, token_map) = token_tree_to_syntax_node(&expanded, FragmentKind::Items).unwrap(); + let content = node.syntax_node().to_string(); + + let get_text = |id, kind| -> String { + content[token_map.range_by_token(id).unwrap().by_kind(kind).unwrap()].to_string() + }; + + assert_eq!(expanded.token_trees.len(), 4); + // {($e:ident) => { fn $e() {} }} + // 012345 67 8 9 T12 3 + + assert_eq!(get_text(tt::TokenId(9), IDENT), "fn"); + assert_eq!(get_text(tt::TokenId(12), T!['(']), "("); + assert_eq!(get_text(tt::TokenId(13), T!['{']), "{"); +} + +#[test] +fn test_convert_tt() { + parse_macro(r#" +macro_rules! impl_froms { + ($e:ident: $($v:ident),*) => { + $( + impl From<$v> for $e { + fn from(it: $v) -> $e { + $e::$v(it) + } + } + )* + } +} +"#) + .assert_expand_tt( + "impl_froms!(TokenTree: Leaf, Subtree);", + "impl From for TokenTree {fn from (it : Leaf) -> TokenTree {TokenTree ::Leaf (it)}} \ + impl From for TokenTree {fn from (it : Subtree) -> TokenTree {TokenTree ::Subtree (it)}}" + ); +} + +#[test] +fn test_convert_tt2() { + parse_macro( + r#" +macro_rules! impl_froms { + ($e:ident: $($v:ident),*) => { + $( + impl From<$v> for $e { + fn from(it: $v) -> $e { + $e::$v(it) + } + } + )* + } +} +"#, + ) + .assert_expand( + "impl_froms!(TokenTree: Leaf, Subtree);", + r#" +SUBTREE $ + IDENT impl 20 + IDENT From 21 + PUNCH < [joint] 22 + IDENT Leaf 53 + PUNCH > [alone] 25 + IDENT for 26 + IDENT TokenTree 51 + SUBTREE {} 29 + IDENT fn 30 + IDENT from 31 + SUBTREE () 32 + IDENT it 33 + PUNCH : [alone] 34 + IDENT Leaf 53 + PUNCH - [joint] 37 + PUNCH > [alone] 38 + IDENT TokenTree 51 + SUBTREE {} 41 + IDENT TokenTree 51 + PUNCH : [joint] 44 + PUNCH : [joint] 45 + IDENT Leaf 53 + SUBTREE () 48 + IDENT it 49 + IDENT impl 20 + IDENT From 21 + PUNCH < [joint] 22 + IDENT Subtree 55 + PUNCH > [alone] 25 + IDENT for 26 + IDENT TokenTree 51 + SUBTREE {} 29 + IDENT fn 30 + IDENT from 31 + SUBTREE () 32 + IDENT it 33 + PUNCH : [alone] 34 + IDENT Subtree 55 + PUNCH - [joint] 37 + PUNCH > [alone] 38 + IDENT TokenTree 51 + SUBTREE {} 41 + IDENT TokenTree 51 + PUNCH : [joint] 44 + PUNCH : [joint] 45 + IDENT Subtree 55 + SUBTREE () 48 + IDENT it 49 +"#, + ); +} + +#[test] +fn test_lifetime_split() { + parse_macro( + r#" +macro_rules! foo { + ($($t:tt)*) => { $($t)*} +} +"#, + ) + .assert_expand( + r#"foo!(static bar: &'static str = "hello";);"#, + r#" +SUBTREE $ + IDENT static 17 + IDENT bar 18 + PUNCH : [alone] 19 + PUNCH & [alone] 20 + PUNCH ' [joint] 21 + IDENT static 22 + IDENT str 23 + PUNCH = [alone] 24 + LITERAL "hello" 25 + PUNCH ; [joint] 26 +"#, + ); +} + +#[test] +fn test_expr_order() { + let expanded = parse_macro( + r#" + macro_rules! foo { + ($ i:expr) => { + fn bar() { $ i * 2; } + } + } +"#, + ) + .expand_items("foo! { 1 + 1}"); + + let dump = format!("{:#?}", expanded); + assert_eq_text!( + dump.trim(), + r#"MACRO_ITEMS@0..15 + FN@0..15 + FN_KW@0..2 "fn" + NAME@2..5 + IDENT@2..5 "bar" + PARAM_LIST@5..7 + L_PAREN@5..6 "(" + R_PAREN@6..7 ")" + BLOCK_EXPR@7..15 + L_CURLY@7..8 "{" + EXPR_STMT@8..14 + BIN_EXPR@8..13 + BIN_EXPR@8..11 + LITERAL@8..9 + INT_NUMBER@8..9 "1" + PLUS@9..10 "+" + LITERAL@10..11 + INT_NUMBER@10..11 "1" + STAR@11..12 "*" + LITERAL@12..13 + INT_NUMBER@12..13 "2" + SEMICOLON@13..14 ";" + R_CURLY@14..15 "}""#, + ); +} + +#[test] +fn test_fail_match_pattern_by_first_token() { + parse_macro( + r#" + macro_rules! foo { + ($ i:ident) => ( + mod $ i {} + ); + (= $ i:ident) => ( + fn $ i() {} + ); + (+ $ i:ident) => ( + struct $ i; + ) + } +"#, + ) + .assert_expand_items("foo! { foo }", "mod foo {}") + .assert_expand_items("foo! { = bar }", "fn bar () {}") + .assert_expand_items("foo! { + Baz }", "struct Baz ;"); +} + +#[test] +fn test_fail_match_pattern_by_last_token() { + parse_macro( + r#" + macro_rules! foo { + ($ i:ident) => ( + mod $ i {} + ); + ($ i:ident =) => ( + fn $ i() {} + ); + ($ i:ident +) => ( + struct $ i; + ) + } +"#, + ) + .assert_expand_items("foo! { foo }", "mod foo {}") + .assert_expand_items("foo! { bar = }", "fn bar () {}") + .assert_expand_items("foo! { Baz + }", "struct Baz ;"); +} + +#[test] +fn test_fail_match_pattern_by_word_token() { + parse_macro( + r#" + macro_rules! foo { + ($ i:ident) => ( + mod $ i {} + ); + (spam $ i:ident) => ( + fn $ i() {} + ); + (eggs $ i:ident) => ( + struct $ i; + ) + } +"#, + ) + .assert_expand_items("foo! { foo }", "mod foo {}") + .assert_expand_items("foo! { spam bar }", "fn bar () {}") + .assert_expand_items("foo! { eggs Baz }", "struct Baz ;"); +} + +#[test] +fn test_match_group_pattern_by_separator_token() { + parse_macro( + r#" + macro_rules! foo { + ($ ($ i:ident),*) => ($ ( + mod $ i {} + )*); + ($ ($ i:ident)#*) => ($ ( + fn $ i() {} + )*); + ($ i:ident ,# $ j:ident) => ( + struct $ i; + struct $ j; + ) + } +"#, + ) + .assert_expand_items("foo! { foo, bar }", "mod foo {} mod bar {}") + .assert_expand_items("foo! { foo# bar }", "fn foo () {} fn bar () {}") + .assert_expand_items("foo! { Foo,# Bar }", "struct Foo ; struct Bar ;"); +} + +#[test] +fn test_match_group_pattern_with_multiple_defs() { + parse_macro( + r#" + macro_rules! foo { + ($ ($ i:ident),*) => ( struct Bar { $ ( + fn $ i {} + )*} ); + } +"#, + ) + .assert_expand_items("foo! { foo, bar }", "struct Bar {fn foo {} fn bar {}}"); +} + +#[test] +fn test_match_group_pattern_with_multiple_statement() { + parse_macro( + r#" + macro_rules! foo { + ($ ($ i:ident),*) => ( fn baz { $ ( + $ i (); + )*} ); + } +"#, + ) + .assert_expand_items("foo! { foo, bar }", "fn baz {foo () ; bar () ;}"); +} + +#[test] +fn test_match_group_pattern_with_multiple_statement_without_semi() { + parse_macro( + r#" + macro_rules! foo { + ($ ($ i:ident),*) => ( fn baz { $ ( + $i() + );*} ); + } +"#, + ) + .assert_expand_items("foo! { foo, bar }", "fn baz {foo () ;bar ()}"); +} + +#[test] +fn test_match_group_empty_fixed_token() { + parse_macro( + r#" + macro_rules! foo { + ($ ($ i:ident)* #abc) => ( fn baz { $ ( + $ i (); + )*} ); + } +"#, + ) + .assert_expand_items("foo! {#abc}", "fn baz {}"); +} + +#[test] +fn test_match_group_in_subtree() { + parse_macro( + r#" + macro_rules! foo { + (fn $name:ident {$($i:ident)*} ) => ( fn $name() { $ ( + $ i (); + )*} ); + }"#, + ) + .assert_expand_items("foo! {fn baz {a b} }", "fn baz () {a () ; b () ;}"); +} + +#[test] +fn test_match_group_with_multichar_sep() { + parse_macro( + r#" + macro_rules! foo { + (fn $name:ident {$($i:literal)*} ) => ( fn $name() -> bool { $($i)&&*} ); + }"#, + ) + .assert_expand_items("foo! (fn baz {true true} );", "fn baz () -> bool {true &&true}"); +} + +#[test] +fn test_match_group_zero_match() { + parse_macro( + r#" + macro_rules! foo { + ( $($i:ident)* ) => (); + }"#, + ) + .assert_expand_items("foo! ();", ""); +} + +#[test] +fn test_match_group_in_group() { + parse_macro( + r#" + macro_rules! foo { + { $( ( $($i:ident)* ) )* } => ( $( ( $($i)* ) )* ); + }"#, + ) + .assert_expand_items("foo! ( (a b) );", "(a b)"); +} + +#[test] +fn test_expand_to_item_list() { + let tree = parse_macro( + " + macro_rules! structs { + ($($i:ident),*) => { + $(struct $i { field: u32 } )* + } + } + ", + ) + .expand_items("structs!(Foo, Bar);"); + assert_eq!( + format!("{:#?}", tree).trim(), + r#" +MACRO_ITEMS@0..40 + STRUCT@0..20 + STRUCT_KW@0..6 "struct" + NAME@6..9 + IDENT@6..9 "Foo" + RECORD_FIELD_LIST@9..20 + L_CURLY@9..10 "{" + RECORD_FIELD@10..19 + NAME@10..15 + IDENT@10..15 "field" + COLON@15..16 ":" + PATH_TYPE@16..19 + PATH@16..19 + PATH_SEGMENT@16..19 + NAME_REF@16..19 + IDENT@16..19 "u32" + R_CURLY@19..20 "}" + STRUCT@20..40 + STRUCT_KW@20..26 "struct" + NAME@26..29 + IDENT@26..29 "Bar" + RECORD_FIELD_LIST@29..40 + L_CURLY@29..30 "{" + RECORD_FIELD@30..39 + NAME@30..35 + IDENT@30..35 "field" + COLON@35..36 ":" + PATH_TYPE@36..39 + PATH@36..39 + PATH_SEGMENT@36..39 + NAME_REF@36..39 + IDENT@36..39 "u32" + R_CURLY@39..40 "}""# + .trim() + ); +} + +fn to_subtree(tt: &tt::TokenTree) -> &tt::Subtree { + if let tt::TokenTree::Subtree(subtree) = tt { + return &subtree; + } + unreachable!("It is not a subtree"); +} +fn to_literal(tt: &tt::TokenTree) -> &tt::Literal { + if let tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) = tt { + return lit; + } + unreachable!("It is not a literal"); +} + +fn to_punct(tt: &tt::TokenTree) -> &tt::Punct { + if let tt::TokenTree::Leaf(tt::Leaf::Punct(lit)) = tt { + return lit; + } + unreachable!("It is not a Punct"); +} + +#[test] +fn test_expand_literals_to_token_tree() { + let expansion = parse_macro( + r#" + macro_rules! literals { + ($i:ident) => { + { + let a = 'c'; + let c = 1000; + let f = 12E+99_f64; + let s = "rust1"; + } + } + } + "#, + ) + .expand_tt("literals!(foo);"); + let stm_tokens = &to_subtree(&expansion.token_trees[0]).token_trees; + + // [let] [a] [=] ['c'] [;] + assert_eq!(to_literal(&stm_tokens[3]).text, "'c'"); + // [let] [c] [=] [1000] [;] + assert_eq!(to_literal(&stm_tokens[5 + 3]).text, "1000"); + // [let] [f] [=] [12E+99_f64] [;] + assert_eq!(to_literal(&stm_tokens[10 + 3]).text, "12E+99_f64"); + // [let] [s] [=] ["rust1"] [;] + assert_eq!(to_literal(&stm_tokens[15 + 3]).text, "\"rust1\""); +} + +#[test] +fn test_attr_to_token_tree() { + let expansion = parse_to_token_tree_by_syntax( + r#" + #[derive(Copy)] + struct Foo; + "#, + ); + + assert_eq!(to_punct(&expansion.token_trees[0]).char, '#'); + assert_eq!( + to_subtree(&expansion.token_trees[1]).delimiter_kind(), + Some(tt::DelimiterKind::Bracket) + ); +} + +#[test] +fn test_two_idents() { + parse_macro( + r#" + macro_rules! foo { + ($ i:ident, $ j:ident) => { + fn foo() { let a = $ i; let b = $j; } + } + } +"#, + ) + .assert_expand_items("foo! { foo, bar }", "fn foo () {let a = foo ; let b = bar ;}"); +} + +#[test] +fn test_tt_to_stmts() { + let stmts = parse_macro( + r#" + macro_rules! foo { + () => { + let a = 0; + a = 10 + 1; + a + } + } +"#, + ) + .expand_statements("foo!{}"); + + assert_eq!( + format!("{:#?}", stmts).trim(), + r#"MACRO_STMTS@0..15 + LET_STMT@0..7 + LET_KW@0..3 "let" + IDENT_PAT@3..4 + NAME@3..4 + IDENT@3..4 "a" + EQ@4..5 "=" + LITERAL@5..6 + INT_NUMBER@5..6 "0" + SEMICOLON@6..7 ";" + EXPR_STMT@7..14 + BIN_EXPR@7..13 + PATH_EXPR@7..8 + PATH@7..8 + PATH_SEGMENT@7..8 + NAME_REF@7..8 + IDENT@7..8 "a" + EQ@8..9 "=" + BIN_EXPR@9..13 + LITERAL@9..11 + INT_NUMBER@9..11 "10" + PLUS@11..12 "+" + LITERAL@12..13 + INT_NUMBER@12..13 "1" + SEMICOLON@13..14 ";" + EXPR_STMT@14..15 + PATH_EXPR@14..15 + PATH@14..15 + PATH_SEGMENT@14..15 + NAME_REF@14..15 + IDENT@14..15 "a""#, + ); +} + +#[test] +fn test_match_literal() { + parse_macro( + r#" + macro_rules! foo { + ('(') => { + fn foo() {} + } + } +"#, + ) + .assert_expand_items("foo! ['('];", "fn foo () {}"); +} + +// The following tests are port from intellij-rust directly +// https://github.com/intellij-rust/intellij-rust/blob/c4e9feee4ad46e7953b1948c112533360b6087bb/src/test/kotlin/org/rust/lang/core/macros/RsMacroExpansionTest.kt + +#[test] +fn test_path() { + parse_macro( + r#" + macro_rules! foo { + ($ i:path) => { + fn foo() { let a = $ i; } + } + } +"#, + ) + .assert_expand_items("foo! { foo }", "fn foo () {let a = foo ;}") + .assert_expand_items( + "foo! { bar::::baz:: }", + "fn foo () {let a = bar ::< u8 >:: baz ::< u8 > ;}", + ); +} + +#[test] +fn test_two_paths() { + parse_macro( + r#" + macro_rules! foo { + ($ i:path, $ j:path) => { + fn foo() { let a = $ i; let b = $j; } + } + } +"#, + ) + .assert_expand_items("foo! { foo, bar }", "fn foo () {let a = foo ; let b = bar ;}"); +} + +#[test] +fn test_path_with_path() { + parse_macro( + r#" + macro_rules! foo { + ($ i:path) => { + fn foo() { let a = $ i :: bar; } + } + } +"#, + ) + .assert_expand_items("foo! { foo }", "fn foo () {let a = foo :: bar ;}"); +} + +#[test] +fn test_expr() { + parse_macro( + r#" + macro_rules! foo { + ($ i:expr) => { + fn bar() { $ i; } + } + } +"#, + ) + .assert_expand_items( + "foo! { 2 + 2 * baz(3).quux() }", + "fn bar () {2 + 2 * baz (3) . quux () ;}", + ); +} + +#[test] +fn test_last_expr() { + parse_macro( + r#" + macro_rules! vec { + ($($item:expr),*) => { + { + let mut v = Vec::new(); + $( + v.push($item); + )* + v + } + }; + } +"#, + ) + .assert_expand_items( + "vec!(1,2,3);", + "{let mut v = Vec :: new () ; v . push (1) ; v . push (2) ; v . push (3) ; v}", + ); +} + +#[test] +fn test_ty() { + parse_macro( + r#" + macro_rules! foo { + ($ i:ty) => ( + fn bar() -> $ i { unimplemented!() } + ) + } +"#, + ) + .assert_expand_items("foo! { Baz }", "fn bar () -> Baz < u8 > {unimplemented ! ()}"); +} + +#[test] +fn test_ty_with_complex_type() { + parse_macro( + r#" + macro_rules! foo { + ($ i:ty) => ( + fn bar() -> $ i { unimplemented!() } + ) + } +"#, + ) + // Reference lifetime struct with generic type + .assert_expand_items( + "foo! { &'a Baz }", + "fn bar () -> & 'a Baz < u8 > {unimplemented ! ()}", + ) + // extern "Rust" func type + .assert_expand_items( + r#"foo! { extern "Rust" fn() -> Ret }"#, + r#"fn bar () -> extern "Rust" fn () -> Ret {unimplemented ! ()}"#, + ); +} + +#[test] +fn test_pat_() { + parse_macro( + r#" + macro_rules! foo { + ($ i:pat) => { fn foo() { let $ i; } } + } +"#, + ) + .assert_expand_items("foo! { (a, b) }", "fn foo () {let (a , b) ;}"); +} + +#[test] +fn test_stmt() { + parse_macro( + r#" + macro_rules! foo { + ($ i:stmt) => ( + fn bar() { $ i; } + ) + } +"#, + ) + .assert_expand_items("foo! { 2 }", "fn bar () {2 ;}") + .assert_expand_items("foo! { let a = 0 }", "fn bar () {let a = 0 ;}"); +} + +#[test] +fn test_single_item() { + parse_macro( + r#" + macro_rules! foo { + ($ i:item) => ( + $ i + ) + } +"#, + ) + .assert_expand_items("foo! {mod c {}}", "mod c {}"); +} + +#[test] +fn test_all_items() { + parse_macro( + r#" + macro_rules! foo { + ($ ($ i:item)*) => ($ ( + $ i + )*) + } +"#, + ). + assert_expand_items( + r#" + foo! { + extern crate a; + mod b; + mod c {} + use d; + const E: i32 = 0; + static F: i32 = 0; + impl G {} + struct H; + enum I { Foo } + trait J {} + fn h() {} + extern {} + type T = u8; + } +"#, + r#"extern crate a ; mod b ; mod c {} use d ; const E : i32 = 0 ; static F : i32 = 0 ; impl G {} struct H ; enum I {Foo} trait J {} fn h () {} extern {} type T = u8 ;"#, + ); +} + +#[test] +fn test_block() { + parse_macro( + r#" + macro_rules! foo { + ($ i:block) => { fn foo() $ i } + } +"#, + ) + .assert_expand_statements("foo! { { 1; } }", "fn foo () {1 ;}"); +} + +#[test] +fn test_meta() { + parse_macro( + r#" + macro_rules! foo { + ($ i:meta) => ( + #[$ i] + fn bar() {} + ) + } +"#, + ) + .assert_expand_items( + r#"foo! { cfg(target_os = "windows") }"#, + r#"# [cfg (target_os = "windows")] fn bar () {}"#, + ); +} + +#[test] +fn test_meta_doc_comments() { + parse_macro( + r#" + macro_rules! foo { + ($(#[$ i:meta])+) => ( + $(#[$ i])+ + fn bar() {} + ) + } +"#, + ). + assert_expand_items( + r#"foo! { + /// Single Line Doc 1 + /** + MultiLines Doc + */ + }"#, + "# [doc = \" Single Line Doc 1\"] # [doc = \"\\\\n MultiLines Doc\\\\n \"] fn bar () {}", + ); +} + +#[test] +fn test_tt_block() { + parse_macro( + r#" + macro_rules! foo { + ($ i:tt) => { fn foo() $ i } + } + "#, + ) + .assert_expand_items(r#"foo! { { 1; } }"#, r#"fn foo () {1 ;}"#); +} + +#[test] +fn test_tt_group() { + parse_macro( + r#" + macro_rules! foo { + ($($ i:tt)*) => { $($ i)* } + } + "#, + ) + .assert_expand_items(r#"foo! { fn foo() {} }"#, r#"fn foo () {}"#); +} + +#[test] +fn test_tt_composite() { + parse_macro( + r#" + macro_rules! foo { + ($i:tt) => { 0 } + } + "#, + ) + .assert_expand_items(r#"foo! { => }"#, r#"0"#); +} + +#[test] +fn test_tt_composite2() { + let node = parse_macro( + r#" + macro_rules! foo { + ($($tt:tt)*) => { abs!(=> $($tt)*) } + } + "#, + ) + .expand_items(r#"foo!{#}"#); + + let res = format!("{:#?}", &node); + assert_eq_text!( + res.trim(), + r###"MACRO_ITEMS@0..10 + MACRO_CALL@0..10 + PATH@0..3 + PATH_SEGMENT@0..3 + NAME_REF@0..3 + IDENT@0..3 "abs" + BANG@3..4 "!" + TOKEN_TREE@4..10 + L_PAREN@4..5 "(" + EQ@5..6 "=" + R_ANGLE@6..7 ">" + WHITESPACE@7..8 " " + POUND@8..9 "#" + R_PAREN@9..10 ")""### + ); +} + +#[test] +fn test_lifetime() { + parse_macro( + r#" + macro_rules! foo { + ($ lt:lifetime) => { struct Ref<$ lt>{ s: &$ lt str } } + } +"#, + ) + .assert_expand_items(r#"foo!{'a}"#, r#"struct Ref <'a > {s : &'a str}"#); +} + +#[test] +fn test_literal() { + parse_macro( + r#" + macro_rules! foo { + ($ type:ty $ lit:literal) => { const VALUE: $ type = $ lit;}; + } +"#, + ) + .assert_expand_items(r#"foo!(u8 0);"#, r#"const VALUE : u8 = 0 ;"#); +} + +#[test] +fn test_boolean_is_ident() { + parse_macro( + r#" + macro_rules! foo { + ($lit0:literal, $lit1:literal) => { const VALUE: (bool,bool) = ($lit0,$lit1); }; + } +"#, + ) + .assert_expand( + r#"foo!(true,false);"#, + r#" +SUBTREE $ + IDENT const 14 + IDENT VALUE 15 + PUNCH : [alone] 16 + SUBTREE () 17 + IDENT bool 18 + PUNCH , [alone] 19 + IDENT bool 20 + PUNCH = [alone] 21 + SUBTREE () 22 + IDENT true 29 + PUNCH , [joint] 25 + IDENT false 31 + PUNCH ; [alone] 28 +"#, + ); +} + +#[test] +fn test_vis() { + parse_macro( + r#" + macro_rules! foo { + ($ vis:vis $ name:ident) => { $ vis fn $ name() {}}; + } +"#, + ) + .assert_expand_items(r#"foo!(pub foo);"#, r#"pub fn foo () {}"#) + // test optional cases + .assert_expand_items(r#"foo!(foo);"#, r#"fn foo () {}"#); +} + +#[test] +fn test_inner_macro_rules() { + parse_macro( + r#" +macro_rules! foo { + ($a:ident, $b:ident, $c:tt) => { + + macro_rules! bar { + ($bi:ident) => { + fn $bi() -> u8 {$c} + } + } + + bar!($a); + fn $b() -> u8 {$c} + } +} +"#, + ). + assert_expand_items( + r#"foo!(x,y, 1);"#, + r#"macro_rules ! bar {($ bi : ident) => {fn $ bi () -> u8 {1}}} bar ! (x) ; fn y () -> u8 {1}"#, + ); +} + +// The following tests are based on real world situations +#[test] +fn test_vec() { + let fixture = parse_macro( + r#" + macro_rules! vec { + ($($item:expr),*) => { + { + let mut v = Vec::new(); + $( + v.push($item); + )* + v + } + }; +} +"#, + ); + fixture + .assert_expand_items(r#"vec!();"#, r#"{let mut v = Vec :: new () ; v}"#) + .assert_expand_items( + r#"vec![1u32,2];"#, + r#"{let mut v = Vec :: new () ; v . push (1u32) ; v . push (2) ; v}"#, + ); + + let tree = fixture.expand_expr(r#"vec![1u32,2];"#); + + assert_eq!( + format!("{:#?}", tree).trim(), + r#"BLOCK_EXPR@0..45 + L_CURLY@0..1 "{" + LET_STMT@1..20 + LET_KW@1..4 "let" + IDENT_PAT@4..8 + MUT_KW@4..7 "mut" + NAME@7..8 + IDENT@7..8 "v" + EQ@8..9 "=" + CALL_EXPR@9..19 + PATH_EXPR@9..17 + PATH@9..17 + PATH@9..12 + PATH_SEGMENT@9..12 + NAME_REF@9..12 + IDENT@9..12 "Vec" + COLON2@12..14 "::" + PATH_SEGMENT@14..17 + NAME_REF@14..17 + IDENT@14..17 "new" + ARG_LIST@17..19 + L_PAREN@17..18 "(" + R_PAREN@18..19 ")" + SEMICOLON@19..20 ";" + EXPR_STMT@20..33 + METHOD_CALL_EXPR@20..32 + PATH_EXPR@20..21 + PATH@20..21 + PATH_SEGMENT@20..21 + NAME_REF@20..21 + IDENT@20..21 "v" + DOT@21..22 "." + NAME_REF@22..26 + IDENT@22..26 "push" + ARG_LIST@26..32 + L_PAREN@26..27 "(" + LITERAL@27..31 + INT_NUMBER@27..31 "1u32" + R_PAREN@31..32 ")" + SEMICOLON@32..33 ";" + EXPR_STMT@33..43 + METHOD_CALL_EXPR@33..42 + PATH_EXPR@33..34 + PATH@33..34 + PATH_SEGMENT@33..34 + NAME_REF@33..34 + IDENT@33..34 "v" + DOT@34..35 "." + NAME_REF@35..39 + IDENT@35..39 "push" + ARG_LIST@39..42 + L_PAREN@39..40 "(" + LITERAL@40..41 + INT_NUMBER@40..41 "2" + R_PAREN@41..42 ")" + SEMICOLON@42..43 ";" + PATH_EXPR@43..44 + PATH@43..44 + PATH_SEGMENT@43..44 + NAME_REF@43..44 + IDENT@43..44 "v" + R_CURLY@44..45 "}""# + ); +} + +#[test] +fn test_winapi_struct() { + // from https://github.com/retep998/winapi-rs/blob/a7ef2bca086aae76cf6c4ce4c2552988ed9798ad/src/macros.rs#L366 + + parse_macro( + r#" +macro_rules! STRUCT { + ($(#[$attrs:meta])* struct $name:ident { + $($field:ident: $ftype:ty,)+ + }) => ( + #[repr(C)] #[derive(Copy)] $(#[$attrs])* + pub struct $name { + $(pub $field: $ftype,)+ + } + impl Clone for $name { + #[inline] + fn clone(&self) -> $name { *self } + } + #[cfg(feature = "impl-default")] + impl Default for $name { + #[inline] + fn default() -> $name { unsafe { $crate::_core::mem::zeroed() } } + } + ); +} +"#, + ). + // from https://github.com/retep998/winapi-rs/blob/a7ef2bca086aae76cf6c4ce4c2552988ed9798ad/src/shared/d3d9caps.rs + assert_expand_items(r#"STRUCT!{struct D3DVSHADERCAPS2_0 {Caps: u8,}}"#, + "# [repr (C)] # [derive (Copy)] pub struct D3DVSHADERCAPS2_0 {pub Caps : u8 ,} impl Clone for D3DVSHADERCAPS2_0 {# [inline] fn clone (& self) -> D3DVSHADERCAPS2_0 {* self}} # [cfg (feature = \"impl-default\")] impl Default for D3DVSHADERCAPS2_0 {# [inline] fn default () -> D3DVSHADERCAPS2_0 {unsafe {$crate :: _core :: mem :: zeroed ()}}}" + ) + .assert_expand_items(r#"STRUCT!{#[cfg_attr(target_arch = "x86", repr(packed))] struct D3DCONTENTPROTECTIONCAPS {Caps : u8 ,}}"#, + "# [repr (C)] # [derive (Copy)] # [cfg_attr (target_arch = \"x86\" , repr (packed))] pub struct D3DCONTENTPROTECTIONCAPS {pub Caps : u8 ,} impl Clone for D3DCONTENTPROTECTIONCAPS {# [inline] fn clone (& self) -> D3DCONTENTPROTECTIONCAPS {* self}} # [cfg (feature = \"impl-default\")] impl Default for D3DCONTENTPROTECTIONCAPS {# [inline] fn default () -> D3DCONTENTPROTECTIONCAPS {unsafe {$crate :: _core :: mem :: zeroed ()}}}" + ); +} + +#[test] +fn test_int_base() { + parse_macro( + r#" +macro_rules! int_base { + ($Trait:ident for $T:ident as $U:ident -> $Radix:ident) => { + #[stable(feature = "rust1", since = "1.0.0")] + impl fmt::$Trait for $T { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + $Radix.fmt_int(*self as $U, f) + } + } + } +} +"#, + ).assert_expand_items(r#" int_base!{Binary for isize as usize -> Binary}"#, + "# [stable (feature = \"rust1\" , since = \"1.0.0\")] impl fmt ::Binary for isize {fn fmt (& self , f : & mut fmt :: Formatter < \'_ >) -> fmt :: Result {Binary . fmt_int (* self as usize , f)}}" + ); +} + +#[test] +fn test_generate_pattern_iterators() { + // from https://github.com/rust-lang/rust/blob/316a391dcb7d66dc25f1f9a4ec9d368ef7615005/src/libcore/str/mod.rs + parse_macro( + r#" +macro_rules! generate_pattern_iterators { + { double ended; with $(#[$common_stability_attribute:meta])*, + $forward_iterator:ident, + $reverse_iterator:ident, $iterty:ty + } => { + fn foo(){} + } +} +"#, + ).assert_expand_items( + r#"generate_pattern_iterators ! ( double ended ; with # [ stable ( feature = "rust1" , since = "1.0.0" ) ] , Split , RSplit , & 'a str );"#, + "fn foo () {}", + ); +} + +#[test] +fn test_impl_fn_for_zst() { + // from https://github.com/rust-lang/rust/blob/5d20ff4d2718c820632b38c1e49d4de648a9810b/src/libcore/internal_macros.rs + parse_macro( + r#" +macro_rules! impl_fn_for_zst { + { $( $( #[$attr: meta] )* + struct $Name: ident impl$( <$( $lifetime : lifetime ),+> )? Fn = + |$( $arg: ident: $ArgTy: ty ),*| -> $ReturnTy: ty +$body: block; )+ + } => { + $( + $( #[$attr] )* + struct $Name; + + impl $( <$( $lifetime ),+> )? Fn<($( $ArgTy, )*)> for $Name { + #[inline] + extern "rust-call" fn call(&self, ($( $arg, )*): ($( $ArgTy, )*)) -> $ReturnTy { + $body + } + } + + impl $( <$( $lifetime ),+> )? FnMut<($( $ArgTy, )*)> for $Name { + #[inline] + extern "rust-call" fn call_mut( + &mut self, + ($( $arg, )*): ($( $ArgTy, )*) + ) -> $ReturnTy { + Fn::call(&*self, ($( $arg, )*)) + } + } + + impl $( <$( $lifetime ),+> )? FnOnce<($( $ArgTy, )*)> for $Name { + type Output = $ReturnTy; + + #[inline] + extern "rust-call" fn call_once(self, ($( $arg, )*): ($( $ArgTy, )*)) -> $ReturnTy { + Fn::call(&self, ($( $arg, )*)) + } + } + )+ +} + } +"#, + ).assert_expand_items(r#" +impl_fn_for_zst ! { + # [ derive ( Clone ) ] + struct CharEscapeDebugContinue impl Fn = | c : char | -> char :: EscapeDebug { + c . escape_debug_ext ( false ) + } ; + + # [ derive ( Clone ) ] + struct CharEscapeUnicode impl Fn = | c : char | -> char :: EscapeUnicode { + c . escape_unicode ( ) + } ; + # [ derive ( Clone ) ] + struct CharEscapeDefault impl Fn = | c : char | -> char :: EscapeDefault { + c . escape_default ( ) + } ; + } +"#, + "# [derive (Clone)] struct CharEscapeDebugContinue ; impl Fn < (char ,) > for CharEscapeDebugContinue {# [inline] extern \"rust-call\" fn call (& self , (c ,) : (char ,)) -> char :: EscapeDebug {{c . escape_debug_ext (false)}}} impl FnMut < (char ,) > for CharEscapeDebugContinue {# [inline] extern \"rust-call\" fn call_mut (& mut self , (c ,) : (char ,)) -> char :: EscapeDebug {Fn :: call (&* self , (c ,))}} impl FnOnce < (char ,) > for CharEscapeDebugContinue {type Output = char :: EscapeDebug ; # [inline] extern \"rust-call\" fn call_once (self , (c ,) : (char ,)) -> char :: EscapeDebug {Fn :: call (& self , (c ,))}} # [derive (Clone)] struct CharEscapeUnicode ; impl Fn < (char ,) > for CharEscapeUnicode {# [inline] extern \"rust-call\" fn call (& self , (c ,) : (char ,)) -> char :: EscapeUnicode {{c . escape_unicode ()}}} impl FnMut < (char ,) > for CharEscapeUnicode {# [inline] extern \"rust-call\" fn call_mut (& mut self , (c ,) : (char ,)) -> char :: EscapeUnicode {Fn :: call (&* self , (c ,))}} impl FnOnce < (char ,) > for CharEscapeUnicode {type Output = char :: EscapeUnicode ; # [inline] extern \"rust-call\" fn call_once (self , (c ,) : (char ,)) -> char :: EscapeUnicode {Fn :: call (& self , (c ,))}} # [derive (Clone)] struct CharEscapeDefault ; impl Fn < (char ,) > for CharEscapeDefault {# [inline] extern \"rust-call\" fn call (& self , (c ,) : (char ,)) -> char :: EscapeDefault {{c . escape_default ()}}} impl FnMut < (char ,) > for CharEscapeDefault {# [inline] extern \"rust-call\" fn call_mut (& mut self , (c ,) : (char ,)) -> char :: EscapeDefault {Fn :: call (&* self , (c ,))}} impl FnOnce < (char ,) > for CharEscapeDefault {type Output = char :: EscapeDefault ; # [inline] extern \"rust-call\" fn call_once (self , (c ,) : (char ,)) -> char :: EscapeDefault {Fn :: call (& self , (c ,))}}" + ); +} + +#[test] +fn test_impl_nonzero_fmt() { + // from https://github.com/rust-lang/rust/blob/316a391dcb7d66dc25f1f9a4ec9d368ef7615005/src/libcore/num/mod.rs#L12 + parse_macro( + r#" + macro_rules! impl_nonzero_fmt { + ( #[$stability: meta] ( $( $Trait: ident ),+ ) for $Ty: ident ) => { + fn foo () {} + } + } +"#, + ).assert_expand_items( + r#"impl_nonzero_fmt! { # [stable(feature= "nonzero",since="1.28.0")] (Debug,Display,Binary,Octal,LowerHex,UpperHex) for NonZeroU8}"#, + "fn foo () {}", + ); +} + +#[test] +fn test_cfg_if_items() { + // from https://github.com/rust-lang/rust/blob/33fe1131cadba69d317156847be9a402b89f11bb/src/libstd/macros.rs#L986 + parse_macro( + r#" + macro_rules! __cfg_if_items { + (($($not:meta,)*) ; ) => {}; + (($($not:meta,)*) ; ( ($($m:meta),*) ($($it:item)*) ), $($rest:tt)*) => { + __cfg_if_items! { ($($not,)* $($m,)*) ; $($rest)* } + } + } +"#, + ).assert_expand_items( + r#"__cfg_if_items ! { ( rustdoc , ) ; ( ( ) ( # [ cfg ( any ( target_os = "redox" , unix ) ) ] # [ stable ( feature = "rust1" , since = "1.0.0" ) ] pub use sys :: ext as unix ; # [ cfg ( windows ) ] # [ stable ( feature = "rust1" , since = "1.0.0" ) ] pub use sys :: ext as windows ; # [ cfg ( any ( target_os = "linux" , target_os = "l4re" ) ) ] pub mod linux ; ) ) , }"#, + "__cfg_if_items ! {(rustdoc ,) ;}", + ); +} + +#[test] +fn test_cfg_if_main() { + // from https://github.com/rust-lang/rust/blob/3d211248393686e0f73851fc7548f6605220fbe1/src/libpanic_unwind/macros.rs#L9 + parse_macro( + r#" + macro_rules! cfg_if { + ($( + if #[cfg($($meta:meta),*)] { $($it:item)* } + ) else * else { + $($it2:item)* + }) => { + __cfg_if_items! { + () ; + $( ( ($($meta),*) ($($it)*) ), )* + ( () ($($it2)*) ), + } + }; + + // Internal macro to Apply a cfg attribute to a list of items + (@__apply $m:meta, $($it:item)*) => { + $(#[$m] $it)* + }; + } +"#, + ).assert_expand_items(r#" +cfg_if ! { + if # [ cfg ( target_env = "msvc" ) ] { + // no extra unwinder support needed + } else if # [ cfg ( all ( target_arch = "wasm32" , not ( target_os = "emscripten" ) ) ) ] { + // no unwinder on the system! + } else { + mod libunwind ; + pub use libunwind :: * ; + } + } +"#, + "__cfg_if_items ! {() ; ((target_env = \"msvc\") ()) , ((all (target_arch = \"wasm32\" , not (target_os = \"emscripten\"))) ()) , (() (mod libunwind ; pub use libunwind :: * ;)) ,}" + ).assert_expand_items( + r#" +cfg_if ! { @ __apply cfg ( all ( not ( any ( not ( any ( target_os = "solaris" , target_os = "illumos" ) ) ) ) ) ) , } +"#, + "", + ); +} + +#[test] +fn test_proptest_arbitrary() { + // from https://github.com/AltSysrq/proptest/blob/d1c4b049337d2f75dd6f49a095115f7c532e5129/proptest/src/arbitrary/macros.rs#L16 + parse_macro( + r#" +macro_rules! arbitrary { + ([$($bounds : tt)*] $typ: ty, $strat: ty, $params: ty; + $args: ident => $logic: expr) => { + impl<$($bounds)*> $crate::arbitrary::Arbitrary for $typ { + type Parameters = $params; + type Strategy = $strat; + fn arbitrary_with($args: Self::Parameters) -> Self::Strategy { + $logic + } + } + }; + +}"#, + ).assert_expand_items(r#"arbitrary ! ( [ A : Arbitrary ] + Vec < A > , + VecStrategy < A :: Strategy > , + RangedParams1 < A :: Parameters > ; + args => { let product_unpack ! [ range , a ] = args ; vec ( any_with :: < A > ( a ) , range ) } + ) ;"#, + "impl $crate :: arbitrary :: Arbitrary for Vec < A > {type Parameters = RangedParams1 < A :: Parameters > ; type Strategy = VecStrategy < A :: Strategy > ; fn arbitrary_with (args : Self :: Parameters) -> Self :: Strategy {{let product_unpack ! [range , a] = args ; vec (any_with :: < A > (a) , range)}}}" + ); +} + +#[test] +fn test_old_ridl() { + // This is from winapi 2.8, which do not have a link from github + // + let expanded = parse_macro( + r#" +#[macro_export] +macro_rules! RIDL { + (interface $interface:ident ($vtbl:ident) : $pinterface:ident ($pvtbl:ident) + {$( + fn $method:ident(&mut self $(,$p:ident : $t:ty)*) -> $rtr:ty + ),+} + ) => { + impl $interface { + $(pub unsafe fn $method(&mut self) -> $rtr { + ((*self.lpVtbl).$method)(self $(,$p)*) + })+ + } + }; +}"#, + ).expand_tt(r#" + RIDL!{interface ID3D11Asynchronous(ID3D11AsynchronousVtbl): ID3D11DeviceChild(ID3D11DeviceChildVtbl) { + fn GetDataSize(&mut self) -> UINT + }}"#); + + assert_eq!(expanded.to_string(), "impl ID3D11Asynchronous {pub unsafe fn GetDataSize (& mut self) -> UINT {((* self . lpVtbl) .GetDataSize) (self)}}"); +} + +#[test] +fn test_quick_error() { + let expanded = parse_macro( + r#" +macro_rules! quick_error { + + (SORT [enum $name:ident $( #[$meta:meta] )*] + items [$($( #[$imeta:meta] )* + => $iitem:ident: $imode:tt [$( $ivar:ident: $ityp:ty ),*] + {$( $ifuncs:tt )*} )* ] + buf [ ] + queue [ ] + ) => { + quick_error!(ENUMINITION [enum $name $( #[$meta] )*] + body [] + queue [$( + $( #[$imeta] )* + => + $iitem: $imode [$( $ivar: $ityp ),*] + )*] + ); +}; + +} +"#, + ) + .expand_tt( + r#" +quick_error ! (SORT [enum Wrapped # [derive (Debug)]] items [ + => One : UNIT [] {} + => Two : TUPLE [s :String] {display ("two: {}" , s) from ()} + ] buf [] queue []) ; +"#, + ); + + assert_eq!(expanded.to_string(), "quick_error ! (ENUMINITION [enum Wrapped # [derive (Debug)]] body [] queue [=> One : UNIT [] => Two : TUPLE [s : String]]) ;"); +} + +#[test] +fn test_empty_repeat_vars_in_empty_repeat_vars() { + parse_macro( + r#" +macro_rules! delegate_impl { + ([$self_type:ident, $self_wrap:ty, $self_map:ident] + pub trait $name:ident $(: $sup:ident)* $(+ $more_sup:ident)* { + + // "Escaped" associated types. Stripped before making the `trait` + // itself, but forwarded when delegating impls. + $( + @escape [type $assoc_name_ext:ident] + // Associated types. Forwarded. + )* + $( + @section type + $( + $(#[$_assoc_attr:meta])* + type $assoc_name:ident $(: $assoc_bound:ty)*; + )+ + )* + // Methods. Forwarded. Using $self_map!(self) around the self argument. + // Methods must use receiver `self` or explicit type like `self: &Self` + // &self and &mut self are _not_ supported. + $( + @section self + $( + $(#[$_method_attr:meta])* + fn $method_name:ident(self $(: $self_selftype:ty)* $(,$marg:ident : $marg_ty:ty)*) -> $mret:ty; + )+ + )* + // Arbitrary tail that is ignored when forwarding. + $( + @section nodelegate + $($tail:tt)* + )* + }) => { + impl<> $name for $self_wrap where $self_type: $name { + $( + $( + fn $method_name(self $(: $self_selftype)* $(,$marg: $marg_ty)*) -> $mret { + $self_map!(self).$method_name($($marg),*) + } + )* + )* + } + } +} +"#, + ).assert_expand_items( + r#"delegate_impl ! {[G , & 'a mut G , deref] pub trait Data : GraphBase {@ section type type NodeWeight ;}}"#, + "impl <> Data for & \'a mut G where G : Data {}", + ); +} + +#[test] +fn expr_interpolation() { + let expanded = parse_macro( + r#" + macro_rules! id { + ($expr:expr) => { + map($expr) + } + } + "#, + ) + .expand_expr("id!(x + foo);"); + + assert_eq!(expanded.to_string(), "map(x+foo)"); +} + +pub(crate) struct MacroFixture { + rules: MacroRules, +} + +impl MacroFixture { + pub(crate) fn expand_tt(&self, invocation: &str) -> tt::Subtree { + self.try_expand_tt(invocation).unwrap() + } + + fn try_expand_tt(&self, invocation: &str) -> Result { + let source_file = ast::SourceFile::parse(invocation).tree(); + let macro_invocation = + source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); + + let (invocation_tt, _) = ast_to_token_tree(¯o_invocation.token_tree().unwrap()) + .ok_or_else(|| ExpandError::ConversionError)?; + + self.rules.expand(&invocation_tt).result() + } + + fn assert_expand_err(&self, invocation: &str, err: &ExpandError) { + assert_eq!(self.try_expand_tt(invocation).as_ref(), Err(err)); + } + + fn expand_items(&self, invocation: &str) -> SyntaxNode { + let expanded = self.expand_tt(invocation); + token_tree_to_syntax_node(&expanded, FragmentKind::Items).unwrap().0.syntax_node() + } + + fn expand_statements(&self, invocation: &str) -> SyntaxNode { + let expanded = self.expand_tt(invocation); + token_tree_to_syntax_node(&expanded, FragmentKind::Statements).unwrap().0.syntax_node() + } + + fn expand_expr(&self, invocation: &str) -> SyntaxNode { + let expanded = self.expand_tt(invocation); + token_tree_to_syntax_node(&expanded, FragmentKind::Expr).unwrap().0.syntax_node() + } + + fn assert_expand_tt(&self, invocation: &str, expected: &str) { + let expansion = self.expand_tt(invocation); + assert_eq!(expansion.to_string(), expected); + } + + fn assert_expand(&self, invocation: &str, expected: &str) { + let expansion = self.expand_tt(invocation); + let actual = format!("{:?}", expansion); + test_utils::assert_eq_text!(&actual.trim(), &expected.trim()); + } + + fn assert_expand_items(&self, invocation: &str, expected: &str) -> &MacroFixture { + self.assert_expansion(FragmentKind::Items, invocation, expected); + self + } + + fn assert_expand_statements(&self, invocation: &str, expected: &str) -> &MacroFixture { + self.assert_expansion(FragmentKind::Statements, invocation, expected); + self + } + + fn assert_expansion(&self, kind: FragmentKind, invocation: &str, expected: &str) { + let expanded = self.expand_tt(invocation); + assert_eq!(expanded.to_string(), expected); + + let expected = expected.replace("$crate", "C_C__C"); + + // wrap the given text to a macro call + let expected = { + let wrapped = format!("wrap_macro!( {} )", expected); + let wrapped = ast::SourceFile::parse(&wrapped); + let wrapped = + wrapped.tree().syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); + let mut wrapped = ast_to_token_tree(&wrapped).unwrap().0; + wrapped.delimiter = None; + wrapped + }; + + let expanded_tree = token_tree_to_syntax_node(&expanded, kind).unwrap().0.syntax_node(); + let expanded_tree = debug_dump_ignore_spaces(&expanded_tree).trim().to_string(); + + let expected_tree = token_tree_to_syntax_node(&expected, kind).unwrap().0.syntax_node(); + let expected_tree = debug_dump_ignore_spaces(&expected_tree).trim().to_string(); + + let expected_tree = expected_tree.replace("C_C__C", "$crate"); + assert_eq!( + expanded_tree, expected_tree, + "\nleft:\n{}\nright:\n{}", + expanded_tree, expected_tree, + ); + } +} + +fn parse_macro_to_tt(ra_fixture: &str) -> tt::Subtree { + let source_file = ast::SourceFile::parse(ra_fixture).ok().unwrap(); + let macro_definition = + source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); + + let (definition_tt, _) = ast_to_token_tree(¯o_definition.token_tree().unwrap()).unwrap(); + + let parsed = parse_to_token_tree( + &ra_fixture[macro_definition.token_tree().unwrap().syntax().text_range()], + ) + .unwrap() + .0; + assert_eq!(definition_tt, parsed); + + definition_tt +} + +pub(crate) fn parse_macro(ra_fixture: &str) -> MacroFixture { + let definition_tt = parse_macro_to_tt(ra_fixture); + let rules = MacroRules::parse(&definition_tt).unwrap(); + MacroFixture { rules } +} + +pub(crate) fn parse_macro_error(ra_fixture: &str) -> ParseError { + let definition_tt = parse_macro_to_tt(ra_fixture); + + match MacroRules::parse(&definition_tt) { + Ok(_) => panic!("Expect error"), + Err(err) => err, + } +} + +pub(crate) fn parse_to_token_tree_by_syntax(ra_fixture: &str) -> tt::Subtree { + let source_file = ast::SourceFile::parse(ra_fixture).ok().unwrap(); + let tt = syntax_node_to_token_tree(source_file.syntax()).unwrap().0; + + let parsed = parse_to_token_tree(ra_fixture).unwrap().0; + assert_eq!(tt, parsed); + + parsed +} + +fn debug_dump_ignore_spaces(node: &syntax::SyntaxNode) -> String { + let mut level = 0; + let mut buf = String::new(); + macro_rules! indent { + () => { + for _ in 0..level { + buf.push_str(" "); + } + }; + } + + for event in node.preorder_with_tokens() { + match event { + WalkEvent::Enter(element) => { + match element { + NodeOrToken::Node(node) => { + indent!(); + writeln!(buf, "{:?}", node.kind()).unwrap(); + } + NodeOrToken::Token(token) => match token.kind() { + syntax::SyntaxKind::WHITESPACE => {} + _ => { + indent!(); + writeln!(buf, "{:?}", token.kind()).unwrap(); + } + }, + } + level += 1; + } + WalkEvent::Leave(_) => level -= 1, + } + } + + buf +} + +#[test] +fn test_issue_2520() { + let macro_fixture = parse_macro( + r#" + macro_rules! my_macro { + { + ( $( + $( [] $sname:ident : $stype:ty )? + $( [$expr:expr] $nname:ident : $ntype:ty )? + ),* ) + } => { + Test { + $( + $( $sname, )? + )* + } + }; + } + "#, + ); + + macro_fixture.assert_expand_items( + r#"my_macro ! { + ([] p1 : u32 , [|_| S0K0] s : S0K0 , [] k0 : i32) + }"#, + "Test {p1 , k0 ,}", + ); +} + +#[test] +fn test_issue_3861() { + let macro_fixture = parse_macro( + r#" + macro_rules! rgb_color { + ($p:expr, $t: ty) => { + pub fn new() { + let _ = 0 as $t << $p; + } + }; + } + "#, + ); + + macro_fixture.expand_items(r#"rgb_color!(8 + 8, u32);"#); +} + +#[test] +fn test_repeat_bad_var() { + // FIXME: the second rule of the macro should be removed and an error about + // `$( $c )+` raised + parse_macro( + r#" + macro_rules! foo { + ($( $b:ident )+) => { + $( $c )+ + }; + ($( $b:ident )+) => { + $( $b )+ + } + } + "#, + ) + .assert_expand_items("foo!(b0 b1);", "b0 b1"); +} + +#[test] +fn test_no_space_after_semi_colon() { + let expanded = parse_macro( + r#" + macro_rules! with_std { ($($i:item)*) => ($(#[cfg(feature = "std")]$i)*) } + "#, + ) + .expand_items(r#"with_std! {mod m;mod f;}"#); + + let dump = format!("{:#?}", expanded); + assert_eq_text!( + dump.trim(), + r###"MACRO_ITEMS@0..52 + MODULE@0..26 + ATTR@0..21 + POUND@0..1 "#" + L_BRACK@1..2 "[" + PATH@2..5 + PATH_SEGMENT@2..5 + NAME_REF@2..5 + IDENT@2..5 "cfg" + TOKEN_TREE@5..20 + L_PAREN@5..6 "(" + IDENT@6..13 "feature" + EQ@13..14 "=" + STRING@14..19 "\"std\"" + R_PAREN@19..20 ")" + R_BRACK@20..21 "]" + MOD_KW@21..24 "mod" + NAME@24..25 + IDENT@24..25 "m" + SEMICOLON@25..26 ";" + MODULE@26..52 + ATTR@26..47 + POUND@26..27 "#" + L_BRACK@27..28 "[" + PATH@28..31 + PATH_SEGMENT@28..31 + NAME_REF@28..31 + IDENT@28..31 "cfg" + TOKEN_TREE@31..46 + L_PAREN@31..32 "(" + IDENT@32..39 "feature" + EQ@39..40 "=" + STRING@40..45 "\"std\"" + R_PAREN@45..46 ")" + R_BRACK@46..47 "]" + MOD_KW@47..50 "mod" + NAME@50..51 + IDENT@50..51 "f" + SEMICOLON@51..52 ";""###, + ); +} + +// https://github.com/rust-lang/rust/blob/master/src/test/ui/issues/issue-57597.rs +#[test] +fn test_rustc_issue_57597() { + fn test_error(fixture: &str) { + assert_eq!(parse_macro_error(fixture), ParseError::RepetitionEmtpyTokenTree); + } + + test_error("macro_rules! foo { ($($($i:ident)?)+) => {}; }"); + test_error("macro_rules! foo { ($($($i:ident)?)*) => {}; }"); + test_error("macro_rules! foo { ($($($i:ident)?)?) => {}; }"); + test_error("macro_rules! foo { ($($($($i:ident)?)?)?) => {}; }"); + test_error("macro_rules! foo { ($($($($i:ident)*)?)?) => {}; }"); + test_error("macro_rules! foo { ($($($($i:ident)?)*)?) => {}; }"); + test_error("macro_rules! foo { ($($($($i:ident)?)?)*) => {}; }"); + test_error("macro_rules! foo { ($($($($i:ident)*)*)?) => {}; }"); + test_error("macro_rules! foo { ($($($($i:ident)?)*)*) => {}; }"); + test_error("macro_rules! foo { ($($($($i:ident)?)*)+) => {}; }"); + test_error("macro_rules! foo { ($($($($i:ident)+)?)*) => {}; }"); + test_error("macro_rules! foo { ($($($($i:ident)+)*)?) => {}; }"); +} + +#[test] +fn test_expand_bad_literal() { + parse_macro( + r#" + macro_rules! foo { ($i:literal) => {}; } + "#, + ) + .assert_expand_err(r#"foo!(&k");"#, &ExpandError::BindingError("".into())); +} + +#[test] +fn test_empty_comments() { + parse_macro( + r#" + macro_rules! one_arg_macro { ($fmt:expr) => (); } + "#, + ) + .assert_expand_err( + r#"one_arg_macro!(/**/)"#, + &ExpandError::BindingError("expected Expr".into()), + ); +} diff --git a/crates/mbe/src/tt_iter.rs b/crates/mbe/src/tt_iter.rs new file mode 100644 index 000000000..46c420718 --- /dev/null +++ b/crates/mbe/src/tt_iter.rs @@ -0,0 +1,75 @@ +//! FIXME: write short doc here + +#[derive(Debug, Clone)] +pub(crate) struct TtIter<'a> { + pub(crate) inner: std::slice::Iter<'a, tt::TokenTree>, +} + +impl<'a> TtIter<'a> { + pub(crate) fn new(subtree: &'a tt::Subtree) -> TtIter<'a> { + TtIter { inner: subtree.token_trees.iter() } + } + + pub(crate) fn expect_char(&mut self, char: char) -> Result<(), ()> { + match self.next() { + Some(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: c, .. }))) if *c == char => { + Ok(()) + } + _ => Err(()), + } + } + + pub(crate) fn expect_subtree(&mut self) -> Result<&'a tt::Subtree, ()> { + match self.next() { + Some(tt::TokenTree::Subtree(it)) => Ok(it), + _ => Err(()), + } + } + + pub(crate) fn expect_leaf(&mut self) -> Result<&'a tt::Leaf, ()> { + match self.next() { + Some(tt::TokenTree::Leaf(it)) => Ok(it), + _ => Err(()), + } + } + + pub(crate) fn expect_ident(&mut self) -> Result<&'a tt::Ident, ()> { + match self.expect_leaf()? { + tt::Leaf::Ident(it) => Ok(it), + _ => Err(()), + } + } + + pub(crate) fn expect_literal(&mut self) -> Result<&'a tt::Leaf, ()> { + let it = self.expect_leaf()?; + match it { + tt::Leaf::Literal(_) => Ok(it), + tt::Leaf::Ident(ident) if ident.text == "true" || ident.text == "false" => Ok(it), + _ => Err(()), + } + } + + pub(crate) fn expect_punct(&mut self) -> Result<&'a tt::Punct, ()> { + match self.expect_leaf()? { + tt::Leaf::Punct(it) => Ok(it), + _ => Err(()), + } + } + + pub(crate) fn peek_n(&self, n: usize) -> Option<&tt::TokenTree> { + self.inner.as_slice().get(n) + } +} + +impl<'a> Iterator for TtIter<'a> { + type Item = &'a tt::TokenTree; + fn next(&mut self) -> Option { + self.inner.next() + } + + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} + +impl<'a> std::iter::ExactSizeIterator for TtIter<'a> {} diff --git a/crates/proc_macro_srv/Cargo.toml b/crates/proc_macro_srv/Cargo.toml index f7d481cba..0954ffb66 100644 --- a/crates/proc_macro_srv/Cargo.toml +++ b/crates/proc_macro_srv/Cargo.toml @@ -14,7 +14,7 @@ libloading = "0.6.0" memmap = "0.7" tt = { path = "../tt" } -ra_mbe = { path = "../ra_mbe" } +mbe = { path = "../mbe" } ra_proc_macro = { path = "../ra_proc_macro" } test_utils = { path = "../test_utils" } diff --git a/crates/proc_macro_srv/src/rustc_server.rs b/crates/proc_macro_srv/src/rustc_server.rs index d534d1337..7d1695c86 100644 --- a/crates/proc_macro_srv/src/rustc_server.rs +++ b/crates/proc_macro_srv/src/rustc_server.rs @@ -182,7 +182,7 @@ pub mod token_stream { fn from_str(src: &str) -> Result { let (subtree, _token_map) = - ra_mbe::parse_to_token_tree(src).ok_or("Failed to parse from mbe")?; + mbe::parse_to_token_tree(src).ok_or("Failed to parse from mbe")?; let tt: tt::TokenTree = subtree.into(); Ok(tt.into()) diff --git a/crates/ra_cfg/Cargo.toml b/crates/ra_cfg/Cargo.toml index cb0d2b9d7..e77c7bd72 100644 --- a/crates/ra_cfg/Cargo.toml +++ b/crates/ra_cfg/Cargo.toml @@ -15,4 +15,4 @@ syntax = { path = "../syntax" } tt = { path = "../tt" } [dev-dependencies] -mbe = { path = "../ra_mbe", package = "ra_mbe" } +mbe = { path = "../mbe" } diff --git a/crates/ra_cfg/src/cfg_expr.rs b/crates/ra_cfg/src/cfg_expr.rs index 940091465..fc93730d9 100644 --- a/crates/ra_cfg/src/cfg_expr.rs +++ b/crates/ra_cfg/src/cfg_expr.rs @@ -4,7 +4,7 @@ use std::slice::Iter as SliceIter; -use syntax::SmolStr; +use tt::SmolStr; #[derive(Debug, Clone, PartialEq, Eq)] pub enum CfgExpr { diff --git a/crates/ra_hir_def/Cargo.toml b/crates/ra_hir_def/Cargo.toml index 38129782f..ba7916c30 100644 --- a/crates/ra_hir_def/Cargo.toml +++ b/crates/ra_hir_def/Cargo.toml @@ -28,7 +28,7 @@ syntax = { path = "../syntax" } profile = { path = "../profile" } hir_expand = { path = "../ra_hir_expand", package = "ra_hir_expand" } test_utils = { path = "../test_utils" } -mbe = { path = "../ra_mbe", package = "ra_mbe" } +mbe = { path = "../mbe" } ra_cfg = { path = "../ra_cfg" } tt = { path = "../tt" } diff --git a/crates/ra_hir_expand/Cargo.toml b/crates/ra_hir_expand/Cargo.toml index 153a70bdf..cbb0ac29b 100644 --- a/crates/ra_hir_expand/Cargo.toml +++ b/crates/ra_hir_expand/Cargo.toml @@ -19,5 +19,5 @@ syntax = { path = "../syntax" } parser = { path = "../parser" } profile = { path = "../profile" } tt = { path = "../tt" } -mbe = { path = "../ra_mbe", package = "ra_mbe" } +mbe = { path = "../mbe" } test_utils = { path = "../test_utils"} diff --git a/crates/ra_mbe/Cargo.toml b/crates/ra_mbe/Cargo.toml deleted file mode 100644 index 4a4be65eb..000000000 --- a/crates/ra_mbe/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -edition = "2018" -name = "ra_mbe" -version = "0.1.0" -authors = ["rust-analyzer developers"] -license = "MIT OR Apache-2.0" - -[lib] -doctest = false - -[dependencies] -syntax = { path = "../syntax" } -parser = { path = "../parser" } -tt = { path = "../tt" } -rustc-hash = "1.1.0" -smallvec = "1.2.0" -log = "0.4.8" - -[dev-dependencies] -test_utils = { path = "../test_utils" } diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs deleted file mode 100644 index f854ca09a..000000000 --- a/crates/ra_mbe/src/lib.rs +++ /dev/null @@ -1,278 +0,0 @@ -//! `mbe` (short for Macro By Example) crate contains code for handling -//! `macro_rules` macros. It uses `TokenTree` (from `tt` package) as the -//! interface, although it contains some code to bridge `SyntaxNode`s and -//! `TokenTree`s as well! - -mod parser; -mod mbe_expander; -mod syntax_bridge; -mod tt_iter; -mod subtree_source; - -#[cfg(test)] -mod tests; - -pub use tt::{Delimiter, Punct}; - -use crate::{ - parser::{parse_pattern, Op}, - tt_iter::TtIter, -}; - -#[derive(Debug, PartialEq, Eq)] -pub enum ParseError { - Expected(String), - RepetitionEmtpyTokenTree, -} - -#[derive(Debug, PartialEq, Eq, Clone)] -pub enum ExpandError { - NoMatchingRule, - UnexpectedToken, - BindingError(String), - ConversionError, - InvalidRepeat, - ProcMacroError(tt::ExpansionError), -} - -impl From for ExpandError { - fn from(it: tt::ExpansionError) -> Self { - ExpandError::ProcMacroError(it) - } -} - -pub use crate::syntax_bridge::{ - ast_to_token_tree, parse_to_token_tree, syntax_node_to_token_tree, token_tree_to_syntax_node, - TokenMap, -}; - -/// This struct contains AST for a single `macro_rules` definition. What might -/// be very confusing is that AST has almost exactly the same shape as -/// `tt::TokenTree`, but there's a crucial difference: in macro rules, `$ident` -/// and `$()*` have special meaning (see `Var` and `Repeat` data structures) -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct MacroRules { - rules: Vec, - /// Highest id of the token we have in TokenMap - shift: Shift, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -struct Rule { - lhs: tt::Subtree, - rhs: tt::Subtree, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct Shift(u32); - -impl Shift { - fn new(tt: &tt::Subtree) -> Shift { - // Note that TokenId is started from zero, - // We have to add 1 to prevent duplication. - let value = max_id(tt).map_or(0, |it| it + 1); - return Shift(value); - - // Find the max token id inside a subtree - fn max_id(subtree: &tt::Subtree) -> Option { - subtree - .token_trees - .iter() - .filter_map(|tt| match tt { - tt::TokenTree::Subtree(subtree) => { - let tree_id = max_id(subtree); - match subtree.delimiter { - Some(it) if it.id != tt::TokenId::unspecified() => { - Some(tree_id.map_or(it.id.0, |t| t.max(it.id.0))) - } - _ => tree_id, - } - } - tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) - if ident.id != tt::TokenId::unspecified() => - { - Some(ident.id.0) - } - _ => None, - }) - .max() - } - } - - /// Shift given TokenTree token id - fn shift_all(self, tt: &mut tt::Subtree) { - for t in tt.token_trees.iter_mut() { - match t { - tt::TokenTree::Leaf(leaf) => match leaf { - tt::Leaf::Ident(ident) => ident.id = self.shift(ident.id), - tt::Leaf::Punct(punct) => punct.id = self.shift(punct.id), - tt::Leaf::Literal(lit) => lit.id = self.shift(lit.id), - }, - tt::TokenTree::Subtree(tt) => { - if let Some(it) = tt.delimiter.as_mut() { - it.id = self.shift(it.id); - }; - self.shift_all(tt) - } - } - } - } - - fn shift(self, id: tt::TokenId) -> tt::TokenId { - if id == tt::TokenId::unspecified() { - return id; - } - tt::TokenId(id.0 + self.0) - } - - fn unshift(self, id: tt::TokenId) -> Option { - id.0.checked_sub(self.0).map(tt::TokenId) - } -} - -#[derive(Debug, Eq, PartialEq)] -pub enum Origin { - Def, - Call, -} - -impl MacroRules { - pub fn parse(tt: &tt::Subtree) -> Result { - // Note: this parsing can be implemented using mbe machinery itself, by - // matching against `$($lhs:tt => $rhs:tt);*` pattern, but implementing - // manually seems easier. - let mut src = TtIter::new(tt); - let mut rules = Vec::new(); - while src.len() > 0 { - let rule = Rule::parse(&mut src)?; - rules.push(rule); - if let Err(()) = src.expect_char(';') { - if src.len() > 0 { - return Err(ParseError::Expected("expected `:`".to_string())); - } - break; - } - } - - for rule in rules.iter() { - validate(&rule.lhs)?; - } - - Ok(MacroRules { rules, shift: Shift::new(tt) }) - } - - pub fn expand(&self, tt: &tt::Subtree) -> ExpandResult { - // apply shift - let mut tt = tt.clone(); - self.shift.shift_all(&mut tt); - mbe_expander::expand(self, &tt) - } - - pub fn map_id_down(&self, id: tt::TokenId) -> tt::TokenId { - self.shift.shift(id) - } - - pub fn map_id_up(&self, id: tt::TokenId) -> (tt::TokenId, Origin) { - match self.shift.unshift(id) { - Some(id) => (id, Origin::Call), - None => (id, Origin::Def), - } - } -} - -impl Rule { - fn parse(src: &mut TtIter) -> Result { - let mut lhs = src - .expect_subtree() - .map_err(|()| ParseError::Expected("expected subtree".to_string()))? - .clone(); - lhs.delimiter = None; - src.expect_char('=').map_err(|()| ParseError::Expected("expected `=`".to_string()))?; - src.expect_char('>').map_err(|()| ParseError::Expected("expected `>`".to_string()))?; - let mut rhs = src - .expect_subtree() - .map_err(|()| ParseError::Expected("expected subtree".to_string()))? - .clone(); - rhs.delimiter = None; - Ok(crate::Rule { lhs, rhs }) - } -} - -fn to_parse_error(e: ExpandError) -> ParseError { - let msg = match e { - ExpandError::InvalidRepeat => "invalid repeat".to_string(), - _ => "invalid macro definition".to_string(), - }; - ParseError::Expected(msg) -} - -fn validate(pattern: &tt::Subtree) -> Result<(), ParseError> { - for op in parse_pattern(pattern) { - let op = op.map_err(to_parse_error)?; - - match op { - Op::TokenTree(tt::TokenTree::Subtree(subtree)) => validate(subtree)?, - Op::Repeat { subtree, separator, .. } => { - // Checks that no repetition which could match an empty token - // https://github.com/rust-lang/rust/blob/a58b1ed44f5e06976de2bdc4d7dc81c36a96934f/src/librustc_expand/mbe/macro_rules.rs#L558 - - if separator.is_none() { - if parse_pattern(subtree).all(|child_op| { - match child_op.map_err(to_parse_error) { - Ok(Op::Var { kind, .. }) => { - // vis is optional - if kind.map_or(false, |it| it == "vis") { - return true; - } - } - Ok(Op::Repeat { kind, .. }) => { - return matches!( - kind, - parser::RepeatKind::ZeroOrMore | parser::RepeatKind::ZeroOrOne - ) - } - _ => {} - } - false - }) { - return Err(ParseError::RepetitionEmtpyTokenTree); - } - } - validate(subtree)? - } - _ => (), - } - } - Ok(()) -} - -#[derive(Debug)] -pub struct ExpandResult(pub T, pub Option); - -impl ExpandResult { - pub fn ok(t: T) -> ExpandResult { - ExpandResult(t, None) - } - - pub fn only_err(err: ExpandError) -> ExpandResult - where - T: Default, - { - ExpandResult(Default::default(), Some(err)) - } - - pub fn map(self, f: impl FnOnce(T) -> U) -> ExpandResult { - ExpandResult(f(self.0), self.1) - } - - pub fn result(self) -> Result { - self.1.map(Err).unwrap_or(Ok(self.0)) - } -} - -impl From> for ExpandResult { - fn from(result: Result) -> ExpandResult { - result - .map_or_else(|e| ExpandResult(Default::default(), Some(e)), |it| ExpandResult(it, None)) - } -} diff --git a/crates/ra_mbe/src/mbe_expander.rs b/crates/ra_mbe/src/mbe_expander.rs deleted file mode 100644 index 1ad8b9f8a..000000000 --- a/crates/ra_mbe/src/mbe_expander.rs +++ /dev/null @@ -1,180 +0,0 @@ -//! This module takes a (parsed) definition of `macro_rules` invocation, a -//! `tt::TokenTree` representing an argument of macro invocation, and produces a -//! `tt::TokenTree` for the result of the expansion. - -mod matcher; -mod transcriber; - -use rustc_hash::FxHashMap; -use syntax::SmolStr; - -use crate::{ExpandError, ExpandResult}; - -pub(crate) fn expand(rules: &crate::MacroRules, input: &tt::Subtree) -> ExpandResult { - expand_rules(&rules.rules, input) -} - -fn expand_rules(rules: &[crate::Rule], input: &tt::Subtree) -> ExpandResult { - let mut match_: Option<(matcher::Match, &crate::Rule)> = None; - for rule in rules { - let new_match = match matcher::match_(&rule.lhs, input) { - Ok(m) => m, - Err(_e) => { - // error in pattern parsing - continue; - } - }; - if new_match.err.is_none() { - // If we find a rule that applies without errors, we're done. - // Unconditionally returning the transcription here makes the - // `test_repeat_bad_var` test fail. - let ExpandResult(res, transcribe_err) = - transcriber::transcribe(&rule.rhs, &new_match.bindings); - if transcribe_err.is_none() { - return ExpandResult::ok(res); - } - } - // Use the rule if we matched more tokens, or had fewer errors - if let Some((prev_match, _)) = &match_ { - if (new_match.unmatched_tts, new_match.err_count) - < (prev_match.unmatched_tts, prev_match.err_count) - { - match_ = Some((new_match, rule)); - } - } else { - match_ = Some((new_match, rule)); - } - } - if let Some((match_, rule)) = match_ { - // if we got here, there was no match without errors - let ExpandResult(result, transcribe_err) = - transcriber::transcribe(&rule.rhs, &match_.bindings); - ExpandResult(result, match_.err.or(transcribe_err)) - } else { - ExpandResult(tt::Subtree::default(), Some(ExpandError::NoMatchingRule)) - } -} - -/// The actual algorithm for expansion is not too hard, but is pretty tricky. -/// `Bindings` structure is the key to understanding what we are doing here. -/// -/// On the high level, it stores mapping from meta variables to the bits of -/// syntax it should be substituted with. For example, if `$e:expr` is matched -/// with `1 + 1` by macro_rules, the `Binding` will store `$e -> 1 + 1`. -/// -/// The tricky bit is dealing with repetitions (`$()*`). Consider this example: -/// -/// ```not_rust -/// macro_rules! foo { -/// ($($ i:ident $($ e:expr),*);*) => { -/// $(fn $ i() { $($ e);*; })* -/// } -/// } -/// foo! { foo 1,2,3; bar 4,5,6 } -/// ``` -/// -/// Here, the `$i` meta variable is matched first with `foo` and then with -/// `bar`, and `$e` is matched in turn with `1`, `2`, `3`, `4`, `5`, `6`. -/// -/// To represent such "multi-mappings", we use a recursive structures: we map -/// variables not to values, but to *lists* of values or other lists (that is, -/// to the trees). -/// -/// For the above example, the bindings would store -/// -/// ```not_rust -/// i -> [foo, bar] -/// e -> [[1, 2, 3], [4, 5, 6]] -/// ``` -/// -/// We construct `Bindings` in the `match_lhs`. The interesting case is -/// `TokenTree::Repeat`, where we use `push_nested` to create the desired -/// nesting structure. -/// -/// The other side of the puzzle is `expand_subtree`, where we use the bindings -/// to substitute meta variables in the output template. When expanding, we -/// maintain a `nesting` stack of indices which tells us which occurrence from -/// the `Bindings` we should take. We push to the stack when we enter a -/// repetition. -/// -/// In other words, `Bindings` is a *multi* mapping from `SmolStr` to -/// `tt::TokenTree`, where the index to select a particular `TokenTree` among -/// many is not a plain `usize`, but an `&[usize]`. -#[derive(Debug, Default)] -struct Bindings { - inner: FxHashMap, -} - -#[derive(Debug)] -enum Binding { - Fragment(Fragment), - Nested(Vec), - Empty, -} - -#[derive(Debug, Clone)] -enum Fragment { - /// token fragments are just copy-pasted into the output - Tokens(tt::TokenTree), - /// Ast fragments are inserted with fake delimiters, so as to make things - /// like `$i * 2` where `$i = 1 + 1` work as expectd. - Ast(tt::TokenTree), -} - -#[cfg(test)] -mod tests { - use syntax::{ast, AstNode}; - - use super::*; - use crate::ast_to_token_tree; - - #[test] - fn test_expand_rule() { - assert_err( - "($($i:ident);*) => ($i)", - "foo!{a}", - ExpandError::BindingError(String::from( - "expected simple binding, found nested binding `i`", - )), - ); - - // FIXME: - // Add an err test case for ($($i:ident)) => ($()) - } - - fn assert_err(macro_body: &str, invocation: &str, err: ExpandError) { - assert_eq!(expand_first(&create_rules(&format_macro(macro_body)), invocation).1, Some(err)); - } - - fn format_macro(macro_body: &str) -> String { - format!( - " - macro_rules! foo {{ - {} - }} -", - macro_body - ) - } - - fn create_rules(macro_definition: &str) -> crate::MacroRules { - let source_file = ast::SourceFile::parse(macro_definition).ok().unwrap(); - let macro_definition = - source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); - - let (definition_tt, _) = - ast_to_token_tree(¯o_definition.token_tree().unwrap()).unwrap(); - crate::MacroRules::parse(&definition_tt).unwrap() - } - - fn expand_first(rules: &crate::MacroRules, invocation: &str) -> ExpandResult { - let source_file = ast::SourceFile::parse(invocation).ok().unwrap(); - let macro_invocation = - source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); - - let (invocation_tt, _) = - ast_to_token_tree(¯o_invocation.token_tree().unwrap()).unwrap(); - - expand_rules(&rules.rules, &invocation_tt) - } -} diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs deleted file mode 100644 index b698b9832..000000000 --- a/crates/ra_mbe/src/mbe_expander/matcher.rs +++ /dev/null @@ -1,477 +0,0 @@ -//! FIXME: write short doc here - -use crate::{ - mbe_expander::{Binding, Bindings, Fragment}, - parser::{parse_pattern, Op, RepeatKind, Separator}, - subtree_source::SubtreeTokenSource, - tt_iter::TtIter, - ExpandError, -}; - -use super::ExpandResult; -use parser::{FragmentKind::*, TreeSink}; -use syntax::{SmolStr, SyntaxKind}; -use tt::buffer::{Cursor, TokenBuffer}; - -impl Bindings { - fn push_optional(&mut self, name: &SmolStr) { - // FIXME: Do we have a better way to represent an empty token ? - // Insert an empty subtree for empty token - let tt = tt::Subtree::default().into(); - self.inner.insert(name.clone(), Binding::Fragment(Fragment::Tokens(tt))); - } - - fn push_empty(&mut self, name: &SmolStr) { - self.inner.insert(name.clone(), Binding::Empty); - } - - fn push_nested(&mut self, idx: usize, nested: Bindings) -> Result<(), ExpandError> { - for (key, value) in nested.inner { - if !self.inner.contains_key(&key) { - self.inner.insert(key.clone(), Binding::Nested(Vec::new())); - } - match self.inner.get_mut(&key) { - Some(Binding::Nested(it)) => { - // insert empty nested bindings before this one - while it.len() < idx { - it.push(Binding::Nested(vec![])); - } - it.push(value); - } - _ => { - return Err(ExpandError::BindingError(format!( - "could not find binding `{}`", - key - ))); - } - } - } - Ok(()) - } -} - -macro_rules! err { - () => { - ExpandError::BindingError(format!("")) - }; - ($($tt:tt)*) => { - ExpandError::BindingError(format!($($tt)*)) - }; -} - -#[derive(Debug, Default)] -pub(super) struct Match { - pub bindings: Bindings, - /// We currently just keep the first error and count the rest to compare matches. - pub err: Option, - pub err_count: usize, - /// How many top-level token trees were left to match. - pub unmatched_tts: usize, -} - -impl Match { - pub fn add_err(&mut self, err: ExpandError) { - let prev_err = self.err.take(); - self.err = prev_err.or(Some(err)); - self.err_count += 1; - } -} - -// General note: These functions have two channels to return errors, a `Result` -// return value and the `&mut Match`. The returned Result is for pattern parsing -// errors; if a branch of the macro definition doesn't parse, it doesn't make -// sense to try using it. Matching errors are added to the `Match`. It might -// make sense to make pattern parsing a separate step? - -pub(super) fn match_(pattern: &tt::Subtree, src: &tt::Subtree) -> Result { - assert!(pattern.delimiter == None); - - let mut res = Match::default(); - let mut src = TtIter::new(src); - - match_subtree(&mut res, pattern, &mut src)?; - - if src.len() > 0 { - res.unmatched_tts += src.len(); - res.add_err(err!("leftover tokens")); - } - - Ok(res) -} - -fn match_subtree( - res: &mut Match, - pattern: &tt::Subtree, - src: &mut TtIter, -) -> Result<(), ExpandError> { - for op in parse_pattern(pattern) { - match op? { - Op::TokenTree(tt::TokenTree::Leaf(lhs)) => { - let rhs = match src.expect_leaf() { - Ok(l) => l, - Err(()) => { - res.add_err(err!("expected leaf: `{}`", lhs)); - continue; - } - }; - match (lhs, rhs) { - ( - tt::Leaf::Punct(tt::Punct { char: lhs, .. }), - tt::Leaf::Punct(tt::Punct { char: rhs, .. }), - ) if lhs == rhs => (), - ( - tt::Leaf::Ident(tt::Ident { text: lhs, .. }), - tt::Leaf::Ident(tt::Ident { text: rhs, .. }), - ) if lhs == rhs => (), - ( - tt::Leaf::Literal(tt::Literal { text: lhs, .. }), - tt::Leaf::Literal(tt::Literal { text: rhs, .. }), - ) if lhs == rhs => (), - _ => { - res.add_err(ExpandError::UnexpectedToken); - } - } - } - Op::TokenTree(tt::TokenTree::Subtree(lhs)) => { - let rhs = match src.expect_subtree() { - Ok(s) => s, - Err(()) => { - res.add_err(err!("expected subtree")); - continue; - } - }; - if lhs.delimiter_kind() != rhs.delimiter_kind() { - res.add_err(err!("mismatched delimiter")); - continue; - } - let mut src = TtIter::new(rhs); - match_subtree(res, lhs, &mut src)?; - if src.len() > 0 { - res.add_err(err!("leftover tokens")); - } - } - Op::Var { name, kind } => { - let kind = match kind { - Some(k) => k, - None => { - res.add_err(ExpandError::UnexpectedToken); - continue; - } - }; - let ExpandResult(matched, match_err) = match_meta_var(kind.as_str(), src); - match matched { - Some(fragment) => { - res.bindings.inner.insert(name.clone(), Binding::Fragment(fragment)); - } - None if match_err.is_none() => res.bindings.push_optional(name), - _ => {} - } - if let Some(err) = match_err { - res.add_err(err); - } - } - Op::Repeat { subtree, kind, separator } => { - match_repeat(res, subtree, kind, separator, src)?; - } - } - } - Ok(()) -} - -impl<'a> TtIter<'a> { - fn eat_separator(&mut self, separator: &Separator) -> bool { - let mut fork = self.clone(); - let ok = match separator { - Separator::Ident(lhs) => match fork.expect_ident() { - Ok(rhs) => rhs.text == lhs.text, - _ => false, - }, - Separator::Literal(lhs) => match fork.expect_literal() { - Ok(rhs) => match rhs { - tt::Leaf::Literal(rhs) => rhs.text == lhs.text, - tt::Leaf::Ident(rhs) => rhs.text == lhs.text, - tt::Leaf::Punct(_) => false, - }, - _ => false, - }, - Separator::Puncts(lhss) => lhss.iter().all(|lhs| match fork.expect_punct() { - Ok(rhs) => rhs.char == lhs.char, - _ => false, - }), - }; - if ok { - *self = fork; - } - ok - } - - pub(crate) fn expect_tt(&mut self) -> Result { - match self.peek_n(0) { - Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '\'' => { - return self.expect_lifetime(); - } - _ => (), - } - - let tt = self.next().ok_or_else(|| ())?.clone(); - let punct = match tt { - tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if punct.spacing == tt::Spacing::Joint => { - punct - } - _ => return Ok(tt), - }; - - let (second, third) = match (self.peek_n(0), self.peek_n(1)) { - ( - Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p2))), - Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p3))), - ) if p2.spacing == tt::Spacing::Joint => (p2.char, Some(p3.char)), - (Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p2))), _) => (p2.char, None), - _ => return Ok(tt), - }; - - match (punct.char, second, third) { - ('.', '.', Some('.')) - | ('.', '.', Some('=')) - | ('<', '<', Some('=')) - | ('>', '>', Some('=')) => { - let tt2 = self.next().unwrap().clone(); - let tt3 = self.next().unwrap().clone(); - Ok(tt::Subtree { delimiter: None, token_trees: vec![tt, tt2, tt3] }.into()) - } - ('-', '=', None) - | ('-', '>', None) - | (':', ':', None) - | ('!', '=', None) - | ('.', '.', None) - | ('*', '=', None) - | ('/', '=', None) - | ('&', '&', None) - | ('&', '=', None) - | ('%', '=', None) - | ('^', '=', None) - | ('+', '=', None) - | ('<', '<', None) - | ('<', '=', None) - | ('=', '=', None) - | ('=', '>', None) - | ('>', '=', None) - | ('>', '>', None) - | ('|', '=', None) - | ('|', '|', None) => { - let tt2 = self.next().unwrap().clone(); - Ok(tt::Subtree { delimiter: None, token_trees: vec![tt, tt2] }.into()) - } - _ => Ok(tt), - } - } - - pub(crate) fn expect_lifetime(&mut self) -> Result { - let punct = self.expect_punct()?; - if punct.char != '\'' { - return Err(()); - } - let ident = self.expect_ident()?; - - Ok(tt::Subtree { - delimiter: None, - token_trees: vec![ - tt::Leaf::Punct(*punct).into(), - tt::Leaf::Ident(ident.clone()).into(), - ], - } - .into()) - } - - pub(crate) fn expect_fragment( - &mut self, - fragment_kind: parser::FragmentKind, - ) -> ExpandResult> { - pub(crate) struct OffsetTokenSink<'a> { - pub(crate) cursor: Cursor<'a>, - pub(crate) error: bool, - } - - impl<'a> TreeSink for OffsetTokenSink<'a> { - fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) { - if kind == SyntaxKind::LIFETIME { - n_tokens = 2; - } - for _ in 0..n_tokens { - self.cursor = self.cursor.bump_subtree(); - } - } - fn start_node(&mut self, _kind: SyntaxKind) {} - fn finish_node(&mut self) {} - fn error(&mut self, _error: parser::ParseError) { - self.error = true; - } - } - - let buffer = TokenBuffer::new(&self.inner.as_slice()); - let mut src = SubtreeTokenSource::new(&buffer); - let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false }; - - parser::parse_fragment(&mut src, &mut sink, fragment_kind); - - let mut err = None; - if !sink.cursor.is_root() || sink.error { - err = Some(err!("expected {:?}", fragment_kind)); - } - - let mut curr = buffer.begin(); - let mut res = vec![]; - - if sink.cursor.is_root() { - while curr != sink.cursor { - if let Some(token) = curr.token_tree() { - res.push(token); - } - curr = curr.bump(); - } - } - self.inner = self.inner.as_slice()[res.len()..].iter(); - if res.len() == 0 && err.is_none() { - err = Some(err!("no tokens consumed")); - } - let res = match res.len() { - 1 => Some(res[0].clone()), - 0 => None, - _ => Some(tt::TokenTree::Subtree(tt::Subtree { - delimiter: None, - token_trees: res.into_iter().cloned().collect(), - })), - }; - ExpandResult(res, err) - } - - pub(crate) fn eat_vis(&mut self) -> Option { - let mut fork = self.clone(); - match fork.expect_fragment(Visibility) { - ExpandResult(tt, None) => { - *self = fork; - tt - } - ExpandResult(_, Some(_)) => None, - } - } -} - -pub(super) fn match_repeat( - res: &mut Match, - pattern: &tt::Subtree, - kind: RepeatKind, - separator: Option, - src: &mut TtIter, -) -> Result<(), ExpandError> { - // Dirty hack to make macro-expansion terminate. - // This should be replaced by a propper macro-by-example implementation - let mut limit = 65536; - let mut counter = 0; - - for i in 0.. { - let mut fork = src.clone(); - - if let Some(separator) = &separator { - if i != 0 && !fork.eat_separator(separator) { - break; - } - } - - let mut nested = Match::default(); - match_subtree(&mut nested, pattern, &mut fork)?; - if nested.err.is_none() { - limit -= 1; - if limit == 0 { - log::warn!( - "match_lhs exceeded repeat pattern limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", - pattern, - src, - kind, - separator - ); - break; - } - *src = fork; - - if let Err(err) = res.bindings.push_nested(counter, nested.bindings) { - res.add_err(err); - } - counter += 1; - if counter == 1 { - if let RepeatKind::ZeroOrOne = kind { - break; - } - } - } else { - break; - } - } - - match (kind, counter) { - (RepeatKind::OneOrMore, 0) => { - res.add_err(ExpandError::UnexpectedToken); - } - (_, 0) => { - // Collect all empty variables in subtrees - let mut vars = Vec::new(); - collect_vars(&mut vars, pattern)?; - for var in vars { - res.bindings.push_empty(&var) - } - } - _ => (), - } - Ok(()) -} - -fn match_meta_var(kind: &str, input: &mut TtIter) -> ExpandResult> { - let fragment = match kind { - "path" => Path, - "expr" => Expr, - "ty" => Type, - "pat" => Pattern, - "stmt" => Statement, - "block" => Block, - "meta" => MetaItem, - "item" => Item, - _ => { - let tt_result = match kind { - "ident" => input - .expect_ident() - .map(|ident| Some(tt::Leaf::from(ident.clone()).into())) - .map_err(|()| err!("expected ident")), - "tt" => input.expect_tt().map(Some).map_err(|()| err!()), - "lifetime" => input - .expect_lifetime() - .map(|tt| Some(tt)) - .map_err(|()| err!("expected lifetime")), - "literal" => input - .expect_literal() - .map(|literal| Some(tt::Leaf::from(literal.clone()).into())) - .map_err(|()| err!()), - // `vis` is optional - "vis" => match input.eat_vis() { - Some(vis) => Ok(Some(vis)), - None => Ok(None), - }, - _ => Err(ExpandError::UnexpectedToken), - }; - return tt_result.map(|it| it.map(Fragment::Tokens)).into(); - } - }; - let result = input.expect_fragment(fragment); - result.map(|tt| if kind == "expr" { tt.map(Fragment::Ast) } else { tt.map(Fragment::Tokens) }) -} - -fn collect_vars(buf: &mut Vec, pattern: &tt::Subtree) -> Result<(), ExpandError> { - for op in parse_pattern(pattern) { - match op? { - Op::Var { name, .. } => buf.push(name.clone()), - Op::TokenTree(tt::TokenTree::Leaf(_)) => (), - Op::TokenTree(tt::TokenTree::Subtree(subtree)) => collect_vars(buf, subtree)?, - Op::Repeat { subtree, .. } => collect_vars(buf, subtree)?, - } - } - Ok(()) -} diff --git a/crates/ra_mbe/src/mbe_expander/transcriber.rs b/crates/ra_mbe/src/mbe_expander/transcriber.rs deleted file mode 100644 index c9525c5bf..000000000 --- a/crates/ra_mbe/src/mbe_expander/transcriber.rs +++ /dev/null @@ -1,254 +0,0 @@ -//! Transcriber takes a template, like `fn $ident() {}`, a set of bindings like -//! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}` - -use syntax::SmolStr; - -use super::ExpandResult; -use crate::{ - mbe_expander::{Binding, Bindings, Fragment}, - parser::{parse_template, Op, RepeatKind, Separator}, - ExpandError, -}; - -impl Bindings { - fn contains(&self, name: &str) -> bool { - self.inner.contains_key(name) - } - - fn get(&self, name: &str, nesting: &mut [NestingState]) -> Result<&Fragment, ExpandError> { - let mut b = self.inner.get(name).ok_or_else(|| { - ExpandError::BindingError(format!("could not find binding `{}`", name)) - })?; - for nesting_state in nesting.iter_mut() { - nesting_state.hit = true; - b = match b { - Binding::Fragment(_) => break, - Binding::Nested(bs) => bs.get(nesting_state.idx).ok_or_else(|| { - nesting_state.at_end = true; - ExpandError::BindingError(format!("could not find nested binding `{}`", name)) - })?, - Binding::Empty => { - nesting_state.at_end = true; - return Err(ExpandError::BindingError(format!( - "could not find empty binding `{}`", - name - ))); - } - }; - } - match b { - Binding::Fragment(it) => Ok(it), - Binding::Nested(_) => Err(ExpandError::BindingError(format!( - "expected simple binding, found nested binding `{}`", - name - ))), - Binding::Empty => Err(ExpandError::BindingError(format!( - "expected simple binding, found empty binding `{}`", - name - ))), - } - } -} - -pub(super) fn transcribe(template: &tt::Subtree, bindings: &Bindings) -> ExpandResult { - assert!(template.delimiter == None); - let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new() }; - let mut arena: Vec = Vec::new(); - expand_subtree(&mut ctx, template, &mut arena) -} - -#[derive(Debug)] -struct NestingState { - idx: usize, - /// `hit` is currently necessary to tell `expand_repeat` if it should stop - /// because there is no variable in use by the current repetition - hit: bool, - /// `at_end` is currently necessary to tell `expand_repeat` if it should stop - /// because there is no more value avaible for the current repetition - at_end: bool, -} - -#[derive(Debug)] -struct ExpandCtx<'a> { - bindings: &'a Bindings, - nesting: Vec, -} - -fn expand_subtree( - ctx: &mut ExpandCtx, - template: &tt::Subtree, - arena: &mut Vec, -) -> ExpandResult { - // remember how many elements are in the arena now - when returning, we want to drain exactly how many elements we added. This way, the recursive uses of the arena get their own "view" of the arena, but will reuse the allocation - let start_elements = arena.len(); - let mut err = None; - for op in parse_template(template) { - let op = match op { - Ok(op) => op, - Err(e) => { - err = Some(e); - break; - } - }; - match op { - Op::TokenTree(tt @ tt::TokenTree::Leaf(..)) => arena.push(tt.clone()), - Op::TokenTree(tt::TokenTree::Subtree(tt)) => { - let ExpandResult(tt, e) = expand_subtree(ctx, tt, arena); - err = err.or(e); - arena.push(tt.into()); - } - Op::Var { name, kind: _ } => { - let ExpandResult(fragment, e) = expand_var(ctx, name); - err = err.or(e); - push_fragment(arena, fragment); - } - Op::Repeat { subtree, kind, separator } => { - let ExpandResult(fragment, e) = expand_repeat(ctx, subtree, kind, separator, arena); - err = err.or(e); - push_fragment(arena, fragment) - } - } - } - // drain the elements added in this instance of expand_subtree - let tts = arena.drain(start_elements..arena.len()).collect(); - ExpandResult(tt::Subtree { delimiter: template.delimiter, token_trees: tts }, err) -} - -fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> ExpandResult { - if v == "crate" { - // We simply produce identifier `$crate` here. And it will be resolved when lowering ast to Path. - let tt = - tt::Leaf::from(tt::Ident { text: "$crate".into(), id: tt::TokenId::unspecified() }) - .into(); - ExpandResult::ok(Fragment::Tokens(tt)) - } else if !ctx.bindings.contains(v) { - // Note that it is possible to have a `$var` inside a macro which is not bound. - // For example: - // ``` - // macro_rules! foo { - // ($a:ident, $b:ident, $c:tt) => { - // macro_rules! bar { - // ($bi:ident) => { - // fn $bi() -> u8 {$c} - // } - // } - // } - // ``` - // We just treat it a normal tokens - let tt = tt::Subtree { - delimiter: None, - token_trees: vec![ - tt::Leaf::from(tt::Punct { - char: '$', - spacing: tt::Spacing::Alone, - id: tt::TokenId::unspecified(), - }) - .into(), - tt::Leaf::from(tt::Ident { text: v.clone(), id: tt::TokenId::unspecified() }) - .into(), - ], - } - .into(); - ExpandResult::ok(Fragment::Tokens(tt)) - } else { - ctx.bindings.get(&v, &mut ctx.nesting).map_or_else( - |e| ExpandResult(Fragment::Tokens(tt::TokenTree::empty()), Some(e)), - |b| ExpandResult::ok(b.clone()), - ) - } -} - -fn expand_repeat( - ctx: &mut ExpandCtx, - template: &tt::Subtree, - kind: RepeatKind, - separator: Option, - arena: &mut Vec, -) -> ExpandResult { - let mut buf: Vec = Vec::new(); - ctx.nesting.push(NestingState { idx: 0, at_end: false, hit: false }); - // Dirty hack to make macro-expansion terminate. - // This should be replaced by a proper macro-by-example implementation - let limit = 65536; - let mut has_seps = 0; - let mut counter = 0; - - loop { - let ExpandResult(mut t, e) = expand_subtree(ctx, template, arena); - let nesting_state = ctx.nesting.last_mut().unwrap(); - if nesting_state.at_end || !nesting_state.hit { - break; - } - nesting_state.idx += 1; - nesting_state.hit = false; - - counter += 1; - if counter == limit { - log::warn!( - "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}", - template, - ctx - ); - break; - } - - if e.is_some() { - continue; - } - - t.delimiter = None; - push_subtree(&mut buf, t); - - if let Some(ref sep) = separator { - match sep { - Separator::Ident(ident) => { - has_seps = 1; - buf.push(tt::Leaf::from(ident.clone()).into()); - } - Separator::Literal(lit) => { - has_seps = 1; - buf.push(tt::Leaf::from(lit.clone()).into()); - } - - Separator::Puncts(puncts) => { - has_seps = puncts.len(); - for punct in puncts { - buf.push(tt::Leaf::from(*punct).into()); - } - } - } - } - - if RepeatKind::ZeroOrOne == kind { - break; - } - } - - ctx.nesting.pop().unwrap(); - for _ in 0..has_seps { - buf.pop(); - } - - // Check if it is a single token subtree without any delimiter - // e.g {Delimiter:None> ['>'] /Delimiter:None>} - let tt = tt::Subtree { delimiter: None, token_trees: buf }.into(); - - if RepeatKind::OneOrMore == kind && counter == 0 { - return ExpandResult(Fragment::Tokens(tt), Some(ExpandError::UnexpectedToken)); - } - ExpandResult::ok(Fragment::Tokens(tt)) -} - -fn push_fragment(buf: &mut Vec, fragment: Fragment) { - match fragment { - Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt), - Fragment::Tokens(tt) | Fragment::Ast(tt) => buf.push(tt), - } -} - -fn push_subtree(buf: &mut Vec, tt: tt::Subtree) { - match tt.delimiter { - None => buf.extend(tt.token_trees), - _ => buf.push(tt.into()), - } -} diff --git a/crates/ra_mbe/src/parser.rs b/crates/ra_mbe/src/parser.rs deleted file mode 100644 index 6b46a1673..000000000 --- a/crates/ra_mbe/src/parser.rs +++ /dev/null @@ -1,184 +0,0 @@ -//! Parser recognizes special macro syntax, `$var` and `$(repeat)*`, in token -//! trees. - -use smallvec::SmallVec; -use syntax::SmolStr; - -use crate::{tt_iter::TtIter, ExpandError}; - -#[derive(Debug)] -pub(crate) enum Op<'a> { - Var { name: &'a SmolStr, kind: Option<&'a SmolStr> }, - Repeat { subtree: &'a tt::Subtree, kind: RepeatKind, separator: Option }, - TokenTree(&'a tt::TokenTree), -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) enum RepeatKind { - ZeroOrMore, - OneOrMore, - ZeroOrOne, -} - -#[derive(Clone, Debug, Eq)] -pub(crate) enum Separator { - Literal(tt::Literal), - Ident(tt::Ident), - Puncts(SmallVec<[tt::Punct; 3]>), -} - -// Note that when we compare a Separator, we just care about its textual value. -impl PartialEq for Separator { - fn eq(&self, other: &Separator) -> bool { - use Separator::*; - - match (self, other) { - (Ident(ref a), Ident(ref b)) => a.text == b.text, - (Literal(ref a), Literal(ref b)) => a.text == b.text, - (Puncts(ref a), Puncts(ref b)) if a.len() == b.len() => { - let a_iter = a.iter().map(|a| a.char); - let b_iter = b.iter().map(|b| b.char); - a_iter.eq(b_iter) - } - _ => false, - } - } -} - -pub(crate) fn parse_template( - template: &tt::Subtree, -) -> impl Iterator, ExpandError>> { - parse_inner(template, Mode::Template) -} - -pub(crate) fn parse_pattern( - pattern: &tt::Subtree, -) -> impl Iterator, ExpandError>> { - parse_inner(pattern, Mode::Pattern) -} - -#[derive(Clone, Copy)] -enum Mode { - Pattern, - Template, -} - -fn parse_inner(src: &tt::Subtree, mode: Mode) -> impl Iterator, ExpandError>> { - let mut src = TtIter::new(src); - std::iter::from_fn(move || { - let first = src.next()?; - Some(next_op(first, &mut src, mode)) - }) -} - -macro_rules! err { - ($($tt:tt)*) => { - ExpandError::UnexpectedToken - }; -} - -macro_rules! bail { - ($($tt:tt)*) => { - return Err(err!($($tt)*)) - }; -} - -fn next_op<'a>( - first: &'a tt::TokenTree, - src: &mut TtIter<'a>, - mode: Mode, -) -> Result, ExpandError> { - let res = match first { - tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: '$', .. })) => { - // Note that the '$' itself is a valid token inside macro_rules. - let second = match src.next() { - None => return Ok(Op::TokenTree(first)), - Some(it) => it, - }; - match second { - tt::TokenTree::Subtree(subtree) => { - let (separator, kind) = parse_repeat(src)?; - Op::Repeat { subtree, separator, kind } - } - tt::TokenTree::Leaf(leaf) => match leaf { - tt::Leaf::Punct(..) => return Err(ExpandError::UnexpectedToken), - tt::Leaf::Ident(ident) => { - let name = &ident.text; - let kind = eat_fragment_kind(src, mode)?; - Op::Var { name, kind } - } - tt::Leaf::Literal(lit) => { - if is_boolean_literal(lit) { - let name = &lit.text; - let kind = eat_fragment_kind(src, mode)?; - Op::Var { name, kind } - } else { - bail!("bad var 2"); - } - } - }, - } - } - tt => Op::TokenTree(tt), - }; - Ok(res) -} - -fn eat_fragment_kind<'a>( - src: &mut TtIter<'a>, - mode: Mode, -) -> Result, ExpandError> { - if let Mode::Pattern = mode { - src.expect_char(':').map_err(|()| err!("bad fragment specifier 1"))?; - let ident = src.expect_ident().map_err(|()| err!("bad fragment specifier 1"))?; - return Ok(Some(&ident.text)); - }; - Ok(None) -} - -fn is_boolean_literal(lit: &tt::Literal) -> bool { - matches!(lit.text.as_str(), "true" | "false") -} - -fn parse_repeat(src: &mut TtIter) -> Result<(Option, RepeatKind), ExpandError> { - let mut separator = Separator::Puncts(SmallVec::new()); - for tt in src { - let tt = match tt { - tt::TokenTree::Leaf(leaf) => leaf, - tt::TokenTree::Subtree(_) => return Err(ExpandError::InvalidRepeat), - }; - let has_sep = match &separator { - Separator::Puncts(puncts) => !puncts.is_empty(), - _ => true, - }; - match tt { - tt::Leaf::Ident(_) | tt::Leaf::Literal(_) if has_sep => { - return Err(ExpandError::InvalidRepeat) - } - tt::Leaf::Ident(ident) => separator = Separator::Ident(ident.clone()), - tt::Leaf::Literal(lit) => separator = Separator::Literal(lit.clone()), - tt::Leaf::Punct(punct) => { - let repeat_kind = match punct.char { - '*' => RepeatKind::ZeroOrMore, - '+' => RepeatKind::OneOrMore, - '?' => RepeatKind::ZeroOrOne, - _ => { - match &mut separator { - Separator::Puncts(puncts) => { - if puncts.len() == 3 { - return Err(ExpandError::InvalidRepeat); - } - puncts.push(punct.clone()) - } - _ => return Err(ExpandError::InvalidRepeat), - } - continue; - } - }; - let separator = if has_sep { Some(separator) } else { None }; - return Ok((separator, repeat_kind)); - } - } - } - Err(ExpandError::InvalidRepeat) -} diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs deleted file mode 100644 index 41461b315..000000000 --- a/crates/ra_mbe/src/subtree_source.rs +++ /dev/null @@ -1,197 +0,0 @@ -//! FIXME: write short doc here - -use parser::{Token, TokenSource}; -use std::cell::{Cell, Ref, RefCell}; -use syntax::{lex_single_syntax_kind, SmolStr, SyntaxKind, SyntaxKind::*, T}; -use tt::buffer::{Cursor, TokenBuffer}; - -#[derive(Debug, Clone, Eq, PartialEq)] -struct TtToken { - pub kind: SyntaxKind, - pub is_joint_to_next: bool, - pub text: SmolStr, -} - -pub(crate) struct SubtreeTokenSource<'a> { - cached_cursor: Cell>, - cached: RefCell>>, - curr: (Token, usize), -} - -impl<'a> SubtreeTokenSource<'a> { - // Helper function used in test - #[cfg(test)] - pub fn text(&self) -> SmolStr { - match *self.get(self.curr.1) { - Some(ref tt) => tt.text.clone(), - _ => SmolStr::new(""), - } - } -} - -impl<'a> SubtreeTokenSource<'a> { - pub fn new(buffer: &'a TokenBuffer) -> SubtreeTokenSource<'a> { - let cursor = buffer.begin(); - - let mut res = SubtreeTokenSource { - curr: (Token { kind: EOF, is_jointed_to_next: false }, 0), - cached_cursor: Cell::new(cursor), - cached: RefCell::new(Vec::with_capacity(10)), - }; - res.curr = (res.mk_token(0), 0); - res - } - - fn mk_token(&self, pos: usize) -> Token { - match *self.get(pos) { - Some(ref tt) => Token { kind: tt.kind, is_jointed_to_next: tt.is_joint_to_next }, - None => Token { kind: EOF, is_jointed_to_next: false }, - } - } - - fn get(&self, pos: usize) -> Ref> { - fn is_lifetime(c: Cursor) -> Option<(Cursor, SmolStr)> { - let tkn = c.token_tree(); - - if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = tkn { - if punct.char == '\'' { - let next = c.bump(); - if let Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) = next.token_tree() { - let res_cursor = next.bump(); - let text = SmolStr::new("'".to_string() + &ident.to_string()); - - return Some((res_cursor, text)); - } else { - panic!("Next token must be ident : {:#?}", next.token_tree()); - } - } - } - - None - } - - if pos < self.cached.borrow().len() { - return Ref::map(self.cached.borrow(), |c| &c[pos]); - } - - { - let mut cached = self.cached.borrow_mut(); - while pos >= cached.len() { - let cursor = self.cached_cursor.get(); - if cursor.eof() { - cached.push(None); - continue; - } - - if let Some((curr, text)) = is_lifetime(cursor) { - cached.push(Some(TtToken { kind: LIFETIME, is_joint_to_next: false, text })); - self.cached_cursor.set(curr); - continue; - } - - match cursor.token_tree() { - Some(tt::TokenTree::Leaf(leaf)) => { - cached.push(Some(convert_leaf(&leaf))); - self.cached_cursor.set(cursor.bump()); - } - Some(tt::TokenTree::Subtree(subtree)) => { - self.cached_cursor.set(cursor.subtree().unwrap()); - cached.push(Some(convert_delim(subtree.delimiter_kind(), false))); - } - None => { - if let Some(subtree) = cursor.end() { - cached.push(Some(convert_delim(subtree.delimiter_kind(), true))); - self.cached_cursor.set(cursor.bump()); - } - } - } - } - } - - Ref::map(self.cached.borrow(), |c| &c[pos]) - } -} - -impl<'a> TokenSource for SubtreeTokenSource<'a> { - fn current(&self) -> Token { - self.curr.0 - } - - /// Lookahead n token - fn lookahead_nth(&self, n: usize) -> Token { - self.mk_token(self.curr.1 + n) - } - - /// bump cursor to next token - fn bump(&mut self) { - if self.current().kind == EOF { - return; - } - - self.curr = (self.mk_token(self.curr.1 + 1), self.curr.1 + 1); - } - - /// Is the current token a specified keyword? - fn is_keyword(&self, kw: &str) -> bool { - match *self.get(self.curr.1) { - Some(ref t) => t.text == *kw, - _ => false, - } - } -} - -fn convert_delim(d: Option, closing: bool) -> TtToken { - let (kinds, texts) = match d { - Some(tt::DelimiterKind::Parenthesis) => ([T!['('], T![')']], "()"), - Some(tt::DelimiterKind::Brace) => ([T!['{'], T!['}']], "{}"), - Some(tt::DelimiterKind::Bracket) => ([T!['['], T![']']], "[]"), - None => ([L_DOLLAR, R_DOLLAR], ""), - }; - - let idx = closing as usize; - let kind = kinds[idx]; - let text = if !texts.is_empty() { &texts[idx..texts.len() - (1 - idx)] } else { "" }; - TtToken { kind, is_joint_to_next: false, text: SmolStr::new(text) } -} - -fn convert_literal(l: &tt::Literal) -> TtToken { - let kind = lex_single_syntax_kind(&l.text) - .map(|(kind, _error)| kind) - .filter(|kind| kind.is_literal()) - .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &l)); - - TtToken { kind, is_joint_to_next: false, text: l.text.clone() } -} - -fn convert_ident(ident: &tt::Ident) -> TtToken { - let kind = match ident.text.as_ref() { - "true" => T![true], - "false" => T![false], - i if i.starts_with('\'') => LIFETIME, - _ => SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT), - }; - - TtToken { kind, is_joint_to_next: false, text: ident.text.clone() } -} - -fn convert_punct(p: tt::Punct) -> TtToken { - let kind = match SyntaxKind::from_char(p.char) { - None => panic!("{:#?} is not a valid punct", p), - Some(kind) => kind, - }; - - let text = { - let mut buf = [0u8; 4]; - let s: &str = p.char.encode_utf8(&mut buf); - SmolStr::new(s) - }; - TtToken { kind, is_joint_to_next: p.spacing == tt::Spacing::Joint, text } -} - -fn convert_leaf(leaf: &tt::Leaf) -> TtToken { - match leaf { - tt::Leaf::Literal(l) => convert_literal(l), - tt::Leaf::Ident(ident) => convert_ident(ident), - tt::Leaf::Punct(punct) => convert_punct(*punct), - } -} diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs deleted file mode 100644 index a8ad917fb..000000000 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ /dev/null @@ -1,832 +0,0 @@ -//! FIXME: write short doc here - -use parser::{FragmentKind, ParseError, TreeSink}; -use rustc_hash::FxHashMap; -use syntax::{ - ast::{self, make::tokens::doc_comment}, - tokenize, AstToken, Parse, SmolStr, SyntaxKind, - SyntaxKind::*, - SyntaxNode, SyntaxToken, SyntaxTreeBuilder, TextRange, TextSize, Token as RawToken, T, -}; -use tt::buffer::{Cursor, TokenBuffer}; - -use crate::subtree_source::SubtreeTokenSource; -use crate::ExpandError; - -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub enum TokenTextRange { - Token(TextRange), - Delimiter(TextRange, TextRange), -} - -impl TokenTextRange { - pub fn by_kind(self, kind: SyntaxKind) -> Option { - match self { - TokenTextRange::Token(it) => Some(it), - TokenTextRange::Delimiter(open, close) => match kind { - T!['{'] | T!['('] | T!['['] => Some(open), - T!['}'] | T![')'] | T![']'] => Some(close), - _ => None, - }, - } - } -} - -/// Maps `tt::TokenId` to the relative range of the original token. -#[derive(Debug, PartialEq, Eq, Clone, Default)] -pub struct TokenMap { - /// Maps `tt::TokenId` to the *relative* source range. - entries: Vec<(tt::TokenId, TokenTextRange)>, -} - -/// Convert the syntax tree (what user has written) to a `TokenTree` (what macro -/// will consume). -pub fn ast_to_token_tree(ast: &impl ast::AstNode) -> Option<(tt::Subtree, TokenMap)> { - syntax_node_to_token_tree(ast.syntax()) -} - -/// Convert the syntax node to a `TokenTree` (what macro -/// will consume). -pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, TokenMap)> { - let global_offset = node.text_range().start(); - let mut c = Convertor::new(node, global_offset); - let subtree = c.go()?; - Some((subtree, c.id_alloc.map)) -} - -// The following items are what `rustc` macro can be parsed into : -// link: https://github.com/rust-lang/rust/blob/9ebf47851a357faa4cd97f4b1dc7835f6376e639/src/libsyntax/ext/expand.rs#L141 -// * Expr(P) -> token_tree_to_expr -// * Pat(P) -> token_tree_to_pat -// * Ty(P) -> token_tree_to_ty -// * Stmts(SmallVec<[ast::Stmt; 1]>) -> token_tree_to_stmts -// * Items(SmallVec<[P; 1]>) -> token_tree_to_items -// -// * TraitItems(SmallVec<[ast::TraitItem; 1]>) -// * AssocItems(SmallVec<[ast::AssocItem; 1]>) -// * ForeignItems(SmallVec<[ast::ForeignItem; 1]> - -pub fn token_tree_to_syntax_node( - tt: &tt::Subtree, - fragment_kind: FragmentKind, -) -> Result<(Parse, TokenMap), ExpandError> { - let tmp; - let tokens = match tt { - tt::Subtree { delimiter: None, token_trees } => token_trees.as_slice(), - _ => { - tmp = [tt.clone().into()]; - &tmp[..] - } - }; - let buffer = TokenBuffer::new(&tokens); - let mut token_source = SubtreeTokenSource::new(&buffer); - let mut tree_sink = TtTreeSink::new(buffer.begin()); - parser::parse_fragment(&mut token_source, &mut tree_sink, fragment_kind); - if tree_sink.roots.len() != 1 { - return Err(ExpandError::ConversionError); - } - //FIXME: would be cool to report errors - let (parse, range_map) = tree_sink.finish(); - Ok((parse, range_map)) -} - -/// Convert a string to a `TokenTree` -pub fn parse_to_token_tree(text: &str) -> Option<(tt::Subtree, TokenMap)> { - let (tokens, errors) = tokenize(text); - if !errors.is_empty() { - return None; - } - - let mut conv = RawConvertor { - text, - offset: TextSize::default(), - inner: tokens.iter(), - id_alloc: TokenIdAlloc { - map: Default::default(), - global_offset: TextSize::default(), - next_id: 0, - }, - }; - - let subtree = conv.go()?; - Some((subtree, conv.id_alloc.map)) -} - -impl TokenMap { - pub fn token_by_range(&self, relative_range: TextRange) -> Option { - let &(token_id, _) = self.entries.iter().find(|(_, range)| match range { - TokenTextRange::Token(it) => *it == relative_range, - TokenTextRange::Delimiter(open, close) => { - *open == relative_range || *close == relative_range - } - })?; - Some(token_id) - } - - pub fn range_by_token(&self, token_id: tt::TokenId) -> Option { - let &(_, range) = self.entries.iter().find(|(tid, _)| *tid == token_id)?; - Some(range) - } - - fn insert(&mut self, token_id: tt::TokenId, relative_range: TextRange) { - self.entries.push((token_id, TokenTextRange::Token(relative_range))); - } - - fn insert_delim( - &mut self, - token_id: tt::TokenId, - open_relative_range: TextRange, - close_relative_range: TextRange, - ) -> usize { - let res = self.entries.len(); - self.entries - .push((token_id, TokenTextRange::Delimiter(open_relative_range, close_relative_range))); - res - } - - fn update_close_delim(&mut self, idx: usize, close_relative_range: TextRange) { - let (_, token_text_range) = &mut self.entries[idx]; - if let TokenTextRange::Delimiter(dim, _) = token_text_range { - *token_text_range = TokenTextRange::Delimiter(*dim, close_relative_range); - } - } - - fn remove_delim(&mut self, idx: usize) { - // FIXME: This could be accidently quadratic - self.entries.remove(idx); - } -} - -/// Returns the textual content of a doc comment block as a quoted string -/// That is, strips leading `///` (or `/**`, etc) -/// and strips the ending `*/` -/// And then quote the string, which is needed to convert to `tt::Literal` -fn doc_comment_text(comment: &ast::Comment) -> SmolStr { - let prefix_len = comment.prefix().len(); - let mut text = &comment.text()[prefix_len..]; - - // Remove ending "*/" - if comment.kind().shape == ast::CommentShape::Block { - text = &text[0..text.len() - 2]; - } - - // Quote the string - // Note that `tt::Literal` expect an escaped string - let text = format!("{:?}", text.escape_default().to_string()); - text.into() -} - -fn convert_doc_comment(token: &syntax::SyntaxToken) -> Option> { - let comment = ast::Comment::cast(token.clone())?; - let doc = comment.kind().doc?; - - // Make `doc="\" Comments\"" - let mut meta_tkns = Vec::new(); - meta_tkns.push(mk_ident("doc")); - meta_tkns.push(mk_punct('=')); - meta_tkns.push(mk_doc_literal(&comment)); - - // Make `#![]` - let mut token_trees = Vec::new(); - token_trees.push(mk_punct('#')); - if let ast::CommentPlacement::Inner = doc { - token_trees.push(mk_punct('!')); - } - token_trees.push(tt::TokenTree::from(tt::Subtree { - delimiter: Some(tt::Delimiter { - kind: tt::DelimiterKind::Bracket, - id: tt::TokenId::unspecified(), - }), - token_trees: meta_tkns, - })); - - return Some(token_trees); - - // Helper functions - fn mk_ident(s: &str) -> tt::TokenTree { - tt::TokenTree::from(tt::Leaf::from(tt::Ident { - text: s.into(), - id: tt::TokenId::unspecified(), - })) - } - - fn mk_punct(c: char) -> tt::TokenTree { - tt::TokenTree::from(tt::Leaf::from(tt::Punct { - char: c, - spacing: tt::Spacing::Alone, - id: tt::TokenId::unspecified(), - })) - } - - fn mk_doc_literal(comment: &ast::Comment) -> tt::TokenTree { - let lit = tt::Literal { text: doc_comment_text(comment), id: tt::TokenId::unspecified() }; - - tt::TokenTree::from(tt::Leaf::from(lit)) - } -} - -struct TokenIdAlloc { - map: TokenMap, - global_offset: TextSize, - next_id: u32, -} - -impl TokenIdAlloc { - fn alloc(&mut self, absolute_range: TextRange) -> tt::TokenId { - let relative_range = absolute_range - self.global_offset; - let token_id = tt::TokenId(self.next_id); - self.next_id += 1; - self.map.insert(token_id, relative_range); - token_id - } - - fn open_delim(&mut self, open_abs_range: TextRange) -> (tt::TokenId, usize) { - let token_id = tt::TokenId(self.next_id); - self.next_id += 1; - let idx = self.map.insert_delim( - token_id, - open_abs_range - self.global_offset, - open_abs_range - self.global_offset, - ); - (token_id, idx) - } - - fn close_delim(&mut self, idx: usize, close_abs_range: Option) { - match close_abs_range { - None => { - self.map.remove_delim(idx); - } - Some(close) => { - self.map.update_close_delim(idx, close - self.global_offset); - } - } - } -} - -/// A Raw Token (straightly from lexer) convertor -struct RawConvertor<'a> { - text: &'a str, - offset: TextSize, - id_alloc: TokenIdAlloc, - inner: std::slice::Iter<'a, RawToken>, -} - -trait SrcToken: std::fmt::Debug { - fn kind(&self) -> SyntaxKind; - - fn to_char(&self) -> Option; - - fn to_text(&self) -> SmolStr; -} - -trait TokenConvertor { - type Token: SrcToken; - - fn go(&mut self) -> Option { - let mut subtree = tt::Subtree::default(); - subtree.delimiter = None; - while self.peek().is_some() { - self.collect_leaf(&mut subtree.token_trees); - } - if subtree.token_trees.is_empty() { - return None; - } - if subtree.token_trees.len() == 1 { - if let tt::TokenTree::Subtree(first) = &subtree.token_trees[0] { - return Some(first.clone()); - } - } - Some(subtree) - } - - fn collect_leaf(&mut self, result: &mut Vec) { - let (token, range) = match self.bump() { - None => return, - Some(it) => it, - }; - - let k: SyntaxKind = token.kind(); - if k == COMMENT { - if let Some(tokens) = self.convert_doc_comment(&token) { - result.extend(tokens); - } - return; - } - - result.push(if k.is_punct() { - assert_eq!(range.len(), TextSize::of('.')); - let delim = match k { - T!['('] => Some((tt::DelimiterKind::Parenthesis, T![')'])), - T!['{'] => Some((tt::DelimiterKind::Brace, T!['}'])), - T!['['] => Some((tt::DelimiterKind::Bracket, T![']'])), - _ => None, - }; - - if let Some((kind, closed)) = delim { - let mut subtree = tt::Subtree::default(); - let (id, idx) = self.id_alloc().open_delim(range); - subtree.delimiter = Some(tt::Delimiter { kind, id }); - - while self.peek().map(|it| it.kind() != closed).unwrap_or(false) { - self.collect_leaf(&mut subtree.token_trees); - } - let last_range = match self.bump() { - None => { - // For error resilience, we insert an char punct for the opening delim here - self.id_alloc().close_delim(idx, None); - let leaf: tt::Leaf = tt::Punct { - id: self.id_alloc().alloc(range), - char: token.to_char().unwrap(), - spacing: tt::Spacing::Alone, - } - .into(); - result.push(leaf.into()); - result.extend(subtree.token_trees); - return; - } - Some(it) => it.1, - }; - self.id_alloc().close_delim(idx, Some(last_range)); - subtree.into() - } else { - let spacing = match self.peek() { - Some(next) - if next.kind().is_trivia() - || next.kind() == T!['['] - || next.kind() == T!['{'] - || next.kind() == T!['('] => - { - tt::Spacing::Alone - } - Some(next) if next.kind().is_punct() => tt::Spacing::Joint, - _ => tt::Spacing::Alone, - }; - let char = match token.to_char() { - Some(c) => c, - None => { - panic!("Token from lexer must be single char: token = {:#?}", token); - } - }; - tt::Leaf::from(tt::Punct { char, spacing, id: self.id_alloc().alloc(range) }).into() - } - } else { - macro_rules! make_leaf { - ($i:ident) => { - tt::$i { id: self.id_alloc().alloc(range), text: token.to_text() }.into() - }; - } - let leaf: tt::Leaf = match k { - T![true] | T![false] => make_leaf!(Ident), - IDENT => make_leaf!(Ident), - k if k.is_keyword() => make_leaf!(Ident), - k if k.is_literal() => make_leaf!(Literal), - LIFETIME => { - let char_unit = TextSize::of('\''); - let r = TextRange::at(range.start(), char_unit); - let apostrophe = tt::Leaf::from(tt::Punct { - char: '\'', - spacing: tt::Spacing::Joint, - id: self.id_alloc().alloc(r), - }); - result.push(apostrophe.into()); - - let r = TextRange::at(range.start() + char_unit, range.len() - char_unit); - let ident = tt::Leaf::from(tt::Ident { - text: SmolStr::new(&token.to_text()[1..]), - id: self.id_alloc().alloc(r), - }); - result.push(ident.into()); - return; - } - _ => return, - }; - - leaf.into() - }); - } - - fn convert_doc_comment(&self, token: &Self::Token) -> Option>; - - fn bump(&mut self) -> Option<(Self::Token, TextRange)>; - - fn peek(&self) -> Option; - - fn id_alloc(&mut self) -> &mut TokenIdAlloc; -} - -impl<'a> SrcToken for (RawToken, &'a str) { - fn kind(&self) -> SyntaxKind { - self.0.kind - } - - fn to_char(&self) -> Option { - self.1.chars().next() - } - - fn to_text(&self) -> SmolStr { - self.1.into() - } -} - -impl RawConvertor<'_> {} - -impl<'a> TokenConvertor for RawConvertor<'a> { - type Token = (RawToken, &'a str); - - fn convert_doc_comment(&self, token: &Self::Token) -> Option> { - convert_doc_comment(&doc_comment(token.1)) - } - - fn bump(&mut self) -> Option<(Self::Token, TextRange)> { - let token = self.inner.next()?; - let range = TextRange::at(self.offset, token.len); - self.offset += token.len; - - Some(((*token, &self.text[range]), range)) - } - - fn peek(&self) -> Option { - let token = self.inner.as_slice().get(0).cloned(); - - token.map(|it| { - let range = TextRange::at(self.offset, it.len); - (it, &self.text[range]) - }) - } - - fn id_alloc(&mut self) -> &mut TokenIdAlloc { - &mut self.id_alloc - } -} - -struct Convertor { - id_alloc: TokenIdAlloc, - current: Option, - range: TextRange, - punct_offset: Option<(SyntaxToken, TextSize)>, -} - -impl Convertor { - fn new(node: &SyntaxNode, global_offset: TextSize) -> Convertor { - Convertor { - id_alloc: { TokenIdAlloc { map: TokenMap::default(), global_offset, next_id: 0 } }, - current: node.first_token(), - range: node.text_range(), - punct_offset: None, - } - } -} - -#[derive(Debug)] -enum SynToken { - Ordiniary(SyntaxToken), - Punch(SyntaxToken, TextSize), -} - -impl SynToken { - fn token(&self) -> &SyntaxToken { - match self { - SynToken::Ordiniary(it) => it, - SynToken::Punch(it, _) => it, - } - } -} - -impl SrcToken for SynToken { - fn kind(&self) -> SyntaxKind { - self.token().kind() - } - fn to_char(&self) -> Option { - match self { - SynToken::Ordiniary(_) => None, - SynToken::Punch(it, i) => it.text().chars().nth((*i).into()), - } - } - fn to_text(&self) -> SmolStr { - self.token().text().clone() - } -} - -impl TokenConvertor for Convertor { - type Token = SynToken; - fn convert_doc_comment(&self, token: &Self::Token) -> Option> { - convert_doc_comment(token.token()) - } - - fn bump(&mut self) -> Option<(Self::Token, TextRange)> { - if let Some((punct, offset)) = self.punct_offset.clone() { - if usize::from(offset) + 1 < punct.text().len() { - let offset = offset + TextSize::of('.'); - let range = punct.text_range(); - self.punct_offset = Some((punct.clone(), offset)); - let range = TextRange::at(range.start() + offset, TextSize::of('.')); - return Some((SynToken::Punch(punct, offset), range)); - } - } - - let curr = self.current.clone()?; - if !&self.range.contains_range(curr.text_range()) { - return None; - } - self.current = curr.next_token(); - - let token = if curr.kind().is_punct() { - let range = curr.text_range(); - let range = TextRange::at(range.start(), TextSize::of('.')); - self.punct_offset = Some((curr.clone(), 0.into())); - (SynToken::Punch(curr, 0.into()), range) - } else { - self.punct_offset = None; - let range = curr.text_range(); - (SynToken::Ordiniary(curr), range) - }; - - Some(token) - } - - fn peek(&self) -> Option { - if let Some((punct, mut offset)) = self.punct_offset.clone() { - offset = offset + TextSize::of('.'); - if usize::from(offset) < punct.text().len() { - return Some(SynToken::Punch(punct, offset)); - } - } - - let curr = self.current.clone()?; - if !self.range.contains_range(curr.text_range()) { - return None; - } - - let token = if curr.kind().is_punct() { - SynToken::Punch(curr, 0.into()) - } else { - SynToken::Ordiniary(curr) - }; - Some(token) - } - - fn id_alloc(&mut self) -> &mut TokenIdAlloc { - &mut self.id_alloc - } -} - -struct TtTreeSink<'a> { - buf: String, - cursor: Cursor<'a>, - open_delims: FxHashMap, - text_pos: TextSize, - inner: SyntaxTreeBuilder, - token_map: TokenMap, - - // Number of roots - // Use for detect ill-form tree which is not single root - roots: smallvec::SmallVec<[usize; 1]>, -} - -impl<'a> TtTreeSink<'a> { - fn new(cursor: Cursor<'a>) -> Self { - TtTreeSink { - buf: String::new(), - cursor, - open_delims: FxHashMap::default(), - text_pos: 0.into(), - inner: SyntaxTreeBuilder::default(), - roots: smallvec::SmallVec::new(), - token_map: TokenMap::default(), - } - } - - fn finish(self) -> (Parse, TokenMap) { - (self.inner.finish(), self.token_map) - } -} - -fn delim_to_str(d: Option, closing: bool) -> SmolStr { - let texts = match d { - Some(tt::DelimiterKind::Parenthesis) => "()", - Some(tt::DelimiterKind::Brace) => "{}", - Some(tt::DelimiterKind::Bracket) => "[]", - None => return "".into(), - }; - - let idx = closing as usize; - let text = &texts[idx..texts.len() - (1 - idx)]; - text.into() -} - -impl<'a> TreeSink for TtTreeSink<'a> { - fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) { - if kind == L_DOLLAR || kind == R_DOLLAR { - self.cursor = self.cursor.bump_subtree(); - return; - } - if kind == LIFETIME { - n_tokens = 2; - } - - let mut last = self.cursor; - for _ in 0..n_tokens { - if self.cursor.eof() { - break; - } - last = self.cursor; - let text: SmolStr = match self.cursor.token_tree() { - Some(tt::TokenTree::Leaf(leaf)) => { - // Mark the range if needed - let (text, id) = match leaf { - tt::Leaf::Ident(ident) => (ident.text.clone(), ident.id), - tt::Leaf::Punct(punct) => { - (SmolStr::new_inline_from_ascii(1, &[punct.char as u8]), punct.id) - } - tt::Leaf::Literal(lit) => (lit.text.clone(), lit.id), - }; - let range = TextRange::at(self.text_pos, TextSize::of(text.as_str())); - self.token_map.insert(id, range); - self.cursor = self.cursor.bump(); - text - } - Some(tt::TokenTree::Subtree(subtree)) => { - self.cursor = self.cursor.subtree().unwrap(); - if let Some(id) = subtree.delimiter.map(|it| it.id) { - self.open_delims.insert(id, self.text_pos); - } - delim_to_str(subtree.delimiter_kind(), false) - } - None => { - if let Some(parent) = self.cursor.end() { - self.cursor = self.cursor.bump(); - if let Some(id) = parent.delimiter.map(|it| it.id) { - if let Some(open_delim) = self.open_delims.get(&id) { - let open_range = TextRange::at(*open_delim, TextSize::of('(')); - let close_range = TextRange::at(self.text_pos, TextSize::of('(')); - self.token_map.insert_delim(id, open_range, close_range); - } - } - delim_to_str(parent.delimiter_kind(), true) - } else { - continue; - } - } - }; - self.buf += &text; - self.text_pos += TextSize::of(text.as_str()); - } - - let text = SmolStr::new(self.buf.as_str()); - self.buf.clear(); - self.inner.token(kind, text); - - // Add whitespace between adjoint puncts - let next = last.bump(); - if let ( - Some(tt::TokenTree::Leaf(tt::Leaf::Punct(curr))), - Some(tt::TokenTree::Leaf(tt::Leaf::Punct(_))), - ) = (last.token_tree(), next.token_tree()) - { - // Note: We always assume the semi-colon would be the last token in - // other parts of RA such that we don't add whitespace here. - if curr.spacing == tt::Spacing::Alone && curr.char != ';' { - self.inner.token(WHITESPACE, " ".into()); - self.text_pos += TextSize::of(' '); - } - } - } - - fn start_node(&mut self, kind: SyntaxKind) { - self.inner.start_node(kind); - - match self.roots.last_mut() { - None | Some(0) => self.roots.push(1), - Some(ref mut n) => **n += 1, - }; - } - - fn finish_node(&mut self) { - self.inner.finish_node(); - *self.roots.last_mut().unwrap() -= 1; - } - - fn error(&mut self, error: ParseError) { - self.inner.error(error, self.text_pos) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::tests::parse_macro; - use parser::TokenSource; - use syntax::{ - algo::{insert_children, InsertPosition}, - ast::AstNode, - }; - - #[test] - fn convert_tt_token_source() { - let expansion = parse_macro( - r#" - macro_rules! literals { - ($i:ident) => { - { - let a = 'c'; - let c = 1000; - let f = 12E+99_f64; - let s = "rust1"; - } - } - } - "#, - ) - .expand_tt("literals!(foo);"); - let tts = &[expansion.into()]; - let buffer = tt::buffer::TokenBuffer::new(tts); - let mut tt_src = SubtreeTokenSource::new(&buffer); - let mut tokens = vec![]; - while tt_src.current().kind != EOF { - tokens.push((tt_src.current().kind, tt_src.text())); - tt_src.bump(); - } - - // [${] - // [let] [a] [=] ['c'] [;] - assert_eq!(tokens[2 + 3].1, "'c'"); - assert_eq!(tokens[2 + 3].0, CHAR); - // [let] [c] [=] [1000] [;] - assert_eq!(tokens[2 + 5 + 3].1, "1000"); - assert_eq!(tokens[2 + 5 + 3].0, INT_NUMBER); - // [let] [f] [=] [12E+99_f64] [;] - assert_eq!(tokens[2 + 10 + 3].1, "12E+99_f64"); - assert_eq!(tokens[2 + 10 + 3].0, FLOAT_NUMBER); - - // [let] [s] [=] ["rust1"] [;] - assert_eq!(tokens[2 + 15 + 3].1, "\"rust1\""); - assert_eq!(tokens[2 + 15 + 3].0, STRING); - } - - #[test] - fn stmts_token_trees_to_expr_is_err() { - let expansion = parse_macro( - r#" - macro_rules! stmts { - () => { - let a = 0; - let b = 0; - let c = 0; - let d = 0; - } - } - "#, - ) - .expand_tt("stmts!();"); - assert!(token_tree_to_syntax_node(&expansion, FragmentKind::Expr).is_err()); - } - - #[test] - fn test_token_tree_last_child_is_white_space() { - let source_file = ast::SourceFile::parse("f!({} );").ok().unwrap(); - let macro_call = source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); - let token_tree = macro_call.token_tree().unwrap(); - - // Token Tree now is : - // TokenTree - // - T!['('] - // - TokenTree - // - T!['{'] - // - T!['}'] - // - WHITE_SPACE - // - T![')'] - - let rbrace = - token_tree.syntax().descendants_with_tokens().find(|it| it.kind() == T!['}']).unwrap(); - let space = token_tree - .syntax() - .descendants_with_tokens() - .find(|it| it.kind() == SyntaxKind::WHITESPACE) - .unwrap(); - - // reorder th white space, such that the white is inside the inner token-tree. - let token_tree = insert_children( - &rbrace.parent().unwrap(), - InsertPosition::Last, - std::iter::once(space), - ); - - // Token Tree now is : - // TokenTree - // - T!['{'] - // - T!['}'] - // - WHITE_SPACE - let token_tree = ast::TokenTree::cast(token_tree).unwrap(); - let tt = ast_to_token_tree(&token_tree).unwrap().0; - - assert_eq!(tt.delimiter_kind(), Some(tt::DelimiterKind::Brace)); - } - - #[test] - fn test_token_tree_multi_char_punct() { - let source_file = ast::SourceFile::parse("struct Foo { a: x::Y }").ok().unwrap(); - let struct_def = source_file.syntax().descendants().find_map(ast::Struct::cast).unwrap(); - let tt = ast_to_token_tree(&struct_def).unwrap().0; - token_tree_to_syntax_node(&tt, FragmentKind::Item).unwrap(); - } -} diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs deleted file mode 100644 index 0796ceee1..000000000 --- a/crates/ra_mbe/src/tests.rs +++ /dev/null @@ -1,1898 +0,0 @@ -use std::fmt::Write; - -use ::parser::FragmentKind; -use syntax::{ast, AstNode, NodeOrToken, SyntaxKind::IDENT, SyntaxNode, WalkEvent, T}; -use test_utils::assert_eq_text; - -use super::*; - -mod rule_parsing { - use syntax::{ast, AstNode}; - - use crate::ast_to_token_tree; - - use super::*; - - #[test] - fn test_valid_arms() { - fn check(macro_body: &str) { - let m = parse_macro_arm(macro_body); - m.unwrap(); - } - - check("($i:ident) => ()"); - check("($($i:ident)*) => ($_)"); - check("($($true:ident)*) => ($true)"); - check("($($false:ident)*) => ($false)"); - check("($) => ($)"); - } - - #[test] - fn test_invalid_arms() { - fn check(macro_body: &str, err: &str) { - let m = parse_macro_arm(macro_body); - assert_eq!(m, Err(ParseError::Expected(String::from(err)))); - } - - check("invalid", "expected subtree"); - - check("$i:ident => ()", "expected subtree"); - check("($i:ident) ()", "expected `=`"); - check("($($i:ident)_) => ()", "invalid repeat"); - - check("($i) => ($i)", "invalid macro definition"); - check("($i:) => ($i)", "invalid macro definition"); - } - - fn parse_macro_arm(arm_definition: &str) -> Result { - let macro_definition = format!(" macro_rules! m {{ {} }} ", arm_definition); - let source_file = ast::SourceFile::parse(¯o_definition).ok().unwrap(); - let macro_definition = - source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); - - let (definition_tt, _) = - ast_to_token_tree(¯o_definition.token_tree().unwrap()).unwrap(); - crate::MacroRules::parse(&definition_tt) - } -} - -// Good first issue (although a slightly challenging one): -// -// * Pick a random test from here -// https://github.com/intellij-rust/intellij-rust/blob/c4e9feee4ad46e7953b1948c112533360b6087bb/src/test/kotlin/org/rust/lang/core/macros/RsMacroExpansionTest.kt -// * Port the test to rust and add it to this module -// * Make it pass :-) - -#[test] -fn test_token_id_shift() { - let expansion = parse_macro( - r#" -macro_rules! foobar { - ($e:ident) => { foo bar $e } -} -"#, - ) - .expand_tt("foobar!(baz);"); - - fn get_id(t: &tt::TokenTree) -> Option { - if let tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) = t { - return Some(ident.id.0); - } - None - } - - assert_eq!(expansion.token_trees.len(), 3); - // {($e:ident) => { foo bar $e }} - // 012345 67 8 9 T 12 - assert_eq!(get_id(&expansion.token_trees[0]), Some(9)); - assert_eq!(get_id(&expansion.token_trees[1]), Some(10)); - - // The input args of macro call include parentheses: - // (baz) - // So baz should be 12+1+1 - assert_eq!(get_id(&expansion.token_trees[2]), Some(14)); -} - -#[test] -fn test_token_map() { - let expanded = parse_macro( - r#" -macro_rules! foobar { - ($e:ident) => { fn $e() {} } -} -"#, - ) - .expand_tt("foobar!(baz);"); - - let (node, token_map) = token_tree_to_syntax_node(&expanded, FragmentKind::Items).unwrap(); - let content = node.syntax_node().to_string(); - - let get_text = |id, kind| -> String { - content[token_map.range_by_token(id).unwrap().by_kind(kind).unwrap()].to_string() - }; - - assert_eq!(expanded.token_trees.len(), 4); - // {($e:ident) => { fn $e() {} }} - // 012345 67 8 9 T12 3 - - assert_eq!(get_text(tt::TokenId(9), IDENT), "fn"); - assert_eq!(get_text(tt::TokenId(12), T!['(']), "("); - assert_eq!(get_text(tt::TokenId(13), T!['{']), "{"); -} - -#[test] -fn test_convert_tt() { - parse_macro(r#" -macro_rules! impl_froms { - ($e:ident: $($v:ident),*) => { - $( - impl From<$v> for $e { - fn from(it: $v) -> $e { - $e::$v(it) - } - } - )* - } -} -"#) - .assert_expand_tt( - "impl_froms!(TokenTree: Leaf, Subtree);", - "impl From for TokenTree {fn from (it : Leaf) -> TokenTree {TokenTree ::Leaf (it)}} \ - impl From for TokenTree {fn from (it : Subtree) -> TokenTree {TokenTree ::Subtree (it)}}" - ); -} - -#[test] -fn test_convert_tt2() { - parse_macro( - r#" -macro_rules! impl_froms { - ($e:ident: $($v:ident),*) => { - $( - impl From<$v> for $e { - fn from(it: $v) -> $e { - $e::$v(it) - } - } - )* - } -} -"#, - ) - .assert_expand( - "impl_froms!(TokenTree: Leaf, Subtree);", - r#" -SUBTREE $ - IDENT impl 20 - IDENT From 21 - PUNCH < [joint] 22 - IDENT Leaf 53 - PUNCH > [alone] 25 - IDENT for 26 - IDENT TokenTree 51 - SUBTREE {} 29 - IDENT fn 30 - IDENT from 31 - SUBTREE () 32 - IDENT it 33 - PUNCH : [alone] 34 - IDENT Leaf 53 - PUNCH - [joint] 37 - PUNCH > [alone] 38 - IDENT TokenTree 51 - SUBTREE {} 41 - IDENT TokenTree 51 - PUNCH : [joint] 44 - PUNCH : [joint] 45 - IDENT Leaf 53 - SUBTREE () 48 - IDENT it 49 - IDENT impl 20 - IDENT From 21 - PUNCH < [joint] 22 - IDENT Subtree 55 - PUNCH > [alone] 25 - IDENT for 26 - IDENT TokenTree 51 - SUBTREE {} 29 - IDENT fn 30 - IDENT from 31 - SUBTREE () 32 - IDENT it 33 - PUNCH : [alone] 34 - IDENT Subtree 55 - PUNCH - [joint] 37 - PUNCH > [alone] 38 - IDENT TokenTree 51 - SUBTREE {} 41 - IDENT TokenTree 51 - PUNCH : [joint] 44 - PUNCH : [joint] 45 - IDENT Subtree 55 - SUBTREE () 48 - IDENT it 49 -"#, - ); -} - -#[test] -fn test_lifetime_split() { - parse_macro( - r#" -macro_rules! foo { - ($($t:tt)*) => { $($t)*} -} -"#, - ) - .assert_expand( - r#"foo!(static bar: &'static str = "hello";);"#, - r#" -SUBTREE $ - IDENT static 17 - IDENT bar 18 - PUNCH : [alone] 19 - PUNCH & [alone] 20 - PUNCH ' [joint] 21 - IDENT static 22 - IDENT str 23 - PUNCH = [alone] 24 - LITERAL "hello" 25 - PUNCH ; [joint] 26 -"#, - ); -} - -#[test] -fn test_expr_order() { - let expanded = parse_macro( - r#" - macro_rules! foo { - ($ i:expr) => { - fn bar() { $ i * 2; } - } - } -"#, - ) - .expand_items("foo! { 1 + 1}"); - - let dump = format!("{:#?}", expanded); - assert_eq_text!( - dump.trim(), - r#"MACRO_ITEMS@0..15 - FN@0..15 - FN_KW@0..2 "fn" - NAME@2..5 - IDENT@2..5 "bar" - PARAM_LIST@5..7 - L_PAREN@5..6 "(" - R_PAREN@6..7 ")" - BLOCK_EXPR@7..15 - L_CURLY@7..8 "{" - EXPR_STMT@8..14 - BIN_EXPR@8..13 - BIN_EXPR@8..11 - LITERAL@8..9 - INT_NUMBER@8..9 "1" - PLUS@9..10 "+" - LITERAL@10..11 - INT_NUMBER@10..11 "1" - STAR@11..12 "*" - LITERAL@12..13 - INT_NUMBER@12..13 "2" - SEMICOLON@13..14 ";" - R_CURLY@14..15 "}""#, - ); -} - -#[test] -fn test_fail_match_pattern_by_first_token() { - parse_macro( - r#" - macro_rules! foo { - ($ i:ident) => ( - mod $ i {} - ); - (= $ i:ident) => ( - fn $ i() {} - ); - (+ $ i:ident) => ( - struct $ i; - ) - } -"#, - ) - .assert_expand_items("foo! { foo }", "mod foo {}") - .assert_expand_items("foo! { = bar }", "fn bar () {}") - .assert_expand_items("foo! { + Baz }", "struct Baz ;"); -} - -#[test] -fn test_fail_match_pattern_by_last_token() { - parse_macro( - r#" - macro_rules! foo { - ($ i:ident) => ( - mod $ i {} - ); - ($ i:ident =) => ( - fn $ i() {} - ); - ($ i:ident +) => ( - struct $ i; - ) - } -"#, - ) - .assert_expand_items("foo! { foo }", "mod foo {}") - .assert_expand_items("foo! { bar = }", "fn bar () {}") - .assert_expand_items("foo! { Baz + }", "struct Baz ;"); -} - -#[test] -fn test_fail_match_pattern_by_word_token() { - parse_macro( - r#" - macro_rules! foo { - ($ i:ident) => ( - mod $ i {} - ); - (spam $ i:ident) => ( - fn $ i() {} - ); - (eggs $ i:ident) => ( - struct $ i; - ) - } -"#, - ) - .assert_expand_items("foo! { foo }", "mod foo {}") - .assert_expand_items("foo! { spam bar }", "fn bar () {}") - .assert_expand_items("foo! { eggs Baz }", "struct Baz ;"); -} - -#[test] -fn test_match_group_pattern_by_separator_token() { - parse_macro( - r#" - macro_rules! foo { - ($ ($ i:ident),*) => ($ ( - mod $ i {} - )*); - ($ ($ i:ident)#*) => ($ ( - fn $ i() {} - )*); - ($ i:ident ,# $ j:ident) => ( - struct $ i; - struct $ j; - ) - } -"#, - ) - .assert_expand_items("foo! { foo, bar }", "mod foo {} mod bar {}") - .assert_expand_items("foo! { foo# bar }", "fn foo () {} fn bar () {}") - .assert_expand_items("foo! { Foo,# Bar }", "struct Foo ; struct Bar ;"); -} - -#[test] -fn test_match_group_pattern_with_multiple_defs() { - parse_macro( - r#" - macro_rules! foo { - ($ ($ i:ident),*) => ( struct Bar { $ ( - fn $ i {} - )*} ); - } -"#, - ) - .assert_expand_items("foo! { foo, bar }", "struct Bar {fn foo {} fn bar {}}"); -} - -#[test] -fn test_match_group_pattern_with_multiple_statement() { - parse_macro( - r#" - macro_rules! foo { - ($ ($ i:ident),*) => ( fn baz { $ ( - $ i (); - )*} ); - } -"#, - ) - .assert_expand_items("foo! { foo, bar }", "fn baz {foo () ; bar () ;}"); -} - -#[test] -fn test_match_group_pattern_with_multiple_statement_without_semi() { - parse_macro( - r#" - macro_rules! foo { - ($ ($ i:ident),*) => ( fn baz { $ ( - $i() - );*} ); - } -"#, - ) - .assert_expand_items("foo! { foo, bar }", "fn baz {foo () ;bar ()}"); -} - -#[test] -fn test_match_group_empty_fixed_token() { - parse_macro( - r#" - macro_rules! foo { - ($ ($ i:ident)* #abc) => ( fn baz { $ ( - $ i (); - )*} ); - } -"#, - ) - .assert_expand_items("foo! {#abc}", "fn baz {}"); -} - -#[test] -fn test_match_group_in_subtree() { - parse_macro( - r#" - macro_rules! foo { - (fn $name:ident {$($i:ident)*} ) => ( fn $name() { $ ( - $ i (); - )*} ); - }"#, - ) - .assert_expand_items("foo! {fn baz {a b} }", "fn baz () {a () ; b () ;}"); -} - -#[test] -fn test_match_group_with_multichar_sep() { - parse_macro( - r#" - macro_rules! foo { - (fn $name:ident {$($i:literal)*} ) => ( fn $name() -> bool { $($i)&&*} ); - }"#, - ) - .assert_expand_items("foo! (fn baz {true true} );", "fn baz () -> bool {true &&true}"); -} - -#[test] -fn test_match_group_zero_match() { - parse_macro( - r#" - macro_rules! foo { - ( $($i:ident)* ) => (); - }"#, - ) - .assert_expand_items("foo! ();", ""); -} - -#[test] -fn test_match_group_in_group() { - parse_macro( - r#" - macro_rules! foo { - { $( ( $($i:ident)* ) )* } => ( $( ( $($i)* ) )* ); - }"#, - ) - .assert_expand_items("foo! ( (a b) );", "(a b)"); -} - -#[test] -fn test_expand_to_item_list() { - let tree = parse_macro( - " - macro_rules! structs { - ($($i:ident),*) => { - $(struct $i { field: u32 } )* - } - } - ", - ) - .expand_items("structs!(Foo, Bar);"); - assert_eq!( - format!("{:#?}", tree).trim(), - r#" -MACRO_ITEMS@0..40 - STRUCT@0..20 - STRUCT_KW@0..6 "struct" - NAME@6..9 - IDENT@6..9 "Foo" - RECORD_FIELD_LIST@9..20 - L_CURLY@9..10 "{" - RECORD_FIELD@10..19 - NAME@10..15 - IDENT@10..15 "field" - COLON@15..16 ":" - PATH_TYPE@16..19 - PATH@16..19 - PATH_SEGMENT@16..19 - NAME_REF@16..19 - IDENT@16..19 "u32" - R_CURLY@19..20 "}" - STRUCT@20..40 - STRUCT_KW@20..26 "struct" - NAME@26..29 - IDENT@26..29 "Bar" - RECORD_FIELD_LIST@29..40 - L_CURLY@29..30 "{" - RECORD_FIELD@30..39 - NAME@30..35 - IDENT@30..35 "field" - COLON@35..36 ":" - PATH_TYPE@36..39 - PATH@36..39 - PATH_SEGMENT@36..39 - NAME_REF@36..39 - IDENT@36..39 "u32" - R_CURLY@39..40 "}""# - .trim() - ); -} - -fn to_subtree(tt: &tt::TokenTree) -> &tt::Subtree { - if let tt::TokenTree::Subtree(subtree) = tt { - return &subtree; - } - unreachable!("It is not a subtree"); -} -fn to_literal(tt: &tt::TokenTree) -> &tt::Literal { - if let tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) = tt { - return lit; - } - unreachable!("It is not a literal"); -} - -fn to_punct(tt: &tt::TokenTree) -> &tt::Punct { - if let tt::TokenTree::Leaf(tt::Leaf::Punct(lit)) = tt { - return lit; - } - unreachable!("It is not a Punct"); -} - -#[test] -fn test_expand_literals_to_token_tree() { - let expansion = parse_macro( - r#" - macro_rules! literals { - ($i:ident) => { - { - let a = 'c'; - let c = 1000; - let f = 12E+99_f64; - let s = "rust1"; - } - } - } - "#, - ) - .expand_tt("literals!(foo);"); - let stm_tokens = &to_subtree(&expansion.token_trees[0]).token_trees; - - // [let] [a] [=] ['c'] [;] - assert_eq!(to_literal(&stm_tokens[3]).text, "'c'"); - // [let] [c] [=] [1000] [;] - assert_eq!(to_literal(&stm_tokens[5 + 3]).text, "1000"); - // [let] [f] [=] [12E+99_f64] [;] - assert_eq!(to_literal(&stm_tokens[10 + 3]).text, "12E+99_f64"); - // [let] [s] [=] ["rust1"] [;] - assert_eq!(to_literal(&stm_tokens[15 + 3]).text, "\"rust1\""); -} - -#[test] -fn test_attr_to_token_tree() { - let expansion = parse_to_token_tree_by_syntax( - r#" - #[derive(Copy)] - struct Foo; - "#, - ); - - assert_eq!(to_punct(&expansion.token_trees[0]).char, '#'); - assert_eq!( - to_subtree(&expansion.token_trees[1]).delimiter_kind(), - Some(tt::DelimiterKind::Bracket) - ); -} - -#[test] -fn test_two_idents() { - parse_macro( - r#" - macro_rules! foo { - ($ i:ident, $ j:ident) => { - fn foo() { let a = $ i; let b = $j; } - } - } -"#, - ) - .assert_expand_items("foo! { foo, bar }", "fn foo () {let a = foo ; let b = bar ;}"); -} - -#[test] -fn test_tt_to_stmts() { - let stmts = parse_macro( - r#" - macro_rules! foo { - () => { - let a = 0; - a = 10 + 1; - a - } - } -"#, - ) - .expand_statements("foo!{}"); - - assert_eq!( - format!("{:#?}", stmts).trim(), - r#"MACRO_STMTS@0..15 - LET_STMT@0..7 - LET_KW@0..3 "let" - IDENT_PAT@3..4 - NAME@3..4 - IDENT@3..4 "a" - EQ@4..5 "=" - LITERAL@5..6 - INT_NUMBER@5..6 "0" - SEMICOLON@6..7 ";" - EXPR_STMT@7..14 - BIN_EXPR@7..13 - PATH_EXPR@7..8 - PATH@7..8 - PATH_SEGMENT@7..8 - NAME_REF@7..8 - IDENT@7..8 "a" - EQ@8..9 "=" - BIN_EXPR@9..13 - LITERAL@9..11 - INT_NUMBER@9..11 "10" - PLUS@11..12 "+" - LITERAL@12..13 - INT_NUMBER@12..13 "1" - SEMICOLON@13..14 ";" - EXPR_STMT@14..15 - PATH_EXPR@14..15 - PATH@14..15 - PATH_SEGMENT@14..15 - NAME_REF@14..15 - IDENT@14..15 "a""#, - ); -} - -#[test] -fn test_match_literal() { - parse_macro( - r#" - macro_rules! foo { - ('(') => { - fn foo() {} - } - } -"#, - ) - .assert_expand_items("foo! ['('];", "fn foo () {}"); -} - -// The following tests are port from intellij-rust directly -// https://github.com/intellij-rust/intellij-rust/blob/c4e9feee4ad46e7953b1948c112533360b6087bb/src/test/kotlin/org/rust/lang/core/macros/RsMacroExpansionTest.kt - -#[test] -fn test_path() { - parse_macro( - r#" - macro_rules! foo { - ($ i:path) => { - fn foo() { let a = $ i; } - } - } -"#, - ) - .assert_expand_items("foo! { foo }", "fn foo () {let a = foo ;}") - .assert_expand_items( - "foo! { bar::::baz:: }", - "fn foo () {let a = bar ::< u8 >:: baz ::< u8 > ;}", - ); -} - -#[test] -fn test_two_paths() { - parse_macro( - r#" - macro_rules! foo { - ($ i:path, $ j:path) => { - fn foo() { let a = $ i; let b = $j; } - } - } -"#, - ) - .assert_expand_items("foo! { foo, bar }", "fn foo () {let a = foo ; let b = bar ;}"); -} - -#[test] -fn test_path_with_path() { - parse_macro( - r#" - macro_rules! foo { - ($ i:path) => { - fn foo() { let a = $ i :: bar; } - } - } -"#, - ) - .assert_expand_items("foo! { foo }", "fn foo () {let a = foo :: bar ;}"); -} - -#[test] -fn test_expr() { - parse_macro( - r#" - macro_rules! foo { - ($ i:expr) => { - fn bar() { $ i; } - } - } -"#, - ) - .assert_expand_items( - "foo! { 2 + 2 * baz(3).quux() }", - "fn bar () {2 + 2 * baz (3) . quux () ;}", - ); -} - -#[test] -fn test_last_expr() { - parse_macro( - r#" - macro_rules! vec { - ($($item:expr),*) => { - { - let mut v = Vec::new(); - $( - v.push($item); - )* - v - } - }; - } -"#, - ) - .assert_expand_items( - "vec!(1,2,3);", - "{let mut v = Vec :: new () ; v . push (1) ; v . push (2) ; v . push (3) ; v}", - ); -} - -#[test] -fn test_ty() { - parse_macro( - r#" - macro_rules! foo { - ($ i:ty) => ( - fn bar() -> $ i { unimplemented!() } - ) - } -"#, - ) - .assert_expand_items("foo! { Baz }", "fn bar () -> Baz < u8 > {unimplemented ! ()}"); -} - -#[test] -fn test_ty_with_complex_type() { - parse_macro( - r#" - macro_rules! foo { - ($ i:ty) => ( - fn bar() -> $ i { unimplemented!() } - ) - } -"#, - ) - // Reference lifetime struct with generic type - .assert_expand_items( - "foo! { &'a Baz }", - "fn bar () -> & 'a Baz < u8 > {unimplemented ! ()}", - ) - // extern "Rust" func type - .assert_expand_items( - r#"foo! { extern "Rust" fn() -> Ret }"#, - r#"fn bar () -> extern "Rust" fn () -> Ret {unimplemented ! ()}"#, - ); -} - -#[test] -fn test_pat_() { - parse_macro( - r#" - macro_rules! foo { - ($ i:pat) => { fn foo() { let $ i; } } - } -"#, - ) - .assert_expand_items("foo! { (a, b) }", "fn foo () {let (a , b) ;}"); -} - -#[test] -fn test_stmt() { - parse_macro( - r#" - macro_rules! foo { - ($ i:stmt) => ( - fn bar() { $ i; } - ) - } -"#, - ) - .assert_expand_items("foo! { 2 }", "fn bar () {2 ;}") - .assert_expand_items("foo! { let a = 0 }", "fn bar () {let a = 0 ;}"); -} - -#[test] -fn test_single_item() { - parse_macro( - r#" - macro_rules! foo { - ($ i:item) => ( - $ i - ) - } -"#, - ) - .assert_expand_items("foo! {mod c {}}", "mod c {}"); -} - -#[test] -fn test_all_items() { - parse_macro( - r#" - macro_rules! foo { - ($ ($ i:item)*) => ($ ( - $ i - )*) - } -"#, - ). - assert_expand_items( - r#" - foo! { - extern crate a; - mod b; - mod c {} - use d; - const E: i32 = 0; - static F: i32 = 0; - impl G {} - struct H; - enum I { Foo } - trait J {} - fn h() {} - extern {} - type T = u8; - } -"#, - r#"extern crate a ; mod b ; mod c {} use d ; const E : i32 = 0 ; static F : i32 = 0 ; impl G {} struct H ; enum I {Foo} trait J {} fn h () {} extern {} type T = u8 ;"#, - ); -} - -#[test] -fn test_block() { - parse_macro( - r#" - macro_rules! foo { - ($ i:block) => { fn foo() $ i } - } -"#, - ) - .assert_expand_statements("foo! { { 1; } }", "fn foo () {1 ;}"); -} - -#[test] -fn test_meta() { - parse_macro( - r#" - macro_rules! foo { - ($ i:meta) => ( - #[$ i] - fn bar() {} - ) - } -"#, - ) - .assert_expand_items( - r#"foo! { cfg(target_os = "windows") }"#, - r#"# [cfg (target_os = "windows")] fn bar () {}"#, - ); -} - -#[test] -fn test_meta_doc_comments() { - parse_macro( - r#" - macro_rules! foo { - ($(#[$ i:meta])+) => ( - $(#[$ i])+ - fn bar() {} - ) - } -"#, - ). - assert_expand_items( - r#"foo! { - /// Single Line Doc 1 - /** - MultiLines Doc - */ - }"#, - "# [doc = \" Single Line Doc 1\"] # [doc = \"\\\\n MultiLines Doc\\\\n \"] fn bar () {}", - ); -} - -#[test] -fn test_tt_block() { - parse_macro( - r#" - macro_rules! foo { - ($ i:tt) => { fn foo() $ i } - } - "#, - ) - .assert_expand_items(r#"foo! { { 1; } }"#, r#"fn foo () {1 ;}"#); -} - -#[test] -fn test_tt_group() { - parse_macro( - r#" - macro_rules! foo { - ($($ i:tt)*) => { $($ i)* } - } - "#, - ) - .assert_expand_items(r#"foo! { fn foo() {} }"#, r#"fn foo () {}"#); -} - -#[test] -fn test_tt_composite() { - parse_macro( - r#" - macro_rules! foo { - ($i:tt) => { 0 } - } - "#, - ) - .assert_expand_items(r#"foo! { => }"#, r#"0"#); -} - -#[test] -fn test_tt_composite2() { - let node = parse_macro( - r#" - macro_rules! foo { - ($($tt:tt)*) => { abs!(=> $($tt)*) } - } - "#, - ) - .expand_items(r#"foo!{#}"#); - - let res = format!("{:#?}", &node); - assert_eq_text!( - res.trim(), - r###"MACRO_ITEMS@0..10 - MACRO_CALL@0..10 - PATH@0..3 - PATH_SEGMENT@0..3 - NAME_REF@0..3 - IDENT@0..3 "abs" - BANG@3..4 "!" - TOKEN_TREE@4..10 - L_PAREN@4..5 "(" - EQ@5..6 "=" - R_ANGLE@6..7 ">" - WHITESPACE@7..8 " " - POUND@8..9 "#" - R_PAREN@9..10 ")""### - ); -} - -#[test] -fn test_lifetime() { - parse_macro( - r#" - macro_rules! foo { - ($ lt:lifetime) => { struct Ref<$ lt>{ s: &$ lt str } } - } -"#, - ) - .assert_expand_items(r#"foo!{'a}"#, r#"struct Ref <'a > {s : &'a str}"#); -} - -#[test] -fn test_literal() { - parse_macro( - r#" - macro_rules! foo { - ($ type:ty $ lit:literal) => { const VALUE: $ type = $ lit;}; - } -"#, - ) - .assert_expand_items(r#"foo!(u8 0);"#, r#"const VALUE : u8 = 0 ;"#); -} - -#[test] -fn test_boolean_is_ident() { - parse_macro( - r#" - macro_rules! foo { - ($lit0:literal, $lit1:literal) => { const VALUE: (bool,bool) = ($lit0,$lit1); }; - } -"#, - ) - .assert_expand( - r#"foo!(true,false);"#, - r#" -SUBTREE $ - IDENT const 14 - IDENT VALUE 15 - PUNCH : [alone] 16 - SUBTREE () 17 - IDENT bool 18 - PUNCH , [alone] 19 - IDENT bool 20 - PUNCH = [alone] 21 - SUBTREE () 22 - IDENT true 29 - PUNCH , [joint] 25 - IDENT false 31 - PUNCH ; [alone] 28 -"#, - ); -} - -#[test] -fn test_vis() { - parse_macro( - r#" - macro_rules! foo { - ($ vis:vis $ name:ident) => { $ vis fn $ name() {}}; - } -"#, - ) - .assert_expand_items(r#"foo!(pub foo);"#, r#"pub fn foo () {}"#) - // test optional cases - .assert_expand_items(r#"foo!(foo);"#, r#"fn foo () {}"#); -} - -#[test] -fn test_inner_macro_rules() { - parse_macro( - r#" -macro_rules! foo { - ($a:ident, $b:ident, $c:tt) => { - - macro_rules! bar { - ($bi:ident) => { - fn $bi() -> u8 {$c} - } - } - - bar!($a); - fn $b() -> u8 {$c} - } -} -"#, - ). - assert_expand_items( - r#"foo!(x,y, 1);"#, - r#"macro_rules ! bar {($ bi : ident) => {fn $ bi () -> u8 {1}}} bar ! (x) ; fn y () -> u8 {1}"#, - ); -} - -// The following tests are based on real world situations -#[test] -fn test_vec() { - let fixture = parse_macro( - r#" - macro_rules! vec { - ($($item:expr),*) => { - { - let mut v = Vec::new(); - $( - v.push($item); - )* - v - } - }; -} -"#, - ); - fixture - .assert_expand_items(r#"vec!();"#, r#"{let mut v = Vec :: new () ; v}"#) - .assert_expand_items( - r#"vec![1u32,2];"#, - r#"{let mut v = Vec :: new () ; v . push (1u32) ; v . push (2) ; v}"#, - ); - - let tree = fixture.expand_expr(r#"vec![1u32,2];"#); - - assert_eq!( - format!("{:#?}", tree).trim(), - r#"BLOCK_EXPR@0..45 - L_CURLY@0..1 "{" - LET_STMT@1..20 - LET_KW@1..4 "let" - IDENT_PAT@4..8 - MUT_KW@4..7 "mut" - NAME@7..8 - IDENT@7..8 "v" - EQ@8..9 "=" - CALL_EXPR@9..19 - PATH_EXPR@9..17 - PATH@9..17 - PATH@9..12 - PATH_SEGMENT@9..12 - NAME_REF@9..12 - IDENT@9..12 "Vec" - COLON2@12..14 "::" - PATH_SEGMENT@14..17 - NAME_REF@14..17 - IDENT@14..17 "new" - ARG_LIST@17..19 - L_PAREN@17..18 "(" - R_PAREN@18..19 ")" - SEMICOLON@19..20 ";" - EXPR_STMT@20..33 - METHOD_CALL_EXPR@20..32 - PATH_EXPR@20..21 - PATH@20..21 - PATH_SEGMENT@20..21 - NAME_REF@20..21 - IDENT@20..21 "v" - DOT@21..22 "." - NAME_REF@22..26 - IDENT@22..26 "push" - ARG_LIST@26..32 - L_PAREN@26..27 "(" - LITERAL@27..31 - INT_NUMBER@27..31 "1u32" - R_PAREN@31..32 ")" - SEMICOLON@32..33 ";" - EXPR_STMT@33..43 - METHOD_CALL_EXPR@33..42 - PATH_EXPR@33..34 - PATH@33..34 - PATH_SEGMENT@33..34 - NAME_REF@33..34 - IDENT@33..34 "v" - DOT@34..35 "." - NAME_REF@35..39 - IDENT@35..39 "push" - ARG_LIST@39..42 - L_PAREN@39..40 "(" - LITERAL@40..41 - INT_NUMBER@40..41 "2" - R_PAREN@41..42 ")" - SEMICOLON@42..43 ";" - PATH_EXPR@43..44 - PATH@43..44 - PATH_SEGMENT@43..44 - NAME_REF@43..44 - IDENT@43..44 "v" - R_CURLY@44..45 "}""# - ); -} - -#[test] -fn test_winapi_struct() { - // from https://github.com/retep998/winapi-rs/blob/a7ef2bca086aae76cf6c4ce4c2552988ed9798ad/src/macros.rs#L366 - - parse_macro( - r#" -macro_rules! STRUCT { - ($(#[$attrs:meta])* struct $name:ident { - $($field:ident: $ftype:ty,)+ - }) => ( - #[repr(C)] #[derive(Copy)] $(#[$attrs])* - pub struct $name { - $(pub $field: $ftype,)+ - } - impl Clone for $name { - #[inline] - fn clone(&self) -> $name { *self } - } - #[cfg(feature = "impl-default")] - impl Default for $name { - #[inline] - fn default() -> $name { unsafe { $crate::_core::mem::zeroed() } } - } - ); -} -"#, - ). - // from https://github.com/retep998/winapi-rs/blob/a7ef2bca086aae76cf6c4ce4c2552988ed9798ad/src/shared/d3d9caps.rs - assert_expand_items(r#"STRUCT!{struct D3DVSHADERCAPS2_0 {Caps: u8,}}"#, - "# [repr (C)] # [derive (Copy)] pub struct D3DVSHADERCAPS2_0 {pub Caps : u8 ,} impl Clone for D3DVSHADERCAPS2_0 {# [inline] fn clone (& self) -> D3DVSHADERCAPS2_0 {* self}} # [cfg (feature = \"impl-default\")] impl Default for D3DVSHADERCAPS2_0 {# [inline] fn default () -> D3DVSHADERCAPS2_0 {unsafe {$crate :: _core :: mem :: zeroed ()}}}" - ) - .assert_expand_items(r#"STRUCT!{#[cfg_attr(target_arch = "x86", repr(packed))] struct D3DCONTENTPROTECTIONCAPS {Caps : u8 ,}}"#, - "# [repr (C)] # [derive (Copy)] # [cfg_attr (target_arch = \"x86\" , repr (packed))] pub struct D3DCONTENTPROTECTIONCAPS {pub Caps : u8 ,} impl Clone for D3DCONTENTPROTECTIONCAPS {# [inline] fn clone (& self) -> D3DCONTENTPROTECTIONCAPS {* self}} # [cfg (feature = \"impl-default\")] impl Default for D3DCONTENTPROTECTIONCAPS {# [inline] fn default () -> D3DCONTENTPROTECTIONCAPS {unsafe {$crate :: _core :: mem :: zeroed ()}}}" - ); -} - -#[test] -fn test_int_base() { - parse_macro( - r#" -macro_rules! int_base { - ($Trait:ident for $T:ident as $U:ident -> $Radix:ident) => { - #[stable(feature = "rust1", since = "1.0.0")] - impl fmt::$Trait for $T { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - $Radix.fmt_int(*self as $U, f) - } - } - } -} -"#, - ).assert_expand_items(r#" int_base!{Binary for isize as usize -> Binary}"#, - "# [stable (feature = \"rust1\" , since = \"1.0.0\")] impl fmt ::Binary for isize {fn fmt (& self , f : & mut fmt :: Formatter < \'_ >) -> fmt :: Result {Binary . fmt_int (* self as usize , f)}}" - ); -} - -#[test] -fn test_generate_pattern_iterators() { - // from https://github.com/rust-lang/rust/blob/316a391dcb7d66dc25f1f9a4ec9d368ef7615005/src/libcore/str/mod.rs - parse_macro( - r#" -macro_rules! generate_pattern_iterators { - { double ended; with $(#[$common_stability_attribute:meta])*, - $forward_iterator:ident, - $reverse_iterator:ident, $iterty:ty - } => { - fn foo(){} - } -} -"#, - ).assert_expand_items( - r#"generate_pattern_iterators ! ( double ended ; with # [ stable ( feature = "rust1" , since = "1.0.0" ) ] , Split , RSplit , & 'a str );"#, - "fn foo () {}", - ); -} - -#[test] -fn test_impl_fn_for_zst() { - // from https://github.com/rust-lang/rust/blob/5d20ff4d2718c820632b38c1e49d4de648a9810b/src/libcore/internal_macros.rs - parse_macro( - r#" -macro_rules! impl_fn_for_zst { - { $( $( #[$attr: meta] )* - struct $Name: ident impl$( <$( $lifetime : lifetime ),+> )? Fn = - |$( $arg: ident: $ArgTy: ty ),*| -> $ReturnTy: ty -$body: block; )+ - } => { - $( - $( #[$attr] )* - struct $Name; - - impl $( <$( $lifetime ),+> )? Fn<($( $ArgTy, )*)> for $Name { - #[inline] - extern "rust-call" fn call(&self, ($( $arg, )*): ($( $ArgTy, )*)) -> $ReturnTy { - $body - } - } - - impl $( <$( $lifetime ),+> )? FnMut<($( $ArgTy, )*)> for $Name { - #[inline] - extern "rust-call" fn call_mut( - &mut self, - ($( $arg, )*): ($( $ArgTy, )*) - ) -> $ReturnTy { - Fn::call(&*self, ($( $arg, )*)) - } - } - - impl $( <$( $lifetime ),+> )? FnOnce<($( $ArgTy, )*)> for $Name { - type Output = $ReturnTy; - - #[inline] - extern "rust-call" fn call_once(self, ($( $arg, )*): ($( $ArgTy, )*)) -> $ReturnTy { - Fn::call(&self, ($( $arg, )*)) - } - } - )+ -} - } -"#, - ).assert_expand_items(r#" -impl_fn_for_zst ! { - # [ derive ( Clone ) ] - struct CharEscapeDebugContinue impl Fn = | c : char | -> char :: EscapeDebug { - c . escape_debug_ext ( false ) - } ; - - # [ derive ( Clone ) ] - struct CharEscapeUnicode impl Fn = | c : char | -> char :: EscapeUnicode { - c . escape_unicode ( ) - } ; - # [ derive ( Clone ) ] - struct CharEscapeDefault impl Fn = | c : char | -> char :: EscapeDefault { - c . escape_default ( ) - } ; - } -"#, - "# [derive (Clone)] struct CharEscapeDebugContinue ; impl Fn < (char ,) > for CharEscapeDebugContinue {# [inline] extern \"rust-call\" fn call (& self , (c ,) : (char ,)) -> char :: EscapeDebug {{c . escape_debug_ext (false)}}} impl FnMut < (char ,) > for CharEscapeDebugContinue {# [inline] extern \"rust-call\" fn call_mut (& mut self , (c ,) : (char ,)) -> char :: EscapeDebug {Fn :: call (&* self , (c ,))}} impl FnOnce < (char ,) > for CharEscapeDebugContinue {type Output = char :: EscapeDebug ; # [inline] extern \"rust-call\" fn call_once (self , (c ,) : (char ,)) -> char :: EscapeDebug {Fn :: call (& self , (c ,))}} # [derive (Clone)] struct CharEscapeUnicode ; impl Fn < (char ,) > for CharEscapeUnicode {# [inline] extern \"rust-call\" fn call (& self , (c ,) : (char ,)) -> char :: EscapeUnicode {{c . escape_unicode ()}}} impl FnMut < (char ,) > for CharEscapeUnicode {# [inline] extern \"rust-call\" fn call_mut (& mut self , (c ,) : (char ,)) -> char :: EscapeUnicode {Fn :: call (&* self , (c ,))}} impl FnOnce < (char ,) > for CharEscapeUnicode {type Output = char :: EscapeUnicode ; # [inline] extern \"rust-call\" fn call_once (self , (c ,) : (char ,)) -> char :: EscapeUnicode {Fn :: call (& self , (c ,))}} # [derive (Clone)] struct CharEscapeDefault ; impl Fn < (char ,) > for CharEscapeDefault {# [inline] extern \"rust-call\" fn call (& self , (c ,) : (char ,)) -> char :: EscapeDefault {{c . escape_default ()}}} impl FnMut < (char ,) > for CharEscapeDefault {# [inline] extern \"rust-call\" fn call_mut (& mut self , (c ,) : (char ,)) -> char :: EscapeDefault {Fn :: call (&* self , (c ,))}} impl FnOnce < (char ,) > for CharEscapeDefault {type Output = char :: EscapeDefault ; # [inline] extern \"rust-call\" fn call_once (self , (c ,) : (char ,)) -> char :: EscapeDefault {Fn :: call (& self , (c ,))}}" - ); -} - -#[test] -fn test_impl_nonzero_fmt() { - // from https://github.com/rust-lang/rust/blob/316a391dcb7d66dc25f1f9a4ec9d368ef7615005/src/libcore/num/mod.rs#L12 - parse_macro( - r#" - macro_rules! impl_nonzero_fmt { - ( #[$stability: meta] ( $( $Trait: ident ),+ ) for $Ty: ident ) => { - fn foo () {} - } - } -"#, - ).assert_expand_items( - r#"impl_nonzero_fmt! { # [stable(feature= "nonzero",since="1.28.0")] (Debug,Display,Binary,Octal,LowerHex,UpperHex) for NonZeroU8}"#, - "fn foo () {}", - ); -} - -#[test] -fn test_cfg_if_items() { - // from https://github.com/rust-lang/rust/blob/33fe1131cadba69d317156847be9a402b89f11bb/src/libstd/macros.rs#L986 - parse_macro( - r#" - macro_rules! __cfg_if_items { - (($($not:meta,)*) ; ) => {}; - (($($not:meta,)*) ; ( ($($m:meta),*) ($($it:item)*) ), $($rest:tt)*) => { - __cfg_if_items! { ($($not,)* $($m,)*) ; $($rest)* } - } - } -"#, - ).assert_expand_items( - r#"__cfg_if_items ! { ( rustdoc , ) ; ( ( ) ( # [ cfg ( any ( target_os = "redox" , unix ) ) ] # [ stable ( feature = "rust1" , since = "1.0.0" ) ] pub use sys :: ext as unix ; # [ cfg ( windows ) ] # [ stable ( feature = "rust1" , since = "1.0.0" ) ] pub use sys :: ext as windows ; # [ cfg ( any ( target_os = "linux" , target_os = "l4re" ) ) ] pub mod linux ; ) ) , }"#, - "__cfg_if_items ! {(rustdoc ,) ;}", - ); -} - -#[test] -fn test_cfg_if_main() { - // from https://github.com/rust-lang/rust/blob/3d211248393686e0f73851fc7548f6605220fbe1/src/libpanic_unwind/macros.rs#L9 - parse_macro( - r#" - macro_rules! cfg_if { - ($( - if #[cfg($($meta:meta),*)] { $($it:item)* } - ) else * else { - $($it2:item)* - }) => { - __cfg_if_items! { - () ; - $( ( ($($meta),*) ($($it)*) ), )* - ( () ($($it2)*) ), - } - }; - - // Internal macro to Apply a cfg attribute to a list of items - (@__apply $m:meta, $($it:item)*) => { - $(#[$m] $it)* - }; - } -"#, - ).assert_expand_items(r#" -cfg_if ! { - if # [ cfg ( target_env = "msvc" ) ] { - // no extra unwinder support needed - } else if # [ cfg ( all ( target_arch = "wasm32" , not ( target_os = "emscripten" ) ) ) ] { - // no unwinder on the system! - } else { - mod libunwind ; - pub use libunwind :: * ; - } - } -"#, - "__cfg_if_items ! {() ; ((target_env = \"msvc\") ()) , ((all (target_arch = \"wasm32\" , not (target_os = \"emscripten\"))) ()) , (() (mod libunwind ; pub use libunwind :: * ;)) ,}" - ).assert_expand_items( - r#" -cfg_if ! { @ __apply cfg ( all ( not ( any ( not ( any ( target_os = "solaris" , target_os = "illumos" ) ) ) ) ) ) , } -"#, - "", - ); -} - -#[test] -fn test_proptest_arbitrary() { - // from https://github.com/AltSysrq/proptest/blob/d1c4b049337d2f75dd6f49a095115f7c532e5129/proptest/src/arbitrary/macros.rs#L16 - parse_macro( - r#" -macro_rules! arbitrary { - ([$($bounds : tt)*] $typ: ty, $strat: ty, $params: ty; - $args: ident => $logic: expr) => { - impl<$($bounds)*> $crate::arbitrary::Arbitrary for $typ { - type Parameters = $params; - type Strategy = $strat; - fn arbitrary_with($args: Self::Parameters) -> Self::Strategy { - $logic - } - } - }; - -}"#, - ).assert_expand_items(r#"arbitrary ! ( [ A : Arbitrary ] - Vec < A > , - VecStrategy < A :: Strategy > , - RangedParams1 < A :: Parameters > ; - args => { let product_unpack ! [ range , a ] = args ; vec ( any_with :: < A > ( a ) , range ) } - ) ;"#, - "impl $crate :: arbitrary :: Arbitrary for Vec < A > {type Parameters = RangedParams1 < A :: Parameters > ; type Strategy = VecStrategy < A :: Strategy > ; fn arbitrary_with (args : Self :: Parameters) -> Self :: Strategy {{let product_unpack ! [range , a] = args ; vec (any_with :: < A > (a) , range)}}}" - ); -} - -#[test] -fn test_old_ridl() { - // This is from winapi 2.8, which do not have a link from github - // - let expanded = parse_macro( - r#" -#[macro_export] -macro_rules! RIDL { - (interface $interface:ident ($vtbl:ident) : $pinterface:ident ($pvtbl:ident) - {$( - fn $method:ident(&mut self $(,$p:ident : $t:ty)*) -> $rtr:ty - ),+} - ) => { - impl $interface { - $(pub unsafe fn $method(&mut self) -> $rtr { - ((*self.lpVtbl).$method)(self $(,$p)*) - })+ - } - }; -}"#, - ).expand_tt(r#" - RIDL!{interface ID3D11Asynchronous(ID3D11AsynchronousVtbl): ID3D11DeviceChild(ID3D11DeviceChildVtbl) { - fn GetDataSize(&mut self) -> UINT - }}"#); - - assert_eq!(expanded.to_string(), "impl ID3D11Asynchronous {pub unsafe fn GetDataSize (& mut self) -> UINT {((* self . lpVtbl) .GetDataSize) (self)}}"); -} - -#[test] -fn test_quick_error() { - let expanded = parse_macro( - r#" -macro_rules! quick_error { - - (SORT [enum $name:ident $( #[$meta:meta] )*] - items [$($( #[$imeta:meta] )* - => $iitem:ident: $imode:tt [$( $ivar:ident: $ityp:ty ),*] - {$( $ifuncs:tt )*} )* ] - buf [ ] - queue [ ] - ) => { - quick_error!(ENUMINITION [enum $name $( #[$meta] )*] - body [] - queue [$( - $( #[$imeta] )* - => - $iitem: $imode [$( $ivar: $ityp ),*] - )*] - ); -}; - -} -"#, - ) - .expand_tt( - r#" -quick_error ! (SORT [enum Wrapped # [derive (Debug)]] items [ - => One : UNIT [] {} - => Two : TUPLE [s :String] {display ("two: {}" , s) from ()} - ] buf [] queue []) ; -"#, - ); - - assert_eq!(expanded.to_string(), "quick_error ! (ENUMINITION [enum Wrapped # [derive (Debug)]] body [] queue [=> One : UNIT [] => Two : TUPLE [s : String]]) ;"); -} - -#[test] -fn test_empty_repeat_vars_in_empty_repeat_vars() { - parse_macro( - r#" -macro_rules! delegate_impl { - ([$self_type:ident, $self_wrap:ty, $self_map:ident] - pub trait $name:ident $(: $sup:ident)* $(+ $more_sup:ident)* { - - // "Escaped" associated types. Stripped before making the `trait` - // itself, but forwarded when delegating impls. - $( - @escape [type $assoc_name_ext:ident] - // Associated types. Forwarded. - )* - $( - @section type - $( - $(#[$_assoc_attr:meta])* - type $assoc_name:ident $(: $assoc_bound:ty)*; - )+ - )* - // Methods. Forwarded. Using $self_map!(self) around the self argument. - // Methods must use receiver `self` or explicit type like `self: &Self` - // &self and &mut self are _not_ supported. - $( - @section self - $( - $(#[$_method_attr:meta])* - fn $method_name:ident(self $(: $self_selftype:ty)* $(,$marg:ident : $marg_ty:ty)*) -> $mret:ty; - )+ - )* - // Arbitrary tail that is ignored when forwarding. - $( - @section nodelegate - $($tail:tt)* - )* - }) => { - impl<> $name for $self_wrap where $self_type: $name { - $( - $( - fn $method_name(self $(: $self_selftype)* $(,$marg: $marg_ty)*) -> $mret { - $self_map!(self).$method_name($($marg),*) - } - )* - )* - } - } -} -"#, - ).assert_expand_items( - r#"delegate_impl ! {[G , & 'a mut G , deref] pub trait Data : GraphBase {@ section type type NodeWeight ;}}"#, - "impl <> Data for & \'a mut G where G : Data {}", - ); -} - -#[test] -fn expr_interpolation() { - let expanded = parse_macro( - r#" - macro_rules! id { - ($expr:expr) => { - map($expr) - } - } - "#, - ) - .expand_expr("id!(x + foo);"); - - assert_eq!(expanded.to_string(), "map(x+foo)"); -} - -pub(crate) struct MacroFixture { - rules: MacroRules, -} - -impl MacroFixture { - pub(crate) fn expand_tt(&self, invocation: &str) -> tt::Subtree { - self.try_expand_tt(invocation).unwrap() - } - - fn try_expand_tt(&self, invocation: &str) -> Result { - let source_file = ast::SourceFile::parse(invocation).tree(); - let macro_invocation = - source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); - - let (invocation_tt, _) = ast_to_token_tree(¯o_invocation.token_tree().unwrap()) - .ok_or_else(|| ExpandError::ConversionError)?; - - self.rules.expand(&invocation_tt).result() - } - - fn assert_expand_err(&self, invocation: &str, err: &ExpandError) { - assert_eq!(self.try_expand_tt(invocation).as_ref(), Err(err)); - } - - fn expand_items(&self, invocation: &str) -> SyntaxNode { - let expanded = self.expand_tt(invocation); - token_tree_to_syntax_node(&expanded, FragmentKind::Items).unwrap().0.syntax_node() - } - - fn expand_statements(&self, invocation: &str) -> SyntaxNode { - let expanded = self.expand_tt(invocation); - token_tree_to_syntax_node(&expanded, FragmentKind::Statements).unwrap().0.syntax_node() - } - - fn expand_expr(&self, invocation: &str) -> SyntaxNode { - let expanded = self.expand_tt(invocation); - token_tree_to_syntax_node(&expanded, FragmentKind::Expr).unwrap().0.syntax_node() - } - - fn assert_expand_tt(&self, invocation: &str, expected: &str) { - let expansion = self.expand_tt(invocation); - assert_eq!(expansion.to_string(), expected); - } - - fn assert_expand(&self, invocation: &str, expected: &str) { - let expansion = self.expand_tt(invocation); - let actual = format!("{:?}", expansion); - test_utils::assert_eq_text!(&actual.trim(), &expected.trim()); - } - - fn assert_expand_items(&self, invocation: &str, expected: &str) -> &MacroFixture { - self.assert_expansion(FragmentKind::Items, invocation, expected); - self - } - - fn assert_expand_statements(&self, invocation: &str, expected: &str) -> &MacroFixture { - self.assert_expansion(FragmentKind::Statements, invocation, expected); - self - } - - fn assert_expansion(&self, kind: FragmentKind, invocation: &str, expected: &str) { - let expanded = self.expand_tt(invocation); - assert_eq!(expanded.to_string(), expected); - - let expected = expected.replace("$crate", "C_C__C"); - - // wrap the given text to a macro call - let expected = { - let wrapped = format!("wrap_macro!( {} )", expected); - let wrapped = ast::SourceFile::parse(&wrapped); - let wrapped = - wrapped.tree().syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - let mut wrapped = ast_to_token_tree(&wrapped).unwrap().0; - wrapped.delimiter = None; - wrapped - }; - - let expanded_tree = token_tree_to_syntax_node(&expanded, kind).unwrap().0.syntax_node(); - let expanded_tree = debug_dump_ignore_spaces(&expanded_tree).trim().to_string(); - - let expected_tree = token_tree_to_syntax_node(&expected, kind).unwrap().0.syntax_node(); - let expected_tree = debug_dump_ignore_spaces(&expected_tree).trim().to_string(); - - let expected_tree = expected_tree.replace("C_C__C", "$crate"); - assert_eq!( - expanded_tree, expected_tree, - "\nleft:\n{}\nright:\n{}", - expanded_tree, expected_tree, - ); - } -} - -fn parse_macro_to_tt(ra_fixture: &str) -> tt::Subtree { - let source_file = ast::SourceFile::parse(ra_fixture).ok().unwrap(); - let macro_definition = - source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); - - let (definition_tt, _) = ast_to_token_tree(¯o_definition.token_tree().unwrap()).unwrap(); - - let parsed = parse_to_token_tree( - &ra_fixture[macro_definition.token_tree().unwrap().syntax().text_range()], - ) - .unwrap() - .0; - assert_eq!(definition_tt, parsed); - - definition_tt -} - -pub(crate) fn parse_macro(ra_fixture: &str) -> MacroFixture { - let definition_tt = parse_macro_to_tt(ra_fixture); - let rules = MacroRules::parse(&definition_tt).unwrap(); - MacroFixture { rules } -} - -pub(crate) fn parse_macro_error(ra_fixture: &str) -> ParseError { - let definition_tt = parse_macro_to_tt(ra_fixture); - - match MacroRules::parse(&definition_tt) { - Ok(_) => panic!("Expect error"), - Err(err) => err, - } -} - -pub(crate) fn parse_to_token_tree_by_syntax(ra_fixture: &str) -> tt::Subtree { - let source_file = ast::SourceFile::parse(ra_fixture).ok().unwrap(); - let tt = syntax_node_to_token_tree(source_file.syntax()).unwrap().0; - - let parsed = parse_to_token_tree(ra_fixture).unwrap().0; - assert_eq!(tt, parsed); - - parsed -} - -fn debug_dump_ignore_spaces(node: &syntax::SyntaxNode) -> String { - let mut level = 0; - let mut buf = String::new(); - macro_rules! indent { - () => { - for _ in 0..level { - buf.push_str(" "); - } - }; - } - - for event in node.preorder_with_tokens() { - match event { - WalkEvent::Enter(element) => { - match element { - NodeOrToken::Node(node) => { - indent!(); - writeln!(buf, "{:?}", node.kind()).unwrap(); - } - NodeOrToken::Token(token) => match token.kind() { - syntax::SyntaxKind::WHITESPACE => {} - _ => { - indent!(); - writeln!(buf, "{:?}", token.kind()).unwrap(); - } - }, - } - level += 1; - } - WalkEvent::Leave(_) => level -= 1, - } - } - - buf -} - -#[test] -fn test_issue_2520() { - let macro_fixture = parse_macro( - r#" - macro_rules! my_macro { - { - ( $( - $( [] $sname:ident : $stype:ty )? - $( [$expr:expr] $nname:ident : $ntype:ty )? - ),* ) - } => { - Test { - $( - $( $sname, )? - )* - } - }; - } - "#, - ); - - macro_fixture.assert_expand_items( - r#"my_macro ! { - ([] p1 : u32 , [|_| S0K0] s : S0K0 , [] k0 : i32) - }"#, - "Test {p1 , k0 ,}", - ); -} - -#[test] -fn test_issue_3861() { - let macro_fixture = parse_macro( - r#" - macro_rules! rgb_color { - ($p:expr, $t: ty) => { - pub fn new() { - let _ = 0 as $t << $p; - } - }; - } - "#, - ); - - macro_fixture.expand_items(r#"rgb_color!(8 + 8, u32);"#); -} - -#[test] -fn test_repeat_bad_var() { - // FIXME: the second rule of the macro should be removed and an error about - // `$( $c )+` raised - parse_macro( - r#" - macro_rules! foo { - ($( $b:ident )+) => { - $( $c )+ - }; - ($( $b:ident )+) => { - $( $b )+ - } - } - "#, - ) - .assert_expand_items("foo!(b0 b1);", "b0 b1"); -} - -#[test] -fn test_no_space_after_semi_colon() { - let expanded = parse_macro( - r#" - macro_rules! with_std { ($($i:item)*) => ($(#[cfg(feature = "std")]$i)*) } - "#, - ) - .expand_items(r#"with_std! {mod m;mod f;}"#); - - let dump = format!("{:#?}", expanded); - assert_eq_text!( - dump.trim(), - r###"MACRO_ITEMS@0..52 - MODULE@0..26 - ATTR@0..21 - POUND@0..1 "#" - L_BRACK@1..2 "[" - PATH@2..5 - PATH_SEGMENT@2..5 - NAME_REF@2..5 - IDENT@2..5 "cfg" - TOKEN_TREE@5..20 - L_PAREN@5..6 "(" - IDENT@6..13 "feature" - EQ@13..14 "=" - STRING@14..19 "\"std\"" - R_PAREN@19..20 ")" - R_BRACK@20..21 "]" - MOD_KW@21..24 "mod" - NAME@24..25 - IDENT@24..25 "m" - SEMICOLON@25..26 ";" - MODULE@26..52 - ATTR@26..47 - POUND@26..27 "#" - L_BRACK@27..28 "[" - PATH@28..31 - PATH_SEGMENT@28..31 - NAME_REF@28..31 - IDENT@28..31 "cfg" - TOKEN_TREE@31..46 - L_PAREN@31..32 "(" - IDENT@32..39 "feature" - EQ@39..40 "=" - STRING@40..45 "\"std\"" - R_PAREN@45..46 ")" - R_BRACK@46..47 "]" - MOD_KW@47..50 "mod" - NAME@50..51 - IDENT@50..51 "f" - SEMICOLON@51..52 ";""###, - ); -} - -// https://github.com/rust-lang/rust/blob/master/src/test/ui/issues/issue-57597.rs -#[test] -fn test_rustc_issue_57597() { - fn test_error(fixture: &str) { - assert_eq!(parse_macro_error(fixture), ParseError::RepetitionEmtpyTokenTree); - } - - test_error("macro_rules! foo { ($($($i:ident)?)+) => {}; }"); - test_error("macro_rules! foo { ($($($i:ident)?)*) => {}; }"); - test_error("macro_rules! foo { ($($($i:ident)?)?) => {}; }"); - test_error("macro_rules! foo { ($($($($i:ident)?)?)?) => {}; }"); - test_error("macro_rules! foo { ($($($($i:ident)*)?)?) => {}; }"); - test_error("macro_rules! foo { ($($($($i:ident)?)*)?) => {}; }"); - test_error("macro_rules! foo { ($($($($i:ident)?)?)*) => {}; }"); - test_error("macro_rules! foo { ($($($($i:ident)*)*)?) => {}; }"); - test_error("macro_rules! foo { ($($($($i:ident)?)*)*) => {}; }"); - test_error("macro_rules! foo { ($($($($i:ident)?)*)+) => {}; }"); - test_error("macro_rules! foo { ($($($($i:ident)+)?)*) => {}; }"); - test_error("macro_rules! foo { ($($($($i:ident)+)*)?) => {}; }"); -} - -#[test] -fn test_expand_bad_literal() { - parse_macro( - r#" - macro_rules! foo { ($i:literal) => {}; } - "#, - ) - .assert_expand_err(r#"foo!(&k");"#, &ExpandError::BindingError("".into())); -} - -#[test] -fn test_empty_comments() { - parse_macro( - r#" - macro_rules! one_arg_macro { ($fmt:expr) => (); } - "#, - ) - .assert_expand_err( - r#"one_arg_macro!(/**/)"#, - &ExpandError::BindingError("expected Expr".into()), - ); -} diff --git a/crates/ra_mbe/src/tt_iter.rs b/crates/ra_mbe/src/tt_iter.rs deleted file mode 100644 index 46c420718..000000000 --- a/crates/ra_mbe/src/tt_iter.rs +++ /dev/null @@ -1,75 +0,0 @@ -//! FIXME: write short doc here - -#[derive(Debug, Clone)] -pub(crate) struct TtIter<'a> { - pub(crate) inner: std::slice::Iter<'a, tt::TokenTree>, -} - -impl<'a> TtIter<'a> { - pub(crate) fn new(subtree: &'a tt::Subtree) -> TtIter<'a> { - TtIter { inner: subtree.token_trees.iter() } - } - - pub(crate) fn expect_char(&mut self, char: char) -> Result<(), ()> { - match self.next() { - Some(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: c, .. }))) if *c == char => { - Ok(()) - } - _ => Err(()), - } - } - - pub(crate) fn expect_subtree(&mut self) -> Result<&'a tt::Subtree, ()> { - match self.next() { - Some(tt::TokenTree::Subtree(it)) => Ok(it), - _ => Err(()), - } - } - - pub(crate) fn expect_leaf(&mut self) -> Result<&'a tt::Leaf, ()> { - match self.next() { - Some(tt::TokenTree::Leaf(it)) => Ok(it), - _ => Err(()), - } - } - - pub(crate) fn expect_ident(&mut self) -> Result<&'a tt::Ident, ()> { - match self.expect_leaf()? { - tt::Leaf::Ident(it) => Ok(it), - _ => Err(()), - } - } - - pub(crate) fn expect_literal(&mut self) -> Result<&'a tt::Leaf, ()> { - let it = self.expect_leaf()?; - match it { - tt::Leaf::Literal(_) => Ok(it), - tt::Leaf::Ident(ident) if ident.text == "true" || ident.text == "false" => Ok(it), - _ => Err(()), - } - } - - pub(crate) fn expect_punct(&mut self) -> Result<&'a tt::Punct, ()> { - match self.expect_leaf()? { - tt::Leaf::Punct(it) => Ok(it), - _ => Err(()), - } - } - - pub(crate) fn peek_n(&self, n: usize) -> Option<&tt::TokenTree> { - self.inner.as_slice().get(n) - } -} - -impl<'a> Iterator for TtIter<'a> { - type Item = &'a tt::TokenTree; - fn next(&mut self) -> Option { - self.inner.next() - } - - fn size_hint(&self) -> (usize, Option) { - self.inner.size_hint() - } -} - -impl<'a> std::iter::ExactSizeIterator for TtIter<'a> {} diff --git a/crates/rust-analyzer/Cargo.toml b/crates/rust-analyzer/Cargo.toml index 7713ed7ea..156f8d538 100644 --- a/crates/rust-analyzer/Cargo.toml +++ b/crates/rust-analyzer/Cargo.toml @@ -60,5 +60,5 @@ winapi = "0.3.8" [dev-dependencies] expect = { path = "../expect" } test_utils = { path = "../test_utils" } -mbe = { path = "../ra_mbe", package = "ra_mbe" } +mbe = { path = "../mbe" } tt = { path = "../tt" } diff --git a/xtask/tests/tidy.rs b/xtask/tests/tidy.rs index 8c608284a..0188aaa2e 100644 --- a/xtask/tests/tidy.rs +++ b/xtask/tests/tidy.rs @@ -195,7 +195,7 @@ impl TidyDocs { "ra_hir", "ra_hir_expand", "ra_ide", - "ra_mbe", + "mbe", "parser", "profile", "ra_project_model", -- cgit v1.2.3 From 3615758f8ebf5d2cdd5d82f10daa596acfc1a64f Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Thu, 13 Aug 2020 10:15:45 +0200 Subject: Minimize deps --- crates/ra_cfg/Cargo.toml | 2 +- crates/ra_cfg/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/ra_cfg/Cargo.toml b/crates/ra_cfg/Cargo.toml index e77c7bd72..661e53904 100644 --- a/crates/ra_cfg/Cargo.toml +++ b/crates/ra_cfg/Cargo.toml @@ -11,8 +11,8 @@ doctest = false [dependencies] rustc-hash = "1.1.0" -syntax = { path = "../syntax" } tt = { path = "../tt" } [dev-dependencies] mbe = { path = "../mbe" } +syntax = { path = "../syntax" } diff --git a/crates/ra_cfg/src/lib.rs b/crates/ra_cfg/src/lib.rs index 7e025143b..6fb37bf9f 100644 --- a/crates/ra_cfg/src/lib.rs +++ b/crates/ra_cfg/src/lib.rs @@ -3,7 +3,7 @@ mod cfg_expr; use rustc_hash::FxHashSet; -use syntax::SmolStr; +use tt::SmolStr; pub use cfg_expr::CfgExpr; -- cgit v1.2.3 From 5734cc85868bf5fe2e4e023b40913b2063a78e31 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Thu, 13 Aug 2020 10:17:59 +0200 Subject: Simplify --- crates/ra_cfg/src/cfg_expr.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/crates/ra_cfg/src/cfg_expr.rs b/crates/ra_cfg/src/cfg_expr.rs index fc93730d9..336fe25bc 100644 --- a/crates/ra_cfg/src/cfg_expr.rs +++ b/crates/ra_cfg/src/cfg_expr.rs @@ -86,17 +86,15 @@ fn next_cfg_expr(it: &mut SliceIter) -> Option { mod tests { use super::*; - use mbe::{ast_to_token_tree, TokenMap}; + use mbe::ast_to_token_tree; use syntax::ast::{self, AstNode}; - fn get_token_tree_generated(input: &str) -> (tt::Subtree, TokenMap) { - let source_file = ast::SourceFile::parse(input).ok().unwrap(); - let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - ast_to_token_tree(&tt).unwrap() - } - fn assert_parse_result(input: &str, expected: CfgExpr) { - let (tt, _) = get_token_tree_generated(input); + let (tt, _) = { + let source_file = ast::SourceFile::parse(input).ok().unwrap(); + let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); + ast_to_token_tree(&tt).unwrap() + }; let cfg = CfgExpr::parse(&tt); assert_eq!(cfg, expected); } -- cgit v1.2.3 From 68c223872562a8d746d4f1045d508887a0cbca5e Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Thu, 13 Aug 2020 10:19:09 +0200 Subject: Rename ra_cfg -> cfg --- Cargo.lock | 30 +++--- crates/cfg/Cargo.toml | 18 ++++ crates/cfg/src/cfg_expr.rs | 133 ++++++++++++++++++++++++++ crates/cfg/src/lib.rs | 51 ++++++++++ crates/ra_cfg/Cargo.toml | 18 ---- crates/ra_cfg/src/cfg_expr.rs | 133 -------------------------- crates/ra_cfg/src/lib.rs | 51 ---------- crates/ra_db/Cargo.toml | 2 +- crates/ra_db/src/fixture.rs | 2 +- crates/ra_db/src/input.rs | 2 +- crates/ra_hir_def/Cargo.toml | 2 +- crates/ra_hir_def/src/adt.rs | 2 +- crates/ra_hir_def/src/attr.rs | 2 +- crates/ra_hir_def/src/body.rs | 2 +- crates/ra_hir_def/src/nameres/collector.rs | 2 +- crates/ra_ide/Cargo.toml | 2 +- crates/ra_ide/src/lib.rs | 2 +- crates/ra_ide/src/mock_analysis.rs | 2 +- crates/ra_ide/src/parent_module.rs | 2 +- crates/ra_ide/src/runnables.rs | 2 +- crates/ra_project_model/Cargo.toml | 2 +- crates/ra_project_model/src/cfg_flag.rs | 2 +- crates/ra_project_model/src/lib.rs | 2 +- crates/rust-analyzer/Cargo.toml | 2 +- crates/rust-analyzer/src/cargo_target_spec.rs | 4 +- 25 files changed, 236 insertions(+), 236 deletions(-) create mode 100644 crates/cfg/Cargo.toml create mode 100644 crates/cfg/src/cfg_expr.rs create mode 100644 crates/cfg/src/lib.rs delete mode 100644 crates/ra_cfg/Cargo.toml delete mode 100644 crates/ra_cfg/src/cfg_expr.rs delete mode 100644 crates/ra_cfg/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index a63cd58f2..18c979b39 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -112,6 +112,16 @@ version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9a06fb2e53271d7c279ec1efea6ab691c35a2ae67ec0d91d7acec0caf13b518" +[[package]] +name = "cfg" +version = "0.0.0" +dependencies = [ + "mbe", + "rustc-hash", + "syntax", + "tt", +] + [[package]] name = "cfg-if" version = "0.1.10" @@ -955,22 +965,12 @@ dependencies = [ "text_edit", ] -[[package]] -name = "ra_cfg" -version = "0.1.0" -dependencies = [ - "mbe", - "rustc-hash", - "syntax", - "tt", -] - [[package]] name = "ra_db" version = "0.1.0" dependencies = [ + "cfg", "profile", - "ra_cfg", "rustc-hash", "salsa", "stdx", @@ -1012,6 +1012,7 @@ version = "0.1.0" dependencies = [ "anymap", "arena", + "cfg", "drop_bomb", "either", "expect", @@ -1022,7 +1023,6 @@ dependencies = [ "mbe", "once_cell", "profile", - "ra_cfg", "ra_db", "ra_hir_expand", "rustc-hash", @@ -1082,6 +1082,7 @@ dependencies = [ name = "ra_ide" version = "0.1.0" dependencies = [ + "cfg", "either", "expect", "indexmap", @@ -1090,7 +1091,6 @@ dependencies = [ "oorandom", "profile", "ra_assists", - "ra_cfg", "ra_db", "ra_fmt", "ra_hir", @@ -1141,9 +1141,9 @@ dependencies = [ "anyhow", "arena", "cargo_metadata", + "cfg", "log", "paths", - "ra_cfg", "ra_db", "ra_proc_macro", "rustc-hash", @@ -1240,6 +1240,7 @@ name = "rust-analyzer" version = "0.1.0" dependencies = [ "anyhow", + "cfg", "crossbeam-channel", "env_logger", "expect", @@ -1256,7 +1257,6 @@ dependencies = [ "pico-args", "proc_macro_srv", "profile", - "ra_cfg", "ra_db", "ra_hir", "ra_hir_def", diff --git a/crates/cfg/Cargo.toml b/crates/cfg/Cargo.toml new file mode 100644 index 000000000..d2ea551d1 --- /dev/null +++ b/crates/cfg/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "cfg" +version = "0.0.0" +license = "MIT OR Apache-2.0" +authors = ["rust-analyzer developers"] +edition = "2018" + +[lib] +doctest = false + +[dependencies] +rustc-hash = "1.1.0" + +tt = { path = "../tt" } + +[dev-dependencies] +mbe = { path = "../mbe" } +syntax = { path = "../syntax" } diff --git a/crates/cfg/src/cfg_expr.rs b/crates/cfg/src/cfg_expr.rs new file mode 100644 index 000000000..336fe25bc --- /dev/null +++ b/crates/cfg/src/cfg_expr.rs @@ -0,0 +1,133 @@ +//! The condition expression used in `#[cfg(..)]` attributes. +//! +//! See: https://doc.rust-lang.org/reference/conditional-compilation.html#conditional-compilation + +use std::slice::Iter as SliceIter; + +use tt::SmolStr; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CfgExpr { + Invalid, + Atom(SmolStr), + KeyValue { key: SmolStr, value: SmolStr }, + All(Vec), + Any(Vec), + Not(Box), +} + +impl CfgExpr { + pub fn parse(tt: &tt::Subtree) -> CfgExpr { + next_cfg_expr(&mut tt.token_trees.iter()).unwrap_or(CfgExpr::Invalid) + } + /// Fold the cfg by querying all basic `Atom` and `KeyValue` predicates. + pub fn fold(&self, query: &dyn Fn(&SmolStr, Option<&SmolStr>) -> bool) -> Option { + match self { + CfgExpr::Invalid => None, + CfgExpr::Atom(name) => Some(query(name, None)), + CfgExpr::KeyValue { key, value } => Some(query(key, Some(value))), + CfgExpr::All(preds) => { + preds.iter().try_fold(true, |s, pred| Some(s && pred.fold(query)?)) + } + CfgExpr::Any(preds) => { + preds.iter().try_fold(false, |s, pred| Some(s || pred.fold(query)?)) + } + CfgExpr::Not(pred) => pred.fold(query).map(|s| !s), + } + } +} + +fn next_cfg_expr(it: &mut SliceIter) -> Option { + let name = match it.next() { + None => return None, + Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.text.clone(), + Some(_) => return Some(CfgExpr::Invalid), + }; + + // Peek + let ret = match it.as_slice().first() { + Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '=' => { + match it.as_slice().get(1) { + Some(tt::TokenTree::Leaf(tt::Leaf::Literal(literal))) => { + it.next(); + it.next(); + // FIXME: escape? raw string? + let value = + SmolStr::new(literal.text.trim_start_matches('"').trim_end_matches('"')); + CfgExpr::KeyValue { key: name, value } + } + _ => return Some(CfgExpr::Invalid), + } + } + Some(tt::TokenTree::Subtree(subtree)) => { + it.next(); + let mut sub_it = subtree.token_trees.iter(); + let mut subs = std::iter::from_fn(|| next_cfg_expr(&mut sub_it)).collect(); + match name.as_str() { + "all" => CfgExpr::All(subs), + "any" => CfgExpr::Any(subs), + "not" => CfgExpr::Not(Box::new(subs.pop().unwrap_or(CfgExpr::Invalid))), + _ => CfgExpr::Invalid, + } + } + _ => CfgExpr::Atom(name), + }; + + // Eat comma separator + if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = it.as_slice().first() { + if punct.char == ',' { + it.next(); + } + } + Some(ret) +} + +#[cfg(test)] +mod tests { + use super::*; + + use mbe::ast_to_token_tree; + use syntax::ast::{self, AstNode}; + + fn assert_parse_result(input: &str, expected: CfgExpr) { + let (tt, _) = { + let source_file = ast::SourceFile::parse(input).ok().unwrap(); + let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); + ast_to_token_tree(&tt).unwrap() + }; + let cfg = CfgExpr::parse(&tt); + assert_eq!(cfg, expected); + } + + #[test] + fn test_cfg_expr_parser() { + assert_parse_result("#![cfg(foo)]", CfgExpr::Atom("foo".into())); + assert_parse_result("#![cfg(foo,)]", CfgExpr::Atom("foo".into())); + assert_parse_result( + "#![cfg(not(foo))]", + CfgExpr::Not(Box::new(CfgExpr::Atom("foo".into()))), + ); + assert_parse_result("#![cfg(foo(bar))]", CfgExpr::Invalid); + + // Only take the first + assert_parse_result(r#"#![cfg(foo, bar = "baz")]"#, CfgExpr::Atom("foo".into())); + + assert_parse_result( + r#"#![cfg(all(foo, bar = "baz"))]"#, + CfgExpr::All(vec![ + CfgExpr::Atom("foo".into()), + CfgExpr::KeyValue { key: "bar".into(), value: "baz".into() }, + ]), + ); + + assert_parse_result( + r#"#![cfg(any(not(), all(), , bar = "baz",))]"#, + CfgExpr::Any(vec![ + CfgExpr::Not(Box::new(CfgExpr::Invalid)), + CfgExpr::All(vec![]), + CfgExpr::Invalid, + CfgExpr::KeyValue { key: "bar".into(), value: "baz".into() }, + ]), + ); + } +} diff --git a/crates/cfg/src/lib.rs b/crates/cfg/src/lib.rs new file mode 100644 index 000000000..a9d50e698 --- /dev/null +++ b/crates/cfg/src/lib.rs @@ -0,0 +1,51 @@ +//! cfg defines conditional compiling options, `cfg` attibute parser and evaluator + +mod cfg_expr; + +use rustc_hash::FxHashSet; +use tt::SmolStr; + +pub use cfg_expr::CfgExpr; + +/// Configuration options used for conditional compilition on items with `cfg` attributes. +/// We have two kind of options in different namespaces: atomic options like `unix`, and +/// key-value options like `target_arch="x86"`. +/// +/// Note that for key-value options, one key can have multiple values (but not none). +/// `feature` is an example. We have both `feature="foo"` and `feature="bar"` if features +/// `foo` and `bar` are both enabled. And here, we store key-value options as a set of tuple +/// of key and value in `key_values`. +/// +/// See: https://doc.rust-lang.org/reference/conditional-compilation.html#set-configuration-options +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct CfgOptions { + atoms: FxHashSet, + key_values: FxHashSet<(SmolStr, SmolStr)>, +} + +impl CfgOptions { + pub fn check(&self, cfg: &CfgExpr) -> Option { + cfg.fold(&|key, value| match value { + None => self.atoms.contains(key), + Some(value) => self.key_values.contains(&(key.clone(), value.clone())), + }) + } + + pub fn insert_atom(&mut self, key: SmolStr) { + self.atoms.insert(key); + } + + pub fn insert_key_value(&mut self, key: SmolStr, value: SmolStr) { + self.key_values.insert((key, value)); + } + + pub fn append(&mut self, other: &CfgOptions) { + for atom in &other.atoms { + self.atoms.insert(atom.clone()); + } + + for (key, value) in &other.key_values { + self.key_values.insert((key.clone(), value.clone())); + } + } +} diff --git a/crates/ra_cfg/Cargo.toml b/crates/ra_cfg/Cargo.toml deleted file mode 100644 index 661e53904..000000000 --- a/crates/ra_cfg/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -edition = "2018" -name = "ra_cfg" -version = "0.1.0" -authors = ["rust-analyzer developers"] -license = "MIT OR Apache-2.0" - -[lib] -doctest = false - -[dependencies] -rustc-hash = "1.1.0" - -tt = { path = "../tt" } - -[dev-dependencies] -mbe = { path = "../mbe" } -syntax = { path = "../syntax" } diff --git a/crates/ra_cfg/src/cfg_expr.rs b/crates/ra_cfg/src/cfg_expr.rs deleted file mode 100644 index 336fe25bc..000000000 --- a/crates/ra_cfg/src/cfg_expr.rs +++ /dev/null @@ -1,133 +0,0 @@ -//! The condition expression used in `#[cfg(..)]` attributes. -//! -//! See: https://doc.rust-lang.org/reference/conditional-compilation.html#conditional-compilation - -use std::slice::Iter as SliceIter; - -use tt::SmolStr; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum CfgExpr { - Invalid, - Atom(SmolStr), - KeyValue { key: SmolStr, value: SmolStr }, - All(Vec), - Any(Vec), - Not(Box), -} - -impl CfgExpr { - pub fn parse(tt: &tt::Subtree) -> CfgExpr { - next_cfg_expr(&mut tt.token_trees.iter()).unwrap_or(CfgExpr::Invalid) - } - /// Fold the cfg by querying all basic `Atom` and `KeyValue` predicates. - pub fn fold(&self, query: &dyn Fn(&SmolStr, Option<&SmolStr>) -> bool) -> Option { - match self { - CfgExpr::Invalid => None, - CfgExpr::Atom(name) => Some(query(name, None)), - CfgExpr::KeyValue { key, value } => Some(query(key, Some(value))), - CfgExpr::All(preds) => { - preds.iter().try_fold(true, |s, pred| Some(s && pred.fold(query)?)) - } - CfgExpr::Any(preds) => { - preds.iter().try_fold(false, |s, pred| Some(s || pred.fold(query)?)) - } - CfgExpr::Not(pred) => pred.fold(query).map(|s| !s), - } - } -} - -fn next_cfg_expr(it: &mut SliceIter) -> Option { - let name = match it.next() { - None => return None, - Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.text.clone(), - Some(_) => return Some(CfgExpr::Invalid), - }; - - // Peek - let ret = match it.as_slice().first() { - Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '=' => { - match it.as_slice().get(1) { - Some(tt::TokenTree::Leaf(tt::Leaf::Literal(literal))) => { - it.next(); - it.next(); - // FIXME: escape? raw string? - let value = - SmolStr::new(literal.text.trim_start_matches('"').trim_end_matches('"')); - CfgExpr::KeyValue { key: name, value } - } - _ => return Some(CfgExpr::Invalid), - } - } - Some(tt::TokenTree::Subtree(subtree)) => { - it.next(); - let mut sub_it = subtree.token_trees.iter(); - let mut subs = std::iter::from_fn(|| next_cfg_expr(&mut sub_it)).collect(); - match name.as_str() { - "all" => CfgExpr::All(subs), - "any" => CfgExpr::Any(subs), - "not" => CfgExpr::Not(Box::new(subs.pop().unwrap_or(CfgExpr::Invalid))), - _ => CfgExpr::Invalid, - } - } - _ => CfgExpr::Atom(name), - }; - - // Eat comma separator - if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = it.as_slice().first() { - if punct.char == ',' { - it.next(); - } - } - Some(ret) -} - -#[cfg(test)] -mod tests { - use super::*; - - use mbe::ast_to_token_tree; - use syntax::ast::{self, AstNode}; - - fn assert_parse_result(input: &str, expected: CfgExpr) { - let (tt, _) = { - let source_file = ast::SourceFile::parse(input).ok().unwrap(); - let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - ast_to_token_tree(&tt).unwrap() - }; - let cfg = CfgExpr::parse(&tt); - assert_eq!(cfg, expected); - } - - #[test] - fn test_cfg_expr_parser() { - assert_parse_result("#![cfg(foo)]", CfgExpr::Atom("foo".into())); - assert_parse_result("#![cfg(foo,)]", CfgExpr::Atom("foo".into())); - assert_parse_result( - "#![cfg(not(foo))]", - CfgExpr::Not(Box::new(CfgExpr::Atom("foo".into()))), - ); - assert_parse_result("#![cfg(foo(bar))]", CfgExpr::Invalid); - - // Only take the first - assert_parse_result(r#"#![cfg(foo, bar = "baz")]"#, CfgExpr::Atom("foo".into())); - - assert_parse_result( - r#"#![cfg(all(foo, bar = "baz"))]"#, - CfgExpr::All(vec![ - CfgExpr::Atom("foo".into()), - CfgExpr::KeyValue { key: "bar".into(), value: "baz".into() }, - ]), - ); - - assert_parse_result( - r#"#![cfg(any(not(), all(), , bar = "baz",))]"#, - CfgExpr::Any(vec![ - CfgExpr::Not(Box::new(CfgExpr::Invalid)), - CfgExpr::All(vec![]), - CfgExpr::Invalid, - CfgExpr::KeyValue { key: "bar".into(), value: "baz".into() }, - ]), - ); - } -} diff --git a/crates/ra_cfg/src/lib.rs b/crates/ra_cfg/src/lib.rs deleted file mode 100644 index 6fb37bf9f..000000000 --- a/crates/ra_cfg/src/lib.rs +++ /dev/null @@ -1,51 +0,0 @@ -//! ra_cfg defines conditional compiling options, `cfg` attibute parser and evaluator - -mod cfg_expr; - -use rustc_hash::FxHashSet; -use tt::SmolStr; - -pub use cfg_expr::CfgExpr; - -/// Configuration options used for conditional compilition on items with `cfg` attributes. -/// We have two kind of options in different namespaces: atomic options like `unix`, and -/// key-value options like `target_arch="x86"`. -/// -/// Note that for key-value options, one key can have multiple values (but not none). -/// `feature` is an example. We have both `feature="foo"` and `feature="bar"` if features -/// `foo` and `bar` are both enabled. And here, we store key-value options as a set of tuple -/// of key and value in `key_values`. -/// -/// See: https://doc.rust-lang.org/reference/conditional-compilation.html#set-configuration-options -#[derive(Debug, Clone, PartialEq, Eq, Default)] -pub struct CfgOptions { - atoms: FxHashSet, - key_values: FxHashSet<(SmolStr, SmolStr)>, -} - -impl CfgOptions { - pub fn check(&self, cfg: &CfgExpr) -> Option { - cfg.fold(&|key, value| match value { - None => self.atoms.contains(key), - Some(value) => self.key_values.contains(&(key.clone(), value.clone())), - }) - } - - pub fn insert_atom(&mut self, key: SmolStr) { - self.atoms.insert(key); - } - - pub fn insert_key_value(&mut self, key: SmolStr, value: SmolStr) { - self.key_values.insert((key, value)); - } - - pub fn append(&mut self, other: &CfgOptions) { - for atom in &other.atoms { - self.atoms.insert(atom.clone()); - } - - for (key, value) in &other.key_values { - self.key_values.insert((key.clone(), value.clone())); - } - } -} diff --git a/crates/ra_db/Cargo.toml b/crates/ra_db/Cargo.toml index 156ea1ee4..ad432f096 100644 --- a/crates/ra_db/Cargo.toml +++ b/crates/ra_db/Cargo.toml @@ -13,7 +13,7 @@ salsa = "0.15.2" rustc-hash = "1.1.0" syntax = { path = "../syntax" } -ra_cfg = { path = "../ra_cfg" } +cfg = { path = "../cfg" } profile = { path = "../profile" } tt = { path = "../tt" } test_utils = { path = "../test_utils" } diff --git a/crates/ra_db/src/fixture.rs b/crates/ra_db/src/fixture.rs index 2aafb9965..5ff8ead0e 100644 --- a/crates/ra_db/src/fixture.rs +++ b/crates/ra_db/src/fixture.rs @@ -59,7 +59,7 @@ //! ``` use std::{str::FromStr, sync::Arc}; -use ra_cfg::CfgOptions; +use cfg::CfgOptions; use rustc_hash::FxHashMap; use test_utils::{extract_range_or_offset, Fixture, RangeOrOffset, CURSOR_MARKER}; use vfs::{file_set::FileSet, VfsPath}; diff --git a/crates/ra_db/src/input.rs b/crates/ra_db/src/input.rs index 12a863499..f3d65cdf0 100644 --- a/crates/ra_db/src/input.rs +++ b/crates/ra_db/src/input.rs @@ -8,7 +8,7 @@ use std::{fmt, iter::FromIterator, ops, str::FromStr, sync::Arc}; -use ra_cfg::CfgOptions; +use cfg::CfgOptions; use rustc_hash::{FxHashMap, FxHashSet}; use syntax::SmolStr; use tt::TokenExpander; diff --git a/crates/ra_hir_def/Cargo.toml b/crates/ra_hir_def/Cargo.toml index ba7916c30..e7d3c4d5b 100644 --- a/crates/ra_hir_def/Cargo.toml +++ b/crates/ra_hir_def/Cargo.toml @@ -29,7 +29,7 @@ profile = { path = "../profile" } hir_expand = { path = "../ra_hir_expand", package = "ra_hir_expand" } test_utils = { path = "../test_utils" } mbe = { path = "../mbe" } -ra_cfg = { path = "../ra_cfg" } +cfg = { path = "../cfg" } tt = { path = "../tt" } [dev-dependencies] diff --git a/crates/ra_hir_def/src/adt.rs b/crates/ra_hir_def/src/adt.rs index c83219d77..d69ff2fc7 100644 --- a/crates/ra_hir_def/src/adt.rs +++ b/crates/ra_hir_def/src/adt.rs @@ -23,7 +23,7 @@ use crate::{ EnumId, HasModule, LocalEnumVariantId, LocalFieldId, Lookup, ModuleId, StructId, UnionId, VariantId, }; -use ra_cfg::CfgOptions; +use cfg::CfgOptions; /// Note that we use `StructData` for unions as well! #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/crates/ra_hir_def/src/attr.rs b/crates/ra_hir_def/src/attr.rs index 36dc8b816..1e5b06ca0 100644 --- a/crates/ra_hir_def/src/attr.rs +++ b/crates/ra_hir_def/src/attr.rs @@ -5,7 +5,7 @@ use std::{ops, sync::Arc}; use either::Either; use hir_expand::{hygiene::Hygiene, AstId, InFile}; use mbe::ast_to_token_tree; -use ra_cfg::{CfgExpr, CfgOptions}; +use cfg::{CfgExpr, CfgOptions}; use syntax::{ ast::{self, AstNode, AttrsOwner}, SmolStr, diff --git a/crates/ra_hir_def/src/body.rs b/crates/ra_hir_def/src/body.rs index 7c33966a7..3568513d1 100644 --- a/crates/ra_hir_def/src/body.rs +++ b/crates/ra_hir_def/src/body.rs @@ -9,7 +9,7 @@ use arena::{map::ArenaMap, Arena}; use drop_bomb::DropBomb; use either::Either; use hir_expand::{ast_id_map::AstIdMap, hygiene::Hygiene, AstId, HirFileId, InFile, MacroDefId}; -use ra_cfg::CfgOptions; +use cfg::CfgOptions; use ra_db::CrateId; use rustc_hash::FxHashMap; use syntax::{ast, AstNode, AstPtr}; diff --git a/crates/ra_hir_def/src/nameres/collector.rs b/crates/ra_hir_def/src/nameres/collector.rs index f7270ec91..fa2dadfc5 100644 --- a/crates/ra_hir_def/src/nameres/collector.rs +++ b/crates/ra_hir_def/src/nameres/collector.rs @@ -11,7 +11,7 @@ use hir_expand::{ proc_macro::ProcMacroExpander, HirFileId, MacroCallId, MacroDefId, MacroDefKind, }; -use ra_cfg::CfgOptions; +use cfg::CfgOptions; use ra_db::{CrateId, FileId, ProcMacroId}; use rustc_hash::FxHashMap; use syntax::ast; diff --git a/crates/ra_ide/Cargo.toml b/crates/ra_ide/Cargo.toml index 8e0fa5917..c60e55545 100644 --- a/crates/ra_ide/Cargo.toml +++ b/crates/ra_ide/Cargo.toml @@ -25,7 +25,7 @@ syntax = { path = "../syntax" } text_edit = { path = "../text_edit" } ra_db = { path = "../ra_db" } ra_ide_db = { path = "../ra_ide_db" } -ra_cfg = { path = "../ra_cfg" } +cfg = { path = "../cfg" } ra_fmt = { path = "../ra_fmt" } profile = { path = "../profile" } test_utils = { path = "../test_utils" } diff --git a/crates/ra_ide/src/lib.rs b/crates/ra_ide/src/lib.rs index 20967ba99..1fdf17800 100644 --- a/crates/ra_ide/src/lib.rs +++ b/crates/ra_ide/src/lib.rs @@ -47,7 +47,7 @@ mod typing; use std::sync::Arc; -use ra_cfg::CfgOptions; +use cfg::CfgOptions; use ra_db::{ salsa::{self, ParallelDatabase}, CheckCanceled, Env, FileLoader, FileSet, SourceDatabase, VfsPath, diff --git a/crates/ra_ide/src/mock_analysis.rs b/crates/ra_ide/src/mock_analysis.rs index c7e0f4b58..a4691f028 100644 --- a/crates/ra_ide/src/mock_analysis.rs +++ b/crates/ra_ide/src/mock_analysis.rs @@ -1,7 +1,7 @@ //! FIXME: write short doc here use std::sync::Arc; -use ra_cfg::CfgOptions; +use cfg::CfgOptions; use ra_db::{CrateName, FileSet, SourceRoot, VfsPath}; use test_utils::{ extract_annotations, extract_range_or_offset, Fixture, RangeOrOffset, CURSOR_MARKER, diff --git a/crates/ra_ide/src/parent_module.rs b/crates/ra_ide/src/parent_module.rs index 69af0c86a..b78388e6b 100644 --- a/crates/ra_ide/src/parent_module.rs +++ b/crates/ra_ide/src/parent_module.rs @@ -63,7 +63,7 @@ pub(crate) fn crate_for(db: &RootDatabase, file_id: FileId) -> Vec { #[cfg(test)] mod tests { - use ra_cfg::CfgOptions; + use cfg::CfgOptions; use ra_db::Env; use test_utils::mark; diff --git a/crates/ra_ide/src/runnables.rs b/crates/ra_ide/src/runnables.rs index 54cb3b309..7d8a210b7 100644 --- a/crates/ra_ide/src/runnables.rs +++ b/crates/ra_ide/src/runnables.rs @@ -2,7 +2,7 @@ use std::fmt; use hir::{AsAssocItem, Attrs, HirFileId, InFile, Semantics}; use itertools::Itertools; -use ra_cfg::CfgExpr; +use cfg::CfgExpr; use ra_ide_db::RootDatabase; use syntax::{ ast::{self, AstNode, AttrsOwner, DocCommentsOwner, ModuleItemOwner, NameOwner}, diff --git a/crates/ra_project_model/Cargo.toml b/crates/ra_project_model/Cargo.toml index 171fe8626..52f2d57b3 100644 --- a/crates/ra_project_model/Cargo.toml +++ b/crates/ra_project_model/Cargo.toml @@ -15,7 +15,7 @@ rustc-hash = "1.1.0" cargo_metadata = "0.11.1" arena = { path = "../arena" } -ra_cfg = { path = "../ra_cfg" } +cfg = { path = "../cfg" } ra_db = { path = "../ra_db" } toolchain = { path = "../toolchain" } ra_proc_macro = { path = "../ra_proc_macro" } diff --git a/crates/ra_project_model/src/cfg_flag.rs b/crates/ra_project_model/src/cfg_flag.rs index bd50056c6..e92962cf6 100644 --- a/crates/ra_project_model/src/cfg_flag.rs +++ b/crates/ra_project_model/src/cfg_flag.rs @@ -3,7 +3,7 @@ //! rustc main.rs --cfg foo --cfg 'feature="bar"' use std::str::FromStr; -use ra_cfg::CfgOptions; +use cfg::CfgOptions; use stdx::split_once; #[derive(Clone, Eq, PartialEq, Debug)] diff --git a/crates/ra_project_model/src/lib.rs b/crates/ra_project_model/src/lib.rs index 46f44910c..47e7d2420 100644 --- a/crates/ra_project_model/src/lib.rs +++ b/crates/ra_project_model/src/lib.rs @@ -13,7 +13,7 @@ use std::{ use anyhow::{bail, Context, Result}; use paths::{AbsPath, AbsPathBuf}; -use ra_cfg::CfgOptions; +use cfg::CfgOptions; use ra_db::{CrateGraph, CrateId, CrateName, Edition, Env, FileId}; use rustc_hash::{FxHashMap, FxHashSet}; diff --git a/crates/rust-analyzer/Cargo.toml b/crates/rust-analyzer/Cargo.toml index 156f8d538..440b1cd13 100644 --- a/crates/rust-analyzer/Cargo.toml +++ b/crates/rust-analyzer/Cargo.toml @@ -42,7 +42,7 @@ syntax = { path = "../syntax" } text_edit = { path = "../text_edit" } vfs = { path = "../vfs" } vfs-notify = { path = "../vfs-notify" } -ra_cfg = { path = "../ra_cfg" } +cfg = { path = "../cfg" } toolchain = { path = "../toolchain" } # This should only be used in CLI diff --git a/crates/rust-analyzer/src/cargo_target_spec.rs b/crates/rust-analyzer/src/cargo_target_spec.rs index 7929368c0..87ddd25bf 100644 --- a/crates/rust-analyzer/src/cargo_target_spec.rs +++ b/crates/rust-analyzer/src/cargo_target_spec.rs @@ -1,6 +1,6 @@ //! See `CargoTargetSpec` -use ra_cfg::CfgExpr; +use cfg::CfgExpr; use ra_ide::{FileId, RunnableKind, TestId}; use ra_project_model::{self, TargetKind}; use vfs::AbsPathBuf; @@ -178,7 +178,7 @@ mod tests { use super::*; use mbe::ast_to_token_tree; - use ra_cfg::CfgExpr; + use cfg::CfgExpr; use syntax::{ ast::{self, AstNode}, SmolStr, -- cgit v1.2.3 From 7d9480c6eb43b3ef1bd75ad26e99c14cca5cb366 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Thu, 13 Aug 2020 10:32:19 +0200 Subject: fmt --- crates/ra_hir_def/src/attr.rs | 2 +- crates/ra_hir_def/src/body.rs | 2 +- crates/ra_hir_def/src/nameres/collector.rs | 2 +- crates/ra_ide/src/runnables.rs | 2 +- crates/ra_project_model/src/lib.rs | 2 +- crates/rust-analyzer/src/cargo_target_spec.rs | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/ra_hir_def/src/attr.rs b/crates/ra_hir_def/src/attr.rs index 1e5b06ca0..dea552a60 100644 --- a/crates/ra_hir_def/src/attr.rs +++ b/crates/ra_hir_def/src/attr.rs @@ -2,10 +2,10 @@ use std::{ops, sync::Arc}; +use cfg::{CfgExpr, CfgOptions}; use either::Either; use hir_expand::{hygiene::Hygiene, AstId, InFile}; use mbe::ast_to_token_tree; -use cfg::{CfgExpr, CfgOptions}; use syntax::{ ast::{self, AstNode, AttrsOwner}, SmolStr, diff --git a/crates/ra_hir_def/src/body.rs b/crates/ra_hir_def/src/body.rs index 3568513d1..fe659386a 100644 --- a/crates/ra_hir_def/src/body.rs +++ b/crates/ra_hir_def/src/body.rs @@ -6,10 +6,10 @@ pub mod scope; use std::{mem, ops::Index, sync::Arc}; use arena::{map::ArenaMap, Arena}; +use cfg::CfgOptions; use drop_bomb::DropBomb; use either::Either; use hir_expand::{ast_id_map::AstIdMap, hygiene::Hygiene, AstId, HirFileId, InFile, MacroDefId}; -use cfg::CfgOptions; use ra_db::CrateId; use rustc_hash::FxHashMap; use syntax::{ast, AstNode, AstPtr}; diff --git a/crates/ra_hir_def/src/nameres/collector.rs b/crates/ra_hir_def/src/nameres/collector.rs index fa2dadfc5..6a5891936 100644 --- a/crates/ra_hir_def/src/nameres/collector.rs +++ b/crates/ra_hir_def/src/nameres/collector.rs @@ -3,6 +3,7 @@ //! `DefCollector::collect` contains the fixed-point iteration loop which //! resolves imports and expands macros. +use cfg::CfgOptions; use hir_expand::{ ast_id_map::FileAstId, builtin_derive::find_builtin_derive, @@ -11,7 +12,6 @@ use hir_expand::{ proc_macro::ProcMacroExpander, HirFileId, MacroCallId, MacroDefId, MacroDefKind, }; -use cfg::CfgOptions; use ra_db::{CrateId, FileId, ProcMacroId}; use rustc_hash::FxHashMap; use syntax::ast; diff --git a/crates/ra_ide/src/runnables.rs b/crates/ra_ide/src/runnables.rs index 7d8a210b7..fb40762cf 100644 --- a/crates/ra_ide/src/runnables.rs +++ b/crates/ra_ide/src/runnables.rs @@ -1,8 +1,8 @@ use std::fmt; +use cfg::CfgExpr; use hir::{AsAssocItem, Attrs, HirFileId, InFile, Semantics}; use itertools::Itertools; -use cfg::CfgExpr; use ra_ide_db::RootDatabase; use syntax::{ ast::{self, AstNode, AttrsOwner, DocCommentsOwner, ModuleItemOwner, NameOwner}, diff --git a/crates/ra_project_model/src/lib.rs b/crates/ra_project_model/src/lib.rs index 47e7d2420..ee42198f3 100644 --- a/crates/ra_project_model/src/lib.rs +++ b/crates/ra_project_model/src/lib.rs @@ -12,8 +12,8 @@ use std::{ }; use anyhow::{bail, Context, Result}; -use paths::{AbsPath, AbsPathBuf}; use cfg::CfgOptions; +use paths::{AbsPath, AbsPathBuf}; use ra_db::{CrateGraph, CrateId, CrateName, Edition, Env, FileId}; use rustc_hash::{FxHashMap, FxHashSet}; diff --git a/crates/rust-analyzer/src/cargo_target_spec.rs b/crates/rust-analyzer/src/cargo_target_spec.rs index 87ddd25bf..9c7a9cce6 100644 --- a/crates/rust-analyzer/src/cargo_target_spec.rs +++ b/crates/rust-analyzer/src/cargo_target_spec.rs @@ -177,8 +177,8 @@ fn required_features(cfg_expr: &CfgExpr, features: &mut Vec) { mod tests { use super::*; - use mbe::ast_to_token_tree; use cfg::CfgExpr; + use mbe::ast_to_token_tree; use syntax::{ ast::{self, AstNode}, SmolStr, -- cgit v1.2.3