From 550d7fbe3cbf2af4a47fca6c9bbefaf798cd7b7b Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Wed, 12 Aug 2020 16:46:20 +0200 Subject: Rename ra_tt -> tt --- crates/tt/src/lib.rs | 246 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 246 insertions(+) create mode 100644 crates/tt/src/lib.rs (limited to 'crates/tt/src/lib.rs') diff --git a/crates/tt/src/lib.rs b/crates/tt/src/lib.rs new file mode 100644 index 000000000..20c3f5eab --- /dev/null +++ b/crates/tt/src/lib.rs @@ -0,0 +1,246 @@ +//! `tt` crate defines a `TokenTree` data structure: this is the interface (both +//! input and output) of macros. It closely mirrors `proc_macro` crate's +//! `TokenTree`. +use std::{ + fmt::{self, Debug}, + panic::RefUnwindSafe, +}; + +use stdx::impl_from; + +pub use smol_str::SmolStr; + +/// Represents identity of the token. +/// +/// For hygiene purposes, we need to track which expanded tokens originated from +/// which source tokens. We do it by assigning an distinct identity to each +/// source token and making sure that identities are preserved during macro +/// expansion. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct TokenId(pub u32); + +impl TokenId { + pub const fn unspecified() -> TokenId { + TokenId(!0) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum TokenTree { + Leaf(Leaf), + Subtree(Subtree), +} +impl_from!(Leaf, Subtree for TokenTree); + +impl TokenTree { + pub fn empty() -> Self { + TokenTree::Subtree(Subtree::default()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum Leaf { + Literal(Literal), + Punct(Punct), + Ident(Ident), +} +impl_from!(Literal, Punct, Ident for Leaf); + +#[derive(Clone, PartialEq, Eq, Hash, Default)] +pub struct Subtree { + pub delimiter: Option, + pub token_trees: Vec, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Delimiter { + pub id: TokenId, + pub kind: DelimiterKind, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum DelimiterKind { + Parenthesis, + Brace, + Bracket, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Literal { + pub text: SmolStr, + pub id: TokenId, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct Punct { + pub char: char, + pub spacing: Spacing, + pub id: TokenId, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Spacing { + Alone, + Joint, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Ident { + pub text: SmolStr, + pub id: TokenId, +} + +fn print_debug_subtree(f: &mut fmt::Formatter<'_>, subtree: &Subtree, level: usize) -> fmt::Result { + let align = std::iter::repeat(" ").take(level).collect::(); + + let aux = match subtree.delimiter.map(|it| (it.kind, it.id.0)) { + None => "$".to_string(), + Some((DelimiterKind::Parenthesis, id)) => format!("() {}", id), + Some((DelimiterKind::Brace, id)) => format!("{{}} {}", id), + Some((DelimiterKind::Bracket, id)) => format!("[] {}", id), + }; + + if subtree.token_trees.is_empty() { + write!(f, "{}SUBTREE {}", align, aux)?; + } else { + writeln!(f, "{}SUBTREE {}", align, aux)?; + for (idx, child) in subtree.token_trees.iter().enumerate() { + print_debug_token(f, child, level + 1)?; + if idx != subtree.token_trees.len() - 1 { + writeln!(f)?; + } + } + } + + Ok(()) +} + +fn print_debug_token(f: &mut fmt::Formatter<'_>, tkn: &TokenTree, level: usize) -> fmt::Result { + let align = std::iter::repeat(" ").take(level).collect::(); + + match tkn { + TokenTree::Leaf(leaf) => match leaf { + Leaf::Literal(lit) => write!(f, "{}LITERAL {} {}", align, lit.text, lit.id.0)?, + Leaf::Punct(punct) => write!( + f, + "{}PUNCH {} [{}] {}", + align, + punct.char, + if punct.spacing == Spacing::Alone { "alone" } else { "joint" }, + punct.id.0 + )?, + Leaf::Ident(ident) => write!(f, "{}IDENT {} {}", align, ident.text, ident.id.0)?, + }, + TokenTree::Subtree(subtree) => { + print_debug_subtree(f, subtree, level)?; + } + } + + Ok(()) +} + +impl Debug for Subtree { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + print_debug_subtree(f, self, 0) + } +} + +impl fmt::Display for TokenTree { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + TokenTree::Leaf(it) => fmt::Display::fmt(it, f), + TokenTree::Subtree(it) => fmt::Display::fmt(it, f), + } + } +} + +impl fmt::Display for Subtree { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let (l, r) = match self.delimiter_kind() { + Some(DelimiterKind::Parenthesis) => ("(", ")"), + Some(DelimiterKind::Brace) => ("{", "}"), + Some(DelimiterKind::Bracket) => ("[", "]"), + None => ("", ""), + }; + f.write_str(l)?; + let mut needs_space = false; + for tt in self.token_trees.iter() { + if needs_space { + f.write_str(" ")?; + } + needs_space = true; + match tt { + TokenTree::Leaf(Leaf::Punct(p)) => { + needs_space = p.spacing == Spacing::Alone; + fmt::Display::fmt(p, f)? + } + tt => fmt::Display::fmt(tt, f)?, + } + } + f.write_str(r)?; + Ok(()) + } +} + +impl fmt::Display for Leaf { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Leaf::Ident(it) => fmt::Display::fmt(it, f), + Leaf::Literal(it) => fmt::Display::fmt(it, f), + Leaf::Punct(it) => fmt::Display::fmt(it, f), + } + } +} + +impl fmt::Display for Ident { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(&self.text, f) + } +} + +impl fmt::Display for Literal { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(&self.text, f) + } +} + +impl fmt::Display for Punct { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Display::fmt(&self.char, f) + } +} + +impl Subtree { + /// Count the number of tokens recursively + pub fn count(&self) -> usize { + let children_count = self + .token_trees + .iter() + .map(|c| match c { + TokenTree::Subtree(c) => c.count(), + _ => 0, + }) + .sum::(); + + self.token_trees.len() + children_count + } + + pub fn delimiter_kind(&self) -> Option { + self.delimiter.map(|it| it.kind) + } +} + +pub mod buffer; + +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum ExpansionError { + IOError(String), + JsonError(String), + Unknown(String), + ExpansionError(String), +} + +pub trait TokenExpander: Debug + Send + Sync + RefUnwindSafe { + fn expand(&self, subtree: &Subtree, attrs: Option<&Subtree>) + -> Result; +} -- cgit v1.2.3