Merge #59

59: Moved TokenSet into it's own file. r=matklad a=Plasticcaz

As discussed in Issue #11, the only thing left in that issue that hasn't been fixed appears to be that TokenSet is not in it's own file. This pull request pulls TokenSet, it's macros and it's test into it's own file.

Co-authored-by: Zac Winter <[email protected]>

1 files changed, 1 insertions, 36 deletions

diff --git a/crates/libsyntax2/src/parser_api.rs b/crates/libsyntax2/src/parser_api.rs
index 9bc58e7f7..c4753140e 100644
--- a/crates/libsyntax2/src/parser_api.rs
+++ b/crates/libsyntax2/src/parser_api.rs
@@ -1,45 +1,10 @@
 use {
+    token_set::TokenSet,
     parser_impl::ParserImpl,
     SyntaxKind::{self, ERROR},
     drop_bomb::DropBomb,
 };
 
-#[derive(Clone, Copy)]
-pub(crate) struct TokenSet(pub(crate) u128);
-
-fn mask(kind: SyntaxKind) -> u128 {
-    1u128 << (kind as usize)
-}
-
-impl TokenSet {
-    pub const EMPTY: TokenSet = TokenSet(0);
-
-    pub fn contains(&self, kind: SyntaxKind) -> bool {
-        self.0 & mask(kind) != 0
-    }
-}
-
-#[macro_export]
-macro_rules! token_set {
-    ($($t:ident),*) => { TokenSet($(1u128 << ($t as usize))|*) };
-    ($($t:ident),* ,) => { token_set!($($t),*) };
-}
-
-#[macro_export]
-macro_rules! token_set_union {
-    ($($ts:expr),*) => { TokenSet($($ts.0)|*) };
-    ($($ts:expr),* ,) => { token_set_union!($($ts),*) };
-}
-
-#[test]
-fn token_set_works_for_tokens() {
-    use SyntaxKind::*;
-    let ts = token_set! { EOF, SHEBANG };
-    assert!(ts.contains(EOF));
-    assert!(ts.contains(SHEBANG));
-    assert!(!ts.contains(PLUS));
-}
-
 /// `Parser` struct provides the low-level API for
 /// navigating through the stream of tokens and
 /// constructing the parse tree. The actual parsing