Merge #59

59: Moved TokenSet into it's own file. r=matklad a=Plasticcaz

As discussed in Issue #11, the only thing left in that issue that hasn't been fixed appears to be that TokenSet is not in it's own file. This pull request pulls TokenSet, it's macros and it's test into it's own file.

Co-authored-by: Zac Winter <[email protected]>

4 files changed, 41 insertions, 37 deletions

diff --git a/crates/libsyntax2/src/grammar/mod.rs b/crates/libsyntax2/src/grammar/mod.rs
index 339664af3..e19805b9d 100644
--- a/crates/libsyntax2/src/grammar/mod.rs
+++ b/crates/libsyntax2/src/grammar/mod.rs
@@ -32,7 +32,8 @@ mod type_params;
 mod types;
 
 use {
-    parser_api::{Marker, CompletedMarker, Parser, TokenSet},
+    token_set::TokenSet,
+    parser_api::{Marker, CompletedMarker, Parser},
     SyntaxKind::{self, *},
 };
 pub(crate) use self::{
diff --git a/crates/libsyntax2/src/lib.rs b/crates/libsyntax2/src/lib.rs
index 3b5115110..7a30f5d38 100644
--- a/crates/libsyntax2/src/lib.rs
+++ b/crates/libsyntax2/src/lib.rs
@@ -31,6 +31,7 @@ pub mod algo;
 pub mod ast;
 mod lexer;
 #[macro_use]
+mod token_set;
 mod parser_api;
 mod grammar;
 mod parser_impl;
diff --git a/crates/libsyntax2/src/parser_api.rs b/crates/libsyntax2/src/parser_api.rs
index 9bc58e7f7..c4753140e 100644
--- a/crates/libsyntax2/src/parser_api.rs
+++ b/crates/libsyntax2/src/parser_api.rs
@@ -1,45 +1,10 @@
 use {
+    token_set::TokenSet,
     parser_impl::ParserImpl,
     SyntaxKind::{self, ERROR},
     drop_bomb::DropBomb,
 };
 
-#[derive(Clone, Copy)]
-pub(crate) struct TokenSet(pub(crate) u128);
-
-fn mask(kind: SyntaxKind) -> u128 {
-    1u128 << (kind as usize)
-}
-
-impl TokenSet {
-    pub const EMPTY: TokenSet = TokenSet(0);
-
-    pub fn contains(&self, kind: SyntaxKind) -> bool {
-        self.0 & mask(kind) != 0
-    }
-}
-
-#[macro_export]
-macro_rules! token_set {
-    ($($t:ident),*) => { TokenSet($(1u128 << ($t as usize))|*) };
-    ($($t:ident),* ,) => { token_set!($($t),*) };
-}
-
-#[macro_export]
-macro_rules! token_set_union {
-    ($($ts:expr),*) => { TokenSet($($ts.0)|*) };
-    ($($ts:expr),* ,) => { token_set_union!($($ts),*) };
-}
-
-#[test]
-fn token_set_works_for_tokens() {
-    use SyntaxKind::*;
-    let ts = token_set! { EOF, SHEBANG };
-    assert!(ts.contains(EOF));
-    assert!(ts.contains(SHEBANG));
-    assert!(!ts.contains(PLUS));
-}
-
 /// `Parser` struct provides the low-level API for
 /// navigating through the stream of tokens and
 /// constructing the parse tree. The actual parsing
diff --git a/crates/libsyntax2/src/token_set.rs b/crates/libsyntax2/src/token_set.rs
new file mode 100644
index 000000000..c83fba81b
--- /dev/null
+++ b/crates/libsyntax2/src/token_set.rs
@@ -0,0 +1,37 @@
+use SyntaxKind;
+
+#[derive(Clone, Copy)]
+pub(crate) struct TokenSet(pub(crate) u128);
+
+fn mask(kind: SyntaxKind) -> u128 {
+    1u128 << (kind as usize)
+}
+
+impl TokenSet {
+    pub const EMPTY: TokenSet = TokenSet(0);
+
+    pub fn contains(&self, kind: SyntaxKind) -> bool {
+        self.0 & mask(kind) != 0
+    }
+}
+
+#[macro_export]
+macro_rules! token_set {
+    ($($t:ident),*) => { TokenSet($(1u128 << ($t as usize))|*) };
+    ($($t:ident),* ,) => { token_set!($($t),*) };
+}
+
+#[macro_export]
+macro_rules! token_set_union {
+    ($($ts:expr),*) => { TokenSet($($ts.0)|*) };
+    ($($ts:expr),* ,) => { token_set_union!($($ts),*) };
+}
+
+#[test]
+fn token_set_works_for_tokens() {
+    use SyntaxKind::*;
+    let ts = token_set! { EOF, SHEBANG };
+    assert!(ts.contains(EOF));
+    assert!(ts.contains(SHEBANG));
+    assert!(!ts.contains(PLUS));
+}