aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorZac Winter <[email protected]>2018-09-06 14:54:54 +0100
committerZac Winter <[email protected]>2018-09-06 14:57:04 +0100
commit518cc87496494d639f0a6bf189f1a3567bcfa328 (patch)
treeac9a756c5cfc83fed671f57e20a2f05d8b00e539
parent751562d2f77a7bc3eeeffbf8e53c9f3515fa9653 (diff)
Moved TokenSet into it's own file.
-rw-r--r--crates/libsyntax2/src/grammar/mod.rs3
-rw-r--r--crates/libsyntax2/src/lib.rs1
-rw-r--r--crates/libsyntax2/src/parser_api.rs37
-rw-r--r--crates/libsyntax2/src/token_set.rs37
4 files changed, 41 insertions, 37 deletions
diff --git a/crates/libsyntax2/src/grammar/mod.rs b/crates/libsyntax2/src/grammar/mod.rs
index 339664af3..e19805b9d 100644
--- a/crates/libsyntax2/src/grammar/mod.rs
+++ b/crates/libsyntax2/src/grammar/mod.rs
@@ -32,7 +32,8 @@ mod type_params;
32mod types; 32mod types;
33 33
34use { 34use {
35 parser_api::{Marker, CompletedMarker, Parser, TokenSet}, 35 token_set::TokenSet,
36 parser_api::{Marker, CompletedMarker, Parser},
36 SyntaxKind::{self, *}, 37 SyntaxKind::{self, *},
37}; 38};
38pub(crate) use self::{ 39pub(crate) use self::{
diff --git a/crates/libsyntax2/src/lib.rs b/crates/libsyntax2/src/lib.rs
index 3b5115110..7a30f5d38 100644
--- a/crates/libsyntax2/src/lib.rs
+++ b/crates/libsyntax2/src/lib.rs
@@ -31,6 +31,7 @@ pub mod algo;
31pub mod ast; 31pub mod ast;
32mod lexer; 32mod lexer;
33#[macro_use] 33#[macro_use]
34mod token_set;
34mod parser_api; 35mod parser_api;
35mod grammar; 36mod grammar;
36mod parser_impl; 37mod parser_impl;
diff --git a/crates/libsyntax2/src/parser_api.rs b/crates/libsyntax2/src/parser_api.rs
index 9bc58e7f7..c4753140e 100644
--- a/crates/libsyntax2/src/parser_api.rs
+++ b/crates/libsyntax2/src/parser_api.rs
@@ -1,45 +1,10 @@
1use { 1use {
2 token_set::TokenSet,
2 parser_impl::ParserImpl, 3 parser_impl::ParserImpl,
3 SyntaxKind::{self, ERROR}, 4 SyntaxKind::{self, ERROR},
4 drop_bomb::DropBomb, 5 drop_bomb::DropBomb,
5}; 6};
6 7
7#[derive(Clone, Copy)]
8pub(crate) struct TokenSet(pub(crate) u128);
9
10fn mask(kind: SyntaxKind) -> u128 {
11 1u128 << (kind as usize)
12}
13
14impl TokenSet {
15 pub const EMPTY: TokenSet = TokenSet(0);
16
17 pub fn contains(&self, kind: SyntaxKind) -> bool {
18 self.0 & mask(kind) != 0
19 }
20}
21
22#[macro_export]
23macro_rules! token_set {
24 ($($t:ident),*) => { TokenSet($(1u128 << ($t as usize))|*) };
25 ($($t:ident),* ,) => { token_set!($($t),*) };
26}
27
28#[macro_export]
29macro_rules! token_set_union {
30 ($($ts:expr),*) => { TokenSet($($ts.0)|*) };
31 ($($ts:expr),* ,) => { token_set_union!($($ts),*) };
32}
33
34#[test]
35fn token_set_works_for_tokens() {
36 use SyntaxKind::*;
37 let ts = token_set! { EOF, SHEBANG };
38 assert!(ts.contains(EOF));
39 assert!(ts.contains(SHEBANG));
40 assert!(!ts.contains(PLUS));
41}
42
43/// `Parser` struct provides the low-level API for 8/// `Parser` struct provides the low-level API for
44/// navigating through the stream of tokens and 9/// navigating through the stream of tokens and
45/// constructing the parse tree. The actual parsing 10/// constructing the parse tree. The actual parsing
diff --git a/crates/libsyntax2/src/token_set.rs b/crates/libsyntax2/src/token_set.rs
new file mode 100644
index 000000000..c83fba81b
--- /dev/null
+++ b/crates/libsyntax2/src/token_set.rs
@@ -0,0 +1,37 @@
1use SyntaxKind;
2
3#[derive(Clone, Copy)]
4pub(crate) struct TokenSet(pub(crate) u128);
5
6fn mask(kind: SyntaxKind) -> u128 {
7 1u128 << (kind as usize)
8}
9
10impl TokenSet {
11 pub const EMPTY: TokenSet = TokenSet(0);
12
13 pub fn contains(&self, kind: SyntaxKind) -> bool {
14 self.0 & mask(kind) != 0
15 }
16}
17
18#[macro_export]
19macro_rules! token_set {
20 ($($t:ident),*) => { TokenSet($(1u128 << ($t as usize))|*) };
21 ($($t:ident),* ,) => { token_set!($($t),*) };
22}
23
24#[macro_export]
25macro_rules! token_set_union {
26 ($($ts:expr),*) => { TokenSet($($ts.0)|*) };
27 ($($ts:expr),* ,) => { token_set_union!($($ts),*) };
28}
29
30#[test]
31fn token_set_works_for_tokens() {
32 use SyntaxKind::*;
33 let ts = token_set! { EOF, SHEBANG };
34 assert!(ts.contains(EOF));
35 assert!(ts.contains(SHEBANG));
36 assert!(!ts.contains(PLUS));
37}