Fix panic with EOF token

3 files changed, 22 insertions, 12 deletions

diff --git a/src/parser_api.rs b/src/parser_api.rs
index 58002a8fc..c78c6e43a 100644
--- a/src/parser_api.rs
+++ b/src/parser_api.rs
@@ -29,6 +29,15 @@ macro_rules! token_set_union {
     ($($ts:expr),* ,) => { token_set_union!($($ts),*) };
 }
 
+#[test]
+fn token_set_works_for_tokens() {
+    use SyntaxKind::*;
+    let ts = token_set! { EOF, SHEBANG };
+    assert!(ts.contains(EOF));
+    assert!(ts.contains(SHEBANG));
+    assert!(!ts.contains(PLUS));
+}
+
 /// `Parser` struct provides the low-level API for
 /// navigating through the stream of tokens and
 /// constructing the parse tree. The actual parsing
diff --git a/src/syntax_kinds/generated.rs b/src/syntax_kinds/generated.rs
index 9a9c1f223..655ec4ea5 100644
--- a/src/syntax_kinds/generated.rs
+++ b/src/syntax_kinds/generated.rs
@@ -5,6 +5,12 @@ use super::SyntaxInfo;
 /// The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT_DEF`.
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum SyntaxKind {
+    // Technical SyntaxKinds: they appear temporally during parsing,
+    // but never end up in the final tree
+    #[doc(hidden)]
+    TOMBSTONE,
+    #[doc(hidden)]
+    EOF,
     SEMI,
     COMMA,
     L_PAREN,
@@ -192,12 +198,6 @@ pub enum SyntaxKind {
     PARAM,
     SELF_PARAM,
     ARG_LIST,
-    // Technical SyntaxKinds: they appear temporally during parsing,
-    // but never end up in the final tree
-    #[doc(hidden)]
-    TOMBSTONE,
-    #[doc(hidden)]
-    EOF,
 }
 use self::SyntaxKind::*;
 
diff --git a/src/syntax_kinds/generated.rs.tera b/src/syntax_kinds/generated.rs.tera
index 4e2ee56c3..197f720e1 100644
--- a/src/syntax_kinds/generated.rs.tera
+++ b/src/syntax_kinds/generated.rs.tera
@@ -5,6 +5,13 @@ use super::SyntaxInfo;
 /// The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT_DEF`.
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum SyntaxKind {
+    // Technical SyntaxKinds: they appear temporally during parsing,
+    // but never end up in the final tree
+    #[doc(hidden)]
+    TOMBSTONE,
+    #[doc(hidden)]
+    EOF,
+
 {%- for t in concat(a=single_byte_tokens, b=multi_byte_tokens) %}
     {{t.1}},
 {%- endfor -%}
@@ -14,12 +21,6 @@ pub enum SyntaxKind {
 {% for t in concat(a=tokens, b=nodes) %}
     {{t}},
 {%- endfor %}
-    // Technical SyntaxKinds: they appear temporally during parsing,
-    // but never end up in the final tree
-    #[doc(hidden)]
-    TOMBSTONE,
-    #[doc(hidden)]
-    EOF,
 }
 use self::SyntaxKind::*;