4 files changed, 127 insertions, 20 deletions

diff --git a/crates/ra_hir/src/macros.rs b/crates/ra_hir/src/macros.rs
index 9ba381685..c23ad53cc 100644
--- a/crates/ra_hir/src/macros.rs
+++ b/crates/ra_hir/src/macros.rs
@@ -221,12 +221,24 @@ fn convert_tt(tt: &SyntaxNode) -> Option<tt::Subtree> {
         if child == first_child || child == last_child || child.kind().is_trivia() {
             continue;
         }
-        let child = if child.kind() == TOKEN_TREE {
+        let child: tt::TokenTree = if child.kind() == TOKEN_TREE {
             convert_tt(child)?.into()
-        } else if child.kind().is_keyword() {
+        } else if child.kind().is_keyword() || child.kind() == IDENT {
             let text = child.leaf_text().unwrap().clone();
             tt::Leaf::from(tt::Ident { text }).into()
+        } else if child.kind().is_punct() {
+            // FIXME: multibyte tokens
+            tt::Leaf::from(tt::Punct {
+                char: child.text().char_at(0)?,
+            })
+            .into()
+        } else if child.kind().is_literal() {
+            tt::Leaf::from(tt::Literal {
+                text: child.leaf_text().unwrap().clone(),
+            })
+            .into()
         } else {
+            log::error!("unknown kind: {:?}", child);
             return None;
         };
         token_trees.push(child)
diff --git a/crates/ra_syntax/src/grammar.ron b/crates/ra_syntax/src/grammar.ron
index d4c863705..47334bdf0 100644
--- a/crates/ra_syntax/src/grammar.ron
+++ b/crates/ra_syntax/src/grammar.ron
@@ -24,6 +24,7 @@ Grammar(
         ["/", "SLASH"],
         ["^", "CARET"],
         ["%", "PERCENT"],
+        ["_", "UNDERSCORE"],
     ],
     // Tokens for which the longest match must be chosen (e.g. `..` is a DOTDOT, but `.` is a DOT)
     multi_byte_tokens: [
@@ -99,20 +100,21 @@ Grammar(
         "default",
         "union",
     ],
-    tokens: [
-        "ERROR",
-        "IDENT",
-        "UNDERSCORE",
-        "WHITESPACE",
+    literals: [
         "INT_NUMBER",
         "FLOAT_NUMBER",
-        "LIFETIME",
         "CHAR",
         "BYTE",
         "STRING",
         "RAW_STRING",
         "BYTE_STRING",
         "RAW_BYTE_STRING",
+    ],
+    tokens: [
+        "ERROR",
+        "IDENT",
+        "WHITESPACE",
+        "LIFETIME",
         "COMMENT",
         "SHEBANG",
     ],
diff --git a/crates/ra_syntax/src/syntax_kinds/generated.rs b/crates/ra_syntax/src/syntax_kinds/generated.rs
index 14f610891..fea513458 100644
--- a/crates/ra_syntax/src/syntax_kinds/generated.rs
+++ b/crates/ra_syntax/src/syntax_kinds/generated.rs
@@ -36,6 +36,7 @@ pub enum SyntaxKind {
     SLASH,
     CARET,
     PERCENT,
+    UNDERSCORE,
     DOT,
     DOTDOT,
     DOTDOTDOT,
@@ -103,19 +104,18 @@ pub enum SyntaxKind {
     AUTO_KW,
     DEFAULT_KW,
     UNION_KW,
-    ERROR,
-    IDENT,
-    UNDERSCORE,
-    WHITESPACE,
     INT_NUMBER,
     FLOAT_NUMBER,
-    LIFETIME,
     CHAR,
     BYTE,
     STRING,
     RAW_STRING,
     BYTE_STRING,
     RAW_BYTE_STRING,
+    ERROR,
+    IDENT,
+    WHITESPACE,
+    LIFETIME,
     COMMENT,
     SHEBANG,
     SOURCE_FILE,
@@ -275,6 +275,79 @@ impl SyntaxKind {
         }
     }
 
+    pub fn is_punct(self) -> bool {
+        match self {
+            | SEMI
+            | COMMA
+            | L_PAREN
+            | R_PAREN
+            | L_CURLY
+            | R_CURLY
+            | L_BRACK
+            | R_BRACK
+            | L_ANGLE
+            | R_ANGLE
+            | AT
+            | POUND
+            | TILDE
+            | QUESTION
+            | DOLLAR
+            | AMP
+            | PIPE
+            | PLUS
+            | STAR
+            | SLASH
+            | CARET
+            | PERCENT
+            | UNDERSCORE
+            | DOT
+            | DOTDOT
+            | DOTDOTDOT
+            | DOTDOTEQ
+            | COLON
+            | COLONCOLON
+            | EQ
+            | EQEQ
+            | FAT_ARROW
+            | EXCL
+            | NEQ
+            | MINUS
+            | THIN_ARROW
+            | LTEQ
+            | GTEQ
+            | PLUSEQ
+            | MINUSEQ
+            | PIPEEQ
+            | AMPEQ
+            | CARETEQ
+            | SLASHEQ
+            | STAREQ
+            | PERCENTEQ
+            | AMPAMP
+            | PIPEPIPE
+            | SHL
+            | SHR
+            | SHLEQ
+            | SHREQ
+                => true,
+            _ => false
+        }
+    }
+    pub fn is_literal(self) -> bool {
+        match self {
+            | INT_NUMBER
+            | FLOAT_NUMBER
+            | CHAR
+            | BYTE
+            | STRING
+            | RAW_STRING
+            | BYTE_STRING
+            | RAW_BYTE_STRING
+                => true,
+            _ => false
+        }
+    }
+
     pub(crate) fn info(self) -> &'static SyntaxInfo {
         match self {
             SEMI => &SyntaxInfo { name: "SEMI" },
@@ -299,6 +372,7 @@ impl SyntaxKind {
             SLASH => &SyntaxInfo { name: "SLASH" },
             CARET => &SyntaxInfo { name: "CARET" },
             PERCENT => &SyntaxInfo { name: "PERCENT" },
+            UNDERSCORE => &SyntaxInfo { name: "UNDERSCORE" },
             DOT => &SyntaxInfo { name: "DOT" },
             DOTDOT => &SyntaxInfo { name: "DOTDOT" },
             DOTDOTDOT => &SyntaxInfo { name: "DOTDOTDOT" },
@@ -366,19 +440,18 @@ impl SyntaxKind {
             AUTO_KW => &SyntaxInfo { name: "AUTO_KW" },
             DEFAULT_KW => &SyntaxInfo { name: "DEFAULT_KW" },
             UNION_KW => &SyntaxInfo { name: "UNION_KW" },
-            ERROR => &SyntaxInfo { name: "ERROR" },
-            IDENT => &SyntaxInfo { name: "IDENT" },
-            UNDERSCORE => &SyntaxInfo { name: "UNDERSCORE" },
-            WHITESPACE => &SyntaxInfo { name: "WHITESPACE" },
             INT_NUMBER => &SyntaxInfo { name: "INT_NUMBER" },
             FLOAT_NUMBER => &SyntaxInfo { name: "FLOAT_NUMBER" },
-            LIFETIME => &SyntaxInfo { name: "LIFETIME" },
             CHAR => &SyntaxInfo { name: "CHAR" },
             BYTE => &SyntaxInfo { name: "BYTE" },
             STRING => &SyntaxInfo { name: "STRING" },
             RAW_STRING => &SyntaxInfo { name: "RAW_STRING" },
             BYTE_STRING => &SyntaxInfo { name: "BYTE_STRING" },
             RAW_BYTE_STRING => &SyntaxInfo { name: "RAW_BYTE_STRING" },
+            ERROR => &SyntaxInfo { name: "ERROR" },
+            IDENT => &SyntaxInfo { name: "IDENT" },
+            WHITESPACE => &SyntaxInfo { name: "WHITESPACE" },
+            LIFETIME => &SyntaxInfo { name: "LIFETIME" },
             COMMENT => &SyntaxInfo { name: "COMMENT" },
             SHEBANG => &SyntaxInfo { name: "SHEBANG" },
             SOURCE_FILE => &SyntaxInfo { name: "SOURCE_FILE" },
@@ -559,6 +632,7 @@ impl SyntaxKind {
             '/' => SLASH,
             '^' => CARET,
             '%' => PERCENT,
+            '_' => UNDERSCORE,
             _ => return None,
         };
         Some(tok)
diff --git a/crates/ra_syntax/src/syntax_kinds/generated.rs.tera b/crates/ra_syntax/src/syntax_kinds/generated.rs.tera
index 21f9444b1..837437136 100644
--- a/crates/ra_syntax/src/syntax_kinds/generated.rs.tera
+++ b/crates/ra_syntax/src/syntax_kinds/generated.rs.tera
@@ -23,7 +23,7 @@ pub enum SyntaxKind {
 {% for kw in concat(a=keywords, b=contextual_keywords) %}
     {{kw | upper}}_KW,
 {%- endfor -%}
-{% for t in concat(a=tokens, b=nodes) %}
+{% for t in concat(a=literals, b=tokens, c=nodes) %}
     {{t}},
 {%- endfor %}
 }
@@ -40,6 +40,25 @@ impl SyntaxKind {
         }
     }
 
+    pub fn is_punct(self) -> bool {
+        match self {
+{%- for t in concat(a=single_byte_tokens, b=multi_byte_tokens) %}
+            | {{t.1}}
+{%- endfor %}
+                => true,
+            _ => false
+        }
+    }
+    pub fn is_literal(self) -> bool {
+        match self {
+{%- for t in literals %}
+            | {{t}}
+{%- endfor %}
+                => true,
+            _ => false
+        }
+    }
+
     pub(crate) fn info(self) -> &'static SyntaxInfo {
         match self {
 {%- for t in concat(a=single_byte_tokens, b=multi_byte_tokens) %}
@@ -48,7 +67,7 @@ impl SyntaxKind {
 {% for kw in concat(a=keywords, b=contextual_keywords) %}
             {{kw | upper}}_KW => &SyntaxInfo { name: "{{kw | upper}}_KW" },
 {%- endfor -%}
-{% for t in concat(a=tokens, b=nodes) %}
+{% for t in concat(a=literals, b=tokens, c=nodes) %}
             {{t}} => &SyntaxInfo { name: "{{t}}" },
 {%- endfor %}
             TOMBSTONE => &SyntaxInfo { name: "TOMBSTONE" },