Merge #1902

1902: simplify r=matklad a=matklad



Co-authored-by: Aleksey Kladov <[email protected]>

4 files changed, 25 insertions, 39 deletions

diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs
index 9d6d0133f..cf7458905 100644
--- a/crates/ra_mbe/src/subtree_source.rs
+++ b/crates/ra_mbe/src/subtree_source.rs
@@ -148,15 +148,7 @@ fn convert_ident(ident: &tt::Ident) -> TtToken {
 }
 
 fn convert_punct(p: tt::Punct) -> TtToken {
-    let kind = match p.char {
-        // lexer may produce compound tokens for these ones
-        '.' => T![.],
-        ':' => T![:],
-        '=' => T![=],
-        '!' => T![!],
-        '-' => T![-],
-        c => SyntaxKind::from_char(c).unwrap(),
-    };
+    let kind = SyntaxKind::from_char(p.char).unwrap();
     let text = {
         let mut buf = [0u8; 4];
         let s: &str = p.char.encode_utf8(&mut buf);
diff --git a/crates/ra_parser/src/syntax_kind/generated.rs b/crates/ra_parser/src/syntax_kind/generated.rs
index 23eb3c9cb..8b43d93fe 100644
--- a/crates/ra_parser/src/syntax_kind/generated.rs
+++ b/crates/ra_parser/src/syntax_kind/generated.rs
@@ -342,6 +342,11 @@ impl SyntaxKind {
             '^' => CARET,
             '%' => PERCENT,
             '_' => UNDERSCORE,
+            '.' => DOT,
+            ':' => COLON,
+            '=' => EQ,
+            '!' => EXCL,
+            '-' => MINUS,
             _ => return None,
         };
         Some(tok)
diff --git a/crates/ra_syntax/src/grammar.ron b/crates/ra_syntax/src/grammar.ron
index da9de2214..5f395501a 100644
--- a/crates/ra_syntax/src/grammar.ron
+++ b/crates/ra_syntax/src/grammar.ron
@@ -1,7 +1,7 @@
 // Stores definitions which must be used in multiple places
 // See `cargo gen-syntax` (defined in crates/tools/src/main.rs)
 Grammar(
-    single_byte_tokens: [
+    punct: [
         (";", "SEMI"),
         (",", "COMMA"),
         ("(", "L_PAREN"),
@@ -25,9 +25,6 @@ Grammar(
         ("^", "CARET"),
         ("%", "PERCENT"),
         ("_", "UNDERSCORE"),
-    ],
-    // Tokens for which the longest match must be chosen (e.g. `..` is a DOTDOT, but `.` is a DOT)
-    multi_byte_tokens: [
         (".", "DOT"),
         ("..", "DOTDOT"),
         ("...", "DOTDOTDOT"),
diff --git a/crates/ra_tools/src/boilerplate_gen.rs b/crates/ra_tools/src/boilerplate_gen.rs
index 578f13a3e..1d112c0af 100644
--- a/crates/ra_tools/src/boilerplate_gen.rs
+++ b/crates/ra_tools/src/boilerplate_gen.rs
@@ -160,31 +160,24 @@ fn generate_ast(grammar: &Grammar) -> Result<String> {
 }
 
 fn generate_syntax_kinds(grammar: &Grammar) -> Result<String> {
-    let single_byte_tokens_values =
-        grammar.single_byte_tokens.iter().map(|(token, _name)| token.chars().next().unwrap());
-    let single_byte_tokens = grammar
-        .single_byte_tokens
+    let (single_byte_tokens_values, single_byte_tokens): (Vec<_>, Vec<_>) = grammar
+        .punct
         .iter()
-        .map(|(_token, name)| format_ident!("{}", name))
-        .collect::<Vec<_>>();
-
-    let punctuation_values =
-        grammar.single_byte_tokens.iter().chain(grammar.multi_byte_tokens.iter()).map(
-            |(token, _name)| {
-                if "{}[]()".contains(token) {
-                    let c = token.chars().next().unwrap();
-                    quote! { #c }
-                } else {
-                    let cs = token.chars().map(|c| Punct::new(c, Spacing::Joint));
-                    quote! { #(#cs)* }
-                }
-            },
-        );
-    let punctuation = single_byte_tokens
-        .clone()
-        .into_iter()
-        .chain(grammar.multi_byte_tokens.iter().map(|(_token, name)| format_ident!("{}", name)))
-        .collect::<Vec<_>>();
+        .filter(|(token, _name)| token.len() == 1)
+        .map(|(token, name)| (token.chars().next().unwrap(), format_ident!("{}", name)))
+        .unzip();
+
+    let punctuation_values = grammar.punct.iter().map(|(token, _name)| {
+        if "{}[]()".contains(token) {
+            let c = token.chars().next().unwrap();
+            quote! { #c }
+        } else {
+            let cs = token.chars().map(|c| Punct::new(c, Spacing::Joint));
+            quote! { #(#cs)* }
+        }
+    });
+    let punctuation =
+        grammar.punct.iter().map(|(_token, name)| format_ident!("{}", name)).collect::<Vec<_>>();
 
     let full_keywords_values = &grammar.keywords;
     let full_keywords =
@@ -294,8 +287,7 @@ fn reformat(text: impl std::fmt::Display) -> Result<String> {
 
 #[derive(Deserialize, Debug)]
 struct Grammar {
-    single_byte_tokens: Vec<(String, String)>,
-    multi_byte_tokens: Vec<(String, String)>,
+    punct: Vec<(String, String)>,
     keywords: Vec<String>,
     contextual_keywords: Vec<String>,
     literals: Vec<String>,