Merge #4925

4925: Syntax highlighting for escape sequences in strings r=matklad a=ltentrup

I have added a new semantic token type `ESCAPE_SEQUENCE` as the LSP specification does not seem to have an appropriate token type. This may actually be a regression for some users, as the TextMate Rust grammar has a scope `constant.character.escape.rust` which highlights escape sequences (which caused problems with semantic highlighting, see #4138).

Fixes #2604.

Co-authored-by: Leander Tentrup <[email protected]>

12 files changed, 38 insertions, 5 deletions

diff --git a/crates/ra_ide/src/snapshots/highlight_doctest.html b/crates/ra_ide/src/snapshots/highlight_doctest.html
index 13a5d1b12..f92a0aba5 100644
--- a/crates/ra_ide/src/snapshots/highlight_doctest.html
+++ b/crates/ra_ide/src/snapshots/highlight_doctest.html
@@ -26,6 +26,7 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .format_specifier   { color: #CC696B; }
 .mutable            { text-decoration: underline; }
 .unresolved_reference { color: #FC5555; }
+.escape_sequence    { color: #94BFF3; }
 
 .keyword            { color: #F0DFAF; font-weight: bold; }
 .keyword.unsafe     { color: #BC8383; font-weight: bold; }
diff --git a/crates/ra_ide/src/snapshots/highlight_injection.html b/crates/ra_ide/src/snapshots/highlight_injection.html
index e1c9d3523..47dbd7bc8 100644
--- a/crates/ra_ide/src/snapshots/highlight_injection.html
+++ b/crates/ra_ide/src/snapshots/highlight_injection.html
@@ -26,6 +26,7 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .format_specifier   { color: #CC696B; }
 .mutable            { text-decoration: underline; }
 .unresolved_reference { color: #FC5555; }
+.escape_sequence    { color: #94BFF3; }
 
 .keyword            { color: #F0DFAF; font-weight: bold; }
 .keyword.unsafe     { color: #BC8383; font-weight: bold; }
diff --git a/crates/ra_ide/src/snapshots/highlight_strings.html b/crates/ra_ide/src/snapshots/highlight_strings.html
index 666b48fd0..b46fa44c6 100644
--- a/crates/ra_ide/src/snapshots/highlight_strings.html
+++ b/crates/ra_ide/src/snapshots/highlight_strings.html
@@ -26,6 +26,7 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .format_specifier   { color: #CC696B; }
 .mutable            { text-decoration: underline; }
 .unresolved_reference { color: #FC5555; }
+.escape_sequence    { color: #94BFF3; }
 
 .keyword            { color: #F0DFAF; font-weight: bold; }
 .keyword.unsafe     { color: #BC8383; font-weight: bold; }
@@ -83,6 +84,10 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 
     <span class="macro">println!</span>(<span class="string_literal">r"Hello, </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">!"</span>, <span class="string_literal">"world"</span>);
 
-    <span class="macro">println!</span>(<span class="string_literal">"</span><span class="format_specifier">{</span><span class="variable">\x41</span><span class="format_specifier">}</span><span class="string_literal">"</span>, A = <span class="numeric_literal">92</span>);
+    <span class="comment">// escape sequences</span>
+    <span class="macro">println!</span>(<span class="string_literal">"Hello</span><span class="escape_sequence">\n</span><span class="string_literal">World"</span>);
+    <span class="macro">println!</span>(<span class="string_literal">"</span><span class="escape_sequence">\u{48}</span><span class="escape_sequence">\x65</span><span class="escape_sequence">\x6C</span><span class="escape_sequence">\x6C</span><span class="escape_sequence">\x6F</span><span class="string_literal"> World"</span>);
+
+    <span class="macro">println!</span>(<span class="string_literal">"</span><span class="format_specifier">{</span><span class="escape_sequence">\x41</span><span class="format_specifier">}</span><span class="string_literal">"</span>, A = <span class="numeric_literal">92</span>);
     <span class="macro">println!</span>(<span class="string_literal">"</span><span class="format_specifier">{</span><span class="variable">ничоси</span><span class="format_specifier">}</span><span class="string_literal">"</span>, ничоси = <span class="numeric_literal">92</span>);
 }</code></pre>
\ No newline at end of file
diff --git a/crates/ra_ide/src/snapshots/highlight_unsafe.html b/crates/ra_ide/src/snapshots/highlight_unsafe.html
index e1540499b..73438fbb4 100644
--- a/crates/ra_ide/src/snapshots/highlight_unsafe.html
+++ b/crates/ra_ide/src/snapshots/highlight_unsafe.html
@@ -26,6 +26,7 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .format_specifier   { color: #CC696B; }
 .mutable            { text-decoration: underline; }
 .unresolved_reference { color: #FC5555; }
+.escape_sequence    { color: #94BFF3; }
 
 .keyword            { color: #F0DFAF; font-weight: bold; }
 .keyword.unsafe     { color: #BC8383; font-weight: bold; }
diff --git a/crates/ra_ide/src/snapshots/highlighting.html b/crates/ra_ide/src/snapshots/highlighting.html
index 52912dc93..0c4f0a018 100644
--- a/crates/ra_ide/src/snapshots/highlighting.html
+++ b/crates/ra_ide/src/snapshots/highlighting.html
@@ -26,6 +26,7 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .format_specifier   { color: #CC696B; }
 .mutable            { text-decoration: underline; }
 .unresolved_reference { color: #FC5555; }
+.escape_sequence    { color: #94BFF3; }
 
 .keyword            { color: #F0DFAF; font-weight: bold; }
 .keyword.unsafe     { color: #BC8383; font-weight: bold; }
diff --git a/crates/ra_ide/src/snapshots/rainbow_highlighting.html b/crates/ra_ide/src/snapshots/rainbow_highlighting.html
index 1d7f04882..a74a70069 100644
--- a/crates/ra_ide/src/snapshots/rainbow_highlighting.html
+++ b/crates/ra_ide/src/snapshots/rainbow_highlighting.html
@@ -26,6 +26,7 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .format_specifier   { color: #CC696B; }
 .mutable            { text-decoration: underline; }
 .unresolved_reference { color: #FC5555; }
+.escape_sequence    { color: #94BFF3; }
 
 .keyword            { color: #F0DFAF; font-weight: bold; }
 .keyword.unsafe     { color: #BC8383; font-weight: bold; }
diff --git a/crates/ra_ide/src/syntax_highlighting.rs b/crates/ra_ide/src/syntax_highlighting.rs
index 5a4de450c..854b6cc6d 100644
--- a/crates/ra_ide/src/syntax_highlighting.rs
+++ b/crates/ra_ide/src/syntax_highlighting.rs
@@ -212,8 +212,8 @@ pub(crate) fn highlight(
             if let Some(string) =
                 element_to_highlight.as_token().cloned().and_then(ast::String::cast)
             {
-                stack.push();
                 if is_format_string {
+                    stack.push();
                     string.lex_format_specifier(|piece_range, kind| {
                         if let Some(highlight) = highlight_format_specifier(kind) {
                             stack.add(HighlightedRange {
@@ -223,13 +223,27 @@ pub(crate) fn highlight(
                             });
                         }
                     });
+                    stack.pop();
+                }
+                // Highlight escape sequences
+                if let Some(char_ranges) = string.char_ranges() {
+                    stack.push();
+                    for (piece_range, _) in char_ranges.iter().filter(|(_, char)| char.is_ok()) {
+                        if string.text()[piece_range.start().into()..].starts_with('\\') {
+                            stack.add(HighlightedRange {
+                                range: piece_range + range.start(),
+                                highlight: HighlightTag::EscapeSequence.into(),
+                                binding_hash: None,
+                            });
+                        }
+                    }
+                    stack.pop_and_inject(false);
                 }
-                stack.pop();
             } else if let Some(string) =
                 element_to_highlight.as_token().cloned().and_then(ast::RawString::cast)
             {
-                stack.push();
                 if is_format_string {
+                    stack.push();
                     string.lex_format_specifier(|piece_range, kind| {
                         if let Some(highlight) = highlight_format_specifier(kind) {
                             stack.add(HighlightedRange {
@@ -239,8 +253,8 @@ pub(crate) fn highlight(
                             });
                         }
                     });
+                    stack.pop();
                 }
-                stack.pop();
             }
         }
     }
diff --git a/crates/ra_ide/src/syntax_highlighting/html.rs b/crates/ra_ide/src/syntax_highlighting/html.rs
index 853b4a20f..99b6b25ab 100644
--- a/crates/ra_ide/src/syntax_highlighting/html.rs
+++ b/crates/ra_ide/src/syntax_highlighting/html.rs
@@ -85,6 +85,7 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .format_specifier   { color: #CC696B; }
 .mutable            { text-decoration: underline; }
 .unresolved_reference { color: #FC5555; }
+.escape_sequence    { color: #94BFF3; }
 
 .keyword            { color: #F0DFAF; font-weight: bold; }
 .keyword.unsafe     { color: #BC8383; font-weight: bold; }
diff --git a/crates/ra_ide/src/syntax_highlighting/tags.rs b/crates/ra_ide/src/syntax_highlighting/tags.rs
index 94f466966..400d22fb6 100644
--- a/crates/ra_ide/src/syntax_highlighting/tags.rs
+++ b/crates/ra_ide/src/syntax_highlighting/tags.rs
@@ -23,6 +23,7 @@ pub enum HighlightTag {
     Constant,
     Enum,
     EnumVariant,
+    EscapeSequence,
     Field,
     FormatSpecifier,
     Function,
@@ -71,6 +72,7 @@ impl HighlightTag {
             HighlightTag::Constant => "constant",
             HighlightTag::Enum => "enum",
             HighlightTag::EnumVariant => "enum_variant",
+            HighlightTag::EscapeSequence => "escape_sequence",
             HighlightTag::Field => "field",
             HighlightTag::FormatSpecifier => "format_specifier",
             HighlightTag::Function => "function",
diff --git a/crates/ra_ide/src/syntax_highlighting/tests.rs b/crates/ra_ide/src/syntax_highlighting/tests.rs
index ebf5b50ac..b4d56a7a0 100644
--- a/crates/ra_ide/src/syntax_highlighting/tests.rs
+++ b/crates/ra_ide/src/syntax_highlighting/tests.rs
@@ -246,6 +246,10 @@ fn main() {
 
     println!(r"Hello, {}!", "world");
 
+    // escape sequences
+    println!("Hello\nWorld");
+    println!("\u{48}\x65\x6C\x6C\x6F World");
+
     println!("{\x41}", A = 92);
     println!("{ничоси}", ничоси = 92);
 }"#
diff --git a/crates/rust-analyzer/src/semantic_tokens.rs b/crates/rust-analyzer/src/semantic_tokens.rs
index 6f125c37c..2ea63d33b 100644
--- a/crates/rust-analyzer/src/semantic_tokens.rs
+++ b/crates/rust-analyzer/src/semantic_tokens.rs
@@ -45,6 +45,7 @@ define_semantic_token_types![
     (UNION, "union"),
     (UNRESOLVED_REFERENCE, "unresolvedReference"),
     (FORMAT_SPECIFIER, "formatSpecifier"),
+    (ESCAPE_SEQUENCE, "escapeSequence"),
 ];
 
 macro_rules! define_semantic_token_modifiers {
diff --git a/crates/rust-analyzer/src/to_proto.rs b/crates/rust-analyzer/src/to_proto.rs
index 2851b4d31..055c97455 100644
--- a/crates/rust-analyzer/src/to_proto.rs
+++ b/crates/rust-analyzer/src/to_proto.rs
@@ -324,6 +324,7 @@ fn semantic_token_type_and_modifiers(
         HighlightTag::UnresolvedReference => semantic_tokens::UNRESOLVED_REFERENCE,
         HighlightTag::FormatSpecifier => semantic_tokens::FORMAT_SPECIFIER,
         HighlightTag::Operator => lsp_types::SemanticTokenType::OPERATOR,
+        HighlightTag::EscapeSequence => semantic_tokens::ESCAPE_SEQUENCE,
     };
 
     for modifier in highlight.modifiers.iter() {