4 files changed, 374 insertions, 44 deletions

diff --git a/crates/ra_ide/src/syntax_highlighting/html.rs b/crates/ra_ide/src/syntax_highlighting/html.rs
index edfe61f39..0be55bca9 100644
--- a/crates/ra_ide/src/syntax_highlighting/html.rs
+++ b/crates/ra_ide/src/syntax_highlighting/html.rs
@@ -19,7 +19,7 @@ pub(crate) fn highlight_as_html(db: &RootDatabase, file_id: FileId, rainbow: boo
         )
     }
 
-    let ranges = highlight(db, file_id, None);
+    let ranges = highlight(db, file_id, None, false);
     let text = parse.tree().syntax().to_string();
     let mut prev_pos = TextSize::from(0);
     let mut buf = String::new();
@@ -64,11 +64,15 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 
 .lifetime           { color: #DFAF8F; font-style: italic; }
 .comment            { color: #7F9F7F; }
+.documentation      { color: #629755; }
+.injected           { opacity: 0.65 ; }
 .struct, .enum      { color: #7CB8BB; }
 .enum_variant       { color: #BDE0F3; }
 .string_literal     { color: #CC9393; }
 .field              { color: #94BFF3; }
 .function           { color: #93E0E3; }
+.function.unsafe    { color: #BC8383; }
+.operator.unsafe    { color: #BC8383; }
 .parameter          { color: #94BFF3; }
 .text               { color: #DCDCCC; }
 .type               { color: #7CB8BB; }
@@ -79,12 +83,15 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .bool_literal       { color: #BFE6EB; }
 .macro              { color: #94BFF3; }
 .module             { color: #AFD8AF; }
+.value_param        { color: #DCDCCC; }
 .variable           { color: #DCDCCC; }
 .format_specifier   { color: #CC696B; }
 .mutable            { text-decoration: underline; }
-
+.escape_sequence    { color: #94BFF3; }
 .keyword            { color: #F0DFAF; font-weight: bold; }
 .keyword.unsafe     { color: #BC8383; font-weight: bold; }
 .control            { font-style: italic; }
+
+.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
 </style>
 ";
diff --git a/crates/ra_ide/src/syntax_highlighting/injection.rs b/crates/ra_ide/src/syntax_highlighting/injection.rs
new file mode 100644
index 000000000..8665b480f
--- /dev/null
+++ b/crates/ra_ide/src/syntax_highlighting/injection.rs
@@ -0,0 +1,188 @@
+//! Syntax highlighting injections such as highlighting of documentation tests.
+
+use std::{collections::BTreeMap, convert::TryFrom};
+
+use ast::{HasQuotes, HasStringValue};
+use hir::Semantics;
+use ra_syntax::{ast, AstToken, SyntaxNode, SyntaxToken, TextRange, TextSize};
+use stdx::SepBy;
+
+use crate::{
+    call_info::ActiveParameter, Analysis, Highlight, HighlightModifier, HighlightTag,
+    HighlightedRange, RootDatabase,
+};
+
+use super::HighlightedRangeStack;
+
+pub(super) fn highlight_injection(
+    acc: &mut HighlightedRangeStack,
+    sema: &Semantics<RootDatabase>,
+    literal: ast::RawString,
+    expanded: SyntaxToken,
+) -> Option<()> {
+    let active_parameter = ActiveParameter::at_token(&sema, expanded)?;
+    if !active_parameter.name.starts_with("ra_fixture") {
+        return None;
+    }
+    let value = literal.value()?;
+    let (analysis, tmp_file_id) = Analysis::from_single_file(value.into_owned());
+
+    if let Some(range) = literal.open_quote_text_range() {
+        acc.add(HighlightedRange {
+            range,
+            highlight: HighlightTag::StringLiteral.into(),
+            binding_hash: None,
+        })
+    }
+
+    for mut h in analysis.highlight(tmp_file_id).unwrap() {
+        if let Some(r) = literal.map_range_up(h.range) {
+            h.range = r;
+            acc.add(h)
+        }
+    }
+
+    if let Some(range) = literal.close_quote_text_range() {
+        acc.add(HighlightedRange {
+            range,
+            highlight: HighlightTag::StringLiteral.into(),
+            binding_hash: None,
+        })
+    }
+
+    Some(())
+}
+
+/// Mapping from extracted documentation code to original code
+type RangesMap = BTreeMap<TextSize, TextSize>;
+
+const RUSTDOC_FENCE: &'static str = "```";
+const RUSTDOC_FENCE_TOKENS: &[&'static str] =
+    &["", "rust", "should_panic", "ignore", "no_run", "compile_fail", "edition2015", "edition2018"];
+
+/// Extracts Rust code from documentation comments as well as a mapping from
+/// the extracted source code back to the original source ranges.
+/// Lastly, a vector of new comment highlight ranges (spanning only the
+/// comment prefix) is returned which is used in the syntax highlighting
+/// injection to replace the previous (line-spanning) comment ranges.
+pub(super) fn extract_doc_comments(
+    node: &SyntaxNode,
+) -> Option<(String, RangesMap, Vec<HighlightedRange>)> {
+    // wrap the doctest into function body to get correct syntax highlighting
+    let prefix = "fn doctest() {\n";
+    let suffix = "}\n";
+    // Mapping from extracted documentation code to original code
+    let mut range_mapping: RangesMap = BTreeMap::new();
+    let mut line_start = TextSize::try_from(prefix.len()).unwrap();
+    let mut is_codeblock = false;
+    let mut is_doctest = false;
+    // Replace the original, line-spanning comment ranges by new, only comment-prefix
+    // spanning comment ranges.
+    let mut new_comments = Vec::new();
+    let doctest = node
+        .children_with_tokens()
+        .filter_map(|el| el.into_token().and_then(ast::Comment::cast))
+        .filter(|comment| comment.kind().doc.is_some())
+        .filter(|comment| {
+            if let Some(idx) = comment.text().find(RUSTDOC_FENCE) {
+                is_codeblock = !is_codeblock;
+                // Check whether code is rust by inspecting fence guards
+                let guards = &comment.text()[idx + RUSTDOC_FENCE.len()..];
+                let is_rust =
+                    guards.split(',').all(|sub| RUSTDOC_FENCE_TOKENS.contains(&sub.trim()));
+                is_doctest = is_codeblock && is_rust;
+                false
+            } else {
+                is_doctest
+            }
+        })
+        .map(|comment| {
+            let prefix_len = comment.prefix().len();
+            let line: &str = comment.text().as_str();
+            let range = comment.syntax().text_range();
+
+            // whitespace after comment is ignored
+            let pos = if let Some(ws) = line.chars().nth(prefix_len).filter(|c| c.is_whitespace()) {
+                prefix_len + ws.len_utf8()
+            } else {
+                prefix_len
+            };
+
+            // lines marked with `#` should be ignored in output, we skip the `#` char
+            let pos = if let Some(ws) = line.chars().nth(pos).filter(|&c| c == '#') {
+                pos + ws.len_utf8()
+            } else {
+                pos
+            };
+
+            range_mapping.insert(line_start, range.start() + TextSize::try_from(pos).unwrap());
+            new_comments.push(HighlightedRange {
+                range: TextRange::new(
+                    range.start(),
+                    range.start() + TextSize::try_from(pos).unwrap(),
+                ),
+                highlight: HighlightTag::Comment | HighlightModifier::Documentation,
+                binding_hash: None,
+            });
+            line_start += range.len() - TextSize::try_from(pos).unwrap();
+            line_start += TextSize::try_from('\n'.len_utf8()).unwrap();
+
+            line[pos..].to_owned()
+        })
+        .sep_by("\n")
+        .to_string();
+
+    if doctest.is_empty() {
+        return None;
+    }
+
+    let doctest = format!("{}{}{}", prefix, doctest, suffix);
+    Some((doctest, range_mapping, new_comments))
+}
+
+/// Injection of syntax highlighting of doctests.
+pub(super) fn highlight_doc_comment(
+    text: String,
+    range_mapping: RangesMap,
+    new_comments: Vec<HighlightedRange>,
+    stack: &mut HighlightedRangeStack,
+) {
+    let (analysis, tmp_file_id) = Analysis::from_single_file(text);
+
+    stack.push();
+    for mut h in analysis.with_db(|db| super::highlight(db, tmp_file_id, None, true)).unwrap() {
+        // Determine start offset and end offset in case of multi-line ranges
+        let mut start_offset = None;
+        let mut end_offset = None;
+        for (line_start, orig_line_start) in range_mapping.range(..h.range.end()).rev() {
+            // It's possible for orig_line_start - line_start to be negative. Add h.range.start()
+            // here and remove it from the end range after the loop below so that the values are
+            // always non-negative.
+            let offset = h.range.start() + orig_line_start - line_start;
+            if line_start <= &h.range.start() {
+                start_offset.get_or_insert(offset);
+                break;
+            } else {
+                end_offset.get_or_insert(offset);
+            }
+        }
+        if let Some(start_offset) = start_offset {
+            h.range = TextRange::new(
+                start_offset,
+                h.range.end() + end_offset.unwrap_or(start_offset) - h.range.start(),
+            );
+
+            h.highlight |= HighlightModifier::Injected;
+            stack.add(h);
+        }
+    }
+
+    // Inject the comment prefix highlight ranges
+    stack.push();
+    for comment in new_comments {
+        stack.add(comment);
+    }
+    stack.pop_and_inject(None);
+    stack
+        .pop_and_inject(Some(Highlight::from(HighlightTag::Generic) | HighlightModifier::Injected));
+}
diff --git a/crates/ra_ide/src/syntax_highlighting/tags.rs b/crates/ra_ide/src/syntax_highlighting/tags.rs
index 1514531de..49ec94bdc 100644
--- a/crates/ra_ide/src/syntax_highlighting/tags.rs
+++ b/crates/ra_ide/src/syntax_highlighting/tags.rs
@@ -23,13 +23,16 @@ pub enum HighlightTag {
     Constant,
     Enum,
     EnumVariant,
+    EscapeSequence,
     Field,
     Function,
+    Generic,
     Keyword,
     Lifetime,
     Macro,
     Module,
     NumericLiteral,
+    Punctuation,
     SelfKeyword,
     SelfType,
     Static,
@@ -39,6 +42,7 @@ pub enum HighlightTag {
     TypeAlias,
     TypeParam,
     Union,
+    ValueParam,
     Local,
     UnresolvedReference,
     FormatSpecifier,
@@ -55,6 +59,8 @@ pub enum HighlightModifier {
     /// `foo` in `fn foo(x: i32)` is a definition, `foo` in `foo(90 + 2)` is
     /// not.
     Definition,
+    Documentation,
+    Injected,
     Mutable,
     Unsafe,
 }
@@ -71,13 +77,18 @@ impl HighlightTag {
             HighlightTag::Constant => "constant",
             HighlightTag::Enum => "enum",
             HighlightTag::EnumVariant => "enum_variant",
+            HighlightTag::EscapeSequence => "escape_sequence",
             HighlightTag::Field => "field",
+            HighlightTag::FormatSpecifier => "format_specifier",
             HighlightTag::Function => "function",
+            HighlightTag::Generic => "generic",
             HighlightTag::Keyword => "keyword",
             HighlightTag::Lifetime => "lifetime",
+            HighlightTag::Punctuation => "punctuation",
             HighlightTag::Macro => "macro",
             HighlightTag::Module => "module",
             HighlightTag::NumericLiteral => "numeric_literal",
+            HighlightTag::Operator => "operator",
             HighlightTag::SelfKeyword => "self_keyword",
             HighlightTag::SelfType => "self_type",
             HighlightTag::Static => "static",
@@ -87,10 +98,9 @@ impl HighlightTag {
             HighlightTag::TypeAlias => "type_alias",
             HighlightTag::TypeParam => "type_param",
             HighlightTag::Union => "union",
+            HighlightTag::ValueParam => "value_param",
             HighlightTag::Local => "variable",
             HighlightTag::UnresolvedReference => "unresolved_reference",
-            HighlightTag::FormatSpecifier => "format_specifier",
-            HighlightTag::Operator => "operator",
         }
     }
 }
@@ -106,6 +116,8 @@ impl HighlightModifier {
         HighlightModifier::Attribute,
         HighlightModifier::ControlFlow,
         HighlightModifier::Definition,
+        HighlightModifier::Documentation,
+        HighlightModifier::Injected,
         HighlightModifier::Mutable,
         HighlightModifier::Unsafe,
     ];
@@ -115,6 +127,8 @@ impl HighlightModifier {
             HighlightModifier::Attribute => "attribute",
             HighlightModifier::ControlFlow => "control",
             HighlightModifier::Definition => "declaration",
+            HighlightModifier::Documentation => "documentation",
+            HighlightModifier::Injected => "injected",
             HighlightModifier::Mutable => "mutable",
             HighlightModifier::Unsafe => "unsafe",
         }
diff --git a/crates/ra_ide/src/syntax_highlighting/tests.rs b/crates/ra_ide/src/syntax_highlighting/tests.rs
index eb43a23da..87a6e2523 100644
--- a/crates/ra_ide/src/syntax_highlighting/tests.rs
+++ b/crates/ra_ide/src/syntax_highlighting/tests.rs
@@ -1,15 +1,13 @@
 use std::fs;
 
-use test_utils::{assert_eq_text, project_dir, read_text};
+use expect::{expect_file, ExpectFile};
+use test_utils::project_dir;
 
-use crate::{
-    mock_analysis::{single_file, MockAnalysis},
-    FileRange, TextRange,
-};
+use crate::{mock_analysis::single_file, FileRange, TextRange};
 
 #[test]
 fn test_highlighting() {
-    let (analysis, file_id) = single_file(
+    check_highlighting(
         r#"
 #[derive(Clone, Debug)]
 struct Foo {
@@ -27,6 +25,16 @@ impl Bar for Foo {
     }
 }
 
+impl Foo {
+    fn baz(mut self) -> i32 {
+        self.x
+    }
+
+    fn qux(&mut self) {
+        self.x = 0;
+    }
+}
+
 static mut STATIC_MUT: i32 = 0;
 
 fn foo<'a, T>() -> T {
@@ -43,6 +51,12 @@ def_fn! {
     }
 }
 
+macro_rules! noop {
+    ($expr:expr) => {
+        $expr
+    }
+}
+
 // comment
 fn main() {
     println!("Hello, {}!", 92);
@@ -61,10 +75,14 @@ fn main() {
         // Do nothing
     }
 
+    noop!(noop!(1));
+
     let mut x = 42;
     let y = &mut x;
     let z = &y;
 
+    let Foo { x: z, y } = Foo { x: z, y };
+
     y;
 }
 
@@ -84,17 +102,14 @@ impl<T> Option<T> {
 }
 "#
         .trim(),
+        expect_file!["crates/ra_ide/test_data/highlighting.html"],
+        false,
     );
-    let dst_file = project_dir().join("crates/ra_ide/src/snapshots/highlighting.html");
-    let actual_html = &analysis.highlight_as_html(file_id, false).unwrap();
-    let expected_html = &read_text(&dst_file);
-    fs::write(dst_file, &actual_html).unwrap();
-    assert_eq_text!(expected_html, actual_html);
 }
 
 #[test]
 fn test_rainbow_highlighting() {
-    let (analysis, file_id) = single_file(
+    check_highlighting(
         r#"
 fn main() {
     let hello = "hello";
@@ -110,12 +125,9 @@ fn bar() {
 }
 "#
         .trim(),
+        expect_file!["crates/ra_ide/test_data/rainbow_highlighting.html"],
+        true,
     );
-    let dst_file = project_dir().join("crates/ra_ide/src/snapshots/rainbow_highlighting.html");
-    let actual_html = &analysis.highlight_as_html(file_id, true).unwrap();
-    let expected_html = &read_text(&dst_file);
-    fs::write(dst_file, &actual_html).unwrap();
-    assert_eq_text!(expected_html, actual_html);
 }
 
 #[test]
@@ -123,12 +135,10 @@ fn accidentally_quadratic() {
     let file = project_dir().join("crates/ra_syntax/test_data/accidentally_quadratic");
     let src = fs::read_to_string(file).unwrap();
 
-    let mut mock = MockAnalysis::new();
-    let file_id = mock.add_file("/main.rs", &src);
-    let host = mock.analysis_host();
+    let (analysis, file_id) = single_file(&src);
 
     // let t = std::time::Instant::now();
-    let _ = host.analysis().highlight(file_id).unwrap();
+    let _ = analysis.highlight(file_id).unwrap();
     // eprintln!("elapsed: {:?}", t.elapsed());
 }
 
@@ -136,16 +146,17 @@ fn accidentally_quadratic() {
 fn test_ranges() {
     let (analysis, file_id) = single_file(
         r#"
-            #[derive(Clone, Debug)]
-            struct Foo {
-                pub x: i32,
-                pub y: i32,
-            }"#,
+#[derive(Clone, Debug)]
+struct Foo {
+    pub x: i32,
+    pub y: i32,
+}
+"#,
     );
 
     // The "x"
     let highlights = &analysis
-        .highlight_range(FileRange { file_id, range: TextRange::at(82.into(), 1.into()) })
+        .highlight_range(FileRange { file_id, range: TextRange::at(45.into(), 1.into()) })
         .unwrap();
 
     assert_eq!(&highlights[0].highlight.to_string(), "field.declaration");
@@ -153,7 +164,7 @@ fn test_ranges() {
 
 #[test]
 fn test_flattening() {
-    let (analysis, file_id) = single_file(
+    check_highlighting(
         r##"
 fn fixture(ra_fixture: &str) {}
 
@@ -167,13 +178,9 @@ fn main() {
     );
 }"##
         .trim(),
+        expect_file!["crates/ra_ide/test_data/highlight_injection.html"],
+        false,
     );
-
-    let dst_file = project_dir().join("crates/ra_ide/src/snapshots/highlight_injection.html");
-    let actual_html = &analysis.highlight_as_html(file_id, false).unwrap();
-    let expected_html = &read_text(&dst_file);
-    fs::write(dst_file, &actual_html).unwrap();
-    assert_eq_text!(expected_html, actual_html);
 }
 
 #[test]
@@ -192,7 +199,7 @@ macro_rules! test {}
 fn test_string_highlighting() {
     // The format string detection is based on macro-expansion,
     // thus, we have to copy the macro definition from `std`
-    let (analysis, file_id) = single_file(
+    check_highlighting(
         r#"
 macro_rules! println {
     ($($arg:tt)*) => ({
@@ -218,6 +225,7 @@ fn main() {
     println!("{argument}", argument = "test");   // => "test"
     println!("{name} {}", 1, name = 2);          // => "2 1"
     println!("{a} {c} {b}", a="a", b='b', c=3);  // => "a 3 b"
+    println!("{{{}}}", 2);                       // => "{2}"
     println!("Hello {:5}!", "x");
     println!("Hello {:1$}!", "x", 5);
     println!("Hello {1:0$}!", 5, "x");
@@ -245,15 +253,128 @@ fn main() {
 
     println!(r"Hello, {}!", "world");
 
+    // escape sequences
+    println!("Hello\nWorld");
+    println!("\u{48}\x65\x6C\x6C\x6F World");
+
     println!("{\x41}", A = 92);
     println!("{ничоси}", ничоси = 92);
 }"#
         .trim(),
+        expect_file!["crates/ra_ide/test_data/highlight_strings.html"],
+        false,
+    );
+}
+
+#[test]
+fn test_unsafe_highlighting() {
+    check_highlighting(
+        r#"
+unsafe fn unsafe_fn() {}
+
+struct HasUnsafeFn;
+
+impl HasUnsafeFn {
+    unsafe fn unsafe_method(&self) {}
+}
+
+fn main() {
+    let x = &5 as *const usize;
+    unsafe {
+        unsafe_fn();
+        HasUnsafeFn.unsafe_method();
+        let y = *(x);
+        let z = -x;
+    }
+}
+"#
+        .trim(),
+        expect_file!["crates/ra_ide/test_data/highlight_unsafe.html"],
+        false,
+    );
+}
+
+#[test]
+fn test_highlight_doctest() {
+    check_highlighting(
+        r#"
+/// ```
+/// let _ = "early doctests should not go boom";
+/// ```
+struct Foo {
+    bar: bool,
+}
+
+impl Foo {
+    pub const bar: bool = true;
+
+    /// Constructs a new `Foo`.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # #![allow(unused_mut)]
+    /// let mut foo: Foo = Foo::new();
+    /// ```
+    pub const fn new() -> Foo {
+        Foo { bar: true }
+    }
+
+    /// `bar` method on `Foo`.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use x::y;
+    ///
+    /// let foo = Foo::new();
+    ///
+    /// // calls bar on foo
+    /// assert!(foo.bar());
+    ///
+    /// let bar = foo.bar || Foo::bar;
+    ///
+    /// /* multi-line
+    ///        comment */
+    ///
+    /// let multi_line_string = "Foo
+    ///   bar
+    ///          ";
+    ///
+    /// ```
+    ///
+    /// ```rust,no_run
+    /// let foobar = Foo::new().bar();
+    /// ```
+    ///
+    /// ```sh
+    /// echo 1
+    /// ```
+    pub fn foo(&self) -> bool {
+        true
+    }
+}
+
+/// ```
+/// noop!(1);
+/// ```
+macro_rules! noop {
+    ($expr:expr) => {
+        $expr
+    }
+}
+"#
+        .trim(),
+        expect_file!["crates/ra_ide/test_data/highlight_doctest.html"],
+        false,
     );
+}
 
-    let dst_file = project_dir().join("crates/ra_ide/src/snapshots/highlight_strings.html");
-    let actual_html = &analysis.highlight_as_html(file_id, false).unwrap();
-    let expected_html = &read_text(&dst_file);
-    fs::write(dst_file, &actual_html).unwrap();
-    assert_eq_text!(expected_html, actual_html);
+/// Highlights the code given by the `ra_fixture` argument, renders the
+/// result as HTML, and compares it with the HTML file given as `snapshot`.
+/// Note that the `snapshot` file is overwritten by the rendered HTML.
+fn check_highlighting(ra_fixture: &str, expect: ExpectFile, rainbow: bool) {
+    let (analysis, file_id) = single_file(ra_fixture);
+    let actual_html = &analysis.highlight_as_html(file_id, rainbow).unwrap();
+    expect.assert_eq(actual_html)
 }