From 4a2efb2f42494f62891ac801e0a27d246bd36684 Mon Sep 17 00:00:00 2001 From: Leander Tentrup Date: Tue, 28 Apr 2020 11:01:51 +0200 Subject: Implement syntax highlighting for doctests --- crates/ra_ide/src/snapshots/highlight_doctest.html | 70 +++++++++ crates/ra_ide/src/syntax_highlighting.rs | 126 ++++++++++------ crates/ra_ide/src/syntax_highlighting/injection.rs | 168 +++++++++++++++++++++ crates/ra_ide/src/syntax_highlighting/tests.rs | 50 ++++++ 4 files changed, 368 insertions(+), 46 deletions(-) create mode 100644 crates/ra_ide/src/snapshots/highlight_doctest.html create mode 100644 crates/ra_ide/src/syntax_highlighting/injection.rs (limited to 'crates/ra_ide') diff --git a/crates/ra_ide/src/snapshots/highlight_doctest.html b/crates/ra_ide/src/snapshots/highlight_doctest.html new file mode 100644 index 000000000..2f2d8c900 --- /dev/null +++ b/crates/ra_ide/src/snapshots/highlight_doctest.html @@ -0,0 +1,70 @@ + + +
impl Foo {
+    /// Constructs a new `Foo`.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # #![allow(unused_mut)]
+    /// let mut foo: Foo = Foo::new();
+    /// ```
+    pub const fn new() -> Foo {
+        Foo { }
+    }
+
+    /// `bar` method on `Foo`.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let foo = Foo::new();
+    ///
+    /// // calls bar on foo
+    /// assert!(foo.bar());
+    ///
+    /// /* multi-line
+    ///        comment */
+    ///
+    /// let multi_line_string = "Foo
+    ///   bar
+    ///          ";
+    ///
+    /// ```
+    ///
+    /// ```
+    /// let foobar = Foo::new().bar();
+    /// ```
+    pub fn foo(&self) -> bool {
+        true
+    }
+}
\ No newline at end of file diff --git a/crates/ra_ide/src/syntax_highlighting.rs b/crates/ra_ide/src/syntax_highlighting.rs index 19ecd54d6..6903403b2 100644 --- a/crates/ra_ide/src/syntax_highlighting.rs +++ b/crates/ra_ide/src/syntax_highlighting.rs @@ -1,5 +1,6 @@ mod tags; mod html; +mod injection; #[cfg(test)] mod tests; @@ -10,14 +11,14 @@ use ra_ide_db::{ }; use ra_prof::profile; use ra_syntax::{ - ast::{self, HasFormatSpecifier, HasQuotes, HasStringValue}, + ast::{self, HasFormatSpecifier}, AstNode, AstToken, Direction, NodeOrToken, SyntaxElement, SyntaxKind::*, - SyntaxToken, TextRange, WalkEvent, T, + TextRange, WalkEvent, T, }; use rustc_hash::FxHashMap; -use crate::{call_info::ActiveParameter, Analysis, FileId}; +use crate::FileId; use ast::FormatSpecifier; pub(crate) use html::highlight_as_html; @@ -123,6 +124,23 @@ pub(crate) fn highlight( _ => (), } + // Check for Rust code in documentation + match &event { + WalkEvent::Leave(NodeOrToken::Node(node)) => { + if let Some((doctest, range_mapping, new_comments)) = + injection::extract_doc_comments(node) + { + injection::highlight_doc_comment( + doctest, + range_mapping, + new_comments, + &mut stack, + ); + } + } + _ => (), + } + let element = match event { WalkEvent::Enter(it) => it, WalkEvent::Leave(_) => continue, @@ -173,7 +191,7 @@ pub(crate) fn highlight( if let Some(token) = element.as_token().cloned().and_then(ast::RawString::cast) { let expanded = element_to_highlight.as_token().unwrap().clone(); - if highlight_injection(&mut stack, &sema, token, expanded).is_some() { + if injection::highlight_injection(&mut stack, &sema, token, expanded).is_some() { continue; } } @@ -259,9 +277,8 @@ impl HighlightedRangeStack { let mut parent = prev.pop().unwrap(); for ele in children { assert!(parent.range.contains_range(ele.range)); - let mut cloned = parent.clone(); - parent.range = TextRange::new(parent.range.start(), ele.range.start()); - cloned.range = TextRange::new(ele.range.end(), cloned.range.end()); + + let cloned = Self::intersect(&mut parent, &ele); if !parent.range.is_empty() { prev.push(parent); } @@ -274,6 +291,62 @@ impl HighlightedRangeStack { } } + /// Intersects the `HighlightedRange` `parent` with `child`. + /// `parent` is mutated in place, becoming the range before `child`. + /// Returns the range (of the same type as `parent`) *after* `child`. + fn intersect(parent: &mut HighlightedRange, child: &HighlightedRange) -> HighlightedRange { + assert!(parent.range.contains_range(child.range)); + + let mut cloned = parent.clone(); + parent.range = TextRange::new(parent.range.start(), child.range.start()); + cloned.range = TextRange::new(child.range.end(), cloned.range.end()); + + cloned + } + + /// Similar to `pop`, but can modify arbitrary prior ranges (where `pop`) + /// can only modify the last range currently on the stack. + /// Can be used to do injections that span multiple ranges, like the + /// doctest injection below. + /// If `delete` is set to true, the parent range is deleted instead of + /// intersected. + /// + /// Note that `pop` can be simulated by `pop_and_inject(false)` but the + /// latter is computationally more expensive. + fn pop_and_inject(&mut self, delete: bool) { + let mut children = self.stack.pop().unwrap(); + let prev = self.stack.last_mut().unwrap(); + children.sort_by_key(|range| range.range.start()); + prev.sort_by_key(|range| range.range.start()); + + for child in children { + if let Some(idx) = + prev.iter().position(|parent| parent.range.contains_range(child.range)) + { + let cloned = Self::intersect(&mut prev[idx], &child); + let insert_idx = if delete || prev[idx].range.is_empty() { + prev.remove(idx); + idx + } else { + idx + 1 + }; + prev.insert(insert_idx, child); + if !delete && !cloned.range.is_empty() { + prev.insert(insert_idx + 1, cloned); + } + } else if let Some(_idx) = + prev.iter().position(|parent| parent.range.contains(child.range.start())) + { + unreachable!("child range should be completely contained in parent range"); + } else { + let idx = prev + .binary_search_by_key(&child.range.start(), |range| range.range.start()) + .unwrap_or_else(|x| x); + prev.insert(idx, child); + } + } + } + fn add(&mut self, range: HighlightedRange) { self.stack .last_mut() @@ -539,42 +612,3 @@ fn highlight_name_by_syntax(name: ast::Name) -> Highlight { tag.into() } - -fn highlight_injection( - acc: &mut HighlightedRangeStack, - sema: &Semantics, - literal: ast::RawString, - expanded: SyntaxToken, -) -> Option<()> { - let active_parameter = ActiveParameter::at_token(&sema, expanded)?; - if !active_parameter.name.starts_with("ra_fixture") { - return None; - } - let value = literal.value()?; - let (analysis, tmp_file_id) = Analysis::from_single_file(value); - - if let Some(range) = literal.open_quote_text_range() { - acc.add(HighlightedRange { - range, - highlight: HighlightTag::StringLiteral.into(), - binding_hash: None, - }) - } - - for mut h in analysis.highlight(tmp_file_id).unwrap() { - if let Some(r) = literal.map_range_up(h.range) { - h.range = r; - acc.add(h) - } - } - - if let Some(range) = literal.close_quote_text_range() { - acc.add(HighlightedRange { - range, - highlight: HighlightTag::StringLiteral.into(), - binding_hash: None, - }) - } - - Some(()) -} diff --git a/crates/ra_ide/src/syntax_highlighting/injection.rs b/crates/ra_ide/src/syntax_highlighting/injection.rs new file mode 100644 index 000000000..3575a0fc6 --- /dev/null +++ b/crates/ra_ide/src/syntax_highlighting/injection.rs @@ -0,0 +1,168 @@ +//! Syntax highlighting injections such as highlighting of documentation tests. + +use std::{collections::BTreeMap, convert::TryFrom}; + +use ast::{HasQuotes, HasStringValue}; +use hir::Semantics; +use ra_syntax::{ast, AstToken, SyntaxNode, SyntaxToken, TextRange, TextSize}; +use stdx::SepBy; + +use crate::{call_info::ActiveParameter, Analysis, HighlightTag, HighlightedRange, RootDatabase}; + +use super::HighlightedRangeStack; + +pub(super) fn highlight_injection( + acc: &mut HighlightedRangeStack, + sema: &Semantics, + literal: ast::RawString, + expanded: SyntaxToken, +) -> Option<()> { + let active_parameter = ActiveParameter::at_token(&sema, expanded)?; + if !active_parameter.name.starts_with("ra_fixture") { + return None; + } + let value = literal.value()?; + let (analysis, tmp_file_id) = Analysis::from_single_file(value); + + if let Some(range) = literal.open_quote_text_range() { + acc.add(HighlightedRange { + range, + highlight: HighlightTag::StringLiteral.into(), + binding_hash: None, + }) + } + + for mut h in analysis.highlight(tmp_file_id).unwrap() { + if let Some(r) = literal.map_range_up(h.range) { + h.range = r; + acc.add(h) + } + } + + if let Some(range) = literal.close_quote_text_range() { + acc.add(HighlightedRange { + range, + highlight: HighlightTag::StringLiteral.into(), + binding_hash: None, + }) + } + + Some(()) +} + +/// Mapping from extracted documentation code to original code +type RangesMap = BTreeMap; + +/// Extracts Rust code from documentation comments as well as a mapping from +/// the extracted source code back to the original source ranges. +/// Lastly, a vector of new comment highlight ranges (spanning only the +/// comment prefix) is returned which is used in the syntax highlighting +/// injection to replace the previous (line-spanning) comment ranges. +pub(super) fn extract_doc_comments( + node: &SyntaxNode, +) -> Option<(String, RangesMap, Vec)> { + // wrap the doctest into function body to get correct syntax highlighting + let prefix = "fn doctest() {\n"; + let suffix = "}\n"; + // Mapping from extracted documentation code to original code + let mut range_mapping: RangesMap = BTreeMap::new(); + let mut line_start = TextSize::try_from(prefix.len()).unwrap(); + let mut is_doctest = false; + // Replace the original, line-spanning comment ranges by new, only comment-prefix + // spanning comment ranges. + let mut new_comments = Vec::new(); + let doctest = node + .children_with_tokens() + .filter_map(|el| el.into_token().and_then(ast::Comment::cast)) + .filter(|comment| comment.kind().doc.is_some()) + .filter(|comment| { + if comment.text().contains("```") { + is_doctest = !is_doctest; + false + } else { + is_doctest + } + }) + .map(|comment| { + let prefix_len = comment.prefix().len(); + let line: &str = comment.text().as_str(); + let range = comment.syntax().text_range(); + + // whitespace after comment is ignored + let pos = if let Some(ws) = line.chars().nth(prefix_len).filter(|c| c.is_whitespace()) { + prefix_len + ws.len_utf8() + } else { + prefix_len + }; + + // lines marked with `#` should be ignored in output, we skip the `#` char + let pos = if let Some(ws) = line.chars().nth(pos).filter(|&c| c == '#') { + pos + ws.len_utf8() + } else { + pos + }; + + range_mapping.insert(line_start, range.start() + TextSize::try_from(pos).unwrap()); + new_comments.push(HighlightedRange { + range: TextRange::new( + range.start(), + range.start() + TextSize::try_from(pos).unwrap(), + ), + highlight: HighlightTag::Comment.into(), + binding_hash: None, + }); + line_start += range.len() - TextSize::try_from(pos).unwrap(); + line_start += TextSize::try_from('\n'.len_utf8()).unwrap(); + + line[pos..].to_owned() + }) + .sep_by("\n") + .to_string(); + + if doctest.is_empty() { + return None; + } + + let doctest = format!("{}{}{}", prefix, doctest, suffix); + Some((doctest, range_mapping, new_comments)) +} + +/// Injection of syntax highlighting of doctests. +pub(super) fn highlight_doc_comment( + text: String, + range_mapping: RangesMap, + new_comments: Vec, + stack: &mut HighlightedRangeStack, +) { + let (analysis, tmp_file_id) = Analysis::from_single_file(text); + + stack.push(); + for mut h in analysis.highlight(tmp_file_id).unwrap() { + // Determine start offset and end offset in case of multi-line ranges + let mut start_offset = None; + let mut end_offset = None; + for (line_start, orig_line_start) in range_mapping.range(..h.range.end()).rev() { + if line_start <= &h.range.start() { + start_offset.get_or_insert(orig_line_start - line_start); + break; + } else { + end_offset.get_or_insert(orig_line_start - line_start); + } + } + if let Some(start_offset) = start_offset { + h.range = TextRange::new( + h.range.start() + start_offset, + h.range.end() + end_offset.unwrap_or(start_offset), + ); + stack.add(h); + } + } + + // Inject the comment prefix highlight ranges + stack.push(); + for comment in new_comments { + stack.add(comment); + } + stack.pop_and_inject(false); + stack.pop_and_inject(true); +} diff --git a/crates/ra_ide/src/syntax_highlighting/tests.rs b/crates/ra_ide/src/syntax_highlighting/tests.rs index 5e42c5b55..ba345d90a 100644 --- a/crates/ra_ide/src/syntax_highlighting/tests.rs +++ b/crates/ra_ide/src/syntax_highlighting/tests.rs @@ -284,3 +284,53 @@ fn main() { false, ); } + +#[test] +fn test_highlight_doctest() { + check_highlighting( + r#" +impl Foo { + /// Constructs a new `Foo`. + /// + /// # Examples + /// + /// ``` + /// # #![allow(unused_mut)] + /// let mut foo: Foo = Foo::new(); + /// ``` + pub const fn new() -> Foo { + Foo { } + } + + /// `bar` method on `Foo`. + /// + /// # Examples + /// + /// ``` + /// let foo = Foo::new(); + /// + /// // calls bar on foo + /// assert!(foo.bar()); + /// + /// /* multi-line + /// comment */ + /// + /// let multi_line_string = "Foo + /// bar + /// "; + /// + /// ``` + /// + /// ``` + /// let foobar = Foo::new().bar(); + /// ``` + pub fn foo(&self) -> bool { + true + } +} +"# + .trim(), + "crates/ra_ide/src/snapshots/highlight_doctest.html", + false, + ) +} -- cgit v1.2.3