4 files changed, 60 insertions, 63 deletions

diff --git a/crates/ra_syntax/src/parsing/lexer.rs b/crates/ra_syntax/src/parsing/lexer.rs
index 1a5a6dc06..fa3be1016 100644
--- a/crates/ra_syntax/src/parsing/lexer.rs
+++ b/crates/ra_syntax/src/parsing/lexer.rs
@@ -1,6 +1,8 @@
 //! Lexer analyzes raw input string and produces lexemes (tokens).
 //! It is just a bridge to `rustc_lexer`.
 
+use rustc_lexer::{LiteralKind as LK, RawStrError};
+
 use std::convert::TryInto;
 
 use crate::{
@@ -180,8 +182,6 @@ fn rustc_token_kind_to_syntax_kind(
     return (syntax_kind, None);
 
     fn match_literal_kind(kind: &rustc_lexer::LiteralKind) -> (SyntaxKind, Option<&'static str>) {
-        use rustc_lexer::{LexRawStrError, LiteralKind as LK};
-
         #[rustfmt::skip]
         let syntax_kind = match *kind {
             LK::Int { empty_int: false, .. } => INT_NUMBER,
@@ -215,27 +215,27 @@ fn rustc_token_kind_to_syntax_kind(
                 return (BYTE_STRING, Some("Missing trailing `\"` symbol to terminate the byte string literal"))
             }
 
-            LK::RawStr(str) => match str.validate() {
-                Ok(_) => RAW_STRING,
-                Err(LexRawStrError::InvalidStarter) => return (RAW_STRING, Some("Missing `\"` symbol after `#` symbols to begin the raw string literal")),
-                Err(LexRawStrError::NoTerminator { expected, found, .. }) => if expected == found {
+            LK::RawStr { err, .. } => match err {
+                None => RAW_STRING,
+                Some(RawStrError::InvalidStarter { .. }) => return (RAW_STRING, Some("Missing `\"` symbol after `#` symbols to begin the raw string literal")),
+                Some(RawStrError::NoTerminator { expected, found, .. }) => if expected == found {
                     return (RAW_STRING, Some("Missing trailing `\"` to terminate the raw string literal"))
                 } else {
                     return (RAW_STRING, Some("Missing trailing `\"` with `#` symbols to terminate the raw string literal"))
 
                 },
-                Err(LexRawStrError::TooManyDelimiters { .. }) => return (RAW_STRING, Some("Too many `#` symbols: raw strings may be delimited by up to 65535 `#` symbols")),
+                Some(RawStrError::TooManyDelimiters { .. }) => return (RAW_STRING, Some("Too many `#` symbols: raw strings may be delimited by up to 65535 `#` symbols")),
             },
-            LK::RawByteStr(str) => match str.validate() {
-                Ok(_) => RAW_BYTE_STRING,
-                Err(LexRawStrError::InvalidStarter) => return (RAW_BYTE_STRING, Some("Missing `\"` symbol after `#` symbols to begin the raw byte string literal")),
-                Err(LexRawStrError::NoTerminator { expected, found, .. }) => if expected == found {
+            LK::RawByteStr { err, .. } => match err {
+                None => RAW_BYTE_STRING,
+                Some(RawStrError::InvalidStarter { .. }) => return (RAW_BYTE_STRING, Some("Missing `\"` symbol after `#` symbols to begin the raw byte string literal")),
+                Some(RawStrError::NoTerminator { expected, found, .. }) => if expected == found {
                     return (RAW_BYTE_STRING, Some("Missing trailing `\"` to terminate the raw byte string literal"))
                 } else {
                     return (RAW_BYTE_STRING, Some("Missing trailing `\"` with `#` symbols to terminate the raw byte string literal"))
 
                 },
-                Err(LexRawStrError::TooManyDelimiters { .. }) => return (RAW_BYTE_STRING, Some("Too many `#` symbols: raw byte strings may be delimited by up to 65535 `#` symbols")),
+                Some(RawStrError::TooManyDelimiters { .. }) => return (RAW_BYTE_STRING, Some("Too many `#` symbols: raw byte strings may be delimited by up to 65535 `#` symbols")),
             },
         };
 
diff --git a/crates/ra_syntax/src/parsing/reparsing.rs b/crates/ra_syntax/src/parsing/reparsing.rs
index edbc190f8..ed5a42ea3 100644
--- a/crates/ra_syntax/src/parsing/reparsing.rs
+++ b/crates/ra_syntax/src/parsing/reparsing.rs
@@ -120,10 +120,7 @@ fn get_text_after_edit(element: SyntaxElement, edit: &Indel) -> String {
 }
 
 fn is_contextual_kw(text: &str) -> bool {
-    match text {
-        "auto" | "default" | "union" => true,
-        _ => false,
-    }
+    matches!(text, "auto" | "default" | "union")
 }
 
 fn find_reparsable_node(node: &SyntaxNode, range: TextRange) -> Option<(SyntaxNode, Reparser)> {
diff --git a/crates/ra_syntax/src/parsing/text_token_source.rs b/crates/ra_syntax/src/parsing/text_token_source.rs
index 7ddc2c2c3..97aa3e795 100644
--- a/crates/ra_syntax/src/parsing/text_token_source.rs
+++ b/crates/ra_syntax/src/parsing/text_token_source.rs
@@ -1,40 +1,35 @@
-//! FIXME: write short doc here
+//! See `TextTokenSource` docs.
 
-use ra_parser::Token as PToken;
 use ra_parser::TokenSource;
 
 use crate::{parsing::lexer::Token, SyntaxKind::EOF, TextRange, TextSize};
 
+/// Implementation of `ra_parser::TokenSource` that takes tokens from source code text.
 pub(crate) struct TextTokenSource<'t> {
     text: &'t str,
-    /// start position of each token(expect whitespace and comment)
+    /// token and its start position (non-whitespace/comment tokens)
     /// ```non-rust
     ///  struct Foo;
-    /// ^------^---
-    /// |      |  ^-
-    /// 0      7  10
+    ///  ^------^--^-
+    ///  |      |    \________
+    ///  |      \____         \
+    ///  |           \         |
+    ///  (struct, 0) (Foo, 7) (;, 10)
     /// ```
-    /// (token, start_offset): `[(struct, 0), (Foo, 7), (;, 10)]`
-    start_offsets: Vec<TextSize>,
-    /// non-whitespace/comment tokens
-    /// ```non-rust
-    /// struct Foo {}
-    /// ^^^^^^ ^^^ ^^
-    /// ```
-    /// tokens: `[struct, Foo, {, }]`
-    tokens: Vec<Token>,
+    /// `[(struct, 0), (Foo, 7), (;, 10)]`
+    token_offset_pairs: Vec<(Token, TextSize)>,
 
     /// Current token and position
-    curr: (PToken, usize),
+    curr: (ra_parser::Token, usize),
 }
 
 impl<'t> TokenSource for TextTokenSource<'t> {
-    fn current(&self) -> PToken {
+    fn current(&self) -> ra_parser::Token {
         self.curr.0
     }
 
-    fn lookahead_nth(&self, n: usize) -> PToken {
-        mk_token(self.curr.1 + n, &self.start_offsets, &self.tokens)
+    fn lookahead_nth(&self, n: usize) -> ra_parser::Token {
+        mk_token(self.curr.1 + n, &self.token_offset_pairs)
     }
 
     fn bump(&mut self) {
@@ -43,45 +38,47 @@ impl<'t> TokenSource for TextTokenSource<'t> {
         }
 
         let pos = self.curr.1 + 1;
-        self.curr = (mk_token(pos, &self.start_offsets, &self.tokens), pos);
+        self.curr = (mk_token(pos, &self.token_offset_pairs), pos);
     }
 
     fn is_keyword(&self, kw: &str) -> bool {
-        let pos = self.curr.1;
-        if pos >= self.tokens.len() {
-            return false;
-        }
-        let range = TextRange::at(self.start_offsets[pos], self.tokens[pos].len);
-        self.text[range] == *kw
+        self.token_offset_pairs
+            .get(self.curr.1)
+            .map(|(token, offset)| &self.text[TextRange::at(*offset, token.len)] == kw)
+            .unwrap_or(false)
     }
 }
 
-fn mk_token(pos: usize, start_offsets: &[TextSize], tokens: &[Token]) -> PToken {
-    let kind = tokens.get(pos).map(|t| t.kind).unwrap_or(EOF);
-    let is_jointed_to_next = if pos + 1 < start_offsets.len() {
-        start_offsets[pos] + tokens[pos].len == start_offsets[pos + 1]
-    } else {
-        false
+fn mk_token(pos: usize, token_offset_pairs: &[(Token, TextSize)]) -> ra_parser::Token {
+    let (kind, is_jointed_to_next) = match token_offset_pairs.get(pos) {
+        Some((token, offset)) => (
+            token.kind,
+            token_offset_pairs
+                .get(pos + 1)
+                .map(|(_, next_offset)| offset + token.len == *next_offset)
+                .unwrap_or(false),
+        ),
+        None => (EOF, false),
     };
-
-    PToken { kind, is_jointed_to_next }
+    ra_parser::Token { kind, is_jointed_to_next }
 }
 
 impl<'t> TextTokenSource<'t> {
     /// Generate input from tokens(expect comment and whitespace).
     pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> TextTokenSource<'t> {
-        let mut tokens = Vec::new();
-        let mut start_offsets = Vec::new();
-        let mut len = 0.into();
-        for &token in raw_tokens.iter() {
-            if !token.kind.is_trivia() {
-                tokens.push(token);
-                start_offsets.push(len);
-            }
-            len += token.len;
-        }
+        let token_offset_pairs: Vec<_> = raw_tokens
+            .iter()
+            .filter_map({
+                let mut len = 0.into();
+                move |token| {
+                    let pair = if token.kind.is_trivia() { None } else { Some((*token, len)) };
+                    len += token.len;
+                    pair
+                }
+            })
+            .collect();
 
-        let first = mk_token(0, &start_offsets, &tokens);
-        TextTokenSource { text, start_offsets, tokens, curr: (first, 0) }
+        let first = mk_token(0, &token_offset_pairs);
+        TextTokenSource { text, token_offset_pairs, curr: (first, 0) }
     }
 }
diff --git a/crates/ra_syntax/src/parsing/text_tree_sink.rs b/crates/ra_syntax/src/parsing/text_tree_sink.rs
index 22aed1db1..c6b30a02a 100644
--- a/crates/ra_syntax/src/parsing/text_tree_sink.rs
+++ b/crates/ra_syntax/src/parsing/text_tree_sink.rs
@@ -160,7 +160,10 @@ fn n_attached_trivias<'a>(
                             if let Some((peek_kind, peek_text)) =
                                 trivias.peek().map(|(_, pair)| pair)
                             {
-                                if *peek_kind == COMMENT && peek_text.starts_with("///") {
+                                if *peek_kind == COMMENT
+                                    && peek_text.starts_with("///")
+                                    && !peek_text.starts_with("////")
+                                {
                                     continue;
                                 }
                             }