6 files changed, 94 insertions, 49 deletions

diff --git a/crates/ra_syntax/Cargo.toml b/crates/ra_syntax/Cargo.toml
index c07ff488e..a9a5cc7bc 100644
--- a/crates/ra_syntax/Cargo.toml
+++ b/crates/ra_syntax/Cargo.toml
@@ -13,7 +13,7 @@ doctest = false
 [dependencies]
 itertools = "0.9.0"
 rowan = "0.10.0"
-rustc_lexer = { version = "656.0.0", package = "rustc-ap-rustc_lexer" }
+rustc_lexer = { version = "660.0.0", package = "rustc-ap-rustc_lexer" }
 rustc-hash = "1.1.0"
 arrayvec = "0.5.1"
 once_cell = "1.3.1"
diff --git a/crates/ra_syntax/src/algo.rs b/crates/ra_syntax/src/algo.rs
index 2a8dac757..664894d1f 100644
--- a/crates/ra_syntax/src/algo.rs
+++ b/crates/ra_syntax/src/algo.rs
@@ -266,6 +266,15 @@ impl<'a> SyntaxRewriter<'a> {
         let replacement = Replacement::Single(with.clone().into());
         self.replacements.insert(what, replacement);
     }
+    pub fn replace_with_many<T: Clone + Into<SyntaxElement>>(
+        &mut self,
+        what: &T,
+        with: Vec<SyntaxElement>,
+    ) {
+        let what = what.clone().into();
+        let replacement = Replacement::Many(with);
+        self.replacements.insert(what, replacement);
+    }
     pub fn replace_ast<T: AstNode>(&mut self, what: &T, with: &T) {
         self.replace(what.syntax(), with.syntax())
     }
@@ -302,31 +311,41 @@ impl<'a> SyntaxRewriter<'a> {
 
     fn rewrite_children(&self, node: &SyntaxNode) -> SyntaxNode {
         //  FIXME: this could be made much faster.
-        let new_children =
-            node.children_with_tokens().flat_map(|it| self.rewrite_self(&it)).collect::<Vec<_>>();
+        let mut new_children = Vec::new();
+        for child in node.children_with_tokens() {
+            self.rewrite_self(&mut new_children, &child);
+        }
         with_children(node, new_children)
     }
 
     fn rewrite_self(
         &self,
+        acc: &mut Vec<NodeOrToken<rowan::GreenNode, rowan::GreenToken>>,
         element: &SyntaxElement,
-    ) -> Option<NodeOrToken<rowan::GreenNode, rowan::GreenToken>> {
+    ) {
         if let Some(replacement) = self.replacement(&element) {
-            return match replacement {
+            match replacement {
                 Replacement::Single(NodeOrToken::Node(it)) => {
-                    Some(NodeOrToken::Node(it.green().clone()))
+                    acc.push(NodeOrToken::Node(it.green().clone()))
                 }
                 Replacement::Single(NodeOrToken::Token(it)) => {
-                    Some(NodeOrToken::Token(it.green().clone()))
+                    acc.push(NodeOrToken::Token(it.green().clone()))
                 }
-                Replacement::Delete => None,
+                Replacement::Many(replacements) => {
+                    acc.extend(replacements.iter().map(|it| match it {
+                        NodeOrToken::Node(it) => NodeOrToken::Node(it.green().clone()),
+                        NodeOrToken::Token(it) => NodeOrToken::Token(it.green().clone()),
+                    }))
+                }
+                Replacement::Delete => (),
             };
+            return;
         }
         let res = match element {
             NodeOrToken::Token(it) => NodeOrToken::Token(it.green().clone()),
             NodeOrToken::Node(it) => NodeOrToken::Node(self.rewrite_children(it).green().clone()),
         };
-        Some(res)
+        acc.push(res)
     }
 }
 
@@ -341,6 +360,7 @@ impl ops::AddAssign for SyntaxRewriter<'_> {
 enum Replacement {
     Delete,
     Single(SyntaxElement),
+    Many(Vec<SyntaxElement>),
 }
 
 fn with_children(
diff --git a/crates/ra_syntax/src/ast/edit.rs b/crates/ra_syntax/src/ast/edit.rs
index 24a1e1d91..29eb3fcb9 100644
--- a/crates/ra_syntax/src/ast/edit.rs
+++ b/crates/ra_syntax/src/ast/edit.rs
@@ -1,7 +1,10 @@
 //! This module contains functions for editing syntax trees. As the trees are
 //! immutable, all function here return a fresh copy of the tree, instead of
 //! doing an in-place modification.
-use std::{iter, ops::RangeInclusive};
+use std::{
+    fmt, iter,
+    ops::{self, RangeInclusive},
+};
 
 use arrayvec::ArrayVec;
 
@@ -437,6 +440,28 @@ impl From<u8> for IndentLevel {
     }
 }
 
+impl fmt::Display for IndentLevel {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let spaces = "                                        ";
+        let buf;
+        let len = self.0 as usize * 4;
+        let indent = if len <= spaces.len() {
+            &spaces[..len]
+        } else {
+            buf = iter::repeat(' ').take(len).collect::<String>();
+            &buf
+        };
+        fmt::Display::fmt(indent, f)
+    }
+}
+
+impl ops::Add<u8> for IndentLevel {
+    type Output = IndentLevel;
+    fn add(self, rhs: u8) -> IndentLevel {
+        IndentLevel(self.0 + rhs)
+    }
+}
+
 impl IndentLevel {
     pub fn from_node(node: &SyntaxNode) -> IndentLevel {
         let first_token = match node.first_token() {
@@ -453,6 +478,14 @@ impl IndentLevel {
         IndentLevel(0)
     }
 
+    /// XXX: this intentionally doesn't change the indent of the very first token.
+    /// Ie, in something like
+    /// ```
+    /// fn foo() {
+    ///    92
+    /// }
+    /// ```
+    /// if you indent the block, the `{` token would stay put.
     fn increase_indent(self, node: SyntaxNode) -> SyntaxNode {
         let mut rewriter = SyntaxRewriter::default();
         node.descendants_with_tokens()
@@ -463,12 +496,7 @@ impl IndentLevel {
                 text.contains('\n')
             })
             .for_each(|ws| {
-                let new_ws = make::tokens::whitespace(&format!(
-                    "{}{:width$}",
-                    ws.syntax().text(),
-                    "",
-                    width = self.0 as usize * 4
-                ));
+                let new_ws = make::tokens::whitespace(&format!("{}{}", ws.syntax(), self,));
                 rewriter.replace(ws.syntax(), &new_ws)
             });
         rewriter.rewrite(&node)
@@ -485,7 +513,7 @@ impl IndentLevel {
             })
             .for_each(|ws| {
                 let new_ws = make::tokens::whitespace(
-                    &ws.syntax().text().replace(&format!("\n{:1$}", "", self.0 as usize * 4), "\n"),
+                    &ws.syntax().text().replace(&format!("\n{}", self), "\n"),
                 );
                 rewriter.replace(ws.syntax(), &new_ws)
             });
diff --git a/crates/ra_syntax/src/ast/make.rs b/crates/ra_syntax/src/ast/make.rs
index d0e960fb4..da0eb0926 100644
--- a/crates/ra_syntax/src/ast/make.rs
+++ b/crates/ra_syntax/src/ast/make.rs
@@ -1,5 +1,9 @@
 //! This module contains free-standing functions for creating AST fragments out
 //! of smaller pieces.
+//!
+//! Note that all functions here intended to be stupid constructors, which just
+//! assemble a finish node from immediate children. If you want to do something
+//! smarter than that, it probably doesn't belong in this module.
 use itertools::Itertools;
 use stdx::format_to;
 
@@ -95,6 +99,9 @@ pub fn expr_empty_block() -> ast::Expr {
 pub fn expr_unimplemented() -> ast::Expr {
     expr_from_text("unimplemented!()")
 }
+pub fn expr_unreachable() -> ast::Expr {
+    expr_from_text("unreachable!()")
+}
 pub fn expr_todo() -> ast::Expr {
     expr_from_text("todo!()")
 }
@@ -264,10 +271,6 @@ pub fn token(kind: SyntaxKind) -> SyntaxToken {
         .unwrap_or_else(|| panic!("unhandled token: {:?}", kind))
 }
 
-pub fn unreachable_macro_call() -> ast::MacroCall {
-    ast_from_text(&format!("unreachable!()"))
-}
-
 pub fn param(name: String, ty: String) -> ast::Param {
     ast_from_text(&format!("fn f({}: {}) {{ }}", name, ty))
 }
@@ -277,7 +280,12 @@ pub fn param_list(pats: impl IntoIterator<Item = ast::Param>) -> ast::ParamList
     ast_from_text(&format!("fn f({}) {{ }}", args))
 }
 
+pub fn visibility_pub_crate() -> ast::Visibility {
+    ast_from_text("pub(crate) struct S")
+}
+
 pub fn fn_def(
+    visibility: Option<ast::Visibility>,
     fn_name: ast::Name,
     type_params: Option<ast::TypeParamList>,
     params: ast::ParamList,
@@ -285,21 +293,11 @@ pub fn fn_def(
 ) -> ast::FnDef {
     let type_params =
         if let Some(type_params) = type_params { format!("<{}>", type_params) } else { "".into() };
-    ast_from_text(&format!("fn {}{}{} {}", fn_name, type_params, params, body))
-}
-
-pub fn add_leading_newlines(amount_of_newlines: usize, t: impl AstNode) -> ast::SourceFile {
-    let newlines = "\n".repeat(amount_of_newlines);
-    ast_from_text(&format!("{}{}", newlines, t.syntax()))
-}
-
-pub fn add_trailing_newlines(amount_of_newlines: usize, t: impl AstNode) -> ast::SourceFile {
-    let newlines = "\n".repeat(amount_of_newlines);
-    ast_from_text(&format!("{}{}", t.syntax(), newlines))
-}
-
-pub fn add_pub_crate_modifier(fn_def: ast::FnDef) -> ast::FnDef {
-    ast_from_text(&format!("pub(crate) {}", fn_def))
+    let visibility = match visibility {
+        None => String::new(),
+        Some(it) => format!("{} ", it),
+    };
+    ast_from_text(&format!("{}fn {}{}{} {}", visibility, fn_name, type_params, params, body))
 }
 
 fn ast_from_text<N: AstNode>(text: &str) -> N {
diff --git a/crates/ra_syntax/src/ast/tokens.rs b/crates/ra_syntax/src/ast/tokens.rs
index 74906d8a6..3cd6d99c3 100644
--- a/crates/ra_syntax/src/ast/tokens.rs
+++ b/crates/ra_syntax/src/ast/tokens.rs
@@ -6,6 +6,7 @@ use crate::{
     ast::{AstToken, Comment, RawString, String, Whitespace},
     TextRange, TextSize,
 };
+use rustc_lexer::unescape::{unescape_literal, Mode};
 
 impl Comment {
     pub fn kind(&self) -> CommentKind {
@@ -147,7 +148,7 @@ impl HasStringValue for String {
 
         let mut buf = std::string::String::with_capacity(text.len());
         let mut has_error = false;
-        rustc_lexer::unescape::unescape_str(text, &mut |_, unescaped_char| match unescaped_char {
+        unescape_literal(text, Mode::Str, &mut |_, unescaped_char| match unescaped_char {
             Ok(c) => buf.push(c),
             Err(_) => has_error = true,
         });
@@ -498,7 +499,7 @@ impl HasFormatSpecifier for String {
         let offset = self.text_range_between_quotes()?.start() - self.syntax().text_range().start();
 
         let mut res = Vec::with_capacity(text.len());
-        rustc_lexer::unescape::unescape_str(text, &mut |range, unescaped_char| {
+        unescape_literal(text, Mode::Str, &mut |range, unescaped_char| {
             res.push((
                 TextRange::new(range.start.try_into().unwrap(), range.end.try_into().unwrap())
                     + offset,
diff --git a/crates/ra_syntax/src/validation.rs b/crates/ra_syntax/src/validation.rs
index d68cf0a82..fdec48fb0 100644
--- a/crates/ra_syntax/src/validation.rs
+++ b/crates/ra_syntax/src/validation.rs
@@ -2,15 +2,15 @@
 
 mod block;
 
-use std::convert::TryFrom;
-
-use rustc_lexer::unescape;
-
 use crate::{
     ast, match_ast, AstNode, SyntaxError,
     SyntaxKind::{BYTE, BYTE_STRING, CHAR, CONST_DEF, FN_DEF, INT_NUMBER, STRING, TYPE_ALIAS_DEF},
     SyntaxNode, SyntaxToken, TextSize, T,
 };
+use rustc_lexer::unescape::{
+    self, unescape_byte, unescape_byte_literal, unescape_char, unescape_literal, Mode,
+};
+use std::convert::TryFrom;
 
 fn rustc_unescape_error_to_string(err: unescape::EscapeError) -> &'static str {
     use unescape::EscapeError as EE;
@@ -81,10 +81,8 @@ fn rustc_unescape_error_to_string(err: unescape::EscapeError) -> &'static str {
 
 pub(crate) fn validate(root: &SyntaxNode) -> Vec<SyntaxError> {
     // FIXME:
-    // * Add validation of character literal containing only a single char
-    // * Add validation of `crate` keyword not appearing in the middle of the symbol path
+    // * Add unescape validation of raw string literals and raw byte string literals
     // * Add validation of doc comments are being attached to nodes
-    // * Remove validation of unterminated literals (it is already implemented in `tokenize()`)
 
     let mut errors = Vec::new();
     for node in root.descendants() {
@@ -121,18 +119,18 @@ fn validate_literal(literal: ast::Literal, acc: &mut Vec<SyntaxError>) {
 
     match token.kind() {
         BYTE => {
-            if let Some(Err(e)) = unquote(text, 2, '\'').map(unescape::unescape_byte) {
+            if let Some(Err(e)) = unquote(text, 2, '\'').map(unescape_byte) {
                 push_err(2, e);
             }
         }
         CHAR => {
-            if let Some(Err(e)) = unquote(text, 1, '\'').map(unescape::unescape_char) {
+            if let Some(Err(e)) = unquote(text, 1, '\'').map(unescape_char) {
                 push_err(1, e);
             }
         }
         BYTE_STRING => {
             if let Some(without_quotes) = unquote(text, 2, '"') {
-                unescape::unescape_byte_str(without_quotes, &mut |range, char| {
+                unescape_byte_literal(without_quotes, Mode::ByteStr, &mut |range, char| {
                     if let Err(err) = char {
                         push_err(2, (range.start, err));
                     }
@@ -141,7 +139,7 @@ fn validate_literal(literal: ast::Literal, acc: &mut Vec<SyntaxError>) {
         }
         STRING => {
             if let Some(without_quotes) = unquote(text, 1, '"') {
-                unescape::unescape_str(without_quotes, &mut |range, char| {
+                unescape_literal(without_quotes, Mode::Str, &mut |range, char| {
                     if let Err(err) = char {
                         push_err(1, (range.start, err));
                     }