From ac798e1f7cfbc6d27c87bb28e3f1d5b6801796aa Mon Sep 17 00:00:00 2001 From: Leander Tentrup Date: Fri, 17 Apr 2020 09:37:18 +0200 Subject: Implement syntax highlighting for format strings Detailed changes: 1) Implement a lexer for string literals that divides the string in format specifier `{}` including the format specifier modifier. 2) Adapt syntax highlighting to add ranges for the detected sequences. 3) Add a test case for the format string syntax highlighting. --- crates/ra_syntax/src/ast/tokens.rs | 324 +++++++++++++++++++++++++++++++++++++ 1 file changed, 324 insertions(+) (limited to 'crates/ra_syntax/src') diff --git a/crates/ra_syntax/src/ast/tokens.rs b/crates/ra_syntax/src/ast/tokens.rs index e8320b57e..ec3b4e553 100644 --- a/crates/ra_syntax/src/ast/tokens.rs +++ b/crates/ra_syntax/src/ast/tokens.rs @@ -172,3 +172,327 @@ impl RawString { Some(range + contents_range.start()) } } + +#[derive(Debug)] +pub enum FormatSpecifier { + Open, + Close, + Integer, + Identifier, + Colon, + Fill, + Align, + Sign, + NumberSign, + Zero, + DollarSign, + Dot, + Asterisk, + QuestionMark, +} + +pub trait HasFormatSpecifier: AstToken { + fn lex_format_specifier(&self, callback: &mut F) + where + F: FnMut(TextRange, FormatSpecifier), + { + let src = self.text().as_str(); + let initial_len = src.len(); + let mut chars = src.chars(); + + while let Some(first_char) = chars.next() { + match first_char { + '{' => { + // Format specifier, see syntax at https://doc.rust-lang.org/std/fmt/index.html#syntax + if chars.clone().next() == Some('{') { + // Escaped format specifier, `{{` + chars.next(); + continue; + } + + let start = initial_len - chars.as_str().len() - first_char.len_utf8(); + let end = initial_len - chars.as_str().len(); + callback( + TextRange::from_to(TextUnit::from_usize(start), TextUnit::from_usize(end)), + FormatSpecifier::Open, + ); + + let next_char = if let Some(c) = chars.clone().next() { + c + } else { + break; + }; + + // check for integer/identifier + match next_char { + '0'..='9' => { + // integer + read_integer(&mut chars, initial_len, callback); + } + 'a'..='z' | 'A'..='Z' | '_' => { + // identifier + read_identifier(&mut chars, initial_len, callback); + } + _ => {} + } + + if chars.clone().next() == Some(':') { + skip_char_and_emit( + &mut chars, + initial_len, + FormatSpecifier::Colon, + callback, + ); + + // check for fill/align + let mut cloned = chars.clone().take(2); + let first = cloned.next().unwrap_or_default(); + let second = cloned.next().unwrap_or_default(); + match second { + '<' | '^' | '>' => { + // alignment specifier, first char specifies fillment + skip_char_and_emit( + &mut chars, + initial_len, + FormatSpecifier::Fill, + callback, + ); + skip_char_and_emit( + &mut chars, + initial_len, + FormatSpecifier::Align, + callback, + ); + } + _ => match first { + '<' | '^' | '>' => { + skip_char_and_emit( + &mut chars, + initial_len, + FormatSpecifier::Align, + callback, + ); + } + _ => {} + }, + } + + // check for sign + match chars.clone().next().unwrap_or_default() { + '+' | '-' => { + skip_char_and_emit( + &mut chars, + initial_len, + FormatSpecifier::Sign, + callback, + ); + } + _ => {} + } + + // check for `#` + if let Some('#') = chars.clone().next() { + skip_char_and_emit( + &mut chars, + initial_len, + FormatSpecifier::NumberSign, + callback, + ); + } + + // check for `0` + let mut cloned = chars.clone().take(2); + let first = cloned.next(); + let second = cloned.next(); + + if first == Some('0') && second != Some('$') { + skip_char_and_emit( + &mut chars, + initial_len, + FormatSpecifier::Zero, + callback, + ); + } + + // width + match chars.clone().next().unwrap_or_default() { + '0'..='9' => { + read_integer(&mut chars, initial_len, callback); + if chars.clone().next() == Some('$') { + skip_char_and_emit( + &mut chars, + initial_len, + FormatSpecifier::DollarSign, + callback, + ); + } + } + 'a'..='z' | 'A'..='Z' | '_' => { + read_identifier(&mut chars, initial_len, callback); + if chars.clone().next() != Some('$') { + continue; + } + skip_char_and_emit( + &mut chars, + initial_len, + FormatSpecifier::DollarSign, + callback, + ); + } + _ => {} + } + + // precision + if chars.clone().next() == Some('.') { + skip_char_and_emit( + &mut chars, + initial_len, + FormatSpecifier::Dot, + callback, + ); + + match chars.clone().next().unwrap_or_default() { + '*' => { + skip_char_and_emit( + &mut chars, + initial_len, + FormatSpecifier::Asterisk, + callback, + ); + } + '0'..='9' => { + read_integer(&mut chars, initial_len, callback); + if chars.clone().next() == Some('$') { + skip_char_and_emit( + &mut chars, + initial_len, + FormatSpecifier::DollarSign, + callback, + ); + } + } + 'a'..='z' | 'A'..='Z' | '_' => { + read_identifier(&mut chars, initial_len, callback); + if chars.clone().next() != Some('$') { + continue; + } + skip_char_and_emit( + &mut chars, + initial_len, + FormatSpecifier::DollarSign, + callback, + ); + } + _ => { + continue; + } + } + } + + // type + match chars.clone().next().unwrap_or_default() { + '?' => { + skip_char_and_emit( + &mut chars, + initial_len, + FormatSpecifier::QuestionMark, + callback, + ); + } + 'a'..='z' | 'A'..='Z' | '_' => { + read_identifier(&mut chars, initial_len, callback); + } + _ => {} + } + } + + let mut cloned = chars.clone().take(2); + let first = cloned.next(); + let second = cloned.next(); + if first != Some('}') { + continue; + } + if second == Some('}') { + // Escaped format end specifier, `}}` + continue; + } + skip_char_and_emit(&mut chars, initial_len, FormatSpecifier::Close, callback); + } + _ => { + while let Some(next_char) = chars.clone().next() { + match next_char { + '{' => break, + _ => {} + } + chars.next(); + } + } + }; + } + + fn skip_char_and_emit( + chars: &mut std::str::Chars, + initial_len: usize, + emit: FormatSpecifier, + callback: &mut F, + ) where + F: FnMut(TextRange, FormatSpecifier), + { + let start = initial_len - chars.as_str().len(); + chars.next(); + let end = initial_len - chars.as_str().len(); + callback( + TextRange::from_to(TextUnit::from_usize(start), TextUnit::from_usize(end)), + emit, + ); + } + + fn read_integer(chars: &mut std::str::Chars, initial_len: usize, callback: &mut F) + where + F: FnMut(TextRange, FormatSpecifier), + { + let start = initial_len - chars.as_str().len(); + chars.next(); + while let Some(next_char) = chars.clone().next() { + match next_char { + '0'..='9' => { + chars.next(); + } + _ => { + break; + } + } + } + let end = initial_len - chars.as_str().len(); + callback( + TextRange::from_to(TextUnit::from_usize(start), TextUnit::from_usize(end)), + FormatSpecifier::Integer, + ); + } + fn read_identifier(chars: &mut std::str::Chars, initial_len: usize, callback: &mut F) + where + F: FnMut(TextRange, FormatSpecifier), + { + let start = initial_len - chars.as_str().len(); + chars.next(); + while let Some(next_char) = chars.clone().next() { + match next_char { + 'a'..='z' | 'A'..='Z' | '0'..='9' | '_' => { + chars.next(); + } + _ => { + break; + } + } + } + let end = initial_len - chars.as_str().len(); + callback( + TextRange::from_to(TextUnit::from_usize(start), TextUnit::from_usize(end)), + FormatSpecifier::Identifier, + ); + } + } +} + +impl HasFormatSpecifier for String {} +impl HasFormatSpecifier for RawString {} -- cgit v1.2.3 From 317fc650d5c1267a8c192041efe6b591d900808f Mon Sep 17 00:00:00 2001 From: Timo Freiberg Date: Thu, 16 Apr 2020 23:53:25 +0200 Subject: Make add_function generate functions in other modules via qualified path --- crates/ra_syntax/src/ast/make.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'crates/ra_syntax/src') diff --git a/crates/ra_syntax/src/ast/make.rs b/crates/ra_syntax/src/ast/make.rs index 0f4a50be4..b0f4803f3 100644 --- a/crates/ra_syntax/src/ast/make.rs +++ b/crates/ra_syntax/src/ast/make.rs @@ -293,11 +293,16 @@ pub fn fn_def( ast_from_text(&format!("fn {}{}{} {}", fn_name, type_params, params, body)) } -pub fn add_newlines(amount_of_newlines: usize, t: impl AstNode) -> ast::SourceFile { +pub fn add_leading_newlines(amount_of_newlines: usize, t: impl AstNode) -> ast::SourceFile { let newlines = "\n".repeat(amount_of_newlines); ast_from_text(&format!("{}{}", newlines, t.syntax())) } +pub fn add_trailing_newlines(amount_of_newlines: usize, t: impl AstNode) -> ast::SourceFile { + let newlines = "\n".repeat(amount_of_newlines); + ast_from_text(&format!("{}{}", t.syntax(), newlines)) +} + fn ast_from_text(text: &str) -> N { let parse = SourceFile::parse(text); let node = parse.tree().syntax().descendants().find_map(N::cast).unwrap(); -- cgit v1.2.3 From f2f882bc44a85eb13276a8fbda7533d94e92e3af Mon Sep 17 00:00:00 2001 From: Timo Freiberg Date: Mon, 20 Apr 2020 18:36:12 +0200 Subject: Add `pub(crate)` to functions generated in other module --- crates/ra_syntax/src/ast/make.rs | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'crates/ra_syntax/src') diff --git a/crates/ra_syntax/src/ast/make.rs b/crates/ra_syntax/src/ast/make.rs index b0f4803f3..ee0f5cc40 100644 --- a/crates/ra_syntax/src/ast/make.rs +++ b/crates/ra_syntax/src/ast/make.rs @@ -303,6 +303,10 @@ pub fn add_trailing_newlines(amount_of_newlines: usize, t: impl AstNode) -> ast: ast_from_text(&format!("{}{}", t.syntax(), newlines)) } +pub fn add_pub_crate_modifier(fn_def: ast::FnDef) -> ast::FnDef { + ast_from_text(&format!("pub(crate) {}", fn_def)) +} + fn ast_from_text(text: &str) -> N { let parse = SourceFile::parse(text); let node = parse.tree().syntax().descendants().find_map(N::cast).unwrap(); -- cgit v1.2.3 From b2829a52161bc414f3b361c06b66633a234bba16 Mon Sep 17 00:00:00 2001 From: Leander Tentrup Date: Wed, 22 Apr 2020 10:08:46 +0200 Subject: Apply suggestions from code review Co-Authored-By: bjorn3 --- crates/ra_syntax/src/ast/tokens.rs | 59 +++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 30 deletions(-) (limited to 'crates/ra_syntax/src') diff --git a/crates/ra_syntax/src/ast/tokens.rs b/crates/ra_syntax/src/ast/tokens.rs index ec3b4e553..3e5c56b19 100644 --- a/crates/ra_syntax/src/ast/tokens.rs +++ b/crates/ra_syntax/src/ast/tokens.rs @@ -192,7 +192,7 @@ pub enum FormatSpecifier { } pub trait HasFormatSpecifier: AstToken { - fn lex_format_specifier(&self, callback: &mut F) + fn lex_format_specifier(&self, mut callback: F) where F: FnMut(TextRange, FormatSpecifier), { @@ -217,21 +217,15 @@ pub trait HasFormatSpecifier: AstToken { FormatSpecifier::Open, ); - let next_char = if let Some(c) = chars.clone().next() { - c - } else { - break; - }; - // check for integer/identifier - match next_char { + match chars.clone().next().unwrap_or_default() { '0'..='9' => { // integer - read_integer(&mut chars, initial_len, callback); + read_integer(&mut chars, initial_len, &mut callback); } 'a'..='z' | 'A'..='Z' | '_' => { // identifier - read_identifier(&mut chars, initial_len, callback); + read_identifier(&mut chars, initial_len, &mut callback); } _ => {} } @@ -241,7 +235,7 @@ pub trait HasFormatSpecifier: AstToken { &mut chars, initial_len, FormatSpecifier::Colon, - callback, + &mut callback, ); // check for fill/align @@ -255,13 +249,13 @@ pub trait HasFormatSpecifier: AstToken { &mut chars, initial_len, FormatSpecifier::Fill, - callback, + &mut callback, ); skip_char_and_emit( &mut chars, initial_len, FormatSpecifier::Align, - callback, + &mut callback, ); } _ => match first { @@ -270,7 +264,7 @@ pub trait HasFormatSpecifier: AstToken { &mut chars, initial_len, FormatSpecifier::Align, - callback, + &mut callback, ); } _ => {} @@ -284,7 +278,7 @@ pub trait HasFormatSpecifier: AstToken { &mut chars, initial_len, FormatSpecifier::Sign, - callback, + &mut callback, ); } _ => {} @@ -296,7 +290,7 @@ pub trait HasFormatSpecifier: AstToken { &mut chars, initial_len, FormatSpecifier::NumberSign, - callback, + &mut callback, ); } @@ -310,25 +304,25 @@ pub trait HasFormatSpecifier: AstToken { &mut chars, initial_len, FormatSpecifier::Zero, - callback, + &mut callback, ); } // width match chars.clone().next().unwrap_or_default() { '0'..='9' => { - read_integer(&mut chars, initial_len, callback); + read_integer(&mut chars, initial_len, &mut callback); if chars.clone().next() == Some('$') { skip_char_and_emit( &mut chars, initial_len, FormatSpecifier::DollarSign, - callback, + &mut callback, ); } } 'a'..='z' | 'A'..='Z' | '_' => { - read_identifier(&mut chars, initial_len, callback); + read_identifier(&mut chars, initial_len, &mut callback); if chars.clone().next() != Some('$') { continue; } @@ -336,7 +330,7 @@ pub trait HasFormatSpecifier: AstToken { &mut chars, initial_len, FormatSpecifier::DollarSign, - callback, + &mut callback, ); } _ => {} @@ -348,7 +342,7 @@ pub trait HasFormatSpecifier: AstToken { &mut chars, initial_len, FormatSpecifier::Dot, - callback, + &mut callback, ); match chars.clone().next().unwrap_or_default() { @@ -357,22 +351,22 @@ pub trait HasFormatSpecifier: AstToken { &mut chars, initial_len, FormatSpecifier::Asterisk, - callback, + &mut callback, ); } '0'..='9' => { - read_integer(&mut chars, initial_len, callback); + read_integer(&mut chars, initial_len, &mut callback); if chars.clone().next() == Some('$') { skip_char_and_emit( &mut chars, initial_len, FormatSpecifier::DollarSign, - callback, + &mut callback, ); } } 'a'..='z' | 'A'..='Z' | '_' => { - read_identifier(&mut chars, initial_len, callback); + read_identifier(&mut chars, initial_len, &mut callback); if chars.clone().next() != Some('$') { continue; } @@ -380,7 +374,7 @@ pub trait HasFormatSpecifier: AstToken { &mut chars, initial_len, FormatSpecifier::DollarSign, - callback, + &mut callback, ); } _ => { @@ -396,11 +390,11 @@ pub trait HasFormatSpecifier: AstToken { &mut chars, initial_len, FormatSpecifier::QuestionMark, - callback, + &mut callback, ); } 'a'..='z' | 'A'..='Z' | '_' => { - read_identifier(&mut chars, initial_len, callback); + read_identifier(&mut chars, initial_len, &mut callback); } _ => {} } @@ -416,7 +410,12 @@ pub trait HasFormatSpecifier: AstToken { // Escaped format end specifier, `}}` continue; } - skip_char_and_emit(&mut chars, initial_len, FormatSpecifier::Close, callback); + skip_char_and_emit( + &mut chars, + initial_len, + FormatSpecifier::Close, + &mut callback, + ); } _ => { while let Some(next_char) = chars.clone().next() { -- cgit v1.2.3 From 445052f6d426043b543033f3fa4594fc1a09d7fa Mon Sep 17 00:00:00 2001 From: Leander Tentrup Date: Wed, 22 Apr 2020 15:28:35 +0200 Subject: Adapt format specifier highlighting to support escaped squences and unicode identifiers --- crates/ra_syntax/src/ast/tokens.rs | 280 +++++++++++++++++++++---------------- 1 file changed, 158 insertions(+), 122 deletions(-) (limited to 'crates/ra_syntax/src') diff --git a/crates/ra_syntax/src/ast/tokens.rs b/crates/ra_syntax/src/ast/tokens.rs index 3e5c56b19..aa34b682d 100644 --- a/crates/ra_syntax/src/ast/tokens.rs +++ b/crates/ra_syntax/src/ast/tokens.rs @@ -192,68 +192,76 @@ pub enum FormatSpecifier { } pub trait HasFormatSpecifier: AstToken { + fn char_ranges( + &self, + ) -> Option)>>; + fn lex_format_specifier(&self, mut callback: F) where F: FnMut(TextRange, FormatSpecifier), { - let src = self.text().as_str(); - let initial_len = src.len(); - let mut chars = src.chars(); + let char_ranges = if let Some(char_ranges) = self.char_ranges() { + char_ranges + } else { + return; + }; + let mut chars = char_ranges.iter().peekable(); - while let Some(first_char) = chars.next() { + while let Some((range, first_char)) = chars.next() { match first_char { - '{' => { + Ok('{') => { // Format specifier, see syntax at https://doc.rust-lang.org/std/fmt/index.html#syntax - if chars.clone().next() == Some('{') { + if let Some((_, Ok('{'))) = chars.peek() { // Escaped format specifier, `{{` chars.next(); continue; } - let start = initial_len - chars.as_str().len() - first_char.len_utf8(); - let end = initial_len - chars.as_str().len(); - callback( - TextRange::from_to(TextUnit::from_usize(start), TextUnit::from_usize(end)), - FormatSpecifier::Open, - ); + callback(*range, FormatSpecifier::Open); // check for integer/identifier - match chars.clone().next().unwrap_or_default() { + match chars + .peek() + .and_then(|next| next.1.as_ref().ok()) + .copied() + .unwrap_or_default() + { '0'..='9' => { // integer - read_integer(&mut chars, initial_len, &mut callback); + read_integer(&mut chars, &mut callback); } - 'a'..='z' | 'A'..='Z' | '_' => { + c if c == '_' || c.is_alphabetic() => { // identifier - read_identifier(&mut chars, initial_len, &mut callback); + read_identifier(&mut chars, &mut callback); } _ => {} } - if chars.clone().next() == Some(':') { - skip_char_and_emit( - &mut chars, - initial_len, - FormatSpecifier::Colon, - &mut callback, - ); + if let Some((_, Ok(':'))) = chars.peek() { + skip_char_and_emit(&mut chars, FormatSpecifier::Colon, &mut callback); // check for fill/align let mut cloned = chars.clone().take(2); - let first = cloned.next().unwrap_or_default(); - let second = cloned.next().unwrap_or_default(); + let first = cloned + .next() + .and_then(|next| next.1.as_ref().ok()) + .copied() + .unwrap_or_default(); + let second = cloned + .next() + .and_then(|next| next.1.as_ref().ok()) + .copied() + .unwrap_or_default(); match second { '<' | '^' | '>' => { // alignment specifier, first char specifies fillment skip_char_and_emit( &mut chars, - initial_len, FormatSpecifier::Fill, &mut callback, ); skip_char_and_emit( &mut chars, - initial_len, FormatSpecifier::Align, &mut callback, ); @@ -262,7 +270,6 @@ pub trait HasFormatSpecifier: AstToken { '<' | '^' | '>' => { skip_char_and_emit( &mut chars, - initial_len, FormatSpecifier::Align, &mut callback, ); @@ -272,11 +279,15 @@ pub trait HasFormatSpecifier: AstToken { } // check for sign - match chars.clone().next().unwrap_or_default() { + match chars + .peek() + .and_then(|next| next.1.as_ref().ok()) + .copied() + .unwrap_or_default() + { '+' | '-' => { skip_char_and_emit( &mut chars, - initial_len, FormatSpecifier::Sign, &mut callback, ); @@ -285,10 +296,9 @@ pub trait HasFormatSpecifier: AstToken { } // check for `#` - if let Some('#') = chars.clone().next() { + if let Some((_, Ok('#'))) = chars.peek() { skip_char_and_emit( &mut chars, - initial_len, FormatSpecifier::NumberSign, &mut callback, ); @@ -296,39 +306,39 @@ pub trait HasFormatSpecifier: AstToken { // check for `0` let mut cloned = chars.clone().take(2); - let first = cloned.next(); - let second = cloned.next(); + let first = cloned.next().and_then(|next| next.1.as_ref().ok()).copied(); + let second = cloned.next().and_then(|next| next.1.as_ref().ok()).copied(); if first == Some('0') && second != Some('$') { - skip_char_and_emit( - &mut chars, - initial_len, - FormatSpecifier::Zero, - &mut callback, - ); + skip_char_and_emit(&mut chars, FormatSpecifier::Zero, &mut callback); } // width - match chars.clone().next().unwrap_or_default() { + match chars + .peek() + .and_then(|next| next.1.as_ref().ok()) + .copied() + .unwrap_or_default() + { '0'..='9' => { - read_integer(&mut chars, initial_len, &mut callback); - if chars.clone().next() == Some('$') { + read_integer(&mut chars, &mut callback); + if let Some((_, Ok('$'))) = chars.peek() { skip_char_and_emit( &mut chars, - initial_len, FormatSpecifier::DollarSign, &mut callback, ); } } - 'a'..='z' | 'A'..='Z' | '_' => { - read_identifier(&mut chars, initial_len, &mut callback); - if chars.clone().next() != Some('$') { + c if c == '_' || c.is_alphabetic() => { + read_identifier(&mut chars, &mut callback); + if chars.peek().and_then(|next| next.1.as_ref().ok()).copied() + != Some('$') + { continue; } skip_char_and_emit( &mut chars, - initial_len, FormatSpecifier::DollarSign, &mut callback, ); @@ -337,42 +347,41 @@ pub trait HasFormatSpecifier: AstToken { } // precision - if chars.clone().next() == Some('.') { - skip_char_and_emit( - &mut chars, - initial_len, - FormatSpecifier::Dot, - &mut callback, - ); - - match chars.clone().next().unwrap_or_default() { + if let Some((_, Ok('.'))) = chars.peek() { + skip_char_and_emit(&mut chars, FormatSpecifier::Dot, &mut callback); + + match chars + .peek() + .and_then(|next| next.1.as_ref().ok()) + .copied() + .unwrap_or_default() + { '*' => { skip_char_and_emit( &mut chars, - initial_len, FormatSpecifier::Asterisk, &mut callback, ); } '0'..='9' => { - read_integer(&mut chars, initial_len, &mut callback); - if chars.clone().next() == Some('$') { + read_integer(&mut chars, &mut callback); + if let Some((_, Ok('$'))) = chars.peek() { skip_char_and_emit( &mut chars, - initial_len, FormatSpecifier::DollarSign, &mut callback, ); } } - 'a'..='z' | 'A'..='Z' | '_' => { - read_identifier(&mut chars, initial_len, &mut callback); - if chars.clone().next() != Some('$') { + c if c == '_' || c.is_alphabetic() => { + read_identifier(&mut chars, &mut callback); + if chars.peek().and_then(|next| next.1.as_ref().ok()).copied() + != Some('$') + { continue; } skip_char_and_emit( &mut chars, - initial_len, FormatSpecifier::DollarSign, &mut callback, ); @@ -384,25 +393,29 @@ pub trait HasFormatSpecifier: AstToken { } // type - match chars.clone().next().unwrap_or_default() { + match chars + .peek() + .and_then(|next| next.1.as_ref().ok()) + .copied() + .unwrap_or_default() + { '?' => { skip_char_and_emit( &mut chars, - initial_len, FormatSpecifier::QuestionMark, &mut callback, ); } - 'a'..='z' | 'A'..='Z' | '_' => { - read_identifier(&mut chars, initial_len, &mut callback); + c if c == '_' || c.is_alphabetic() => { + read_identifier(&mut chars, &mut callback); } _ => {} } } let mut cloned = chars.clone().take(2); - let first = cloned.next(); - let second = cloned.next(); + let first = cloned.next().and_then(|next| next.1.as_ref().ok()).copied(); + let second = cloned.next().and_then(|next| next.1.as_ref().ok()).copied(); if first != Some('}') { continue; } @@ -410,15 +423,10 @@ pub trait HasFormatSpecifier: AstToken { // Escaped format end specifier, `}}` continue; } - skip_char_and_emit( - &mut chars, - initial_len, - FormatSpecifier::Close, - &mut callback, - ); + skip_char_and_emit(&mut chars, FormatSpecifier::Close, &mut callback); } _ => { - while let Some(next_char) = chars.clone().next() { + while let Some((_, Ok(next_char))) = chars.peek() { match next_char { '{' => break, _ => {} @@ -429,69 +437,97 @@ pub trait HasFormatSpecifier: AstToken { }; } - fn skip_char_and_emit( - chars: &mut std::str::Chars, - initial_len: usize, + fn skip_char_and_emit<'a, I, F>( + chars: &mut std::iter::Peekable, emit: FormatSpecifier, callback: &mut F, ) where + I: Iterator)>, F: FnMut(TextRange, FormatSpecifier), { - let start = initial_len - chars.as_str().len(); - chars.next(); - let end = initial_len - chars.as_str().len(); - callback( - TextRange::from_to(TextUnit::from_usize(start), TextUnit::from_usize(end)), - emit, - ); + let (range, _) = chars.next().unwrap(); + callback(*range, emit); } - fn read_integer(chars: &mut std::str::Chars, initial_len: usize, callback: &mut F) + fn read_integer<'a, I, F>(chars: &mut std::iter::Peekable, callback: &mut F) where + I: Iterator)>, F: FnMut(TextRange, FormatSpecifier), { - let start = initial_len - chars.as_str().len(); - chars.next(); - while let Some(next_char) = chars.clone().next() { - match next_char { - '0'..='9' => { - chars.next(); - } - _ => { - break; - } + let (mut range, c) = chars.next().unwrap(); + assert!(c.as_ref().unwrap().is_ascii_digit()); + while let Some((r, Ok(next_char))) = chars.peek() { + if next_char.is_ascii_digit() { + chars.next(); + range = range.extend_to(r); + } else { + break; } } - let end = initial_len - chars.as_str().len(); - callback( - TextRange::from_to(TextUnit::from_usize(start), TextUnit::from_usize(end)), - FormatSpecifier::Integer, - ); + callback(range, FormatSpecifier::Integer); } - fn read_identifier(chars: &mut std::str::Chars, initial_len: usize, callback: &mut F) + + fn read_identifier<'a, I, F>(chars: &mut std::iter::Peekable, callback: &mut F) where + I: Iterator)>, F: FnMut(TextRange, FormatSpecifier), { - let start = initial_len - chars.as_str().len(); - chars.next(); - while let Some(next_char) = chars.clone().next() { - match next_char { - 'a'..='z' | 'A'..='Z' | '0'..='9' | '_' => { - chars.next(); - } - _ => { - break; - } + let (mut range, c) = chars.next().unwrap(); + assert!(c.as_ref().unwrap().is_alphabetic() || *c.as_ref().unwrap() == '_'); + while let Some((r, Ok(next_char))) = chars.peek() { + if *next_char == '_' || next_char.is_ascii_digit() || next_char.is_alphabetic() { + chars.next(); + range = range.extend_to(r); + } else { + break; } } - let end = initial_len - chars.as_str().len(); - callback( - TextRange::from_to(TextUnit::from_usize(start), TextUnit::from_usize(end)), - FormatSpecifier::Identifier, - ); + callback(range, FormatSpecifier::Identifier); } } } -impl HasFormatSpecifier for String {} -impl HasFormatSpecifier for RawString {} +impl HasFormatSpecifier for String { + fn char_ranges( + &self, + ) -> Option)>> { + let text = self.text().as_str(); + let text = &text[self.text_range_between_quotes()? - self.syntax().text_range().start()]; + let offset = self.text_range_between_quotes()?.start() - self.syntax().text_range().start(); + + let mut res = Vec::with_capacity(text.len()); + rustc_lexer::unescape::unescape_str(text, &mut |range, unescaped_char| { + res.push(( + TextRange::from_to( + TextUnit::from_usize(range.start), + TextUnit::from_usize(range.end), + ) + offset, + unescaped_char, + )) + }); + + Some(res) + } +} + +impl HasFormatSpecifier for RawString { + fn char_ranges( + &self, + ) -> Option)>> { + let text = self.text().as_str(); + let text = &text[self.text_range_between_quotes()? - self.syntax().text_range().start()]; + let offset = self.text_range_between_quotes()?.start() - self.syntax().text_range().start(); + + let mut res = Vec::with_capacity(text.len()); + for (idx, c) in text.char_indices() { + res.push(( + TextRange::from_to( + TextUnit::from_usize(idx), + TextUnit::from_usize(idx + c.len_utf8()), + ) + offset, + Ok(c), + )); + } + Some(res) + } +} -- cgit v1.2.3