Merge #7657

7657: utf8 r=matklad a=matklad

- Prepare for utf-8 offsets
- reduce code duplication in tests
- Make utf8 default, implement utf16 in terms of it
- Make it easy to add additional context for offset conversion
- Implement utf8 offsets

closes #7453

Co-authored-by: Aleksey Kladov <[email protected]>

1 files changed, 47 insertions, 43 deletions

diff --git a/crates/rust-analyzer/src/to_proto.rs b/crates/rust-analyzer/src/to_proto.rs
index b0ddb603d..70cb7fbab 100644
--- a/crates/rust-analyzer/src/to_proto.rs
+++ b/crates/rust-analyzer/src/to_proto.rs
@@ -7,22 +7,29 @@ use std::{
 use ide::{
     Annotation, AnnotationKind, Assist, AssistKind, CallInfo, CompletionItem, CompletionItemKind,
     Documentation, FileId, FileRange, FileSystemEdit, Fold, FoldKind, Highlight, HlMod, HlPunct,
-    HlRange, HlTag, Indel, InlayHint, InlayKind, InsertTextFormat, LineIndex, Markup,
-    NavigationTarget, ReferenceAccess, RenameError, Runnable, Severity, SourceChange, TextEdit,
-    TextRange, TextSize,
+    HlRange, HlTag, Indel, InlayHint, InlayKind, InsertTextFormat, Markup, NavigationTarget,
+    ReferenceAccess, RenameError, Runnable, Severity, SourceChange, TextEdit, TextRange, TextSize,
 };
 use ide_db::SymbolKind;
 use itertools::Itertools;
 use serde_json::to_value;
 
 use crate::{
-    cargo_target_spec::CargoTargetSpec, global_state::GlobalStateSnapshot,
-    line_endings::LineEndings, lsp_ext, semantic_tokens, Result,
+    cargo_target_spec::CargoTargetSpec,
+    global_state::GlobalStateSnapshot,
+    line_index::{LineEndings, LineIndex, OffsetEncoding},
+    lsp_ext, semantic_tokens, Result,
 };
 
 pub(crate) fn position(line_index: &LineIndex, offset: TextSize) -> lsp_types::Position {
-    let line_col = line_index.line_col(offset);
-    lsp_types::Position::new(line_col.line, line_col.col_utf16)
+    let line_col = line_index.index.line_col(offset);
+    match line_index.encoding {
+        OffsetEncoding::Utf8 => lsp_types::Position::new(line_col.line, line_col.col),
+        OffsetEncoding::Utf16 => {
+            let line_col = line_index.index.to_utf16(line_col);
+            lsp_types::Position::new(line_col.line, line_col.col)
+        }
+    }
 }
 
 pub(crate) fn range(line_index: &LineIndex, range: TextRange) -> lsp_types::Range {
@@ -122,13 +129,9 @@ pub(crate) fn completion_item_kind(
     }
 }
 
-pub(crate) fn text_edit(
-    line_index: &LineIndex,
-    line_endings: LineEndings,
-    indel: Indel,
-) -> lsp_types::TextEdit {
+pub(crate) fn text_edit(line_index: &LineIndex, indel: Indel) -> lsp_types::TextEdit {
     let range = range(line_index, indel.delete);
-    let new_text = match line_endings {
+    let new_text = match line_index.endings {
         LineEndings::Unix => indel.insert,
         LineEndings::Dos => indel.insert.replace('\n', "\r\n"),
     };
@@ -137,11 +140,10 @@ pub(crate) fn text_edit(
 
 pub(crate) fn snippet_text_edit(
     line_index: &LineIndex,
-    line_endings: LineEndings,
     is_snippet: bool,
     indel: Indel,
 ) -> lsp_ext::SnippetTextEdit {
-    let text_edit = text_edit(line_index, line_endings, indel);
+    let text_edit = text_edit(line_index, indel);
     let insert_text_format =
         if is_snippet { Some(lsp_types::InsertTextFormat::Snippet) } else { None };
     lsp_ext::SnippetTextEdit {
@@ -153,27 +155,24 @@ pub(crate) fn snippet_text_edit(
 
 pub(crate) fn text_edit_vec(
     line_index: &LineIndex,
-    line_endings: LineEndings,
     text_edit: TextEdit,
 ) -> Vec<lsp_types::TextEdit> {
-    text_edit.into_iter().map(|indel| self::text_edit(line_index, line_endings, indel)).collect()
+    text_edit.into_iter().map(|indel| self::text_edit(line_index, indel)).collect()
 }
 
 pub(crate) fn snippet_text_edit_vec(
     line_index: &LineIndex,
-    line_endings: LineEndings,
     is_snippet: bool,
     text_edit: TextEdit,
 ) -> Vec<lsp_ext::SnippetTextEdit> {
     text_edit
         .into_iter()
-        .map(|indel| self::snippet_text_edit(line_index, line_endings, is_snippet, indel))
+        .map(|indel| self::snippet_text_edit(line_index, is_snippet, indel))
         .collect()
 }
 
 pub(crate) fn completion_item(
     line_index: &LineIndex,
-    line_endings: LineEndings,
     completion_item: CompletionItem,
 ) -> Vec<lsp_types::CompletionItem> {
     fn set_score(res: &mut lsp_types::CompletionItem, label: &str) {
@@ -190,19 +189,19 @@ pub(crate) fn completion_item(
     for indel in completion_item.text_edit().iter() {
         if indel.delete.contains_range(source_range) {
             text_edit = Some(if indel.delete == source_range {
-                self::text_edit(line_index, line_endings, indel.clone())
+                self::text_edit(line_index, indel.clone())
             } else {
                 assert!(source_range.end() == indel.delete.end());
                 let range1 = TextRange::new(indel.delete.start(), source_range.start());
                 let range2 = source_range;
                 let indel1 = Indel::replace(range1, String::new());
                 let indel2 = Indel::replace(range2, indel.insert.clone());
-                additional_text_edits.push(self::text_edit(line_index, line_endings, indel1));
-                self::text_edit(line_index, line_endings, indel2)
+                additional_text_edits.push(self::text_edit(line_index, indel1));
+                self::text_edit(line_index, indel2)
             })
         } else {
             assert!(source_range.intersect(indel.delete).is_none());
-            let text_edit = self::text_edit(line_index, line_endings, indel.clone());
+            let text_edit = self::text_edit(line_index, indel.clone());
             additional_text_edits.push(text_edit);
         }
     }
@@ -358,7 +357,7 @@ pub(crate) fn semantic_tokens(
         let token_index = semantic_tokens::type_index(type_);
         let modifier_bitset = mods.0;
 
-        for mut text_range in line_index.lines(highlight_range.range) {
+        for mut text_range in line_index.index.lines(highlight_range.range) {
             if text[text_range].ends_with('\n') {
                 text_range =
                     TextRange::new(text_range.start(), text_range.end() - TextSize::of('\n'));
@@ -565,7 +564,7 @@ pub(crate) fn location(
     frange: FileRange,
 ) -> Result<lsp_types::Location> {
     let url = url(snap, frange.file_id);
-    let line_index = snap.analysis.file_line_index(frange.file_id)?;
+    let line_index = snap.file_line_index(frange.file_id)?;
     let range = range(&line_index, frange.range);
     let loc = lsp_types::Location::new(url, range);
     Ok(loc)
@@ -577,7 +576,7 @@ pub(crate) fn location_from_nav(
     nav: NavigationTarget,
 ) -> Result<lsp_types::Location> {
     let url = url(snap, nav.file_id);
-    let line_index = snap.analysis.file_line_index(nav.file_id)?;
+    let line_index = snap.file_line_index(nav.file_id)?;
     let range = range(&line_index, nav.full_range);
     let loc = lsp_types::Location::new(url, range);
     Ok(loc)
@@ -590,7 +589,7 @@ pub(crate) fn location_link(
 ) -> Result<lsp_types::LocationLink> {
     let origin_selection_range = match src {
         Some(src) => {
-            let line_index = snap.analysis.file_line_index(src.file_id)?;
+            let line_index = snap.file_line_index(src.file_id)?;
             let range = range(&line_index, src.range);
             Some(range)
         }
@@ -610,7 +609,7 @@ fn location_info(
     snap: &GlobalStateSnapshot,
     target: NavigationTarget,
 ) -> Result<(lsp_types::Url, lsp_types::Range, lsp_types::Range)> {
-    let line_index = snap.analysis.file_line_index(target.file_id)?;
+    let line_index = snap.file_line_index(target.file_id)?;
 
     let target_uri = url(snap, target.file_id);
     let target_range = range(&line_index, target.full_range);
@@ -648,12 +647,8 @@ pub(crate) fn snippet_text_document_edit(
     edit: TextEdit,
 ) -> Result<lsp_ext::SnippetTextDocumentEdit> {
     let text_document = optional_versioned_text_document_identifier(snap, file_id);
-    let line_index = snap.analysis.file_line_index(file_id)?;
-    let line_endings = snap.file_line_endings(file_id);
-    let edits = edit
-        .into_iter()
-        .map(|it| snippet_text_edit(&line_index, line_endings, is_snippet, it))
-        .collect();
+    let line_index = snap.file_line_index(file_id)?;
+    let edits = edit.into_iter().map(|it| snippet_text_edit(&line_index, is_snippet, it)).collect();
     Ok(lsp_ext::SnippetTextDocumentEdit { text_document, edits })
 }
 
@@ -674,9 +669,8 @@ pub(crate) fn snippet_text_document_ops(
             if !initial_contents.is_empty() {
                 let text_document =
                     lsp_types::OptionalVersionedTextDocumentIdentifier { uri, version: None };
-                let range = range(&LineIndex::new(""), TextRange::empty(TextSize::from(0)));
                 let text_edit = lsp_ext::SnippetTextEdit {
-                    range,
+                    range: lsp_types::Range::default(),
                     new_text: initial_contents,
                     insert_text_format: Some(lsp_types::InsertTextFormat::PlainText),
                 };
@@ -867,7 +861,7 @@ pub(crate) fn code_lens(
 ) -> Result<lsp_types::CodeLens> {
     match annotation.kind {
         AnnotationKind::Runnable { debug, runnable: run } => {
-            let line_index = snap.analysis.file_line_index(run.nav.file_id)?;
+            let line_index = snap.file_line_index(run.nav.file_id)?;
             let annotation_range = range(&line_index, annotation.range);
 
             let action = run.action();
@@ -883,7 +877,7 @@ pub(crate) fn code_lens(
             Ok(lsp_types::CodeLens { range: annotation_range, command: Some(command), data: None })
         }
         AnnotationKind::HasImpls { position: file_position, data } => {
-            let line_index = snap.analysis.file_line_index(file_position.file_id)?;
+            let line_index = snap.file_line_index(file_position.file_id)?;
             let annotation_range = range(&line_index, annotation.range);
             let url = url(snap, file_position.file_id);
 
@@ -926,7 +920,7 @@ pub(crate) fn code_lens(
             })
         }
         AnnotationKind::HasReferences { position: file_position, data } => {
-            let line_index = snap.analysis.file_line_index(file_position.file_id)?;
+            let line_index = snap.file_line_index(file_position.file_id)?;
             let annotation_range = range(&line_index, annotation.range);
             let url = url(snap, file_position.file_id);
 
@@ -1060,6 +1054,8 @@ pub(crate) fn rename_error(err: RenameError) -> crate::LspError {
 
 #[cfg(test)]
 mod tests {
+    use std::sync::Arc;
+
     use hir::PrefixKind;
     use ide::Analysis;
     use ide_db::helpers::{insert_use::InsertUseConfig, SnippetCap};
@@ -1077,7 +1073,11 @@ mod tests {
         }"#;
 
         let (offset, text) = test_utils::extract_offset(fixture);
-        let line_index = LineIndex::new(&text);
+        let line_index = LineIndex {
+            index: Arc::new(ide::LineIndex::new(&text)),
+            endings: LineEndings::Unix,
+            encoding: OffsetEncoding::Utf16,
+        };
         let (analysis, file_id) = Analysis::from_single_file(text);
         let completions: Vec<(String, Option<String>)> = analysis
             .completions(
@@ -1095,7 +1095,7 @@ mod tests {
             .unwrap()
             .into_iter()
             .filter(|c| c.label().ends_with("arg"))
-            .map(|c| completion_item(&line_index, LineEndings::Unix, c))
+            .map(|c| completion_item(&line_index, c))
             .flat_map(|comps| comps.into_iter().map(|c| (c.label, c.sort_text)))
             .collect();
         expect_test::expect![[r#"
@@ -1133,7 +1133,11 @@ fn main() {
         let folds = analysis.folding_ranges(file_id).unwrap();
         assert_eq!(folds.len(), 4);
 
-        let line_index = LineIndex::new(&text);
+        let line_index = LineIndex {
+            index: Arc::new(ide::LineIndex::new(&text)),
+            endings: LineEndings::Unix,
+            encoding: OffsetEncoding::Utf16,
+        };
         let converted: Vec<lsp_types::FoldingRange> =
             folds.into_iter().map(|it| folding_range(&text, &line_index, true, it)).collect();