Implement utf8 offsets

5 files changed, 45 insertions, 15 deletions

diff --git a/crates/rust-analyzer/src/from_proto.rs b/crates/rust-analyzer/src/from_proto.rs
index c671b5f64..4f3ae8cc3 100644
--- a/crates/rust-analyzer/src/from_proto.rs
+++ b/crates/rust-analyzer/src/from_proto.rs
@@ -1,13 +1,16 @@
 //! Conversion lsp_types types to rust-analyzer specific ones.
 use std::convert::TryFrom;
 
-use ide::{Annotation, AnnotationKind, AssistKind, LineColUtf16};
+use ide::{Annotation, AnnotationKind, AssistKind, LineCol, LineColUtf16};
 use ide_db::base_db::{FileId, FilePosition, FileRange};
 use syntax::{TextRange, TextSize};
 use vfs::AbsPathBuf;
 
 use crate::{
-    from_json, global_state::GlobalStateSnapshot, line_endings::LineIndex, lsp_ext, Result,
+    from_json,
+    global_state::GlobalStateSnapshot,
+    line_endings::{LineIndex, OffsetEncoding},
+    lsp_ext, Result,
 };
 
 pub(crate) fn abs_path(url: &lsp_types::Url) -> Result<AbsPathBuf> {
@@ -20,8 +23,16 @@ pub(crate) fn vfs_path(url: &lsp_types::Url) -> Result<vfs::VfsPath> {
 }
 
 pub(crate) fn offset(line_index: &LineIndex, position: lsp_types::Position) -> TextSize {
-    let line_col = LineColUtf16 { line: position.line as u32, col: position.character as u32 };
-    let line_col = line_index.index.to_utf8(line_col);
+    let line_col = match line_index.encoding {
+        OffsetEncoding::Utf8 => {
+            LineCol { line: position.line as u32, col: position.character as u32 }
+        }
+        OffsetEncoding::Utf16 => {
+            let line_col =
+                LineColUtf16 { line: position.line as u32, col: position.character as u32 };
+            line_index.index.to_utf8(line_col)
+        }
+    };
     line_index.index.offset(line_col)
 }
 
diff --git a/crates/rust-analyzer/src/global_state.rs b/crates/rust-analyzer/src/global_state.rs
index ffef33430..d26e5ef48 100644
--- a/crates/rust-analyzer/src/global_state.rs
+++ b/crates/rust-analyzer/src/global_state.rs
@@ -22,7 +22,7 @@ use crate::{
     diagnostics::{CheckFixes, DiagnosticCollection},
     document::DocumentData,
     from_proto,
-    line_endings::{LineEndings, LineIndex},
+    line_endings::{LineEndings, LineIndex, OffsetEncoding},
     main_loop::Task,
     op_queue::OpQueue,
     reload::SourceRootConfig,
@@ -274,7 +274,7 @@ impl GlobalStateSnapshot {
     pub(crate) fn file_line_index(&self, file_id: FileId) -> Cancelable<LineIndex> {
         let endings = self.vfs.read().1[&file_id];
         let index = self.analysis.file_line_index(file_id)?;
-        let res = LineIndex { index, endings };
+        let res = LineIndex { index, endings, encoding: OffsetEncoding::Utf16 };
         Ok(res)
     }
 
diff --git a/crates/rust-analyzer/src/line_endings.rs b/crates/rust-analyzer/src/line_endings.rs
index cc152c529..7b6cba43e 100644
--- a/crates/rust-analyzer/src/line_endings.rs
+++ b/crates/rust-analyzer/src/line_endings.rs
@@ -4,9 +4,16 @@
 
 use std::sync::Arc;
 
+pub(crate) enum OffsetEncoding {
+    #[allow(unused)]
+    Utf8,
+    Utf16,
+}
+
 pub(crate) struct LineIndex {
     pub(crate) index: Arc<ide::LineIndex>,
     pub(crate) endings: LineEndings,
+    pub(crate) encoding: OffsetEncoding,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
diff --git a/crates/rust-analyzer/src/lsp_utils.rs b/crates/rust-analyzer/src/lsp_utils.rs
index 25162185e..6b8e347b9 100644
--- a/crates/rust-analyzer/src/lsp_utils.rs
+++ b/crates/rust-analyzer/src/lsp_utils.rs
@@ -7,7 +7,7 @@ use lsp_server::Notification;
 use crate::{
     from_proto,
     global_state::GlobalState,
-    line_endings::{LineEndings, LineIndex},
+    line_endings::{LineEndings, LineIndex, OffsetEncoding},
 };
 
 pub(crate) fn is_canceled(e: &(dyn Error + 'static)) -> bool {
@@ -95,8 +95,9 @@ pub(crate) fn apply_document_changes(
 ) {
     let mut line_index = LineIndex {
         index: Arc::new(ide::LineIndex::new(old_text)),
-        // We don't care about line endings here.
+        // We don't care about line endings or offset encoding here.
         endings: LineEndings::Unix,
+        encoding: OffsetEncoding::Utf16,
     };
 
     // The changes we got must be applied sequentially, but can cross lines so we
diff --git a/crates/rust-analyzer/src/to_proto.rs b/crates/rust-analyzer/src/to_proto.rs
index 43e29ef04..368d916e7 100644
--- a/crates/rust-analyzer/src/to_proto.rs
+++ b/crates/rust-analyzer/src/to_proto.rs
@@ -17,14 +17,19 @@ use serde_json::to_value;
 use crate::{
     cargo_target_spec::CargoTargetSpec,
     global_state::GlobalStateSnapshot,
-    line_endings::{LineEndings, LineIndex},
+    line_endings::{LineEndings, LineIndex, OffsetEncoding},
     lsp_ext, semantic_tokens, Result,
 };
 
 pub(crate) fn position(line_index: &LineIndex, offset: TextSize) -> lsp_types::Position {
     let line_col = line_index.index.line_col(offset);
-    let line_col = line_index.index.to_utf16(line_col);
-    lsp_types::Position::new(line_col.line, line_col.col)
+    match line_index.encoding {
+        OffsetEncoding::Utf8 => lsp_types::Position::new(line_col.line, line_col.col),
+        OffsetEncoding::Utf16 => {
+            let line_col = line_index.index.to_utf16(line_col);
+            lsp_types::Position::new(line_col.line, line_col.col)
+        }
+    }
 }
 
 pub(crate) fn range(line_index: &LineIndex, range: TextRange) -> lsp_types::Range {
@@ -1068,8 +1073,11 @@ mod tests {
         }"#;
 
         let (offset, text) = test_utils::extract_offset(fixture);
-        let line_index =
-            LineIndex { index: Arc::new(ide::LineIndex::new(&text)), endings: LineEndings::Unix };
+        let line_index = LineIndex {
+            index: Arc::new(ide::LineIndex::new(&text)),
+            endings: LineEndings::Unix,
+            encoding: OffsetEncoding::Utf16,
+        };
         let (analysis, file_id) = Analysis::from_single_file(text);
         let completions: Vec<(String, Option<String>)> = analysis
             .completions(
@@ -1125,8 +1133,11 @@ fn main() {
         let folds = analysis.folding_ranges(file_id).unwrap();
         assert_eq!(folds.len(), 4);
 
-        let line_index =
-            LineIndex { index: Arc::new(ide::LineIndex::new(&text)), endings: LineEndings::Unix };
+        let line_index = LineIndex {
+            index: Arc::new(ide::LineIndex::new(&text)),
+            endings: LineEndings::Unix,
+            encoding: OffsetEncoding::Utf16,
+        };
         let converted: Vec<lsp_types::FoldingRange> =
             folds.into_iter().map(|it| folding_range(&text, &line_index, true, it)).collect();