From 136d1864bcb5046e7f334ac347a8a94946d1ba90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adolfo=20Ochagav=C3=ADa?= Date: Thu, 15 Nov 2018 17:34:05 +0100 Subject: Support UTF-16 chars in LineIndex --- crates/ra_editor/src/line_index.rs | 303 ++++++++++++++++++++++++------------- crates/ra_lsp_server/src/conv.rs | 2 - 2 files changed, 197 insertions(+), 108 deletions(-) diff --git a/crates/ra_editor/src/line_index.rs b/crates/ra_editor/src/line_index.rs index 9abbb0d09..0b3a28cd4 100644 --- a/crates/ra_editor/src/line_index.rs +++ b/crates/ra_editor/src/line_index.rs @@ -1,43 +1,124 @@ use crate::TextUnit; +use rustc_hash::FxHashMap; use superslice::Ext; -#[derive(Clone, Debug, Hash, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct LineIndex { newlines: Vec, + utf16_lines: FxHashMap>, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub struct LineCol { pub line: u32, - pub col: TextUnit, + pub col: u32, +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq)] +struct Utf16Char { + start: TextUnit, + end: TextUnit, +} + +impl Utf16Char { + fn len(&self) -> TextUnit { + self.end - self.start + } } impl LineIndex { pub fn new(text: &str) -> LineIndex { + let mut utf16_lines = FxHashMap::default(); + let mut utf16_chars = Vec::new(); + let mut newlines = vec![0.into()]; - let mut curr = 0.into(); + let mut curr_row = 0.into(); + let mut curr_col = 0.into(); + let mut line = 0; for c in text.chars() { - curr += TextUnit::of_char(c); + curr_row += TextUnit::of_char(c); if c == '\n' { - newlines.push(curr); + newlines.push(curr_row); + + // Save any utf-16 characters seen in the previous line + if utf16_chars.len() > 0 { + utf16_lines.insert(line, utf16_chars); + utf16_chars = Vec::new(); + } + + // Prepare for processing the next line + curr_col = 0.into(); + line += 1; + continue; } + + let char_len = TextUnit::of_char(c); + if char_len.to_usize() > 1 { + utf16_chars.push(Utf16Char { + start: curr_col, + end: curr_col + char_len, + }); + } + + curr_col += char_len; + } + LineIndex { + newlines, + utf16_lines, } - LineIndex { newlines } } pub fn line_col(&self, offset: TextUnit) -> LineCol { let line = self.newlines.upper_bound(&offset) - 1; let line_start_offset = self.newlines[line]; let col = offset - line_start_offset; + LineCol { line: line as u32, - col, + col: self.utf8_to_utf16_col(line as u32, col) as u32, } } pub fn offset(&self, line_col: LineCol) -> TextUnit { //TODO: return Result - self.newlines[line_col.line as usize] + line_col.col + let col = self.utf16_to_utf8_col(line_col.line, line_col.col); + self.newlines[line_col.line as usize] + col + } + + fn utf8_to_utf16_col(&self, line: u32, mut col: TextUnit) -> usize { + if let Some(utf16_chars) = self.utf16_lines.get(&line) { + let mut correction = TextUnit::from_usize(0); + for c in utf16_chars { + if col >= c.end { + correction += c.len() - TextUnit::from_usize(1); + } else { + // From here on, all utf16 characters come *after* the character we are mapping, + // so we don't need to take them into account + break; + } + } + + col -= correction; + } + + col.to_usize() + } + + fn utf16_to_utf8_col(&self, line: u32, col: u32) -> TextUnit { + let mut col: TextUnit = col.into(); + if let Some(utf16_chars) = self.utf16_lines.get(&line) { + for c in utf16_chars { + if col >= c.start { + col += c.len() - TextUnit::from_usize(1); + } else { + // From here on, all utf16 characters come *after* the character we are mapping, + // so we don't need to take them into account + break; + } + } + } + + col } } @@ -45,105 +126,115 @@ impl LineIndex { fn test_line_index() { let text = "hello\nworld"; let index = LineIndex::new(text); - assert_eq!( - index.line_col(0.into()), - LineCol { - line: 0, - col: 0.into() - } - ); - assert_eq!( - index.line_col(1.into()), - LineCol { - line: 0, - col: 1.into() - } - ); - assert_eq!( - index.line_col(5.into()), - LineCol { - line: 0, - col: 5.into() - } - ); - assert_eq!( - index.line_col(6.into()), - LineCol { - line: 1, - col: 0.into() - } - ); - assert_eq!( - index.line_col(7.into()), - LineCol { - line: 1, - col: 1.into() - } - ); - assert_eq!( - index.line_col(8.into()), - LineCol { - line: 1, - col: 2.into() - } - ); - assert_eq!( - index.line_col(10.into()), - LineCol { - line: 1, - col: 4.into() - } - ); - assert_eq!( - index.line_col(11.into()), - LineCol { - line: 1, - col: 5.into() - } - ); - assert_eq!( - index.line_col(12.into()), - LineCol { - line: 1, - col: 6.into() - } - ); + assert_eq!(index.line_col(0.into()), LineCol { line: 0, col: 0 }); + assert_eq!(index.line_col(1.into()), LineCol { line: 0, col: 1 }); + assert_eq!(index.line_col(5.into()), LineCol { line: 0, col: 5 }); + assert_eq!(index.line_col(6.into()), LineCol { line: 1, col: 0 }); + assert_eq!(index.line_col(7.into()), LineCol { line: 1, col: 1 }); + assert_eq!(index.line_col(8.into()), LineCol { line: 1, col: 2 }); + assert_eq!(index.line_col(10.into()), LineCol { line: 1, col: 4 }); + assert_eq!(index.line_col(11.into()), LineCol { line: 1, col: 5 }); + assert_eq!(index.line_col(12.into()), LineCol { line: 1, col: 6 }); let text = "\nhello\nworld"; let index = LineIndex::new(text); - assert_eq!( - index.line_col(0.into()), - LineCol { - line: 0, - col: 0.into() - } - ); - assert_eq!( - index.line_col(1.into()), - LineCol { - line: 1, - col: 0.into() - } - ); - assert_eq!( - index.line_col(2.into()), - LineCol { - line: 1, - col: 1.into() - } - ); - assert_eq!( - index.line_col(6.into()), - LineCol { - line: 1, - col: 5.into() - } - ); - assert_eq!( - index.line_col(7.into()), - LineCol { - line: 2, - col: 0.into() - } - ); + assert_eq!(index.line_col(0.into()), LineCol { line: 0, col: 0 }); + assert_eq!(index.line_col(1.into()), LineCol { line: 1, col: 0 }); + assert_eq!(index.line_col(2.into()), LineCol { line: 1, col: 1 }); + assert_eq!(index.line_col(6.into()), LineCol { line: 1, col: 5 }); + assert_eq!(index.line_col(7.into()), LineCol { line: 2, col: 0 }); +} + +#[cfg(test)] +mod test_utf8_utf16_conv { + use super::*; + + #[test] + fn test_char_len() { + assert_eq!('メ'.len_utf8(), 3); + assert_eq!('メ'.len_utf16(), 1); + } + + #[test] + fn test_empty_index() { + let col_index = LineIndex::new( + " +const C: char = 'x'; +", + ); + assert_eq!(col_index.utf16_lines.len(), 0); + } + + #[test] + fn test_single_char() { + let col_index = LineIndex::new( + " +const C: char = 'メ'; +", + ); + + assert_eq!(col_index.utf16_lines.len(), 1); + assert_eq!(col_index.utf16_lines[&1].len(), 1); + assert_eq!( + col_index.utf16_lines[&1][0], + Utf16Char { + start: 17.into(), + end: 20.into() + } + ); + + // UTF-8 to UTF-16, no changes + assert_eq!(col_index.utf8_to_utf16_col(1, 15.into()), 15); + + // UTF-8 to UTF-16 + assert_eq!(col_index.utf8_to_utf16_col(1, 22.into()), 20); + + // UTF-16 to UTF-8, no changes + assert_eq!(col_index.utf16_to_utf8_col(1, 15), TextUnit::from(15)); + + // UTF-16 to UTF-8 + assert_eq!(col_index.utf16_to_utf8_col(1, 19), TextUnit::from(21)); + } + + #[test] + fn test_string() { + let col_index = LineIndex::new( + " +const C: char = \"メ メ\"; +", + ); + + assert_eq!(col_index.utf16_lines.len(), 1); + assert_eq!(col_index.utf16_lines[&1].len(), 2); + assert_eq!( + col_index.utf16_lines[&1][0], + Utf16Char { + start: 17.into(), + end: 20.into() + } + ); + assert_eq!( + col_index.utf16_lines[&1][1], + Utf16Char { + start: 21.into(), + end: 24.into() + } + ); + + // UTF-8 to UTF-16 + assert_eq!(col_index.utf8_to_utf16_col(1, 15.into()), 15); + + assert_eq!(col_index.utf8_to_utf16_col(1, 21.into()), 19); + assert_eq!(col_index.utf8_to_utf16_col(1, 25.into()), 21); + + assert!(col_index.utf8_to_utf16_col(2, 15.into()) == 15); + + // UTF-16 to UTF-8 + assert_eq!(col_index.utf16_to_utf8_col(1, 15), TextUnit::from_usize(15)); + + assert_eq!(col_index.utf16_to_utf8_col(1, 18), TextUnit::from_usize(20)); + assert_eq!(col_index.utf16_to_utf8_col(1, 19), TextUnit::from_usize(23)); + + assert_eq!(col_index.utf16_to_utf8_col(2, 15), TextUnit::from_usize(15)); + } } diff --git a/crates/ra_lsp_server/src/conv.rs b/crates/ra_lsp_server/src/conv.rs index e5a2449c2..a102b9105 100644 --- a/crates/ra_lsp_server/src/conv.rs +++ b/crates/ra_lsp_server/src/conv.rs @@ -49,7 +49,6 @@ impl ConvWith for Position { type Output = TextUnit; fn conv_with(self, line_index: &LineIndex) -> TextUnit { - // TODO: UTF-16 let line_col = LineCol { line: self.line as u32, col: (self.character as u32).into(), @@ -64,7 +63,6 @@ impl ConvWith for TextUnit { fn conv_with(self, line_index: &LineIndex) -> Position { let line_col = line_index.line_col(self); - // TODO: UTF-16 Position::new(u64::from(line_col.line), u64::from(u32::from(line_col.col))) } } -- cgit v1.2.3