From 136d1864bcb5046e7f334ac347a8a94946d1ba90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adolfo=20Ochagav=C3=ADa?= Date: Thu, 15 Nov 2018 17:34:05 +0100 Subject: Support UTF-16 chars in LineIndex --- crates/ra_editor/src/line_index.rs | 303 ++++++++++++++++++++++++------------- 1 file changed, 197 insertions(+), 106 deletions(-) (limited to 'crates/ra_editor') diff --git a/crates/ra_editor/src/line_index.rs b/crates/ra_editor/src/line_index.rs index 9abbb0d09..0b3a28cd4 100644 --- a/crates/ra_editor/src/line_index.rs +++ b/crates/ra_editor/src/line_index.rs @@ -1,43 +1,124 @@ use crate::TextUnit; +use rustc_hash::FxHashMap; use superslice::Ext; -#[derive(Clone, Debug, Hash, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct LineIndex { newlines: Vec, + utf16_lines: FxHashMap>, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub struct LineCol { pub line: u32, - pub col: TextUnit, + pub col: u32, +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq)] +struct Utf16Char { + start: TextUnit, + end: TextUnit, +} + +impl Utf16Char { + fn len(&self) -> TextUnit { + self.end - self.start + } } impl LineIndex { pub fn new(text: &str) -> LineIndex { + let mut utf16_lines = FxHashMap::default(); + let mut utf16_chars = Vec::new(); + let mut newlines = vec![0.into()]; - let mut curr = 0.into(); + let mut curr_row = 0.into(); + let mut curr_col = 0.into(); + let mut line = 0; for c in text.chars() { - curr += TextUnit::of_char(c); + curr_row += TextUnit::of_char(c); if c == '\n' { - newlines.push(curr); + newlines.push(curr_row); + + // Save any utf-16 characters seen in the previous line + if utf16_chars.len() > 0 { + utf16_lines.insert(line, utf16_chars); + utf16_chars = Vec::new(); + } + + // Prepare for processing the next line + curr_col = 0.into(); + line += 1; + continue; } + + let char_len = TextUnit::of_char(c); + if char_len.to_usize() > 1 { + utf16_chars.push(Utf16Char { + start: curr_col, + end: curr_col + char_len, + }); + } + + curr_col += char_len; + } + LineIndex { + newlines, + utf16_lines, } - LineIndex { newlines } } pub fn line_col(&self, offset: TextUnit) -> LineCol { let line = self.newlines.upper_bound(&offset) - 1; let line_start_offset = self.newlines[line]; let col = offset - line_start_offset; + LineCol { line: line as u32, - col, + col: self.utf8_to_utf16_col(line as u32, col) as u32, } } pub fn offset(&self, line_col: LineCol) -> TextUnit { //TODO: return Result - self.newlines[line_col.line as usize] + line_col.col + let col = self.utf16_to_utf8_col(line_col.line, line_col.col); + self.newlines[line_col.line as usize] + col + } + + fn utf8_to_utf16_col(&self, line: u32, mut col: TextUnit) -> usize { + if let Some(utf16_chars) = self.utf16_lines.get(&line) { + let mut correction = TextUnit::from_usize(0); + for c in utf16_chars { + if col >= c.end { + correction += c.len() - TextUnit::from_usize(1); + } else { + // From here on, all utf16 characters come *after* the character we are mapping, + // so we don't need to take them into account + break; + } + } + + col -= correction; + } + + col.to_usize() + } + + fn utf16_to_utf8_col(&self, line: u32, col: u32) -> TextUnit { + let mut col: TextUnit = col.into(); + if let Some(utf16_chars) = self.utf16_lines.get(&line) { + for c in utf16_chars { + if col >= c.start { + col += c.len() - TextUnit::from_usize(1); + } else { + // From here on, all utf16 characters come *after* the character we are mapping, + // so we don't need to take them into account + break; + } + } + } + + col } } @@ -45,105 +126,115 @@ impl LineIndex { fn test_line_index() { let text = "hello\nworld"; let index = LineIndex::new(text); - assert_eq!( - index.line_col(0.into()), - LineCol { - line: 0, - col: 0.into() - } - ); - assert_eq!( - index.line_col(1.into()), - LineCol { - line: 0, - col: 1.into() - } - ); - assert_eq!( - index.line_col(5.into()), - LineCol { - line: 0, - col: 5.into() - } - ); - assert_eq!( - index.line_col(6.into()), - LineCol { - line: 1, - col: 0.into() - } - ); - assert_eq!( - index.line_col(7.into()), - LineCol { - line: 1, - col: 1.into() - } - ); - assert_eq!( - index.line_col(8.into()), - LineCol { - line: 1, - col: 2.into() - } - ); - assert_eq!( - index.line_col(10.into()), - LineCol { - line: 1, - col: 4.into() - } - ); - assert_eq!( - index.line_col(11.into()), - LineCol { - line: 1, - col: 5.into() - } - ); - assert_eq!( - index.line_col(12.into()), - LineCol { - line: 1, - col: 6.into() - } - ); + assert_eq!(index.line_col(0.into()), LineCol { line: 0, col: 0 }); + assert_eq!(index.line_col(1.into()), LineCol { line: 0, col: 1 }); + assert_eq!(index.line_col(5.into()), LineCol { line: 0, col: 5 }); + assert_eq!(index.line_col(6.into()), LineCol { line: 1, col: 0 }); + assert_eq!(index.line_col(7.into()), LineCol { line: 1, col: 1 }); + assert_eq!(index.line_col(8.into()), LineCol { line: 1, col: 2 }); + assert_eq!(index.line_col(10.into()), LineCol { line: 1, col: 4 }); + assert_eq!(index.line_col(11.into()), LineCol { line: 1, col: 5 }); + assert_eq!(index.line_col(12.into()), LineCol { line: 1, col: 6 }); let text = "\nhello\nworld"; let index = LineIndex::new(text); - assert_eq!( - index.line_col(0.into()), - LineCol { - line: 0, - col: 0.into() - } - ); - assert_eq!( - index.line_col(1.into()), - LineCol { - line: 1, - col: 0.into() - } - ); - assert_eq!( - index.line_col(2.into()), - LineCol { - line: 1, - col: 1.into() - } - ); - assert_eq!( - index.line_col(6.into()), - LineCol { - line: 1, - col: 5.into() - } - ); - assert_eq!( - index.line_col(7.into()), - LineCol { - line: 2, - col: 0.into() - } - ); + assert_eq!(index.line_col(0.into()), LineCol { line: 0, col: 0 }); + assert_eq!(index.line_col(1.into()), LineCol { line: 1, col: 0 }); + assert_eq!(index.line_col(2.into()), LineCol { line: 1, col: 1 }); + assert_eq!(index.line_col(6.into()), LineCol { line: 1, col: 5 }); + assert_eq!(index.line_col(7.into()), LineCol { line: 2, col: 0 }); +} + +#[cfg(test)] +mod test_utf8_utf16_conv { + use super::*; + + #[test] + fn test_char_len() { + assert_eq!('メ'.len_utf8(), 3); + assert_eq!('メ'.len_utf16(), 1); + } + + #[test] + fn test_empty_index() { + let col_index = LineIndex::new( + " +const C: char = 'x'; +", + ); + assert_eq!(col_index.utf16_lines.len(), 0); + } + + #[test] + fn test_single_char() { + let col_index = LineIndex::new( + " +const C: char = 'メ'; +", + ); + + assert_eq!(col_index.utf16_lines.len(), 1); + assert_eq!(col_index.utf16_lines[&1].len(), 1); + assert_eq!( + col_index.utf16_lines[&1][0], + Utf16Char { + start: 17.into(), + end: 20.into() + } + ); + + // UTF-8 to UTF-16, no changes + assert_eq!(col_index.utf8_to_utf16_col(1, 15.into()), 15); + + // UTF-8 to UTF-16 + assert_eq!(col_index.utf8_to_utf16_col(1, 22.into()), 20); + + // UTF-16 to UTF-8, no changes + assert_eq!(col_index.utf16_to_utf8_col(1, 15), TextUnit::from(15)); + + // UTF-16 to UTF-8 + assert_eq!(col_index.utf16_to_utf8_col(1, 19), TextUnit::from(21)); + } + + #[test] + fn test_string() { + let col_index = LineIndex::new( + " +const C: char = \"メ メ\"; +", + ); + + assert_eq!(col_index.utf16_lines.len(), 1); + assert_eq!(col_index.utf16_lines[&1].len(), 2); + assert_eq!( + col_index.utf16_lines[&1][0], + Utf16Char { + start: 17.into(), + end: 20.into() + } + ); + assert_eq!( + col_index.utf16_lines[&1][1], + Utf16Char { + start: 21.into(), + end: 24.into() + } + ); + + // UTF-8 to UTF-16 + assert_eq!(col_index.utf8_to_utf16_col(1, 15.into()), 15); + + assert_eq!(col_index.utf8_to_utf16_col(1, 21.into()), 19); + assert_eq!(col_index.utf8_to_utf16_col(1, 25.into()), 21); + + assert!(col_index.utf8_to_utf16_col(2, 15.into()) == 15); + + // UTF-16 to UTF-8 + assert_eq!(col_index.utf16_to_utf8_col(1, 15), TextUnit::from_usize(15)); + + assert_eq!(col_index.utf16_to_utf8_col(1, 18), TextUnit::from_usize(20)); + assert_eq!(col_index.utf16_to_utf8_col(1, 19), TextUnit::from_usize(23)); + + assert_eq!(col_index.utf16_to_utf8_col(2, 15), TextUnit::from_usize(15)); + } } -- cgit v1.2.3 From bccbee5ba3eea3decaf3d00a5dc09dd6cbc632f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adolfo=20Ochagav=C3=ADa?= Date: Fri, 16 Nov 2018 12:02:45 +0100 Subject: Rename col to col_utf16 --- crates/ra_editor/src/line_index.rs | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) (limited to 'crates/ra_editor') diff --git a/crates/ra_editor/src/line_index.rs b/crates/ra_editor/src/line_index.rs index 0b3a28cd4..26b309b78 100644 --- a/crates/ra_editor/src/line_index.rs +++ b/crates/ra_editor/src/line_index.rs @@ -11,7 +11,7 @@ pub struct LineIndex { #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub struct LineCol { pub line: u32, - pub col: u32, + pub col_utf16: u32, } #[derive(Clone, Debug, Hash, PartialEq, Eq)] @@ -75,13 +75,13 @@ impl LineIndex { LineCol { line: line as u32, - col: self.utf8_to_utf16_col(line as u32, col) as u32, + col_utf16: self.utf8_to_utf16_col(line as u32, col) as u32, } } pub fn offset(&self, line_col: LineCol) -> TextUnit { //TODO: return Result - let col = self.utf16_to_utf8_col(line_col.line, line_col.col); + let col = self.utf16_to_utf8_col(line_col.line, line_col.col_utf16); self.newlines[line_col.line as usize] + col } @@ -126,23 +126,23 @@ impl LineIndex { fn test_line_index() { let text = "hello\nworld"; let index = LineIndex::new(text); - assert_eq!(index.line_col(0.into()), LineCol { line: 0, col: 0 }); - assert_eq!(index.line_col(1.into()), LineCol { line: 0, col: 1 }); - assert_eq!(index.line_col(5.into()), LineCol { line: 0, col: 5 }); - assert_eq!(index.line_col(6.into()), LineCol { line: 1, col: 0 }); - assert_eq!(index.line_col(7.into()), LineCol { line: 1, col: 1 }); - assert_eq!(index.line_col(8.into()), LineCol { line: 1, col: 2 }); - assert_eq!(index.line_col(10.into()), LineCol { line: 1, col: 4 }); - assert_eq!(index.line_col(11.into()), LineCol { line: 1, col: 5 }); - assert_eq!(index.line_col(12.into()), LineCol { line: 1, col: 6 }); + assert_eq!(index.line_col(0.into()), LineCol { line: 0, col_utf16: 0 }); + assert_eq!(index.line_col(1.into()), LineCol { line: 0, col_utf16: 1 }); + assert_eq!(index.line_col(5.into()), LineCol { line: 0, col_utf16: 5 }); + assert_eq!(index.line_col(6.into()), LineCol { line: 1, col_utf16: 0 }); + assert_eq!(index.line_col(7.into()), LineCol { line: 1, col_utf16: 1 }); + assert_eq!(index.line_col(8.into()), LineCol { line: 1, col_utf16: 2 }); + assert_eq!(index.line_col(10.into()), LineCol { line: 1, col_utf16: 4 }); + assert_eq!(index.line_col(11.into()), LineCol { line: 1, col_utf16: 5 }); + assert_eq!(index.line_col(12.into()), LineCol { line: 1, col_utf16: 6 }); let text = "\nhello\nworld"; let index = LineIndex::new(text); - assert_eq!(index.line_col(0.into()), LineCol { line: 0, col: 0 }); - assert_eq!(index.line_col(1.into()), LineCol { line: 1, col: 0 }); - assert_eq!(index.line_col(2.into()), LineCol { line: 1, col: 1 }); - assert_eq!(index.line_col(6.into()), LineCol { line: 1, col: 5 }); - assert_eq!(index.line_col(7.into()), LineCol { line: 2, col: 0 }); + assert_eq!(index.line_col(0.into()), LineCol { line: 0, col_utf16: 0 }); + assert_eq!(index.line_col(1.into()), LineCol { line: 1, col_utf16: 0 }); + assert_eq!(index.line_col(2.into()), LineCol { line: 1, col_utf16: 1 }); + assert_eq!(index.line_col(6.into()), LineCol { line: 1, col_utf16: 5 }); + assert_eq!(index.line_col(7.into()), LineCol { line: 2, col_utf16: 0 }); } #[cfg(test)] -- cgit v1.2.3 From acd51cb361720458615bd1ceb909b0f4461328df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adolfo=20Ochagav=C3=ADa?= Date: Fri, 16 Nov 2018 12:16:20 +0100 Subject: cargo format --- crates/ra_editor/src/line_index.rs | 112 ++++++++++++++++++++++++++++++++----- 1 file changed, 98 insertions(+), 14 deletions(-) (limited to 'crates/ra_editor') diff --git a/crates/ra_editor/src/line_index.rs b/crates/ra_editor/src/line_index.rs index 26b309b78..aab7e4081 100644 --- a/crates/ra_editor/src/line_index.rs +++ b/crates/ra_editor/src/line_index.rs @@ -126,23 +126,107 @@ impl LineIndex { fn test_line_index() { let text = "hello\nworld"; let index = LineIndex::new(text); - assert_eq!(index.line_col(0.into()), LineCol { line: 0, col_utf16: 0 }); - assert_eq!(index.line_col(1.into()), LineCol { line: 0, col_utf16: 1 }); - assert_eq!(index.line_col(5.into()), LineCol { line: 0, col_utf16: 5 }); - assert_eq!(index.line_col(6.into()), LineCol { line: 1, col_utf16: 0 }); - assert_eq!(index.line_col(7.into()), LineCol { line: 1, col_utf16: 1 }); - assert_eq!(index.line_col(8.into()), LineCol { line: 1, col_utf16: 2 }); - assert_eq!(index.line_col(10.into()), LineCol { line: 1, col_utf16: 4 }); - assert_eq!(index.line_col(11.into()), LineCol { line: 1, col_utf16: 5 }); - assert_eq!(index.line_col(12.into()), LineCol { line: 1, col_utf16: 6 }); + assert_eq!( + index.line_col(0.into()), + LineCol { + line: 0, + col_utf16: 0 + } + ); + assert_eq!( + index.line_col(1.into()), + LineCol { + line: 0, + col_utf16: 1 + } + ); + assert_eq!( + index.line_col(5.into()), + LineCol { + line: 0, + col_utf16: 5 + } + ); + assert_eq!( + index.line_col(6.into()), + LineCol { + line: 1, + col_utf16: 0 + } + ); + assert_eq!( + index.line_col(7.into()), + LineCol { + line: 1, + col_utf16: 1 + } + ); + assert_eq!( + index.line_col(8.into()), + LineCol { + line: 1, + col_utf16: 2 + } + ); + assert_eq!( + index.line_col(10.into()), + LineCol { + line: 1, + col_utf16: 4 + } + ); + assert_eq!( + index.line_col(11.into()), + LineCol { + line: 1, + col_utf16: 5 + } + ); + assert_eq!( + index.line_col(12.into()), + LineCol { + line: 1, + col_utf16: 6 + } + ); let text = "\nhello\nworld"; let index = LineIndex::new(text); - assert_eq!(index.line_col(0.into()), LineCol { line: 0, col_utf16: 0 }); - assert_eq!(index.line_col(1.into()), LineCol { line: 1, col_utf16: 0 }); - assert_eq!(index.line_col(2.into()), LineCol { line: 1, col_utf16: 1 }); - assert_eq!(index.line_col(6.into()), LineCol { line: 1, col_utf16: 5 }); - assert_eq!(index.line_col(7.into()), LineCol { line: 2, col_utf16: 0 }); + assert_eq!( + index.line_col(0.into()), + LineCol { + line: 0, + col_utf16: 0 + } + ); + assert_eq!( + index.line_col(1.into()), + LineCol { + line: 1, + col_utf16: 0 + } + ); + assert_eq!( + index.line_col(2.into()), + LineCol { + line: 1, + col_utf16: 1 + } + ); + assert_eq!( + index.line_col(6.into()), + LineCol { + line: 1, + col_utf16: 5 + } + ); + assert_eq!( + index.line_col(7.into()), + LineCol { + line: 2, + col_utf16: 0 + } + ); } #[cfg(test)] -- cgit v1.2.3