aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--Cargo.toml1
-rw-r--r--src/lexer/ptr.rs4
-rw-r--r--src/lib.rs4
-rw-r--r--src/parser/event.rs8
-rw-r--r--src/parser/input.rs4
-rw-r--r--src/text.rs136
-rw-r--r--src/tree/file_builder.rs6
-rw-r--r--src/tree/mod.rs8
8 files changed, 18 insertions, 153 deletions
diff --git a/Cargo.toml b/Cargo.toml
index 622abcc42..954f3e94b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,6 +9,7 @@ members = [ "tools" ]
9 9
10[dependencies] 10[dependencies]
11unicode-xid = "0.1.0" 11unicode-xid = "0.1.0"
12text_unit = "0.1.1"
12 13
13[dev-dependencies] 14[dev-dependencies]
14testutils = { path = "./tests/testutils" } 15testutils = { path = "./tests/testutils" }
diff --git a/src/lexer/ptr.rs b/src/lexer/ptr.rs
index 99d55b283..d1391fd5f 100644
--- a/src/lexer/ptr.rs
+++ b/src/lexer/ptr.rs
@@ -11,7 +11,7 @@ impl<'s> Ptr<'s> {
11 pub fn new(text: &'s str) -> Ptr<'s> { 11 pub fn new(text: &'s str) -> Ptr<'s> {
12 Ptr { 12 Ptr {
13 text, 13 text,
14 len: TextUnit::new(0), 14 len: 0.into(),
15 } 15 }
16 } 16 }
17 17
@@ -47,7 +47,7 @@ impl<'s> Ptr<'s> {
47 47
48 pub fn bump(&mut self) -> Option<char> { 48 pub fn bump(&mut self) -> Option<char> {
49 let ch = self.chars().next()?; 49 let ch = self.chars().next()?;
50 self.len += TextUnit::len_of_char(ch); 50 self.len += TextUnit::of_char(ch);
51 Some(ch) 51 Some(ch)
52 } 52 }
53 53
diff --git a/src/lib.rs b/src/lib.rs
index 153458644..b90b70c05 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -16,14 +16,14 @@
16//#![warn(unreachable_pub)] // rust-lang/rust#47816 16//#![warn(unreachable_pub)] // rust-lang/rust#47816
17 17
18extern crate unicode_xid; 18extern crate unicode_xid;
19extern crate text_unit;
19 20
20mod text;
21mod tree; 21mod tree;
22mod lexer; 22mod lexer;
23mod parser; 23mod parser;
24 24
25pub mod syntax_kinds; 25pub mod syntax_kinds;
26pub use text::{TextRange, TextUnit}; 26pub use text_unit::{TextRange, TextUnit};
27pub use tree::{File, Node, SyntaxKind, Token}; 27pub use tree::{File, Node, SyntaxKind, Token};
28pub(crate) use tree::{ErrorMsg, FileBuilder, Sink}; 28pub(crate) use tree::{ErrorMsg, FileBuilder, Sink};
29pub use lexer::{next_token, tokenize}; 29pub use lexer::{next_token, tokenize};
diff --git a/src/parser/event.rs b/src/parser/event.rs
index 1c0905a38..ac8a55de9 100644
--- a/src/parser/event.rs
+++ b/src/parser/event.rs
@@ -1,5 +1,7 @@
1use {ErrorMsg, File, FileBuilder, Sink, SyntaxKind, TextUnit, Token}; 1use {
2use syntax_kinds::TOMBSTONE; 2 ErrorMsg, File, FileBuilder, Sink, SyntaxKind, Token,
3 syntax_kinds::TOMBSTONE,
4};
3use super::is_insignificant; 5use super::is_insignificant;
4 6
5/// `Parser` produces a flat list of `Event`s. 7/// `Parser` produces a flat list of `Event`s.
@@ -133,7 +135,7 @@ pub(super) fn to_file(text: String, tokens: &[Token], events: Vec<Event>) -> Fil
133 builder.leaf(token.kind, token.len); 135 builder.leaf(token.kind, token.len);
134 idx += 1 136 idx += 1
135 } 137 }
136 let mut len = TextUnit::new(0); 138 let mut len = 0.into();
137 for _ in 0..n_raw_tokens { 139 for _ in 0..n_raw_tokens {
138 len += tokens[idx].len; 140 len += tokens[idx].len;
139 idx += 1; 141 idx += 1;
diff --git a/src/parser/input.rs b/src/parser/input.rs
index 13589467b..9b400b959 100644
--- a/src/parser/input.rs
+++ b/src/parser/input.rs
@@ -14,7 +14,7 @@ impl<'t> ParserInput<'t> {
14 pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> ParserInput<'t> { 14 pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> ParserInput<'t> {
15 let mut tokens = Vec::new(); 15 let mut tokens = Vec::new();
16 let mut start_offsets = Vec::new(); 16 let mut start_offsets = Vec::new();
17 let mut len = TextUnit::new(0); 17 let mut len = 0.into();
18 for &token in raw_tokens.iter() { 18 for &token in raw_tokens.iter() {
19 if !is_insignificant(token.kind) { 19 if !is_insignificant(token.kind) {
20 tokens.push(token); 20 tokens.push(token);
@@ -44,7 +44,7 @@ impl<'t> ParserInput<'t> {
44 if !(idx < self.tokens.len()) { 44 if !(idx < self.tokens.len()) {
45 return ""; 45 return "";
46 } 46 }
47 let range = TextRange::from_len(self.start_offsets[idx], self.tokens[idx].len); 47 let range = TextRange::offset_len(self.start_offsets[idx], self.tokens[idx].len);
48 &self.text[range] 48 &self.text[range]
49 } 49 }
50} 50}
diff --git a/src/text.rs b/src/text.rs
deleted file mode 100644
index 4084bf44e..000000000
--- a/src/text.rs
+++ /dev/null
@@ -1,136 +0,0 @@
1use std::fmt;
2use std::ops;
3
4/// An text position in a source file
5#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
6pub struct TextUnit(u32);
7
8impl TextUnit {
9 /// The positional offset required for one character
10 pub fn len_of_char(c: char) -> TextUnit {
11 TextUnit(c.len_utf8() as u32)
12 }
13
14 #[allow(missing_docs)]
15 pub fn new(val: u32) -> TextUnit {
16 TextUnit(val)
17 }
18}
19
20impl fmt::Debug for TextUnit {
21 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
22 <Self as fmt::Display>::fmt(self, f)
23 }
24}
25
26impl fmt::Display for TextUnit {
27 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
28 self.0.fmt(f)
29 }
30}
31
32impl From<TextUnit> for u32 {
33 fn from(tu: TextUnit) -> u32 {
34 tu.0
35 }
36}
37
38impl From<u32> for TextUnit {
39 fn from(tu: u32) -> TextUnit {
40 TextUnit::new(tu)
41 }
42}
43
44impl ops::Add<TextUnit> for TextUnit {
45 type Output = TextUnit;
46 fn add(self, rhs: TextUnit) -> TextUnit {
47 TextUnit(self.0 + rhs.0)
48 }
49}
50
51impl ops::AddAssign<TextUnit> for TextUnit {
52 fn add_assign(&mut self, rhs: TextUnit) {
53 self.0 += rhs.0
54 }
55}
56
57impl ops::Sub<TextUnit> for TextUnit {
58 type Output = TextUnit;
59 fn sub(self, rhs: TextUnit) -> TextUnit {
60 TextUnit(self.0 - rhs.0)
61 }
62}
63
64impl ops::SubAssign<TextUnit> for TextUnit {
65 fn sub_assign(&mut self, rhs: TextUnit) {
66 self.0 -= rhs.0
67 }
68}
69
70/// A range of text in a source file
71#[derive(Clone, Copy, PartialEq, Eq)]
72pub struct TextRange {
73 start: TextUnit,
74 end: TextUnit,
75}
76
77impl fmt::Debug for TextRange {
78 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
79 <Self as fmt::Display>::fmt(self, f)
80 }
81}
82
83impl fmt::Display for TextRange {
84 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
85 write!(f, "[{}; {})", self.start(), self.end())
86 }
87}
88
89impl TextRange {
90 /// An length-0 range of text
91 pub fn empty() -> TextRange {
92 TextRange::from_to(TextUnit::new(0), TextUnit::new(0))
93 }
94
95 /// The left-inclusive range (`[from..to)`) between to points in the text
96 pub fn from_to(from: TextUnit, to: TextUnit) -> TextRange {
97 assert!(from <= to, "Invalid text range [{}; {})", from, to);
98 TextRange {
99 start: from,
100 end: to,
101 }
102 }
103
104 /// The range from some point over some length
105 pub fn from_len(from: TextUnit, len: TextUnit) -> TextRange {
106 TextRange::from_to(from, from + len)
107 }
108
109 /// The starting position of this range
110 pub fn start(&self) -> TextUnit {
111 self.start
112 }
113
114 /// The end position of this range
115 pub fn end(&self) -> TextUnit {
116 self.end
117 }
118
119 /// The length of this range
120 pub fn len(&self) -> TextUnit {
121 self.end - self.start
122 }
123
124 /// Is this range empty of any content?
125 pub fn is_empty(&self) -> bool {
126 self.start() == self.end()
127 }
128}
129
130impl ops::Index<TextRange> for str {
131 type Output = str;
132
133 fn index(&self, index: TextRange) -> &str {
134 &self[index.start().0 as usize..index.end().0 as usize]
135 }
136}
diff --git a/src/tree/file_builder.rs b/src/tree/file_builder.rs
index f831676c7..712602168 100644
--- a/src/tree/file_builder.rs
+++ b/src/tree/file_builder.rs
@@ -31,7 +31,7 @@ impl Sink for FileBuilder {
31 fn leaf(&mut self, kind: SyntaxKind, len: TextUnit) { 31 fn leaf(&mut self, kind: SyntaxKind, len: TextUnit) {
32 let leaf = NodeData { 32 let leaf = NodeData {
33 kind, 33 kind,
34 range: TextRange::from_len(self.pos, len), 34 range: TextRange::offset_len(self.pos, len),
35 parent: None, 35 parent: None,
36 first_child: None, 36 first_child: None,
37 next_sibling: None, 37 next_sibling: None,
@@ -44,7 +44,7 @@ impl Sink for FileBuilder {
44 fn start_internal(&mut self, kind: SyntaxKind) { 44 fn start_internal(&mut self, kind: SyntaxKind) {
45 let node = NodeData { 45 let node = NodeData {
46 kind, 46 kind,
47 range: TextRange::from_len(self.pos, 0.into()), 47 range: TextRange::offset_len(self.pos, 0.into()),
48 parent: None, 48 parent: None,
49 first_child: None, 49 first_child: None,
50 next_sibling: None, 50 next_sibling: None,
@@ -83,7 +83,7 @@ impl FileBuilder {
83 nodes: Vec::new(), 83 nodes: Vec::new(),
84 errors: Vec::new(), 84 errors: Vec::new(),
85 in_progress: Vec::new(), 85 in_progress: Vec::new(),
86 pos: TextUnit::new(0), 86 pos: 0.into(),
87 } 87 }
88 } 88 }
89 89
diff --git a/src/tree/mod.rs b/src/tree/mod.rs
index ebf26777b..f7b16d7b5 100644
--- a/src/tree/mod.rs
+++ b/src/tree/mod.rs
@@ -1,9 +1,7 @@
1use text::{TextRange, TextUnit};
2
3use std::fmt;
4use std::cmp;
5
6mod file_builder; 1mod file_builder;
2
3use ::{TextRange, TextUnit};
4use std::{fmt, cmp};
7pub(crate) use self::file_builder::{ErrorMsg, FileBuilder, Sink}; 5pub(crate) use self::file_builder::{ErrorMsg, FileBuilder, Sink};
8 6
9pub use syntax_kinds::SyntaxKind; 7pub use syntax_kinds::SyntaxKind;