diff options
-rw-r--r-- | Cargo.toml | 1 | ||||
-rw-r--r-- | src/lexer/ptr.rs | 4 | ||||
-rw-r--r-- | src/lib.rs | 4 | ||||
-rw-r--r-- | src/parser/event.rs | 8 | ||||
-rw-r--r-- | src/parser/input.rs | 4 | ||||
-rw-r--r-- | src/text.rs | 136 | ||||
-rw-r--r-- | src/tree/file_builder.rs | 6 | ||||
-rw-r--r-- | src/tree/mod.rs | 8 |
8 files changed, 18 insertions, 153 deletions
diff --git a/Cargo.toml b/Cargo.toml index 622abcc42..954f3e94b 100644 --- a/Cargo.toml +++ b/Cargo.toml | |||
@@ -9,6 +9,7 @@ members = [ "tools" ] | |||
9 | 9 | ||
10 | [dependencies] | 10 | [dependencies] |
11 | unicode-xid = "0.1.0" | 11 | unicode-xid = "0.1.0" |
12 | text_unit = "0.1.1" | ||
12 | 13 | ||
13 | [dev-dependencies] | 14 | [dev-dependencies] |
14 | testutils = { path = "./tests/testutils" } | 15 | testutils = { path = "./tests/testutils" } |
diff --git a/src/lexer/ptr.rs b/src/lexer/ptr.rs index 99d55b283..d1391fd5f 100644 --- a/src/lexer/ptr.rs +++ b/src/lexer/ptr.rs | |||
@@ -11,7 +11,7 @@ impl<'s> Ptr<'s> { | |||
11 | pub fn new(text: &'s str) -> Ptr<'s> { | 11 | pub fn new(text: &'s str) -> Ptr<'s> { |
12 | Ptr { | 12 | Ptr { |
13 | text, | 13 | text, |
14 | len: TextUnit::new(0), | 14 | len: 0.into(), |
15 | } | 15 | } |
16 | } | 16 | } |
17 | 17 | ||
@@ -47,7 +47,7 @@ impl<'s> Ptr<'s> { | |||
47 | 47 | ||
48 | pub fn bump(&mut self) -> Option<char> { | 48 | pub fn bump(&mut self) -> Option<char> { |
49 | let ch = self.chars().next()?; | 49 | let ch = self.chars().next()?; |
50 | self.len += TextUnit::len_of_char(ch); | 50 | self.len += TextUnit::of_char(ch); |
51 | Some(ch) | 51 | Some(ch) |
52 | } | 52 | } |
53 | 53 | ||
diff --git a/src/lib.rs b/src/lib.rs index 153458644..b90b70c05 100644 --- a/src/lib.rs +++ b/src/lib.rs | |||
@@ -16,14 +16,14 @@ | |||
16 | //#![warn(unreachable_pub)] // rust-lang/rust#47816 | 16 | //#![warn(unreachable_pub)] // rust-lang/rust#47816 |
17 | 17 | ||
18 | extern crate unicode_xid; | 18 | extern crate unicode_xid; |
19 | extern crate text_unit; | ||
19 | 20 | ||
20 | mod text; | ||
21 | mod tree; | 21 | mod tree; |
22 | mod lexer; | 22 | mod lexer; |
23 | mod parser; | 23 | mod parser; |
24 | 24 | ||
25 | pub mod syntax_kinds; | 25 | pub mod syntax_kinds; |
26 | pub use text::{TextRange, TextUnit}; | 26 | pub use text_unit::{TextRange, TextUnit}; |
27 | pub use tree::{File, Node, SyntaxKind, Token}; | 27 | pub use tree::{File, Node, SyntaxKind, Token}; |
28 | pub(crate) use tree::{ErrorMsg, FileBuilder, Sink}; | 28 | pub(crate) use tree::{ErrorMsg, FileBuilder, Sink}; |
29 | pub use lexer::{next_token, tokenize}; | 29 | pub use lexer::{next_token, tokenize}; |
diff --git a/src/parser/event.rs b/src/parser/event.rs index 1c0905a38..ac8a55de9 100644 --- a/src/parser/event.rs +++ b/src/parser/event.rs | |||
@@ -1,5 +1,7 @@ | |||
1 | use {ErrorMsg, File, FileBuilder, Sink, SyntaxKind, TextUnit, Token}; | 1 | use { |
2 | use syntax_kinds::TOMBSTONE; | 2 | ErrorMsg, File, FileBuilder, Sink, SyntaxKind, Token, |
3 | syntax_kinds::TOMBSTONE, | ||
4 | }; | ||
3 | use super::is_insignificant; | 5 | use super::is_insignificant; |
4 | 6 | ||
5 | /// `Parser` produces a flat list of `Event`s. | 7 | /// `Parser` produces a flat list of `Event`s. |
@@ -133,7 +135,7 @@ pub(super) fn to_file(text: String, tokens: &[Token], events: Vec<Event>) -> Fil | |||
133 | builder.leaf(token.kind, token.len); | 135 | builder.leaf(token.kind, token.len); |
134 | idx += 1 | 136 | idx += 1 |
135 | } | 137 | } |
136 | let mut len = TextUnit::new(0); | 138 | let mut len = 0.into(); |
137 | for _ in 0..n_raw_tokens { | 139 | for _ in 0..n_raw_tokens { |
138 | len += tokens[idx].len; | 140 | len += tokens[idx].len; |
139 | idx += 1; | 141 | idx += 1; |
diff --git a/src/parser/input.rs b/src/parser/input.rs index 13589467b..9b400b959 100644 --- a/src/parser/input.rs +++ b/src/parser/input.rs | |||
@@ -14,7 +14,7 @@ impl<'t> ParserInput<'t> { | |||
14 | pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> ParserInput<'t> { | 14 | pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> ParserInput<'t> { |
15 | let mut tokens = Vec::new(); | 15 | let mut tokens = Vec::new(); |
16 | let mut start_offsets = Vec::new(); | 16 | let mut start_offsets = Vec::new(); |
17 | let mut len = TextUnit::new(0); | 17 | let mut len = 0.into(); |
18 | for &token in raw_tokens.iter() { | 18 | for &token in raw_tokens.iter() { |
19 | if !is_insignificant(token.kind) { | 19 | if !is_insignificant(token.kind) { |
20 | tokens.push(token); | 20 | tokens.push(token); |
@@ -44,7 +44,7 @@ impl<'t> ParserInput<'t> { | |||
44 | if !(idx < self.tokens.len()) { | 44 | if !(idx < self.tokens.len()) { |
45 | return ""; | 45 | return ""; |
46 | } | 46 | } |
47 | let range = TextRange::from_len(self.start_offsets[idx], self.tokens[idx].len); | 47 | let range = TextRange::offset_len(self.start_offsets[idx], self.tokens[idx].len); |
48 | &self.text[range] | 48 | &self.text[range] |
49 | } | 49 | } |
50 | } | 50 | } |
diff --git a/src/text.rs b/src/text.rs deleted file mode 100644 index 4084bf44e..000000000 --- a/src/text.rs +++ /dev/null | |||
@@ -1,136 +0,0 @@ | |||
1 | use std::fmt; | ||
2 | use std::ops; | ||
3 | |||
4 | /// An text position in a source file | ||
5 | #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] | ||
6 | pub struct TextUnit(u32); | ||
7 | |||
8 | impl TextUnit { | ||
9 | /// The positional offset required for one character | ||
10 | pub fn len_of_char(c: char) -> TextUnit { | ||
11 | TextUnit(c.len_utf8() as u32) | ||
12 | } | ||
13 | |||
14 | #[allow(missing_docs)] | ||
15 | pub fn new(val: u32) -> TextUnit { | ||
16 | TextUnit(val) | ||
17 | } | ||
18 | } | ||
19 | |||
20 | impl fmt::Debug for TextUnit { | ||
21 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { | ||
22 | <Self as fmt::Display>::fmt(self, f) | ||
23 | } | ||
24 | } | ||
25 | |||
26 | impl fmt::Display for TextUnit { | ||
27 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { | ||
28 | self.0.fmt(f) | ||
29 | } | ||
30 | } | ||
31 | |||
32 | impl From<TextUnit> for u32 { | ||
33 | fn from(tu: TextUnit) -> u32 { | ||
34 | tu.0 | ||
35 | } | ||
36 | } | ||
37 | |||
38 | impl From<u32> for TextUnit { | ||
39 | fn from(tu: u32) -> TextUnit { | ||
40 | TextUnit::new(tu) | ||
41 | } | ||
42 | } | ||
43 | |||
44 | impl ops::Add<TextUnit> for TextUnit { | ||
45 | type Output = TextUnit; | ||
46 | fn add(self, rhs: TextUnit) -> TextUnit { | ||
47 | TextUnit(self.0 + rhs.0) | ||
48 | } | ||
49 | } | ||
50 | |||
51 | impl ops::AddAssign<TextUnit> for TextUnit { | ||
52 | fn add_assign(&mut self, rhs: TextUnit) { | ||
53 | self.0 += rhs.0 | ||
54 | } | ||
55 | } | ||
56 | |||
57 | impl ops::Sub<TextUnit> for TextUnit { | ||
58 | type Output = TextUnit; | ||
59 | fn sub(self, rhs: TextUnit) -> TextUnit { | ||
60 | TextUnit(self.0 - rhs.0) | ||
61 | } | ||
62 | } | ||
63 | |||
64 | impl ops::SubAssign<TextUnit> for TextUnit { | ||
65 | fn sub_assign(&mut self, rhs: TextUnit) { | ||
66 | self.0 -= rhs.0 | ||
67 | } | ||
68 | } | ||
69 | |||
70 | /// A range of text in a source file | ||
71 | #[derive(Clone, Copy, PartialEq, Eq)] | ||
72 | pub struct TextRange { | ||
73 | start: TextUnit, | ||
74 | end: TextUnit, | ||
75 | } | ||
76 | |||
77 | impl fmt::Debug for TextRange { | ||
78 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { | ||
79 | <Self as fmt::Display>::fmt(self, f) | ||
80 | } | ||
81 | } | ||
82 | |||
83 | impl fmt::Display for TextRange { | ||
84 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { | ||
85 | write!(f, "[{}; {})", self.start(), self.end()) | ||
86 | } | ||
87 | } | ||
88 | |||
89 | impl TextRange { | ||
90 | /// An length-0 range of text | ||
91 | pub fn empty() -> TextRange { | ||
92 | TextRange::from_to(TextUnit::new(0), TextUnit::new(0)) | ||
93 | } | ||
94 | |||
95 | /// The left-inclusive range (`[from..to)`) between to points in the text | ||
96 | pub fn from_to(from: TextUnit, to: TextUnit) -> TextRange { | ||
97 | assert!(from <= to, "Invalid text range [{}; {})", from, to); | ||
98 | TextRange { | ||
99 | start: from, | ||
100 | end: to, | ||
101 | } | ||
102 | } | ||
103 | |||
104 | /// The range from some point over some length | ||
105 | pub fn from_len(from: TextUnit, len: TextUnit) -> TextRange { | ||
106 | TextRange::from_to(from, from + len) | ||
107 | } | ||
108 | |||
109 | /// The starting position of this range | ||
110 | pub fn start(&self) -> TextUnit { | ||
111 | self.start | ||
112 | } | ||
113 | |||
114 | /// The end position of this range | ||
115 | pub fn end(&self) -> TextUnit { | ||
116 | self.end | ||
117 | } | ||
118 | |||
119 | /// The length of this range | ||
120 | pub fn len(&self) -> TextUnit { | ||
121 | self.end - self.start | ||
122 | } | ||
123 | |||
124 | /// Is this range empty of any content? | ||
125 | pub fn is_empty(&self) -> bool { | ||
126 | self.start() == self.end() | ||
127 | } | ||
128 | } | ||
129 | |||
130 | impl ops::Index<TextRange> for str { | ||
131 | type Output = str; | ||
132 | |||
133 | fn index(&self, index: TextRange) -> &str { | ||
134 | &self[index.start().0 as usize..index.end().0 as usize] | ||
135 | } | ||
136 | } | ||
diff --git a/src/tree/file_builder.rs b/src/tree/file_builder.rs index f831676c7..712602168 100644 --- a/src/tree/file_builder.rs +++ b/src/tree/file_builder.rs | |||
@@ -31,7 +31,7 @@ impl Sink for FileBuilder { | |||
31 | fn leaf(&mut self, kind: SyntaxKind, len: TextUnit) { | 31 | fn leaf(&mut self, kind: SyntaxKind, len: TextUnit) { |
32 | let leaf = NodeData { | 32 | let leaf = NodeData { |
33 | kind, | 33 | kind, |
34 | range: TextRange::from_len(self.pos, len), | 34 | range: TextRange::offset_len(self.pos, len), |
35 | parent: None, | 35 | parent: None, |
36 | first_child: None, | 36 | first_child: None, |
37 | next_sibling: None, | 37 | next_sibling: None, |
@@ -44,7 +44,7 @@ impl Sink for FileBuilder { | |||
44 | fn start_internal(&mut self, kind: SyntaxKind) { | 44 | fn start_internal(&mut self, kind: SyntaxKind) { |
45 | let node = NodeData { | 45 | let node = NodeData { |
46 | kind, | 46 | kind, |
47 | range: TextRange::from_len(self.pos, 0.into()), | 47 | range: TextRange::offset_len(self.pos, 0.into()), |
48 | parent: None, | 48 | parent: None, |
49 | first_child: None, | 49 | first_child: None, |
50 | next_sibling: None, | 50 | next_sibling: None, |
@@ -83,7 +83,7 @@ impl FileBuilder { | |||
83 | nodes: Vec::new(), | 83 | nodes: Vec::new(), |
84 | errors: Vec::new(), | 84 | errors: Vec::new(), |
85 | in_progress: Vec::new(), | 85 | in_progress: Vec::new(), |
86 | pos: TextUnit::new(0), | 86 | pos: 0.into(), |
87 | } | 87 | } |
88 | } | 88 | } |
89 | 89 | ||
diff --git a/src/tree/mod.rs b/src/tree/mod.rs index ebf26777b..f7b16d7b5 100644 --- a/src/tree/mod.rs +++ b/src/tree/mod.rs | |||
@@ -1,9 +1,7 @@ | |||
1 | use text::{TextRange, TextUnit}; | ||
2 | |||
3 | use std::fmt; | ||
4 | use std::cmp; | ||
5 | |||
6 | mod file_builder; | 1 | mod file_builder; |
2 | |||
3 | use ::{TextRange, TextUnit}; | ||
4 | use std::{fmt, cmp}; | ||
7 | pub(crate) use self::file_builder::{ErrorMsg, FileBuilder, Sink}; | 5 | pub(crate) use self::file_builder::{ErrorMsg, FileBuilder, Sink}; |
8 | 6 | ||
9 | pub use syntax_kinds::SyntaxKind; | 7 | pub use syntax_kinds::SyntaxKind; |