aboutsummaryrefslogtreecommitdiff
path: root/crates/ra_syntax/src
diff options
context:
space:
mode:
Diffstat (limited to 'crates/ra_syntax/src')
-rw-r--r--crates/ra_syntax/src/algo.rs12
-rw-r--r--crates/ra_syntax/src/ast/make.rs11
-rw-r--r--crates/ra_syntax/src/ast/tokens.rs372
-rw-r--r--crates/ra_syntax/src/fuzz.rs15
-rw-r--r--crates/ra_syntax/src/lib.rs4
-rw-r--r--crates/ra_syntax/src/parsing/lexer.rs32
-rw-r--r--crates/ra_syntax/src/parsing/reparsing.rs4
-rw-r--r--crates/ra_syntax/src/parsing/text_token_source.rs8
-rw-r--r--crates/ra_syntax/src/parsing/text_tree_sink.rs14
-rw-r--r--crates/ra_syntax/src/ptr.rs2
-rw-r--r--crates/ra_syntax/src/syntax_error.rs6
-rw-r--r--crates/ra_syntax/src/syntax_node.rs4
-rw-r--r--crates/ra_syntax/src/tests.rs8
-rw-r--r--crates/ra_syntax/src/validation.rs6
14 files changed, 435 insertions, 63 deletions
diff --git a/crates/ra_syntax/src/algo.rs b/crates/ra_syntax/src/algo.rs
index 06df8495c..2a8dac757 100644
--- a/crates/ra_syntax/src/algo.rs
+++ b/crates/ra_syntax/src/algo.rs
@@ -11,7 +11,7 @@ use rustc_hash::FxHashMap;
11 11
12use crate::{ 12use crate::{
13 AstNode, Direction, NodeOrToken, SyntaxElement, SyntaxKind, SyntaxNode, SyntaxNodePtr, 13 AstNode, Direction, NodeOrToken, SyntaxElement, SyntaxKind, SyntaxNode, SyntaxNodePtr,
14 SyntaxToken, TextRange, TextUnit, 14 SyntaxToken, TextRange, TextSize,
15}; 15};
16 16
17/// Returns ancestors of the node at the offset, sorted by length. This should 17/// Returns ancestors of the node at the offset, sorted by length. This should
@@ -21,7 +21,7 @@ use crate::{
21/// t.parent().ancestors())`. 21/// t.parent().ancestors())`.
22pub fn ancestors_at_offset( 22pub fn ancestors_at_offset(
23 node: &SyntaxNode, 23 node: &SyntaxNode,
24 offset: TextUnit, 24 offset: TextSize,
25) -> impl Iterator<Item = SyntaxNode> { 25) -> impl Iterator<Item = SyntaxNode> {
26 node.token_at_offset(offset) 26 node.token_at_offset(offset)
27 .map(|token| token.parent().ancestors()) 27 .map(|token| token.parent().ancestors())
@@ -37,7 +37,7 @@ pub fn ancestors_at_offset(
37/// ``` 37/// ```
38/// 38///
39/// then the shorter node will be silently preferred. 39/// then the shorter node will be silently preferred.
40pub fn find_node_at_offset<N: AstNode>(syntax: &SyntaxNode, offset: TextUnit) -> Option<N> { 40pub fn find_node_at_offset<N: AstNode>(syntax: &SyntaxNode, offset: TextSize) -> Option<N> {
41 ancestors_at_offset(syntax, offset).find_map(N::cast) 41 ancestors_at_offset(syntax, offset).find_map(N::cast)
42} 42}
43 43
@@ -180,7 +180,7 @@ fn _insert_children(
180 position: InsertPosition<SyntaxElement>, 180 position: InsertPosition<SyntaxElement>,
181 to_insert: &mut dyn Iterator<Item = SyntaxElement>, 181 to_insert: &mut dyn Iterator<Item = SyntaxElement>,
182) -> SyntaxNode { 182) -> SyntaxNode {
183 let mut delta = TextUnit::default(); 183 let mut delta = TextSize::default();
184 let to_insert = to_insert.map(|element| { 184 let to_insert = to_insert.map(|element| {
185 delta += element.text_range().len(); 185 delta += element.text_range().len();
186 to_green_element(element) 186 to_green_element(element)
@@ -347,7 +347,7 @@ fn with_children(
347 parent: &SyntaxNode, 347 parent: &SyntaxNode,
348 new_children: Vec<NodeOrToken<rowan::GreenNode, rowan::GreenToken>>, 348 new_children: Vec<NodeOrToken<rowan::GreenNode, rowan::GreenToken>>,
349) -> SyntaxNode { 349) -> SyntaxNode {
350 let len = new_children.iter().map(|it| it.text_len()).sum::<TextUnit>(); 350 let len = new_children.iter().map(|it| it.text_len()).sum::<TextSize>();
351 let new_node = rowan::GreenNode::new(rowan::SyntaxKind(parent.kind() as u16), new_children); 351 let new_node = rowan::GreenNode::new(rowan::SyntaxKind(parent.kind() as u16), new_children);
352 let new_root_node = parent.replace_with(new_node); 352 let new_root_node = parent.replace_with(new_node);
353 let new_root_node = SyntaxNode::new_root(new_root_node); 353 let new_root_node = SyntaxNode::new_root(new_root_node);
@@ -355,7 +355,7 @@ fn with_children(
355 // FIXME: use a more elegant way to re-fetch the node (#1185), make 355 // FIXME: use a more elegant way to re-fetch the node (#1185), make
356 // `range` private afterwards 356 // `range` private afterwards
357 let mut ptr = SyntaxNodePtr::new(parent); 357 let mut ptr = SyntaxNodePtr::new(parent);
358 ptr.range = TextRange::offset_len(ptr.range.start(), len); 358 ptr.range = TextRange::at(ptr.range.start(), len);
359 ptr.to_node(&new_root_node) 359 ptr.to_node(&new_root_node)
360} 360}
361 361
diff --git a/crates/ra_syntax/src/ast/make.rs b/crates/ra_syntax/src/ast/make.rs
index 0f4a50be4..ee0f5cc40 100644
--- a/crates/ra_syntax/src/ast/make.rs
+++ b/crates/ra_syntax/src/ast/make.rs
@@ -293,11 +293,20 @@ pub fn fn_def(
293 ast_from_text(&format!("fn {}{}{} {}", fn_name, type_params, params, body)) 293 ast_from_text(&format!("fn {}{}{} {}", fn_name, type_params, params, body))
294} 294}
295 295
296pub fn add_newlines(amount_of_newlines: usize, t: impl AstNode) -> ast::SourceFile { 296pub fn add_leading_newlines(amount_of_newlines: usize, t: impl AstNode) -> ast::SourceFile {
297 let newlines = "\n".repeat(amount_of_newlines); 297 let newlines = "\n".repeat(amount_of_newlines);
298 ast_from_text(&format!("{}{}", newlines, t.syntax())) 298 ast_from_text(&format!("{}{}", newlines, t.syntax()))
299} 299}
300 300
301pub fn add_trailing_newlines(amount_of_newlines: usize, t: impl AstNode) -> ast::SourceFile {
302 let newlines = "\n".repeat(amount_of_newlines);
303 ast_from_text(&format!("{}{}", t.syntax(), newlines))
304}
305
306pub fn add_pub_crate_modifier(fn_def: ast::FnDef) -> ast::FnDef {
307 ast_from_text(&format!("pub(crate) {}", fn_def))
308}
309
301fn ast_from_text<N: AstNode>(text: &str) -> N { 310fn ast_from_text<N: AstNode>(text: &str) -> N {
302 let parse = SourceFile::parse(text); 311 let parse = SourceFile::parse(text);
303 let node = parse.tree().syntax().descendants().find_map(N::cast).unwrap(); 312 let node = parse.tree().syntax().descendants().find_map(N::cast).unwrap();
diff --git a/crates/ra_syntax/src/ast/tokens.rs b/crates/ra_syntax/src/ast/tokens.rs
index e8320b57e..3865729b8 100644
--- a/crates/ra_syntax/src/ast/tokens.rs
+++ b/crates/ra_syntax/src/ast/tokens.rs
@@ -1,8 +1,10 @@
1//! There are many AstNodes, but only a few tokens, so we hand-write them here. 1//! There are many AstNodes, but only a few tokens, so we hand-write them here.
2 2
3use std::convert::{TryFrom, TryInto};
4
3use crate::{ 5use crate::{
4 ast::{AstToken, Comment, RawString, String, Whitespace}, 6 ast::{AstToken, Comment, RawString, String, Whitespace},
5 TextRange, TextUnit, 7 TextRange, TextSize,
6}; 8};
7 9
8impl Comment { 10impl Comment {
@@ -56,6 +58,9 @@ const COMMENT_PREFIX_TO_KIND: &[(&str, CommentKind)] = {
56}; 58};
57 59
58fn kind_by_prefix(text: &str) -> CommentKind { 60fn kind_by_prefix(text: &str) -> CommentKind {
61 if text == "/**/" {
62 return CommentKind { shape: CommentShape::Block, doc: None };
63 }
59 for (prefix, kind) in COMMENT_PREFIX_TO_KIND.iter() { 64 for (prefix, kind) in COMMENT_PREFIX_TO_KIND.iter() {
60 if text.starts_with(prefix) { 65 if text.starts_with(prefix) {
61 return *kind; 66 return *kind;
@@ -94,14 +99,14 @@ impl QuoteOffsets {
94 return None; 99 return None;
95 } 100 }
96 101
97 let start = TextUnit::from(0); 102 let start = TextSize::from(0);
98 let left_quote = TextUnit::from_usize(left_quote) + TextUnit::of_char('"'); 103 let left_quote = TextSize::try_from(left_quote).unwrap() + TextSize::of('"');
99 let right_quote = TextUnit::from_usize(right_quote); 104 let right_quote = TextSize::try_from(right_quote).unwrap();
100 let end = TextUnit::of_str(literal); 105 let end = TextSize::of(literal);
101 106
102 let res = QuoteOffsets { 107 let res = QuoteOffsets {
103 quotes: [TextRange::from_to(start, left_quote), TextRange::from_to(right_quote, end)], 108 quotes: [TextRange::new(start, left_quote), TextRange::new(right_quote, end)],
104 contents: TextRange::from_to(left_quote, right_quote), 109 contents: TextRange::new(left_quote, right_quote),
105 }; 110 };
106 Some(res) 111 Some(res)
107 } 112 }
@@ -168,7 +173,358 @@ impl HasStringValue for RawString {
168impl RawString { 173impl RawString {
169 pub fn map_range_up(&self, range: TextRange) -> Option<TextRange> { 174 pub fn map_range_up(&self, range: TextRange) -> Option<TextRange> {
170 let contents_range = self.text_range_between_quotes()?; 175 let contents_range = self.text_range_between_quotes()?;
171 assert!(range.is_subrange(&TextRange::offset_len(0.into(), contents_range.len()))); 176 assert!(TextRange::up_to(contents_range.len()).contains_range(range));
172 Some(range + contents_range.start()) 177 Some(range + contents_range.start())
173 } 178 }
174} 179}
180
181#[derive(Debug)]
182pub enum FormatSpecifier {
183 Open,
184 Close,
185 Integer,
186 Identifier,
187 Colon,
188 Fill,
189 Align,
190 Sign,
191 NumberSign,
192 Zero,
193 DollarSign,
194 Dot,
195 Asterisk,
196 QuestionMark,
197}
198
199pub trait HasFormatSpecifier: AstToken {
200 fn char_ranges(
201 &self,
202 ) -> Option<Vec<(TextRange, Result<char, rustc_lexer::unescape::EscapeError>)>>;
203
204 fn lex_format_specifier<F>(&self, mut callback: F)
205 where
206 F: FnMut(TextRange, FormatSpecifier),
207 {
208 let char_ranges = if let Some(char_ranges) = self.char_ranges() {
209 char_ranges
210 } else {
211 return;
212 };
213 let mut chars = char_ranges.iter().peekable();
214
215 while let Some((range, first_char)) = chars.next() {
216 match first_char {
217 Ok('{') => {
218 // Format specifier, see syntax at https://doc.rust-lang.org/std/fmt/index.html#syntax
219 if let Some((_, Ok('{'))) = chars.peek() {
220 // Escaped format specifier, `{{`
221 chars.next();
222 continue;
223 }
224
225 callback(*range, FormatSpecifier::Open);
226
227 // check for integer/identifier
228 match chars
229 .peek()
230 .and_then(|next| next.1.as_ref().ok())
231 .copied()
232 .unwrap_or_default()
233 {
234 '0'..='9' => {
235 // integer
236 read_integer(&mut chars, &mut callback);
237 }
238 c if c == '_' || c.is_alphabetic() => {
239 // identifier
240 read_identifier(&mut chars, &mut callback);
241 }
242 _ => {}
243 }
244
245 if let Some((_, Ok(':'))) = chars.peek() {
246 skip_char_and_emit(&mut chars, FormatSpecifier::Colon, &mut callback);
247
248 // check for fill/align
249 let mut cloned = chars.clone().take(2);
250 let first = cloned
251 .next()
252 .and_then(|next| next.1.as_ref().ok())
253 .copied()
254 .unwrap_or_default();
255 let second = cloned
256 .next()
257 .and_then(|next| next.1.as_ref().ok())
258 .copied()
259 .unwrap_or_default();
260 match second {
261 '<' | '^' | '>' => {
262 // alignment specifier, first char specifies fillment
263 skip_char_and_emit(
264 &mut chars,
265 FormatSpecifier::Fill,
266 &mut callback,
267 );
268 skip_char_and_emit(
269 &mut chars,
270 FormatSpecifier::Align,
271 &mut callback,
272 );
273 }
274 _ => match first {
275 '<' | '^' | '>' => {
276 skip_char_and_emit(
277 &mut chars,
278 FormatSpecifier::Align,
279 &mut callback,
280 );
281 }
282 _ => {}
283 },
284 }
285
286 // check for sign
287 match chars
288 .peek()
289 .and_then(|next| next.1.as_ref().ok())
290 .copied()
291 .unwrap_or_default()
292 {
293 '+' | '-' => {
294 skip_char_and_emit(
295 &mut chars,
296 FormatSpecifier::Sign,
297 &mut callback,
298 );
299 }
300 _ => {}
301 }
302
303 // check for `#`
304 if let Some((_, Ok('#'))) = chars.peek() {
305 skip_char_and_emit(
306 &mut chars,
307 FormatSpecifier::NumberSign,
308 &mut callback,
309 );
310 }
311
312 // check for `0`
313 let mut cloned = chars.clone().take(2);
314 let first = cloned.next().and_then(|next| next.1.as_ref().ok()).copied();
315 let second = cloned.next().and_then(|next| next.1.as_ref().ok()).copied();
316
317 if first == Some('0') && second != Some('$') {
318 skip_char_and_emit(&mut chars, FormatSpecifier::Zero, &mut callback);
319 }
320
321 // width
322 match chars
323 .peek()
324 .and_then(|next| next.1.as_ref().ok())
325 .copied()
326 .unwrap_or_default()
327 {
328 '0'..='9' => {
329 read_integer(&mut chars, &mut callback);
330 if let Some((_, Ok('$'))) = chars.peek() {
331 skip_char_and_emit(
332 &mut chars,
333 FormatSpecifier::DollarSign,
334 &mut callback,
335 );
336 }
337 }
338 c if c == '_' || c.is_alphabetic() => {
339 read_identifier(&mut chars, &mut callback);
340 if chars.peek().and_then(|next| next.1.as_ref().ok()).copied()
341 != Some('$')
342 {
343 continue;
344 }
345 skip_char_and_emit(
346 &mut chars,
347 FormatSpecifier::DollarSign,
348 &mut callback,
349 );
350 }
351 _ => {}
352 }
353
354 // precision
355 if let Some((_, Ok('.'))) = chars.peek() {
356 skip_char_and_emit(&mut chars, FormatSpecifier::Dot, &mut callback);
357
358 match chars
359 .peek()
360 .and_then(|next| next.1.as_ref().ok())
361 .copied()
362 .unwrap_or_default()
363 {
364 '*' => {
365 skip_char_and_emit(
366 &mut chars,
367 FormatSpecifier::Asterisk,
368 &mut callback,
369 );
370 }
371 '0'..='9' => {
372 read_integer(&mut chars, &mut callback);
373 if let Some((_, Ok('$'))) = chars.peek() {
374 skip_char_and_emit(
375 &mut chars,
376 FormatSpecifier::DollarSign,
377 &mut callback,
378 );
379 }
380 }
381 c if c == '_' || c.is_alphabetic() => {
382 read_identifier(&mut chars, &mut callback);
383 if chars.peek().and_then(|next| next.1.as_ref().ok()).copied()
384 != Some('$')
385 {
386 continue;
387 }
388 skip_char_and_emit(
389 &mut chars,
390 FormatSpecifier::DollarSign,
391 &mut callback,
392 );
393 }
394 _ => {
395 continue;
396 }
397 }
398 }
399
400 // type
401 match chars
402 .peek()
403 .and_then(|next| next.1.as_ref().ok())
404 .copied()
405 .unwrap_or_default()
406 {
407 '?' => {
408 skip_char_and_emit(
409 &mut chars,
410 FormatSpecifier::QuestionMark,
411 &mut callback,
412 );
413 }
414 c if c == '_' || c.is_alphabetic() => {
415 read_identifier(&mut chars, &mut callback);
416 }
417 _ => {}
418 }
419 }
420
421 let mut cloned = chars.clone().take(2);
422 let first = cloned.next().and_then(|next| next.1.as_ref().ok()).copied();
423 let second = cloned.next().and_then(|next| next.1.as_ref().ok()).copied();
424 if first != Some('}') {
425 continue;
426 }
427 if second == Some('}') {
428 // Escaped format end specifier, `}}`
429 continue;
430 }
431 skip_char_and_emit(&mut chars, FormatSpecifier::Close, &mut callback);
432 }
433 _ => {
434 while let Some((_, Ok(next_char))) = chars.peek() {
435 match next_char {
436 '{' => break,
437 _ => {}
438 }
439 chars.next();
440 }
441 }
442 };
443 }
444
445 fn skip_char_and_emit<'a, I, F>(
446 chars: &mut std::iter::Peekable<I>,
447 emit: FormatSpecifier,
448 callback: &mut F,
449 ) where
450 I: Iterator<Item = &'a (TextRange, Result<char, rustc_lexer::unescape::EscapeError>)>,
451 F: FnMut(TextRange, FormatSpecifier),
452 {
453 let (range, _) = chars.next().unwrap();
454 callback(*range, emit);
455 }
456
457 fn read_integer<'a, I, F>(chars: &mut std::iter::Peekable<I>, callback: &mut F)
458 where
459 I: Iterator<Item = &'a (TextRange, Result<char, rustc_lexer::unescape::EscapeError>)>,
460 F: FnMut(TextRange, FormatSpecifier),
461 {
462 let (mut range, c) = chars.next().unwrap();
463 assert!(c.as_ref().unwrap().is_ascii_digit());
464 while let Some((r, Ok(next_char))) = chars.peek() {
465 if next_char.is_ascii_digit() {
466 chars.next();
467 range = range.cover(*r);
468 } else {
469 break;
470 }
471 }
472 callback(range, FormatSpecifier::Integer);
473 }
474
475 fn read_identifier<'a, I, F>(chars: &mut std::iter::Peekable<I>, callback: &mut F)
476 where
477 I: Iterator<Item = &'a (TextRange, Result<char, rustc_lexer::unescape::EscapeError>)>,
478 F: FnMut(TextRange, FormatSpecifier),
479 {
480 let (mut range, c) = chars.next().unwrap();
481 assert!(c.as_ref().unwrap().is_alphabetic() || *c.as_ref().unwrap() == '_');
482 while let Some((r, Ok(next_char))) = chars.peek() {
483 if *next_char == '_' || next_char.is_ascii_digit() || next_char.is_alphabetic() {
484 chars.next();
485 range = range.cover(*r);
486 } else {
487 break;
488 }
489 }
490 callback(range, FormatSpecifier::Identifier);
491 }
492 }
493}
494
495impl HasFormatSpecifier for String {
496 fn char_ranges(
497 &self,
498 ) -> Option<Vec<(TextRange, Result<char, rustc_lexer::unescape::EscapeError>)>> {
499 let text = self.text().as_str();
500 let text = &text[self.text_range_between_quotes()? - self.syntax().text_range().start()];
501 let offset = self.text_range_between_quotes()?.start() - self.syntax().text_range().start();
502
503 let mut res = Vec::with_capacity(text.len());
504 rustc_lexer::unescape::unescape_str(text, &mut |range, unescaped_char| {
505 res.push((
506 TextRange::new(range.start.try_into().unwrap(), range.end.try_into().unwrap())
507 + offset,
508 unescaped_char,
509 ))
510 });
511
512 Some(res)
513 }
514}
515
516impl HasFormatSpecifier for RawString {
517 fn char_ranges(
518 &self,
519 ) -> Option<Vec<(TextRange, Result<char, rustc_lexer::unescape::EscapeError>)>> {
520 let text = self.text().as_str();
521 let text = &text[self.text_range_between_quotes()? - self.syntax().text_range().start()];
522 let offset = self.text_range_between_quotes()?.start() - self.syntax().text_range().start();
523
524 let mut res = Vec::with_capacity(text.len());
525 for (idx, c) in text.char_indices() {
526 res.push((TextRange::at(idx.try_into().unwrap(), TextSize::of(c)) + offset, Ok(c)));
527 }
528 Some(res)
529 }
530}
diff --git a/crates/ra_syntax/src/fuzz.rs b/crates/ra_syntax/src/fuzz.rs
index 7012df7f0..10fbe3176 100644
--- a/crates/ra_syntax/src/fuzz.rs
+++ b/crates/ra_syntax/src/fuzz.rs
@@ -1,8 +1,13 @@
1//! FIXME: write short doc here 1//! FIXME: write short doc here
2 2
3use crate::{validation, AstNode, SourceFile, TextRange, TextUnit}; 3use std::{
4 convert::TryInto,
5 str::{self, FromStr},
6};
7
4use ra_text_edit::AtomTextEdit; 8use ra_text_edit::AtomTextEdit;
5use std::str::{self, FromStr}; 9
10use crate::{validation, AstNode, SourceFile, TextRange};
6 11
7fn check_file_invariants(file: &SourceFile) { 12fn check_file_invariants(file: &SourceFile) {
8 let root = file.syntax(); 13 let root = file.syntax();
@@ -34,10 +39,8 @@ impl CheckReparse {
34 let text = lines.collect::<Vec<_>>().join("\n"); 39 let text = lines.collect::<Vec<_>>().join("\n");
35 let text = format!("{}{}{}", PREFIX, text, SUFFIX); 40 let text = format!("{}{}{}", PREFIX, text, SUFFIX);
36 text.get(delete_start..delete_start.checked_add(delete_len)?)?; // make sure delete is a valid range 41 text.get(delete_start..delete_start.checked_add(delete_len)?)?; // make sure delete is a valid range
37 let delete = TextRange::offset_len( 42 let delete =
38 TextUnit::from_usize(delete_start), 43 TextRange::at(delete_start.try_into().unwrap(), delete_len.try_into().unwrap());
39 TextUnit::from_usize(delete_len),
40 );
41 let edited_text = 44 let edited_text =
42 format!("{}{}{}", &text[..delete_start], &insert, &text[delete_start + delete_len..]); 45 format!("{}{}{}", &text[..delete_start], &insert, &text[delete_start + delete_len..]);
43 let edit = AtomTextEdit { delete, insert }; 46 let edit = AtomTextEdit { delete, insert };
diff --git a/crates/ra_syntax/src/lib.rs b/crates/ra_syntax/src/lib.rs
index a796e78b1..ceeb2bde9 100644
--- a/crates/ra_syntax/src/lib.rs
+++ b/crates/ra_syntax/src/lib.rs
@@ -55,7 +55,7 @@ pub use crate::{
55 }, 55 },
56}; 56};
57pub use ra_parser::{SyntaxKind, T}; 57pub use ra_parser::{SyntaxKind, T};
58pub use rowan::{SmolStr, SyntaxText, TextRange, TextUnit, TokenAtOffset, WalkEvent}; 58pub use rowan::{SmolStr, SyntaxText, TextRange, TextSize, TokenAtOffset, WalkEvent};
59 59
60/// `Parse` is the result of the parsing: a syntax tree and a collection of 60/// `Parse` is the result of the parsing: a syntax tree and a collection of
61/// errors. 61/// errors.
@@ -266,7 +266,7 @@ fn api_walkthrough() {
266 assert_eq!(expr_syntax.kind(), SyntaxKind::BIN_EXPR); 266 assert_eq!(expr_syntax.kind(), SyntaxKind::BIN_EXPR);
267 267
268 // And text range: 268 // And text range:
269 assert_eq!(expr_syntax.text_range(), TextRange::from_to(32.into(), 37.into())); 269 assert_eq!(expr_syntax.text_range(), TextRange::new(32.into(), 37.into()));
270 270
271 // You can get node's text as a `SyntaxText` object, which will traverse the 271 // You can get node's text as a `SyntaxText` object, which will traverse the
272 // tree collecting token's text: 272 // tree collecting token's text:
diff --git a/crates/ra_syntax/src/parsing/lexer.rs b/crates/ra_syntax/src/parsing/lexer.rs
index 67c1f1b48..f450ef4a2 100644
--- a/crates/ra_syntax/src/parsing/lexer.rs
+++ b/crates/ra_syntax/src/parsing/lexer.rs
@@ -1,10 +1,12 @@
1//! Lexer analyzes raw input string and produces lexemes (tokens). 1//! Lexer analyzes raw input string and produces lexemes (tokens).
2//! It is just a bridge to `rustc_lexer`. 2//! It is just a bridge to `rustc_lexer`.
3 3
4use std::convert::TryInto;
5
4use crate::{ 6use crate::{
5 SyntaxError, 7 SyntaxError,
6 SyntaxKind::{self, *}, 8 SyntaxKind::{self, *},
7 TextRange, TextUnit, T, 9 TextRange, TextSize, T,
8}; 10};
9 11
10/// A token of Rust source. 12/// A token of Rust source.
@@ -13,7 +15,7 @@ pub struct Token {
13 /// The kind of token. 15 /// The kind of token.
14 pub kind: SyntaxKind, 16 pub kind: SyntaxKind,
15 /// The length of the token. 17 /// The length of the token.
16 pub len: TextUnit, 18 pub len: TextSize,
17} 19}
18 20
19/// Break a string up into its component tokens. 21/// Break a string up into its component tokens.
@@ -28,18 +30,19 @@ pub fn tokenize(text: &str) -> (Vec<Token>, Vec<SyntaxError>) {
28 let mut tokens = Vec::new(); 30 let mut tokens = Vec::new();
29 let mut errors = Vec::new(); 31 let mut errors = Vec::new();
30 32
31 let mut offset: usize = rustc_lexer::strip_shebang(text) 33 let mut offset = match rustc_lexer::strip_shebang(text) {
32 .map(|shebang_len| { 34 Some(shebang_len) => {
33 tokens.push(Token { kind: SHEBANG, len: TextUnit::from_usize(shebang_len) }); 35 tokens.push(Token { kind: SHEBANG, len: shebang_len.try_into().unwrap() });
34 shebang_len 36 shebang_len
35 }) 37 }
36 .unwrap_or(0); 38 None => 0,
39 };
37 40
38 let text_without_shebang = &text[offset..]; 41 let text_without_shebang = &text[offset..];
39 42
40 for rustc_token in rustc_lexer::tokenize(text_without_shebang) { 43 for rustc_token in rustc_lexer::tokenize(text_without_shebang) {
41 let token_len = TextUnit::from_usize(rustc_token.len); 44 let token_len: TextSize = rustc_token.len.try_into().unwrap();
42 let token_range = TextRange::offset_len(TextUnit::from_usize(offset), token_len); 45 let token_range = TextRange::at(offset.try_into().unwrap(), token_len);
43 46
44 let (syntax_kind, err_message) = 47 let (syntax_kind, err_message) =
45 rustc_token_kind_to_syntax_kind(&rustc_token.kind, &text[token_range]); 48 rustc_token_kind_to_syntax_kind(&rustc_token.kind, &text[token_range]);
@@ -65,7 +68,7 @@ pub fn tokenize(text: &str) -> (Vec<Token>, Vec<SyntaxError>) {
65/// Beware that unescape errors are not checked at tokenization time. 68/// Beware that unescape errors are not checked at tokenization time.
66pub fn lex_single_syntax_kind(text: &str) -> Option<(SyntaxKind, Option<SyntaxError>)> { 69pub fn lex_single_syntax_kind(text: &str) -> Option<(SyntaxKind, Option<SyntaxError>)> {
67 lex_first_token(text) 70 lex_first_token(text)
68 .filter(|(token, _)| token.len == TextUnit::of_str(text)) 71 .filter(|(token, _)| token.len == TextSize::of(text))
69 .map(|(token, error)| (token.kind, error)) 72 .map(|(token, error)| (token.kind, error))
70} 73}
71 74
@@ -75,7 +78,7 @@ pub fn lex_single_syntax_kind(text: &str) -> Option<(SyntaxKind, Option<SyntaxEr
75/// Beware that unescape errors are not checked at tokenization time. 78/// Beware that unescape errors are not checked at tokenization time.
76pub fn lex_single_valid_syntax_kind(text: &str) -> Option<SyntaxKind> { 79pub fn lex_single_valid_syntax_kind(text: &str) -> Option<SyntaxKind> {
77 lex_first_token(text) 80 lex_first_token(text)
78 .filter(|(token, error)| !error.is_some() && token.len == TextUnit::of_str(text)) 81 .filter(|(token, error)| !error.is_some() && token.len == TextSize::of(text))
79 .map(|(token, _error)| token.kind) 82 .map(|(token, _error)| token.kind)
80} 83}
81 84
@@ -96,10 +99,9 @@ fn lex_first_token(text: &str) -> Option<(Token, Option<SyntaxError>)> {
96 let rustc_token = rustc_lexer::first_token(text); 99 let rustc_token = rustc_lexer::first_token(text);
97 let (syntax_kind, err_message) = rustc_token_kind_to_syntax_kind(&rustc_token.kind, text); 100 let (syntax_kind, err_message) = rustc_token_kind_to_syntax_kind(&rustc_token.kind, text);
98 101
99 let token = Token { kind: syntax_kind, len: TextUnit::from_usize(rustc_token.len) }; 102 let token = Token { kind: syntax_kind, len: rustc_token.len.try_into().unwrap() };
100 let optional_error = err_message.map(|err_message| { 103 let optional_error = err_message
101 SyntaxError::new(err_message, TextRange::from_to(0.into(), TextUnit::of_str(text))) 104 .map(|err_message| SyntaxError::new(err_message, TextRange::up_to(TextSize::of(text))));
102 });
103 105
104 Some((token, optional_error)) 106 Some((token, optional_error))
105} 107}
diff --git a/crates/ra_syntax/src/parsing/reparsing.rs b/crates/ra_syntax/src/parsing/reparsing.rs
index 2d65b91f1..ffff0a7b2 100644
--- a/crates/ra_syntax/src/parsing/reparsing.rs
+++ b/crates/ra_syntax/src/parsing/reparsing.rs
@@ -19,7 +19,7 @@ use crate::{
19 syntax_node::{GreenNode, GreenToken, NodeOrToken, SyntaxElement, SyntaxNode}, 19 syntax_node::{GreenNode, GreenToken, NodeOrToken, SyntaxElement, SyntaxNode},
20 SyntaxError, 20 SyntaxError,
21 SyntaxKind::*, 21 SyntaxKind::*,
22 TextRange, TextUnit, T, 22 TextRange, TextSize, T,
23}; 23};
24 24
25pub(crate) fn incremental_reparse( 25pub(crate) fn incremental_reparse(
@@ -176,7 +176,7 @@ fn merge_errors(
176 if old_err_range.end() <= range_before_reparse.start() { 176 if old_err_range.end() <= range_before_reparse.start() {
177 res.push(old_err); 177 res.push(old_err);
178 } else if old_err_range.start() >= range_before_reparse.end() { 178 } else if old_err_range.start() >= range_before_reparse.end() {
179 let inserted_len = TextUnit::of_str(&edit.insert); 179 let inserted_len = TextSize::of(&edit.insert);
180 res.push(old_err.with_range((old_err_range + inserted_len) - edit.delete.len())); 180 res.push(old_err.with_range((old_err_range + inserted_len) - edit.delete.len()));
181 // Note: extra parens are intentional to prevent uint underflow, HWAB (here was a bug) 181 // Note: extra parens are intentional to prevent uint underflow, HWAB (here was a bug)
182 } 182 }
diff --git a/crates/ra_syntax/src/parsing/text_token_source.rs b/crates/ra_syntax/src/parsing/text_token_source.rs
index e2433913c..7ddc2c2c3 100644
--- a/crates/ra_syntax/src/parsing/text_token_source.rs
+++ b/crates/ra_syntax/src/parsing/text_token_source.rs
@@ -3,7 +3,7 @@
3use ra_parser::Token as PToken; 3use ra_parser::Token as PToken;
4use ra_parser::TokenSource; 4use ra_parser::TokenSource;
5 5
6use crate::{parsing::lexer::Token, SyntaxKind::EOF, TextRange, TextUnit}; 6use crate::{parsing::lexer::Token, SyntaxKind::EOF, TextRange, TextSize};
7 7
8pub(crate) struct TextTokenSource<'t> { 8pub(crate) struct TextTokenSource<'t> {
9 text: &'t str, 9 text: &'t str,
@@ -15,7 +15,7 @@ pub(crate) struct TextTokenSource<'t> {
15 /// 0 7 10 15 /// 0 7 10
16 /// ``` 16 /// ```
17 /// (token, start_offset): `[(struct, 0), (Foo, 7), (;, 10)]` 17 /// (token, start_offset): `[(struct, 0), (Foo, 7), (;, 10)]`
18 start_offsets: Vec<TextUnit>, 18 start_offsets: Vec<TextSize>,
19 /// non-whitespace/comment tokens 19 /// non-whitespace/comment tokens
20 /// ```non-rust 20 /// ```non-rust
21 /// struct Foo {} 21 /// struct Foo {}
@@ -51,12 +51,12 @@ impl<'t> TokenSource for TextTokenSource<'t> {
51 if pos >= self.tokens.len() { 51 if pos >= self.tokens.len() {
52 return false; 52 return false;
53 } 53 }
54 let range = TextRange::offset_len(self.start_offsets[pos], self.tokens[pos].len); 54 let range = TextRange::at(self.start_offsets[pos], self.tokens[pos].len);
55 self.text[range] == *kw 55 self.text[range] == *kw
56 } 56 }
57} 57}
58 58
59fn mk_token(pos: usize, start_offsets: &[TextUnit], tokens: &[Token]) -> PToken { 59fn mk_token(pos: usize, start_offsets: &[TextSize], tokens: &[Token]) -> PToken {
60 let kind = tokens.get(pos).map(|t| t.kind).unwrap_or(EOF); 60 let kind = tokens.get(pos).map(|t| t.kind).unwrap_or(EOF);
61 let is_jointed_to_next = if pos + 1 < start_offsets.len() { 61 let is_jointed_to_next = if pos + 1 < start_offsets.len() {
62 start_offsets[pos] + tokens[pos].len == start_offsets[pos + 1] 62 start_offsets[pos] + tokens[pos].len == start_offsets[pos + 1]
diff --git a/crates/ra_syntax/src/parsing/text_tree_sink.rs b/crates/ra_syntax/src/parsing/text_tree_sink.rs
index 87bb21cd9..22aed1db1 100644
--- a/crates/ra_syntax/src/parsing/text_tree_sink.rs
+++ b/crates/ra_syntax/src/parsing/text_tree_sink.rs
@@ -9,7 +9,7 @@ use crate::{
9 syntax_node::GreenNode, 9 syntax_node::GreenNode,
10 SmolStr, SyntaxError, 10 SmolStr, SyntaxError,
11 SyntaxKind::{self, *}, 11 SyntaxKind::{self, *},
12 SyntaxTreeBuilder, TextRange, TextUnit, 12 SyntaxTreeBuilder, TextRange, TextSize,
13}; 13};
14 14
15/// Bridges the parser with our specific syntax tree representation. 15/// Bridges the parser with our specific syntax tree representation.
@@ -18,7 +18,7 @@ use crate::{
18pub(crate) struct TextTreeSink<'a> { 18pub(crate) struct TextTreeSink<'a> {
19 text: &'a str, 19 text: &'a str,
20 tokens: &'a [Token], 20 tokens: &'a [Token],
21 text_pos: TextUnit, 21 text_pos: TextSize,
22 token_pos: usize, 22 token_pos: usize,
23 state: State, 23 state: State,
24 inner: SyntaxTreeBuilder, 24 inner: SyntaxTreeBuilder,
@@ -42,7 +42,7 @@ impl<'a> TreeSink for TextTreeSink<'a> {
42 let len = self.tokens[self.token_pos..self.token_pos + n_tokens] 42 let len = self.tokens[self.token_pos..self.token_pos + n_tokens]
43 .iter() 43 .iter()
44 .map(|it| it.len) 44 .map(|it| it.len)
45 .sum::<TextUnit>(); 45 .sum::<TextSize>();
46 self.do_token(kind, len, n_tokens); 46 self.do_token(kind, len, n_tokens);
47 } 47 }
48 48
@@ -62,12 +62,12 @@ impl<'a> TreeSink for TextTreeSink<'a> {
62 self.tokens[self.token_pos..].iter().take_while(|it| it.kind.is_trivia()).count(); 62 self.tokens[self.token_pos..].iter().take_while(|it| it.kind.is_trivia()).count();
63 let leading_trivias = &self.tokens[self.token_pos..self.token_pos + n_trivias]; 63 let leading_trivias = &self.tokens[self.token_pos..self.token_pos + n_trivias];
64 let mut trivia_end = 64 let mut trivia_end =
65 self.text_pos + leading_trivias.iter().map(|it| it.len).sum::<TextUnit>(); 65 self.text_pos + leading_trivias.iter().map(|it| it.len).sum::<TextSize>();
66 66
67 let n_attached_trivias = { 67 let n_attached_trivias = {
68 let leading_trivias = leading_trivias.iter().rev().map(|it| { 68 let leading_trivias = leading_trivias.iter().rev().map(|it| {
69 let next_end = trivia_end - it.len; 69 let next_end = trivia_end - it.len;
70 let range = TextRange::from_to(next_end, trivia_end); 70 let range = TextRange::new(next_end, trivia_end);
71 trivia_end = next_end; 71 trivia_end = next_end;
72 (it.kind, &self.text[range]) 72 (it.kind, &self.text[range])
73 }); 73 });
@@ -132,8 +132,8 @@ impl<'a> TextTreeSink<'a> {
132 } 132 }
133 } 133 }
134 134
135 fn do_token(&mut self, kind: SyntaxKind, len: TextUnit, n_tokens: usize) { 135 fn do_token(&mut self, kind: SyntaxKind, len: TextSize, n_tokens: usize) {
136 let range = TextRange::offset_len(self.text_pos, len); 136 let range = TextRange::at(self.text_pos, len);
137 let text: SmolStr = self.text[range].into(); 137 let text: SmolStr = self.text[range].into();
138 self.text_pos += len; 138 self.text_pos += len;
139 self.token_pos += n_tokens; 139 self.token_pos += n_tokens;
diff --git a/crates/ra_syntax/src/ptr.rs b/crates/ra_syntax/src/ptr.rs
index ecbfffcf4..62f03e93d 100644
--- a/crates/ra_syntax/src/ptr.rs
+++ b/crates/ra_syntax/src/ptr.rs
@@ -24,7 +24,7 @@ impl SyntaxNodePtr {
24 pub fn to_node(&self, root: &SyntaxNode) -> SyntaxNode { 24 pub fn to_node(&self, root: &SyntaxNode) -> SyntaxNode {
25 assert!(root.parent().is_none()); 25 assert!(root.parent().is_none());
26 successors(Some(root.clone()), |node| { 26 successors(Some(root.clone()), |node| {
27 node.children().find(|it| self.range.is_subrange(&it.text_range())) 27 node.children().find(|it| it.text_range().contains_range(self.range))
28 }) 28 })
29 .find(|it| it.text_range() == self.range && it.kind() == self.kind) 29 .find(|it| it.text_range() == self.range && it.kind() == self.kind)
30 .unwrap_or_else(|| panic!("can't resolve local ptr to SyntaxNode: {:?}", self)) 30 .unwrap_or_else(|| panic!("can't resolve local ptr to SyntaxNode: {:?}", self))
diff --git a/crates/ra_syntax/src/syntax_error.rs b/crates/ra_syntax/src/syntax_error.rs
index 54acf7847..7c4511fec 100644
--- a/crates/ra_syntax/src/syntax_error.rs
+++ b/crates/ra_syntax/src/syntax_error.rs
@@ -2,7 +2,7 @@
2 2
3use std::fmt; 3use std::fmt;
4 4
5use crate::{TextRange, TextUnit}; 5use crate::{TextRange, TextSize};
6 6
7/// Represents the result of unsuccessful tokenization, parsing 7/// Represents the result of unsuccessful tokenization, parsing
8/// or tree validation. 8/// or tree validation.
@@ -23,8 +23,8 @@ impl SyntaxError {
23 pub fn new(message: impl Into<String>, range: TextRange) -> Self { 23 pub fn new(message: impl Into<String>, range: TextRange) -> Self {
24 Self(message.into(), range) 24 Self(message.into(), range)
25 } 25 }
26 pub fn new_at_offset(message: impl Into<String>, offset: TextUnit) -> Self { 26 pub fn new_at_offset(message: impl Into<String>, offset: TextSize) -> Self {
27 Self(message.into(), TextRange::offset_len(offset, 0.into())) 27 Self(message.into(), TextRange::empty(offset))
28 } 28 }
29 29
30 pub fn range(&self) -> TextRange { 30 pub fn range(&self) -> TextRange {
diff --git a/crates/ra_syntax/src/syntax_node.rs b/crates/ra_syntax/src/syntax_node.rs
index 4e3a1460d..f9d379abf 100644
--- a/crates/ra_syntax/src/syntax_node.rs
+++ b/crates/ra_syntax/src/syntax_node.rs
@@ -8,7 +8,7 @@
8 8
9use rowan::{GreenNodeBuilder, Language}; 9use rowan::{GreenNodeBuilder, Language};
10 10
11use crate::{Parse, SmolStr, SyntaxError, SyntaxKind, TextUnit}; 11use crate::{Parse, SmolStr, SyntaxError, SyntaxKind, TextSize};
12 12
13pub(crate) use rowan::{GreenNode, GreenToken}; 13pub(crate) use rowan::{GreenNode, GreenToken};
14 14
@@ -69,7 +69,7 @@ impl SyntaxTreeBuilder {
69 self.inner.finish_node() 69 self.inner.finish_node()
70 } 70 }
71 71
72 pub fn error(&mut self, error: ra_parser::ParseError, text_pos: TextUnit) { 72 pub fn error(&mut self, error: ra_parser::ParseError, text_pos: TextSize) {
73 self.errors.push(SyntaxError::new_at_offset(error.0, text_pos)) 73 self.errors.push(SyntaxError::new_at_offset(error.0, text_pos))
74 } 74 }
75} 75}
diff --git a/crates/ra_syntax/src/tests.rs b/crates/ra_syntax/src/tests.rs
index 355843b94..aee57db62 100644
--- a/crates/ra_syntax/src/tests.rs
+++ b/crates/ra_syntax/src/tests.rs
@@ -5,7 +5,7 @@ use std::{
5 5
6use test_utils::{collect_rust_files, dir_tests, project_dir, read_text}; 6use test_utils::{collect_rust_files, dir_tests, project_dir, read_text};
7 7
8use crate::{fuzz, tokenize, SourceFile, SyntaxError, TextRange, TextUnit, Token}; 8use crate::{fuzz, tokenize, SourceFile, SyntaxError, TextRange, TextSize, Token};
9 9
10#[test] 10#[test]
11fn lexer_tests() { 11fn lexer_tests() {
@@ -121,12 +121,12 @@ fn assert_errors_are_absent(errors: &[SyntaxError], path: &Path) {
121 121
122fn dump_tokens_and_errors(tokens: &[Token], errors: &[SyntaxError], text: &str) -> String { 122fn dump_tokens_and_errors(tokens: &[Token], errors: &[SyntaxError], text: &str) -> String {
123 let mut acc = String::new(); 123 let mut acc = String::new();
124 let mut offset = TextUnit::from_usize(0); 124 let mut offset: TextSize = 0.into();
125 for token in tokens { 125 for token in tokens {
126 let token_len = token.len; 126 let token_len = token.len;
127 let token_text = &text[TextRange::offset_len(offset, token.len)]; 127 let token_text = &text[TextRange::at(offset, token.len)];
128 offset += token.len; 128 offset += token.len;
129 writeln!(acc, "{:?} {} {:?}", token.kind, token_len, token_text).unwrap(); 129 writeln!(acc, "{:?} {:?} {:?}", token.kind, token_len, token_text).unwrap();
130 } 130 }
131 for err in errors { 131 for err in errors {
132 writeln!(acc, "> error{:?} token({:?}) msg({})", err.range(), &text[err.range()], err) 132 writeln!(acc, "> error{:?} token({:?}) msg({})", err.range(), &text[err.range()], err)
diff --git a/crates/ra_syntax/src/validation.rs b/crates/ra_syntax/src/validation.rs
index f85b3e61b..5e93895ec 100644
--- a/crates/ra_syntax/src/validation.rs
+++ b/crates/ra_syntax/src/validation.rs
@@ -2,12 +2,14 @@
2 2
3mod block; 3mod block;
4 4
5use std::convert::TryFrom;
6
5use rustc_lexer::unescape; 7use rustc_lexer::unescape;
6 8
7use crate::{ 9use crate::{
8 ast, match_ast, AstNode, SyntaxError, 10 ast, match_ast, AstNode, SyntaxError,
9 SyntaxKind::{BYTE, BYTE_STRING, CHAR, CONST_DEF, FN_DEF, INT_NUMBER, STRING, TYPE_ALIAS_DEF}, 11 SyntaxKind::{BYTE, BYTE_STRING, CHAR, CONST_DEF, FN_DEF, INT_NUMBER, STRING, TYPE_ALIAS_DEF},
10 SyntaxNode, SyntaxToken, TextUnit, T, 12 SyntaxNode, SyntaxToken, TextSize, T,
11}; 13};
12 14
13fn rustc_unescape_error_to_string(err: unescape::EscapeError) -> &'static str { 15fn rustc_unescape_error_to_string(err: unescape::EscapeError) -> &'static str {
@@ -112,7 +114,7 @@ fn validate_literal(literal: ast::Literal, acc: &mut Vec<SyntaxError>) {
112 114
113 // FIXME: lift this lambda refactor to `fn` (https://github.com/rust-analyzer/rust-analyzer/pull/2834#discussion_r366199205) 115 // FIXME: lift this lambda refactor to `fn` (https://github.com/rust-analyzer/rust-analyzer/pull/2834#discussion_r366199205)
114 let mut push_err = |prefix_len, (off, err): (usize, unescape::EscapeError)| { 116 let mut push_err = |prefix_len, (off, err): (usize, unescape::EscapeError)| {
115 let off = token.text_range().start() + TextUnit::from_usize(off + prefix_len); 117 let off = token.text_range().start() + TextSize::try_from(off + prefix_len).unwrap();
116 acc.push(SyntaxError::new_at_offset(rustc_unescape_error_to_string(err), off)); 118 acc.push(SyntaxError::new_at_offset(rustc_unescape_error_to_string(err), off));
117 }; 119 };
118 120