From 50a02eb3593591a02677e1b56f24d7ff0459b9d0 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Wed, 12 Aug 2020 17:06:49 +0200 Subject: Rename ra_parser -> parser --- crates/ra_syntax/src/parsing/text_token_source.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'crates/ra_syntax/src/parsing/text_token_source.rs') diff --git a/crates/ra_syntax/src/parsing/text_token_source.rs b/crates/ra_syntax/src/parsing/text_token_source.rs index 97aa3e795..df866dc2b 100644 --- a/crates/ra_syntax/src/parsing/text_token_source.rs +++ b/crates/ra_syntax/src/parsing/text_token_source.rs @@ -1,10 +1,10 @@ //! See `TextTokenSource` docs. -use ra_parser::TokenSource; +use parser::TokenSource; use crate::{parsing::lexer::Token, SyntaxKind::EOF, TextRange, TextSize}; -/// Implementation of `ra_parser::TokenSource` that takes tokens from source code text. +/// Implementation of `parser::TokenSource` that takes tokens from source code text. pub(crate) struct TextTokenSource<'t> { text: &'t str, /// token and its start position (non-whitespace/comment tokens) @@ -20,15 +20,15 @@ pub(crate) struct TextTokenSource<'t> { token_offset_pairs: Vec<(Token, TextSize)>, /// Current token and position - curr: (ra_parser::Token, usize), + curr: (parser::Token, usize), } impl<'t> TokenSource for TextTokenSource<'t> { - fn current(&self) -> ra_parser::Token { + fn current(&self) -> parser::Token { self.curr.0 } - fn lookahead_nth(&self, n: usize) -> ra_parser::Token { + fn lookahead_nth(&self, n: usize) -> parser::Token { mk_token(self.curr.1 + n, &self.token_offset_pairs) } @@ -49,7 +49,7 @@ impl<'t> TokenSource for TextTokenSource<'t> { } } -fn mk_token(pos: usize, token_offset_pairs: &[(Token, TextSize)]) -> ra_parser::Token { +fn mk_token(pos: usize, token_offset_pairs: &[(Token, TextSize)]) -> parser::Token { let (kind, is_jointed_to_next) = match token_offset_pairs.get(pos) { Some((token, offset)) => ( token.kind, @@ -60,7 +60,7 @@ fn mk_token(pos: usize, token_offset_pairs: &[(Token, TextSize)]) -> ra_parser:: ), None => (EOF, false), }; - ra_parser::Token { kind, is_jointed_to_next } + parser::Token { kind, is_jointed_to_next } } impl<'t> TextTokenSource<'t> { -- cgit v1.2.3 From a1c187eef3ba08076aedb5154929f7eda8d1b424 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Wed, 12 Aug 2020 18:26:51 +0200 Subject: Rename ra_syntax -> syntax --- crates/ra_syntax/src/parsing/text_token_source.rs | 84 ----------------------- 1 file changed, 84 deletions(-) delete mode 100644 crates/ra_syntax/src/parsing/text_token_source.rs (limited to 'crates/ra_syntax/src/parsing/text_token_source.rs') diff --git a/crates/ra_syntax/src/parsing/text_token_source.rs b/crates/ra_syntax/src/parsing/text_token_source.rs deleted file mode 100644 index df866dc2b..000000000 --- a/crates/ra_syntax/src/parsing/text_token_source.rs +++ /dev/null @@ -1,84 +0,0 @@ -//! See `TextTokenSource` docs. - -use parser::TokenSource; - -use crate::{parsing::lexer::Token, SyntaxKind::EOF, TextRange, TextSize}; - -/// Implementation of `parser::TokenSource` that takes tokens from source code text. -pub(crate) struct TextTokenSource<'t> { - text: &'t str, - /// token and its start position (non-whitespace/comment tokens) - /// ```non-rust - /// struct Foo; - /// ^------^--^- - /// | | \________ - /// | \____ \ - /// | \ | - /// (struct, 0) (Foo, 7) (;, 10) - /// ``` - /// `[(struct, 0), (Foo, 7), (;, 10)]` - token_offset_pairs: Vec<(Token, TextSize)>, - - /// Current token and position - curr: (parser::Token, usize), -} - -impl<'t> TokenSource for TextTokenSource<'t> { - fn current(&self) -> parser::Token { - self.curr.0 - } - - fn lookahead_nth(&self, n: usize) -> parser::Token { - mk_token(self.curr.1 + n, &self.token_offset_pairs) - } - - fn bump(&mut self) { - if self.curr.0.kind == EOF { - return; - } - - let pos = self.curr.1 + 1; - self.curr = (mk_token(pos, &self.token_offset_pairs), pos); - } - - fn is_keyword(&self, kw: &str) -> bool { - self.token_offset_pairs - .get(self.curr.1) - .map(|(token, offset)| &self.text[TextRange::at(*offset, token.len)] == kw) - .unwrap_or(false) - } -} - -fn mk_token(pos: usize, token_offset_pairs: &[(Token, TextSize)]) -> parser::Token { - let (kind, is_jointed_to_next) = match token_offset_pairs.get(pos) { - Some((token, offset)) => ( - token.kind, - token_offset_pairs - .get(pos + 1) - .map(|(_, next_offset)| offset + token.len == *next_offset) - .unwrap_or(false), - ), - None => (EOF, false), - }; - parser::Token { kind, is_jointed_to_next } -} - -impl<'t> TextTokenSource<'t> { - /// Generate input from tokens(expect comment and whitespace). - pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> TextTokenSource<'t> { - let token_offset_pairs: Vec<_> = raw_tokens - .iter() - .filter_map({ - let mut len = 0.into(); - move |token| { - let pair = if token.kind.is_trivia() { None } else { Some((*token, len)) }; - len += token.len; - pair - } - }) - .collect(); - - let first = mk_token(0, &token_offset_pairs); - TextTokenSource { text, token_offset_pairs, curr: (first, 0) } - } -} -- cgit v1.2.3