From 0c81b9deeed81bfb2cf8142af9d748317d5d71a1 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Wed, 20 Feb 2019 21:50:07 +0300 Subject: route parsing via TokenSource trait --- crates/ra_syntax/src/parsing/parser_impl/input.rs | 69 ++++++++++------------- 1 file changed, 31 insertions(+), 38 deletions(-) (limited to 'crates/ra_syntax/src/parsing/parser_impl/input.rs') diff --git a/crates/ra_syntax/src/parsing/parser_impl/input.rs b/crates/ra_syntax/src/parsing/parser_impl/input.rs index 275d94918..8ebbd3825 100644 --- a/crates/ra_syntax/src/parsing/parser_impl/input.rs +++ b/crates/ra_syntax/src/parsing/parser_impl/input.rs @@ -1,10 +1,40 @@ use crate::{ SyntaxKind, SyntaxKind::EOF, TextRange, TextUnit, - parsing::lexer::Token, + parsing::{ + parser_impl::TokenSource, + lexer::Token, + }, }; use std::ops::{Add, AddAssign}; +impl<'t> TokenSource for ParserInput<'t> { + fn token_kind(&self, pos: InputPosition) -> SyntaxKind { + let idx = pos.0 as usize; + if !(idx < self.tokens.len()) { + return EOF; + } + self.tokens[idx].kind + } + fn is_token_joint_to_next(&self, pos: InputPosition) -> bool { + let idx_curr = pos.0 as usize; + let idx_next = pos.0 as usize; + if !(idx_next < self.tokens.len()) { + return true; + } + self.start_offsets[idx_curr] + self.tokens[idx_curr].len == self.start_offsets[idx_next] + } + fn is_keyword(&self, pos: InputPosition, kw: &str) -> bool { + let idx = pos.0 as usize; + if !(idx < self.tokens.len()) { + return false; + } + let range = TextRange::offset_len(self.start_offsets[idx], self.tokens[idx].len); + + self.text[range] == *kw + } +} + pub(crate) struct ParserInput<'t> { text: &'t str, /// start position of each token(expect whitespace and comment) @@ -41,43 +71,6 @@ impl<'t> ParserInput<'t> { ParserInput { text, start_offsets, tokens } } - - /// Get the syntax kind of token at given input position. - pub fn kind(&self, pos: InputPosition) -> SyntaxKind { - let idx = pos.0 as usize; - if !(idx < self.tokens.len()) { - return EOF; - } - self.tokens[idx].kind - } - - /// Get the length of a token at given input position. - pub fn token_len(&self, pos: InputPosition) -> TextUnit { - let idx = pos.0 as usize; - if !(idx < self.tokens.len()) { - return 0.into(); - } - self.tokens[idx].len - } - - /// Get the start position of a taken at given input position. - pub fn token_start_at(&self, pos: InputPosition) -> TextUnit { - let idx = pos.0 as usize; - if !(idx < self.tokens.len()) { - return 0.into(); - } - self.start_offsets[idx] - } - - /// Get the raw text of a token at given input position. - pub fn token_text(&self, pos: InputPosition) -> &'t str { - let idx = pos.0 as usize; - if !(idx < self.tokens.len()) { - return ""; - } - let range = TextRange::offset_len(self.start_offsets[idx], self.tokens[idx].len); - &self.text[range] - } } #[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)] -- cgit v1.2.3 From 2b5e336ce7172914686b33c8ac1522911366fcf0 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Wed, 20 Feb 2019 22:19:12 +0300 Subject: move abstract traits to top --- crates/ra_syntax/src/parsing/parser_impl/input.rs | 32 +++-------------------- 1 file changed, 4 insertions(+), 28 deletions(-) (limited to 'crates/ra_syntax/src/parsing/parser_impl/input.rs') diff --git a/crates/ra_syntax/src/parsing/parser_impl/input.rs b/crates/ra_syntax/src/parsing/parser_impl/input.rs index 8ebbd3825..e9735e526 100644 --- a/crates/ra_syntax/src/parsing/parser_impl/input.rs +++ b/crates/ra_syntax/src/parsing/parser_impl/input.rs @@ -1,22 +1,21 @@ use crate::{ SyntaxKind, SyntaxKind::EOF, TextRange, TextUnit, parsing::{ + TokenPos, parser_impl::TokenSource, lexer::Token, }, }; -use std::ops::{Add, AddAssign}; - impl<'t> TokenSource for ParserInput<'t> { - fn token_kind(&self, pos: InputPosition) -> SyntaxKind { + fn token_kind(&self, pos: TokenPos) -> SyntaxKind { let idx = pos.0 as usize; if !(idx < self.tokens.len()) { return EOF; } self.tokens[idx].kind } - fn is_token_joint_to_next(&self, pos: InputPosition) -> bool { + fn is_token_joint_to_next(&self, pos: TokenPos) -> bool { let idx_curr = pos.0 as usize; let idx_next = pos.0 as usize; if !(idx_next < self.tokens.len()) { @@ -24,7 +23,7 @@ impl<'t> TokenSource for ParserInput<'t> { } self.start_offsets[idx_curr] + self.tokens[idx_curr].len == self.start_offsets[idx_next] } - fn is_keyword(&self, pos: InputPosition, kw: &str) -> bool { + fn is_keyword(&self, pos: TokenPos, kw: &str) -> bool { let idx = pos.0 as usize; if !(idx < self.tokens.len()) { return false; @@ -72,26 +71,3 @@ impl<'t> ParserInput<'t> { ParserInput { text, start_offsets, tokens } } } - -#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)] -pub(crate) struct InputPosition(u32); - -impl InputPosition { - pub fn new() -> Self { - InputPosition(0) - } -} - -impl Add for InputPosition { - type Output = InputPosition; - - fn add(self, rhs: u32) -> InputPosition { - InputPosition(self.0 + rhs) - } -} - -impl AddAssign for InputPosition { - fn add_assign(&mut self, rhs: u32) { - self.0 += rhs - } -} -- cgit v1.2.3 From e72ad0a2faac98972544dd42316ccf8090717102 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Wed, 20 Feb 2019 22:27:49 +0300 Subject: fix off by one error --- crates/ra_syntax/src/parsing/parser_impl/input.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'crates/ra_syntax/src/parsing/parser_impl/input.rs') diff --git a/crates/ra_syntax/src/parsing/parser_impl/input.rs b/crates/ra_syntax/src/parsing/parser_impl/input.rs index e9735e526..11b32b9ce 100644 --- a/crates/ra_syntax/src/parsing/parser_impl/input.rs +++ b/crates/ra_syntax/src/parsing/parser_impl/input.rs @@ -17,7 +17,7 @@ impl<'t> TokenSource for ParserInput<'t> { } fn is_token_joint_to_next(&self, pos: TokenPos) -> bool { let idx_curr = pos.0 as usize; - let idx_next = pos.0 as usize; + let idx_next = pos.0 as usize + 1; if !(idx_next < self.tokens.len()) { return true; } -- cgit v1.2.3 From cce23fddba4241202ebd29cce44db4ce9a08793a Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Wed, 20 Feb 2019 22:52:32 +0300 Subject: flattern module structure --- crates/ra_syntax/src/parsing/parser_impl/input.rs | 73 ----------------------- 1 file changed, 73 deletions(-) delete mode 100644 crates/ra_syntax/src/parsing/parser_impl/input.rs (limited to 'crates/ra_syntax/src/parsing/parser_impl/input.rs') diff --git a/crates/ra_syntax/src/parsing/parser_impl/input.rs b/crates/ra_syntax/src/parsing/parser_impl/input.rs deleted file mode 100644 index 11b32b9ce..000000000 --- a/crates/ra_syntax/src/parsing/parser_impl/input.rs +++ /dev/null @@ -1,73 +0,0 @@ -use crate::{ - SyntaxKind, SyntaxKind::EOF, TextRange, TextUnit, - parsing::{ - TokenPos, - parser_impl::TokenSource, - lexer::Token, - }, -}; - -impl<'t> TokenSource for ParserInput<'t> { - fn token_kind(&self, pos: TokenPos) -> SyntaxKind { - let idx = pos.0 as usize; - if !(idx < self.tokens.len()) { - return EOF; - } - self.tokens[idx].kind - } - fn is_token_joint_to_next(&self, pos: TokenPos) -> bool { - let idx_curr = pos.0 as usize; - let idx_next = pos.0 as usize + 1; - if !(idx_next < self.tokens.len()) { - return true; - } - self.start_offsets[idx_curr] + self.tokens[idx_curr].len == self.start_offsets[idx_next] - } - fn is_keyword(&self, pos: TokenPos, kw: &str) -> bool { - let idx = pos.0 as usize; - if !(idx < self.tokens.len()) { - return false; - } - let range = TextRange::offset_len(self.start_offsets[idx], self.tokens[idx].len); - - self.text[range] == *kw - } -} - -pub(crate) struct ParserInput<'t> { - text: &'t str, - /// start position of each token(expect whitespace and comment) - /// ```non-rust - /// struct Foo; - /// ^------^--- - /// | | ^- - /// 0 7 10 - /// ``` - /// (token, start_offset): `[(struct, 0), (Foo, 7), (;, 10)]` - start_offsets: Vec, - /// non-whitespace/comment tokens - /// ```non-rust - /// struct Foo {} - /// ^^^^^^ ^^^ ^^ - /// ``` - /// tokens: `[struct, Foo, {, }]` - tokens: Vec, -} - -impl<'t> ParserInput<'t> { - /// Generate input from tokens(expect comment and whitespace). - pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> ParserInput<'t> { - let mut tokens = Vec::new(); - let mut start_offsets = Vec::new(); - let mut len = 0.into(); - for &token in raw_tokens.iter() { - if !token.kind.is_trivia() { - tokens.push(token); - start_offsets.push(len); - } - len += token.len; - } - - ParserInput { text, start_offsets, tokens } - } -} -- cgit v1.2.3