diff options
author | Edwin Cheng <[email protected]> | 2019-04-07 17:12:07 +0100 |
---|---|---|
committer | Edwin Cheng <[email protected]> | 2019-04-07 17:12:07 +0100 |
commit | a7254201df07fb929ca689857d7472564d484c3e (patch) | |
tree | bb32649bf733ed892971ff5ada4fabbe58acc77c /crates | |
parent | 7f1e93a3c934342a9b9c418865ff395b8ad245ce (diff) |
Combine all tokensource to one and refactoring
Diffstat (limited to 'crates')
-rw-r--r-- | crates/ra_mbe/src/lib.rs | 1 | ||||
-rw-r--r-- | crates/ra_mbe/src/subtree_parser.rs | 59 | ||||
-rw-r--r-- | crates/ra_mbe/src/subtree_source.rs | 56 | ||||
-rw-r--r-- | crates/ra_mbe/src/syntax_bridge.rs | 6 | ||||
-rw-r--r-- | crates/ra_mbe/src/tt_cursor.rs | 58 |
5 files changed, 94 insertions, 86 deletions
diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index 38d3ec7e1..84ce2b783 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs | |||
@@ -21,6 +21,7 @@ mod mbe_expander; | |||
21 | mod syntax_bridge; | 21 | mod syntax_bridge; |
22 | mod tt_cursor; | 22 | mod tt_cursor; |
23 | mod subtree_source; | 23 | mod subtree_source; |
24 | mod subtree_parser; | ||
24 | 25 | ||
25 | use ra_syntax::SmolStr; | 26 | use ra_syntax::SmolStr; |
26 | 27 | ||
diff --git a/crates/ra_mbe/src/subtree_parser.rs b/crates/ra_mbe/src/subtree_parser.rs new file mode 100644 index 000000000..48eee6fa7 --- /dev/null +++ b/crates/ra_mbe/src/subtree_parser.rs | |||
@@ -0,0 +1,59 @@ | |||
1 | use crate::subtree_source::SubtreeTokenSource; | ||
2 | |||
3 | use ra_parser::{TokenSource, TreeSink}; | ||
4 | use ra_syntax::{SyntaxKind}; | ||
5 | |||
6 | struct OffsetTokenSink { | ||
7 | token_pos: usize, | ||
8 | } | ||
9 | |||
10 | impl TreeSink for OffsetTokenSink { | ||
11 | fn token(&mut self, _kind: SyntaxKind, n_tokens: u8) { | ||
12 | self.token_pos += n_tokens as usize; | ||
13 | } | ||
14 | fn start_node(&mut self, _kind: SyntaxKind) {} | ||
15 | fn finish_node(&mut self) {} | ||
16 | fn error(&mut self, _error: ra_parser::ParseError) {} | ||
17 | } | ||
18 | |||
19 | pub(crate) struct Parser<'a> { | ||
20 | subtree: &'a tt::Subtree, | ||
21 | pos: &'a mut usize, | ||
22 | } | ||
23 | |||
24 | impl<'a> Parser<'a> { | ||
25 | pub fn new(pos: &'a mut usize, subtree: &'a tt::Subtree) -> Parser<'a> { | ||
26 | Parser { pos, subtree } | ||
27 | } | ||
28 | |||
29 | pub fn parse_path(self) -> Option<tt::TokenTree> { | ||
30 | self.parse(ra_parser::parse_path) | ||
31 | } | ||
32 | |||
33 | fn parse<F>(self, f: F) -> Option<tt::TokenTree> | ||
34 | where | ||
35 | F: FnOnce(&dyn TokenSource, &mut dyn TreeSink), | ||
36 | { | ||
37 | let mut src = SubtreeTokenSource::new(self.subtree); | ||
38 | src.advance(*self.pos, true); | ||
39 | let mut sink = OffsetTokenSink { token_pos: 0 }; | ||
40 | |||
41 | f(&src, &mut sink); | ||
42 | |||
43 | self.finish(sink.token_pos, &mut src) | ||
44 | } | ||
45 | |||
46 | fn finish(self, parsed_token: usize, src: &mut SubtreeTokenSource) -> Option<tt::TokenTree> { | ||
47 | let res = src.bump_n(parsed_token, self.pos); | ||
48 | let res: Vec<_> = res.into_iter().cloned().collect(); | ||
49 | |||
50 | match res.len() { | ||
51 | 0 => None, | ||
52 | 1 => Some(res[0].clone()), | ||
53 | _ => Some(tt::TokenTree::Subtree(tt::Subtree { | ||
54 | delimiter: tt::Delimiter::None, | ||
55 | token_trees: res, | ||
56 | })), | ||
57 | } | ||
58 | } | ||
59 | } | ||
diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs index 8f5ce4ed5..d9ba5d3d0 100644 --- a/crates/ra_mbe/src/subtree_source.rs +++ b/crates/ra_mbe/src/subtree_source.rs | |||
@@ -9,12 +9,12 @@ struct TtToken { | |||
9 | pub n_tokens: usize, | 9 | pub n_tokens: usize, |
10 | } | 10 | } |
11 | 11 | ||
12 | /// SubtreeSourceQuerier let outside to query internal tokens as string | 12 | /// Querier let outside to query internal tokens as string |
13 | pub(crate) struct SubtreeSourceQuerier<'a> { | 13 | pub(crate) struct Querier<'a> { |
14 | src: &'a SubtreeTokenSource<'a>, | 14 | src: &'a SubtreeTokenSource<'a>, |
15 | } | 15 | } |
16 | 16 | ||
17 | impl<'a> SubtreeSourceQuerier<'a> { | 17 | impl<'a> Querier<'a> { |
18 | pub(crate) fn token(&self, uidx: usize) -> (SyntaxKind, &SmolStr) { | 18 | pub(crate) fn token(&self, uidx: usize) -> (SyntaxKind, &SmolStr) { |
19 | let tkn = &self.src.tokens[uidx]; | 19 | let tkn = &self.src.tokens[uidx]; |
20 | (tkn.kind, &tkn.text) | 20 | (tkn.kind, &tkn.text) |
@@ -32,7 +32,8 @@ impl<'a> SubtreeTokenSource<'a> { | |||
32 | SubtreeTokenSource { tokens: TtTokenBuilder::build(subtree), tt_pos: 0, subtree } | 32 | SubtreeTokenSource { tokens: TtTokenBuilder::build(subtree), tt_pos: 0, subtree } |
33 | } | 33 | } |
34 | 34 | ||
35 | pub fn advance(&mut self, curr: usize, skip_first_delimiter: bool) { | 35 | // Advance token source and skip the first delimiter |
36 | pub fn advance(&mut self, n_token: usize, skip_first_delimiter: bool) { | ||
36 | if skip_first_delimiter { | 37 | if skip_first_delimiter { |
37 | self.tt_pos += 1; | 38 | self.tt_pos += 1; |
38 | } | 39 | } |
@@ -47,32 +48,20 @@ impl<'a> SubtreeTokenSource<'a> { | |||
47 | // Such that we cannot simpliy advance the cursor | 48 | // Such that we cannot simpliy advance the cursor |
48 | // We have to bump it one by one | 49 | // We have to bump it one by one |
49 | let mut pos = 0; | 50 | let mut pos = 0; |
50 | while pos < curr { | 51 | while pos < n_token { |
51 | pos += self.bump(&self.subtree.token_trees[pos]); | 52 | pos += self.bump(&self.subtree.token_trees[pos]); |
52 | } | 53 | } |
53 | } | 54 | } |
54 | 55 | ||
55 | pub fn querier(&self) -> SubtreeSourceQuerier { | 56 | pub fn querier(&self) -> Querier { |
56 | SubtreeSourceQuerier { src: self } | 57 | Querier { src: self } |
57 | } | ||
58 | |||
59 | fn count(&self, tt: &tt::TokenTree) -> usize { | ||
60 | assert!(!self.tokens.is_empty()); | ||
61 | TtTokenBuilder::count_tt_tokens(tt, None) | ||
62 | } | ||
63 | |||
64 | pub(crate) fn bump(&mut self, tt: &tt::TokenTree) -> usize { | ||
65 | let cur = &self.tokens[self.tt_pos]; | ||
66 | let n_tokens = cur.n_tokens; | ||
67 | self.tt_pos += self.count(tt); | ||
68 | n_tokens | ||
69 | } | 58 | } |
70 | 59 | ||
71 | pub(crate) fn bump_n( | 60 | pub(crate) fn bump_n( |
72 | &mut self, | 61 | &mut self, |
73 | n_tokens: usize, | 62 | n_tt_tokens: usize, |
74 | mut token_pos: usize, | 63 | token_pos: &mut usize, |
75 | ) -> (usize, Vec<&tt::TokenTree>) { | 64 | ) -> Vec<&tt::TokenTree> { |
76 | let mut res = vec![]; | 65 | let mut res = vec![]; |
77 | // Matching `TtToken` cursor to `tt::TokenTree` cursor | 66 | // Matching `TtToken` cursor to `tt::TokenTree` cursor |
78 | // It is because TtToken is not One to One mapping to tt::Token | 67 | // It is because TtToken is not One to One mapping to tt::Token |
@@ -83,17 +72,28 @@ impl<'a> SubtreeTokenSource<'a> { | |||
83 | // | 72 | // |
84 | // Such that we cannot simpliy advance the cursor | 73 | // Such that we cannot simpliy advance the cursor |
85 | // We have to bump it one by one | 74 | // We have to bump it one by one |
86 | let next_pos = self.tt_pos + n_tokens; | 75 | let next_pos = self.tt_pos + n_tt_tokens; |
87 | let old_token_pos = token_pos; | ||
88 | 76 | ||
89 | while self.tt_pos < next_pos { | 77 | while self.tt_pos < next_pos { |
90 | let current = &self.subtree.token_trees[token_pos]; | 78 | let current = &self.subtree.token_trees[*token_pos]; |
91 | let n = self.bump(current); | 79 | let n = self.bump(current); |
92 | res.extend((0..n).map(|i| &self.subtree.token_trees[token_pos + i])); | 80 | res.extend((0..n).map(|i| &self.subtree.token_trees[*token_pos + i])); |
93 | token_pos += n; | 81 | *token_pos += n; |
94 | } | 82 | } |
95 | 83 | ||
96 | (token_pos - old_token_pos, res) | 84 | res |
85 | } | ||
86 | |||
87 | fn count(&self, tt: &tt::TokenTree) -> usize { | ||
88 | assert!(!self.tokens.is_empty()); | ||
89 | TtTokenBuilder::count_tt_tokens(tt, None) | ||
90 | } | ||
91 | |||
92 | fn bump(&mut self, tt: &tt::TokenTree) -> usize { | ||
93 | let cur = &self.tokens[self.tt_pos]; | ||
94 | let n_tokens = cur.n_tokens; | ||
95 | self.tt_pos += self.count(tt); | ||
96 | n_tokens | ||
97 | } | 97 | } |
98 | } | 98 | } |
99 | 99 | ||
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 102bba341..b0fb91a63 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs | |||
@@ -4,7 +4,7 @@ use ra_syntax::{ | |||
4 | ast, SyntaxKind::*, TextUnit | 4 | ast, SyntaxKind::*, TextUnit |
5 | }; | 5 | }; |
6 | 6 | ||
7 | use crate::subtree_source::{SubtreeTokenSource, SubtreeSourceQuerier}; | 7 | use crate::subtree_source::{SubtreeTokenSource, Querier}; |
8 | 8 | ||
9 | /// Maps `tt::TokenId` to the relative range of the original token. | 9 | /// Maps `tt::TokenId` to the relative range of the original token. |
10 | #[derive(Default)] | 10 | #[derive(Default)] |
@@ -107,14 +107,14 @@ fn convert_tt( | |||
107 | 107 | ||
108 | struct TtTreeSink<'a> { | 108 | struct TtTreeSink<'a> { |
109 | buf: String, | 109 | buf: String, |
110 | src_querier: SubtreeSourceQuerier<'a>, | 110 | src_querier: Querier<'a>, |
111 | text_pos: TextUnit, | 111 | text_pos: TextUnit, |
112 | token_pos: usize, | 112 | token_pos: usize, |
113 | inner: SyntaxTreeBuilder, | 113 | inner: SyntaxTreeBuilder, |
114 | } | 114 | } |
115 | 115 | ||
116 | impl<'a> TtTreeSink<'a> { | 116 | impl<'a> TtTreeSink<'a> { |
117 | fn new(src_querier: SubtreeSourceQuerier<'a>) -> TtTreeSink { | 117 | fn new(src_querier: Querier<'a>) -> TtTreeSink { |
118 | TtTreeSink { | 118 | TtTreeSink { |
119 | buf: String::new(), | 119 | buf: String::new(), |
120 | src_querier, | 120 | src_querier, |
diff --git a/crates/ra_mbe/src/tt_cursor.rs b/crates/ra_mbe/src/tt_cursor.rs index 52e072599..d29faa77c 100644 --- a/crates/ra_mbe/src/tt_cursor.rs +++ b/crates/ra_mbe/src/tt_cursor.rs | |||
@@ -1,25 +1,5 @@ | |||
1 | use crate::ParseError; | 1 | use crate::ParseError; |
2 | use crate::subtree_source::SubtreeTokenSource; | 2 | use crate::subtree_parser::Parser; |
3 | |||
4 | use ra_parser::{TokenSource, TreeSink}; | ||
5 | |||
6 | use ra_syntax::{ | ||
7 | SyntaxKind | ||
8 | }; | ||
9 | |||
10 | struct SubtreeTokenSink { | ||
11 | token_pos: usize, | ||
12 | } | ||
13 | |||
14 | impl TreeSink for SubtreeTokenSink { | ||
15 | fn token(&mut self, _kind: SyntaxKind, n_tokens: u8) { | ||
16 | self.token_pos += n_tokens as usize; | ||
17 | } | ||
18 | |||
19 | fn start_node(&mut self, _kind: SyntaxKind) {} | ||
20 | fn finish_node(&mut self) {} | ||
21 | fn error(&mut self, _error: ra_parser::ParseError) {} | ||
22 | } | ||
23 | 3 | ||
24 | #[derive(Clone)] | 4 | #[derive(Clone)] |
25 | pub(crate) struct TtCursor<'a> { | 5 | pub(crate) struct TtCursor<'a> { |
@@ -99,41 +79,9 @@ impl<'a> TtCursor<'a> { | |||
99 | }) | 79 | }) |
100 | } | 80 | } |
101 | 81 | ||
102 | fn eat_parse_result( | ||
103 | &mut self, | ||
104 | parsed_token: usize, | ||
105 | src: &mut SubtreeTokenSource, | ||
106 | ) -> Option<tt::TokenTree> { | ||
107 | let (adv, res) = src.bump_n(parsed_token, self.pos); | ||
108 | self.pos += adv; | ||
109 | |||
110 | let res: Vec<_> = res.into_iter().cloned().collect(); | ||
111 | |||
112 | match res.len() { | ||
113 | 0 => None, | ||
114 | 1 => Some(res[0].clone()), | ||
115 | _ => Some(tt::TokenTree::Subtree(tt::Subtree { | ||
116 | delimiter: tt::Delimiter::None, | ||
117 | token_trees: res, | ||
118 | })), | ||
119 | } | ||
120 | } | ||
121 | |||
122 | fn eat_parse<F>(&mut self, f: F) -> Option<tt::TokenTree> | ||
123 | where | ||
124 | F: FnOnce(&dyn TokenSource, &mut dyn TreeSink), | ||
125 | { | ||
126 | let mut src = SubtreeTokenSource::new(self.subtree); | ||
127 | src.advance(self.pos, true); | ||
128 | let mut sink = SubtreeTokenSink { token_pos: 0 }; | ||
129 | |||
130 | f(&src, &mut sink); | ||
131 | |||
132 | self.eat_parse_result(sink.token_pos, &mut src) | ||
133 | } | ||
134 | |||
135 | pub(crate) fn eat_path(&mut self) -> Option<tt::TokenTree> { | 82 | pub(crate) fn eat_path(&mut self) -> Option<tt::TokenTree> { |
136 | self.eat_parse(ra_parser::parse_path) | 83 | let parser = Parser::new(&mut self.pos, self.subtree); |
84 | parser.parse_path() | ||
137 | } | 85 | } |
138 | 86 | ||
139 | pub(crate) fn expect_char(&mut self, char: char) -> Result<(), ParseError> { | 87 | pub(crate) fn expect_char(&mut self, char: char) -> Result<(), ParseError> { |