aboutsummaryrefslogtreecommitdiff
path: root/crates
diff options
context:
space:
mode:
authorEdwin Cheng <[email protected]>2019-04-07 17:12:07 +0100
committerEdwin Cheng <[email protected]>2019-04-07 17:12:07 +0100
commita7254201df07fb929ca689857d7472564d484c3e (patch)
treebb32649bf733ed892971ff5ada4fabbe58acc77c /crates
parent7f1e93a3c934342a9b9c418865ff395b8ad245ce (diff)
Combine all tokensource to one and refactoring
Diffstat (limited to 'crates')
-rw-r--r--crates/ra_mbe/src/lib.rs1
-rw-r--r--crates/ra_mbe/src/subtree_parser.rs59
-rw-r--r--crates/ra_mbe/src/subtree_source.rs56
-rw-r--r--crates/ra_mbe/src/syntax_bridge.rs6
-rw-r--r--crates/ra_mbe/src/tt_cursor.rs58
5 files changed, 94 insertions, 86 deletions
diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs
index 38d3ec7e1..84ce2b783 100644
--- a/crates/ra_mbe/src/lib.rs
+++ b/crates/ra_mbe/src/lib.rs
@@ -21,6 +21,7 @@ mod mbe_expander;
21mod syntax_bridge; 21mod syntax_bridge;
22mod tt_cursor; 22mod tt_cursor;
23mod subtree_source; 23mod subtree_source;
24mod subtree_parser;
24 25
25use ra_syntax::SmolStr; 26use ra_syntax::SmolStr;
26 27
diff --git a/crates/ra_mbe/src/subtree_parser.rs b/crates/ra_mbe/src/subtree_parser.rs
new file mode 100644
index 000000000..48eee6fa7
--- /dev/null
+++ b/crates/ra_mbe/src/subtree_parser.rs
@@ -0,0 +1,59 @@
1use crate::subtree_source::SubtreeTokenSource;
2
3use ra_parser::{TokenSource, TreeSink};
4use ra_syntax::{SyntaxKind};
5
6struct OffsetTokenSink {
7 token_pos: usize,
8}
9
10impl TreeSink for OffsetTokenSink {
11 fn token(&mut self, _kind: SyntaxKind, n_tokens: u8) {
12 self.token_pos += n_tokens as usize;
13 }
14 fn start_node(&mut self, _kind: SyntaxKind) {}
15 fn finish_node(&mut self) {}
16 fn error(&mut self, _error: ra_parser::ParseError) {}
17}
18
19pub(crate) struct Parser<'a> {
20 subtree: &'a tt::Subtree,
21 pos: &'a mut usize,
22}
23
24impl<'a> Parser<'a> {
25 pub fn new(pos: &'a mut usize, subtree: &'a tt::Subtree) -> Parser<'a> {
26 Parser { pos, subtree }
27 }
28
29 pub fn parse_path(self) -> Option<tt::TokenTree> {
30 self.parse(ra_parser::parse_path)
31 }
32
33 fn parse<F>(self, f: F) -> Option<tt::TokenTree>
34 where
35 F: FnOnce(&dyn TokenSource, &mut dyn TreeSink),
36 {
37 let mut src = SubtreeTokenSource::new(self.subtree);
38 src.advance(*self.pos, true);
39 let mut sink = OffsetTokenSink { token_pos: 0 };
40
41 f(&src, &mut sink);
42
43 self.finish(sink.token_pos, &mut src)
44 }
45
46 fn finish(self, parsed_token: usize, src: &mut SubtreeTokenSource) -> Option<tt::TokenTree> {
47 let res = src.bump_n(parsed_token, self.pos);
48 let res: Vec<_> = res.into_iter().cloned().collect();
49
50 match res.len() {
51 0 => None,
52 1 => Some(res[0].clone()),
53 _ => Some(tt::TokenTree::Subtree(tt::Subtree {
54 delimiter: tt::Delimiter::None,
55 token_trees: res,
56 })),
57 }
58 }
59}
diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs
index 8f5ce4ed5..d9ba5d3d0 100644
--- a/crates/ra_mbe/src/subtree_source.rs
+++ b/crates/ra_mbe/src/subtree_source.rs
@@ -9,12 +9,12 @@ struct TtToken {
9 pub n_tokens: usize, 9 pub n_tokens: usize,
10} 10}
11 11
12/// SubtreeSourceQuerier let outside to query internal tokens as string 12/// Querier let outside to query internal tokens as string
13pub(crate) struct SubtreeSourceQuerier<'a> { 13pub(crate) struct Querier<'a> {
14 src: &'a SubtreeTokenSource<'a>, 14 src: &'a SubtreeTokenSource<'a>,
15} 15}
16 16
17impl<'a> SubtreeSourceQuerier<'a> { 17impl<'a> Querier<'a> {
18 pub(crate) fn token(&self, uidx: usize) -> (SyntaxKind, &SmolStr) { 18 pub(crate) fn token(&self, uidx: usize) -> (SyntaxKind, &SmolStr) {
19 let tkn = &self.src.tokens[uidx]; 19 let tkn = &self.src.tokens[uidx];
20 (tkn.kind, &tkn.text) 20 (tkn.kind, &tkn.text)
@@ -32,7 +32,8 @@ impl<'a> SubtreeTokenSource<'a> {
32 SubtreeTokenSource { tokens: TtTokenBuilder::build(subtree), tt_pos: 0, subtree } 32 SubtreeTokenSource { tokens: TtTokenBuilder::build(subtree), tt_pos: 0, subtree }
33 } 33 }
34 34
35 pub fn advance(&mut self, curr: usize, skip_first_delimiter: bool) { 35 // Advance token source and skip the first delimiter
36 pub fn advance(&mut self, n_token: usize, skip_first_delimiter: bool) {
36 if skip_first_delimiter { 37 if skip_first_delimiter {
37 self.tt_pos += 1; 38 self.tt_pos += 1;
38 } 39 }
@@ -47,32 +48,20 @@ impl<'a> SubtreeTokenSource<'a> {
47 // Such that we cannot simpliy advance the cursor 48 // Such that we cannot simpliy advance the cursor
48 // We have to bump it one by one 49 // We have to bump it one by one
49 let mut pos = 0; 50 let mut pos = 0;
50 while pos < curr { 51 while pos < n_token {
51 pos += self.bump(&self.subtree.token_trees[pos]); 52 pos += self.bump(&self.subtree.token_trees[pos]);
52 } 53 }
53 } 54 }
54 55
55 pub fn querier(&self) -> SubtreeSourceQuerier { 56 pub fn querier(&self) -> Querier {
56 SubtreeSourceQuerier { src: self } 57 Querier { src: self }
57 }
58
59 fn count(&self, tt: &tt::TokenTree) -> usize {
60 assert!(!self.tokens.is_empty());
61 TtTokenBuilder::count_tt_tokens(tt, None)
62 }
63
64 pub(crate) fn bump(&mut self, tt: &tt::TokenTree) -> usize {
65 let cur = &self.tokens[self.tt_pos];
66 let n_tokens = cur.n_tokens;
67 self.tt_pos += self.count(tt);
68 n_tokens
69 } 58 }
70 59
71 pub(crate) fn bump_n( 60 pub(crate) fn bump_n(
72 &mut self, 61 &mut self,
73 n_tokens: usize, 62 n_tt_tokens: usize,
74 mut token_pos: usize, 63 token_pos: &mut usize,
75 ) -> (usize, Vec<&tt::TokenTree>) { 64 ) -> Vec<&tt::TokenTree> {
76 let mut res = vec![]; 65 let mut res = vec![];
77 // Matching `TtToken` cursor to `tt::TokenTree` cursor 66 // Matching `TtToken` cursor to `tt::TokenTree` cursor
78 // It is because TtToken is not One to One mapping to tt::Token 67 // It is because TtToken is not One to One mapping to tt::Token
@@ -83,17 +72,28 @@ impl<'a> SubtreeTokenSource<'a> {
83 // 72 //
84 // Such that we cannot simpliy advance the cursor 73 // Such that we cannot simpliy advance the cursor
85 // We have to bump it one by one 74 // We have to bump it one by one
86 let next_pos = self.tt_pos + n_tokens; 75 let next_pos = self.tt_pos + n_tt_tokens;
87 let old_token_pos = token_pos;
88 76
89 while self.tt_pos < next_pos { 77 while self.tt_pos < next_pos {
90 let current = &self.subtree.token_trees[token_pos]; 78 let current = &self.subtree.token_trees[*token_pos];
91 let n = self.bump(current); 79 let n = self.bump(current);
92 res.extend((0..n).map(|i| &self.subtree.token_trees[token_pos + i])); 80 res.extend((0..n).map(|i| &self.subtree.token_trees[*token_pos + i]));
93 token_pos += n; 81 *token_pos += n;
94 } 82 }
95 83
96 (token_pos - old_token_pos, res) 84 res
85 }
86
87 fn count(&self, tt: &tt::TokenTree) -> usize {
88 assert!(!self.tokens.is_empty());
89 TtTokenBuilder::count_tt_tokens(tt, None)
90 }
91
92 fn bump(&mut self, tt: &tt::TokenTree) -> usize {
93 let cur = &self.tokens[self.tt_pos];
94 let n_tokens = cur.n_tokens;
95 self.tt_pos += self.count(tt);
96 n_tokens
97 } 97 }
98} 98}
99 99
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index 102bba341..b0fb91a63 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -4,7 +4,7 @@ use ra_syntax::{
4 ast, SyntaxKind::*, TextUnit 4 ast, SyntaxKind::*, TextUnit
5}; 5};
6 6
7use crate::subtree_source::{SubtreeTokenSource, SubtreeSourceQuerier}; 7use crate::subtree_source::{SubtreeTokenSource, Querier};
8 8
9/// Maps `tt::TokenId` to the relative range of the original token. 9/// Maps `tt::TokenId` to the relative range of the original token.
10#[derive(Default)] 10#[derive(Default)]
@@ -107,14 +107,14 @@ fn convert_tt(
107 107
108struct TtTreeSink<'a> { 108struct TtTreeSink<'a> {
109 buf: String, 109 buf: String,
110 src_querier: SubtreeSourceQuerier<'a>, 110 src_querier: Querier<'a>,
111 text_pos: TextUnit, 111 text_pos: TextUnit,
112 token_pos: usize, 112 token_pos: usize,
113 inner: SyntaxTreeBuilder, 113 inner: SyntaxTreeBuilder,
114} 114}
115 115
116impl<'a> TtTreeSink<'a> { 116impl<'a> TtTreeSink<'a> {
117 fn new(src_querier: SubtreeSourceQuerier<'a>) -> TtTreeSink { 117 fn new(src_querier: Querier<'a>) -> TtTreeSink {
118 TtTreeSink { 118 TtTreeSink {
119 buf: String::new(), 119 buf: String::new(),
120 src_querier, 120 src_querier,
diff --git a/crates/ra_mbe/src/tt_cursor.rs b/crates/ra_mbe/src/tt_cursor.rs
index 52e072599..d29faa77c 100644
--- a/crates/ra_mbe/src/tt_cursor.rs
+++ b/crates/ra_mbe/src/tt_cursor.rs
@@ -1,25 +1,5 @@
1use crate::ParseError; 1use crate::ParseError;
2use crate::subtree_source::SubtreeTokenSource; 2use crate::subtree_parser::Parser;
3
4use ra_parser::{TokenSource, TreeSink};
5
6use ra_syntax::{
7 SyntaxKind
8};
9
10struct SubtreeTokenSink {
11 token_pos: usize,
12}
13
14impl TreeSink for SubtreeTokenSink {
15 fn token(&mut self, _kind: SyntaxKind, n_tokens: u8) {
16 self.token_pos += n_tokens as usize;
17 }
18
19 fn start_node(&mut self, _kind: SyntaxKind) {}
20 fn finish_node(&mut self) {}
21 fn error(&mut self, _error: ra_parser::ParseError) {}
22}
23 3
24#[derive(Clone)] 4#[derive(Clone)]
25pub(crate) struct TtCursor<'a> { 5pub(crate) struct TtCursor<'a> {
@@ -99,41 +79,9 @@ impl<'a> TtCursor<'a> {
99 }) 79 })
100 } 80 }
101 81
102 fn eat_parse_result(
103 &mut self,
104 parsed_token: usize,
105 src: &mut SubtreeTokenSource,
106 ) -> Option<tt::TokenTree> {
107 let (adv, res) = src.bump_n(parsed_token, self.pos);
108 self.pos += adv;
109
110 let res: Vec<_> = res.into_iter().cloned().collect();
111
112 match res.len() {
113 0 => None,
114 1 => Some(res[0].clone()),
115 _ => Some(tt::TokenTree::Subtree(tt::Subtree {
116 delimiter: tt::Delimiter::None,
117 token_trees: res,
118 })),
119 }
120 }
121
122 fn eat_parse<F>(&mut self, f: F) -> Option<tt::TokenTree>
123 where
124 F: FnOnce(&dyn TokenSource, &mut dyn TreeSink),
125 {
126 let mut src = SubtreeTokenSource::new(self.subtree);
127 src.advance(self.pos, true);
128 let mut sink = SubtreeTokenSink { token_pos: 0 };
129
130 f(&src, &mut sink);
131
132 self.eat_parse_result(sink.token_pos, &mut src)
133 }
134
135 pub(crate) fn eat_path(&mut self) -> Option<tt::TokenTree> { 82 pub(crate) fn eat_path(&mut self) -> Option<tt::TokenTree> {
136 self.eat_parse(ra_parser::parse_path) 83 let parser = Parser::new(&mut self.pos, self.subtree);
84 parser.parse_path()
137 } 85 }
138 86
139 pub(crate) fn expect_char(&mut self, char: char) -> Result<(), ParseError> { 87 pub(crate) fn expect_char(&mut self, char: char) -> Result<(), ParseError> {