aboutsummaryrefslogtreecommitdiff
path: root/crates/ra_mbe/src
diff options
context:
space:
mode:
authorEdwin Cheng <[email protected]>2019-05-25 13:31:53 +0100
committerEdwin Cheng <[email protected]>2019-05-25 13:41:03 +0100
commitfcb1eef3232c3fc673bf5f98595708e108c3950c (patch)
tree326a3961c5ca99cb50a12fb94f3398116af4a2f1 /crates/ra_mbe/src
parentef00b5af1c7a7a7cac685eff661a10252825d84a (diff)
Change TokenSource to iteration based
Diffstat (limited to 'crates/ra_mbe/src')
-rw-r--r--crates/ra_mbe/src/subtree_parser.rs4
-rw-r--r--crates/ra_mbe/src/subtree_source.rs59
-rw-r--r--crates/ra_mbe/src/syntax_bridge.rs42
3 files changed, 64 insertions, 41 deletions
diff --git a/crates/ra_mbe/src/subtree_parser.rs b/crates/ra_mbe/src/subtree_parser.rs
index 709b87a38..9cc989b23 100644
--- a/crates/ra_mbe/src/subtree_parser.rs
+++ b/crates/ra_mbe/src/subtree_parser.rs
@@ -68,13 +68,13 @@ impl<'a> Parser<'a> {
68 68
69 fn parse<F>(self, f: F) -> Option<tt::TokenTree> 69 fn parse<F>(self, f: F) -> Option<tt::TokenTree>
70 where 70 where
71 F: FnOnce(&dyn TokenSource, &mut dyn TreeSink), 71 F: FnOnce(&mut dyn TokenSource, &mut dyn TreeSink),
72 { 72 {
73 let buffer = TokenBuffer::new(&self.subtree.token_trees[*self.cur_pos..]); 73 let buffer = TokenBuffer::new(&self.subtree.token_trees[*self.cur_pos..]);
74 let mut src = SubtreeTokenSource::new(&buffer); 74 let mut src = SubtreeTokenSource::new(&buffer);
75 let mut sink = OffsetTokenSink { token_pos: 0, error: false }; 75 let mut sink = OffsetTokenSink { token_pos: 0, error: false };
76 76
77 f(&src, &mut sink); 77 f(&mut src, &mut sink);
78 78
79 let r = self.finish(sink.token_pos, &mut src); 79 let r = self.finish(sink.token_pos, &mut src);
80 if sink.error { 80 if sink.error {
diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs
index 972af4a7c..c4f79f38a 100644
--- a/crates/ra_mbe/src/subtree_source.rs
+++ b/crates/ra_mbe/src/subtree_source.rs
@@ -1,6 +1,7 @@
1use ra_parser::{TokenSource}; 1use ra_parser::{TokenSource, Token};
2use ra_syntax::{classify_literal, SmolStr, SyntaxKind, SyntaxKind::*, T}; 2use ra_syntax::{classify_literal, SmolStr, SyntaxKind, SyntaxKind::*, T};
3use std::cell::{RefCell, Cell}; 3use std::cell::{RefCell, Cell};
4use std::sync::Arc;
4use tt::buffer::{TokenBuffer, Cursor}; 5use tt::buffer::{TokenBuffer, Cursor};
5 6
6pub(crate) trait Querier { 7pub(crate) trait Querier {
@@ -65,7 +66,7 @@ impl<'a> SubtreeWalk<'a> {
65 return cached[pos].clone(); 66 return cached[pos].clone();
66 } 67 }
67 68
68 fn collect_token_trees(&mut self, n: usize) -> Vec<tt::TokenTree> { 69 fn collect_token_trees(&self, n: usize) -> Vec<tt::TokenTree> {
69 let mut res = vec![]; 70 let mut res = vec![];
70 71
71 let mut pos = 0; 72 let mut pos = 0;
@@ -117,43 +118,59 @@ impl<'a> Querier for SubtreeWalk<'a> {
117} 118}
118 119
119pub(crate) struct SubtreeTokenSource<'a> { 120pub(crate) struct SubtreeTokenSource<'a> {
120 walker: SubtreeWalk<'a>, 121 walker: Arc<SubtreeWalk<'a>>,
122 curr: (Token, usize),
121} 123}
122 124
123impl<'a> SubtreeTokenSource<'a> { 125impl<'a> SubtreeTokenSource<'a> {
124 pub fn new(buffer: &'a TokenBuffer) -> SubtreeTokenSource<'a> { 126 pub fn new(buffer: &'a TokenBuffer) -> SubtreeTokenSource<'a> {
125 SubtreeTokenSource { walker: SubtreeWalk::new(buffer.begin()) } 127 let mut res = SubtreeTokenSource {
128 walker: Arc::new(SubtreeWalk::new(buffer.begin())),
129 curr: (Token { kind: EOF, is_jointed_to_next: false }, 0),
130 };
131 res.curr = (res.mk_token(0), 0);
132 res
126 } 133 }
127 134
128 pub fn querier<'b>(&'a self) -> &'b SubtreeWalk<'a> 135 pub fn querier(&self) -> Arc<SubtreeWalk<'a>> {
129 where 136 self.walker.clone()
130 'a: 'b,
131 {
132 &self.walker
133 } 137 }
134 138
135 pub(crate) fn bump_n(&mut self, parsed_tokens: usize) -> Vec<tt::TokenTree> { 139 pub(crate) fn bump_n(&mut self, parsed_tokens: usize) -> Vec<tt::TokenTree> {
136 let res = self.walker.collect_token_trees(parsed_tokens); 140 let res = self.walker.collect_token_trees(parsed_tokens);
137 res 141 res
138 } 142 }
143
144 fn mk_token(&self, pos: usize) -> Token {
145 match self.walker.get(pos) {
146 Some(tt) => Token { kind: tt.kind, is_jointed_to_next: tt.is_joint_to_next },
147 None => Token { kind: EOF, is_jointed_to_next: false },
148 }
149 }
139} 150}
140 151
141impl<'a> TokenSource for SubtreeTokenSource<'a> { 152impl<'a> TokenSource for SubtreeTokenSource<'a> {
142 fn token_kind(&self, pos: usize) -> SyntaxKind { 153 fn current(&self) -> Token {
143 if let Some(tok) = self.walker.get(pos) { 154 self.curr.0
144 tok.kind
145 } else {
146 SyntaxKind::EOF
147 }
148 } 155 }
149 fn is_token_joint_to_next(&self, pos: usize) -> bool { 156
150 match self.walker.get(pos) { 157 /// Lookahead n token
151 Some(t) => t.is_joint_to_next, 158 fn lookahead_nth(&self, n: usize) -> Token {
152 _ => false, 159 self.mk_token(self.curr.1 + n)
160 }
161
162 /// bump cursor to next token
163 fn bump(&mut self) {
164 if self.current().kind == EOF {
165 return;
153 } 166 }
167
168 self.curr = (self.mk_token(self.curr.1 + 1), self.curr.1 + 1)
154 } 169 }
155 fn is_keyword(&self, pos: usize, kw: &str) -> bool { 170
156 match self.walker.get(pos) { 171 /// Is the current token a specified keyword?
172 fn is_keyword(&self, kw: &str) -> bool {
173 match self.walker.get(self.curr.1) {
157 Some(t) => t.text == *kw, 174 Some(t) => t.text == *kw,
158 _ => false, 175 _ => false,
159 } 176 }
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index 0a75305b4..0aab5ea8b 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -48,9 +48,10 @@ pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, Toke
48/// Parses the token tree (result of macro expansion) to an expression 48/// Parses the token tree (result of macro expansion) to an expression
49pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result<TreeArc<ast::Expr>, ExpandError> { 49pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result<TreeArc<ast::Expr>, ExpandError> {
50 let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); 50 let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
51 let token_source = SubtreeTokenSource::new(&buffer); 51 let mut token_source = SubtreeTokenSource::new(&buffer);
52 let mut tree_sink = TtTreeSink::new(token_source.querier()); 52 let querier = token_source.querier();
53 ra_parser::parse_expr(&token_source, &mut tree_sink); 53 let mut tree_sink = TtTreeSink::new(querier.as_ref());
54 ra_parser::parse_expr(&mut token_source, &mut tree_sink);
54 if tree_sink.roots.len() != 1 { 55 if tree_sink.roots.len() != 1 {
55 return Err(ExpandError::ConversionError); 56 return Err(ExpandError::ConversionError);
56 } 57 }
@@ -64,9 +65,10 @@ pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result<TreeArc<ast::Expr>, Expand
64/// Parses the token tree (result of macro expansion) to a Pattern 65/// Parses the token tree (result of macro expansion) to a Pattern
65pub fn token_tree_to_pat(tt: &tt::Subtree) -> Result<TreeArc<ast::Pat>, ExpandError> { 66pub fn token_tree_to_pat(tt: &tt::Subtree) -> Result<TreeArc<ast::Pat>, ExpandError> {
66 let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); 67 let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
67 let token_source = SubtreeTokenSource::new(&buffer); 68 let mut token_source = SubtreeTokenSource::new(&buffer);
68 let mut tree_sink = TtTreeSink::new(token_source.querier()); 69 let querier = token_source.querier();
69 ra_parser::parse_pat(&token_source, &mut tree_sink); 70 let mut tree_sink = TtTreeSink::new(querier.as_ref());
71 ra_parser::parse_pat(&mut token_source, &mut tree_sink);
70 if tree_sink.roots.len() != 1 { 72 if tree_sink.roots.len() != 1 {
71 return Err(ExpandError::ConversionError); 73 return Err(ExpandError::ConversionError);
72 } 74 }
@@ -78,9 +80,10 @@ pub fn token_tree_to_pat(tt: &tt::Subtree) -> Result<TreeArc<ast::Pat>, ExpandEr
78/// Parses the token tree (result of macro expansion) to a Type 80/// Parses the token tree (result of macro expansion) to a Type
79pub fn token_tree_to_ty(tt: &tt::Subtree) -> Result<TreeArc<ast::TypeRef>, ExpandError> { 81pub fn token_tree_to_ty(tt: &tt::Subtree) -> Result<TreeArc<ast::TypeRef>, ExpandError> {
80 let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); 82 let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
81 let token_source = SubtreeTokenSource::new(&buffer); 83 let mut token_source = SubtreeTokenSource::new(&buffer);
82 let mut tree_sink = TtTreeSink::new(token_source.querier()); 84 let querier = token_source.querier();
83 ra_parser::parse_ty(&token_source, &mut tree_sink); 85 let mut tree_sink = TtTreeSink::new(querier.as_ref());
86 ra_parser::parse_ty(&mut token_source, &mut tree_sink);
84 if tree_sink.roots.len() != 1 { 87 if tree_sink.roots.len() != 1 {
85 return Err(ExpandError::ConversionError); 88 return Err(ExpandError::ConversionError);
86 } 89 }
@@ -93,9 +96,10 @@ pub fn token_tree_to_macro_stmts(
93 tt: &tt::Subtree, 96 tt: &tt::Subtree,
94) -> Result<TreeArc<ast::MacroStmts>, ExpandError> { 97) -> Result<TreeArc<ast::MacroStmts>, ExpandError> {
95 let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); 98 let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
96 let token_source = SubtreeTokenSource::new(&buffer); 99 let mut token_source = SubtreeTokenSource::new(&buffer);
97 let mut tree_sink = TtTreeSink::new(token_source.querier()); 100 let querier = token_source.querier();
98 ra_parser::parse_macro_stmts(&token_source, &mut tree_sink); 101 let mut tree_sink = TtTreeSink::new(querier.as_ref());
102 ra_parser::parse_macro_stmts(&mut token_source, &mut tree_sink);
99 if tree_sink.roots.len() != 1 { 103 if tree_sink.roots.len() != 1 {
100 return Err(ExpandError::ConversionError); 104 return Err(ExpandError::ConversionError);
101 } 105 }
@@ -108,9 +112,10 @@ pub fn token_tree_to_macro_items(
108 tt: &tt::Subtree, 112 tt: &tt::Subtree,
109) -> Result<TreeArc<ast::MacroItems>, ExpandError> { 113) -> Result<TreeArc<ast::MacroItems>, ExpandError> {
110 let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); 114 let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
111 let token_source = SubtreeTokenSource::new(&buffer); 115 let mut token_source = SubtreeTokenSource::new(&buffer);
112 let mut tree_sink = TtTreeSink::new(token_source.querier()); 116 let querier = token_source.querier();
113 ra_parser::parse_macro_items(&token_source, &mut tree_sink); 117 let mut tree_sink = TtTreeSink::new(querier.as_ref());
118 ra_parser::parse_macro_items(&mut token_source, &mut tree_sink);
114 if tree_sink.roots.len() != 1 { 119 if tree_sink.roots.len() != 1 {
115 return Err(ExpandError::ConversionError); 120 return Err(ExpandError::ConversionError);
116 } 121 }
@@ -121,9 +126,10 @@ pub fn token_tree_to_macro_items(
121/// Parses the token tree (result of macro expansion) as a sequence of items 126/// Parses the token tree (result of macro expansion) as a sequence of items
122pub fn token_tree_to_ast_item_list(tt: &tt::Subtree) -> TreeArc<ast::SourceFile> { 127pub fn token_tree_to_ast_item_list(tt: &tt::Subtree) -> TreeArc<ast::SourceFile> {
123 let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); 128 let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
124 let token_source = SubtreeTokenSource::new(&buffer); 129 let mut token_source = SubtreeTokenSource::new(&buffer);
125 let mut tree_sink = TtTreeSink::new(token_source.querier()); 130 let querier = token_source.querier();
126 ra_parser::parse(&token_source, &mut tree_sink); 131 let mut tree_sink = TtTreeSink::new(querier.as_ref());
132 ra_parser::parse(&mut token_source, &mut tree_sink);
127 let syntax = tree_sink.inner.finish(); 133 let syntax = tree_sink.inner.finish();
128 ast::SourceFile::cast(&syntax).unwrap().to_owned() 134 ast::SourceFile::cast(&syntax).unwrap().to_owned()
129} 135}