diff options
author | bors[bot] <bors[bot]@users.noreply.github.com> | 2019-05-23 15:31:26 +0100 |
---|---|---|
committer | bors[bot] <bors[bot]@users.noreply.github.com> | 2019-05-23 15:31:26 +0100 |
commit | ef00b5af1c7a7a7cac685eff661a10252825d84a (patch) | |
tree | b48cf769d66b2c9366c5488cfa254520730a98a3 /crates/ra_mbe/src/syntax_bridge.rs | |
parent | eef24bddc96ddcdbcad5fddb9c0cf0e2ccad7681 (diff) | |
parent | 63b67134fdbfedfeab655ac9ff03730575fdf475 (diff) |
Merge #1312
1312: Introduce TokenBuffer r=matklad a=edwin0cheng
As discussed in Zulip, this PR Introduce `TokenBuffer` , a safe version of `syn` crate `TokenBuffer` which support cursor based traversal of `tt::TokenTree`. This is the basis of incoming refactoring of `TokenSource` iterator based API.
This PR do the following things:
* Add TokenBuffer in `ra_tt` crate.
* Try to use this new API to refactor the `SubtreeSource` to prove it usage.
Co-authored-by: Edwin Cheng <[email protected]>
Diffstat (limited to 'crates/ra_mbe/src/syntax_bridge.rs')
-rw-r--r-- | crates/ra_mbe/src/syntax_bridge.rs | 21 |
1 files changed, 14 insertions, 7 deletions
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index d8e344557..0a75305b4 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs | |||
@@ -47,7 +47,8 @@ pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, Toke | |||
47 | 47 | ||
48 | /// Parses the token tree (result of macro expansion) to an expression | 48 | /// Parses the token tree (result of macro expansion) to an expression |
49 | pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result<TreeArc<ast::Expr>, ExpandError> { | 49 | pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result<TreeArc<ast::Expr>, ExpandError> { |
50 | let token_source = SubtreeTokenSource::new(tt); | 50 | let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); |
51 | let token_source = SubtreeTokenSource::new(&buffer); | ||
51 | let mut tree_sink = TtTreeSink::new(token_source.querier()); | 52 | let mut tree_sink = TtTreeSink::new(token_source.querier()); |
52 | ra_parser::parse_expr(&token_source, &mut tree_sink); | 53 | ra_parser::parse_expr(&token_source, &mut tree_sink); |
53 | if tree_sink.roots.len() != 1 { | 54 | if tree_sink.roots.len() != 1 { |
@@ -62,7 +63,8 @@ pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result<TreeArc<ast::Expr>, Expand | |||
62 | 63 | ||
63 | /// Parses the token tree (result of macro expansion) to a Pattern | 64 | /// Parses the token tree (result of macro expansion) to a Pattern |
64 | pub fn token_tree_to_pat(tt: &tt::Subtree) -> Result<TreeArc<ast::Pat>, ExpandError> { | 65 | pub fn token_tree_to_pat(tt: &tt::Subtree) -> Result<TreeArc<ast::Pat>, ExpandError> { |
65 | let token_source = SubtreeTokenSource::new(tt); | 66 | let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); |
67 | let token_source = SubtreeTokenSource::new(&buffer); | ||
66 | let mut tree_sink = TtTreeSink::new(token_source.querier()); | 68 | let mut tree_sink = TtTreeSink::new(token_source.querier()); |
67 | ra_parser::parse_pat(&token_source, &mut tree_sink); | 69 | ra_parser::parse_pat(&token_source, &mut tree_sink); |
68 | if tree_sink.roots.len() != 1 { | 70 | if tree_sink.roots.len() != 1 { |
@@ -75,7 +77,8 @@ pub fn token_tree_to_pat(tt: &tt::Subtree) -> Result<TreeArc<ast::Pat>, ExpandEr | |||
75 | 77 | ||
76 | /// Parses the token tree (result of macro expansion) to a Type | 78 | /// Parses the token tree (result of macro expansion) to a Type |
77 | pub fn token_tree_to_ty(tt: &tt::Subtree) -> Result<TreeArc<ast::TypeRef>, ExpandError> { | 79 | pub fn token_tree_to_ty(tt: &tt::Subtree) -> Result<TreeArc<ast::TypeRef>, ExpandError> { |
78 | let token_source = SubtreeTokenSource::new(tt); | 80 | let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); |
81 | let token_source = SubtreeTokenSource::new(&buffer); | ||
79 | let mut tree_sink = TtTreeSink::new(token_source.querier()); | 82 | let mut tree_sink = TtTreeSink::new(token_source.querier()); |
80 | ra_parser::parse_ty(&token_source, &mut tree_sink); | 83 | ra_parser::parse_ty(&token_source, &mut tree_sink); |
81 | if tree_sink.roots.len() != 1 { | 84 | if tree_sink.roots.len() != 1 { |
@@ -89,7 +92,8 @@ pub fn token_tree_to_ty(tt: &tt::Subtree) -> Result<TreeArc<ast::TypeRef>, Expan | |||
89 | pub fn token_tree_to_macro_stmts( | 92 | pub fn token_tree_to_macro_stmts( |
90 | tt: &tt::Subtree, | 93 | tt: &tt::Subtree, |
91 | ) -> Result<TreeArc<ast::MacroStmts>, ExpandError> { | 94 | ) -> Result<TreeArc<ast::MacroStmts>, ExpandError> { |
92 | let token_source = SubtreeTokenSource::new(tt); | 95 | let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); |
96 | let token_source = SubtreeTokenSource::new(&buffer); | ||
93 | let mut tree_sink = TtTreeSink::new(token_source.querier()); | 97 | let mut tree_sink = TtTreeSink::new(token_source.querier()); |
94 | ra_parser::parse_macro_stmts(&token_source, &mut tree_sink); | 98 | ra_parser::parse_macro_stmts(&token_source, &mut tree_sink); |
95 | if tree_sink.roots.len() != 1 { | 99 | if tree_sink.roots.len() != 1 { |
@@ -103,7 +107,8 @@ pub fn token_tree_to_macro_stmts( | |||
103 | pub fn token_tree_to_macro_items( | 107 | pub fn token_tree_to_macro_items( |
104 | tt: &tt::Subtree, | 108 | tt: &tt::Subtree, |
105 | ) -> Result<TreeArc<ast::MacroItems>, ExpandError> { | 109 | ) -> Result<TreeArc<ast::MacroItems>, ExpandError> { |
106 | let token_source = SubtreeTokenSource::new(tt); | 110 | let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); |
111 | let token_source = SubtreeTokenSource::new(&buffer); | ||
107 | let mut tree_sink = TtTreeSink::new(token_source.querier()); | 112 | let mut tree_sink = TtTreeSink::new(token_source.querier()); |
108 | ra_parser::parse_macro_items(&token_source, &mut tree_sink); | 113 | ra_parser::parse_macro_items(&token_source, &mut tree_sink); |
109 | if tree_sink.roots.len() != 1 { | 114 | if tree_sink.roots.len() != 1 { |
@@ -115,7 +120,8 @@ pub fn token_tree_to_macro_items( | |||
115 | 120 | ||
116 | /// Parses the token tree (result of macro expansion) as a sequence of items | 121 | /// Parses the token tree (result of macro expansion) as a sequence of items |
117 | pub fn token_tree_to_ast_item_list(tt: &tt::Subtree) -> TreeArc<ast::SourceFile> { | 122 | pub fn token_tree_to_ast_item_list(tt: &tt::Subtree) -> TreeArc<ast::SourceFile> { |
118 | let token_source = SubtreeTokenSource::new(tt); | 123 | let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); |
124 | let token_source = SubtreeTokenSource::new(&buffer); | ||
119 | let mut tree_sink = TtTreeSink::new(token_source.querier()); | 125 | let mut tree_sink = TtTreeSink::new(token_source.querier()); |
120 | ra_parser::parse(&token_source, &mut tree_sink); | 126 | ra_parser::parse(&token_source, &mut tree_sink); |
121 | let syntax = tree_sink.inner.finish(); | 127 | let syntax = tree_sink.inner.finish(); |
@@ -381,7 +387,8 @@ mod tests { | |||
381 | "#, | 387 | "#, |
382 | ); | 388 | ); |
383 | let expansion = expand(&rules, "literals!(foo)"); | 389 | let expansion = expand(&rules, "literals!(foo)"); |
384 | let tt_src = SubtreeTokenSource::new(&expansion); | 390 | let buffer = tt::buffer::TokenBuffer::new(&[expansion.clone().into()]); |
391 | let tt_src = SubtreeTokenSource::new(&buffer); | ||
385 | 392 | ||
386 | let query = tt_src.querier(); | 393 | let query = tt_src.querier(); |
387 | 394 | ||