From 74e846b9ecffd819af3109c50e48517b560b17cf Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Thu, 11 Apr 2019 11:02:41 +0800 Subject: Add L_DOLLAR and R_DOLLAR --- crates/ra_parser/src/syntax_kind/generated.rs | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'crates/ra_parser') diff --git a/crates/ra_parser/src/syntax_kind/generated.rs b/crates/ra_parser/src/syntax_kind/generated.rs index 547af1b27..b16177de2 100644 --- a/crates/ra_parser/src/syntax_kind/generated.rs +++ b/crates/ra_parser/src/syntax_kind/generated.rs @@ -119,6 +119,8 @@ pub enum SyntaxKind { LIFETIME, COMMENT, SHEBANG, + L_DOLLAR, + R_DOLLAR, SOURCE_FILE, STRUCT_DEF, ENUM_DEF, @@ -460,6 +462,8 @@ impl SyntaxKind { LIFETIME => &SyntaxInfo { name: "LIFETIME" }, COMMENT => &SyntaxInfo { name: "COMMENT" }, SHEBANG => &SyntaxInfo { name: "SHEBANG" }, + L_DOLLAR => &SyntaxInfo { name: "L_DOLLAR" }, + R_DOLLAR => &SyntaxInfo { name: "R_DOLLAR" }, SOURCE_FILE => &SyntaxInfo { name: "SOURCE_FILE" }, STRUCT_DEF => &SyntaxInfo { name: "STRUCT_DEF" }, ENUM_DEF => &SyntaxInfo { name: "ENUM_DEF" }, -- cgit v1.2.3 From f66300ccd1e6ef05b633cda06c87f913d1c91a1e Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Sat, 13 Apr 2019 01:50:05 +0800 Subject: Remove skip Delimiter::None and handle Dollars --- crates/ra_parser/src/parser.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'crates/ra_parser') diff --git a/crates/ra_parser/src/parser.rs b/crates/ra_parser/src/parser.rs index 56f8b7126..3cb57ed9c 100644 --- a/crates/ra_parser/src/parser.rs +++ b/crates/ra_parser/src/parser.rs @@ -99,6 +99,8 @@ impl<'t> Parser<'t> { /// consumed between the `start` and the corresponding `Marker::complete` /// belong to the same node. pub(crate) fn start(&mut self) -> Marker { + self.eat_dollars(); + let pos = self.events.len() as u32; self.push_event(Event::tombstone()); Marker::new(pos) @@ -180,13 +182,23 @@ impl<'t> Parser<'t> { } fn do_bump(&mut self, kind: SyntaxKind, n_raw_tokens: u8) { + self.eat_dollars(); self.token_pos += usize::from(n_raw_tokens); self.push_event(Event::Token { kind, n_raw_tokens }); + self.eat_dollars(); } fn push_event(&mut self, event: Event) { self.events.push(event) } + + fn eat_dollars(&mut self) { + while self.nth(0) == SyntaxKind::L_DOLLAR || self.nth(0) == SyntaxKind::R_DOLLAR { + let kind = self.nth(0); + self.token_pos += 1; + self.push_event(Event::Token { kind, n_raw_tokens: 1 }); + } + } } /// See `Parser::start`. -- cgit v1.2.3 From 6646d49f238bb92d55fcb4900830f19faa2994a5 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Sat, 13 Apr 2019 18:38:31 +0800 Subject: Fix bug and add expr , pat , ty matcher --- crates/ra_parser/src/grammar.rs | 12 ++++ crates/ra_parser/src/grammar/expressions.rs | 50 +++++++++++++--- crates/ra_parser/src/grammar/patterns.rs | 2 +- crates/ra_parser/src/lib.rs | 33 ++++++++--- crates/ra_parser/src/parser.rs | 92 +++++++++++++++++++++++++---- 5 files changed, 159 insertions(+), 30 deletions(-) (limited to 'crates/ra_parser') diff --git a/crates/ra_parser/src/grammar.rs b/crates/ra_parser/src/grammar.rs index c5f510e6b..5a7a55141 100644 --- a/crates/ra_parser/src/grammar.rs +++ b/crates/ra_parser/src/grammar.rs @@ -53,6 +53,18 @@ pub(crate) fn path(p: &mut Parser) { paths::type_path(p); } +pub(crate) fn expr(p: &mut Parser) { + expressions::expr(p); +} + +pub(crate) fn type_(p: &mut Parser) { + types::type_(p) +} + +pub(crate) fn pattern(p: &mut Parser) { + patterns::pattern(p) +} + pub(crate) fn reparser( node: SyntaxKind, first_child: Option, diff --git a/crates/ra_parser/src/grammar/expressions.rs b/crates/ra_parser/src/grammar/expressions.rs index 9b38b0a31..295577325 100644 --- a/crates/ra_parser/src/grammar/expressions.rs +++ b/crates/ra_parser/src/grammar/expressions.rs @@ -8,17 +8,20 @@ const EXPR_FIRST: TokenSet = LHS_FIRST; pub(super) fn expr(p: &mut Parser) -> BlockLike { let r = Restrictions { forbid_structs: false, prefer_stmt: false }; - expr_bp(p, r, 1).1 + let mut dollar_lvl = 0; + expr_bp(p, r, 1, &mut dollar_lvl).1 } pub(super) fn expr_stmt(p: &mut Parser) -> (Option, BlockLike) { let r = Restrictions { forbid_structs: false, prefer_stmt: true }; - expr_bp(p, r, 1) + let mut dollar_lvl = 0; + expr_bp(p, r, 1, &mut dollar_lvl) } fn expr_no_struct(p: &mut Parser) { let r = Restrictions { forbid_structs: true, prefer_stmt: false }; - expr_bp(p, r, 1); + let mut dollar_lvl = 0; + expr_bp(p, r, 1, &mut dollar_lvl); } // test block @@ -206,8 +209,23 @@ fn current_op(p: &Parser) -> (u8, Op) { } // Parses expression with binding power of at least bp. -fn expr_bp(p: &mut Parser, r: Restrictions, bp: u8) -> (Option, BlockLike) { - let mut lhs = match lhs(p, r) { +fn expr_bp( + p: &mut Parser, + r: Restrictions, + mut bp: u8, + dollar_lvl: &mut usize, +) -> (Option, BlockLike) { + // `newly_dollar_open` is a flag indicated that dollar is just closed after lhs, e.g. + // `$1$ + a` + // We use this flag to skip handling it. + let mut newly_dollar_open = false; + + if p.at_l_dollar() { + *dollar_lvl += p.eat_l_dollars(); + newly_dollar_open = true; + } + + let mut lhs = match lhs(p, r, dollar_lvl) { Some((lhs, blocklike)) => { // test stmt_bin_expr_ambiguity // fn foo() { @@ -223,6 +241,15 @@ fn expr_bp(p: &mut Parser, r: Restrictions, bp: u8) -> (Option, }; loop { + if *dollar_lvl > 0 && p.at_r_dollar() { + *dollar_lvl -= p.eat_r_dollars(*dollar_lvl); + if !newly_dollar_open { + // We "pump" bp for make it highest priority + bp = 255; + } + newly_dollar_open = false; + } + let is_range = p.current() == DOTDOT || p.current() == DOTDOTEQ; let (op_bp, op) = current_op(p); if op_bp < bp { @@ -235,7 +262,8 @@ fn expr_bp(p: &mut Parser, r: Restrictions, bp: u8) -> (Option, p.bump_compound(kind, n); } } - expr_bp(p, r, op_bp + 1); + + expr_bp(p, r, op_bp + 1, dollar_lvl); lhs = m.complete(p, if is_range { RANGE_EXPR } else { BIN_EXPR }); } (Some(lhs), BlockLike::NotBlock) @@ -244,7 +272,11 @@ fn expr_bp(p: &mut Parser, r: Restrictions, bp: u8) -> (Option, const LHS_FIRST: TokenSet = atom::ATOM_EXPR_FIRST.union(token_set![AMP, STAR, EXCL, DOTDOT, DOTDOTEQ, MINUS]); -fn lhs(p: &mut Parser, r: Restrictions) -> Option<(CompletedMarker, BlockLike)> { +fn lhs( + p: &mut Parser, + r: Restrictions, + dollar_lvl: &mut usize, +) -> Option<(CompletedMarker, BlockLike)> { let m; let kind = match p.current() { // test ref_expr @@ -275,7 +307,7 @@ fn lhs(p: &mut Parser, r: Restrictions) -> Option<(CompletedMarker, BlockLike)> m = p.start(); p.bump(); if p.at_ts(EXPR_FIRST) { - expr_bp(p, r, 2); + expr_bp(p, r, 2, dollar_lvl); } return Some((m.complete(p, RANGE_EXPR), BlockLike::NotBlock)); } @@ -287,7 +319,7 @@ fn lhs(p: &mut Parser, r: Restrictions) -> Option<(CompletedMarker, BlockLike)> )); } }; - expr_bp(p, r, 255); + expr_bp(p, r, 255, dollar_lvl); Some((m.complete(p, kind), BlockLike::NotBlock)) } diff --git a/crates/ra_parser/src/grammar/patterns.rs b/crates/ra_parser/src/grammar/patterns.rs index 9a307559b..03fa9b71e 100644 --- a/crates/ra_parser/src/grammar/patterns.rs +++ b/crates/ra_parser/src/grammar/patterns.rs @@ -5,7 +5,7 @@ pub(super) const PATTERN_FIRST: TokenSet = expressions::LITERAL_FIRST .union(token_set![REF_KW, MUT_KW, L_PAREN, L_BRACK, AMP, UNDERSCORE, MINUS]); pub(super) fn pattern(p: &mut Parser) { - pattern_r(p, PAT_RECOVERY_SET) + pattern_r(p, PAT_RECOVERY_SET); } /// Parses a pattern list separated by pipes `|` diff --git a/crates/ra_parser/src/lib.rs b/crates/ra_parser/src/lib.rs index 3ceeeebd7..56755c394 100644 --- a/crates/ra_parser/src/lib.rs +++ b/crates/ra_parser/src/lib.rs @@ -53,20 +53,39 @@ pub trait TreeSink { fn error(&mut self, error: ParseError); } -/// Parse given tokens into the given sink as a rust file. -pub fn parse(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { +fn parse_from_tokens(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink, f: F) +where + F: FnOnce(&mut parser::Parser), +{ let mut p = parser::Parser::new(token_source); - grammar::root(&mut p); + f(&mut p); let events = p.finish(); event::process(tree_sink, events); } +/// Parse given tokens into the given sink as a rust file. +pub fn parse(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { + parse_from_tokens(token_source, tree_sink, grammar::root); +} + /// Parse given tokens into the given sink as a path pub fn parse_path(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { - let mut p = parser::Parser::new(token_source); - grammar::path(&mut p); - let events = p.finish(); - event::process(tree_sink, events); + parse_from_tokens(token_source, tree_sink, grammar::path); +} + +/// Parse given tokens into the given sink as a expression +pub fn parse_expr(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { + parse_from_tokens(token_source, tree_sink, grammar::expr); +} + +/// Parse given tokens into the given sink as a ty +pub fn parse_ty(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { + parse_from_tokens(token_source, tree_sink, grammar::type_); +} + +/// Parse given tokens into the given sink as a pattern +pub fn parse_pat(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { + parse_from_tokens(token_source, tree_sink, grammar::pattern); } /// A parsing function for a specific braced-block. diff --git a/crates/ra_parser/src/parser.rs b/crates/ra_parser/src/parser.rs index 3cb57ed9c..71f1f8b30 100644 --- a/crates/ra_parser/src/parser.rs +++ b/crates/ra_parser/src/parser.rs @@ -45,8 +45,9 @@ impl<'t> Parser<'t> { /// /// Useful for parsing things like `>>`. pub(crate) fn current2(&self) -> Option<(SyntaxKind, SyntaxKind)> { - let c1 = self.token_source.token_kind(self.token_pos); - let c2 = self.token_source.token_kind(self.token_pos + 1); + let c1 = self.nth(0); + let c2 = self.nth(1); + if self.token_source.is_token_joint_to_next(self.token_pos) { Some((c1, c2)) } else { @@ -59,9 +60,9 @@ impl<'t> Parser<'t> { /// /// Useful for parsing things like `=>>`. pub(crate) fn current3(&self) -> Option<(SyntaxKind, SyntaxKind, SyntaxKind)> { - let c1 = self.token_source.token_kind(self.token_pos); - let c2 = self.token_source.token_kind(self.token_pos + 1); - let c3 = self.token_source.token_kind(self.token_pos + 2); + let c1 = self.nth(0); + let c2 = self.nth(1); + let c3 = self.nth(2); if self.token_source.is_token_joint_to_next(self.token_pos) && self.token_source.is_token_joint_to_next(self.token_pos + 1) { @@ -77,7 +78,23 @@ impl<'t> Parser<'t> { let steps = self.steps.get(); assert!(steps <= 10_000_000, "the parser seems stuck"); self.steps.set(steps + 1); - self.token_source.token_kind(self.token_pos + n) + + // It is beecause the Dollar will appear between nth + // Following code skips through it + let mut non_dollars_count = 0; + let mut i = 0; + + loop { + let kind = self.token_source.token_kind(self.token_pos + i); + i += 1; + + match kind { + EOF => return EOF, + SyntaxKind::L_DOLLAR | SyntaxKind::R_DOLLAR => {} + _ if non_dollars_count == n => return kind, + _ => non_dollars_count += 1, + } + } } /// Checks if the current token is `kind`. @@ -99,8 +116,6 @@ impl<'t> Parser<'t> { /// consumed between the `start` and the corresponding `Marker::complete` /// belong to the same node. pub(crate) fn start(&mut self) -> Marker { - self.eat_dollars(); - let pos = self.events.len() as u32; self.push_event(Event::tombstone()); Marker::new(pos) @@ -185,7 +200,6 @@ impl<'t> Parser<'t> { self.eat_dollars(); self.token_pos += usize::from(n_raw_tokens); self.push_event(Event::Token { kind, n_raw_tokens }); - self.eat_dollars(); } fn push_event(&mut self, event: Event) { @@ -193,12 +207,64 @@ impl<'t> Parser<'t> { } fn eat_dollars(&mut self) { - while self.nth(0) == SyntaxKind::L_DOLLAR || self.nth(0) == SyntaxKind::R_DOLLAR { - let kind = self.nth(0); - self.token_pos += 1; - self.push_event(Event::Token { kind, n_raw_tokens: 1 }); + loop { + match self.token_source.token_kind(self.token_pos) { + k @ SyntaxKind::L_DOLLAR | k @ SyntaxKind::R_DOLLAR => { + self.token_pos += 1; + self.push_event(Event::Token { kind: k, n_raw_tokens: 1 }); + } + _ => { + return; + } + } + } + } + + pub(crate) fn eat_l_dollars(&mut self) -> usize { + let mut ate_count = 0; + loop { + match self.token_source.token_kind(self.token_pos) { + k @ SyntaxKind::L_DOLLAR => { + self.token_pos += 1; + self.push_event(Event::Token { kind: k, n_raw_tokens: 1 }); + ate_count += 1; + } + _ => { + return ate_count; + } + } } } + + pub(crate) fn eat_r_dollars(&mut self, max_count: usize) -> usize { + let mut ate_count = 0; + loop { + match self.token_source.token_kind(self.token_pos) { + k @ SyntaxKind::R_DOLLAR => { + self.token_pos += 1; + self.push_event(Event::Token { kind: k, n_raw_tokens: 1 }); + ate_count += 1; + + if max_count >= ate_count { + return ate_count; + } + } + _ => { + return ate_count; + } + } + } + } + + pub(crate) fn at_l_dollar(&self) -> bool { + let kind = self.token_source.token_kind(self.token_pos); + (kind == SyntaxKind::L_DOLLAR) + } + + pub(crate) fn at_r_dollar(&self) -> bool { + let kind = self.token_source.token_kind(self.token_pos); + (kind == SyntaxKind::R_DOLLAR) + } } /// See `Parser::start`. -- cgit v1.2.3