From 50a02eb3593591a02677e1b56f24d7ff0459b9d0 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Wed, 12 Aug 2020 17:06:49 +0200 Subject: Rename ra_parser -> parser --- crates/parser/Cargo.toml | 12 + crates/parser/src/event.rs | 130 +++++ crates/parser/src/grammar.rs | 293 ++++++++++ crates/parser/src/grammar/attributes.rs | 48 ++ crates/parser/src/grammar/expressions.rs | 651 ++++++++++++++++++++++ crates/parser/src/grammar/expressions/atom.rs | 611 ++++++++++++++++++++ crates/parser/src/grammar/items.rs | 432 ++++++++++++++ crates/parser/src/grammar/items/adt.rs | 178 ++++++ crates/parser/src/grammar/items/consts.rs | 33 ++ crates/parser/src/grammar/items/traits.rs | 153 +++++ crates/parser/src/grammar/items/use_item.rs | 132 +++++ crates/parser/src/grammar/params.rs | 188 +++++++ crates/parser/src/grammar/paths.rs | 115 ++++ crates/parser/src/grammar/patterns.rs | 379 +++++++++++++ crates/parser/src/grammar/type_args.rs | 63 +++ crates/parser/src/grammar/type_params.rs | 209 +++++++ crates/parser/src/grammar/types.rs | 324 +++++++++++ crates/parser/src/lib.rs | 149 +++++ crates/parser/src/parser.rs | 350 ++++++++++++ crates/parser/src/syntax_kind.rs | 25 + crates/parser/src/syntax_kind/generated.rs | 367 ++++++++++++ crates/parser/src/token_set.rs | 42 ++ crates/ra_hir_expand/Cargo.toml | 2 +- crates/ra_hir_expand/src/builtin_derive.rs | 2 +- crates/ra_hir_expand/src/builtin_macro.rs | 2 +- crates/ra_hir_expand/src/db.rs | 2 +- crates/ra_hir_expand/src/eager.rs | 2 +- crates/ra_hir_expand/src/lib.rs | 2 +- crates/ra_mbe/Cargo.toml | 2 +- crates/ra_mbe/src/mbe_expander/matcher.rs | 8 +- crates/ra_mbe/src/subtree_source.rs | 2 +- crates/ra_mbe/src/syntax_bridge.rs | 6 +- crates/ra_mbe/src/tests.rs | 5 +- crates/ra_parser/Cargo.toml | 13 - crates/ra_parser/src/event.rs | 130 ----- crates/ra_parser/src/grammar.rs | 293 ---------- crates/ra_parser/src/grammar/attributes.rs | 48 -- crates/ra_parser/src/grammar/expressions.rs | 651 ---------------------- crates/ra_parser/src/grammar/expressions/atom.rs | 611 -------------------- crates/ra_parser/src/grammar/items.rs | 432 -------------- crates/ra_parser/src/grammar/items/adt.rs | 178 ------ crates/ra_parser/src/grammar/items/consts.rs | 33 -- crates/ra_parser/src/grammar/items/traits.rs | 153 ----- crates/ra_parser/src/grammar/items/use_item.rs | 132 ----- crates/ra_parser/src/grammar/params.rs | 188 ------- crates/ra_parser/src/grammar/paths.rs | 115 ---- crates/ra_parser/src/grammar/patterns.rs | 379 ------------- crates/ra_parser/src/grammar/type_args.rs | 63 --- crates/ra_parser/src/grammar/type_params.rs | 209 ------- crates/ra_parser/src/grammar/types.rs | 324 ----------- crates/ra_parser/src/lib.rs | 149 ----- crates/ra_parser/src/parser.rs | 350 ------------ crates/ra_parser/src/syntax_kind.rs | 25 - crates/ra_parser/src/syntax_kind/generated.rs | 367 ------------ crates/ra_parser/src/token_set.rs | 42 -- crates/ra_syntax/Cargo.toml | 2 +- crates/ra_syntax/src/ast/node_ext.rs | 2 +- crates/ra_syntax/src/lib.rs | 14 +- crates/ra_syntax/src/parsing.rs | 14 +- crates/ra_syntax/src/parsing/reparsing.rs | 2 +- crates/ra_syntax/src/parsing/text_token_source.rs | 14 +- crates/ra_syntax/src/parsing/text_tree_sink.rs | 2 +- crates/ra_syntax/src/syntax_node.rs | 2 +- 63 files changed, 4928 insertions(+), 4928 deletions(-) create mode 100644 crates/parser/Cargo.toml create mode 100644 crates/parser/src/event.rs create mode 100644 crates/parser/src/grammar.rs create mode 100644 crates/parser/src/grammar/attributes.rs create mode 100644 crates/parser/src/grammar/expressions.rs create mode 100644 crates/parser/src/grammar/expressions/atom.rs create mode 100644 crates/parser/src/grammar/items.rs create mode 100644 crates/parser/src/grammar/items/adt.rs create mode 100644 crates/parser/src/grammar/items/consts.rs create mode 100644 crates/parser/src/grammar/items/traits.rs create mode 100644 crates/parser/src/grammar/items/use_item.rs create mode 100644 crates/parser/src/grammar/params.rs create mode 100644 crates/parser/src/grammar/paths.rs create mode 100644 crates/parser/src/grammar/patterns.rs create mode 100644 crates/parser/src/grammar/type_args.rs create mode 100644 crates/parser/src/grammar/type_params.rs create mode 100644 crates/parser/src/grammar/types.rs create mode 100644 crates/parser/src/lib.rs create mode 100644 crates/parser/src/parser.rs create mode 100644 crates/parser/src/syntax_kind.rs create mode 100644 crates/parser/src/syntax_kind/generated.rs create mode 100644 crates/parser/src/token_set.rs delete mode 100644 crates/ra_parser/Cargo.toml delete mode 100644 crates/ra_parser/src/event.rs delete mode 100644 crates/ra_parser/src/grammar.rs delete mode 100644 crates/ra_parser/src/grammar/attributes.rs delete mode 100644 crates/ra_parser/src/grammar/expressions.rs delete mode 100644 crates/ra_parser/src/grammar/expressions/atom.rs delete mode 100644 crates/ra_parser/src/grammar/items.rs delete mode 100644 crates/ra_parser/src/grammar/items/adt.rs delete mode 100644 crates/ra_parser/src/grammar/items/consts.rs delete mode 100644 crates/ra_parser/src/grammar/items/traits.rs delete mode 100644 crates/ra_parser/src/grammar/items/use_item.rs delete mode 100644 crates/ra_parser/src/grammar/params.rs delete mode 100644 crates/ra_parser/src/grammar/paths.rs delete mode 100644 crates/ra_parser/src/grammar/patterns.rs delete mode 100644 crates/ra_parser/src/grammar/type_args.rs delete mode 100644 crates/ra_parser/src/grammar/type_params.rs delete mode 100644 crates/ra_parser/src/grammar/types.rs delete mode 100644 crates/ra_parser/src/lib.rs delete mode 100644 crates/ra_parser/src/parser.rs delete mode 100644 crates/ra_parser/src/syntax_kind.rs delete mode 100644 crates/ra_parser/src/syntax_kind/generated.rs delete mode 100644 crates/ra_parser/src/token_set.rs (limited to 'crates') diff --git a/crates/parser/Cargo.toml b/crates/parser/Cargo.toml new file mode 100644 index 000000000..358be92d1 --- /dev/null +++ b/crates/parser/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "parser" +version = "0.0.0" +license = "MIT OR Apache-2.0" +authors = ["rust-analyzer developers"] +edition = "2018" + +[lib] +doctest = false + +[dependencies] +drop_bomb = "0.1.4" diff --git a/crates/parser/src/event.rs b/crates/parser/src/event.rs new file mode 100644 index 000000000..a7d06a815 --- /dev/null +++ b/crates/parser/src/event.rs @@ -0,0 +1,130 @@ +//! This module provides a way to construct a `File`. +//! It is intended to be completely decoupled from the +//! parser, so as to allow to evolve the tree representation +//! and the parser algorithm independently. +//! +//! The `TreeSink` trait is the bridge between the parser and the +//! tree builder: the parser produces a stream of events like +//! `start node`, `finish node`, and `FileBuilder` converts +//! this stream to a real tree. +use std::mem; + +use crate::{ + ParseError, + SyntaxKind::{self, *}, + TreeSink, +}; + +/// `Parser` produces a flat list of `Event`s. +/// They are converted to a tree-structure in +/// a separate pass, via `TreeBuilder`. +#[derive(Debug)] +pub(crate) enum Event { + /// This event signifies the start of the node. + /// It should be either abandoned (in which case the + /// `kind` is `TOMBSTONE`, and the event is ignored), + /// or completed via a `Finish` event. + /// + /// All tokens between a `Start` and a `Finish` would + /// become the children of the respective node. + /// + /// For left-recursive syntactic constructs, the parser produces + /// a child node before it sees a parent. `forward_parent` + /// saves the position of current event's parent. + /// + /// Consider this path + /// + /// foo::bar + /// + /// The events for it would look like this: + /// + /// + /// START(PATH) IDENT('foo') FINISH START(PATH) T![::] IDENT('bar') FINISH + /// | /\ + /// | | + /// +------forward-parent------+ + /// + /// And the tree would look like this + /// + /// +--PATH---------+ + /// | | | + /// | | | + /// | '::' 'bar' + /// | + /// PATH + /// | + /// 'foo' + /// + /// See also `CompletedMarker::precede`. + Start { + kind: SyntaxKind, + forward_parent: Option, + }, + + /// Complete the previous `Start` event + Finish, + + /// Produce a single leaf-element. + /// `n_raw_tokens` is used to glue complex contextual tokens. + /// For example, lexer tokenizes `>>` as `>`, `>`, and + /// `n_raw_tokens = 2` is used to produced a single `>>`. + Token { + kind: SyntaxKind, + n_raw_tokens: u8, + }, + + Error { + msg: ParseError, + }, +} + +impl Event { + pub(crate) fn tombstone() -> Self { + Event::Start { kind: TOMBSTONE, forward_parent: None } + } +} + +/// Generate the syntax tree with the control of events. +pub(super) fn process(sink: &mut dyn TreeSink, mut events: Vec) { + let mut forward_parents = Vec::new(); + + for i in 0..events.len() { + match mem::replace(&mut events[i], Event::tombstone()) { + Event::Start { kind: TOMBSTONE, .. } => (), + + Event::Start { kind, forward_parent } => { + // For events[A, B, C], B is A's forward_parent, C is B's forward_parent, + // in the normal control flow, the parent-child relation: `A -> B -> C`, + // while with the magic forward_parent, it writes: `C <- B <- A`. + + // append `A` into parents. + forward_parents.push(kind); + let mut idx = i; + let mut fp = forward_parent; + while let Some(fwd) = fp { + idx += fwd as usize; + // append `A`'s forward_parent `B` + fp = match mem::replace(&mut events[idx], Event::tombstone()) { + Event::Start { kind, forward_parent } => { + if kind != TOMBSTONE { + forward_parents.push(kind); + } + forward_parent + } + _ => unreachable!(), + }; + // append `B`'s forward_parent `C` in the next stage. + } + + for kind in forward_parents.drain(..).rev() { + sink.start_node(kind); + } + } + Event::Finish => sink.finish_node(), + Event::Token { kind, n_raw_tokens } => { + sink.token(kind, n_raw_tokens); + } + Event::Error { msg } => sink.error(msg), + } + } +} diff --git a/crates/parser/src/grammar.rs b/crates/parser/src/grammar.rs new file mode 100644 index 000000000..88468bc97 --- /dev/null +++ b/crates/parser/src/grammar.rs @@ -0,0 +1,293 @@ +//! This is the actual "grammar" of the Rust language. +//! +//! Each function in this module and its children corresponds +//! to a production of the formal grammar. Submodules roughly +//! correspond to different *areas* of the grammar. By convention, +//! each submodule starts with `use super::*` import and exports +//! "public" productions via `pub(super)`. +//! +//! See docs for `Parser` to learn about API, available to the grammar, +//! and see docs for `Event` to learn how this actually manages to +//! produce parse trees. +//! +//! Code in this module also contains inline tests, which start with +//! `// test name-of-the-test` comment and look like this: +//! +//! ``` +//! // test function_with_zero_parameters +//! // fn foo() {} +//! ``` +//! +//! After adding a new inline-test, run `cargo xtask codegen` to +//! extract it as a standalone text-fixture into +//! `crates/ra_syntax/test_data/parser/`, and run `cargo test` once to +//! create the "gold" value. +//! +//! Coding convention: rules like `where_clause` always produce either a +//! node or an error, rules like `opt_where_clause` may produce nothing. +//! Non-opt rules typically start with `assert!(p.at(FIRST_TOKEN))`, the +//! caller is responsible for branching on the first token. +mod attributes; +mod expressions; +mod items; +mod params; +mod paths; +mod patterns; +mod type_args; +mod type_params; +mod types; + +use crate::{ + parser::{CompletedMarker, Marker, Parser}, + SyntaxKind::{self, *}, + TokenSet, +}; + +pub(crate) fn root(p: &mut Parser) { + let m = p.start(); + p.eat(SHEBANG); + items::mod_contents(p, false); + m.complete(p, SOURCE_FILE); +} + +/// Various pieces of syntax that can be parsed by macros by example +pub(crate) mod fragments { + use super::*; + + pub(crate) use super::{ + expressions::block_expr, paths::type_path as path, patterns::pattern, types::type_, + }; + + pub(crate) fn expr(p: &mut Parser) { + let _ = expressions::expr(p); + } + + pub(crate) fn stmt(p: &mut Parser) { + expressions::stmt(p, expressions::StmtWithSemi::No) + } + + pub(crate) fn opt_visibility(p: &mut Parser) { + let _ = super::opt_visibility(p); + } + + // Parse a meta item , which excluded [], e.g : #[ MetaItem ] + pub(crate) fn meta_item(p: &mut Parser) { + fn is_delimiter(p: &mut Parser) -> bool { + matches!(p.current(), T!['{'] | T!['('] | T!['[']) + } + + if is_delimiter(p) { + items::token_tree(p); + return; + } + + let m = p.start(); + while !p.at(EOF) { + if is_delimiter(p) { + items::token_tree(p); + break; + } else { + // https://doc.rust-lang.org/reference/attributes.html + // https://doc.rust-lang.org/reference/paths.html#simple-paths + // The start of an meta must be a simple path + match p.current() { + IDENT | T![::] | T![super] | T![self] | T![crate] => p.bump_any(), + T![=] => { + p.bump_any(); + match p.current() { + c if c.is_literal() => p.bump_any(), + T![true] | T![false] => p.bump_any(), + _ => {} + } + break; + } + _ => break, + } + } + } + + m.complete(p, TOKEN_TREE); + } + + pub(crate) fn item(p: &mut Parser) { + items::item_or_macro(p, true) + } + + pub(crate) fn macro_items(p: &mut Parser) { + let m = p.start(); + items::mod_contents(p, false); + m.complete(p, MACRO_ITEMS); + } + + pub(crate) fn macro_stmts(p: &mut Parser) { + let m = p.start(); + + while !p.at(EOF) { + if p.at(T![;]) { + p.bump(T![;]); + continue; + } + + expressions::stmt(p, expressions::StmtWithSemi::Optional); + } + + m.complete(p, MACRO_STMTS); + } +} + +pub(crate) fn reparser( + node: SyntaxKind, + first_child: Option, + parent: Option, +) -> Option { + let res = match node { + BLOCK_EXPR => expressions::block_expr, + RECORD_FIELD_LIST => items::record_field_def_list, + RECORD_EXPR_FIELD_LIST => items::record_field_list, + VARIANT_LIST => items::enum_variant_list, + MATCH_ARM_LIST => items::match_arm_list, + USE_TREE_LIST => items::use_tree_list, + EXTERN_ITEM_LIST => items::extern_item_list, + TOKEN_TREE if first_child? == T!['{'] => items::token_tree, + ASSOC_ITEM_LIST => match parent? { + IMPL => items::impl_item_list, + TRAIT => items::trait_item_list, + _ => return None, + }, + ITEM_LIST => items::mod_item_list, + _ => return None, + }; + Some(res) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum BlockLike { + Block, + NotBlock, +} + +impl BlockLike { + fn is_block(self) -> bool { + self == BlockLike::Block + } +} + +fn opt_visibility(p: &mut Parser) -> bool { + match p.current() { + T![pub] => { + let m = p.start(); + p.bump(T![pub]); + if p.at(T!['(']) { + match p.nth(1) { + // test crate_visibility + // pub(crate) struct S; + // pub(self) struct S; + // pub(self) struct S; + // pub(self) struct S; + T![crate] | T![self] | T![super] => { + p.bump_any(); + p.bump_any(); + p.expect(T![')']); + } + T![in] => { + p.bump_any(); + p.bump_any(); + paths::use_path(p); + p.expect(T![')']); + } + _ => (), + } + } + m.complete(p, VISIBILITY); + } + // test crate_keyword_vis + // crate fn main() { } + // struct S { crate field: u32 } + // struct T(crate u32); + // + // test crate_keyword_path + // fn foo() { crate::foo(); } + T![crate] if !p.nth_at(1, T![::]) => { + let m = p.start(); + p.bump(T![crate]); + m.complete(p, VISIBILITY); + } + _ => return false, + } + true +} + +fn opt_alias(p: &mut Parser) { + if p.at(T![as]) { + let m = p.start(); + p.bump(T![as]); + if !p.eat(T![_]) { + name(p); + } + m.complete(p, RENAME); + } +} + +fn abi(p: &mut Parser) { + assert!(p.at(T![extern])); + let abi = p.start(); + p.bump(T![extern]); + match p.current() { + STRING | RAW_STRING => p.bump_any(), + _ => (), + } + abi.complete(p, ABI); +} + +fn opt_fn_ret_type(p: &mut Parser) -> bool { + if p.at(T![->]) { + let m = p.start(); + p.bump(T![->]); + types::type_no_bounds(p); + m.complete(p, RET_TYPE); + true + } else { + false + } +} + +fn name_r(p: &mut Parser, recovery: TokenSet) { + if p.at(IDENT) { + let m = p.start(); + p.bump(IDENT); + m.complete(p, NAME); + } else { + p.err_recover("expected a name", recovery); + } +} + +fn name(p: &mut Parser) { + name_r(p, TokenSet::EMPTY) +} + +fn name_ref(p: &mut Parser) { + if p.at(IDENT) { + let m = p.start(); + p.bump(IDENT); + m.complete(p, NAME_REF); + } else { + p.err_and_bump("expected identifier"); + } +} + +fn name_ref_or_index(p: &mut Parser) { + assert!(p.at(IDENT) || p.at(INT_NUMBER)); + let m = p.start(); + p.bump_any(); + m.complete(p, NAME_REF); +} + +fn error_block(p: &mut Parser, message: &str) { + assert!(p.at(T!['{'])); + let m = p.start(); + p.error(message); + p.bump(T!['{']); + expressions::expr_block_contents(p); + p.eat(T!['}']); + m.complete(p, ERROR); +} diff --git a/crates/parser/src/grammar/attributes.rs b/crates/parser/src/grammar/attributes.rs new file mode 100644 index 000000000..f3158ade3 --- /dev/null +++ b/crates/parser/src/grammar/attributes.rs @@ -0,0 +1,48 @@ +//! FIXME: write short doc here + +use super::*; + +pub(super) fn inner_attributes(p: &mut Parser) { + while p.at(T![#]) && p.nth(1) == T![!] { + attribute(p, true) + } +} + +pub(super) fn outer_attributes(p: &mut Parser) { + while p.at(T![#]) { + attribute(p, false) + } +} + +fn attribute(p: &mut Parser, inner: bool) { + let attr = p.start(); + assert!(p.at(T![#])); + p.bump(T![#]); + + if inner { + assert!(p.at(T![!])); + p.bump(T![!]); + } + + if p.eat(T!['[']) { + paths::use_path(p); + + match p.current() { + T![=] => { + p.bump(T![=]); + if expressions::literal(p).is_none() { + p.error("expected literal"); + } + } + T!['('] | T!['['] | T!['{'] => items::token_tree(p), + _ => {} + } + + if !p.eat(T![']']) { + p.error("expected `]`"); + } + } else { + p.error("expected `[`"); + } + attr.complete(p, ATTR); +} diff --git a/crates/parser/src/grammar/expressions.rs b/crates/parser/src/grammar/expressions.rs new file mode 100644 index 000000000..3291e3f14 --- /dev/null +++ b/crates/parser/src/grammar/expressions.rs @@ -0,0 +1,651 @@ +//! FIXME: write short doc here + +mod atom; + +pub(crate) use self::atom::{block_expr, match_arm_list}; +pub(super) use self::atom::{literal, LITERAL_FIRST}; +use super::*; + +pub(super) enum StmtWithSemi { + Yes, + No, + Optional, +} + +const EXPR_FIRST: TokenSet = LHS_FIRST; + +pub(super) fn expr(p: &mut Parser) -> (Option, BlockLike) { + let r = Restrictions { forbid_structs: false, prefer_stmt: false }; + expr_bp(p, r, 1) +} + +pub(super) fn expr_with_attrs(p: &mut Parser) -> bool { + let m = p.start(); + let has_attrs = p.at(T![#]); + attributes::outer_attributes(p); + + let (cm, _block_like) = expr(p); + let success = cm.is_some(); + + match (has_attrs, cm) { + (true, Some(cm)) => { + let kind = cm.kind(); + cm.undo_completion(p).abandon(p); + m.complete(p, kind); + } + _ => m.abandon(p), + } + + success +} + +pub(super) fn expr_stmt(p: &mut Parser) -> (Option, BlockLike) { + let r = Restrictions { forbid_structs: false, prefer_stmt: true }; + expr_bp(p, r, 1) +} + +fn expr_no_struct(p: &mut Parser) { + let r = Restrictions { forbid_structs: true, prefer_stmt: false }; + expr_bp(p, r, 1); +} + +fn is_expr_stmt_attr_allowed(kind: SyntaxKind) -> bool { + let forbid = matches!(kind, BIN_EXPR | RANGE_EXPR); + !forbid +} + +pub(super) fn stmt(p: &mut Parser, with_semi: StmtWithSemi) { + let m = p.start(); + // test attr_on_expr_stmt + // fn foo() { + // #[A] foo(); + // #[B] bar!{} + // #[C] #[D] {} + // #[D] return (); + // } + let has_attrs = p.at(T![#]); + attributes::outer_attributes(p); + + if p.at(T![let]) { + let_stmt(p, m, with_semi); + return; + } + + // test block_items + // fn a() { fn b() {} } + let m = match items::maybe_item(p, m) { + Ok(()) => return, + Err(m) => m, + }; + + let (cm, blocklike) = expr_stmt(p); + let kind = cm.as_ref().map(|cm| cm.kind()).unwrap_or(ERROR); + + if has_attrs && !is_expr_stmt_attr_allowed(kind) { + // test_err attr_on_expr_not_allowed + // fn foo() { + // #[A] 1 + 2; + // #[B] if true {}; + // } + p.error(format!("attributes are not allowed on {:?}", kind)); + } + + if p.at(T!['}']) { + // test attr_on_last_expr_in_block + // fn foo() { + // { #[A] bar!()? } + // #[B] &() + // } + if let Some(cm) = cm { + cm.undo_completion(p).abandon(p); + m.complete(p, kind); + } else { + m.abandon(p); + } + } else { + // test no_semi_after_block + // fn foo() { + // if true {} + // loop {} + // match () {} + // while true {} + // for _ in () {} + // {} + // {} + // macro_rules! test { + // () => {} + // } + // test!{} + // } + + match with_semi { + StmtWithSemi::Yes => { + if blocklike.is_block() { + p.eat(T![;]); + } else { + p.expect(T![;]); + } + } + StmtWithSemi::No => {} + StmtWithSemi::Optional => { + if p.at(T![;]) { + p.eat(T![;]); + } + } + } + + m.complete(p, EXPR_STMT); + } + + // test let_stmt + // fn foo() { + // let a; + // let b: i32; + // let c = 92; + // let d: i32 = 92; + // let e: !; + // let _: ! = {}; + // let f = #[attr]||{}; + // } + fn let_stmt(p: &mut Parser, m: Marker, with_semi: StmtWithSemi) { + assert!(p.at(T![let])); + p.bump(T![let]); + patterns::pattern(p); + if p.at(T![:]) { + types::ascription(p); + } + if p.eat(T![=]) { + expressions::expr_with_attrs(p); + } + + match with_semi { + StmtWithSemi::Yes => { + p.expect(T![;]); + } + StmtWithSemi::No => {} + StmtWithSemi::Optional => { + if p.at(T![;]) { + p.eat(T![;]); + } + } + } + m.complete(p, LET_STMT); + } +} + +pub(super) fn expr_block_contents(p: &mut Parser) { + // This is checked by a validator + attributes::inner_attributes(p); + + while !p.at(EOF) && !p.at(T!['}']) { + // test nocontentexpr + // fn foo(){ + // ;;;some_expr();;;;{;;;};;;;Ok(()) + // } + + // test nocontentexpr_after_item + // fn simple_function() { + // enum LocalEnum { + // One, + // Two, + // }; + // fn f() {}; + // struct S {}; + // } + + if p.at(T![;]) { + p.bump(T![;]); + continue; + } + + stmt(p, StmtWithSemi::Yes) + } +} + +#[derive(Clone, Copy)] +struct Restrictions { + forbid_structs: bool, + prefer_stmt: bool, +} + +/// Binding powers of operators for a Pratt parser. +/// +/// See https://www.oilshell.org/blog/2016/11/03.html +#[rustfmt::skip] +fn current_op(p: &Parser) -> (u8, SyntaxKind) { + const NOT_AN_OP: (u8, SyntaxKind) = (0, T![@]); + match p.current() { + T![|] if p.at(T![||]) => (3, T![||]), + T![|] if p.at(T![|=]) => (1, T![|=]), + T![|] => (6, T![|]), + T![>] if p.at(T![>>=]) => (1, T![>>=]), + T![>] if p.at(T![>>]) => (9, T![>>]), + T![>] if p.at(T![>=]) => (5, T![>=]), + T![>] => (5, T![>]), + T![=] if p.at(T![=>]) => NOT_AN_OP, + T![=] if p.at(T![==]) => (5, T![==]), + T![=] => (1, T![=]), + T![<] if p.at(T![<=]) => (5, T![<=]), + T![<] if p.at(T![<<=]) => (1, T![<<=]), + T![<] if p.at(T![<<]) => (9, T![<<]), + T![<] => (5, T![<]), + T![+] if p.at(T![+=]) => (1, T![+=]), + T![+] => (10, T![+]), + T![^] if p.at(T![^=]) => (1, T![^=]), + T![^] => (7, T![^]), + T![%] if p.at(T![%=]) => (1, T![%=]), + T![%] => (11, T![%]), + T![&] if p.at(T![&=]) => (1, T![&=]), + T![&] if p.at(T![&&]) => (4, T![&&]), + T![&] => (8, T![&]), + T![/] if p.at(T![/=]) => (1, T![/=]), + T![/] => (11, T![/]), + T![*] if p.at(T![*=]) => (1, T![*=]), + T![*] => (11, T![*]), + T![.] if p.at(T![..=]) => (2, T![..=]), + T![.] if p.at(T![..]) => (2, T![..]), + T![!] if p.at(T![!=]) => (5, T![!=]), + T![-] if p.at(T![-=]) => (1, T![-=]), + T![-] => (10, T![-]), + T![as] => (12, T![as]), + + _ => NOT_AN_OP + } +} + +// Parses expression with binding power of at least bp. +fn expr_bp(p: &mut Parser, mut r: Restrictions, bp: u8) -> (Option, BlockLike) { + let mut lhs = match lhs(p, r) { + Some((lhs, blocklike)) => { + // test stmt_bin_expr_ambiguity + // fn foo() { + // let _ = {1} & 2; + // {1} &2; + // } + if r.prefer_stmt && blocklike.is_block() { + return (Some(lhs), BlockLike::Block); + } + lhs + } + None => return (None, BlockLike::NotBlock), + }; + + loop { + let is_range = p.at(T![..]) || p.at(T![..=]); + let (op_bp, op) = current_op(p); + if op_bp < bp { + break; + } + // test as_precedence + // fn foo() { + // let _ = &1 as *const i32; + // } + if p.at(T![as]) { + lhs = cast_expr(p, lhs); + continue; + } + let m = lhs.precede(p); + p.bump(op); + + // test binop_resets_statementness + // fn foo() { + // v = {1}&2; + // } + r = Restrictions { prefer_stmt: false, ..r }; + + if is_range { + // test postfix_range + // fn foo() { + // let x = 1..; + // match 1.. { _ => () }; + // match a.b()..S { _ => () }; + // } + let has_trailing_expression = + p.at_ts(EXPR_FIRST) && !(r.forbid_structs && p.at(T!['{'])); + if !has_trailing_expression { + // no RHS + lhs = m.complete(p, RANGE_EXPR); + break; + } + } + + expr_bp(p, Restrictions { prefer_stmt: false, ..r }, op_bp + 1); + lhs = m.complete(p, if is_range { RANGE_EXPR } else { BIN_EXPR }); + } + (Some(lhs), BlockLike::NotBlock) +} + +const LHS_FIRST: TokenSet = + atom::ATOM_EXPR_FIRST.union(token_set![T![&], T![*], T![!], T![.], T![-]]); + +fn lhs(p: &mut Parser, r: Restrictions) -> Option<(CompletedMarker, BlockLike)> { + let m; + let kind = match p.current() { + // test ref_expr + // fn foo() { + // // reference operator + // let _ = &1; + // let _ = &mut &f(); + // let _ = &raw; + // let _ = &raw.0; + // // raw reference operator + // let _ = &raw mut foo; + // let _ = &raw const foo; + // } + T![&] => { + m = p.start(); + p.bump(T![&]); + if p.at(IDENT) + && p.at_contextual_kw("raw") + && (p.nth_at(1, T![mut]) || p.nth_at(1, T![const])) + { + p.bump_remap(T![raw]); + p.bump_any(); + } else { + p.eat(T![mut]); + } + REF_EXPR + } + // test unary_expr + // fn foo() { + // **&1; + // !!true; + // --1; + // } + T![*] | T![!] | T![-] => { + m = p.start(); + p.bump_any(); + PREFIX_EXPR + } + _ => { + // test full_range_expr + // fn foo() { xs[..]; } + for &op in [T![..=], T![..]].iter() { + if p.at(op) { + m = p.start(); + p.bump(op); + if p.at_ts(EXPR_FIRST) && !(r.forbid_structs && p.at(T!['{'])) { + expr_bp(p, r, 2); + } + return Some((m.complete(p, RANGE_EXPR), BlockLike::NotBlock)); + } + } + + // test expression_after_block + // fn foo() { + // let mut p = F{x: 5}; + // {p}.x = 10; + // } + // + let (lhs, blocklike) = atom::atom_expr(p, r)?; + return Some(postfix_expr(p, lhs, blocklike, !(r.prefer_stmt && blocklike.is_block()))); + } + }; + // parse the interior of the unary expression + expr_bp(p, r, 255); + Some((m.complete(p, kind), BlockLike::NotBlock)) +} + +fn postfix_expr( + p: &mut Parser, + mut lhs: CompletedMarker, + // Calls are disallowed if the type is a block and we prefer statements because the call cannot be disambiguated from a tuple + // E.g. `while true {break}();` is parsed as + // `while true {break}; ();` + mut block_like: BlockLike, + mut allow_calls: bool, +) -> (CompletedMarker, BlockLike) { + loop { + lhs = match p.current() { + // test stmt_postfix_expr_ambiguity + // fn foo() { + // match () { + // _ => {} + // () => {} + // [] => {} + // } + // } + T!['('] if allow_calls => call_expr(p, lhs), + T!['['] if allow_calls => index_expr(p, lhs), + T![.] => match postfix_dot_expr(p, lhs) { + Ok(it) => it, + Err(it) => { + lhs = it; + break; + } + }, + T![?] => try_expr(p, lhs), + _ => break, + }; + allow_calls = true; + block_like = BlockLike::NotBlock; + } + return (lhs, block_like); + + fn postfix_dot_expr( + p: &mut Parser, + lhs: CompletedMarker, + ) -> Result { + assert!(p.at(T![.])); + if p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::])) { + return Ok(method_call_expr(p, lhs)); + } + + // test await_expr + // fn foo() { + // x.await; + // x.0.await; + // x.0().await?.hello(); + // } + if p.nth(1) == T![await] { + let m = lhs.precede(p); + p.bump(T![.]); + p.bump(T![await]); + return Ok(m.complete(p, AWAIT_EXPR)); + } + + if p.at(T![..=]) || p.at(T![..]) { + return Err(lhs); + } + + Ok(field_expr(p, lhs)) + } +} + +// test call_expr +// fn foo() { +// let _ = f(); +// let _ = f()(1)(1, 2,); +// let _ = f(::func()); +// f(::func()); +// } +fn call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { + assert!(p.at(T!['('])); + let m = lhs.precede(p); + arg_list(p); + m.complete(p, CALL_EXPR) +} + +// test index_expr +// fn foo() { +// x[1][2]; +// } +fn index_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { + assert!(p.at(T!['['])); + let m = lhs.precede(p); + p.bump(T!['[']); + expr(p); + p.expect(T![']']); + m.complete(p, INDEX_EXPR) +} + +// test method_call_expr +// fn foo() { +// x.foo(); +// y.bar::(1, 2,); +// } +fn method_call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { + assert!(p.at(T![.]) && p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::]))); + let m = lhs.precede(p); + p.bump_any(); + name_ref(p); + type_args::opt_type_arg_list(p, true); + if p.at(T!['(']) { + arg_list(p); + } + m.complete(p, METHOD_CALL_EXPR) +} + +// test field_expr +// fn foo() { +// x.foo; +// x.0.bar; +// x.0(); +// } + +// test_err bad_tuple_index_expr +// fn foo() { +// x.0.; +// x.1i32; +// x.0x01; +// } +fn field_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { + assert!(p.at(T![.])); + let m = lhs.precede(p); + p.bump(T![.]); + if p.at(IDENT) || p.at(INT_NUMBER) { + name_ref_or_index(p) + } else if p.at(FLOAT_NUMBER) { + // FIXME: How to recover and instead parse INT + T![.]? + p.bump_any(); + } else { + p.error("expected field name or number") + } + m.complete(p, FIELD_EXPR) +} + +// test try_expr +// fn foo() { +// x?; +// } +fn try_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { + assert!(p.at(T![?])); + let m = lhs.precede(p); + p.bump(T![?]); + m.complete(p, TRY_EXPR) +} + +// test cast_expr +// fn foo() { +// 82 as i32; +// 81 as i8 + 1; +// 79 as i16 - 1; +// 0x36 as u8 <= 0x37; +// } +fn cast_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { + assert!(p.at(T![as])); + let m = lhs.precede(p); + p.bump(T![as]); + // Use type_no_bounds(), because cast expressions are not + // allowed to have bounds. + types::type_no_bounds(p); + m.complete(p, CAST_EXPR) +} + +fn arg_list(p: &mut Parser) { + assert!(p.at(T!['('])); + let m = p.start(); + p.bump(T!['(']); + while !p.at(T![')']) && !p.at(EOF) { + // test arg_with_attr + // fn main() { + // foo(#[attr] 92) + // } + if !expr_with_attrs(p) { + break; + } + if !p.at(T![')']) && !p.expect(T![,]) { + break; + } + } + p.eat(T![')']); + m.complete(p, ARG_LIST); +} + +// test path_expr +// fn foo() { +// let _ = a; +// let _ = a::b; +// let _ = ::a::; +// let _ = format!(); +// } +fn path_expr(p: &mut Parser, r: Restrictions) -> (CompletedMarker, BlockLike) { + assert!(paths::is_path_start(p)); + let m = p.start(); + paths::expr_path(p); + match p.current() { + T!['{'] if !r.forbid_structs => { + record_field_list(p); + (m.complete(p, RECORD_EXPR), BlockLike::NotBlock) + } + T![!] if !p.at(T![!=]) => { + let block_like = items::macro_call_after_excl(p); + (m.complete(p, MACRO_CALL), block_like) + } + _ => (m.complete(p, PATH_EXPR), BlockLike::NotBlock), + } +} + +// test record_lit +// fn foo() { +// S {}; +// S { x, y: 32, }; +// S { x, y: 32, ..Default::default() }; +// TupleStruct { 0: 1 }; +// } +pub(crate) fn record_field_list(p: &mut Parser) { + assert!(p.at(T!['{'])); + let m = p.start(); + p.bump(T!['{']); + while !p.at(EOF) && !p.at(T!['}']) { + let m = p.start(); + // test record_literal_field_with_attr + // fn main() { + // S { #[cfg(test)] field: 1 } + // } + attributes::outer_attributes(p); + + match p.current() { + IDENT | INT_NUMBER => { + // test_err record_literal_before_ellipsis_recovery + // fn main() { + // S { field ..S::default() } + // } + if p.nth_at(1, T![:]) || p.nth_at(1, T![..]) { + name_ref_or_index(p); + p.expect(T![:]); + } + expr(p); + m.complete(p, RECORD_EXPR_FIELD); + } + T![.] if p.at(T![..]) => { + m.abandon(p); + p.bump(T![..]); + expr(p); + } + T!['{'] => { + error_block(p, "expected a field"); + m.abandon(p); + } + _ => { + p.err_and_bump("expected identifier"); + m.abandon(p); + } + } + if !p.at(T!['}']) { + p.expect(T![,]); + } + } + p.expect(T!['}']); + m.complete(p, RECORD_EXPR_FIELD_LIST); +} diff --git a/crates/parser/src/grammar/expressions/atom.rs b/crates/parser/src/grammar/expressions/atom.rs new file mode 100644 index 000000000..0b01d3bc6 --- /dev/null +++ b/crates/parser/src/grammar/expressions/atom.rs @@ -0,0 +1,611 @@ +//! FIXME: write short doc here + +use super::*; + +// test expr_literals +// fn foo() { +// let _ = true; +// let _ = false; +// let _ = 1; +// let _ = 2.0; +// let _ = b'a'; +// let _ = 'b'; +// let _ = "c"; +// let _ = r"d"; +// let _ = b"e"; +// let _ = br"f"; +// } +pub(crate) const LITERAL_FIRST: TokenSet = token_set![ + TRUE_KW, + FALSE_KW, + INT_NUMBER, + FLOAT_NUMBER, + BYTE, + CHAR, + STRING, + RAW_STRING, + BYTE_STRING, + RAW_BYTE_STRING +]; + +pub(crate) fn literal(p: &mut Parser) -> Option { + if !p.at_ts(LITERAL_FIRST) { + return None; + } + let m = p.start(); + p.bump_any(); + Some(m.complete(p, LITERAL)) +} + +// E.g. for after the break in `if break {}`, this should not match +pub(super) const ATOM_EXPR_FIRST: TokenSet = + LITERAL_FIRST.union(paths::PATH_FIRST).union(token_set![ + T!['('], + T!['{'], + T!['['], + L_DOLLAR, + T![|], + T![move], + T![box], + T![if], + T![while], + T![match], + T![unsafe], + T![return], + T![break], + T![continue], + T![async], + T![try], + T![loop], + T![for], + LIFETIME, + ]); + +const EXPR_RECOVERY_SET: TokenSet = token_set![LET_KW, R_DOLLAR]; + +pub(super) fn atom_expr(p: &mut Parser, r: Restrictions) -> Option<(CompletedMarker, BlockLike)> { + if let Some(m) = literal(p) { + return Some((m, BlockLike::NotBlock)); + } + if paths::is_path_start(p) { + return Some(path_expr(p, r)); + } + let la = p.nth(1); + let done = match p.current() { + T!['('] => tuple_expr(p), + T!['['] => array_expr(p), + L_DOLLAR => meta_var_expr(p), + T![|] => lambda_expr(p), + T![move] if la == T![|] => lambda_expr(p), + T![async] if la == T![|] || (la == T![move] && p.nth(2) == T![|]) => lambda_expr(p), + T![if] => if_expr(p), + + T![loop] => loop_expr(p, None), + T![box] => box_expr(p, None), + T![for] => for_expr(p, None), + T![while] => while_expr(p, None), + T![try] => try_block_expr(p, None), + LIFETIME if la == T![:] => { + let m = p.start(); + label(p); + match p.current() { + T![loop] => loop_expr(p, Some(m)), + T![for] => for_expr(p, Some(m)), + T![while] => while_expr(p, Some(m)), + // test labeled_block + // fn f() { 'label: {}; } + T!['{'] => { + block_expr(p); + m.complete(p, EFFECT_EXPR) + } + _ => { + // test_err misplaced_label_err + // fn main() { + // 'loop: impl + // } + p.error("expected a loop"); + m.complete(p, ERROR); + return None; + } + } + } + T![async] if la == T!['{'] || (la == T![move] && p.nth(2) == T!['{']) => { + let m = p.start(); + p.bump(T![async]); + p.eat(T![move]); + block_expr(p); + m.complete(p, EFFECT_EXPR) + } + T![match] => match_expr(p), + // test unsafe_block + // fn f() { unsafe { } } + T![unsafe] if la == T!['{'] => { + let m = p.start(); + p.bump(T![unsafe]); + block_expr(p); + m.complete(p, EFFECT_EXPR) + } + T!['{'] => { + // test for_range_from + // fn foo() { + // for x in 0 .. { + // break; + // } + // } + block_expr_unchecked(p) + } + T![return] => return_expr(p), + T![continue] => continue_expr(p), + T![break] => break_expr(p, r), + _ => { + p.err_recover("expected expression", EXPR_RECOVERY_SET); + return None; + } + }; + let blocklike = match done.kind() { + IF_EXPR | WHILE_EXPR | FOR_EXPR | LOOP_EXPR | MATCH_EXPR | BLOCK_EXPR | EFFECT_EXPR => { + BlockLike::Block + } + _ => BlockLike::NotBlock, + }; + Some((done, blocklike)) +} + +// test tuple_expr +// fn foo() { +// (); +// (1); +// (1,); +// } +fn tuple_expr(p: &mut Parser) -> CompletedMarker { + assert!(p.at(T!['('])); + let m = p.start(); + p.expect(T!['(']); + + let mut saw_comma = false; + let mut saw_expr = false; + while !p.at(EOF) && !p.at(T![')']) { + saw_expr = true; + if !p.at_ts(EXPR_FIRST) { + p.error("expected expression"); + break; + } + expr(p); + if !p.at(T![')']) { + saw_comma = true; + p.expect(T![,]); + } + } + p.expect(T![')']); + m.complete(p, if saw_expr && !saw_comma { PAREN_EXPR } else { TUPLE_EXPR }) +} + +// test array_expr +// fn foo() { +// []; +// [1]; +// [1, 2,]; +// [1; 2]; +// } +fn array_expr(p: &mut Parser) -> CompletedMarker { + assert!(p.at(T!['['])); + let m = p.start(); + + let mut n_exprs = 0u32; + let mut has_semi = false; + + p.bump(T!['[']); + while !p.at(EOF) && !p.at(T![']']) { + n_exprs += 1; + + // test array_attrs + // const A: &[i64] = &[1, #[cfg(test)] 2]; + if !expr_with_attrs(p) { + break; + } + + if n_exprs == 1 && p.eat(T![;]) { + has_semi = true; + continue; + } + + if has_semi || !p.at(T![']']) && !p.expect(T![,]) { + break; + } + } + p.expect(T![']']); + + m.complete(p, ARRAY_EXPR) +} + +// test lambda_expr +// fn foo() { +// || (); +// || -> i32 { 92 }; +// |x| x; +// move |x: i32,| x; +// async || {}; +// move || {}; +// async move || {}; +// } +fn lambda_expr(p: &mut Parser) -> CompletedMarker { + assert!( + p.at(T![|]) + || (p.at(T![move]) && p.nth(1) == T![|]) + || (p.at(T![async]) && p.nth(1) == T![|]) + || (p.at(T![async]) && p.nth(1) == T![move] && p.nth(2) == T![|]) + ); + let m = p.start(); + p.eat(T![async]); + p.eat(T![move]); + params::param_list_closure(p); + if opt_fn_ret_type(p) { + // test lambda_ret_block + // fn main() { || -> i32 { 92 }(); } + block_expr(p); + } else { + if p.at_ts(EXPR_FIRST) { + expr(p); + } else { + p.error("expected expression"); + } + } + m.complete(p, CLOSURE_EXPR) +} + +// test if_expr +// fn foo() { +// if true {}; +// if true {} else {}; +// if true {} else if false {} else {}; +// if S {}; +// if { true } { } else { }; +// } +fn if_expr(p: &mut Parser) -> CompletedMarker { + assert!(p.at(T![if])); + let m = p.start(); + p.bump(T![if]); + cond(p); + block_expr(p); + if p.at(T![else]) { + p.bump(T![else]); + if p.at(T![if]) { + if_expr(p); + } else { + block_expr(p); + } + } + m.complete(p, IF_EXPR) +} + +// test label +// fn foo() { +// 'a: loop {} +// 'b: while true {} +// 'c: for x in () {} +// } +fn label(p: &mut Parser) { + assert!(p.at(LIFETIME) && p.nth(1) == T![:]); + let m = p.start(); + p.bump(LIFETIME); + p.bump_any(); + m.complete(p, LABEL); +} + +// test loop_expr +// fn foo() { +// loop {}; +// } +fn loop_expr(p: &mut Parser, m: Option) -> CompletedMarker { + assert!(p.at(T![loop])); + let m = m.unwrap_or_else(|| p.start()); + p.bump(T![loop]); + block_expr(p); + m.complete(p, LOOP_EXPR) +} + +// test while_expr +// fn foo() { +// while true {}; +// while let Some(x) = it.next() {}; +// while { true } {}; +// } +fn while_expr(p: &mut Parser, m: Option) -> CompletedMarker { + assert!(p.at(T![while])); + let m = m.unwrap_or_else(|| p.start()); + p.bump(T![while]); + cond(p); + block_expr(p); + m.complete(p, WHILE_EXPR) +} + +// test for_expr +// fn foo() { +// for x in [] {}; +// } +fn for_expr(p: &mut Parser, m: Option) -> CompletedMarker { + assert!(p.at(T![for])); + let m = m.unwrap_or_else(|| p.start()); + p.bump(T![for]); + patterns::pattern(p); + p.expect(T![in]); + expr_no_struct(p); + block_expr(p); + m.complete(p, FOR_EXPR) +} + +// test cond +// fn foo() { if let Some(_) = None {} } +// fn bar() { +// if let Some(_) | Some(_) = None {} +// if let | Some(_) = None {} +// while let Some(_) | Some(_) = None {} +// while let | Some(_) = None {} +// } +fn cond(p: &mut Parser) { + let m = p.start(); + if p.eat(T![let]) { + patterns::pattern_top(p); + p.expect(T![=]); + } + expr_no_struct(p); + m.complete(p, CONDITION); +} + +// test match_expr +// fn foo() { +// match () { }; +// match S {}; +// match { } { _ => () }; +// match { S {} } {}; +// } +fn match_expr(p: &mut Parser) -> CompletedMarker { + assert!(p.at(T![match])); + let m = p.start(); + p.bump(T![match]); + expr_no_struct(p); + if p.at(T!['{']) { + match_arm_list(p); + } else { + p.error("expected `{`") + } + m.complete(p, MATCH_EXPR) +} + +pub(crate) fn match_arm_list(p: &mut Parser) { + assert!(p.at(T!['{'])); + let m = p.start(); + p.eat(T!['{']); + + // test match_arms_inner_attribute + // fn foo() { + // match () { + // #![doc("Inner attribute")] + // #![doc("Can be")] + // #![doc("Stacked")] + // _ => (), + // } + // } + attributes::inner_attributes(p); + + while !p.at(EOF) && !p.at(T!['}']) { + if p.at(T!['{']) { + error_block(p, "expected match arm"); + continue; + } + + // test match_arms_commas + // fn foo() { + // match () { + // _ => (), + // _ => {} + // _ => () + // } + // } + if match_arm(p).is_block() { + p.eat(T![,]); + } else if !p.at(T!['}']) { + p.expect(T![,]); + } + } + p.expect(T!['}']); + m.complete(p, MATCH_ARM_LIST); +} + +// test match_arm +// fn foo() { +// match () { +// _ => (), +// _ if Test > Test{field: 0} => (), +// X | Y if Z => (), +// | X | Y if Z => (), +// | X => (), +// }; +// } +fn match_arm(p: &mut Parser) -> BlockLike { + let m = p.start(); + // test match_arms_outer_attributes + // fn foo() { + // match () { + // #[cfg(feature = "some")] + // _ => (), + // #[cfg(feature = "other")] + // _ => (), + // #[cfg(feature = "many")] + // #[cfg(feature = "attributes")] + // #[cfg(feature = "before")] + // _ => (), + // } + // } + attributes::outer_attributes(p); + + patterns::pattern_top_r(p, TokenSet::EMPTY); + if p.at(T![if]) { + match_guard(p); + } + p.expect(T![=>]); + let blocklike = expr_stmt(p).1; + m.complete(p, MATCH_ARM); + blocklike +} + +// test match_guard +// fn foo() { +// match () { +// _ if foo => (), +// } +// } +fn match_guard(p: &mut Parser) -> CompletedMarker { + assert!(p.at(T![if])); + let m = p.start(); + p.bump(T![if]); + expr(p); + m.complete(p, MATCH_GUARD) +} + +// test block +// fn a() {} +// fn b() { let _ = 1; } +// fn c() { 1; 2; } +// fn d() { 1; 2 } +pub(crate) fn block_expr(p: &mut Parser) { + if !p.at(T!['{']) { + p.error("expected a block"); + return; + } + block_expr_unchecked(p); +} + +fn block_expr_unchecked(p: &mut Parser) -> CompletedMarker { + assert!(p.at(T!['{'])); + let m = p.start(); + p.bump(T!['{']); + expr_block_contents(p); + p.expect(T!['}']); + m.complete(p, BLOCK_EXPR) +} + +// test return_expr +// fn foo() { +// return; +// return 92; +// } +fn return_expr(p: &mut Parser) -> CompletedMarker { + assert!(p.at(T![return])); + let m = p.start(); + p.bump(T![return]); + if p.at_ts(EXPR_FIRST) { + expr(p); + } + m.complete(p, RETURN_EXPR) +} + +// test continue_expr +// fn foo() { +// loop { +// continue; +// continue 'l; +// } +// } +fn continue_expr(p: &mut Parser) -> CompletedMarker { + assert!(p.at(T![continue])); + let m = p.start(); + p.bump(T![continue]); + p.eat(LIFETIME); + m.complete(p, CONTINUE_EXPR) +} + +// test break_expr +// fn foo() { +// loop { +// break; +// break 'l; +// break 92; +// break 'l 92; +// } +// } +fn break_expr(p: &mut Parser, r: Restrictions) -> CompletedMarker { + assert!(p.at(T![break])); + let m = p.start(); + p.bump(T![break]); + p.eat(LIFETIME); + // test break_ambiguity + // fn foo(){ + // if break {} + // while break {} + // for i in break {} + // match break {} + // } + if p.at_ts(EXPR_FIRST) && !(r.forbid_structs && p.at(T!['{'])) { + expr(p); + } + m.complete(p, BREAK_EXPR) +} + +// test try_block_expr +// fn foo() { +// let _ = try {}; +// } +fn try_block_expr(p: &mut Parser, m: Option) -> CompletedMarker { + assert!(p.at(T![try])); + let m = m.unwrap_or_else(|| p.start()); + // Special-case `try!` as macro. + // This is a hack until we do proper edition support + if p.nth_at(1, T![!]) { + // test try_macro_fallback + // fn foo() { try!(Ok(())); } + let path = p.start(); + let path_segment = p.start(); + let name_ref = p.start(); + p.bump_remap(IDENT); + name_ref.complete(p, NAME_REF); + path_segment.complete(p, PATH_SEGMENT); + path.complete(p, PATH); + let _block_like = items::macro_call_after_excl(p); + return m.complete(p, MACRO_CALL); + } + + p.bump(T![try]); + block_expr(p); + m.complete(p, EFFECT_EXPR) +} + +// test box_expr +// fn foo() { +// let x = box 1i32; +// let y = (box 1i32, box 2i32); +// let z = Foo(box 1i32, box 2i32); +// } +fn box_expr(p: &mut Parser, m: Option) -> CompletedMarker { + assert!(p.at(T![box])); + let m = m.unwrap_or_else(|| p.start()); + p.bump(T![box]); + if p.at_ts(EXPR_FIRST) { + expr(p); + } + m.complete(p, BOX_EXPR) +} + +/// Expression from `$var` macro expansion, wrapped in dollars +fn meta_var_expr(p: &mut Parser) -> CompletedMarker { + assert!(p.at(L_DOLLAR)); + let m = p.start(); + p.bump(L_DOLLAR); + let (completed, _is_block) = + expr_bp(p, Restrictions { forbid_structs: false, prefer_stmt: false }, 1); + + match (completed, p.current()) { + (Some(it), R_DOLLAR) => { + p.bump(R_DOLLAR); + m.abandon(p); + it + } + _ => { + while !p.at(R_DOLLAR) { + p.bump_any() + } + p.bump(R_DOLLAR); + m.complete(p, ERROR) + } + } +} diff --git a/crates/parser/src/grammar/items.rs b/crates/parser/src/grammar/items.rs new file mode 100644 index 000000000..d091b0fbb --- /dev/null +++ b/crates/parser/src/grammar/items.rs @@ -0,0 +1,432 @@ +//! FIXME: write short doc here + +mod consts; +mod adt; +mod traits; +mod use_item; + +pub(crate) use self::{ + adt::{enum_variant_list, record_field_def_list}, + expressions::{match_arm_list, record_field_list}, + traits::{impl_item_list, trait_item_list}, + use_item::use_tree_list, +}; +use super::*; + +// test mod_contents +// fn foo() {} +// macro_rules! foo {} +// foo::bar!(); +// super::baz! {} +// struct S; +pub(super) fn mod_contents(p: &mut Parser, stop_on_r_curly: bool) { + attributes::inner_attributes(p); + while !(stop_on_r_curly && p.at(T!['}']) || p.at(EOF)) { + item_or_macro(p, stop_on_r_curly) + } +} + +pub(super) const ITEM_RECOVERY_SET: TokenSet = token_set![ + FN_KW, STRUCT_KW, ENUM_KW, IMPL_KW, TRAIT_KW, CONST_KW, STATIC_KW, LET_KW, MOD_KW, PUB_KW, + CRATE_KW, USE_KW, MACRO_KW +]; + +pub(super) fn item_or_macro(p: &mut Parser, stop_on_r_curly: bool) { + let m = p.start(); + attributes::outer_attributes(p); + let m = match maybe_item(p, m) { + Ok(()) => { + if p.at(T![;]) { + p.err_and_bump( + "expected item, found `;`\n\ + consider removing this semicolon", + ); + } + return; + } + Err(m) => m, + }; + if paths::is_use_path_start(p) { + match macro_call(p) { + BlockLike::Block => (), + BlockLike::NotBlock => { + p.expect(T![;]); + } + } + m.complete(p, MACRO_CALL); + } else { + m.abandon(p); + if p.at(T!['{']) { + error_block(p, "expected an item"); + } else if p.at(T!['}']) && !stop_on_r_curly { + let e = p.start(); + p.error("unmatched `}`"); + p.bump(T!['}']); + e.complete(p, ERROR); + } else if !p.at(EOF) && !p.at(T!['}']) { + p.err_and_bump("expected an item"); + } else { + p.error("expected an item"); + } + } +} + +pub(super) fn maybe_item(p: &mut Parser, m: Marker) -> Result<(), Marker> { + // test_err pub_expr + // fn foo() { pub 92; } + let has_visibility = opt_visibility(p); + + let m = match items_without_modifiers(p, m) { + Ok(()) => return Ok(()), + Err(m) => m, + }; + + let mut has_mods = false; + + // modifiers + has_mods |= p.eat(T![const]); + + // test_err async_without_semicolon + // fn foo() { let _ = async {} } + if p.at(T![async]) && p.nth(1) != T!['{'] && p.nth(1) != T![move] && p.nth(1) != T![|] { + p.eat(T![async]); + has_mods = true; + } + + // test_err unsafe_block_in_mod + // fn foo(){} unsafe { } fn bar(){} + if p.at(T![unsafe]) && p.nth(1) != T!['{'] { + p.eat(T![unsafe]); + has_mods = true; + } + + if p.at(T![extern]) { + has_mods = true; + abi(p); + } + if p.at(IDENT) && p.at_contextual_kw("auto") && p.nth(1) == T![trait] { + p.bump_remap(T![auto]); + has_mods = true; + } + + // test default_item + // default impl T for Foo {} + if p.at(IDENT) && p.at_contextual_kw("default") { + match p.nth(1) { + T![fn] | T![type] | T![const] | T![impl] => { + p.bump_remap(T![default]); + has_mods = true; + } + T![unsafe] => { + // test default_unsafe_item + // default unsafe impl T for Foo { + // default unsafe fn foo() {} + // } + if matches!(p.nth(2), T![impl] | T![fn]) { + p.bump_remap(T![default]); + p.bump(T![unsafe]); + has_mods = true; + } + } + _ => (), + } + } + + // test existential_type + // existential type Foo: Fn() -> usize; + if p.at(IDENT) && p.at_contextual_kw("existential") && p.nth(1) == T![type] { + p.bump_remap(T![existential]); + has_mods = true; + } + + // items + match p.current() { + // test fn + // fn foo() {} + T![fn] => { + fn_def(p); + m.complete(p, FN); + } + + // test trait + // trait T {} + T![trait] => { + traits::trait_def(p); + m.complete(p, TRAIT); + } + + T![const] => { + consts::const_def(p, m); + } + + // test impl + // impl T for S {} + T![impl] => { + traits::impl_def(p); + m.complete(p, IMPL); + } + + T![type] => { + type_def(p, m); + } + _ => { + if !has_visibility && !has_mods { + return Err(m); + } else { + if has_mods { + p.error("expected existential, fn, trait or impl"); + } else { + p.error("expected an item"); + } + m.complete(p, ERROR); + } + } + } + Ok(()) +} + +fn items_without_modifiers(p: &mut Parser, m: Marker) -> Result<(), Marker> { + let la = p.nth(1); + match p.current() { + // test extern_crate + // extern crate foo; + T![extern] if la == T![crate] => extern_crate_item(p, m), + T![type] => { + type_def(p, m); + } + T![mod] => mod_item(p, m), + T![struct] => { + // test struct_items + // struct Foo; + // struct Foo {} + // struct Foo(); + // struct Foo(String, usize); + // struct Foo { + // a: i32, + // b: f32, + // } + adt::struct_def(p, m); + } + // test pub_macro_def + // pub macro m($:ident) {} + T![macro] => { + macro_def(p, m); + } + IDENT if p.at_contextual_kw("union") && p.nth(1) == IDENT => { + // test union_items + // union Foo {} + // union Foo { + // a: i32, + // b: f32, + // } + adt::union_def(p, m); + } + T![enum] => adt::enum_def(p, m), + T![use] => use_item::use_item(p, m), + T![const] if (la == IDENT || la == T![_] || la == T![mut]) => consts::const_def(p, m), + T![static] => consts::static_def(p, m), + // test extern_block + // extern {} + T![extern] + if la == T!['{'] || ((la == STRING || la == RAW_STRING) && p.nth(2) == T!['{']) => + { + abi(p); + extern_item_list(p); + m.complete(p, EXTERN_BLOCK); + } + _ => return Err(m), + }; + Ok(()) +} + +fn extern_crate_item(p: &mut Parser, m: Marker) { + assert!(p.at(T![extern])); + p.bump(T![extern]); + assert!(p.at(T![crate])); + p.bump(T![crate]); + + if p.at(T![self]) { + p.bump(T![self]); + } else { + name_ref(p); + } + + opt_alias(p); + p.expect(T![;]); + m.complete(p, EXTERN_CRATE); +} + +pub(crate) fn extern_item_list(p: &mut Parser) { + assert!(p.at(T!['{'])); + let m = p.start(); + p.bump(T!['{']); + mod_contents(p, true); + p.expect(T!['}']); + m.complete(p, EXTERN_ITEM_LIST); +} + +fn fn_def(p: &mut Parser) { + assert!(p.at(T![fn])); + p.bump(T![fn]); + + name_r(p, ITEM_RECOVERY_SET); + // test function_type_params + // fn foo(){} + type_params::opt_type_param_list(p); + + if p.at(T!['(']) { + params::param_list_fn_def(p); + } else { + p.error("expected function arguments"); + } + // test function_ret_type + // fn foo() {} + // fn bar() -> () {} + opt_fn_ret_type(p); + + // test function_where_clause + // fn foo() where T: Copy {} + type_params::opt_where_clause(p); + + // test fn_decl + // trait T { fn foo(); } + if p.at(T![;]) { + p.bump(T![;]); + } else { + expressions::block_expr(p) + } +} + +// test type_item +// type Foo = Bar; +fn type_def(p: &mut Parser, m: Marker) { + assert!(p.at(T![type])); + p.bump(T![type]); + + name(p); + + // test type_item_type_params + // type Result = (); + type_params::opt_type_param_list(p); + + if p.at(T![:]) { + type_params::bounds(p); + } + + // test type_item_where_clause + // type Foo where Foo: Copy = (); + type_params::opt_where_clause(p); + if p.eat(T![=]) { + types::type_(p); + } + p.expect(T![;]); + m.complete(p, TYPE_ALIAS); +} + +pub(crate) fn mod_item(p: &mut Parser, m: Marker) { + assert!(p.at(T![mod])); + p.bump(T![mod]); + + name(p); + if p.at(T!['{']) { + mod_item_list(p); + } else if !p.eat(T![;]) { + p.error("expected `;` or `{`"); + } + m.complete(p, MODULE); +} + +pub(crate) fn mod_item_list(p: &mut Parser) { + assert!(p.at(T!['{'])); + let m = p.start(); + p.bump(T!['{']); + mod_contents(p, true); + p.expect(T!['}']); + m.complete(p, ITEM_LIST); +} + +// test macro_def +// macro m { ($i:ident) => {} } +// macro m($i:ident) {} +fn macro_def(p: &mut Parser, m: Marker) { + p.expect(T![macro]); + name_r(p, ITEM_RECOVERY_SET); + if p.at(T!['{']) { + token_tree(p); + } else if !p.at(T!['(']) { + p.error("unmatched `(`"); + } else { + let m = p.start(); + token_tree(p); + match p.current() { + T!['{'] | T!['['] | T!['('] => token_tree(p), + _ => p.error("expected `{`, `[`, `(`"), + } + m.complete(p, TOKEN_TREE); + } + + m.complete(p, MACRO_DEF); +} + +fn macro_call(p: &mut Parser) -> BlockLike { + assert!(paths::is_use_path_start(p)); + paths::use_path(p); + macro_call_after_excl(p) +} + +pub(super) fn macro_call_after_excl(p: &mut Parser) -> BlockLike { + p.expect(T![!]); + if p.at(IDENT) { + name(p); + } + // Special-case `macro_rules! try`. + // This is a hack until we do proper edition support + + // test try_macro_rules + // macro_rules! try { () => {} } + if p.at(T![try]) { + let m = p.start(); + p.bump_remap(IDENT); + m.complete(p, NAME); + } + + match p.current() { + T!['{'] => { + token_tree(p); + BlockLike::Block + } + T!['('] | T!['['] => { + token_tree(p); + BlockLike::NotBlock + } + _ => { + p.error("expected `{`, `[`, `(`"); + BlockLike::NotBlock + } + } +} + +pub(crate) fn token_tree(p: &mut Parser) { + let closing_paren_kind = match p.current() { + T!['{'] => T!['}'], + T!['('] => T![')'], + T!['['] => T![']'], + _ => unreachable!(), + }; + let m = p.start(); + p.bump_any(); + while !p.at(EOF) && !p.at(closing_paren_kind) { + match p.current() { + T!['{'] | T!['('] | T!['['] => token_tree(p), + T!['}'] => { + p.error("unmatched `}`"); + m.complete(p, TOKEN_TREE); + return; + } + T![')'] | T![']'] => p.err_and_bump("unmatched brace"), + _ => p.bump_any(), + } + } + p.expect(closing_paren_kind); + m.complete(p, TOKEN_TREE); +} diff --git a/crates/parser/src/grammar/items/adt.rs b/crates/parser/src/grammar/items/adt.rs new file mode 100644 index 000000000..addfb59d4 --- /dev/null +++ b/crates/parser/src/grammar/items/adt.rs @@ -0,0 +1,178 @@ +//! FIXME: write short doc here + +use super::*; + +pub(super) fn struct_def(p: &mut Parser, m: Marker) { + assert!(p.at(T![struct])); + p.bump(T![struct]); + struct_or_union(p, m, T![struct], STRUCT); +} + +pub(super) fn union_def(p: &mut Parser, m: Marker) { + assert!(p.at_contextual_kw("union")); + p.bump_remap(T![union]); + struct_or_union(p, m, T![union], UNION); +} + +fn struct_or_union(p: &mut Parser, m: Marker, kw: SyntaxKind, def: SyntaxKind) { + name_r(p, ITEM_RECOVERY_SET); + type_params::opt_type_param_list(p); + match p.current() { + T![where] => { + type_params::opt_where_clause(p); + match p.current() { + T![;] => { + p.bump(T![;]); + } + T!['{'] => record_field_def_list(p), + _ => { + //FIXME: special case `(` error message + p.error("expected `;` or `{`"); + } + } + } + T![;] if kw == T![struct] => { + p.bump(T![;]); + } + T!['{'] => record_field_def_list(p), + T!['('] if kw == T![struct] => { + tuple_field_def_list(p); + // test tuple_struct_where + // struct Test(T) where T: Clone; + // struct Test(T); + type_params::opt_where_clause(p); + p.expect(T![;]); + } + _ if kw == T![struct] => { + p.error("expected `;`, `{`, or `(`"); + } + _ => { + p.error("expected `{`"); + } + } + m.complete(p, def); +} + +pub(super) fn enum_def(p: &mut Parser, m: Marker) { + assert!(p.at(T![enum])); + p.bump(T![enum]); + name_r(p, ITEM_RECOVERY_SET); + type_params::opt_type_param_list(p); + type_params::opt_where_clause(p); + if p.at(T!['{']) { + enum_variant_list(p); + } else { + p.error("expected `{`") + } + m.complete(p, ENUM); +} + +pub(crate) fn enum_variant_list(p: &mut Parser) { + assert!(p.at(T!['{'])); + let m = p.start(); + p.bump(T!['{']); + while !p.at(EOF) && !p.at(T!['}']) { + if p.at(T!['{']) { + error_block(p, "expected enum variant"); + continue; + } + let var = p.start(); + attributes::outer_attributes(p); + if p.at(IDENT) { + name(p); + match p.current() { + T!['{'] => record_field_def_list(p), + T!['('] => tuple_field_def_list(p), + _ => (), + } + + // test variant_discriminant + // enum E { X(i32) = 10 } + if p.eat(T![=]) { + expressions::expr(p); + } + var.complete(p, VARIANT); + } else { + var.abandon(p); + p.err_and_bump("expected enum variant"); + } + if !p.at(T!['}']) { + p.expect(T![,]); + } + } + p.expect(T!['}']); + m.complete(p, VARIANT_LIST); +} + +pub(crate) fn record_field_def_list(p: &mut Parser) { + assert!(p.at(T!['{'])); + let m = p.start(); + p.bump(T!['{']); + while !p.at(T!['}']) && !p.at(EOF) { + if p.at(T!['{']) { + error_block(p, "expected field"); + continue; + } + record_field_def(p); + if !p.at(T!['}']) { + p.expect(T![,]); + } + } + p.expect(T!['}']); + m.complete(p, RECORD_FIELD_LIST); + + fn record_field_def(p: &mut Parser) { + let m = p.start(); + // test record_field_attrs + // struct S { + // #[serde(with = "url_serde")] + // pub uri: Uri, + // } + attributes::outer_attributes(p); + opt_visibility(p); + if p.at(IDENT) { + name(p); + p.expect(T![:]); + types::type_(p); + m.complete(p, RECORD_FIELD); + } else { + m.abandon(p); + p.err_and_bump("expected field declaration"); + } + } +} + +fn tuple_field_def_list(p: &mut Parser) { + assert!(p.at(T!['('])); + let m = p.start(); + if !p.expect(T!['(']) { + return; + } + while !p.at(T![')']) && !p.at(EOF) { + let m = p.start(); + // test tuple_field_attrs + // struct S ( + // #[serde(with = "url_serde")] + // pub Uri, + // ); + // + // enum S { + // Uri(#[serde(with = "url_serde")] Uri), + // } + attributes::outer_attributes(p); + opt_visibility(p); + if !p.at_ts(types::TYPE_FIRST) { + p.error("expected a type"); + m.complete(p, ERROR); + break; + } + types::type_(p); + m.complete(p, TUPLE_FIELD); + + if !p.at(T![')']) { + p.expect(T![,]); + } + } + p.expect(T![')']); + m.complete(p, TUPLE_FIELD_LIST); +} diff --git a/crates/parser/src/grammar/items/consts.rs b/crates/parser/src/grammar/items/consts.rs new file mode 100644 index 000000000..35ad766dc --- /dev/null +++ b/crates/parser/src/grammar/items/consts.rs @@ -0,0 +1,33 @@ +//! FIXME: write short doc here + +use super::*; + +pub(super) fn static_def(p: &mut Parser, m: Marker) { + const_or_static(p, m, T![static], STATIC) +} + +pub(super) fn const_def(p: &mut Parser, m: Marker) { + const_or_static(p, m, T![const], CONST) +} + +fn const_or_static(p: &mut Parser, m: Marker, kw: SyntaxKind, def: SyntaxKind) { + assert!(p.at(kw)); + p.bump(kw); + p.eat(T![mut]); // FIXME: validator to forbid const mut + + // Allow `_` in place of an identifier in a `const`. + let is_const_underscore = kw == T![const] && p.eat(T![_]); + if !is_const_underscore { + name(p); + } + + // test_err static_underscore + // static _: i32 = 5; + + types::ascription(p); + if p.eat(T![=]) { + expressions::expr(p); + } + p.expect(T![;]); + m.complete(p, def); +} diff --git a/crates/parser/src/grammar/items/traits.rs b/crates/parser/src/grammar/items/traits.rs new file mode 100644 index 000000000..751ce65f2 --- /dev/null +++ b/crates/parser/src/grammar/items/traits.rs @@ -0,0 +1,153 @@ +//! FIXME: write short doc here + +use super::*; + +// test trait_item +// trait T: Hash + Clone where U: Copy {} +// trait X: Hash + Clone where U: Copy {} +pub(super) fn trait_def(p: &mut Parser) { + assert!(p.at(T![trait])); + p.bump(T![trait]); + name_r(p, ITEM_RECOVERY_SET); + type_params::opt_type_param_list(p); + // test trait_alias + // trait Z = T; + // trait Z = T where U: Copy; + // trait Z = where Self: T; + if p.eat(T![=]) { + type_params::bounds_without_colon(p); + type_params::opt_where_clause(p); + p.expect(T![;]); + return; + } + if p.at(T![:]) { + type_params::bounds(p); + } + type_params::opt_where_clause(p); + if p.at(T!['{']) { + trait_item_list(p); + } else { + p.error("expected `{`"); + } +} + +// test trait_item_list +// impl F { +// type A: Clone; +// const B: i32; +// fn foo() {} +// fn bar(&self); +// } +pub(crate) fn trait_item_list(p: &mut Parser) { + assert!(p.at(T!['{'])); + let m = p.start(); + p.bump(T!['{']); + while !p.at(EOF) && !p.at(T!['}']) { + if p.at(T!['{']) { + error_block(p, "expected an item"); + continue; + } + item_or_macro(p, true); + } + p.expect(T!['}']); + m.complete(p, ASSOC_ITEM_LIST); +} + +// test impl_def +// impl Foo {} +pub(super) fn impl_def(p: &mut Parser) { + assert!(p.at(T![impl])); + p.bump(T![impl]); + if choose_type_params_over_qpath(p) { + type_params::opt_type_param_list(p); + } + + // FIXME: never type + // impl ! {} + + // test impl_def_neg + // impl !Send for X {} + p.eat(T![!]); + impl_type(p); + if p.eat(T![for]) { + impl_type(p); + } + type_params::opt_where_clause(p); + if p.at(T!['{']) { + impl_item_list(p); + } else { + p.error("expected `{`"); + } +} + +// test impl_item_list +// impl F { +// type A = i32; +// const B: i32 = 92; +// fn foo() {} +// fn bar(&self) {} +// } +pub(crate) fn impl_item_list(p: &mut Parser) { + assert!(p.at(T!['{'])); + let m = p.start(); + p.bump(T!['{']); + // test impl_inner_attributes + // enum F{} + // impl F { + // //! This is a doc comment + // #![doc("This is also a doc comment")] + // } + attributes::inner_attributes(p); + + while !p.at(EOF) && !p.at(T!['}']) { + if p.at(T!['{']) { + error_block(p, "expected an item"); + continue; + } + item_or_macro(p, true); + } + p.expect(T!['}']); + m.complete(p, ASSOC_ITEM_LIST); +} + +// test impl_type_params +// impl Bar {} +fn choose_type_params_over_qpath(p: &Parser) -> bool { + // There's an ambiguity between generic parameters and qualified paths in impls. + // If we see `<` it may start both, so we have to inspect some following tokens. + // The following combinations can only start generics, + // but not qualified paths (with one exception): + // `<` `>` - empty generic parameters + // `<` `#` - generic parameters with attributes + // `<` `const` - const generic parameters + // `<` (LIFETIME|IDENT) `>` - single generic parameter + // `<` (LIFETIME|IDENT) `,` - first generic parameter in a list + // `<` (LIFETIME|IDENT) `:` - generic parameter with bounds + // `<` (LIFETIME|IDENT) `=` - generic parameter with a default + // The only truly ambiguous case is + // `<` IDENT `>` `::` IDENT ... + // we disambiguate it in favor of generics (`impl ::absolute::Path { ... }`) + // because this is what almost always expected in practice, qualified paths in impls + // (`impl ::AssocTy { ... }`) aren't even allowed by type checker at the moment. + if !p.at(T![<]) { + return false; + } + if p.nth(1) == T![#] || p.nth(1) == T![>] || p.nth(1) == CONST_KW { + return true; + } + (p.nth(1) == LIFETIME || p.nth(1) == IDENT) + && (p.nth(2) == T![>] || p.nth(2) == T![,] || p.nth(2) == T![:] || p.nth(2) == T![=]) +} + +// test_err impl_type +// impl Type {} +// impl Trait1 for T {} +// impl impl NotType {} +// impl Trait2 for impl NotType {} +pub(crate) fn impl_type(p: &mut Parser) { + if p.at(T![impl]) { + p.error("expected trait or type"); + return; + } + types::type_(p); +} diff --git a/crates/parser/src/grammar/items/use_item.rs b/crates/parser/src/grammar/items/use_item.rs new file mode 100644 index 000000000..8e836a77e --- /dev/null +++ b/crates/parser/src/grammar/items/use_item.rs @@ -0,0 +1,132 @@ +//! FIXME: write short doc here + +use super::*; + +pub(super) fn use_item(p: &mut Parser, m: Marker) { + assert!(p.at(T![use])); + p.bump(T![use]); + use_tree(p, true); + p.expect(T![;]); + m.complete(p, USE); +} + +/// Parse a use 'tree', such as `some::path` in `use some::path;` +/// Note that this is called both by `use_item` and `use_tree_list`, +/// so handles both `some::path::{inner::path}` and `inner::path` in +/// `use some::path::{inner::path};` +fn use_tree(p: &mut Parser, top_level: bool) { + let m = p.start(); + match p.current() { + // Finish the use_tree for cases of e.g. + // `use some::path::{self, *};` or `use *;` + // This does not handle cases such as `use some::path::*` + // N.B. in Rust 2015 `use *;` imports all from crate root + // however in Rust 2018 `use *;` errors: ('cannot glob-import all possible crates') + // FIXME: Add this error (if not out of scope) + + // test use_star + // use *; + // use ::*; + // use some::path::{*}; + // use some::path::{::*}; + T![*] => p.bump(T![*]), + T![:] if p.at(T![::]) && p.nth(2) == T![*] => { + // Parse `use ::*;`, which imports all from the crate root in Rust 2015 + // This is invalid inside a use_tree_list, (e.g. `use some::path::{::*}`) + // but still parses and errors later: ('crate root in paths can only be used in start position') + // FIXME: Add this error (if not out of scope) + // In Rust 2018, it is always invalid (see above) + p.bump(T![::]); + p.bump(T![*]); + } + // Open a use tree list + // Handles cases such as `use {some::path};` or `{inner::path}` in + // `use some::path::{{inner::path}, other::path}` + + // test use_tree_list + // use {crate::path::from::root, or::path::from::crate_name}; // Rust 2018 (with a crate named `or`) + // use {path::from::root}; // Rust 2015 + // use ::{some::arbritrary::path}; // Rust 2015 + // use ::{{{root::export}}}; // Nonsensical but perfectly legal nesting + T!['{'] => { + use_tree_list(p); + } + T![:] if p.at(T![::]) && p.nth(2) == T!['{'] => { + p.bump(T![::]); + use_tree_list(p); + } + // Parse a 'standard' path. + // Also handles aliases (e.g. `use something as something_else`) + + // test use_path + // use ::crate_name; // Rust 2018 - All flavours + // use crate_name; // Rust 2018 - Anchored paths + // use item_in_scope_or_crate_name; // Rust 2018 - Uniform Paths + // + // use self::module::Item; + // use crate::Item; + // use self::some::Struct; + // use crate_name::some_item; + _ if paths::is_use_path_start(p) => { + paths::use_path(p); + match p.current() { + T![as] => { + // test use_alias + // use some::path as some_name; + // use some::{ + // other::path as some_other_name, + // different::path as different_name, + // yet::another::path, + // running::out::of::synonyms::for_::different::* + // }; + // use Trait as _; + opt_alias(p); + } + T![:] if p.at(T![::]) => { + p.bump(T![::]); + match p.current() { + T![*] => { + p.bump(T![*]); + } + // test use_tree_list_after_path + // use crate::{Item}; + // use self::{Item}; + T!['{'] => use_tree_list(p), + _ => { + // is this unreachable? + p.error("expected `{` or `*`"); + } + } + } + _ => (), + } + } + _ => { + m.abandon(p); + let msg = "expected one of `*`, `::`, `{`, `self`, `super` or an identifier"; + if top_level { + p.err_recover(msg, ITEM_RECOVERY_SET); + } else { + // if we are parsing a nested tree, we have to eat a token to + // main balanced `{}` + p.err_and_bump(msg); + } + return; + } + } + m.complete(p, USE_TREE); +} + +pub(crate) fn use_tree_list(p: &mut Parser) { + assert!(p.at(T!['{'])); + let m = p.start(); + p.bump(T!['{']); + while !p.at(EOF) && !p.at(T!['}']) { + use_tree(p, false); + if !p.at(T!['}']) { + p.expect(T![,]); + } + } + p.expect(T!['}']); + m.complete(p, USE_TREE_LIST); +} diff --git a/crates/parser/src/grammar/params.rs b/crates/parser/src/grammar/params.rs new file mode 100644 index 000000000..f0da173cc --- /dev/null +++ b/crates/parser/src/grammar/params.rs @@ -0,0 +1,188 @@ +//! FIXME: write short doc here + +use super::*; + +// test param_list +// fn a() {} +// fn b(x: i32) {} +// fn c(x: i32, ) {} +// fn d(x: i32, y: ()) {} +pub(super) fn param_list_fn_def(p: &mut Parser) { + list_(p, Flavor::FnDef) +} + +// test param_list_opt_patterns +// fn foo)>(){} +pub(super) fn param_list_fn_trait(p: &mut Parser) { + list_(p, Flavor::FnTrait) +} + +pub(super) fn param_list_fn_ptr(p: &mut Parser) { + list_(p, Flavor::FnPointer) +} + +pub(super) fn param_list_closure(p: &mut Parser) { + list_(p, Flavor::Closure) +} + +#[derive(Debug, Clone, Copy)] +enum Flavor { + FnDef, // Includes trait fn params; omitted param idents are not supported + FnTrait, // Params for `Fn(...)`/`FnMut(...)`/`FnOnce(...)` annotations + FnPointer, + Closure, +} + +fn list_(p: &mut Parser, flavor: Flavor) { + use Flavor::*; + + let (bra, ket) = match flavor { + Closure => (T![|], T![|]), + FnDef | FnTrait | FnPointer => (T!['('], T![')']), + }; + + let m = p.start(); + p.bump(bra); + + if let FnDef = flavor { + // test self_param_outer_attr + // fn f(#[must_use] self) {} + attributes::outer_attributes(p); + opt_self_param(p); + } + + while !p.at(EOF) && !p.at(ket) { + // test param_outer_arg + // fn f(#[attr1] pat: Type) {} + attributes::outer_attributes(p); + + if !p.at_ts(VALUE_PARAMETER_FIRST) { + p.error("expected value parameter"); + break; + } + let param = value_parameter(p, flavor); + if !p.at(ket) { + p.expect(T![,]); + } + if let Variadic(true) = param { + break; + } + } + + p.expect(ket); + m.complete(p, PARAM_LIST); +} + +const VALUE_PARAMETER_FIRST: TokenSet = patterns::PATTERN_FIRST.union(types::TYPE_FIRST); + +struct Variadic(bool); + +fn value_parameter(p: &mut Parser, flavor: Flavor) -> Variadic { + let mut res = Variadic(false); + let m = p.start(); + match flavor { + // test param_list_vararg + // extern "C" { fn printf(format: *const i8, ...) -> i32; } + Flavor::FnDef | Flavor::FnPointer if p.eat(T![...]) => res = Variadic(true), + + // test fn_def_param + // fn foo((x, y): (i32, i32)) {} + Flavor::FnDef => { + patterns::pattern(p); + if variadic_param(p) { + res = Variadic(true) + } else { + types::ascription(p); + } + } + // test value_parameters_no_patterns + // type F = Box; + Flavor::FnTrait => { + types::type_(p); + } + // test fn_pointer_param_ident_path + // type Foo = fn(Bar::Baz); + // type Qux = fn(baz: Bar::Baz); + + // test fn_pointer_unnamed_arg + // type Foo = fn(_: bar); + Flavor::FnPointer => { + if (p.at(IDENT) || p.at(UNDERSCORE)) && p.nth(1) == T![:] && !p.nth_at(1, T![::]) { + patterns::pattern_single(p); + if variadic_param(p) { + res = Variadic(true) + } else { + types::ascription(p); + } + } else { + types::type_(p); + } + } + // test closure_params + // fn main() { + // let foo = |bar, baz: Baz, qux: Qux::Quux| (); + // } + Flavor::Closure => { + patterns::pattern_single(p); + if p.at(T![:]) && !p.at(T![::]) { + types::ascription(p); + } + } + } + m.complete(p, PARAM); + res +} + +fn variadic_param(p: &mut Parser) -> bool { + if p.at(T![:]) && p.nth_at(1, T![...]) { + p.bump(T![:]); + p.bump(T![...]); + true + } else { + false + } +} + +// test self_param +// impl S { +// fn a(self) {} +// fn b(&self,) {} +// fn c(&'a self,) {} +// fn d(&'a mut self, x: i32) {} +// fn e(mut self) {} +// } +fn opt_self_param(p: &mut Parser) { + let m; + if p.at(T![self]) || p.at(T![mut]) && p.nth(1) == T![self] { + m = p.start(); + p.eat(T![mut]); + p.eat(T![self]); + // test arb_self_types + // impl S { + // fn a(self: &Self) {} + // fn b(mut self: Box) {} + // } + if p.at(T![:]) { + types::ascription(p); + } + } else { + let la1 = p.nth(1); + let la2 = p.nth(2); + let la3 = p.nth(3); + let n_toks = match (p.current(), la1, la2, la3) { + (T![&], T![self], _, _) => 2, + (T![&], T![mut], T![self], _) => 3, + (T![&], LIFETIME, T![self], _) => 3, + (T![&], LIFETIME, T![mut], T![self]) => 4, + _ => return, + }; + m = p.start(); + for _ in 0..n_toks { + p.bump_any(); + } + } + m.complete(p, SELF_PARAM); + if !p.at(T![')']) { + p.expect(T![,]); + } +} diff --git a/crates/parser/src/grammar/paths.rs b/crates/parser/src/grammar/paths.rs new file mode 100644 index 000000000..b503af1dc --- /dev/null +++ b/crates/parser/src/grammar/paths.rs @@ -0,0 +1,115 @@ +//! FIXME: write short doc here + +use super::*; + +pub(super) const PATH_FIRST: TokenSet = + token_set![IDENT, T![self], T![super], T![crate], T![:], T![<]]; + +pub(super) fn is_path_start(p: &Parser) -> bool { + is_use_path_start(p) || p.at(T![<]) +} + +pub(super) fn is_use_path_start(p: &Parser) -> bool { + match p.current() { + IDENT | T![self] | T![super] | T![crate] => true, + T![:] if p.at(T![::]) => true, + _ => false, + } +} + +pub(super) fn use_path(p: &mut Parser) { + path(p, Mode::Use) +} + +pub(crate) fn type_path(p: &mut Parser) { + path(p, Mode::Type) +} + +pub(super) fn expr_path(p: &mut Parser) { + path(p, Mode::Expr) +} + +#[derive(Clone, Copy, Eq, PartialEq)] +enum Mode { + Use, + Type, + Expr, +} + +fn path(p: &mut Parser, mode: Mode) { + let path = p.start(); + path_segment(p, mode, true); + let mut qual = path.complete(p, PATH); + loop { + let use_tree = matches!(p.nth(2), T![*] | T!['{']); + if p.at(T![::]) && !use_tree { + let path = qual.precede(p); + p.bump(T![::]); + path_segment(p, mode, false); + let path = path.complete(p, PATH); + qual = path; + } else { + break; + } + } +} + +fn path_segment(p: &mut Parser, mode: Mode, first: bool) { + let m = p.start(); + // test qual_paths + // type X = ::Output; + // fn foo() { ::default(); } + if first && p.eat(T![<]) { + types::type_(p); + if p.eat(T![as]) { + if is_use_path_start(p) { + types::path_type(p); + } else { + p.error("expected a trait"); + } + } + p.expect(T![>]); + } else { + let mut empty = true; + if first { + p.eat(T![::]); + empty = false; + } + match p.current() { + IDENT => { + name_ref(p); + opt_path_type_args(p, mode); + } + // test crate_path + // use crate::foo; + T![self] | T![super] | T![crate] => p.bump_any(), + _ => { + p.err_recover("expected identifier", items::ITEM_RECOVERY_SET); + if empty { + // test_err empty_segment + // use crate::; + m.abandon(p); + return; + } + } + }; + } + m.complete(p, PATH_SEGMENT); +} + +fn opt_path_type_args(p: &mut Parser, mode: Mode) { + match mode { + Mode::Use => {} + Mode::Type => { + // test path_fn_trait_args + // type F = Box ()>; + if p.at(T!['(']) { + params::param_list_fn_trait(p); + opt_fn_ret_type(p); + } else { + type_args::opt_type_arg_list(p, false) + } + } + Mode::Expr => type_args::opt_type_arg_list(p, true), + } +} diff --git a/crates/parser/src/grammar/patterns.rs b/crates/parser/src/grammar/patterns.rs new file mode 100644 index 000000000..716bdc978 --- /dev/null +++ b/crates/parser/src/grammar/patterns.rs @@ -0,0 +1,379 @@ +//! FIXME: write short doc here + +use super::*; + +pub(super) const PATTERN_FIRST: TokenSet = expressions::LITERAL_FIRST + .union(paths::PATH_FIRST) + .union(token_set![T![box], T![ref], T![mut], T!['('], T!['['], T![&], T![_], T![-], T![.]]); + +pub(crate) fn pattern(p: &mut Parser) { + pattern_r(p, PAT_RECOVERY_SET); +} + +/// Parses a pattern list separated by pipes `|` +pub(super) fn pattern_top(p: &mut Parser) { + pattern_top_r(p, PAT_RECOVERY_SET) +} + +pub(crate) fn pattern_single(p: &mut Parser) { + pattern_single_r(p, PAT_RECOVERY_SET); +} + +/// Parses a pattern list separated by pipes `|` +/// using the given `recovery_set` +pub(super) fn pattern_top_r(p: &mut Parser, recovery_set: TokenSet) { + p.eat(T![|]); + pattern_r(p, recovery_set); +} + +/// Parses a pattern list separated by pipes `|`, with no leading `|`,using the +/// given `recovery_set` +// test or_pattern +// fn main() { +// match () { +// (_ | _) => (), +// &(_ | _) => (), +// (_ | _,) => (), +// [_ | _,] => (), +// } +// } +fn pattern_r(p: &mut Parser, recovery_set: TokenSet) { + let m = p.start(); + pattern_single_r(p, recovery_set); + + if !p.at(T![|]) { + m.abandon(p); + return; + } + while p.eat(T![|]) { + pattern_single_r(p, recovery_set); + } + m.complete(p, OR_PAT); +} + +fn pattern_single_r(p: &mut Parser, recovery_set: TokenSet) { + if let Some(lhs) = atom_pat(p, recovery_set) { + // test range_pat + // fn main() { + // match 92 { + // 0 ... 100 => (), + // 101 ..= 200 => (), + // 200 .. 301=> (), + // } + // } + for &range_op in [T![...], T![..=], T![..]].iter() { + if p.at(range_op) { + let m = lhs.precede(p); + p.bump(range_op); + atom_pat(p, recovery_set); + m.complete(p, RANGE_PAT); + return; + } + } + } +} + +const PAT_RECOVERY_SET: TokenSet = + token_set![LET_KW, IF_KW, WHILE_KW, LOOP_KW, MATCH_KW, R_PAREN, COMMA]; + +fn atom_pat(p: &mut Parser, recovery_set: TokenSet) -> Option { + let m = match p.nth(0) { + T![box] => box_pat(p), + T![ref] | T![mut] => bind_pat(p, true), + IDENT => match p.nth(1) { + // Checks the token after an IDENT to see if a pattern is a path (Struct { .. }) or macro + // (T![x]). + T!['('] | T!['{'] | T![!] => path_or_macro_pat(p), + T![:] if p.nth_at(1, T![::]) => path_or_macro_pat(p), + _ => bind_pat(p, true), + }, + + // test type_path_in_pattern + // fn main() { let <_>::Foo = (); } + _ if paths::is_path_start(p) => path_or_macro_pat(p), + _ if is_literal_pat_start(p) => literal_pat(p), + + T![.] if p.at(T![..]) => dot_dot_pat(p), + T![_] => placeholder_pat(p), + T![&] => ref_pat(p), + T!['('] => tuple_pat(p), + T!['['] => slice_pat(p), + + _ => { + p.err_recover("expected pattern", recovery_set); + return None; + } + }; + + Some(m) +} + +fn is_literal_pat_start(p: &Parser) -> bool { + p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER) + || p.at_ts(expressions::LITERAL_FIRST) +} + +// test literal_pattern +// fn main() { +// match () { +// -1 => (), +// 92 => (), +// 'c' => (), +// "hello" => (), +// } +// } +fn literal_pat(p: &mut Parser) -> CompletedMarker { + assert!(is_literal_pat_start(p)); + let m = p.start(); + if p.at(T![-]) { + p.bump(T![-]); + } + expressions::literal(p); + m.complete(p, LITERAL_PAT) +} + +// test path_part +// fn foo() { +// let foo::Bar = (); +// let ::Bar = (); +// let Bar { .. } = (); +// let Bar(..) = (); +// } +fn path_or_macro_pat(p: &mut Parser) -> CompletedMarker { + assert!(paths::is_path_start(p)); + let m = p.start(); + paths::expr_path(p); + let kind = match p.current() { + T!['('] => { + tuple_pat_fields(p); + TUPLE_STRUCT_PAT + } + T!['{'] => { + record_field_pat_list(p); + RECORD_PAT + } + // test marco_pat + // fn main() { + // let m!(x) = 0; + // } + T![!] => { + items::macro_call_after_excl(p); + return m.complete(p, MACRO_CALL).precede(p).complete(p, MACRO_PAT); + } + _ => PATH_PAT, + }; + m.complete(p, kind) +} + +// test tuple_pat_fields +// fn foo() { +// let S() = (); +// let S(_) = (); +// let S(_,) = (); +// let S(_, .. , x) = (); +// } +fn tuple_pat_fields(p: &mut Parser) { + assert!(p.at(T!['('])); + p.bump(T!['(']); + pat_list(p, T![')']); + p.expect(T![')']); +} + +// test record_field_pat_list +// fn foo() { +// let S {} = (); +// let S { f, ref mut g } = (); +// let S { h: _, ..} = (); +// let S { h: _, } = (); +// } +fn record_field_pat_list(p: &mut Parser) { + assert!(p.at(T!['{'])); + let m = p.start(); + p.bump(T!['{']); + while !p.at(EOF) && !p.at(T!['}']) { + match p.current() { + // A trailing `..` is *not* treated as a REST_PAT. + T![.] if p.at(T![..]) => p.bump(T![..]), + T!['{'] => error_block(p, "expected ident"), + + c => { + let m = p.start(); + match c { + // test record_field_pat + // fn foo() { + // let S { 0: 1 } = (); + // let S { x: 1 } = (); + // } + IDENT | INT_NUMBER if p.nth(1) == T![:] => { + name_ref_or_index(p); + p.bump(T![:]); + pattern(p); + } + T![box] => { + // FIXME: not all box patterns should be allowed + box_pat(p); + } + _ => { + bind_pat(p, false); + } + } + m.complete(p, RECORD_PAT_FIELD); + } + } + if !p.at(T!['}']) { + p.expect(T![,]); + } + } + p.expect(T!['}']); + m.complete(p, RECORD_PAT_FIELD_LIST); +} + +// test placeholder_pat +// fn main() { let _ = (); } +fn placeholder_pat(p: &mut Parser) -> CompletedMarker { + assert!(p.at(T![_])); + let m = p.start(); + p.bump(T![_]); + m.complete(p, WILDCARD_PAT) +} + +// test dot_dot_pat +// fn main() { +// let .. = (); +// // +// // Tuples +// // +// let (a, ..) = (); +// let (a, ..,) = (); +// let Tuple(a, ..) = (); +// let Tuple(a, ..,) = (); +// let (.., ..) = (); +// let Tuple(.., ..) = (); +// let (.., a, ..) = (); +// let Tuple(.., a, ..) = (); +// // +// // Slices +// // +// let [..] = (); +// let [head, ..] = (); +// let [head, tail @ ..] = (); +// let [head, .., cons] = (); +// let [head, mid @ .., cons] = (); +// let [head, .., .., cons] = (); +// let [head, .., mid, tail @ ..] = (); +// let [head, .., mid, .., cons] = (); +// } +fn dot_dot_pat(p: &mut Parser) -> CompletedMarker { + assert!(p.at(T![..])); + let m = p.start(); + p.bump(T![..]); + m.complete(p, REST_PAT) +} + +// test ref_pat +// fn main() { +// let &a = (); +// let &mut b = (); +// } +fn ref_pat(p: &mut Parser) -> CompletedMarker { + assert!(p.at(T![&])); + let m = p.start(); + p.bump(T![&]); + p.eat(T![mut]); + pattern_single(p); + m.complete(p, REF_PAT) +} + +// test tuple_pat +// fn main() { +// let (a, b, ..) = (); +// let (a,) = (); +// let (..) = (); +// let () = (); +// } +fn tuple_pat(p: &mut Parser) -> CompletedMarker { + assert!(p.at(T!['('])); + let m = p.start(); + p.bump(T!['(']); + let mut has_comma = false; + let mut has_pat = false; + let mut has_rest = false; + while !p.at(EOF) && !p.at(T![')']) { + has_pat = true; + if !p.at_ts(PATTERN_FIRST) { + p.error("expected a pattern"); + break; + } + has_rest |= p.at(T![..]); + + pattern(p); + if !p.at(T![')']) { + has_comma = true; + p.expect(T![,]); + } + } + p.expect(T![')']); + + m.complete(p, if !has_comma && !has_rest && has_pat { PAREN_PAT } else { TUPLE_PAT }) +} + +// test slice_pat +// fn main() { +// let [a, b, ..] = []; +// } +fn slice_pat(p: &mut Parser) -> CompletedMarker { + assert!(p.at(T!['['])); + let m = p.start(); + p.bump(T!['[']); + pat_list(p, T![']']); + p.expect(T![']']); + m.complete(p, SLICE_PAT) +} + +fn pat_list(p: &mut Parser, ket: SyntaxKind) { + while !p.at(EOF) && !p.at(ket) { + if !p.at_ts(PATTERN_FIRST) { + p.error("expected a pattern"); + break; + } + + pattern(p); + if !p.at(ket) { + p.expect(T![,]); + } + } +} + +// test bind_pat +// fn main() { +// let a = (); +// let mut b = (); +// let ref c = (); +// let ref mut d = (); +// let e @ _ = (); +// let ref mut f @ g @ _ = (); +// } +fn bind_pat(p: &mut Parser, with_at: bool) -> CompletedMarker { + let m = p.start(); + p.eat(T![ref]); + p.eat(T![mut]); + name(p); + if with_at && p.eat(T![@]) { + pattern_single(p); + } + m.complete(p, IDENT_PAT) +} + +// test box_pat +// fn main() { +// let box i = (); +// let box Outer { box i, j: box Inner(box &x) } = (); +// let box ref mut i = (); +// } +fn box_pat(p: &mut Parser) -> CompletedMarker { + assert!(p.at(T![box])); + let m = p.start(); + p.bump(T![box]); + pattern_single(p); + m.complete(p, BOX_PAT) +} diff --git a/crates/parser/src/grammar/type_args.rs b/crates/parser/src/grammar/type_args.rs new file mode 100644 index 000000000..aef7cd6fb --- /dev/null +++ b/crates/parser/src/grammar/type_args.rs @@ -0,0 +1,63 @@ +//! FIXME: write short doc here + +use super::*; + +pub(super) fn opt_type_arg_list(p: &mut Parser, colon_colon_required: bool) { + let m; + if p.at(T![::]) && p.nth(2) == T![<] { + m = p.start(); + p.bump(T![::]); + p.bump(T![<]); + } else if !colon_colon_required && p.at(T![<]) && p.nth(1) != T![=] { + m = p.start(); + p.bump(T![<]); + } else { + return; + } + + while !p.at(EOF) && !p.at(T![>]) { + type_arg(p); + if !p.at(T![>]) && !p.expect(T![,]) { + break; + } + } + p.expect(T![>]); + m.complete(p, GENERIC_ARG_LIST); +} + +// test type_arg +// type A = B<'static, i32, 1, { 2 }, Item=u64>; +fn type_arg(p: &mut Parser) { + let m = p.start(); + match p.current() { + LIFETIME => { + p.bump(LIFETIME); + m.complete(p, LIFETIME_ARG); + } + // test associated_type_bounds + // fn print_all>(printables: T) {} + IDENT if p.nth(1) == T![:] && p.nth(2) != T![:] => { + name_ref(p); + type_params::bounds(p); + m.complete(p, ASSOC_TYPE_ARG); + } + IDENT if p.nth(1) == T![=] => { + name_ref(p); + p.bump_any(); + types::type_(p); + m.complete(p, ASSOC_TYPE_ARG); + } + T!['{'] => { + expressions::block_expr(p); + m.complete(p, CONST_ARG); + } + k if k.is_literal() => { + expressions::literal(p); + m.complete(p, CONST_ARG); + } + _ => { + types::type_(p); + m.complete(p, TYPE_ARG); + } + } +} diff --git a/crates/parser/src/grammar/type_params.rs b/crates/parser/src/grammar/type_params.rs new file mode 100644 index 000000000..90dabb4c0 --- /dev/null +++ b/crates/parser/src/grammar/type_params.rs @@ -0,0 +1,209 @@ +//! FIXME: write short doc here + +use super::*; + +pub(super) fn opt_type_param_list(p: &mut Parser) { + if !p.at(T![<]) { + return; + } + type_param_list(p); +} + +fn type_param_list(p: &mut Parser) { + assert!(p.at(T![<])); + let m = p.start(); + p.bump(T![<]); + + while !p.at(EOF) && !p.at(T![>]) { + let m = p.start(); + + // test generic_lifetime_type_attribute + // fn foo<#[derive(Lifetime)] 'a, #[derive(Type)] T>(_: &'a T) { + // } + attributes::outer_attributes(p); + + match p.current() { + LIFETIME => lifetime_param(p, m), + IDENT => type_param(p, m), + CONST_KW => type_const_param(p, m), + _ => { + m.abandon(p); + p.err_and_bump("expected type parameter") + } + } + if !p.at(T![>]) && !p.expect(T![,]) { + break; + } + } + p.expect(T![>]); + m.complete(p, GENERIC_PARAM_LIST); +} + +fn lifetime_param(p: &mut Parser, m: Marker) { + assert!(p.at(LIFETIME)); + p.bump(LIFETIME); + if p.at(T![:]) { + lifetime_bounds(p); + } + m.complete(p, LIFETIME_PARAM); +} + +fn type_param(p: &mut Parser, m: Marker) { + assert!(p.at(IDENT)); + name(p); + if p.at(T![:]) { + bounds(p); + } + // test type_param_default + // struct S; + if p.at(T![=]) { + p.bump(T![=]); + types::type_(p) + } + m.complete(p, TYPE_PARAM); +} + +// test const_param +// struct S; +fn type_const_param(p: &mut Parser, m: Marker) { + assert!(p.at(CONST_KW)); + p.bump(T![const]); + name(p); + types::ascription(p); + m.complete(p, CONST_PARAM); +} + +// test type_param_bounds +// struct S; +pub(super) fn bounds(p: &mut Parser) { + assert!(p.at(T![:])); + p.bump(T![:]); + bounds_without_colon(p); +} + +fn lifetime_bounds(p: &mut Parser) { + assert!(p.at(T![:])); + p.bump(T![:]); + while p.at(LIFETIME) { + p.bump(LIFETIME); + if !p.eat(T![+]) { + break; + } + } +} + +pub(super) fn bounds_without_colon_m(p: &mut Parser, marker: Marker) -> CompletedMarker { + while type_bound(p) { + if !p.eat(T![+]) { + break; + } + } + + marker.complete(p, TYPE_BOUND_LIST) +} + +pub(super) fn bounds_without_colon(p: &mut Parser) { + let m = p.start(); + bounds_without_colon_m(p, m); +} + +fn type_bound(p: &mut Parser) -> bool { + let m = p.start(); + let has_paren = p.eat(T!['(']); + p.eat(T![?]); + match p.current() { + LIFETIME => p.bump(LIFETIME), + T![for] => types::for_type(p), + _ if paths::is_use_path_start(p) => types::path_type_(p, false), + _ => { + m.abandon(p); + return false; + } + } + if has_paren { + p.expect(T![')']); + } + m.complete(p, TYPE_BOUND); + + true +} + +// test where_clause +// fn foo() +// where +// 'a: 'b + 'c, +// T: Clone + Copy + 'static, +// Iterator::Item: 'a, +// ::Item: 'a +// {} +pub(super) fn opt_where_clause(p: &mut Parser) { + if !p.at(T![where]) { + return; + } + let m = p.start(); + p.bump(T![where]); + + while is_where_predicate(p) { + where_predicate(p); + + let comma = p.eat(T![,]); + + if is_where_clause_end(p) { + break; + } + + if !comma { + p.error("expected comma"); + } + } + + m.complete(p, WHERE_CLAUSE); +} + +fn is_where_predicate(p: &mut Parser) -> bool { + match p.current() { + LIFETIME => true, + T![impl] => false, + token => types::TYPE_FIRST.contains(token), + } +} + +fn is_where_clause_end(p: &mut Parser) -> bool { + matches!(p.current(), T!['{'] | T![;] | T![=]) +} + +fn where_predicate(p: &mut Parser) { + let m = p.start(); + match p.current() { + LIFETIME => { + p.bump(LIFETIME); + if p.at(T![:]) { + bounds(p); + } else { + p.error("expected colon"); + } + } + T![impl] => { + p.error("expected lifetime or type"); + } + _ => { + // test where_pred_for + // fn for_trait() + // where + // for<'a> F: Fn(&'a str) + // { } + if p.at(T![for]) { + types::for_binder(p); + } + + types::type_(p); + + if p.at(T![:]) { + bounds(p); + } else { + p.error("expected colon"); + } + } + } + m.complete(p, WHERE_PRED); +} diff --git a/crates/parser/src/grammar/types.rs b/crates/parser/src/grammar/types.rs new file mode 100644 index 000000000..0aa173a52 --- /dev/null +++ b/crates/parser/src/grammar/types.rs @@ -0,0 +1,324 @@ +//! FIXME: write short doc here + +use super::*; + +pub(super) const TYPE_FIRST: TokenSet = paths::PATH_FIRST.union(token_set![ + T!['('], + T!['['], + T![<], + T![!], + T![*], + T![&], + T![_], + T![fn], + T![unsafe], + T![extern], + T![for], + T![impl], + T![dyn], +]); + +const TYPE_RECOVERY_SET: TokenSet = token_set![R_PAREN, COMMA, L_DOLLAR]; + +pub(crate) fn type_(p: &mut Parser) { + type_with_bounds_cond(p, true); +} + +pub(super) fn type_no_bounds(p: &mut Parser) { + type_with_bounds_cond(p, false); +} + +fn type_with_bounds_cond(p: &mut Parser, allow_bounds: bool) { + match p.current() { + T!['('] => paren_or_tuple_type(p), + T![!] => never_type(p), + T![*] => pointer_type(p), + T!['['] => array_or_slice_type(p), + T![&] => reference_type(p), + T![_] => placeholder_type(p), + T![fn] | T![unsafe] | T![extern] => fn_pointer_type(p), + T![for] => for_type(p), + T![impl] => impl_trait_type(p), + T![dyn] => dyn_trait_type(p), + // Some path types are not allowed to have bounds (no plus) + T![<] => path_type_(p, allow_bounds), + _ if paths::is_use_path_start(p) => path_or_macro_type_(p, allow_bounds), + _ => { + p.err_recover("expected type", TYPE_RECOVERY_SET); + } + } +} + +pub(super) fn ascription(p: &mut Parser) { + p.expect(T![:]); + type_(p) +} + +fn paren_or_tuple_type(p: &mut Parser) { + assert!(p.at(T!['('])); + let m = p.start(); + p.bump(T!['(']); + let mut n_types: u32 = 0; + let mut trailing_comma: bool = false; + while !p.at(EOF) && !p.at(T![')']) { + n_types += 1; + type_(p); + if p.eat(T![,]) { + trailing_comma = true; + } else { + trailing_comma = false; + break; + } + } + p.expect(T![')']); + + let kind = if n_types == 1 && !trailing_comma { + // test paren_type + // type T = (i32); + PAREN_TYPE + } else { + // test unit_type + // type T = (); + + // test singleton_tuple_type + // type T = (i32,); + TUPLE_TYPE + }; + m.complete(p, kind); +} + +// test never_type +// type Never = !; +fn never_type(p: &mut Parser) { + assert!(p.at(T![!])); + let m = p.start(); + p.bump(T![!]); + m.complete(p, NEVER_TYPE); +} + +fn pointer_type(p: &mut Parser) { + assert!(p.at(T![*])); + let m = p.start(); + p.bump(T![*]); + + match p.current() { + // test pointer_type_mut + // type M = *mut (); + // type C = *mut (); + T![mut] | T![const] => p.bump_any(), + _ => { + // test_err pointer_type_no_mutability + // type T = *(); + p.error( + "expected mut or const in raw pointer type \ + (use `*mut T` or `*const T` as appropriate)", + ); + } + }; + + type_no_bounds(p); + m.complete(p, PTR_TYPE); +} + +fn array_or_slice_type(p: &mut Parser) { + assert!(p.at(T!['['])); + let m = p.start(); + p.bump(T!['[']); + + type_(p); + let kind = match p.current() { + // test slice_type + // type T = [()]; + T![']'] => { + p.bump(T![']']); + SLICE_TYPE + } + + // test array_type + // type T = [(); 92]; + T![;] => { + p.bump(T![;]); + expressions::expr(p); + p.expect(T![']']); + ARRAY_TYPE + } + // test_err array_type_missing_semi + // type T = [() 92]; + _ => { + p.error("expected `;` or `]`"); + SLICE_TYPE + } + }; + m.complete(p, kind); +} + +// test reference_type; +// type A = &(); +// type B = &'static (); +// type C = &mut (); +fn reference_type(p: &mut Parser) { + assert!(p.at(T![&])); + let m = p.start(); + p.bump(T![&]); + p.eat(LIFETIME); + p.eat(T![mut]); + type_no_bounds(p); + m.complete(p, REF_TYPE); +} + +// test placeholder_type +// type Placeholder = _; +fn placeholder_type(p: &mut Parser) { + assert!(p.at(T![_])); + let m = p.start(); + p.bump(T![_]); + m.complete(p, INFER_TYPE); +} + +// test fn_pointer_type +// type A = fn(); +// type B = unsafe fn(); +// type C = unsafe extern "C" fn(); +// type D = extern "C" fn ( u8 , ... ) -> u8; +fn fn_pointer_type(p: &mut Parser) { + let m = p.start(); + p.eat(T![unsafe]); + if p.at(T![extern]) { + abi(p); + } + // test_err fn_pointer_type_missing_fn + // type F = unsafe (); + if !p.eat(T![fn]) { + m.abandon(p); + p.error("expected `fn`"); + return; + } + if p.at(T!['(']) { + params::param_list_fn_ptr(p); + } else { + p.error("expected parameters") + } + // test fn_pointer_type_with_ret + // type F = fn() -> (); + opt_fn_ret_type(p); + m.complete(p, FN_PTR_TYPE); +} + +pub(super) fn for_binder(p: &mut Parser) { + assert!(p.at(T![for])); + p.bump(T![for]); + if p.at(T![<]) { + type_params::opt_type_param_list(p); + } else { + p.error("expected `<`"); + } +} + +// test for_type +// type A = for<'a> fn() -> (); +// type B = for<'a> unsafe extern "C" fn(&'a ()) -> (); +// type Obj = for<'a> PartialEq<&'a i32>; +pub(super) fn for_type(p: &mut Parser) { + assert!(p.at(T![for])); + let m = p.start(); + for_binder(p); + match p.current() { + T![fn] | T![unsafe] | T![extern] => {} + // OK: legacy trait object format + _ if paths::is_use_path_start(p) => {} + _ => { + p.error("expected a function pointer or path"); + } + } + type_no_bounds(p); + m.complete(p, FOR_TYPE); +} + +// test impl_trait_type +// type A = impl Iterator> + 'a; +fn impl_trait_type(p: &mut Parser) { + assert!(p.at(T![impl])); + let m = p.start(); + p.bump(T![impl]); + type_params::bounds_without_colon(p); + m.complete(p, IMPL_TRAIT_TYPE); +} + +// test dyn_trait_type +// type A = dyn Iterator> + 'a; +fn dyn_trait_type(p: &mut Parser) { + assert!(p.at(T![dyn])); + let m = p.start(); + p.bump(T![dyn]); + type_params::bounds_without_colon(p); + m.complete(p, DYN_TRAIT_TYPE); +} + +// test path_type +// type A = Foo; +// type B = ::Foo; +// type C = self::Foo; +// type D = super::Foo; +pub(super) fn path_type(p: &mut Parser) { + path_type_(p, true) +} + +// test macro_call_type +// type A = foo!(); +// type B = crate::foo!(); +fn path_or_macro_type_(p: &mut Parser, allow_bounds: bool) { + assert!(paths::is_path_start(p)); + let m = p.start(); + paths::type_path(p); + + let kind = if p.at(T![!]) && !p.at(T![!=]) { + items::macro_call_after_excl(p); + MACRO_CALL + } else { + PATH_TYPE + }; + + let path = m.complete(p, kind); + + if allow_bounds { + opt_path_type_bounds_as_dyn_trait_type(p, path); + } +} + +pub(super) fn path_type_(p: &mut Parser, allow_bounds: bool) { + assert!(paths::is_path_start(p)); + let m = p.start(); + paths::type_path(p); + + // test path_type_with_bounds + // fn foo() -> Box {} + // fn foo() -> Box {} + let path = m.complete(p, PATH_TYPE); + if allow_bounds { + opt_path_type_bounds_as_dyn_trait_type(p, path); + } +} + +/// This turns a parsed PATH_TYPE optionally into a DYN_TRAIT_TYPE +/// with a TYPE_BOUND_LIST +fn opt_path_type_bounds_as_dyn_trait_type(p: &mut Parser, path_type_marker: CompletedMarker) { + if !p.at(T![+]) { + return; + } + + // First create a TYPE_BOUND from the completed PATH_TYPE + let m = path_type_marker.precede(p).complete(p, TYPE_BOUND); + + // Next setup a marker for the TYPE_BOUND_LIST + let m = m.precede(p); + + // This gets consumed here so it gets properly set + // in the TYPE_BOUND_LIST + p.eat(T![+]); + + // Parse rest of the bounds into the TYPE_BOUND_LIST + let m = type_params::bounds_without_colon_m(p, m); + + // Finally precede everything with DYN_TRAIT_TYPE + m.precede(p).complete(p, DYN_TRAIT_TYPE); +} diff --git a/crates/parser/src/lib.rs b/crates/parser/src/lib.rs new file mode 100644 index 000000000..eeb8ad66b --- /dev/null +++ b/crates/parser/src/lib.rs @@ -0,0 +1,149 @@ +//! The Rust parser. +//! +//! The parser doesn't know about concrete representation of tokens and syntax +//! trees. Abstract `TokenSource` and `TreeSink` traits are used instead. As a +//! consequence, this crates does not contain a lexer. +//! +//! The `Parser` struct from the `parser` module is a cursor into the sequence +//! of tokens. Parsing routines use `Parser` to inspect current state and +//! advance the parsing. +//! +//! The actual parsing happens in the `grammar` module. +//! +//! Tests for this crate live in `ra_syntax` crate. + +#[macro_use] +mod token_set; +#[macro_use] +mod syntax_kind; +mod event; +mod parser; +mod grammar; + +pub(crate) use token_set::TokenSet; + +pub use syntax_kind::SyntaxKind; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ParseError(pub Box); + +/// `TokenSource` abstracts the source of the tokens parser operates on. +/// +/// Hopefully this will allow us to treat text and token trees in the same way! +pub trait TokenSource { + fn current(&self) -> Token; + + /// Lookahead n token + fn lookahead_nth(&self, n: usize) -> Token; + + /// bump cursor to next token + fn bump(&mut self); + + /// Is the current token a specified keyword? + fn is_keyword(&self, kw: &str) -> bool; +} + +/// `Token` abstracts the cursor of `TokenSource` operates on. +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +pub struct Token { + /// What is the current token? + pub kind: SyntaxKind, + + /// Is the current token joined to the next one (`> >` vs `>>`). + pub is_jointed_to_next: bool, +} + +/// `TreeSink` abstracts details of a particular syntax tree implementation. +pub trait TreeSink { + /// Adds new token to the current branch. + fn token(&mut self, kind: SyntaxKind, n_tokens: u8); + + /// Start new branch and make it current. + fn start_node(&mut self, kind: SyntaxKind); + + /// Finish current branch and restore previous + /// branch as current. + fn finish_node(&mut self); + + fn error(&mut self, error: ParseError); +} + +fn parse_from_tokens(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink, f: F) +where + F: FnOnce(&mut parser::Parser), +{ + let mut p = parser::Parser::new(token_source); + f(&mut p); + let events = p.finish(); + event::process(tree_sink, events); +} + +/// Parse given tokens into the given sink as a rust file. +pub fn parse(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) { + parse_from_tokens(token_source, tree_sink, grammar::root); +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +pub enum FragmentKind { + Path, + Expr, + Statement, + Type, + Pattern, + Item, + Block, + Visibility, + MetaItem, + + // These kinds are used when parsing the result of expansion + // FIXME: use separate fragment kinds for macro inputs and outputs? + Items, + Statements, +} + +pub fn parse_fragment( + token_source: &mut dyn TokenSource, + tree_sink: &mut dyn TreeSink, + fragment_kind: FragmentKind, +) { + let parser: fn(&'_ mut parser::Parser) = match fragment_kind { + FragmentKind::Path => grammar::fragments::path, + FragmentKind::Expr => grammar::fragments::expr, + FragmentKind::Type => grammar::fragments::type_, + FragmentKind::Pattern => grammar::fragments::pattern, + FragmentKind::Item => grammar::fragments::item, + FragmentKind::Block => grammar::fragments::block_expr, + FragmentKind::Visibility => grammar::fragments::opt_visibility, + FragmentKind::MetaItem => grammar::fragments::meta_item, + FragmentKind::Statement => grammar::fragments::stmt, + FragmentKind::Items => grammar::fragments::macro_items, + FragmentKind::Statements => grammar::fragments::macro_stmts, + }; + parse_from_tokens(token_source, tree_sink, parser) +} + +/// A parsing function for a specific braced-block. +pub struct Reparser(fn(&mut parser::Parser)); + +impl Reparser { + /// If the node is a braced block, return the corresponding `Reparser`. + pub fn for_node( + node: SyntaxKind, + first_child: Option, + parent: Option, + ) -> Option { + grammar::reparser(node, first_child, parent).map(Reparser) + } + + /// Re-parse given tokens using this `Reparser`. + /// + /// Tokens must start with `{`, end with `}` and form a valid brace + /// sequence. + pub fn parse(self, token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) { + let Reparser(r) = self; + let mut p = parser::Parser::new(token_source); + r(&mut p); + let events = p.finish(); + event::process(tree_sink, events); + } +} diff --git a/crates/parser/src/parser.rs b/crates/parser/src/parser.rs new file mode 100644 index 000000000..d2487acc3 --- /dev/null +++ b/crates/parser/src/parser.rs @@ -0,0 +1,350 @@ +//! FIXME: write short doc here + +use std::cell::Cell; + +use drop_bomb::DropBomb; + +use crate::{ + event::Event, + ParseError, + SyntaxKind::{self, EOF, ERROR, TOMBSTONE}, + TokenSet, TokenSource, T, +}; + +/// `Parser` struct provides the low-level API for +/// navigating through the stream of tokens and +/// constructing the parse tree. The actual parsing +/// happens in the `grammar` module. +/// +/// However, the result of this `Parser` is not a real +/// tree, but rather a flat stream of events of the form +/// "start expression, consume number literal, +/// finish expression". See `Event` docs for more. +pub(crate) struct Parser<'t> { + token_source: &'t mut dyn TokenSource, + events: Vec, + steps: Cell, +} + +impl<'t> Parser<'t> { + pub(super) fn new(token_source: &'t mut dyn TokenSource) -> Parser<'t> { + Parser { token_source, events: Vec::new(), steps: Cell::new(0) } + } + + pub(crate) fn finish(self) -> Vec { + self.events + } + + /// Returns the kind of the current token. + /// If parser has already reached the end of input, + /// the special `EOF` kind is returned. + pub(crate) fn current(&self) -> SyntaxKind { + self.nth(0) + } + + /// Lookahead operation: returns the kind of the next nth + /// token. + pub(crate) fn nth(&self, n: usize) -> SyntaxKind { + assert!(n <= 3); + + let steps = self.steps.get(); + assert!(steps <= 10_000_000, "the parser seems stuck"); + self.steps.set(steps + 1); + + self.token_source.lookahead_nth(n).kind + } + + /// Checks if the current token is `kind`. + pub(crate) fn at(&self, kind: SyntaxKind) -> bool { + self.nth_at(0, kind) + } + + pub(crate) fn nth_at(&self, n: usize, kind: SyntaxKind) -> bool { + match kind { + T![-=] => self.at_composite2(n, T![-], T![=]), + T![->] => self.at_composite2(n, T![-], T![>]), + T![::] => self.at_composite2(n, T![:], T![:]), + T![!=] => self.at_composite2(n, T![!], T![=]), + T![..] => self.at_composite2(n, T![.], T![.]), + T![*=] => self.at_composite2(n, T![*], T![=]), + T![/=] => self.at_composite2(n, T![/], T![=]), + T![&&] => self.at_composite2(n, T![&], T![&]), + T![&=] => self.at_composite2(n, T![&], T![=]), + T![%=] => self.at_composite2(n, T![%], T![=]), + T![^=] => self.at_composite2(n, T![^], T![=]), + T![+=] => self.at_composite2(n, T![+], T![=]), + T![<<] => self.at_composite2(n, T![<], T![<]), + T![<=] => self.at_composite2(n, T![<], T![=]), + T![==] => self.at_composite2(n, T![=], T![=]), + T![=>] => self.at_composite2(n, T![=], T![>]), + T![>=] => self.at_composite2(n, T![>], T![=]), + T![>>] => self.at_composite2(n, T![>], T![>]), + T![|=] => self.at_composite2(n, T![|], T![=]), + T![||] => self.at_composite2(n, T![|], T![|]), + + T![...] => self.at_composite3(n, T![.], T![.], T![.]), + T![..=] => self.at_composite3(n, T![.], T![.], T![=]), + T![<<=] => self.at_composite3(n, T![<], T![<], T![=]), + T![>>=] => self.at_composite3(n, T![>], T![>], T![=]), + + _ => self.token_source.lookahead_nth(n).kind == kind, + } + } + + /// Consume the next token if `kind` matches. + pub(crate) fn eat(&mut self, kind: SyntaxKind) -> bool { + if !self.at(kind) { + return false; + } + let n_raw_tokens = match kind { + T![-=] + | T![->] + | T![::] + | T![!=] + | T![..] + | T![*=] + | T![/=] + | T![&&] + | T![&=] + | T![%=] + | T![^=] + | T![+=] + | T![<<] + | T![<=] + | T![==] + | T![=>] + | T![>=] + | T![>>] + | T![|=] + | T![||] => 2, + + T![...] | T![..=] | T![<<=] | T![>>=] => 3, + _ => 1, + }; + self.do_bump(kind, n_raw_tokens); + true + } + + fn at_composite2(&self, n: usize, k1: SyntaxKind, k2: SyntaxKind) -> bool { + let t1 = self.token_source.lookahead_nth(n); + if t1.kind != k1 || !t1.is_jointed_to_next { + return false; + } + let t2 = self.token_source.lookahead_nth(n + 1); + t2.kind == k2 + } + + fn at_composite3(&self, n: usize, k1: SyntaxKind, k2: SyntaxKind, k3: SyntaxKind) -> bool { + let t1 = self.token_source.lookahead_nth(n); + if t1.kind != k1 || !t1.is_jointed_to_next { + return false; + } + let t2 = self.token_source.lookahead_nth(n + 1); + if t2.kind != k2 || !t2.is_jointed_to_next { + return false; + } + let t3 = self.token_source.lookahead_nth(n + 2); + t3.kind == k3 + } + + /// Checks if the current token is in `kinds`. + pub(crate) fn at_ts(&self, kinds: TokenSet) -> bool { + kinds.contains(self.current()) + } + + /// Checks if the current token is contextual keyword with text `t`. + pub(crate) fn at_contextual_kw(&self, kw: &str) -> bool { + self.token_source.is_keyword(kw) + } + + /// Starts a new node in the syntax tree. All nodes and tokens + /// consumed between the `start` and the corresponding `Marker::complete` + /// belong to the same node. + pub(crate) fn start(&mut self) -> Marker { + let pos = self.events.len() as u32; + self.push_event(Event::tombstone()); + Marker::new(pos) + } + + /// Consume the next token if `kind` matches. + pub(crate) fn bump(&mut self, kind: SyntaxKind) { + assert!(self.eat(kind)); + } + + /// Advances the parser by one token + pub(crate) fn bump_any(&mut self) { + let kind = self.nth(0); + if kind == EOF { + return; + } + self.do_bump(kind, 1) + } + + /// Advances the parser by one token, remapping its kind. + /// This is useful to create contextual keywords from + /// identifiers. For example, the lexer creates an `union` + /// *identifier* token, but the parser remaps it to the + /// `union` keyword, and keyword is what ends up in the + /// final tree. + pub(crate) fn bump_remap(&mut self, kind: SyntaxKind) { + if self.nth(0) == EOF { + // FIXME: panic!? + return; + } + self.do_bump(kind, 1); + } + + /// Emit error with the `message` + /// FIXME: this should be much more fancy and support + /// structured errors with spans and notes, like rustc + /// does. + pub(crate) fn error>(&mut self, message: T) { + let msg = ParseError(Box::new(message.into())); + self.push_event(Event::Error { msg }) + } + + /// Consume the next token if it is `kind` or emit an error + /// otherwise. + pub(crate) fn expect(&mut self, kind: SyntaxKind) -> bool { + if self.eat(kind) { + return true; + } + self.error(format!("expected {:?}", kind)); + false + } + + /// Create an error node and consume the next token. + pub(crate) fn err_and_bump(&mut self, message: &str) { + self.err_recover(message, TokenSet::EMPTY); + } + + /// Create an error node and consume the next token. + pub(crate) fn err_recover(&mut self, message: &str, recovery: TokenSet) { + match self.current() { + T!['{'] | T!['}'] => { + self.error(message); + return; + } + _ => (), + } + + if self.at_ts(recovery) { + self.error(message); + return; + } + + let m = self.start(); + self.error(message); + self.bump_any(); + m.complete(self, ERROR); + } + + fn do_bump(&mut self, kind: SyntaxKind, n_raw_tokens: u8) { + for _ in 0..n_raw_tokens { + self.token_source.bump(); + } + + self.push_event(Event::Token { kind, n_raw_tokens }); + } + + fn push_event(&mut self, event: Event) { + self.events.push(event) + } +} + +/// See `Parser::start`. +pub(crate) struct Marker { + pos: u32, + bomb: DropBomb, +} + +impl Marker { + fn new(pos: u32) -> Marker { + Marker { pos, bomb: DropBomb::new("Marker must be either completed or abandoned") } + } + + /// Finishes the syntax tree node and assigns `kind` to it, + /// and mark the create a `CompletedMarker` for possible future + /// operation like `.precede()` to deal with forward_parent. + pub(crate) fn complete(mut self, p: &mut Parser, kind: SyntaxKind) -> CompletedMarker { + self.bomb.defuse(); + let idx = self.pos as usize; + match &mut p.events[idx] { + Event::Start { kind: slot, .. } => { + *slot = kind; + } + _ => unreachable!(), + } + let finish_pos = p.events.len() as u32; + p.push_event(Event::Finish); + CompletedMarker::new(self.pos, finish_pos, kind) + } + + /// Abandons the syntax tree node. All its children + /// are attached to its parent instead. + pub(crate) fn abandon(mut self, p: &mut Parser) { + self.bomb.defuse(); + let idx = self.pos as usize; + if idx == p.events.len() - 1 { + match p.events.pop() { + Some(Event::Start { kind: TOMBSTONE, forward_parent: None }) => (), + _ => unreachable!(), + } + } + } +} + +pub(crate) struct CompletedMarker { + start_pos: u32, + finish_pos: u32, + kind: SyntaxKind, +} + +impl CompletedMarker { + fn new(start_pos: u32, finish_pos: u32, kind: SyntaxKind) -> Self { + CompletedMarker { start_pos, finish_pos, kind } + } + + /// This method allows to create a new node which starts + /// *before* the current one. That is, parser could start + /// node `A`, then complete it, and then after parsing the + /// whole `A`, decide that it should have started some node + /// `B` before starting `A`. `precede` allows to do exactly + /// that. See also docs about `forward_parent` in `Event::Start`. + /// + /// Given completed events `[START, FINISH]` and its corresponding + /// `CompletedMarker(pos: 0, _)`. + /// Append a new `START` events as `[START, FINISH, NEWSTART]`, + /// then mark `NEWSTART` as `START`'s parent with saving its relative + /// distance to `NEWSTART` into forward_parent(=2 in this case); + pub(crate) fn precede(self, p: &mut Parser) -> Marker { + let new_pos = p.start(); + let idx = self.start_pos as usize; + match &mut p.events[idx] { + Event::Start { forward_parent, .. } => { + *forward_parent = Some(new_pos.pos - self.start_pos); + } + _ => unreachable!(), + } + new_pos + } + + /// Undo this completion and turns into a `Marker` + pub(crate) fn undo_completion(self, p: &mut Parser) -> Marker { + let start_idx = self.start_pos as usize; + let finish_idx = self.finish_pos as usize; + match &mut p.events[start_idx] { + Event::Start { kind, forward_parent: None } => *kind = TOMBSTONE, + _ => unreachable!(), + } + match &mut p.events[finish_idx] { + slot @ Event::Finish => *slot = Event::tombstone(), + _ => unreachable!(), + } + Marker::new(self.start_pos) + } + + pub(crate) fn kind(&self) -> SyntaxKind { + self.kind + } +} diff --git a/crates/parser/src/syntax_kind.rs b/crates/parser/src/syntax_kind.rs new file mode 100644 index 000000000..63204436c --- /dev/null +++ b/crates/parser/src/syntax_kind.rs @@ -0,0 +1,25 @@ +//! FIXME: write short doc here + +#[macro_use] +mod generated; + +pub use self::generated::SyntaxKind; + +impl From for SyntaxKind { + fn from(d: u16) -> SyntaxKind { + assert!(d <= (SyntaxKind::__LAST as u16)); + unsafe { std::mem::transmute::(d) } + } +} + +impl From for u16 { + fn from(k: SyntaxKind) -> u16 { + k as u16 + } +} + +impl SyntaxKind { + pub fn is_trivia(self) -> bool { + matches!(self, SyntaxKind::WHITESPACE | SyntaxKind::COMMENT) + } +} diff --git a/crates/parser/src/syntax_kind/generated.rs b/crates/parser/src/syntax_kind/generated.rs new file mode 100644 index 000000000..192ecd864 --- /dev/null +++ b/crates/parser/src/syntax_kind/generated.rs @@ -0,0 +1,367 @@ +//! Generated file, do not edit by hand, see `xtask/src/codegen` + +#![allow(bad_style, missing_docs, unreachable_pub)] +#[doc = r" The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT`."] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +#[repr(u16)] +pub enum SyntaxKind { + #[doc(hidden)] + TOMBSTONE, + #[doc(hidden)] + EOF, + SEMICOLON, + COMMA, + L_PAREN, + R_PAREN, + L_CURLY, + R_CURLY, + L_BRACK, + R_BRACK, + L_ANGLE, + R_ANGLE, + AT, + POUND, + TILDE, + QUESTION, + DOLLAR, + AMP, + PIPE, + PLUS, + STAR, + SLASH, + CARET, + PERCENT, + UNDERSCORE, + DOT, + DOT2, + DOT3, + DOT2EQ, + COLON, + COLON2, + EQ, + EQ2, + FAT_ARROW, + BANG, + NEQ, + MINUS, + THIN_ARROW, + LTEQ, + GTEQ, + PLUSEQ, + MINUSEQ, + PIPEEQ, + AMPEQ, + CARETEQ, + SLASHEQ, + STAREQ, + PERCENTEQ, + AMP2, + PIPE2, + SHL, + SHR, + SHLEQ, + SHREQ, + AS_KW, + ASYNC_KW, + AWAIT_KW, + BOX_KW, + BREAK_KW, + CONST_KW, + CONTINUE_KW, + CRATE_KW, + DYN_KW, + ELSE_KW, + ENUM_KW, + EXTERN_KW, + FALSE_KW, + FN_KW, + FOR_KW, + IF_KW, + IMPL_KW, + IN_KW, + LET_KW, + LOOP_KW, + MACRO_KW, + MATCH_KW, + MOD_KW, + MOVE_KW, + MUT_KW, + PUB_KW, + REF_KW, + RETURN_KW, + SELF_KW, + STATIC_KW, + STRUCT_KW, + SUPER_KW, + TRAIT_KW, + TRUE_KW, + TRY_KW, + TYPE_KW, + UNSAFE_KW, + USE_KW, + WHERE_KW, + WHILE_KW, + AUTO_KW, + DEFAULT_KW, + EXISTENTIAL_KW, + UNION_KW, + RAW_KW, + INT_NUMBER, + FLOAT_NUMBER, + CHAR, + BYTE, + STRING, + RAW_STRING, + BYTE_STRING, + RAW_BYTE_STRING, + ERROR, + IDENT, + WHITESPACE, + LIFETIME, + COMMENT, + SHEBANG, + L_DOLLAR, + R_DOLLAR, + SOURCE_FILE, + STRUCT, + UNION, + ENUM, + FN, + RET_TYPE, + EXTERN_CRATE, + MODULE, + USE, + STATIC, + CONST, + TRAIT, + IMPL, + TYPE_ALIAS, + MACRO_CALL, + TOKEN_TREE, + MACRO_DEF, + PAREN_TYPE, + TUPLE_TYPE, + NEVER_TYPE, + PATH_TYPE, + PTR_TYPE, + ARRAY_TYPE, + SLICE_TYPE, + REF_TYPE, + INFER_TYPE, + FN_PTR_TYPE, + FOR_TYPE, + IMPL_TRAIT_TYPE, + DYN_TRAIT_TYPE, + OR_PAT, + PAREN_PAT, + REF_PAT, + BOX_PAT, + IDENT_PAT, + WILDCARD_PAT, + REST_PAT, + PATH_PAT, + RECORD_PAT, + RECORD_PAT_FIELD_LIST, + RECORD_PAT_FIELD, + TUPLE_STRUCT_PAT, + TUPLE_PAT, + SLICE_PAT, + RANGE_PAT, + LITERAL_PAT, + MACRO_PAT, + TUPLE_EXPR, + ARRAY_EXPR, + PAREN_EXPR, + PATH_EXPR, + CLOSURE_EXPR, + IF_EXPR, + WHILE_EXPR, + CONDITION, + LOOP_EXPR, + FOR_EXPR, + CONTINUE_EXPR, + BREAK_EXPR, + LABEL, + BLOCK_EXPR, + RETURN_EXPR, + MATCH_EXPR, + MATCH_ARM_LIST, + MATCH_ARM, + MATCH_GUARD, + RECORD_EXPR, + RECORD_EXPR_FIELD_LIST, + RECORD_EXPR_FIELD, + EFFECT_EXPR, + BOX_EXPR, + CALL_EXPR, + INDEX_EXPR, + METHOD_CALL_EXPR, + FIELD_EXPR, + AWAIT_EXPR, + TRY_EXPR, + CAST_EXPR, + REF_EXPR, + PREFIX_EXPR, + RANGE_EXPR, + BIN_EXPR, + EXTERN_BLOCK, + EXTERN_ITEM_LIST, + VARIANT, + RECORD_FIELD_LIST, + RECORD_FIELD, + TUPLE_FIELD_LIST, + TUPLE_FIELD, + VARIANT_LIST, + ITEM_LIST, + ASSOC_ITEM_LIST, + ATTR, + META_ITEM, + USE_TREE, + USE_TREE_LIST, + PATH, + PATH_SEGMENT, + LITERAL, + RENAME, + VISIBILITY, + WHERE_CLAUSE, + WHERE_PRED, + ABI, + NAME, + NAME_REF, + LET_STMT, + EXPR_STMT, + GENERIC_PARAM_LIST, + GENERIC_PARAM, + LIFETIME_PARAM, + TYPE_PARAM, + CONST_PARAM, + GENERIC_ARG_LIST, + LIFETIME_ARG, + TYPE_ARG, + ASSOC_TYPE_ARG, + CONST_ARG, + PARAM_LIST, + PARAM, + SELF_PARAM, + ARG_LIST, + TYPE_BOUND, + TYPE_BOUND_LIST, + MACRO_ITEMS, + MACRO_STMTS, + #[doc(hidden)] + __LAST, +} +use self::SyntaxKind::*; +impl SyntaxKind { + pub fn is_keyword(self) -> bool { + match self { + AS_KW | ASYNC_KW | AWAIT_KW | BOX_KW | BREAK_KW | CONST_KW | CONTINUE_KW | CRATE_KW + | DYN_KW | ELSE_KW | ENUM_KW | EXTERN_KW | FALSE_KW | FN_KW | FOR_KW | IF_KW + | IMPL_KW | IN_KW | LET_KW | LOOP_KW | MACRO_KW | MATCH_KW | MOD_KW | MOVE_KW + | MUT_KW | PUB_KW | REF_KW | RETURN_KW | SELF_KW | STATIC_KW | STRUCT_KW | SUPER_KW + | TRAIT_KW | TRUE_KW | TRY_KW | TYPE_KW | UNSAFE_KW | USE_KW | WHERE_KW | WHILE_KW + | AUTO_KW | DEFAULT_KW | EXISTENTIAL_KW | UNION_KW | RAW_KW => true, + _ => false, + } + } + pub fn is_punct(self) -> bool { + match self { + SEMICOLON | COMMA | L_PAREN | R_PAREN | L_CURLY | R_CURLY | L_BRACK | R_BRACK + | L_ANGLE | R_ANGLE | AT | POUND | TILDE | QUESTION | DOLLAR | AMP | PIPE | PLUS + | STAR | SLASH | CARET | PERCENT | UNDERSCORE | DOT | DOT2 | DOT3 | DOT2EQ | COLON + | COLON2 | EQ | EQ2 | FAT_ARROW | BANG | NEQ | MINUS | THIN_ARROW | LTEQ | GTEQ + | PLUSEQ | MINUSEQ | PIPEEQ | AMPEQ | CARETEQ | SLASHEQ | STAREQ | PERCENTEQ | AMP2 + | PIPE2 | SHL | SHR | SHLEQ | SHREQ => true, + _ => false, + } + } + pub fn is_literal(self) -> bool { + match self { + INT_NUMBER | FLOAT_NUMBER | CHAR | BYTE | STRING | RAW_STRING | BYTE_STRING + | RAW_BYTE_STRING => true, + _ => false, + } + } + pub fn from_keyword(ident: &str) -> Option { + let kw = match ident { + "as" => AS_KW, + "async" => ASYNC_KW, + "await" => AWAIT_KW, + "box" => BOX_KW, + "break" => BREAK_KW, + "const" => CONST_KW, + "continue" => CONTINUE_KW, + "crate" => CRATE_KW, + "dyn" => DYN_KW, + "else" => ELSE_KW, + "enum" => ENUM_KW, + "extern" => EXTERN_KW, + "false" => FALSE_KW, + "fn" => FN_KW, + "for" => FOR_KW, + "if" => IF_KW, + "impl" => IMPL_KW, + "in" => IN_KW, + "let" => LET_KW, + "loop" => LOOP_KW, + "macro" => MACRO_KW, + "match" => MATCH_KW, + "mod" => MOD_KW, + "move" => MOVE_KW, + "mut" => MUT_KW, + "pub" => PUB_KW, + "ref" => REF_KW, + "return" => RETURN_KW, + "self" => SELF_KW, + "static" => STATIC_KW, + "struct" => STRUCT_KW, + "super" => SUPER_KW, + "trait" => TRAIT_KW, + "true" => TRUE_KW, + "try" => TRY_KW, + "type" => TYPE_KW, + "unsafe" => UNSAFE_KW, + "use" => USE_KW, + "where" => WHERE_KW, + "while" => WHILE_KW, + _ => return None, + }; + Some(kw) + } + pub fn from_char(c: char) -> Option { + let tok = match c { + ';' => SEMICOLON, + ',' => COMMA, + '(' => L_PAREN, + ')' => R_PAREN, + '{' => L_CURLY, + '}' => R_CURLY, + '[' => L_BRACK, + ']' => R_BRACK, + '<' => L_ANGLE, + '>' => R_ANGLE, + '@' => AT, + '#' => POUND, + '~' => TILDE, + '?' => QUESTION, + '$' => DOLLAR, + '&' => AMP, + '|' => PIPE, + '+' => PLUS, + '*' => STAR, + '/' => SLASH, + '^' => CARET, + '%' => PERCENT, + '_' => UNDERSCORE, + '.' => DOT, + ':' => COLON, + '=' => EQ, + '!' => BANG, + '-' => MINUS, + _ => return None, + }; + Some(tok) + } +} +#[macro_export] +macro_rules ! T { [ ; ] => { $ crate :: SyntaxKind :: SEMICOLON } ; [ , ] => { $ crate :: SyntaxKind :: COMMA } ; [ '(' ] => { $ crate :: SyntaxKind :: L_PAREN } ; [ ')' ] => { $ crate :: SyntaxKind :: R_PAREN } ; [ '{' ] => { $ crate :: SyntaxKind :: L_CURLY } ; [ '}' ] => { $ crate :: SyntaxKind :: R_CURLY } ; [ '[' ] => { $ crate :: SyntaxKind :: L_BRACK } ; [ ']' ] => { $ crate :: SyntaxKind :: R_BRACK } ; [ < ] => { $ crate :: SyntaxKind :: L_ANGLE } ; [ > ] => { $ crate :: SyntaxKind :: R_ANGLE } ; [ @ ] => { $ crate :: SyntaxKind :: AT } ; [ # ] => { $ crate :: SyntaxKind :: POUND } ; [ ~ ] => { $ crate :: SyntaxKind :: TILDE } ; [ ? ] => { $ crate :: SyntaxKind :: QUESTION } ; [ $ ] => { $ crate :: SyntaxKind :: DOLLAR } ; [ & ] => { $ crate :: SyntaxKind :: AMP } ; [ | ] => { $ crate :: SyntaxKind :: PIPE } ; [ + ] => { $ crate :: SyntaxKind :: PLUS } ; [ * ] => { $ crate :: SyntaxKind :: STAR } ; [ / ] => { $ crate :: SyntaxKind :: SLASH } ; [ ^ ] => { $ crate :: SyntaxKind :: CARET } ; [ % ] => { $ crate :: SyntaxKind :: PERCENT } ; [ _ ] => { $ crate :: SyntaxKind :: UNDERSCORE } ; [ . ] => { $ crate :: SyntaxKind :: DOT } ; [ .. ] => { $ crate :: SyntaxKind :: DOT2 } ; [ ... ] => { $ crate :: SyntaxKind :: DOT3 } ; [ ..= ] => { $ crate :: SyntaxKind :: DOT2EQ } ; [ : ] => { $ crate :: SyntaxKind :: COLON } ; [ :: ] => { $ crate :: SyntaxKind :: COLON2 } ; [ = ] => { $ crate :: SyntaxKind :: EQ } ; [ == ] => { $ crate :: SyntaxKind :: EQ2 } ; [ => ] => { $ crate :: SyntaxKind :: FAT_ARROW } ; [ ! ] => { $ crate :: SyntaxKind :: BANG } ; [ != ] => { $ crate :: SyntaxKind :: NEQ } ; [ - ] => { $ crate :: SyntaxKind :: MINUS } ; [ -> ] => { $ crate :: SyntaxKind :: THIN_ARROW } ; [ <= ] => { $ crate :: SyntaxKind :: LTEQ } ; [ >= ] => { $ crate :: SyntaxKind :: GTEQ } ; [ += ] => { $ crate :: SyntaxKind :: PLUSEQ } ; [ -= ] => { $ crate :: SyntaxKind :: MINUSEQ } ; [ |= ] => { $ crate :: SyntaxKind :: PIPEEQ } ; [ &= ] => { $ crate :: SyntaxKind :: AMPEQ } ; [ ^= ] => { $ crate :: SyntaxKind :: CARETEQ } ; [ /= ] => { $ crate :: SyntaxKind :: SLASHEQ } ; [ *= ] => { $ crate :: SyntaxKind :: STAREQ } ; [ %= ] => { $ crate :: SyntaxKind :: PERCENTEQ } ; [ && ] => { $ crate :: SyntaxKind :: AMP2 } ; [ || ] => { $ crate :: SyntaxKind :: PIPE2 } ; [ << ] => { $ crate :: SyntaxKind :: SHL } ; [ >> ] => { $ crate :: SyntaxKind :: SHR } ; [ <<= ] => { $ crate :: SyntaxKind :: SHLEQ } ; [ >>= ] => { $ crate :: SyntaxKind :: SHREQ } ; [ as ] => { $ crate :: SyntaxKind :: AS_KW } ; [ async ] => { $ crate :: SyntaxKind :: ASYNC_KW } ; [ await ] => { $ crate :: SyntaxKind :: AWAIT_KW } ; [ box ] => { $ crate :: SyntaxKind :: BOX_KW } ; [ break ] => { $ crate :: SyntaxKind :: BREAK_KW } ; [ const ] => { $ crate :: SyntaxKind :: CONST_KW } ; [ continue ] => { $ crate :: SyntaxKind :: CONTINUE_KW } ; [ crate ] => { $ crate :: SyntaxKind :: CRATE_KW } ; [ dyn ] => { $ crate :: SyntaxKind :: DYN_KW } ; [ else ] => { $ crate :: SyntaxKind :: ELSE_KW } ; [ enum ] => { $ crate :: SyntaxKind :: ENUM_KW } ; [ extern ] => { $ crate :: SyntaxKind :: EXTERN_KW } ; [ false ] => { $ crate :: SyntaxKind :: FALSE_KW } ; [ fn ] => { $ crate :: SyntaxKind :: FN_KW } ; [ for ] => { $ crate :: SyntaxKind :: FOR_KW } ; [ if ] => { $ crate :: SyntaxKind :: IF_KW } ; [ impl ] => { $ crate :: SyntaxKind :: IMPL_KW } ; [ in ] => { $ crate :: SyntaxKind :: IN_KW } ; [ let ] => { $ crate :: SyntaxKind :: LET_KW } ; [ loop ] => { $ crate :: SyntaxKind :: LOOP_KW } ; [ macro ] => { $ crate :: SyntaxKind :: MACRO_KW } ; [ match ] => { $ crate :: SyntaxKind :: MATCH_KW } ; [ mod ] => { $ crate :: SyntaxKind :: MOD_KW } ; [ move ] => { $ crate :: SyntaxKind :: MOVE_KW } ; [ mut ] => { $ crate :: SyntaxKind :: MUT_KW } ; [ pub ] => { $ crate :: SyntaxKind :: PUB_KW } ; [ ref ] => { $ crate :: SyntaxKind :: REF_KW } ; [ return ] => { $ crate :: SyntaxKind :: RETURN_KW } ; [ self ] => { $ crate :: SyntaxKind :: SELF_KW } ; [ static ] => { $ crate :: SyntaxKind :: STATIC_KW } ; [ struct ] => { $ crate :: SyntaxKind :: STRUCT_KW } ; [ super ] => { $ crate :: SyntaxKind :: SUPER_KW } ; [ trait ] => { $ crate :: SyntaxKind :: TRAIT_KW } ; [ true ] => { $ crate :: SyntaxKind :: TRUE_KW } ; [ try ] => { $ crate :: SyntaxKind :: TRY_KW } ; [ type ] => { $ crate :: SyntaxKind :: TYPE_KW } ; [ unsafe ] => { $ crate :: SyntaxKind :: UNSAFE_KW } ; [ use ] => { $ crate :: SyntaxKind :: USE_KW } ; [ where ] => { $ crate :: SyntaxKind :: WHERE_KW } ; [ while ] => { $ crate :: SyntaxKind :: WHILE_KW } ; [ auto ] => { $ crate :: SyntaxKind :: AUTO_KW } ; [ default ] => { $ crate :: SyntaxKind :: DEFAULT_KW } ; [ existential ] => { $ crate :: SyntaxKind :: EXISTENTIAL_KW } ; [ union ] => { $ crate :: SyntaxKind :: UNION_KW } ; [ raw ] => { $ crate :: SyntaxKind :: RAW_KW } ; [ lifetime ] => { $ crate :: SyntaxKind :: LIFETIME } ; [ ident ] => { $ crate :: SyntaxKind :: IDENT } ; [ shebang ] => { $ crate :: SyntaxKind :: SHEBANG } ; } diff --git a/crates/parser/src/token_set.rs b/crates/parser/src/token_set.rs new file mode 100644 index 000000000..994017acf --- /dev/null +++ b/crates/parser/src/token_set.rs @@ -0,0 +1,42 @@ +//! A bit-set of `SyntaxKind`s. + +use crate::SyntaxKind; + +/// A bit-set of `SyntaxKind`s +#[derive(Clone, Copy)] +pub(crate) struct TokenSet(u128); + +impl TokenSet { + pub(crate) const EMPTY: TokenSet = TokenSet(0); + + pub(crate) const fn singleton(kind: SyntaxKind) -> TokenSet { + TokenSet(mask(kind)) + } + + pub(crate) const fn union(self, other: TokenSet) -> TokenSet { + TokenSet(self.0 | other.0) + } + + pub(crate) fn contains(&self, kind: SyntaxKind) -> bool { + self.0 & mask(kind) != 0 + } +} + +const fn mask(kind: SyntaxKind) -> u128 { + 1u128 << (kind as usize) +} + +#[macro_export] +macro_rules! token_set { + ($($t:expr),*) => { TokenSet::EMPTY$(.union(TokenSet::singleton($t)))* }; + ($($t:expr),* ,) => { token_set!($($t),*) }; +} + +#[test] +fn token_set_works_for_tokens() { + use crate::SyntaxKind::*; + let ts = token_set![EOF, SHEBANG]; + assert!(ts.contains(EOF)); + assert!(ts.contains(SHEBANG)); + assert!(!ts.contains(PLUS)); +} diff --git a/crates/ra_hir_expand/Cargo.toml b/crates/ra_hir_expand/Cargo.toml index 7d8ccd56f..052330fde 100644 --- a/crates/ra_hir_expand/Cargo.toml +++ b/crates/ra_hir_expand/Cargo.toml @@ -16,7 +16,7 @@ rustc-hash = "1.0.0" arena = { path = "../arena" } ra_db = { path = "../ra_db" } ra_syntax = { path = "../ra_syntax" } -ra_parser = { path = "../ra_parser" } +parser = { path = "../parser" } profile = { path = "../profile" } tt = { path = "../tt" } mbe = { path = "../ra_mbe", package = "ra_mbe" } diff --git a/crates/ra_hir_expand/src/builtin_derive.rs b/crates/ra_hir_expand/src/builtin_derive.rs index 69fa907cb..95e6977f2 100644 --- a/crates/ra_hir_expand/src/builtin_derive.rs +++ b/crates/ra_hir_expand/src/builtin_derive.rs @@ -2,7 +2,7 @@ use log::debug; -use ra_parser::FragmentKind; +use parser::FragmentKind; use ra_syntax::{ ast::{self, AstNode, GenericParamsOwner, ModuleItemOwner, NameOwner}, match_ast, diff --git a/crates/ra_hir_expand/src/builtin_macro.rs b/crates/ra_hir_expand/src/builtin_macro.rs index 9f50569dc..24dc0b4e7 100644 --- a/crates/ra_hir_expand/src/builtin_macro.rs +++ b/crates/ra_hir_expand/src/builtin_macro.rs @@ -6,8 +6,8 @@ use crate::{ use either::Either; use mbe::parse_to_token_tree; +use parser::FragmentKind; use ra_db::FileId; -use ra_parser::FragmentKind; use ra_syntax::ast::{self, AstToken, HasStringValue}; macro_rules! register_builtin { diff --git a/crates/ra_hir_expand/src/db.rs b/crates/ra_hir_expand/src/db.rs index f30528b3e..d83c391a9 100644 --- a/crates/ra_hir_expand/src/db.rs +++ b/crates/ra_hir_expand/src/db.rs @@ -3,8 +3,8 @@ use std::sync::Arc; use mbe::{ExpandResult, MacroRules}; +use parser::FragmentKind; use ra_db::{salsa, SourceDatabase}; -use ra_parser::FragmentKind; use ra_syntax::{algo::diff, AstNode, GreenNode, Parse, SyntaxKind::*, SyntaxNode}; use crate::{ diff --git a/crates/ra_hir_expand/src/eager.rs b/crates/ra_hir_expand/src/eager.rs index 302d2b3e0..dc83044ea 100644 --- a/crates/ra_hir_expand/src/eager.rs +++ b/crates/ra_hir_expand/src/eager.rs @@ -25,8 +25,8 @@ use crate::{ EagerCallLoc, EagerMacroId, InFile, MacroCallId, MacroCallKind, MacroDefId, MacroDefKind, }; +use parser::FragmentKind; use ra_db::CrateId; -use ra_parser::FragmentKind; use ra_syntax::{algo::SyntaxRewriter, SyntaxNode}; use std::sync::Arc; diff --git a/crates/ra_hir_expand/src/lib.rs b/crates/ra_hir_expand/src/lib.rs index 8bb735fc6..38f0ffff8 100644 --- a/crates/ra_hir_expand/src/lib.rs +++ b/crates/ra_hir_expand/src/lib.rs @@ -317,7 +317,7 @@ pub struct ExpansionInfo { } pub use mbe::Origin; -use ra_parser::FragmentKind; +use parser::FragmentKind; impl ExpansionInfo { pub fn call_node(&self) -> Option> { diff --git a/crates/ra_mbe/Cargo.toml b/crates/ra_mbe/Cargo.toml index 23315910c..e518f73e3 100644 --- a/crates/ra_mbe/Cargo.toml +++ b/crates/ra_mbe/Cargo.toml @@ -10,7 +10,7 @@ doctest = false [dependencies] ra_syntax = { path = "../ra_syntax" } -ra_parser = { path = "../ra_parser" } +parser = { path = "../parser" } tt = { path = "../tt" } rustc-hash = "1.1.0" smallvec = "1.2.0" diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs index 933a3a3b5..c752804b2 100644 --- a/crates/ra_mbe/src/mbe_expander/matcher.rs +++ b/crates/ra_mbe/src/mbe_expander/matcher.rs @@ -9,7 +9,7 @@ use crate::{ }; use super::ExpandResult; -use ra_parser::{FragmentKind::*, TreeSink}; +use parser::{FragmentKind::*, TreeSink}; use ra_syntax::{SmolStr, SyntaxKind}; use tt::buffer::{Cursor, TokenBuffer}; @@ -285,7 +285,7 @@ impl<'a> TtIter<'a> { pub(crate) fn expect_fragment( &mut self, - fragment_kind: ra_parser::FragmentKind, + fragment_kind: parser::FragmentKind, ) -> ExpandResult> { pub(crate) struct OffsetTokenSink<'a> { pub(crate) cursor: Cursor<'a>, @@ -303,7 +303,7 @@ impl<'a> TtIter<'a> { } fn start_node(&mut self, _kind: SyntaxKind) {} fn finish_node(&mut self) {} - fn error(&mut self, _error: ra_parser::ParseError) { + fn error(&mut self, _error: parser::ParseError) { self.error = true; } } @@ -312,7 +312,7 @@ impl<'a> TtIter<'a> { let mut src = SubtreeTokenSource::new(&buffer); let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false }; - ra_parser::parse_fragment(&mut src, &mut sink, fragment_kind); + parser::parse_fragment(&mut src, &mut sink, fragment_kind); let mut err = None; if !sink.cursor.is_root() || sink.error { diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs index d7866452d..1a1cb08cf 100644 --- a/crates/ra_mbe/src/subtree_source.rs +++ b/crates/ra_mbe/src/subtree_source.rs @@ -1,6 +1,6 @@ //! FIXME: write short doc here -use ra_parser::{Token, TokenSource}; +use parser::{Token, TokenSource}; use ra_syntax::{lex_single_syntax_kind, SmolStr, SyntaxKind, SyntaxKind::*, T}; use std::cell::{Cell, Ref, RefCell}; use tt::buffer::{Cursor, TokenBuffer}; diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 5fc48507f..7b9c88ae6 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -1,6 +1,6 @@ //! FIXME: write short doc here -use ra_parser::{FragmentKind, ParseError, TreeSink}; +use parser::{FragmentKind, ParseError, TreeSink}; use ra_syntax::{ ast::{self, make::tokens::doc_comment}, tokenize, AstToken, Parse, SmolStr, SyntaxKind, @@ -81,7 +81,7 @@ pub fn token_tree_to_syntax_node( let buffer = TokenBuffer::new(&tokens); let mut token_source = SubtreeTokenSource::new(&buffer); let mut tree_sink = TtTreeSink::new(buffer.begin()); - ra_parser::parse_fragment(&mut token_source, &mut tree_sink, fragment_kind); + parser::parse_fragment(&mut token_source, &mut tree_sink, fragment_kind); if tree_sink.roots.len() != 1 { return Err(ExpandError::ConversionError); } @@ -715,7 +715,7 @@ impl<'a> TreeSink for TtTreeSink<'a> { mod tests { use super::*; use crate::tests::parse_macro; - use ra_parser::TokenSource; + use parser::TokenSource; use ra_syntax::{ algo::{insert_children, InsertPosition}, ast::AstNode, diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs index 286983d60..be39b0e45 100644 --- a/crates/ra_mbe/src/tests.rs +++ b/crates/ra_mbe/src/tests.rs @@ -1,6 +1,6 @@ use std::fmt::Write; -use ra_parser::FragmentKind; +use ::parser::FragmentKind; use ra_syntax::{ast, AstNode, NodeOrToken, SyntaxKind::IDENT, SyntaxNode, WalkEvent, T}; use test_utils::assert_eq_text; @@ -9,9 +9,10 @@ use super::*; mod rule_parsing { use ra_syntax::{ast, AstNode}; - use super::*; use crate::ast_to_token_tree; + use super::*; + #[test] fn test_valid_arms() { fn check(macro_body: &str) { diff --git a/crates/ra_parser/Cargo.toml b/crates/ra_parser/Cargo.toml deleted file mode 100644 index 72ec3e4d9..000000000 --- a/crates/ra_parser/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -edition = "2018" -name = "ra_parser" -version = "0.1.0" -authors = ["rust-analyzer developers"] -publish = false -license = "MIT OR Apache-2.0" - -[lib] -doctest = false - -[dependencies] -drop_bomb = "0.1.4" diff --git a/crates/ra_parser/src/event.rs b/crates/ra_parser/src/event.rs deleted file mode 100644 index a7d06a815..000000000 --- a/crates/ra_parser/src/event.rs +++ /dev/null @@ -1,130 +0,0 @@ -//! This module provides a way to construct a `File`. -//! It is intended to be completely decoupled from the -//! parser, so as to allow to evolve the tree representation -//! and the parser algorithm independently. -//! -//! The `TreeSink` trait is the bridge between the parser and the -//! tree builder: the parser produces a stream of events like -//! `start node`, `finish node`, and `FileBuilder` converts -//! this stream to a real tree. -use std::mem; - -use crate::{ - ParseError, - SyntaxKind::{self, *}, - TreeSink, -}; - -/// `Parser` produces a flat list of `Event`s. -/// They are converted to a tree-structure in -/// a separate pass, via `TreeBuilder`. -#[derive(Debug)] -pub(crate) enum Event { - /// This event signifies the start of the node. - /// It should be either abandoned (in which case the - /// `kind` is `TOMBSTONE`, and the event is ignored), - /// or completed via a `Finish` event. - /// - /// All tokens between a `Start` and a `Finish` would - /// become the children of the respective node. - /// - /// For left-recursive syntactic constructs, the parser produces - /// a child node before it sees a parent. `forward_parent` - /// saves the position of current event's parent. - /// - /// Consider this path - /// - /// foo::bar - /// - /// The events for it would look like this: - /// - /// - /// START(PATH) IDENT('foo') FINISH START(PATH) T![::] IDENT('bar') FINISH - /// | /\ - /// | | - /// +------forward-parent------+ - /// - /// And the tree would look like this - /// - /// +--PATH---------+ - /// | | | - /// | | | - /// | '::' 'bar' - /// | - /// PATH - /// | - /// 'foo' - /// - /// See also `CompletedMarker::precede`. - Start { - kind: SyntaxKind, - forward_parent: Option, - }, - - /// Complete the previous `Start` event - Finish, - - /// Produce a single leaf-element. - /// `n_raw_tokens` is used to glue complex contextual tokens. - /// For example, lexer tokenizes `>>` as `>`, `>`, and - /// `n_raw_tokens = 2` is used to produced a single `>>`. - Token { - kind: SyntaxKind, - n_raw_tokens: u8, - }, - - Error { - msg: ParseError, - }, -} - -impl Event { - pub(crate) fn tombstone() -> Self { - Event::Start { kind: TOMBSTONE, forward_parent: None } - } -} - -/// Generate the syntax tree with the control of events. -pub(super) fn process(sink: &mut dyn TreeSink, mut events: Vec) { - let mut forward_parents = Vec::new(); - - for i in 0..events.len() { - match mem::replace(&mut events[i], Event::tombstone()) { - Event::Start { kind: TOMBSTONE, .. } => (), - - Event::Start { kind, forward_parent } => { - // For events[A, B, C], B is A's forward_parent, C is B's forward_parent, - // in the normal control flow, the parent-child relation: `A -> B -> C`, - // while with the magic forward_parent, it writes: `C <- B <- A`. - - // append `A` into parents. - forward_parents.push(kind); - let mut idx = i; - let mut fp = forward_parent; - while let Some(fwd) = fp { - idx += fwd as usize; - // append `A`'s forward_parent `B` - fp = match mem::replace(&mut events[idx], Event::tombstone()) { - Event::Start { kind, forward_parent } => { - if kind != TOMBSTONE { - forward_parents.push(kind); - } - forward_parent - } - _ => unreachable!(), - }; - // append `B`'s forward_parent `C` in the next stage. - } - - for kind in forward_parents.drain(..).rev() { - sink.start_node(kind); - } - } - Event::Finish => sink.finish_node(), - Event::Token { kind, n_raw_tokens } => { - sink.token(kind, n_raw_tokens); - } - Event::Error { msg } => sink.error(msg), - } - } -} diff --git a/crates/ra_parser/src/grammar.rs b/crates/ra_parser/src/grammar.rs deleted file mode 100644 index 88468bc97..000000000 --- a/crates/ra_parser/src/grammar.rs +++ /dev/null @@ -1,293 +0,0 @@ -//! This is the actual "grammar" of the Rust language. -//! -//! Each function in this module and its children corresponds -//! to a production of the formal grammar. Submodules roughly -//! correspond to different *areas* of the grammar. By convention, -//! each submodule starts with `use super::*` import and exports -//! "public" productions via `pub(super)`. -//! -//! See docs for `Parser` to learn about API, available to the grammar, -//! and see docs for `Event` to learn how this actually manages to -//! produce parse trees. -//! -//! Code in this module also contains inline tests, which start with -//! `// test name-of-the-test` comment and look like this: -//! -//! ``` -//! // test function_with_zero_parameters -//! // fn foo() {} -//! ``` -//! -//! After adding a new inline-test, run `cargo xtask codegen` to -//! extract it as a standalone text-fixture into -//! `crates/ra_syntax/test_data/parser/`, and run `cargo test` once to -//! create the "gold" value. -//! -//! Coding convention: rules like `where_clause` always produce either a -//! node or an error, rules like `opt_where_clause` may produce nothing. -//! Non-opt rules typically start with `assert!(p.at(FIRST_TOKEN))`, the -//! caller is responsible for branching on the first token. -mod attributes; -mod expressions; -mod items; -mod params; -mod paths; -mod patterns; -mod type_args; -mod type_params; -mod types; - -use crate::{ - parser::{CompletedMarker, Marker, Parser}, - SyntaxKind::{self, *}, - TokenSet, -}; - -pub(crate) fn root(p: &mut Parser) { - let m = p.start(); - p.eat(SHEBANG); - items::mod_contents(p, false); - m.complete(p, SOURCE_FILE); -} - -/// Various pieces of syntax that can be parsed by macros by example -pub(crate) mod fragments { - use super::*; - - pub(crate) use super::{ - expressions::block_expr, paths::type_path as path, patterns::pattern, types::type_, - }; - - pub(crate) fn expr(p: &mut Parser) { - let _ = expressions::expr(p); - } - - pub(crate) fn stmt(p: &mut Parser) { - expressions::stmt(p, expressions::StmtWithSemi::No) - } - - pub(crate) fn opt_visibility(p: &mut Parser) { - let _ = super::opt_visibility(p); - } - - // Parse a meta item , which excluded [], e.g : #[ MetaItem ] - pub(crate) fn meta_item(p: &mut Parser) { - fn is_delimiter(p: &mut Parser) -> bool { - matches!(p.current(), T!['{'] | T!['('] | T!['[']) - } - - if is_delimiter(p) { - items::token_tree(p); - return; - } - - let m = p.start(); - while !p.at(EOF) { - if is_delimiter(p) { - items::token_tree(p); - break; - } else { - // https://doc.rust-lang.org/reference/attributes.html - // https://doc.rust-lang.org/reference/paths.html#simple-paths - // The start of an meta must be a simple path - match p.current() { - IDENT | T![::] | T![super] | T![self] | T![crate] => p.bump_any(), - T![=] => { - p.bump_any(); - match p.current() { - c if c.is_literal() => p.bump_any(), - T![true] | T![false] => p.bump_any(), - _ => {} - } - break; - } - _ => break, - } - } - } - - m.complete(p, TOKEN_TREE); - } - - pub(crate) fn item(p: &mut Parser) { - items::item_or_macro(p, true) - } - - pub(crate) fn macro_items(p: &mut Parser) { - let m = p.start(); - items::mod_contents(p, false); - m.complete(p, MACRO_ITEMS); - } - - pub(crate) fn macro_stmts(p: &mut Parser) { - let m = p.start(); - - while !p.at(EOF) { - if p.at(T![;]) { - p.bump(T![;]); - continue; - } - - expressions::stmt(p, expressions::StmtWithSemi::Optional); - } - - m.complete(p, MACRO_STMTS); - } -} - -pub(crate) fn reparser( - node: SyntaxKind, - first_child: Option, - parent: Option, -) -> Option { - let res = match node { - BLOCK_EXPR => expressions::block_expr, - RECORD_FIELD_LIST => items::record_field_def_list, - RECORD_EXPR_FIELD_LIST => items::record_field_list, - VARIANT_LIST => items::enum_variant_list, - MATCH_ARM_LIST => items::match_arm_list, - USE_TREE_LIST => items::use_tree_list, - EXTERN_ITEM_LIST => items::extern_item_list, - TOKEN_TREE if first_child? == T!['{'] => items::token_tree, - ASSOC_ITEM_LIST => match parent? { - IMPL => items::impl_item_list, - TRAIT => items::trait_item_list, - _ => return None, - }, - ITEM_LIST => items::mod_item_list, - _ => return None, - }; - Some(res) -} - -#[derive(Clone, Copy, PartialEq, Eq)] -enum BlockLike { - Block, - NotBlock, -} - -impl BlockLike { - fn is_block(self) -> bool { - self == BlockLike::Block - } -} - -fn opt_visibility(p: &mut Parser) -> bool { - match p.current() { - T![pub] => { - let m = p.start(); - p.bump(T![pub]); - if p.at(T!['(']) { - match p.nth(1) { - // test crate_visibility - // pub(crate) struct S; - // pub(self) struct S; - // pub(self) struct S; - // pub(self) struct S; - T![crate] | T![self] | T![super] => { - p.bump_any(); - p.bump_any(); - p.expect(T![')']); - } - T![in] => { - p.bump_any(); - p.bump_any(); - paths::use_path(p); - p.expect(T![')']); - } - _ => (), - } - } - m.complete(p, VISIBILITY); - } - // test crate_keyword_vis - // crate fn main() { } - // struct S { crate field: u32 } - // struct T(crate u32); - // - // test crate_keyword_path - // fn foo() { crate::foo(); } - T![crate] if !p.nth_at(1, T![::]) => { - let m = p.start(); - p.bump(T![crate]); - m.complete(p, VISIBILITY); - } - _ => return false, - } - true -} - -fn opt_alias(p: &mut Parser) { - if p.at(T![as]) { - let m = p.start(); - p.bump(T![as]); - if !p.eat(T![_]) { - name(p); - } - m.complete(p, RENAME); - } -} - -fn abi(p: &mut Parser) { - assert!(p.at(T![extern])); - let abi = p.start(); - p.bump(T![extern]); - match p.current() { - STRING | RAW_STRING => p.bump_any(), - _ => (), - } - abi.complete(p, ABI); -} - -fn opt_fn_ret_type(p: &mut Parser) -> bool { - if p.at(T![->]) { - let m = p.start(); - p.bump(T![->]); - types::type_no_bounds(p); - m.complete(p, RET_TYPE); - true - } else { - false - } -} - -fn name_r(p: &mut Parser, recovery: TokenSet) { - if p.at(IDENT) { - let m = p.start(); - p.bump(IDENT); - m.complete(p, NAME); - } else { - p.err_recover("expected a name", recovery); - } -} - -fn name(p: &mut Parser) { - name_r(p, TokenSet::EMPTY) -} - -fn name_ref(p: &mut Parser) { - if p.at(IDENT) { - let m = p.start(); - p.bump(IDENT); - m.complete(p, NAME_REF); - } else { - p.err_and_bump("expected identifier"); - } -} - -fn name_ref_or_index(p: &mut Parser) { - assert!(p.at(IDENT) || p.at(INT_NUMBER)); - let m = p.start(); - p.bump_any(); - m.complete(p, NAME_REF); -} - -fn error_block(p: &mut Parser, message: &str) { - assert!(p.at(T!['{'])); - let m = p.start(); - p.error(message); - p.bump(T!['{']); - expressions::expr_block_contents(p); - p.eat(T!['}']); - m.complete(p, ERROR); -} diff --git a/crates/ra_parser/src/grammar/attributes.rs b/crates/ra_parser/src/grammar/attributes.rs deleted file mode 100644 index f3158ade3..000000000 --- a/crates/ra_parser/src/grammar/attributes.rs +++ /dev/null @@ -1,48 +0,0 @@ -//! FIXME: write short doc here - -use super::*; - -pub(super) fn inner_attributes(p: &mut Parser) { - while p.at(T![#]) && p.nth(1) == T![!] { - attribute(p, true) - } -} - -pub(super) fn outer_attributes(p: &mut Parser) { - while p.at(T![#]) { - attribute(p, false) - } -} - -fn attribute(p: &mut Parser, inner: bool) { - let attr = p.start(); - assert!(p.at(T![#])); - p.bump(T![#]); - - if inner { - assert!(p.at(T![!])); - p.bump(T![!]); - } - - if p.eat(T!['[']) { - paths::use_path(p); - - match p.current() { - T![=] => { - p.bump(T![=]); - if expressions::literal(p).is_none() { - p.error("expected literal"); - } - } - T!['('] | T!['['] | T!['{'] => items::token_tree(p), - _ => {} - } - - if !p.eat(T![']']) { - p.error("expected `]`"); - } - } else { - p.error("expected `[`"); - } - attr.complete(p, ATTR); -} diff --git a/crates/ra_parser/src/grammar/expressions.rs b/crates/ra_parser/src/grammar/expressions.rs deleted file mode 100644 index 3291e3f14..000000000 --- a/crates/ra_parser/src/grammar/expressions.rs +++ /dev/null @@ -1,651 +0,0 @@ -//! FIXME: write short doc here - -mod atom; - -pub(crate) use self::atom::{block_expr, match_arm_list}; -pub(super) use self::atom::{literal, LITERAL_FIRST}; -use super::*; - -pub(super) enum StmtWithSemi { - Yes, - No, - Optional, -} - -const EXPR_FIRST: TokenSet = LHS_FIRST; - -pub(super) fn expr(p: &mut Parser) -> (Option, BlockLike) { - let r = Restrictions { forbid_structs: false, prefer_stmt: false }; - expr_bp(p, r, 1) -} - -pub(super) fn expr_with_attrs(p: &mut Parser) -> bool { - let m = p.start(); - let has_attrs = p.at(T![#]); - attributes::outer_attributes(p); - - let (cm, _block_like) = expr(p); - let success = cm.is_some(); - - match (has_attrs, cm) { - (true, Some(cm)) => { - let kind = cm.kind(); - cm.undo_completion(p).abandon(p); - m.complete(p, kind); - } - _ => m.abandon(p), - } - - success -} - -pub(super) fn expr_stmt(p: &mut Parser) -> (Option, BlockLike) { - let r = Restrictions { forbid_structs: false, prefer_stmt: true }; - expr_bp(p, r, 1) -} - -fn expr_no_struct(p: &mut Parser) { - let r = Restrictions { forbid_structs: true, prefer_stmt: false }; - expr_bp(p, r, 1); -} - -fn is_expr_stmt_attr_allowed(kind: SyntaxKind) -> bool { - let forbid = matches!(kind, BIN_EXPR | RANGE_EXPR); - !forbid -} - -pub(super) fn stmt(p: &mut Parser, with_semi: StmtWithSemi) { - let m = p.start(); - // test attr_on_expr_stmt - // fn foo() { - // #[A] foo(); - // #[B] bar!{} - // #[C] #[D] {} - // #[D] return (); - // } - let has_attrs = p.at(T![#]); - attributes::outer_attributes(p); - - if p.at(T![let]) { - let_stmt(p, m, with_semi); - return; - } - - // test block_items - // fn a() { fn b() {} } - let m = match items::maybe_item(p, m) { - Ok(()) => return, - Err(m) => m, - }; - - let (cm, blocklike) = expr_stmt(p); - let kind = cm.as_ref().map(|cm| cm.kind()).unwrap_or(ERROR); - - if has_attrs && !is_expr_stmt_attr_allowed(kind) { - // test_err attr_on_expr_not_allowed - // fn foo() { - // #[A] 1 + 2; - // #[B] if true {}; - // } - p.error(format!("attributes are not allowed on {:?}", kind)); - } - - if p.at(T!['}']) { - // test attr_on_last_expr_in_block - // fn foo() { - // { #[A] bar!()? } - // #[B] &() - // } - if let Some(cm) = cm { - cm.undo_completion(p).abandon(p); - m.complete(p, kind); - } else { - m.abandon(p); - } - } else { - // test no_semi_after_block - // fn foo() { - // if true {} - // loop {} - // match () {} - // while true {} - // for _ in () {} - // {} - // {} - // macro_rules! test { - // () => {} - // } - // test!{} - // } - - match with_semi { - StmtWithSemi::Yes => { - if blocklike.is_block() { - p.eat(T![;]); - } else { - p.expect(T![;]); - } - } - StmtWithSemi::No => {} - StmtWithSemi::Optional => { - if p.at(T![;]) { - p.eat(T![;]); - } - } - } - - m.complete(p, EXPR_STMT); - } - - // test let_stmt - // fn foo() { - // let a; - // let b: i32; - // let c = 92; - // let d: i32 = 92; - // let e: !; - // let _: ! = {}; - // let f = #[attr]||{}; - // } - fn let_stmt(p: &mut Parser, m: Marker, with_semi: StmtWithSemi) { - assert!(p.at(T![let])); - p.bump(T![let]); - patterns::pattern(p); - if p.at(T![:]) { - types::ascription(p); - } - if p.eat(T![=]) { - expressions::expr_with_attrs(p); - } - - match with_semi { - StmtWithSemi::Yes => { - p.expect(T![;]); - } - StmtWithSemi::No => {} - StmtWithSemi::Optional => { - if p.at(T![;]) { - p.eat(T![;]); - } - } - } - m.complete(p, LET_STMT); - } -} - -pub(super) fn expr_block_contents(p: &mut Parser) { - // This is checked by a validator - attributes::inner_attributes(p); - - while !p.at(EOF) && !p.at(T!['}']) { - // test nocontentexpr - // fn foo(){ - // ;;;some_expr();;;;{;;;};;;;Ok(()) - // } - - // test nocontentexpr_after_item - // fn simple_function() { - // enum LocalEnum { - // One, - // Two, - // }; - // fn f() {}; - // struct S {}; - // } - - if p.at(T![;]) { - p.bump(T![;]); - continue; - } - - stmt(p, StmtWithSemi::Yes) - } -} - -#[derive(Clone, Copy)] -struct Restrictions { - forbid_structs: bool, - prefer_stmt: bool, -} - -/// Binding powers of operators for a Pratt parser. -/// -/// See https://www.oilshell.org/blog/2016/11/03.html -#[rustfmt::skip] -fn current_op(p: &Parser) -> (u8, SyntaxKind) { - const NOT_AN_OP: (u8, SyntaxKind) = (0, T![@]); - match p.current() { - T![|] if p.at(T![||]) => (3, T![||]), - T![|] if p.at(T![|=]) => (1, T![|=]), - T![|] => (6, T![|]), - T![>] if p.at(T![>>=]) => (1, T![>>=]), - T![>] if p.at(T![>>]) => (9, T![>>]), - T![>] if p.at(T![>=]) => (5, T![>=]), - T![>] => (5, T![>]), - T![=] if p.at(T![=>]) => NOT_AN_OP, - T![=] if p.at(T![==]) => (5, T![==]), - T![=] => (1, T![=]), - T![<] if p.at(T![<=]) => (5, T![<=]), - T![<] if p.at(T![<<=]) => (1, T![<<=]), - T![<] if p.at(T![<<]) => (9, T![<<]), - T![<] => (5, T![<]), - T![+] if p.at(T![+=]) => (1, T![+=]), - T![+] => (10, T![+]), - T![^] if p.at(T![^=]) => (1, T![^=]), - T![^] => (7, T![^]), - T![%] if p.at(T![%=]) => (1, T![%=]), - T![%] => (11, T![%]), - T![&] if p.at(T![&=]) => (1, T![&=]), - T![&] if p.at(T![&&]) => (4, T![&&]), - T![&] => (8, T![&]), - T![/] if p.at(T![/=]) => (1, T![/=]), - T![/] => (11, T![/]), - T![*] if p.at(T![*=]) => (1, T![*=]), - T![*] => (11, T![*]), - T![.] if p.at(T![..=]) => (2, T![..=]), - T![.] if p.at(T![..]) => (2, T![..]), - T![!] if p.at(T![!=]) => (5, T![!=]), - T![-] if p.at(T![-=]) => (1, T![-=]), - T![-] => (10, T![-]), - T![as] => (12, T![as]), - - _ => NOT_AN_OP - } -} - -// Parses expression with binding power of at least bp. -fn expr_bp(p: &mut Parser, mut r: Restrictions, bp: u8) -> (Option, BlockLike) { - let mut lhs = match lhs(p, r) { - Some((lhs, blocklike)) => { - // test stmt_bin_expr_ambiguity - // fn foo() { - // let _ = {1} & 2; - // {1} &2; - // } - if r.prefer_stmt && blocklike.is_block() { - return (Some(lhs), BlockLike::Block); - } - lhs - } - None => return (None, BlockLike::NotBlock), - }; - - loop { - let is_range = p.at(T![..]) || p.at(T![..=]); - let (op_bp, op) = current_op(p); - if op_bp < bp { - break; - } - // test as_precedence - // fn foo() { - // let _ = &1 as *const i32; - // } - if p.at(T![as]) { - lhs = cast_expr(p, lhs); - continue; - } - let m = lhs.precede(p); - p.bump(op); - - // test binop_resets_statementness - // fn foo() { - // v = {1}&2; - // } - r = Restrictions { prefer_stmt: false, ..r }; - - if is_range { - // test postfix_range - // fn foo() { - // let x = 1..; - // match 1.. { _ => () }; - // match a.b()..S { _ => () }; - // } - let has_trailing_expression = - p.at_ts(EXPR_FIRST) && !(r.forbid_structs && p.at(T!['{'])); - if !has_trailing_expression { - // no RHS - lhs = m.complete(p, RANGE_EXPR); - break; - } - } - - expr_bp(p, Restrictions { prefer_stmt: false, ..r }, op_bp + 1); - lhs = m.complete(p, if is_range { RANGE_EXPR } else { BIN_EXPR }); - } - (Some(lhs), BlockLike::NotBlock) -} - -const LHS_FIRST: TokenSet = - atom::ATOM_EXPR_FIRST.union(token_set![T![&], T![*], T![!], T![.], T![-]]); - -fn lhs(p: &mut Parser, r: Restrictions) -> Option<(CompletedMarker, BlockLike)> { - let m; - let kind = match p.current() { - // test ref_expr - // fn foo() { - // // reference operator - // let _ = &1; - // let _ = &mut &f(); - // let _ = &raw; - // let _ = &raw.0; - // // raw reference operator - // let _ = &raw mut foo; - // let _ = &raw const foo; - // } - T![&] => { - m = p.start(); - p.bump(T![&]); - if p.at(IDENT) - && p.at_contextual_kw("raw") - && (p.nth_at(1, T![mut]) || p.nth_at(1, T![const])) - { - p.bump_remap(T![raw]); - p.bump_any(); - } else { - p.eat(T![mut]); - } - REF_EXPR - } - // test unary_expr - // fn foo() { - // **&1; - // !!true; - // --1; - // } - T![*] | T![!] | T![-] => { - m = p.start(); - p.bump_any(); - PREFIX_EXPR - } - _ => { - // test full_range_expr - // fn foo() { xs[..]; } - for &op in [T![..=], T![..]].iter() { - if p.at(op) { - m = p.start(); - p.bump(op); - if p.at_ts(EXPR_FIRST) && !(r.forbid_structs && p.at(T!['{'])) { - expr_bp(p, r, 2); - } - return Some((m.complete(p, RANGE_EXPR), BlockLike::NotBlock)); - } - } - - // test expression_after_block - // fn foo() { - // let mut p = F{x: 5}; - // {p}.x = 10; - // } - // - let (lhs, blocklike) = atom::atom_expr(p, r)?; - return Some(postfix_expr(p, lhs, blocklike, !(r.prefer_stmt && blocklike.is_block()))); - } - }; - // parse the interior of the unary expression - expr_bp(p, r, 255); - Some((m.complete(p, kind), BlockLike::NotBlock)) -} - -fn postfix_expr( - p: &mut Parser, - mut lhs: CompletedMarker, - // Calls are disallowed if the type is a block and we prefer statements because the call cannot be disambiguated from a tuple - // E.g. `while true {break}();` is parsed as - // `while true {break}; ();` - mut block_like: BlockLike, - mut allow_calls: bool, -) -> (CompletedMarker, BlockLike) { - loop { - lhs = match p.current() { - // test stmt_postfix_expr_ambiguity - // fn foo() { - // match () { - // _ => {} - // () => {} - // [] => {} - // } - // } - T!['('] if allow_calls => call_expr(p, lhs), - T!['['] if allow_calls => index_expr(p, lhs), - T![.] => match postfix_dot_expr(p, lhs) { - Ok(it) => it, - Err(it) => { - lhs = it; - break; - } - }, - T![?] => try_expr(p, lhs), - _ => break, - }; - allow_calls = true; - block_like = BlockLike::NotBlock; - } - return (lhs, block_like); - - fn postfix_dot_expr( - p: &mut Parser, - lhs: CompletedMarker, - ) -> Result { - assert!(p.at(T![.])); - if p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::])) { - return Ok(method_call_expr(p, lhs)); - } - - // test await_expr - // fn foo() { - // x.await; - // x.0.await; - // x.0().await?.hello(); - // } - if p.nth(1) == T![await] { - let m = lhs.precede(p); - p.bump(T![.]); - p.bump(T![await]); - return Ok(m.complete(p, AWAIT_EXPR)); - } - - if p.at(T![..=]) || p.at(T![..]) { - return Err(lhs); - } - - Ok(field_expr(p, lhs)) - } -} - -// test call_expr -// fn foo() { -// let _ = f(); -// let _ = f()(1)(1, 2,); -// let _ = f(::func()); -// f(::func()); -// } -fn call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { - assert!(p.at(T!['('])); - let m = lhs.precede(p); - arg_list(p); - m.complete(p, CALL_EXPR) -} - -// test index_expr -// fn foo() { -// x[1][2]; -// } -fn index_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { - assert!(p.at(T!['['])); - let m = lhs.precede(p); - p.bump(T!['[']); - expr(p); - p.expect(T![']']); - m.complete(p, INDEX_EXPR) -} - -// test method_call_expr -// fn foo() { -// x.foo(); -// y.bar::(1, 2,); -// } -fn method_call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { - assert!(p.at(T![.]) && p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::]))); - let m = lhs.precede(p); - p.bump_any(); - name_ref(p); - type_args::opt_type_arg_list(p, true); - if p.at(T!['(']) { - arg_list(p); - } - m.complete(p, METHOD_CALL_EXPR) -} - -// test field_expr -// fn foo() { -// x.foo; -// x.0.bar; -// x.0(); -// } - -// test_err bad_tuple_index_expr -// fn foo() { -// x.0.; -// x.1i32; -// x.0x01; -// } -fn field_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { - assert!(p.at(T![.])); - let m = lhs.precede(p); - p.bump(T![.]); - if p.at(IDENT) || p.at(INT_NUMBER) { - name_ref_or_index(p) - } else if p.at(FLOAT_NUMBER) { - // FIXME: How to recover and instead parse INT + T![.]? - p.bump_any(); - } else { - p.error("expected field name or number") - } - m.complete(p, FIELD_EXPR) -} - -// test try_expr -// fn foo() { -// x?; -// } -fn try_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { - assert!(p.at(T![?])); - let m = lhs.precede(p); - p.bump(T![?]); - m.complete(p, TRY_EXPR) -} - -// test cast_expr -// fn foo() { -// 82 as i32; -// 81 as i8 + 1; -// 79 as i16 - 1; -// 0x36 as u8 <= 0x37; -// } -fn cast_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { - assert!(p.at(T![as])); - let m = lhs.precede(p); - p.bump(T![as]); - // Use type_no_bounds(), because cast expressions are not - // allowed to have bounds. - types::type_no_bounds(p); - m.complete(p, CAST_EXPR) -} - -fn arg_list(p: &mut Parser) { - assert!(p.at(T!['('])); - let m = p.start(); - p.bump(T!['(']); - while !p.at(T![')']) && !p.at(EOF) { - // test arg_with_attr - // fn main() { - // foo(#[attr] 92) - // } - if !expr_with_attrs(p) { - break; - } - if !p.at(T![')']) && !p.expect(T![,]) { - break; - } - } - p.eat(T![')']); - m.complete(p, ARG_LIST); -} - -// test path_expr -// fn foo() { -// let _ = a; -// let _ = a::b; -// let _ = ::a::; -// let _ = format!(); -// } -fn path_expr(p: &mut Parser, r: Restrictions) -> (CompletedMarker, BlockLike) { - assert!(paths::is_path_start(p)); - let m = p.start(); - paths::expr_path(p); - match p.current() { - T!['{'] if !r.forbid_structs => { - record_field_list(p); - (m.complete(p, RECORD_EXPR), BlockLike::NotBlock) - } - T![!] if !p.at(T![!=]) => { - let block_like = items::macro_call_after_excl(p); - (m.complete(p, MACRO_CALL), block_like) - } - _ => (m.complete(p, PATH_EXPR), BlockLike::NotBlock), - } -} - -// test record_lit -// fn foo() { -// S {}; -// S { x, y: 32, }; -// S { x, y: 32, ..Default::default() }; -// TupleStruct { 0: 1 }; -// } -pub(crate) fn record_field_list(p: &mut Parser) { - assert!(p.at(T!['{'])); - let m = p.start(); - p.bump(T!['{']); - while !p.at(EOF) && !p.at(T!['}']) { - let m = p.start(); - // test record_literal_field_with_attr - // fn main() { - // S { #[cfg(test)] field: 1 } - // } - attributes::outer_attributes(p); - - match p.current() { - IDENT | INT_NUMBER => { - // test_err record_literal_before_ellipsis_recovery - // fn main() { - // S { field ..S::default() } - // } - if p.nth_at(1, T![:]) || p.nth_at(1, T![..]) { - name_ref_or_index(p); - p.expect(T![:]); - } - expr(p); - m.complete(p, RECORD_EXPR_FIELD); - } - T![.] if p.at(T![..]) => { - m.abandon(p); - p.bump(T![..]); - expr(p); - } - T!['{'] => { - error_block(p, "expected a field"); - m.abandon(p); - } - _ => { - p.err_and_bump("expected identifier"); - m.abandon(p); - } - } - if !p.at(T!['}']) { - p.expect(T![,]); - } - } - p.expect(T!['}']); - m.complete(p, RECORD_EXPR_FIELD_LIST); -} diff --git a/crates/ra_parser/src/grammar/expressions/atom.rs b/crates/ra_parser/src/grammar/expressions/atom.rs deleted file mode 100644 index 0b01d3bc6..000000000 --- a/crates/ra_parser/src/grammar/expressions/atom.rs +++ /dev/null @@ -1,611 +0,0 @@ -//! FIXME: write short doc here - -use super::*; - -// test expr_literals -// fn foo() { -// let _ = true; -// let _ = false; -// let _ = 1; -// let _ = 2.0; -// let _ = b'a'; -// let _ = 'b'; -// let _ = "c"; -// let _ = r"d"; -// let _ = b"e"; -// let _ = br"f"; -// } -pub(crate) const LITERAL_FIRST: TokenSet = token_set![ - TRUE_KW, - FALSE_KW, - INT_NUMBER, - FLOAT_NUMBER, - BYTE, - CHAR, - STRING, - RAW_STRING, - BYTE_STRING, - RAW_BYTE_STRING -]; - -pub(crate) fn literal(p: &mut Parser) -> Option { - if !p.at_ts(LITERAL_FIRST) { - return None; - } - let m = p.start(); - p.bump_any(); - Some(m.complete(p, LITERAL)) -} - -// E.g. for after the break in `if break {}`, this should not match -pub(super) const ATOM_EXPR_FIRST: TokenSet = - LITERAL_FIRST.union(paths::PATH_FIRST).union(token_set![ - T!['('], - T!['{'], - T!['['], - L_DOLLAR, - T![|], - T![move], - T![box], - T![if], - T![while], - T![match], - T![unsafe], - T![return], - T![break], - T![continue], - T![async], - T![try], - T![loop], - T![for], - LIFETIME, - ]); - -const EXPR_RECOVERY_SET: TokenSet = token_set![LET_KW, R_DOLLAR]; - -pub(super) fn atom_expr(p: &mut Parser, r: Restrictions) -> Option<(CompletedMarker, BlockLike)> { - if let Some(m) = literal(p) { - return Some((m, BlockLike::NotBlock)); - } - if paths::is_path_start(p) { - return Some(path_expr(p, r)); - } - let la = p.nth(1); - let done = match p.current() { - T!['('] => tuple_expr(p), - T!['['] => array_expr(p), - L_DOLLAR => meta_var_expr(p), - T![|] => lambda_expr(p), - T![move] if la == T![|] => lambda_expr(p), - T![async] if la == T![|] || (la == T![move] && p.nth(2) == T![|]) => lambda_expr(p), - T![if] => if_expr(p), - - T![loop] => loop_expr(p, None), - T![box] => box_expr(p, None), - T![for] => for_expr(p, None), - T![while] => while_expr(p, None), - T![try] => try_block_expr(p, None), - LIFETIME if la == T![:] => { - let m = p.start(); - label(p); - match p.current() { - T![loop] => loop_expr(p, Some(m)), - T![for] => for_expr(p, Some(m)), - T![while] => while_expr(p, Some(m)), - // test labeled_block - // fn f() { 'label: {}; } - T!['{'] => { - block_expr(p); - m.complete(p, EFFECT_EXPR) - } - _ => { - // test_err misplaced_label_err - // fn main() { - // 'loop: impl - // } - p.error("expected a loop"); - m.complete(p, ERROR); - return None; - } - } - } - T![async] if la == T!['{'] || (la == T![move] && p.nth(2) == T!['{']) => { - let m = p.start(); - p.bump(T![async]); - p.eat(T![move]); - block_expr(p); - m.complete(p, EFFECT_EXPR) - } - T![match] => match_expr(p), - // test unsafe_block - // fn f() { unsafe { } } - T![unsafe] if la == T!['{'] => { - let m = p.start(); - p.bump(T![unsafe]); - block_expr(p); - m.complete(p, EFFECT_EXPR) - } - T!['{'] => { - // test for_range_from - // fn foo() { - // for x in 0 .. { - // break; - // } - // } - block_expr_unchecked(p) - } - T![return] => return_expr(p), - T![continue] => continue_expr(p), - T![break] => break_expr(p, r), - _ => { - p.err_recover("expected expression", EXPR_RECOVERY_SET); - return None; - } - }; - let blocklike = match done.kind() { - IF_EXPR | WHILE_EXPR | FOR_EXPR | LOOP_EXPR | MATCH_EXPR | BLOCK_EXPR | EFFECT_EXPR => { - BlockLike::Block - } - _ => BlockLike::NotBlock, - }; - Some((done, blocklike)) -} - -// test tuple_expr -// fn foo() { -// (); -// (1); -// (1,); -// } -fn tuple_expr(p: &mut Parser) -> CompletedMarker { - assert!(p.at(T!['('])); - let m = p.start(); - p.expect(T!['(']); - - let mut saw_comma = false; - let mut saw_expr = false; - while !p.at(EOF) && !p.at(T![')']) { - saw_expr = true; - if !p.at_ts(EXPR_FIRST) { - p.error("expected expression"); - break; - } - expr(p); - if !p.at(T![')']) { - saw_comma = true; - p.expect(T![,]); - } - } - p.expect(T![')']); - m.complete(p, if saw_expr && !saw_comma { PAREN_EXPR } else { TUPLE_EXPR }) -} - -// test array_expr -// fn foo() { -// []; -// [1]; -// [1, 2,]; -// [1; 2]; -// } -fn array_expr(p: &mut Parser) -> CompletedMarker { - assert!(p.at(T!['['])); - let m = p.start(); - - let mut n_exprs = 0u32; - let mut has_semi = false; - - p.bump(T!['[']); - while !p.at(EOF) && !p.at(T![']']) { - n_exprs += 1; - - // test array_attrs - // const A: &[i64] = &[1, #[cfg(test)] 2]; - if !expr_with_attrs(p) { - break; - } - - if n_exprs == 1 && p.eat(T![;]) { - has_semi = true; - continue; - } - - if has_semi || !p.at(T![']']) && !p.expect(T![,]) { - break; - } - } - p.expect(T![']']); - - m.complete(p, ARRAY_EXPR) -} - -// test lambda_expr -// fn foo() { -// || (); -// || -> i32 { 92 }; -// |x| x; -// move |x: i32,| x; -// async || {}; -// move || {}; -// async move || {}; -// } -fn lambda_expr(p: &mut Parser) -> CompletedMarker { - assert!( - p.at(T![|]) - || (p.at(T![move]) && p.nth(1) == T![|]) - || (p.at(T![async]) && p.nth(1) == T![|]) - || (p.at(T![async]) && p.nth(1) == T![move] && p.nth(2) == T![|]) - ); - let m = p.start(); - p.eat(T![async]); - p.eat(T![move]); - params::param_list_closure(p); - if opt_fn_ret_type(p) { - // test lambda_ret_block - // fn main() { || -> i32 { 92 }(); } - block_expr(p); - } else { - if p.at_ts(EXPR_FIRST) { - expr(p); - } else { - p.error("expected expression"); - } - } - m.complete(p, CLOSURE_EXPR) -} - -// test if_expr -// fn foo() { -// if true {}; -// if true {} else {}; -// if true {} else if false {} else {}; -// if S {}; -// if { true } { } else { }; -// } -fn if_expr(p: &mut Parser) -> CompletedMarker { - assert!(p.at(T![if])); - let m = p.start(); - p.bump(T![if]); - cond(p); - block_expr(p); - if p.at(T![else]) { - p.bump(T![else]); - if p.at(T![if]) { - if_expr(p); - } else { - block_expr(p); - } - } - m.complete(p, IF_EXPR) -} - -// test label -// fn foo() { -// 'a: loop {} -// 'b: while true {} -// 'c: for x in () {} -// } -fn label(p: &mut Parser) { - assert!(p.at(LIFETIME) && p.nth(1) == T![:]); - let m = p.start(); - p.bump(LIFETIME); - p.bump_any(); - m.complete(p, LABEL); -} - -// test loop_expr -// fn foo() { -// loop {}; -// } -fn loop_expr(p: &mut Parser, m: Option) -> CompletedMarker { - assert!(p.at(T![loop])); - let m = m.unwrap_or_else(|| p.start()); - p.bump(T![loop]); - block_expr(p); - m.complete(p, LOOP_EXPR) -} - -// test while_expr -// fn foo() { -// while true {}; -// while let Some(x) = it.next() {}; -// while { true } {}; -// } -fn while_expr(p: &mut Parser, m: Option) -> CompletedMarker { - assert!(p.at(T![while])); - let m = m.unwrap_or_else(|| p.start()); - p.bump(T![while]); - cond(p); - block_expr(p); - m.complete(p, WHILE_EXPR) -} - -// test for_expr -// fn foo() { -// for x in [] {}; -// } -fn for_expr(p: &mut Parser, m: Option) -> CompletedMarker { - assert!(p.at(T![for])); - let m = m.unwrap_or_else(|| p.start()); - p.bump(T![for]); - patterns::pattern(p); - p.expect(T![in]); - expr_no_struct(p); - block_expr(p); - m.complete(p, FOR_EXPR) -} - -// test cond -// fn foo() { if let Some(_) = None {} } -// fn bar() { -// if let Some(_) | Some(_) = None {} -// if let | Some(_) = None {} -// while let Some(_) | Some(_) = None {} -// while let | Some(_) = None {} -// } -fn cond(p: &mut Parser) { - let m = p.start(); - if p.eat(T![let]) { - patterns::pattern_top(p); - p.expect(T![=]); - } - expr_no_struct(p); - m.complete(p, CONDITION); -} - -// test match_expr -// fn foo() { -// match () { }; -// match S {}; -// match { } { _ => () }; -// match { S {} } {}; -// } -fn match_expr(p: &mut Parser) -> CompletedMarker { - assert!(p.at(T![match])); - let m = p.start(); - p.bump(T![match]); - expr_no_struct(p); - if p.at(T!['{']) { - match_arm_list(p); - } else { - p.error("expected `{`") - } - m.complete(p, MATCH_EXPR) -} - -pub(crate) fn match_arm_list(p: &mut Parser) { - assert!(p.at(T!['{'])); - let m = p.start(); - p.eat(T!['{']); - - // test match_arms_inner_attribute - // fn foo() { - // match () { - // #![doc("Inner attribute")] - // #![doc("Can be")] - // #![doc("Stacked")] - // _ => (), - // } - // } - attributes::inner_attributes(p); - - while !p.at(EOF) && !p.at(T!['}']) { - if p.at(T!['{']) { - error_block(p, "expected match arm"); - continue; - } - - // test match_arms_commas - // fn foo() { - // match () { - // _ => (), - // _ => {} - // _ => () - // } - // } - if match_arm(p).is_block() { - p.eat(T![,]); - } else if !p.at(T!['}']) { - p.expect(T![,]); - } - } - p.expect(T!['}']); - m.complete(p, MATCH_ARM_LIST); -} - -// test match_arm -// fn foo() { -// match () { -// _ => (), -// _ if Test > Test{field: 0} => (), -// X | Y if Z => (), -// | X | Y if Z => (), -// | X => (), -// }; -// } -fn match_arm(p: &mut Parser) -> BlockLike { - let m = p.start(); - // test match_arms_outer_attributes - // fn foo() { - // match () { - // #[cfg(feature = "some")] - // _ => (), - // #[cfg(feature = "other")] - // _ => (), - // #[cfg(feature = "many")] - // #[cfg(feature = "attributes")] - // #[cfg(feature = "before")] - // _ => (), - // } - // } - attributes::outer_attributes(p); - - patterns::pattern_top_r(p, TokenSet::EMPTY); - if p.at(T![if]) { - match_guard(p); - } - p.expect(T![=>]); - let blocklike = expr_stmt(p).1; - m.complete(p, MATCH_ARM); - blocklike -} - -// test match_guard -// fn foo() { -// match () { -// _ if foo => (), -// } -// } -fn match_guard(p: &mut Parser) -> CompletedMarker { - assert!(p.at(T![if])); - let m = p.start(); - p.bump(T![if]); - expr(p); - m.complete(p, MATCH_GUARD) -} - -// test block -// fn a() {} -// fn b() { let _ = 1; } -// fn c() { 1; 2; } -// fn d() { 1; 2 } -pub(crate) fn block_expr(p: &mut Parser) { - if !p.at(T!['{']) { - p.error("expected a block"); - return; - } - block_expr_unchecked(p); -} - -fn block_expr_unchecked(p: &mut Parser) -> CompletedMarker { - assert!(p.at(T!['{'])); - let m = p.start(); - p.bump(T!['{']); - expr_block_contents(p); - p.expect(T!['}']); - m.complete(p, BLOCK_EXPR) -} - -// test return_expr -// fn foo() { -// return; -// return 92; -// } -fn return_expr(p: &mut Parser) -> CompletedMarker { - assert!(p.at(T![return])); - let m = p.start(); - p.bump(T![return]); - if p.at_ts(EXPR_FIRST) { - expr(p); - } - m.complete(p, RETURN_EXPR) -} - -// test continue_expr -// fn foo() { -// loop { -// continue; -// continue 'l; -// } -// } -fn continue_expr(p: &mut Parser) -> CompletedMarker { - assert!(p.at(T![continue])); - let m = p.start(); - p.bump(T![continue]); - p.eat(LIFETIME); - m.complete(p, CONTINUE_EXPR) -} - -// test break_expr -// fn foo() { -// loop { -// break; -// break 'l; -// break 92; -// break 'l 92; -// } -// } -fn break_expr(p: &mut Parser, r: Restrictions) -> CompletedMarker { - assert!(p.at(T![break])); - let m = p.start(); - p.bump(T![break]); - p.eat(LIFETIME); - // test break_ambiguity - // fn foo(){ - // if break {} - // while break {} - // for i in break {} - // match break {} - // } - if p.at_ts(EXPR_FIRST) && !(r.forbid_structs && p.at(T!['{'])) { - expr(p); - } - m.complete(p, BREAK_EXPR) -} - -// test try_block_expr -// fn foo() { -// let _ = try {}; -// } -fn try_block_expr(p: &mut Parser, m: Option) -> CompletedMarker { - assert!(p.at(T![try])); - let m = m.unwrap_or_else(|| p.start()); - // Special-case `try!` as macro. - // This is a hack until we do proper edition support - if p.nth_at(1, T![!]) { - // test try_macro_fallback - // fn foo() { try!(Ok(())); } - let path = p.start(); - let path_segment = p.start(); - let name_ref = p.start(); - p.bump_remap(IDENT); - name_ref.complete(p, NAME_REF); - path_segment.complete(p, PATH_SEGMENT); - path.complete(p, PATH); - let _block_like = items::macro_call_after_excl(p); - return m.complete(p, MACRO_CALL); - } - - p.bump(T![try]); - block_expr(p); - m.complete(p, EFFECT_EXPR) -} - -// test box_expr -// fn foo() { -// let x = box 1i32; -// let y = (box 1i32, box 2i32); -// let z = Foo(box 1i32, box 2i32); -// } -fn box_expr(p: &mut Parser, m: Option) -> CompletedMarker { - assert!(p.at(T![box])); - let m = m.unwrap_or_else(|| p.start()); - p.bump(T![box]); - if p.at_ts(EXPR_FIRST) { - expr(p); - } - m.complete(p, BOX_EXPR) -} - -/// Expression from `$var` macro expansion, wrapped in dollars -fn meta_var_expr(p: &mut Parser) -> CompletedMarker { - assert!(p.at(L_DOLLAR)); - let m = p.start(); - p.bump(L_DOLLAR); - let (completed, _is_block) = - expr_bp(p, Restrictions { forbid_structs: false, prefer_stmt: false }, 1); - - match (completed, p.current()) { - (Some(it), R_DOLLAR) => { - p.bump(R_DOLLAR); - m.abandon(p); - it - } - _ => { - while !p.at(R_DOLLAR) { - p.bump_any() - } - p.bump(R_DOLLAR); - m.complete(p, ERROR) - } - } -} diff --git a/crates/ra_parser/src/grammar/items.rs b/crates/ra_parser/src/grammar/items.rs deleted file mode 100644 index d091b0fbb..000000000 --- a/crates/ra_parser/src/grammar/items.rs +++ /dev/null @@ -1,432 +0,0 @@ -//! FIXME: write short doc here - -mod consts; -mod adt; -mod traits; -mod use_item; - -pub(crate) use self::{ - adt::{enum_variant_list, record_field_def_list}, - expressions::{match_arm_list, record_field_list}, - traits::{impl_item_list, trait_item_list}, - use_item::use_tree_list, -}; -use super::*; - -// test mod_contents -// fn foo() {} -// macro_rules! foo {} -// foo::bar!(); -// super::baz! {} -// struct S; -pub(super) fn mod_contents(p: &mut Parser, stop_on_r_curly: bool) { - attributes::inner_attributes(p); - while !(stop_on_r_curly && p.at(T!['}']) || p.at(EOF)) { - item_or_macro(p, stop_on_r_curly) - } -} - -pub(super) const ITEM_RECOVERY_SET: TokenSet = token_set![ - FN_KW, STRUCT_KW, ENUM_KW, IMPL_KW, TRAIT_KW, CONST_KW, STATIC_KW, LET_KW, MOD_KW, PUB_KW, - CRATE_KW, USE_KW, MACRO_KW -]; - -pub(super) fn item_or_macro(p: &mut Parser, stop_on_r_curly: bool) { - let m = p.start(); - attributes::outer_attributes(p); - let m = match maybe_item(p, m) { - Ok(()) => { - if p.at(T![;]) { - p.err_and_bump( - "expected item, found `;`\n\ - consider removing this semicolon", - ); - } - return; - } - Err(m) => m, - }; - if paths::is_use_path_start(p) { - match macro_call(p) { - BlockLike::Block => (), - BlockLike::NotBlock => { - p.expect(T![;]); - } - } - m.complete(p, MACRO_CALL); - } else { - m.abandon(p); - if p.at(T!['{']) { - error_block(p, "expected an item"); - } else if p.at(T!['}']) && !stop_on_r_curly { - let e = p.start(); - p.error("unmatched `}`"); - p.bump(T!['}']); - e.complete(p, ERROR); - } else if !p.at(EOF) && !p.at(T!['}']) { - p.err_and_bump("expected an item"); - } else { - p.error("expected an item"); - } - } -} - -pub(super) fn maybe_item(p: &mut Parser, m: Marker) -> Result<(), Marker> { - // test_err pub_expr - // fn foo() { pub 92; } - let has_visibility = opt_visibility(p); - - let m = match items_without_modifiers(p, m) { - Ok(()) => return Ok(()), - Err(m) => m, - }; - - let mut has_mods = false; - - // modifiers - has_mods |= p.eat(T![const]); - - // test_err async_without_semicolon - // fn foo() { let _ = async {} } - if p.at(T![async]) && p.nth(1) != T!['{'] && p.nth(1) != T![move] && p.nth(1) != T![|] { - p.eat(T![async]); - has_mods = true; - } - - // test_err unsafe_block_in_mod - // fn foo(){} unsafe { } fn bar(){} - if p.at(T![unsafe]) && p.nth(1) != T!['{'] { - p.eat(T![unsafe]); - has_mods = true; - } - - if p.at(T![extern]) { - has_mods = true; - abi(p); - } - if p.at(IDENT) && p.at_contextual_kw("auto") && p.nth(1) == T![trait] { - p.bump_remap(T![auto]); - has_mods = true; - } - - // test default_item - // default impl T for Foo {} - if p.at(IDENT) && p.at_contextual_kw("default") { - match p.nth(1) { - T![fn] | T![type] | T![const] | T![impl] => { - p.bump_remap(T![default]); - has_mods = true; - } - T![unsafe] => { - // test default_unsafe_item - // default unsafe impl T for Foo { - // default unsafe fn foo() {} - // } - if matches!(p.nth(2), T![impl] | T![fn]) { - p.bump_remap(T![default]); - p.bump(T![unsafe]); - has_mods = true; - } - } - _ => (), - } - } - - // test existential_type - // existential type Foo: Fn() -> usize; - if p.at(IDENT) && p.at_contextual_kw("existential") && p.nth(1) == T![type] { - p.bump_remap(T![existential]); - has_mods = true; - } - - // items - match p.current() { - // test fn - // fn foo() {} - T![fn] => { - fn_def(p); - m.complete(p, FN); - } - - // test trait - // trait T {} - T![trait] => { - traits::trait_def(p); - m.complete(p, TRAIT); - } - - T![const] => { - consts::const_def(p, m); - } - - // test impl - // impl T for S {} - T![impl] => { - traits::impl_def(p); - m.complete(p, IMPL); - } - - T![type] => { - type_def(p, m); - } - _ => { - if !has_visibility && !has_mods { - return Err(m); - } else { - if has_mods { - p.error("expected existential, fn, trait or impl"); - } else { - p.error("expected an item"); - } - m.complete(p, ERROR); - } - } - } - Ok(()) -} - -fn items_without_modifiers(p: &mut Parser, m: Marker) -> Result<(), Marker> { - let la = p.nth(1); - match p.current() { - // test extern_crate - // extern crate foo; - T![extern] if la == T![crate] => extern_crate_item(p, m), - T![type] => { - type_def(p, m); - } - T![mod] => mod_item(p, m), - T![struct] => { - // test struct_items - // struct Foo; - // struct Foo {} - // struct Foo(); - // struct Foo(String, usize); - // struct Foo { - // a: i32, - // b: f32, - // } - adt::struct_def(p, m); - } - // test pub_macro_def - // pub macro m($:ident) {} - T![macro] => { - macro_def(p, m); - } - IDENT if p.at_contextual_kw("union") && p.nth(1) == IDENT => { - // test union_items - // union Foo {} - // union Foo { - // a: i32, - // b: f32, - // } - adt::union_def(p, m); - } - T![enum] => adt::enum_def(p, m), - T![use] => use_item::use_item(p, m), - T![const] if (la == IDENT || la == T![_] || la == T![mut]) => consts::const_def(p, m), - T![static] => consts::static_def(p, m), - // test extern_block - // extern {} - T![extern] - if la == T!['{'] || ((la == STRING || la == RAW_STRING) && p.nth(2) == T!['{']) => - { - abi(p); - extern_item_list(p); - m.complete(p, EXTERN_BLOCK); - } - _ => return Err(m), - }; - Ok(()) -} - -fn extern_crate_item(p: &mut Parser, m: Marker) { - assert!(p.at(T![extern])); - p.bump(T![extern]); - assert!(p.at(T![crate])); - p.bump(T![crate]); - - if p.at(T![self]) { - p.bump(T![self]); - } else { - name_ref(p); - } - - opt_alias(p); - p.expect(T![;]); - m.complete(p, EXTERN_CRATE); -} - -pub(crate) fn extern_item_list(p: &mut Parser) { - assert!(p.at(T!['{'])); - let m = p.start(); - p.bump(T!['{']); - mod_contents(p, true); - p.expect(T!['}']); - m.complete(p, EXTERN_ITEM_LIST); -} - -fn fn_def(p: &mut Parser) { - assert!(p.at(T![fn])); - p.bump(T![fn]); - - name_r(p, ITEM_RECOVERY_SET); - // test function_type_params - // fn foo(){} - type_params::opt_type_param_list(p); - - if p.at(T!['(']) { - params::param_list_fn_def(p); - } else { - p.error("expected function arguments"); - } - // test function_ret_type - // fn foo() {} - // fn bar() -> () {} - opt_fn_ret_type(p); - - // test function_where_clause - // fn foo() where T: Copy {} - type_params::opt_where_clause(p); - - // test fn_decl - // trait T { fn foo(); } - if p.at(T![;]) { - p.bump(T![;]); - } else { - expressions::block_expr(p) - } -} - -// test type_item -// type Foo = Bar; -fn type_def(p: &mut Parser, m: Marker) { - assert!(p.at(T![type])); - p.bump(T![type]); - - name(p); - - // test type_item_type_params - // type Result = (); - type_params::opt_type_param_list(p); - - if p.at(T![:]) { - type_params::bounds(p); - } - - // test type_item_where_clause - // type Foo where Foo: Copy = (); - type_params::opt_where_clause(p); - if p.eat(T![=]) { - types::type_(p); - } - p.expect(T![;]); - m.complete(p, TYPE_ALIAS); -} - -pub(crate) fn mod_item(p: &mut Parser, m: Marker) { - assert!(p.at(T![mod])); - p.bump(T![mod]); - - name(p); - if p.at(T!['{']) { - mod_item_list(p); - } else if !p.eat(T![;]) { - p.error("expected `;` or `{`"); - } - m.complete(p, MODULE); -} - -pub(crate) fn mod_item_list(p: &mut Parser) { - assert!(p.at(T!['{'])); - let m = p.start(); - p.bump(T!['{']); - mod_contents(p, true); - p.expect(T!['}']); - m.complete(p, ITEM_LIST); -} - -// test macro_def -// macro m { ($i:ident) => {} } -// macro m($i:ident) {} -fn macro_def(p: &mut Parser, m: Marker) { - p.expect(T![macro]); - name_r(p, ITEM_RECOVERY_SET); - if p.at(T!['{']) { - token_tree(p); - } else if !p.at(T!['(']) { - p.error("unmatched `(`"); - } else { - let m = p.start(); - token_tree(p); - match p.current() { - T!['{'] | T!['['] | T!['('] => token_tree(p), - _ => p.error("expected `{`, `[`, `(`"), - } - m.complete(p, TOKEN_TREE); - } - - m.complete(p, MACRO_DEF); -} - -fn macro_call(p: &mut Parser) -> BlockLike { - assert!(paths::is_use_path_start(p)); - paths::use_path(p); - macro_call_after_excl(p) -} - -pub(super) fn macro_call_after_excl(p: &mut Parser) -> BlockLike { - p.expect(T![!]); - if p.at(IDENT) { - name(p); - } - // Special-case `macro_rules! try`. - // This is a hack until we do proper edition support - - // test try_macro_rules - // macro_rules! try { () => {} } - if p.at(T![try]) { - let m = p.start(); - p.bump_remap(IDENT); - m.complete(p, NAME); - } - - match p.current() { - T!['{'] => { - token_tree(p); - BlockLike::Block - } - T!['('] | T!['['] => { - token_tree(p); - BlockLike::NotBlock - } - _ => { - p.error("expected `{`, `[`, `(`"); - BlockLike::NotBlock - } - } -} - -pub(crate) fn token_tree(p: &mut Parser) { - let closing_paren_kind = match p.current() { - T!['{'] => T!['}'], - T!['('] => T![')'], - T!['['] => T![']'], - _ => unreachable!(), - }; - let m = p.start(); - p.bump_any(); - while !p.at(EOF) && !p.at(closing_paren_kind) { - match p.current() { - T!['{'] | T!['('] | T!['['] => token_tree(p), - T!['}'] => { - p.error("unmatched `}`"); - m.complete(p, TOKEN_TREE); - return; - } - T![')'] | T![']'] => p.err_and_bump("unmatched brace"), - _ => p.bump_any(), - } - } - p.expect(closing_paren_kind); - m.complete(p, TOKEN_TREE); -} diff --git a/crates/ra_parser/src/grammar/items/adt.rs b/crates/ra_parser/src/grammar/items/adt.rs deleted file mode 100644 index addfb59d4..000000000 --- a/crates/ra_parser/src/grammar/items/adt.rs +++ /dev/null @@ -1,178 +0,0 @@ -//! FIXME: write short doc here - -use super::*; - -pub(super) fn struct_def(p: &mut Parser, m: Marker) { - assert!(p.at(T![struct])); - p.bump(T![struct]); - struct_or_union(p, m, T![struct], STRUCT); -} - -pub(super) fn union_def(p: &mut Parser, m: Marker) { - assert!(p.at_contextual_kw("union")); - p.bump_remap(T![union]); - struct_or_union(p, m, T![union], UNION); -} - -fn struct_or_union(p: &mut Parser, m: Marker, kw: SyntaxKind, def: SyntaxKind) { - name_r(p, ITEM_RECOVERY_SET); - type_params::opt_type_param_list(p); - match p.current() { - T![where] => { - type_params::opt_where_clause(p); - match p.current() { - T![;] => { - p.bump(T![;]); - } - T!['{'] => record_field_def_list(p), - _ => { - //FIXME: special case `(` error message - p.error("expected `;` or `{`"); - } - } - } - T![;] if kw == T![struct] => { - p.bump(T![;]); - } - T!['{'] => record_field_def_list(p), - T!['('] if kw == T![struct] => { - tuple_field_def_list(p); - // test tuple_struct_where - // struct Test(T) where T: Clone; - // struct Test(T); - type_params::opt_where_clause(p); - p.expect(T![;]); - } - _ if kw == T![struct] => { - p.error("expected `;`, `{`, or `(`"); - } - _ => { - p.error("expected `{`"); - } - } - m.complete(p, def); -} - -pub(super) fn enum_def(p: &mut Parser, m: Marker) { - assert!(p.at(T![enum])); - p.bump(T![enum]); - name_r(p, ITEM_RECOVERY_SET); - type_params::opt_type_param_list(p); - type_params::opt_where_clause(p); - if p.at(T!['{']) { - enum_variant_list(p); - } else { - p.error("expected `{`") - } - m.complete(p, ENUM); -} - -pub(crate) fn enum_variant_list(p: &mut Parser) { - assert!(p.at(T!['{'])); - let m = p.start(); - p.bump(T!['{']); - while !p.at(EOF) && !p.at(T!['}']) { - if p.at(T!['{']) { - error_block(p, "expected enum variant"); - continue; - } - let var = p.start(); - attributes::outer_attributes(p); - if p.at(IDENT) { - name(p); - match p.current() { - T!['{'] => record_field_def_list(p), - T!['('] => tuple_field_def_list(p), - _ => (), - } - - // test variant_discriminant - // enum E { X(i32) = 10 } - if p.eat(T![=]) { - expressions::expr(p); - } - var.complete(p, VARIANT); - } else { - var.abandon(p); - p.err_and_bump("expected enum variant"); - } - if !p.at(T!['}']) { - p.expect(T![,]); - } - } - p.expect(T!['}']); - m.complete(p, VARIANT_LIST); -} - -pub(crate) fn record_field_def_list(p: &mut Parser) { - assert!(p.at(T!['{'])); - let m = p.start(); - p.bump(T!['{']); - while !p.at(T!['}']) && !p.at(EOF) { - if p.at(T!['{']) { - error_block(p, "expected field"); - continue; - } - record_field_def(p); - if !p.at(T!['}']) { - p.expect(T![,]); - } - } - p.expect(T!['}']); - m.complete(p, RECORD_FIELD_LIST); - - fn record_field_def(p: &mut Parser) { - let m = p.start(); - // test record_field_attrs - // struct S { - // #[serde(with = "url_serde")] - // pub uri: Uri, - // } - attributes::outer_attributes(p); - opt_visibility(p); - if p.at(IDENT) { - name(p); - p.expect(T![:]); - types::type_(p); - m.complete(p, RECORD_FIELD); - } else { - m.abandon(p); - p.err_and_bump("expected field declaration"); - } - } -} - -fn tuple_field_def_list(p: &mut Parser) { - assert!(p.at(T!['('])); - let m = p.start(); - if !p.expect(T!['(']) { - return; - } - while !p.at(T![')']) && !p.at(EOF) { - let m = p.start(); - // test tuple_field_attrs - // struct S ( - // #[serde(with = "url_serde")] - // pub Uri, - // ); - // - // enum S { - // Uri(#[serde(with = "url_serde")] Uri), - // } - attributes::outer_attributes(p); - opt_visibility(p); - if !p.at_ts(types::TYPE_FIRST) { - p.error("expected a type"); - m.complete(p, ERROR); - break; - } - types::type_(p); - m.complete(p, TUPLE_FIELD); - - if !p.at(T![')']) { - p.expect(T![,]); - } - } - p.expect(T![')']); - m.complete(p, TUPLE_FIELD_LIST); -} diff --git a/crates/ra_parser/src/grammar/items/consts.rs b/crates/ra_parser/src/grammar/items/consts.rs deleted file mode 100644 index 35ad766dc..000000000 --- a/crates/ra_parser/src/grammar/items/consts.rs +++ /dev/null @@ -1,33 +0,0 @@ -//! FIXME: write short doc here - -use super::*; - -pub(super) fn static_def(p: &mut Parser, m: Marker) { - const_or_static(p, m, T![static], STATIC) -} - -pub(super) fn const_def(p: &mut Parser, m: Marker) { - const_or_static(p, m, T![const], CONST) -} - -fn const_or_static(p: &mut Parser, m: Marker, kw: SyntaxKind, def: SyntaxKind) { - assert!(p.at(kw)); - p.bump(kw); - p.eat(T![mut]); // FIXME: validator to forbid const mut - - // Allow `_` in place of an identifier in a `const`. - let is_const_underscore = kw == T![const] && p.eat(T![_]); - if !is_const_underscore { - name(p); - } - - // test_err static_underscore - // static _: i32 = 5; - - types::ascription(p); - if p.eat(T![=]) { - expressions::expr(p); - } - p.expect(T![;]); - m.complete(p, def); -} diff --git a/crates/ra_parser/src/grammar/items/traits.rs b/crates/ra_parser/src/grammar/items/traits.rs deleted file mode 100644 index 751ce65f2..000000000 --- a/crates/ra_parser/src/grammar/items/traits.rs +++ /dev/null @@ -1,153 +0,0 @@ -//! FIXME: write short doc here - -use super::*; - -// test trait_item -// trait T: Hash + Clone where U: Copy {} -// trait X: Hash + Clone where U: Copy {} -pub(super) fn trait_def(p: &mut Parser) { - assert!(p.at(T![trait])); - p.bump(T![trait]); - name_r(p, ITEM_RECOVERY_SET); - type_params::opt_type_param_list(p); - // test trait_alias - // trait Z = T; - // trait Z = T where U: Copy; - // trait Z = where Self: T; - if p.eat(T![=]) { - type_params::bounds_without_colon(p); - type_params::opt_where_clause(p); - p.expect(T![;]); - return; - } - if p.at(T![:]) { - type_params::bounds(p); - } - type_params::opt_where_clause(p); - if p.at(T!['{']) { - trait_item_list(p); - } else { - p.error("expected `{`"); - } -} - -// test trait_item_list -// impl F { -// type A: Clone; -// const B: i32; -// fn foo() {} -// fn bar(&self); -// } -pub(crate) fn trait_item_list(p: &mut Parser) { - assert!(p.at(T!['{'])); - let m = p.start(); - p.bump(T!['{']); - while !p.at(EOF) && !p.at(T!['}']) { - if p.at(T!['{']) { - error_block(p, "expected an item"); - continue; - } - item_or_macro(p, true); - } - p.expect(T!['}']); - m.complete(p, ASSOC_ITEM_LIST); -} - -// test impl_def -// impl Foo {} -pub(super) fn impl_def(p: &mut Parser) { - assert!(p.at(T![impl])); - p.bump(T![impl]); - if choose_type_params_over_qpath(p) { - type_params::opt_type_param_list(p); - } - - // FIXME: never type - // impl ! {} - - // test impl_def_neg - // impl !Send for X {} - p.eat(T![!]); - impl_type(p); - if p.eat(T![for]) { - impl_type(p); - } - type_params::opt_where_clause(p); - if p.at(T!['{']) { - impl_item_list(p); - } else { - p.error("expected `{`"); - } -} - -// test impl_item_list -// impl F { -// type A = i32; -// const B: i32 = 92; -// fn foo() {} -// fn bar(&self) {} -// } -pub(crate) fn impl_item_list(p: &mut Parser) { - assert!(p.at(T!['{'])); - let m = p.start(); - p.bump(T!['{']); - // test impl_inner_attributes - // enum F{} - // impl F { - // //! This is a doc comment - // #![doc("This is also a doc comment")] - // } - attributes::inner_attributes(p); - - while !p.at(EOF) && !p.at(T!['}']) { - if p.at(T!['{']) { - error_block(p, "expected an item"); - continue; - } - item_or_macro(p, true); - } - p.expect(T!['}']); - m.complete(p, ASSOC_ITEM_LIST); -} - -// test impl_type_params -// impl Bar {} -fn choose_type_params_over_qpath(p: &Parser) -> bool { - // There's an ambiguity between generic parameters and qualified paths in impls. - // If we see `<` it may start both, so we have to inspect some following tokens. - // The following combinations can only start generics, - // but not qualified paths (with one exception): - // `<` `>` - empty generic parameters - // `<` `#` - generic parameters with attributes - // `<` `const` - const generic parameters - // `<` (LIFETIME|IDENT) `>` - single generic parameter - // `<` (LIFETIME|IDENT) `,` - first generic parameter in a list - // `<` (LIFETIME|IDENT) `:` - generic parameter with bounds - // `<` (LIFETIME|IDENT) `=` - generic parameter with a default - // The only truly ambiguous case is - // `<` IDENT `>` `::` IDENT ... - // we disambiguate it in favor of generics (`impl ::absolute::Path { ... }`) - // because this is what almost always expected in practice, qualified paths in impls - // (`impl ::AssocTy { ... }`) aren't even allowed by type checker at the moment. - if !p.at(T![<]) { - return false; - } - if p.nth(1) == T![#] || p.nth(1) == T![>] || p.nth(1) == CONST_KW { - return true; - } - (p.nth(1) == LIFETIME || p.nth(1) == IDENT) - && (p.nth(2) == T![>] || p.nth(2) == T![,] || p.nth(2) == T![:] || p.nth(2) == T![=]) -} - -// test_err impl_type -// impl Type {} -// impl Trait1 for T {} -// impl impl NotType {} -// impl Trait2 for impl NotType {} -pub(crate) fn impl_type(p: &mut Parser) { - if p.at(T![impl]) { - p.error("expected trait or type"); - return; - } - types::type_(p); -} diff --git a/crates/ra_parser/src/grammar/items/use_item.rs b/crates/ra_parser/src/grammar/items/use_item.rs deleted file mode 100644 index 8e836a77e..000000000 --- a/crates/ra_parser/src/grammar/items/use_item.rs +++ /dev/null @@ -1,132 +0,0 @@ -//! FIXME: write short doc here - -use super::*; - -pub(super) fn use_item(p: &mut Parser, m: Marker) { - assert!(p.at(T![use])); - p.bump(T![use]); - use_tree(p, true); - p.expect(T![;]); - m.complete(p, USE); -} - -/// Parse a use 'tree', such as `some::path` in `use some::path;` -/// Note that this is called both by `use_item` and `use_tree_list`, -/// so handles both `some::path::{inner::path}` and `inner::path` in -/// `use some::path::{inner::path};` -fn use_tree(p: &mut Parser, top_level: bool) { - let m = p.start(); - match p.current() { - // Finish the use_tree for cases of e.g. - // `use some::path::{self, *};` or `use *;` - // This does not handle cases such as `use some::path::*` - // N.B. in Rust 2015 `use *;` imports all from crate root - // however in Rust 2018 `use *;` errors: ('cannot glob-import all possible crates') - // FIXME: Add this error (if not out of scope) - - // test use_star - // use *; - // use ::*; - // use some::path::{*}; - // use some::path::{::*}; - T![*] => p.bump(T![*]), - T![:] if p.at(T![::]) && p.nth(2) == T![*] => { - // Parse `use ::*;`, which imports all from the crate root in Rust 2015 - // This is invalid inside a use_tree_list, (e.g. `use some::path::{::*}`) - // but still parses and errors later: ('crate root in paths can only be used in start position') - // FIXME: Add this error (if not out of scope) - // In Rust 2018, it is always invalid (see above) - p.bump(T![::]); - p.bump(T![*]); - } - // Open a use tree list - // Handles cases such as `use {some::path};` or `{inner::path}` in - // `use some::path::{{inner::path}, other::path}` - - // test use_tree_list - // use {crate::path::from::root, or::path::from::crate_name}; // Rust 2018 (with a crate named `or`) - // use {path::from::root}; // Rust 2015 - // use ::{some::arbritrary::path}; // Rust 2015 - // use ::{{{root::export}}}; // Nonsensical but perfectly legal nesting - T!['{'] => { - use_tree_list(p); - } - T![:] if p.at(T![::]) && p.nth(2) == T!['{'] => { - p.bump(T![::]); - use_tree_list(p); - } - // Parse a 'standard' path. - // Also handles aliases (e.g. `use something as something_else`) - - // test use_path - // use ::crate_name; // Rust 2018 - All flavours - // use crate_name; // Rust 2018 - Anchored paths - // use item_in_scope_or_crate_name; // Rust 2018 - Uniform Paths - // - // use self::module::Item; - // use crate::Item; - // use self::some::Struct; - // use crate_name::some_item; - _ if paths::is_use_path_start(p) => { - paths::use_path(p); - match p.current() { - T![as] => { - // test use_alias - // use some::path as some_name; - // use some::{ - // other::path as some_other_name, - // different::path as different_name, - // yet::another::path, - // running::out::of::synonyms::for_::different::* - // }; - // use Trait as _; - opt_alias(p); - } - T![:] if p.at(T![::]) => { - p.bump(T![::]); - match p.current() { - T![*] => { - p.bump(T![*]); - } - // test use_tree_list_after_path - // use crate::{Item}; - // use self::{Item}; - T!['{'] => use_tree_list(p), - _ => { - // is this unreachable? - p.error("expected `{` or `*`"); - } - } - } - _ => (), - } - } - _ => { - m.abandon(p); - let msg = "expected one of `*`, `::`, `{`, `self`, `super` or an identifier"; - if top_level { - p.err_recover(msg, ITEM_RECOVERY_SET); - } else { - // if we are parsing a nested tree, we have to eat a token to - // main balanced `{}` - p.err_and_bump(msg); - } - return; - } - } - m.complete(p, USE_TREE); -} - -pub(crate) fn use_tree_list(p: &mut Parser) { - assert!(p.at(T!['{'])); - let m = p.start(); - p.bump(T!['{']); - while !p.at(EOF) && !p.at(T!['}']) { - use_tree(p, false); - if !p.at(T!['}']) { - p.expect(T![,]); - } - } - p.expect(T!['}']); - m.complete(p, USE_TREE_LIST); -} diff --git a/crates/ra_parser/src/grammar/params.rs b/crates/ra_parser/src/grammar/params.rs deleted file mode 100644 index f0da173cc..000000000 --- a/crates/ra_parser/src/grammar/params.rs +++ /dev/null @@ -1,188 +0,0 @@ -//! FIXME: write short doc here - -use super::*; - -// test param_list -// fn a() {} -// fn b(x: i32) {} -// fn c(x: i32, ) {} -// fn d(x: i32, y: ()) {} -pub(super) fn param_list_fn_def(p: &mut Parser) { - list_(p, Flavor::FnDef) -} - -// test param_list_opt_patterns -// fn foo)>(){} -pub(super) fn param_list_fn_trait(p: &mut Parser) { - list_(p, Flavor::FnTrait) -} - -pub(super) fn param_list_fn_ptr(p: &mut Parser) { - list_(p, Flavor::FnPointer) -} - -pub(super) fn param_list_closure(p: &mut Parser) { - list_(p, Flavor::Closure) -} - -#[derive(Debug, Clone, Copy)] -enum Flavor { - FnDef, // Includes trait fn params; omitted param idents are not supported - FnTrait, // Params for `Fn(...)`/`FnMut(...)`/`FnOnce(...)` annotations - FnPointer, - Closure, -} - -fn list_(p: &mut Parser, flavor: Flavor) { - use Flavor::*; - - let (bra, ket) = match flavor { - Closure => (T![|], T![|]), - FnDef | FnTrait | FnPointer => (T!['('], T![')']), - }; - - let m = p.start(); - p.bump(bra); - - if let FnDef = flavor { - // test self_param_outer_attr - // fn f(#[must_use] self) {} - attributes::outer_attributes(p); - opt_self_param(p); - } - - while !p.at(EOF) && !p.at(ket) { - // test param_outer_arg - // fn f(#[attr1] pat: Type) {} - attributes::outer_attributes(p); - - if !p.at_ts(VALUE_PARAMETER_FIRST) { - p.error("expected value parameter"); - break; - } - let param = value_parameter(p, flavor); - if !p.at(ket) { - p.expect(T![,]); - } - if let Variadic(true) = param { - break; - } - } - - p.expect(ket); - m.complete(p, PARAM_LIST); -} - -const VALUE_PARAMETER_FIRST: TokenSet = patterns::PATTERN_FIRST.union(types::TYPE_FIRST); - -struct Variadic(bool); - -fn value_parameter(p: &mut Parser, flavor: Flavor) -> Variadic { - let mut res = Variadic(false); - let m = p.start(); - match flavor { - // test param_list_vararg - // extern "C" { fn printf(format: *const i8, ...) -> i32; } - Flavor::FnDef | Flavor::FnPointer if p.eat(T![...]) => res = Variadic(true), - - // test fn_def_param - // fn foo((x, y): (i32, i32)) {} - Flavor::FnDef => { - patterns::pattern(p); - if variadic_param(p) { - res = Variadic(true) - } else { - types::ascription(p); - } - } - // test value_parameters_no_patterns - // type F = Box; - Flavor::FnTrait => { - types::type_(p); - } - // test fn_pointer_param_ident_path - // type Foo = fn(Bar::Baz); - // type Qux = fn(baz: Bar::Baz); - - // test fn_pointer_unnamed_arg - // type Foo = fn(_: bar); - Flavor::FnPointer => { - if (p.at(IDENT) || p.at(UNDERSCORE)) && p.nth(1) == T![:] && !p.nth_at(1, T![::]) { - patterns::pattern_single(p); - if variadic_param(p) { - res = Variadic(true) - } else { - types::ascription(p); - } - } else { - types::type_(p); - } - } - // test closure_params - // fn main() { - // let foo = |bar, baz: Baz, qux: Qux::Quux| (); - // } - Flavor::Closure => { - patterns::pattern_single(p); - if p.at(T![:]) && !p.at(T![::]) { - types::ascription(p); - } - } - } - m.complete(p, PARAM); - res -} - -fn variadic_param(p: &mut Parser) -> bool { - if p.at(T![:]) && p.nth_at(1, T![...]) { - p.bump(T![:]); - p.bump(T![...]); - true - } else { - false - } -} - -// test self_param -// impl S { -// fn a(self) {} -// fn b(&self,) {} -// fn c(&'a self,) {} -// fn d(&'a mut self, x: i32) {} -// fn e(mut self) {} -// } -fn opt_self_param(p: &mut Parser) { - let m; - if p.at(T![self]) || p.at(T![mut]) && p.nth(1) == T![self] { - m = p.start(); - p.eat(T![mut]); - p.eat(T![self]); - // test arb_self_types - // impl S { - // fn a(self: &Self) {} - // fn b(mut self: Box) {} - // } - if p.at(T![:]) { - types::ascription(p); - } - } else { - let la1 = p.nth(1); - let la2 = p.nth(2); - let la3 = p.nth(3); - let n_toks = match (p.current(), la1, la2, la3) { - (T![&], T![self], _, _) => 2, - (T![&], T![mut], T![self], _) => 3, - (T![&], LIFETIME, T![self], _) => 3, - (T![&], LIFETIME, T![mut], T![self]) => 4, - _ => return, - }; - m = p.start(); - for _ in 0..n_toks { - p.bump_any(); - } - } - m.complete(p, SELF_PARAM); - if !p.at(T![')']) { - p.expect(T![,]); - } -} diff --git a/crates/ra_parser/src/grammar/paths.rs b/crates/ra_parser/src/grammar/paths.rs deleted file mode 100644 index b503af1dc..000000000 --- a/crates/ra_parser/src/grammar/paths.rs +++ /dev/null @@ -1,115 +0,0 @@ -//! FIXME: write short doc here - -use super::*; - -pub(super) const PATH_FIRST: TokenSet = - token_set![IDENT, T![self], T![super], T![crate], T![:], T![<]]; - -pub(super) fn is_path_start(p: &Parser) -> bool { - is_use_path_start(p) || p.at(T![<]) -} - -pub(super) fn is_use_path_start(p: &Parser) -> bool { - match p.current() { - IDENT | T![self] | T![super] | T![crate] => true, - T![:] if p.at(T![::]) => true, - _ => false, - } -} - -pub(super) fn use_path(p: &mut Parser) { - path(p, Mode::Use) -} - -pub(crate) fn type_path(p: &mut Parser) { - path(p, Mode::Type) -} - -pub(super) fn expr_path(p: &mut Parser) { - path(p, Mode::Expr) -} - -#[derive(Clone, Copy, Eq, PartialEq)] -enum Mode { - Use, - Type, - Expr, -} - -fn path(p: &mut Parser, mode: Mode) { - let path = p.start(); - path_segment(p, mode, true); - let mut qual = path.complete(p, PATH); - loop { - let use_tree = matches!(p.nth(2), T![*] | T!['{']); - if p.at(T![::]) && !use_tree { - let path = qual.precede(p); - p.bump(T![::]); - path_segment(p, mode, false); - let path = path.complete(p, PATH); - qual = path; - } else { - break; - } - } -} - -fn path_segment(p: &mut Parser, mode: Mode, first: bool) { - let m = p.start(); - // test qual_paths - // type X = ::Output; - // fn foo() { ::default(); } - if first && p.eat(T![<]) { - types::type_(p); - if p.eat(T![as]) { - if is_use_path_start(p) { - types::path_type(p); - } else { - p.error("expected a trait"); - } - } - p.expect(T![>]); - } else { - let mut empty = true; - if first { - p.eat(T![::]); - empty = false; - } - match p.current() { - IDENT => { - name_ref(p); - opt_path_type_args(p, mode); - } - // test crate_path - // use crate::foo; - T![self] | T![super] | T![crate] => p.bump_any(), - _ => { - p.err_recover("expected identifier", items::ITEM_RECOVERY_SET); - if empty { - // test_err empty_segment - // use crate::; - m.abandon(p); - return; - } - } - }; - } - m.complete(p, PATH_SEGMENT); -} - -fn opt_path_type_args(p: &mut Parser, mode: Mode) { - match mode { - Mode::Use => {} - Mode::Type => { - // test path_fn_trait_args - // type F = Box ()>; - if p.at(T!['(']) { - params::param_list_fn_trait(p); - opt_fn_ret_type(p); - } else { - type_args::opt_type_arg_list(p, false) - } - } - Mode::Expr => type_args::opt_type_arg_list(p, true), - } -} diff --git a/crates/ra_parser/src/grammar/patterns.rs b/crates/ra_parser/src/grammar/patterns.rs deleted file mode 100644 index 716bdc978..000000000 --- a/crates/ra_parser/src/grammar/patterns.rs +++ /dev/null @@ -1,379 +0,0 @@ -//! FIXME: write short doc here - -use super::*; - -pub(super) const PATTERN_FIRST: TokenSet = expressions::LITERAL_FIRST - .union(paths::PATH_FIRST) - .union(token_set![T![box], T![ref], T![mut], T!['('], T!['['], T![&], T![_], T![-], T![.]]); - -pub(crate) fn pattern(p: &mut Parser) { - pattern_r(p, PAT_RECOVERY_SET); -} - -/// Parses a pattern list separated by pipes `|` -pub(super) fn pattern_top(p: &mut Parser) { - pattern_top_r(p, PAT_RECOVERY_SET) -} - -pub(crate) fn pattern_single(p: &mut Parser) { - pattern_single_r(p, PAT_RECOVERY_SET); -} - -/// Parses a pattern list separated by pipes `|` -/// using the given `recovery_set` -pub(super) fn pattern_top_r(p: &mut Parser, recovery_set: TokenSet) { - p.eat(T![|]); - pattern_r(p, recovery_set); -} - -/// Parses a pattern list separated by pipes `|`, with no leading `|`,using the -/// given `recovery_set` -// test or_pattern -// fn main() { -// match () { -// (_ | _) => (), -// &(_ | _) => (), -// (_ | _,) => (), -// [_ | _,] => (), -// } -// } -fn pattern_r(p: &mut Parser, recovery_set: TokenSet) { - let m = p.start(); - pattern_single_r(p, recovery_set); - - if !p.at(T![|]) { - m.abandon(p); - return; - } - while p.eat(T![|]) { - pattern_single_r(p, recovery_set); - } - m.complete(p, OR_PAT); -} - -fn pattern_single_r(p: &mut Parser, recovery_set: TokenSet) { - if let Some(lhs) = atom_pat(p, recovery_set) { - // test range_pat - // fn main() { - // match 92 { - // 0 ... 100 => (), - // 101 ..= 200 => (), - // 200 .. 301=> (), - // } - // } - for &range_op in [T![...], T![..=], T![..]].iter() { - if p.at(range_op) { - let m = lhs.precede(p); - p.bump(range_op); - atom_pat(p, recovery_set); - m.complete(p, RANGE_PAT); - return; - } - } - } -} - -const PAT_RECOVERY_SET: TokenSet = - token_set![LET_KW, IF_KW, WHILE_KW, LOOP_KW, MATCH_KW, R_PAREN, COMMA]; - -fn atom_pat(p: &mut Parser, recovery_set: TokenSet) -> Option { - let m = match p.nth(0) { - T![box] => box_pat(p), - T![ref] | T![mut] => bind_pat(p, true), - IDENT => match p.nth(1) { - // Checks the token after an IDENT to see if a pattern is a path (Struct { .. }) or macro - // (T![x]). - T!['('] | T!['{'] | T![!] => path_or_macro_pat(p), - T![:] if p.nth_at(1, T![::]) => path_or_macro_pat(p), - _ => bind_pat(p, true), - }, - - // test type_path_in_pattern - // fn main() { let <_>::Foo = (); } - _ if paths::is_path_start(p) => path_or_macro_pat(p), - _ if is_literal_pat_start(p) => literal_pat(p), - - T![.] if p.at(T![..]) => dot_dot_pat(p), - T![_] => placeholder_pat(p), - T![&] => ref_pat(p), - T!['('] => tuple_pat(p), - T!['['] => slice_pat(p), - - _ => { - p.err_recover("expected pattern", recovery_set); - return None; - } - }; - - Some(m) -} - -fn is_literal_pat_start(p: &Parser) -> bool { - p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER) - || p.at_ts(expressions::LITERAL_FIRST) -} - -// test literal_pattern -// fn main() { -// match () { -// -1 => (), -// 92 => (), -// 'c' => (), -// "hello" => (), -// } -// } -fn literal_pat(p: &mut Parser) -> CompletedMarker { - assert!(is_literal_pat_start(p)); - let m = p.start(); - if p.at(T![-]) { - p.bump(T![-]); - } - expressions::literal(p); - m.complete(p, LITERAL_PAT) -} - -// test path_part -// fn foo() { -// let foo::Bar = (); -// let ::Bar = (); -// let Bar { .. } = (); -// let Bar(..) = (); -// } -fn path_or_macro_pat(p: &mut Parser) -> CompletedMarker { - assert!(paths::is_path_start(p)); - let m = p.start(); - paths::expr_path(p); - let kind = match p.current() { - T!['('] => { - tuple_pat_fields(p); - TUPLE_STRUCT_PAT - } - T!['{'] => { - record_field_pat_list(p); - RECORD_PAT - } - // test marco_pat - // fn main() { - // let m!(x) = 0; - // } - T![!] => { - items::macro_call_after_excl(p); - return m.complete(p, MACRO_CALL).precede(p).complete(p, MACRO_PAT); - } - _ => PATH_PAT, - }; - m.complete(p, kind) -} - -// test tuple_pat_fields -// fn foo() { -// let S() = (); -// let S(_) = (); -// let S(_,) = (); -// let S(_, .. , x) = (); -// } -fn tuple_pat_fields(p: &mut Parser) { - assert!(p.at(T!['('])); - p.bump(T!['(']); - pat_list(p, T![')']); - p.expect(T![')']); -} - -// test record_field_pat_list -// fn foo() { -// let S {} = (); -// let S { f, ref mut g } = (); -// let S { h: _, ..} = (); -// let S { h: _, } = (); -// } -fn record_field_pat_list(p: &mut Parser) { - assert!(p.at(T!['{'])); - let m = p.start(); - p.bump(T!['{']); - while !p.at(EOF) && !p.at(T!['}']) { - match p.current() { - // A trailing `..` is *not* treated as a REST_PAT. - T![.] if p.at(T![..]) => p.bump(T![..]), - T!['{'] => error_block(p, "expected ident"), - - c => { - let m = p.start(); - match c { - // test record_field_pat - // fn foo() { - // let S { 0: 1 } = (); - // let S { x: 1 } = (); - // } - IDENT | INT_NUMBER if p.nth(1) == T![:] => { - name_ref_or_index(p); - p.bump(T![:]); - pattern(p); - } - T![box] => { - // FIXME: not all box patterns should be allowed - box_pat(p); - } - _ => { - bind_pat(p, false); - } - } - m.complete(p, RECORD_PAT_FIELD); - } - } - if !p.at(T!['}']) { - p.expect(T![,]); - } - } - p.expect(T!['}']); - m.complete(p, RECORD_PAT_FIELD_LIST); -} - -// test placeholder_pat -// fn main() { let _ = (); } -fn placeholder_pat(p: &mut Parser) -> CompletedMarker { - assert!(p.at(T![_])); - let m = p.start(); - p.bump(T![_]); - m.complete(p, WILDCARD_PAT) -} - -// test dot_dot_pat -// fn main() { -// let .. = (); -// // -// // Tuples -// // -// let (a, ..) = (); -// let (a, ..,) = (); -// let Tuple(a, ..) = (); -// let Tuple(a, ..,) = (); -// let (.., ..) = (); -// let Tuple(.., ..) = (); -// let (.., a, ..) = (); -// let Tuple(.., a, ..) = (); -// // -// // Slices -// // -// let [..] = (); -// let [head, ..] = (); -// let [head, tail @ ..] = (); -// let [head, .., cons] = (); -// let [head, mid @ .., cons] = (); -// let [head, .., .., cons] = (); -// let [head, .., mid, tail @ ..] = (); -// let [head, .., mid, .., cons] = (); -// } -fn dot_dot_pat(p: &mut Parser) -> CompletedMarker { - assert!(p.at(T![..])); - let m = p.start(); - p.bump(T![..]); - m.complete(p, REST_PAT) -} - -// test ref_pat -// fn main() { -// let &a = (); -// let &mut b = (); -// } -fn ref_pat(p: &mut Parser) -> CompletedMarker { - assert!(p.at(T![&])); - let m = p.start(); - p.bump(T![&]); - p.eat(T![mut]); - pattern_single(p); - m.complete(p, REF_PAT) -} - -// test tuple_pat -// fn main() { -// let (a, b, ..) = (); -// let (a,) = (); -// let (..) = (); -// let () = (); -// } -fn tuple_pat(p: &mut Parser) -> CompletedMarker { - assert!(p.at(T!['('])); - let m = p.start(); - p.bump(T!['(']); - let mut has_comma = false; - let mut has_pat = false; - let mut has_rest = false; - while !p.at(EOF) && !p.at(T![')']) { - has_pat = true; - if !p.at_ts(PATTERN_FIRST) { - p.error("expected a pattern"); - break; - } - has_rest |= p.at(T![..]); - - pattern(p); - if !p.at(T![')']) { - has_comma = true; - p.expect(T![,]); - } - } - p.expect(T![')']); - - m.complete(p, if !has_comma && !has_rest && has_pat { PAREN_PAT } else { TUPLE_PAT }) -} - -// test slice_pat -// fn main() { -// let [a, b, ..] = []; -// } -fn slice_pat(p: &mut Parser) -> CompletedMarker { - assert!(p.at(T!['['])); - let m = p.start(); - p.bump(T!['[']); - pat_list(p, T![']']); - p.expect(T![']']); - m.complete(p, SLICE_PAT) -} - -fn pat_list(p: &mut Parser, ket: SyntaxKind) { - while !p.at(EOF) && !p.at(ket) { - if !p.at_ts(PATTERN_FIRST) { - p.error("expected a pattern"); - break; - } - - pattern(p); - if !p.at(ket) { - p.expect(T![,]); - } - } -} - -// test bind_pat -// fn main() { -// let a = (); -// let mut b = (); -// let ref c = (); -// let ref mut d = (); -// let e @ _ = (); -// let ref mut f @ g @ _ = (); -// } -fn bind_pat(p: &mut Parser, with_at: bool) -> CompletedMarker { - let m = p.start(); - p.eat(T![ref]); - p.eat(T![mut]); - name(p); - if with_at && p.eat(T![@]) { - pattern_single(p); - } - m.complete(p, IDENT_PAT) -} - -// test box_pat -// fn main() { -// let box i = (); -// let box Outer { box i, j: box Inner(box &x) } = (); -// let box ref mut i = (); -// } -fn box_pat(p: &mut Parser) -> CompletedMarker { - assert!(p.at(T![box])); - let m = p.start(); - p.bump(T![box]); - pattern_single(p); - m.complete(p, BOX_PAT) -} diff --git a/crates/ra_parser/src/grammar/type_args.rs b/crates/ra_parser/src/grammar/type_args.rs deleted file mode 100644 index aef7cd6fb..000000000 --- a/crates/ra_parser/src/grammar/type_args.rs +++ /dev/null @@ -1,63 +0,0 @@ -//! FIXME: write short doc here - -use super::*; - -pub(super) fn opt_type_arg_list(p: &mut Parser, colon_colon_required: bool) { - let m; - if p.at(T![::]) && p.nth(2) == T![<] { - m = p.start(); - p.bump(T![::]); - p.bump(T![<]); - } else if !colon_colon_required && p.at(T![<]) && p.nth(1) != T![=] { - m = p.start(); - p.bump(T![<]); - } else { - return; - } - - while !p.at(EOF) && !p.at(T![>]) { - type_arg(p); - if !p.at(T![>]) && !p.expect(T![,]) { - break; - } - } - p.expect(T![>]); - m.complete(p, GENERIC_ARG_LIST); -} - -// test type_arg -// type A = B<'static, i32, 1, { 2 }, Item=u64>; -fn type_arg(p: &mut Parser) { - let m = p.start(); - match p.current() { - LIFETIME => { - p.bump(LIFETIME); - m.complete(p, LIFETIME_ARG); - } - // test associated_type_bounds - // fn print_all>(printables: T) {} - IDENT if p.nth(1) == T![:] && p.nth(2) != T![:] => { - name_ref(p); - type_params::bounds(p); - m.complete(p, ASSOC_TYPE_ARG); - } - IDENT if p.nth(1) == T![=] => { - name_ref(p); - p.bump_any(); - types::type_(p); - m.complete(p, ASSOC_TYPE_ARG); - } - T!['{'] => { - expressions::block_expr(p); - m.complete(p, CONST_ARG); - } - k if k.is_literal() => { - expressions::literal(p); - m.complete(p, CONST_ARG); - } - _ => { - types::type_(p); - m.complete(p, TYPE_ARG); - } - } -} diff --git a/crates/ra_parser/src/grammar/type_params.rs b/crates/ra_parser/src/grammar/type_params.rs deleted file mode 100644 index 90dabb4c0..000000000 --- a/crates/ra_parser/src/grammar/type_params.rs +++ /dev/null @@ -1,209 +0,0 @@ -//! FIXME: write short doc here - -use super::*; - -pub(super) fn opt_type_param_list(p: &mut Parser) { - if !p.at(T![<]) { - return; - } - type_param_list(p); -} - -fn type_param_list(p: &mut Parser) { - assert!(p.at(T![<])); - let m = p.start(); - p.bump(T![<]); - - while !p.at(EOF) && !p.at(T![>]) { - let m = p.start(); - - // test generic_lifetime_type_attribute - // fn foo<#[derive(Lifetime)] 'a, #[derive(Type)] T>(_: &'a T) { - // } - attributes::outer_attributes(p); - - match p.current() { - LIFETIME => lifetime_param(p, m), - IDENT => type_param(p, m), - CONST_KW => type_const_param(p, m), - _ => { - m.abandon(p); - p.err_and_bump("expected type parameter") - } - } - if !p.at(T![>]) && !p.expect(T![,]) { - break; - } - } - p.expect(T![>]); - m.complete(p, GENERIC_PARAM_LIST); -} - -fn lifetime_param(p: &mut Parser, m: Marker) { - assert!(p.at(LIFETIME)); - p.bump(LIFETIME); - if p.at(T![:]) { - lifetime_bounds(p); - } - m.complete(p, LIFETIME_PARAM); -} - -fn type_param(p: &mut Parser, m: Marker) { - assert!(p.at(IDENT)); - name(p); - if p.at(T![:]) { - bounds(p); - } - // test type_param_default - // struct S; - if p.at(T![=]) { - p.bump(T![=]); - types::type_(p) - } - m.complete(p, TYPE_PARAM); -} - -// test const_param -// struct S; -fn type_const_param(p: &mut Parser, m: Marker) { - assert!(p.at(CONST_KW)); - p.bump(T![const]); - name(p); - types::ascription(p); - m.complete(p, CONST_PARAM); -} - -// test type_param_bounds -// struct S; -pub(super) fn bounds(p: &mut Parser) { - assert!(p.at(T![:])); - p.bump(T![:]); - bounds_without_colon(p); -} - -fn lifetime_bounds(p: &mut Parser) { - assert!(p.at(T![:])); - p.bump(T![:]); - while p.at(LIFETIME) { - p.bump(LIFETIME); - if !p.eat(T![+]) { - break; - } - } -} - -pub(super) fn bounds_without_colon_m(p: &mut Parser, marker: Marker) -> CompletedMarker { - while type_bound(p) { - if !p.eat(T![+]) { - break; - } - } - - marker.complete(p, TYPE_BOUND_LIST) -} - -pub(super) fn bounds_without_colon(p: &mut Parser) { - let m = p.start(); - bounds_without_colon_m(p, m); -} - -fn type_bound(p: &mut Parser) -> bool { - let m = p.start(); - let has_paren = p.eat(T!['(']); - p.eat(T![?]); - match p.current() { - LIFETIME => p.bump(LIFETIME), - T![for] => types::for_type(p), - _ if paths::is_use_path_start(p) => types::path_type_(p, false), - _ => { - m.abandon(p); - return false; - } - } - if has_paren { - p.expect(T![')']); - } - m.complete(p, TYPE_BOUND); - - true -} - -// test where_clause -// fn foo() -// where -// 'a: 'b + 'c, -// T: Clone + Copy + 'static, -// Iterator::Item: 'a, -// ::Item: 'a -// {} -pub(super) fn opt_where_clause(p: &mut Parser) { - if !p.at(T![where]) { - return; - } - let m = p.start(); - p.bump(T![where]); - - while is_where_predicate(p) { - where_predicate(p); - - let comma = p.eat(T![,]); - - if is_where_clause_end(p) { - break; - } - - if !comma { - p.error("expected comma"); - } - } - - m.complete(p, WHERE_CLAUSE); -} - -fn is_where_predicate(p: &mut Parser) -> bool { - match p.current() { - LIFETIME => true, - T![impl] => false, - token => types::TYPE_FIRST.contains(token), - } -} - -fn is_where_clause_end(p: &mut Parser) -> bool { - matches!(p.current(), T!['{'] | T![;] | T![=]) -} - -fn where_predicate(p: &mut Parser) { - let m = p.start(); - match p.current() { - LIFETIME => { - p.bump(LIFETIME); - if p.at(T![:]) { - bounds(p); - } else { - p.error("expected colon"); - } - } - T![impl] => { - p.error("expected lifetime or type"); - } - _ => { - // test where_pred_for - // fn for_trait() - // where - // for<'a> F: Fn(&'a str) - // { } - if p.at(T![for]) { - types::for_binder(p); - } - - types::type_(p); - - if p.at(T![:]) { - bounds(p); - } else { - p.error("expected colon"); - } - } - } - m.complete(p, WHERE_PRED); -} diff --git a/crates/ra_parser/src/grammar/types.rs b/crates/ra_parser/src/grammar/types.rs deleted file mode 100644 index 0aa173a52..000000000 --- a/crates/ra_parser/src/grammar/types.rs +++ /dev/null @@ -1,324 +0,0 @@ -//! FIXME: write short doc here - -use super::*; - -pub(super) const TYPE_FIRST: TokenSet = paths::PATH_FIRST.union(token_set![ - T!['('], - T!['['], - T![<], - T![!], - T![*], - T![&], - T![_], - T![fn], - T![unsafe], - T![extern], - T![for], - T![impl], - T![dyn], -]); - -const TYPE_RECOVERY_SET: TokenSet = token_set![R_PAREN, COMMA, L_DOLLAR]; - -pub(crate) fn type_(p: &mut Parser) { - type_with_bounds_cond(p, true); -} - -pub(super) fn type_no_bounds(p: &mut Parser) { - type_with_bounds_cond(p, false); -} - -fn type_with_bounds_cond(p: &mut Parser, allow_bounds: bool) { - match p.current() { - T!['('] => paren_or_tuple_type(p), - T![!] => never_type(p), - T![*] => pointer_type(p), - T!['['] => array_or_slice_type(p), - T![&] => reference_type(p), - T![_] => placeholder_type(p), - T![fn] | T![unsafe] | T![extern] => fn_pointer_type(p), - T![for] => for_type(p), - T![impl] => impl_trait_type(p), - T![dyn] => dyn_trait_type(p), - // Some path types are not allowed to have bounds (no plus) - T![<] => path_type_(p, allow_bounds), - _ if paths::is_use_path_start(p) => path_or_macro_type_(p, allow_bounds), - _ => { - p.err_recover("expected type", TYPE_RECOVERY_SET); - } - } -} - -pub(super) fn ascription(p: &mut Parser) { - p.expect(T![:]); - type_(p) -} - -fn paren_or_tuple_type(p: &mut Parser) { - assert!(p.at(T!['('])); - let m = p.start(); - p.bump(T!['(']); - let mut n_types: u32 = 0; - let mut trailing_comma: bool = false; - while !p.at(EOF) && !p.at(T![')']) { - n_types += 1; - type_(p); - if p.eat(T![,]) { - trailing_comma = true; - } else { - trailing_comma = false; - break; - } - } - p.expect(T![')']); - - let kind = if n_types == 1 && !trailing_comma { - // test paren_type - // type T = (i32); - PAREN_TYPE - } else { - // test unit_type - // type T = (); - - // test singleton_tuple_type - // type T = (i32,); - TUPLE_TYPE - }; - m.complete(p, kind); -} - -// test never_type -// type Never = !; -fn never_type(p: &mut Parser) { - assert!(p.at(T![!])); - let m = p.start(); - p.bump(T![!]); - m.complete(p, NEVER_TYPE); -} - -fn pointer_type(p: &mut Parser) { - assert!(p.at(T![*])); - let m = p.start(); - p.bump(T![*]); - - match p.current() { - // test pointer_type_mut - // type M = *mut (); - // type C = *mut (); - T![mut] | T![const] => p.bump_any(), - _ => { - // test_err pointer_type_no_mutability - // type T = *(); - p.error( - "expected mut or const in raw pointer type \ - (use `*mut T` or `*const T` as appropriate)", - ); - } - }; - - type_no_bounds(p); - m.complete(p, PTR_TYPE); -} - -fn array_or_slice_type(p: &mut Parser) { - assert!(p.at(T!['['])); - let m = p.start(); - p.bump(T!['[']); - - type_(p); - let kind = match p.current() { - // test slice_type - // type T = [()]; - T![']'] => { - p.bump(T![']']); - SLICE_TYPE - } - - // test array_type - // type T = [(); 92]; - T![;] => { - p.bump(T![;]); - expressions::expr(p); - p.expect(T![']']); - ARRAY_TYPE - } - // test_err array_type_missing_semi - // type T = [() 92]; - _ => { - p.error("expected `;` or `]`"); - SLICE_TYPE - } - }; - m.complete(p, kind); -} - -// test reference_type; -// type A = &(); -// type B = &'static (); -// type C = &mut (); -fn reference_type(p: &mut Parser) { - assert!(p.at(T![&])); - let m = p.start(); - p.bump(T![&]); - p.eat(LIFETIME); - p.eat(T![mut]); - type_no_bounds(p); - m.complete(p, REF_TYPE); -} - -// test placeholder_type -// type Placeholder = _; -fn placeholder_type(p: &mut Parser) { - assert!(p.at(T![_])); - let m = p.start(); - p.bump(T![_]); - m.complete(p, INFER_TYPE); -} - -// test fn_pointer_type -// type A = fn(); -// type B = unsafe fn(); -// type C = unsafe extern "C" fn(); -// type D = extern "C" fn ( u8 , ... ) -> u8; -fn fn_pointer_type(p: &mut Parser) { - let m = p.start(); - p.eat(T![unsafe]); - if p.at(T![extern]) { - abi(p); - } - // test_err fn_pointer_type_missing_fn - // type F = unsafe (); - if !p.eat(T![fn]) { - m.abandon(p); - p.error("expected `fn`"); - return; - } - if p.at(T!['(']) { - params::param_list_fn_ptr(p); - } else { - p.error("expected parameters") - } - // test fn_pointer_type_with_ret - // type F = fn() -> (); - opt_fn_ret_type(p); - m.complete(p, FN_PTR_TYPE); -} - -pub(super) fn for_binder(p: &mut Parser) { - assert!(p.at(T![for])); - p.bump(T![for]); - if p.at(T![<]) { - type_params::opt_type_param_list(p); - } else { - p.error("expected `<`"); - } -} - -// test for_type -// type A = for<'a> fn() -> (); -// type B = for<'a> unsafe extern "C" fn(&'a ()) -> (); -// type Obj = for<'a> PartialEq<&'a i32>; -pub(super) fn for_type(p: &mut Parser) { - assert!(p.at(T![for])); - let m = p.start(); - for_binder(p); - match p.current() { - T![fn] | T![unsafe] | T![extern] => {} - // OK: legacy trait object format - _ if paths::is_use_path_start(p) => {} - _ => { - p.error("expected a function pointer or path"); - } - } - type_no_bounds(p); - m.complete(p, FOR_TYPE); -} - -// test impl_trait_type -// type A = impl Iterator> + 'a; -fn impl_trait_type(p: &mut Parser) { - assert!(p.at(T![impl])); - let m = p.start(); - p.bump(T![impl]); - type_params::bounds_without_colon(p); - m.complete(p, IMPL_TRAIT_TYPE); -} - -// test dyn_trait_type -// type A = dyn Iterator> + 'a; -fn dyn_trait_type(p: &mut Parser) { - assert!(p.at(T![dyn])); - let m = p.start(); - p.bump(T![dyn]); - type_params::bounds_without_colon(p); - m.complete(p, DYN_TRAIT_TYPE); -} - -// test path_type -// type A = Foo; -// type B = ::Foo; -// type C = self::Foo; -// type D = super::Foo; -pub(super) fn path_type(p: &mut Parser) { - path_type_(p, true) -} - -// test macro_call_type -// type A = foo!(); -// type B = crate::foo!(); -fn path_or_macro_type_(p: &mut Parser, allow_bounds: bool) { - assert!(paths::is_path_start(p)); - let m = p.start(); - paths::type_path(p); - - let kind = if p.at(T![!]) && !p.at(T![!=]) { - items::macro_call_after_excl(p); - MACRO_CALL - } else { - PATH_TYPE - }; - - let path = m.complete(p, kind); - - if allow_bounds { - opt_path_type_bounds_as_dyn_trait_type(p, path); - } -} - -pub(super) fn path_type_(p: &mut Parser, allow_bounds: bool) { - assert!(paths::is_path_start(p)); - let m = p.start(); - paths::type_path(p); - - // test path_type_with_bounds - // fn foo() -> Box {} - // fn foo() -> Box {} - let path = m.complete(p, PATH_TYPE); - if allow_bounds { - opt_path_type_bounds_as_dyn_trait_type(p, path); - } -} - -/// This turns a parsed PATH_TYPE optionally into a DYN_TRAIT_TYPE -/// with a TYPE_BOUND_LIST -fn opt_path_type_bounds_as_dyn_trait_type(p: &mut Parser, path_type_marker: CompletedMarker) { - if !p.at(T![+]) { - return; - } - - // First create a TYPE_BOUND from the completed PATH_TYPE - let m = path_type_marker.precede(p).complete(p, TYPE_BOUND); - - // Next setup a marker for the TYPE_BOUND_LIST - let m = m.precede(p); - - // This gets consumed here so it gets properly set - // in the TYPE_BOUND_LIST - p.eat(T![+]); - - // Parse rest of the bounds into the TYPE_BOUND_LIST - let m = type_params::bounds_without_colon_m(p, m); - - // Finally precede everything with DYN_TRAIT_TYPE - m.precede(p).complete(p, DYN_TRAIT_TYPE); -} diff --git a/crates/ra_parser/src/lib.rs b/crates/ra_parser/src/lib.rs deleted file mode 100644 index eeb8ad66b..000000000 --- a/crates/ra_parser/src/lib.rs +++ /dev/null @@ -1,149 +0,0 @@ -//! The Rust parser. -//! -//! The parser doesn't know about concrete representation of tokens and syntax -//! trees. Abstract `TokenSource` and `TreeSink` traits are used instead. As a -//! consequence, this crates does not contain a lexer. -//! -//! The `Parser` struct from the `parser` module is a cursor into the sequence -//! of tokens. Parsing routines use `Parser` to inspect current state and -//! advance the parsing. -//! -//! The actual parsing happens in the `grammar` module. -//! -//! Tests for this crate live in `ra_syntax` crate. - -#[macro_use] -mod token_set; -#[macro_use] -mod syntax_kind; -mod event; -mod parser; -mod grammar; - -pub(crate) use token_set::TokenSet; - -pub use syntax_kind::SyntaxKind; - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct ParseError(pub Box); - -/// `TokenSource` abstracts the source of the tokens parser operates on. -/// -/// Hopefully this will allow us to treat text and token trees in the same way! -pub trait TokenSource { - fn current(&self) -> Token; - - /// Lookahead n token - fn lookahead_nth(&self, n: usize) -> Token; - - /// bump cursor to next token - fn bump(&mut self); - - /// Is the current token a specified keyword? - fn is_keyword(&self, kw: &str) -> bool; -} - -/// `Token` abstracts the cursor of `TokenSource` operates on. -#[derive(Debug, Copy, Clone, Eq, PartialEq)] -pub struct Token { - /// What is the current token? - pub kind: SyntaxKind, - - /// Is the current token joined to the next one (`> >` vs `>>`). - pub is_jointed_to_next: bool, -} - -/// `TreeSink` abstracts details of a particular syntax tree implementation. -pub trait TreeSink { - /// Adds new token to the current branch. - fn token(&mut self, kind: SyntaxKind, n_tokens: u8); - - /// Start new branch and make it current. - fn start_node(&mut self, kind: SyntaxKind); - - /// Finish current branch and restore previous - /// branch as current. - fn finish_node(&mut self); - - fn error(&mut self, error: ParseError); -} - -fn parse_from_tokens(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink, f: F) -where - F: FnOnce(&mut parser::Parser), -{ - let mut p = parser::Parser::new(token_source); - f(&mut p); - let events = p.finish(); - event::process(tree_sink, events); -} - -/// Parse given tokens into the given sink as a rust file. -pub fn parse(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) { - parse_from_tokens(token_source, tree_sink, grammar::root); -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] -pub enum FragmentKind { - Path, - Expr, - Statement, - Type, - Pattern, - Item, - Block, - Visibility, - MetaItem, - - // These kinds are used when parsing the result of expansion - // FIXME: use separate fragment kinds for macro inputs and outputs? - Items, - Statements, -} - -pub fn parse_fragment( - token_source: &mut dyn TokenSource, - tree_sink: &mut dyn TreeSink, - fragment_kind: FragmentKind, -) { - let parser: fn(&'_ mut parser::Parser) = match fragment_kind { - FragmentKind::Path => grammar::fragments::path, - FragmentKind::Expr => grammar::fragments::expr, - FragmentKind::Type => grammar::fragments::type_, - FragmentKind::Pattern => grammar::fragments::pattern, - FragmentKind::Item => grammar::fragments::item, - FragmentKind::Block => grammar::fragments::block_expr, - FragmentKind::Visibility => grammar::fragments::opt_visibility, - FragmentKind::MetaItem => grammar::fragments::meta_item, - FragmentKind::Statement => grammar::fragments::stmt, - FragmentKind::Items => grammar::fragments::macro_items, - FragmentKind::Statements => grammar::fragments::macro_stmts, - }; - parse_from_tokens(token_source, tree_sink, parser) -} - -/// A parsing function for a specific braced-block. -pub struct Reparser(fn(&mut parser::Parser)); - -impl Reparser { - /// If the node is a braced block, return the corresponding `Reparser`. - pub fn for_node( - node: SyntaxKind, - first_child: Option, - parent: Option, - ) -> Option { - grammar::reparser(node, first_child, parent).map(Reparser) - } - - /// Re-parse given tokens using this `Reparser`. - /// - /// Tokens must start with `{`, end with `}` and form a valid brace - /// sequence. - pub fn parse(self, token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) { - let Reparser(r) = self; - let mut p = parser::Parser::new(token_source); - r(&mut p); - let events = p.finish(); - event::process(tree_sink, events); - } -} diff --git a/crates/ra_parser/src/parser.rs b/crates/ra_parser/src/parser.rs deleted file mode 100644 index d2487acc3..000000000 --- a/crates/ra_parser/src/parser.rs +++ /dev/null @@ -1,350 +0,0 @@ -//! FIXME: write short doc here - -use std::cell::Cell; - -use drop_bomb::DropBomb; - -use crate::{ - event::Event, - ParseError, - SyntaxKind::{self, EOF, ERROR, TOMBSTONE}, - TokenSet, TokenSource, T, -}; - -/// `Parser` struct provides the low-level API for -/// navigating through the stream of tokens and -/// constructing the parse tree. The actual parsing -/// happens in the `grammar` module. -/// -/// However, the result of this `Parser` is not a real -/// tree, but rather a flat stream of events of the form -/// "start expression, consume number literal, -/// finish expression". See `Event` docs for more. -pub(crate) struct Parser<'t> { - token_source: &'t mut dyn TokenSource, - events: Vec, - steps: Cell, -} - -impl<'t> Parser<'t> { - pub(super) fn new(token_source: &'t mut dyn TokenSource) -> Parser<'t> { - Parser { token_source, events: Vec::new(), steps: Cell::new(0) } - } - - pub(crate) fn finish(self) -> Vec { - self.events - } - - /// Returns the kind of the current token. - /// If parser has already reached the end of input, - /// the special `EOF` kind is returned. - pub(crate) fn current(&self) -> SyntaxKind { - self.nth(0) - } - - /// Lookahead operation: returns the kind of the next nth - /// token. - pub(crate) fn nth(&self, n: usize) -> SyntaxKind { - assert!(n <= 3); - - let steps = self.steps.get(); - assert!(steps <= 10_000_000, "the parser seems stuck"); - self.steps.set(steps + 1); - - self.token_source.lookahead_nth(n).kind - } - - /// Checks if the current token is `kind`. - pub(crate) fn at(&self, kind: SyntaxKind) -> bool { - self.nth_at(0, kind) - } - - pub(crate) fn nth_at(&self, n: usize, kind: SyntaxKind) -> bool { - match kind { - T![-=] => self.at_composite2(n, T![-], T![=]), - T![->] => self.at_composite2(n, T![-], T![>]), - T![::] => self.at_composite2(n, T![:], T![:]), - T![!=] => self.at_composite2(n, T![!], T![=]), - T![..] => self.at_composite2(n, T![.], T![.]), - T![*=] => self.at_composite2(n, T![*], T![=]), - T![/=] => self.at_composite2(n, T![/], T![=]), - T![&&] => self.at_composite2(n, T![&], T![&]), - T![&=] => self.at_composite2(n, T![&], T![=]), - T![%=] => self.at_composite2(n, T![%], T![=]), - T![^=] => self.at_composite2(n, T![^], T![=]), - T![+=] => self.at_composite2(n, T![+], T![=]), - T![<<] => self.at_composite2(n, T![<], T![<]), - T![<=] => self.at_composite2(n, T![<], T![=]), - T![==] => self.at_composite2(n, T![=], T![=]), - T![=>] => self.at_composite2(n, T![=], T![>]), - T![>=] => self.at_composite2(n, T![>], T![=]), - T![>>] => self.at_composite2(n, T![>], T![>]), - T![|=] => self.at_composite2(n, T![|], T![=]), - T![||] => self.at_composite2(n, T![|], T![|]), - - T![...] => self.at_composite3(n, T![.], T![.], T![.]), - T![..=] => self.at_composite3(n, T![.], T![.], T![=]), - T![<<=] => self.at_composite3(n, T![<], T![<], T![=]), - T![>>=] => self.at_composite3(n, T![>], T![>], T![=]), - - _ => self.token_source.lookahead_nth(n).kind == kind, - } - } - - /// Consume the next token if `kind` matches. - pub(crate) fn eat(&mut self, kind: SyntaxKind) -> bool { - if !self.at(kind) { - return false; - } - let n_raw_tokens = match kind { - T![-=] - | T![->] - | T![::] - | T![!=] - | T![..] - | T![*=] - | T![/=] - | T![&&] - | T![&=] - | T![%=] - | T![^=] - | T![+=] - | T![<<] - | T![<=] - | T![==] - | T![=>] - | T![>=] - | T![>>] - | T![|=] - | T![||] => 2, - - T![...] | T![..=] | T![<<=] | T![>>=] => 3, - _ => 1, - }; - self.do_bump(kind, n_raw_tokens); - true - } - - fn at_composite2(&self, n: usize, k1: SyntaxKind, k2: SyntaxKind) -> bool { - let t1 = self.token_source.lookahead_nth(n); - if t1.kind != k1 || !t1.is_jointed_to_next { - return false; - } - let t2 = self.token_source.lookahead_nth(n + 1); - t2.kind == k2 - } - - fn at_composite3(&self, n: usize, k1: SyntaxKind, k2: SyntaxKind, k3: SyntaxKind) -> bool { - let t1 = self.token_source.lookahead_nth(n); - if t1.kind != k1 || !t1.is_jointed_to_next { - return false; - } - let t2 = self.token_source.lookahead_nth(n + 1); - if t2.kind != k2 || !t2.is_jointed_to_next { - return false; - } - let t3 = self.token_source.lookahead_nth(n + 2); - t3.kind == k3 - } - - /// Checks if the current token is in `kinds`. - pub(crate) fn at_ts(&self, kinds: TokenSet) -> bool { - kinds.contains(self.current()) - } - - /// Checks if the current token is contextual keyword with text `t`. - pub(crate) fn at_contextual_kw(&self, kw: &str) -> bool { - self.token_source.is_keyword(kw) - } - - /// Starts a new node in the syntax tree. All nodes and tokens - /// consumed between the `start` and the corresponding `Marker::complete` - /// belong to the same node. - pub(crate) fn start(&mut self) -> Marker { - let pos = self.events.len() as u32; - self.push_event(Event::tombstone()); - Marker::new(pos) - } - - /// Consume the next token if `kind` matches. - pub(crate) fn bump(&mut self, kind: SyntaxKind) { - assert!(self.eat(kind)); - } - - /// Advances the parser by one token - pub(crate) fn bump_any(&mut self) { - let kind = self.nth(0); - if kind == EOF { - return; - } - self.do_bump(kind, 1) - } - - /// Advances the parser by one token, remapping its kind. - /// This is useful to create contextual keywords from - /// identifiers. For example, the lexer creates an `union` - /// *identifier* token, but the parser remaps it to the - /// `union` keyword, and keyword is what ends up in the - /// final tree. - pub(crate) fn bump_remap(&mut self, kind: SyntaxKind) { - if self.nth(0) == EOF { - // FIXME: panic!? - return; - } - self.do_bump(kind, 1); - } - - /// Emit error with the `message` - /// FIXME: this should be much more fancy and support - /// structured errors with spans and notes, like rustc - /// does. - pub(crate) fn error>(&mut self, message: T) { - let msg = ParseError(Box::new(message.into())); - self.push_event(Event::Error { msg }) - } - - /// Consume the next token if it is `kind` or emit an error - /// otherwise. - pub(crate) fn expect(&mut self, kind: SyntaxKind) -> bool { - if self.eat(kind) { - return true; - } - self.error(format!("expected {:?}", kind)); - false - } - - /// Create an error node and consume the next token. - pub(crate) fn err_and_bump(&mut self, message: &str) { - self.err_recover(message, TokenSet::EMPTY); - } - - /// Create an error node and consume the next token. - pub(crate) fn err_recover(&mut self, message: &str, recovery: TokenSet) { - match self.current() { - T!['{'] | T!['}'] => { - self.error(message); - return; - } - _ => (), - } - - if self.at_ts(recovery) { - self.error(message); - return; - } - - let m = self.start(); - self.error(message); - self.bump_any(); - m.complete(self, ERROR); - } - - fn do_bump(&mut self, kind: SyntaxKind, n_raw_tokens: u8) { - for _ in 0..n_raw_tokens { - self.token_source.bump(); - } - - self.push_event(Event::Token { kind, n_raw_tokens }); - } - - fn push_event(&mut self, event: Event) { - self.events.push(event) - } -} - -/// See `Parser::start`. -pub(crate) struct Marker { - pos: u32, - bomb: DropBomb, -} - -impl Marker { - fn new(pos: u32) -> Marker { - Marker { pos, bomb: DropBomb::new("Marker must be either completed or abandoned") } - } - - /// Finishes the syntax tree node and assigns `kind` to it, - /// and mark the create a `CompletedMarker` for possible future - /// operation like `.precede()` to deal with forward_parent. - pub(crate) fn complete(mut self, p: &mut Parser, kind: SyntaxKind) -> CompletedMarker { - self.bomb.defuse(); - let idx = self.pos as usize; - match &mut p.events[idx] { - Event::Start { kind: slot, .. } => { - *slot = kind; - } - _ => unreachable!(), - } - let finish_pos = p.events.len() as u32; - p.push_event(Event::Finish); - CompletedMarker::new(self.pos, finish_pos, kind) - } - - /// Abandons the syntax tree node. All its children - /// are attached to its parent instead. - pub(crate) fn abandon(mut self, p: &mut Parser) { - self.bomb.defuse(); - let idx = self.pos as usize; - if idx == p.events.len() - 1 { - match p.events.pop() { - Some(Event::Start { kind: TOMBSTONE, forward_parent: None }) => (), - _ => unreachable!(), - } - } - } -} - -pub(crate) struct CompletedMarker { - start_pos: u32, - finish_pos: u32, - kind: SyntaxKind, -} - -impl CompletedMarker { - fn new(start_pos: u32, finish_pos: u32, kind: SyntaxKind) -> Self { - CompletedMarker { start_pos, finish_pos, kind } - } - - /// This method allows to create a new node which starts - /// *before* the current one. That is, parser could start - /// node `A`, then complete it, and then after parsing the - /// whole `A`, decide that it should have started some node - /// `B` before starting `A`. `precede` allows to do exactly - /// that. See also docs about `forward_parent` in `Event::Start`. - /// - /// Given completed events `[START, FINISH]` and its corresponding - /// `CompletedMarker(pos: 0, _)`. - /// Append a new `START` events as `[START, FINISH, NEWSTART]`, - /// then mark `NEWSTART` as `START`'s parent with saving its relative - /// distance to `NEWSTART` into forward_parent(=2 in this case); - pub(crate) fn precede(self, p: &mut Parser) -> Marker { - let new_pos = p.start(); - let idx = self.start_pos as usize; - match &mut p.events[idx] { - Event::Start { forward_parent, .. } => { - *forward_parent = Some(new_pos.pos - self.start_pos); - } - _ => unreachable!(), - } - new_pos - } - - /// Undo this completion and turns into a `Marker` - pub(crate) fn undo_completion(self, p: &mut Parser) -> Marker { - let start_idx = self.start_pos as usize; - let finish_idx = self.finish_pos as usize; - match &mut p.events[start_idx] { - Event::Start { kind, forward_parent: None } => *kind = TOMBSTONE, - _ => unreachable!(), - } - match &mut p.events[finish_idx] { - slot @ Event::Finish => *slot = Event::tombstone(), - _ => unreachable!(), - } - Marker::new(self.start_pos) - } - - pub(crate) fn kind(&self) -> SyntaxKind { - self.kind - } -} diff --git a/crates/ra_parser/src/syntax_kind.rs b/crates/ra_parser/src/syntax_kind.rs deleted file mode 100644 index 63204436c..000000000 --- a/crates/ra_parser/src/syntax_kind.rs +++ /dev/null @@ -1,25 +0,0 @@ -//! FIXME: write short doc here - -#[macro_use] -mod generated; - -pub use self::generated::SyntaxKind; - -impl From for SyntaxKind { - fn from(d: u16) -> SyntaxKind { - assert!(d <= (SyntaxKind::__LAST as u16)); - unsafe { std::mem::transmute::(d) } - } -} - -impl From for u16 { - fn from(k: SyntaxKind) -> u16 { - k as u16 - } -} - -impl SyntaxKind { - pub fn is_trivia(self) -> bool { - matches!(self, SyntaxKind::WHITESPACE | SyntaxKind::COMMENT) - } -} diff --git a/crates/ra_parser/src/syntax_kind/generated.rs b/crates/ra_parser/src/syntax_kind/generated.rs deleted file mode 100644 index 192ecd864..000000000 --- a/crates/ra_parser/src/syntax_kind/generated.rs +++ /dev/null @@ -1,367 +0,0 @@ -//! Generated file, do not edit by hand, see `xtask/src/codegen` - -#![allow(bad_style, missing_docs, unreachable_pub)] -#[doc = r" The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT`."] -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] -#[repr(u16)] -pub enum SyntaxKind { - #[doc(hidden)] - TOMBSTONE, - #[doc(hidden)] - EOF, - SEMICOLON, - COMMA, - L_PAREN, - R_PAREN, - L_CURLY, - R_CURLY, - L_BRACK, - R_BRACK, - L_ANGLE, - R_ANGLE, - AT, - POUND, - TILDE, - QUESTION, - DOLLAR, - AMP, - PIPE, - PLUS, - STAR, - SLASH, - CARET, - PERCENT, - UNDERSCORE, - DOT, - DOT2, - DOT3, - DOT2EQ, - COLON, - COLON2, - EQ, - EQ2, - FAT_ARROW, - BANG, - NEQ, - MINUS, - THIN_ARROW, - LTEQ, - GTEQ, - PLUSEQ, - MINUSEQ, - PIPEEQ, - AMPEQ, - CARETEQ, - SLASHEQ, - STAREQ, - PERCENTEQ, - AMP2, - PIPE2, - SHL, - SHR, - SHLEQ, - SHREQ, - AS_KW, - ASYNC_KW, - AWAIT_KW, - BOX_KW, - BREAK_KW, - CONST_KW, - CONTINUE_KW, - CRATE_KW, - DYN_KW, - ELSE_KW, - ENUM_KW, - EXTERN_KW, - FALSE_KW, - FN_KW, - FOR_KW, - IF_KW, - IMPL_KW, - IN_KW, - LET_KW, - LOOP_KW, - MACRO_KW, - MATCH_KW, - MOD_KW, - MOVE_KW, - MUT_KW, - PUB_KW, - REF_KW, - RETURN_KW, - SELF_KW, - STATIC_KW, - STRUCT_KW, - SUPER_KW, - TRAIT_KW, - TRUE_KW, - TRY_KW, - TYPE_KW, - UNSAFE_KW, - USE_KW, - WHERE_KW, - WHILE_KW, - AUTO_KW, - DEFAULT_KW, - EXISTENTIAL_KW, - UNION_KW, - RAW_KW, - INT_NUMBER, - FLOAT_NUMBER, - CHAR, - BYTE, - STRING, - RAW_STRING, - BYTE_STRING, - RAW_BYTE_STRING, - ERROR, - IDENT, - WHITESPACE, - LIFETIME, - COMMENT, - SHEBANG, - L_DOLLAR, - R_DOLLAR, - SOURCE_FILE, - STRUCT, - UNION, - ENUM, - FN, - RET_TYPE, - EXTERN_CRATE, - MODULE, - USE, - STATIC, - CONST, - TRAIT, - IMPL, - TYPE_ALIAS, - MACRO_CALL, - TOKEN_TREE, - MACRO_DEF, - PAREN_TYPE, - TUPLE_TYPE, - NEVER_TYPE, - PATH_TYPE, - PTR_TYPE, - ARRAY_TYPE, - SLICE_TYPE, - REF_TYPE, - INFER_TYPE, - FN_PTR_TYPE, - FOR_TYPE, - IMPL_TRAIT_TYPE, - DYN_TRAIT_TYPE, - OR_PAT, - PAREN_PAT, - REF_PAT, - BOX_PAT, - IDENT_PAT, - WILDCARD_PAT, - REST_PAT, - PATH_PAT, - RECORD_PAT, - RECORD_PAT_FIELD_LIST, - RECORD_PAT_FIELD, - TUPLE_STRUCT_PAT, - TUPLE_PAT, - SLICE_PAT, - RANGE_PAT, - LITERAL_PAT, - MACRO_PAT, - TUPLE_EXPR, - ARRAY_EXPR, - PAREN_EXPR, - PATH_EXPR, - CLOSURE_EXPR, - IF_EXPR, - WHILE_EXPR, - CONDITION, - LOOP_EXPR, - FOR_EXPR, - CONTINUE_EXPR, - BREAK_EXPR, - LABEL, - BLOCK_EXPR, - RETURN_EXPR, - MATCH_EXPR, - MATCH_ARM_LIST, - MATCH_ARM, - MATCH_GUARD, - RECORD_EXPR, - RECORD_EXPR_FIELD_LIST, - RECORD_EXPR_FIELD, - EFFECT_EXPR, - BOX_EXPR, - CALL_EXPR, - INDEX_EXPR, - METHOD_CALL_EXPR, - FIELD_EXPR, - AWAIT_EXPR, - TRY_EXPR, - CAST_EXPR, - REF_EXPR, - PREFIX_EXPR, - RANGE_EXPR, - BIN_EXPR, - EXTERN_BLOCK, - EXTERN_ITEM_LIST, - VARIANT, - RECORD_FIELD_LIST, - RECORD_FIELD, - TUPLE_FIELD_LIST, - TUPLE_FIELD, - VARIANT_LIST, - ITEM_LIST, - ASSOC_ITEM_LIST, - ATTR, - META_ITEM, - USE_TREE, - USE_TREE_LIST, - PATH, - PATH_SEGMENT, - LITERAL, - RENAME, - VISIBILITY, - WHERE_CLAUSE, - WHERE_PRED, - ABI, - NAME, - NAME_REF, - LET_STMT, - EXPR_STMT, - GENERIC_PARAM_LIST, - GENERIC_PARAM, - LIFETIME_PARAM, - TYPE_PARAM, - CONST_PARAM, - GENERIC_ARG_LIST, - LIFETIME_ARG, - TYPE_ARG, - ASSOC_TYPE_ARG, - CONST_ARG, - PARAM_LIST, - PARAM, - SELF_PARAM, - ARG_LIST, - TYPE_BOUND, - TYPE_BOUND_LIST, - MACRO_ITEMS, - MACRO_STMTS, - #[doc(hidden)] - __LAST, -} -use self::SyntaxKind::*; -impl SyntaxKind { - pub fn is_keyword(self) -> bool { - match self { - AS_KW | ASYNC_KW | AWAIT_KW | BOX_KW | BREAK_KW | CONST_KW | CONTINUE_KW | CRATE_KW - | DYN_KW | ELSE_KW | ENUM_KW | EXTERN_KW | FALSE_KW | FN_KW | FOR_KW | IF_KW - | IMPL_KW | IN_KW | LET_KW | LOOP_KW | MACRO_KW | MATCH_KW | MOD_KW | MOVE_KW - | MUT_KW | PUB_KW | REF_KW | RETURN_KW | SELF_KW | STATIC_KW | STRUCT_KW | SUPER_KW - | TRAIT_KW | TRUE_KW | TRY_KW | TYPE_KW | UNSAFE_KW | USE_KW | WHERE_KW | WHILE_KW - | AUTO_KW | DEFAULT_KW | EXISTENTIAL_KW | UNION_KW | RAW_KW => true, - _ => false, - } - } - pub fn is_punct(self) -> bool { - match self { - SEMICOLON | COMMA | L_PAREN | R_PAREN | L_CURLY | R_CURLY | L_BRACK | R_BRACK - | L_ANGLE | R_ANGLE | AT | POUND | TILDE | QUESTION | DOLLAR | AMP | PIPE | PLUS - | STAR | SLASH | CARET | PERCENT | UNDERSCORE | DOT | DOT2 | DOT3 | DOT2EQ | COLON - | COLON2 | EQ | EQ2 | FAT_ARROW | BANG | NEQ | MINUS | THIN_ARROW | LTEQ | GTEQ - | PLUSEQ | MINUSEQ | PIPEEQ | AMPEQ | CARETEQ | SLASHEQ | STAREQ | PERCENTEQ | AMP2 - | PIPE2 | SHL | SHR | SHLEQ | SHREQ => true, - _ => false, - } - } - pub fn is_literal(self) -> bool { - match self { - INT_NUMBER | FLOAT_NUMBER | CHAR | BYTE | STRING | RAW_STRING | BYTE_STRING - | RAW_BYTE_STRING => true, - _ => false, - } - } - pub fn from_keyword(ident: &str) -> Option { - let kw = match ident { - "as" => AS_KW, - "async" => ASYNC_KW, - "await" => AWAIT_KW, - "box" => BOX_KW, - "break" => BREAK_KW, - "const" => CONST_KW, - "continue" => CONTINUE_KW, - "crate" => CRATE_KW, - "dyn" => DYN_KW, - "else" => ELSE_KW, - "enum" => ENUM_KW, - "extern" => EXTERN_KW, - "false" => FALSE_KW, - "fn" => FN_KW, - "for" => FOR_KW, - "if" => IF_KW, - "impl" => IMPL_KW, - "in" => IN_KW, - "let" => LET_KW, - "loop" => LOOP_KW, - "macro" => MACRO_KW, - "match" => MATCH_KW, - "mod" => MOD_KW, - "move" => MOVE_KW, - "mut" => MUT_KW, - "pub" => PUB_KW, - "ref" => REF_KW, - "return" => RETURN_KW, - "self" => SELF_KW, - "static" => STATIC_KW, - "struct" => STRUCT_KW, - "super" => SUPER_KW, - "trait" => TRAIT_KW, - "true" => TRUE_KW, - "try" => TRY_KW, - "type" => TYPE_KW, - "unsafe" => UNSAFE_KW, - "use" => USE_KW, - "where" => WHERE_KW, - "while" => WHILE_KW, - _ => return None, - }; - Some(kw) - } - pub fn from_char(c: char) -> Option { - let tok = match c { - ';' => SEMICOLON, - ',' => COMMA, - '(' => L_PAREN, - ')' => R_PAREN, - '{' => L_CURLY, - '}' => R_CURLY, - '[' => L_BRACK, - ']' => R_BRACK, - '<' => L_ANGLE, - '>' => R_ANGLE, - '@' => AT, - '#' => POUND, - '~' => TILDE, - '?' => QUESTION, - '$' => DOLLAR, - '&' => AMP, - '|' => PIPE, - '+' => PLUS, - '*' => STAR, - '/' => SLASH, - '^' => CARET, - '%' => PERCENT, - '_' => UNDERSCORE, - '.' => DOT, - ':' => COLON, - '=' => EQ, - '!' => BANG, - '-' => MINUS, - _ => return None, - }; - Some(tok) - } -} -#[macro_export] -macro_rules ! T { [ ; ] => { $ crate :: SyntaxKind :: SEMICOLON } ; [ , ] => { $ crate :: SyntaxKind :: COMMA } ; [ '(' ] => { $ crate :: SyntaxKind :: L_PAREN } ; [ ')' ] => { $ crate :: SyntaxKind :: R_PAREN } ; [ '{' ] => { $ crate :: SyntaxKind :: L_CURLY } ; [ '}' ] => { $ crate :: SyntaxKind :: R_CURLY } ; [ '[' ] => { $ crate :: SyntaxKind :: L_BRACK } ; [ ']' ] => { $ crate :: SyntaxKind :: R_BRACK } ; [ < ] => { $ crate :: SyntaxKind :: L_ANGLE } ; [ > ] => { $ crate :: SyntaxKind :: R_ANGLE } ; [ @ ] => { $ crate :: SyntaxKind :: AT } ; [ # ] => { $ crate :: SyntaxKind :: POUND } ; [ ~ ] => { $ crate :: SyntaxKind :: TILDE } ; [ ? ] => { $ crate :: SyntaxKind :: QUESTION } ; [ $ ] => { $ crate :: SyntaxKind :: DOLLAR } ; [ & ] => { $ crate :: SyntaxKind :: AMP } ; [ | ] => { $ crate :: SyntaxKind :: PIPE } ; [ + ] => { $ crate :: SyntaxKind :: PLUS } ; [ * ] => { $ crate :: SyntaxKind :: STAR } ; [ / ] => { $ crate :: SyntaxKind :: SLASH } ; [ ^ ] => { $ crate :: SyntaxKind :: CARET } ; [ % ] => { $ crate :: SyntaxKind :: PERCENT } ; [ _ ] => { $ crate :: SyntaxKind :: UNDERSCORE } ; [ . ] => { $ crate :: SyntaxKind :: DOT } ; [ .. ] => { $ crate :: SyntaxKind :: DOT2 } ; [ ... ] => { $ crate :: SyntaxKind :: DOT3 } ; [ ..= ] => { $ crate :: SyntaxKind :: DOT2EQ } ; [ : ] => { $ crate :: SyntaxKind :: COLON } ; [ :: ] => { $ crate :: SyntaxKind :: COLON2 } ; [ = ] => { $ crate :: SyntaxKind :: EQ } ; [ == ] => { $ crate :: SyntaxKind :: EQ2 } ; [ => ] => { $ crate :: SyntaxKind :: FAT_ARROW } ; [ ! ] => { $ crate :: SyntaxKind :: BANG } ; [ != ] => { $ crate :: SyntaxKind :: NEQ } ; [ - ] => { $ crate :: SyntaxKind :: MINUS } ; [ -> ] => { $ crate :: SyntaxKind :: THIN_ARROW } ; [ <= ] => { $ crate :: SyntaxKind :: LTEQ } ; [ >= ] => { $ crate :: SyntaxKind :: GTEQ } ; [ += ] => { $ crate :: SyntaxKind :: PLUSEQ } ; [ -= ] => { $ crate :: SyntaxKind :: MINUSEQ } ; [ |= ] => { $ crate :: SyntaxKind :: PIPEEQ } ; [ &= ] => { $ crate :: SyntaxKind :: AMPEQ } ; [ ^= ] => { $ crate :: SyntaxKind :: CARETEQ } ; [ /= ] => { $ crate :: SyntaxKind :: SLASHEQ } ; [ *= ] => { $ crate :: SyntaxKind :: STAREQ } ; [ %= ] => { $ crate :: SyntaxKind :: PERCENTEQ } ; [ && ] => { $ crate :: SyntaxKind :: AMP2 } ; [ || ] => { $ crate :: SyntaxKind :: PIPE2 } ; [ << ] => { $ crate :: SyntaxKind :: SHL } ; [ >> ] => { $ crate :: SyntaxKind :: SHR } ; [ <<= ] => { $ crate :: SyntaxKind :: SHLEQ } ; [ >>= ] => { $ crate :: SyntaxKind :: SHREQ } ; [ as ] => { $ crate :: SyntaxKind :: AS_KW } ; [ async ] => { $ crate :: SyntaxKind :: ASYNC_KW } ; [ await ] => { $ crate :: SyntaxKind :: AWAIT_KW } ; [ box ] => { $ crate :: SyntaxKind :: BOX_KW } ; [ break ] => { $ crate :: SyntaxKind :: BREAK_KW } ; [ const ] => { $ crate :: SyntaxKind :: CONST_KW } ; [ continue ] => { $ crate :: SyntaxKind :: CONTINUE_KW } ; [ crate ] => { $ crate :: SyntaxKind :: CRATE_KW } ; [ dyn ] => { $ crate :: SyntaxKind :: DYN_KW } ; [ else ] => { $ crate :: SyntaxKind :: ELSE_KW } ; [ enum ] => { $ crate :: SyntaxKind :: ENUM_KW } ; [ extern ] => { $ crate :: SyntaxKind :: EXTERN_KW } ; [ false ] => { $ crate :: SyntaxKind :: FALSE_KW } ; [ fn ] => { $ crate :: SyntaxKind :: FN_KW } ; [ for ] => { $ crate :: SyntaxKind :: FOR_KW } ; [ if ] => { $ crate :: SyntaxKind :: IF_KW } ; [ impl ] => { $ crate :: SyntaxKind :: IMPL_KW } ; [ in ] => { $ crate :: SyntaxKind :: IN_KW } ; [ let ] => { $ crate :: SyntaxKind :: LET_KW } ; [ loop ] => { $ crate :: SyntaxKind :: LOOP_KW } ; [ macro ] => { $ crate :: SyntaxKind :: MACRO_KW } ; [ match ] => { $ crate :: SyntaxKind :: MATCH_KW } ; [ mod ] => { $ crate :: SyntaxKind :: MOD_KW } ; [ move ] => { $ crate :: SyntaxKind :: MOVE_KW } ; [ mut ] => { $ crate :: SyntaxKind :: MUT_KW } ; [ pub ] => { $ crate :: SyntaxKind :: PUB_KW } ; [ ref ] => { $ crate :: SyntaxKind :: REF_KW } ; [ return ] => { $ crate :: SyntaxKind :: RETURN_KW } ; [ self ] => { $ crate :: SyntaxKind :: SELF_KW } ; [ static ] => { $ crate :: SyntaxKind :: STATIC_KW } ; [ struct ] => { $ crate :: SyntaxKind :: STRUCT_KW } ; [ super ] => { $ crate :: SyntaxKind :: SUPER_KW } ; [ trait ] => { $ crate :: SyntaxKind :: TRAIT_KW } ; [ true ] => { $ crate :: SyntaxKind :: TRUE_KW } ; [ try ] => { $ crate :: SyntaxKind :: TRY_KW } ; [ type ] => { $ crate :: SyntaxKind :: TYPE_KW } ; [ unsafe ] => { $ crate :: SyntaxKind :: UNSAFE_KW } ; [ use ] => { $ crate :: SyntaxKind :: USE_KW } ; [ where ] => { $ crate :: SyntaxKind :: WHERE_KW } ; [ while ] => { $ crate :: SyntaxKind :: WHILE_KW } ; [ auto ] => { $ crate :: SyntaxKind :: AUTO_KW } ; [ default ] => { $ crate :: SyntaxKind :: DEFAULT_KW } ; [ existential ] => { $ crate :: SyntaxKind :: EXISTENTIAL_KW } ; [ union ] => { $ crate :: SyntaxKind :: UNION_KW } ; [ raw ] => { $ crate :: SyntaxKind :: RAW_KW } ; [ lifetime ] => { $ crate :: SyntaxKind :: LIFETIME } ; [ ident ] => { $ crate :: SyntaxKind :: IDENT } ; [ shebang ] => { $ crate :: SyntaxKind :: SHEBANG } ; } diff --git a/crates/ra_parser/src/token_set.rs b/crates/ra_parser/src/token_set.rs deleted file mode 100644 index 994017acf..000000000 --- a/crates/ra_parser/src/token_set.rs +++ /dev/null @@ -1,42 +0,0 @@ -//! A bit-set of `SyntaxKind`s. - -use crate::SyntaxKind; - -/// A bit-set of `SyntaxKind`s -#[derive(Clone, Copy)] -pub(crate) struct TokenSet(u128); - -impl TokenSet { - pub(crate) const EMPTY: TokenSet = TokenSet(0); - - pub(crate) const fn singleton(kind: SyntaxKind) -> TokenSet { - TokenSet(mask(kind)) - } - - pub(crate) const fn union(self, other: TokenSet) -> TokenSet { - TokenSet(self.0 | other.0) - } - - pub(crate) fn contains(&self, kind: SyntaxKind) -> bool { - self.0 & mask(kind) != 0 - } -} - -const fn mask(kind: SyntaxKind) -> u128 { - 1u128 << (kind as usize) -} - -#[macro_export] -macro_rules! token_set { - ($($t:expr),*) => { TokenSet::EMPTY$(.union(TokenSet::singleton($t)))* }; - ($($t:expr),* ,) => { token_set!($($t),*) }; -} - -#[test] -fn token_set_works_for_tokens() { - use crate::SyntaxKind::*; - let ts = token_set![EOF, SHEBANG]; - assert!(ts.contains(EOF)); - assert!(ts.contains(SHEBANG)); - assert!(!ts.contains(PLUS)); -} diff --git a/crates/ra_syntax/Cargo.toml b/crates/ra_syntax/Cargo.toml index f2789e6a3..eec4bd845 100644 --- a/crates/ra_syntax/Cargo.toml +++ b/crates/ra_syntax/Cargo.toml @@ -21,7 +21,7 @@ once_cell = "1.3.1" stdx = { path = "../stdx" } text_edit = { path = "../text_edit" } -ra_parser = { path = "../ra_parser" } +parser = { path = "../parser" } # This crate transitively depends on `smol_str` via `rowan`. # ideally, `serde` should be enabled by `rust-analyzer`, but we enable it here diff --git a/crates/ra_syntax/src/ast/node_ext.rs b/crates/ra_syntax/src/ast/node_ext.rs index 733e97877..50c1c157d 100644 --- a/crates/ra_syntax/src/ast/node_ext.rs +++ b/crates/ra_syntax/src/ast/node_ext.rs @@ -4,7 +4,7 @@ use std::fmt; use itertools::Itertools; -use ra_parser::SyntaxKind; +use parser::SyntaxKind; use crate::{ ast::{self, support, AstNode, NameOwner, SyntaxNode}, diff --git a/crates/ra_syntax/src/lib.rs b/crates/ra_syntax/src/lib.rs index 465607f55..7f8da66af 100644 --- a/crates/ra_syntax/src/lib.rs +++ b/crates/ra_syntax/src/lib.rs @@ -11,7 +11,7 @@ //! //! The most interesting modules here are `syntax_node` (which defines concrete //! syntax tree) and `ast` (which defines abstract syntax tree on top of the -//! CST). The actual parser live in a separate `ra_parser` crate, though the +//! CST). The actual parser live in a separate `parser` crate, though the //! lexer lives in this crate. //! //! See `api_walkthrough` test in this file for a quick API tour! @@ -53,7 +53,7 @@ pub use crate::{ SyntaxNodeChildren, SyntaxToken, SyntaxTreeBuilder, }, }; -pub use ra_parser::{SyntaxKind, T}; +pub use parser::{SyntaxKind, T}; pub use rowan::{SmolStr, SyntaxText, TextRange, TextSize, TokenAtOffset, WalkEvent}; /// `Parse` is the result of the parsing: a syntax tree and a collection of @@ -169,35 +169,35 @@ impl SourceFile { impl ast::Path { /// Returns `text`, parsed as a path, but only if it has no errors. pub fn parse(text: &str) -> Result { - parsing::parse_text_fragment(text, ra_parser::FragmentKind::Path) + parsing::parse_text_fragment(text, parser::FragmentKind::Path) } } impl ast::Pat { /// Returns `text`, parsed as a pattern, but only if it has no errors. pub fn parse(text: &str) -> Result { - parsing::parse_text_fragment(text, ra_parser::FragmentKind::Pattern) + parsing::parse_text_fragment(text, parser::FragmentKind::Pattern) } } impl ast::Expr { /// Returns `text`, parsed as an expression, but only if it has no errors. pub fn parse(text: &str) -> Result { - parsing::parse_text_fragment(text, ra_parser::FragmentKind::Expr) + parsing::parse_text_fragment(text, parser::FragmentKind::Expr) } } impl ast::Item { /// Returns `text`, parsed as an item, but only if it has no errors. pub fn parse(text: &str) -> Result { - parsing::parse_text_fragment(text, ra_parser::FragmentKind::Item) + parsing::parse_text_fragment(text, parser::FragmentKind::Item) } } impl ast::Type { /// Returns `text`, parsed as an type reference, but only if it has no errors. pub fn parse(text: &str) -> Result { - parsing::parse_text_fragment(text, ra_parser::FragmentKind::Type) + parsing::parse_text_fragment(text, parser::FragmentKind::Type) } } diff --git a/crates/ra_syntax/src/parsing.rs b/crates/ra_syntax/src/parsing.rs index 0ed3c20ef..68a39eb21 100644 --- a/crates/ra_syntax/src/parsing.rs +++ b/crates/ra_syntax/src/parsing.rs @@ -1,4 +1,4 @@ -//! Lexing, bridging to ra_parser (which does the actual parsing) and +//! Lexing, bridging to parser (which does the actual parsing) and //! incremental reparsing. mod lexer; @@ -13,7 +13,7 @@ use text_tree_sink::TextTreeSink; pub use lexer::*; pub(crate) use self::reparsing::incremental_reparse; -use ra_parser::SyntaxKind; +use parser::SyntaxKind; pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec) { let (tokens, lexer_errors) = tokenize(&text); @@ -21,7 +21,7 @@ pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec) { let mut token_source = TextTokenSource::new(text, &tokens); let mut tree_sink = TextTreeSink::new(text, &tokens); - ra_parser::parse(&mut token_source, &mut tree_sink); + parser::parse(&mut token_source, &mut tree_sink); let (tree, mut parser_errors) = tree_sink.finish(); parser_errors.extend(lexer_errors); @@ -32,7 +32,7 @@ pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec) { /// Returns `text` parsed as a `T` provided there are no parse errors. pub(crate) fn parse_text_fragment( text: &str, - fragment_kind: ra_parser::FragmentKind, + fragment_kind: parser::FragmentKind, ) -> Result { let (tokens, lexer_errors) = tokenize(&text); if !lexer_errors.is_empty() { @@ -44,13 +44,13 @@ pub(crate) fn parse_text_fragment( // TextTreeSink assumes that there's at least some root node to which it can attach errors and // tokens. We arbitrarily give it a SourceFile. - use ra_parser::TreeSink; + use parser::TreeSink; tree_sink.start_node(SyntaxKind::SOURCE_FILE); - ra_parser::parse_fragment(&mut token_source, &mut tree_sink, fragment_kind); + parser::parse_fragment(&mut token_source, &mut tree_sink, fragment_kind); tree_sink.finish_node(); let (tree, parser_errors) = tree_sink.finish(); - use ra_parser::TokenSource; + use parser::TokenSource; if !parser_errors.is_empty() || token_source.current().kind != SyntaxKind::EOF { return Err(()); } diff --git a/crates/ra_syntax/src/parsing/reparsing.rs b/crates/ra_syntax/src/parsing/reparsing.rs index 6644ffca4..4149f856a 100644 --- a/crates/ra_syntax/src/parsing/reparsing.rs +++ b/crates/ra_syntax/src/parsing/reparsing.rs @@ -6,7 +6,7 @@ //! - otherwise, we search for the nearest `{}` block which contains the edit //! and try to parse only this block. -use ra_parser::Reparser; +use parser::Reparser; use text_edit::Indel; use crate::{ diff --git a/crates/ra_syntax/src/parsing/text_token_source.rs b/crates/ra_syntax/src/parsing/text_token_source.rs index 97aa3e795..df866dc2b 100644 --- a/crates/ra_syntax/src/parsing/text_token_source.rs +++ b/crates/ra_syntax/src/parsing/text_token_source.rs @@ -1,10 +1,10 @@ //! See `TextTokenSource` docs. -use ra_parser::TokenSource; +use parser::TokenSource; use crate::{parsing::lexer::Token, SyntaxKind::EOF, TextRange, TextSize}; -/// Implementation of `ra_parser::TokenSource` that takes tokens from source code text. +/// Implementation of `parser::TokenSource` that takes tokens from source code text. pub(crate) struct TextTokenSource<'t> { text: &'t str, /// token and its start position (non-whitespace/comment tokens) @@ -20,15 +20,15 @@ pub(crate) struct TextTokenSource<'t> { token_offset_pairs: Vec<(Token, TextSize)>, /// Current token and position - curr: (ra_parser::Token, usize), + curr: (parser::Token, usize), } impl<'t> TokenSource for TextTokenSource<'t> { - fn current(&self) -> ra_parser::Token { + fn current(&self) -> parser::Token { self.curr.0 } - fn lookahead_nth(&self, n: usize) -> ra_parser::Token { + fn lookahead_nth(&self, n: usize) -> parser::Token { mk_token(self.curr.1 + n, &self.token_offset_pairs) } @@ -49,7 +49,7 @@ impl<'t> TokenSource for TextTokenSource<'t> { } } -fn mk_token(pos: usize, token_offset_pairs: &[(Token, TextSize)]) -> ra_parser::Token { +fn mk_token(pos: usize, token_offset_pairs: &[(Token, TextSize)]) -> parser::Token { let (kind, is_jointed_to_next) = match token_offset_pairs.get(pos) { Some((token, offset)) => ( token.kind, @@ -60,7 +60,7 @@ fn mk_token(pos: usize, token_offset_pairs: &[(Token, TextSize)]) -> ra_parser:: ), None => (EOF, false), }; - ra_parser::Token { kind, is_jointed_to_next } + parser::Token { kind, is_jointed_to_next } } impl<'t> TextTokenSource<'t> { diff --git a/crates/ra_syntax/src/parsing/text_tree_sink.rs b/crates/ra_syntax/src/parsing/text_tree_sink.rs index 6d1828d20..c1b5f246d 100644 --- a/crates/ra_syntax/src/parsing/text_tree_sink.rs +++ b/crates/ra_syntax/src/parsing/text_tree_sink.rs @@ -2,7 +2,7 @@ use std::mem; -use ra_parser::{ParseError, TreeSink}; +use parser::{ParseError, TreeSink}; use crate::{ parsing::Token, diff --git a/crates/ra_syntax/src/syntax_node.rs b/crates/ra_syntax/src/syntax_node.rs index a7dbdba7b..b2abcbfbb 100644 --- a/crates/ra_syntax/src/syntax_node.rs +++ b/crates/ra_syntax/src/syntax_node.rs @@ -71,7 +71,7 @@ impl SyntaxTreeBuilder { self.inner.finish_node() } - pub fn error(&mut self, error: ra_parser::ParseError, text_pos: TextSize) { + pub fn error(&mut self, error: parser::ParseError, text_pos: TextSize) { self.errors.push(SyntaxError::new_at_offset(*error.0, text_pos)) } } -- cgit v1.2.3