From 0b5d39f2a204e5ec6cd6205440e4cdc763162814 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 20 Jan 2018 23:25:34 +0300 Subject: Markers API --- src/parser/event_parser/grammar/attributes.rs | 8 +- src/parser/event_parser/grammar/expressions.rs | 4 +- src/parser/event_parser/grammar/items.rs | 151 ++++++++++++------------- src/parser/event_parser/grammar/mod.rs | 12 +- src/parser/event_parser/grammar/paths.rs | 17 ++- src/parser/event_parser/mod.rs | 2 +- src/parser/event_parser/parser.rs | 112 ++++++++++++------ src/parser/mod.rs | 5 +- src/tree/file_builder.rs | 8 +- src/tree/mod.rs | 15 ++- 10 files changed, 189 insertions(+), 145 deletions(-) (limited to 'src') diff --git a/src/parser/event_parser/grammar/attributes.rs b/src/parser/event_parser/grammar/attributes.rs index 8b5e5bcfe..2d04a1a41 100644 --- a/src/parser/event_parser/grammar/attributes.rs +++ b/src/parser/event_parser/grammar/attributes.rs @@ -19,13 +19,13 @@ fn attribute(p: &mut Parser, kind: AttrKind) -> bool { if kind == AttrKind::Inner && p.raw_lookahead(1) != EXCL { return false; } - p.start(ATTR); + let attr = p.start(); p.bump(); if kind == AttrKind::Inner { p.bump(); } p.expect(L_BRACK) && meta_item(p) && p.expect(R_BRACK); - p.finish(); + attr.complete(p, ATTR); true } else { false @@ -34,7 +34,7 @@ fn attribute(p: &mut Parser, kind: AttrKind) -> bool { fn meta_item(p: &mut Parser) -> bool { if p.at(IDENT) { - p.start(META_ITEM); + let meta_item = p.start(); p.bump(); if p.eat(EQ) { if !expressions::literal(p) { @@ -46,7 +46,7 @@ fn meta_item(p: &mut Parser) -> bool { comma_list(p, R_PAREN, meta_item_inner); p.expect(R_PAREN); } - p.finish(); + meta_item.complete(p, META_ITEM); true } else { false diff --git a/src/parser/event_parser/grammar/expressions.rs b/src/parser/event_parser/grammar/expressions.rs index 0f65193c9..a943b8c81 100644 --- a/src/parser/event_parser/grammar/expressions.rs +++ b/src/parser/event_parser/grammar/expressions.rs @@ -6,9 +6,9 @@ pub(super) fn literal(p: &mut Parser) -> bool { INT_NUMBER | FLOAT_NUMBER | BYTE | CHAR | STRING | RAW_STRING | BYTE_STRING | RAW_BYTE_STRING => { - p.start(LITERAL); + let lit = p.start(); p.bump(); - p.finish(); + lit.complete(p, LITERAL); true } _ => false diff --git a/src/parser/event_parser/grammar/items.rs b/src/parser/event_parser/grammar/items.rs index 4514f0dab..c9a890553 100644 --- a/src/parser/event_parser/grammar/items.rs +++ b/src/parser/event_parser/grammar/items.rs @@ -8,19 +8,34 @@ pub(super) fn mod_contents(p: &mut Parser) { } fn item(p: &mut Parser) { - let attrs_start = p.mark(); + let item = p.start(); attributes::outer_attributes(p); visibility(p); let la = p.raw_lookahead(1); - let item_start = p.mark(); - match p.current() { - EXTERN_KW if la == CRATE_KW => extern_crate_item(p), - MOD_KW => mod_item(p), - USE_KW => use_item(p), - STRUCT_KW => struct_item(p), - FN_KW => fn_item(p), + let item_kind = match p.current() { + EXTERN_KW if la == CRATE_KW => { + extern_crate_item(p); + EXTERN_CRATE_ITEM + } + MOD_KW => { + mod_item(p); + MOD_ITEM + } + USE_KW => { + use_item(p); + USE_ITEM + } + STRUCT_KW => { + struct_item(p); + STRUCT_ITEM + } + FN_KW => { + fn_item(p); + FN_ITEM + } err_token => { - p.start(ERROR); + item.abandon(p); + let err = p.start(); let message = if err_token == SEMI { //TODO: if the item is incomplete, this message is misleading "expected item, found `;`\n\ @@ -32,60 +47,52 @@ fn item(p: &mut Parser) { .message(message) .emit(); p.bump(); - p.finish(); + err.complete(p, ERROR); return; } }; - p.forward_parent(attrs_start, item_start); + item.complete(p, item_kind); } fn struct_item(p: &mut Parser) { - p.start(STRUCT_ITEM); - assert!(p.at(STRUCT_KW)); p.bump(); - struct_inner(p); - p.finish(); - - fn struct_inner(p: &mut Parser) { - if !p.expect(IDENT) { - p.finish(); - return; - } - generic_parameters(p); - match p.current() { - WHERE_KW => { - where_clause(p); - match p.current() { - SEMI => { - p.bump(); - return; - } - L_CURLY => named_fields(p), - _ => { //TODO: special case `(` error message - p.error() - .message("expected `;` or `{`") - .emit(); - return; - } + if !p.expect(IDENT) { + return; + } + generic_parameters(p); + match p.current() { + WHERE_KW => { + where_clause(p); + match p.current() { + SEMI => { + p.bump(); + return; + } + L_CURLY => named_fields(p), + _ => { //TODO: special case `(` error message + p.error() + .message("expected `;` or `{`") + .emit(); + return; } } - SEMI => { - p.bump(); - return; - } - L_CURLY => named_fields(p), - L_PAREN => { - tuple_fields(p); - p.expect(SEMI); - } - _ => { - p.error() - .message("expected `;`, `{`, or `(`") - .emit(); - return; - } + } + SEMI => { + p.bump(); + return; + } + L_CURLY => named_fields(p), + L_PAREN => { + pos_fields(p); + p.expect(SEMI); + } + _ => { + p.error() + .message("expected `;`, `{`, or `(`") + .emit(); + return; } } } @@ -97,30 +104,30 @@ fn named_fields(p: &mut Parser) { })); fn named_field(p: &mut Parser) { - p.start(NAMED_FIELD); + let field = p.start(); visibility(p); if p.expect(IDENT) && p.expect(COLON) { types::type_ref(p); }; - p.finish() + field.complete(p, NAMED_FIELD); } } -fn tuple_fields(p: &mut Parser) { +fn pos_fields(p: &mut Parser) { if !p.expect(L_PAREN) { return; } comma_list(p, R_PAREN, |p| { - tuple_field(p); + pos_field(p); true }); p.expect(R_PAREN); - fn tuple_field(p: &mut Parser) { - p.start(POS_FIELD); + fn pos_field(p: &mut Parser) { + let pos_field = p.start(); visibility(p); types::type_ref(p); - p.finish(); + pos_field.complete(p, POS_FIELD); } } @@ -129,28 +136,21 @@ fn generic_parameters(_: &mut Parser) {} fn where_clause(_: &mut Parser) {} fn extern_crate_item(p: &mut Parser) { - p.start(EXTERN_CRATE_ITEM); - assert!(p.at(EXTERN_KW)); p.bump(); - assert!(p.at(CRATE_KW)); p.bump(); p.expect(IDENT) && alias(p) && p.expect(SEMI); - p.finish(); } fn mod_item(p: &mut Parser) { - p.start(MOD_ITEM); - assert!(p.at(MOD_KW)); p.bump(); if p.expect(IDENT) && !p.eat(SEMI) { p.curly_block(mod_contents); } - p.finish() } pub(super) fn is_use_tree_start(kind: SyntaxKind) -> bool { @@ -158,28 +158,24 @@ pub(super) fn is_use_tree_start(kind: SyntaxKind) -> bool { } fn use_item(p: &mut Parser) { - p.start(USE_ITEM); - assert!(p.at(USE_KW)); p.bump(); + use_tree(p); p.expect(SEMI); - p.finish(); fn use_tree(p: &mut Parser) -> bool { let la = p.raw_lookahead(1); + let m = p.start(); match (p.current(), la) { (STAR, _) => { - p.start(USE_TREE); p.bump(); } (COLONCOLON, STAR) => { - p.start(USE_TREE); p.bump(); p.bump(); } (L_CURLY, _) | (COLONCOLON, L_CURLY) => { - p.start(USE_TREE); if p.at(COLONCOLON) { p.bump(); } @@ -188,7 +184,6 @@ fn use_item(p: &mut Parser) { }); } _ if paths::is_path_start(p) => { - p.start(USE_TREE); paths::use_path(p); match p.current() { AS_KW => { @@ -216,23 +211,23 @@ fn use_item(p: &mut Parser) { _ => (), } } - _ => return false, + _ => { + m.abandon(p); + return false + }, } - p.finish(); + m.complete(p, USE_TREE); return true; } } fn fn_item(p: &mut Parser) { - p.start(FN_ITEM); - assert!(p.at(FN_KW)); p.bump(); p.expect(IDENT) && p.expect(L_PAREN) && p.expect(R_PAREN) && p.curly_block(|_| ()); - p.finish(); } diff --git a/src/parser/event_parser/grammar/mod.rs b/src/parser/event_parser/grammar/mod.rs index c3d0c8c10..d3b63c4c1 100644 --- a/src/parser/event_parser/grammar/mod.rs +++ b/src/parser/event_parser/grammar/mod.rs @@ -10,15 +10,15 @@ mod types; mod paths; pub(crate) fn file(p: &mut Parser) { - p.start(FILE); + let file = p.start(); p.eat(SHEBANG); items::mod_contents(p); - p.finish() + file.complete(p, FILE); } fn visibility(p: &mut Parser) { if p.at(PUB_KW) { - p.start(VISIBILITY); + let vis = p.start(); p.bump(); if p.at(L_PAREN) { match p.raw_lookahead(1) { @@ -32,16 +32,16 @@ fn visibility(p: &mut Parser) { _ => () } } - p.finish(); + vis.complete(p, VISIBILITY); } } fn alias(p: &mut Parser) -> bool { if p.at(AS_KW) { - p.start(ALIAS); + let alias = p.start(); p.bump(); p.expect(IDENT); - p.finish(); + alias.complete(p, ALIAS); } true //FIXME: return false if three are errors } diff --git a/src/parser/event_parser/grammar/paths.rs b/src/parser/event_parser/grammar/paths.rs index f5124cfce..b58c59aef 100644 --- a/src/parser/event_parser/grammar/paths.rs +++ b/src/parser/event_parser/grammar/paths.rs @@ -8,19 +8,16 @@ pub(crate) fn use_path(p: &mut Parser) { if !is_path_start(p) { return; } - let mut prev = p.mark(); - p.start(PATH); + let path = p.start(); path_segment(p, true); - p.finish(); + let mut qual = path.complete(p, PATH); loop { - let curr = p.mark(); if p.at(COLONCOLON) && !items::is_use_tree_start(p.raw_lookahead(1)) { - p.start(PATH); + let path = qual.precede(p); p.bump(); path_segment(p, false); - p.forward_parent(prev, curr); - prev = curr; - p.finish(); + let path = path.complete(p, PATH); + qual = path; } else { break; } @@ -28,7 +25,7 @@ pub(crate) fn use_path(p: &mut Parser) { } fn path_segment(p: &mut Parser, first: bool) { - p.start(PATH_SEGMENT); + let segment = p.start(); if first { p.eat(COLONCOLON); } @@ -42,5 +39,5 @@ fn path_segment(p: &mut Parser, first: bool) { .emit(); } }; - p.finish(); + segment.complete(p, PATH_SEGMENT); } diff --git a/src/parser/event_parser/mod.rs b/src/parser/event_parser/mod.rs index c89a3ebe7..7c81182e3 100644 --- a/src/parser/event_parser/mod.rs +++ b/src/parser/event_parser/mod.rs @@ -23,4 +23,4 @@ pub(crate) fn parse<'t>(text: &'t str, raw_tokens: &'t [Token]) -> Vec { let mut parser = parser::Parser::new(text, raw_tokens); grammar::file(&mut parser); parser.into_events() -} \ No newline at end of file +} diff --git a/src/parser/event_parser/parser.rs b/src/parser/event_parser/parser.rs index 6171b3579..2bc9dd34f 100644 --- a/src/parser/event_parser/parser.rs +++ b/src/parser/event_parser/parser.rs @@ -1,8 +1,71 @@ use {Token, SyntaxKind, TextUnit}; -use super::{Event}; +use super::Event; use super::super::is_insignificant; use syntax_kinds::{L_CURLY, R_CURLY, ERROR}; -use tree::EOF; +use tree::{EOF, TOMBSTONE}; + +pub(crate) struct Marker { + pos: u32 +} + +impl Marker { + pub fn complete(self, p: &mut Parser, kind: SyntaxKind) -> CompleteMarker { + match self.event(p) { + &mut Event::Start { kind: ref mut slot, ..} => { + *slot = kind; + } + _ => unreachable!(), + } + p.event(Event::Finish); + let result = CompleteMarker { pos: self.pos }; + ::std::mem::forget(self); + result + } + + pub fn abandon(self, p: &mut Parser) { + let idx = self.pos as usize; + if idx == p.events.len() - 1 { + match p.events.pop() { + Some(Event::Start { kind: TOMBSTONE, forward_parent: None }) => (), + _ => unreachable!() + } + } + ::std::mem::forget(self); + } + + fn event<'p>(&self, p: &'p mut Parser) -> &'p mut Event { + &mut p.events[self.idx()] + } + + fn idx(&self) -> usize { + self.pos as usize + } +} + +impl Drop for Marker { + fn drop(&mut self) { + if !::std::thread::panicking() { + panic!("Each marker should be eithe completed or abandoned"); + } + } +} + +pub(crate) struct CompleteMarker { + pos: u32 +} + +impl CompleteMarker { + pub(crate) fn precede(self, p: &mut Parser) -> Marker { + let m = p.start(); + match p.events[self.pos as usize] { + Event::Start { ref mut forward_parent, ..} => { + *forward_parent = Some(m.pos - self.pos); + } + _ => unreachable!(), + } + m + } +} pub(crate) struct Parser<'t> { @@ -19,12 +82,9 @@ pub(crate) struct Parser<'t> { curly_limit: Option, } -#[derive(Debug, Clone, Copy,PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct Pos(u32); -#[derive(Debug, Clone, Copy,PartialEq, Eq)] -pub(crate) struct Mark(u32); - impl<'t> Parser<'t> { pub(crate) fn new(text: &'t str, raw_tokens: &'t [Token]) -> Parser<'t> { let mut tokens = Vec::new(); @@ -50,31 +110,13 @@ impl<'t> Parser<'t> { } } - pub(crate) fn mark(&self) -> Mark { - Mark(self.events.len() as u32) - } - - pub(crate) fn forward_parent(&mut self, child: Mark, parent: Mark) { - if child == parent || parent == self.mark() { - return - } - assert!(child.0 < parent.0); - let diff = parent.0 - child.0; - match self.events[child.0 as usize] { - Event::Start { ref mut forward_parent, .. } => { - *forward_parent = Some(diff); - } - _ => unreachable!() - } - } - pub(crate) fn pos(&self) -> Pos { Pos(self.pos as u32) } pub(crate) fn into_events(self) -> Vec { assert!(self.curly_limit.is_none()); - assert!(self.current() == EOF); + assert_eq!(self.current(), EOF); self.events } @@ -85,18 +127,16 @@ impl<'t> Parser<'t> { let token = self.tokens[self.pos]; if let Some(limit) = self.curly_limit { if limit == self.curly_level && token.kind == R_CURLY { - return EOF + return EOF; } } token.kind } - pub(crate) fn start(&mut self, kind: SyntaxKind) { - self.event(Event::Start { kind, forward_parent: None }); - } - - pub(crate) fn finish(&mut self) { - self.event(Event::Finish); + pub(crate) fn start(&mut self) -> Marker { + let m = Marker { pos: self.events.len() as u32 }; + self.event(Event::Start { kind: TOMBSTONE, forward_parent: None }); + m } pub(crate) fn error<'p>(&'p mut self) -> ErrorBuilder<'p, 't> { @@ -124,20 +164,20 @@ impl<'t> Parser<'t> { let old_level = self.curly_level; let old_limit = self.curly_limit; if !self.expect(L_CURLY) { - return false + return false; } self.curly_limit = Some(self.curly_level); f(self); assert!(self.curly_level > old_level); self.curly_limit = old_limit; if !self.expect(R_CURLY) { - self.start(ERROR); + let err = self.start(); while self.curly_level > old_level { if self.bump() == EOF { break; } } - self.finish(); + err.complete(self, ERROR); } true } @@ -149,7 +189,7 @@ impl<'t> Parser<'t> { pub(crate) struct ErrorBuilder<'p, 't: 'p> { message: Option, - parser: &'p mut Parser<'t> + parser: &'p mut Parser<'t>, } impl<'t, 'p> ErrorBuilder<'p, 't> { diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 0c29442f9..5ec4b8e93 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -1,6 +1,7 @@ use {Token, File, FileBuilder, Sink, SyntaxKind}; use syntax_kinds::*; +use tree::TOMBSTONE; mod event_parser; use self::event_parser::Event; @@ -29,6 +30,8 @@ fn from_events_to_file( } match event { + &Event::Start { kind: TOMBSTONE, .. } => (), + &Event::Start { .. } => { forward_parents.clear(); let mut idx = i; @@ -62,7 +65,7 @@ fn from_events_to_file( } builder.finish_internal() }, - &Event::Token { kind, mut n_raw_tokens } => loop { + &Event::Token { kind: _, mut n_raw_tokens } => loop { let token = tokens[idx]; if !is_insignificant(token.kind) { n_raw_tokens -= 1; diff --git a/src/tree/file_builder.rs b/src/tree/file_builder.rs index 37bd5b2c8..35702ddd7 100644 --- a/src/tree/file_builder.rs +++ b/src/tree/file_builder.rs @@ -48,7 +48,9 @@ impl Sink for FileBuilder { } fn finish_internal(&mut self) { - let (id, _) = self.in_progress.pop().unwrap(); + let (id, _) = self.in_progress.pop().expect( + "trying to complete a node, but there are no in-progress nodes" + ); if !self.in_progress.is_empty() { self.add_len(id); } @@ -77,8 +79,8 @@ impl FileBuilder { self.in_progress.iter().map(|&(idx, _)| self.nodes[idx].kind) .collect::>() ); - assert!( - self.pos == (self.text.len() as u32).into(), + assert_eq!( + self.pos, (self.text.len() as u32).into(), "nodes in FileBuilder do not cover the whole file" ); File { diff --git a/src/tree/mod.rs b/src/tree/mod.rs index d8f843737..3315b926e 100644 --- a/src/tree/mod.rs +++ b/src/tree/mod.rs @@ -10,17 +10,24 @@ pub use self::file_builder::{FileBuilder, Sink}; #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct SyntaxKind(pub(crate) u32); -pub(crate) const EOF: SyntaxKind = SyntaxKind(10000); +pub(crate) const EOF: SyntaxKind = SyntaxKind(!0); pub(crate) const EOF_INFO: SyntaxInfo = SyntaxInfo { name: "EOF" }; +pub(crate) const TOMBSTONE: SyntaxKind = SyntaxKind(!0 - 1); +pub(crate) const TOMBSTONE_INFO: SyntaxInfo = SyntaxInfo { + name: "TOMBSTONE" +}; + + impl SyntaxKind { fn info(self) -> &'static SyntaxInfo { - if self == EOF { - return &EOF_INFO; + match self { + EOF => &EOF_INFO, + TOMBSTONE => &TOMBSTONE_INFO, + _ => syntax_info(self), } - syntax_info(self) } } -- cgit v1.2.3