From 5562931e4f3c6c57e9122c3ce34d941fb1ff6e5b Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sun, 7 Jan 2018 21:09:05 +0300 Subject: Introduce EOF token --- src/parser/event_parser/grammar.rs | 23 ++-- src/parser/event_parser/parser.rs | 35 +++--- src/tree/mod.rs | 218 +++++++++++++++++++------------------ 3 files changed, 136 insertions(+), 140 deletions(-) diff --git a/src/parser/event_parser/grammar.rs b/src/parser/event_parser/grammar.rs index f676a183c..64c6718cb 100644 --- a/src/parser/event_parser/grammar.rs +++ b/src/parser/event_parser/grammar.rs @@ -1,5 +1,6 @@ use super::parser::Parser; use {SyntaxKind}; +use tree::EOF; use syntax_kinds::*; // Items // @@ -18,11 +19,7 @@ pub(crate) fn file(p: &mut Parser) { } fn item_first(p: &Parser) -> bool { - let current = match p.current() { - Some(c) => c, - None => return false, - }; - match current { + match p.current() { STRUCT_KW | FN_KW => true, _ => false, } @@ -79,7 +76,7 @@ fn visibility(_: &mut Parser) { // Error recovery and high-order utils // fn node_if(p: &mut Parser, first: SyntaxKind, node_kind: SyntaxKind, rest: F) -> bool { - p.current_is(first) && { node(p, node_kind, |p| { p.bump(); rest(p); }); true } + p.current() == first && { node(p, node_kind, |p| { p.bump(); rest(p); }); true } } fn node(p: &mut Parser, node_kind: SyntaxKind, rest: F) { @@ -95,7 +92,7 @@ fn many bool>(p: &mut Parser, f: F) { fn comma_list bool>(p: &mut Parser, f: F) { many(p, |p| { f(p); - if p.is_eof() { + if p.current() == EOF { false } else { p.expect(COMMA); @@ -119,7 +116,7 @@ where f(p); return true; } - if p.is_eof() { + if p.current() == EOF { if skipped { p.finish(); } @@ -131,18 +128,14 @@ where .message(message) .emit(); } - p.bump().unwrap(); + p.bump(); skipped = true; } } impl<'p> Parser<'p> { - fn current_is(&self, kind: SyntaxKind) -> bool { - self.current() == Some(kind) - } - pub(crate) fn expect(&mut self, kind: SyntaxKind) -> bool { - if self.current_is(kind) { + if self.current() == kind { self.bump(); true } else { @@ -154,7 +147,7 @@ impl<'p> Parser<'p> { } fn optional(&mut self, kind: SyntaxKind) { - if self.current_is(kind) { + if self.current() == kind { self.bump(); } } diff --git a/src/parser/event_parser/parser.rs b/src/parser/event_parser/parser.rs index f0d1d358b..bec9dbab4 100644 --- a/src/parser/event_parser/parser.rs +++ b/src/parser/event_parser/parser.rs @@ -2,8 +2,7 @@ use {Token, SyntaxKind, TextUnit}; use super::{Event}; use super::super::is_insignificant; use syntax_kinds::{L_CURLY, R_CURLY, ERROR}; - -pub(crate) const EOF: SyntaxKind = SyntaxKind(10000); +use tree::EOF; pub(crate) struct Parser<'t> { @@ -46,19 +45,22 @@ impl<'t> Parser<'t> { } pub(crate) fn into_events(self) -> Vec { - assert!(self.is_eof()); + assert!(self.curly_limit.is_none()); + assert!(self.current() == EOF); self.events } - pub(crate) fn is_eof(&self) -> bool { + pub(crate) fn current(&self) -> SyntaxKind { if self.pos == self.tokens.len() { - return true + return EOF; } + let token = self.tokens[self.pos]; if let Some(limit) = self.curly_limit { - let token = self.tokens[self.pos]; - return limit == self.curly_level && token.kind == R_CURLY; + if limit == self.curly_level && token.kind == R_CURLY { + return EOF + } } - false + token.kind } pub(crate) fn start(&mut self, kind: SyntaxKind) { @@ -73,24 +75,17 @@ impl<'t> Parser<'t> { ErrorBuilder::new(self) } - pub(crate) fn current(&self) -> Option { - if self.is_eof() { - return None; - } - let token = self.tokens[self.pos]; - Some(token.kind) - } - - pub(crate) fn bump(&mut self) -> Option { - let kind = self.current()?; + pub(crate) fn bump(&mut self) -> SyntaxKind { + let kind = self.current(); match kind { L_CURLY => self.curly_level += 1, R_CURLY => self.curly_level -= 1, + EOF => return EOF, _ => (), } self.pos += 1; self.event(Event::Token { kind, n_raw_tokens: 1 }); - Some(kind) + kind } pub(crate) fn lookahead(&self, kinds: &[SyntaxKind]) -> bool { @@ -114,7 +109,7 @@ impl<'t> Parser<'t> { if !self.expect(R_CURLY) { self.start(ERROR); while self.curly_level > old_level { - if self.bump().is_none() { + if self.bump() == EOF { break; } } diff --git a/src/tree/mod.rs b/src/tree/mod.rs index 7f4d427ba..d8f843737 100644 --- a/src/tree/mod.rs +++ b/src/tree/mod.rs @@ -10,8 +10,16 @@ pub use self::file_builder::{FileBuilder, Sink}; #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct SyntaxKind(pub(crate) u32); +pub(crate) const EOF: SyntaxKind = SyntaxKind(10000); +pub(crate) const EOF_INFO: SyntaxInfo = SyntaxInfo { + name: "EOF" +}; + impl SyntaxKind { fn info(self) -> &'static SyntaxInfo { + if self == EOF { + return &EOF_INFO; + } syntax_info(self) } } @@ -35,72 +43,72 @@ pub struct Token { } pub struct File { - text: String, - nodes: Vec, - errors: Vec, + text: String, + nodes: Vec, + errors: Vec, } impl File { - pub fn root<'f>(&'f self) -> Node<'f> { - assert!(!self.nodes.is_empty()); - Node { file: self, idx: NodeIdx(0) } - } + pub fn root<'f>(&'f self) -> Node<'f> { + assert!(!self.nodes.is_empty()); + Node { file: self, idx: NodeIdx(0) } + } } #[derive(Clone, Copy)] pub struct Node<'f> { - file: &'f File, - idx: NodeIdx, + file: &'f File, + idx: NodeIdx, } impl<'f> Node<'f> { - pub fn kind(&self) -> SyntaxKind { - self.data().kind - } + pub fn kind(&self) -> SyntaxKind { + self.data().kind + } - pub fn range(&self) -> TextRange { - self.data().range - } + pub fn range(&self) -> TextRange { + self.data().range + } - pub fn text(&self) -> &'f str { - &self.file.text.as_str()[self.range()] - } + pub fn text(&self) -> &'f str { + &self.file.text.as_str()[self.range()] + } - pub fn parent(&self) -> Option> { - self.as_node(self.data().parent) - } + pub fn parent(&self) -> Option> { + self.as_node(self.data().parent) + } - pub fn children(&self) -> Children<'f> { - Children { next: self.as_node(self.data().first_child) } - } + pub fn children(&self) -> Children<'f> { + Children { next: self.as_node(self.data().first_child) } + } - pub fn errors(&self) -> SyntaxErrors<'f> { - let pos = self.file.errors.iter().position(|e| e.node == self.idx); - let next = pos - .map(|i| ErrorIdx(i as u32)) - .map(|idx| SyntaxError { file: self.file, idx }); - SyntaxErrors { next } - } + pub fn errors(&self) -> SyntaxErrors<'f> { + let pos = self.file.errors.iter().position(|e| e.node == self.idx); + let next = pos + .map(|i| ErrorIdx(i as u32)) + .map(|idx| SyntaxError { file: self.file, idx }); + SyntaxErrors { next } + } - fn data(&self) -> &'f NodeData { - &self.file.nodes[self.idx] - } + fn data(&self) -> &'f NodeData { + &self.file.nodes[self.idx] + } - fn as_node(&self, idx: Option) -> Option> { - idx.map(|idx| Node { file: self.file, idx }) - } + fn as_node(&self, idx: Option) -> Option> { + idx.map(|idx| Node { file: self.file, idx }) + } } impl<'f> fmt::Debug for Node<'f> { - fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - write!(fmt, "{:?}@{:?}", self.kind(), self.range()) - } + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + write!(fmt, "{:?}@{:?}", self.kind(), self.range()) + } } impl<'f> cmp::PartialEq> for Node<'f> { - fn eq(&self, other: &Node<'f>) -> bool { - self.idx == other.idx && ::std::ptr::eq(self.file, other.file) - } + fn eq(&self, other: &Node<'f>) -> bool { + self.idx == other.idx && ::std::ptr::eq(self.file, other.file) + } } impl<'f> cmp::Eq for Node<'f> { @@ -108,66 +116,66 @@ impl<'f> cmp::Eq for Node<'f> { #[derive(Clone, Copy)] pub struct SyntaxError<'f> { - file: &'f File, - idx: ErrorIdx, + file: &'f File, + idx: ErrorIdx, } impl<'f> SyntaxError<'f> { - pub fn message(&self) -> &'f str { - self.data().message.as_str() - } - - pub fn after_child(&self) -> Option> { - let idx = self.data().after_child?; - Some(Node { file: self.file, idx }) - } - - fn data(&self) -> &'f SyntaxErrorData { - &self.file.errors[self.idx] - } - - fn next(&self) -> Option> { - let next_idx = self.idx.0 + 1; - if !((next_idx as usize) < self.file.errors.len()) { - return None; - } - let result = SyntaxError { - file: self.file, - idx: ErrorIdx(next_idx) - }; - if result.data().node != self.data().node { - return None; - } - Some(result) - } + pub fn message(&self) -> &'f str { + self.data().message.as_str() + } + + pub fn after_child(&self) -> Option> { + let idx = self.data().after_child?; + Some(Node { file: self.file, idx }) + } + + fn data(&self) -> &'f SyntaxErrorData { + &self.file.errors[self.idx] + } + + fn next(&self) -> Option> { + let next_idx = self.idx.0 + 1; + if !((next_idx as usize) < self.file.errors.len()) { + return None; + } + let result = SyntaxError { + file: self.file, + idx: ErrorIdx(next_idx) + }; + if result.data().node != self.data().node { + return None; + } + Some(result) + } } pub struct Children<'f> { - next: Option>, + next: Option>, } impl<'f> Iterator for Children<'f> { - type Item = Node<'f>; + type Item = Node<'f>; - fn next(&mut self) -> Option> { - let next = self.next; - self.next = next.and_then(|node| node.as_node(node.data().next_sibling)); - next - } + fn next(&mut self) -> Option> { + let next = self.next; + self.next = next.and_then(|node| node.as_node(node.data().next_sibling)); + next + } } pub struct SyntaxErrors<'f> { - next: Option>, + next: Option>, } impl<'f> Iterator for SyntaxErrors<'f> { - type Item = SyntaxError<'f>; + type Item = SyntaxError<'f>; - fn next(&mut self) -> Option> { - let next = self.next; - self.next = next.as_ref().and_then(SyntaxError::next); - next - } + fn next(&mut self) -> Option> { + let next = self.next; + self.next = next.as_ref().and_then(SyntaxError::next); + next + } } @@ -175,40 +183,40 @@ impl<'f> Iterator for SyntaxErrors<'f> { struct NodeIdx(u32); struct NodeData { - kind: SyntaxKind, - range: TextRange, - parent: Option, - first_child: Option, - next_sibling: Option, + kind: SyntaxKind, + range: TextRange, + parent: Option, + first_child: Option, + next_sibling: Option, } impl ::std::ops::Index for Vec { - type Output = NodeData; + type Output = NodeData; - fn index(&self, NodeIdx(idx): NodeIdx) -> &NodeData { - &self[idx as usize] - } + fn index(&self, NodeIdx(idx): NodeIdx) -> &NodeData { + &self[idx as usize] + } } impl ::std::ops::IndexMut for Vec { - fn index_mut(&mut self, NodeIdx(idx): NodeIdx) -> &mut NodeData { - &mut self[idx as usize] - } + fn index_mut(&mut self, NodeIdx(idx): NodeIdx) -> &mut NodeData { + &mut self[idx as usize] + } } #[derive(Clone, Copy)] struct ErrorIdx(u32); struct SyntaxErrorData { - node: NodeIdx, - message: String, - after_child: Option, + node: NodeIdx, + message: String, + after_child: Option, } impl ::std::ops::Index for Vec { - type Output = SyntaxErrorData; + type Output = SyntaxErrorData; - fn index(&self, ErrorIdx(idx): ErrorIdx) -> &SyntaxErrorData { - &self[idx as usize] - } + fn index(&self, ErrorIdx(idx): ErrorIdx) -> &SyntaxErrorData { + &self[idx as usize] + } } -- cgit v1.2.3