Parser: first scraches

6 files changed, 81 insertions, 17 deletions

diff --git a/src/parser/event_parser/grammar.rs b/src/parser/event_parser/grammar.rs
index 5219ed535..0896506fb 100644
--- a/src/parser/event_parser/grammar.rs
+++ b/src/parser/event_parser/grammar.rs
@@ -40,6 +40,12 @@ fn mod_items(p: &mut Parser) {
 fn item(p: &mut Parser) -> Result {
     outer_attributes(p)?;
     visibility(p)?;
+    if p.current_is(STRUCT_KW) {
+        p.start(STRUCT_ITEM);
+        p.bump();
+        p.finish();
+        return OK;
+    }
     ERR
 }
 
diff --git a/src/parser/event_parser/mod.rs b/src/parser/event_parser/mod.rs
index bdfd23974..1228236a9 100644
--- a/src/parser/event_parser/mod.rs
+++ b/src/parser/event_parser/mod.rs
@@ -4,6 +4,7 @@ use syntax_kinds::*;
 mod grammar;
 mod parser;
 
+#[derive(Debug)]
 pub(crate) enum Event {
     Start { kind: SyntaxKind },
     Finish,
diff --git a/src/parser/event_parser/parser.rs b/src/parser/event_parser/parser.rs
index 0e4d44b79..2d5418a29 100644
--- a/src/parser/event_parser/parser.rs
+++ b/src/parser/event_parser/parser.rs
@@ -1,5 +1,6 @@
 use {Token, SyntaxKind, TextUnit};
-use super::Event;
+use super::{Event};
+use super::super::is_insignificant;
 use syntax_kinds::{WHITESPACE, COMMENT};
 
 pub struct Parser<'t> {
@@ -16,9 +17,8 @@ impl<'t> Parser<'t> {
         let mut non_ws_tokens = Vec::new();
         let mut len = TextUnit::new(0);
         for (idx, &token) in raw_tokens.iter().enumerate() {
-            match token.kind {
-                WHITESPACE | COMMENT => (),
-                _ => non_ws_tokens.push((idx, len)),
+            if !is_insignificant(token.kind) {
+                non_ws_tokens.push((idx, len))
             }
             len += token.len;
         }
@@ -50,15 +50,25 @@ impl<'t> Parser<'t> {
         self.event(Event::Finish);
     }
 
-    pub(crate) fn bump(&mut self) -> Option<SyntaxKind> {
+    pub(crate) fn current(&self) -> Option<SyntaxKind> {
         if self.is_eof() {
             return None;
         }
         let idx = self.non_ws_tokens[self.pos].0;
-        self.pos += 1;
         Some(self.raw_tokens[idx].kind)
     }
 
+    pub(crate) fn current_is(&self, kind: SyntaxKind) -> bool {
+        self.current() == Some(kind)
+    }
+
+    pub(crate) fn bump(&mut self) -> Option<SyntaxKind> {
+        let kind = self.current()?;
+        self.pos += 1;
+        self.event(Event::Token { kind, n_raw_tokens: 1 });
+        Some(kind)
+    }
+
     fn event(&mut self, event: Event) {
         self.events.push(event)
     }
diff --git a/src/parser/mod.rs b/src/parser/mod.rs
index ccccd78f9..a632fbc01 100644
--- a/src/parser/mod.rs
+++ b/src/parser/mod.rs
@@ -8,17 +8,50 @@ use self::event_parser::Event;
 
 pub fn parse(text: String, tokens: &[Token]) -> File {
     let events = event_parser::parse(&text, tokens);
-    from_events_to_file(text, events)
+    from_events_to_file(text, tokens, events)
 }
 
-fn from_events_to_file(text: String, events: Vec<Event>) -> File {
+fn from_events_to_file(
+    text: String,
+    tokens: &[Token],
+    events: Vec<Event>,
+) -> File {
     let mut builder = FileBuilder::new(text);
+    let mut idx = 0;
     for event in events {
         match event {
             Event::Start { kind } => builder.start_internal(kind),
-            Event::Finish => builder.finish_internal(),
-            Event::Token { .. } => unimplemented!(),
+            Event::Finish => {
+                while idx < tokens.len() {
+                    let token = tokens[idx];
+                    if is_insignificant(token.kind) {
+                        idx += 1;
+                        builder.leaf(token.kind, token.len);
+                    } else {
+                        break;
+                    }
+                }
+                builder.finish_internal()
+            },
+            Event::Token { kind, mut n_raw_tokens } => loop {
+                let token = tokens[idx];
+                if !is_insignificant(token.kind) {
+                    n_raw_tokens -= 1;
+                }
+                idx += 1;
+                builder.leaf(token.kind, token.len);
+                if n_raw_tokens == 0 {
+                    break;
+                }
+            },
         }
     }
     builder.finish()
 }
+
+fn is_insignificant(kind: SyntaxKind) -> bool {
+    match kind {
+        WHITESPACE | COMMENT => true,
+        _ => false,
+    }
+}
\ No newline at end of file
diff --git a/src/syntax_kinds.rs b/src/syntax_kinds.rs
index a1bcad062..d9b1462a7 100644
--- a/src/syntax_kinds.rs
+++ b/src/syntax_kinds.rs
@@ -59,8 +59,9 @@ pub const COMMENT: SyntaxKind = SyntaxKind(54);
 pub const DOC_COMMENT: SyntaxKind = SyntaxKind(55);
 pub const SHEBANG: SyntaxKind = SyntaxKind(56);
 pub const FILE: SyntaxKind = SyntaxKind(57);
+pub const STRUCT_ITEM: SyntaxKind = SyntaxKind(58);
 
-static INFOS: [SyntaxInfo; 58] = [
+static INFOS: [SyntaxInfo; 59] = [
     SyntaxInfo { name: "USE_KW" },
     SyntaxInfo { name: "FN_KW" },
     SyntaxInfo { name: "STRUCT_KW" },
@@ -119,6 +120,7 @@ static INFOS: [SyntaxInfo; 58] = [
     SyntaxInfo { name: "DOC_COMMENT" },
     SyntaxInfo { name: "SHEBANG" },
     SyntaxInfo { name: "FILE" },
+    SyntaxInfo { name: "STRUCT_ITEM" },
 ];
 
 pub(crate) fn syntax_info(kind: SyntaxKind) -> &'static SyntaxInfo {
diff --git a/src/tree/file_builder.rs b/src/tree/file_builder.rs
index eba850fef..da8b8f824 100644
--- a/src/tree/file_builder.rs
+++ b/src/tree/file_builder.rs
@@ -64,8 +64,14 @@ impl FileBuilder {
     }
 
     pub fn finish(self) -> File {
-        assert!(self.in_progress.is_empty());
-        assert!(self.pos == (self.text.len() as u32).into());
+        assert!(
+            self.in_progress.is_empty(),
+            "some nodes in FileBuilder are unfinished"
+        );
+        assert!(
+            self.pos == (self.text.len() as u32).into(),
+            "nodes in FileBuilder do not cover the whole file"
+        );
         File {
             text: self.text,
             nodes: self.nodes,
@@ -81,11 +87,17 @@ impl FileBuilder {
     fn push_child(&mut self, mut child: NodeData) -> NodeIdx {
         child.parent = Some(self.current_id());
         let id = self.new_node(child);
-        if let Some(sibling) = self.current_sibling() {
-            fill(&mut sibling.next_sibling, id);
-            return id
+        {
+
+            let (parent, sibling) = *self.in_progress.last().unwrap();
+            let slot = if let Some(idx) = sibling {
+                &mut self.nodes[idx].next_sibling
+            } else {
+                &mut self.nodes[parent].first_child
+            };
+            fill(slot, id);
         }
-        fill(&mut self.current_parent().first_child, id);
+        self.in_progress.last_mut().unwrap().1 = Some(id);
         id
     }