Support contextual tokens

4 files changed, 59 insertions, 16 deletions

diff --git a/src/parser/event.rs b/src/parser/event.rs
index fd6bdc086..64d751d63 100644
--- a/src/parser/event.rs
+++ b/src/parser/event.rs
@@ -1,4 +1,4 @@
-use {File, FileBuilder, Sink, SyntaxKind, Token};
+use {File, FileBuilder, Sink, SyntaxKind, Token, TextUnit};
 use syntax_kinds::TOMBSTONE;
 use super::is_insignificant;
 
@@ -120,18 +120,25 @@ pub(super) fn to_file(text: String, tokens: &[Token], events: Vec<Event>) -> Fil
                 builder.finish_internal()
             }
             &Event::Token {
-                kind: _,
+                kind,
                 mut n_raw_tokens,
-            } => loop {
-                let token = tokens[idx];
-                if !is_insignificant(token.kind) {
-                    n_raw_tokens -= 1;
+            } => {
+                // FIXME: currently, we attach whitespace to some random node
+                // this should be done in a sensible manner instead
+                loop {
+                    let token = tokens[idx];
+                    if !is_insignificant(token.kind) {
+                        break;
+                    }
+                    builder.leaf(token.kind, token.len);
+                    idx += 1
                 }
-                idx += 1;
-                builder.leaf(token.kind, token.len);
-                if n_raw_tokens == 0 {
-                    break;
+                let mut len = TextUnit::new(0);
+                for _ in 0..n_raw_tokens {
+                    len += tokens[idx].len;
+                    idx += 1;
                 }
+                builder.leaf(kind, len);
             },
             &Event::Error { ref message } => builder.error().message(message.clone()).emit(),
         }
diff --git a/src/parser/grammar/items/mod.rs b/src/parser/grammar/items/mod.rs
index 3612802e1..4afe2e418 100644
--- a/src/parser/grammar/items/mod.rs
+++ b/src/parser/grammar/items/mod.rs
@@ -81,7 +81,6 @@ fn item(p: &mut Parser) {
                 CONST_ITEM
             }
         },
-        // TODO: auto trait
         // test unsafe_trait
         // unsafe trait T {}
         UNSAFE_KW if la == TRAIT_KW => {
@@ -89,7 +88,16 @@ fn item(p: &mut Parser) {
             traits::trait_item(p);
             TRAIT_ITEM
         }
-        // TODO: default impl
+
+        // test unsafe_auto_trait
+        // unsafe auto trait T {}
+        UNSAFE_KW if p.at_kw(1, "auto") && p.nth(2) == TRAIT_KW => {
+            p.bump();
+            p.bump_remap(AUTO_KW);
+            traits::trait_item(p);
+            TRAIT_ITEM
+        }
+
         // test unsafe_impl
         // unsafe impl Foo {}
         UNSAFE_KW if la == IMPL_KW => {
@@ -97,6 +105,16 @@ fn item(p: &mut Parser) {
             traits::impl_item(p);
             IMPL_ITEM
         }
+
+        // test unsafe_default_impl
+        // unsafe default impl Foo {}
+        UNSAFE_KW if p.at_kw(1, "default") && p.nth(2) == IMPL_KW => {
+            p.bump();
+            p.bump_remap(DEFAULT_KW);
+            traits::impl_item(p);
+            IMPL_ITEM
+        }
+
         MOD_KW => {
             mod_item(p);
             MOD_ITEM
diff --git a/src/parser/input.rs b/src/parser/input.rs
index 162b9ef5f..2ad621166 100644
--- a/src/parser/input.rs
+++ b/src/parser/input.rs
@@ -46,9 +46,10 @@ impl<'t> ParserInput<'t> {
         if !(idx < self.tokens.len()) {
             return "";
         }
-        let start_offset = self.start_offsets[idx];
-        let end_offset = self.tokens[idx].len;
-        let range = TextRange::from_to(start_offset, end_offset);
+        let range = TextRange::from_len(
+            self.start_offsets[idx],
+            self.tokens[idx].len
+        );
         &self.text[range]
     }
 }
diff --git a/src/parser/parser.rs b/src/parser/parser.rs
index bb775c4a5..7e1b22ee5 100644
--- a/src/parser/parser.rs
+++ b/src/parser/parser.rs
@@ -1,6 +1,6 @@
 use super::Event;
 use super::input::{InputPosition, ParserInput};
-use SyntaxKind::{self, EOF, TOMBSTONE};
+use SyntaxKind::{self, EOF, TOMBSTONE, IDENT};
 
 pub(crate) struct Marker {
     pos: u32,
@@ -145,14 +145,31 @@ impl<'t> Parser<'t> {
         });
     }
 
+    pub(crate) fn bump_remap(&mut self, kind: SyntaxKind) {
+        if self.current() == EOF {
+            // TODO: panic!?
+            return;
+        }
+        self.pos += 1;
+        self.event(Event::Token {
+            kind,
+            n_raw_tokens: 1,
+        });
+    }
+
     pub(crate) fn nth(&self, n: u32) -> SyntaxKind {
         self.inp.kind(self.pos + n)
     }
 
+    pub(crate) fn at_kw(&self, n: u32, t: &str) -> bool {
+        self.nth(n) == IDENT && self.inp.text(self.pos + n) == t
+    }
+
     pub(crate) fn current(&self) -> SyntaxKind {
         self.nth(0)
     }
 
+
     fn event(&mut self, event: Event) {
         self.events.push(event)
     }