From 85c42fba1291f1cc41fb7bfec63117895b394fc5 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sun, 4 Feb 2018 16:46:26 +0300 Subject: Support contextual tokens --- src/parser/event.rs | 27 +++++++++++++++++---------- src/parser/grammar/items/mod.rs | 22 ++++++++++++++++++++-- src/parser/input.rs | 7 ++++--- src/parser/parser.rs | 19 ++++++++++++++++++- 4 files changed, 59 insertions(+), 16 deletions(-) (limited to 'src/parser') diff --git a/src/parser/event.rs b/src/parser/event.rs index fd6bdc086..64d751d63 100644 --- a/src/parser/event.rs +++ b/src/parser/event.rs @@ -1,4 +1,4 @@ -use {File, FileBuilder, Sink, SyntaxKind, Token}; +use {File, FileBuilder, Sink, SyntaxKind, Token, TextUnit}; use syntax_kinds::TOMBSTONE; use super::is_insignificant; @@ -120,18 +120,25 @@ pub(super) fn to_file(text: String, tokens: &[Token], events: Vec) -> Fil builder.finish_internal() } &Event::Token { - kind: _, + kind, mut n_raw_tokens, - } => loop { - let token = tokens[idx]; - if !is_insignificant(token.kind) { - n_raw_tokens -= 1; + } => { + // FIXME: currently, we attach whitespace to some random node + // this should be done in a sensible manner instead + loop { + let token = tokens[idx]; + if !is_insignificant(token.kind) { + break; + } + builder.leaf(token.kind, token.len); + idx += 1 } - idx += 1; - builder.leaf(token.kind, token.len); - if n_raw_tokens == 0 { - break; + let mut len = TextUnit::new(0); + for _ in 0..n_raw_tokens { + len += tokens[idx].len; + idx += 1; } + builder.leaf(kind, len); }, &Event::Error { ref message } => builder.error().message(message.clone()).emit(), } diff --git a/src/parser/grammar/items/mod.rs b/src/parser/grammar/items/mod.rs index 3612802e1..4afe2e418 100644 --- a/src/parser/grammar/items/mod.rs +++ b/src/parser/grammar/items/mod.rs @@ -81,7 +81,6 @@ fn item(p: &mut Parser) { CONST_ITEM } }, - // TODO: auto trait // test unsafe_trait // unsafe trait T {} UNSAFE_KW if la == TRAIT_KW => { @@ -89,7 +88,16 @@ fn item(p: &mut Parser) { traits::trait_item(p); TRAIT_ITEM } - // TODO: default impl + + // test unsafe_auto_trait + // unsafe auto trait T {} + UNSAFE_KW if p.at_kw(1, "auto") && p.nth(2) == TRAIT_KW => { + p.bump(); + p.bump_remap(AUTO_KW); + traits::trait_item(p); + TRAIT_ITEM + } + // test unsafe_impl // unsafe impl Foo {} UNSAFE_KW if la == IMPL_KW => { @@ -97,6 +105,16 @@ fn item(p: &mut Parser) { traits::impl_item(p); IMPL_ITEM } + + // test unsafe_default_impl + // unsafe default impl Foo {} + UNSAFE_KW if p.at_kw(1, "default") && p.nth(2) == IMPL_KW => { + p.bump(); + p.bump_remap(DEFAULT_KW); + traits::impl_item(p); + IMPL_ITEM + } + MOD_KW => { mod_item(p); MOD_ITEM diff --git a/src/parser/input.rs b/src/parser/input.rs index 162b9ef5f..2ad621166 100644 --- a/src/parser/input.rs +++ b/src/parser/input.rs @@ -46,9 +46,10 @@ impl<'t> ParserInput<'t> { if !(idx < self.tokens.len()) { return ""; } - let start_offset = self.start_offsets[idx]; - let end_offset = self.tokens[idx].len; - let range = TextRange::from_to(start_offset, end_offset); + let range = TextRange::from_len( + self.start_offsets[idx], + self.tokens[idx].len + ); &self.text[range] } } diff --git a/src/parser/parser.rs b/src/parser/parser.rs index bb775c4a5..7e1b22ee5 100644 --- a/src/parser/parser.rs +++ b/src/parser/parser.rs @@ -1,6 +1,6 @@ use super::Event; use super::input::{InputPosition, ParserInput}; -use SyntaxKind::{self, EOF, TOMBSTONE}; +use SyntaxKind::{self, EOF, TOMBSTONE, IDENT}; pub(crate) struct Marker { pos: u32, @@ -145,14 +145,31 @@ impl<'t> Parser<'t> { }); } + pub(crate) fn bump_remap(&mut self, kind: SyntaxKind) { + if self.current() == EOF { + // TODO: panic!? + return; + } + self.pos += 1; + self.event(Event::Token { + kind, + n_raw_tokens: 1, + }); + } + pub(crate) fn nth(&self, n: u32) -> SyntaxKind { self.inp.kind(self.pos + n) } + pub(crate) fn at_kw(&self, n: u32, t: &str) -> bool { + self.nth(n) == IDENT && self.inp.text(self.pos + n) == t + } + pub(crate) fn current(&self) -> SyntaxKind { self.nth(0) } + fn event(&mut self, event: Event) { self.events.push(event) } -- cgit v1.2.3 From c119e8fd8b0106bb2e031e76ac1be89bd10a0674 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sun, 4 Feb 2018 17:06:00 +0300 Subject: G: unsafe fn --- src/parser/grammar/items/mod.rs | 92 ++++++++++++++++++++++++++++------------- src/parser/parser.rs | 6 +-- 2 files changed, 67 insertions(+), 31 deletions(-) (limited to 'src/parser') diff --git a/src/parser/grammar/items/mod.rs b/src/parser/grammar/items/mod.rs index 4afe2e418..588fca4c6 100644 --- a/src/parser/grammar/items/mod.rs +++ b/src/parser/grammar/items/mod.rs @@ -81,38 +81,74 @@ fn item(p: &mut Parser) { CONST_ITEM } }, - // test unsafe_trait - // unsafe trait T {} - UNSAFE_KW if la == TRAIT_KW => { + UNSAFE_KW => { p.bump(); - traits::trait_item(p); - TRAIT_ITEM - } + let la = p.nth(1); + match p.current() { + // test unsafe_trait + // unsafe trait T {} + TRAIT_KW => { + traits::trait_item(p); + TRAIT_ITEM + } - // test unsafe_auto_trait - // unsafe auto trait T {} - UNSAFE_KW if p.at_kw(1, "auto") && p.nth(2) == TRAIT_KW => { - p.bump(); - p.bump_remap(AUTO_KW); - traits::trait_item(p); - TRAIT_ITEM - } + // test unsafe_auto_trait + // unsafe auto trait T {} + IDENT if p.at_kw("auto") && la == TRAIT_KW => { + p.bump_remap(AUTO_KW); + traits::trait_item(p); + TRAIT_ITEM + } - // test unsafe_impl - // unsafe impl Foo {} - UNSAFE_KW if la == IMPL_KW => { - p.bump(); - traits::impl_item(p); - IMPL_ITEM - } + // test unsafe_impl + // unsafe impl Foo {} + IMPL_KW => { + traits::impl_item(p); + IMPL_ITEM + } - // test unsafe_default_impl - // unsafe default impl Foo {} - UNSAFE_KW if p.at_kw(1, "default") && p.nth(2) == IMPL_KW => { - p.bump(); - p.bump_remap(DEFAULT_KW); - traits::impl_item(p); - IMPL_ITEM + // test unsafe_default_impl + // unsafe default impl Foo {} + IDENT if p.at_kw("default") && la == IMPL_KW => { + p.bump_remap(DEFAULT_KW); + traits::impl_item(p); + IMPL_ITEM + } + + // test unsafe_extern_fn + // unsafe extern "C" fn foo() {} + EXTERN_KW => { + abi(p); + if !p.at(FN_KW) { + item.abandon(p); + p.error().message("expected function").emit(); + return + } + fn_item(p); + FN_ITEM + } + + // test unsafe_fn + // unsafe fn foo() {} + FN_KW => { + fn_item(p); + FN_ITEM + } + + t => { + item.abandon(p); + let message = "expected `trait`, `impl` or `fn`"; + + // test unsafe_block_in_mod + // fn foo(){} unsafe { } fn bar(){} + if t == L_CURLY { + error_block(p, message); + } else { + p.error().message(message).emit(); + } + return; + } + } } MOD_KW => { diff --git a/src/parser/parser.rs b/src/parser/parser.rs index 7e1b22ee5..53a116fb7 100644 --- a/src/parser/parser.rs +++ b/src/parser/parser.rs @@ -1,6 +1,6 @@ use super::Event; use super::input::{InputPosition, ParserInput}; -use SyntaxKind::{self, EOF, TOMBSTONE, IDENT}; +use SyntaxKind::{self, EOF, TOMBSTONE}; pub(crate) struct Marker { pos: u32, @@ -161,8 +161,8 @@ impl<'t> Parser<'t> { self.inp.kind(self.pos + n) } - pub(crate) fn at_kw(&self, n: u32, t: &str) -> bool { - self.nth(n) == IDENT && self.inp.text(self.pos + n) == t + pub(crate) fn at_kw(&self, t: &str) -> bool { + self.inp.text(self.pos) == t } pub(crate) fn current(&self) -> SyntaxKind { -- cgit v1.2.3 From 351107d0b143e2c3497bd0f424f0d76bc51df0c0 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sun, 4 Feb 2018 17:06:43 +0300 Subject: Reformat --- src/parser/event.rs | 4 ++-- src/parser/grammar/items/mod.rs | 2 +- src/parser/input.rs | 5 +---- src/parser/parser.rs | 1 - 4 files changed, 4 insertions(+), 8 deletions(-) (limited to 'src/parser') diff --git a/src/parser/event.rs b/src/parser/event.rs index 64d751d63..e97350c89 100644 --- a/src/parser/event.rs +++ b/src/parser/event.rs @@ -1,4 +1,4 @@ -use {File, FileBuilder, Sink, SyntaxKind, Token, TextUnit}; +use {File, FileBuilder, Sink, SyntaxKind, TextUnit, Token}; use syntax_kinds::TOMBSTONE; use super::is_insignificant; @@ -139,7 +139,7 @@ pub(super) fn to_file(text: String, tokens: &[Token], events: Vec) -> Fil idx += 1; } builder.leaf(kind, len); - }, + } &Event::Error { ref message } => builder.error().message(message.clone()).emit(), } } diff --git a/src/parser/grammar/items/mod.rs b/src/parser/grammar/items/mod.rs index 588fca4c6..37f2ab132 100644 --- a/src/parser/grammar/items/mod.rs +++ b/src/parser/grammar/items/mod.rs @@ -122,7 +122,7 @@ fn item(p: &mut Parser) { if !p.at(FN_KW) { item.abandon(p); p.error().message("expected function").emit(); - return + return; } fn_item(p); FN_ITEM diff --git a/src/parser/input.rs b/src/parser/input.rs index 2ad621166..4d74ac1b1 100644 --- a/src/parser/input.rs +++ b/src/parser/input.rs @@ -46,10 +46,7 @@ impl<'t> ParserInput<'t> { if !(idx < self.tokens.len()) { return ""; } - let range = TextRange::from_len( - self.start_offsets[idx], - self.tokens[idx].len - ); + let range = TextRange::from_len(self.start_offsets[idx], self.tokens[idx].len); &self.text[range] } } diff --git a/src/parser/parser.rs b/src/parser/parser.rs index 53a116fb7..752d532d0 100644 --- a/src/parser/parser.rs +++ b/src/parser/parser.rs @@ -169,7 +169,6 @@ impl<'t> Parser<'t> { self.nth(0) } - fn event(&mut self, event: Event) { self.events.push(event) } -- cgit v1.2.3