From ac37a11f04b31f792068a1cb50dbbf5ccd4d982d Mon Sep 17 00:00:00 2001 From: Veetaha Date: Sun, 26 Jan 2020 20:44:49 +0200 Subject: Reimplemented lexer with vectors instead of iterators --- crates/ra_mbe/src/subtree_source.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'crates/ra_mbe/src/subtree_source.rs') diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs index b841c39d3..72ac8df03 100644 --- a/crates/ra_mbe/src/subtree_source.rs +++ b/crates/ra_mbe/src/subtree_source.rs @@ -1,7 +1,7 @@ //! FIXME: write short doc here use ra_parser::{Token, TokenSource}; -use ra_syntax::{classify_literal, SmolStr, SyntaxKind, SyntaxKind::*, T}; +use ra_syntax::{single_token, SmolStr, SyntaxKind, SyntaxKind::*, T}; use std::cell::{Cell, Ref, RefCell}; use tt::buffer::{Cursor, TokenBuffer}; @@ -129,8 +129,10 @@ fn convert_delim(d: Option, closing: bool) -> TtToken { } fn convert_literal(l: &tt::Literal) -> TtToken { - let kind = - classify_literal(&l.text).map(|tkn| tkn.kind).unwrap_or_else(|| match l.text.as_ref() { + let kind = single_token(&l.text) + .map(|parsed| parsed.token.kind) + .filter(|kind| kind.is_literal()) + .unwrap_or_else(|| match l.text.as_ref() { "true" => T![true], "false" => T![false], _ => panic!("Fail to convert given literal {:#?}", &l), -- cgit v1.2.3 From 9e7eaa959f9dc368a55f1a80b35651b78b3d0883 Mon Sep 17 00:00:00 2001 From: Veetaha Date: Tue, 28 Jan 2020 07:09:13 +0200 Subject: ra_syntax: refactored the lexer design as per @matklad and @kiljacken PR review --- crates/ra_mbe/src/subtree_source.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'crates/ra_mbe/src/subtree_source.rs') diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs index 72ac8df03..c9f42b3dd 100644 --- a/crates/ra_mbe/src/subtree_source.rs +++ b/crates/ra_mbe/src/subtree_source.rs @@ -1,7 +1,7 @@ //! FIXME: write short doc here use ra_parser::{Token, TokenSource}; -use ra_syntax::{single_token, SmolStr, SyntaxKind, SyntaxKind::*, T}; +use ra_syntax::{lex_single_valid_syntax_kind, SmolStr, SyntaxKind, SyntaxKind::*, T}; use std::cell::{Cell, Ref, RefCell}; use tt::buffer::{Cursor, TokenBuffer}; @@ -129,8 +129,7 @@ fn convert_delim(d: Option, closing: bool) -> TtToken { } fn convert_literal(l: &tt::Literal) -> TtToken { - let kind = single_token(&l.text) - .map(|parsed| parsed.token.kind) + let kind = lex_single_valid_syntax_kind(&l.text) .filter(|kind| kind.is_literal()) .unwrap_or_else(|| match l.text.as_ref() { "true" => T![true], -- cgit v1.2.3