From f4a6d92050fd1e3b3a92883f0cbde04876fe12dc Mon Sep 17 00:00:00 2001 From: Evgenii P Date: Fri, 9 Aug 2019 14:08:34 +0700 Subject: Reduce code duplication in lexer by introducing small macro --- crates/ra_syntax/src/parsing/lexer.rs | 37 ++++++++++++++++------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/crates/ra_syntax/src/parsing/lexer.rs b/crates/ra_syntax/src/parsing/lexer.rs index 2a4343b0a..926811d96 100644 --- a/crates/ra_syntax/src/parsing/lexer.rs +++ b/crates/ra_syntax/src/parsing/lexer.rs @@ -12,6 +12,21 @@ pub struct Token { pub len: TextUnit, } +macro_rules! match_literal_kind { + ($kind:expr) => { + match $kind { + ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER, + ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER, + ra_rustc_lexer::LiteralKind::Char { .. } => CHAR, + ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE, + ra_rustc_lexer::LiteralKind::Str { .. } => STRING, + ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING, + ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING, + ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING, + } + } +} + /// Break a string up into its component tokens pub fn tokenize(text: &str) -> Vec { if text.is_empty() { @@ -53,16 +68,7 @@ pub fn tokenize(text: &str) -> Vec { } } ra_rustc_lexer::TokenKind::RawIdent => IDENT, - ra_rustc_lexer::TokenKind::Literal { kind, .. } => match kind { - ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER, - ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER, - ra_rustc_lexer::LiteralKind::Char { .. } => CHAR, - ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE, - ra_rustc_lexer::LiteralKind::Str { .. } => STRING, - ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING, - ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING, - ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING, - }, + ra_rustc_lexer::TokenKind::Literal { kind, .. } => match_literal_kind!(kind), ra_rustc_lexer::TokenKind::Lifetime { .. } => LIFETIME, ra_rustc_lexer::TokenKind::Semi => SEMI, ra_rustc_lexer::TokenKind::Comma => COMMA, @@ -131,16 +137,7 @@ pub fn classify_literal(text: &str) -> Option { return None; } let kind = match t.kind { - ra_rustc_lexer::TokenKind::Literal { kind, .. } => match kind { - ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER, - ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER, - ra_rustc_lexer::LiteralKind::Char { .. } => CHAR, - ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE, - ra_rustc_lexer::LiteralKind::Str { .. } => STRING, - ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING, - ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING, - ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING, - }, + ra_rustc_lexer::TokenKind::Literal { kind, .. } => match_literal_kind!(kind), _ => return None, }; Some(Token { kind, len: TextUnit::from_usize(t.len) }) -- cgit v1.2.3 From 073cf423916c8854e060a1127856dc4736fe83b1 Mon Sep 17 00:00:00 2001 From: Evgenii P Date: Fri, 9 Aug 2019 14:23:13 +0700 Subject: rustfmt --- crates/ra_syntax/src/parsing/lexer.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/ra_syntax/src/parsing/lexer.rs b/crates/ra_syntax/src/parsing/lexer.rs index 926811d96..7a9bc355d 100644 --- a/crates/ra_syntax/src/parsing/lexer.rs +++ b/crates/ra_syntax/src/parsing/lexer.rs @@ -15,16 +15,16 @@ pub struct Token { macro_rules! match_literal_kind { ($kind:expr) => { match $kind { - ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER, - ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER, - ra_rustc_lexer::LiteralKind::Char { .. } => CHAR, - ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE, - ra_rustc_lexer::LiteralKind::Str { .. } => STRING, - ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING, - ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING, - ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING, - } - } + ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER, + ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER, + ra_rustc_lexer::LiteralKind::Char { .. } => CHAR, + ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE, + ra_rustc_lexer::LiteralKind::Str { .. } => STRING, + ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING, + ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING, + ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING, + } + }; } /// Break a string up into its component tokens -- cgit v1.2.3 From db4839033c1903f0926100692b69641b506c05d1 Mon Sep 17 00:00:00 2001 From: Evgenii P Date: Fri, 9 Aug 2019 15:04:13 +0700 Subject: Change macro to function --- crates/ra_syntax/src/parsing/lexer.rs | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/crates/ra_syntax/src/parsing/lexer.rs b/crates/ra_syntax/src/parsing/lexer.rs index 7a9bc355d..06822ea91 100644 --- a/crates/ra_syntax/src/parsing/lexer.rs +++ b/crates/ra_syntax/src/parsing/lexer.rs @@ -12,19 +12,17 @@ pub struct Token { pub len: TextUnit, } -macro_rules! match_literal_kind { - ($kind:expr) => { - match $kind { - ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER, - ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER, - ra_rustc_lexer::LiteralKind::Char { .. } => CHAR, - ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE, - ra_rustc_lexer::LiteralKind::Str { .. } => STRING, - ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING, - ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING, - ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING, - } - }; +fn match_literal_kind(kind: ra_rustc_lexer::LiteralKind) -> SyntaxKind { + match kind { + ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER, + ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER, + ra_rustc_lexer::LiteralKind::Char { .. } => CHAR, + ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE, + ra_rustc_lexer::LiteralKind::Str { .. } => STRING, + ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING, + ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING, + ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING, + } } /// Break a string up into its component tokens @@ -68,7 +66,7 @@ pub fn tokenize(text: &str) -> Vec { } } ra_rustc_lexer::TokenKind::RawIdent => IDENT, - ra_rustc_lexer::TokenKind::Literal { kind, .. } => match_literal_kind!(kind), + ra_rustc_lexer::TokenKind::Literal { kind, .. } => match_literal_kind(kind), ra_rustc_lexer::TokenKind::Lifetime { .. } => LIFETIME, ra_rustc_lexer::TokenKind::Semi => SEMI, ra_rustc_lexer::TokenKind::Comma => COMMA, @@ -137,7 +135,7 @@ pub fn classify_literal(text: &str) -> Option { return None; } let kind = match t.kind { - ra_rustc_lexer::TokenKind::Literal { kind, .. } => match_literal_kind!(kind), + ra_rustc_lexer::TokenKind::Literal { kind, .. } => match_literal_kind(kind), _ => return None, }; Some(Token { kind, len: TextUnit::from_usize(t.len) }) -- cgit v1.2.3