diff options
-rw-r--r-- | crates/ra_syntax/src/parsing/lexer.rs | 37 |
1 files changed, 17 insertions, 20 deletions
diff --git a/crates/ra_syntax/src/parsing/lexer.rs b/crates/ra_syntax/src/parsing/lexer.rs index 2a4343b0a..926811d96 100644 --- a/crates/ra_syntax/src/parsing/lexer.rs +++ b/crates/ra_syntax/src/parsing/lexer.rs | |||
@@ -12,6 +12,21 @@ pub struct Token { | |||
12 | pub len: TextUnit, | 12 | pub len: TextUnit, |
13 | } | 13 | } |
14 | 14 | ||
15 | macro_rules! match_literal_kind { | ||
16 | ($kind:expr) => { | ||
17 | match $kind { | ||
18 | ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER, | ||
19 | ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER, | ||
20 | ra_rustc_lexer::LiteralKind::Char { .. } => CHAR, | ||
21 | ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE, | ||
22 | ra_rustc_lexer::LiteralKind::Str { .. } => STRING, | ||
23 | ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING, | ||
24 | ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING, | ||
25 | ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING, | ||
26 | } | ||
27 | } | ||
28 | } | ||
29 | |||
15 | /// Break a string up into its component tokens | 30 | /// Break a string up into its component tokens |
16 | pub fn tokenize(text: &str) -> Vec<Token> { | 31 | pub fn tokenize(text: &str) -> Vec<Token> { |
17 | if text.is_empty() { | 32 | if text.is_empty() { |
@@ -53,16 +68,7 @@ pub fn tokenize(text: &str) -> Vec<Token> { | |||
53 | } | 68 | } |
54 | } | 69 | } |
55 | ra_rustc_lexer::TokenKind::RawIdent => IDENT, | 70 | ra_rustc_lexer::TokenKind::RawIdent => IDENT, |
56 | ra_rustc_lexer::TokenKind::Literal { kind, .. } => match kind { | 71 | ra_rustc_lexer::TokenKind::Literal { kind, .. } => match_literal_kind!(kind), |
57 | ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER, | ||
58 | ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER, | ||
59 | ra_rustc_lexer::LiteralKind::Char { .. } => CHAR, | ||
60 | ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE, | ||
61 | ra_rustc_lexer::LiteralKind::Str { .. } => STRING, | ||
62 | ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING, | ||
63 | ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING, | ||
64 | ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING, | ||
65 | }, | ||
66 | ra_rustc_lexer::TokenKind::Lifetime { .. } => LIFETIME, | 72 | ra_rustc_lexer::TokenKind::Lifetime { .. } => LIFETIME, |
67 | ra_rustc_lexer::TokenKind::Semi => SEMI, | 73 | ra_rustc_lexer::TokenKind::Semi => SEMI, |
68 | ra_rustc_lexer::TokenKind::Comma => COMMA, | 74 | ra_rustc_lexer::TokenKind::Comma => COMMA, |
@@ -131,16 +137,7 @@ pub fn classify_literal(text: &str) -> Option<Token> { | |||
131 | return None; | 137 | return None; |
132 | } | 138 | } |
133 | let kind = match t.kind { | 139 | let kind = match t.kind { |
134 | ra_rustc_lexer::TokenKind::Literal { kind, .. } => match kind { | 140 | ra_rustc_lexer::TokenKind::Literal { kind, .. } => match_literal_kind!(kind), |
135 | ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER, | ||
136 | ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER, | ||
137 | ra_rustc_lexer::LiteralKind::Char { .. } => CHAR, | ||
138 | ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE, | ||
139 | ra_rustc_lexer::LiteralKind::Str { .. } => STRING, | ||
140 | ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING, | ||
141 | ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING, | ||
142 | ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING, | ||
143 | }, | ||
144 | _ => return None, | 141 | _ => return None, |
145 | }; | 142 | }; |
146 | Some(Token { kind, len: TextUnit::from_usize(t.len) }) | 143 | Some(Token { kind, len: TextUnit::from_usize(t.len) }) |