aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--crates/ra_syntax/src/parsing/lexer.rs37
1 files changed, 17 insertions, 20 deletions
diff --git a/crates/ra_syntax/src/parsing/lexer.rs b/crates/ra_syntax/src/parsing/lexer.rs
index 2a4343b0a..926811d96 100644
--- a/crates/ra_syntax/src/parsing/lexer.rs
+++ b/crates/ra_syntax/src/parsing/lexer.rs
@@ -12,6 +12,21 @@ pub struct Token {
12 pub len: TextUnit, 12 pub len: TextUnit,
13} 13}
14 14
15macro_rules! match_literal_kind {
16 ($kind:expr) => {
17 match $kind {
18 ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER,
19 ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER,
20 ra_rustc_lexer::LiteralKind::Char { .. } => CHAR,
21 ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE,
22 ra_rustc_lexer::LiteralKind::Str { .. } => STRING,
23 ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING,
24 ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING,
25 ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING,
26 }
27 }
28}
29
15/// Break a string up into its component tokens 30/// Break a string up into its component tokens
16pub fn tokenize(text: &str) -> Vec<Token> { 31pub fn tokenize(text: &str) -> Vec<Token> {
17 if text.is_empty() { 32 if text.is_empty() {
@@ -53,16 +68,7 @@ pub fn tokenize(text: &str) -> Vec<Token> {
53 } 68 }
54 } 69 }
55 ra_rustc_lexer::TokenKind::RawIdent => IDENT, 70 ra_rustc_lexer::TokenKind::RawIdent => IDENT,
56 ra_rustc_lexer::TokenKind::Literal { kind, .. } => match kind { 71 ra_rustc_lexer::TokenKind::Literal { kind, .. } => match_literal_kind!(kind),
57 ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER,
58 ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER,
59 ra_rustc_lexer::LiteralKind::Char { .. } => CHAR,
60 ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE,
61 ra_rustc_lexer::LiteralKind::Str { .. } => STRING,
62 ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING,
63 ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING,
64 ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING,
65 },
66 ra_rustc_lexer::TokenKind::Lifetime { .. } => LIFETIME, 72 ra_rustc_lexer::TokenKind::Lifetime { .. } => LIFETIME,
67 ra_rustc_lexer::TokenKind::Semi => SEMI, 73 ra_rustc_lexer::TokenKind::Semi => SEMI,
68 ra_rustc_lexer::TokenKind::Comma => COMMA, 74 ra_rustc_lexer::TokenKind::Comma => COMMA,
@@ -131,16 +137,7 @@ pub fn classify_literal(text: &str) -> Option<Token> {
131 return None; 137 return None;
132 } 138 }
133 let kind = match t.kind { 139 let kind = match t.kind {
134 ra_rustc_lexer::TokenKind::Literal { kind, .. } => match kind { 140 ra_rustc_lexer::TokenKind::Literal { kind, .. } => match_literal_kind!(kind),
135 ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER,
136 ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER,
137 ra_rustc_lexer::LiteralKind::Char { .. } => CHAR,
138 ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE,
139 ra_rustc_lexer::LiteralKind::Str { .. } => STRING,
140 ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING,
141 ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING,
142 ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING,
143 },
144 _ => return None, 141 _ => return None,
145 }; 142 };
146 Some(Token { kind, len: TextUnit::from_usize(t.len) }) 143 Some(Token { kind, len: TextUnit::from_usize(t.len) })