aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--crates/ra_syntax/src/parsing/lexer.rs35
1 files changed, 15 insertions, 20 deletions
diff --git a/crates/ra_syntax/src/parsing/lexer.rs b/crates/ra_syntax/src/parsing/lexer.rs
index 2a4343b0a..06822ea91 100644
--- a/crates/ra_syntax/src/parsing/lexer.rs
+++ b/crates/ra_syntax/src/parsing/lexer.rs
@@ -12,6 +12,19 @@ pub struct Token {
12 pub len: TextUnit, 12 pub len: TextUnit,
13} 13}
14 14
15fn match_literal_kind(kind: ra_rustc_lexer::LiteralKind) -> SyntaxKind {
16 match kind {
17 ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER,
18 ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER,
19 ra_rustc_lexer::LiteralKind::Char { .. } => CHAR,
20 ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE,
21 ra_rustc_lexer::LiteralKind::Str { .. } => STRING,
22 ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING,
23 ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING,
24 ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING,
25 }
26}
27
15/// Break a string up into its component tokens 28/// Break a string up into its component tokens
16pub fn tokenize(text: &str) -> Vec<Token> { 29pub fn tokenize(text: &str) -> Vec<Token> {
17 if text.is_empty() { 30 if text.is_empty() {
@@ -53,16 +66,7 @@ pub fn tokenize(text: &str) -> Vec<Token> {
53 } 66 }
54 } 67 }
55 ra_rustc_lexer::TokenKind::RawIdent => IDENT, 68 ra_rustc_lexer::TokenKind::RawIdent => IDENT,
56 ra_rustc_lexer::TokenKind::Literal { kind, .. } => match kind { 69 ra_rustc_lexer::TokenKind::Literal { kind, .. } => match_literal_kind(kind),
57 ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER,
58 ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER,
59 ra_rustc_lexer::LiteralKind::Char { .. } => CHAR,
60 ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE,
61 ra_rustc_lexer::LiteralKind::Str { .. } => STRING,
62 ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING,
63 ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING,
64 ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING,
65 },
66 ra_rustc_lexer::TokenKind::Lifetime { .. } => LIFETIME, 70 ra_rustc_lexer::TokenKind::Lifetime { .. } => LIFETIME,
67 ra_rustc_lexer::TokenKind::Semi => SEMI, 71 ra_rustc_lexer::TokenKind::Semi => SEMI,
68 ra_rustc_lexer::TokenKind::Comma => COMMA, 72 ra_rustc_lexer::TokenKind::Comma => COMMA,
@@ -131,16 +135,7 @@ pub fn classify_literal(text: &str) -> Option<Token> {
131 return None; 135 return None;
132 } 136 }
133 let kind = match t.kind { 137 let kind = match t.kind {
134 ra_rustc_lexer::TokenKind::Literal { kind, .. } => match kind { 138 ra_rustc_lexer::TokenKind::Literal { kind, .. } => match_literal_kind(kind),
135 ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER,
136 ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER,
137 ra_rustc_lexer::LiteralKind::Char { .. } => CHAR,
138 ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE,
139 ra_rustc_lexer::LiteralKind::Str { .. } => STRING,
140 ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING,
141 ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING,
142 ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING,
143 },
144 _ => return None, 139 _ => return None,
145 }; 140 };
146 Some(Token { kind, len: TextUnit::from_usize(t.len) }) 141 Some(Token { kind, len: TextUnit::from_usize(t.len) })