1 files changed, 244 insertions, 0 deletions

diff --git a/crates/syntax/src/parsing/lexer.rs b/crates/syntax/src/parsing/lexer.rs
new file mode 100644
index 000000000..fa3be1016
--- /dev/null
+++ b/crates/syntax/src/parsing/lexer.rs
@@ -0,0 +1,244 @@
+//! Lexer analyzes raw input string and produces lexemes (tokens).
+//! It is just a bridge to `rustc_lexer`.
+
+use rustc_lexer::{LiteralKind as LK, RawStrError};
+
+use std::convert::TryInto;
+
+use crate::{
+    SyntaxError,
+    SyntaxKind::{self, *},
+    TextRange, TextSize, T,
+};
+
+/// A token of Rust source.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct Token {
+    /// The kind of token.
+    pub kind: SyntaxKind,
+    /// The length of the token.
+    pub len: TextSize,
+}
+
+/// Break a string up into its component tokens.
+/// Beware that it checks for shebang first and its length contributes to resulting
+/// tokens offsets.
+pub fn tokenize(text: &str) -> (Vec<Token>, Vec<SyntaxError>) {
+    // non-empty string is a precondtion of `rustc_lexer::strip_shebang()`.
+    if text.is_empty() {
+        return Default::default();
+    }
+
+    let mut tokens = Vec::new();
+    let mut errors = Vec::new();
+
+    let mut offset = match rustc_lexer::strip_shebang(text) {
+        Some(shebang_len) => {
+            tokens.push(Token { kind: SHEBANG, len: shebang_len.try_into().unwrap() });
+            shebang_len
+        }
+        None => 0,
+    };
+
+    let text_without_shebang = &text[offset..];
+
+    for rustc_token in rustc_lexer::tokenize(text_without_shebang) {
+        let token_len: TextSize = rustc_token.len.try_into().unwrap();
+        let token_range = TextRange::at(offset.try_into().unwrap(), token_len);
+
+        let (syntax_kind, err_message) =
+            rustc_token_kind_to_syntax_kind(&rustc_token.kind, &text[token_range]);
+
+        tokens.push(Token { kind: syntax_kind, len: token_len });
+
+        if let Some(err_message) = err_message {
+            errors.push(SyntaxError::new(err_message, token_range));
+        }
+
+        offset += rustc_token.len;
+    }
+
+    (tokens, errors)
+}
+
+/// Returns `SyntaxKind` and `Option<SyntaxError>` of the first token
+/// encountered at the beginning of the string.
+///
+/// Returns `None` if the string contains zero *or two or more* tokens.
+/// The token is malformed if the returned error is not `None`.
+///
+/// Beware that unescape errors are not checked at tokenization time.
+pub fn lex_single_syntax_kind(text: &str) -> Option<(SyntaxKind, Option<SyntaxError>)> {
+    lex_first_token(text)
+        .filter(|(token, _)| token.len == TextSize::of(text))
+        .map(|(token, error)| (token.kind, error))
+}
+
+/// The same as `lex_single_syntax_kind()` but returns only `SyntaxKind` and
+/// returns `None` if any tokenization error occured.
+///
+/// Beware that unescape errors are not checked at tokenization time.
+pub fn lex_single_valid_syntax_kind(text: &str) -> Option<SyntaxKind> {
+    lex_first_token(text)
+        .filter(|(token, error)| !error.is_some() && token.len == TextSize::of(text))
+        .map(|(token, _error)| token.kind)
+}
+
+/// Returns `SyntaxKind` and `Option<SyntaxError>` of the first token
+/// encountered at the beginning of the string.
+///
+/// Returns `None` if the string contains zero tokens or if the token was parsed
+/// with an error.
+/// The token is malformed if the returned error is not `None`.
+///
+/// Beware that unescape errors are not checked at tokenization time.
+fn lex_first_token(text: &str) -> Option<(Token, Option<SyntaxError>)> {
+    // non-empty string is a precondtion of `rustc_lexer::first_token()`.
+    if text.is_empty() {
+        return None;
+    }
+
+    let rustc_token = rustc_lexer::first_token(text);
+    let (syntax_kind, err_message) = rustc_token_kind_to_syntax_kind(&rustc_token.kind, text);
+
+    let token = Token { kind: syntax_kind, len: rustc_token.len.try_into().unwrap() };
+    let optional_error = err_message
+        .map(|err_message| SyntaxError::new(err_message, TextRange::up_to(TextSize::of(text))));
+
+    Some((token, optional_error))
+}
+
+/// Returns `SyntaxKind` and an optional tokenize error message.
+fn rustc_token_kind_to_syntax_kind(
+    rustc_token_kind: &rustc_lexer::TokenKind,
+    token_text: &str,
+) -> (SyntaxKind, Option<&'static str>) {
+    // A note on an intended tradeoff:
+    // We drop some useful infromation here (see patterns with double dots `..`)
+    // Storing that info in `SyntaxKind` is not possible due to its layout requirements of
+    // being `u16` that come from `rowan::SyntaxKind`.
+
+    let syntax_kind = {
+        match rustc_token_kind {
+            rustc_lexer::TokenKind::LineComment => COMMENT,
+
+            rustc_lexer::TokenKind::BlockComment { terminated: true } => COMMENT,
+            rustc_lexer::TokenKind::BlockComment { terminated: false } => {
+                return (
+                    COMMENT,
+                    Some("Missing trailing `*/` symbols to terminate the block comment"),
+                );
+            }
+
+            rustc_lexer::TokenKind::Whitespace => WHITESPACE,
+
+            rustc_lexer::TokenKind::Ident => {
+                if token_text == "_" {
+                    UNDERSCORE
+                } else {
+                    SyntaxKind::from_keyword(token_text).unwrap_or(IDENT)
+                }
+            }
+
+            rustc_lexer::TokenKind::RawIdent => IDENT,
+            rustc_lexer::TokenKind::Literal { kind, .. } => return match_literal_kind(&kind),
+
+            rustc_lexer::TokenKind::Lifetime { starts_with_number: false } => LIFETIME,
+            rustc_lexer::TokenKind::Lifetime { starts_with_number: true } => {
+                return (LIFETIME, Some("Lifetime name cannot start with a number"))
+            }
+
+            rustc_lexer::TokenKind::Semi => T![;],
+            rustc_lexer::TokenKind::Comma => T![,],
+            rustc_lexer::TokenKind::Dot => T![.],
+            rustc_lexer::TokenKind::OpenParen => T!['('],
+            rustc_lexer::TokenKind::CloseParen => T![')'],
+            rustc_lexer::TokenKind::OpenBrace => T!['{'],
+            rustc_lexer::TokenKind::CloseBrace => T!['}'],
+            rustc_lexer::TokenKind::OpenBracket => T!['['],
+            rustc_lexer::TokenKind::CloseBracket => T![']'],
+            rustc_lexer::TokenKind::At => T![@],
+            rustc_lexer::TokenKind::Pound => T![#],
+            rustc_lexer::TokenKind::Tilde => T![~],
+            rustc_lexer::TokenKind::Question => T![?],
+            rustc_lexer::TokenKind::Colon => T![:],
+            rustc_lexer::TokenKind::Dollar => T![$],
+            rustc_lexer::TokenKind::Eq => T![=],
+            rustc_lexer::TokenKind::Not => T![!],
+            rustc_lexer::TokenKind::Lt => T![<],
+            rustc_lexer::TokenKind::Gt => T![>],
+            rustc_lexer::TokenKind::Minus => T![-],
+            rustc_lexer::TokenKind::And => T![&],
+            rustc_lexer::TokenKind::Or => T![|],
+            rustc_lexer::TokenKind::Plus => T![+],
+            rustc_lexer::TokenKind::Star => T![*],
+            rustc_lexer::TokenKind::Slash => T![/],
+            rustc_lexer::TokenKind::Caret => T![^],
+            rustc_lexer::TokenKind::Percent => T![%],
+            rustc_lexer::TokenKind::Unknown => ERROR,
+        }
+    };
+
+    return (syntax_kind, None);
+
+    fn match_literal_kind(kind: &rustc_lexer::LiteralKind) -> (SyntaxKind, Option<&'static str>) {
+        #[rustfmt::skip]
+        let syntax_kind = match *kind {
+            LK::Int { empty_int: false, .. } => INT_NUMBER,
+            LK::Int { empty_int: true, .. } => {
+                return (INT_NUMBER, Some("Missing digits after the integer base prefix"))
+            }
+
+            LK::Float { empty_exponent: false, .. } => FLOAT_NUMBER,
+            LK::Float { empty_exponent: true, .. } => {
+                return (FLOAT_NUMBER, Some("Missing digits after the exponent symbol"))
+            }
+
+            LK::Char { terminated: true } => CHAR,
+            LK::Char { terminated: false } => {
+                return (CHAR, Some("Missing trailing `'` symbol to terminate the character literal"))
+            }
+
+            LK::Byte { terminated: true } => BYTE,
+            LK::Byte { terminated: false } => {
+                return (BYTE, Some("Missing trailing `'` symbol to terminate the byte literal"))
+            }
+
+            LK::Str { terminated: true } => STRING,
+            LK::Str { terminated: false } => {
+                return (STRING, Some("Missing trailing `\"` symbol to terminate the string literal"))
+            }
+
+
+            LK::ByteStr { terminated: true } => BYTE_STRING,
+            LK::ByteStr { terminated: false } => {
+                return (BYTE_STRING, Some("Missing trailing `\"` symbol to terminate the byte string literal"))
+            }
+
+            LK::RawStr { err, .. } => match err {
+                None => RAW_STRING,
+                Some(RawStrError::InvalidStarter { .. }) => return (RAW_STRING, Some("Missing `\"` symbol after `#` symbols to begin the raw string literal")),
+                Some(RawStrError::NoTerminator { expected, found, .. }) => if expected == found {
+                    return (RAW_STRING, Some("Missing trailing `\"` to terminate the raw string literal"))
+                } else {
+                    return (RAW_STRING, Some("Missing trailing `\"` with `#` symbols to terminate the raw string literal"))
+
+                },
+                Some(RawStrError::TooManyDelimiters { .. }) => return (RAW_STRING, Some("Too many `#` symbols: raw strings may be delimited by up to 65535 `#` symbols")),
+            },
+            LK::RawByteStr { err, .. } => match err {
+                None => RAW_BYTE_STRING,
+                Some(RawStrError::InvalidStarter { .. }) => return (RAW_BYTE_STRING, Some("Missing `\"` symbol after `#` symbols to begin the raw byte string literal")),
+                Some(RawStrError::NoTerminator { expected, found, .. }) => if expected == found {
+                    return (RAW_BYTE_STRING, Some("Missing trailing `\"` to terminate the raw byte string literal"))
+                } else {
+                    return (RAW_BYTE_STRING, Some("Missing trailing `\"` with `#` symbols to terminate the raw byte string literal"))
+
+                },
+                Some(RawStrError::TooManyDelimiters { .. }) => return (RAW_BYTE_STRING, Some("Too many `#` symbols: raw byte strings may be delimited by up to 65535 `#` symbols")),
+            },
+        };
+
+        (syntax_kind, None)
+    }
+}