From a1c187eef3ba08076aedb5154929f7eda8d1b424 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Wed, 12 Aug 2020 18:26:51 +0200 Subject: Rename ra_syntax -> syntax --- crates/syntax/src/parsing/lexer.rs | 244 +++++++++++++++++++++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 crates/syntax/src/parsing/lexer.rs (limited to 'crates/syntax/src/parsing/lexer.rs') diff --git a/crates/syntax/src/parsing/lexer.rs b/crates/syntax/src/parsing/lexer.rs new file mode 100644 index 000000000..fa3be1016 --- /dev/null +++ b/crates/syntax/src/parsing/lexer.rs @@ -0,0 +1,244 @@ +//! Lexer analyzes raw input string and produces lexemes (tokens). +//! It is just a bridge to `rustc_lexer`. + +use rustc_lexer::{LiteralKind as LK, RawStrError}; + +use std::convert::TryInto; + +use crate::{ + SyntaxError, + SyntaxKind::{self, *}, + TextRange, TextSize, T, +}; + +/// A token of Rust source. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Token { + /// The kind of token. + pub kind: SyntaxKind, + /// The length of the token. + pub len: TextSize, +} + +/// Break a string up into its component tokens. +/// Beware that it checks for shebang first and its length contributes to resulting +/// tokens offsets. +pub fn tokenize(text: &str) -> (Vec, Vec) { + // non-empty string is a precondtion of `rustc_lexer::strip_shebang()`. + if text.is_empty() { + return Default::default(); + } + + let mut tokens = Vec::new(); + let mut errors = Vec::new(); + + let mut offset = match rustc_lexer::strip_shebang(text) { + Some(shebang_len) => { + tokens.push(Token { kind: SHEBANG, len: shebang_len.try_into().unwrap() }); + shebang_len + } + None => 0, + }; + + let text_without_shebang = &text[offset..]; + + for rustc_token in rustc_lexer::tokenize(text_without_shebang) { + let token_len: TextSize = rustc_token.len.try_into().unwrap(); + let token_range = TextRange::at(offset.try_into().unwrap(), token_len); + + let (syntax_kind, err_message) = + rustc_token_kind_to_syntax_kind(&rustc_token.kind, &text[token_range]); + + tokens.push(Token { kind: syntax_kind, len: token_len }); + + if let Some(err_message) = err_message { + errors.push(SyntaxError::new(err_message, token_range)); + } + + offset += rustc_token.len; + } + + (tokens, errors) +} + +/// Returns `SyntaxKind` and `Option` of the first token +/// encountered at the beginning of the string. +/// +/// Returns `None` if the string contains zero *or two or more* tokens. +/// The token is malformed if the returned error is not `None`. +/// +/// Beware that unescape errors are not checked at tokenization time. +pub fn lex_single_syntax_kind(text: &str) -> Option<(SyntaxKind, Option)> { + lex_first_token(text) + .filter(|(token, _)| token.len == TextSize::of(text)) + .map(|(token, error)| (token.kind, error)) +} + +/// The same as `lex_single_syntax_kind()` but returns only `SyntaxKind` and +/// returns `None` if any tokenization error occured. +/// +/// Beware that unescape errors are not checked at tokenization time. +pub fn lex_single_valid_syntax_kind(text: &str) -> Option { + lex_first_token(text) + .filter(|(token, error)| !error.is_some() && token.len == TextSize::of(text)) + .map(|(token, _error)| token.kind) +} + +/// Returns `SyntaxKind` and `Option` of the first token +/// encountered at the beginning of the string. +/// +/// Returns `None` if the string contains zero tokens or if the token was parsed +/// with an error. +/// The token is malformed if the returned error is not `None`. +/// +/// Beware that unescape errors are not checked at tokenization time. +fn lex_first_token(text: &str) -> Option<(Token, Option)> { + // non-empty string is a precondtion of `rustc_lexer::first_token()`. + if text.is_empty() { + return None; + } + + let rustc_token = rustc_lexer::first_token(text); + let (syntax_kind, err_message) = rustc_token_kind_to_syntax_kind(&rustc_token.kind, text); + + let token = Token { kind: syntax_kind, len: rustc_token.len.try_into().unwrap() }; + let optional_error = err_message + .map(|err_message| SyntaxError::new(err_message, TextRange::up_to(TextSize::of(text)))); + + Some((token, optional_error)) +} + +/// Returns `SyntaxKind` and an optional tokenize error message. +fn rustc_token_kind_to_syntax_kind( + rustc_token_kind: &rustc_lexer::TokenKind, + token_text: &str, +) -> (SyntaxKind, Option<&'static str>) { + // A note on an intended tradeoff: + // We drop some useful infromation here (see patterns with double dots `..`) + // Storing that info in `SyntaxKind` is not possible due to its layout requirements of + // being `u16` that come from `rowan::SyntaxKind`. + + let syntax_kind = { + match rustc_token_kind { + rustc_lexer::TokenKind::LineComment => COMMENT, + + rustc_lexer::TokenKind::BlockComment { terminated: true } => COMMENT, + rustc_lexer::TokenKind::BlockComment { terminated: false } => { + return ( + COMMENT, + Some("Missing trailing `*/` symbols to terminate the block comment"), + ); + } + + rustc_lexer::TokenKind::Whitespace => WHITESPACE, + + rustc_lexer::TokenKind::Ident => { + if token_text == "_" { + UNDERSCORE + } else { + SyntaxKind::from_keyword(token_text).unwrap_or(IDENT) + } + } + + rustc_lexer::TokenKind::RawIdent => IDENT, + rustc_lexer::TokenKind::Literal { kind, .. } => return match_literal_kind(&kind), + + rustc_lexer::TokenKind::Lifetime { starts_with_number: false } => LIFETIME, + rustc_lexer::TokenKind::Lifetime { starts_with_number: true } => { + return (LIFETIME, Some("Lifetime name cannot start with a number")) + } + + rustc_lexer::TokenKind::Semi => T![;], + rustc_lexer::TokenKind::Comma => T![,], + rustc_lexer::TokenKind::Dot => T![.], + rustc_lexer::TokenKind::OpenParen => T!['('], + rustc_lexer::TokenKind::CloseParen => T![')'], + rustc_lexer::TokenKind::OpenBrace => T!['{'], + rustc_lexer::TokenKind::CloseBrace => T!['}'], + rustc_lexer::TokenKind::OpenBracket => T!['['], + rustc_lexer::TokenKind::CloseBracket => T![']'], + rustc_lexer::TokenKind::At => T![@], + rustc_lexer::TokenKind::Pound => T![#], + rustc_lexer::TokenKind::Tilde => T![~], + rustc_lexer::TokenKind::Question => T![?], + rustc_lexer::TokenKind::Colon => T![:], + rustc_lexer::TokenKind::Dollar => T![$], + rustc_lexer::TokenKind::Eq => T![=], + rustc_lexer::TokenKind::Not => T![!], + rustc_lexer::TokenKind::Lt => T![<], + rustc_lexer::TokenKind::Gt => T![>], + rustc_lexer::TokenKind::Minus => T![-], + rustc_lexer::TokenKind::And => T![&], + rustc_lexer::TokenKind::Or => T![|], + rustc_lexer::TokenKind::Plus => T![+], + rustc_lexer::TokenKind::Star => T![*], + rustc_lexer::TokenKind::Slash => T![/], + rustc_lexer::TokenKind::Caret => T![^], + rustc_lexer::TokenKind::Percent => T![%], + rustc_lexer::TokenKind::Unknown => ERROR, + } + }; + + return (syntax_kind, None); + + fn match_literal_kind(kind: &rustc_lexer::LiteralKind) -> (SyntaxKind, Option<&'static str>) { + #[rustfmt::skip] + let syntax_kind = match *kind { + LK::Int { empty_int: false, .. } => INT_NUMBER, + LK::Int { empty_int: true, .. } => { + return (INT_NUMBER, Some("Missing digits after the integer base prefix")) + } + + LK::Float { empty_exponent: false, .. } => FLOAT_NUMBER, + LK::Float { empty_exponent: true, .. } => { + return (FLOAT_NUMBER, Some("Missing digits after the exponent symbol")) + } + + LK::Char { terminated: true } => CHAR, + LK::Char { terminated: false } => { + return (CHAR, Some("Missing trailing `'` symbol to terminate the character literal")) + } + + LK::Byte { terminated: true } => BYTE, + LK::Byte { terminated: false } => { + return (BYTE, Some("Missing trailing `'` symbol to terminate the byte literal")) + } + + LK::Str { terminated: true } => STRING, + LK::Str { terminated: false } => { + return (STRING, Some("Missing trailing `\"` symbol to terminate the string literal")) + } + + + LK::ByteStr { terminated: true } => BYTE_STRING, + LK::ByteStr { terminated: false } => { + return (BYTE_STRING, Some("Missing trailing `\"` symbol to terminate the byte string literal")) + } + + LK::RawStr { err, .. } => match err { + None => RAW_STRING, + Some(RawStrError::InvalidStarter { .. }) => return (RAW_STRING, Some("Missing `\"` symbol after `#` symbols to begin the raw string literal")), + Some(RawStrError::NoTerminator { expected, found, .. }) => if expected == found { + return (RAW_STRING, Some("Missing trailing `\"` to terminate the raw string literal")) + } else { + return (RAW_STRING, Some("Missing trailing `\"` with `#` symbols to terminate the raw string literal")) + + }, + Some(RawStrError::TooManyDelimiters { .. }) => return (RAW_STRING, Some("Too many `#` symbols: raw strings may be delimited by up to 65535 `#` symbols")), + }, + LK::RawByteStr { err, .. } => match err { + None => RAW_BYTE_STRING, + Some(RawStrError::InvalidStarter { .. }) => return (RAW_BYTE_STRING, Some("Missing `\"` symbol after `#` symbols to begin the raw byte string literal")), + Some(RawStrError::NoTerminator { expected, found, .. }) => if expected == found { + return (RAW_BYTE_STRING, Some("Missing trailing `\"` to terminate the raw byte string literal")) + } else { + return (RAW_BYTE_STRING, Some("Missing trailing `\"` with `#` symbols to terminate the raw byte string literal")) + + }, + Some(RawStrError::TooManyDelimiters { .. }) => return (RAW_BYTE_STRING, Some("Too many `#` symbols: raw byte strings may be delimited by up to 65535 `#` symbols")), + }, + }; + + (syntax_kind, None) + } +} -- cgit v1.2.3