aboutsummaryrefslogtreecommitdiff
path: root/crates/syntax/src/parsing/lexer.rs
diff options
context:
space:
mode:
Diffstat (limited to 'crates/syntax/src/parsing/lexer.rs')
-rw-r--r--crates/syntax/src/parsing/lexer.rs8
1 files changed, 4 insertions, 4 deletions
diff --git a/crates/syntax/src/parsing/lexer.rs b/crates/syntax/src/parsing/lexer.rs
index 0cbba73c5..7c8d0a4c4 100644
--- a/crates/syntax/src/parsing/lexer.rs
+++ b/crates/syntax/src/parsing/lexer.rs
@@ -24,7 +24,7 @@ pub struct Token {
24/// Beware that it checks for shebang first and its length contributes to resulting 24/// Beware that it checks for shebang first and its length contributes to resulting
25/// tokens offsets. 25/// tokens offsets.
26pub fn tokenize(text: &str) -> (Vec<Token>, Vec<SyntaxError>) { 26pub fn tokenize(text: &str) -> (Vec<Token>, Vec<SyntaxError>) {
27 // non-empty string is a precondtion of `rustc_lexer::strip_shebang()`. 27 // non-empty string is a precondition of `rustc_lexer::strip_shebang()`.
28 if text.is_empty() { 28 if text.is_empty() {
29 return Default::default(); 29 return Default::default();
30 } 30 }
@@ -76,7 +76,7 @@ pub fn lex_single_syntax_kind(text: &str) -> Option<(SyntaxKind, Option<SyntaxEr
76} 76}
77 77
78/// The same as `lex_single_syntax_kind()` but returns only `SyntaxKind` and 78/// The same as `lex_single_syntax_kind()` but returns only `SyntaxKind` and
79/// returns `None` if any tokenization error occured. 79/// returns `None` if any tokenization error occurred.
80/// 80///
81/// Beware that unescape errors are not checked at tokenization time. 81/// Beware that unescape errors are not checked at tokenization time.
82pub fn lex_single_valid_syntax_kind(text: &str) -> Option<SyntaxKind> { 82pub fn lex_single_valid_syntax_kind(text: &str) -> Option<SyntaxKind> {
@@ -96,7 +96,7 @@ pub fn lex_single_valid_syntax_kind(text: &str) -> Option<SyntaxKind> {
96/// 96///
97/// Beware that unescape errors are not checked at tokenization time. 97/// Beware that unescape errors are not checked at tokenization time.
98fn lex_first_token(text: &str) -> Option<(Token, Option<SyntaxError>)> { 98fn lex_first_token(text: &str) -> Option<(Token, Option<SyntaxError>)> {
99 // non-empty string is a precondtion of `rustc_lexer::first_token()`. 99 // non-empty string is a precondition of `rustc_lexer::first_token()`.
100 if text.is_empty() { 100 if text.is_empty() {
101 return None; 101 return None;
102 } 102 }
@@ -117,7 +117,7 @@ fn rustc_token_kind_to_syntax_kind(
117 token_text: &str, 117 token_text: &str,
118) -> (SyntaxKind, Option<&'static str>) { 118) -> (SyntaxKind, Option<&'static str>) {
119 // A note on an intended tradeoff: 119 // A note on an intended tradeoff:
120 // We drop some useful infromation here (see patterns with double dots `..`) 120 // We drop some useful information here (see patterns with double dots `..`)
121 // Storing that info in `SyntaxKind` is not possible due to its layout requirements of 121 // Storing that info in `SyntaxKind` is not possible due to its layout requirements of
122 // being `u16` that come from `rowan::SyntaxKind`. 122 // being `u16` that come from `rowan::SyntaxKind`.
123 123