From 89cf7d874393dc22056fe8bdcbff4ec43e5f0bb8 Mon Sep 17 00:00:00 2001 From: DJMcNab <36049421+DJMcNab@users.noreply.github.com> Date: Sat, 8 Dec 2018 14:46:15 +0000 Subject: Clarify and correct comment about multi_byte_tokens --- crates/ra_syntax/src/grammar.ron | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/ra_syntax/src/grammar.ron b/crates/ra_syntax/src/grammar.ron index 53cd2118f..eed67637e 100644 --- a/crates/ra_syntax/src/grammar.ron +++ b/crates/ra_syntax/src/grammar.ron @@ -25,9 +25,9 @@ Grammar( ["^", "CARET"], ["%", "PERCENT"], ], - // TODO: Confirm surmision: the tokens which cannot be recorded in a single UTF-8 byte + // Tokens for which the longest match must be chosen (e.g. `..` is a DOTDOT, but `.` is a DOT) multi_byte_tokens: [ - [".", "DOT"], // Note: DOT is here because + [".", "DOT"], ["..", "DOTDOT"], ["...", "DOTDOTDOT"], ["..=", "DOTDOTEQ"], -- cgit v1.2.3