Add validator for byte

8 files changed, 416 insertions, 90 deletions

diff --git a/crates/ra_syntax/src/ast/generated.rs b/crates/ra_syntax/src/ast/generated.rs
index 2e9ae263a..75236153d 100644
--- a/crates/ra_syntax/src/ast/generated.rs
+++ b/crates/ra_syntax/src/ast/generated.rs
@@ -372,6 +372,43 @@ impl<R: TreeRoot<RaTypes>> BreakExprNode<R> {
 
 impl<'a> BreakExpr<'a> {}
 
+// Byte
+#[derive(Debug, Clone, Copy,)]
+pub struct ByteNode<R: TreeRoot<RaTypes> = OwnedRoot> {
+    pub(crate) syntax: SyntaxNode<R>,
+}
+pub type Byte<'a> = ByteNode<RefRoot<'a>>;
+
+impl<R1: TreeRoot<RaTypes>, R2: TreeRoot<RaTypes>> PartialEq<ByteNode<R1>> for ByteNode<R2> {
+    fn eq(&self, other: &ByteNode<R1>) -> bool { self.syntax == other.syntax }
+}
+impl<R: TreeRoot<RaTypes>> Eq for ByteNode<R> {}
+impl<R: TreeRoot<RaTypes>> Hash for ByteNode<R> {
+    fn hash<H: Hasher>(&self, state: &mut H) { self.syntax.hash(state) }
+}
+
+impl<'a> AstNode<'a> for Byte<'a> {
+    fn cast(syntax: SyntaxNodeRef<'a>) -> Option<Self> {
+        match syntax.kind() {
+            BYTE => Some(Byte { syntax }),
+            _ => None,
+        }
+    }
+    fn syntax(self) -> SyntaxNodeRef<'a> { self.syntax }
+}
+
+impl<R: TreeRoot<RaTypes>> ByteNode<R> {
+    pub fn borrowed(&self) -> Byte {
+        ByteNode { syntax: self.syntax.borrowed() }
+    }
+    pub fn owned(&self) -> ByteNode {
+        ByteNode { syntax: self.syntax.owned() }
+    }
+}
+
+
+impl<'a> Byte<'a> {}
+
 // CallExpr
 #[derive(Debug, Clone, Copy,)]
 pub struct CallExprNode<R: TreeRoot<RaTypes> = OwnedRoot> {
diff --git a/crates/ra_syntax/src/ast/mod.rs b/crates/ra_syntax/src/ast/mod.rs
index f20714ede..686b5cf04 100644
--- a/crates/ra_syntax/src/ast/mod.rs
+++ b/crates/ra_syntax/src/ast/mod.rs
@@ -134,6 +134,12 @@ impl<'a> Char<'a> {
     }
 }
 
+impl<'a> Byte<'a> {
+    pub fn text(&self) -> &SmolStr {
+        &self.syntax().leaf_text().unwrap()
+    }
+}
+
 impl<'a> String<'a> {
     pub fn text(&self) -> &SmolStr {
         &self.syntax().leaf_text().unwrap()
diff --git a/crates/ra_syntax/src/grammar.ron b/crates/ra_syntax/src/grammar.ron
index c3184667e..2c2ed1aeb 100644
--- a/crates/ra_syntax/src/grammar.ron
+++ b/crates/ra_syntax/src/grammar.ron
@@ -412,6 +412,7 @@ Grammar(
         "RangeExpr": (),
         "BinExpr": (),
         "String": (),
+        "Byte": (),
         "Char": (),
         "Literal": (),
 
diff --git a/crates/ra_syntax/src/string_lexing.rs b/crates/ra_syntax/src/string_lexing.rs
index d613bb042..4e8c3a91c 100644
--- a/crates/ra_syntax/src/string_lexing.rs
+++ b/crates/ra_syntax/src/string_lexing.rs
@@ -63,6 +63,56 @@ impl<'a> Iterator for StringComponentIterator<'a> {
     }
 }
 
+pub fn parse_byte_literal(src: &str) -> ByteComponentIterator {
+    ByteComponentIterator {
+        parser: Parser::new(src),
+        has_closing_quote: false,
+    }
+}
+
+pub struct ByteComponentIterator<'a> {
+    parser: Parser<'a>,
+    pub has_closing_quote: bool,
+}
+
+impl<'a> Iterator for ByteComponentIterator<'a> {
+    type Item = CharComponent;
+    fn next(&mut self) -> Option<CharComponent> {
+        if self.parser.pos == 0 {
+            assert!(
+                self.parser.advance() == 'b',
+                "Byte literal should start with a b"
+            );
+
+            assert!(
+                self.parser.advance() == '\'',
+                "Byte literal should start with a b, followed by a quote"
+            );
+        }
+
+
+        if let Some(component) = self.parser.parse_char_component() {
+            return Some(component);
+        }
+
+        // We get here when there are no char components left to parse
+        if self.parser.peek() == Some('\'') {
+            self.parser.advance();
+            self.has_closing_quote = true;
+        }
+
+        assert!(
+            self.parser.peek() == None,
+            "byte literal should leave no unparsed input: src = {}, pos = {}, length = {}",
+            self.parser.src,
+            self.parser.pos,
+            self.parser.src.len()
+        );
+
+        None
+    }
+}
+
 pub fn parse_char_literal(src: &str) -> CharComponentIterator {
     CharComponentIterator {
         parser: Parser::new(src),
diff --git a/crates/ra_syntax/src/validation/byte.rs b/crates/ra_syntax/src/validation/byte.rs
new file mode 100644
index 000000000..3d2806c4e
--- /dev/null
+++ b/crates/ra_syntax/src/validation/byte.rs
@@ -0,0 +1,202 @@
+//! Validation of byte literals
+
+use crate::{
+    ast::{self, AstNode},
+    string_lexing::{self, CharComponentKind},
+    TextRange,
+    validation::char,
+    yellow::{
+        SyntaxError,
+        SyntaxErrorKind::*,
+    },
+};
+
+pub(super) fn validate_byte_node(node: ast::Byte, errors: &mut Vec<SyntaxError>) {
+    let literal_text = node.text();
+    let literal_range = node.syntax().range();
+    let mut components = string_lexing::parse_byte_literal(literal_text);
+    let mut len = 0;
+    for component in &mut components {
+        len += 1;
+        let text = &literal_text[component.range];
+        let range = component.range + literal_range.start();
+
+        use self::CharComponentKind::*;
+        match component.kind {
+            AsciiEscape => validate_byte_escape(text, range, errors),
+            AsciiCodeEscape => validate_byte_code_escape(text, range, errors),
+            UnicodeEscape => errors.push(SyntaxError::new(UnicodeEscapeForbidden, range)),
+            CodePoint => {
+                let c = text.chars().next().expect("Code points should be one character long");
+
+                // These bytes must always be escaped
+                if c == '\t' || c == '\r' || c == '\n' {
+                    errors.push(SyntaxError::new(UnescapedByte, range));
+                }
+
+                // Only ASCII bytes are allowed
+                if c > 0x7F as char {
+                    errors.push(SyntaxError::new(ByteOutOfRange, range));
+                }
+            }
+        }
+    }
+
+    if !components.has_closing_quote {
+        errors.push(SyntaxError::new(UnclosedByte, literal_range));
+    }
+
+    if len == 0 {
+        errors.push(SyntaxError::new(EmptyByte, literal_range));
+    }
+
+    if len > 1 {
+        errors.push(SyntaxError::new(OverlongByte, literal_range));
+    }
+}
+
+fn validate_byte_escape(text: &str, range: TextRange, errors: &mut Vec<SyntaxError>) {
+    if text.len() == 1 {
+        // Escape sequence consists only of leading `\`
+        errors.push(SyntaxError::new(EmptyByteEscape, range));
+    } else {
+        let escape_code = text.chars().skip(1).next().unwrap();
+        if !char::is_ascii_escape(escape_code) {
+            errors.push(SyntaxError::new(InvalidByteEscape, range));
+        }
+    }
+}
+
+fn validate_byte_code_escape(text: &str, range: TextRange, errors: &mut Vec<SyntaxError>) {
+    // A ByteCodeEscape has 4 chars, example: `\xDD`
+    if text.len() < 4 {
+        errors.push(SyntaxError::new(TooShortByteCodeEscape, range));
+    } else {
+        assert!(
+            text.chars().count() == 4,
+            "ByteCodeEscape cannot be longer than 4 chars"
+        );
+
+        if u8::from_str_radix(&text[2..], 16).is_err() {
+            errors.push(SyntaxError::new(MalformedByteCodeEscape, range));
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use crate::SourceFileNode;
+
+    fn build_file(literal: &str) -> SourceFileNode {
+        let src = format!("const C: u8 = b'{}';", literal);
+        SourceFileNode::parse(&src)
+    }
+
+    fn assert_valid_byte(literal: &str) {
+        let file = build_file(literal);
+        assert!(
+            file.errors().len() == 0,
+            "Errors for literal '{}': {:?}",
+            literal,
+            file.errors()
+        );
+    }
+
+    fn assert_invalid_byte(literal: &str) {
+        let file = build_file(literal);
+        assert!(file.errors().len() > 0);
+    }
+
+    #[test]
+    fn test_ansi_codepoints() {
+        for byte in 0..128 {
+            match byte {
+                b'\n' | b'\r' | b'\t' => assert_invalid_byte(&(byte as char).to_string()),
+                b'\'' | b'\\' => { /* Ignore character close and backslash */ }
+                _ => assert_valid_byte(&(byte as char).to_string()),
+            }
+        }
+
+        for byte in 128..=255u8 {
+            assert_invalid_byte(&(byte as char).to_string());
+        }
+    }
+
+    #[test]
+    fn test_unicode_codepoints() {
+        let invalid = ["Ƒ", "バ", "メ", "﷽"];
+        for c in &invalid {
+            assert_invalid_byte(c);
+        }
+    }
+
+    #[test]
+    fn test_unicode_multiple_codepoints() {
+        let invalid = ["नी", "👨‍👨‍"];
+        for c in &invalid {
+            assert_invalid_byte(c);
+        }
+    }
+
+    #[test]
+    fn test_valid_byte_escape() {
+        let valid = [
+            r"\'", "\"", "\\\\", "\\\"", r"\n", r"\r", r"\t", r"\0", "a", "b",
+        ];
+        for c in &valid {
+            assert_valid_byte(c);
+        }
+    }
+
+    #[test]
+    fn test_invalid_byte_escape() {
+        let invalid = [r"\a", r"\?", r"\"];
+        for c in &invalid {
+            assert_invalid_byte(c);
+        }
+    }
+
+    #[test]
+    fn test_valid_byte_code_escape() {
+        let valid = [r"\x00", r"\x7F", r"\x55", r"\xF0"];
+        for c in &valid {
+            assert_valid_byte(c);
+        }
+    }
+
+    #[test]
+    fn test_invalid_byte_code_escape() {
+        let invalid = [r"\x", r"\x7"];
+        for c in &invalid {
+            assert_invalid_byte(c);
+        }
+    }
+
+    #[test]
+    fn test_invalid_unicode_escape() {
+        let well_formed = [
+            r"\u{FF}",
+            r"\u{0}",
+            r"\u{F}",
+            r"\u{10FFFF}",
+            r"\u{1_0__FF___FF_____}",
+        ];
+        for c in &well_formed {
+            assert_invalid_byte(c);
+        }
+
+        let invalid = [
+            r"\u",
+            r"\u{}",
+            r"\u{",
+            r"\u{FF",
+            r"\u{FFFFFF}",
+            r"\u{_F}",
+            r"\u{00FFFFF}",
+            r"\u{110000}",
+        ];
+        for c in &invalid {
+            assert_invalid_byte(c);
+        }
+    }
+}
diff --git a/crates/ra_syntax/src/validation/char.rs b/crates/ra_syntax/src/validation/char.rs
index 63f9bad24..793539b3a 100644
--- a/crates/ra_syntax/src/validation/char.rs
+++ b/crates/ra_syntax/src/validation/char.rs
@@ -1,3 +1,5 @@
+//! Validation of char literals
+
 use std::u32;
 
 use arrayvec::ArrayString;
@@ -12,7 +14,7 @@ use crate::{
     },
 };
 
-pub(crate) fn validate_char_node(node: ast::Char, errors: &mut Vec<SyntaxError>) {
+pub(super) fn validate_char_node(node: ast::Char, errors: &mut Vec<SyntaxError>) {
     let literal_text = node.text();
     let literal_range = node.syntax().range();
     let mut components = string_lexing::parse_char_literal(literal_text);
@@ -37,7 +39,7 @@ pub(crate) fn validate_char_node(node: ast::Char, errors: &mut Vec<SyntaxError>)
     }
 }
 
-pub(crate) fn validate_char_component(
+pub(super) fn validate_char_component(
     text: &str,
     kind: CharComponentKind,
     range: TextRange,
@@ -46,109 +48,115 @@ pub(crate) fn validate_char_component(
     // Validate escapes
     use self::CharComponentKind::*;
     match kind {
-        AsciiEscape => {
-            if text.len() == 1 {
-                // Escape sequence consists only of leading `\`
-                errors.push(SyntaxError::new(EmptyAsciiEscape, range));
-            } else {
-                let escape_code = text.chars().skip(1).next().unwrap();
-                if !is_ascii_escape(escape_code) {
-                    errors.push(SyntaxError::new(InvalidAsciiEscape, range));
-                }
+        AsciiEscape => validate_ascii_escape(text, range, errors),
+        AsciiCodeEscape => validate_ascii_code_escape(text, range, errors),
+        UnicodeEscape => validate_unicode_escape(text, range, errors),
+        CodePoint => {
+            // These code points must always be escaped
+            if text == "\t" || text == "\r" || text == "\n" {
+                errors.push(SyntaxError::new(UnescapedCodepoint, range));
             }
         }
-        AsciiCodeEscape => {
-            // An AsciiCodeEscape has 4 chars, example: `\xDD`
-            if text.len() < 4 {
-                errors.push(SyntaxError::new(TooShortAsciiCodeEscape, range));
-            } else {
-                assert!(
-                    text.chars().count() == 4,
-                    "AsciiCodeEscape cannot be longer than 4 chars"
-                );
-
-                match u8::from_str_radix(&text[2..], 16) {
-                    Ok(code) if code < 128 => { /* Escape code is valid */ }
-                    Ok(_) => errors.push(SyntaxError::new(AsciiCodeEscapeOutOfRange, range)),
-                    Err(_) => errors.push(SyntaxError::new(MalformedAsciiCodeEscape, range)),
-                }
-            }
+    }
+}
+
+fn validate_ascii_escape(text: &str, range: TextRange, errors: &mut Vec<SyntaxError>) {
+    if text.len() == 1 {
+        // Escape sequence consists only of leading `\`
+        errors.push(SyntaxError::new(EmptyAsciiEscape, range));
+    } else {
+        let escape_code = text.chars().skip(1).next().unwrap();
+        if !is_ascii_escape(escape_code) {
+            errors.push(SyntaxError::new(InvalidAsciiEscape, range));
         }
-        UnicodeEscape => {
-            assert!(&text[..2] == "\\u", "UnicodeEscape always starts with \\u");
+    }
+}
 
-            if text.len() == 2 {
-                // No starting `{`
-                errors.push(SyntaxError::new(MalformedUnicodeEscape, range));
-                return;
-            }
+pub(super) fn is_ascii_escape(code: char) -> bool {
+    match code {
+        '\\' | '\'' | '"' | 'n' | 'r' | 't' | '0' => true,
+        _ => false,
+    }
+}
 
-            if text.len() == 3 {
-                // Only starting `{`
-                errors.push(SyntaxError::new(UnclosedUnicodeEscape, range));
-                return;
-            }
+fn validate_ascii_code_escape(text: &str, range: TextRange, errors: &mut Vec<SyntaxError>) {
+    // An AsciiCodeEscape has 4 chars, example: `\xDD`
+    if text.len() < 4 {
+        errors.push(SyntaxError::new(TooShortAsciiCodeEscape, range));
+    } else {
+        assert!(
+            text.chars().count() == 4,
+            "AsciiCodeEscape cannot be longer than 4 chars"
+        );
 
-            let mut code = ArrayString::<[_; 6]>::new();
-            let mut closed = false;
-            for c in text[3..].chars() {
-                assert!(!closed, "no characters after escape is closed");
-
-                if c.is_digit(16) {
-                    if code.len() == 6 {
-                        errors.push(SyntaxError::new(OverlongUnicodeEscape, range));
-                        return;
-                    }
-
-                    code.push(c);
-                } else if c == '_' {
-                    // Reject leading _
-                    if code.len() == 0 {
-                        errors.push(SyntaxError::new(MalformedUnicodeEscape, range));
-                        return;
-                    }
-                } else if c == '}' {
-                    closed = true;
-                } else {
-                    errors.push(SyntaxError::new(MalformedUnicodeEscape, range));
-                    return;
-                }
-            }
+        match u8::from_str_radix(&text[2..], 16) {
+            Ok(code) if code < 128 => { /* Escape code is valid */ }
+            Ok(_) => errors.push(SyntaxError::new(AsciiCodeEscapeOutOfRange, range)),
+            Err(_) => errors.push(SyntaxError::new(MalformedAsciiCodeEscape, range)),
+        }
+    }
+}
 
-            if !closed {
-                errors.push(SyntaxError::new(UnclosedUnicodeEscape, range))
-            }
+fn validate_unicode_escape(text: &str, range: TextRange, errors: &mut Vec<SyntaxError>) {
+    assert!(&text[..2] == "\\u", "UnicodeEscape always starts with \\u");
 
-            if code.len() == 0 {
-                errors.push(SyntaxError::new(EmptyUnicodeEcape, range));
+    if text.len() == 2 {
+        // No starting `{`
+        errors.push(SyntaxError::new(MalformedUnicodeEscape, range));
+        return;
+    }
+
+    if text.len() == 3 {
+        // Only starting `{`
+        errors.push(SyntaxError::new(UnclosedUnicodeEscape, range));
+        return;
+    }
+
+    let mut code = ArrayString::<[_; 6]>::new();
+    let mut closed = false;
+    for c in text[3..].chars() {
+        assert!(!closed, "no characters after escape is closed");
+
+        if c.is_digit(16) {
+            if code.len() == 6 {
+                errors.push(SyntaxError::new(OverlongUnicodeEscape, range));
                 return;
             }
 
-            match u32::from_str_radix(&code, 16) {
-                Ok(code_u32) if code_u32 > 0x10FFFF => {
-                    errors.push(SyntaxError::new(UnicodeEscapeOutOfRange, range));
-                }
-                Ok(_) => {
-                    // Valid escape code
-                }
-                Err(_) => {
-                    errors.push(SyntaxError::new(MalformedUnicodeEscape, range));
-                }
-            }
-        }
-        CodePoint => {
-            // These code points must always be escaped
-            if text == "\t" || text == "\r" {
-                errors.push(SyntaxError::new(UnescapedCodepoint, range));
+            code.push(c);
+        } else if c == '_' {
+            // Reject leading _
+            if code.len() == 0 {
+                errors.push(SyntaxError::new(MalformedUnicodeEscape, range));
+                return;
             }
+        } else if c == '}' {
+            closed = true;
+        } else {
+            errors.push(SyntaxError::new(MalformedUnicodeEscape, range));
+            return;
         }
     }
-}
 
-fn is_ascii_escape(code: char) -> bool {
-    match code {
-        '\\' | '\'' | '"' | 'n' | 'r' | 't' | '0' => true,
-        _ => false,
+    if !closed {
+        errors.push(SyntaxError::new(UnclosedUnicodeEscape, range))
+    }
+
+    if code.len() == 0 {
+        errors.push(SyntaxError::new(EmptyUnicodeEcape, range));
+        return;
+    }
+
+    match u32::from_str_radix(&code, 16) {
+        Ok(code_u32) if code_u32 > 0x10FFFF => {
+            errors.push(SyntaxError::new(UnicodeEscapeOutOfRange, range));
+        }
+        Ok(_) => {
+            // Valid escape code
+        }
+        Err(_) => {
+            errors.push(SyntaxError::new(MalformedUnicodeEscape, range));
+        }
     }
 }
 
diff --git a/crates/ra_syntax/src/validation/mod.rs b/crates/ra_syntax/src/validation/mod.rs
index 2ff0bc26d..acad7cb7f 100644
--- a/crates/ra_syntax/src/validation/mod.rs
+++ b/crates/ra_syntax/src/validation/mod.rs
@@ -5,6 +5,7 @@ use crate::{
     yellow::SyntaxError,
 };
 
+mod byte;
 mod char;
 mod string;
 
@@ -12,6 +13,7 @@ pub(crate) fn validate(file: &SourceFileNode) -> Vec<SyntaxError> {
     let mut errors = Vec::new();
     for node in file.syntax().descendants() {
         let _ = visitor_ctx(&mut errors)
+            .visit::<ast::Byte, _>(self::byte::validate_byte_node)
             .visit::<ast::Char, _>(self::char::validate_char_node)
             .visit::<ast::String, _>(self::string::validate_string_node)
             .accept(node);
diff --git a/crates/ra_syntax/src/yellow/syntax_error.rs b/crates/ra_syntax/src/yellow/syntax_error.rs
index cf7b1d495..df230293b 100644
--- a/crates/ra_syntax/src/yellow/syntax_error.rs
+++ b/crates/ra_syntax/src/yellow/syntax_error.rs
@@ -72,6 +72,16 @@ pub enum SyntaxErrorKind {
     EmptyChar,
     UnclosedChar,
     OverlongChar,
+    EmptyByte,
+    UnclosedByte,
+    OverlongByte,
+    ByteOutOfRange,
+    UnescapedByte,
+    EmptyByteEscape,
+    InvalidByteEscape,
+    TooShortByteCodeEscape,
+    MalformedByteCodeEscape,
+    UnicodeEscapeForbidden,
     EmptyAsciiEscape,
     InvalidAsciiEscape,
     TooShortAsciiCodeEscape,
@@ -98,6 +108,16 @@ impl fmt::Display for SyntaxErrorKind {
             EmptyChar => write!(f, "Empty char literal"),
             UnclosedChar => write!(f, "Unclosed char literal"),
             OverlongChar => write!(f, "Char literal should be one character long"),
+            EmptyByte => write!(f, "Empty byte literal"),
+            UnclosedByte => write!(f, "Unclosed byte literal"),
+            OverlongByte => write!(f, "Byte literal should be one character long"),
+            ByteOutOfRange => write!(f, "Byte should be a valid ASCII character"),
+            UnescapedByte => write!(f, "This byte should always be escaped"),
+            EmptyByteEscape => write!(f, "Empty escape sequence"),
+            InvalidByteEscape => write!(f, "Invalid escape sequence"),
+            TooShortByteCodeEscape => write!(f, "Escape sequence should have two digits"),
+            MalformedByteCodeEscape => write!(f, "Escape sequence should be a hexadecimal number"),
+            UnicodeEscapeForbidden => write!(f, "Unicode escapes are not allowed in byte literals or byte strings"),
             TooShortAsciiCodeEscape => write!(f, "Escape sequence should have two digits"),
             AsciiCodeEscapeOutOfRange => {
                 write!(f, "Escape sequence should be between \\x00 and \\x7F")