From a1c187eef3ba08076aedb5154929f7eda8d1b424 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Wed, 12 Aug 2020 18:26:51 +0200 Subject: Rename ra_syntax -> syntax --- crates/syntax/src/tests.rs | 280 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 280 insertions(+) create mode 100644 crates/syntax/src/tests.rs (limited to 'crates/syntax/src/tests.rs') diff --git a/crates/syntax/src/tests.rs b/crates/syntax/src/tests.rs new file mode 100644 index 000000000..ddc718369 --- /dev/null +++ b/crates/syntax/src/tests.rs @@ -0,0 +1,280 @@ +use std::{ + fmt::Write, + fs, + path::{Path, PathBuf}, +}; + +use expect::expect_file; +use rayon::prelude::*; +use test_utils::project_dir; + +use crate::{fuzz, tokenize, SourceFile, SyntaxError, TextRange, TextSize, Token}; + +#[test] +fn lexer_tests() { + // FIXME: + // * Add tests for unicode escapes in byte-character and [raw]-byte-string literals + // * Add tests for unescape errors + + dir_tests(&test_data_dir(), &["lexer/ok"], "txt", |text, path| { + let (tokens, errors) = tokenize(text); + assert_errors_are_absent(&errors, path); + dump_tokens_and_errors(&tokens, &errors, text) + }); + dir_tests(&test_data_dir(), &["lexer/err"], "txt", |text, path| { + let (tokens, errors) = tokenize(text); + assert_errors_are_present(&errors, path); + dump_tokens_and_errors(&tokens, &errors, text) + }); +} + +#[test] +fn parse_smoke_test() { + let code = r##" +fn main() { + println!("Hello, world!") +} + "##; + + let parse = SourceFile::parse(code); + // eprintln!("{:#?}", parse.syntax_node()); + assert!(parse.ok().is_ok()); +} + +#[test] +fn parser_tests() { + dir_tests(&test_data_dir(), &["parser/inline/ok", "parser/ok"], "rast", |text, path| { + let parse = SourceFile::parse(text); + let errors = parse.errors(); + assert_errors_are_absent(&errors, path); + parse.debug_dump() + }); + dir_tests(&test_data_dir(), &["parser/err", "parser/inline/err"], "rast", |text, path| { + let parse = SourceFile::parse(text); + let errors = parse.errors(); + assert_errors_are_present(&errors, path); + parse.debug_dump() + }); +} + +#[test] +fn expr_parser_tests() { + fragment_parser_dir_test( + &["parser/fragments/expr/ok"], + &["parser/fragments/expr/err"], + crate::ast::Expr::parse, + ); +} + +#[test] +fn path_parser_tests() { + fragment_parser_dir_test( + &["parser/fragments/path/ok"], + &["parser/fragments/path/err"], + crate::ast::Path::parse, + ); +} + +#[test] +fn pattern_parser_tests() { + fragment_parser_dir_test( + &["parser/fragments/pattern/ok"], + &["parser/fragments/pattern/err"], + crate::ast::Pat::parse, + ); +} + +#[test] +fn item_parser_tests() { + fragment_parser_dir_test( + &["parser/fragments/item/ok"], + &["parser/fragments/item/err"], + crate::ast::Item::parse, + ); +} + +#[test] +fn type_parser_tests() { + fragment_parser_dir_test( + &["parser/fragments/type/ok"], + &["parser/fragments/type/err"], + crate::ast::Type::parse, + ); +} + +#[test] +fn parser_fuzz_tests() { + for (_, text) in collect_rust_files(&test_data_dir(), &["parser/fuzz-failures"]) { + fuzz::check_parser(&text) + } +} + +#[test] +fn reparse_fuzz_tests() { + for (_, text) in collect_rust_files(&test_data_dir(), &["reparse/fuzz-failures"]) { + let check = fuzz::CheckReparse::from_data(text.as_bytes()).unwrap(); + println!("{:?}", check); + check.run(); + } +} + +/// Test that Rust-analyzer can parse and validate the rust-analyzer +/// FIXME: Use this as a benchmark +#[test] +fn self_hosting_parsing() { + let dir = project_dir().join("crates"); + let files = walkdir::WalkDir::new(dir) + .into_iter() + .filter_entry(|entry| { + // Get all files which are not in the crates/syntax/test_data folder + !entry.path().components().any(|component| component.as_os_str() == "test_data") + }) + .map(|e| e.unwrap()) + .filter(|entry| { + // Get all `.rs ` files + !entry.path().is_dir() && (entry.path().extension().unwrap_or_default() == "rs") + }) + .map(|entry| entry.into_path()) + .collect::>(); + assert!( + files.len() > 100, + "self_hosting_parsing found too few files - is it running in the right directory?" + ); + + let errors = files + .into_par_iter() + .filter_map(|file| { + let text = read_text(&file); + match SourceFile::parse(&text).ok() { + Ok(_) => None, + Err(err) => Some((file, err)), + } + }) + .collect::>(); + + if !errors.is_empty() { + let errors = errors + .into_iter() + .map(|(path, err)| format!("{}: {:?}\n", path.display(), err)) + .collect::(); + panic!("Parsing errors:\n{}\n", errors); + } +} + +fn test_data_dir() -> PathBuf { + project_dir().join("crates/syntax/test_data") +} + +fn assert_errors_are_present(errors: &[SyntaxError], path: &Path) { + assert!(!errors.is_empty(), "There should be errors in the file {:?}", path.display()); +} +fn assert_errors_are_absent(errors: &[SyntaxError], path: &Path) { + assert_eq!( + errors, + &[] as &[SyntaxError], + "There should be no errors in the file {:?}", + path.display(), + ); +} + +fn dump_tokens_and_errors(tokens: &[Token], errors: &[SyntaxError], text: &str) -> String { + let mut acc = String::new(); + let mut offset: TextSize = 0.into(); + for token in tokens { + let token_len = token.len; + let token_text = &text[TextRange::at(offset, token.len)]; + offset += token.len; + writeln!(acc, "{:?} {:?} {:?}", token.kind, token_len, token_text).unwrap(); + } + for err in errors { + writeln!(acc, "> error{:?} token({:?}) msg({})", err.range(), &text[err.range()], err) + .unwrap(); + } + acc +} + +fn fragment_parser_dir_test(ok_paths: &[&str], err_paths: &[&str], f: F) +where + T: crate::AstNode, + F: Fn(&str) -> Result, +{ + dir_tests(&test_data_dir(), ok_paths, "rast", |text, path| { + if let Ok(node) = f(text) { + format!("{:#?}", crate::ast::AstNode::syntax(&node)) + } else { + panic!("Failed to parse '{:?}'", path); + } + }); + dir_tests(&test_data_dir(), err_paths, "rast", |text, path| { + if let Ok(_) = f(text) { + panic!("'{:?}' successfully parsed when it should have errored", path); + } else { + "ERROR\n".to_owned() + } + }); +} + +/// Calls callback `f` with input code and file paths for each `.rs` file in `test_data_dir` +/// subdirectories defined by `paths`. +/// +/// If the content of the matching output file differs from the output of `f()` +/// the test will fail. +/// +/// If there is no matching output file it will be created and filled with the +/// output of `f()`, but the test will fail. +fn dir_tests(test_data_dir: &Path, paths: &[&str], outfile_extension: &str, f: F) +where + F: Fn(&str, &Path) -> String, +{ + for (path, input_code) in collect_rust_files(test_data_dir, paths) { + let actual = f(&input_code, &path); + let path = path.with_extension(outfile_extension); + expect_file![path].assert_eq(&actual) + } +} + +/// Collects all `.rs` files from `dir` subdirectories defined by `paths`. +fn collect_rust_files(root_dir: &Path, paths: &[&str]) -> Vec<(PathBuf, String)> { + paths + .iter() + .flat_map(|path| { + let path = root_dir.to_owned().join(path); + rust_files_in_dir(&path).into_iter() + }) + .map(|path| { + let text = read_text(&path); + (path, text) + }) + .collect() +} + +/// Collects paths to all `.rs` files from `dir` in a sorted `Vec`. +fn rust_files_in_dir(dir: &Path) -> Vec { + let mut acc = Vec::new(); + for file in fs::read_dir(&dir).unwrap() { + let file = file.unwrap(); + let path = file.path(); + if path.extension().unwrap_or_default() == "rs" { + acc.push(path); + } + } + acc.sort(); + acc +} + +/// Read file and normalize newlines. +/// +/// `rustc` seems to always normalize `\r\n` newlines to `\n`: +/// +/// ``` +/// let s = " +/// "; +/// assert_eq!(s.as_bytes(), &[10]); +/// ``` +/// +/// so this should always be correct. +fn read_text(path: &Path) -> String { + fs::read_to_string(path) + .unwrap_or_else(|_| panic!("File at {:?} should be valid", path)) + .replace("\r\n", "\n") +} -- cgit v1.2.3