From a80e252193096f22ae79fa03e66a0853ddae050e Mon Sep 17 00:00:00 2001 From: Akshay Date: Sat, 19 Feb 2022 20:44:59 +0530 Subject: parallelize statix-check --- Cargo.lock | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ bin/Cargo.toml | 9 ++++--- bin/src/lint.rs | 9 +++++-- flake.lock | 18 ++++++------- flake.nix | 4 +-- lib/src/lib.rs | 2 ++ vfs/Cargo.toml | 2 +- vfs/src/lib.rs | 8 ++++++ 8 files changed, 117 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b4e82be..e23afee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -123,6 +123,40 @@ version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "328b822bdcba4d4e402be8d9adb6eebf269f969f8eadef977a553ff3c4fbcb58" +[[package]] +name = "crossbeam-channel" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e54ea8bc3fb1ee042f5aace6e3c6e025d3874866da222930f70ce62aceba0bfa" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c00d6d2ea26e8b151d99093005cb442fb9a37aeaca582a03ec70946f49ab5ed9" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + [[package]] name = "crossbeam-utils" version = "0.8.5" @@ -139,6 +173,12 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + [[package]] name = "encode_unicode" version = "0.3.6" @@ -322,6 +362,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "once_cell" version = "1.8.0" @@ -379,6 +429,31 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "lazy_static", + "num_cpus", +] + [[package]] name = "regex" version = "1.5.4" @@ -441,6 +516,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + [[package]] name = "serde" version = "1.0.130" @@ -514,6 +595,7 @@ dependencies = [ "ignore", "insta", "lib", + "rayon", "rnix", "serde", "serde_json", @@ -672,6 +754,7 @@ name = "vfs" version = "0.0.0" dependencies = [ "indexmap", + "rayon", ] [[package]] diff --git a/bin/Cargo.toml b/bin/Cargo.toml index bf39a3a..715e466 100644 --- a/bin/Cargo.toml +++ b/bin/Cargo.toml @@ -16,14 +16,15 @@ path = "src/main.rs" [dependencies] ariadne = "0.1.3" -rnix = "0.10.1" clap = "3.0.0-beta.4" ignore = "0.4.18" -thiserror = "1.0.30" -similar = "2.1.0" -vfs = { path = "../vfs" } lib = { path = "../lib" } +rayon = "1.5.1" +rnix = "0.10.1" +similar = "2.1.0" +thiserror = "1.0.30" toml = "0.5.8" +vfs = { path = "../vfs" } [dependencies.serde] version = "1.0.68" diff --git a/bin/src/lint.rs b/bin/src/lint.rs index 23a1d10..e1544e1 100644 --- a/bin/src/lint.rs +++ b/bin/src/lint.rs @@ -50,6 +50,7 @@ pub mod main { }; use lib::session::SessionInfo; + use rayon::prelude::*; pub fn main(check_config: CheckConfig) -> Result<(), StatixErr> { let vfs = check_config.vfs()?; @@ -59,9 +60,13 @@ pub mod main { let version = conf_file.version()?; let session = SessionInfo::from_version(version); let lint = |vfs_entry| lint_with(vfs_entry, &lints, &session); - let results = vfs.iter().map(lint).collect::>(); + let results = vfs + .par_iter() + .map(lint) + .filter(|lr| !lr.reports.is_empty()) + .collect::>(); - if results.iter().map(|r| r.reports.len()).sum::() != 0 { + if results.len() != 0 { for r in &results { stdout.write(&r, &vfs, check_config.format).unwrap(); } diff --git a/flake.lock b/flake.lock index 1e575e3..a7c083f 100644 --- a/flake.lock +++ b/flake.lock @@ -8,11 +8,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1642141613, - "narHash": "sha256-WFS7T5oOLD8anu/77iz7EPz7zQsJZtP5qVa5485k3dI=", + "lastModified": 1645251813, + "narHash": "sha256-cQ66tGjnZclBCS3nD26mZ5fUH+3/HnysGffBiWXUSHk=", "owner": "nix-community", "repo": "fenix", - "rev": "d75ffc9f1d3b60bb597e48ae01f486a6de27d30e", + "rev": "9892337b588c38ec59466a1c89befce464aae7f8", "type": "github" }, "original": { @@ -43,11 +43,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1642069818, - "narHash": "sha256-666w6j8wl/bojfgpp0k58/UJ5rbrdYFbI2RFT2BXbSQ=", + "lastModified": 1645013224, + "narHash": "sha256-b7OEC8vwzJv3rsz9pwnTX2LQDkeOWz2DbKypkVvNHXc=", "owner": "nixos", "repo": "nixpkgs", - "rev": "46821ea01c8f54d2a20f5a503809abfc605269d7", + "rev": "b66b39216b1fef2d8c33cc7a5c72d8da80b79970", "type": "github" }, "original": { @@ -67,11 +67,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1642101527, - "narHash": "sha256-C4zcaWULWlrirpEt/pA85CCMBt+7SWZweMaMJ7EvrXw=", + "lastModified": 1645205556, + "narHash": "sha256-e4lZW3qRyOEJ+vLKFQP7m2Dxh5P44NrnekZYLxlucww=", "owner": "rust-analyzer", "repo": "rust-analyzer", - "rev": "b4c31481a554d0132003228ba319bd9476fe85ae", + "rev": "acf5874b39f3dc5262317a6074d9fc7285081161", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 9bb9332..f6099e0 100644 --- a/flake.nix +++ b/flake.nix @@ -33,9 +33,9 @@ }); chanspec = { - date = "2021-12-01"; + date = "2022-02-06"; channel = "nightly"; - sha256 = "DhIP1w63/hMbWlgElJGBumEK/ExFWCdLaeBV5F8uWHc="; # set zeros after modifying channel or date + sha256 = "oKkTWopCDx4tphzTtRn+zDDtvmIZrL/H44tV2ruSfDw="; # set zeros after modifying channel or date }; rustChannel = p: (fenix.overlay p p).fenix.toolchainOf chanspec; diff --git a/lib/src/lib.rs b/lib/src/lib.rs index cc4818a..8c05706 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -201,6 +201,8 @@ impl Suggestion { } } +unsafe impl Send for Suggestion {} + #[cfg(feature = "json-out")] impl Serialize for Suggestion { fn serialize(&self, serializer: S) -> Result diff --git a/vfs/Cargo.toml b/vfs/Cargo.toml index 7a5dc6b..d1d654c 100644 --- a/vfs/Cargo.toml +++ b/vfs/Cargo.toml @@ -6,4 +6,4 @@ license = "MIT" [dependencies] indexmap = "1.6.2" - +rayon = "1.5.1" diff --git a/vfs/src/lib.rs b/vfs/src/lib.rs index cd6cc03..2d7577d 100644 --- a/vfs/src/lib.rs +++ b/vfs/src/lib.rs @@ -5,6 +5,7 @@ use std::{ }; use indexmap::IndexSet; +use rayon::prelude::*; #[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash)] pub struct FileId(pub u32); @@ -70,6 +71,13 @@ impl ReadOnlyVfs { contents: self.get_str(*file_id), }) } + pub fn par_iter(&self) -> impl ParallelIterator { + self.data.par_iter().map(move |(file_id, _)| VfsEntry { + file_id: *file_id, + file_path: self.file_path(*file_id), + contents: self.get_str(*file_id), + }) + } } pub struct VfsEntry<'ρ> { -- cgit v1.2.3