From 73a344c0acd6df049422f5035af7378a2d9cb562 Mon Sep 17 00:00:00 2001 From: Dandaedre Date: Tue, 29 Sep 2026 17:36:49 +0200 Subject: [PATCH] perf: add scaling benchmark suite --- Cargo.lock | 197 ++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 5 ++ README.md | 1 + benches/core.rs | 119 ++++++++++++++++++++++++++ docs/README.md | 2 + docs/benchmarks.md | 41 +++++++++ docs/development.md | 7 ++ tasks.md | 2 +- 8 files changed, 373 insertions(+), 1 deletion(-) create mode 100644 benches/core.rs create mode 100644 docs/benchmarks.md diff --git a/Cargo.lock b/Cargo.lock index d4d0e3f..4a65cfe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,6 +37,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstyle" version = "1.0.14" @@ -52,6 +67,12 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + [[package]] name = "backtrace" version = "0.3.76" @@ -135,6 +156,12 @@ dependencies = [ "libbz2-rs-sys", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.5.1" @@ -153,6 +180,33 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "cipher" version = "0.5.2" @@ -244,12 +298,51 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" +dependencies = [ + "alloca", + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools", + "num-traits", + "oorandom", + "page_size", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" +dependencies = [ + "cast", + "itertools", +] + [[package]] name = "crossbeam-utils" version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.2.2" @@ -339,6 +432,12 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + [[package]] name = "equivalent" version = "1.0.2" @@ -507,6 +606,17 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -672,6 +782,15 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -693,6 +812,7 @@ name = "john_slop" version = "0.1.0" dependencies = [ "clap", + "criterion", "directories", "figment", "futures", @@ -845,6 +965,15 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "object" version = "0.37.3" @@ -860,6 +989,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "option-ext" version = "0.2.0" @@ -872,6 +1007,16 @@ version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c45bb4a6ae1280ec0803b1ef9d3455eb50f01efbbe1447ab020f1d54fba9d8" +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "parking_lot_core" version = "0.9.12" @@ -1337,6 +1482,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tokio" version = "1.53.1" @@ -1635,6 +1790,22 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -1644,6 +1815,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-link" version = "0.2.1" @@ -1703,6 +1880,26 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zerocopy" +version = "0.8.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6df92bf3d9227be3d53173901ddbffac2babc27ae50f397776ffd6dc33f800cb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac4f328cf2f05d084e496c3e9c3f33ed0a183656a16e1fcec4d464d8373aec82" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "zerofrom" version = "0.1.8" diff --git a/Cargo.toml b/Cargo.toml index f8e0335..b1a6aa1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,11 @@ walkdir = "2.5.0" zip = "8.6.0" [dev-dependencies] +criterion = { version = "0.8.2", default-features = false } futures = { version = "0.3.34", default-features = false, features = ["std"] } insta = { version = "1.48.0", default-features = false } tower = { version = "0.5.3", default-features = false, features = ["util"] } + +[[bench]] +name = "core" +harness = false diff --git a/README.md b/README.md index e6532c8..c19179d 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,7 @@ untrusted inputs, but John Slop is not a sandbox or malware scanner. - [Evaluation corpus and current metrics](docs/evaluation.md) - [Limitations and responsible interpretation](docs/limitations.md) - [Development, tests, features, and coverage](docs/development.md) +- [Performance benchmarks](docs/benchmarks.md) - [Design decision records](docs/decisions/) - [Background reading](docs/background.md) - [Ambiguous and invisible Unicode data sources](docs/unicode-confusables.md) diff --git a/benches/core.rs b/benches/core.rs new file mode 100644 index 0000000..151323c --- /dev/null +++ b/benches/core.rs @@ -0,0 +1,119 @@ +use std::{ + fs::{self, File}, + io::Write, +}; + +use criterion::{Criterion, Throughput, criterion_group, criterion_main}; +use john_slop::{AnalysisContext, Config, Detector, read_inputs}; +use tempfile::TempDir; +use zip::write::FileOptions; + +fn benchmark_large_document(criterion: &mut Criterion) { + let config = Config::embedded(); + let detector = Detector::new(&config).expect("embedded detector"); + let source = "Reviewed prose without configured signals.\n".repeat(25_000); + let context = AnalysisContext::for_path("large.txt"); + let mut group = criterion.benchmark_group("analysis"); + group.throughput(Throughput::Bytes(source.len() as u64)); + group.bench_function("one-megabyte-document", |bencher| { + bencher.iter(|| detector.analyze_with(&source, &context)); + }); + group.finish(); +} + +fn benchmark_many_small_files(criterion: &mut Criterion) { + let directory = small_file_tree(250); + let config = Config::embedded(); + let mut group = criterion.benchmark_group("input"); + group.throughput(Throughput::Elements(250)); + group.bench_function("250-small-files", |bencher| { + bencher.iter(|| { + read_inputs(&config, directory.path()) + .expect("open fixture directory") + .collect::, _>>() + .expect("read fixture directory") + }); + }); + group.finish(); +} + +fn benchmark_rule_scaling(criterion: &mut Criterion) { + let mut config = Config::embedded(); + let original = config.rules.clone(); + for generation in 1..10 { + config + .rules + .extend(original.iter().cloned().map(|mut rule| { + rule.id = format!("{}-{generation}", rule.id); + rule + })); + } + config.validate().expect("scaled config remains valid"); + let source = "Assistant: Certainly! Here's a result—review it.\n".repeat(100); + let context = AnalysisContext::for_path("scaled.txt"); + + criterion.bench_function("rules/build-and-analyze-110-rules", |bencher| { + bencher.iter(|| { + Detector::new(&config) + .expect("build scaled detector") + .analyze_with(&source, &context) + }); + }); +} + +fn benchmark_archive_limits(criterion: &mut Criterion) { + let directory = archive_with_members(100); + let path = directory.path().join("many.zip"); + let config = Config::embedded(); + let mut group = criterion.benchmark_group("input"); + group.throughput(Throughput::Elements(100)); + group.bench_function("100-member-archive", |bencher| { + bencher.iter(|| { + read_inputs(&config, &path) + .expect("open fixture archive") + .collect::, _>>() + .expect("read fixture archive") + }); + }); + group.finish(); +} + +fn small_file_tree(count: usize) -> TempDir { + let directory = tempfile::tempdir().expect("create benchmark directory"); + for index in 0..count { + fs::write( + directory.path().join(format!("document-{index:04}.txt")), + "Reviewed prose without configured signals.\n", + ) + .expect("write benchmark document"); + } + directory +} + +fn archive_with_members(count: usize) -> TempDir { + let directory = tempfile::tempdir().expect("create benchmark directory"); + let file = File::create(directory.path().join("many.zip")).expect("create benchmark archive"); + let mut archive = zip::ZipWriter::new(file); + let options: FileOptions<()> = FileOptions::default() + .compression_method(zip::CompressionMethod::Stored) + .unix_permissions(0o644); + for index in 0..count { + archive + .start_file(format!("document-{index:04}.txt"), options) + .expect("start benchmark member"); + archive + .write_all(b"Reviewed prose without configured signals.\n") + .expect("write benchmark member"); + } + archive.finish().expect("finish benchmark archive"); + directory +} + +criterion_group!( + benches, + benchmark_large_document, + benchmark_many_small_files, + benchmark_rule_scaling, + benchmark_archive_limits +); +criterion_main!(benches); diff --git a/docs/README.md b/docs/README.md index 0fa49d2..648d4ee 100644 --- a/docs/README.md +++ b/docs/README.md @@ -28,6 +28,8 @@ the details needed to use, embed, and develop John Slop: score semantics, evaluation limits, and operational boundaries. - [Development](development.md) covers the supported Rust version, feature matrix, snapshots, coverage, badges, and CI quality gates. +- [Performance benchmarks](benchmarks.md) defines scaling workloads and the + evidence required before introducing parallelism. - [Background](background.md) collects external context on observable signs of low-care AI-assisted writing. - [Ambiguous and invisible Unicode references](unicode-confusables.md) records diff --git a/docs/benchmarks.md b/docs/benchmarks.md new file mode 100644 index 0000000..21d9737 --- /dev/null +++ b/docs/benchmarks.md @@ -0,0 +1,41 @@ +# Performance benchmarks + +Criterion benchmarks cover the scaling boundaries most likely to change +architectural decisions: + +- analysis of an approximately 1 MiB clean prose document; +- streaming and decoding 250 small files in deterministic path order; +- compiling and analyzing with 110 rules (10× the embedded catalog); and +- opening, validating, and reading a 100-member stored ZIP archive. + +Run them on an otherwise idle machine: + +```sh +cargo bench --locked --bench core +``` + +Use `-- --quick` for a smoke measurement. Criterion output under `target/` is +not a portable performance artifact: record the CPU, OS, Rust version, commit, +and exact command when comparing results. + +The Milestone 7 smoke baseline was measured on 2026-09-29 with +`cargo bench --locked --bench core -- --quick`, Rust 1.100.0-nightly, Linux +7.2.7, and an AMD Ryzen 9 9950X3D: + +| Workload | Criterion estimate | +| --- | ---: | +| approximately 1 MiB document | 15.686 ms / 65.36 MiB/s | +| 250 small files | 286.54 µs / 872.48 Kfiles/s | +| build and analyze 110 rules | 9.474 ms | +| 100-member stored archive | 116.37 µs / 859.34 Kmembers/s | + +These quick-mode figures are orientation data, not a cross-machine performance +guarantee. A release decision should use Criterion's normal sampling mode. + +Milestone 7 retains the ordered single-threaded implementation. The baseline +workloads complete in milliseconds on the development machine, and directory +and archive results already stream rather than accumulating the entire input +tree before analysis. Parallel traversal would add scheduling and output-order +coordination without an established user-visible bottleneck. Reconsider it only +with a representative workload, a reproducible before/after benchmark, and +tests proving byte-for-byte output order remains unchanged. diff --git a/docs/development.md b/docs/development.md index 05fa4cc..2dba4e1 100644 --- a/docs/development.md +++ b/docs/development.md @@ -78,3 +78,10 @@ reviewed seeds for configuration merging, rule validation/compilation, UTF-8 span conversion, replacement invariants, and hostile archive bytes. See its README for invocation. Fuzz artifacts remain ignored until a failure has been minimized and promoted to a normal regression test. + +## Benchmarks + +Run `cargo bench --locked --bench core` for the large-document, many-file, +rule-scaling, and archive-limit workloads described in +[Performance benchmarks](benchmarks.md). Benchmark reports are local artifacts; +only methodology and contextualized measurements belong in version control. diff --git a/tasks.md b/tasks.md index 25bafc2..1fadc59 100644 --- a/tasks.md +++ b/tasks.md @@ -223,7 +223,7 @@ same config. - [x] Add fuzz targets for TOML/config merging, regex/rule loading, UTF-8 span mapping, replacement planning, and malicious ZIP metadata. Seed them with regression fixtures. -- [ ] Benchmark large files, many small files, rule scaling, and archive limits. +- [x] Benchmark large files, many small files, rule scaling, and archive limits. Add parallelism only if measurement shows a benefit and output remains ordered. - [ ] Review dependency features, licenses, advisories, unsafe code, and release binary size. Generate an SBOM/checksums if releases distribute binaries.