perf: add scaling benchmark suite

This commit is contained in:
Dandaedre 2026-09-29 17:36:49 +02:00
parent d4a2967ac8
commit 73a344c0ac
Signed by: Dandaedre
GPG Key ID: 440C6592E8A26D99
8 changed files with 373 additions and 1 deletions

197
Cargo.lock generated
View File

@ -37,6 +37,21 @@ dependencies = [
"memchr",
]
[[package]]
name = "alloca"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4"
dependencies = [
"cc",
]
[[package]]
name = "anes"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
[[package]]
name = "anstyle"
version = "1.0.14"
@ -52,6 +67,12 @@ dependencies = [
"bytemuck",
]
[[package]]
name = "autocfg"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
[[package]]
name = "backtrace"
version = "0.3.76"
@ -135,6 +156,12 @@ dependencies = [
"libbz2-rs-sys",
]
[[package]]
name = "cast"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "cc"
version = "1.5.1"
@ -153,6 +180,33 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
[[package]]
name = "ciborium"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e"
dependencies = [
"ciborium-io",
"ciborium-ll",
"serde",
]
[[package]]
name = "ciborium-io"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757"
[[package]]
name = "ciborium-ll"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9"
dependencies = [
"ciborium-io",
"half",
]
[[package]]
name = "cipher"
version = "0.5.2"
@ -244,12 +298,51 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "criterion"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3"
dependencies = [
"alloca",
"anes",
"cast",
"ciborium",
"clap",
"criterion-plot",
"itertools",
"num-traits",
"oorandom",
"page_size",
"regex",
"serde",
"serde_json",
"tinytemplate",
"walkdir",
]
[[package]]
name = "criterion-plot"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea"
dependencies = [
"cast",
"itertools",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
[[package]]
name = "crunchy"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
[[package]]
name = "crypto-common"
version = "0.2.2"
@ -339,6 +432,12 @@ dependencies = [
"syn 3.0.6",
]
[[package]]
name = "either"
version = "1.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34"
[[package]]
name = "equivalent"
version = "1.0.2"
@ -507,6 +606,17 @@ dependencies = [
"regex-syntax",
]
[[package]]
name = "half"
version = "2.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
dependencies = [
"cfg-if",
"crunchy",
"zerocopy",
]
[[package]]
name = "hashbrown"
version = "0.14.5"
@ -672,6 +782,15 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45"
[[package]]
name = "itertools"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
dependencies = [
"either",
]
[[package]]
name = "itoa"
version = "1.0.18"
@ -693,6 +812,7 @@ name = "john_slop"
version = "0.1.0"
dependencies = [
"clap",
"criterion",
"directories",
"figment",
"futures",
@ -845,6 +965,15 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441"
[[package]]
name = "num-traits"
version = "0.2.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
dependencies = [
"autocfg",
]
[[package]]
name = "object"
version = "0.37.3"
@ -860,6 +989,12 @@ version = "1.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
[[package]]
name = "oorandom"
version = "11.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
[[package]]
name = "option-ext"
version = "0.2.0"
@ -872,6 +1007,16 @@ version = "4.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13c45bb4a6ae1280ec0803b1ef9d3455eb50f01efbbe1447ab020f1d54fba9d8"
[[package]]
name = "page_size"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "parking_lot_core"
version = "0.9.12"
@ -1337,6 +1482,16 @@ dependencies = [
"zerovec",
]
[[package]]
name = "tinytemplate"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
dependencies = [
"serde",
"serde_json",
]
[[package]]
name = "tokio"
version = "1.53.1"
@ -1635,6 +1790,22 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-util"
version = "0.1.11"
@ -1644,6 +1815,12 @@ dependencies = [
"windows-sys",
]
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows-link"
version = "0.2.1"
@ -1703,6 +1880,26 @@ dependencies = [
"synstructure",
]
[[package]]
name = "zerocopy"
version = "0.8.59"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6df92bf3d9227be3d53173901ddbffac2babc27ae50f397776ffd6dc33f800cb"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.8.59"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac4f328cf2f05d084e496c3e9c3f33ed0a183656a16e1fcec4d464d8373aec82"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "zerofrom"
version = "0.1.8"

View File

@ -42,6 +42,11 @@ walkdir = "2.5.0"
zip = "8.6.0"
[dev-dependencies]
criterion = { version = "0.8.2", default-features = false }
futures = { version = "0.3.34", default-features = false, features = ["std"] }
insta = { version = "1.48.0", default-features = false }
tower = { version = "0.5.3", default-features = false, features = ["util"] }
[[bench]]
name = "core"
harness = false

View File

@ -93,6 +93,7 @@ untrusted inputs, but John Slop is not a sandbox or malware scanner.
- [Evaluation corpus and current metrics](docs/evaluation.md)
- [Limitations and responsible interpretation](docs/limitations.md)
- [Development, tests, features, and coverage](docs/development.md)
- [Performance benchmarks](docs/benchmarks.md)
- [Design decision records](docs/decisions/)
- [Background reading](docs/background.md)
- [Ambiguous and invisible Unicode data sources](docs/unicode-confusables.md)

119
benches/core.rs Normal file
View File

@ -0,0 +1,119 @@
use std::{
fs::{self, File},
io::Write,
};
use criterion::{Criterion, Throughput, criterion_group, criterion_main};
use john_slop::{AnalysisContext, Config, Detector, read_inputs};
use tempfile::TempDir;
use zip::write::FileOptions;
fn benchmark_large_document(criterion: &mut Criterion) {
let config = Config::embedded();
let detector = Detector::new(&config).expect("embedded detector");
let source = "Reviewed prose without configured signals.\n".repeat(25_000);
let context = AnalysisContext::for_path("large.txt");
let mut group = criterion.benchmark_group("analysis");
group.throughput(Throughput::Bytes(source.len() as u64));
group.bench_function("one-megabyte-document", |bencher| {
bencher.iter(|| detector.analyze_with(&source, &context));
});
group.finish();
}
fn benchmark_many_small_files(criterion: &mut Criterion) {
let directory = small_file_tree(250);
let config = Config::embedded();
let mut group = criterion.benchmark_group("input");
group.throughput(Throughput::Elements(250));
group.bench_function("250-small-files", |bencher| {
bencher.iter(|| {
read_inputs(&config, directory.path())
.expect("open fixture directory")
.collect::<Result<Vec<_>, _>>()
.expect("read fixture directory")
});
});
group.finish();
}
fn benchmark_rule_scaling(criterion: &mut Criterion) {
let mut config = Config::embedded();
let original = config.rules.clone();
for generation in 1..10 {
config
.rules
.extend(original.iter().cloned().map(|mut rule| {
rule.id = format!("{}-{generation}", rule.id);
rule
}));
}
config.validate().expect("scaled config remains valid");
let source = "Assistant: Certainly! Here's a result—review it.\n".repeat(100);
let context = AnalysisContext::for_path("scaled.txt");
criterion.bench_function("rules/build-and-analyze-110-rules", |bencher| {
bencher.iter(|| {
Detector::new(&config)
.expect("build scaled detector")
.analyze_with(&source, &context)
});
});
}
fn benchmark_archive_limits(criterion: &mut Criterion) {
let directory = archive_with_members(100);
let path = directory.path().join("many.zip");
let config = Config::embedded();
let mut group = criterion.benchmark_group("input");
group.throughput(Throughput::Elements(100));
group.bench_function("100-member-archive", |bencher| {
bencher.iter(|| {
read_inputs(&config, &path)
.expect("open fixture archive")
.collect::<Result<Vec<_>, _>>()
.expect("read fixture archive")
});
});
group.finish();
}
fn small_file_tree(count: usize) -> TempDir {
let directory = tempfile::tempdir().expect("create benchmark directory");
for index in 0..count {
fs::write(
directory.path().join(format!("document-{index:04}.txt")),
"Reviewed prose without configured signals.\n",
)
.expect("write benchmark document");
}
directory
}
fn archive_with_members(count: usize) -> TempDir {
let directory = tempfile::tempdir().expect("create benchmark directory");
let file = File::create(directory.path().join("many.zip")).expect("create benchmark archive");
let mut archive = zip::ZipWriter::new(file);
let options: FileOptions<()> = FileOptions::default()
.compression_method(zip::CompressionMethod::Stored)
.unix_permissions(0o644);
for index in 0..count {
archive
.start_file(format!("document-{index:04}.txt"), options)
.expect("start benchmark member");
archive
.write_all(b"Reviewed prose without configured signals.\n")
.expect("write benchmark member");
}
archive.finish().expect("finish benchmark archive");
directory
}
criterion_group!(
benches,
benchmark_large_document,
benchmark_many_small_files,
benchmark_rule_scaling,
benchmark_archive_limits
);
criterion_main!(benches);

View File

@ -28,6 +28,8 @@ the details needed to use, embed, and develop John Slop:
score semantics, evaluation limits, and operational boundaries.
- [Development](development.md) covers the supported Rust version, feature
matrix, snapshots, coverage, badges, and CI quality gates.
- [Performance benchmarks](benchmarks.md) defines scaling workloads and the
evidence required before introducing parallelism.
- [Background](background.md) collects external context on observable signs of
low-care AI-assisted writing.
- [Ambiguous and invisible Unicode references](unicode-confusables.md) records

41
docs/benchmarks.md Normal file
View File

@ -0,0 +1,41 @@
# Performance benchmarks
Criterion benchmarks cover the scaling boundaries most likely to change
architectural decisions:
- analysis of an approximately 1 MiB clean prose document;
- streaming and decoding 250 small files in deterministic path order;
- compiling and analyzing with 110 rules (10× the embedded catalog); and
- opening, validating, and reading a 100-member stored ZIP archive.
Run them on an otherwise idle machine:
```sh
cargo bench --locked --bench core
```
Use `-- --quick` for a smoke measurement. Criterion output under `target/` is
not a portable performance artifact: record the CPU, OS, Rust version, commit,
and exact command when comparing results.
The Milestone 7 smoke baseline was measured on 2026-09-29 with
`cargo bench --locked --bench core -- --quick`, Rust 1.100.0-nightly, Linux
7.2.7, and an AMD Ryzen 9 9950X3D:
| Workload | Criterion estimate |
| --- | ---: |
| approximately 1 MiB document | 15.686 ms / 65.36 MiB/s |
| 250 small files | 286.54 µs / 872.48 Kfiles/s |
| build and analyze 110 rules | 9.474 ms |
| 100-member stored archive | 116.37 µs / 859.34 Kmembers/s |
These quick-mode figures are orientation data, not a cross-machine performance
guarantee. A release decision should use Criterion's normal sampling mode.
Milestone 7 retains the ordered single-threaded implementation. The baseline
workloads complete in milliseconds on the development machine, and directory
and archive results already stream rather than accumulating the entire input
tree before analysis. Parallel traversal would add scheduling and output-order
coordination without an established user-visible bottleneck. Reconsider it only
with a representative workload, a reproducible before/after benchmark, and
tests proving byte-for-byte output order remains unchanged.

View File

@ -78,3 +78,10 @@ reviewed seeds for configuration merging, rule validation/compilation, UTF-8
span conversion, replacement invariants, and hostile archive bytes. See its
README for invocation. Fuzz artifacts remain ignored until a failure has been
minimized and promoted to a normal regression test.
## Benchmarks
Run `cargo bench --locked --bench core` for the large-document, many-file,
rule-scaling, and archive-limit workloads described in
[Performance benchmarks](benchmarks.md). Benchmark reports are local artifacts;
only methodology and contextualized measurements belong in version control.

View File

@ -223,7 +223,7 @@ same config.
- [x] Add fuzz targets for TOML/config merging, regex/rule loading, UTF-8 span
mapping, replacement planning, and malicious ZIP metadata. Seed them with
regression fixtures.
- [ ] Benchmark large files, many small files, rule scaling, and archive limits.
- [x] Benchmark large files, many small files, rule scaling, and archive limits.
Add parallelism only if measurement shows a benefit and output remains ordered.
- [ ] Review dependency features, licenses, advisories, unsafe code, and release
binary size. Generate an SBOM/checksums if releases distribute binaries.