perf: add scaling benchmark suite
This commit is contained in:
parent
d4a2967ac8
commit
73a344c0ac
|
|
@ -37,6 +37,21 @@ dependencies = [
|
|||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "alloca"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anes"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
|
||||
|
||||
[[package]]
|
||||
name = "anstyle"
|
||||
version = "1.0.14"
|
||||
|
|
@ -52,6 +67,12 @@ dependencies = [
|
|||
"bytemuck",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||
|
||||
[[package]]
|
||||
name = "backtrace"
|
||||
version = "0.3.76"
|
||||
|
|
@ -135,6 +156,12 @@ dependencies = [
|
|||
"libbz2-rs-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cast"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.5.1"
|
||||
|
|
@ -153,6 +180,33 @@ version = "1.0.5"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
|
||||
|
||||
[[package]]
|
||||
name = "ciborium"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e"
|
||||
dependencies = [
|
||||
"ciborium-io",
|
||||
"ciborium-ll",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ciborium-io"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757"
|
||||
|
||||
[[package]]
|
||||
name = "ciborium-ll"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9"
|
||||
dependencies = [
|
||||
"ciborium-io",
|
||||
"half",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cipher"
|
||||
version = "0.5.2"
|
||||
|
|
@ -244,12 +298,51 @@ dependencies = [
|
|||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "criterion"
|
||||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3"
|
||||
dependencies = [
|
||||
"alloca",
|
||||
"anes",
|
||||
"cast",
|
||||
"ciborium",
|
||||
"clap",
|
||||
"criterion-plot",
|
||||
"itertools",
|
||||
"num-traits",
|
||||
"oorandom",
|
||||
"page_size",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tinytemplate",
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "criterion-plot"
|
||||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea"
|
||||
dependencies = [
|
||||
"cast",
|
||||
"itertools",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
|
||||
|
||||
[[package]]
|
||||
name = "crunchy"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.2.2"
|
||||
|
|
@ -339,6 +432,12 @@ dependencies = [
|
|||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.18.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34"
|
||||
|
||||
[[package]]
|
||||
name = "equivalent"
|
||||
version = "1.0.2"
|
||||
|
|
@ -507,6 +606,17 @@ dependencies = [
|
|||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "half"
|
||||
version = "2.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crunchy",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.14.5"
|
||||
|
|
@ -672,6 +782,15 @@ version = "1.2.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45"
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
|
||||
dependencies = [
|
||||
"either",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.18"
|
||||
|
|
@ -693,6 +812,7 @@ name = "john_slop"
|
|||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"criterion",
|
||||
"directories",
|
||||
"figment",
|
||||
"futures",
|
||||
|
|
@ -845,6 +965,15 @@ version = "0.2.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441"
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "object"
|
||||
version = "0.37.3"
|
||||
|
|
@ -860,6 +989,12 @@ version = "1.21.4"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
||||
|
||||
[[package]]
|
||||
name = "oorandom"
|
||||
version = "11.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
|
||||
|
||||
[[package]]
|
||||
name = "option-ext"
|
||||
version = "0.2.0"
|
||||
|
|
@ -872,6 +1007,16 @@ version = "4.4.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13c45bb4a6ae1280ec0803b1ef9d3455eb50f01efbbe1447ab020f1d54fba9d8"
|
||||
|
||||
[[package]]
|
||||
name = "page_size"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.9.12"
|
||||
|
|
@ -1337,6 +1482,16 @@ dependencies = [
|
|||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinytemplate"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.53.1"
|
||||
|
|
@ -1635,6 +1790,22 @@ dependencies = [
|
|||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
|
||||
dependencies = [
|
||||
"winapi-i686-pc-windows-gnu",
|
||||
"winapi-x86_64-pc-windows-gnu",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-i686-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||
|
||||
[[package]]
|
||||
name = "winapi-util"
|
||||
version = "0.1.11"
|
||||
|
|
@ -1644,6 +1815,12 @@ dependencies = [
|
|||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-x86_64-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.2.1"
|
||||
|
|
@ -1703,6 +1880,26 @@ dependencies = [
|
|||
"synstructure",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.8.59"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6df92bf3d9227be3d53173901ddbffac2babc27ae50f397776ffd6dc33f800cb"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.8.59"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac4f328cf2f05d084e496c3e9c3f33ed0a183656a16e1fcec4d464d8373aec82"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerofrom"
|
||||
version = "0.1.8"
|
||||
|
|
|
|||
|
|
@ -42,6 +42,11 @@ walkdir = "2.5.0"
|
|||
zip = "8.6.0"
|
||||
|
||||
[dev-dependencies]
|
||||
criterion = { version = "0.8.2", default-features = false }
|
||||
futures = { version = "0.3.34", default-features = false, features = ["std"] }
|
||||
insta = { version = "1.48.0", default-features = false }
|
||||
tower = { version = "0.5.3", default-features = false, features = ["util"] }
|
||||
|
||||
[[bench]]
|
||||
name = "core"
|
||||
harness = false
|
||||
|
|
|
|||
|
|
@ -93,6 +93,7 @@ untrusted inputs, but John Slop is not a sandbox or malware scanner.
|
|||
- [Evaluation corpus and current metrics](docs/evaluation.md)
|
||||
- [Limitations and responsible interpretation](docs/limitations.md)
|
||||
- [Development, tests, features, and coverage](docs/development.md)
|
||||
- [Performance benchmarks](docs/benchmarks.md)
|
||||
- [Design decision records](docs/decisions/)
|
||||
- [Background reading](docs/background.md)
|
||||
- [Ambiguous and invisible Unicode data sources](docs/unicode-confusables.md)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,119 @@
|
|||
use std::{
|
||||
fs::{self, File},
|
||||
io::Write,
|
||||
};
|
||||
|
||||
use criterion::{Criterion, Throughput, criterion_group, criterion_main};
|
||||
use john_slop::{AnalysisContext, Config, Detector, read_inputs};
|
||||
use tempfile::TempDir;
|
||||
use zip::write::FileOptions;
|
||||
|
||||
fn benchmark_large_document(criterion: &mut Criterion) {
|
||||
let config = Config::embedded();
|
||||
let detector = Detector::new(&config).expect("embedded detector");
|
||||
let source = "Reviewed prose without configured signals.\n".repeat(25_000);
|
||||
let context = AnalysisContext::for_path("large.txt");
|
||||
let mut group = criterion.benchmark_group("analysis");
|
||||
group.throughput(Throughput::Bytes(source.len() as u64));
|
||||
group.bench_function("one-megabyte-document", |bencher| {
|
||||
bencher.iter(|| detector.analyze_with(&source, &context));
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn benchmark_many_small_files(criterion: &mut Criterion) {
|
||||
let directory = small_file_tree(250);
|
||||
let config = Config::embedded();
|
||||
let mut group = criterion.benchmark_group("input");
|
||||
group.throughput(Throughput::Elements(250));
|
||||
group.bench_function("250-small-files", |bencher| {
|
||||
bencher.iter(|| {
|
||||
read_inputs(&config, directory.path())
|
||||
.expect("open fixture directory")
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.expect("read fixture directory")
|
||||
});
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn benchmark_rule_scaling(criterion: &mut Criterion) {
|
||||
let mut config = Config::embedded();
|
||||
let original = config.rules.clone();
|
||||
for generation in 1..10 {
|
||||
config
|
||||
.rules
|
||||
.extend(original.iter().cloned().map(|mut rule| {
|
||||
rule.id = format!("{}-{generation}", rule.id);
|
||||
rule
|
||||
}));
|
||||
}
|
||||
config.validate().expect("scaled config remains valid");
|
||||
let source = "Assistant: Certainly! Here's a result—review it.\n".repeat(100);
|
||||
let context = AnalysisContext::for_path("scaled.txt");
|
||||
|
||||
criterion.bench_function("rules/build-and-analyze-110-rules", |bencher| {
|
||||
bencher.iter(|| {
|
||||
Detector::new(&config)
|
||||
.expect("build scaled detector")
|
||||
.analyze_with(&source, &context)
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
fn benchmark_archive_limits(criterion: &mut Criterion) {
|
||||
let directory = archive_with_members(100);
|
||||
let path = directory.path().join("many.zip");
|
||||
let config = Config::embedded();
|
||||
let mut group = criterion.benchmark_group("input");
|
||||
group.throughput(Throughput::Elements(100));
|
||||
group.bench_function("100-member-archive", |bencher| {
|
||||
bencher.iter(|| {
|
||||
read_inputs(&config, &path)
|
||||
.expect("open fixture archive")
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.expect("read fixture archive")
|
||||
});
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn small_file_tree(count: usize) -> TempDir {
|
||||
let directory = tempfile::tempdir().expect("create benchmark directory");
|
||||
for index in 0..count {
|
||||
fs::write(
|
||||
directory.path().join(format!("document-{index:04}.txt")),
|
||||
"Reviewed prose without configured signals.\n",
|
||||
)
|
||||
.expect("write benchmark document");
|
||||
}
|
||||
directory
|
||||
}
|
||||
|
||||
fn archive_with_members(count: usize) -> TempDir {
|
||||
let directory = tempfile::tempdir().expect("create benchmark directory");
|
||||
let file = File::create(directory.path().join("many.zip")).expect("create benchmark archive");
|
||||
let mut archive = zip::ZipWriter::new(file);
|
||||
let options: FileOptions<()> = FileOptions::default()
|
||||
.compression_method(zip::CompressionMethod::Stored)
|
||||
.unix_permissions(0o644);
|
||||
for index in 0..count {
|
||||
archive
|
||||
.start_file(format!("document-{index:04}.txt"), options)
|
||||
.expect("start benchmark member");
|
||||
archive
|
||||
.write_all(b"Reviewed prose without configured signals.\n")
|
||||
.expect("write benchmark member");
|
||||
}
|
||||
archive.finish().expect("finish benchmark archive");
|
||||
directory
|
||||
}
|
||||
|
||||
criterion_group!(
|
||||
benches,
|
||||
benchmark_large_document,
|
||||
benchmark_many_small_files,
|
||||
benchmark_rule_scaling,
|
||||
benchmark_archive_limits
|
||||
);
|
||||
criterion_main!(benches);
|
||||
|
|
@ -28,6 +28,8 @@ the details needed to use, embed, and develop John Slop:
|
|||
score semantics, evaluation limits, and operational boundaries.
|
||||
- [Development](development.md) covers the supported Rust version, feature
|
||||
matrix, snapshots, coverage, badges, and CI quality gates.
|
||||
- [Performance benchmarks](benchmarks.md) defines scaling workloads and the
|
||||
evidence required before introducing parallelism.
|
||||
- [Background](background.md) collects external context on observable signs of
|
||||
low-care AI-assisted writing.
|
||||
- [Ambiguous and invisible Unicode references](unicode-confusables.md) records
|
||||
|
|
|
|||
|
|
@ -0,0 +1,41 @@
|
|||
# Performance benchmarks
|
||||
|
||||
Criterion benchmarks cover the scaling boundaries most likely to change
|
||||
architectural decisions:
|
||||
|
||||
- analysis of an approximately 1 MiB clean prose document;
|
||||
- streaming and decoding 250 small files in deterministic path order;
|
||||
- compiling and analyzing with 110 rules (10× the embedded catalog); and
|
||||
- opening, validating, and reading a 100-member stored ZIP archive.
|
||||
|
||||
Run them on an otherwise idle machine:
|
||||
|
||||
```sh
|
||||
cargo bench --locked --bench core
|
||||
```
|
||||
|
||||
Use `-- --quick` for a smoke measurement. Criterion output under `target/` is
|
||||
not a portable performance artifact: record the CPU, OS, Rust version, commit,
|
||||
and exact command when comparing results.
|
||||
|
||||
The Milestone 7 smoke baseline was measured on 2026-09-29 with
|
||||
`cargo bench --locked --bench core -- --quick`, Rust 1.100.0-nightly, Linux
|
||||
7.2.7, and an AMD Ryzen 9 9950X3D:
|
||||
|
||||
| Workload | Criterion estimate |
|
||||
| --- | ---: |
|
||||
| approximately 1 MiB document | 15.686 ms / 65.36 MiB/s |
|
||||
| 250 small files | 286.54 µs / 872.48 Kfiles/s |
|
||||
| build and analyze 110 rules | 9.474 ms |
|
||||
| 100-member stored archive | 116.37 µs / 859.34 Kmembers/s |
|
||||
|
||||
These quick-mode figures are orientation data, not a cross-machine performance
|
||||
guarantee. A release decision should use Criterion's normal sampling mode.
|
||||
|
||||
Milestone 7 retains the ordered single-threaded implementation. The baseline
|
||||
workloads complete in milliseconds on the development machine, and directory
|
||||
and archive results already stream rather than accumulating the entire input
|
||||
tree before analysis. Parallel traversal would add scheduling and output-order
|
||||
coordination without an established user-visible bottleneck. Reconsider it only
|
||||
with a representative workload, a reproducible before/after benchmark, and
|
||||
tests proving byte-for-byte output order remains unchanged.
|
||||
|
|
@ -78,3 +78,10 @@ reviewed seeds for configuration merging, rule validation/compilation, UTF-8
|
|||
span conversion, replacement invariants, and hostile archive bytes. See its
|
||||
README for invocation. Fuzz artifacts remain ignored until a failure has been
|
||||
minimized and promoted to a normal regression test.
|
||||
|
||||
## Benchmarks
|
||||
|
||||
Run `cargo bench --locked --bench core` for the large-document, many-file,
|
||||
rule-scaling, and archive-limit workloads described in
|
||||
[Performance benchmarks](benchmarks.md). Benchmark reports are local artifacts;
|
||||
only methodology and contextualized measurements belong in version control.
|
||||
|
|
|
|||
2
tasks.md
2
tasks.md
|
|
@ -223,7 +223,7 @@ same config.
|
|||
- [x] Add fuzz targets for TOML/config merging, regex/rule loading, UTF-8 span
|
||||
mapping, replacement planning, and malicious ZIP metadata. Seed them with
|
||||
regression fixtures.
|
||||
- [ ] Benchmark large files, many small files, rule scaling, and archive limits.
|
||||
- [x] Benchmark large files, many small files, rule scaling, and archive limits.
|
||||
Add parallelism only if measurement shows a benefit and output remains ordered.
|
||||
- [ ] Review dependency features, licenses, advisories, unsafe code, and release
|
||||
binary size. Generate an SBOM/checksums if releases distribute binaries.
|
||||
|
|
|
|||
Loading…
Reference in New Issue