From 02af5e9a55c02554de138563e05b50f46d795e8e Mon Sep 17 00:00:00 2001 From: Dandaedre Date: Wed, 16 Sep 2026 19:14:16 +0200 Subject: [PATCH] Guard authored and diagnostic output parity --- reporter | 2 +- reports | 2 +- scripts/check-components.sh | 2 + scripts/check-diagnostic-parity.py | 65 ++++++++++++++++++++++++++++++ scripts/check-regular-parity.py | 47 +++++++++++++++++++++ 5 files changed, 116 insertions(+), 2 deletions(-) create mode 100644 scripts/check-diagnostic-parity.py create mode 100644 scripts/check-regular-parity.py diff --git a/reporter b/reporter index 2eebf62..127a74a 160000 --- a/reporter +++ b/reporter @@ -1 +1 @@ -Subproject commit 2eebf62f122631f72234438f6b5be15955021300 +Subproject commit 127a74ab29afacaa90c3683fea8f556f8207d92a diff --git a/reports b/reports index c42ba8c..e8eb3e3 160000 --- a/reports +++ b/reports @@ -1 +1 @@ -Subproject commit c42ba8c095d770d8d4620169587389f9d247a7e5 +Subproject commit e8eb3e381305da0b97cc241e95c33a14c211a01d diff --git a/scripts/check-components.sh b/scripts/check-components.sh index fcccd3c..ad07c68 100755 --- a/scripts/check-components.sh +++ b/scripts/check-components.sh @@ -44,6 +44,8 @@ BRANDING_DIRECTORY="$(pwd)/reports/plugins/diagnostics/runtime/themes" \ echo 'Checking stored Reportist YAML' reporter/tools/validate-all.sh reports/reports +python3 scripts/check-regular-parity.py +python3 scripts/check-diagnostic-parity.py pipeline_check_parent="$(mktemp -d "${TMPDIR:-/tmp}/beffroi-pipeline-check.XXXXXX")" trap 'rm -rf -- "$pipeline_check_parent"' EXIT diff --git a/scripts/check-diagnostic-parity.py b/scripts/check-diagnostic-parity.py new file mode 100644 index 0000000..549b314 --- /dev/null +++ b/scripts/check-diagnostic-parity.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Guard reviewed finding counts from the pre-plugin diagnostic renderer.""" + +import json +from pathlib import Path +import subprocess +import sys +import tempfile + + +ROOT = Path(__file__).resolve().parents[1] +CAPTURES = ROOT / "reports/diagnostics" +COUNTS = { + "hardware-17-tA6kwqAg-windows-system-report-desktop-2osf051-20260904-134702": (2, 10, 0), + "hardware-18-WHA8oUN2-windows-system-report-pcvid008-20260904-134729": (1, 9, 1), + "hardware-19-BOOso6wz-windows-system-report-desktop-dh4hot6-20260904-173202": (2, 11, 0), + "hardware-20-dlBVl76W-windows-system-report-desktop-pr71lgi-20260904-162618": (2, 7, 0), + "hardware-35-NSmkLXTb-windows-system-report-blin2-20260904-161637": (2, 6, 0), + "hardware-51-6M9gNOPR-windows-system-report-desktop-sau019h-20260904-164514": (3, 10, 0), + "hardware-53-AbCX84Q8-windows-system-report-desktop-3pptg49-20260904-135829": (1, 7, 0), + "windows_system_report_DESKTOP-SAU019H_20260914_163753": (2, 5, 0), + "windows_system_report_DESKTOP-SAU019H_20260915_113124": (0, 7, 0), + "windows_system_report_DESKTOP-SAU019H_20260915_121057": (1, 16, 0), +} + + +def main(): + captures = {path.stem: path for path in CAPTURES.glob("*.json")} + if set(captures) != set(COUNTS): + print("Stored diagnostic set changed; review the parity baseline", file=sys.stderr) + return 1 + with tempfile.TemporaryDirectory(prefix="beffroi-diagnostic-parity.") as directory: + latest_diagnostic = None + for name, expected in COUNTS.items(): + capture = captures[name] + collected = json.loads(capture.read_text(encoding="utf-8"))["generated_at"] + output = Path(directory) / f"{name}.json" + subprocess.run([str(ROOT / "reports/plugins/diagnostics/analyse.sh"), str(capture), + "--evidence-root", str(CAPTURES), "--output", str(output), + "--reference-time", collected, "--producer-version", "parity-check"], + check=True, stdout=subprocess.DEVNULL) + findings = json.loads(output.read_text(encoding="utf-8"))["findings"] + actual = tuple(sum(item["severity"] == severity for item in findings) + for severity in ("critical", "warning", "unanalysed")) + if actual != expected: + print(f"{name}: expected {expected}, got {actual}", file=sys.stderr) + return 1 + if name == "windows_system_report_DESKTOP-SAU019H_20260915_121057": + latest_diagnostic = output + ir = Path(directory) / "latest.report.json" + subprocess.run([str(ROOT / "reporter/render.sh"), str(latest_diagnostic), + "--plugin", str(ROOT / "reports/plugins/diagnostics"), + "--output", str(Path(directory) / "latest.pdf"), "--emit-ir", str(ir)], + check=True, stdout=subprocess.DEVNULL) + report = json.loads(ir.read_text(encoding="utf-8")) + score = report["sections"][0]["content"][1]["value"] + if score != 62: + print(f"Latest capture: expected pre-plugin signal score 62, got {score}", file=sys.stderr) + return 1 + print(f"All {len(COUNTS)} diagnostics retain pre-plugin critical/warning/unanalysed counts") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check-regular-parity.py b/scripts/check-regular-parity.py new file mode 100644 index 0000000..e880f17 --- /dev/null +++ b/scripts/check-regular-parity.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Guard the seven authored PDFs against the pre-plugin release's extracted text.""" + +import hashlib +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + + +ROOT = Path(__file__).resolve().parents[1] +TEXT_SHA256 = { + "2026-09-04-clevo-handoff": "6bb91e92caf442eaabacca61fbafca6b967bd4692877eee9efa5d3ff54686e5c", + "2026-09-04-clevo-initial": "2a05b6f93f8e5a5e0adf48fa2d83d3072fc404755654e8feb27eb5c592ec2ddf", + "2026-09-04-report": "e7dbaf7fa722fa18e333387f463622f7e082bd9a39dd785d5700965456a743ab", + "2026-09-14-system-stability-diagnostic-notes": "0fbe2dc9bd6044663c0bc026dc132eb69a7f8e4f8c4a9640914da9860aa9c11e", + "2026-09-15-clevo-diagnostic-synthesis": "c729ceca5c6e7481f16593af1113c94d03cee33d51af2556bd4c74a26155215a", + "2026-09-15-clevo-manufacturer-report": "01c522b83a14524bee1646c3f55c45bc0c96eec28696416054224958027d4d3d", + "2026-09-15-clevo-system-stability-diagnostic-notes": "32c88e56144aa15392e929a5d07a7f737e6397e37e65dae43d648429f8da1a7f", +} + + +def main(): + if not shutil.which("pdftotext"): + print("pdftotext is required for regular report parity checks", file=sys.stderr) + return 2 + inputs = {path.stem: path for path in (ROOT / "reports/reports").glob("*.yaml")} + if set(inputs) != set(TEXT_SHA256): + print("Authored report set changed; update the parity baseline deliberately", file=sys.stderr) + return 1 + with tempfile.TemporaryDirectory(prefix="beffroi-regular-parity.") as directory: + for name, expected in TEXT_SHA256.items(): + pdf = Path(directory) / f"{name}.pdf" + subprocess.run([str(ROOT / "reporter/render.sh"), str(inputs[name]), "--output", str(pdf)], + check=True, stdout=subprocess.DEVNULL) + extracted = subprocess.check_output(["pdftotext", "-layout", str(pdf), "-"]) + actual = hashlib.sha256(extracted).hexdigest() + if actual != expected: + print(f"{name}: extracted text differs from pre-plugin PDF", file=sys.stderr) + return 1 + print(f"All {len(TEXT_SHA256)} authored reports retain pre-plugin extracted text") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())