Guard authored and diagnostic output parity

This commit is contained in:
Dandaedre 2026-09-16 19:14:16 +02:00
parent 22a6d477c5
commit 02af5e9a55
Signed by: Dandaedre
GPG Key ID: 440C6592E8A26D99
5 changed files with 116 additions and 2 deletions

@ -1 +1 @@
Subproject commit 2eebf62f122631f72234438f6b5be15955021300
Subproject commit 127a74ab29afacaa90c3683fea8f556f8207d92a

@ -1 +1 @@
Subproject commit c42ba8c095d770d8d4620169587389f9d247a7e5
Subproject commit e8eb3e381305da0b97cc241e95c33a14c211a01d

View File

@ -44,6 +44,8 @@ BRANDING_DIRECTORY="$(pwd)/reports/plugins/diagnostics/runtime/themes" \
echo 'Checking stored Reportist YAML'
reporter/tools/validate-all.sh reports/reports
python3 scripts/check-regular-parity.py
python3 scripts/check-diagnostic-parity.py
pipeline_check_parent="$(mktemp -d "${TMPDIR:-/tmp}/beffroi-pipeline-check.XXXXXX")"
trap 'rm -rf -- "$pipeline_check_parent"' EXIT

View File

@ -0,0 +1,65 @@
#!/usr/bin/env python3
"""Guard reviewed finding counts from the pre-plugin diagnostic renderer."""
import json
from pathlib import Path
import subprocess
import sys
import tempfile
ROOT = Path(__file__).resolve().parents[1]
CAPTURES = ROOT / "reports/diagnostics"
COUNTS = {
"hardware-17-tA6kwqAg-windows-system-report-desktop-2osf051-20260904-134702": (2, 10, 0),
"hardware-18-WHA8oUN2-windows-system-report-pcvid008-20260904-134729": (1, 9, 1),
"hardware-19-BOOso6wz-windows-system-report-desktop-dh4hot6-20260904-173202": (2, 11, 0),
"hardware-20-dlBVl76W-windows-system-report-desktop-pr71lgi-20260904-162618": (2, 7, 0),
"hardware-35-NSmkLXTb-windows-system-report-blin2-20260904-161637": (2, 6, 0),
"hardware-51-6M9gNOPR-windows-system-report-desktop-sau019h-20260904-164514": (3, 10, 0),
"hardware-53-AbCX84Q8-windows-system-report-desktop-3pptg49-20260904-135829": (1, 7, 0),
"windows_system_report_DESKTOP-SAU019H_20260914_163753": (2, 5, 0),
"windows_system_report_DESKTOP-SAU019H_20260915_113124": (0, 7, 0),
"windows_system_report_DESKTOP-SAU019H_20260915_121057": (1, 16, 0),
}
def main():
captures = {path.stem: path for path in CAPTURES.glob("*.json")}
if set(captures) != set(COUNTS):
print("Stored diagnostic set changed; review the parity baseline", file=sys.stderr)
return 1
with tempfile.TemporaryDirectory(prefix="beffroi-diagnostic-parity.") as directory:
latest_diagnostic = None
for name, expected in COUNTS.items():
capture = captures[name]
collected = json.loads(capture.read_text(encoding="utf-8"))["generated_at"]
output = Path(directory) / f"{name}.json"
subprocess.run([str(ROOT / "reports/plugins/diagnostics/analyse.sh"), str(capture),
"--evidence-root", str(CAPTURES), "--output", str(output),
"--reference-time", collected, "--producer-version", "parity-check"],
check=True, stdout=subprocess.DEVNULL)
findings = json.loads(output.read_text(encoding="utf-8"))["findings"]
actual = tuple(sum(item["severity"] == severity for item in findings)
for severity in ("critical", "warning", "unanalysed"))
if actual != expected:
print(f"{name}: expected {expected}, got {actual}", file=sys.stderr)
return 1
if name == "windows_system_report_DESKTOP-SAU019H_20260915_121057":
latest_diagnostic = output
ir = Path(directory) / "latest.report.json"
subprocess.run([str(ROOT / "reporter/render.sh"), str(latest_diagnostic),
"--plugin", str(ROOT / "reports/plugins/diagnostics"),
"--output", str(Path(directory) / "latest.pdf"), "--emit-ir", str(ir)],
check=True, stdout=subprocess.DEVNULL)
report = json.loads(ir.read_text(encoding="utf-8"))
score = report["sections"][0]["content"][1]["value"]
if score != 62:
print(f"Latest capture: expected pre-plugin signal score 62, got {score}", file=sys.stderr)
return 1
print(f"All {len(COUNTS)} diagnostics retain pre-plugin critical/warning/unanalysed counts")
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,47 @@
#!/usr/bin/env python3
"""Guard the seven authored PDFs against the pre-plugin release's extracted text."""
import hashlib
from pathlib import Path
import shutil
import subprocess
import sys
import tempfile
ROOT = Path(__file__).resolve().parents[1]
TEXT_SHA256 = {
"2026-09-04-clevo-handoff": "6bb91e92caf442eaabacca61fbafca6b967bd4692877eee9efa5d3ff54686e5c",
"2026-09-04-clevo-initial": "2a05b6f93f8e5a5e0adf48fa2d83d3072fc404755654e8feb27eb5c592ec2ddf",
"2026-09-04-report": "e7dbaf7fa722fa18e333387f463622f7e082bd9a39dd785d5700965456a743ab",
"2026-09-14-system-stability-diagnostic-notes": "0fbe2dc9bd6044663c0bc026dc132eb69a7f8e4f8c4a9640914da9860aa9c11e",
"2026-09-15-clevo-diagnostic-synthesis": "c729ceca5c6e7481f16593af1113c94d03cee33d51af2556bd4c74a26155215a",
"2026-09-15-clevo-manufacturer-report": "01c522b83a14524bee1646c3f55c45bc0c96eec28696416054224958027d4d3d",
"2026-09-15-clevo-system-stability-diagnostic-notes": "32c88e56144aa15392e929a5d07a7f737e6397e37e65dae43d648429f8da1a7f",
}
def main():
if not shutil.which("pdftotext"):
print("pdftotext is required for regular report parity checks", file=sys.stderr)
return 2
inputs = {path.stem: path for path in (ROOT / "reports/reports").glob("*.yaml")}
if set(inputs) != set(TEXT_SHA256):
print("Authored report set changed; update the parity baseline deliberately", file=sys.stderr)
return 1
with tempfile.TemporaryDirectory(prefix="beffroi-regular-parity.") as directory:
for name, expected in TEXT_SHA256.items():
pdf = Path(directory) / f"{name}.pdf"
subprocess.run([str(ROOT / "reporter/render.sh"), str(inputs[name]), "--output", str(pdf)],
check=True, stdout=subprocess.DEVNULL)
extracted = subprocess.check_output(["pdftotext", "-layout", str(pdf), "-"])
actual = hashlib.sha256(extracted).hexdigest()
if actual != expected:
print(f"{name}: extracted text differs from pre-plugin PDF", file=sys.stderr)
return 1
print(f"All {len(TEXT_SHA256)} authored reports retain pre-plugin extracted text")
return 0
if __name__ == "__main__":
raise SystemExit(main())