Guard authored and diagnostic output parity
This commit is contained in:
parent
22a6d477c5
commit
02af5e9a55
2
reporter
2
reporter
|
|
@ -1 +1 @@
|
|||
Subproject commit 2eebf62f122631f72234438f6b5be15955021300
|
||||
Subproject commit 127a74ab29afacaa90c3683fea8f556f8207d92a
|
||||
2
reports
2
reports
|
|
@ -1 +1 @@
|
|||
Subproject commit c42ba8c095d770d8d4620169587389f9d247a7e5
|
||||
Subproject commit e8eb3e381305da0b97cc241e95c33a14c211a01d
|
||||
|
|
@ -44,6 +44,8 @@ BRANDING_DIRECTORY="$(pwd)/reports/plugins/diagnostics/runtime/themes" \
|
|||
|
||||
echo 'Checking stored Reportist YAML'
|
||||
reporter/tools/validate-all.sh reports/reports
|
||||
python3 scripts/check-regular-parity.py
|
||||
python3 scripts/check-diagnostic-parity.py
|
||||
|
||||
pipeline_check_parent="$(mktemp -d "${TMPDIR:-/tmp}/beffroi-pipeline-check.XXXXXX")"
|
||||
trap 'rm -rf -- "$pipeline_check_parent"' EXIT
|
||||
|
|
|
|||
|
|
@ -0,0 +1,65 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Guard reviewed finding counts from the pre-plugin diagnostic renderer."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CAPTURES = ROOT / "reports/diagnostics"
|
||||
COUNTS = {
|
||||
"hardware-17-tA6kwqAg-windows-system-report-desktop-2osf051-20260904-134702": (2, 10, 0),
|
||||
"hardware-18-WHA8oUN2-windows-system-report-pcvid008-20260904-134729": (1, 9, 1),
|
||||
"hardware-19-BOOso6wz-windows-system-report-desktop-dh4hot6-20260904-173202": (2, 11, 0),
|
||||
"hardware-20-dlBVl76W-windows-system-report-desktop-pr71lgi-20260904-162618": (2, 7, 0),
|
||||
"hardware-35-NSmkLXTb-windows-system-report-blin2-20260904-161637": (2, 6, 0),
|
||||
"hardware-51-6M9gNOPR-windows-system-report-desktop-sau019h-20260904-164514": (3, 10, 0),
|
||||
"hardware-53-AbCX84Q8-windows-system-report-desktop-3pptg49-20260904-135829": (1, 7, 0),
|
||||
"windows_system_report_DESKTOP-SAU019H_20260914_163753": (2, 5, 0),
|
||||
"windows_system_report_DESKTOP-SAU019H_20260915_113124": (0, 7, 0),
|
||||
"windows_system_report_DESKTOP-SAU019H_20260915_121057": (1, 16, 0),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
captures = {path.stem: path for path in CAPTURES.glob("*.json")}
|
||||
if set(captures) != set(COUNTS):
|
||||
print("Stored diagnostic set changed; review the parity baseline", file=sys.stderr)
|
||||
return 1
|
||||
with tempfile.TemporaryDirectory(prefix="beffroi-diagnostic-parity.") as directory:
|
||||
latest_diagnostic = None
|
||||
for name, expected in COUNTS.items():
|
||||
capture = captures[name]
|
||||
collected = json.loads(capture.read_text(encoding="utf-8"))["generated_at"]
|
||||
output = Path(directory) / f"{name}.json"
|
||||
subprocess.run([str(ROOT / "reports/plugins/diagnostics/analyse.sh"), str(capture),
|
||||
"--evidence-root", str(CAPTURES), "--output", str(output),
|
||||
"--reference-time", collected, "--producer-version", "parity-check"],
|
||||
check=True, stdout=subprocess.DEVNULL)
|
||||
findings = json.loads(output.read_text(encoding="utf-8"))["findings"]
|
||||
actual = tuple(sum(item["severity"] == severity for item in findings)
|
||||
for severity in ("critical", "warning", "unanalysed"))
|
||||
if actual != expected:
|
||||
print(f"{name}: expected {expected}, got {actual}", file=sys.stderr)
|
||||
return 1
|
||||
if name == "windows_system_report_DESKTOP-SAU019H_20260915_121057":
|
||||
latest_diagnostic = output
|
||||
ir = Path(directory) / "latest.report.json"
|
||||
subprocess.run([str(ROOT / "reporter/render.sh"), str(latest_diagnostic),
|
||||
"--plugin", str(ROOT / "reports/plugins/diagnostics"),
|
||||
"--output", str(Path(directory) / "latest.pdf"), "--emit-ir", str(ir)],
|
||||
check=True, stdout=subprocess.DEVNULL)
|
||||
report = json.loads(ir.read_text(encoding="utf-8"))
|
||||
score = report["sections"][0]["content"][1]["value"]
|
||||
if score != 62:
|
||||
print(f"Latest capture: expected pre-plugin signal score 62, got {score}", file=sys.stderr)
|
||||
return 1
|
||||
print(f"All {len(COUNTS)} diagnostics retain pre-plugin critical/warning/unanalysed counts")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Guard the seven authored PDFs against the pre-plugin release's extracted text."""
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
TEXT_SHA256 = {
|
||||
"2026-09-04-clevo-handoff": "6bb91e92caf442eaabacca61fbafca6b967bd4692877eee9efa5d3ff54686e5c",
|
||||
"2026-09-04-clevo-initial": "2a05b6f93f8e5a5e0adf48fa2d83d3072fc404755654e8feb27eb5c592ec2ddf",
|
||||
"2026-09-04-report": "e7dbaf7fa722fa18e333387f463622f7e082bd9a39dd785d5700965456a743ab",
|
||||
"2026-09-14-system-stability-diagnostic-notes": "0fbe2dc9bd6044663c0bc026dc132eb69a7f8e4f8c4a9640914da9860aa9c11e",
|
||||
"2026-09-15-clevo-diagnostic-synthesis": "c729ceca5c6e7481f16593af1113c94d03cee33d51af2556bd4c74a26155215a",
|
||||
"2026-09-15-clevo-manufacturer-report": "01c522b83a14524bee1646c3f55c45bc0c96eec28696416054224958027d4d3d",
|
||||
"2026-09-15-clevo-system-stability-diagnostic-notes": "32c88e56144aa15392e929a5d07a7f737e6397e37e65dae43d648429f8da1a7f",
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
if not shutil.which("pdftotext"):
|
||||
print("pdftotext is required for regular report parity checks", file=sys.stderr)
|
||||
return 2
|
||||
inputs = {path.stem: path for path in (ROOT / "reports/reports").glob("*.yaml")}
|
||||
if set(inputs) != set(TEXT_SHA256):
|
||||
print("Authored report set changed; update the parity baseline deliberately", file=sys.stderr)
|
||||
return 1
|
||||
with tempfile.TemporaryDirectory(prefix="beffroi-regular-parity.") as directory:
|
||||
for name, expected in TEXT_SHA256.items():
|
||||
pdf = Path(directory) / f"{name}.pdf"
|
||||
subprocess.run([str(ROOT / "reporter/render.sh"), str(inputs[name]), "--output", str(pdf)],
|
||||
check=True, stdout=subprocess.DEVNULL)
|
||||
extracted = subprocess.check_output(["pdftotext", "-layout", str(pdf), "-"])
|
||||
actual = hashlib.sha256(extracted).hexdigest()
|
||||
if actual != expected:
|
||||
print(f"{name}: extracted text differs from pre-plugin PDF", file=sys.stderr)
|
||||
return 1
|
||||
print(f"All {len(TEXT_SHA256)} authored reports retain pre-plugin extracted text")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Loading…
Reference in New Issue