beffroi-reporting/scripts/check-diagnostic-parity.py

74 lines
3.8 KiB
Python

#!/usr/bin/env python3
"""Guard reviewed finding counts from the pre-plugin diagnostic renderer."""
import json
from pathlib import Path
import subprocess
import sys
import tempfile
ROOT = Path(__file__).resolve().parents[1]
CAPTURES = ROOT / "reports/diagnostics"
COUNTS = {
"hardware-17-tA6kwqAg-windows-system-report-desktop-2osf051-20260904-134702": (2, 10, 0),
"hardware-18-WHA8oUN2-windows-system-report-pcvid008-20260904-134729": (1, 9, 1),
"hardware-19-BOOso6wz-windows-system-report-desktop-dh4hot6-20260904-173202": (2, 11, 0),
"hardware-20-dlBVl76W-windows-system-report-desktop-pr71lgi-20260904-162618": (2, 7, 0),
"hardware-35-NSmkLXTb-windows-system-report-blin2-20260904-161637": (2, 6, 0),
"hardware-51-6M9gNOPR-windows-system-report-desktop-sau019h-20260904-164514": (3, 10, 0),
"hardware-53-AbCX84Q8-windows-system-report-desktop-3pptg49-20260904-135829": (1, 7, 0),
"windows_system_report_DESKTOP-SAU019H_20260914_163753": (2, 5, 0),
"windows_system_report_DESKTOP-SAU019H_20260915_113124": (0, 7, 0),
"windows_system_report_DESKTOP-SAU019H_20260915_121057": (1, 16, 0),
}
def main():
captures = {path.stem: path for path in CAPTURES.glob("*.json")}
if set(captures) != set(COUNTS):
print("Stored diagnostic set changed; review the parity baseline", file=sys.stderr)
return 1
with tempfile.TemporaryDirectory(prefix="beffroi-diagnostic-parity.") as directory:
latest_diagnostic = None
for name, expected in COUNTS.items():
capture = captures[name]
collected = json.loads(capture.read_text(encoding="utf-8"))["generated_at"]
output = Path(directory) / f"{name}.json"
subprocess.run([str(ROOT / "reports/plugins/diagnostics/analyse.sh"), str(capture),
"--evidence-root", str(CAPTURES), "--output", str(output),
"--reference-time", collected, "--producer-version", "parity-check"],
check=True, stdout=subprocess.DEVNULL)
findings = json.loads(output.read_text(encoding="utf-8"))["findings"]
actual = tuple(sum(item["severity"] == severity for item in findings)
for severity in ("critical", "warning", "unanalysed"))
if actual != expected:
print(f"{name}: expected {expected}, got {actual}", file=sys.stderr)
return 1
if name == "windows_system_report_DESKTOP-SAU019H_20260915_121057":
latest_diagnostic = output
ir = Path(directory) / "latest.report.json"
subprocess.run([str(ROOT / "reporter/render.sh"), str(latest_diagnostic),
"--plugin", str(ROOT / "reports/plugins/diagnostics"),
"--output", str(Path(directory) / "latest.pdf"), "--emit-ir", str(ir)],
check=True, stdout=subprocess.DEVNULL)
report = json.loads(ir.read_text(encoding="utf-8"))
assessment = report["sections"][0]["content"][0]
if assessment["title"] != "Diagnostic assessment" or "leading finding" not in assessment["text"]:
print("Latest capture: diagnostic assessment is missing", file=sys.stderr)
return 1
if not report["sections"][0]["content"][3]["content"][0]["items"][0]["title"].startswith(
"Microsoft-Windows-WER-SystemErrorReporting · Event 1001"):
print("Latest capture: reliability event identity was lost", file=sys.stderr)
return 1
score = report["sections"][0]["content"][1]["value"]
if score != 62:
print(f"Latest capture: expected pre-plugin signal score 62, got {score}", file=sys.stderr)
return 1
print(f"All {len(COUNTS)} diagnostics retain pre-plugin critical/warning/unanalysed counts")
return 0
if __name__ == "__main__":
raise SystemExit(main())