beffroi-reporting/tests/check-legacy-baseline.py

140 lines
7.1 KiB
Python

#!/usr/bin/env python3
"""Reproduce the migration baseline without modifying the legacy repositories."""
import argparse
import hashlib
import json
from pathlib import Path
import shutil
import subprocess
import tempfile
ROOT = Path(__file__).resolve().parents[1]
SOURCE = ROOT / "references/typist_reporter"
REVISION = "1d381904fff3d378a7c0e04588e75310a9b7ed60"
FIXTURES = ROOT / "diagnostics/tests/fixtures/legacy"
REPORTS = ROOT / "reporter/tests/fixtures/legacy"
EXPECTED = ROOT / "diagnostics/tests/legacy-baseline.json"
# Explicit allowlist: never materialize real reports or diagnostic captures.
SOURCE_FILES = (
"LICENSE", "report.typ", "diagnostics.typ", "modules/model.typ",
"modules/diagnostics-model.typ", "modules/diagnostics-normalize.typ",
"config/themes.yaml", "config/diagnostics-rules.yaml",
"config/diagnostics-watchlist.yaml", "assets/beffroi.png",
)
BRIDGE = '''#import "modules/diagnostics-normalize.typ": normalize
#let data = normalize(json(sys.inputs.at("fixture")),
rules: yaml("config/diagnostics-rules.yaml"), source-name: "synthetic fixture")
#metadata((
schema_version: data.schema_version,
mode: data.mode,
collection_incomplete: data.collection_incomplete,
analysis_confidence: data.analysis_confidence,
health_score: data.health_score,
event_counts: data.event_counts,
issue_counts: data.issue_counts,
system: data.system,
findings: data.findings,
event_groups: data.event_groups,
coverage: data.coverage,
)) <baseline>
'''
def run(*args, cwd=None):
return subprocess.run(args, cwd=cwd, check=True, capture_output=True).stdout
def verify_semantics(results):
v1 = results["windiagnostics-v1"]
assert v1["system"]["ram_total_gb"] == 31.82
assert any(f["title"] == "Service Control Manager · Event 7000"
for f in v1["findings"])
full = results["windiagnostics-v2-full"]
assert full["event_counts"] == dict(critical=0, error=1, warning=2, information=1, unknown=0)
assert full["collection_incomplete"] and full["analysis_confidence"] <= 50
assert any(e["level"] == 3 and e["count"] == 2 for e in full["event_groups"])
compact = results["windiagnostics-v2-compact"]
assert not any(f["title"] == "Devices report problems" for f in compact["findings"])
partial = results["windiagnostics-v2-partial"]
assert any(c["section"] == "services" and c["severity"] == "warning"
for c in partial["coverage"])
watch = results["windiagnostics-watchlist"]
assert [f["watch_id"] for f in watch["findings"] if "watch_id" in f] == [
"winring0", "dell-dbutil", "gigabyte-oc-guru", "obs-overlay", "obs-audio-hooks", "smb1"]
assert all(f["weight_factor"] == 0 for f in watch["findings"] if "watch_id" in f)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output", type=Path, required=True, help="New directory for PDFs and observed JSON")
args = parser.parse_args()
for tool in ("git", "typst", "pdftotext"):
if not shutil.which(tool):
parser.error(f"{tool} is required; install it explicitly before running this check")
if args.output.exists():
parser.error("output directory already exists; choose a new directory")
output = args.output.resolve()
output.mkdir(parents=True)
results = {}
manifest = {"source_revision": REVISION, "typst": run("typst", "--version").decode().strip(),
"fixture_sha256": {}, "pdfs": []}
with tempfile.TemporaryDirectory(prefix="beffroi-legacy-baseline-") as directory:
work = Path(directory)
for name in SOURCE_FILES:
target = work / name
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(run("git", "-C", str(SOURCE), "show", f"{REVISION}:{name}"))
(work / "baseline.typ").write_text(BRIDGE)
for fixture in sorted(FIXTURES.glob("*.json")):
shutil.copyfile(fixture, work / fixture.name)
manifest["fixture_sha256"][fixture.name] = hashlib.sha256(fixture.read_bytes()).hexdigest()
raw = run("typst", "query", "--root", directory, "--input", f"fixture={fixture.name}",
"baseline.typ", "<baseline>", "--field", "value", "--one", cwd=work)
results[fixture.stem] = json.loads(raw)
pdf = output / f"{fixture.stem}.pdf"
run("typst", "compile", "--root", directory, "--input", f"diagnostics={fixture.name}",
"--input", "author=Synthetic baseline", "diagnostics.typ", str(pdf), cwd=work)
manifest["pdfs"].append(pdf.name)
verify_semantics(results)
for fixture in sorted(REPORTS.glob("*.yaml")):
shutil.copyfile(fixture, work / fixture.name)
manifest["fixture_sha256"][fixture.name] = hashlib.sha256(fixture.read_bytes()).hexdigest()
pdf = output / f"report-{fixture.stem}.pdf"
run("typst", "compile", "--root", directory, "--input", f"report={fixture.name}",
"report.typ", str(pdf), cwd=work)
manifest["pdfs"].append(pdf.name)
for name, content, message in (
("missing-author", 'metadata:\n Date: "2026-09-15"\n', "metadata.Author"),
("missing-date", 'metadata:\n Author: "Synthetic"\n', "metadata.Date"),
("unknown-theme", 'theme: unknown\nmetadata:\n Author: "Synthetic"\n Date: "2026-09-15"\n', "Unknown report theme"),
):
(work / "invalid.yaml").write_text(content)
result = subprocess.run(["typst", "compile", "--input", "report=invalid.yaml",
"report.typ", str(work / "invalid.pdf")], cwd=work, capture_output=True)
assert result.returncode != 0 and message in result.stderr.decode(), name
manifest["negative_checks"] = ["missing-author", "missing-date", "unknown-theme"]
for name in manifest["pdfs"]:
assert (output / name).read_bytes().startswith(b"%PDF-"), name
text = run("pdftotext", str(output / name), "-").decode()
expected_title = {"report-minimal.pdf": "Short report title",
"report-technical.pdf": "Service Interruption Investigation"}.get(
name, "Windows diagnostic assessment")
assert expected_title in " ".join(text.split()), f"Missing rendered title: {name}"
manifest["fonts"] = [font for font in run("typst", "fonts").decode().splitlines()
if "DejaVu" in font]
(output / "diagnostics.json").write_text(json.dumps(results, indent=2, ensure_ascii=False) + "\n")
(output / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")
if not EXPECTED.exists():
parser.exit(1, f"Observed baseline written to {output}; reviewed expected snapshot is missing\n")
assert results == json.loads(EXPECTED.read_text()), "Diagnostic baseline differs; inspect diagnostics.json"
print(f"PASS: 5 diagnostic snapshots, 7 PDFs, 3 invalid inputs; artifacts: {output}")
if __name__ == "__main__":
try:
main()
except subprocess.CalledProcessError as error:
raise SystemExit(error.stderr.decode(errors="replace")) from error