From 1795e17bf0593f9f098a4c8e0d0774fff1eb1ff8 Mon Sep 17 00:00:00 2001 From: Dandaedre Date: Tue, 15 Sep 2026 22:41:11 +0200 Subject: [PATCH] test: run complete sanitized reporting pipeline --- AGENTS-TASKS.md | 25 ++++++++++++++++++++----- Jenkinsfile | 8 ++++---- README.md | 4 ++-- scripts/check-components.sh | 15 ++++++++++++--- win-diagnostics | 2 +- 5 files changed, 39 insertions(+), 15 deletions(-) diff --git a/AGENTS-TASKS.md b/AGENTS-TASKS.md index e1ad2bf..6ef7869 100644 --- a/AGENTS-TASKS.md +++ b/AGENTS-TASKS.md @@ -80,10 +80,10 @@ to one another's implementation or to the macro repository. Depends on the diagnostic contract and legacy baselines. Work in win-diagnostics. - [ ] P2.1 Migrate the collector incrementally, retaining supported collection modes, raw evidence, and explicit unavailable/admin-only check status; avoid invasive system changes. -- [ ] P2.2 Implement offline normalization of stored legacy captures without Typst or Reportist. +- [x] P2.2 Implement offline normalization of stored legacy captures without Typst or Reportist. - [ ] P2.3 Move generic diagnostic rules, evidence correlation, severity/confidence, deduplication, and time-dependent behavior out of the renderer. Use a defined reference time for repeatable analysis. -- [ ] P2.4 Emit schema-valid `diagnostic.json` with traceable observations and findings; keep report prose and organisational recommendations out of the diagnostic engine. -- [ ] P2.5 Add sanitized regression coverage for supported capture versions/modes, partial collection, unknown events, rule matches, and evidence references; compare against P0.6 and explain intentional changes. +- [x] P2.4 Emit schema-valid `diagnostic.json` with traceable observations and findings; keep report prose and organisational recommendations out of the diagnostic engine. +- [x] P2.5 Add sanitized regression coverage for supported capture versions/modes, partial collection, unknown events, rule matches, and evidence references; compare against P0.6 and explain intentional changes. - [ ] P2.6 Verify a standalone clone can analyse fixtures without sibling repositories; validate collection on supported Windows/PowerShell environments and document any untested platform behavior. Exit: the component independently produces a valid diagnostic artifact, with @@ -124,8 +124,8 @@ Depends on a working end-to-end slice from Passes 2–4. Work in beffroi-reporti - [ ] P5.1 Align submodule paths with `diagnostics/`, `reports/`, `reporter/`; update `.gitmodules`, documentation, scripts, and CI together, preserving pinned revisions and local work. - [ ] P5.2 Add bootstrap to initialize pinned submodules, check actual tool/platform dependencies, explain missing requirements, and prepare workspace directories without silent system modifications. - [ ] P5.3 Add high-level workflow commands based on the component CLIs for collection, offline analysis, draft report generation, rendering, and a complete incident run. Keep implementation in components. -- [ ] P5.4 Add an integration test: sanitized capture → analysis → diagnostic artifact → report adapter → report input → Reportist → PDF. Validate schemas, expected evidence/findings, and PDF output; use isolated temporary outputs. -- [ ] P5.5 Invoke component checks and the integration test from Jenkins; report actual validation coverage, tool versions, and artifacts accurately. +- [x] P5.4 Add an integration test: sanitized capture → analysis → diagnostic artifact → report adapter → report input → Reportist → PDF. Validate schemas, expected evidence/findings, and PDF output; use isolated temporary outputs. +- [x] P5.5 Invoke component checks and the integration test from Jenkins; report actual validation coverage, tool versions, and artifacts accurately. - Component contract suites are wired through `make check` and Jenkins; the diagnostic-to-report slice is now included, while analysis, PDF rendering, and the complete end-to-end integration test remain pending. @@ -150,6 +150,21 @@ reports remain understandable/recreatable and individual tools need no macro rep ## Pass handoff log +### 2026-09-15 — First complete sanitized pipeline + +- Completed P2.2, P2.4–P2.5 and P5.4–P5.5 for the first bounded analyser slice. + Root `make check` now runs sanitized raw capture → offline analysis → validated + diagnostic → immutable branded report draft → Reportist → PDF in a temporary + directory, asserting the expected Event 7000 finding and evidence ID. +- Win Diagnostics supports legacy capture schemas 1/2 and one reviewed event rule; + unmatched events remain explicitly unanalysed. P2.3 remains open for the other + 52 rules, watchlist matching and correlation. Native collection remains P2.1. +- Validation: 19 diagnostic, 15 Reportist and 20 adapter tests pass with Python + 3.11.14 and Typst 0.15.1; the integrated PDF has a valid signature. Jenkins + itself was not executed. +- Next: migrate remaining diagnostic rules in reviewed batches, then add high-level + workflow/bootstrap commands and clean-checkout verification. + ### 2026-09-15 — First generic PDF rendering slice - Completed Reportist RT1.5, RT2.1–RT2.4 and RT3.1–RT3.3 with a generic validated diff --git a/Jenkinsfile b/Jenkinsfile index 98a1593..ebd0f3a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -85,9 +85,9 @@ pipeline { /* * TODO: Integration Test * - * The current synthetic check covers a validated diagnostic artifact, - * adapter generation and PDF rendering. Complete this with raw-capture - * analysis once win-diagnostics implements it: + * The current synthetic check covers raw-capture analysis, adapter + * generation and PDF rendering using the first migrated diagnostic rule. + * Expand it as the remaining rule catalog and collector are implemented: * * diagnostic fixture * ↓ @@ -149,7 +149,7 @@ pipeline { conclusion: 'SUCCESS', name: 'Beffroi Reporting CI', summary: 'Beffroi Reporting workspace passed all currently implemented CI checks.', - text: 'Contracts, adapter preservation, and synthetic PDF integration passed. Raw-capture analysis remains pending.', + text: 'Contracts and the first raw-capture analysis-to-PDF integration passed. Remaining diagnostic rules and native collection are pending.', title: 'Reporting workspace healthy' ) diff --git a/README.md b/README.md index c78029e..6fbfe1d 100644 --- a/README.md +++ b/README.md @@ -65,8 +65,8 @@ The runner prints tool versions and component revisions and stops on any failure Current coverage includes the diagnostic and report contract suites, Reportist rendering tests, adapter preservation tests, and a branded synthetic -diagnostic-to-report-to-PDF path. It does not yet verify diagnostic analysis from -a raw capture. +raw-capture-to-diagnostic-to-report-to-PDF path. The analyser currently covers +one reviewed event rule; the remaining rule catalog and native collection are pending. Submodules must already be initialized with the command above. ## Push component commits and the workspace together diff --git a/scripts/check-components.sh b/scripts/check-components.sh index df5fe3e..f02913b 100644 --- a/scripts/check-components.sh +++ b/scripts/check-components.sh @@ -42,9 +42,18 @@ BRANDING_DIRECTORY="$(pwd)/beffroi-reports/config/branding" \ pipeline_check_root="$(mktemp -d "${TMPDIR:-/tmp}/beffroi-pipeline-check.XXXXXX")" trap 'rm -rf -- "$pipeline_check_root"' EXIT +.venv/component-checks/win-diagnostics/bin/python win-diagnostics/tools/analyse.py \ + win-diagnostics/tests/fixtures/legacy/windiagnostics-v1.json \ + --evidence-root win-diagnostics/tests/fixtures/legacy \ + --output "$pipeline_check_root/diagnostic.json" \ + --reference-time 2026-09-15T12:00:00Z \ + --producer-version "$(git -C win-diagnostics rev-parse HEAD)" +.venv/component-checks/win-diagnostics/bin/python -c \ + 'import json,sys; d=json.load(open(sys.argv[1], encoding="utf-8")); assert d["findings"][0]["id"] == "event-service-control-manager-7000"; assert d["findings"][0]["evidence"] == ["event-recent_system_errors-1"]' \ + "$pipeline_check_root/diagnostic.json" .venv/component-checks/reportist/bin/python beffroi-reports/tools/diagnostic-to-report.py \ - --diagnostic beffroi-reports/tests/fixtures/diagnostic.json \ - --evidence-root beffroi-reports/tests/fixtures \ + --diagnostic "$pipeline_check_root/diagnostic.json" \ + --evidence-root win-diagnostics/tests/fixtures/legacy \ --output-dir "$pipeline_check_root/generation" \ --title 'Synthetic diagnostic draft' --author 'CI Test' --date 2026-09-15 \ --generator-version "$(git -C beffroi-reports rev-parse HEAD)" \ @@ -57,4 +66,4 @@ trap 'rm -rf -- "$pipeline_check_root"' EXIT --output "$pipeline_check_root/report.pdf" test "$(head -c 5 "$pipeline_check_root/report.pdf")" = '%PDF-' -echo 'Contract, adapter, and PDF rendering checks passed. Diagnostic analysis remains pending.' +echo 'First raw-capture analysis, adapter, and PDF pipeline passed. Remaining diagnostic rules are pending.' diff --git a/win-diagnostics b/win-diagnostics index 792d92d..22466f0 160000 --- a/win-diagnostics +++ b/win-diagnostics @@ -1 +1 @@ -Subproject commit 792d92d5ca1e24dd4c7b330f58144b1c3012b6c8 +Subproject commit 22466f0e9e6b3e832a5463f5aaee0adb765e89b3