From 22466f0e9e6b3e832a5463f5aaee0adb765e89b3 Mon Sep 17 00:00:00 2001 From: Dandaedre Date: Tue, 15 Sep 2026 22:40:12 +0200 Subject: [PATCH] feat: analyse stored legacy captures --- AGENTS-TASKS.md | 32 ++++-- Jenkinsfile | 2 +- README.md | 17 ++- docs/diagnostic-contract.md | 23 +++- tests/test_analyse.py | 124 ++++++++++++++++++++ tools/analyse.py | 219 ++++++++++++++++++++++++++++++++++++ 6 files changed, 406 insertions(+), 11 deletions(-) create mode 100644 tests/test_analyse.py create mode 100644 tools/analyse.py diff --git a/AGENTS-TASKS.md b/AGENTS-TASKS.md index a4844d1..c5fad24 100644 --- a/AGENTS-TASKS.md +++ b/AGENTS-TASKS.md @@ -41,7 +41,7 @@ Depends on Pass 0 for the selected slice. - [x] WD1.1 Define `schemas/diagnostic.schema.json` with explicit version, metadata, system, collection status, observations, findings, and warnings; adapt fields to actual evidence. - [x] WD1.2 Define stable finding IDs, severity/confidence semantics, evidence references, and how raw evidence is retained. Missing, skipped, and failed checks must not imply health. - [x] WD1.3 Document mapping from legacy capture schemas 1 and 2, including unsupported versions and information that cannot be recovered. -- [ ] WD1.4 Define independent collection/offline-analysis CLI contracts: input/output paths, errors, exit codes, dependencies, and compatibility policy. +- [x] WD1.4 Define independent collection/offline-analysis CLI contracts: input/output paths, errors, exit codes, dependencies, and compatibility policy. - [x] WD1.5 Add synthetic valid/invalid schema examples and validation tests; document the interface for downstream report adapters without importing their implementation. Exit: a consumer can understand and validate diagnostic output using a documented @@ -53,8 +53,8 @@ Depends on Pass 1. - [ ] WD2.1 Migrate collection incrementally, preserving Compact/Mid/Full behavior and explicit administrator-only/unavailable checks; do not silently modify the system. - [ ] WD2.2 Separate structured evidence from console presentation; retain raw records where practical and avoid machine-specific absolute paths. -- [ ] WD2.3 Implement offline normalization of stored captures without Typst, live Windows access, or sibling repositories. -- [ ] WD2.4 Test supported capture versions/modes, malformed input, missing sections, partial collection, and preservation of evidence references against the baseline. +- [x] WD2.3 Implement offline normalization of stored captures without Typst, live Windows access, or sibling repositories. +- [x] WD2.4 Test supported capture versions/modes, malformed input, missing sections, partial collection, and preservation of evidence references against the baseline. Exit: supported evidence can be collected and stored captures normalized through the public interface, with collection limitations documented. @@ -65,9 +65,9 @@ Depends on normalized evidence and the diagnostic contract. - [ ] WD3.1 Port reusable rules and watchlist matching with traceable evidence; keep investigation-specific conclusions and organisational recommendations outside this engine. - [ ] WD3.2 Preserve and test event grouping, deduplication, correlation, unknown-event behavior, collection completeness, severity, and confidence. -- [ ] WD3.3 Make time-dependent analysis repeatable using a defined reference time; document intentional differences from legacy behavior. -- [ ] WD3.4 Emit schema-valid `diagnostic.json`; verify every finding's evidence reference resolves and raw observations remain available. -- [ ] WD3.5 Run sanitized regression fixtures for known diagnostic situations without the original machine; record exact commands and results. +- [x] WD3.3 Make time-dependent analysis repeatable using a defined reference time; document intentional differences from legacy behavior. +- [x] WD3.4 Emit schema-valid `diagnostic.json`; verify every finding's evidence reference resolves and raw observations remain available. +- [x] WD3.5 Run sanitized regression fixtures for known diagnostic situations without the original machine; record exact commands and results. Exit: offline analysis produces deterministic, evidence-backed diagnostic output without any rendering or report-content dependency. @@ -76,7 +76,7 @@ without any rendering or report-content dependency. Depends on the working diagnostic slice. -- [ ] WD4.1 Correct Jenkins' instruction-file check; enable schema, fixture, and analysis checks as they become available, with accurate coverage reporting. +- [x] WD4.1 Correct Jenkins' instruction-file check; enable schema, fixture, and analysis checks as they become available, with accurate coverage reporting. - [ ] WD4.2 Validate native collection on supported Windows/PowerShell environments; clearly document anything not tested by the non-Windows Jenkins agent. - [ ] WD4.3 Document installation, collection, offline analysis, output schema, privacy, dependencies, and troubleshooting from a fresh standalone clone. - [ ] WD4.4 Verify the standalone clone needs neither Reportist, beffroi-reports, the macro repository, nor legacy sources; audit fixtures for identifying data. @@ -87,6 +87,24 @@ valid diagnostic result independently of reporting tools. ## Handoff log +### 2026-09-15 — First offline analyser slice + +- Completed WD1.4, WD2.3–WD2.4, WD3.3–WD3.5 and WD4.1 for the bounded first + slice. `tools/analyse.py` accepts legacy schemas 1/2, normalizes section status, + preserves raw event values/pointers, groups duplicate events and emits validated + diagnostic v1 JSON with explicit reference time and source hash. +- Migrated the reviewed Service Control Manager Event 7000 rule. Unmatched events + are retained as `unanalysed` with zero confidence; partial collection caps known + rule confidence at 50. The other 52 rules and watchlist matching remain WD3.1. +- Added nine analysis tests across all five sanitized fixtures, deterministic + output, evidence links, malformed/missing input, incomplete/unsupported states, + and outside-CWD CLI use. +- Validation: 19 component tests pass on Python 3.11.14. The integration workspace + runs raw fixture → analyser → adapter → Reportist → PDF. Native Windows collection + and full legacy-rule parity remain untested. +- Next: migrate remaining generic rules in reviewed groups and implement watchlist + matching/correlation before native collector work. + ### 2026-09-15 — Planning pass - Completed WD0.1–WD0.2; added roadmap and corrected README navigation. diff --git a/Jenkinsfile b/Jenkinsfile index 62e34cf..ae6d7c6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -91,7 +91,7 @@ pipeline { conclusion: 'SUCCESS', name: 'Win Diagnostics CI', summary: 'Win Diagnostics passed all currently implemented CI checks.', - text: 'Repository and diagnostic contract tests passed. Collector and analysis-engine execution remain pending.', + text: 'Repository, diagnostic contract, and first offline-analysis fixture tests passed. Native collection and remaining rules remain pending.', title: 'Win Diagnostics CI passed' ) diff --git a/README.md b/README.md index fcf557e..f59a5f3 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,22 @@ acceptance criteria, and the next-pass handoff. The version-1 schema, standalone validator, and synthetic examples are available. See [contract documentation](docs/diagnostic-contract.md) for setup and validation commands. -Collection/analysis and PDF rendering are still separate migration tasks. +The first offline analysis slice supports stored legacy capture schemas 1 and 2: + +```bash +python tools/analyse.py /path/to/capture.json \ + --evidence-root /path/to \ + --output /path/to/diagnostic.json \ + --reference-time 2026-09-15T12:00:00Z \ + --producer-version "$(git rev-parse HEAD)" +``` + +It normalizes collection coverage, preserves event values and JSON Pointers, +implements the reviewed Service Control Manager Event 7000 rule, and leaves all +other events explicitly `unanalysed`. Reference time and producer version are +required for reproducibility. Existing output requires `--force`. Exit codes are +0 for validated output, 1 for input, validation, or I/O failure, and 2 for invalid +CLI usage. Remaining rules and native Windows collection are still migration work. ## License diff --git a/docs/diagnostic-contract.md b/docs/diagnostic-contract.md index e2f32cb..9629689 100644 --- a/docs/diagnostic-contract.md +++ b/docs/diagnostic-contract.md @@ -82,7 +82,25 @@ structure and internal relationships only; it does not prove raw evidence exists With it, the validator checks the capture hash/version, resolves every observation pointer, compares raw values, and rejects paths whose symlinks escape the root. -## Mapping legacy captures (contract, not yet an implemented converter) +## Offline analysis command + +The first converter/analyser slice is available as: + +```bash +python tools/analyse.py capture.json \ + --evidence-root . \ + --output diagnostic.json \ + --reference-time 2026-09-15T12:00:00Z \ + --producer-version VERSION +``` + +Input must be inside the explicit evidence root so output retains a portable +source path. Existing output requires `--force`. Exit codes are 0 for a validated +artifact, 1 for input, validation, or I/O failure, and 2 for invalid invocation. +The current rule catalog contains only Service Control Manager Event 7000; other +captured events remain traceable `unanalysed` findings. + +## Mapping legacy captures | Legacy input | Version-1 artifact mapping | | --- | --- | @@ -106,7 +124,8 @@ remain migration comparisons, not new-schema examples. Consumers select by explicit artifact version and reject unsupported versions. Breaking field or semantic changes require a new version and documented migration. Do not relabel raw captures or silently reshape historical artifacts. Contract v1 -is the initial implementation; no converter or production producer is claimed yet. +is the initial implementation. The offline analyser is a bounded migration slice, +not yet the complete legacy rule/watchlist engine or collector. Schema and format behavior follow [JSON Schema Draft 2020-12](https://json-schema.org/draft/2020-12/json-schema-validation) and the [jsonschema validator documentation](https://python-jsonschema.readthedocs.io/en/stable/validate/). diff --git a/tests/test_analyse.py b/tests/test_analyse.py new file mode 100644 index 0000000..169690c --- /dev/null +++ b/tests/test_analyse.py @@ -0,0 +1,124 @@ +import copy +import importlib.util +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +FIXTURES = ROOT / "tests" / "fixtures" / "legacy" +spec = importlib.util.spec_from_file_location("diagnostic_analyse", ROOT / "tools" / "analyse.py") +analyser = importlib.util.module_from_spec(spec) +spec.loader.exec_module(analyser) + + +class AnalyseTests(unittest.TestCase): + reference_time = "2026-09-15T12:00:00Z" + + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.work = Path(self.temporary.name) + + def tearDown(self): + self.temporary.cleanup() + + def run_fixture(self, name, output_name=None): + return analyser.analyse_file( + FIXTURES / name, FIXTURES, self.work / (output_name or name), + self.reference_time, "test-revision") + + def test_all_sanitized_legacy_fixtures_produce_valid_artifacts(self): + expected = { + "windiagnostics-v1.json": ("complete", 1, 1), + "windiagnostics-v2-compact.json": ("partial", 0, 0), + "windiagnostics-v2-full.json": ("partial", 3, 1), + "windiagnostics-v2-partial.json": ("partial", 3, 1), + "windiagnostics-watchlist.json": ("partial", 3, 1), + } + for name, values in expected.items(): + with self.subTest(name=name): + result = self.run_fixture(name) + self.assertEqual((result["collection"]["status"], len(result["observations"]), + len(result["findings"])), values) + + def test_service_event_rule_is_traceable(self): + result = self.run_fixture("windiagnostics-v1.json") + finding = result["findings"][0] + observation = result["observations"][0] + self.assertEqual((finding["severity"], finding["confidence"]), ("warning", 90)) + self.assertEqual(finding["evidence"], [observation["id"]]) + self.assertEqual(observation["source_pointer"], "/sections/recent_system_errors/data/0") + self.assertEqual(observation["value"]["Id"], 7000) + + def test_unknown_duplicate_events_are_grouped_without_interpretation(self): + result = self.run_fixture("windiagnostics-v2-full.json") + self.assertEqual(len(result["findings"]), 1) + self.assertEqual(result["findings"][0]["severity"], "unanalysed") + self.assertEqual(result["findings"][0]["confidence"], 0) + self.assertEqual(len(result["findings"][0]["evidence"]), 3) + + def test_partial_collection_caps_known_rule_confidence(self): + capture = analyser.validation.load_json(FIXTURES / "windiagnostics-v1.json") + capture = copy.deepcopy(capture) + capture["sections"]["storage"] = {"status": "unavailable", "count": 0, + "data": [], "error": "Synthetic failure"} + result = analyser.analyse(capture, "capture.json", "0" * 64, + self.reference_time, "test-revision") + self.assertEqual(result["collection"]["status"], "partial") + self.assertEqual(result["findings"][0]["confidence"], 50) + + def test_unsupported_status_is_preserved_as_limitation(self): + result = self.run_fixture("windiagnostics-v2-partial.json") + unexpected = [section for section in result["collection"]["sections"].values() + if section.get("reason", "").startswith("Legacy capture used unsupported")] + self.assertTrue(unexpected) + self.assertTrue(any("incomplete" in warning for warning in result["warnings"])) + + def test_unsupported_version_and_bad_reference_time_fail(self): + capture = analyser.validation.load_json(FIXTURES / "windiagnostics-v1.json") + invalid = copy.deepcopy(capture) + invalid["schema_version"] = 3 + with self.assertRaisesRegex(ValueError, "unsupported"): + analyser.analyse(invalid, "capture.json", "0" * 64, + self.reference_time, "test-revision") + with self.assertRaisesRegex(ValueError, "reference-time"): + analyser.analyse(capture, "capture.json", "0" * 64, + "2026-09-15", "test-revision") + + def test_missing_sections_and_malformed_json_fail(self): + capture = analyser.validation.load_json(FIXTURES / "windiagnostics-v1.json") + del capture["sections"] + with self.assertRaisesRegex(ValueError, "/sections"): + analyser.analyse(capture, "capture.json", "0" * 64, + self.reference_time, "test-revision") + malformed = self.work / "malformed.json" + malformed.write_text('{"schema_version":', encoding="utf-8") + with self.assertRaises(ValueError): + analyser.analyse_file(malformed, self.work, self.work / "output.json", + self.reference_time, "test-revision") + + def test_output_is_deterministic_and_not_overwritten_implicitly(self): + first = self.run_fixture("windiagnostics-v1.json", "first.json") + second = self.run_fixture("windiagnostics-v1.json", "second.json") + self.assertEqual(first, second) + self.assertEqual((self.work / "first.json").read_bytes(), + (self.work / "second.json").read_bytes()) + with self.assertRaisesRegex(ValueError, "output exists"): + self.run_fixture("windiagnostics-v1.json", "first.json") + + def test_cli_runs_outside_repository(self): + output = self.work / "diagnostic.json" + result = subprocess.run([ + sys.executable, str(ROOT / "tools" / "analyse.py"), + str(FIXTURES / "windiagnostics-v1.json"), + "--evidence-root", str(FIXTURES), "--output", str(output), + "--reference-time", self.reference_time, "--producer-version", "test-revision", + ], cwd=self.work, text=True, capture_output=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(output.is_file()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/analyse.py b/tools/analyse.py new file mode 100644 index 0000000..6c2e333 --- /dev/null +++ b/tools/analyse.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +"""Analyse a stored legacy capture into a validated diagnostic v1 artifact.""" + +import argparse +from collections import OrderedDict +from datetime import datetime +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import re +import sys +import tempfile + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location("diagnostic_validation", ROOT / "tools" / "validate.py") +validation = importlib.util.module_from_spec(spec) +spec.loader.exec_module(validation) + +EVENT_RULES = { + ("Service Control Manager", 7000): { + "severity": "warning", + "confidence": 90, + "title": "Service Control Manager event 7000", + "description": "A configured service or driver failed to start.", + }, +} + + +def pointer(*parts): + return "/" + "/".join(str(part).replace("~", "~0").replace("/", "~1") for part in parts) + + +def slug(value): + result = re.sub(r"[^A-Za-z0-9._:-]+", "-", str(value)).strip("-").lower() + return result or "unknown" + + +def valid_timestamp(value): + if not isinstance(value, str): + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + return value if parsed.tzinfo is not None else None + + +def normalize_section(section): + if not isinstance(section, dict): + return {"status": "unavailable", "record_count": None, "truncated": False, + "reason": "Legacy section is not an object."} + source_status = section.get("status") + status = source_status if source_status in ("ok", "skipped", "unavailable") else "unavailable" + count = section.get("count") + if type(count) is not int or count < 0: + data = section.get("data") + count = len(data) if isinstance(data, list) else None + result = {"status": status, "record_count": count, + "truncated": bool(section.get("truncated", False))} + if status != "ok": + reason = section.get("error") + if not isinstance(reason, str) or not reason.strip(): + reason = (f"Legacy capture used unsupported section status: {source_status}." + if source_status not in ("skipped", "unavailable") + else f"Legacy capture marked section {status} without a reason.") + result["reason"] = reason + return result + + +def selected_event_sections(capture): + sections = capture["sections"] + version = capture["schema_version"] + names = (("recent_system_errors", "recent_application_errors") if version == 1 else + ("recent_system_events", "recent_application_events")) + return [(name, sections[name]) for name in names if name in sections] + + +def analyse(capture, artifact, artifact_hash, reference_time, producer_version): + version = capture.get("schema_version") + if type(version) is not int or version not in (1, 2): + raise ValueError(f"unsupported legacy capture schema version: {version!r}") + if not isinstance(capture.get("sections"), dict): + raise ValueError("legacy capture requires an object at /sections") + collected_at = valid_timestamp(capture.get("generated_at")) + if collected_at is None: + raise ValueError("/generated_at: expected a timezone-qualified ISO timestamp") + if valid_timestamp(reference_time) is None: + raise ValueError("--reference-time: expected a timezone-qualified ISO timestamp") + + collection_sections = {name: normalize_section(section) + for name, section in capture["sections"].items()} + complete = bool(collection_sections) and all( + section["status"] == "ok" and not section["truncated"] + for section in collection_sections.values()) + collection_status = "complete" if complete else "partial" + observations = [] + groups = OrderedDict() + for section_name, section in selected_event_sections(capture): + data = section.get("data", []) if isinstance(section, dict) else [] + if not isinstance(data, list): + continue + for index, event in enumerate(data): + if not isinstance(event, dict): + continue + observation_id = f"event-{slug(section_name)}-{index + 1}" + observation = {"id": observation_id, "section": section_name, + "source_pointer": pointer("sections", section_name, "data", index), + "value": event} + observed_at = valid_timestamp(event.get("TimeCreated")) + if observed_at: + observation["observed_at"] = observed_at + observations.append(observation) + provider = str(event.get("ProviderName", "Unknown provider")) + raw_event_id = event.get("Id", "unknown") + event_id = raw_event_id if type(raw_event_id) in (int, str) else "unknown" + groups.setdefault((provider, event_id), []).append(observation_id) + + findings = [] + for (provider, event_id), evidence in groups.items(): + rule = EVENT_RULES.get((provider, event_id)) + if rule: + confidence = min(rule["confidence"], 50) if collection_status != "complete" else rule["confidence"] + findings.append({"id": f"event-{slug(provider)}-{slug(event_id)}", + "severity": rule["severity"], "confidence": confidence, + "title": rule["title"], "description": rule["description"], + "evidence": evidence}) + else: + findings.append({"id": f"event-{slug(provider)}-{slug(event_id)}", + "severity": "unanalysed", "confidence": 0, + "title": f"Unanalysed event: {provider} / {event_id}", + "description": "The event is preserved for review; no migrated technical rule matches it.", + "evidence": evidence}) + + warnings = [str(item) for item in capture.get("warnings", []) if str(item).strip()] + if collection_status != "complete": + warnings.append("Legacy capture collection is incomplete; finding confidence is capped at 50.") + warnings.append(f"Initial migrated analyser supports {len(EVENT_RULES)} event rule; unmatched events remain unanalysed.") + system = {"host": capture.get("host", {})} + if isinstance(capture.get("summary"), dict): + system["legacy_summary"] = capture["summary"] + return { + "schema_version": 1, + "metadata": {"producer": {"name": "win-diagnostics", "version": producer_version}, + "collected_at": collected_at, "analysis_reference_time": reference_time, + "source": {"artifact": artifact, "schema_version": version, + "sha256": artifact_hash}}, + "system": system, + "collection": {"mode": capture.get("mode", "Full"), "status": collection_status, + "sections": collection_sections}, + "observations": observations, + "findings": findings, + "warnings": warnings, + } + + +def analyse_file(input_path, evidence_root, output_path, reference_time, producer_version, force=False): + if input_path.is_symlink(): + raise ValueError(f"{input_path}: input symlinks are not supported") + input_path = input_path.resolve() + evidence_root = evidence_root.resolve() + output_path = output_path.resolve() + if not input_path.is_relative_to(evidence_root): + raise ValueError(f"{input_path}: input must be inside --evidence-root") + if output_path == input_path: + raise ValueError("input and output paths must differ") + artifact = input_path.relative_to(evidence_root).as_posix() + capture = validation.load_json(input_path) + result = analyse(capture, artifact, hashlib.sha256(input_path.read_bytes()).hexdigest(), + reference_time, producer_version) + errors = validation.validate_document(result) + if not errors: + errors = validation.verify_evidence(result, evidence_root) + if errors: + raise ValueError("generated diagnostic is invalid:\n" + "\n".join(errors)) + if output_path.exists() and not force: + raise ValueError(f"{output_path}: output exists; use --force to replace it") + output_path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp(prefix=f".{output_path.name}.", dir=output_path.parent) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + json.dump(result, stream, indent=2, ensure_ascii=False, allow_nan=False) + stream.write("\n") + if force: + os.replace(temporary, output_path) + else: + os.link(temporary, output_path) + finally: + temporary.unlink(missing_ok=True) + return result + + +def parser(): + result = argparse.ArgumentParser(description=__doc__) + result.add_argument("input", type=Path) + result.add_argument("--evidence-root", required=True, type=Path) + result.add_argument("--output", required=True, type=Path) + result.add_argument("--reference-time", required=True) + result.add_argument("--producer-version", required=True) + result.add_argument("--force", action="store_true") + return result + + +def main(argv=None): + args = parser().parse_args(argv) + try: + analyse_file(args.input, args.evidence_root, args.output, args.reference_time, + args.producer_version, args.force) + except (OSError, ValueError, KeyError, TypeError, RecursionError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + print(f"Wrote validated diagnostic v1: {args.output}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())