From 792d92d5ca1e24dd4c7b330f58144b1c3012b6c8 Mon Sep 17 00:00:00 2001 From: Dandaedre Date: Tue, 15 Sep 2026 16:32:36 +0200 Subject: [PATCH] feat: define diagnostic v1 contract and evidence validator --- .gitignore | 3 + AGENTS-TASKS.md | 16 +- Jenkinsfile | 23 +-- README.md | 6 + docs/diagnostic-contract.md | 113 ++++++++++++++ docs/migration-inventory.md | 8 +- examples/diagnostic.json | 65 +++++++++ requirements.in | 2 + requirements.txt | 24 +++ schemas/diagnostic.schema.json | 260 +++++++++++++++++++++++++++++++++ tests/test_contract.py | 131 +++++++++++++++++ tools/validate.py | 135 +++++++++++++++++ 12 files changed, 768 insertions(+), 18 deletions(-) create mode 100644 docs/diagnostic-contract.md create mode 100644 examples/diagnostic.json create mode 100644 requirements.in create mode 100644 requirements.txt create mode 100644 schemas/diagnostic.schema.json create mode 100644 tests/test_contract.py create mode 100644 tools/validate.py diff --git a/.gitignore b/.gitignore index a7ee804..aea563c 100644 --- a/.gitignore +++ b/.gitignore @@ -149,3 +149,6 @@ core.* # uv.lock / Cargo.lock should normally be committed when applicable. # ============================================================================= + +# Isolated Jenkins validation environment +.ci-venv/ diff --git a/AGENTS-TASKS.md b/AGENTS-TASKS.md index 1a8cec9..a4844d1 100644 --- a/AGENTS-TASKS.md +++ b/AGENTS-TASKS.md @@ -38,11 +38,11 @@ expected results. Existing legacy fixtures are not assumed sanitized. Depends on Pass 0 for the selected slice. -- [ ] WD1.1 Define `schemas/diagnostic.schema.json` with explicit version, metadata, system, collection status, observations, findings, and warnings; adapt fields to actual evidence. -- [ ] WD1.2 Define stable finding IDs, severity/confidence semantics, evidence references, and how raw evidence is retained. Missing, skipped, and failed checks must not imply health. -- [ ] WD1.3 Document mapping from legacy capture schemas 1 and 2, including unsupported versions and information that cannot be recovered. +- [x] WD1.1 Define `schemas/diagnostic.schema.json` with explicit version, metadata, system, collection status, observations, findings, and warnings; adapt fields to actual evidence. +- [x] WD1.2 Define stable finding IDs, severity/confidence semantics, evidence references, and how raw evidence is retained. Missing, skipped, and failed checks must not imply health. +- [x] WD1.3 Document mapping from legacy capture schemas 1 and 2, including unsupported versions and information that cannot be recovered. - [ ] WD1.4 Define independent collection/offline-analysis CLI contracts: input/output paths, errors, exit codes, dependencies, and compatibility policy. -- [ ] WD1.5 Add synthetic valid/invalid schema examples and validation tests; document the interface for downstream report adapters without importing their implementation. +- [x] WD1.5 Add synthetic valid/invalid schema examples and validation tests; document the interface for downstream report adapters without importing their implementation. Exit: a consumer can understand and validate diagnostic output using a documented public contract, independently of collection internals. @@ -102,3 +102,11 @@ valid diagnostic result independently of reporting tools. - Environment: Typst 0.15.1; PowerShell unavailable, so native Windows collection remains untested. Advisory URLs were not revalidated. - Preserved concurrent checkout updates: instructions are now `AGENT.md`; README/tracker links now match. Jenkins file-presence checks pass locally; WD4.1 still requires future schema/analysis CI. - Next: WD1 schema/examples, then one offline normalization slice. Full diagnostic rules and Windows validation remain open. + +### 2026-09-15 — Versioned contract pass + +- Completed WD1.1–WD1.3 and WD1.5: diagnostic v1 schema, synthetic artifact, standalone validator, evidence verification, version/legacy mapping, and pinned dependencies. +- Evidence: [diagnostic contract](docs/diagnostic-contract.md), `schemas/diagnostic.schema.json`, `tools/validate.py`, `examples/diagnostic.json`, `tests/test_contract.py`. +- Validation: 10 unittest cases (plus invalid-input subcases) passed on Python 3.11 in an isolated component copy with only its own requirements. Earlier core tests also passed on Python 3.14. Includes artifact checksums, JSON Pointer/value matching, dangling/duplicate IDs, invalid formats, partial coverage and path containment. +- Jenkins now runs contract tests in an isolated environment; pipeline execution itself was not performed here. This is contract validation, not a migrated analysis engine. +- Next: WD1.4 precise collection/offline-analysis commands and WD2 first normalization slice. Preserve evidence and the existing synthetic legacy snapshots. Native Windows collection remains untested. diff --git a/Jenkinsfile b/Jenkinsfile index 75e668b..62e34cf 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,15 +45,18 @@ pipeline { * available. */ - /* - * TODO: Validate Diagnostic Schema - * - * Once schemas/diagnostic.schema.json exists: - * - * - validate the schema; - * - validate sanitized fixtures; - * - verify schema_version handling. - */ + stage('Validate Contract') { + steps { + sh ''' + set -eu + command -v uv >/dev/null 2>&1 || { echo "uv is required for the isolated validation environment" >&2; exit 1; } + uv venv --python 3.11 --allow-existing .ci-venv + uv pip sync --python .ci-venv/bin/python requirements.txt + .ci-venv/bin/python --version + .ci-venv/bin/python -m unittest discover -s tests -p 'test_*.py' + ''' + } + } /* * TODO: Analysis Tests @@ -88,7 +91,7 @@ pipeline { conclusion: 'SUCCESS', name: 'Win Diagnostics CI', summary: 'Win Diagnostics passed all currently implemented CI checks.', - text: 'Repository validation completed successfully. Diagnostic schema and fixture analysis will be enabled as they are implemented.', + text: 'Repository and diagnostic contract tests passed. Collector and analysis-engine execution remain pending.', title: 'Win Diagnostics CI passed' ) diff --git a/README.md b/README.md index 695c7f6..fcf557e 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,12 @@ See [AGENT.md](AGENT.md) for repository ownership and development rules. [AGENTS-TASKS.md](AGENTS-TASKS.md) tracks migration passes, completed work, acceptance criteria, and the next-pass handoff. +## Contract validation + +The version-1 schema, standalone validator, and synthetic examples are available. +See [contract documentation](docs/diagnostic-contract.md) for setup and validation commands. +Collection/analysis and PDF rendering are still separate migration tasks. + ## License Software source code is licensed under the **MIT License**. diff --git a/docs/diagnostic-contract.md b/docs/diagnostic-contract.md new file mode 100644 index 0000000..e2f32cb --- /dev/null +++ b/docs/diagnostic-contract.md @@ -0,0 +1,113 @@ +# Diagnostic contract version 1 + +The implemented public interface is [diagnostic.schema.json](../schemas/diagnostic.schema.json), +a JSON Schema Draft 2020-12 document. This pass provides validation and synthetic +examples, not a migrated collector or analysis engine. + +## Install and validate independently + +Python 3.11 or newer is required. From a standalone component checkout: + +```bash +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements.txt +.venv/bin/python tools/validate.py examples/diagnostic.json +.venv/bin/python tools/validate.py examples/diagnostic.json --evidence-root tests/fixtures/legacy +.venv/bin/python -m unittest discover -s tests -p 'test_*.py' +``` + +On Windows, use `.venv\Scripts\python.exe` in place of `.venv/bin/python`. +Alternatively, use `uv venv` and `uv pip sync --python .venv/bin/python requirements.txt`. +Dependencies, including transitive dependencies, are pinned in `requirements.txt`. +No Typst, Reportist, macro checkout, network service, or live Windows machine is +needed after dependencies are installed. Validation accepts an explicit JSON path, +writes no artifacts, and exits 0 for valid input, 1 for invalid/unreadable input or +missing dependencies, and 2 for incorrect CLI usage. Errors identify JSON paths. + +## Envelope + +| Field | Meaning | +| --- | --- | +| `schema_version` | Exactly integer `1`, the diagnostic artifact contract version. | +| `metadata.producer` | Tool name and version/revision; synthetic examples label their producer version accordingly. | +| `metadata.collected_at` | RFC 3339 collection timestamp with timezone. | +| `metadata.analysis_reference_time` | Explicit RFC 3339 reference for time-dependent analysis; never implicitly today's date. | +| `metadata.source` | Original capture artifact, its SHA-256, and its separate capture `schema_version` (1 or 2). | +| `system` | Sparse typed JSON system facts. Unknown facts should be omitted or null, not presentation strings. | +| `collection` | Capture mode, coverage status, and named section results. | +| `observations` | Identified raw JSON values with source pointers and section names. | +| `findings` | Identified technical conclusions with severity, confidence, description, and observation IDs. | +| `warnings` | Nonblank diagnostic/collection limitation strings. | + +All envelope fields are required. Most objects reject unknown fields to catch +misspellings; `system` intentionally allows heterogeneous JSON facts. JSON object +keys must be unique and numbers finite. References are local to the schema; +validation performs no remote schema lookup. + +## Collection semantics + +Modes are `Compact`, `Mid`, and `Full`. Coverage status is `complete`, `partial`, +or `unknown`. Each declared section has `status` (`ok`, `skipped`, `unavailable`, +`missing`), `record_count` (nonnegative integer or null), and `truncated` (boolean). +A non-OK section requires a reason. Count is retained-record count, not a health +signal; failed native commands may still provide records worth preserving. + +The validator rejects `complete` with no declared sections, unsuccessful sections, +or truncation. Complete means complete for the declared analysis scope, not proof +of Windows health or a claim that every possible collector section was evaluated. +A future analyzer must declare its actual scope honestly. Null counts and absent +facts cannot establish health; successful zero-count and missing zero-count checks +must remain distinguishable. + +## Evidence and finding identity + +An observation has `id`, `section`, `source_pointer`, and `value`; optional +`observed_at` is a timezone-qualified timestamp. Its value is the exact raw JSON +value at that pointer, not a rewritten prose excerpt. Derived/normalized facts +belong in `system` or a finding's technical description, supported by observations. +JSON Pointers escape `~` as `~0` and `/` as `~1`; the empty pointer names the root. + +A finding has `id`, `severity` (`critical`, `warning`, `info`, `unanalysed`), +`confidence` (number 0–100), `title`, `description`, and a nonempty `evidence` list +of observation IDs. Confidence describes strength of the finding, not the chance +that the system is healthy. IDs are opaque tokens; producers must keep them stable +for repeated analysis of the same evidence/rule. The future analyzer must document +its ID derivation. The validator rejects duplicate IDs and dangling observation +or section references; JSON Schema alone cannot enforce those relationships. + +Artifact paths use relative forward-slash paths without drive letters, parent/dot +segments, backslashes, or control characters. They are relative to the explicit +`--evidence-root`, not a hard-coded workspace. Without that option validation checks +structure and internal relationships only; it does not prove raw evidence exists. +With it, the validator checks the capture hash/version, resolves every observation +pointer, compares raw values, and rejects paths whose symlinks escape the root. + +## Mapping legacy captures (contract, not yet an implemented converter) + +| Legacy input | Version-1 artifact mapping | +| --- | --- | +| Capture `schema_version` 1 or 2 | Preserve under `metadata.source.schema_version`; artifact version remains 1. Other capture versions are unsupported. | +| `generated_at`, host/tool context | Collection/reference timestamps and typed system/provenance facts. Do not invent historical producer revisions. | +| `sections.*.status/count/data/error` | Section state/count/reason and observations referencing original records. Retain native command outputs and nested failures. | +| Unrecognized section status | `unavailable` with a reason recording the original state; preserve original evidence. | +| `limit`, `truncated`, diagnostic_collection | Coverage evidence; absence of an old field must not imply truncation or successful collection. | +| Compact `summary` | Observations can point into `/summary/...`; declare the associated section/scope. Summary zero counts do not override explicit unavailable states. | +| Schema-1 recent_*_errors vs schema-2 recent_*_events | Normalize observed event facts without duplicating the old/new representations; retain pointers to chosen source records. | +| `/Date(milliseconds)/` and ISO timestamps | Preserve raw values in observations; convert valid instants to timezone-qualified timestamps where used as normalized time. Unparseable required timestamps must produce an error, not today's date. | +| Unknown events | Preserve as observations and, when surfaced as findings, `unanalysed`; do not silently classify as healthy. | +| Human prose, executive summaries, remediation | Excluded from the engine contract; handled by the report adapter and human review. | + +The legacy normalizer's health headline/score, display strings, inventory excerpts, +and report remediation are not mandatory diagnostic fields. Existing snapshots +remain migration comparisons, not new-schema examples. + +## Compatibility + +Consumers select by explicit artifact version and reject unsupported versions. +Breaking field or semantic changes require a new version and documented migration. +Do not relabel raw captures or silently reshape historical artifacts. Contract v1 +is the initial implementation; no converter or production producer is claimed yet. + +Schema and format behavior follow [JSON Schema Draft 2020-12](https://json-schema.org/draft/2020-12/json-schema-validation) +and the [jsonschema validator documentation](https://python-jsonschema.readthedocs.io/en/stable/validate/). +The validator enables format checking explicitly, including timezone-qualified dates. diff --git a/docs/migration-inventory.md b/docs/migration-inventory.md index 300aabe..9223051 100644 --- a/docs/migration-inventory.md +++ b/docs/migration-inventory.md @@ -196,7 +196,7 @@ that context in beffroi-reports rather than a generic diagnostic title. All matches need their limitations preserved; zero-weight advisories must not become confirmed failures or silently alter health scoring. -## First diagnostic contract decisions (draft, not implemented) +## First diagnostic contract decisions 1. Keep the raw capture schema version distinct from the new diagnostic schema version. A schema-2 capture does not imply diagnostic contract version 2. @@ -211,9 +211,9 @@ confirmed failures or silently alter health scoring. 6. Use collection time as the explicit analysis reference time; never use wall clock time implicitly for fading. Document rounding/timezone compatibility. -Next: implement the versioned schema and valid/invalid examples, then offline -normalization for one fixture. Field names/CLI are still proposals, not a released -interface. Full native collection support remains a later Windows validation gate. +The version-1 schema and standalone validator are now implemented; see +[diagnostic contract](diagnostic-contract.md). Offline normalization and native +collection remain unimplemented; full Windows support is a later validation gate. ## Provenance and fixture review diff --git a/examples/diagnostic.json b/examples/diagnostic.json new file mode 100644 index 0000000..d193db0 --- /dev/null +++ b/examples/diagnostic.json @@ -0,0 +1,65 @@ +{ + "schema_version": 1, + "metadata": { + "producer": { + "name": "win-diagnostics", + "version": "synthetic-example" + }, + "collected_at": "2026-09-14T16:00:00+02:00", + "analysis_reference_time": "2026-09-14T16:00:00+02:00", + "source": { + "artifact": "windiagnostics-v1.json", + "schema_version": 1, + "sha256": "ab9e786c73bcd5a58bc311ada7a04a97de61f5bbe98f8a8137a698bdb4d652ca" + } + }, + "system": { + "computer_name": "SYNTHETIC-PC", + "memory_total_gb": 31.82 + }, + "collection": { + "mode": "Full", + "status": "partial", + "sections": { + "recent_system_errors": { + "status": "ok", + "record_count": 1, + "truncated": false + }, + "system_file_check": { + "status": "missing", + "record_count": null, + "truncated": false, + "reason": "Not evaluated in this synthetic contract example." + } + } + }, + "observations": [ + { + "id": "event-7000-1", + "section": "recent_system_errors", + "source_pointer": "/sections/recent_system_errors/data/0", + "value": { + "TimeCreated": "2026-09-14T15:00:00+02:00", + "Id": 7000, + "ProviderName": "Service Control Manager", + "Message": "The TestService service failed to start due to the following error: The system cannot find the file specified." + } + } + ], + "findings": [ + { + "id": "service-start-failure", + "severity": "warning", + "confidence": 90, + "title": "A service failed to start", + "description": "The captured Service Control Manager event reports a service startup failure.", + "evidence": [ + "event-7000-1" + ] + } + ], + "warnings": [ + "Synthetic contract example; not generated by a migrated diagnostic engine." + ] +} diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..d3bc7cf --- /dev/null +++ b/requirements.in @@ -0,0 +1,2 @@ +jsonschema==4.26.0 +rfc3339-validator==0.1.4 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8f271c5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,24 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --python-version 3.11 --universal requirements.in -o requirements.txt +attrs==26.1.0 + # via + # jsonschema + # referencing +jsonschema==4.26.0 + # via -r requirements.in +jsonschema-specifications==2025.9.1 + # via jsonschema +referencing==0.37.0 + # via + # jsonschema + # jsonschema-specifications +rfc3339-validator==0.1.4 + # via -r requirements.in +rpds-py==2026.6.3 + # via + # jsonschema + # referencing +six==1.17.0 + # via rfc3339-validator +typing-extensions==4.16.0 ; python_full_version < '3.13' + # via referencing diff --git a/schemas/diagnostic.schema.json b/schemas/diagnostic.schema.json new file mode 100644 index 0000000..4a0e9aa --- /dev/null +++ b/schemas/diagnostic.schema.json @@ -0,0 +1,260 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Win Diagnostics artifact version 1", + "description": "Normalized evidence and technical findings; raw capture schema versions are separate.", + "type": "object", + "properties": { + "schema_version": { + "type": "integer", + "const": 1 + }, + "metadata": { + "type": "object", + "properties": { + "producer": { + "type": "object", + "properties": { + "name": { + "type": "string", + "pattern": "\\S" + }, + "version": { + "type": "string", + "pattern": "\\S" + } + }, + "required": [ + "name", + "version" + ], + "additionalProperties": false + }, + "collected_at": { + "type": "string", + "format": "date-time" + }, + "analysis_reference_time": { + "type": "string", + "format": "date-time" + }, + "source": { + "type": "object", + "properties": { + "artifact": { + "type": "string", + "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.?(?:/|$))[^\\\\:\\x00-\\x1f]+$(?![\\s\\S])" + }, + "schema_version": { + "type": "integer", + "enum": [ + 1, + 2 + ] + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$(?![\\s\\S])" + } + }, + "required": [ + "artifact", + "schema_version", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "producer", + "collected_at", + "analysis_reference_time", + "source" + ], + "additionalProperties": false + }, + "system": { + "type": "object" + }, + "collection": { + "type": "object", + "properties": { + "mode": { + "enum": [ + "Compact", + "Mid", + "Full" + ] + }, + "status": { + "enum": [ + "complete", + "partial", + "unknown" + ] + }, + "sections": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "status": { + "enum": [ + "ok", + "skipped", + "unavailable", + "missing" + ] + }, + "record_count": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "truncated": { + "type": "boolean" + }, + "reason": { + "type": "string", + "pattern": "\\S" + } + }, + "required": [ + "status", + "record_count", + "truncated" + ], + "additionalProperties": false, + "allOf": [ + { + "if": { + "properties": { + "status": { + "enum": [ + "skipped", + "unavailable", + "missing" + ] + } + } + }, + "then": { + "required": [ + "reason" + ] + } + } + ] + } + } + }, + "required": [ + "mode", + "status", + "sections" + ], + "additionalProperties": false + }, + "observations": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]*$(?![\\s\\S])" + }, + "section": { + "type": "string", + "pattern": "\\S" + }, + "source_pointer": { + "type": "string", + "pattern": "^(?:/(?:[^~/]|~[01])*)*$(?![\\s\\S])" + }, + "value": {}, + "observed_at": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "id", + "section", + "source_pointer", + "value" + ], + "additionalProperties": false + } + }, + "findings": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]*$(?![\\s\\S])" + }, + "severity": { + "enum": [ + "critical", + "warning", + "info", + "unanalysed" + ] + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 100 + }, + "title": { + "type": "string", + "pattern": "\\S" + }, + "description": { + "type": "string", + "pattern": "\\S" + }, + "evidence": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]*$(?![\\s\\S])" + }, + "minItems": 1, + "uniqueItems": true + } + }, + "required": [ + "id", + "severity", + "confidence", + "title", + "description", + "evidence" + ], + "additionalProperties": false + } + }, + "warnings": { + "type": "array", + "items": { + "type": "string", + "pattern": "\\S" + } + } + }, + "required": [ + "schema_version", + "metadata", + "system", + "collection", + "observations", + "findings", + "warnings" + ], + "additionalProperties": false +} diff --git a/tests/test_contract.py b/tests/test_contract.py new file mode 100644 index 0000000..85eae39 --- /dev/null +++ b/tests/test_contract.py @@ -0,0 +1,131 @@ +import copy +import importlib.util +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location("diagnostic_validation", ROOT / "tools/validate.py") +validation = importlib.util.module_from_spec(spec) +spec.loader.exec_module(validation) + + +class DiagnosticContractTests(unittest.TestCase): + def setUp(self): + self.document = validation.load_json(ROOT / "examples/diagnostic.json") + + def test_example_and_evidence(self): + self.assertEqual(validation.validate_document(self.document), []) + self.assertEqual(validation.verify_evidence(self.document, ROOT / "tests/fixtures/legacy"), []) + + def test_invalid_shapes_and_versions(self): + for path, value in [ + (("schema_version",), 2), (("schema_version",), True), + (("metadata", "source", "schema_version"), 3), + (("metadata", "collected_at"), "2026-02-30T12:00:00Z"), + (("metadata", "analysis_reference_time"), "2026-09-15T12:00:00"), + (("findings", 0, "confidence"), 101), (("findings", 0, "confidence"), -1), + (("findings", 0, "confidence"), "90"), (("findings", 0, "confidence"), True), + (("findings", 0, "evidence"), []), (("findings", 0, "title"), " "), + (("findings", 0, "severity"), "healthy"), + (("observations", 0, "source_pointer"), "/broken~2escape"), + ]: + with self.subTest(path=path, value=value): + doc = copy.deepcopy(self.document) + target = doc + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + self.assertTrue(validation.validate_document(doc)) + self.assertTrue(validation.validate_document({})) + self.assertTrue(validation.validate_document([])) + + def test_portable_source_paths(self): + for path in ["/capture.json", "../capture.json", "a/../capture.json", "C:/capture.json", + "a\\capture.json", "./capture.json", "capture.json\n"]: + with self.subTest(path=path): + self.document["metadata"]["source"]["artifact"] = path + self.assertTrue(validation.validate_document(self.document)) + + def test_broken_evidence_and_identity_links(self): + self.document["findings"][0]["evidence"] = ["missing"] + self.assertIn("unknown observation", " ".join(validation.validate_document(self.document))) + self.setUp() + self.document["observations"].append(copy.deepcopy(self.document["observations"][0])) + self.assertIn("duplicate observation", " ".join(validation.validate_document(self.document))) + self.setUp() + self.document["findings"].append(copy.deepcopy(self.document["findings"][0])) + self.assertIn("duplicate finding", " ".join(validation.validate_document(self.document))) + self.setUp() + self.document["observations"][0]["section"] = "undeclared" + self.assertIn("unknown collection section", " ".join(validation.validate_document(self.document))) + + def test_partial_collection_cannot_claim_complete(self): + self.document["collection"]["status"] = "complete" + self.assertTrue(validation.validate_document(self.document)) + del self.document["collection"]["sections"]["system_file_check"] + self.assertEqual(validation.validate_document(self.document), []) + self.document["collection"]["sections"]["recent_system_errors"]["truncated"] = True + self.assertTrue(validation.validate_document(self.document)) + self.document["collection"]["sections"] = {} + self.document["observations"] = [] + self.document["findings"] = [] + self.assertTrue(validation.validate_document(self.document)) + + def test_unavailable_checks_need_reason(self): + del self.document["collection"]["sections"]["system_file_check"]["reason"] + self.assertTrue(validation.validate_document(self.document)) + + def test_evidence_integrity(self): + root = ROOT / "tests/fixtures/legacy" + self.document["metadata"]["source"]["sha256"] = "0" * 64 + self.assertIn("checksum", " ".join(validation.verify_evidence(self.document, root))) + self.setUp() + self.document["observations"][0]["source_pointer"] = "/missing" + self.assertIn("does not resolve", " ".join(validation.verify_evidence(self.document, root))) + self.setUp() + self.document["observations"][0]["value"] = {"fabricated": True} + self.assertIn("differs", " ".join(validation.verify_evidence(self.document, root))) + self.setUp() + self.document["metadata"]["source"]["schema_version"] = 2 + self.assertIn("version differs", " ".join(validation.verify_evidence(self.document, root))) + + def test_json_pointer_escape_and_array_rules(self): + doc = {"a/b": {"~": [True]}} + self.assertIs(validation.resolve_pointer(doc, "/a~1b/~0/0"), True) + self.assertEqual(validation.resolve_pointer(doc, ""), doc) + for token in ("01", "-1", "-", "٠"): + with self.subTest(token=token), self.assertRaises(ValueError): + validation.resolve_pointer([1], "/" + token) + + def test_cli_from_another_directory(self): + with tempfile.TemporaryDirectory() as temp: + result = subprocess.run([sys.executable, str(ROOT / "tools/validate.py"), + str(ROOT / "examples/diagnostic.json"), "--evidence-root", + str(ROOT / "tests/fixtures/legacy")], cwd=temp, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + path = Path(temp) / "invalid.json" + for content in ('{"schema_version":1,"schema_version":2}', '{"bad":NaN}', '{"bad":1e999}', '{broken'): + path.write_text(content) + result = subprocess.run([sys.executable, str(ROOT / "tools/validate.py"), str(path)], cwd=temp, capture_output=True, text=True) + self.assertEqual(result.returncode, 1) + self.assertNotIn("Traceback", result.stderr) + + def test_evidence_symlink_cannot_escape_root(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "evidence" + root.mkdir() + target = Path(temp) / "outside.json" + target.write_text("{}") + try: + (root / self.document["metadata"]["source"]["artifact"]).symlink_to(target) + except OSError: + self.skipTest("symlink creation is unavailable on this platform") + self.assertIn("escapes evidence root", " ".join(validation.verify_evidence(self.document, root))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/validate.py b/tools/validate.py new file mode 100644 index 0000000..1fd1dc4 --- /dev/null +++ b/tools/validate.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Validate diagnostic contract v1, optionally verifying its raw evidence.""" + +import argparse +import hashlib +import json +from pathlib import Path +import sys + +try: + import rfc3339_validator # Required so date-time formats cannot be silently ignored. + from jsonschema import Draft202012Validator, FormatChecker +except ImportError: + raise SystemExit("Install validation dependencies with: python -m pip install -r requirements.txt") + +SCHEMA = json.loads((Path(__file__).resolve().parents[1] / "schemas/diagnostic.schema.json").read_text()) +Draft202012Validator.check_schema(SCHEMA) +VALIDATOR = Draft202012Validator(SCHEMA, format_checker=FormatChecker()) + + +def unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key: {key}") + result[key] = value + return result + + +def reject_constant(value): + raise ValueError(f"non-JSON number: {value}") + + +def load_json(path): + document = json.loads(Path(path).read_text(encoding="utf-8"), + object_pairs_hook=unique_object, parse_constant=reject_constant) + json.dumps(document, allow_nan=False) # Also reject exponent overflow such as 1e999. + return document + + +def pointer_path(parts): + return "/" + "/".join(str(p).replace("~", "~0").replace("/", "~1") for p in parts) + + +def validate_document(document): + json.dumps(document, allow_nan=False) + errors = [f"{pointer_path(e.absolute_path)}: {e.message}" for e in VALIDATOR.iter_errors(document)] + if errors: + return sorted(errors) + observations = document["observations"] + ids = set() + for index, observation in enumerate(observations): + if observation["id"] in ids: + errors.append(f"/observations/{index}/id: duplicate observation ID") + ids.add(observation["id"]) + if observation["section"] not in document["collection"]["sections"]: + errors.append(f"/observations/{index}/section: unknown collection section") + findings = set() + for index, finding in enumerate(document["findings"]): + if finding["id"] in findings: + errors.append(f"/findings/{index}/id: duplicate finding ID") + findings.add(finding["id"]) + for evidence in finding["evidence"]: + if evidence not in ids: + errors.append(f"/findings/{index}/evidence: unknown observation {evidence}") + sections = document["collection"]["sections"] + if document["collection"]["status"] == "complete" and ( + not sections or any(s["status"] != "ok" or s["truncated"] for s in sections.values()) + ): + errors.append("/collection/status: complete requires nonempty, successful, untruncated sections") + return errors + + +def resolve_pointer(document, pointer): + value = document + for encoded in pointer.split("/")[1:] if pointer else []: + token = encoded.replace("~1", "/").replace("~0", "~") + if isinstance(value, list): + if not token.isascii() or not token.isdecimal() or (len(token) > 1 and token[0] == "0"): + raise ValueError("invalid array index") + value = value[int(token)] + elif isinstance(value, dict): + value = value[token] + else: + raise ValueError("pointer traverses a scalar") + return value + + +def verify_evidence(document, root): + root = Path(root).resolve() + source = document["metadata"]["source"] + artifact = (root / source["artifact"]).resolve() + if not artifact.is_relative_to(root): + return ["/metadata/source/artifact: resolved path escapes evidence root"] + raw = artifact.read_bytes() + if hashlib.sha256(raw).hexdigest() != source["sha256"]: + return ["/metadata/source/sha256: evidence checksum differs"] + capture = load_json(artifact) + if (not isinstance(capture, dict) or type(capture.get("schema_version")) is not int + or capture["schema_version"] != source["schema_version"]): + return ["/metadata/source/schema_version: capture version differs"] + errors = [] + for index, observation in enumerate(document["observations"]): + try: + actual = resolve_pointer(capture, observation["source_pointer"]) + except (KeyError, IndexError, ValueError): + errors.append(f"/observations/{index}/source_pointer: does not resolve in evidence") + continue + if json.dumps(actual, sort_keys=True) != json.dumps(observation["value"], sort_keys=True): + errors.append(f"/observations/{index}/value: differs from referenced raw evidence") + return errors + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input", type=Path) + parser.add_argument("--evidence-root", type=Path, help="Also check capture SHA-256 and observation pointers/values") + args = parser.parse_args(argv) + try: + document = load_json(args.input) + errors = validate_document(document) + if not errors and args.evidence_root is not None: + errors = verify_evidence(document, args.evidence_root) + except (OSError, ValueError, RecursionError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + if errors: + print("\n".join(errors), file=sys.stderr) + return 1 + print(f"Valid diagnostic v1: {args.input}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())