132 lines
6.7 KiB
Python
132 lines
6.7 KiB
Python
import copy
|
||
import importlib.util
|
||
import json
|
||
from pathlib import Path
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
import unittest
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
spec = importlib.util.spec_from_file_location("diagnostic_validation", ROOT / "tools/validate.py")
|
||
validation = importlib.util.module_from_spec(spec)
|
||
spec.loader.exec_module(validation)
|
||
|
||
|
||
class DiagnosticContractTests(unittest.TestCase):
|
||
def setUp(self):
|
||
self.document = validation.load_json(ROOT / "examples/diagnostic.json")
|
||
|
||
def test_example_and_evidence(self):
|
||
self.assertEqual(validation.validate_document(self.document), [])
|
||
self.assertEqual(validation.verify_evidence(self.document, ROOT / "tests/fixtures/legacy"), [])
|
||
|
||
def test_invalid_shapes_and_versions(self):
|
||
for path, value in [
|
||
(("schema_version",), 2), (("schema_version",), True),
|
||
(("metadata", "source", "schema_version"), 3),
|
||
(("metadata", "collected_at"), "2026-02-30T12:00:00Z"),
|
||
(("metadata", "analysis_reference_time"), "2026-09-15T12:00:00"),
|
||
(("findings", 0, "confidence"), 101), (("findings", 0, "confidence"), -1),
|
||
(("findings", 0, "confidence"), "90"), (("findings", 0, "confidence"), True),
|
||
(("findings", 0, "evidence"), []), (("findings", 0, "title"), " "),
|
||
(("findings", 0, "severity"), "healthy"),
|
||
(("observations", 0, "source_pointer"), "/broken~2escape"),
|
||
]:
|
||
with self.subTest(path=path, value=value):
|
||
doc = copy.deepcopy(self.document)
|
||
target = doc
|
||
for key in path[:-1]:
|
||
target = target[key]
|
||
target[path[-1]] = value
|
||
self.assertTrue(validation.validate_document(doc))
|
||
self.assertTrue(validation.validate_document({}))
|
||
self.assertTrue(validation.validate_document([]))
|
||
|
||
def test_portable_source_paths(self):
|
||
for path in ["/capture.json", "../capture.json", "a/../capture.json", "C:/capture.json",
|
||
"a\\capture.json", "./capture.json", "capture.json\n"]:
|
||
with self.subTest(path=path):
|
||
self.document["metadata"]["source"]["artifact"] = path
|
||
self.assertTrue(validation.validate_document(self.document))
|
||
|
||
def test_broken_evidence_and_identity_links(self):
|
||
self.document["findings"][0]["evidence"] = ["missing"]
|
||
self.assertIn("unknown observation", " ".join(validation.validate_document(self.document)))
|
||
self.setUp()
|
||
self.document["observations"].append(copy.deepcopy(self.document["observations"][0]))
|
||
self.assertIn("duplicate observation", " ".join(validation.validate_document(self.document)))
|
||
self.setUp()
|
||
self.document["findings"].append(copy.deepcopy(self.document["findings"][0]))
|
||
self.assertIn("duplicate finding", " ".join(validation.validate_document(self.document)))
|
||
self.setUp()
|
||
self.document["observations"][0]["section"] = "undeclared"
|
||
self.assertIn("unknown collection section", " ".join(validation.validate_document(self.document)))
|
||
|
||
def test_partial_collection_cannot_claim_complete(self):
|
||
self.document["collection"]["status"] = "complete"
|
||
self.assertTrue(validation.validate_document(self.document))
|
||
del self.document["collection"]["sections"]["system_file_check"]
|
||
self.assertEqual(validation.validate_document(self.document), [])
|
||
self.document["collection"]["sections"]["recent_system_errors"]["truncated"] = True
|
||
self.assertTrue(validation.validate_document(self.document))
|
||
self.document["collection"]["sections"] = {}
|
||
self.document["observations"] = []
|
||
self.document["findings"] = []
|
||
self.assertTrue(validation.validate_document(self.document))
|
||
|
||
def test_unavailable_checks_need_reason(self):
|
||
del self.document["collection"]["sections"]["system_file_check"]["reason"]
|
||
self.assertTrue(validation.validate_document(self.document))
|
||
|
||
def test_evidence_integrity(self):
|
||
root = ROOT / "tests/fixtures/legacy"
|
||
self.document["metadata"]["source"]["sha256"] = "0" * 64
|
||
self.assertIn("checksum", " ".join(validation.verify_evidence(self.document, root)))
|
||
self.setUp()
|
||
self.document["observations"][0]["source_pointer"] = "/missing"
|
||
self.assertIn("does not resolve", " ".join(validation.verify_evidence(self.document, root)))
|
||
self.setUp()
|
||
self.document["observations"][0]["value"] = {"fabricated": True}
|
||
self.assertIn("differs", " ".join(validation.verify_evidence(self.document, root)))
|
||
self.setUp()
|
||
self.document["metadata"]["source"]["schema_version"] = 2
|
||
self.assertIn("version differs", " ".join(validation.verify_evidence(self.document, root)))
|
||
|
||
def test_json_pointer_escape_and_array_rules(self):
|
||
doc = {"a/b": {"~": [True]}}
|
||
self.assertIs(validation.resolve_pointer(doc, "/a~1b/~0/0"), True)
|
||
self.assertEqual(validation.resolve_pointer(doc, ""), doc)
|
||
for token in ("01", "-1", "-", "٠"):
|
||
with self.subTest(token=token), self.assertRaises(ValueError):
|
||
validation.resolve_pointer([1], "/" + token)
|
||
|
||
def test_cli_from_another_directory(self):
|
||
with tempfile.TemporaryDirectory() as temp:
|
||
result = subprocess.run([sys.executable, str(ROOT / "tools/validate.py"),
|
||
str(ROOT / "examples/diagnostic.json"), "--evidence-root",
|
||
str(ROOT / "tests/fixtures/legacy")], cwd=temp, capture_output=True, text=True)
|
||
self.assertEqual(result.returncode, 0, result.stderr)
|
||
path = Path(temp) / "invalid.json"
|
||
for content in ('{"schema_version":1,"schema_version":2}', '{"bad":NaN}', '{"bad":1e999}', '{broken'):
|
||
path.write_text(content)
|
||
result = subprocess.run([sys.executable, str(ROOT / "tools/validate.py"), str(path)], cwd=temp, capture_output=True, text=True)
|
||
self.assertEqual(result.returncode, 1)
|
||
self.assertNotIn("Traceback", result.stderr)
|
||
|
||
def test_evidence_symlink_cannot_escape_root(self):
|
||
with tempfile.TemporaryDirectory() as temp:
|
||
root = Path(temp) / "evidence"
|
||
root.mkdir()
|
||
target = Path(temp) / "outside.json"
|
||
target.write_text("{}")
|
||
try:
|
||
(root / self.document["metadata"]["source"]["artifact"]).symlink_to(target)
|
||
except OSError:
|
||
self.skipTest("symlink creation is unavailable on this platform")
|
||
self.assertIn("escapes evidence root", " ".join(validation.verify_evidence(self.document, root)))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|