reportist/tests/test_contract.py

125 lines
6.2 KiB
Python

import copy
import importlib.util
import json
from pathlib import Path
import subprocess
import sys
import tempfile
import unittest
ROOT = Path(__file__).resolve().parents[1]
spec = importlib.util.spec_from_file_location("report_validation", ROOT / "tests/support_validation.py")
validation = importlib.util.module_from_spec(spec)
spec.loader.exec_module(validation)
class ReportContractTests(unittest.TestCase):
def setUp(self):
self.document = validation.load_report(ROOT / "examples/example-report.json")
def test_yaml_json_parity_and_minimal_report(self):
self.assertEqual(self.document, validation.load_report(ROOT / "examples/example-report.yaml"))
self.assertEqual(validation.validate_document(self.document), [])
self.assertEqual(validation.validate_document(validation.load_report(ROOT / "examples/minimal-report.json")), [])
def test_optional_branding_replacement(self):
self.document["branding"] = {"organization": "Independent Example", "logo": "assets/example.svg",
"theme": {"accent": "#112233", "font": "Example Font"}}
self.assertEqual(validation.validate_document(self.document), [])
del self.document["branding"]
self.assertEqual(validation.validate_document(self.document), [])
def test_required_fields_and_versions(self):
for path, value in [(("schema_version",), 2), (("schema_version",), True),
(("title",), " "), (("metadata", "Author"), ""),
(("metadata", "Date"), "2026-02-30"),
(("metadata", "Date"), "2026-09-15T00:00:00Z")]:
with self.subTest(path=path):
doc = copy.deepcopy(self.document)
target = doc
for key in path[:-1]:
target = target[key]
target[path[-1]] = value
self.assertTrue(validation.validate_document(doc))
for key in ("Date", "Author"):
doc = copy.deepcopy(self.document)
del doc["metadata"][key]
self.assertTrue(validation.validate_document(doc))
def test_legacy_domain_fields_rejected(self):
for key in ("diagnostics", "organization", "theme", "collect"):
with self.subTest(key=key):
doc = copy.deepcopy(self.document)
doc[key] = {}
self.assertTrue(validation.validate_document(doc))
def test_all_blocks_and_unknown_nested_blocks(self):
blocks = [
{"type": "text", "text": "Text"}, {"type": "bullets", "items": ["Item"]},
{"type": "steps", "items": ["Step"]}, {"type": "callout", "text": "Note", "kind": "warning"},
{"type": "code", "text": "echo example"},
]
self.document["sections"] = [{"title": "Blocks", "content": [
{"type": "subsection", "title": "Nested", "content": blocks}]}]
self.assertEqual(validation.validate_document(self.document), [])
blocks.append({"type": "execute", "command": "anything"})
self.assertTrue(validation.validate_document(self.document))
blocks.pop()
blocks[0]["extra"] = "typo"
self.assertTrue(validation.validate_document(self.document))
def test_bad_branding_and_references(self):
for path in ("../secret.svg", "/logo.svg", "C:\\logo.svg", "assets/../logo.svg", "logo.svg\n"):
with self.subTest(path=path):
self.document["branding"]["logo"] = path
self.assertTrue(validation.validate_document(self.document))
self.setUp()
self.document["branding"]["theme"]["accent"] = "not-a-color"
self.assertTrue(validation.validate_document(self.document))
self.setUp()
self.document["findings"][0]["evidence_refs"][0]["pointer"] = "/bad~3escape"
self.assertTrue(validation.validate_document(self.document))
self.setUp()
self.document["findings"].append(copy.deepcopy(self.document["findings"][0]))
self.assertIn("duplicate finding", " ".join(validation.validate_document(self.document)))
def test_cli_parsing_errors_and_non_json_yaml(self):
with tempfile.TemporaryDirectory() as temp:
for suffix, content in [
("yaml", "title: first\ntitle: second\n"),
("yaml", "metadata: {Date: 2026-09-15}\n"),
("yaml", "value: .nan\n"), ("yaml", "value: &cycle [*cycle]\n"),
("yaml", "1: bad key\n"), ("yaml", "value: !!python/object:invalid {}\n"),
("yaml", "base: &base {a: 1}\nvalue: {<<: *base}\n"),
("json", '{"title":"first","title":"second"}'),
("json", '{"value":1e999}'), ("json", '{"value":NaN}'),
("txt", "{}"), ("yaml", "[unclosed"),
]:
with self.subTest(content=content):
path = Path(temp) / ("invalid." + suffix)
path.write_text(content)
result = subprocess.run([sys.executable, str(ROOT / "tests/support_validation.py"), str(path)],
cwd=temp, text=True, capture_output=True)
self.assertEqual(result.returncode, 1, result.stdout)
self.assertNotIn("Traceback", result.stderr)
def test_cli_from_another_directory(self):
with tempfile.TemporaryDirectory() as temp:
result = subprocess.run([sys.executable, str(ROOT / "tests/support_validation.py"),
str(ROOT / "examples/example-report.yaml")], cwd=temp, text=True, capture_output=True)
self.assertEqual(result.returncode, 0, result.stderr)
def test_production_validator_rejects_contract_extensions(self):
with tempfile.TemporaryDirectory() as temp:
document = copy.deepcopy(self.document)
document["collect"] = {}
path = Path(temp) / "invalid.json"
path.write_text(json.dumps(document), encoding="utf-8")
result = subprocess.run(["bash", str(ROOT / "tools/validate.sh"), str(path)],
text=True, capture_output=True)
self.assertEqual(result.returncode, 1)
if __name__ == "__main__":
unittest.main()