162 lines
8.2 KiB
Python
162 lines
8.2 KiB
Python
import copy
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
spec = importlib.util.spec_from_file_location("report_validation", ROOT / "tests/support_validation.py")
|
|
validation = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(validation)
|
|
|
|
|
|
class ReportContractTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.document = validation.load_report(ROOT / "examples/example-report.json")
|
|
|
|
def test_yaml_json_parity_and_minimal_report(self):
|
|
self.assertEqual(self.document, validation.load_report(ROOT / "examples/example-report.yaml"))
|
|
self.assertEqual(validation.validate_document(self.document), [])
|
|
self.assertEqual(validation.validate_document(validation.load_report(ROOT / "examples/minimal-report.json")), [])
|
|
|
|
def test_optional_branding_replacement(self):
|
|
self.document["branding"] = {"organization": "Independent Example", "logo": "assets/example.svg",
|
|
"theme": {"accent": "#112233", "font": "Example Font"}}
|
|
self.assertEqual(validation.validate_document(self.document), [])
|
|
del self.document["branding"]
|
|
self.assertEqual(validation.validate_document(self.document), [])
|
|
|
|
def test_summary_metrics_are_bounded_and_typed(self):
|
|
self.document["metrics"] = [
|
|
{"label": "Critical", "value": 2, "tone": "critical"},
|
|
{"label": "Confidence", "value": "85%", "tone": "info"},
|
|
]
|
|
self.assertEqual(validation.validate_document(self.document), [])
|
|
self.document["metrics"].append({"label": "Invalid", "value": [], "tone": "unknown"})
|
|
self.assertTrue(validation.validate_document(self.document))
|
|
|
|
def test_required_fields_and_versions(self):
|
|
for path, value in [(("schema_version",), 2), (("schema_version",), True),
|
|
(("title",), " "), (("metadata", "Author"), ""),
|
|
(("metadata", "Date"), "2026-02-30"),
|
|
(("metadata", "Date"), "2026-09-15T00:00:00Z")]:
|
|
with self.subTest(path=path):
|
|
doc = copy.deepcopy(self.document)
|
|
target = doc
|
|
for key in path[:-1]:
|
|
target = target[key]
|
|
target[path[-1]] = value
|
|
self.assertTrue(validation.validate_document(doc))
|
|
for key in ("Date", "Author"):
|
|
doc = copy.deepcopy(self.document)
|
|
del doc["metadata"][key]
|
|
self.assertTrue(validation.validate_document(doc))
|
|
|
|
def test_legacy_domain_fields_rejected(self):
|
|
for key in ("diagnostics", "organization", "theme", "collect"):
|
|
with self.subTest(key=key):
|
|
doc = copy.deepcopy(self.document)
|
|
doc[key] = {}
|
|
self.assertTrue(validation.validate_document(doc))
|
|
|
|
def test_all_blocks_and_unknown_nested_blocks(self):
|
|
blocks = [
|
|
{"type": "text", "text": "Text"}, {"type": "bullets", "items": ["Item"]},
|
|
{"type": "steps", "items": ["Step"]}, {"type": "callout", "text": "Note", "kind": "warning"},
|
|
{"type": "code", "text": "echo example"},
|
|
]
|
|
self.document["sections"] = [{"title": "Blocks", "content": [
|
|
{"type": "subsection", "title": "Nested", "content": blocks}]}]
|
|
self.assertEqual(validation.validate_document(self.document), [])
|
|
blocks.append({"type": "execute", "command": "anything"})
|
|
self.assertTrue(validation.validate_document(self.document))
|
|
blocks.pop()
|
|
blocks[0]["extra"] = "typo"
|
|
self.assertTrue(validation.validate_document(self.document))
|
|
|
|
def test_card_block_has_valid_nested_content_and_tone(self):
|
|
card = {"type": "card", "title": "Review", "tone": "warning",
|
|
"content": [{"type": "text", "text": "Check the evidence."},
|
|
{"type": "bullets", "items": ["Preserve the capture."]}]}
|
|
self.document["sections"] = [{"title": "Cards", "content": [card]}]
|
|
self.assertEqual(validation.validate_document(self.document), [])
|
|
card["tone"] = "purple"
|
|
self.assertTrue(validation.validate_document(self.document))
|
|
card["tone"] = "warning"
|
|
card["content"] = []
|
|
self.assertTrue(validation.validate_document(self.document))
|
|
|
|
def test_diagnostic_layout_blocks_share_the_report_contract(self):
|
|
self.document["sections"] = [{"title": "Overview", "page_break_before": True,
|
|
"content": [
|
|
{"type": "progress", "label": "Signal score", "value": 73,
|
|
"tone": "warning"},
|
|
{"type": "table", "columns": ["Property", "Value"],
|
|
"rows": [["Memory", "32 GB"]]},
|
|
{"type": "finding_list", "compact": True,
|
|
"items": [{"title": "Example finding", "severity": "warning"}]},
|
|
{"type": "references", "items": [
|
|
{"label": "Source", "url": "https://example.com/source"}]},
|
|
]}]
|
|
self.assertEqual(validation.validate_document(self.document), [])
|
|
self.document["sections"][0]["content"][1]["rows"] = [["Memory"]]
|
|
self.assertTrue(validation.validate_document(self.document))
|
|
|
|
def test_bad_branding_and_references(self):
|
|
for path in ("../secret.svg", "/logo.svg", "C:\\logo.svg", "assets/../logo.svg", "logo.svg\n"):
|
|
with self.subTest(path=path):
|
|
self.document["branding"]["logo"] = path
|
|
self.assertTrue(validation.validate_document(self.document))
|
|
self.setUp()
|
|
self.document["branding"]["theme"]["accent"] = "not-a-color"
|
|
self.assertTrue(validation.validate_document(self.document))
|
|
self.setUp()
|
|
self.document["findings"][0]["evidence_refs"][0]["pointer"] = "/bad~3escape"
|
|
self.assertTrue(validation.validate_document(self.document))
|
|
self.setUp()
|
|
self.document["findings"].append(copy.deepcopy(self.document["findings"][0]))
|
|
self.assertIn("duplicate finding", " ".join(validation.validate_document(self.document)))
|
|
|
|
def test_cli_parsing_errors_and_non_json_yaml(self):
|
|
with tempfile.TemporaryDirectory() as temp:
|
|
for suffix, content in [
|
|
("yaml", "title: first\ntitle: second\n"),
|
|
("yaml", "metadata: {Date: 2026-09-15}\n"),
|
|
("yaml", "value: .nan\n"), ("yaml", "value: &cycle [*cycle]\n"),
|
|
("yaml", "1: bad key\n"), ("yaml", "value: !!python/object:invalid {}\n"),
|
|
("yaml", "base: &base {a: 1}\nvalue: {<<: *base}\n"),
|
|
("json", '{"title":"first","title":"second"}'),
|
|
("json", '{"value":1e999}'), ("json", '{"value":NaN}'),
|
|
("txt", "{}"), ("yaml", "[unclosed"),
|
|
]:
|
|
with self.subTest(content=content):
|
|
path = Path(temp) / ("invalid." + suffix)
|
|
path.write_text(content)
|
|
result = subprocess.run([sys.executable, str(ROOT / "tests/support_validation.py"), str(path)],
|
|
cwd=temp, text=True, capture_output=True)
|
|
self.assertEqual(result.returncode, 1, result.stdout)
|
|
self.assertNotIn("Traceback", result.stderr)
|
|
|
|
def test_cli_from_another_directory(self):
|
|
with tempfile.TemporaryDirectory() as temp:
|
|
result = subprocess.run([sys.executable, str(ROOT / "tests/support_validation.py"),
|
|
str(ROOT / "examples/example-report.yaml")], cwd=temp, text=True, capture_output=True)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
|
|
def test_production_validator_rejects_contract_extensions(self):
|
|
with tempfile.TemporaryDirectory() as temp:
|
|
document = copy.deepcopy(self.document)
|
|
document["collect"] = {}
|
|
path = Path(temp) / "invalid.json"
|
|
path.write_text(json.dumps(document), encoding="utf-8")
|
|
result = subprocess.run(["bash", str(ROOT / "tools/validate.sh"), str(path)],
|
|
text=True, capture_output=True)
|
|
self.assertEqual(result.returncode, 1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|