reportist/plugins/diagnostics/tests/support_validation.py

136 lines
5.3 KiB
Python

#!/usr/bin/env python3
"""Validate diagnostic contract v1, optionally verifying its raw evidence."""
import argparse
import hashlib
import json
from pathlib import Path
import sys
try:
import rfc3339_validator # Required so date-time formats cannot be silently ignored.
from jsonschema import Draft202012Validator, FormatChecker
except ImportError:
raise SystemExit("Prepare test dependencies with: ./tests/test.sh")
SCHEMA = json.loads((Path(__file__).resolve().parents[1] / "schemas/diagnostic.schema.json").read_text())
Draft202012Validator.check_schema(SCHEMA)
VALIDATOR = Draft202012Validator(SCHEMA, format_checker=FormatChecker())
def unique_object(pairs):
result = {}
for key, value in pairs:
if key in result:
raise ValueError(f"duplicate JSON key: {key}")
result[key] = value
return result
def reject_constant(value):
raise ValueError(f"non-JSON number: {value}")
def load_json(path):
document = json.loads(Path(path).read_text(encoding="utf-8"),
object_pairs_hook=unique_object, parse_constant=reject_constant)
json.dumps(document, allow_nan=False) # Also reject exponent overflow such as 1e999.
return document
def pointer_path(parts):
return "/" + "/".join(str(p).replace("~", "~0").replace("/", "~1") for p in parts)
def validate_document(document):
json.dumps(document, allow_nan=False)
errors = [f"{pointer_path(e.absolute_path)}: {e.message}" for e in VALIDATOR.iter_errors(document)]
if errors:
return sorted(errors)
observations = document["observations"]
ids = set()
for index, observation in enumerate(observations):
if observation["id"] in ids:
errors.append(f"/observations/{index}/id: duplicate observation ID")
ids.add(observation["id"])
if observation["section"] not in document["collection"]["sections"]:
errors.append(f"/observations/{index}/section: unknown collection section")
findings = set()
for index, finding in enumerate(document["findings"]):
if finding["id"] in findings:
errors.append(f"/findings/{index}/id: duplicate finding ID")
findings.add(finding["id"])
for evidence in finding["evidence"]:
if evidence not in ids:
errors.append(f"/findings/{index}/evidence: unknown observation {evidence}")
sections = document["collection"]["sections"]
if document["collection"]["status"] == "complete" and (
not sections or any(s["status"] != "ok" or s["truncated"] for s in sections.values())
):
errors.append("/collection/status: complete requires nonempty, successful, untruncated sections")
return errors
def resolve_pointer(document, pointer):
value = document
for encoded in pointer.split("/")[1:] if pointer else []:
token = encoded.replace("~1", "/").replace("~0", "~")
if isinstance(value, list):
if not token.isascii() or not token.isdecimal() or (len(token) > 1 and token[0] == "0"):
raise ValueError("invalid array index")
value = value[int(token)]
elif isinstance(value, dict):
value = value[token]
else:
raise ValueError("pointer traverses a scalar")
return value
def verify_evidence(document, root):
root = Path(root).resolve()
source = document["metadata"]["source"]
artifact = (root / source["artifact"]).resolve()
if not artifact.is_relative_to(root):
return ["/metadata/source/artifact: resolved path escapes evidence root"]
raw = artifact.read_bytes()
if hashlib.sha256(raw).hexdigest() != source["sha256"]:
return ["/metadata/source/sha256: evidence checksum differs"]
capture = load_json(artifact)
if (not isinstance(capture, dict) or type(capture.get("schema_version")) is not int
or capture["schema_version"] != source["schema_version"]):
return ["/metadata/source/schema_version: capture version differs"]
errors = []
for index, observation in enumerate(document["observations"]):
try:
actual = resolve_pointer(capture, observation["source_pointer"])
except (KeyError, IndexError, ValueError):
errors.append(f"/observations/{index}/source_pointer: does not resolve in evidence")
continue
if json.dumps(actual, sort_keys=True) != json.dumps(observation["value"], sort_keys=True):
errors.append(f"/observations/{index}/value: differs from referenced raw evidence")
return errors
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("input", type=Path)
parser.add_argument("--evidence-root", type=Path, help="Also check capture SHA-256 and observation pointers/values")
args = parser.parse_args(argv)
try:
document = load_json(args.input)
errors = validate_document(document)
if not errors and args.evidence_root is not None:
errors = verify_evidence(document, args.evidence_root)
except (OSError, ValueError, RecursionError) as error:
print(f"error: {error}", file=sys.stderr)
return 1
if errors:
print("\n".join(errors), file=sys.stderr)
return 1
print(f"Valid diagnostic v1: {args.input}")
return 0
if __name__ == "__main__":
sys.exit(main())