274 lines
15 KiB
Python
274 lines
15 KiB
Python
import copy
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
FIXTURES = ROOT / "tests" / "fixtures" / "legacy"
|
|
spec = importlib.util.spec_from_file_location("diagnostic_analyse", ROOT / "tests" / "support_analyse.py")
|
|
analyser = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(analyser)
|
|
|
|
|
|
class AnalyseTests(unittest.TestCase):
|
|
reference_time = "2026-09-15T12:00:00Z"
|
|
|
|
def setUp(self):
|
|
self.temporary = tempfile.TemporaryDirectory()
|
|
self.work = Path(self.temporary.name)
|
|
|
|
def tearDown(self):
|
|
self.temporary.cleanup()
|
|
|
|
def run_fixture(self, name, output_name=None):
|
|
return analyser.analyse_file(
|
|
FIXTURES / name, FIXTURES, self.work / (output_name or name),
|
|
self.reference_time, "test-revision")
|
|
|
|
def test_all_sanitized_legacy_fixtures_produce_valid_artifacts(self):
|
|
expected = {
|
|
"windiagnostics-v1.json": ("complete", 1, 1),
|
|
"windiagnostics-v2-compact.json": ("partial", 0, 0),
|
|
"windiagnostics-v2-full.json": ("partial", 3, 1),
|
|
"windiagnostics-v2-partial.json": ("partial", 3, 1),
|
|
"windiagnostics-watchlist.json": ("partial", 11, 7),
|
|
}
|
|
for name, values in expected.items():
|
|
with self.subTest(name=name):
|
|
result = self.run_fixture(name)
|
|
self.assertEqual((result["collection"]["status"], len(result["observations"]),
|
|
len(result["findings"])), values)
|
|
|
|
def test_service_event_rule_is_traceable(self):
|
|
result = self.run_fixture("windiagnostics-v1.json")
|
|
finding = result["findings"][0]
|
|
observation = result["observations"][0]
|
|
self.assertEqual((finding["severity"], finding["confidence"]), ("warning", 90))
|
|
self.assertEqual(finding["evidence"], [observation["id"]])
|
|
self.assertEqual(observation["source_pointer"], "/sections/recent_system_errors/data/0")
|
|
self.assertEqual(observation["value"]["Id"], 7000)
|
|
|
|
def test_first_rule_batch_matches_reviewed_severity_and_confidence(self):
|
|
expected = {
|
|
("Microsoft-Windows-WER-SystemErrorReporting", 1001): ("critical", 100),
|
|
("volmgr", 162): ("info", 100), ("stornvme", 11): ("critical", 92),
|
|
("Microsoft-Windows-TPM-WMI", 1040): ("critical", 85),
|
|
("Service Control Manager", 7000): ("warning", 90),
|
|
("Service Control Manager", 7009): ("warning", 82),
|
|
("Microsoft-Windows-NDIS", 10317): ("warning", 78),
|
|
("Application Error", 1000): ("warning", 75),
|
|
("Microsoft-Windows-WindowsUpdateClient", 20): ("warning", 88),
|
|
("Application Hang", 1002): ("warning", 70),
|
|
("MsiInstaller", 1043): ("warning", 80), ("Volsnap", 36): ("warning", 82),
|
|
("Service Control Manager", 7043): ("warning", 75),
|
|
("Microsoft-Windows-DistributedCOM", 10010): ("info", 65),
|
|
("EventLog", 6008): ("warning", 95),
|
|
}
|
|
actual = {key: (rule["severity"], rule["confidence"])
|
|
for key, rule in analyser.EVENT_RULES.items()}
|
|
self.assertEqual({key: actual[key] for key in expected}, expected)
|
|
capture = analyser.validation.load_json(FIXTURES / "windiagnostics-v1.json")
|
|
capture = copy.deepcopy(capture)
|
|
events = [{"TimeCreated": "2026-09-14T12:00:00Z", "ProviderName": provider,
|
|
"Id": event_id, "Message": "Synthetic matching event"}
|
|
for provider, event_id in expected]
|
|
capture["sections"]["recent_system_errors"]["data"] = events
|
|
capture["sections"]["recent_system_errors"]["count"] = len(events)
|
|
result = analyser.analyse(capture, "capture.json", "0" * 64,
|
|
self.reference_time, "test-revision")
|
|
self.assertEqual(len(result["findings"]), len(expected))
|
|
self.assertEqual({(finding["title"].rsplit(" event ", 1)[0],
|
|
int(finding["title"].rsplit(" event ", 1)[1])):
|
|
(finding["severity"], finding["confidence"])
|
|
for finding in result["findings"]}, expected)
|
|
|
|
def test_complete_rule_catalog_matches_legacy_classification_metadata(self):
|
|
catalog = sorted((provider, event_id, rule["severity"], rule["confidence"],
|
|
bool(rule.get("fading")), rule.get("fade_after_days", 7),
|
|
rule.get("fade_to_half_days", 90))
|
|
for (provider, event_id), rule in analyser.EVENT_RULES.items())
|
|
encoded = json.dumps(catalog, separators=(",", ":"), ensure_ascii=False).encode()
|
|
self.assertEqual(len(catalog), 53)
|
|
self.assertEqual(hashlib.sha256(encoded).hexdigest(),
|
|
"00c4b492d013a4f10e8a07da1f6a6819010061b3085819f143311b8279b0c7dc")
|
|
|
|
def test_unknown_duplicate_events_are_grouped_without_interpretation(self):
|
|
result = self.run_fixture("windiagnostics-v2-full.json")
|
|
self.assertEqual(len(result["findings"]), 1)
|
|
self.assertEqual(result["findings"][0]["severity"], "unanalysed")
|
|
self.assertEqual(result["findings"][0]["confidence"], 0)
|
|
self.assertEqual(len(result["findings"][0]["evidence"]), 3)
|
|
|
|
def test_reliability_history_adds_actionable_events_without_duplicates(self):
|
|
capture = analyser.validation.load_json(FIXTURES / "windiagnostics-v2-full.json")
|
|
capture = copy.deepcopy(capture)
|
|
event = {"TimeCreated": "2026-09-14T12:00:00.100Z",
|
|
"ProviderName": "Service Control Manager", "Id": 7000,
|
|
"Message": "Synthetic service failure"}
|
|
capture["sections"]["recent_system_events"]["data"] = [event]
|
|
capture["sections"]["recent_system_events"]["count"] = 1
|
|
capture["sections"]["recent_application_events"]["data"] = []
|
|
capture["sections"]["recent_application_events"]["count"] = 0
|
|
capture["sections"]["reliability_records"]["data"] = [
|
|
{"TimeGenerated": "2026-09-14T12:00:00.900Z",
|
|
"SourceName": "Service Control Manager", "EventIdentifier": "7000",
|
|
"Message": "Synthetic service failure"},
|
|
{"TimeGenerated": "2026-09-13T12:00:00Z",
|
|
"SourceName": "Service Control Manager", "EventIdentifier": 7000,
|
|
"Message": "Separate reliability failure"},
|
|
{"TimeGenerated": "2026-09-13T12:00:00Z", "SourceName": "EventLog",
|
|
"EventIdentifier": 1, "Message": "Unreviewed reliability noise"},
|
|
]
|
|
capture["sections"]["reliability_records"]["count"] = 3
|
|
result = analyser.analyse(capture, "capture.json", "0" * 64,
|
|
self.reference_time, "test-revision")
|
|
finding = next(item for item in result["findings"]
|
|
if item["id"] == "event-service-control-manager-7000")
|
|
self.assertEqual(finding["evidence"], ["event-recent_system_events-1",
|
|
"event-reliability-records-2"])
|
|
self.assertNotIn("event-reliability-records-1",
|
|
{item["id"] for item in result["observations"]})
|
|
|
|
def test_watchlist_fixture_emits_six_traceable_candidates(self):
|
|
result = self.run_fixture("windiagnostics-watchlist.json")
|
|
findings = {finding["id"]: finding for finding in result["findings"]
|
|
if finding["id"].startswith("watchlist-")}
|
|
self.assertEqual(set(findings), {
|
|
"watchlist-winring0", "watchlist-dell-dbutil", "watchlist-gigabyte-oc-guru",
|
|
"watchlist-obs-overlay", "watchlist-obs-audio-hooks", "watchlist-smb1",
|
|
})
|
|
self.assertNotIn("watchlist-intel-wifi-observed", findings)
|
|
observations = {item["id"]: item for item in result["observations"]}
|
|
for finding in findings.values():
|
|
self.assertEqual(finding["severity"], "warning")
|
|
self.assertEqual(finding["confidence"], 50)
|
|
self.assertTrue(all(item in observations for item in finding["evidence"]))
|
|
self.assertIn("/sections/system_drivers/data/0",
|
|
{item["source_pointer"] for item in observations.values()})
|
|
|
|
def test_watchlist_exact_version_enabled_and_requirement_gates(self):
|
|
capture = analyser.validation.load_json(FIXTURES / "windiagnostics-watchlist.json")
|
|
capture = copy.deepcopy(capture)
|
|
capture["sections"]["installed_programs"]["data"][2]["DisplayVersion"] = "2.09"
|
|
capture["sections"]["enabled_windows_features"]["data"][0]["State"] = 1
|
|
capture["sections"]["installed_programs"]["data"] = [
|
|
item for item in capture["sections"]["installed_programs"]["data"]
|
|
if item["DisplayName"] != "OBS Studio"]
|
|
result = analyser.analyse(capture, "capture.json", "0" * 64,
|
|
self.reference_time, "test-revision")
|
|
ids = {finding["id"] for finding in result["findings"]}
|
|
self.assertNotIn("watchlist-gigabyte-oc-guru", ids)
|
|
self.assertNotIn("watchlist-smb1", ids)
|
|
self.assertNotIn("watchlist-obs-overlay", ids)
|
|
self.assertNotIn("watchlist-obs-audio-hooks", ids)
|
|
|
|
def test_watchlist_event_gate_requires_repeated_matching_events(self):
|
|
capture = analyser.validation.load_json(FIXTURES / "windiagnostics-watchlist.json")
|
|
capture = copy.deepcopy(capture)
|
|
capture["sections"]["system_drivers"]["data"].append(
|
|
{"Name": "NETWTW10", "PathName": "C:\\Drivers\\netwtw10.sys", "State": "Running"})
|
|
for event, event_id in zip(capture["sections"]["recent_system_events"]["data"], (5002, 5005)):
|
|
event["ProviderName"] = "Netwtw10"
|
|
event["Id"] = event_id
|
|
result = analyser.analyse(capture, "capture.json", "0" * 64,
|
|
self.reference_time, "test-revision")
|
|
finding = next(item for item in result["findings"]
|
|
if item["id"] == "watchlist-intel-wifi-observed")
|
|
self.assertEqual(len(finding["evidence"]), 3)
|
|
self.assertTrue(any(item.startswith("event-") for item in finding["evidence"]))
|
|
|
|
def test_partial_collection_caps_known_rule_confidence(self):
|
|
capture = analyser.validation.load_json(FIXTURES / "windiagnostics-v1.json")
|
|
capture = copy.deepcopy(capture)
|
|
capture["sections"]["storage"] = {"status": "unavailable", "count": 0,
|
|
"data": [], "error": "Synthetic failure"}
|
|
result = analyser.analyse(capture, "capture.json", "0" * 64,
|
|
self.reference_time, "test-revision")
|
|
self.assertEqual(result["collection"]["status"], "partial")
|
|
self.assertEqual(result["findings"][0]["confidence"], 50)
|
|
|
|
def test_fading_uses_explicit_reference_time(self):
|
|
capture = analyser.validation.load_json(FIXTURES / "windiagnostics-v1.json")
|
|
capture = copy.deepcopy(capture)
|
|
capture["sections"]["recent_system_errors"]["data"][0]["TimeCreated"] = "2025-01-01T00:00:00Z"
|
|
result = analyser.analyse(capture, "capture.json", "0" * 64,
|
|
self.reference_time, "test-revision")
|
|
finding = result["findings"][0]
|
|
self.assertGreater(finding["age_days"], 90)
|
|
self.assertEqual(finding["relevance_weight"], 0.5)
|
|
|
|
def test_unsupported_status_is_preserved_as_limitation(self):
|
|
result = self.run_fixture("windiagnostics-v2-partial.json")
|
|
unexpected = [section for section in result["collection"]["sections"].values()
|
|
if section.get("reason", "").startswith("Legacy capture used unsupported")]
|
|
self.assertTrue(unexpected)
|
|
self.assertTrue(any("incomplete" in warning for warning in result["warnings"]))
|
|
|
|
def test_unsupported_version_and_bad_reference_time_fail(self):
|
|
capture = analyser.validation.load_json(FIXTURES / "windiagnostics-v1.json")
|
|
invalid = copy.deepcopy(capture)
|
|
invalid["schema_version"] = 3
|
|
with self.assertRaisesRegex(ValueError, "unsupported"):
|
|
analyser.analyse(invalid, "capture.json", "0" * 64,
|
|
self.reference_time, "test-revision")
|
|
with self.assertRaisesRegex(ValueError, "reference-time"):
|
|
analyser.analyse(capture, "capture.json", "0" * 64,
|
|
"2026-09-15", "test-revision")
|
|
|
|
def test_missing_sections_and_malformed_json_fail(self):
|
|
capture = analyser.validation.load_json(FIXTURES / "windiagnostics-v1.json")
|
|
del capture["sections"]
|
|
with self.assertRaisesRegex(ValueError, "/sections"):
|
|
analyser.analyse(capture, "capture.json", "0" * 64,
|
|
self.reference_time, "test-revision")
|
|
malformed = self.work / "malformed.json"
|
|
malformed.write_text('{"schema_version":', encoding="utf-8")
|
|
with self.assertRaises(ValueError):
|
|
analyser.analyse_file(malformed, self.work, self.work / "output.json",
|
|
self.reference_time, "test-revision")
|
|
|
|
def test_output_is_deterministic_and_not_overwritten_implicitly(self):
|
|
first = self.run_fixture("windiagnostics-v1.json", "first.json")
|
|
second = self.run_fixture("windiagnostics-v1.json", "second.json")
|
|
self.assertEqual(first, second)
|
|
self.assertEqual((self.work / "first.json").read_bytes(),
|
|
(self.work / "second.json").read_bytes())
|
|
with self.assertRaisesRegex(ValueError, "output exists"):
|
|
self.run_fixture("windiagnostics-v1.json", "first.json")
|
|
|
|
def test_cli_runs_outside_repository(self):
|
|
output = self.work / "diagnostic.json"
|
|
result = subprocess.run([
|
|
"bash", str(ROOT / "analyse.sh"),
|
|
str(FIXTURES / "windiagnostics-v1.json"),
|
|
"--evidence-root", str(FIXTURES), "--output", str(output),
|
|
"--reference-time", self.reference_time, "--producer-version", "test-revision",
|
|
], cwd=self.work, text=True, capture_output=True)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
|
|
def test_standalone_component_copy_analyses_fixture(self):
|
|
standalone = self.work / "standalone"
|
|
shutil.copytree(ROOT, standalone, ignore=shutil.ignore_patterns(
|
|
".git", ".ci-venv", "__pycache__", "*.pyc"))
|
|
fixture = standalone / "tests" / "fixtures" / "legacy" / "windiagnostics-v1.json"
|
|
output = self.work / "standalone-diagnostic.json"
|
|
result = subprocess.run([
|
|
"bash", str(standalone / "analyse.sh"), str(fixture),
|
|
"--evidence-root", str(fixture.parent), "--output", str(output),
|
|
"--reference-time", self.reference_time,
|
|
"--producer-version", "standalone-test",
|
|
], cwd=self.work, text=True, capture_output=True)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual(json.loads(output.read_text())["schema_version"], 1)
|
|
self.assertTrue(output.is_file())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|