This repository has been archived on 2026-09-16. You can view files and clone it, but cannot push or open issues or pull requests.
win-diagnostics-analysis/tools/analyse.py

580 lines
29 KiB
Python

#!/usr/bin/env python3
"""Analyse a stored legacy capture into a validated diagnostic v1 artifact."""
import argparse
from collections import OrderedDict
from datetime import datetime
import hashlib
import importlib.util
import json
import os
from pathlib import Path
import re
import sys
import tempfile
ROOT = Path(__file__).resolve().parents[1]
spec = importlib.util.spec_from_file_location("diagnostic_validation", ROOT / "tools" / "validate.py")
validation = importlib.util.module_from_spec(spec)
spec.loader.exec_module(validation)
EVENT_RULES = {
("Microsoft-Windows-WER-SystemErrorReporting", 1001): {
"severity": "critical", "confidence": 100,
"description": "Windows reports a reboot following a kernel bugcheck. The stop code and saved dump confirm a crash but do not identify its cause.",
},
("volmgr", 162): {
"severity": "info", "confidence": 100,
"description": "Windows records successful crash-dump generation; this supports a crash finding rather than proving a storage failure.",
},
("stornvme", 11): {
"severity": "critical", "confidence": 92,
"description": "The storage controller path reported an I/O controller error.",
},
("Microsoft-Windows-TPM-WMI", 1040): {
"severity": "critical", "confidence": 85,
"description": "Pre-attestation health checks reported a critical component failure.",
},
("Service Control Manager", 7000): {
"severity": "warning",
"confidence": 90,
"description": "A configured service or driver failed to start.",
"fading": True,
},
("Service Control Manager", 7009): {
"severity": "warning", "confidence": 82, "fading": True,
"description": "A service did not connect before the Service Control Manager timeout.",
},
("Microsoft-Windows-NDIS", 10317): {
"severity": "warning", "confidence": 78,
"description": "A network miniport failed a transition to operational power; correlation with connectivity symptoms is required.",
},
("Application Error", 1000): {
"severity": "warning", "confidence": 75, "fading": True,
"description": "An application terminated unexpectedly; the application, module and exception code require review.",
},
("Microsoft-Windows-WindowsUpdateClient", 20): {
"severity": "warning", "confidence": 88, "fading": True,
"description": "An update installation failed.",
},
("Application Hang", 1002): {
"severity": "warning", "confidence": 70, "fading": True,
"description": "An application stopped responding and was closed.",
},
("MsiInstaller", 1043): {
"severity": "warning", "confidence": 80, "fading": True,
"description": "Windows Installer could not finish a transaction.",
},
("Volsnap", 36): {
"severity": "warning", "confidence": 82,
"description": "A volume shadow-copy operation was aborted.",
},
("Service Control Manager", 7043): {
"severity": "warning", "confidence": 75, "fading": True,
"description": "A service did not shut down cleanly.",
},
("Microsoft-Windows-DistributedCOM", 10010): {
"severity": "info", "confidence": 65,
"description": "A DCOM registration timeout is contextual unless repeated or correlated with symptoms.",
},
("EventLog", 6008): {
"severity": "warning", "confidence": 95,
"description": "Windows recorded an unexpected shutdown; this confirms an unclean stop but does not identify its cause.",
},
("Service Control Manager", 7023): {
"severity": "warning", "confidence": 88, "fading": True,
"description": "A service terminated and returned an error; impact depends on the named service and whether its function is needed.",
},
("MsiInstaller", 11704): {
"severity": "warning", "confidence": 92, "fading": True,
"description": "Windows Installer reported a suspended installation that blocked another product transaction.",
},
("MsiInstaller", 11722): {
"severity": "warning", "confidence": 92, "fading": True,
"description": "A program executed by an MSI package did not complete as expected, so the installation was not successful.",
},
("VSS", 12344): {
"severity": "warning", "confidence": 82,
"description": "A Volume Shadow Copy writer failed during its freeze operation; backups or restore points near that time may be incomplete.",
},
(".NET Runtime", 1022): {
"severity": "info", "confidence": 80,
"description": "The process could not initialize profiler-attach support; this is contextual unless profiling was expected.",
},
("Waterfox Default Browser Agent", 5): {
"severity": "info", "confidence": 55,
"description": "The captured event has no message, so it provides no actionable evidence by itself.",
},
("Microsoft-Windows-Kernel-Boot", 29): {
"severity": "warning", "confidence": 92,
"description": "Windows failed Fast Startup; the status and adjacent shutdown events do not establish the underlying cause.",
},
("Service Control Manager", 7031): {
"severity": "warning", "confidence": 95,
"description": "A service terminated unexpectedly; Windows may schedule a recovery action.",
},
("Microsoft-Windows-DeviceAssociationService", 3505): {
"severity": "warning", "confidence": 70,
"description": "Device association failed during pairing enumeration; the message does not identify a failed hardware component.",
},
("Microsoft-Windows-HAL", 15): {
"severity": "warning", "confidence": 90,
"description": "The IOMMU reported an error; its raw identifier does not identify a component or prove physical failure.",
},
("Microsoft-Windows-HAL", 21): {
"severity": "warning", "confidence": 90,
"description": "The ACPI time/alarm method failed while Windows attempted to set the hardware real-time clock.",
},
("usbaudio2", 38): {
"severity": "warning", "confidence": 90,
"description": "A USB audio control request failed.",
},
("ESENT", 419): {
"severity": "critical", "confidence": 95,
"description": "The database engine could not read a database page; the recorded error identifies the failure class.",
},
("ESENT", 474): {
"severity": "critical", "confidence": 98,
"description": "A database page failed checksum verification; this confirms corrupt read data but does not isolate its cause.",
},
("ESENT", 907): {
"severity": "critical", "confidence": 95,
"description": "ESENT detected a transient memory problem; this is a stability signal, not proof of a defective RAM module.",
},
("ESENT", 908): {
"severity": "critical", "confidence": 95,
"description": "The database process terminated after a fatal error; its error code and adjacent ESENT events require correlation.",
},
("Microsoft-Windows-Defrag", 264): {
"severity": "warning", "confidence": 92,
"description": "Storage optimization did not complete; an unsupported filesystem is not evidence of a failing disk.",
},
("Microsoft-Windows-WLAN-AutoConfig", 10002): {
"severity": "info", "confidence": 85,
"description": "A WLAN extension stopped; this can accompany adapter shutdown or reset and does not establish a driver defect.",
},
("Microsoft-Windows-DistributedCOM", 10016): {
"severity": "info", "confidence": 90,
"description": "DCOM denied an activation request; common Windows component instances are expected and do not alone establish a fault.",
},
("Netwtw10", 6062): {
"severity": "info", "confidence": 60,
"description": "The driver logged an LSO trigger; the message alone does not establish adapter failure.",
},
("Microsoft-Windows-Hyper-V-Hypervisor", 167): {
"severity": "warning", "confidence": 90,
"description": "VM side-channel mitigations were not enabled with HyperThreading; this is a security configuration signal.",
},
("volmgr", 161): {
"severity": "warning", "confidence": 95,
"description": "Windows could not create a crash dump; this reduces crash evidence without identifying the crash cause.",
},
("Microsoft-Windows-WinRM", 10149): {
"severity": "info", "confidence": 90,
"description": "WinRM is not listening for remote-management requests; this may be intentional.",
},
("winsrvext", 100): {
"severity": "info", "confidence": 85,
"description": "The named process delayed shutdown; a brief delay does not establish system instability.",
},
("Netwtw10", 5005): {
"severity": "warning", "confidence": 95,
"description": "The Intel wireless driver reported an internal adapter error and failure.",
},
("Netwtw10", 5032): {
"severity": "warning", "confidence": 90,
"description": "The Intel wireless miniport reset watchdog was triggered.",
},
("Netwtw10", 5002): {
"severity": "warning", "confidence": 95,
"description": "The Intel driver reported an adapter malfunction.",
},
("Netwtw10", 5010): {
"severity": "warning", "confidence": 95,
"description": "The wireless adapter returned an invalid value to the Intel driver.",
},
("Microsoft-Windows-Perflib", 1008): {
"severity": "warning", "confidence": 90,
"description": "A performance-counter provider failed to open; counters may be unavailable without implying hardware failure.",
},
("Microsoft-Windows-WMI", 63): {
"severity": "info", "confidence": 90,
"description": "A WMI provider registered with LocalSystem privileges; this describes a privilege boundary, not proof of exploitation.",
},
("Microsoft-Windows-Winlogon", 6004): {
"severity": "warning", "confidence": 80,
"description": "A Winlogon notification subscriber failed; impact depends on the subscriber and affected operation.",
},
("Microsoft-Windows-RestartManager", 10010): {
"severity": "info", "confidence": 85,
"description": "Restart Manager could not restart an application across the reported security identity boundary.",
},
("Microsoft-Windows-Search", 10024): {
"severity": "warning", "confidence": 85,
"description": "Windows Search terminated a filter host that stopped responding; indexing or a file-format filter may be affected.",
},
("Microsoft-Windows-CodeIntegrity", 3116): {
"severity": "info", "confidence": 95,
"description": "Policy signature metadata was recorded; this message does not report a blocked driver.",
},
("Microsoft-Windows-CodeIntegrity", 3099): {
"severity": "info", "confidence": 95,
"description": "An App Control policy was loaded; activation is not itself a driver failure.",
},
("Microsoft-Windows-CodeIntegrity", 3085): {
"severity": "info", "confidence": 90,
"description": "WHQL signing requirements were not enforced for this boot; other signature checks may still be active.",
},
("Microsoft-Windows-CodeIntegrity", 3076): {
"severity": "warning", "confidence": 95,
"description": "App Control audit policy would have blocked a file; audit mode did not block it.",
},
("Microsoft-Windows-CodeIntegrity", 3077): {
"severity": "warning", "confidence": 95,
"description": "App Control blocked a file that did not satisfy its policy.",
},
}
WATCHLIST = (
{"id": "winring0", "title": "WinRing0 low-level driver vulnerability candidate",
"match": ({"section": "system_drivers", "field": "PathName", "basename": True,
"values": ("WinRing0.sys", "WinRing0x64.sys")},),
"description": "Microsoft documents WinRing0 as vulnerable; a filename match does not verify the binary or prove exploitation."},
{"id": "dell-dbutil", "title": "Dell dbutil_2_3 driver vulnerability candidate",
"match": ({"section": "system_drivers", "field": "PathName", "basename": True,
"values": ("dbutil_2_3.sys",)},),
"description": "Dell documents an access-control vulnerability in dbutil_2_3.sys; file identity and remediation state require verification."},
{"id": "gigabyte-oc-guru", "title": "GIGABYTE OC GURU II 2.08 vulnerability candidate",
"match": ({"section": "installed_programs", "field": "DisplayName",
"values": ("OC GURU II", "GIGABYTE OC GURU II"),
"version_field": "DisplayVersion", "versions": ("2.08",)},),
"description": "The vendor lists this exact software version as affected; other versions are not inferred to be affected."},
{"id": "intel-wifi-observed", "title": "Intel wireless driver repeated-failure candidate",
"match": ({"section": "system_drivers", "field": "Name", "values": ("Netwtw10",)},),
"events": {"provider": "Netwtw10", "ids": ("5002", "5005", "5010", "5032"), "minimum": 2},
"description": "An installed Intel wireless driver has repeated matching failure signals; correlation does not prove a shared defect or cause."},
{"id": "obs-overlay", "title": "OBS and overlay-tool conflict candidate",
"match": ({"section": "installed_programs", "field": "DisplayName",
"values": ("MSI Afterburner", "RivaTuner Statistics Server", "EVGA Precision X1",
"ASUS GPU Tweak II", "Overwolf", "FRAPS")},),
"requires": ({"section": "installed_programs", "field": "DisplayName", "values": ("OBS Studio",)},),
"description": "OBS documents possible hook conflicts with these overlay tools; co-installation does not prove simultaneous use or failure."},
{"id": "obs-audio-hooks", "title": "OBS and audio-enhancement conflict candidate",
"match": ({"section": "installed_programs", "field": "DisplayName",
"values": ("Nahimic", "Nahimic Audio", "Sonic Radar", "Sonic Studio")},
{"section": "services", "field": "Name", "values": ("NahimicService",)}),
"requires": ({"section": "installed_programs", "field": "DisplayName", "values": ("OBS Studio",)},),
"description": "OBS documents possible interference from these audio tools; installation does not establish an active conflict."},
{"id": "smb1", "title": "SMBv1 enabled security-risk candidate",
"match": ({"section": "enabled_windows_features", "field": "FeatureName",
"values": ("SMB1Protocol", "SMB1Protocol-Client", "SMB1Protocol-Server"),
"enabled_only": True},),
"description": "Microsoft documents significant SMBv1 vulnerabilities; an enabled feature is not evidence of an attack."},
)
def pointer(*parts):
return "/" + "/".join(str(part).replace("~", "~0").replace("/", "~1") for part in parts)
def slug(value):
result = re.sub(r"[^A-Za-z0-9._:-]+", "-", str(value)).strip("-").lower()
return result or "unknown"
def valid_timestamp(value):
if not isinstance(value, str):
return None
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError:
return None
return value if parsed.tzinfo is not None else None
def fading_metrics(rule, evidence, observations, reference_time):
if not rule.get("fading"):
return {}
observed = [datetime.fromisoformat(observations[item]["observed_at"].replace("Z", "+00:00"))
for item in evidence if "observed_at" in observations[item]]
if not observed:
return {"relevance_weight": 1.0}
reference = datetime.fromisoformat(reference_time.replace("Z", "+00:00"))
age = max(0, (reference.date() - max(observed).date()).days)
if age <= 7:
factor = 1.0
else:
factor = max(0.5, 1.0 - 0.5 * (age - 7) / (90 - 7))
return {"relevance_weight": round(factor, 6), "age_days": age}
def normalize_section(section):
if not isinstance(section, dict):
return {"status": "unavailable", "record_count": None, "truncated": False,
"reason": "Legacy section is not an object."}
source_status = section.get("status")
status = source_status if source_status in ("ok", "skipped", "unavailable") else "unavailable"
count = section.get("count")
if type(count) is not int or count < 0:
data = section.get("data")
count = len(data) if isinstance(data, list) else None
result = {"status": status, "record_count": count,
"truncated": bool(section.get("truncated", False))}
if status != "ok":
reason = section.get("error")
if not isinstance(reason, str) or not reason.strip():
reason = (f"Legacy capture used unsupported section status: {source_status}."
if source_status not in ("skipped", "unavailable")
else f"Legacy capture marked section {status} without a reason.")
result["reason"] = reason
return result
def watch_matches(capture, matcher):
section = capture["sections"].get(matcher["section"])
if not isinstance(section, dict) or section.get("status") != "ok":
return []
data = section.get("data")
if not isinstance(data, list):
return []
expected = {str(value).casefold() for value in matcher["values"]}
matches = []
for index, item in enumerate(data):
if not isinstance(item, dict):
continue
value = str(item.get(matcher["field"], "")).strip()
identity = value.strip('"').replace("\\", "/").rsplit("/", 1)[-1] if matcher.get("basename") else value
if identity.casefold() not in expected:
continue
versions = matcher.get("versions")
if versions and str(item.get(matcher.get("version_field", "DriverVersion"), "")) not in versions:
continue
if matcher.get("enabled_only") and item.get("State") not in (2, "Enabled", "enabled"):
continue
matches.append((matcher["section"], index, item))
return matches
def selected_event_sections(capture):
sections = capture["sections"]
version = capture["schema_version"]
names = (("recent_system_errors", "recent_application_errors") if version == 1 else
("recent_system_events", "recent_application_events"))
return [(name, sections[name]) for name in names if name in sections]
def event_identity(event):
provider = str(event.get("ProviderName", "Unknown provider"))
raw_id = event.get("Id", "unknown")
event_id = raw_id if type(raw_id) in (int, str) else "unknown"
return provider, event_id
def event_signature(provider, event_id, timestamp, message):
normalized_message = " ".join(str(message or "").split())
parsed = valid_timestamp(timestamp)
second = datetime.fromisoformat(parsed.replace("Z", "+00:00")).replace(microsecond=0) if parsed else timestamp
return provider, str(event_id), second, normalized_message
def analyse(capture, artifact, artifact_hash, reference_time, producer_version):
version = capture.get("schema_version")
if type(version) is not int or version not in (1, 2):
raise ValueError(f"unsupported legacy capture schema version: {version!r}")
if not isinstance(capture.get("sections"), dict):
raise ValueError("legacy capture requires an object at /sections")
collected_at = valid_timestamp(capture.get("generated_at"))
if collected_at is None:
raise ValueError("/generated_at: expected a timezone-qualified ISO timestamp")
if valid_timestamp(reference_time) is None:
raise ValueError("--reference-time: expected a timezone-qualified ISO timestamp")
collection_sections = {name: normalize_section(section)
for name, section in capture["sections"].items()}
complete = bool(collection_sections) and all(
section["status"] == "ok" and not section["truncated"]
for section in collection_sections.values())
collection_status = "complete" if complete else "partial"
observations = []
groups = OrderedDict()
event_signatures = set()
for section_name, section in selected_event_sections(capture):
data = section.get("data", []) if isinstance(section, dict) else []
if not isinstance(data, list):
continue
for index, event in enumerate(data):
if not isinstance(event, dict):
continue
observation_id = f"event-{slug(section_name)}-{index + 1}"
observation = {"id": observation_id, "section": section_name,
"source_pointer": pointer("sections", section_name, "data", index),
"value": event}
observed_at = valid_timestamp(event.get("TimeCreated"))
if observed_at:
observation["observed_at"] = observed_at
observations.append(observation)
provider, event_id = event_identity(event)
groups.setdefault((provider, event_id), []).append(observation_id)
event_signatures.add(event_signature(provider, event_id, event.get("TimeCreated"),
event.get("Message")))
reliability = capture["sections"].get("reliability_records")
reliability_data = reliability.get("data", []) if isinstance(reliability, dict) else []
if isinstance(reliability_data, list):
for index, record in enumerate(reliability_data):
if not isinstance(record, dict):
continue
provider = str(record.get("SourceName", "Unknown provider"))
raw_id = record.get("EventIdentifier", "unknown")
event_id = int(raw_id) if isinstance(raw_id, str) and raw_id.isdigit() else raw_id
rule = EVENT_RULES.get((provider, event_id))
if not rule or rule["severity"] not in ("warning", "critical"):
continue
signature = event_signature(provider, event_id, record.get("TimeGenerated"),
record.get("Message"))
if signature in event_signatures:
continue
event_signatures.add(signature)
observation_id = f"event-reliability-records-{index + 1}"
observation = {"id": observation_id, "section": "reliability_records",
"source_pointer": pointer("sections", "reliability_records", "data", index),
"value": record}
observed_at = valid_timestamp(record.get("TimeGenerated"))
if observed_at:
observation["observed_at"] = observed_at
observations.append(observation)
groups.setdefault((provider, event_id), []).append(observation_id)
findings = []
observations_by_id = {item["id"]: item for item in observations}
for (provider, event_id), evidence in groups.items():
rule = EVENT_RULES.get((provider, event_id))
if rule:
confidence = min(rule["confidence"], 50) if collection_status != "complete" else rule["confidence"]
finding = {"id": f"event-{slug(provider)}-{slug(event_id)}",
"severity": rule["severity"], "confidence": confidence,
"title": f"{provider} event {event_id}",
"description": rule["description"], "evidence": evidence}
finding.update(fading_metrics(rule, evidence, observations_by_id, reference_time))
findings.append(finding)
else:
findings.append({"id": f"event-{slug(provider)}-{slug(event_id)}",
"severity": "unanalysed", "confidence": 0,
"title": f"Unanalysed event: {provider} / {event_id}",
"description": "The event is preserved for review; no migrated technical rule matches it.",
"evidence": evidence})
for entry in WATCHLIST:
matches = [match for matcher in entry["match"] for match in watch_matches(capture, matcher)]
requirement_groups = [watch_matches(capture, matcher) for matcher in entry.get("requires", ())]
if not matches or any(not group for group in requirement_groups):
continue
event_gate = entry.get("events")
event_evidence = []
if event_gate:
wanted_ids = set(event_gate["ids"])
event_evidence = [item["id"] for item in observations
if str(item["value"].get("ProviderName", "")) == event_gate["provider"]
and str(item["value"].get("Id", "")) in wanted_ids]
if len(event_evidence) < event_gate["minimum"]:
continue
evidence = []
for section_name, index, item in matches + [match for group in requirement_groups for match in group]:
observation_id = f"watch-{slug(entry['id'])}-{slug(section_name)}-{index + 1}"
if observation_id not in {observation["id"] for observation in observations}:
observations.append({"id": observation_id, "section": section_name,
"source_pointer": pointer("sections", section_name, "data", index),
"value": item})
if observation_id not in evidence:
evidence.append(observation_id)
evidence.extend(item for item in event_evidence if item not in evidence)
findings.append({"id": f"watchlist-{entry['id']}", "severity": "warning",
"confidence": 50 if collection_status != "complete" else 85,
"title": entry["title"], "description": entry["description"],
"evidence": evidence})
warnings = [str(item) for item in capture.get("warnings", []) if str(item).strip()]
if collection_status != "complete":
warnings.append("Legacy capture collection is incomplete; finding confidence is capped at 50.")
warnings.append(f"Migrated analyser supports {len(EVENT_RULES)} event rules and {len(WATCHLIST)} candidate watchlist entries; unmatched events remain unanalysed.")
system = {"host": capture.get("host", {})}
if isinstance(capture.get("summary"), dict):
system["legacy_summary"] = capture["summary"]
return {
"schema_version": 1,
"metadata": {"producer": {"name": "win-diagnostics", "version": producer_version},
"collected_at": collected_at, "analysis_reference_time": reference_time,
"source": {"artifact": artifact, "schema_version": version,
"sha256": artifact_hash}},
"system": system,
"collection": {"mode": capture.get("mode", "Full"), "status": collection_status,
"sections": collection_sections},
"observations": observations,
"findings": findings,
"warnings": warnings,
}
def analyse_file(input_path, evidence_root, output_path, reference_time, producer_version, force=False):
if input_path.is_symlink():
raise ValueError(f"{input_path}: input symlinks are not supported")
input_path = input_path.resolve()
evidence_root = evidence_root.resolve()
output_path = output_path.resolve()
if not input_path.is_relative_to(evidence_root):
raise ValueError(f"{input_path}: input must be inside --evidence-root")
if output_path == input_path:
raise ValueError("input and output paths must differ")
artifact = input_path.relative_to(evidence_root).as_posix()
capture = validation.load_json(input_path)
result = analyse(capture, artifact, hashlib.sha256(input_path.read_bytes()).hexdigest(),
reference_time, producer_version)
errors = validation.validate_document(result)
if not errors:
errors = validation.verify_evidence(result, evidence_root)
if errors:
raise ValueError("generated diagnostic is invalid:\n" + "\n".join(errors))
if output_path.exists() and not force:
raise ValueError(f"{output_path}: output exists; use --force to replace it")
output_path.parent.mkdir(parents=True, exist_ok=True)
descriptor, temporary_name = tempfile.mkstemp(prefix=f".{output_path.name}.", dir=output_path.parent)
temporary = Path(temporary_name)
try:
with os.fdopen(descriptor, "w", encoding="utf-8") as stream:
json.dump(result, stream, indent=2, ensure_ascii=False, allow_nan=False)
stream.write("\n")
if force:
os.replace(temporary, output_path)
else:
os.link(temporary, output_path)
finally:
temporary.unlink(missing_ok=True)
return result
def parser():
result = argparse.ArgumentParser(description=__doc__)
result.add_argument("input", type=Path)
result.add_argument("--evidence-root", required=True, type=Path)
result.add_argument("--output", required=True, type=Path)
result.add_argument("--reference-time", required=True)
result.add_argument("--producer-version", required=True)
result.add_argument("--force", action="store_true")
return result
def main(argv=None):
args = parser().parse_args(argv)
try:
analyse_file(args.input, args.evidence_root, args.output, args.reference_time,
args.producer_version, args.force)
except (OSError, ValueError, KeyError, TypeError, RecursionError) as error:
print(f"error: {error}", file=sys.stderr)
return 1
print(f"Wrote validated diagnostic v1: {args.output}")
return 0
if __name__ == "__main__":
sys.exit(main())