Improve diagnostic classification and correlate catalog failures

This commit is contained in:
Dandaedre 2026-09-11 14:38:37 +02:00
parent 67fb198449
commit 51654caf7d
Signed by: Dandaedre
GPG Key ID: 440C6592E8A26D99
4 changed files with 196 additions and 13 deletions

View File

@ -8,7 +8,7 @@
}
#let number(value, default: 0.0) = {
let found = str(value).match(regex("-?\\d+(?:\\.\\d+)?"))
let found = str(value).replace(",", ".").match(regex("-?\\d+(?:\\.\\d+)?"))
if found == none { default } else { float(found.text) }
}
@ -25,14 +25,19 @@
if found == none { none } else { date-index(int(found.captures.at(0)), int(found.captures.at(1)), int(found.captures.at(2))) }
}
#let event-day(value) = {
#let event-day(value, day-first: false) = {
let found = str(value).match(regex("^(\\d{1,2})/(\\d{1,2})/(\\d{4})"))
if found == none { none } else { date-index(int(found.captures.at(2)), int(found.captures.at(0)), int(found.captures.at(1))) }
if found == none { none } else {
let year = int(found.captures.at(2))
let month = int(found.captures.at(if day-first { 1 } else { 0 }))
let day = int(found.captures.at(if day-first { 0 } else { 1 }))
if month < 1 or month > 12 or day < 1 or day > 31 { none } else { date-index(year, month, day) }
}
}
#let fade-weight(rule, event-time, reference-day) = {
#let fade-weight(rule, event-time, reference-day, day-first: false) = {
if rule == none or not get(rule, "fading", false) or reference-day == none { (factor: 1.0, age_days: none) } else {
let observed-day = event-day(event-time)
let observed-day = event-day(event-time, day-first: day-first)
if observed-day == none { (factor: 1.0, age_days: none) } else {
let age = calc.max(0, reference-day - observed-day)
let starts = get(rule, "fade_after_days", 7)
@ -121,8 +126,8 @@
let description = capture(event.message, "-- Error \\d+\\. (.*?)(?:[ \t]+Action[ \t]|$)", default: "Windows Installer did not complete the transaction.")
product + " installation failed with MSI error " + error + ". " + description
} else if event.provider == "Application Error" and event.id == "1000" {
let application = capture(event.message, "Faulting application name: ([^,]+)", default: "Unknown application")
let module = capture(event.message, "Faulting module name: ([^,]+)", default: "unknown module")
let application = capture(event.message, "(?:Faulting application name|Nom de l’application défaillante|Nom de l'application défaillante) *: ([^,]+)", default: "Unknown application")
let module = capture(event.message, "(?:Faulting module name|Nom du module défaillant) *: ([^,]+)", default: "unknown module")
let exception = capture(event.message, "Exception code: ([^ ]+)", default: "unknown code")
application + " terminated unexpectedly in " + module + " (exception " + exception + ")."
} else {
@ -130,7 +135,7 @@
}
}
#let group-events(events, rules, reference-day: none) = {
#let group-events(events, rules, reference-day: none, day-first: false) = {
let groups = ()
for event in events {
let index = groups.position(group => group.provider == event.provider and group.id == event.id)
@ -138,8 +143,9 @@
let item = event
item.message = event-summary(event)
item.insert("count", 1)
item.insert("messages", (item.message,))
let rule = event-rule(rules, event)
let fading = fade-weight(rule, event.time, reference-day)
let fading = fade-weight(rule, event.time, reference-day, day-first: day-first)
item.insert("severity", if rule == none { get(rules, "unknown_event_severity", "unanalysed") } else { get(rule, "severity", "unanalysed") })
item.insert("rationale", if rule == none { "FAIL TO ANALYSE — no matching rule in diagnostics-rules.yaml." } else { get(rule, "rationale", "") })
item.insert("remediation", if rule == none { "Add a reviewed provider/ID rule before attempting cleanup." } else { get(rule, "remediation", "") })
@ -154,12 +160,39 @@
} else {
let item = groups.at(index)
item.count += 1
let summary = event-summary(event)
if not (summary in item.messages) {
item.messages.push(summary)
item.message = item.messages.join(" / ")
}
groups.at(index) = item
}
}
groups
}
// Correlate original records before provider/ID grouping loses timestamps.
#let event-second(value, day-first: false) = {
let day = event-day(value, day-first: day-first)
let clock = str(value).match(regex(" (\\d{1,2}):(\\d{2}):(\\d{2})"))
if day == none or clock == none { none } else {
day * 86400 + int(clock.captures.at(0)) * 3600 + int(clock.captures.at(1)) * 60 + int(clock.captures.at(2))
}
}
#let catalog-correlation(events, day-first: false) = {
let checksums = events.filter(event => event.provider == "ESENT" and event.id == "474" and lower(event.message).contains("catroot2") and event.message.contains("-1018"))
checksums.find(anchor => {
let time = event-second(anchor.time, day-first: day-first)
let nearby = events.filter(event => {
let other = event-second(event.time, day-first: day-first)
time != none and other != none and calc.abs(time - other) <= 120
})
(nearby.any(event => event.provider == "Application Error" and event.id == "1000" and lower(event.message).contains("cryptsvc") and lower(event.message).contains("esent.dll")) and
nearby.any(event => event.provider == "Microsoft-Windows-WindowsUpdateClient" and event.id == "20"))
})
}
#let normalize(source, rules: (:), source-name: "WinDiagnostics dump") = {
let text = source.replace(regex("\\r"), "")
let sections = parse-sections(text)
@ -170,9 +203,20 @@
let battery = section(sections, "BATTERY")
let storage = section(sections, "STORAGE / FREE SPACE / HEALTH")
let generated = field(text, "Generated")
let events = group-events(parse-events(sections), rules, reference-day: report-day(generated))
// Compare the local capture date with the ISO header; do not assume US dates.
let local-date = field(os, "LocalDateTime")
let day-first = event-day(local-date, day-first: true) == report-day(generated) and event-day(local-date) != report-day(generated)
let raw-events = parse-events(sections)
let events = group-events(raw-events, rules, reference-day: report-day(generated), day-first: day-first)
let findings = ()
let catalog-cluster = catalog-correlation(raw-events, day-first: day-first)
if catalog-cluster != none {
findings.push((severity: "critical", title: "Catalog corruption coincides with Cryptographic Services and update failures", detail: "Around " + catalog-cluster.time + ", CatRoot2 checksum failure (-1018), a CryptSvc crash in ESENT.dll, and Windows Update failure occur within two minutes of the checksum event. This supports a related failure sequence, but does not establish which hardware or software component caused the corruption. See the individual event evidence below.", confidence: 90, weight_factor: 0, resolution_steps: ("Back up important data and preserve event logs before resetting update components.", "Investigate storage and memory stability, including any ESENT memory warnings. Retest at stock CPU/RAM settings and review firmware guidance for the captured motherboard and CPU.", "Run Full diagnostics as administrator to obtain DISM and SFC results, then use those results to guide supported Windows component repair.", "Retry the failed update and verify Cryptographic Services remains stable with no new database checksum failures.")))
}
if field(text, "Administrator") == "False" {
findings.push((severity: "warning", title: "Diagnostic coverage is incomplete", detail: "The collector was not elevated. Skipped administrator-only checks, including DISM and SFC, are not evidence of healthy system files.", confidence: 100, resolution_steps: ("Run WinDiagnostics Full from an elevated PowerShell session.", "Review the new DISM, SFC, and other previously skipped results before closing the investigation.")))
}
let dism = section(sections, "WINDOWS COMPONENT STORE STATUS")
let sfc = section(sections, "SYSTEM FILE CHECK - VERIFY ONLY")
if lower(dism).contains("repairable") {
@ -187,12 +231,18 @@
}
}
let problems = section(sections, "DEVICES WITH PROBLEMS")
if problems != "" and not lower(problems).contains("no problem devices found") {
let disabled-displays = useful-lines(problems).filter(line => line.match(regex("^Error[ \t]+Display[ \t]+")) != none and line.contains("CM_PROB_DISABLED"))
for device in disabled-displays {
let name = capture(device, "^Error[ \t]+Display[ \t]+(.*?)[ \t]+PCI", default: "Display adapter")
findings.push((severity: "warning", title: name + " is disabled (Code 22)", detail: "The device reports CM_PROB_DISABLED. This records a disabled device, not proof of GPU hardware failure. The dump does not establish who or what disabled it.", confidence: 99, resolution_steps: ("Open Device Manager > Display adapters and inspect this adapter's status and Events tab.", "If it should be active, select Enable device and record any new status code.", "Restart and verify display acceleration and normal workload stability. If enabling fails or the device becomes disabled again, collect device events and review the installed vendor driver.", "Investigate concurrent IOMMU and memory/storage errors separately; this dump does not map the IOMMU device identifier to the GPU."), reference_url: "https://learn.microsoft.com/en-us/windows-hardware/drivers/install/cm-prob-disabled", reference_label: "Microsoft Code 22 guidance"))
}
let other-problems = useful-lines(problems).filter(line => line.starts-with("Error") and not (line in disabled-displays))
if problems != "" and not lower(problems).contains("no problem devices found") and (disabled-displays.len() == 0 or other-problems.len() > 0) {
findings.push((severity: "warning", title: "Devices report problems", detail: "One or more present devices did not report OK status.", confidence: 90, resolution_steps: ("Identify every reported device in Device Manager.", "Update, repair, or remove the associated driver or disconnected hardware.", "Re-run diagnostics and confirm that no present device reports a problem.")))
}
let disk-free = ()
for match in storage.matches(regex("(?m)^([A-Z]:)[ \\t]+.*?[ \\t](\\d+(?:\\.\\d+)?)[ \\t]+(\\d+(?:\\.\\d+)?)[ \\t]+(\\d+(?:\\.\\d+)?)[ \\t]*$")) {
disk-free.push((drive: match.captures.at(0), size_gb: float(match.captures.at(1)), free_gb: float(match.captures.at(2)), free_percent: float(match.captures.at(3))))
for match in storage.matches(regex("(?m)^([A-Z]:)[ \\t]+.*?[ \\t](\\d+(?:[.,]\\d+)?)[ \\t]+(\\d+(?:[.,]\\d+)?)[ \\t]+(\\d+(?:[.,]\\d+)?)[ \\t]*$")) {
disk-free.push((drive: match.captures.at(0), size_gb: number(match.captures.at(1)), free_gb: number(match.captures.at(2)), free_percent: number(match.captures.at(3))))
}
let inventory-map = (

View File

@ -132,4 +132,80 @@ event_rules:
rationale: "The captured event has no message, so it provides no actionable evidence by itself."
remediation: "No action is recommended from this empty event alone. Update or repair Waterfox only if browser-default handling is visibly failing or the event repeatedly correlates with a symptom."
# Additional provider/ID pairs observed in the SAURON capture.
- provider: "Microsoft-Windows-Kernel-Boot"
id: "29"
severity: "warning"
confidence: 92
rationale: "Windows failed Fast Startup; the status and adjacent shutdown events do not establish the underlying cause."
remediation: "Compare a normal Restart with shutdown/startup. Test with Fast Startup disabled through Power Options, then review firmware and drivers if failures recur. Preserve adjacent shutdown and bug-check events."
- provider: "Service Control Manager"
id: "7031"
severity: "warning"
confidence: 95
rationale: "A service terminated unexpectedly; Windows may schedule a recovery action."
remediation: "Identify each named service and correlate application crashes at the same time. Repair the owning component and verify the service stays running after restart."
- provider: "Microsoft-Windows-DeviceAssociationService"
id: "3505"
severity: "warning"
confidence: 70
rationale: "Device association failed while enumerating pairing ceremonies; the message does not identify a failed hardware component."
remediation: "Check whether device pairing actually fails. Identify the affected peripheral, retry pairing, and review its driver and device-association logs if reproducible."
- provider: "Microsoft-Windows-HAL"
id: "15"
severity: "warning"
confidence: 90
rationale: "The IOMMU reported an error. The raw device identifier alone does not identify the graphics adapter or prove physical hardware failure."
remediation: "Preserve the full event XML and map the device to its PCI location before blaming a component. Review OEM firmware and chipset/device drivers and correlate recurrence with device activity. Keep DMA protection enabled."
- provider: "Microsoft-Windows-HAL"
id: "21"
severity: "warning"
confidence: 90
rationale: "The ACPI time/alarm method failed while Windows attempted to set the hardware real-time clock."
remediation: "Check firmware time and Windows time synchronization after reboot. Review applicable OEM BIOS fixes and retest shutdown/resume. This event alone does not diagnose a dead CMOS battery."
- provider: "usbaudio2"
id: "38"
severity: "warning"
confidence: 90
rationale: "A USB audio control request failed."
remediation: "Identify the USB audio device, reconnect it directly, and test another port or cable. Review device firmware/drivers and confirm normal playback or recording without recurrence."
- provider: "ESENT"
id: "419"
severity: "critical"
confidence: 95
rationale: "The database engine could not read a database page. Inspect the recorded error; -1018 indicates a checksum/read-verification failure."
remediation: "Back up important data and preserve the event/database context. Check storage and memory stability, then use the owning application’s supported recovery procedure. Verify that the database can be read and the workload succeeds."
reference_url: "https://learn.microsoft.com/en-us/windows/win32/extensible-storage-engine/jet-err-enumeration"
reference_label: "Microsoft database error definitions"
- provider: "ESENT"
id: "474"
severity: "critical"
confidence: 98
rationale: "A database page failed checksum verification. This confirms corruption in the read data but does not isolate disk, RAM, CPU, driver, or power as the cause."
remediation: "Back up important data. Check storage health and run memory diagnostics at stock settings. Repair the affected database through its owning component only after investigating stability, then verify the failed workload and absence of new checksum errors."
reference_url: "https://learn.microsoft.com/en-us/windows/win32/extensible-storage-engine/jet-err-enumeration"
reference_label: "Microsoft database error definitions"
- provider: "ESENT"
id: "907"
severity: "critical"
confidence: 95
rationale: "ESENT detected a transient memory problem; this is a stability signal, not proof that a particular RAM module is defective."
remediation: "Back up data and run Windows Memory Diagnostic or OEM memory tests. Retest at stock CPU/RAM settings and review OEM firmware guidance. Investigate CPU, RAM, storage and drivers if corruption recurs."
reference_url: "https://learn.microsoft.com/en-us/windows/win32/extensible-storage-engine/jet-err-enumeration"
reference_label: "Microsoft database error definitions"
- provider: "ESENT"
id: "908"
severity: "critical"
confidence: 95
rationale: "The database process terminated because of a fatal error. Inspect its error code and enforcement tag and correlate adjacent ESENT events."
remediation: "Preserve the full message and investigate associated checksum/memory errors before retrying the workload. Verify stability and repair the owning component through its supported recovery procedure."
reference_url: "https://learn.microsoft.com/en-us/windows/win32/extensible-storage-engine/jet-err-enumeration"
reference_label: "Microsoft database error definitions"
- provider: "Microsoft-Windows-Defrag"
id: "264"
severity: "warning"
confidence: 92
rationale: "Storage optimization did not complete; the message supplies the reason. An unsupported filesystem is not evidence of a failing disk."
remediation: "Check the named volume and error. For 0x8900002F on exFAT, review optimization scheduling; do not format or convert the volume just to remove this event. Investigate other errors according to their actual cause."
unknown_event_severity: "unanalysed"

View File

@ -58,6 +58,20 @@ No Python, JSON conversion, or external normalization step is used. Typst reads
the `.txt` dump directly. It downloads `@preview/primaviz:0.10.0` on the first
build and caches it locally.
The normalizer also correlates CatRoot2 checksum errors with CryptSvc crashes
in ESENT.dll and Windows Update failures within two minutes of a checksum event.
When all three signals are present, the generated PDF includes a priority
finding and a shared investigation plan. Individual events remain visible;
the correlation adds no extra health-score penalty. This identifies a related
failure sequence, not a proven hardware root cause. Disabled display adapters
receive Code 22 guidance, and non-elevated captures receive a coverage warning.
Run the parser and correlation regression checks with:
```bash
typst compile --root . tools/check-diagnostics.typ /tmp/check-diagnostics.pdf
```
## Report structure
The document is divided into three purposeful parts:

View File

@ -0,0 +1,43 @@
#import "../diagnostics-normalize.typ": normalize, event-day, date-index
#let rules = yaml("../diagnostics-rules.yaml")
#let data = normalize(read("../diagnostics/windows_system_report_SAURON_20260911_135414.txt"), rules: rules)
#assert.eq(data.issue_counts.unanalysed, 0)
#for (provider, id, severity) in (
("Microsoft-Windows-DeviceAssociationService", "3505", "warning"),
("Microsoft-Windows-HAL", "15", "warning"),
("usbaudio2", "38", "warning"),
("Microsoft-Windows-HAL", "21", "warning"),
("ESENT", "419", "critical"),
("ESENT", "474", "critical"),
("ESENT", "907", "critical"),
("Microsoft-Windows-Defrag", "264", "warning"),
) {
let event = data.event_groups.find(item => item.provider == provider and item.id == id)
assert(event != none, message: provider + " / " + id + " missing from parsed events")
assert.eq(event.severity, severity)
assert(event.remediation != "")
}
#assert.eq(data.system.ram_total_gb, 31.82)
#assert.eq(data.disk_free.len(), 2)
#assert.eq(data.disk_free.at(0).free_percent, 81.6)
#assert.eq(data.disk_free.at(1).free_percent, 54.4)
#assert(data.findings.any(item => item.title == "AMD Radeon RX 6500 XT is disabled (Code 22)"))
#assert(data.findings.any(item => item.title == "Diagnostic coverage is incomplete"))
#assert(not data.findings.any(item => item.title == "System-file integrity violations"))
#let crashes = data.event_groups.find(item => item.provider == "Application Error")
#assert(crashes.message.contains("svchost.exe_CryptSvc"))
#assert(crashes.message.contains("RtkAudUService64.exe"))
#assert.eq(crashes.age_days, 0)
#assert.eq(event-day("11/09/2026", day-first: true), date-index(2026, 9, 11))
#assert.eq(event-day("09/30/2026"), date-index(2026, 9, 30))
#assert.eq(event-day("09/30/2026", day-first: true), none)
#let unknown = normalize("--- RECENT SYSTEM ERRORS ---\nTimeCreated : 09/11/2026\nId : 999\nProviderName : UnknownProvider\nMessage : Unrecognized event", rules: rules)
#assert.eq(unknown.issue_counts.unanalysed, 1)
#let cluster-title = "Catalog corruption coincides with Cryptographic Services and update failures"
#assert(data.findings.any(item => item.title == cluster-title))
#let no-update = normalize(read("../diagnostics/windows_system_report_SAURON_20260911_135414.txt").replace("Microsoft-Windows-WindowsUpdateClient", "UnrelatedProvider"), rules: rules)
#assert(not no-update.findings.any(item => item.title == cluster-title))
#let distant-update = normalize(read("../diagnostics/windows_system_report_SAURON_20260911_135414.txt").replace("11/09/2026 10:45:56", "10/09/2026 10:45:56"), rules: rules)
#assert(not distant-update.findings.any(item => item.title == cluster-title))
Diagnostics regression checks passed.