From fe5933e9fdf4a31a7dee2e4fbcad49b8fa977ae4 Mon Sep 17 00:00:00 2001 From: Dandaedre Date: Mon, 5 Oct 2026 00:46:57 +0200 Subject: [PATCH] ci: expose silent coverage gate failures --- docs/security-and-ci.md | 13 ++++++++++ scripts/collect-coverage.sh | 23 ++++++++++++++++-- scripts/test-badges.py | 48 ++++++++++++++++++++++++++++++++++++- 3 files changed, 81 insertions(+), 3 deletions(-) diff --git a/docs/security-and-ci.md b/docs/security-and-ci.md index d51616e..963e9d4 100644 --- a/docs/security-and-ci.md +++ b/docs/security-and-ci.md @@ -183,6 +183,19 @@ and deletes the summary, including on failure. Jenkins then creates the current build's test and coverage markers only after unit/integration tests, Rustdoc tests, and the coverage test run succeed. The release smoke test remains a nonfatal diagnostic. +The collector removes any stale summary before running Cargo and prints the +measured percentage before deleting the current report. On a failed Cargo run, +it also prints per-file line counts and percentages, ordered by coverage, and +preserves Cargo's exit status. Missing or invalid summaries fail with an explicit +diagnostic. A failed run never renders a successful coverage badge. + +The supplied FreeBSD Jenkins log for revision `e47feea` shows passing tests and +a coverage report followed by exit 1, without a reported percentage. +The pinned `cargo-llvm-cov` 0.6.16 can return that status silently when its line +coverage floor is missed. The deleted report prevents confirming the measured +value from that log; rerun CI with the collector diagnostics before changing +tests or investigating platform instrumentation. The 85% floor remains enforced. + At checkout, CI recreates ignored `coverage/` to remove stale workspace state. In `post { cleanup { ... } }`, it uses only current-build markers to choose test status and retain measured coverage, falling back to `failing` and neutral `-%` diff --git a/scripts/collect-coverage.sh b/scripts/collect-coverage.sh index 1ad1353..b7a3ba7 100755 --- a/scripts/collect-coverage.sh +++ b/scripts/collect-coverage.sh @@ -5,7 +5,26 @@ summary=coverage/summary.json trap 'rm -f "$summary"' 0 trap 'exit 1' HUP INT TERM # Alias and honest initial floor live in .cargo/config.toml. -cargo coverage -percentage=$(jq -er '.data | if length == 1 then .[0].totals.lines.percent else error("expected one overall total") end | select(type == "number" and . >= 0 and . <= 100)' "$summary") +rm -f "$summary" +coverage_status=0 +cargo coverage || coverage_status=$? +# cargo-llvm-cov 0.6.16 can fail the floor silently after writing its report. +# Print the measurement before cleanup, but never mask a failed Cargo run. +if [ ! -s "$summary" ]; then + echo "ERROR: cargo coverage produced no summary (exit $coverage_status)." >&2 + [ "$coverage_status" -ne 0 ] || coverage_status=1 + exit "$coverage_status" +fi +if ! percentage=$(jq -er '.data | if type == "array" and length == 1 then .[0].totals.lines.percent else error("expected one overall total") end | select(type == "number" and . >= 0 and . <= 100)' "$summary"); then + echo "ERROR: cargo coverage produced an invalid line-coverage summary (exit $coverage_status)." >&2 + [ "$coverage_status" -ne 0 ] || coverage_status=1 + exit "$coverage_status" +fi +echo "Measured overall line coverage: $percentage% (required floor: see cargo coverage in .cargo/config.toml)." +if [ "$coverage_status" -ne 0 ]; then + echo "ERROR: cargo coverage failed (exit $coverage_status); check the measurement against --fail-under-lines and the preceding compiler/test diagnostics." >&2 + jq -r '.data[0].files // [] | sort_by(.summary.lines.percent)[] | " \(.filename): \(.summary.lines.covered)/\(.summary.lines.count) lines (\(.summary.lines.percent)%)"' "$summary" >&2 || : + exit "$coverage_status" +fi scripts/ci-coverage-badges.sh coverage --value "$percentage" rm -f "$summary" diff --git a/scripts/test-badges.py b/scripts/test-badges.py index de27f74..c23f4b3 100644 --- a/scripts/test-badges.py +++ b/scripts/test-badges.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """Isolated renderer/publication contract checks; no public assets are touched.""" import os +import json from pathlib import Path import stat import shlex @@ -17,6 +18,7 @@ PUBLISH = str(REPO / 'scripts/publish-badges.sh') def run(*args, ok=True): result = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) assert (result.returncode == 0) == ok, (args, result.stderr.decode()) + return result with tempfile.TemporaryDirectory() as directory: @@ -156,6 +158,50 @@ with tempfile.TemporaryDirectory() as directory: assert not list(Path('coverage').glob('.*-123')) os.environ['PATH'] = old_path cargo.unlink() + # A coverage-floor failure can write a valid report and exit silently. + # Exercise the collector with real jq and rendering, but no Rust rebuild. + cargo.write_text('''#!/bin/sh +[ "$*" = coverage ] || exit 99 +if [ -n "$FAKE_COVERAGE_REPORT" ]; then + printf '%s\\n' "$FAKE_COVERAGE_REPORT" > coverage/summary.json +fi +exit "$FAKE_COVERAGE_STATUS" +''') + cargo.chmod(0o755) + os.environ['PATH'] = str(tools) + ':' + old_path + for percentage, status in ((90.87, 0), (85, 0), (84.99, 1), (90.87, 7)): + os.environ['FAKE_COVERAGE_REPORT'] = json.dumps({'data': [{ + 'totals': {'lines': {'percent': percentage}}, + 'files': [{'filename': 'src/example.rs', 'summary': { + 'lines': {'percent': 80, 'count': 100, 'covered': 80}}}], + }]}) + os.environ['FAKE_COVERAGE_STATUS'] = str(status) + before = Path('coverage/coverage.svg').read_bytes() + result = run(str(REPO/'scripts/collect-coverage.sh'), ok=status == 0) + assert result.returncode == status + assert f'{percentage}%' in result.stdout.decode() + if status: + assert '--fail-under-lines' in result.stderr.decode() + assert 'src/example.rs: 80/100 lines (80%)' in result.stderr.decode() + assert Path('coverage/coverage.svg').read_bytes() == before + else: + assert ET.parse('coverage/coverage.svg').getroot().attrib['aria-label'] == f'coverage: {percentage:.0f}%' + assert not Path('coverage/summary.json').exists() + for report in ('', '{broken', '{"data": []}', '{"data": [{}, {}]}', + *[json.dumps({'data': [{'totals': {'lines': {'percent': p}}}]}) + for p in (None, '90', -1, 101)]): + for status in (0, 7): + # An old successful report must not hide a missing current report. + Path('coverage/summary.json').write_text('{"data":[{"totals":{"lines":{"percent":99}}}]}') + os.environ['FAKE_COVERAGE_REPORT'] = report + os.environ['FAKE_COVERAGE_STATUS'] = str(status) + result = run(str(REPO/'scripts/collect-coverage.sh'), ok=False) + assert result.returncode == (status or 1) + assert 'ERROR: cargo coverage produced' in result.stderr.decode() + assert not Path('coverage/summary.json').exists() + os.environ['PATH'] = old_path + del os.environ['FAKE_COVERAGE_REPORT'], os.environ['FAKE_COVERAGE_STATUS'] + cargo.unlink() root.chmod(0o555) if os.geteuid() != 0: run(*args, 'test', ok=False) @@ -171,4 +217,4 @@ for name in ('version', 'test', 'coverage'): pipeline = (REPO/'Jenkinsfile').read_text() assert "artifacts: 'coverage/version.svg,coverage/test.svg,coverage/coverage.svg'" in pipeline assert 'fingerprint: true, allowEmptyArchive: false' in pipeline -print('Badge rendering, XML, validation, publication, README and archive checks passed.') +print('Badge rendering, XML, validation, coverage diagnostics, publication, README and archive checks passed.')