ci: expose silent coverage gate failures
Rust CI tiny-wb failed CI.
Gitea/tiny-wb/pipeline/head There was a failure building this commit
Details
Rust CI tiny-wb failed CI.
Gitea/tiny-wb/pipeline/head There was a failure building this commit
Details
This commit is contained in:
parent
e47feea703
commit
fe5933e9fd
|
|
@ -183,6 +183,19 @@ and deletes the summary, including on failure. Jenkins then creates the current
|
|||
build's test and coverage markers only after unit/integration tests, Rustdoc tests, and the coverage test run succeed.
|
||||
The release smoke test remains a nonfatal diagnostic.
|
||||
|
||||
The collector removes any stale summary before running Cargo and prints the
|
||||
measured percentage before deleting the current report. On a failed Cargo run,
|
||||
it also prints per-file line counts and percentages, ordered by coverage, and
|
||||
preserves Cargo's exit status. Missing or invalid summaries fail with an explicit
|
||||
diagnostic. A failed run never renders a successful coverage badge.
|
||||
|
||||
The supplied FreeBSD Jenkins log for revision `e47feea` shows passing tests and
|
||||
a coverage report followed by exit 1, without a reported percentage.
|
||||
The pinned `cargo-llvm-cov` 0.6.16 can return that status silently when its line
|
||||
coverage floor is missed. The deleted report prevents confirming the measured
|
||||
value from that log; rerun CI with the collector diagnostics before changing
|
||||
tests or investigating platform instrumentation. The 85% floor remains enforced.
|
||||
|
||||
At checkout, CI recreates ignored `coverage/` to remove stale workspace state.
|
||||
In `post { cleanup { ... } }`, it uses only current-build markers to choose test
|
||||
status and retain measured coverage, falling back to `failing` and neutral `-%`
|
||||
|
|
|
|||
|
|
@ -5,7 +5,26 @@ summary=coverage/summary.json
|
|||
trap 'rm -f "$summary"' 0
|
||||
trap 'exit 1' HUP INT TERM
|
||||
# Alias and honest initial floor live in .cargo/config.toml.
|
||||
cargo coverage
|
||||
percentage=$(jq -er '.data | if length == 1 then .[0].totals.lines.percent else error("expected one overall total") end | select(type == "number" and . >= 0 and . <= 100)' "$summary")
|
||||
rm -f "$summary"
|
||||
coverage_status=0
|
||||
cargo coverage || coverage_status=$?
|
||||
# cargo-llvm-cov 0.6.16 can fail the floor silently after writing its report.
|
||||
# Print the measurement before cleanup, but never mask a failed Cargo run.
|
||||
if [ ! -s "$summary" ]; then
|
||||
echo "ERROR: cargo coverage produced no summary (exit $coverage_status)." >&2
|
||||
[ "$coverage_status" -ne 0 ] || coverage_status=1
|
||||
exit "$coverage_status"
|
||||
fi
|
||||
if ! percentage=$(jq -er '.data | if type == "array" and length == 1 then .[0].totals.lines.percent else error("expected one overall total") end | select(type == "number" and . >= 0 and . <= 100)' "$summary"); then
|
||||
echo "ERROR: cargo coverage produced an invalid line-coverage summary (exit $coverage_status)." >&2
|
||||
[ "$coverage_status" -ne 0 ] || coverage_status=1
|
||||
exit "$coverage_status"
|
||||
fi
|
||||
echo "Measured overall line coverage: $percentage% (required floor: see cargo coverage in .cargo/config.toml)."
|
||||
if [ "$coverage_status" -ne 0 ]; then
|
||||
echo "ERROR: cargo coverage failed (exit $coverage_status); check the measurement against --fail-under-lines and the preceding compiler/test diagnostics." >&2
|
||||
jq -r '.data[0].files // [] | sort_by(.summary.lines.percent)[] | " \(.filename): \(.summary.lines.covered)/\(.summary.lines.count) lines (\(.summary.lines.percent)%)"' "$summary" >&2 || :
|
||||
exit "$coverage_status"
|
||||
fi
|
||||
scripts/ci-coverage-badges.sh coverage --value "$percentage"
|
||||
rm -f "$summary"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Isolated renderer/publication contract checks; no public assets are touched."""
|
||||
import os
|
||||
import json
|
||||
from pathlib import Path
|
||||
import stat
|
||||
import shlex
|
||||
|
|
@ -17,6 +18,7 @@ PUBLISH = str(REPO / 'scripts/publish-badges.sh')
|
|||
def run(*args, ok=True):
|
||||
result = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
assert (result.returncode == 0) == ok, (args, result.stderr.decode())
|
||||
return result
|
||||
|
||||
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
|
|
@ -156,6 +158,50 @@ with tempfile.TemporaryDirectory() as directory:
|
|||
assert not list(Path('coverage').glob('.*-123'))
|
||||
os.environ['PATH'] = old_path
|
||||
cargo.unlink()
|
||||
# A coverage-floor failure can write a valid report and exit silently.
|
||||
# Exercise the collector with real jq and rendering, but no Rust rebuild.
|
||||
cargo.write_text('''#!/bin/sh
|
||||
[ "$*" = coverage ] || exit 99
|
||||
if [ -n "$FAKE_COVERAGE_REPORT" ]; then
|
||||
printf '%s\\n' "$FAKE_COVERAGE_REPORT" > coverage/summary.json
|
||||
fi
|
||||
exit "$FAKE_COVERAGE_STATUS"
|
||||
''')
|
||||
cargo.chmod(0o755)
|
||||
os.environ['PATH'] = str(tools) + ':' + old_path
|
||||
for percentage, status in ((90.87, 0), (85, 0), (84.99, 1), (90.87, 7)):
|
||||
os.environ['FAKE_COVERAGE_REPORT'] = json.dumps({'data': [{
|
||||
'totals': {'lines': {'percent': percentage}},
|
||||
'files': [{'filename': 'src/example.rs', 'summary': {
|
||||
'lines': {'percent': 80, 'count': 100, 'covered': 80}}}],
|
||||
}]})
|
||||
os.environ['FAKE_COVERAGE_STATUS'] = str(status)
|
||||
before = Path('coverage/coverage.svg').read_bytes()
|
||||
result = run(str(REPO/'scripts/collect-coverage.sh'), ok=status == 0)
|
||||
assert result.returncode == status
|
||||
assert f'{percentage}%' in result.stdout.decode()
|
||||
if status:
|
||||
assert '--fail-under-lines' in result.stderr.decode()
|
||||
assert 'src/example.rs: 80/100 lines (80%)' in result.stderr.decode()
|
||||
assert Path('coverage/coverage.svg').read_bytes() == before
|
||||
else:
|
||||
assert ET.parse('coverage/coverage.svg').getroot().attrib['aria-label'] == f'coverage: {percentage:.0f}%'
|
||||
assert not Path('coverage/summary.json').exists()
|
||||
for report in ('', '{broken', '{"data": []}', '{"data": [{}, {}]}',
|
||||
*[json.dumps({'data': [{'totals': {'lines': {'percent': p}}}]})
|
||||
for p in (None, '90', -1, 101)]):
|
||||
for status in (0, 7):
|
||||
# An old successful report must not hide a missing current report.
|
||||
Path('coverage/summary.json').write_text('{"data":[{"totals":{"lines":{"percent":99}}}]}')
|
||||
os.environ['FAKE_COVERAGE_REPORT'] = report
|
||||
os.environ['FAKE_COVERAGE_STATUS'] = str(status)
|
||||
result = run(str(REPO/'scripts/collect-coverage.sh'), ok=False)
|
||||
assert result.returncode == (status or 1)
|
||||
assert 'ERROR: cargo coverage produced' in result.stderr.decode()
|
||||
assert not Path('coverage/summary.json').exists()
|
||||
os.environ['PATH'] = old_path
|
||||
del os.environ['FAKE_COVERAGE_REPORT'], os.environ['FAKE_COVERAGE_STATUS']
|
||||
cargo.unlink()
|
||||
root.chmod(0o555)
|
||||
if os.geteuid() != 0:
|
||||
run(*args, 'test', ok=False)
|
||||
|
|
@ -171,4 +217,4 @@ for name in ('version', 'test', 'coverage'):
|
|||
pipeline = (REPO/'Jenkinsfile').read_text()
|
||||
assert "artifacts: 'coverage/version.svg,coverage/test.svg,coverage/coverage.svg'" in pipeline
|
||||
assert 'fingerprint: true, allowEmptyArchive: false' in pipeline
|
||||
print('Badge rendering, XML, validation, publication, README and archive checks passed.')
|
||||
print('Badge rendering, XML, validation, coverage diagnostics, publication, README and archive checks passed.')
|
||||
|
|
|
|||
Loading…
Reference in New Issue