tiny-wb/scripts/test-badges.py

240 lines
12 KiB
Python

#!/usr/bin/env python3
"""Isolated renderer/publication contract checks; no public assets are touched."""
import os
import json
import re
from pathlib import Path
import stat
import shlex
import shutil
import subprocess
import tempfile
import xml.etree.ElementTree as ET
REPO = Path(__file__).resolve().parent.parent
RENDER = str(REPO / 'scripts/ci-coverage-badges.sh')
PUBLISH = str(REPO / 'scripts/publish-badges.sh')
def run(*args, ok=True):
result = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
assert (result.returncode == 0) == ok, (args, result.stderr.decode())
return result
with tempfile.TemporaryDirectory() as directory:
os.chdir(directory)
os.umask(0o077)
for kind in ('version', 'test', 'coverage'):
run(RENDER, kind)
for kind, value, color in [
('version', '1.2.3', '#007ec6'),
('version', '1.2.3-very-long-prerelease.123456789', '#007ec6'),
('test', 'passing', '#4c1'), ('test', 'failing', '#e05d44'),
('test', 'pending', '#9f9f9f'), ('coverage', '0', '#e05d44'),
('coverage', '69', '#e05d44'), ('coverage', '70', '#dfb317'),
('coverage', '87.4', '#dfb317'), ('coverage', '90', '#4c1'),
('coverage', '100', '#4c1'), ('coverage', '-', '#9f9f9f'),
]:
run(RENDER, kind, '--value', value)
root = ET.parse(f'coverage/{kind}.svg').getroot()
assert color in Path(f'coverage/{kind}.svg').read_text()
if value == '1.2.3':
normal_width = int(root.attrib['width'])
if value == '1.2.3-very-long-prerelease.123456789':
assert int(root.attrib['width']) > normal_width
if value == '87.4':
assert root.attrib['aria-label'] == 'coverage: 87%'
run(RENDER, 'version', '--text', 'A<&"', '--value', 'B>&"', '--suffix', '&',
'--size', '14', '--height', '30', '--label-width', '80',
'--value-width', '120', '--color', '#abcdef', '--output', 'custom/badge.svg')
root = ET.parse('custom/badge.svg').getroot()
assert root.attrib['width'] == '200' and root.attrib['height'] == '30'
assert root.attrib['aria-label'] == 'A<&": B>&"&'
Path('directory-output').mkdir()
run(RENDER, 'version', '--output', 'directory-output', ok=False)
original = Path('coverage/coverage.svg').read_bytes()
for args in [
['unknown'], ['coverage', '--bogus'], ['coverage', '--value'],
*[['coverage', '--value', v] for v in ('-1', '101', 'nan', '1e2', 'x', '')],
*[['coverage', flag, v] for flag, v in (
('--size', '5'), ('--size', '25'), ('--height', '13'), ('--height', '65'),
('--label-width', '19'), ('--value-width', '1001'),
('--height', '999999999999999999999999999'), ('--color', 'red'),
('--color', '#1234567'), ('--text', 'bad\ntext'), ('--value', '\t'))],
]:
run(RENDER, *args, ok=False)
assert Path('coverage/coverage.svg').read_bytes() == original
assert not list(Path('.').rglob('*.tmp.*'))
for svg in Path('.').rglob('*.svg'):
ET.parse(svg)
root = Path(directory) / 'assets'
root.mkdir()
args = [PUBLISH, str(root), 'idle-org', 'tiny-wb', 'main']
names = ['version', 'test', 'coverage']
run(*args, *names)
destination = root / 'idle-org/tiny-wb/main'
for path in (root/'idle-org', root/'idle-org/tiny-wb', destination):
assert stat.S_IMODE(path.stat().st_mode) == 0o755
old_inode = (destination/'test.svg').stat().st_ino
run(*args, *names)
assert (destination/'test.svg').stat().st_ino != old_inode
for name in names:
target = destination/f'{name}.svg'
assert target.read_bytes() == Path(f'coverage/{name}.svg').read_bytes()
assert stat.S_IMODE(target.stat().st_mode) == 0o644
run(*args, 'test', 'test', ok=False)
run(*args, 'invalid', ok=False)
run(PUBLISH, 'relative', 'o', 'p', 'main', 'test', ok=False)
run(PUBLISH, str(root/'missing'), 'o', 'p', 'main', 'test', ok=False)
run(PUBLISH, str(root), '..', 'p', 'main', 'test', ok=False)
saved = Path('coverage/test.svg').read_bytes()
Path('coverage/test.svg').unlink()
Path('coverage/test.svg').symlink_to('version.svg')
run(*args, 'test', ok=False)
Path('coverage/test.svg').unlink()
Path('coverage/test.svg').touch()
run(*args, 'test', ok=False)
Path('coverage/test.svg').write_bytes(saved)
(root/'linked').symlink_to(root/'idle-org', target_is_directory=True)
run(PUBLISH, str(root), 'linked', 'tiny-wb', 'main', 'test', ok=False)
(root/'idle-org/linked').symlink_to(root/'idle-org/tiny-wb', target_is_directory=True)
run(PUBLISH, str(root), 'idle-org', 'linked', 'main', 'test', ok=False)
(root/'file').touch()
run(PUBLISH, str(root), 'file', 'p', 'main', 'test', ok=False)
# Migration unlinks the old branch link, preserving its target and contents.
old = destination.with_name('old')
destination.rename(old)
destination.symlink_to(old, target_is_directory=True)
run(*args, *names)
assert destination.is_dir() and not destination.is_symlink()
assert (old/'test.svg').read_bytes() == saved
# Failed copy, chmod and rename must preserve the published badge and
# remove the unfinished temporary file.
tools = Path(directory) / 'fake-bin'
tools.mkdir()
old_path = os.environ['PATH']
for command in ('cp', 'chmod', 'mv'):
stub = tools / command
stub.write_text('#!/bin/sh\nexit 1\n')
stub.chmod(0o755)
os.environ['PATH'] = str(tools) + ':' + old_path
run(*args, 'test', ok=False)
os.environ['PATH'] = old_path
stub.unlink()
assert (destination/'test.svg').read_bytes() == saved
assert not list(destination.glob('.*.svg.*'))
stub = tools / 'mv'
stub.write_text('#!/bin/sh\nkill -TERM "$PPID"\nexit 1\n')
stub.chmod(0o755)
os.environ['PATH'] = str(tools) + ':' + old_path
run(*args, 'test', ok=False)
run(RENDER, 'coverage', '--value', '50', ok=False)
os.environ['PATH'] = old_path
stub.unlink()
assert Path('coverage/coverage.svg').read_bytes() == original
assert not list(Path('coverage').glob('*.tmp.*'))
assert not list(destination.glob('.*.svg.*'))
# Exercise cleanup with real Cargo metadata from this repository while
# keeping all generated files in the isolated directory.
Path('scripts').symlink_to(REPO/'scripts', target_is_directory=True)
cargo = tools/'cargo'
cargo.write_text('#!/bin/sh\nexec ' + shlex.quote(shutil.which('cargo')) +
' metadata --manifest-path ' + shlex.quote(str(REPO/'Cargo.toml')) +
' --no-deps --locked --format-version 1\n')
cargo.chmod(0o755)
os.environ['PATH'] = str(tools) + ':' + old_path
os.environ['BUILD_NUMBER'] = '123'
for marker in ('tests-passed', 'coverage-ready'):
Path(f'coverage/.{marker}-122').touch()
run(str(REPO/'scripts/finalize-badges.sh'))
assert ET.parse('coverage/test.svg').getroot().attrib['aria-label'] == 'tests: failing'
assert ET.parse('coverage/coverage.svg').getroot().attrib['aria-label'] == 'coverage: -%'
run(RENDER, 'coverage', '--value', '87.4')
for marker in ('tests-passed', 'coverage-ready'):
Path(f'coverage/.{marker}-123').touch()
run(str(REPO/'scripts/finalize-badges.sh'))
assert ET.parse('coverage/test.svg').getroot().attrib['aria-label'] == 'tests: passing'
assert ET.parse('coverage/coverage.svg').getroot().attrib['aria-label'] == 'coverage: 87%'
assert not list(Path('coverage').glob('.*-123'))
os.environ['PATH'] = old_path
cargo.unlink()
# A coverage-floor failure can write a valid report and exit silently.
# Exercise the collector with real jq and rendering, but no Rust rebuild.
cargo.write_text('''#!/bin/sh
[ "$*" = coverage ] || exit 99
echo "fake compiler output"
echo "fake test output" >&2
if [ -n "$FAKE_COVERAGE_REPORT" ]; then
printf '%s\\n' "$FAKE_COVERAGE_REPORT" > coverage/summary.json
fi
exit "$FAKE_COVERAGE_STATUS"
''')
cargo.chmod(0o755)
os.environ['PATH'] = str(tools) + ':' + old_path
for percentage, status in ((91.49, 0), (85, 0), (84.99, 1), (91.49, 7)):
os.environ['FAKE_COVERAGE_REPORT'] = json.dumps({'data': [{
'totals': {'lines': {'percent': percentage}},
'files': [{'filename': 'src/example.rs', 'summary': {
'lines': {'percent': 80, 'count': 100, 'covered': 80}}}],
}]})
os.environ['FAKE_COVERAGE_STATUS'] = str(status)
before = Path('coverage/coverage.svg').read_bytes()
result = run(str(REPO/'scripts/collect-coverage.sh'), ok=status == 0)
assert result.returncode == status
assert result.stdout.decode() == f'Final line coverage: {percentage:.2f}%\n'
assert 'fake compiler output' in Path('coverage/coverage.log').read_text()
assert 'fake test output' in Path('coverage/coverage.log').read_text()
if status:
assert '--fail-under-lines' in result.stderr.decode()
assert 'src/example.rs' not in result.stderr.decode()
assert 'coverage/coverage.log' in result.stderr.decode()
assert Path('coverage/coverage.svg').read_bytes() == before
else:
assert ET.parse('coverage/coverage.svg').getroot().attrib['aria-label'] == f'coverage: {percentage:.0f}%'
assert not Path('coverage/summary.json').exists()
for report in ('', '{broken', '{"data": []}', '{"data": [{}, {}]}',
*[json.dumps({'data': [{'totals': {'lines': {'percent': p}}}]})
for p in (None, '90', -1, 101)]):
for status in (0, 7):
# An old successful report must not hide a missing current report.
Path('coverage/summary.json').write_text('{"data":[{"totals":{"lines":{"percent":99}}}]}')
os.environ['FAKE_COVERAGE_REPORT'] = report
os.environ['FAKE_COVERAGE_STATUS'] = str(status)
result = run(str(REPO/'scripts/collect-coverage.sh'), ok=False)
assert result.returncode == (status or 1)
assert 'ERROR: cargo coverage produced' in result.stderr.decode()
assert not Path('coverage/summary.json').exists()
os.environ['PATH'] = old_path
del os.environ['FAKE_COVERAGE_REPORT'], os.environ['FAKE_COVERAGE_STATUS']
cargo.unlink()
root.chmod(0o555)
if os.geteuid() != 0:
run(*args, 'test', ok=False)
root.chmod(0o700)
(destination/'test.svg').unlink()
(destination/'test.svg').mkdir()
run(*args, 'test', ok=False)
assert not list(destination.glob('.*.svg.*'))
# Keep the alias's dependency filter portable across cache roots without
# excluding any production module (including low-coverage startup/scan code).
alias = next(line for line in (REPO/'.cargo/config.toml').read_text().splitlines()
if line.startswith('coverage = '))
arguments = shlex.split(alias.split('"', 1)[1].rsplit('"', 1)[0])
pattern = arguments[arguments.index('--ignore-filename-regex') + 1]
assert arguments[arguments.index('--fail-under-lines') + 1] == '85'
for cache in ('/root/.cargo', '/home/user/.cargo', '/custom/cargo-cache'):
assert re.search(pattern, cache + '/registry/src/index.crates.io-123/serde-1.0/src/lib.rs')
for source in (REPO/'src').rglob('*.rs'):
assert not re.search(pattern, str(source))
readme = (REPO/'README.md').read_text()
for name in ('version', 'test', 'coverage'):
assert readme.count(f'https://assets.dandaedre.com/coverage/idle-org/tiny-wb/main/{name}.svg') == 1
pipeline = (REPO/'Jenkinsfile').read_text()
assert "CARGO_LLVM_COV_VERSION = '0.9.1'" in pipeline
assert "artifacts: 'coverage/version.svg,coverage/test.svg,coverage/coverage.svg'" in pipeline
assert 'fingerprint: true, allowEmptyArchive: false' in pipeline
print('Badge rendering, XML, validation, coverage diagnostics, publication, README and archive checks passed.')