Provision parity tools in CI and retain coverage after test failures

This commit is contained in:
Dandaedre 2026-10-20 14:51:55 +02:00
parent 6ad0bc71d1
commit de804a1afe
6 changed files with 152 additions and 23 deletions

21
Jenkinsfile vendored
View File

@ -52,14 +52,19 @@ pipeline {
stage('Tests') {
steps {
sh '''
set -eu
python3 -m unittest discover -s tests -v
cargo test --locked --workspace --all-features
cargo build --locked --workspace --all-features
python3 scripts/parity.py target/debug/pptx-reporter
python3 scripts/ci.py tests-passed
'''
// Keep failures visible while allowing independent coverage measurement.
catchError(buildResult: 'FAILURE', stageResult: 'FAILURE', catchInterruptions: false) {
sh '''
set -eu
python3 -m unittest discover -s tests -v
cargo test --locked --workspace --all-features
cargo build --locked --workspace --all-features
export PATH="$PWD/downloads/ci-tools/bin:$PATH"
python3 scripts/ci_tools.py
python3 scripts/parity.py target/debug/pptx-reporter
python3 scripts/ci.py tests-passed
'''
}
}
}
stage('Coverage') {

View File

@ -1,11 +1,26 @@
# CI and coverage
Jenkins must use a Multibranch Pipeline supplying `BRANCH_NAME`, with the
`rust_no_docker` worker, Python 3.11+, Rust, FFmpeg/FFprobe and the Typst CLI.
`rust_no_docker` worker, Python 3.11+ and Rust.
SCM credentials, webhooks and the publication mount are administrator setup.
CI checks formatting, Clippy, rustdoc, Python/Rust tests, real-media parity and
whole-workspace Rust coverage. It never creates tags or release artifacts.
Parity reuses FFmpeg, FFprobe and Typst from PATH when available. Otherwise
`scripts/ci_tools.py` fetches checksum-verified FFmpeg/FFprobe on Linux x86-64
and installs locked `typst-cli 0.13.1` with Cargo under ignored
`downloads/ci-tools/`. No sudo is needed; missing tools require network access
and a first-time Typst build can take several minutes. To provision locally:
```sh
export PATH="$PWD/downloads/ci-tools/bin:$PATH"
python3 scripts/ci_tools.py
```
A test, provisioning or parity failure still fails the build, but does not skip
the independent coverage stage. The test-passed marker requires all of them to
succeed; fresh measured coverage can therefore accompany a failing test badge.
## Measurement
CI installs pinned `cargo-llvm-cov 0.9.1` and `llvm-tools-preview`. The tracked

40
scripts/ci_tools.py Normal file
View File

@ -0,0 +1,40 @@
#!/usr/bin/env python3
"""Supply missing parity tools locally, without sudo or changing the worker."""
from pathlib import Path
import os
import platform
import shutil
import subprocess
try:
from .fetch_ffprobe import fetch
except ImportError:
from fetch_ffprobe import fetch
ROOT = Path(__file__).resolve().parents[1]
TYPST_VERSION = "0.13.1"
def prepare():
install = ROOT / "downloads/ci-tools"
bindir = install / "bin"
bindir.mkdir(parents=True, exist_ok=True)
os.environ["PATH"] = str(bindir) + os.pathsep + os.environ.get("PATH", "")
for program in ("ffmpeg", "ffprobe"):
if not shutil.which(program):
if platform.system() != "Linux" or platform.machine().lower() not in ("x86_64", "amd64"):
raise RuntimeError(f"Install {program} on PATH; automatic CI provisioning needs Linux x86-64")
binary, _ = fetch("x86_64-unknown-linux-gnu", program=program)
link = bindir / program
link.unlink(missing_ok=True)
link.symlink_to(binary.resolve())
subprocess.run([program, "-version"], check=True, stdout=subprocess.DEVNULL, timeout=30)
print(f"Parity dependency ready: {program}", flush=True)
if not shutil.which("typst"):
subprocess.run(["cargo", "install", "typst-cli", "--version", TYPST_VERSION,
"--locked", "--root", str(install)], check=True)
subprocess.run(["typst", "--version"], check=True, timeout=30)
if __name__ == "__main__":
prepare()

View File

@ -1,5 +1,5 @@
#!/usr/bin/env python3
"""Fetch verified FFprobe binaries for release packaging, never into source control."""
"""Fetch verified FFprobe for releases; share validation with CI FFmpeg setup."""
import argparse
import hashlib
import json
@ -62,9 +62,11 @@ def validate_binary(path, target):
raise ValueError(f"Unsupported target: {target}")
def unpack(archive, destination, target):
"""Copy only FFprobe and upstream notices; never extract archive paths directly."""
executable = "ffprobe.exe" if "windows" in target else "ffprobe"
def unpack(archive, destination, target, program="ffprobe"):
"""Copy the selected program and notices without extracting archive paths."""
if program not in ("ffprobe", "ffmpeg"):
raise ValueError("Unsupported FFmpeg program")
executable = program + (".exe" if "windows" in target else "")
notices = destination / "licenses"
notices.mkdir()
found = False
@ -82,7 +84,7 @@ def unpack(archive, destination, target):
raise ValueError(f"Archive member too large: {name}")
if is_binary:
if found or path.parent.name != "bin":
raise ValueError("Expected exactly one bin/ffprobe executable")
raise ValueError(f"Expected exactly one bin/{executable} executable")
found = True
output = destination / executable
else:
@ -107,17 +109,19 @@ def unpack(archive, destination, target):
continue
member(info.name, info.size, info.isfile(), lambda i=info: bundle.extractfile(i))
if not found or not licensed:
raise ValueError("Archive must include FFprobe and its upstream license")
raise ValueError(f"Archive must include {program} and its upstream license")
binary = destination / executable
validate_binary(binary, target)
binary.chmod(0o755)
return binary, notices
def fetch(target, cache=None, release="latest", expected_sha256=None):
def fetch(target, cache=None, release="latest", expected_sha256=None, program="ffprobe"):
if program not in ("ffprobe", "ffmpeg"):
raise ValueError("Unsupported FFmpeg program")
if target not in ASSETS:
raise ValueError(f"Unsupported target: {target}")
cache = Path(cache) if cache is not None else ROOT / "downloads" / "ffprobe"
cache = Path(cache) if cache is not None else ROOT / "downloads" / program
cache.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix=".fetch-", dir=cache) as temporary:
temporary = Path(temporary)
@ -138,7 +142,9 @@ def fetch(target, cache=None, release="latest", expected_sha256=None):
if not url.startswith("https://github.com/BtbN/FFmpeg-Builds/releases/download/"):
raise ValueError("Unexpected upstream download URL")
folder = cache / target / digest
binary = folder / ("ffprobe.exe" if "windows" in target else "ffprobe")
if program != "ffprobe":
folder = cache / program / target / digest
binary = folder / (program + (".exe" if "windows" in target else ""))
notices = folder / "licenses"
if folder.exists():
if folder.is_symlink():
@ -154,7 +160,7 @@ def fetch(target, cache=None, release="latest", expected_sha256=None):
if binary.name not in receipt["files"] or receipt["archive_sha256"] != digest:
raise ValueError("Invalid FFprobe cache receipt")
validate_binary(binary, target)
print(f"Using verified FFprobe cache: {binary}", flush=True)
print(f"Using verified {program} cache: {binary}", flush=True)
return binary, notices
archive = temporary / asset["name"]
print(f"Downloading {asset['name']} ({asset['size'] / 1048576:.1f} MiB)", flush=True)
@ -163,7 +169,7 @@ def fetch(target, cache=None, release="latest", expected_sha256=None):
raise ValueError("FFprobe archive SHA-256 mismatch; nothing installed")
stage = temporary / "bundle"
stage.mkdir()
staged_binary, staged_notices = unpack(archive, stage, target)
staged_binary, staged_notices = unpack(archive, stage, target, program)
native = platform.machine().lower() in ("amd64", "x86_64") and (
(os.name == "nt" and "windows" in target)
or (platform.system() == "Linux" and "linux" in target))
@ -172,8 +178,8 @@ def fetch(target, cache=None, release="latest", expected_sha256=None):
result = subprocess.run([str(staged_binary.resolve()), "-version"], check=True,
capture_output=True, text=True, timeout=30)
lines = result.stdout.splitlines()
if not lines or not lines[0].startswith("ffprobe version "):
raise ValueError("Downloaded executable did not identify itself as FFprobe")
if not lines or not lines[0].startswith(f"{program} version "):
raise ValueError(f"Downloaded executable did not identify itself as {program}")
version = lines[0]
receipt = {
"provider": "https://github.com/BtbN/FFmpeg-Builds",
@ -186,7 +192,7 @@ def fetch(target, cache=None, release="latest", expected_sha256=None):
(staged_notices / "provenance.json").write_text(json.dumps(receipt, indent=2) + "\n")
folder.parent.mkdir(parents=True, exist_ok=True)
stage.rename(folder)
print(f"Verified FFprobe: {binary}", flush=True)
print(f"Verified {program}: {binary}", flush=True)
return binary, notices

42
tests/test_ci_tools.py Normal file
View File

@ -0,0 +1,42 @@
import os
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from scripts import ci_tools
class CIToolsTests(unittest.TestCase):
def test_installed_tools_are_checked_without_downloads(self):
with tempfile.TemporaryDirectory() as temp, patch.object(ci_tools, "ROOT", Path(temp)), \
patch.dict(os.environ), patch.object(ci_tools.shutil, "which", return_value="/usr/bin/tool"), \
patch.object(ci_tools, "fetch") as fetch, patch.object(ci_tools.subprocess, "run") as run:
ci_tools.prepare()
fetch.assert_not_called()
self.assertEqual([call.args[0][0] for call in run.call_args_list], ["ffmpeg", "ffprobe", "typst"])
def test_missing_tools_are_provisioned_locally_and_typst_is_pinned(self):
with tempfile.TemporaryDirectory() as temp, patch.object(ci_tools, "ROOT", Path(temp)), \
patch.dict(os.environ), patch.object(ci_tools.shutil, "which", return_value=None), \
patch.object(ci_tools.platform, "system", return_value="Linux"), \
patch.object(ci_tools.platform, "machine", return_value="x86_64"), \
patch.object(ci_tools.subprocess, "run") as run:
binary = Path(temp) / "downloaded"
binary.write_bytes(b"fixture")
with patch.object(ci_tools, "fetch", return_value=(binary, Path(temp))) as fetch:
ci_tools.prepare()
self.assertEqual([c.kwargs["program"] for c in fetch.call_args_list], ["ffmpeg", "ffprobe"])
self.assertEqual((Path(temp) / "downloads/ci-tools/bin/ffmpeg").resolve(), binary)
install = next(c.args[0] for c in run.call_args_list if c.args[0][0] == "cargo")
self.assertIn("--locked", install)
self.assertEqual(install[install.index("--version") + 1], "0.13.1")
def test_download_failure_propagates(self):
with tempfile.TemporaryDirectory() as temp, patch.object(ci_tools, "ROOT", Path(temp)), \
patch.dict(os.environ), patch.object(ci_tools.shutil, "which", return_value=None), \
patch.object(ci_tools.platform, "system", return_value="Linux"), \
patch.object(ci_tools.platform, "machine", return_value="x86_64"), \
patch.object(ci_tools, "fetch", side_effect=ValueError("checksum mismatch")):
with self.assertRaisesRegex(ValueError, "checksum mismatch"):
ci_tools.prepare()

View File

@ -66,6 +66,27 @@ class FetchTests(unittest.TestCase):
fetcher.fetch(WINDOWS, cache)
self.assertFalse(list(cache.glob(".fetch-*")))
def test_ci_ffmpeg_selection_and_cache_are_separate(self):
content = bundle([("build/bin/ffprobe.exe", pe()), ("build/bin/ffmpeg.exe", pe()),
("build/LICENSE", b"notice")])
digest = hashlib.sha256(content).hexdigest()
name = fetcher.ASSETS[WINDOWS]
release = {"id": 1, "assets": [{"id": 2, "name": name, "digest": "sha256:" + digest,
"size": len(content), "browser_download_url":
"https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/" + name}]}
def download(url, destination, limit=None):
destination.write_bytes(json.dumps(release).encode() if url.startswith(fetcher.API) else content)
with tempfile.TemporaryDirectory() as temp, patch.object(fetcher, "download", side_effect=download), \
patch.object(fetcher.platform, "machine", return_value="test-host"):
encoder, _ = fetcher.fetch(WINDOWS, Path(temp), program="ffmpeg")
probe, _ = fetcher.fetch(WINDOWS, Path(temp))
self.assertEqual(encoder.name, "ffmpeg.exe")
self.assertEqual(probe.name, "ffprobe.exe")
self.assertNotEqual(encoder.parent, probe.parent)
self.assertEqual(fetcher.fetch(WINDOWS, Path(temp), program="ffmpeg")[0], encoder)
with self.assertRaisesRegex(ValueError, "Unsupported"):
fetcher.fetch(WINDOWS, Path(temp), program="other")
def test_checksum_mismatch_leaves_no_installed_files(self):
name = fetcher.ASSETS[WINDOWS]
release = {"id": 1, "assets": [{"id": 2, "name": name, "digest": "sha256:" + "0" * 64,