Reuse verified pinned FFprobe cache for offline releases
Beffroi/beffroi-tools/pipeline/head This commit looks good Details

This commit is contained in:
Dandaedre 2026-11-05 10:55:07 +01:00
parent e6a2f8f899
commit 2c3e3c3c00
3 changed files with 38 additions and 16 deletions

View File

@ -74,7 +74,8 @@ LGPL 9.0 asset name and published SHA-256 digest per Windows/Linux target. Dated
versioned names, unlike the downloader's default `latest` assets. To update,
review the provider release and its notices, then update all three fields together.
These dated releases are not marked immutable by GitHub; digest verification
fails closed if an asset changes. Metadata access is still needed with a warm cache.
fails closed if an asset changes. Pinned cached binaries and notices are rechecked locally before any network access;
unpinned downloads still refresh upstream metadata.
```sh
rust_realease . --package pptx-reporter --target x86_64-unknown-linux-gnu --compile-only

View File

@ -114,6 +114,26 @@ def unpack(archive, destination, target):
return binary, notices
def verified_cache(folder, target, digest):
binary = folder / ("ffprobe.exe" if "windows" in target else "ffprobe")
notices = folder / "licenses"
if folder.is_symlink():
raise ValueError("Refusing a symlinked FFprobe cache")
receipt = json.loads((notices / "provenance.json").read_text())
for name, checksum in receipt["files"].items():
path = PurePosixPath(name)
if path.is_absolute() or ".." in path.parts or "\\" in name:
raise ValueError("Invalid cache receipt")
cached = folder.joinpath(*path.parts)
if cached.is_symlink() or sha256(cached) != checksum:
raise ValueError(f"Corrupt FFprobe cache: {folder}; remove it and retry")
if binary.name not in receipt["files"] or receipt["archive_sha256"] != digest:
raise ValueError("Invalid FFprobe cache receipt")
validate_binary(binary, target)
print(f"Using verified FFprobe cache: {binary}", flush=True)
return binary, notices
def fetch(target, cache=None, release="latest", expected_sha256=None, *, asset_name=None):
if target not in ASSETS:
raise ValueError(f"Unsupported target: {target}")
@ -123,6 +143,13 @@ def fetch(target, cache=None, release="latest", expected_sha256=None, *, asset_n
raise ValueError("Expected a target-specific non-shared LGPL FFmpeg 9.0 asset")
cache = Path(cache) if cache is not None else ROOT / "downloads" / "ffprobe"
cache.mkdir(parents=True, exist_ok=True)
if expected_sha256 is not None:
digest = expected_sha256.lower()
if not re.fullmatch(r"[0-9a-f]{64}", digest):
raise ValueError("Invalid requested pin")
folder = cache / target / digest
if folder.exists():
return verified_cache(folder, target, digest)
with tempfile.TemporaryDirectory(prefix=".fetch-", dir=cache) as temporary:
temporary = Path(temporary)
metadata = temporary / "release.json"
@ -145,21 +172,7 @@ def fetch(target, cache=None, release="latest", expected_sha256=None, *, asset_n
binary = folder / ("ffprobe.exe" if "windows" in target else "ffprobe")
notices = folder / "licenses"
if folder.exists():
if folder.is_symlink():
raise ValueError("Refusing a symlinked FFprobe cache")
receipt = json.loads((notices / "provenance.json").read_text())
for name, checksum in receipt["files"].items():
path = PurePosixPath(name)
if path.is_absolute() or ".." in path.parts or "\\" in name:
raise ValueError("Invalid cache receipt")
cached = folder.joinpath(*path.parts)
if cached.is_symlink() or sha256(cached) != checksum:
raise ValueError(f"Corrupt FFprobe cache: {folder}; remove it and retry")
if binary.name not in receipt["files"] or receipt["archive_sha256"] != digest:
raise ValueError("Invalid FFprobe cache receipt")
validate_binary(binary, target)
print(f"Using verified FFprobe cache: {binary}", flush=True)
return binary, notices
return verified_cache(folder, target, digest)
archive = temporary / asset["name"]
print(f"Downloading {asset['name']} ({asset['size'] / 1048576:.1f} MiB)", flush=True)
download(url, archive)

View File

@ -60,6 +60,11 @@ class FetchTests(unittest.TestCase):
release["assets"][0]["name"] = dated_name
self.assertEqual(fetcher.fetch(WINDOWS, cache, "autobuild-fixture", digest,
asset_name=dated_name), (binary, notices))
get.reset_mock()
with patch.object(fetcher, "download", side_effect=AssertionError("network used")):
self.assertEqual(fetcher.fetch(WINDOWS, cache, "autobuild-fixture", digest,
asset_name=dated_name), (binary, notices))
self.assertEqual(get.call_count, 0)
release["assets"][0]["name"] = name
get.reset_mock()
self.assertEqual(fetcher.fetch(WINDOWS, cache), (binary, notices))
@ -67,6 +72,9 @@ class FetchTests(unittest.TestCase):
with self.assertRaisesRegex(ValueError, "requested pin"):
fetcher.fetch(WINDOWS, cache, expected_sha256="0" * 64)
binary.write_bytes(b"corrupted")
with patch.object(fetcher, "download", side_effect=AssertionError("network used")):
with self.assertRaisesRegex(ValueError, "Corrupt"):
fetcher.fetch(WINDOWS, cache, expected_sha256=digest)
with self.assertRaisesRegex(ValueError, "Corrupt"):
fetcher.fetch(WINDOWS, cache)
self.assertFalse(list(cache.glob(".fetch-*")))