Reuse verified pinned FFprobe cache for offline releases
Beffroi/beffroi-tools/pipeline/head This commit looks good
Details
Beffroi/beffroi-tools/pipeline/head This commit looks good
Details
This commit is contained in:
parent
e6a2f8f899
commit
2c3e3c3c00
|
|
@ -74,7 +74,8 @@ LGPL 9.0 asset name and published SHA-256 digest per Windows/Linux target. Dated
|
|||
versioned names, unlike the downloader's default `latest` assets. To update,
|
||||
review the provider release and its notices, then update all three fields together.
|
||||
These dated releases are not marked immutable by GitHub; digest verification
|
||||
fails closed if an asset changes. Metadata access is still needed with a warm cache.
|
||||
fails closed if an asset changes. Pinned cached binaries and notices are rechecked locally before any network access;
|
||||
unpinned downloads still refresh upstream metadata.
|
||||
|
||||
```sh
|
||||
rust_realease . --package pptx-reporter --target x86_64-unknown-linux-gnu --compile-only
|
||||
|
|
|
|||
|
|
@ -114,6 +114,26 @@ def unpack(archive, destination, target):
|
|||
return binary, notices
|
||||
|
||||
|
||||
def verified_cache(folder, target, digest):
|
||||
binary = folder / ("ffprobe.exe" if "windows" in target else "ffprobe")
|
||||
notices = folder / "licenses"
|
||||
if folder.is_symlink():
|
||||
raise ValueError("Refusing a symlinked FFprobe cache")
|
||||
receipt = json.loads((notices / "provenance.json").read_text())
|
||||
for name, checksum in receipt["files"].items():
|
||||
path = PurePosixPath(name)
|
||||
if path.is_absolute() or ".." in path.parts or "\\" in name:
|
||||
raise ValueError("Invalid cache receipt")
|
||||
cached = folder.joinpath(*path.parts)
|
||||
if cached.is_symlink() or sha256(cached) != checksum:
|
||||
raise ValueError(f"Corrupt FFprobe cache: {folder}; remove it and retry")
|
||||
if binary.name not in receipt["files"] or receipt["archive_sha256"] != digest:
|
||||
raise ValueError("Invalid FFprobe cache receipt")
|
||||
validate_binary(binary, target)
|
||||
print(f"Using verified FFprobe cache: {binary}", flush=True)
|
||||
return binary, notices
|
||||
|
||||
|
||||
def fetch(target, cache=None, release="latest", expected_sha256=None, *, asset_name=None):
|
||||
if target not in ASSETS:
|
||||
raise ValueError(f"Unsupported target: {target}")
|
||||
|
|
@ -123,6 +143,13 @@ def fetch(target, cache=None, release="latest", expected_sha256=None, *, asset_n
|
|||
raise ValueError("Expected a target-specific non-shared LGPL FFmpeg 9.0 asset")
|
||||
cache = Path(cache) if cache is not None else ROOT / "downloads" / "ffprobe"
|
||||
cache.mkdir(parents=True, exist_ok=True)
|
||||
if expected_sha256 is not None:
|
||||
digest = expected_sha256.lower()
|
||||
if not re.fullmatch(r"[0-9a-f]{64}", digest):
|
||||
raise ValueError("Invalid requested pin")
|
||||
folder = cache / target / digest
|
||||
if folder.exists():
|
||||
return verified_cache(folder, target, digest)
|
||||
with tempfile.TemporaryDirectory(prefix=".fetch-", dir=cache) as temporary:
|
||||
temporary = Path(temporary)
|
||||
metadata = temporary / "release.json"
|
||||
|
|
@ -145,21 +172,7 @@ def fetch(target, cache=None, release="latest", expected_sha256=None, *, asset_n
|
|||
binary = folder / ("ffprobe.exe" if "windows" in target else "ffprobe")
|
||||
notices = folder / "licenses"
|
||||
if folder.exists():
|
||||
if folder.is_symlink():
|
||||
raise ValueError("Refusing a symlinked FFprobe cache")
|
||||
receipt = json.loads((notices / "provenance.json").read_text())
|
||||
for name, checksum in receipt["files"].items():
|
||||
path = PurePosixPath(name)
|
||||
if path.is_absolute() or ".." in path.parts or "\\" in name:
|
||||
raise ValueError("Invalid cache receipt")
|
||||
cached = folder.joinpath(*path.parts)
|
||||
if cached.is_symlink() or sha256(cached) != checksum:
|
||||
raise ValueError(f"Corrupt FFprobe cache: {folder}; remove it and retry")
|
||||
if binary.name not in receipt["files"] or receipt["archive_sha256"] != digest:
|
||||
raise ValueError("Invalid FFprobe cache receipt")
|
||||
validate_binary(binary, target)
|
||||
print(f"Using verified FFprobe cache: {binary}", flush=True)
|
||||
return binary, notices
|
||||
return verified_cache(folder, target, digest)
|
||||
archive = temporary / asset["name"]
|
||||
print(f"Downloading {asset['name']} ({asset['size'] / 1048576:.1f} MiB)", flush=True)
|
||||
download(url, archive)
|
||||
|
|
|
|||
|
|
@ -60,6 +60,11 @@ class FetchTests(unittest.TestCase):
|
|||
release["assets"][0]["name"] = dated_name
|
||||
self.assertEqual(fetcher.fetch(WINDOWS, cache, "autobuild-fixture", digest,
|
||||
asset_name=dated_name), (binary, notices))
|
||||
get.reset_mock()
|
||||
with patch.object(fetcher, "download", side_effect=AssertionError("network used")):
|
||||
self.assertEqual(fetcher.fetch(WINDOWS, cache, "autobuild-fixture", digest,
|
||||
asset_name=dated_name), (binary, notices))
|
||||
self.assertEqual(get.call_count, 0)
|
||||
release["assets"][0]["name"] = name
|
||||
get.reset_mock()
|
||||
self.assertEqual(fetcher.fetch(WINDOWS, cache), (binary, notices))
|
||||
|
|
@ -67,6 +72,9 @@ class FetchTests(unittest.TestCase):
|
|||
with self.assertRaisesRegex(ValueError, "requested pin"):
|
||||
fetcher.fetch(WINDOWS, cache, expected_sha256="0" * 64)
|
||||
binary.write_bytes(b"corrupted")
|
||||
with patch.object(fetcher, "download", side_effect=AssertionError("network used")):
|
||||
with self.assertRaisesRegex(ValueError, "Corrupt"):
|
||||
fetcher.fetch(WINDOWS, cache, expected_sha256=digest)
|
||||
with self.assertRaisesRegex(ValueError, "Corrupt"):
|
||||
fetcher.fetch(WINDOWS, cache)
|
||||
self.assertFalse(list(cache.glob(".fetch-*")))
|
||||
|
|
|
|||
Loading…
Reference in New Issue