From 96ae48f9f25a1f8cd931da5d983723d5c1024887 Mon Sep 17 00:00:00 2001 From: Dandaedre Date: Tue, 20 Oct 2026 13:19:14 +0200 Subject: [PATCH] Fetch verified FFprobe for portable releases --- AGENTS.md | 3 + docs/rust-reporter-tasks.md | 15 ++- docs/rust-reporter.md | 52 +++++++-- rust-release.toml | 3 + scripts/fetch_ffprobe.py | 207 ++++++++++++++++++++++++++++++++++++ scripts/package-portable.py | 30 +++++- tests/test_fetch_ffprobe.py | 151 ++++++++++++++++++++++++++ tests/test_portable.py | 6 ++ 8 files changed, 454 insertions(+), 13 deletions(-) create mode 100644 scripts/fetch_ffprobe.py create mode 100644 tests/test_fetch_ffprobe.py diff --git a/AGENTS.md b/AGENTS.md index 82dc900..c75fc86 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,6 +40,9 @@ with quick-start examples and links to detailed documentation in `docs/`. three SVGs and publishes them only from main during cleanup. CI never creates releases or uploads release binaries. Cross and `rust_realease` are separate, manually invoked build/release preparation tools. +- `scripts/fetch_ffprobe.py` fetches checksum-verified target FFprobe builds into + ignored `downloads/`. Portable release packaging calls it with + `--fetch-ffprobe`; include upstream notices/provenance, never commit binaries. ## Working conventions diff --git a/docs/rust-reporter-tasks.md b/docs/rust-reporter-tasks.md index 2c4e219..1553c4b 100644 --- a/docs/rust-reporter-tasks.md +++ b/docs/rust-reporter-tasks.md @@ -56,12 +56,21 @@ External checkpoints still required before distributing a Windows release: - [ ] Run on a real Windows machine: drag/drop, file dialogs, scaling, Unicode paths, opening reports, and a standalone target FFprobe beside the app. -- [ ] Choose the FFprobe distribution and collect its licence/build provenance - for a portable bundle; review dependency notices before distribution. +- [x] Add release-time downloading of BtbN's target FFprobe with archive checksum, + executable architecture checks, upstream license and provenance. Keep + downloads and portable bundles ignored by Git. +- [ ] Review dependency notices before distribution. - [ ] Configure/execute the actual Jenkins Multibranch job and confirm that its `rust_no_docker` worker has the documented dependencies and asset mount. - [ ] Confirm actual Jenkins archival, default-branch publication and all three public badge responses after an authorized push/main build. Local tests validate generated SVGs and publication into temporary directories only. -No production asset mount was modified, and no push, tag, or release was made. +No production asset mount was modified and no push or release was made. +The subsequent local `v0.1.0` tag marks the original trial commit `2591252`; +release-time FFprobe fetching is a later follow-up change. + +FFprobe follow-up validation: both target downloads passed checksum and +architecture checks; the Linux binary successfully probed real MPEG4 media. +Both portable archives were built with `--fetch-ffprobe`. Windows executable +imports were inspected; actual Windows execution remains unchecked. diff --git a/docs/rust-reporter.md b/docs/rust-reporter.md index f605e6b..43eec10 100644 --- a/docs/rust-reporter.md +++ b/docs/rust-reporter.md @@ -83,19 +83,57 @@ explicitly created/pushed tag and a clean checkout. Dry-run reports a missing tag as a prerequisite, even while previewing this untagged feature branch. No tag or release is created as part of this trial. -`rust_realease` archives contain Cargo binaries only. They do **not** contain -FFprobe or its notices. Prepare the user-facing portable bundle separately: +`rust_realease` archives contain Cargo binaries only. Its current documented +configuration has no arbitrary packaging hook. Release automation can call the +portable packager after the Cross builds above; it fetches FFprobe on demand: ```sh python3 scripts/package-portable.py --target x86_64-pc-windows-gnu \ - --ffprobe /path/to/windows/ffprobe.exe \ - --ffprobe-licenses /path/to/ffmpeg-licences-and-provenance + --fetch-ffprobe +python3 scripts/package-portable.py --target x86_64-unknown-linux-gnu \ + --fetch-ffprobe ``` This creates a ZIP under `dist/`, with both reporter binaries, FFprobe, -licences, quick-start instructions and a SHA-256 checksum. Supply FFprobe built -for that target, including any needed runtime DLLs in a self-contained -distribution; the packager expects a standalone FFprobe executable. +licences, download provenance, quick-start instructions and a SHA-256 checksum +(Linux uses `.tar.gz`). These commands expect the reporter binaries under +`target//release/`. `rust_realease` can use a separate build directory +for dirty checkouts; use the explicit Cross commands above for these paths. + +The standalone downloader is also callable from a release script: + +```sh +python3 scripts/fetch_ffprobe.py --target x86_64-pc-windows-gnu +``` + +It selects BtbN's **FFmpeg 9.0 release-branch, non-shared LGPL** builds, +checks the archive against the SHA-256 reported by GitHub's release API, +and validates PE/ELF format and x86-64 machine type. On a matching host it also +runs `ffprobe -version` before installing the cache entry. Foreign executables +are inspected without execution. Downloads are temporary; the extracted +binary, upstream notices, and provenance are cached under ignored +`downloads/ffprobe///`. Cached file hashes are verified +on reuse. A failed download/check does not install a cache entry. + +The default upstream tag `latest` moves. For repeatable release builds, pass +`--ffprobe-release TAG --ffprobe-sha256 HASH` to the packager (or `--release TAG +--sha256 HASH` to the downloader). Provenance records the release/asset IDs, +download URL and digest. A missing asset or mismatched digest fails rather +than silently choosing another build. GitHub API rate limits/network failures +are errors; a previously cached download does not bypass metadata verification. + +[BtbN's compatibility notes](https://github.com/BtbN/FFmpeg-Builds#readme) +specify Windows 10 22H2 or newer, and Linux glibc 2.28/kernel 4.18 or newer. +These builds are linked from [FFmpeg's download page](https://ffmpeg.org/download.html). +For another provider or offline packaging, supply a standalone target binary: + +```sh +python3 scripts/package-portable.py --target x86_64-pc-windows-gnu \ + --ffprobe /path/to/ffprobe.exe --ffprobe-licenses /path/to/notices +``` + +No FFprobe binary is committed. Only the downloader and packaging code are +versioned; `downloads/`, `target/` and `dist/` are ignored. The app/report fonts are already embedded. Do not describe a binary-only `rust_realease` archive as the complete portable distribution. diff --git a/rust-release.toml b/rust-release.toml index 0bdc096..91999a0 100644 --- a/rust-release.toml +++ b/rust-release.toml @@ -8,6 +8,9 @@ locked = true [package] format = "tgz" +# rust_realease packages Cargo binaries only. Release automation can additionally +# call: python3 scripts/package-portable.py --target TARGET --fetch-ffprobe +# after Cross builds populate target/TARGET/release (see docs/rust-reporter.md). [[repository.fallbacks]] origin = "https://git.dandaedre.com/Beffroi/beffroi-tools" diff --git a/scripts/fetch_ffprobe.py b/scripts/fetch_ffprobe.py new file mode 100644 index 0000000..597ec82 --- /dev/null +++ b/scripts/fetch_ffprobe.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +"""Fetch verified FFprobe binaries for release packaging, never into source control.""" +import argparse +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import platform +import re +import shutil +import struct +import subprocess +import tarfile +import tempfile +import urllib.parse +import urllib.request +import zipfile + +ROOT = Path(__file__).resolve().parents[1] +API = "https://api.github.com/repos/BtbN/FFmpeg-Builds/releases/tags/" +ASSETS = { + "x86_64-pc-windows-gnu": "ffmpeg-n9.0-latest-win64-lgpl-9.0.zip", + "x86_64-unknown-linux-gnu": "ffmpeg-n9.0-latest-linux64-lgpl-9.0.tar.xz", +} + + +def sha256(path): + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def download(url, destination, limit=512 * 1024 * 1024): + if urllib.parse.urlsplit(url).scheme != "https": + raise ValueError("Downloads must use HTTPS") + request = urllib.request.Request(url, headers={"User-Agent": "pptx-reporter-release"}) + with urllib.request.urlopen(request, timeout=60) as response, destination.open("wb") as out: + if urllib.parse.urlsplit(response.url).scheme != "https": + raise ValueError("Refusing a non-HTTPS redirect") + size = 0 + while chunk := response.read(1024 * 1024): + size += len(chunk) + if size > limit: + raise ValueError("Download exceeds size limit") + out.write(chunk) + + +def validate_binary(path, target): + """Check the executable format AND x86-64 machine type without executing it.""" + with path.open("rb") as stream: + header = stream.read(64) + if target == "x86_64-pc-windows-gnu": + if len(header) != 64 or header[:2] != b"MZ": + raise ValueError("Expected a Windows PE executable") + stream.seek(struct.unpack_from(" (256 * 1024 * 1024 if is_binary else 8 * 1024 * 1024): + raise ValueError(f"Archive member too large: {name}") + if is_binary: + if found or path.parent.name != "bin": + raise ValueError("Expected exactly one bin/ffprobe executable") + found = True + output = destination / executable + else: + output = notices.joinpath(*path.parts) + licensed |= path.name.lower().startswith(("license", "copying")) + output.parent.mkdir(parents=True, exist_ok=True) + with opener() as source, output.open("xb") as out: + shutil.copyfileobj(source, out) + + if archive.suffix == ".zip": + with zipfile.ZipFile(archive) as bundle: + for info in bundle.infolist(): + if info.is_dir(): + continue + mode = info.external_attr >> 16 + regular = not info.is_dir() and (mode & 0o170000) in (0, 0o100000) + member(info.filename, info.file_size, regular, lambda i=info: bundle.open(i)) + else: + with tarfile.open(archive, "r:xz") as bundle: + for info in bundle: + if info.isdir(): + continue + member(info.name, info.size, info.isfile(), lambda i=info: bundle.extractfile(i)) + if not found or not licensed: + raise ValueError("Archive must include FFprobe and its upstream license") + binary = destination / executable + validate_binary(binary, target) + binary.chmod(0o755) + return binary, notices + + +def fetch(target, cache=None, release="latest", expected_sha256=None): + if target not in ASSETS: + raise ValueError(f"Unsupported target: {target}") + cache = Path(cache) if cache is not None else ROOT / "downloads" / "ffprobe" + cache.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=".fetch-", dir=cache) as temporary: + temporary = Path(temporary) + metadata = temporary / "release.json" + download(API + urllib.parse.quote(release, safe=""), metadata, 8 * 1024 * 1024) + release_data = json.loads(metadata.read_text()) + assets = [a for a in release_data["assets"] if a["name"] == ASSETS[target]] + if len(assets) != 1: + raise ValueError(f"Release {release} does not contain {ASSETS[target]}") + asset = assets[0] + digest = asset.get("digest", "") + if not re.fullmatch(r"sha256:[0-9a-f]{64}", digest): + raise ValueError("Upstream asset has no usable SHA-256 digest") + digest = digest.removeprefix("sha256:") + if expected_sha256 is not None and digest != expected_sha256.lower(): + raise ValueError("Upstream SHA-256 differs from the requested pin") + url = asset["browser_download_url"] + if not url.startswith("https://github.com/BtbN/FFmpeg-Builds/releases/download/"): + raise ValueError("Unexpected upstream download URL") + folder = cache / target / digest + binary = folder / ("ffprobe.exe" if "windows" in target else "ffprobe") + notices = folder / "licenses" + if folder.exists(): + if folder.is_symlink(): + raise ValueError("Refusing a symlinked FFprobe cache") + receipt = json.loads((notices / "provenance.json").read_text()) + for name, checksum in receipt["files"].items(): + path = PurePosixPath(name) + if path.is_absolute() or ".." in path.parts or "\\" in name: + raise ValueError("Invalid cache receipt") + cached = folder.joinpath(*path.parts) + if cached.is_symlink() or sha256(cached) != checksum: + raise ValueError(f"Corrupt FFprobe cache: {folder}; remove it and retry") + if binary.name not in receipt["files"] or receipt["archive_sha256"] != digest: + raise ValueError("Invalid FFprobe cache receipt") + validate_binary(binary, target) + print(f"Using verified FFprobe cache: {binary}", flush=True) + return binary, notices + archive = temporary / asset["name"] + print(f"Downloading {asset['name']} ({asset['size'] / 1048576:.1f} MiB)", flush=True) + download(url, archive) + if sha256(archive) != digest: + raise ValueError("FFprobe archive SHA-256 mismatch; nothing installed") + stage = temporary / "bundle" + stage.mkdir() + staged_binary, staged_notices = unpack(archive, stage, target) + native = platform.machine().lower() in ("amd64", "x86_64") and ( + (os.name == "nt" and "windows" in target) + or (platform.system() == "Linux" and "linux" in target)) + version = None + if native: + result = subprocess.run([str(staged_binary.resolve()), "-version"], check=True, + capture_output=True, text=True, timeout=30) + lines = result.stdout.splitlines() + if not lines or not lines[0].startswith("ffprobe version "): + raise ValueError("Downloaded executable did not identify itself as FFprobe") + version = lines[0] + receipt = { + "provider": "https://github.com/BtbN/FFmpeg-Builds", + "release": release, "release_id": release_data["id"], "asset_id": asset["id"], + "url": url, "archive_sha256": digest, "target": target, + "native_version_check": version, + "files": {p.relative_to(stage).as_posix(): sha256(p) + for p in stage.rglob("*") if p.is_file()}, + } + (staged_notices / "provenance.json").write_text(json.dumps(receipt, indent=2) + "\n") + folder.parent.mkdir(parents=True, exist_ok=True) + stage.rename(folder) + print(f"Verified FFprobe: {binary}", flush=True) + return binary, notices + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--target", required=True, choices=ASSETS) + parser.add_argument("--cache", type=Path, default=ROOT / "downloads/ffprobe") + parser.add_argument("--release", default="latest", help="BtbN release tag; default latest") + parser.add_argument("--sha256", help="Optional expected archive digest for reproducibility") + args = parser.parse_args() + try: + fetch(args.target, args.cache, args.release, args.sha256) + except (OSError, ValueError, KeyError, subprocess.SubprocessError, zipfile.BadZipFile, tarfile.TarError) as error: + parser.exit(1, f"FFprobe fetch failed: {error}\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/package-portable.py b/scripts/package-portable.py index 21781f8..dafa555 100644 --- a/scripts/package-portable.py +++ b/scripts/package-portable.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Package already-built binaries plus an explicitly supplied target FFprobe.""" +"""Package already-built binaries with supplied or release-time downloaded FFprobe.""" import argparse import hashlib from pathlib import Path @@ -8,22 +8,46 @@ import tempfile import shutil import tomllib import zipfile +import sys +import subprocess ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +from fetch_ffprobe import fetch def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--target", required=True, choices=("x86_64-pc-windows-gnu", "x86_64-unknown-linux-gnu")) - parser.add_argument("--ffprobe", type=Path, required=True, help="FFprobe built for the target platform") - parser.add_argument("--ffprobe-licenses", type=Path, required=True, help="Directory of notices and source/build provenance for that FFmpeg distribution") + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--ffprobe", type=Path, help="FFprobe built for the target platform") + source.add_argument("--fetch-ffprobe", action="store_true", help="Download and verify the target FFprobe at packaging time") + parser.add_argument("--ffprobe-release", default="latest", help="BtbN release tag for --fetch-ffprobe") + parser.add_argument("--ffprobe-sha256", help="Optional archive digest pin for --fetch-ffprobe") + parser.add_argument("--ffprobe-licenses", type=Path, help="Directory of notices and source/build provenance for a supplied FFmpeg distribution") parser.add_argument("--profile", default="release", choices=("debug", "release")) parser.add_argument("--output", type=Path, default=ROOT / "dist") args = parser.parse_args() + if args.fetch_ffprobe and args.ffprobe_licenses: + parser.error("--ffprobe-licenses is only used with --ffprobe") + if args.ffprobe and not args.ffprobe_licenses: + parser.error("--ffprobe requires --ffprobe-licenses") + if not args.fetch_ffprobe and (args.ffprobe_release != "latest" or args.ffprobe_sha256): + parser.error("FFprobe release/digest options require --fetch-ffprobe") windows = "windows" in args.target extension = ".exe" if windows else "" binaries = [ROOT / "target" / args.target / args.profile / (name + extension) for name in ("pptx-reporter", "pptx-reporter-gui")] + for path in binaries: + if not path.is_file(): + parser.error(f"Missing reporter binary: {path}; build the target first") + if args.fetch_ffprobe: + try: + args.ffprobe, args.ffprobe_licenses = fetch( + args.target, ROOT / "downloads/ffprobe", args.ffprobe_release, args.ffprobe_sha256) + except (OSError, ValueError, KeyError, subprocess.SubprocessError, + zipfile.BadZipFile, tarfile.TarError) as error: + parser.exit(1, f"FFprobe fetch failed: {error}\n") for path in [*binaries, args.ffprobe]: if not path.is_file() or not path.stat().st_size or path.is_symlink(): parser.error(f"Missing or unsafe binary: {path}") diff --git a/tests/test_fetch_ffprobe.py b/tests/test_fetch_ffprobe.py new file mode 100644 index 0000000..d3a3d5b --- /dev/null +++ b/tests/test_fetch_ffprobe.py @@ -0,0 +1,151 @@ +import hashlib +import io +import json +from pathlib import Path +import struct +import subprocess +import tarfile +import tempfile +import unittest +from unittest.mock import patch +import zipfile + +from scripts import fetch_ffprobe as fetcher + +WINDOWS = "x86_64-pc-windows-gnu" +LINUX = "x86_64-unknown-linux-gnu" + + +def pe(): + header = bytearray(90) + header[:2] = b"MZ" + struct.pack_into("