pptx-reporter/scripts/fetch_ffprobe.py

225 lines
10 KiB
Python

#!/usr/bin/env python3
"""Fetch verified FFprobe binaries for release packaging, never into source control."""
import argparse
import hashlib
import json
import os
from pathlib import Path, PurePosixPath
import platform
import re
import shutil
import struct
import subprocess
import tarfile
import tempfile
import urllib.parse
import urllib.request
import zipfile
ROOT = Path(__file__).resolve().parents[1]
API = "https://api.github.com/repos/BtbN/FFmpeg-Builds/releases/tags/"
ASSETS = {
"x86_64-pc-windows-gnu": "ffmpeg-n9.0-latest-win64-lgpl-9.0.zip",
"x86_64-unknown-linux-gnu": "ffmpeg-n9.0-latest-linux64-lgpl-9.0.tar.xz",
}
def sha256(path):
with path.open("rb") as stream:
return hashlib.file_digest(stream, "sha256").hexdigest()
def download(url, destination, limit=512 * 1024 * 1024):
if urllib.parse.urlsplit(url).scheme != "https":
raise ValueError("Downloads must use HTTPS")
request = urllib.request.Request(url, headers={"User-Agent": "pptx-reporter-release"})
with urllib.request.urlopen(request, timeout=60) as response, destination.open("wb") as out:
if urllib.parse.urlsplit(response.url).scheme != "https":
raise ValueError("Refusing a non-HTTPS redirect")
size = 0
while chunk := response.read(1024 * 1024):
size += len(chunk)
if size > limit:
raise ValueError("Download exceeds size limit")
out.write(chunk)
def validate_binary(path, target):
"""Check the executable format AND x86-64 machine type without executing it."""
with path.open("rb") as stream:
header = stream.read(64)
if target == "x86_64-pc-windows-gnu":
if len(header) != 64 or header[:2] != b"MZ":
raise ValueError("Expected a Windows PE executable")
stream.seek(struct.unpack_from("<I", header, 60)[0])
pe = stream.read(26)
if len(pe) != 26 or pe[:6] != b"PE\0\0\x64\x86" or pe[24:] != b"\x0b\x02":
raise ValueError("Expected an x86-64 PE32+ executable")
elif target == "x86_64-unknown-linux-gnu":
if len(header) != 64 or header[:6] != b"\x7fELF\x02\x01" or header[18:20] != b"\x3e\0":
raise ValueError("Expected an x86-64 little-endian ELF executable")
else:
raise ValueError(f"Unsupported target: {target}")
def unpack(archive, destination, target):
"""Copy only FFprobe and upstream notices; never extract archive paths directly."""
executable = "ffprobe.exe" if "windows" in target else "ffprobe"
notices = destination / "licenses"
notices.mkdir()
found = False
licensed = False
def member(name, size, regular, opener):
nonlocal found, licensed
path = PurePosixPath(name)
if path.name != executable and not path.name.lower().startswith(("license", "copying", "readme")):
return
if not regular or path.is_absolute() or ".." in path.parts or "\\" in name:
raise ValueError(f"Unsafe archive member: {name}")
is_binary = path.name == executable
if size > (256 * 1024 * 1024 if is_binary else 8 * 1024 * 1024):
raise ValueError(f"Archive member too large: {name}")
if is_binary:
if found or path.parent.name != "bin":
raise ValueError("Expected exactly one bin/ffprobe executable")
found = True
output = destination / executable
else:
output = notices.joinpath(*path.parts)
licensed |= path.name.lower().startswith(("license", "copying"))
output.parent.mkdir(parents=True, exist_ok=True)
with opener() as source, output.open("xb") as out:
shutil.copyfileobj(source, out)
if archive.suffix == ".zip":
with zipfile.ZipFile(archive) as bundle:
for info in bundle.infolist():
if info.is_dir():
continue
mode = info.external_attr >> 16
regular = not info.is_dir() and (mode & 0o170000) in (0, 0o100000)
member(info.filename, info.file_size, regular, lambda i=info: bundle.open(i))
else:
with tarfile.open(archive, "r:xz") as bundle:
for info in bundle:
if info.isdir():
continue
member(info.name, info.size, info.isfile(), lambda i=info: bundle.extractfile(i))
if not found or not licensed:
raise ValueError("Archive must include FFprobe and its upstream license")
binary = destination / executable
validate_binary(binary, target)
binary.chmod(0o755)
return binary, notices
def verified_cache(folder, target, digest):
binary = folder / ("ffprobe.exe" if "windows" in target else "ffprobe")
notices = folder / "licenses"
if folder.is_symlink():
raise ValueError("Refusing a symlinked FFprobe cache")
receipt = json.loads((notices / "provenance.json").read_text())
for name, checksum in receipt["files"].items():
path = PurePosixPath(name)
if path.is_absolute() or ".." in path.parts or "\\" in name:
raise ValueError("Invalid cache receipt")
cached = folder.joinpath(*path.parts)
if cached.is_symlink() or sha256(cached) != checksum:
raise ValueError(f"Corrupt FFprobe cache: {folder}; remove it and retry")
if binary.name not in receipt["files"] or receipt["archive_sha256"] != digest:
raise ValueError("Invalid FFprobe cache receipt")
validate_binary(binary, target)
print(f"Using verified FFprobe cache: {binary}", flush=True)
return binary, notices
def fetch(target, cache=None, release="latest", expected_sha256=None, *, asset_name=None):
if target not in ASSETS:
raise ValueError(f"Unsupported target: {target}")
asset_name = asset_name or ASSETS[target]
platform_suffix = "win64-lgpl-9.0.zip" if "windows" in target else "linux64-lgpl-9.0.tar.xz"
if not re.fullmatch(r"ffmpeg-n9\.0[-.A-Za-z0-9]*-" + re.escape(platform_suffix), asset_name):
raise ValueError("Expected a target-specific non-shared LGPL FFmpeg 9.0 asset")
cache = Path(cache) if cache is not None else ROOT / "downloads" / "ffprobe"
cache.mkdir(parents=True, exist_ok=True)
if expected_sha256 is not None:
digest = expected_sha256.lower()
if not re.fullmatch(r"[0-9a-f]{64}", digest):
raise ValueError("Invalid requested pin")
folder = cache / target / digest
if folder.exists():
return verified_cache(folder, target, digest)
with tempfile.TemporaryDirectory(prefix=".fetch-", dir=cache) as temporary:
temporary = Path(temporary)
metadata = temporary / "release.json"
download(API + urllib.parse.quote(release, safe=""), metadata, 8 * 1024 * 1024)
release_data = json.loads(metadata.read_text())
assets = [a for a in release_data["assets"] if a["name"] == asset_name]
if len(assets) != 1:
raise ValueError(f"Release {release} does not contain {asset_name}")
asset = assets[0]
digest = asset.get("digest", "")
if not re.fullmatch(r"sha256:[0-9a-f]{64}", digest):
raise ValueError("Upstream asset has no usable SHA-256 digest")
digest = digest.removeprefix("sha256:")
if expected_sha256 is not None and digest != expected_sha256.lower():
raise ValueError("Upstream SHA-256 differs from the requested pin")
url = asset["browser_download_url"]
if not url.startswith("https://github.com/BtbN/FFmpeg-Builds/releases/download/"):
raise ValueError("Unexpected upstream download URL")
folder = cache / target / digest
binary = folder / ("ffprobe.exe" if "windows" in target else "ffprobe")
notices = folder / "licenses"
if folder.exists():
return verified_cache(folder, target, digest)
archive = temporary / asset["name"]
print(f"Downloading {asset['name']} ({asset['size'] / 1048576:.1f} MiB)", flush=True)
download(url, archive)
if sha256(archive) != digest:
raise ValueError("FFprobe archive SHA-256 mismatch; nothing installed")
stage = temporary / "bundle"
stage.mkdir()
staged_binary, staged_notices = unpack(archive, stage, target)
native = platform.machine().lower() in ("amd64", "x86_64") and (
(os.name == "nt" and "windows" in target)
or (platform.system() == "Linux" and "linux" in target))
version = None
if native:
result = subprocess.run([str(staged_binary.resolve()), "-version"], check=True,
capture_output=True, text=True, timeout=30)
lines = result.stdout.splitlines()
if not lines or not lines[0].startswith("ffprobe version "):
raise ValueError("Downloaded executable did not identify itself as FFprobe")
version = lines[0]
receipt = {
"provider": "https://github.com/BtbN/FFmpeg-Builds",
"release": release, "release_id": release_data["id"], "asset_id": asset["id"],
"url": url, "archive_sha256": digest, "target": target,
"native_version_check": version,
"files": {p.relative_to(stage).as_posix(): sha256(p)
for p in stage.rglob("*") if p.is_file()},
}
(staged_notices / "provenance.json").write_text(json.dumps(receipt, indent=2) + "\n")
folder.parent.mkdir(parents=True, exist_ok=True)
stage.rename(folder)
print(f"Verified FFprobe: {binary}", flush=True)
return binary, notices
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--target", required=True, choices=ASSETS)
parser.add_argument("--cache", type=Path, default=ROOT / "downloads/ffprobe")
parser.add_argument("--release", default="latest", help="BtbN release tag; default latest")
parser.add_argument("--sha256", help="Optional expected archive digest for reproducibility")
args = parser.parse_args()
try:
fetch(args.target, args.cache, args.release, args.sha256)
except (OSError, ValueError, KeyError, subprocess.SubprocessError, zipfile.BadZipFile, tarfile.TarError) as error:
parser.exit(1, f"FFprobe fetch failed: {error}\n")
if __name__ == "__main__":
main()