214 lines
9.8 KiB
Python
214 lines
9.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Fetch verified FFprobe for releases; share validation with CI FFmpeg setup."""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path, PurePosixPath
|
|
import platform
|
|
import re
|
|
import shutil
|
|
import struct
|
|
import subprocess
|
|
import tarfile
|
|
import tempfile
|
|
import urllib.parse
|
|
import urllib.request
|
|
import zipfile
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
API = "https://api.github.com/repos/BtbN/FFmpeg-Builds/releases/tags/"
|
|
ASSETS = {
|
|
"x86_64-pc-windows-gnu": "ffmpeg-n9.0-latest-win64-lgpl-9.0.zip",
|
|
"x86_64-unknown-linux-gnu": "ffmpeg-n9.0-latest-linux64-lgpl-9.0.tar.xz",
|
|
}
|
|
|
|
|
|
def sha256(path):
|
|
with path.open("rb") as stream:
|
|
return hashlib.file_digest(stream, "sha256").hexdigest()
|
|
|
|
|
|
def download(url, destination, limit=512 * 1024 * 1024):
|
|
if urllib.parse.urlsplit(url).scheme != "https":
|
|
raise ValueError("Downloads must use HTTPS")
|
|
request = urllib.request.Request(url, headers={"User-Agent": "pptx-reporter-release"})
|
|
with urllib.request.urlopen(request, timeout=60) as response, destination.open("wb") as out:
|
|
if urllib.parse.urlsplit(response.url).scheme != "https":
|
|
raise ValueError("Refusing a non-HTTPS redirect")
|
|
size = 0
|
|
while chunk := response.read(1024 * 1024):
|
|
size += len(chunk)
|
|
if size > limit:
|
|
raise ValueError("Download exceeds size limit")
|
|
out.write(chunk)
|
|
|
|
|
|
def validate_binary(path, target):
|
|
"""Check the executable format AND x86-64 machine type without executing it."""
|
|
with path.open("rb") as stream:
|
|
header = stream.read(64)
|
|
if target == "x86_64-pc-windows-gnu":
|
|
if len(header) != 64 or header[:2] != b"MZ":
|
|
raise ValueError("Expected a Windows PE executable")
|
|
stream.seek(struct.unpack_from("<I", header, 60)[0])
|
|
pe = stream.read(26)
|
|
if len(pe) != 26 or pe[:6] != b"PE\0\0\x64\x86" or pe[24:] != b"\x0b\x02":
|
|
raise ValueError("Expected an x86-64 PE32+ executable")
|
|
elif target == "x86_64-unknown-linux-gnu":
|
|
if len(header) != 64 or header[:6] != b"\x7fELF\x02\x01" or header[18:20] != b"\x3e\0":
|
|
raise ValueError("Expected an x86-64 little-endian ELF executable")
|
|
else:
|
|
raise ValueError(f"Unsupported target: {target}")
|
|
|
|
|
|
def unpack(archive, destination, target, program="ffprobe"):
|
|
"""Copy the selected program and notices without extracting archive paths."""
|
|
if program not in ("ffprobe", "ffmpeg"):
|
|
raise ValueError("Unsupported FFmpeg program")
|
|
executable = program + (".exe" if "windows" in target else "")
|
|
notices = destination / "licenses"
|
|
notices.mkdir()
|
|
found = False
|
|
licensed = False
|
|
|
|
def member(name, size, regular, opener):
|
|
nonlocal found, licensed
|
|
path = PurePosixPath(name)
|
|
if path.name != executable and not path.name.lower().startswith(("license", "copying", "readme")):
|
|
return
|
|
if not regular or path.is_absolute() or ".." in path.parts or "\\" in name:
|
|
raise ValueError(f"Unsafe archive member: {name}")
|
|
is_binary = path.name == executable
|
|
if size > (256 * 1024 * 1024 if is_binary else 8 * 1024 * 1024):
|
|
raise ValueError(f"Archive member too large: {name}")
|
|
if is_binary:
|
|
if found or path.parent.name != "bin":
|
|
raise ValueError(f"Expected exactly one bin/{executable} executable")
|
|
found = True
|
|
output = destination / executable
|
|
else:
|
|
output = notices.joinpath(*path.parts)
|
|
licensed |= path.name.lower().startswith(("license", "copying"))
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
with opener() as source, output.open("xb") as out:
|
|
shutil.copyfileobj(source, out)
|
|
|
|
if archive.suffix == ".zip":
|
|
with zipfile.ZipFile(archive) as bundle:
|
|
for info in bundle.infolist():
|
|
if info.is_dir():
|
|
continue
|
|
mode = info.external_attr >> 16
|
|
regular = not info.is_dir() and (mode & 0o170000) in (0, 0o100000)
|
|
member(info.filename, info.file_size, regular, lambda i=info: bundle.open(i))
|
|
else:
|
|
with tarfile.open(archive, "r:xz") as bundle:
|
|
for info in bundle:
|
|
if info.isdir():
|
|
continue
|
|
member(info.name, info.size, info.isfile(), lambda i=info: bundle.extractfile(i))
|
|
if not found or not licensed:
|
|
raise ValueError(f"Archive must include {program} and its upstream license")
|
|
binary = destination / executable
|
|
validate_binary(binary, target)
|
|
binary.chmod(0o755)
|
|
return binary, notices
|
|
|
|
|
|
def fetch(target, cache=None, release="latest", expected_sha256=None, program="ffprobe"):
|
|
if program not in ("ffprobe", "ffmpeg"):
|
|
raise ValueError("Unsupported FFmpeg program")
|
|
if target not in ASSETS:
|
|
raise ValueError(f"Unsupported target: {target}")
|
|
cache = Path(cache) if cache is not None else ROOT / "downloads" / program
|
|
cache.mkdir(parents=True, exist_ok=True)
|
|
with tempfile.TemporaryDirectory(prefix=".fetch-", dir=cache) as temporary:
|
|
temporary = Path(temporary)
|
|
metadata = temporary / "release.json"
|
|
download(API + urllib.parse.quote(release, safe=""), metadata, 8 * 1024 * 1024)
|
|
release_data = json.loads(metadata.read_text())
|
|
assets = [a for a in release_data["assets"] if a["name"] == ASSETS[target]]
|
|
if len(assets) != 1:
|
|
raise ValueError(f"Release {release} does not contain {ASSETS[target]}")
|
|
asset = assets[0]
|
|
digest = asset.get("digest", "")
|
|
if not re.fullmatch(r"sha256:[0-9a-f]{64}", digest):
|
|
raise ValueError("Upstream asset has no usable SHA-256 digest")
|
|
digest = digest.removeprefix("sha256:")
|
|
if expected_sha256 is not None and digest != expected_sha256.lower():
|
|
raise ValueError("Upstream SHA-256 differs from the requested pin")
|
|
url = asset["browser_download_url"]
|
|
if not url.startswith("https://github.com/BtbN/FFmpeg-Builds/releases/download/"):
|
|
raise ValueError("Unexpected upstream download URL")
|
|
folder = cache / target / digest
|
|
if program != "ffprobe":
|
|
folder = cache / program / target / digest
|
|
binary = folder / (program + (".exe" if "windows" in target else ""))
|
|
notices = folder / "licenses"
|
|
if folder.exists():
|
|
if folder.is_symlink():
|
|
raise ValueError("Refusing a symlinked FFprobe cache")
|
|
receipt = json.loads((notices / "provenance.json").read_text())
|
|
for name, checksum in receipt["files"].items():
|
|
path = PurePosixPath(name)
|
|
if path.is_absolute() or ".." in path.parts or "\\" in name:
|
|
raise ValueError("Invalid cache receipt")
|
|
cached = folder.joinpath(*path.parts)
|
|
if cached.is_symlink() or sha256(cached) != checksum:
|
|
raise ValueError(f"Corrupt FFprobe cache: {folder}; remove it and retry")
|
|
if binary.name not in receipt["files"] or receipt["archive_sha256"] != digest:
|
|
raise ValueError("Invalid FFprobe cache receipt")
|
|
validate_binary(binary, target)
|
|
print(f"Using verified {program} cache: {binary}", flush=True)
|
|
return binary, notices
|
|
archive = temporary / asset["name"]
|
|
print(f"Downloading {asset['name']} ({asset['size'] / 1048576:.1f} MiB)", flush=True)
|
|
download(url, archive)
|
|
if sha256(archive) != digest:
|
|
raise ValueError("FFprobe archive SHA-256 mismatch; nothing installed")
|
|
stage = temporary / "bundle"
|
|
stage.mkdir()
|
|
staged_binary, staged_notices = unpack(archive, stage, target, program)
|
|
native = platform.machine().lower() in ("amd64", "x86_64") and (
|
|
(os.name == "nt" and "windows" in target)
|
|
or (platform.system() == "Linux" and "linux" in target))
|
|
version = None
|
|
if native:
|
|
result = subprocess.run([str(staged_binary.resolve()), "-version"], check=True,
|
|
capture_output=True, text=True, timeout=30)
|
|
lines = result.stdout.splitlines()
|
|
if not lines or not lines[0].startswith(f"{program} version "):
|
|
raise ValueError(f"Downloaded executable did not identify itself as {program}")
|
|
version = lines[0]
|
|
receipt = {
|
|
"provider": "https://github.com/BtbN/FFmpeg-Builds",
|
|
"release": release, "release_id": release_data["id"], "asset_id": asset["id"],
|
|
"url": url, "archive_sha256": digest, "target": target,
|
|
"native_version_check": version,
|
|
"files": {p.relative_to(stage).as_posix(): sha256(p)
|
|
for p in stage.rglob("*") if p.is_file()},
|
|
}
|
|
(staged_notices / "provenance.json").write_text(json.dumps(receipt, indent=2) + "\n")
|
|
folder.parent.mkdir(parents=True, exist_ok=True)
|
|
stage.rename(folder)
|
|
print(f"Verified {program}: {binary}", flush=True)
|
|
return binary, notices
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--target", required=True, choices=ASSETS)
|
|
parser.add_argument("--cache", type=Path, default=ROOT / "downloads/ffprobe")
|
|
parser.add_argument("--release", default="latest", help="BtbN release tag; default latest")
|
|
parser.add_argument("--sha256", help="Optional expected archive digest for reproducibility")
|
|
args = parser.parse_args()
|
|
try:
|
|
fetch(args.target, args.cache, args.release, args.sha256)
|
|
except (OSError, ValueError, KeyError, subprocess.SubprocessError, zipfile.BadZipFile, tarfile.TarError) as error:
|
|
parser.exit(1, f"FFprobe fetch failed: {error}\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|