Add named portable release bundles with pinned FFprobe
This commit is contained in:
parent
a22de66e47
commit
64bdfed396
|
|
@ -32,8 +32,9 @@ with quick-start examples and links to detailed documentation in `docs/`.
|
|||
releases or uploads release binaries. Cross and `rust_realease` are separate,
|
||||
manually invoked build/release preparation tools.
|
||||
- `scripts/fetch_ffprobe.py` fetches checksum-verified target FFprobe builds into
|
||||
ignored `downloads/`. Portable release packaging calls it with
|
||||
`--fetch-ffprobe`; include upstream notices/provenance, never commit binaries.
|
||||
ignored `downloads/`. Named release bundles use `scripts/prepare-release.py` with per-target
|
||||
pins in `crates/pptx-reporter/ffprobe-pins.json`; the standalone packager retains
|
||||
`--fetch-ffprobe`. Include upstream notices/provenance, never commit binaries.
|
||||
|
||||
## Repository layout
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,12 @@
|
|||
{
|
||||
"x86_64-pc-windows-gnu": {
|
||||
"release": "autobuild-2026-10-20-13-08",
|
||||
"asset": "ffmpeg-n9.0.2-29-g6dfaba896f-win64-lgpl-9.0.zip",
|
||||
"sha256": "e7bca5e004c2d13bcb374c3a4bd30c4588b08607a4b1a01369341a4f9678dff3"
|
||||
},
|
||||
"x86_64-unknown-linux-gnu": {
|
||||
"release": "autobuild-2026-10-20-13-08",
|
||||
"asset": "ffmpeg-n9.0.2-29-g6dfaba896f-linux64-lgpl-9.0.tar.xz",
|
||||
"sha256": "5ae5939260c72671383f0a0d8785695bfe06c94c27d4daac2e63eb8576417927"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,8 @@
|
|||
# Cross builds and portable releases
|
||||
|
||||
Run these commands from the repository root. Cross needs Docker or Podman;
|
||||
Run commands from the repository root. The primary release workflow is the
|
||||
`rust_realease` named bundle described below. The standalone packager remains
|
||||
available for manual/offline packaging until native acceptance tests are complete. Cross needs Docker or Podman;
|
||||
`Cross.toml` selects Windows GNU x64 and Linux GNU x64 images. Jenkins does not
|
||||
run release builds or publish releases.
|
||||
|
||||
|
|
@ -54,9 +56,41 @@ rust_realease . --package pptx-reporter --compile-only
|
|||
|
||||
Publication needs a clean checkout and a matching explicitly created/pushed
|
||||
version tag. The existing local `v0.1.0` marks the original trial, not subsequent
|
||||
changes; do not move it implicitly. `rust_realease` currently packages Cargo
|
||||
binaries only: use the portable packager for bundles containing FFprobe.
|
||||
Upstream hook support is tracked separately.
|
||||
changes; do not move it implicitly.
|
||||
|
||||
The `portable` named bundle adds a Windows ZIP and Linux `.tgz`, containing both
|
||||
Cargo binaries, adjacent FFprobe, quick-start instructions and licenses. Canonical
|
||||
cargo-binstall `.tgz` archives remain unchanged and require external FFprobe.
|
||||
Python 3.11+ is required on the build host only.
|
||||
|
||||
The protocol-1 hook `scripts/prepare-release.py` uses the publisher's output and
|
||||
cache directories, copies FFprobe/notices and emits a hashed file manifest. It
|
||||
never guesses Cargo output paths or creates archives. The publisher handles
|
||||
archive validation, checksums and upload. See the upstream
|
||||
[named-bundle contract](https://proxy.dandaedre.com/gitea/idle-org/rust_realease/raw/branch/main/docs/portable-bundles.md).
|
||||
|
||||
`crates/pptx-reporter/ffprobe-pins.json` pins a dated BtbN release, exact non-shared
|
||||
LGPL 9.0 asset name and published SHA-256 digest per target. Dated assets have
|
||||
versioned names, unlike the downloader's default `latest` assets. To update,
|
||||
review the provider release and its notices, then update all three fields together.
|
||||
These dated releases are not marked immutable by GitHub; digest verification
|
||||
fails closed if an asset changes. Metadata access is still needed with a warm cache.
|
||||
|
||||
```sh
|
||||
rust_realease . --package pptx-reporter --target x86_64-unknown-linux-gnu --compile-only
|
||||
```
|
||||
|
||||
Dry-run validates static configuration without downloads. Compile-only prepares
|
||||
and packages bundles without a token, tag creation, push or publication. After a
|
||||
successful invocation, read `dist/latest-artifacts.json`; resolve its `manifest`
|
||||
and `checksums` paths relative to `dist/`. Resolve assets relative to the resulting
|
||||
`artifact-manifest.json` directory and verify `SHA256SUMS`. Do not guess build paths.
|
||||
|
||||
Before publication, extract both portable trees. On Linux, generate a report using
|
||||
adjacent FFprobe without relying on PATH. On Windows, launch the GUI and generate
|
||||
a report without separately installed FFmpeg or Python. Cross compilation and
|
||||
fixture tests do not replace native acceptance tests. Keep the standalone packager
|
||||
until those checks are complete. Jenkins remains checks/coverage/badges only.
|
||||
|
||||
## Application artwork
|
||||
|
||||
|
|
|
|||
|
|
@ -8,9 +8,25 @@ locked = true
|
|||
|
||||
[package]
|
||||
format = "tgz"
|
||||
# rust_realease packages Cargo binaries only. Release automation can additionally
|
||||
# call: python3 scripts/package-portable.py --target TARGET --fetch-ffprobe
|
||||
# after Cross builds populate target/TARGET/release (see docs/releases.md).
|
||||
|
||||
[[package.bundles]]
|
||||
name = "portable"
|
||||
targets = ["x86_64-pc-windows-gnu", "x86_64-unknown-linux-gnu"]
|
||||
formats = { x86_64-pc-windows-gnu = "zip", x86_64-unknown-linux-gnu = "tgz" }
|
||||
archive-name = "{name}-{bundle}-{target}-v{version}"
|
||||
root = "{name}-{bundle}-{target}-v{version}"
|
||||
cargo-binaries = "all"
|
||||
files = [
|
||||
{ source = "LICENSE", destination = "LICENSES/reporter-MIT.txt", workspace-relative = true },
|
||||
{ source = "assets/fonts/LICENSE", destination = "LICENSES/DejaVu.txt" },
|
||||
{ source = "docs/windows-quickstart.txt", destination = "README.txt", workspace-relative = true },
|
||||
]
|
||||
|
||||
[package.bundles.prepare]
|
||||
command = ["python3", "../../scripts/prepare-release.py", "--context", "{context-file}", "--manifest", "{manifest-file}"]
|
||||
inputs = ["ffprobe-pins.json", "workspace:scripts/prepare-release.py", "workspace:scripts/fetch_ffprobe.py"]
|
||||
timeout-seconds = 600
|
||||
required-destinations = ["ffprobe{binary-ext}", "LICENSES/ffmpeg/LICENSE.txt", "LICENSES/ffmpeg/provenance.json"]
|
||||
|
||||
[[repository.fallbacks]]
|
||||
origin = "https://git.dandaedre.com/Beffroi/beffroi-tools"
|
||||
|
|
|
|||
|
|
@ -114,9 +114,13 @@ def unpack(archive, destination, target):
|
|||
return binary, notices
|
||||
|
||||
|
||||
def fetch(target, cache=None, release="latest", expected_sha256=None):
|
||||
def fetch(target, cache=None, release="latest", expected_sha256=None, *, asset_name=None):
|
||||
if target not in ASSETS:
|
||||
raise ValueError(f"Unsupported target: {target}")
|
||||
asset_name = asset_name or ASSETS[target]
|
||||
platform_suffix = "win64-lgpl-9.0.zip" if "windows" in target else "linux64-lgpl-9.0.tar.xz"
|
||||
if not re.fullmatch(r"ffmpeg-n9\.0[-.A-Za-z0-9]*-" + re.escape(platform_suffix), asset_name):
|
||||
raise ValueError("Expected a target-specific non-shared LGPL FFmpeg 9.0 asset")
|
||||
cache = Path(cache) if cache is not None else ROOT / "downloads" / "ffprobe"
|
||||
cache.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(prefix=".fetch-", dir=cache) as temporary:
|
||||
|
|
@ -124,9 +128,9 @@ def fetch(target, cache=None, release="latest", expected_sha256=None):
|
|||
metadata = temporary / "release.json"
|
||||
download(API + urllib.parse.quote(release, safe=""), metadata, 8 * 1024 * 1024)
|
||||
release_data = json.loads(metadata.read_text())
|
||||
assets = [a for a in release_data["assets"] if a["name"] == ASSETS[target]]
|
||||
assets = [a for a in release_data["assets"] if a["name"] == asset_name]
|
||||
if len(assets) != 1:
|
||||
raise ValueError(f"Release {release} does not contain {ASSETS[target]}")
|
||||
raise ValueError(f"Release {release} does not contain {asset_name}")
|
||||
asset = assets[0]
|
||||
digest = asset.get("digest", "")
|
||||
if not re.fullmatch(r"sha256:[0-9a-f]{64}", digest):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,74 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Prepare FFprobe payload files for rust_realease bundle protocol 1."""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from fetch_ffprobe import fetch, sha256
|
||||
|
||||
|
||||
def prepare(context, manifest):
|
||||
if context["schema_version"] != 1:
|
||||
raise ValueError("Unsupported bundle context schema")
|
||||
if Path(manifest).resolve() != Path(context["manifest_path"]).resolve():
|
||||
raise ValueError("Manifest path differs from the bundle context")
|
||||
pins = json.loads((Path(context["package_root"]) / "ffprobe-pins.json").read_text())
|
||||
pin = pins[context["target"]]
|
||||
if pin["release"] == "latest" or not re.fullmatch(r"[0-9a-f]{64}", pin["sha256"]):
|
||||
raise ValueError("FFprobe requires a dated release and SHA-256 pin")
|
||||
binary, notices = fetch(
|
||||
context["target"], Path(context["cache_directory"]),
|
||||
pin["release"], pin["sha256"], asset_name=pin["asset"],
|
||||
)
|
||||
out = Path(context["output_directory"])
|
||||
files = []
|
||||
|
||||
def include(source, relative, destination, kind="file"):
|
||||
if source.is_symlink() or not source.is_file():
|
||||
raise ValueError(f"Not a regular prepared input: {source}")
|
||||
copied = out / relative
|
||||
copied.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(source, copied)
|
||||
files.append({
|
||||
"source": relative, "destination": destination, "kind": kind,
|
||||
"sha256": sha256(copied),
|
||||
})
|
||||
|
||||
name = "ffprobe" + context["binary_ext"]
|
||||
include(binary, name, name, "executable")
|
||||
if notices.is_symlink() or not notices.is_dir():
|
||||
raise ValueError("Missing or symlinked FFprobe notices")
|
||||
licenses = []
|
||||
for path in sorted(notices.rglob("*")):
|
||||
if path.is_symlink() or not (path.is_file() or path.is_dir()):
|
||||
raise ValueError(f"Unsafe notice: {path}")
|
||||
if path.is_file():
|
||||
relative = path.relative_to(notices).as_posix()
|
||||
include(path, "notices/" + relative, "LICENSES/ffmpeg/" + relative)
|
||||
if path.name.lower().startswith(("license", "copying")):
|
||||
licenses.append(path)
|
||||
if not licenses:
|
||||
raise ValueError("Missing upstream license")
|
||||
if not any(f["destination"] == "LICENSES/ffmpeg/LICENSE.txt" for f in files):
|
||||
include(licenses[0], "canonical-LICENSE.txt", "LICENSES/ffmpeg/LICENSE.txt")
|
||||
if not any(f["destination"] == "LICENSES/ffmpeg/provenance.json" for f in files):
|
||||
raise ValueError("Missing FFprobe provenance")
|
||||
Path(manifest).write_text(json.dumps({
|
||||
"schema_version": 1, "target": context["target"], "files": files,
|
||||
}, indent=2) + "\n")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--context", type=Path, required=True)
|
||||
parser.add_argument("--manifest", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
prepare(json.loads(args.context.read_text()), args.manifest)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -56,6 +56,11 @@ class FetchTests(unittest.TestCase):
|
|||
self.assertEqual(receipt["archive_sha256"], digest)
|
||||
self.assertIsNone(receipt["native_version_check"])
|
||||
self.assertFalse((binary.parent / "ffmpeg.exe").exists())
|
||||
dated_name = "ffmpeg-n9.0.2-29-g6dfaba896f-win64-lgpl-9.0.zip"
|
||||
release["assets"][0]["name"] = dated_name
|
||||
self.assertEqual(fetcher.fetch(WINDOWS, cache, "autobuild-fixture", digest,
|
||||
asset_name=dated_name), (binary, notices))
|
||||
release["assets"][0]["name"] = name
|
||||
get.reset_mock()
|
||||
self.assertEqual(fetcher.fetch(WINDOWS, cache), (binary, notices))
|
||||
self.assertEqual(get.call_count, 1) # metadata refreshed, archive reused
|
||||
|
|
@ -120,6 +125,10 @@ class FetchTests(unittest.TestCase):
|
|||
fetcher.download("http://example.invalid/file", Path("/unused"))
|
||||
with self.assertRaises(ValueError):
|
||||
fetcher.fetch("unknown-target")
|
||||
for asset in ("../probe.zip", "ffmpeg-n9.0-latest-win64-gpl-9.0.zip",
|
||||
fetcher.ASSETS[LINUX]):
|
||||
with self.subTest(asset=asset), self.assertRaisesRegex(ValueError, "LGPL"):
|
||||
fetcher.fetch(WINDOWS, asset_name=asset)
|
||||
|
||||
def test_native_smoke_failure_does_not_install_cache(self):
|
||||
header = bytearray(64)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,75 @@
|
|||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
spec = importlib.util.spec_from_file_location("prepare_release", ROOT / "scripts/prepare-release.py")
|
||||
adapter = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(adapter)
|
||||
|
||||
|
||||
class PrepareReleaseTests(unittest.TestCase):
|
||||
def test_payload_for_each_target_and_missing_or_unsafe_notices(self):
|
||||
pins = json.loads((ROOT / "crates/pptx-reporter/ffprobe-pins.json").read_text())
|
||||
for target, pin in pins.items():
|
||||
with self.subTest(target=target), tempfile.TemporaryDirectory(prefix="bundle é ") as temp:
|
||||
root = Path(temp)
|
||||
out = root / "output"
|
||||
out.mkdir()
|
||||
binary = root / "downloaded-probe"
|
||||
binary.write_bytes(b"fixture executable")
|
||||
notices = root / "notices"
|
||||
(notices / "upstream").mkdir(parents=True)
|
||||
license_file = notices / "upstream/COPYING"
|
||||
license_file.write_text("upstream license")
|
||||
provenance = notices / "provenance.json"
|
||||
provenance.write_text('{"provider":"fixture"}')
|
||||
manifest = root / "manifest.json"
|
||||
ext = ".exe" if "windows" in target else ""
|
||||
context = {
|
||||
"schema_version": 1, "target": target, "binary_ext": ext,
|
||||
"package_root": str(ROOT / "crates/pptx-reporter"),
|
||||
"output_directory": str(out), "cache_directory": str(root / "cache"),
|
||||
"manifest_path": str(manifest),
|
||||
}
|
||||
with patch.object(adapter, "fetch", return_value=(binary, notices)) as fetch:
|
||||
adapter.prepare(context, manifest)
|
||||
fetch.assert_called_once_with(target, root / "cache", pin["release"],
|
||||
pin["sha256"], asset_name=pin["asset"])
|
||||
result = json.loads(manifest.read_text())
|
||||
self.assertEqual(result["target"], target)
|
||||
files = {f["destination"]: f for f in result["files"]}
|
||||
self.assertEqual(set(files), {
|
||||
"ffprobe" + ext, "LICENSES/ffmpeg/upstream/COPYING",
|
||||
"LICENSES/ffmpeg/LICENSE.txt", "LICENSES/ffmpeg/provenance.json",
|
||||
})
|
||||
self.assertEqual(files["ffprobe" + ext]["kind"], "executable")
|
||||
for item in files.values():
|
||||
payload = out / item["source"]
|
||||
self.assertEqual(hashlib.sha256(payload.read_bytes()).hexdigest(), item["sha256"])
|
||||
self.assertFalse(payload.samefile(binary))
|
||||
manifest.unlink()
|
||||
provenance.unlink()
|
||||
with self.assertRaisesRegex(ValueError, "provenance"):
|
||||
adapter.prepare(context, manifest)
|
||||
self.assertFalse(manifest.exists())
|
||||
provenance.write_text("{}")
|
||||
license_file.unlink()
|
||||
with self.assertRaisesRegex(ValueError, "license"):
|
||||
adapter.prepare(context, manifest)
|
||||
license_file.symlink_to(binary)
|
||||
with self.assertRaisesRegex(ValueError, "Unsafe notice"):
|
||||
adapter.prepare(context, manifest)
|
||||
self.assertFalse(manifest.exists())
|
||||
|
||||
def test_invalid_context_fails_before_download(self):
|
||||
with patch.object(adapter, "fetch") as fetch:
|
||||
with self.assertRaisesRegex(ValueError, "schema"):
|
||||
adapter.prepare({"schema_version": 2}, Path("/unused"))
|
||||
with self.assertRaisesRegex(ValueError, "Manifest path"):
|
||||
adapter.prepare({"schema_version": 1, "manifest_path": "/different"}, Path("/unused"))
|
||||
fetch.assert_not_called()
|
||||
Loading…
Reference in New Issue