diff --git a/Cargo.lock b/Cargo.lock index 93306f0..17a37a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -67,6 +67,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "arbitrary" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bc62ac97cc33321f50863d514c3bc38a453947a8f9e781137e47c7401020aed" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -224,6 +233,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + [[package]] name = "crypto-common" version = "0.1.7" @@ -234,6 +249,17 @@ dependencies = [ "typenum", ] +[[package]] +name = "derive_arbitrary" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b034bd7d5f032402a2479444dcc6f74e36a03f31854d41680fb240ef682a1ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "digest" version = "0.10.7" @@ -304,6 +330,16 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.12" @@ -659,9 +695,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "js-sys" -version = "0.3.103" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" dependencies = [ "cfg-if", "futures-util", @@ -997,16 +1033,20 @@ dependencies = [ "console", "directories", "flate2", + "libc", "regex", "reqwest", "serde", "serde_json", "sha2", + "signal-hook", "similar", + "tar", "tempfile", "toml", "toml_edit", "url", + "zip", ] [[package]] @@ -1156,6 +1196,26 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + [[package]] name = "simd-adler32" version = "0.3.10" @@ -1250,6 +1310,17 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tempfile" version = "3.27.0" @@ -1531,9 +1602,9 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasm-bindgen" -version = "0.2.126" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" dependencies = [ "cfg-if", "once_cell", @@ -1544,19 +1615,20 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.76" +version = "0.4.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" dependencies = [ "js-sys", + "tokio", "wasm-bindgen", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.126" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1564,31 +1636,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.126" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.126" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" dependencies = [ "unicode-ident", ] [[package]] name = "web-sys" -version = "0.3.103" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" dependencies = [ "js-sys", "wasm-bindgen", @@ -1716,6 +1788,16 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + [[package]] name = "yoke" version = "0.8.3" @@ -1799,6 +1881,21 @@ dependencies = [ "syn 3.0.5", ] +[[package]] +name = "zip" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dcb24d0152526ae49b9b96c1dcf71850ca1e0b882e4e28ed898a93c41334744" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "flate2", + "indexmap", + "memchr", + "zopfli", +] + [[package]] name = "zlib-rs" version = "0.6.7" @@ -1810,3 +1907,15 @@ name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zopfli" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaf7fc5d30c28483d93805c4a5e12b05bbb52407fa67c5f8bd552374cd01fb11" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/Cargo.toml b/Cargo.toml index abf6bf8..41b3331 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,3 +25,9 @@ flate2 = "1.1" sha2 = "0.10" similar = "2" reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "multipart", "rustls-tls"] } +tar = "0.4" +zip = { version = "2.6", default-features = false, features = ["deflate"] } + +[target.'cfg(unix)'.dependencies] +libc = "0.2" +signal-hook = "0.3" diff --git a/docs/automated-macos-releases.md b/docs/automated-macos-releases.md new file mode 100644 index 0000000..27330f3 --- /dev/null +++ b/docs/automated-macos-releases.md @@ -0,0 +1,296 @@ +# Automated macOS releases from Linux + +`rust_realease` can build `aarch64-apple-darwin` and +`x86_64-apple-darwin` on a Linux release host. Apple targets use a pinned +`cargo-zigbuild` container; Linux and Windows targets continue to use Cross. +One invocation may mix both backends and feeds every result into the existing +archive, standalone binary, checksum, collision, upload, and download +verification pipeline. + +This is compile and packaging support for separate Mach-O executables. It does +not create universal binaries or `.app` bundles, sign with Developer ID, +notarize, or support macOS below 13.0. + +## Agent workflow + +1. Add either or both supported Apple triples to `build.targets`. Apple targets + are opt-in; existing configurations retain their current target matrix. +2. Keep project-specific Linux/Windows settings in `Cross.toml` or the existing + `CROSS_CONFIG`. Apple targets do not use Cross configuration or require + Darwin Cross images. +3. Run `rust_realease SOURCE --package PACKAGE --dry-run`. Confirm each target's + printed backend, Apple image digest, SDK path, deployment minimum, artifact + names, and release destination. Dry-run does not start containers, compile, + create artifacts, read a publication token, or contact a release API. +4. Run the project's normal token-free compile-only command. This builds every + selected target and packages all outputs before returning success. For + `tiny-wb`, the entry point remains `./release.sh --compile-only`. +5. Review `SHA256SUMS`, archive layout and modes, and the target-aware check + report. Apple runtime checks are explicitly `skipped` on Linux. +6. Perform native smoke/hardware testing on compatible macOS machines. Publish + later with the normal release command; no separate Apple publisher or manual + `build-macos.sh` step is needed. + +`--target` overrides the configured target list and can be repeated. `--features` +accepts a comma-separated list and overrides `build.features`: + +```console +rust_realease . --package my-tool \ + --target aarch64-apple-darwin \ + --target x86_64-apple-darwin \ + --features serial,ui \ + --artifact-format both --compile-only +``` + +Package name, version and declared binary names come from Cargo metadata. The +selected profile, lock policy, and features are applied identically to Cross +and cargo-zigbuild. + +## Configuration + +The default backend mapping is fixed and target-aware: + +| Target | Backend | +| --- | --- | +| `aarch64-apple-darwin` | pinned cargo-zigbuild container | +| `x86_64-apple-darwin` | pinned cargo-zigbuild container | +| every other valid Rust target | existing Cross command | + +The complete supported syntax is: + +```toml +[build] +targets = [ + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-gnu", + "aarch64-apple-darwin", + "x86_64-apple-darwin", +] +profile = "release" +locked = true +features = ["serial", "ui"] # Optional; default is no explicit features. + +[build.apple] +image = "ghcr.io/rust-cross/cargo-zigbuild@sha256:9b4f6b3eb9e8f9fefb3960fe288ddba576627d66d9eff851ce7f62d2455564a5" +deployment-target = "13.0" +sdk-root = "/opt/MacOSX11.3.sdk" + +[package] +format = "both" +``` + +The `[build.apple]` table is optional because these exact values are built-in +defaults. Declaring it explicitly makes the compiler/SDK contract visible in a +release policy. The image must use an immutable lowercase SHA-256 digest; +mutable tags are rejected. `deployment-target` must parse as a macOS version +at least 13.0. `sdk-root` must be an absolute path inside the image without +parent traversal. Unknown settings, invalid feature names and unsupported +Apple triples fail configuration validation. + +Layered configuration follows the existing rules. A later `build.features` +list replaces an earlier list. Individual values in `[build.apple]` override +earlier values, so a user-level image pin cannot silently erase a project-level +deployment target or SDK path. `--features` replaces the configured feature +list when supplied. + +## Container contract and security + +The publisher invokes the selected Docker or rootless Podman engine directly: + +```text +ENGINE run --rm --user UID:GID [--userns=keep-id for Podman] + -v ABSOLUTE_WORKSPACE:/io + -v ABSOLUTE_FRESH_OUTPUT:/output + -w /io + -e HOME=/tmp + -e CARGO_HOME=/output/cargo-home + -e CARGO_TARGET_DIR=/output/target + -e ZIG_GLOBAL_CACHE_DIR=/output/zig-cache + -e MACOSX_DEPLOYMENT_TARGET=13.0 + -e SDKROOT=/opt/MacOSX11.3.sdk + PINNED_IMAGE + cargo zigbuild --locked --release --package PACKAGE --bin BINARY + --target APPLE_TARGET --message-format=json-render-diagnostics +``` + +Every argument is passed as a distinct process argument, so spaces and Unicode +in checkout/output paths do not undergo shell interpretation. The source is +mounted at `/io`; a fresh publisher-owned directory is mounted at `/output`. +Cargo and Zig caches stay writable there, and generated files retain the +caller's UID/GID. There are no interactive flags or container-engine socket +mounts. Podman receives `--userns=keep-id`; Docker does not. + +Only the listed environment variables enter the compiler container. Gitea's +configured token variable and host variables whose names contain `TOKEN`, +`PASSWORD`, `SECRET`, or `CREDENTIAL` are removed from compiler subprocesses. +They do not appear in arguments or verbose logs. Private Cargo dependencies +therefore need a separately designed non-secret-in-container acquisition +strategy; do not add publication tokens to Apple settings. + +`CROSS_CONTAINER_ENGINE=podman` or `docker` selects the engine for both +backends. Otherwise the existing Docker-then-Podman discovery applies. The +engine must answer `info`. Apple builds require a Linux host; neither a Mac nor +host Xcode is required. `CROSS_CONFIG` continues to reach Cross for Linux and +Windows and is irrelevant to the direct Apple container invocation. + +The pinned image is the environment verified by the downstream reference: +Rust 1.93.0, cargo-zigbuild 0.23.4, Zig 0.16.0, Apple Rust targets, and +`SDKROOT=/opt/MacOSX11.3.sdk`. The SDK is supplied by the upstream +`rust-cross/cargo-zigbuild` image. The digest identifies the actual image; +the SDK directory name alone does not establish the deployment minimum. During +the real verification below, LLVM reported an SDK load-command value of 26.4 +and a minimum of 13.0. The publisher validates the minimum independently. + +## Build discovery and cache identity + +Apple builds request Cargo JSON messages and use each `compiler-artifact` +executable path. `/output` container paths are remapped to the exact mounted +host directory and canonicalized beneath that fresh directory. The publisher +does not reconstruct artifacts from a conventional checkout `target/` path. +This supports invocation outside the checkout, paths with spaces, workspace +package selection, custom profiles, and publisher-owned output directories. + +A clean-commit build cache includes: + +- commit, package, version, declared binaries and complete target list; +- profile, lock policy and Cargo features; +- container engine and active host Rust toolchain; +- Apple image digest, deployment target and SDK root. + +Cached output paths must remain regular files beneath the publisher cache and +their SHA-256 hashes must still match. Cached Apple files are re-inspected as +Mach-O before reuse. Changing any fingerprinted setting, changing a binary, +using a dirty checkout, deleting an output, or passing `--recompile` forces a +fresh build. A backend failure propagates immediately and cannot be reported as +a successful build or create/upload a partial release. + +## Artifacts and target-aware verification + +Apple binary names have no `.exe` suffix. Existing archive and raw-binary +naming applies unchanged. For version `{version}` and package `tiny-wb`: + +```text +tiny-wb-aarch64-apple-darwin-v{version}.tgz +tiny-wb-aarch64-apple-darwin-v{version} +tiny-wb-x86_64-apple-darwin-v{version}.tgz +tiny-wb-x86_64-apple-darwin-v{version} +``` + +Each tgz has the cargo-binstall layout: + +```text +tiny-wb-aarch64-apple-darwin-v{version}/ + tiny-wb +``` + +The member and standalone file retain executable permission. Ownership and +timestamps in tgz files remain normalized. A mixed four-target release using +`format = "both"` produces eight payload assets and one `SHA256SUMS` asset. +All local archives are checked before any release creation. Existing remote +collision handling, replacement policy, upload, authenticated/private download +handling, and checksum verification apply to Apple assets too. + +For every Apple executable, the Linux host parses the Mach-O header and load +commands and requires: + +- a little-endian 64-bit executable matching arm64 or x86-64 as requested; +- macOS `LC_BUILD_VERSION` exactly matching `deployment-target`; +- every declared loaded/re-exported/weak/upward dylib under `/usr/lib/` or + `/System/Library/`; +- at least one system dynamic dependency. + +This detects architecture mistakes, a silently raised deployment minimum, and +build-machine/container library leakage. It does not execute Mach-O files. +`cargo-binstall` verification installs foreign binaries into an isolated +directory but does not run them. Console output records macOS runtime as +`skipped (no compatible macOS runner)`, never passed. A native Mac should test +startup, the application's normal workflow, and relevant hardware. + +## tiny-wb migration + +Apply this configuration change in the downstream checkout; its existing +unprefixed tags, release profile, locked mode, Gitea token setting, repository +fallback and `release/Cross.toml` remain unchanged: + +```diff + [build] +-targets = ["x86_64-unknown-linux-gnu", "x86_64-pc-windows-gnu"] ++targets = [ ++ "x86_64-unknown-linux-gnu", ++ "x86_64-pc-windows-gnu", ++ "aarch64-apple-darwin", ++ "x86_64-apple-darwin", ++] + profile = "release" + locked = true ++ ++[build.apple] ++image = "ghcr.io/rust-cross/cargo-zigbuild@sha256:9b4f6b3eb9e8f9fefb3960fe288ddba576627d66d9eff851ce7f62d2455564a5" ++deployment-target = "13.0" ++sdk-root = "/opt/MacOSX11.3.sdk" +``` + +`release.sh` already passes `--artifact-format both` and selects +`release/Cross.toml`; it needs no extra Apple build command. The publisher +reads version and binaries from Cargo metadata, so do not put `0.1.4` or any +other tested version in release configuration. `scripts/build-macos.sh` remains +a useful diagnostic reference and can be retired from the release procedure +after adopting a publisher version containing this feature. + +Run: + +```console +./release.sh --compile-only +``` + +No Gitea token is required. Publication remains a separate explicit invocation. + +## Verification performed for this implementation + +No release, upload, tag, push, Jenkins job, or global publisher installation was +performed. A disposable clone of `tiny-wb` commit `8ed07cb` under `/tmp` was +built with this checkout's publisher and rootless Podman. + +Real two-target command: + +```console +CROSS_CONTAINER_ENGINE=podman ./target/debug/rust_realease \ + /tmp/tiny-wb-rust-release-macos --package tiny-wb \ + --target aarch64-apple-darwin --target x86_64-apple-darwin \ + --artifact-format both --compile-only --skip-tests +``` + +It completed both targets in 95.7 seconds. A second identical invocation reused +commit `8ed07cb60eb6` from the hash-validated cache. The first artifacts were: + +```text +/tmp/tiny-wb-rust-release-macos/dist/release-IxUtOI/ + tiny-wb-aarch64-apple-darwin-v0.1.4 + tiny-wb-aarch64-apple-darwin-v0.1.4.tgz + tiny-wb-x86_64-apple-darwin-v0.1.4 + tiny-wb-x86_64-apple-darwin-v0.1.4.tgz + SHA256SUMS +``` + +`sha256sum -c SHA256SUMS` passed. `file` identified 64-bit arm64 and x86-64 +Mach-O executables. Both tgz files had one root directory and a mode-0755 +`tiny-wb`. `llvm-objdump` reported macOS minimum 13.0 and these dependencies for +both architectures: IOKit, CoreFoundation, libSystem, libobjc, libiconv and +libcharset, all under Apple system paths. + +The real four-target command additionally set +`CROSS_CONFIG=/tmp/tiny-wb-rust-release-macos/release/Cross.toml` and selected +`x86_64-unknown-linux-gnu` plus `x86_64-pc-windows-gnu`. It completed all four +builds in 169.7 seconds and produced exactly eight payload files in: + +```text +/tmp/tiny-wb-rust-release-macos/dist/release-3XAf1R/ +``` + +The automated suite also covers Docker/Podman argument differences, paths with +spaces and Unicode, mixed routing, credential removal, failure propagation, +dry-run, token-free compile-only operation, cache reuse/invalidation fields, +Mach-O rejection cases, artifact names, permissions, archive layout and eight +checksum entries. Native macOS execution, exact macOS versions/CPUs exercised by +the maintainer's earlier successful testing, hardware behavior, Gatekeeper, +signing, notarization, Jenkins and real publication remain external checks. diff --git a/docs/portable-bundles.md b/docs/portable-bundles.md new file mode 100644 index 0000000..f25ba69 --- /dev/null +++ b/docs/portable-bundles.md @@ -0,0 +1,321 @@ +# Portable application bundles (protocol 1) + +Named bundles add application resources and runtime executables to the release +workflow. They are **additional artifacts**: `package.format`, +`--artifact-format`, canonical tgz filenames, and cargo-binstall metadata retain +their existing meanings. No release was published to test this feature. + +## Agent workflow + +1. Keep `[package] format = "tgz"` for the existing cargo-binstall distribution. +2. Add `[[package.bundles]]` for each additional distribution. Declare every + static inclusion; select Cargo binaries; configure preparation if necessary. +3. Run `rust_realease . --package PACKAGE --dry-run`. This validates static + inputs, configuration, names, and selected targets, and prints commands and + required outputs. Dynamic results remain unresolved. No preparation, dependency + downloads, cache directories, or archives are created. Existing metadata/tag/ + prerequisite diagnostics still apply. +4. Run `rust_realease . --package PACKAGE --compile-only`. This needs no Gitea + token and creates no tag, push, release, or upload. `--stop-after package` + does the same packaging work. `--skip-tests` only skips source tests; + preparation and bundle validation still run. +5. Read the stable local `dist/latest-artifacts.json`. Its `manifest` and + `checksums` paths are relative to `dist/` and identify the latest successful + bundle invocation; a failed invocation leaves the previous locator intact. + Open that `artifact-manifest.json` (beside `SHA256SUMS`). It contains schema + version 1, relative asset filenames, + asset digests, and bundle file destinations, digests, modes, and check statuses. + Resolve filenames relative to this directory; do not guess build paths. +6. Extract and perform native application acceptance tests. Then use the existing + explicit publication workflow. All artifacts are assembled and validated + before release creation. Upload collision handling, `--replace-assets`, and + download/checksum verification include bundles and the artifact manifest. + Unrelated release assets remain untouched. Uploads are not transactional. + +`--target TRIPLE` limits builds and bundle preparation to that target. A bundle's +`targets` filters further; omitted/empty means all selected build targets. Every +selected target must have an entry in `formats`. Supported formats are `zip` and +`tgz` (gzip-compressed tar). Extensions are appended automatically. + +## Configuration contract + +```toml +[package] +format = "tgz" + +[[package.bundles]] +name = "portable" +targets = ["x86_64-pc-windows-gnu", "x86_64-unknown-linux-gnu"] +formats = { x86_64-pc-windows-gnu = "zip", x86_64-unknown-linux-gnu = "tgz" } +archive-name = "{name}-{bundle}-{target}-v{version}" +root = "{name}-{bundle}-{target}-v{version}" +cargo-binaries = "all" # Or ["app", "app-gui"], using declared Cargo target names. +files = [ + { source = "assets", destination = "resources" }, + { source = "LICENSE", destination = "LICENSES/app.txt", workspace-relative = true }, + { source = "runtime/helper.dll", destination = "helper.dll" }, +] + +[package.bundles.prepare] +command = ["python3", "prepare-release.py", "--context", "{context-file}", "--manifest", "{manifest-file}"] +inputs = ["prepare-release.py", "dependency-pins.json", "workspace:scripts/downloader.py"] +timeout-seconds = 600 +required-destinations = ["helper{binary-ext}", "LICENSES/helper/LICENSE.txt"] +``` + +Static sources are relative to the selected Cargo package's directory, even when +configuration is at the workspace root. Set `workspace-relative = true` for +workspace files. Directories are recursively copied as explicitly declared file +trees; empty directories are not included. DLLs/shared libraries/resources use +`kind = "file"` (the default); runnable programs use `kind = "executable"`. +Executable trees apply that kind to every file, so declare mixed trees separately. + +`inputs` lists regular preparation/smoke-test input files and validates their +existence. Strings are package-relative; `workspace:PATH` explicitly selects the +workspace. The hook may read these inputs but is responsible for interpreting +pins. Hooks and bundles are never cached, so each invocation resolves dependency +identity again. A verified Rust build cache may still be reused. Changing a +license, adapter, pin, or downloaded dependency cannot cause a stale bundle hit. +A new source commit can still invalidate the existing Rust build cache. + +Template substitutions in `archive-name`, `root`, static destinations and required +destinations are `{name}` (Cargo package), `{version}`, `{bundle}`, `{target}` and +`{binary-ext}` (`.exe` on Windows, otherwise empty). Archive name and root must +be single components; neither is a source path. Names must not collide with +canonical artifacts or `SHA256SUMS`/`artifact-manifest.json`. Unknown config +fields, formats, binary selections and template placeholders are errors. Layered +configuration replaces the entire bundle list when a later layer specifies one. + +## Host preparation contract + +Preparation runs once per selected bundle/target after compilation or build-cache +reuse, before archive creation. The working directory is the selected package +root. `command` is an argv array, with no shell interpretation. Explicit shell +commands can use `["sh", "-c", "..."]`. The only command substitutions are +`{context-file}`, `{manifest-file}`, and `{bundle-directory}`; paths with spaces +and Unicode stay single arguments. + +The context JSON has these exact keys: + +| Key | Meaning | +| --- | --- | +| `schema_version` | Integer `1` | +| `package_name`, `package_version` | Selected Cargo package identity | +| `bundle`, `target`, `profile`, `binary_ext` | Effective bundle/build settings | +| `workspace_root`, `package_root` | Absolute source roots | +| `source_commit`, `source_dirty` | Git HEAD and pre-build checkout dirty status | +| `binaries` | Object mapping **all declared binary names** to absolute built executable paths | +| `output_directory` | Fresh, initially empty directory for prepared files | +| `cache_directory` | Persistent dependency cache, separated by bundle and target | +| `manifest_path` | Absolute path where the hook must write its result JSON | +| `bundle_directory` | Fresh tree populated after preparation, for optional smoke tests | + +Bundle builds read executable paths from Cargo compiler-artifact JSON and remap +Cross container `/target` paths to the exact host build directory. Cached +builds use validated publisher cache paths. The hook must use `binaries`, never +construct `target/TRIPLE/release/NAME`; this supports workspace members, custom +profiles, `CARGO_TARGET_DIR` and fresh dirty-checkout builds. + +The hook writes outputs only beneath `output_directory`, writes its result to +`manifest_path`, and may maintain the supplied persistent cache. It must not +construct archives, checksums, tags or releases. The host command is trusted +project code, **not an OS filesystem/network sandbox**. Gitea's configured token +variable and environment variables containing TOKEN, PASSWORD, SECRET or +CREDENTIAL are removed from its environment. Output is streamed with target/ +bundle labels and existing secret redaction. Do not put credentials in command +arguments or public provenance files. + +Example result (replace hashes with actual lowercase SHA-256 hex digests): + +```json +{ + "schema_version": 1, + "target": "x86_64-pc-windows-gnu", + "files": [ + {"source": "helper.exe", "destination": "helper.exe", "kind": "executable", "sha256": "<64 hex characters>"}, + {"source": "notices/LICENSE.txt", "destination": "LICENSES/helper/LICENSE.txt", "kind": "file", "sha256": "<64 hex characters>"} + ] +} +``` + +Every manifest source is a regular file relative to `output_directory`; enumerate +files in directories individually. Every destination is relative to the archive +root. Only declared files enter the archive. Required destinations must exist +exactly, including case. The publisher verifies hashes and target/schema identity. +Cargo executables cannot be replaced by static files or hook outputs. + +Paths cannot be absolute, contain `..`, empty components, backslashes or colons. +Symlinks, nonregular files, and hardlinked files are rejected, including directory +traversal through symlinks. Windows destinations additionally reject reserved +names, invalid characters and case-insensitive collisions. File/directory-prefix +collisions also fail. Files are ordered deterministically; ownership and timestamps +are normalized; modes are 0755 for executables and 0644 for data. Identical input +bytes and settings produce identical archives. Each archive is written atomically +and reopened to check its declared contents, modes and hashes. Partial staging is +removed on failure; successful invocation directories are retained under `dist/`. + +Executable validation currently supports little-endian Linux ELF (x86-64, +aarch64, i686, riscv64gc) and Windows PE (x86-64, aarch64, i686). Unsupported +executable targets fail explicitly. This checks format and architecture, not ABI, +DLL completeness, license suitability or runtime behavior. Foreign executables +are never executed automatically by the publisher. Trusted preparation scripts +must follow the same rule themselves. + +Nonzero exit, timeout, SIGINT/SIGTERM cancellation on Unix, malformed JSON, +missing outputs, hash mismatch or architecture mismatch abort before publication. +Unix hooks run in their own process group; descendants are killed on completion, +failure, timeout or cancellation. Native Windows host process-tree cancellation +has not been validated; use a Unix release host for this workflow. + +## Optional native smoke test + +```toml +[package.bundles.smoke-test] +command = ["python3", "smoke-release.py", "{bundle-directory}"] +inputs = ["smoke-release.py"] +timeout-seconds = 60 +``` + +The publisher runs this command only when the target exactly matches `rustc -vV`'s +host triple. The script should invoke programs by absolute path in the supplied +tree, set any necessary environment explicitly, and return nonzero on failure. +Checks are reported as `passed` or `skipped` with a reason in the artifact manifest; +a failed command is reported with its status and stops packaging, so no successful +artifact manifest is emitted for that invocation. A successful smoke test cannot +silently modify declared payload bytes. Preparation may separately record native +checks in its own provenance. Cross-compiling Windows is not Windows validation. + +## Migration recipe: pptx-reporter + +This recipe uses the existing `scripts/fetch_ffprobe.py` API: +`fetch(target, cache=None, release="latest", expected_sha256=None)` returns +`(binary_path, notices_directory)`. It retains that downloader's target selection, +verified archive digests, LGPL/non-shared provider selection and provenance. Do not +copy the host's `ffprobe` or change the publisher into a provider-specific downloader. + +In the consumer workspace's `rust-release.toml`, keep the existing build targets, +release and repository settings. Replace the separate packager comments with: + +```toml +[package] +format = "tgz" + +[[package.bundles]] +name = "portable" +targets = ["x86_64-pc-windows-gnu", "x86_64-unknown-linux-gnu"] +formats = { x86_64-pc-windows-gnu = "zip", x86_64-unknown-linux-gnu = "tgz" } +archive-name = "{name}-portable-{target}-v{version}" +root = "{name}-portable-{target}-v{version}" +cargo-binaries = "all" +files = [ + { source = "LICENSE", destination = "LICENSES/reporter-MIT.txt", workspace-relative = true }, + { source = "assets/fonts/LICENSE", destination = "LICENSES/DejaVu.txt" }, + { source = "docs/windows-quickstart.txt", destination = "README.txt", workspace-relative = true }, +] + +[package.bundles.prepare] +command = ["python3", "prepare-release.py", "--context", "{context-file}", "--manifest", "{manifest-file}"] +inputs = ["prepare-release.py", "ffprobe-pins.json", "workspace:scripts/fetch_ffprobe.py"] +timeout-seconds = 600 +required-destinations = ["ffprobe{binary-ext}", "LICENSES/ffmpeg/LICENSE.txt", "LICENSES/ffmpeg/provenance.json"] +``` + +Create `crates/pptx-reporter/ffprobe-pins.json`, using actual immutable provider +release tags and reviewed SHA-256 archive digests for **each target**: + +```json +{ + "x86_64-pc-windows-gnu": {"release": "", "sha256": ""}, + "x86_64-unknown-linux-gnu": {"release": "", "sha256": ""} +} +``` + +Create `crates/pptx-reporter/prepare-release.py`: + +```python +import argparse +import hashlib +import json +from pathlib import Path +import shutil +import sys + +parser = argparse.ArgumentParser() +parser.add_argument("--context", type=Path, required=True) +parser.add_argument("--manifest", type=Path, required=True) +args = parser.parse_args() +context = json.loads(args.context.read_text()) +assert context["schema_version"] == 1 +sys.path.insert(0, str(Path(context["workspace_root"]) / "scripts")) +from fetch_ffprobe import fetch + +pins = json.loads((Path(context["package_root"]) / "ffprobe-pins.json").read_text()) +pin = pins[context["target"]] +binary, notices = fetch( + context["target"], Path(context["cache_directory"]), + pin["release"], pin["sha256"], +) +out = Path(context["output_directory"]) +files = [] + +def include(source, relative, destination, kind="file"): + if source.is_symlink() or not source.is_file(): + raise ValueError(f"Not a regular prepared input: {source}") + copied = out / relative + copied.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, copied) # Copy bytes, not cache links. + files.append({ + "source": relative, "destination": destination, "kind": kind, + "sha256": hashlib.sha256(copied.read_bytes()).hexdigest(), + }) + +name = "ffprobe" + context["binary_ext"] +include(binary, name, name, "executable") +for path in sorted(notices.rglob("*")): + if path.is_symlink(): + raise ValueError(f"Symlinked notice: {path}") + if path.is_file(): + relative = path.relative_to(notices).as_posix() + include(path, "notices/" + relative, "LICENSES/ffmpeg/" + relative) +# The downloader preserves upstream archive directories, whose names vary. +# Retain them all and add a stable alias for the required license destination. +license_destination = "LICENSES/ffmpeg/LICENSE.txt" +if not any(f["destination"] == license_destination for f in files): + licenses = sorted(p for p in notices.rglob("*") if p.is_file() + and p.name.lower().startswith(("license", "copying"))) + if not licenses: + raise ValueError("Missing upstream license") + include(licenses[0], "canonical-LICENSE.txt", license_destination) +args.manifest.write_text(json.dumps({ + "schema_version": 1, "target": context["target"], "files": files, +}) + "\n") +``` + +The adapter preserves upstream notices and `provenance.json`, including provider +URL, release/asset IDs, archive SHA-256, extracted hashes and the downloader's +native version check. A stable `LICENSE.txt` alias is added while retaining the +original upstream notice hierarchy. It never discovers Cargo output paths or creates archives. +Python is a build-host dependency only. Ignore the target/cache/dist directories +in Git; do not track downloaded executables or generated manifests. + +Run: + +```sh +rust_realease . --package pptx-reporter --dry-run +rust_realease . --package pptx-reporter --compile-only +# To prepare/build just Linux: +rust_realease . --package pptx-reporter --target x86_64-unknown-linux-gnu --compile-only +``` + +Verify checksums and both full portable trees. On Linux, extract the tgz and run +report generation using the adjacent FFprobe without relying on PATH. On a Windows +machine, extract the ZIP, launch the GUI and generate a report without Python, +Typst, PowerPoint or separately installed FFmpeg. These consumer application tests +and Windows native validation are **not performed by the fixture test suite**. +Remove `scripts/package-portable.py` only after completing that consumer migration. +Keep Jenkins as checks/coverage/badges only; publishing remains explicit. + +The canonical cargo-binstall archives still install the declared Rust binaries. +They do **not** install the portable bundle's resources or FFprobe. Document the +external FFprobe requirement for cargo-binstall users. Portable-tree validation, +cargo-binstall installation checks and native-runtime checks are separate results. diff --git a/rust-release.toml b/rust-release.toml index 3b81392..dd93fb3 100644 --- a/rust-release.toml +++ b/rust-release.toml @@ -1,3 +1,18 @@ [[repository.fallbacks]] origin = "git@git.dandaedre.com:idle-org/rust_realease.git" fallback-url = "https://proxy.dandaedre.com/gitea/idle-org/rust_realease" + +[build] +targets = [ + "x86_64-unknown-linux-musl", + "aarch64-unknown-linux-musl", + "x86_64-pc-windows-gnu", + "x86_64-apple-darwin", +] +profile = "release" +locked = true + +[build.apple] +image = "ghcr.io/rust-cross/cargo-zigbuild@sha256:9b4f6b3eb9e8f9fefb3960fe288ddba576627d66d9eff851ce7f62d2455564a5" +deployment-target = "13.0" +sdk-root = "/opt/MacOSX11.3.sdk" diff --git a/src/build.rs b/src/build.rs index 5ff6b7a..dfaf452 100644 --- a/src/build.rs +++ b/src/build.rs @@ -6,7 +6,7 @@ use std::{ time::Instant, }; -#[derive(Debug)] +#[derive(Debug, Clone, Deserialize, Serialize)] pub struct TargetOutputs { pub target: String, pub binaries: Vec<(String, PathBuf)>, @@ -22,14 +22,22 @@ struct BuildFingerprint { targets: Vec, profile: String, locked: bool, + features: Vec, engine: String, toolchain: Option, + apple_image: String, + apple_deployment_target: String, + apple_sdk_root: String, } #[derive(Debug, Deserialize, Serialize)] struct BuildCache { + #[serde(default)] + outputs: Option>, fingerprint: BuildFingerprint, target_dir: PathBuf, + #[serde(default)] + output_hashes: std::collections::BTreeMap, } pub fn prerequisites( @@ -40,8 +48,13 @@ pub fn prerequisites( runner: &Runner, ) -> (Vec, Option) { let mut problems = Vec::new(); - if let Err(error) = validate_cross_config(root) { - problems.push(error.to_string()); + let needs_build = needs_cross; + let needs_cross = needs_build && config.targets.iter().any(|target| !is_apple(target)); + let needs_apple = needs_build && config.targets.iter().any(|target| is_apple(target)); + if needs_cross { + if let Err(error) = validate_cross_config(root) { + problems.push(error.to_string()); + } } for tool in ["git", "cargo"] .into_iter() @@ -54,8 +67,11 @@ pub fn prerequisites( if needs_cross && cross_program().is_none() { problems.push("cross is missing (looked for cross and cargo-cross)".into()); } + if needs_apple && env::consts::OS != "linux" { + problems.push("Apple cross-compilation is supported from a Linux release host".into()); + } let mut engine = None; - if needs_cross { + if needs_build { let requested = env::var("CROSS_CONTAINER_ENGINE").ok(); let candidate = requested.or_else(|| { ["docker", "podman"] @@ -162,6 +178,11 @@ pub fn verify_source( if config.locked { flags.push("--locked"); } + let features; + if !config.features.is_empty() { + features = config.features.join(","); + flags.extend(["--features", &features]); + } runner.output("cargo", &[&["check"], flags.as_slice()].concat(), root)?; if !skip_tests { runner.output("cargo", &[&["test"], flags.as_slice()].concat(), root)?; @@ -184,6 +205,9 @@ pub fn cross_arguments( if config.locked { args.push("--locked".into()); } + if !config.features.is_empty() { + args.extend(["--features".into(), config.features.join(",")]); + } args.extend([ "--target".into(), target.to_owned(), @@ -198,6 +222,95 @@ pub fn cross_arguments( args } +pub fn is_apple(target: &str) -> bool { + matches!(target, "aarch64-apple-darwin" | "x86_64-apple-darwin") +} + +fn apple_arguments( + root: &Path, + output: &Path, + project: &SelectedPackage, + config: &Config, + target: &str, + engine: &str, +) -> Result, Error> { + let root = fs::canonicalize(root) + .map_err(|e| Error(format!("cannot resolve Apple source mount: {e}")))?; + fs::create_dir_all(output) + .map_err(|e| Error(format!("cannot create Apple output directory: {e}")))?; + let output = fs::canonicalize(output) + .map_err(|e| Error(format!("cannot resolve Apple output mount: {e}")))?; + let (uid, gid) = current_user(); + let mut args = vec![ + "run".into(), + "--rm".into(), + "--user".into(), + format!("{uid}:{gid}"), + ]; + if engine == "podman" { + args.push("--userns=keep-id".into()); + } + args.extend([ + "-v".into(), + format!("{}:/io", root.display()), + "-v".into(), + format!("{}:/output", output.display()), + "-w".into(), + "/io".into(), + "-e".into(), + "HOME=/tmp".into(), + "-e".into(), + "CARGO_HOME=/output/cargo-home".into(), + "-e".into(), + "CARGO_TARGET_DIR=/output/target".into(), + "-e".into(), + "ZIG_GLOBAL_CACHE_DIR=/output/zig-cache".into(), + "-e".into(), + format!( + "MACOSX_DEPLOYMENT_TARGET={}", + config.apple.deployment_target + ), + "-e".into(), + format!("SDKROOT={}", config.apple.sdk_root), + config.apple.image.clone(), + "cargo".into(), + "zigbuild".into(), + ]); + if config.locked { + args.push("--locked".into()); + } + if config.profile == "release" { + args.push("--release".into()); + } else { + args.extend(["--profile".into(), config.profile.clone()]); + } + if !config.features.is_empty() { + args.extend(["--features".into(), config.features.join(",")]); + } + args.extend([ + "--package".into(), + project.name.clone(), + "--target".into(), + target.into(), + "--message-format=json-render-diagnostics".into(), + ]); + for binary in &project.binaries { + args.extend(["--bin".into(), binary.clone()]); + } + Ok(args) +} + +#[cfg(unix)] +fn current_user() -> (u32, u32) { + // SAFETY: getuid/getgid have no preconditions and cannot fail. + unsafe { (libc::getuid(), libc::getgid()) } +} + +#[cfg(not(unix))] +fn current_user() -> (u32, u32) { + (0, 0) +} + pub fn build( root: &Path, project: &SelectedPackage, @@ -217,7 +330,7 @@ pub fn build( .and_then(|line| line.split_whitespace().next().map(str::to_owned)) }); let fingerprint = clean_commit.map(|commit| BuildFingerprint { - format_version: 1, + format_version: 2, commit: commit.to_owned(), package: project.name.clone(), version: project.version.clone(), @@ -225,8 +338,12 @@ pub fn build( targets: config.targets.clone(), profile: config.profile.clone(), locked: config.locked, + features: config.features.clone(), engine: engine.to_owned(), toolchain: toolchain.clone(), + apple_image: config.apple.image.clone(), + apple_deployment_target: config.apple.deployment_target.clone(), + apple_sdk_root: config.apple.sdk_root.clone(), }); let cache_path = target_base.join("release-build.json"); if !recompile { @@ -258,9 +375,22 @@ pub fn build( &config.profile }; let mut outputs = Vec::new(); - let cross = cross_program() - .ok_or_else(|| Error("cross is missing (looked for cross and cargo-cross)".into()))?; + let cross = cross_program(); let builds_started = Instant::now(); + let credential_names = env::vars_os() + .filter_map(|(name, _)| { + let name = name.to_string_lossy(); + let upper = name.to_ascii_uppercase(); + ["TOKEN", "PASSWORD", "SECRET", "CREDENTIAL"] + .iter() + .any(|word| upper.contains(word)) + .then(|| name.into_owned()) + }) + .collect::>(); + let credential_removals = credential_names + .iter() + .map(String::as_str) + .collect::>(); for target in &config.targets { println!("Building {target}"); let target_started = Instant::now(); @@ -268,15 +398,42 @@ pub fn build( // build scripts and proc macros isolated so a later image never tries // to execute artifacts produced by an earlier one. let target_build_dir = target_dir.join(target); - let args = cross_arguments(project, config, target, &target_build_dir); + let (program, args, reported_root) = if is_apple(target) { + ( + engine, + apple_arguments(root, &target_build_dir, project, config, target, engine)?, + Path::new("/output"), + ) + } else { + let mut args = cross_arguments(project, config, target, &target_build_dir); + if !config.bundles.is_empty() { + args.push("--message-format=json-render-diagnostics".into()); + } + ( + cross.ok_or_else(|| { + Error("cross is missing (looked for cross and cargo-cross)".into()) + })?, + args, + Path::new("/target"), + ) + }; let borrowed: Vec<&str> = args.iter().map(String::as_str).collect(); - runner.output_with(cross, &borrowed, root, &environment, &[])?; + let messages = + runner.output_with(program, &borrowed, root, &environment, &credential_removals)?; + let reported = cargo_executables(&messages); let mut binaries = Vec::new(); for name in &project.binaries { - let binary = target_build_dir - .join(target) - .join(profile_dir) - .join(policy::binary_filename(name, target)); + let binary = if is_apple(target) || !config.bundles.is_empty() { + let reported = reported + .get(name) + .ok_or_else(|| Error(format!("Cargo did not report executable for {name}")))?; + resolve_executable(reported, reported_root, &target_build_dir)? + } else { + target_build_dir + .join(target) + .join(profile_dir) + .join(policy::binary_filename(name, target)) + }; let metadata = fs::symlink_metadata(&binary).map_err(|_| { Error(format!( "Build did not produce a regular binary: {}", @@ -289,6 +446,9 @@ pub fn build( binary.display() ))); } + if is_apple(target) { + crate::macho::verify_path(&binary, target, &config.apple.deployment_target)?; + } binaries.push((name.clone(), binary)); } outputs.push(TargetOutputs { @@ -300,14 +460,31 @@ pub fn build( target_started.elapsed().as_secs_f64() )); } + let label = if config.targets.iter().all(|target| !is_apple(target)) { + "Cross builds" + } else { + "Release builds" + }; ui::success(format_args!( - "Cross builds complete in {:.1}s", + "{label} complete in {:.1}s", builds_started.elapsed().as_secs_f64() )); if let Some(fingerprint) = fingerprint { let cache = BuildCache { + outputs: Some(outputs.clone()), fingerprint, target_dir, + output_hashes: outputs + .iter() + .flat_map(|output| { + output.binaries.iter().map(|(name, path)| { + Ok(( + format!("{}/{}", output.target, name), + crate::package::checksum(path)?, + )) + }) + }) + .collect::>()?, }; let encoded = serde_json::to_vec_pretty(&cache) .map_err(|e| Error(format!("cannot encode release build cache: {e}")))?; @@ -357,6 +534,37 @@ fn cached_outputs( Ok(path) if path.starts_with(&base) => path, _ => return Ok(None), }; + if let Some(outputs) = cache.outputs { + if outputs.len() != config.targets.len() { + return Ok(None); + } + for (output, target) in outputs.iter().zip(&config.targets) { + if &output.target != target || output.binaries.len() != project.binaries.len() { + return Ok(None); + } + for ((name, path), expected_name) in output.binaries.iter().zip(&project.binaries) { + let expected_hash = cache.output_hashes.get(&format!("{target}/{name}")); + if name != expected_name + || !path.is_file() + || path.is_symlink() + || !fs::canonicalize(path).is_ok_and(|p| p.starts_with(&base)) + || expected_hash.is_none_or(|hash| match crate::package::checksum(path) { + Ok(actual) => &actual != hash, + Err(_) => true, + }) + { + return Ok(None); + } + if is_apple(target) + && crate::macho::verify_path(path, target, &config.apple.deployment_target) + .is_err() + { + return Ok(None); + } + } + } + return Ok(Some(outputs)); + } let profile_dir = if config.profile == "dev" { "debug" } else { @@ -390,6 +598,48 @@ fn cached_outputs( Ok(Some(outputs)) } +fn resolve_executable( + reported: &Path, + reported_root: &Path, + target_dir: &Path, +) -> Result { + // Cross mounts the selected --target-dir at /target inside its container. + // Remap the reported artifact, not a guessed profile/binary layout. + let path = match reported.strip_prefix(reported_root) { + Ok(relative) => target_dir.join(relative), + Err(_) => reported.to_path_buf(), + }; + let base = fs::canonicalize(target_dir)?; + let resolved = fs::canonicalize(&path)?; + if !resolved.starts_with(base) || fs::symlink_metadata(&path)?.file_type().is_symlink() { + return Err(Error( + "Cargo executable is outside the fresh target directory or is a symlink".into(), + )); + } + Ok(resolved) +} + +fn cargo_executables(messages: &[u8]) -> std::collections::BTreeMap { + String::from_utf8_lossy(messages) + .lines() + .filter_map(|line| { + let value: serde_json::Value = serde_json::from_str(line).ok()?; + if value["reason"] != "compiler-artifact" + || !value["target"]["kind"] + .as_array()? + .iter() + .any(|k| k == "bin") + { + return None; + } + Some(( + value["target"]["name"].as_str()?.to_owned(), + PathBuf::from(value["executable"].as_str()?), + )) + }) + .collect() +} + fn short_commit(commit: &str) -> &str { commit.get(..12).unwrap_or(commit) } @@ -414,6 +664,7 @@ mod tests { fn test_project(target_directory: PathBuf) -> SelectedPackage { SelectedPackage { + workspace_root: PathBuf::from("/workspace"), name: "cli".into(), version: "0.1.0".into(), binaries: vec!["cli".into(), "helper".into()], @@ -425,6 +676,9 @@ mod tests { fn test_config() -> Config { Config { + bundles: Vec::new(), + features: Vec::new(), + apple: crate::config::AppleConfig::default(), targets: vec!["x86_64-pc-windows-gnu".into()], profile: "release".into(), locked: true, @@ -465,6 +719,62 @@ mod tests { ); } + #[test] + fn apple_container_arguments_are_pinned_noninteractive_and_path_safe() { + let temporary = tempfile::tempdir().unwrap(); + let root = temporary.path().join("checkout with spaces é"); + let output = temporary.path().join("output with spaces"); + fs::create_dir(&root).unwrap(); + let mut config = test_config(); + config.targets = vec!["aarch64-apple-darwin".into()]; + config.features = vec!["serial".into(), "ui".into()]; + let args = apple_arguments( + &root, + &output, + &test_project(PathBuf::from("target")), + &config, + "aarch64-apple-darwin", + "podman", + ) + .unwrap(); + assert_eq!(args[0], "run"); + assert!(args.contains(&"--rm".into())); + assert!(args.contains(&"--userns=keep-id".into())); + assert!(args.contains(&format!("{}:/io", root.display()))); + assert!(args.contains(&format!("{}:/output", output.display()))); + assert!(args.contains(&config.apple.image)); + assert!(args.contains(&"MACOSX_DEPLOYMENT_TARGET=13.0".into())); + assert!(args.contains(&"SDKROOT=/opt/MacOSX11.3.sdk".into())); + assert!(args + .windows(2) + .any(|args| args == ["--features", "serial,ui"])); + assert!(args.windows(2).any(|args| args == ["--package", "cli"])); + assert!(args + .windows(2) + .any(|args| args == ["--target", "aarch64-apple-darwin"])); + assert!(!args.iter().any(|arg| arg.contains("sock") || arg == "-it")); + + let docker = apple_arguments( + &root, + &output, + &test_project(PathBuf::from("target")), + &config, + "aarch64-apple-darwin", + "docker", + ) + .unwrap(); + assert!(!docker.contains(&"--userns=keep-id".into())); + } + + #[test] + fn backend_mapping_only_routes_supported_apple_targets_to_zig() { + assert!(is_apple("aarch64-apple-darwin")); + assert!(is_apple("x86_64-apple-darwin")); + assert!(!is_apple("x86_64-unknown-linux-gnu")); + assert!(!is_apple("x86_64-pc-windows-gnu")); + assert!(!is_apple("i686-apple-darwin")); + } + #[test] fn completed_clean_build_is_reused_only_for_the_same_fingerprint() { let temporary = tempfile::tempdir().unwrap(); @@ -482,7 +792,7 @@ mod tests { fs::write(path, binary).unwrap(); } let fingerprint = BuildFingerprint { - format_version: 1, + format_version: 2, commit: "0123456789abcdef".into(), package: project.name.clone(), version: project.version.clone(), @@ -490,15 +800,21 @@ mod tests { targets: config.targets.clone(), profile: config.profile.clone(), locked: config.locked, + features: config.features.clone(), engine: "podman".into(), toolchain: Some("stable-x86_64-unknown-linux-gnu".into()), + apple_image: config.apple.image.clone(), + apple_deployment_target: config.apple.deployment_target.clone(), + apple_sdk_root: config.apple.sdk_root.clone(), }; let cache_path = target_base.join("release-build.json"); fs::write( &cache_path, serde_json::to_vec(&BuildCache { + outputs: None, fingerprint: fingerprint.clone(), target_dir, + output_hashes: std::collections::BTreeMap::new(), }) .unwrap(), ) @@ -511,13 +827,37 @@ mod tests { let different = BuildFingerprint { commit: "different".into(), - ..fingerprint + ..fingerprint.clone() }; assert!( cached_outputs(&target_base, &cache_path, &different, &project, &config) .unwrap() .is_none() ); + for incompatible in [ + BuildFingerprint { + features: vec!["new-feature".into()], + ..fingerprint.clone() + }, + BuildFingerprint { + apple_deployment_target: "14.0".into(), + ..fingerprint.clone() + }, + BuildFingerprint { + apple_image: format!("example.invalid/compiler@sha256:{}", "a".repeat(64)), + ..fingerprint.clone() + }, + BuildFingerprint { + engine: "docker".into(), + ..fingerprint.clone() + }, + ] { + assert!( + cached_outputs(&target_base, &cache_path, &incompatible, &project, &config) + .unwrap() + .is_none() + ); + } } #[test] fn cross_pre_build_is_allowed_but_other_executable_options_are_rejected() { diff --git a/src/bundle.rs b/src/bundle.rs new file mode 100644 index 0000000..33fd2da --- /dev/null +++ b/src/bundle.rs @@ -0,0 +1,1125 @@ +//! Host-side portable bundles; preparation is deliberately never cached. +use crate::{ + build::TargetOutputs, + config::Config, + error::Error, + package::{self, Artifact, ArtifactContents}, + policy, + project::SelectedPackage, + runner::Runner, +}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::{BTreeMap, BTreeSet}, + fs, + io::Read, + path::{Path, PathBuf}, +}; + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub struct Bundle { + pub name: String, + #[serde(default)] + pub targets: Vec, + pub formats: BTreeMap, + pub archive_name: String, + pub root: String, + pub cargo_binaries: Selection, + #[serde(default)] + pub files: Vec, + pub prepare: Option, + pub smoke_test: Option, +} +#[derive(Debug, Clone, Copy, Deserialize, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Format { + Tgz, + Zip, +} +#[derive(Debug, Clone, Deserialize)] +#[serde(untagged)] +pub enum Selection { + All(String), + Names(Vec), +} +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub struct Inclusion { + source: String, + destination: String, + #[serde(default)] + workspace_relative: bool, + #[serde(default)] + kind: Kind, +} +#[derive(Debug, Clone, Copy, Default, Deserialize, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Kind { + #[default] + File, + Executable, +} +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub struct Hook { + pub command: Vec, + #[serde(default)] + pub inputs: Vec, + pub timeout_seconds: u64, + #[serde(default)] + pub required_destinations: Vec, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ResultManifest { + schema_version: u32, + target: String, + files: Vec, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Prepared { + source: String, + destination: String, + kind: Kind, + sha256: String, +} +#[derive(Debug, Clone, Serialize)] +pub struct Entry { + pub destination: String, + pub sha256: String, + pub mode: u32, +} +struct Input { + entry: Entry, + source: PathBuf, +} + +fn fail(message: impl Into) -> Result { + Err(Error(message.into())) +} +fn windows(target: &str) -> bool { + target.split('-').any(|x| x == "windows") +} +fn render(s: &str, b: &Bundle, p: &SelectedPackage, target: &str) -> Result { + let mut s = s.to_owned(); + for (key, value) in [ + ("name", p.name.as_str()), + ("version", &p.version), + ("bundle", &b.name), + ("target", target), + ("binary-ext", if windows(target) { ".exe" } else { "" }), + ] { + s = s.replace(&format!("{{{key}}}"), value); + } + if s.contains(['{', '}']) { + return fail(format!("Unknown template placeholder: {s}")); + } + Ok(s) +} +fn safe(path: &str, win: bool) -> Result<(), Error> { + if path.is_empty() || path.contains('\\') || path.starts_with('/') { + return fail(format!("Unsafe bundle path: {path:?}")); + } + for part in path.split('/') { + if part.is_empty() + || part == "." + || part == ".." + || part.chars().any(|c| c.is_control() || c == ':') + { + return fail(format!("Unsafe bundle path: {path:?}")); + } + if win { + let base = part.split('.').next().unwrap().to_uppercase(); + if part.ends_with([' ', '.']) + || part.contains(['<', '>', '"', '|', '?', '*']) + || ["CON", "PRN", "AUX", "NUL", "CONIN$", "CONOUT$"].contains(&base.as_str()) + || ["COM", "LPT"].iter().any(|prefix| { + base.strip_prefix(prefix).is_some_and(|n| { + ["1", "2", "3", "4", "5", "6", "7", "8", "9", "¹", "²", "³"].contains(&n) + }) + }) + { + return fail(format!("Invalid Windows destination: {path}")); + } + } + } + Ok(()) +} +fn component(path: &str, win: bool) -> Result<(), Error> { + safe(path, win)?; + if path.contains('/') { + return fail("Archive name and root must be single path components"); + } + Ok(()) +} +fn source(root: &Path, relative: &str) -> Result { + safe(relative, false)?; + if fs::symlink_metadata(root)?.file_type().is_symlink() { + return fail("Symlinked input root rejected"); + } + let mut path = root.to_path_buf(); + for part in relative.split('/') { + path.push(part); + let m = fs::symlink_metadata(&path)?; + if m.file_type().is_symlink() { + return fail(format!("Symlink input rejected: {}", path.display())); + } + } + Ok(path) +} +fn regular(path: &Path) -> Result<(), Error> { + let m = fs::symlink_metadata(path)?; + if !m.is_file() || m.file_type().is_symlink() { + return fail(format!("Not a regular bundle input: {}", path.display())); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + if m.nlink() != 1 { + return fail(format!( + "Hardlinked bundle input rejected: {}", + path.display() + )); + } + } + Ok(()) +} +fn selected<'a>(b: &Bundle, p: &'a SelectedPackage) -> Result, Error> { + match &b.cargo_binaries { + Selection::All(s) if s == "all" => Ok(p.binaries.iter().collect()), + Selection::Names(names) if !names.is_empty() => { + let mut result = Vec::new(); + for name in names { + let n = p + .binaries + .iter() + .find(|n| *n == name) + .ok_or_else(|| Error(format!("Unknown Cargo binary {name}")))?; + if result.contains(&n) { + return fail("Duplicate Cargo binary selection"); + } + result.push(n); + } + Ok(result) + } + _ => fail("cargo-binaries must be 'all' or a nonempty array of declared binary names"), + } +} + +pub fn plan(config: &Config, p: &SelectedPackage, workspace: &Path) -> Result<(), Error> { + let mut names = BTreeSet::new(); + let mut assets = BTreeSet::from(["SHA256SUMS".to_owned(), "artifact-manifest.json".to_owned()]); + for target in &config.targets { + if config.package_format.archives() { + assets.insert(policy::archive_name(&p.name, target, &p.version)); + } + if config.package_format.binaries() { + for name in &p.binaries { + assets.insert(policy::binary_asset_name(name, target, &p.version)); + } + } + } + for b in &config.bundles { + component(&b.name, true)?; + if b.targets + .iter() + .chain(b.formats.keys()) + .any(|t| !crate::config::valid_target(t)) + { + return fail(format!("Invalid target triple in bundle {}", b.name)); + } + if b.smoke_test + .as_ref() + .is_some_and(|h| !h.required_destinations.is_empty()) + { + return fail("required-destinations belongs to prepare, not smoke-test"); + } + if !names.insert(&b.name) { + return fail("Duplicate bundle name"); + } + selected(b, p)?; + if !b.targets.is_empty() && b.formats.keys().any(|t| !b.targets.contains(t)) { + return fail(format!( + "Bundle {} has a format outside its target filter", + b.name + )); + } + for h in [&b.prepare, &b.smoke_test].into_iter().flatten() { + if h.command.is_empty() || h.command[0].is_empty() || h.timeout_seconds == 0 { + return fail("Hook requires a nonempty command and positive timeout-seconds"); + } + for input in &h.inputs { + let (base, relative) = if let Some(relative) = input.strip_prefix("workspace:") { + (workspace, relative) + } else { + (p.manifest.parent().unwrap(), input.as_str()) + }; + regular(&source(base, relative)?)?; + } + } + for target in &config.targets { + if !b.targets.is_empty() && !b.targets.contains(target) { + continue; + } + let format = b + .formats + .get(target) + .ok_or_else(|| Error(format!("Bundle {} has no format for {target}", b.name)))?; + let root = render(&b.root, b, p, target)?; + component(&root, windows(target))?; + let name = archive_name(b, p, target, *format)?; + if !assets.insert(name.clone()) { + return fail(format!("Artifact filename collision: {name}")); + } + let mut inputs = BTreeMap::new(); + for binary in selected(b, p)? { + insert_destination( + &mut inputs, + &policy::binary_filename(binary, target), + windows(target), + )?; + } + for f in &b.files { + let dest = render(&f.destination, b, p, target)?; + let path = source( + if f.workspace_relative { + workspace + } else { + p.manifest.parent().unwrap() + }, + &f.source, + )?; + validate_tree(&path, &dest, windows(target), &mut inputs)?; + } + println!( + "Bundle {} / {target}: {name}, root {root}, format {format:?}; static files {:?}", + b.name, b.files + ); + for h in [&b.prepare, &b.smoke_test].into_iter().flatten() { + for required in &h.required_destinations { + safe(&render(required, b, p, target)?, windows(target))?; + } + println!( + " host command {:?}; required {:?}; dynamic results unresolved", + h.command, h.required_destinations + ); + } + } + } + Ok(()) +} +fn archive_name(b: &Bundle, p: &SelectedPackage, t: &str, f: Format) -> Result { + let n = render(&b.archive_name, b, p, t)?; + component(&n, windows(t))?; + Ok(format!( + "{n}.{}", + match f { + Format::Tgz => "tgz", + Format::Zip => "zip", + } + )) +} +fn insert_destination(map: &mut BTreeMap, dest: &str, win: bool) -> Result<(), Error> { + safe(dest, win)?; + let key = if win { + dest.to_lowercase() + } else { + dest.into() + }; + if map.keys().any(|k| { + k == &key || k.starts_with(&format!("{key}/")) || key.starts_with(&format!("{k}/")) + }) { + return fail(format!("Bundle destination collision: {dest}")); + } + map.insert(key, ()); + Ok(()) +} +fn validate_tree( + path: &Path, + dest: &str, + win: bool, + map: &mut BTreeMap, +) -> Result<(), Error> { + safe(dest, win)?; + let m = fs::symlink_metadata(path)?; + if m.is_dir() { + for child in fs::read_dir(path)? { + let child = child?; + let name = child + .file_name() + .into_string() + .map_err(|_| Error("Non-UTF8 input filename".into()))?; + validate_tree(&child.path(), &format!("{dest}/{name}"), win, map)?; + } + } else { + regular(path)?; + insert_destination(map, dest, win)?; + } + Ok(()) +} +fn add( + inputs: &mut Vec, + path: &Path, + dest: &str, + kind: Kind, + target: &str, + expected: Option<&str>, + map: &mut BTreeMap, +) -> Result<(), Error> { + regular(path)?; + insert_destination(map, dest, windows(target))?; + let hash = package::checksum(path)?; + if expected.is_some_and(|s| s != hash) { + return fail(format!("SHA-256 mismatch: {dest}")); + } + if matches!(kind, Kind::Executable) { + architecture(path, target)?; + } + inputs.push(Input { + source: path.into(), + entry: Entry { + destination: dest.into(), + sha256: hash, + mode: if matches!(kind, Kind::Executable) { + 0o755 + } else { + 0o644 + }, + }, + }); + Ok(()) +} +fn add_tree( + inputs: &mut Vec, + path: &Path, + dest: &str, + kind: Kind, + target: &str, + map: &mut BTreeMap, +) -> Result<(), Error> { + let m = fs::symlink_metadata(path)?; + if m.is_dir() { + for child in fs::read_dir(path)? { + let child = child?; + let name = child + .file_name() + .into_string() + .map_err(|_| Error("Non-UTF8 filename".into()))?; + add_tree( + inputs, + &child.path(), + &format!("{dest}/{name}"), + kind, + target, + map, + )?; + } + Ok(()) + } else { + add(inputs, path, dest, kind, target, None, map) + } +} +fn architecture(path: &Path, target: &str) -> Result<(), Error> { + use std::io::{Seek, SeekFrom}; + let arch = target.split('-').next().unwrap_or(""); + let mut f = fs::File::open(path)?; + let mut h = [0u8; 64]; + f.read_exact(&mut h)?; + let valid = if windows(target) && &h[..2] == b"MZ" { + let offset = u32::from_le_bytes(h[60..64].try_into().unwrap()); + f.seek(SeekFrom::Start(offset.into()))?; + let mut pe = [0; 6]; + f.read_exact(&mut pe)?; + &pe[..4] == b"PE\0\0" + && Some(u16::from_le_bytes([pe[4], pe[5]])) + == match arch { + "x86_64" => Some(0x8664), + "aarch64" => Some(0xaa64), + "i686" => Some(0x14c), + _ => None, + } + } else if target.contains("linux") && &h[..4] == b"\x7fELF" && h[5] == 1 { + h[4] == if arch == "i686" { 1 } else { 2 } + && Some(u16::from_le_bytes([h[18], h[19]])) + == match arch { + "x86_64" => Some(62), + "aarch64" => Some(183), + "i686" => Some(3), + "riscv64gc" => Some(243), + _ => None, + } + } else { + false + }; + if !valid { + return fail(format!( + "Executable format/architecture mismatch or unsupported target {target}: {}", + path.display() + )); + } + Ok(()) +} + +pub fn assemble( + workspace: &Path, + p: &SelectedPackage, + config: &Config, + outputs: &[TargetOutputs], + commit: &str, + dirty: bool, + runner: &Runner, +) -> Result<(Vec, PathBuf), Error> { + // The outer temporary directory removes all partial artifacts on any failure. + let dist = workspace.join("dist"); + if dist.is_symlink() { + return fail("Symlinked dist rejected"); + } + fs::create_dir_all(&dist)?; + let stage = tempfile::Builder::new() + .prefix("bundles-") + .tempdir_in(&dist)?; + let (mut artifacts, old_sums) = package::package( + stage.path(), + &p.name, + &p.version, + outputs, + config.package_format, + )?; + let artifact_dir = old_sums.parent().unwrap().to_path_buf(); + let mut records = Vec::new(); + for b in &config.bundles { + for output in outputs { + let target = &output.target; + if !b.targets.is_empty() && !b.targets.contains(target) { + continue; + } + let format = *b + .formats + .get(target) + .ok_or_else(|| Error(format!("Missing bundle format for {target}")))?; + let root = render(&b.root, b, p, target)?; + let mut inputs = Vec::new(); + let mut map = BTreeMap::new(); + for binary in selected(b, p)? { + let path = &output + .binaries + .iter() + .find(|(n, _)| n == binary) + .ok_or_else(|| Error(format!("Missing built binary {binary}")))? + .1; + // Cargo output is already validated by the build layer; check its target too. + add( + &mut inputs, + path, + &policy::binary_filename(binary, target), + Kind::Executable, + target, + None, + &mut map, + )?; + } + for f in &b.files { + let path = source( + if f.workspace_relative { + workspace + } else { + p.manifest.parent().unwrap() + }, + &f.source, + )?; + add_tree( + &mut inputs, + &path, + &render(&f.destination, b, p, target)?, + f.kind, + target, + &mut map, + )?; + } + let hook_stage = tempfile::tempdir_in(stage.path())?; + let output_dir = hook_stage.path().join("output"); + fs::create_dir(&output_dir)?; + let cache = p + .target_directory + .join("rust-release-bundle-cache") + .join(&b.name) + .join(target); + fs::create_dir_all(&cache)?; + let context = hook_stage.path().join("context.json"); + let manifest = hook_stage.path().join("result.json"); + let binaries = output + .binaries + .iter() + .map(|(n, path)| Ok((n.clone(), fs::canonicalize(path)?))) + .collect::, Error>>()?; + let tree = hook_stage.path().join("tree"); + fs::create_dir(&tree)?; + let value = serde_json::json!({"schema_version":1,"package_name":p.name,"package_version":p.version,"target":target,"profile":config.profile,"binary_ext":if windows(target){".exe"}else{""},"workspace_root":fs::canonicalize(workspace)?,"package_root":fs::canonicalize(p.manifest.parent().unwrap())?,"source_commit":commit,"source_dirty":dirty,"binaries":binaries,"output_directory":fs::canonicalize(&output_dir)?,"cache_directory":fs::canonicalize(&cache)?,"manifest_path":manifest,"bundle":b.name,"bundle_directory":tree}); + fs::write( + &context, + serde_json::to_vec_pretty(&value).map_err(|e| Error(e.to_string()))?, + )?; + if let Some(h) = &b.prepare { + runner.hook( + &h.command, + p.manifest.parent().unwrap(), + (&context, &manifest, &tree), + h.timeout_seconds, + &format!("{target}/{}", b.name), + )?; + regular(&manifest)?; + let result: ResultManifest = serde_json::from_slice(&fs::read(&manifest)?) + .map_err(|e| Error(format!("Invalid hook manifest: {e}")))?; + if result.schema_version != 1 || result.target != *target { + return fail("Hook manifest schema_version/target mismatch"); + } + for f in result.files { + let path = source(&output_dir, &f.source)?; + add( + &mut inputs, + &path, + &f.destination, + f.kind, + target, + Some(&f.sha256), + &mut map, + )?; + } + for required in &h.required_destinations { + let required = render(required, b, p, target)?; + if !inputs.iter().any(|i| i.entry.destination == required) { + return fail(format!("Missing required hook destination {required}")); + } + } + } + inputs.sort_by(|a, b| a.entry.destination.cmp(&b.entry.destination)); + for input in &inputs { + let dest = tree.join(&input.entry.destination); + fs::create_dir_all(dest.parent().unwrap())?; + fs::copy(&input.source, &dest)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&dest, fs::Permissions::from_mode(input.entry.mode))?; + } + if package::checksum(&dest)? != input.entry.sha256 { + return fail("Input changed during staging"); + } + } + let smoke = if let Some(h) = &b.smoke_test { + let host = runner + .text("rustc", &["-vV"], workspace)? + .lines() + .find_map(|l| l.strip_prefix("host: ").map(str::to_owned)) + .unwrap_or_default(); + if host == *target { + runner.hook( + &h.command, + p.manifest.parent().unwrap(), + (&context, &manifest, &tree), + h.timeout_seconds, + &format!("{target}/{}/smoke", b.name), + )?; + "passed" + } else { + "skipped: target differs from rustc host" + } + } else { + "skipped: no native smoke-test configured" + }; + // Smoke tests must not silently alter the declared tree. + let entries = inputs.iter().map(|i| i.entry.clone()).collect::>(); + let path = artifact_dir.join(archive_name(b, p, target, format)?); + write_archive(&path, &tree, &root, format, &entries)?; + let artifact = Artifact { + target: target.clone(), + sha256: package::checksum(&path)?, + path, + contents: ArtifactContents::Bundle { + root: root.clone(), + format, + entries: entries.clone(), + }, + }; + verify(&artifact.path, &root, format, &entries)?; + println!( + "{}: portable tree/hash/architecture checks passed; native runtime {smoke}", + artifact.path.display() + ); + records.push(serde_json::json!({"bundle":b.name,"target":target,"filename":artifact.path.file_name(),"sha256":artifact.sha256,"root":root,"files":entries,"checks":{"portable_tree":"passed","hashes":"passed","architecture":"passed","native_runtime":smoke}})); + artifacts.push(artifact); + } + } + for a in &artifacts { + if matches!(a.contents, ArtifactContents::Archive { .. }) { + crate::verify::verify_archive(&a.path, a)?; + } + if !matches!(a.contents, ArtifactContents::Bundle { .. }) { + records.push(serde_json::json!({"target":a.target,"filename":a.path.file_name(),"sha256":a.sha256,"checks":{"native_runtime":"skipped","cargo_binstall":"deferred until publication"}})); + } + } + let receipt = artifact_dir.join("artifact-manifest.json"); + fs::write( + &receipt, + serde_json::to_vec_pretty(&serde_json::json!({"schema_version":1,"artifacts":records})) + .map_err(|e| Error(e.to_string()))?, + )?; + artifacts.push(Artifact { + target: "manifest".into(), + sha256: package::checksum(&receipt)?, + path: receipt, + contents: ArtifactContents::Binary, + }); + artifacts.sort_by(|a, b| a.path.file_name().cmp(&b.path.file_name())); + fs::write( + &old_sums, + artifacts + .iter() + .map(|a| { + format!( + "{} {}\n", + a.sha256, + a.path.file_name().unwrap().to_string_lossy() + ) + }) + .collect::(), + )?; + // Stable local discovery entry; keep public manifests free of host paths. + let mut locator = tempfile::NamedTempFile::new_in(&dist)?; + use std::io::Write as _; + locator.write_all(&serde_json::to_vec_pretty(&serde_json::json!({ + "schema_version": 1, + "manifest": artifact_dir.join("artifact-manifest.json").strip_prefix(&dist).map_err(|e| Error(e.to_string()))?, + "checksums": old_sums.strip_prefix(&dist).map_err(|e| Error(e.to_string()))? + })).map_err(|e| Error(e.to_string()))?)?; + locator + .persist(dist.join("latest-artifacts.json")) + .map_err(|e| Error(e.to_string()))?; + let _ = stage.keep(); + Ok((artifacts, old_sums)) +} +fn zip_error(e: zip::result::ZipError) -> Error { + Error(e.to_string()) +} +fn write_archive( + path: &Path, + tree: &Path, + root: &str, + format: Format, + entries: &[Entry], +) -> Result<(), Error> { + let temporary = tempfile::NamedTempFile::new_in(path.parent().unwrap())?; + match format { + Format::Tgz => { + let gz = flate2::GzBuilder::new() + .mtime(0) + .operating_system(255) + .write(temporary.reopen()?, flate2::Compression::default()); + let mut tar = tar::Builder::new(gz); + let mut h = tar::Header::new_ustar(); + h.set_entry_type(tar::EntryType::Directory); + h.set_mode(0o755); + h.set_size(0); + h.set_mtime(0); + h.set_uid(0); + h.set_gid(0); + h.set_cksum(); + tar.append_data(&mut h, root, std::io::empty())?; + for e in entries { + let source = source(tree, &e.destination)?; + regular(&source)?; + if package::checksum(&source)? != e.sha256 { + return fail("Staged content changed"); + } + let mut h = tar::Header::new_ustar(); + h.set_size(fs::metadata(&source)?.len()); + h.set_mode(e.mode); + h.set_uid(0); + h.set_gid(0); + h.set_mtime(0); + h.set_cksum(); + tar.append_data( + &mut h, + format!("{root}/{}", e.destination), + fs::File::open(source)?, + )?; + } + tar.into_inner()?.finish()?; + } + Format::Zip => { + let mut zip = zip::ZipWriter::new(temporary.reopen()?); + for e in entries { + let source = source(tree, &e.destination)?; + regular(&source)?; + if package::checksum(&source)? != e.sha256 { + return fail("Staged content changed"); + } + let options = zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Deflated) + .last_modified_time(zip::DateTime::default()) + .unix_permissions(e.mode); + zip.start_file(format!("{root}/{}", e.destination), options) + .map_err(zip_error)?; + std::io::copy(&mut fs::File::open(source)?, &mut zip)?; + } + zip.finish().map_err(zip_error)?; + } + } + temporary.persist(path).map_err(|e| Error(e.to_string()))?; + Ok(()) +} +pub fn verify(path: &Path, root: &str, format: Format, entries: &[Entry]) -> Result<(), Error> { + use sha2::{Digest, Sha256}; + let mut seen = BTreeSet::new(); + let mut check = |name: String, mode: u32, reader: &mut dyn Read| -> Result<(), Error> { + let dest = name + .strip_prefix(&format!("{root}/")) + .ok_or_else(|| Error("Unexpected archive root".into()))?; + let expected = entries + .iter() + .find(|e| e.destination == dest) + .ok_or_else(|| Error(format!("Unexpected archive member {name}")))?; + if !seen.insert(dest.to_owned()) || mode & 0o777 != expected.mode { + return fail("Duplicate archive member or incorrect permissions"); + } + let mut hash = Sha256::new(); + let mut buf = [0; 65536]; + loop { + let n = reader.read(&mut buf)?; + if n == 0 { + break; + } + hash.update(&buf[..n]); + } + if format!("{:x}", hash.finalize()) != expected.sha256 { + return fail("Archive member SHA-256 mismatch"); + } + Ok(()) + }; + match format { + Format::Tgz => { + let mut tar = tar::Archive::new(flate2::read::GzDecoder::new(fs::File::open(path)?)); + let mut root_seen = false; + for entry in tar.entries()? { + let mut entry = entry?; + let name = entry + .path()? + .to_str() + .ok_or_else(|| Error("Non-UTF8 archive path".into()))? + .to_owned(); + if entry.header().entry_type().is_dir() && name == root && !root_seen { + root_seen = true; + continue; + } + if !entry.header().entry_type().is_file() { + return fail("Unexpected non-file archive member"); + } + check(name, entry.header().mode()?, &mut entry)?; + } + if !root_seen { + return fail("Missing archive root"); + } + } + Format::Zip => { + let mut zip = zip::ZipArchive::new(fs::File::open(path)?).map_err(zip_error)?; + for i in 0..zip.len() { + let mut entry = zip.by_index(i).map_err(zip_error)?; + let mode = entry.unix_mode().unwrap_or(0); + if mode & 0o170000 != 0o100000 { + return fail("Non-regular ZIP member"); + } + check(entry.name().into(), mode, &mut entry)?; + } + } + } + if seen.len() != entries.len() { + return fail("Missing archive members"); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + fn fixture() -> ( + tempfile::TempDir, + SelectedPackage, + Config, + Vec, + ) { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + fs::create_dir(root.join("member")).unwrap(); + fs::write(root.join("member/LICENSE"), "license").unwrap(); + fs::write(root.join("member/README.txt"), "portable instructions").unwrap(); + let mut outputs = Vec::new(); + for target in ["x86_64-unknown-linux-gnu", "x86_64-pc-windows-gnu"] { + let path = root.join(format!("custom output é {target}")); + let mut bytes = vec![0; 128]; + if windows(target) { + bytes[..2].copy_from_slice(b"MZ"); + bytes[60] = 64; + bytes[64..68].copy_from_slice(b"PE\0\0"); + bytes[68..70].copy_from_slice(&0x8664u16.to_le_bytes()); + } else { + bytes[..4].copy_from_slice(b"\x7fELF"); + bytes[4] = 2; + bytes[5] = 1; + bytes[18] = 62; + } + fs::write(&path, bytes).unwrap(); + outputs.push(TargetOutputs { + target: target.into(), + binaries: vec![("demo".into(), path.clone()), ("gui".into(), path)], + }); + } + let p = SelectedPackage { + workspace_root: root.into(), + name: "demo".into(), + version: "1.2.3".into(), + binaries: vec!["demo".into(), "gui".into()], + manifest: root.join("member/Cargo.toml"), + repository: None, + target_directory: root.join("custom-target"), + }; + fs::write(root.join("member/prepare.py"),r#"import json, sys, pathlib, hashlib, shutil, os +c = json.loads(pathlib.Path(sys.argv[1]).read_text()) +assert c['schema_version'] == 1 and c['profile'] == 'small' +assert c['package_version'] == '1.2.3' and c['source_dirty'] +assert pathlib.Path(c['package_root']).name == 'member' +assert pathlib.Path(c['binaries']['demo']).is_absolute() +assert 'RELEASE_TEST_TOKEN' not in os.environ +out = pathlib.Path(c['output_directory']) +files = [] +for name, data, kind in [('runtime'+c['binary_ext'], pathlib.Path(c['binaries']['demo']).read_bytes(), 'executable'), ('LICENSE.txt', b'upstream license', 'file'), ('provenance.json', b'{"release_id":123}', 'file')]: + (out / name).write_bytes(data) + files.append(dict(source=name, destination=name if kind == 'executable' else 'LICENSES/runtime/'+name, kind=kind, sha256=hashlib.sha256(data).hexdigest())) +(out / 'undeclared-cache').write_text('must not be archived') +pathlib.Path(c['manifest_path']).write_text(json.dumps(dict(schema_version=1, target=c['target'], files=files))) +"#).unwrap(); + let bundle:Bundle=toml::from_str(r#" +name='portable' +targets=['x86_64-unknown-linux-gnu','x86_64-pc-windows-gnu'] +formats={x86_64-unknown-linux-gnu='tgz',x86_64-pc-windows-gnu='zip'} +archive-name='{name}-{bundle}-{target}-v{version}' +root='{name}-{bundle}-{target}-v{version}' +cargo-binaries='all' +files=[{source='LICENSE',destination='LICENSES/app.txt'},{source='README.txt',destination='README.txt'}] +[prepare] +command=['python3','prepare.py','{context-file}'] +inputs=['prepare.py'] +timeout-seconds=10 +required-destinations=['runtime{binary-ext}','LICENSES/runtime/LICENSE.txt','LICENSES/runtime/provenance.json'] +"#).unwrap(); + let config = Config { + bundles: vec![bundle], + features: Vec::new(), + apple: crate::config::AppleConfig::default(), + targets: outputs.iter().map(|o| o.target.clone()).collect(), + profile: "small".into(), + locked: true, + tag_prefix: "v".into(), + token_env: "RELEASE_TEST_TOKEN".into(), + package_format: crate::config::PackageFormat::Tgz, + repository_fallbacks: Vec::new(), + }; + (temp, p, config, outputs) + } + #[test] + fn complete_archives_are_deterministic_and_manifest_is_public() { + let (temp, p, c, outputs) = fixture(); + plan(&c, &p, temp.path()).unwrap(); + assert!(!p.target_directory.exists()); // planning does not run preparation + let runner = Runner::new("RELEASE_TEST_TOKEN"); + let (first, sums) = assemble(temp.path(), &p, &c, &outputs, "abc", true, &runner).unwrap(); + let (second, _) = assemble(temp.path(), &p, &c, &outputs, "abc", true, &runner).unwrap(); + assert_eq!(first.len(), 5); + assert_eq!(fs::read_to_string(sums).unwrap().lines().count(), 5); + for (a, b) in first.iter().zip(second.iter()) { + assert_eq!(a.sha256, b.sha256); + if let ArtifactContents::Bundle { + root, + format, + entries, + } = &a.contents + { + assert_eq!(entries.len(), 7); + verify(&a.path, root, *format, entries).unwrap(); + assert!(!entries.iter().any(|e| e.destination.contains("undeclared"))); + } + } + let receipt = first.iter().find(|a| a.target == "manifest").unwrap(); + let text = fs::read_to_string(&receipt.path).unwrap(); + assert!(!text.contains(&temp.path().display().to_string())); + fs::write(temp.path().join("member/LICENSE"), "new license").unwrap(); + let (third, _) = assemble(temp.path(), &p, &c, &outputs, "abc", true, &runner).unwrap(); + assert_ne!( + first + .iter() + .find(|a| matches!(a.contents, ArtifactContents::Bundle { .. })) + .unwrap() + .sha256, + third + .iter() + .find(|a| matches!(a.contents, ArtifactContents::Bundle { .. })) + .unwrap() + .sha256 + ); + } + #[test] + fn invalid_paths_collisions_links_and_executables_are_rejected() { + for path in [ + "/a", + "../a", + "a/../b", + "a\\b", + "C:/a", + "a//b", + "NUL.txt", + "a/COM1.exe", + "x.", + "x ", + ] { + assert!(safe(path, true).is_err(), "{path}"); + } + let mut map = BTreeMap::new(); + insert_destination(&mut map, "Demo.exe", true).unwrap(); + assert!(insert_destination(&mut map, "demo.EXE", true).is_err()); + assert!(insert_destination(&mut map, "Demo.exe/child", true).is_err()); + let (temp, _, _, outputs) = fixture(); + assert!(architecture(&outputs[0].binaries[0].1, &outputs[1].target).is_err()); + #[cfg(unix)] + { + let link = temp.path().join("link"); + std::os::unix::fs::symlink(&outputs[0].binaries[0].1, &link).unwrap(); + assert!(source(temp.path(), "link").is_err()); + fs::remove_file(&link).unwrap(); + fs::hard_link(&outputs[0].binaries[0].1, &link).unwrap(); + assert!(regular(&link).is_err()); + } + } + #[test] + fn failed_hooks_clean_partial_artifacts_and_filter_targets() { + let (temp, p, mut c, outputs) = fixture(); + let runner = Runner::new("RELEASE_TEST_TOKEN"); + c.targets.truncate(1); + plan(&c, &p, temp.path()).unwrap(); + let (artifacts, _) = + assemble(temp.path(), &p, &c, &outputs[..1], "abc", true, &runner).unwrap(); + assert!(!artifacts.iter().any(|a| windows(&a.target))); + let baseline = fs::read_dir(temp.path().join("dist")).unwrap().count(); + let original = fs::read_to_string(temp.path().join("member/prepare.py")).unwrap(); + for tail in ["raise SystemExit(7)","pathlib.Path(c['manifest_path']).write_text('bad json')", "files[0]['sha256']='0'*64; pathlib.Path(c['manifest_path']).write_text(json.dumps(dict(schema_version=1,target=c['target'],files=files)))", "files[0]['destination']='demo'; pathlib.Path(c['manifest_path']).write_text(json.dumps(dict(schema_version=1,target=c['target'],files=files)))", "pathlib.Path(c['manifest_path']).write_text(json.dumps(dict(schema_version=1,target=c['target'],files=[])))", "import time; time.sleep(5)"] { + fs::write(temp.path().join("member/prepare.py"),format!("{original}\n{tail}\n")).unwrap();c.bundles[0].prepare.as_mut().unwrap().timeout_seconds=1; + assert!(assemble(temp.path(),&p,&c,&outputs[..1],"abc",true,&runner).is_err(),"{tail}");assert_eq!(fs::read_dir(temp.path().join("dist")).unwrap().count(),baseline); + } + } + #[test] + fn published_bundle_downloads_validate_full_contents() { + use std::{ + io::{Read, Write}, + net::TcpListener, + thread, + }; + let (temp, p, c, outputs) = fixture(); + let (artifacts, sums) = assemble( + temp.path(), + &p, + &c, + &outputs, + "abc", + true, + &Runner::new("RELEASE_TEST_TOKEN"), + ) + .unwrap(); + let bodies = artifacts + .iter() + .map(|a| fs::read(&a.path).unwrap()) + .chain(std::iter::once(fs::read(&sums).unwrap())) + .collect::>(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let repo = format!("http://{}/team/demo", listener.local_addr().unwrap()); + let server = thread::spawn(move || { + for body in bodies { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0; 4096]; + let _ = stream.read(&mut request).unwrap(); + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .unwrap(); + stream.write_all(&body).unwrap(); + } + }); + crate::verify::verify_published(&repo, "v1.2.3", &artifacts, &sums, None).unwrap(); + server.join().unwrap(); + } + #[test] + fn native_smoke_checks_are_explicit_and_cannot_change_payloads() { + let (temp, p, mut c, outputs) = fixture(); + let host = Runner::new("TOKEN") + .text("rustc", &["-vV"], temp.path()) + .unwrap(); + if !host.lines().any(|l| l == "host: x86_64-unknown-linux-gnu") { + return; + } + c.bundles[0].smoke_test = Some(Hook { + command: vec![ + "python3".into(), + "-c".into(), + "import pathlib,sys; assert (pathlib.Path(sys.argv[1])/'runtime').is_file()".into(), + "{bundle-directory}".into(), + ], + inputs: vec![], + timeout_seconds: 10, + required_destinations: vec![], + }); + let (artifacts, _) = assemble( + temp.path(), + &p, + &c, + &outputs, + "abc", + true, + &Runner::new("TOKEN"), + ) + .unwrap(); + let manifest = fs::read_to_string( + &artifacts + .iter() + .find(|a| a.target == "manifest") + .unwrap() + .path, + ) + .unwrap(); + assert!(manifest.contains("skipped: target differs")); + assert!(manifest.contains(r#""native_runtime": "passed""#)); + c.bundles[0].smoke_test.as_mut().unwrap().command[2] = + "import pathlib,sys; (pathlib.Path(sys.argv[1])/'runtime').write_text('changed')" + .into(); + assert!(assemble( + temp.path(), + &p, + &c, + &outputs[..1], + "abc", + true, + &Runner::new("TOKEN") + ) + .is_err()); + } + #[test] + fn config_rejects_ambiguous_artifacts_and_missing_formats() { + let (temp, p, mut c, _) = fixture(); + c.bundles[0].archive_name = "{name}-{target}-v{version}".into(); + assert!(plan(&c, &p, temp.path()).is_err()); + c.bundles[0].archive_name = "{name}-{bundle}-{target}".into(); + c.bundles[0].formats.clear(); + assert!(plan(&c, &p, temp.path()).is_err()); + assert!(toml::from_str::("name='x'\nunknown=true").is_err()); + } +} diff --git a/src/cli.rs b/src/cli.rs index 888979f..61ab55f 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -27,6 +27,9 @@ pub struct Cli { pub target: Vec, #[arg(long)] pub profile: Option, + /// Enable Cargo features for source checks and every release backend. + #[arg(long, value_delimiter = ',')] + pub features: Vec, #[arg(long, conflicts_with = "unlocked")] pub locked: bool, #[arg(long)] @@ -142,6 +145,7 @@ pub fn run(mut args: Cli) -> Result<(), Error> { unlocked: args.unlocked, tag_prefix: args.tag_prefix, package_format: args.artifact_format, + features: args.features, }, )?; let runner = Runner::new(&config.token_env) @@ -152,6 +156,7 @@ pub fn run(mut args: Cli) -> Result<(), Error> { .to_str() .ok_or_else(|| Error("Project path is not UTF-8.".into()))?; let selected = project::inspect(root, args.package.as_deref(), &runner)?; + crate::bundle::plan(&config, &selected, &selected.workspace_root)?; let publishing = !args.prepare_repo && !args.dry_run && args @@ -267,6 +272,14 @@ pub fn run(mut args: Cli) -> Result<(), Error> { return Ok(()); } for target in &config.targets { + if build::is_apple(target) { + println!( + "{target} build backend: cargo-zigbuild container {} (SDKROOT={}, MACOSX_DEPLOYMENT_TARGET={})", + config.apple.image, config.apple.sdk_root, config.apple.deployment_target + ); + } else { + println!("{target} build backend: Cross"); + } if config.package_format.archives() { println!( "{target} archive: {} ({})", @@ -350,7 +363,8 @@ pub fn run(mut args: Cli) -> Result<(), Error> { if args.stop_after == Some(Stage::Verify) { return Ok(()); } - let engine = engine.ok_or_else(|| Error("No usable Cross container backend.".into()))?; + let engine = + engine.ok_or_else(|| Error("No usable Docker or Podman container backend.".into()))?; let commit = project::head_commit(&checkout.root, &runner)?; let checkout_clean = runner .text( @@ -371,13 +385,41 @@ pub fn run(mut args: Cli) -> Result<(), Error> { if args.stop_after == Some(Stage::Build) { return Ok(()); } - let (artifacts, sums) = crate::package::package( - &checkout.root, - &selected.name, - &selected.version, - &outputs, - config.package_format, - )?; + let (artifacts, sums) = if config.bundles.is_empty() { + crate::package::package( + &checkout.root, + &selected.name, + &selected.version, + &outputs, + config.package_format, + )? + } else { + crate::bundle::assemble( + &selected.workspace_root, + &selected, + &config, + &outputs, + &commit, + !checkout_clean, + &runner, + )? + }; + for artifact in &artifacts { + if matches!( + artifact.contents, + crate::package::ArtifactContents::Archive { .. } + | crate::package::ArtifactContents::Bundle { .. } + ) { + verify::verify_archive(&artifact.path, artifact)?; + } + if build::is_apple(&artifact.target) { + verify::verify_apple_artifact( + &artifact.path, + artifact, + &config.apple.deployment_target, + )?; + } + } ui::detail( "Artifacts", sums.parent().map_or_else( @@ -392,6 +434,11 @@ pub fn run(mut args: Cli) -> Result<(), Error> { artifact.path.display() )); } + for target in &config.targets { + if build::is_apple(target) { + println!("{target} checks: Mach-O architecture, macOS deployment minimum {}, system dynamic dependencies, archive layout and checksum passed; runtime skipped (no compatible macOS runner)", config.apple.deployment_target); + } + } if args.stop_after == Some(Stage::Package) { return Ok(()); } diff --git a/src/config.rs b/src/config.rs index 557526c..9390bce 100644 --- a/src/config.rs +++ b/src/config.rs @@ -7,6 +7,7 @@ use std::{fs, path::Path}; pub const PROJECT_CONFIG_FILE: &str = "rust-release.toml"; #[derive(Debug, Clone, Deserialize, Default)] +#[serde(deny_unknown_fields)] struct FileConfig { build: Option, release: Option, @@ -16,30 +17,46 @@ struct FileConfig { } #[derive(Debug, Clone, Deserialize, Default)] +#[serde(deny_unknown_fields)] struct Build { targets: Option>, profile: Option, locked: Option, + features: Option>, + apple: Option, } #[derive(Debug, Clone, Deserialize, Default)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +struct Apple { + image: Option, + deployment_target: Option, + sdk_root: Option, +} + +#[derive(Debug, Clone, Deserialize, Default)] +#[serde(deny_unknown_fields)] struct Release { #[serde(rename = "tag-prefix")] tag_prefix: Option, } #[derive(Debug, Clone, Deserialize, Default)] +#[serde(deny_unknown_fields)] struct Package { + bundles: Option>, format: Option, } #[derive(Debug, Clone, Deserialize, Default)] +#[serde(deny_unknown_fields)] struct Repository { #[serde(default)] fallbacks: Vec, } #[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] struct FileRepositoryFallback { origin: String, #[serde(rename = "fallback-url")] @@ -47,6 +64,7 @@ struct FileRepositoryFallback { } #[derive(Debug, Clone, Deserialize, Default)] +#[serde(deny_unknown_fields)] struct Gitea { #[serde(rename = "token-env")] token_env: Option, @@ -60,15 +78,37 @@ pub struct RepositoryFallback { #[derive(Debug, Clone)] pub struct Config { + pub bundles: Vec, pub targets: Vec, pub profile: String, pub locked: bool, + pub features: Vec, + pub apple: AppleConfig, pub tag_prefix: String, pub token_env: String, pub package_format: PackageFormat, pub repository_fallbacks: Vec, } +pub const DEFAULT_APPLE_IMAGE: &str = "ghcr.io/rust-cross/cargo-zigbuild@sha256:9b4f6b3eb9e8f9fefb3960fe288ddba576627d66d9eff851ce7f62d2455564a5"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AppleConfig { + pub image: String, + pub deployment_target: String, + pub sdk_root: String, +} + +impl Default for AppleConfig { + fn default() -> Self { + Self { + image: DEFAULT_APPLE_IMAGE.into(), + deployment_target: "13.0".into(), + sdk_root: "/opt/MacOSX11.3.sdk".into(), + } + } +} + impl Config { pub fn resolve_repository(&self, input: &str) -> Result { let normalized = project::normalize_repository(input)?; @@ -109,6 +149,7 @@ pub struct Overrides { pub unlocked: bool, pub tag_prefix: Option, pub package_format: Option, + pub features: Vec, } pub fn load( @@ -173,6 +214,21 @@ fn merge_build(current: &mut Option, later: Option) { if later.locked.is_some() { current.locked = later.locked; } + if later.features.is_some() { + current.features = later.features; + } + if let Some(later) = later.apple { + let apple = current.apple.get_or_insert_with(Apple::default); + if later.image.is_some() { + apple.image = later.image; + } + if later.deployment_target.is_some() { + apple.deployment_target = later.deployment_target; + } + if later.sdk_root.is_some() { + apple.sdk_root = later.sdk_root; + } + } } } @@ -188,6 +244,9 @@ fn merge_release(current: &mut Option, later: Option) { fn merge_package(current: &mut Option, later: Option) { if let Some(later) = later { let current = current.get_or_insert_with(Package::default); + if later.bundles.is_some() { + current.bundles = later.bundles; + } if later.format.is_some() { current.format = later.format; } @@ -214,6 +273,7 @@ fn merge_gitea(current: &mut Option, later: Option) { fn finish(file: FileConfig, overrides: Overrides) -> Result { let build = file.build.unwrap_or_default(); + let apple = build.apple.clone().unwrap_or_default(); let release = file.release.unwrap_or_default(); let package = file.package.unwrap_or_default(); let repository = file.repository.unwrap_or_default(); @@ -244,6 +304,7 @@ fn finish(file: FileConfig, overrides: Overrides) -> Result { }); } let mut config = Config { + bundles: package.bundles.unwrap_or_default(), targets: if overrides.targets.is_empty() { build.targets.unwrap_or_else(|| { [ @@ -266,6 +327,18 @@ fn finish(file: FileConfig, overrides: Overrides) -> Result { } else { build.locked.unwrap_or(true) }, + features: if overrides.features.is_empty() { + build.features.unwrap_or_default() + } else { + overrides.features + }, + apple: AppleConfig { + image: apple.image.unwrap_or_else(|| DEFAULT_APPLE_IMAGE.into()), + deployment_target: apple.deployment_target.unwrap_or_else(|| "13.0".into()), + sdk_root: apple + .sdk_root + .unwrap_or_else(|| "/opt/MacOSX11.3.sdk".into()), + }, tag_prefix: overrides .tag_prefix .or(release.tag_prefix) @@ -286,6 +359,19 @@ fn finish(file: FileConfig, overrides: Overrides) -> Result { } } config.targets = unique; + let mut unique_features = Vec::new(); + for feature in config.features { + if !valid_feature(&feature) { + return Err(Error(format!( + "invalid Cargo feature name in build.features: {feature:?}." + ))); + } + if !unique_features.contains(&feature) { + unique_features.push(feature); + } + } + config.features = unique_features; + validate_apple(&config.apple)?; if !valid_profile(&config.profile) { return Err(Error("build.profile must be a Cargo profile name.".into())); } @@ -307,7 +393,48 @@ fn finish(file: FileConfig, overrides: Overrides) -> Result { Ok(config) } -fn valid_target(s: &str) -> bool { +fn validate_apple(apple: &AppleConfig) -> Result<(), Error> { + let digest = apple + .image + .rsplit_once("@sha256:") + .map(|(_, digest)| digest); + if digest.is_none_or(|digest| { + digest.len() != 64 + || !digest + .bytes() + .all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()) + }) || apple.image.bytes().any(|b| b.is_ascii_whitespace()) + { + return Err(Error( + "build.apple.image must use an immutable @sha256:<64 lowercase hex> digest.".into(), + )); + } + let mut version = apple.deployment_target.split('.'); + let major = version.next().and_then(|value| value.parse::().ok()); + let minor = version.next().and_then(|value| value.parse::().ok()); + let patch = version + .next() + .map_or(Some(0), |value| value.parse::().ok()); + if major + .zip(minor) + .zip(patch) + .is_none_or(|((major, _), _)| major < 13) + || version.next().is_some() + { + return Err(Error("build.apple.deployment-target must be a macOS version at least 13.0 (for example \"13.0\").".into())); + } + if !apple.sdk_root.starts_with('/') + || apple.sdk_root.contains(['\0', '\n', '\r']) + || apple.sdk_root.split('/').any(|part| part == "..") + { + return Err(Error( + "build.apple.sdk-root must be an absolute container path without '..'.".into(), + )); + } + Ok(()) +} + +pub(crate) fn valid_target(s: &str) -> bool { s.split('-').count() >= 3 && s.split('-').all(|part| { !part.is_empty() && part.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_') @@ -320,6 +447,12 @@ fn valid_profile(s: &str) -> bool { .all(|b| b.is_ascii_alphanumeric() || b"_-".contains(&b)) } +fn valid_feature(s: &str) -> bool { + s.as_bytes().first().is_some_and(u8::is_ascii_alphanumeric) + && s.bytes() + .all(|b| b.is_ascii_alphanumeric() || b"_+-".contains(&b)) +} + fn valid_env(s: &str) -> bool { s.as_bytes() .first() @@ -396,4 +529,37 @@ mod tests { assert_eq!(config.profile, "project-profile"); assert_eq!(config.targets.len(), 3); } + + #[test] + fn apple_settings_and_features_are_typed_layered_and_validated() { + let config = parse( + &format!( + "[build]\ntargets=['aarch64-apple-darwin']\nfeatures=['serial','ui']\n[build.apple]\nimage='{DEFAULT_APPLE_IMAGE}'\ndeployment-target='13.2'\nsdk-root='/opt/MacOSX13.sdk'\n" + ), + Overrides::default(), + ) + .unwrap(); + assert_eq!(config.features, ["serial", "ui"]); + assert_eq!(config.apple.deployment_target, "13.2"); + assert_eq!(config.apple.sdk_root, "/opt/MacOSX13.sdk"); + + let overridden = parse( + "[build]\ntargets=['x86_64-apple-darwin']\nfeatures=['old']\n", + Overrides { + features: vec!["new".into()], + ..Overrides::default() + }, + ) + .unwrap(); + assert_eq!(overridden.features, ["new"]); + + for source in [ + "[build.apple]\nimage='cargo-zigbuild:latest'", + &format!("[build.apple]\nimage='{DEFAULT_APPLE_IMAGE}'\ndeployment-target='12.0'"), + &format!("[build.apple]\nimage='{DEFAULT_APPLE_IMAGE}'\nsdk-root='../sdk'"), + "[build]\nfeatures=['bad feature']", + ] { + assert!(parse(source, Overrides::default()).is_err(), "{source}"); + } + } } diff --git a/src/error.rs b/src/error.rs index e99f64c..74791bd 100644 --- a/src/error.rs +++ b/src/error.rs @@ -8,3 +8,9 @@ impl fmt::Display for Error { f.write_str(&self.0) } } + +impl From for Error { + fn from(e: std::io::Error) -> Self { + Self(e.to_string()) + } +} diff --git a/src/macho.rs b/src/macho.rs new file mode 100644 index 0000000..c8cf6b7 --- /dev/null +++ b/src/macho.rs @@ -0,0 +1,181 @@ +use crate::error::Error; +use std::{fs, path::Path}; + +const MH_MAGIC_64_LE: [u8; 4] = [0xcf, 0xfa, 0xed, 0xfe]; +const LC_LOAD_DYLIB: u32 = 0xc; +const LC_LOAD_WEAK_DYLIB: u32 = 0x8000_0018; +const LC_REEXPORT_DYLIB: u32 = 0x8000_001f; +const LC_LOAD_UPWARD_DYLIB: u32 = 0x8000_0023; +const LC_BUILD_VERSION: u32 = 0x32; + +pub fn verify_path(path: &Path, target: &str, minimum: &str) -> Result<(), Error> { + let bytes = fs::read(path) + .map_err(|e| Error(format!("cannot inspect Mach-O {}: {e}", path.display())))?; + verify(&bytes, target, minimum).map_err(|e| Error(format!("{}: {e}", path.display()))) +} + +pub fn verify(bytes: &[u8], target: &str, minimum: &str) -> Result<(), Error> { + if bytes.len() < 32 || bytes[..4] != MH_MAGIC_64_LE { + return Err(Error( + "expected a little-endian 64-bit Mach-O executable".into(), + )); + } + let expected_cpu = match target { + "aarch64-apple-darwin" => 0x0100_000c, + "x86_64-apple-darwin" => 0x0100_0007, + _ => return Err(Error(format!("unsupported Apple target {target}"))), + }; + if word(bytes, 4)? != expected_cpu || word(bytes, 12)? != 2 { + return Err(Error(format!( + "Mach-O architecture or file type does not match {target}" + ))); + } + let commands = word(bytes, 16)? as usize; + let command_bytes = word(bytes, 20)? as usize; + let end = 32usize + .checked_add(command_bytes) + .ok_or_else(|| Error("Mach-O load command size overflow".into()))?; + if end > bytes.len() { + return Err(Error("truncated Mach-O load commands".into())); + } + let expected_minimum = parse_version(minimum)?; + let mut offset = 32usize; + let mut found_minimum = None; + let mut dependencies = Vec::new(); + for _ in 0..commands { + if offset.checked_add(8).is_none_or(|header| header > end) { + return Err(Error("truncated Mach-O load command".into())); + } + let command = word(bytes, offset)?; + let size = word(bytes, offset + 4)? as usize; + let command_end = offset + .checked_add(size) + .ok_or_else(|| Error("Mach-O load command overflow".into()))?; + if size < 8 || command_end > end { + return Err(Error("invalid Mach-O load command size".into())); + } + if command == LC_BUILD_VERSION { + if size < 24 || word(bytes, offset + 8)? != 1 { + return Err(Error("invalid or non-macOS LC_BUILD_VERSION".into())); + } + found_minimum = Some(word(bytes, offset + 12)?); + } else if [ + LC_LOAD_DYLIB, + LC_LOAD_WEAK_DYLIB, + LC_REEXPORT_DYLIB, + LC_LOAD_UPWARD_DYLIB, + ] + .contains(&command) + { + if size < 24 { + return Err(Error("invalid Mach-O dylib command".into())); + } + let name_offset = word(bytes, offset + 8)? as usize; + if name_offset < 24 || name_offset >= size { + return Err(Error("invalid Mach-O dylib name offset".into())); + } + let name = c_string(&bytes[offset + name_offset..command_end])?; + if !name.starts_with("/usr/lib/") && !name.starts_with("/System/Library/") { + return Err(Error(format!("non-system Mach-O dependency: {name}"))); + } + dependencies.push(name.to_owned()); + } + offset = command_end; + } + if offset != end { + return Err(Error("Mach-O load command count/size mismatch".into())); + } + let found = found_minimum.ok_or_else(|| Error("Mach-O has no LC_BUILD_VERSION".into()))?; + if found != expected_minimum { + return Err(Error(format!( + "Mach-O deployment minimum {} does not match configured {minimum}", + display_version(found) + ))); + } + if dependencies.is_empty() { + return Err(Error( + "Mach-O declares no dynamic system dependencies".into(), + )); + } + Ok(()) +} + +fn word(bytes: &[u8], offset: usize) -> Result { + let value = bytes + .get(offset..offset + 4) + .ok_or_else(|| Error("truncated Mach-O field".into()))?; + Ok(u32::from_le_bytes(value.try_into().unwrap())) +} + +fn c_string(bytes: &[u8]) -> Result<&str, Error> { + let end = bytes + .iter() + .position(|byte| *byte == 0) + .ok_or_else(|| Error("unterminated Mach-O dylib name".into()))?; + std::str::from_utf8(&bytes[..end]).map_err(|_| Error("non-UTF-8 Mach-O dylib name".into())) +} + +fn parse_version(value: &str) -> Result { + let mut fields = value.split('.'); + let major = fields.next().and_then(|part| part.parse::().ok()); + let minor = fields.next().and_then(|part| part.parse::().ok()); + let patch = fields + .next() + .map_or(Some(0), |part| part.parse::().ok()); + match (major, minor, patch, fields.next()) { + (Some(major), Some(minor), Some(patch), None) + if major < 65536 && minor < 256 && patch < 256 => + { + Ok((major << 16) | (minor << 8) | patch) + } + _ => Err(Error(format!("invalid macOS deployment version {value:?}"))), + } +} + +fn display_version(value: u32) -> String { + format!("{}.{}.{}", value >> 16, (value >> 8) & 0xff, value & 0xff) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fixture(cpu: u32, minimum: u32, dependency: &str) -> Vec { + let name_size = dependency.len() + 1; + let dylib_size = (24 + name_size).div_ceil(8) * 8; + let mut bytes = vec![0_u8; 32 + 24 + dylib_size]; + bytes[..4].copy_from_slice(&MH_MAGIC_64_LE); + bytes[4..8].copy_from_slice(&cpu.to_le_bytes()); + bytes[12..16].copy_from_slice(&2_u32.to_le_bytes()); + bytes[16..20].copy_from_slice(&2_u32.to_le_bytes()); + bytes[20..24].copy_from_slice(&(24_u32 + dylib_size as u32).to_le_bytes()); + bytes[32..36].copy_from_slice(&LC_BUILD_VERSION.to_le_bytes()); + bytes[36..40].copy_from_slice(&24_u32.to_le_bytes()); + bytes[40..44].copy_from_slice(&1_u32.to_le_bytes()); + bytes[44..48].copy_from_slice(&minimum.to_le_bytes()); + let offset = 56; + bytes[offset..offset + 4].copy_from_slice(&LC_LOAD_DYLIB.to_le_bytes()); + bytes[offset + 4..offset + 8].copy_from_slice(&(dylib_size as u32).to_le_bytes()); + bytes[offset + 8..offset + 12].copy_from_slice(&24_u32.to_le_bytes()); + bytes[offset + 24..offset + 24 + dependency.len()].copy_from_slice(dependency.as_bytes()); + bytes + } + + #[test] + fn validates_architecture_deployment_minimum_and_system_dependencies() { + let arm = fixture(0x0100_000c, 13 << 16, "/usr/lib/libSystem.B.dylib"); + verify(&arm, "aarch64-apple-darwin", "13.0").unwrap(); + assert!(verify(&arm, "x86_64-apple-darwin", "13.0").is_err()); + assert!(verify(&arm, "aarch64-apple-darwin", "14.0").is_err()); + + let intel = fixture( + 0x0100_0007, + (13 << 16) | (2 << 8), + "/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit", + ); + verify(&intel, "x86_64-apple-darwin", "13.2").unwrap(); + let unsafe_dependency = fixture(0x0100_0007, 13 << 16, "/tmp/libInjected.dylib"); + assert!(verify(&unsafe_dependency, "x86_64-apple-darwin", "13.0").is_err()); + assert!(verify(&intel[..40], "x86_64-apple-darwin", "13.2").is_err()); + } +} diff --git a/src/main.rs b/src/main.rs index 405a7be..c1897f3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,10 +1,12 @@ //! Entry point for the incremental Rust replacement. mod build; +mod bundle; mod cli; mod config; mod error; mod gitea; +mod macho; mod manifest; mod model; mod package; diff --git a/src/package.rs b/src/package.rs index f9607c5..9a57cc7 100644 --- a/src/package.rs +++ b/src/package.rs @@ -17,7 +17,15 @@ pub struct Artifact { #[derive(Debug, Clone)] pub enum ArtifactContents { - Archive { root: String, binaries: Vec }, + Bundle { + root: String, + format: crate::bundle::Format, + entries: Vec, + }, + Archive { + root: String, + binaries: Vec, + }, Binary, } diff --git a/src/prepare.rs b/src/prepare.rs index 3c7b890..c76a073 100644 --- a/src/prepare.rs +++ b/src/prepare.rs @@ -473,6 +473,9 @@ mod tests { let runner = Runner::new("GITEA_TOKEN"); let project = project::inspect(root.to_str().unwrap(), None, &runner).unwrap(); let config = Config { + features: Vec::new(), + apple: crate::config::AppleConfig::default(), + bundles: Vec::new(), targets: vec!["x86_64-unknown-linux-musl".into()], profile: "release".into(), locked: true, @@ -534,6 +537,9 @@ mod tests { .unwrap(); let project = project::inspect(root.to_str().unwrap(), None, &runner).unwrap(); let config = Config { + features: Vec::new(), + apple: crate::config::AppleConfig::default(), + bundles: Vec::new(), targets: vec!["x86_64-unknown-linux-musl".into()], profile: "release".into(), locked: true, diff --git a/src/project.rs b/src/project.rs index 0c7c73e..857da15 100644 --- a/src/project.rs +++ b/src/project.rs @@ -7,6 +7,7 @@ use std::{ #[derive(Debug)] pub struct SelectedPackage { + pub workspace_root: PathBuf, pub name: String, pub version: String, pub binaries: Vec, @@ -220,6 +221,7 @@ pub fn inspect( )); } Ok(SelectedPackage { + workspace_root, name: selected.name, version: selected.version, binaries: binaries.iter().map(|binary| binary.name.clone()).collect(), diff --git a/src/runner.rs b/src/runner.rs index 69644a4..43cf4c0 100644 --- a/src/runner.rs +++ b/src/runner.rs @@ -86,6 +86,126 @@ impl Runner { Ok(output.stdout) } + pub fn hook( + &self, + argv: &[String], + cwd: &Path, + paths: (&Path, &Path, &Path), + timeout: u64, + label: &str, + ) -> Result<(), Error> { + let (context, manifest, tree) = paths; + use std::{ + io::{BufRead, BufReader}, + process::Stdio, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::{Duration, Instant}, + }; + let args = argv + .iter() + .map(|a| { + a.replace("{context-file}", &context.to_string_lossy()) + .replace("{manifest-file}", &manifest.to_string_lossy()) + .replace("{bundle-directory}", &tree.to_string_lossy()) + }) + .collect::>(); + let mut command = Command::new(&args[0]); + command + .args(&args[1..]) + .current_dir(cwd) + .env_remove("GITEA_TOKEN") + .env_remove(&self.token_env) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + for (name, _) in std::env::vars_os() { + let upper = name.to_string_lossy().to_uppercase(); + if ["TOKEN", "PASSWORD", "SECRET", "CREDENTIAL"] + .iter() + .any(|s| upper.contains(s)) + { + command.env_remove(name); + } + } + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + command.process_group(0); + } + let cancelled = Arc::new(AtomicBool::new(false)); + #[cfg(unix)] + let registrations = [signal_hook::consts::SIGINT, signal_hook::consts::SIGTERM] + .into_iter() + .map(|signal| signal_hook::flag::register(signal, Arc::clone(&cancelled))) + .collect::, _>>()?; + let result = (|| { + let mut child = command + .spawn() + .map_err(|e| Error(format!("Cannot start hook [{label}] {args:?}: {e}")))?; + let stdout = child.stdout.take().unwrap(); + let stderr = child.stderr.take().unwrap(); + let started = Instant::now(); + std::thread::scope(|scope| { + let stream = |reader: &mut dyn BufRead| { + let mut line = String::new(); + loop { + line.clear(); + match reader.read_line(&mut line) { + Ok(0) | Err(_) => break, + Ok(_) => eprintln!("[{label}] {}", self.redact(line.trim_end())), + } + } + }; + scope.spawn(move || stream(&mut BufReader::new(stdout))); + scope.spawn(move || stream(&mut BufReader::new(stderr))); + let outcome = loop { + if cancelled.load(Ordering::Relaxed) + || started.elapsed() >= Duration::from_secs(timeout) + { + break Err(Error(format!( + "Hook [{label}] {:?} {}", + args, + if cancelled.load(Ordering::Relaxed) { + "cancelled" + } else { + "timed out" + } + ))); + } + match child.try_wait() { + Ok(Some(status)) => { + break if status.success() { + Ok(()) + } else { + Err(Error(format!( + "Hook [{label}] {:?} failed with {status}", + args + ))) + } + } + Ok(None) => std::thread::sleep(Duration::from_millis(25)), + Err(e) => break Err(Error(e.to_string())), + } + }; + // Kill the group even after the leader exits: descendants may retain pipes. + #[cfg(unix)] + unsafe { + libc::kill(-(child.id() as i32), libc::SIGKILL); + } + let _ = child.kill(); + let _ = child.wait(); + outcome + }) + })(); + #[cfg(unix)] + for id in registrations { + signal_hook::low_level::unregister(id); + } + result.map_err(|e: Error| Error(self.redact(&e.to_string()))) + } + pub fn text(&self, program: &str, args: &[&str], cwd: &Path) -> Result { let bytes = self.output(program, args, cwd)?; String::from_utf8(bytes) diff --git a/src/verify.rs b/src/verify.rs index 6d11f81..a038c35 100644 --- a/src/verify.rs +++ b/src/verify.rs @@ -25,6 +25,14 @@ pub fn asset_url(repository: &str, tag: &str, filename: &str) -> String { } pub fn verify_archive(path: &Path, artifact: &Artifact) -> Result<(), Error> { + if let ArtifactContents::Bundle { + root, + format, + entries, + } = &artifact.contents + { + return crate::bundle::verify(path, root, *format, entries); + } let ArtifactContents::Archive { root, binaries } = &artifact.contents else { return Err(Error(format!("{} is not an archive.", path.display()))); }; @@ -93,6 +101,63 @@ pub fn verify_archive(path: &Path, artifact: &Artifact) -> Result<(), Error> { Ok(()) } +pub fn verify_apple_artifact( + path: &Path, + artifact: &Artifact, + deployment_target: &str, +) -> Result<(), Error> { + match &artifact.contents { + ArtifactContents::Binary => { + crate::macho::verify_path(path, &artifact.target, deployment_target) + } + ArtifactContents::Archive { binaries, .. } => { + let gzip = GzDecoder::new( + File::open(path) + .map_err(|e| Error(format!("cannot open {}: {e}", path.display())))?, + ); + let mut archive = tar::Archive::new(gzip); + let mut checked = 0_usize; + for entry in archive.entries().map_err(|e| { + Error(format!( + "cannot inspect Apple archive {}: {e}", + path.display() + )) + })? { + let mut entry = entry.map_err(|e| Error(format!("invalid Apple archive: {e}")))?; + if !entry.header().entry_type().is_file() { + continue; + } + let name = entry + .path() + .map_err(|e| Error(format!("invalid Apple archive path: {e}")))?; + let filename = name.file_name().and_then(|name| name.to_str()); + if !filename.is_some_and(|name| binaries.iter().any(|binary| binary == name)) { + return Err(Error(format!( + "unexpected Apple executable in {}", + path.display() + ))); + } + let mut bytes = Vec::new(); + entry + .read_to_end(&mut bytes) + .map_err(|e| Error(format!("cannot read Apple archive member: {e}")))?; + crate::macho::verify(&bytes, &artifact.target, deployment_target)?; + checked += 1; + } + if checked != binaries.len() { + return Err(Error(format!( + "{} does not contain every declared Apple binary", + path.display() + ))); + } + Ok(()) + } + ArtifactContents::Bundle { .. } => Err(Error( + "Apple named bundles require an explicit Mach-O deployment policy".into(), + )), + } +} + fn member_name(header: &[u8]) -> Result { let name = text_field(&header[..100])?; let prefix = text_field(&header[345..500])?; @@ -199,7 +264,7 @@ pub fn verify_published( } if let Some(artifact) = artifacts.iter().find(|a| a.path == local) { match &artifact.contents { - ArtifactContents::Archive { .. } => { + ArtifactContents::Archive { .. } | ArtifactContents::Bundle { .. } => { verify_archive(&path, artifact)?; println!("Verified URL, checksum and archive: {}", artifact.target); } @@ -344,6 +409,7 @@ mod tests { let root = manifest.parent().unwrap().to_path_buf(); let name = required("RUST_RELEASE_LIVE_NAME"); let project = SelectedPackage { + workspace_root: std::path::PathBuf::new(), name: name.clone(), version: required("RUST_RELEASE_LIVE_VERSION"), binaries: vec![name], @@ -557,6 +623,7 @@ mod tests { } }); let project = SelectedPackage { + workspace_root: std::path::PathBuf::new(), name: "demo".into(), version: "0.1.0".into(), binaries: vec!["demo".into()], @@ -607,6 +674,7 @@ mod tests { .to_string() .contains("unexpected members")); let project = SelectedPackage { + workspace_root: std::path::PathBuf::new(), name: "demo".into(), version: "0.1.0".into(), binaries: vec!["demo".into()], diff --git a/tests/cli_release_flow.rs b/tests/cli_release_flow.rs index 207c526..bc867ca 100644 --- a/tests/cli_release_flow.rs +++ b/tests/cli_release_flow.rs @@ -193,3 +193,309 @@ fn missing_message_file_is_reported_before_project_acquisition() { assert!(!output.status.success()); assert!(String::from_utf8_lossy(&output.stderr).contains("cannot read release message file")); } + +#[test] +fn bundle_cli_uses_reported_paths_and_reruns_preparation_on_cache_hits() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().join("workspace é"); + let bin = tmp.path().join("tools"); + let log = tmp.path().join("cross.log"); + fs::create_dir_all(root.join("member/src")).unwrap(); + fs::create_dir_all(root.join(".cargo")).unwrap(); + fs::create_dir(&bin).unwrap(); + fs::write( + root.join("Cargo.toml"), + "[workspace]\nmembers=['member']\nresolver='2'\n", + ) + .unwrap(); + fs::write( + root.join("member/Cargo.toml"), + r#"[package] +name='demo' +version='0.1.0' +edition='2021' +repository='https://example.com/team/demo' +[package.metadata.binstall] +pkg-url='{ repo }/releases/download/v{ version }/{ name }-{ target }-v{ version }.tgz' +pkg-fmt='tgz' +bin-dir='{ name }-{ target }-v{ version }/{ bin }{ binary-ext }' +"#, + ) + .unwrap(); + fs::write(root.join("member/src/main.rs"), "fn main() {}\n").unwrap(); + fs::write( + root.join(".cargo/config.toml"), + "[build]\ntarget-dir='custom build dir'\n", + ) + .unwrap(); + fs::write(root.join(".gitignore"), "/custom build dir/\n/dist/\n").unwrap(); + fs::write( + root.join("rust-release.toml"), + r#"[build] +targets=['x86_64-unknown-linux-gnu'] +[[package.bundles]] +name='portable' +formats={x86_64-unknown-linux-gnu='tgz'} +archive-name='{name}-{bundle}-{target}-v{version}' +root='{name}-{bundle}-{target}-v{version}' +cargo-binaries='all' +[package.bundles.prepare] +command=['python3','prepare.py','{context-file}'] +inputs=['prepare.py'] +timeout-seconds=10 +required-destinations=['NOTICE'] +"#, + ) + .unwrap(); + fs::write(root.join("member/prepare.py"),r#"import sys,json,pathlib,hashlib +c=json.loads(pathlib.Path(sys.argv[1]).read_text()) +assert pathlib.Path(c['package_root']).name=='member' +assert pathlib.Path(c['workspace_root']).name=='workspace é' +assert 'actual output é' in c['binaries']['demo'] +assert pathlib.Path(c['binaries']['demo']).is_file() +cache=pathlib.Path(c['cache_directory']) +with (cache/'calls').open('a') as f: f.write(str(c['source_dirty'])+'\n') +out=pathlib.Path(c['output_directory'])/'notice' +out.write_bytes(b'notice') +pathlib.Path(c['manifest_path']).write_text(json.dumps(dict(schema_version=1,target=c['target'],files=[dict(source='notice',destination='NOTICE',kind='file',sha256=hashlib.sha256(b'notice').hexdigest())]))) +"#).unwrap(); + executable(&bin.join("docker"), "#!/bin/sh\nexit 0\n"); + executable(&bin.join("cargo-binstall"), "#!/bin/sh\nexit 0\n"); + executable( + &bin.join("cross"), + r#"#!/usr/bin/env python3 +import sys,pathlib,shutil,json,os +a=sys.argv +assert '--message-format=json-render-diagnostics' in a +out=pathlib.Path(a[a.index('--target-dir')+1])/'actual output é' +out.mkdir(parents=True) +binary=out/'demo' +shutil.copyfile('/bin/true',binary) +print(json.dumps(dict(reason='compiler-artifact',target=dict(name='demo',kind=['bin']),executable="/target/actual output é/demo"))) +with open(os.environ['CROSS_LOG'],'a') as f: f.write('build\n') +"#, + ); + git(&root, &["init", "-q"]); + git(&root, &["config", "user.name", "Test"]); + git(&root, &["config", "user.email", "test@example.com"]); + git(&root, &["config", "commit.gpgsign", "false"]); + assert!(run(Command::new("cargo") + .arg("generate-lockfile") + .current_dir(&root)) + .status + .success()); + git(&root, &["add", "."]); + git(&root, &["commit", "-qm", "fixture"]); + git(&root, &["tag", "v0.1.0"]); + assert_success(&publisher( + &root, + &bin, + &log, + &["--package", "demo", "--dry-run"], + )); + assert!(!root.join("dist").exists()); + assert!(!log.exists()); + for _ in 0..2 { + assert_success(&publisher( + &root, + &bin, + &log, + &["--package", "demo", "--compile-only", "--skip-tests"], + )); + } + assert_eq!(fs::read_to_string(&log).unwrap().lines().count(), 1); + let calls = root + .join("custom build dir/rust-release-bundle-cache/portable/x86_64-unknown-linux-gnu/calls"); + assert_eq!(fs::read_to_string(&calls).unwrap(), "False\nFalse\n"); + fs::write( + root.join("member/src/main.rs"), + "fn main() {println!(\"dirty\");}\n", + ) + .unwrap(); + assert_success(&publisher( + &root, + &bin, + &log, + &[ + "--package", + "demo", + "--stop-after", + "package", + "--skip-tests", + ], + )); + assert_eq!(fs::read_to_string(&log).unwrap().lines().count(), 2); + assert!(fs::read_to_string(calls).unwrap().ends_with("True\n")); + // Cancellation is delivered to a disposable publisher process, not the test runner. + fs::write( + root.join("member/prepare.py"), + "import os,signal,time\nos.kill(os.getppid(), signal.SIGTERM)\ntime.sleep(5)\n", + ) + .unwrap(); + let cancelled = publisher( + &root, + &bin, + &log, + &["--package", "demo", "--compile-only", "--skip-tests"], + ); + assert!(!cancelled.status.success()); + assert!(String::from_utf8_lossy(&cancelled.stderr).contains("cancelled")); +} + +#[test] +fn mixed_cross_and_apple_compile_only_routes_and_packages_eight_assets() { + let temporary = TempDir::new().unwrap(); + let project = temporary.path().join("mixed checkout with spaces"); + let tools = temporary.path().join("tools"); + let log = temporary.path().join("routes.log"); + fs::create_dir_all(project.join("src")).unwrap(); + fs::create_dir(&tools).unwrap(); + fs::write( + project.join("Cargo.toml"), + "[package]\nname='demo'\nversion='2.3.4'\nedition='2021'\nrepository='https://example.com/team/demo'\n[features]\nserial=[]\n[package.metadata.binstall]\npkg-url='{ repo }/releases/download/{ version }/{ name }-{ target }-v{ version }.tgz'\npkg-fmt='tgz'\nbin-dir='{ name }-{ target }-v{ version }/{ bin }{ binary-ext }'\n", + ) + .unwrap(); + fs::write(project.join("src/main.rs"), "fn main() {}\n").unwrap(); + fs::write(project.join(".gitignore"), "/target/\n/dist/\n").unwrap(); + fs::write(project.join("release.Cross.toml"), "[build]\n").unwrap(); + fs::write( + project.join("rust-release.toml"), + "[release]\ntag-prefix=''\n[build]\ntargets=['x86_64-unknown-linux-gnu','x86_64-pc-windows-gnu','aarch64-apple-darwin','x86_64-apple-darwin']\nprofile='release'\nlocked=true\nfeatures=['serial']\n[package]\nformat='both'\n", + ) + .unwrap(); + executable(&tools.join("cargo-binstall"), "#!/bin/sh\nexit 0\n"); + executable( + &tools.join("cross"), + "#!/bin/sh\nset -eu\n[ -n \"${CROSS_CONFIG:-}\" ]\ntarget=\ntarget_dir=\nwhile [ $# -gt 0 ]; do case $1 in --target) shift; target=$1;; --target-dir) shift; target_dir=$1;; --features) shift; [ \"$1\" = serial ];; esac; shift; done\ncase $target in *windows*) suffix=.exe;; *) suffix=;; esac\nmkdir -p \"$target_dir/$target/release\"\nprintf cross >\"$target_dir/$target/release/demo$suffix\"\nprintf 'cross:%s\\n' \"$target\" >>\"$CROSS_LOG\"\n", + ); + executable( + &tools.join("docker"), + r#"#!/usr/bin/env python3 +import json, os, pathlib, struct, sys +if sys.argv[1:] == ['info']: + raise SystemExit(0) +assert 'RELEASE_TEST_SECRET' not in os.environ +if os.environ.get('ENGINE_STATUS'): + raise SystemExit(int(os.environ['ENGINE_STATUS'])) +args=sys.argv[1:] +assert args[0:2] == ['run','--rm'] and '-it' not in args +assert not any('sock' in arg for arg in args) +assert '--userns=keep-id' not in args +assert 'serial' in args and '--locked' in args and '--release' in args +mount=next(args[i+1] for i,arg in enumerate(args) if arg=='-v' and args[i+1].endswith(':/output')) +host=pathlib.Path(mount[:-len(':/output')]) +target=args[args.index('--target')+1] +cpu=0x0100000c if target.startswith('aarch64') else 0x01000007 +dependency=b'/usr/lib/libSystem.B.dylib\0' +dylib_size=(24+len(dependency)+7)//8*8 +data=bytearray(32+24+dylib_size) +data[:4]=bytes.fromhex('cffaedfe') +struct.pack_into('