diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4e5bb55..db9a415 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,6 +17,10 @@ jobs: uses: actions/checkout@v7 - name: Validate initial Cargo publication guards run: node --test scripts/initial-publish-context.test.mjs scripts/cargo-publish-context.test.mjs + - name: Validate continuous distribution smoke guards + run: | + node --test scripts/distribution-smoke-context.test.mjs scripts/distribution-smoke-workflow.test.mjs + python3 -m unittest scripts/test_smoke_installed_cli.py - name: Read supported specification revision id: specification diff --git a/.github/workflows/distribution-smoke.yaml b/.github/workflows/distribution-smoke.yaml new file mode 100644 index 0000000..ff45bd9 --- /dev/null +++ b/.github/workflows/distribution-smoke.yaml @@ -0,0 +1,196 @@ +name: Distribution smoke + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + inputs: + version: + description: Exact stable release (empty uses the distribution contract) + type: string + default: '' + workflow_call: + inputs: + version: + type: string + default: '' + scope: + type: string + default: all + source_commit: + type: string + default: '' + +permissions: + contents: read + attestations: read + +concurrency: + group: distribution-smoke-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ inputs.scope || 'all' }}-${{ inputs.version || 'current' }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +defaults: + run: + shell: bash + +jobs: + context: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + version: ${{ steps.context.outputs.version }} + scope: ${{ steps.context.outputs.scope }} + source: ${{ steps.context.outputs.source }} + tap: ${{ steps.context.outputs.tap }} + matrix: ${{ steps.context.outputs.matrix }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - name: Resolve exact published version, source, and supported matrix + id: context + env: + GH_TOKEN: ${{ github.token }} + SMOKE_VERSION: ${{ inputs.version }} + SMOKE_SCOPE: ${{ inputs.scope }} + SMOKE_SOURCE_COMMIT: ${{ inputs.source_commit }} + run: node scripts/distribution-smoke-context.mjs >> "$GITHUB_OUTPUT" + + install: + name: install (${{ matrix.channel }}, ${{ matrix.target }}, Rust ${{ matrix.rust }}) + needs: context + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.context.outputs.matrix) }} + env: + GH_TOKEN: ${{ github.token }} + VERSION: ${{ needs.context.outputs.version }} + SOURCE_COMMIT: ${{ needs.context.outputs.source }} + TAP_COMMIT: ${{ needs.context.outputs.tap }} + CHANNEL: ${{ matrix.channel }} + TARGET: ${{ matrix.target }} + RUST_VERSION: ${{ matrix.rust }} + steps: + - name: Check out current verification code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - name: Check out exact published source for command and catalog comparisons + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + repository: stack-sh/cli + ref: ${{ needs.context.outputs.source }} + path: .release-source + persist-credentials: false + - name: Verify canonical archive checksum, provenance, and package before execution + if: matrix.channel != 'cargo' + run: | + python3 -m scripts.download_smoke_archive --version "$VERSION" --target "$TARGET" \ + --source-commit "$SOURCE_COMMIT" --source-root "$GITHUB_WORKSPACE/.release-source" \ + --destination "$RUNNER_TEMP/canonical-release" + echo "CANONICAL_BINARY=$RUNNER_TEMP/canonical-release/stack-v${VERSION}-${TARGET}/stack" >> "$GITHUB_ENV" + + - name: Install directly into an empty prefix + if: matrix.channel == 'direct' + run: | + mkdir "$RUNNER_TEMP/direct-install" + install -m 0755 "$CANONICAL_BINARY" "$RUNNER_TEMP/direct-install/stack" + echo "STACK_BINARY=$RUNNER_TEMP/direct-install/stack" >> "$GITHUB_ENV" + + - name: Install pinned Aqua + if: matrix.channel == 'aqua' + uses: aquaproj/aqua-installer@96a9bc20066c5bf5e275b41019cfc165b25f4e2e # v4.0.5 + with: + aqua_version: v2.62.3 + enable_aqua_install: false + - name: Install through Aqua in a fresh project and store + if: matrix.channel == 'aqua' + env: + AQUA_ROOT_DIR: ${{ runner.temp }}/aqua-root + XDG_CONFIG_HOME: ${{ runner.temp }}/aqua-config + run: | + project="$RUNNER_TEMP/aqua-project" + node scripts/prepare-aqua-smoke.mjs "$VERSION" "$project" + git init --quiet "$project" + cd "$project" + export AQUA_CONFIG="$project/aqua.yaml" + export AQUA_POLICY_CONFIG="$project/aqua-policy.yaml" + aqua policy allow + aqua update-checksum + aqua install + echo "STACK_BINARY=$(aqua which stack)" >> "$GITHUB_ENV" + + - name: Set up supported Homebrew host + if: matrix.channel == 'homebrew' + uses: Homebrew/actions/setup-homebrew@3cdb78d0f62ad29dd32de765782654f4eedea607 + - name: Install the exact official tap revision into a clean formula prefix + if: matrix.channel == 'homebrew' + env: + HOMEBREW_NO_AUTO_UPDATE: 1 + HOMEBREW_NO_ANALYTICS: 1 + HOMEBREW_CACHE: ${{ runner.temp }}/brew-cache + run: | + brew tap stack-sh/tap + tap_path=$(brew --repository stack-sh/tap) + git -C "$tap_path" fetch origin "$TAP_COMMIT" + git -C "$tap_path" checkout --detach "$TAP_COMMIT" + test "$(git -C "$tap_path" rev-parse HEAD)" = "$TAP_COMMIT" + brew info --json=v2 stack-sh/tap/stack > "$RUNNER_TEMP/formula.json" + node --input-type=module -e 'import fs from "node:fs"; import assert from "node:assert/strict"; const [formula] = JSON.parse(fs.readFileSync(process.env.RUNNER_TEMP + "/formula.json")).formulae; assert.equal(formula.versions.stable, process.env.VERSION); assert.equal(formula.installed.length, 0);' + brew install stack-sh/tap/stack + brew test stack-sh/tap/stack + prefix=$(brew --prefix stack-sh/tap/stack) + cmp "$prefix/etc/bash_completion.d/stack" .release-source/distribution/generated/share/bash-completion/completions/stack + for asset in zsh/site-functions/_stack fish/vendor_completions.d/stack.fish man/man1/stack.1; do + cmp "$prefix/share/$asset" ".release-source/distribution/generated/share/$asset" + done + echo "STACK_BINARY=$prefix/bin/stack" >> "$GITHUB_ENV" + + - name: Install exact registry package with a fresh Cargo cache and build directory + if: matrix.channel == 'cargo' + env: + CARGO_HOME: ${{ runner.temp }}/cargo-registry-home + CARGO_TARGET_DIR: ${{ runner.temp }}/cargo-registry-target + run: | + rustup toolchain install "$RUST_VERSION" --profile minimal + cargo "+$RUST_VERSION" install stack-diagram-cli --version "=$VERSION" --locked --registry crates-io --root "$RUNNER_TEMP/cargo-install" + python3 -m scripts.verify_smoke_cargo_source --cargo-home "$CARGO_HOME" --version "$VERSION" --source-commit "$SOURCE_COMMIT" + echo "STACK_BINARY=$RUNNER_TEMP/cargo-install/bin/stack" >> "$GITHUB_ENV" + + - name: Exercise the installed CLI and a real audited provider import + run: | + comparison=() + if [ "$CHANNEL" != cargo ]; then comparison=(--canonical-binary "$CANONICAL_BINARY"); fi + python3 -m scripts.smoke_installed_cli --binary "$STACK_BINARY" --target "$TARGET" --version "$VERSION" \ + --source-root "$GITHUB_WORKSPACE/.release-source" "${comparison[@]}" > "$RUNNER_TEMP/smoke.json" + cat "$RUNNER_TEMP/smoke.json" >> "$GITHUB_STEP_SUMMARY" + - name: Preserve only verification metadata, never imported artwork + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: install-${{ matrix.channel }}-${{ matrix.target }}-${{ matrix.rust }} + path: ${{ runner.temp }}/smoke.json + if-no-files-found: error + retention-days: 14 + + completion: + name: distribution smoke completion + if: always() + needs: [context, install] + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - name: Fail closed unless every requested installation passed + env: + CONTEXT_RESULT: ${{ needs.context.result }} + INSTALL_RESULT: ${{ needs.install.result }} + SMOKE_SCOPE: ${{ needs.context.outputs.scope }} + SMOKE_VERSION: ${{ needs.context.outputs.version }} + run: | + printf 'Scope: %s; version: %s; context: %s; installs: %s\n' "$SMOKE_SCOPE" "$SMOKE_VERSION" "$CONTEXT_RESULT" "$INSTALL_RESULT" >> "$GITHUB_STEP_SUMMARY" + node --input-type=module -e 'import { requireSuccessfulSmoke } from "./scripts/distribution-smoke-context.mjs"; requireSuccessfulSmoke(process.env.CONTEXT_RESULT, process.env.INSTALL_RESULT);' diff --git a/docs/distribution.md b/docs/distribution.md index 27a97fa..1b64b19 100644 --- a/docs/distribution.md +++ b/docs/distribution.md @@ -183,6 +183,32 @@ The source and published Cargo package names are both `stack-diagram-cli`; the i ## Release activation and rollback +### Continuous clean-install verification + +[`Distribution smoke`](../.github/workflows/distribution-smoke.yaml) runs on pull requests and pushes to `main`, and accepts a manual exact stable version. Without an override it tests `currentReleaseVersion` from the distribution contract, not the possibly unpublished source version. It resolves the immutable release source and the official tap revision once before starting: + +| Channel | Native installation cells | +| --- | --- | +| Direct archive | Four supported targets | +| Aqua 2.62.3 | Four supported targets, fresh Git project and Aqua store | +| Cargo | Four supported targets, each with Rust 1.85.0 and stable; fresh registry cache and build directory | +| Homebrew | Apple Silicon macOS, GNU/Linux arm64 and x86_64; fresh Stack formula prefix and download cache | + +All 19 cells execute the installed binary on the matching native architecture. They check the exact version, help, configuration, doctor, templates, validation, formatting, SVG/JSON output, and completion/manual generation against the **published source**. Each also explicitly imports the audited Simple Icons catalog into a disposable store and renders an imported icon with its attribution. Only result metadata is uploaded; imported artwork and rendered provider examples are not redistributed as CI artifacts. + +Direct, Aqua, and Homebrew binaries must byte-match the canonical archive after checksum, source-bound GitHub provenance, and archive-layout verification. Cargo must install the exact registry package with `--locked` and match its packaged source commit; it is not expected to reproduce prebuilt binary bytes. Homebrew additionally checks installed completion/manual files. Fresh installations do not use the repository's Cargo build output or an existing Stack configuration/icon store. + +The `distribution smoke completion` job always evaluates the context and the whole requested matrix. A failure, cancellation, skip, missing artifact, wrong version, or mismatched digest prevents success. The job summary identifies the version and scope; GitHub Actions reports failure through its normal workflow notifications. Maintainers should watch **Actions** notifications for this repository and inspect the failed matrix cell before retrying; a retry is not a substitute for resolving a reproducible failure. + +The reusable workflow also exposes `native` (Direct + Aqua, eight cells) and `cargo` (eight cells) scopes for publication integration. Cargo-only checks require the exact published source commit and do not assume the GitHub Release already exists. A scoped success is **not** all-channel activation. Full activation requires a successful `all` run for the same stable version after the tap and registry are available. These later results supplement the immutable publication-time manifest; they never rewrite a tag, release asset, or its `verifiedChannels` field. + +Run the negative guards locally with: + +```sh +node --test scripts/distribution-smoke-context.test.mjs scripts/distribution-smoke-workflow.test.mjs +python3 -m unittest scripts/test_smoke_installed_cli.py +``` + A channel becomes available only after all of its target builds and clean-install smoke tests pass. A stable GitHub release additionally requires matching tag/version metadata, complete archive contents, valid checksums and Sigstore bundle, inspectable SPDX SBOMs and provenance, exact generated completion/manual bytes, and successful `stack --version`, `help`, `init`, `check`, and `render` smoke tests on every tier-1 target. Tags and assets are immutable. For a broken release, mark it as withdrawn, exclude it from default update resolution, restore package-manager metadata to the last verified release, and publish a new patch version. Do not overwrite the broken tag or assets. Cargo may yank a broken package version, but yanking is not deletion and the replacement still uses a new version. diff --git a/scripts/distribution-smoke-context.mjs b/scripts/distribution-smoke-context.mjs new file mode 100644 index 0000000..554bb99 --- /dev/null +++ b/scripts/distribution-smoke-context.mjs @@ -0,0 +1,82 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; + +const targets = [ + ["aarch64-apple-darwin", "macos-15"], + ["x86_64-apple-darwin", "macos-15-intel"], + ["aarch64-unknown-linux-gnu", "ubuntu-24.04-arm"], + ["x86_64-unknown-linux-gnu", "ubuntu-24.04"], +]; + +export function smokeMatrix(scope) { + assert.ok(["all", "native", "cargo"].includes(scope), "Unsupported smoke scope"); + const include = []; + for (const [target, runner] of targets) { + if (scope !== "cargo") { + for (const channel of ["direct", "aqua"]) include.push({ channel, target, runner, rust: "none" }); + } + if (scope !== "native") { + for (const rust of ["1.85.0", "stable"]) include.push({ channel: "cargo", target, runner, rust }); + } + if (scope === "all" && target !== "x86_64-apple-darwin") { + include.push({ channel: "homebrew", target, runner: target.endsWith("apple-darwin") ? "macos-26" : runner, rust: "none" }); + } + } + return { include }; +} + +export function validateVersion(version) { + assert.match(version, /^\d+\.\d+\.\d+$/, "Only exact stable versions are supported"); + return version; +} + +export function validateRelease(release, version) { + validateVersion(version); + assert.equal(release.tagName, `v${version}`, "Release version mismatch"); + assert.equal(release.isDraft, false, "Draft release is not installable"); + assert.equal(release.isPrerelease, false, "Prerelease is not a stable channel release"); + const names = release.assets.map(asset => asset.name); + assert.equal(new Set(names).size, names.length, "Duplicate release asset"); + for (const name of [ + `stack-v${version}-checksums.txt`, + `stack-v${version}-checksums.txt.sigstore.json`, + `stack-v${version}-release-manifest.json`, + ...targets.flatMap(([target]) => ["tar.gz", "spdx.json", "provenance.sigstore.json", "sbom.sigstore.json"].map(suffix => `stack-v${version}-${target}.${suffix}`)), + ]) assert.ok(names.includes(name), `Missing release asset: ${name}`); +} + +export function requireSuccessfulSmoke(context, install) { + assert.equal(context, "success", "Smoke context failed or was skipped"); + assert.equal(install, "success", "At least one install failed, was cancelled, or was skipped"); +} + +function gh(...args) { + return JSON.parse(execFileSync("gh", args, { encoding: "utf8", timeout: 60_000 })); +} + +export function resolveContext(env = process.env) { + const scope = env.SMOKE_SCOPE || "all"; + const contract = JSON.parse(readFileSync("distribution/distribution-contract.json", "utf8")); + const version = validateVersion(env.SMOKE_VERSION || contract.product.currentReleaseVersion); + const matrix = smokeMatrix(scope); + let source = env.SMOKE_SOURCE_COMMIT; + if (scope === "cargo") { + assert.match(source || "", /^[0-9a-f]{40}$/, "Cargo-only smoke requires its published source commit"); + } else { + const release = gh("release", "view", `v${version}`, "--repo", "stack-sh/cli", "--json", "tagName,isDraft,isPrerelease,assets"); + validateRelease(release, version); + const tagged = gh("api", `repos/stack-sh/cli/commits/v${version}`).sha; + if (source) assert.equal(source, tagged, "Published source does not match the release tag"); + source = tagged; + } + assert.match(source, /^[0-9a-f]{40}$/); + const tap = scope === "all" ? gh("api", "repos/stack-sh/homebrew-tap/git/ref/heads/main").object.sha : ""; + if (tap) assert.match(tap, /^[0-9a-f]{40}$/); + return { version, scope, source, tap, matrix: JSON.stringify(matrix) }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + for (const [key, value] of Object.entries(resolveContext())) console.log(`${key}=${value}`); +} diff --git a/scripts/distribution-smoke-context.test.mjs b/scripts/distribution-smoke-context.test.mjs new file mode 100644 index 0000000..23b67bd --- /dev/null +++ b/scripts/distribution-smoke-context.test.mjs @@ -0,0 +1,58 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { readFileSync } from "node:fs"; +import { smokeMatrix, validateRelease, validateVersion, requireSuccessfulSmoke } from "./distribution-smoke-context.mjs"; + +const version = "0.5.1"; +const release = { + tagName: `v${version}`, isDraft: false, isPrerelease: false, + assets: [ + `stack-v${version}-checksums.txt`, `stack-v${version}-checksums.txt.sigstore.json`, + `stack-v${version}-release-manifest.json`, + ...smokeMatrix("native").include.filter(row => row.channel === "direct").flatMap(row => ["tar.gz", "spdx.json", "provenance.sigstore.json", "sbom.sigstore.json"].map(suffix => `stack-v${version}-${row.target}.${suffix}`)), + ].map(name => ({ name })), +}; + +test("the all-channel matrix covers the declared supported targets without foreign execution", () => { + const contract = JSON.parse(readFileSync("distribution/distribution-contract.json", "utf8")); + const rows = smokeMatrix("all").include; + assert.equal(rows.length, 19); + assert.equal(new Set(rows.map(row => `${row.channel}/${row.target}/${row.rust}`)).size, 19); + for (const channel of contract.channels) { + const id = channel.id === "github-release" ? "direct" : channel.id; + assert.deepEqual([...new Set(rows.filter(row => row.channel === id).map(row => row.target))].sort(), [...channel.targets].sort()); + } + for (const row of rows) { + assert.ok(row.target.includes("apple") ? row.runner.startsWith("macos-") : row.runner.startsWith("ubuntu-")); + assert.ok(row.target.startsWith("aarch64") ? !row.runner.endsWith("intel") : row.target.includes("apple") ? row.runner.endsWith("intel") : !row.runner.endsWith("arm")); + if (row.target.includes("linux")) assert.equal(row.runner.endsWith("-arm"), row.target.startsWith("aarch64")); + } + assert.equal(smokeMatrix("native").include.length, 8); + assert.equal(smokeMatrix("cargo").include.length, 8); + assert.throws(() => smokeMatrix("skip"), /Unsupported/); +}); + +test("a complete stable release passes, but missing assets and version drift fail", () => { + validateRelease(release, version); + for (let i = 0; i < release.assets.length; i++) { + assert.throws(() => validateRelease({ ...release, assets: release.assets.filter((_, index) => index !== i) }, version), /Missing release asset/); + } + assert.throws(() => validateRelease(release, "0.5.2"), /version mismatch/); + assert.throws(() => validateRelease({ ...release, isDraft: true }, version), /Draft/); + assert.throws(() => validateRelease({ ...release, isPrerelease: true }, version), /Prerelease/); + assert.throws(() => validateRelease({ ...release, assets: [...release.assets, release.assets[0]] }, version), /Duplicate/); +}); + +test("versions cannot select a range, floating release, prerelease, or shell expression", () => { + for (const invalid of ["latest", "^0.5", "0.5.1-rc.1", "0.5.1\n", "$(whoami)", "--help"]) { + assert.throws(() => validateVersion(invalid)); + } +}); + +test("the completion gate never accepts skipped, cancelled, empty, or failing jobs", () => { + requireSuccessfulSmoke("success", "success"); + for (const result of ["failure", "cancelled", "skipped", "", undefined]) { + assert.throws(() => requireSuccessfulSmoke(result, "success")); + assert.throws(() => requireSuccessfulSmoke("success", result)); + } +}); diff --git a/scripts/distribution-smoke-workflow.test.mjs b/scripts/distribution-smoke-workflow.test.mjs new file mode 100644 index 0000000..2d1f546 --- /dev/null +++ b/scripts/distribution-smoke-workflow.test.mjs @@ -0,0 +1,56 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; + +const workflow = readFileSync(".github/workflows/distribution-smoke.yaml", "utf8"); +const download = readFileSync("scripts/download_smoke_archive.py", "utf8"); +const smoke = readFileSync("scripts/smoke_installed_cli.py", "utf8"); + +function validate(source) { + assert.match(source, /^on:\n pull_request:\n push:\n branches: \[main\]/m); + assert.match(source, /^ workflow_dispatch:/m); + assert.match(source, /^ workflow_call:/m); + assert.doesNotMatch(source, /pull_request_target:|workflow_run:|continue-on-error:|\bwrite\b|secrets[.[]/); + assert.match(source, /^permissions:\n contents: read\n attestations: read\n/m); + assert.deepEqual([...source.slice(source.indexOf("\njobs:\n")).matchAll(/^ ([\w-]+):$/gm)].map(match => match[1]), ["context", "install", "completion"]); + assert.equal([...source.matchAll(/persist-credentials: false/g)].length, 4); + const actions = [...source.matchAll(/^\s+(?:- )?uses: (\S+)/gm)].map(match => match[1]); + assert.equal(actions.length, 7); + for (const action of actions) assert.match(action, /@[0-9a-f]{40}$/); + for (const guard of [ + "timeout-minutes: 30", "fail-fast: false", "matrix: ${{ fromJSON(needs.context.outputs.matrix) }}", + "ref: ${{ needs.context.outputs.source }}", "python3 -m scripts.download_smoke_archive", + 'git -C "$tap_path" checkout --detach "$TAP_COMMIT"', "formula.installed.length, 0", + 'cmp "$prefix/etc/bash_completion.d/stack" .release-source/distribution/generated/share/bash-completion/completions/stack', + 'git init --quiet "$project"', "aqua policy allow", "aqua update-checksum", "aqua install", + 'cargo "+$RUST_VERSION" install stack-diagram-cli --version "=$VERSION" --locked --registry crates-io', + "CARGO_HOME: ${{ runner.temp }}/cargo-registry-home", "CARGO_TARGET_DIR: ${{ runner.temp }}/cargo-registry-target", + "python3 -m scripts.verify_smoke_cargo_source", "python3 -m scripts.smoke_installed_cli", + '--canonical-binary "$CANONICAL_BINARY"', "if-no-files-found: error", "path: ${{ runner.temp }}/smoke.json", + "if: always()\n needs: [context, install]", "CONTEXT_RESULT: ${{ needs.context.result }}", + "INSTALL_RESULT: ${{ needs.install.result }}", "requireSuccessfulSmoke(process.env.CONTEXT_RESULT, process.env.INSTALL_RESULT)", + ]) assert.ok(source.includes(guard), `Missing guard: ${guard}`); +} + +test("distribution smoke uses read-only native jobs and a fail-closed aggregate", () => validate(workflow)); + +test("privileged, skipped, unverified, or non-registry smoke mutations are rejected", () => { + for (const [before, after] of [ + ["contents: read", "contents: write"], ["pull_request:", "pull_request_target:"], + ["fail-fast: false", "continue-on-error: true"], ["if: always()", "if: success()"], + ["python3 -m scripts.smoke_installed_cli", "echo skipped"], + ["python3 -m scripts.download_smoke_archive", "echo unverified"], + ["--registry crates-io", "--path ."], ["--canonical-binary", "--unverified-binary"], + ["requireSuccessfulSmoke(process.env.CONTEXT_RESULT, process.env.INSTALL_RESULT)", "console.log('passed')"], + ]) assert.throws(() => validate(workflow.replace(before, after)), `Accepted mutation: ${before}`); +}); + +test("archive authentication precedes extraction and a real provider import precedes rendering", () => { + assert.ok(download.indexOf("verify_checksum(archive, inventory)") < download.indexOf('run("gh", "attestation"')); + assert.ok(download.indexOf('"--source-digest", arguments.source_commit') < download.indexOf('run("tar", "-xzf"')); + assert.ok(download.indexOf('"scripts/package_release.py", "verify"') < download.indexOf('run("tar", "-xzf"')); + assert.match(smoke, /"icons", "import", "simple-icons", "--accept-terms"/); + assert.match(smoke, /data-icon-id="simple-icons:rust"/); + assert.match(smoke, /"simple-icons:rust" in notice.read_text\(\)/); + assert.doesNotMatch(workflow, /path:.*(?:icons|provider\.svg)/); +}); diff --git a/scripts/download_smoke_archive.py b/scripts/download_smoke_archive.py new file mode 100644 index 0000000..6bbb1a6 --- /dev/null +++ b/scripts/download_smoke_archive.py @@ -0,0 +1,41 @@ +"""Download and verify a published archive before any channel binary is executed.""" +import argparse +from pathlib import Path +import subprocess +import sys + +from .smoke_installed_cli import verify_checksum + + +def run(*arguments): + subprocess.run([str(argument) for argument in arguments], check=True, timeout=180) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("version", "target", "source-commit"): + parser.add_argument(f"--{name}", required=True) + parser.add_argument("--source-root", type=Path, required=True) + parser.add_argument("--destination", type=Path, required=True) + arguments = parser.parse_args() + destination = arguments.destination.resolve() + destination.mkdir(parents=True, exist_ok=False) + archive = destination / f"stack-v{arguments.version}-{arguments.target}.tar.gz" + inventory = destination / f"stack-v{arguments.version}-checksums.txt" + run("gh", "release", "download", f"v{arguments.version}", "--repo", "stack-sh/cli", + "--pattern", archive.name, "--pattern", inventory.name, "--dir", destination) + verify_checksum(archive, inventory) + run("gh", "attestation", "verify", archive, "--repo", "stack-sh/cli", + "--signer-workflow", "stack-sh/cli/.github/workflows/release.yaml", + "--source-ref", f"refs/tags/v{arguments.version}", "--source-digest", arguments.source_commit) + timestamp = subprocess.check_output( + ["git", "-C", str(arguments.source_root), "show", "-s", "--format=%ct"], text=True).strip() + # Use the release's notice and generated assets, not an unreleased working tree. + run(sys.executable, arguments.source_root / "scripts/package_release.py", "verify", + "--archive", archive, "--version", arguments.version, "--target", arguments.target, + "--source-date-epoch", timestamp) + run("tar", "-xzf", archive, "-C", destination) + + +if __name__ == "__main__": + main() diff --git a/scripts/prepare-aqua-smoke.mjs b/scripts/prepare-aqua-smoke.mjs new file mode 100644 index 0000000..6cd7e37 --- /dev/null +++ b/scripts/prepare-aqua-smoke.mjs @@ -0,0 +1,10 @@ +import { readFileSync, mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { validateVersion } from "./distribution-smoke-context.mjs"; + +const [version, destination] = process.argv.slice(2); +validateVersion(version); +mkdirSync(destination, { recursive: false }); +const config = readFileSync("tests/aqua/aqua.yaml", "utf8"); +writeFileSync(join(destination, "aqua.yaml"), config.replace(/stack-sh\/cli@v\d+\.\d+\.\d+/, `stack-sh/cli@v${version}`)); +writeFileSync(join(destination, "aqua-policy.yaml"), readFileSync("tests/aqua/aqua-policy.yaml")); diff --git a/scripts/smoke_installed_cli.py b/scripts/smoke_installed_cli.py new file mode 100644 index 0000000..b3e5e36 --- /dev/null +++ b/scripts/smoke_installed_cli.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Exercise the installed binary without reusing an existing config or icon store.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import tempfile + +from .verify_release_binary import command, require, verify_architecture, verify_commands + + +def verify_checksum(archive, inventory): + matches = [] + for line in inventory.read_text().splitlines(): + match = re.fullmatch(r"([0-9a-fA-F]{64}) [ *](.+)", line) + require(match is not None, "Malformed checksum inventory") + if match[2] == archive.name: + matches.append(match[1].lower()) + require(len(matches) == 1, "Missing or duplicate archive checksum") + require(hashlib.sha256(archive.read_bytes()).hexdigest() == matches[0], "Archive checksum mismatch") + + +def verify_import(binary, reference_root): + with tempfile.TemporaryDirectory(prefix="stack-import-smoke-") as temporary: + root = Path(temporary) + environment = os.environ.copy() + for name in ("XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME"): + environment[name] = str(root / name.lower()) + store = root / "icons" + # Only the already-audited catalog is imported; no artwork is uploaded as CI evidence. + command([binary, "icons", "import", "simple-icons", "--accept-terms", "-o", store], root, environment) + pack = store / "simple-icons" + manifest = json.loads((pack / "manifest.json").read_text()) + catalog = json.loads((reference_root / "catalogs/simple-icons.json").read_text()) + require(manifest["provider"]["id"] == "simple-icons", "Imported provider identity mismatch") + require(len(manifest["icons"]) == len(catalog["icons"]), "Imported icon inventory mismatch") + require((pack / "NOTICE.md").stat().st_size > 0, "Import omitted attribution") + source = root / "provider.stack" + source.write_text('stack 1.0\ndiagram "Import smoke" {\n node rust "Rust" { kind service icon "simple-icons:rust" }\n}\n') + output = root / "provider.svg" + notice = root / "provider.NOTICE.md" + command([binary, "render", source, "--provider-pack", store, "--notice", notice, "-o", output], root, environment) + require(output.stat().st_size > 0, "Imported icon did not render") + require('data-icon-id="simple-icons:rust"' in output.read_text(), "Imported icon fell back instead of resolving") + require("simple-icons:rust" in notice.read_text(), "Rendered icon attribution is missing") + return len(manifest["icons"]) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--binary", required=True) + parser.add_argument("--target", required=True) + parser.add_argument("--version", required=True) + parser.add_argument("--source-root", required=True, type=Path) + parser.add_argument("--canonical-binary", type=Path) + arguments = parser.parse_args() + binary = Path(arguments.binary).resolve(strict=True) + verify_architecture(binary, arguments.target) + if arguments.canonical_binary: + require(binary.read_bytes() == arguments.canonical_binary.read_bytes(), "Installed binary differs from the verified release archive") + verify_commands(binary, arguments.version, arguments.source_root / "distribution/generated") + count = verify_import(binary, arguments.source_root) + print(json.dumps({"version": arguments.version, "target": arguments.target, "commands": "passed", "importedIcons": count, "render": "passed"})) + + +if __name__ == "__main__": + main() diff --git a/scripts/test_smoke_installed_cli.py b/scripts/test_smoke_installed_cli.py new file mode 100644 index 0000000..01e92ea --- /dev/null +++ b/scripts/test_smoke_installed_cli.py @@ -0,0 +1,64 @@ +import hashlib +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from scripts.smoke_installed_cli import verify_checksum +from scripts.verify_release_binary import verify_commands +from scripts.verify_smoke_cargo_source import verify_source + + +class ChecksumTests(unittest.TestCase): + def test_registry_source_must_match_one_clean_published_commit(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = "a" * 40 + with self.assertRaisesRegex(ValueError, "exactly one"): + verify_source(root, "0.5.1", source) + vcs = root / "registry/src/crates-io/stack-diagram-cli-0.5.1/.cargo_vcs_info.json" + vcs.parent.mkdir(parents=True) + vcs.write_text(json.dumps({"git": {"sha1": source}})) + verify_source(root, "0.5.1", source) + with self.assertRaisesRegex(ValueError, "differs"): + verify_source(root, "0.5.1", "b" * 40) + vcs.write_text(json.dumps({"git": {"sha1": source, "dirty": True}})) + with self.assertRaisesRegex(ValueError, "dirty"): + verify_source(root, "0.5.1", source) + duplicate = root / "registry/src/other/stack-diagram-cli-0.5.1/.cargo_vcs_info.json" + duplicate.parent.mkdir(parents=True) + duplicate.write_text(vcs.read_text()) + with self.assertRaisesRegex(ValueError, "exactly one"): + verify_source(root, "0.5.1", source) + + def test_wrong_installed_version_is_rejected_before_running_other_commands(self): + with patch("scripts.verify_release_binary.command", return_value=b"stack 0.0.0\n") as run: + with self.assertRaisesRegex(ValueError, "version output"): + verify_commands(Path("unused"), "0.5.1") + self.assertEqual(run.call_count, 1) + + def test_archive_must_exist_and_match_one_exact_checksum(self): + with tempfile.TemporaryDirectory() as directory: + archive = Path(directory) / "stack.tar.gz" + inventory = Path(directory) / "checksums.txt" + archive.write_bytes(b"immutable release bytes") + digest = hashlib.sha256(archive.read_bytes()).hexdigest() + inventory.write_text(f"{digest} {archive.name}\n") + verify_checksum(archive, inventory) + archive.write_bytes(b"tampered") + with self.assertRaisesRegex(ValueError, "mismatch"): + verify_checksum(archive, inventory) + inventory.write_text(f"{digest} missing.tar.gz\n") + with self.assertRaisesRegex(ValueError, "Missing"): + verify_checksum(archive, inventory) + inventory.write_text(f"{digest} {archive.name}\n" * 2) + with self.assertRaisesRegex(ValueError, "duplicate"): + verify_checksum(archive, inventory) + inventory.write_text("bad checksum\n") + with self.assertRaisesRegex(ValueError, "Malformed"): + verify_checksum(archive, inventory) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/verify_release_binary.py b/scripts/verify_release_binary.py index 9381e77..2f86bb8 100644 --- a/scripts/verify_release_binary.py +++ b/scripts/verify_release_binary.py @@ -39,11 +39,12 @@ def command(arguments, working_directory=None, environment=None, allow_stderr=Fa check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + timeout=180, ) if completed.returncode != 0: details = completed.stderr.decode("utf-8", errors="replace").strip() raise ValueError(f"command failed ({completed.returncode}): {' '.join(map(str, arguments))}: {details}") - require(allow_stderr or completed.stderr == b"", f"command emitted unexpected diagnostics: {' '.join(map(str, arguments))}") + require(allow_stderr or completed.stderr == b"", f"command emitted unexpected diagnostics: {' '.join(map(str, arguments))}: {completed.stderr.decode('utf-8', errors='replace').strip()}") return completed.stdout @@ -92,7 +93,9 @@ def verify_macos_runtime(binary): require("Signature=adhoc\n" in details, "macOS release binary must use an ad-hoc signature") -def verify_commands(binary, version): +def verify_commands(binary, version, assets_root=None): + if assets_root is None: + assets_root = ROOT / "distribution/generated" expected_version = f"stack {version}\n".encode() require(command([binary, "--version"]) == expected_version, "--version output does not match Cargo version") require(command([binary, "version"]) == expected_version, "version command output does not match Cargo version") @@ -113,7 +116,7 @@ def verify_commands(binary, version): "config help output is missing usage", ) for relative_path, arguments in GENERATED_COMMANDS.items(): - expected = (ROOT / "distribution/generated" / relative_path).read_bytes() + expected = (assets_root / relative_path).read_bytes() require( command([binary, *arguments]) == expected, f"generated CLI asset differs from source: {relative_path}", diff --git a/scripts/verify_smoke_cargo_source.py b/scripts/verify_smoke_cargo_source.py new file mode 100644 index 0000000..3c98798 --- /dev/null +++ b/scripts/verify_smoke_cargo_source.py @@ -0,0 +1,23 @@ +"""Reject a registry install that does not come from the expected release source.""" +import argparse +import json +from pathlib import Path + +from .verify_release_binary import require + + +def verify_source(cargo_home, version, source_commit): + candidates = list(cargo_home.glob(f"registry/src/*/stack-diagram-cli-{version}/.cargo_vcs_info.json")) + require(len(candidates) == 1, "Expected exactly one fresh registry source package") + vcs = json.loads(candidates[0].read_text()) + require(vcs["git"]["sha1"] == source_commit, "Registry source commit differs from the expected release") + require(vcs["git"].get("dirty", False) is False, "Registry source was packaged from a dirty tree") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cargo-home", type=Path, required=True) + parser.add_argument("--version", required=True) + parser.add_argument("--source-commit", required=True) + arguments = parser.parse_args() + verify_source(arguments.cargo_home, arguments.version, arguments.source_commit)