From b9f9af8bcae0d0422dc94e1c9d8ef5ce39d88d17 Mon Sep 17 00:00:00 2001 From: adamstac Date: Tue, 8 Sep 2026 05:05:22 -0500 Subject: [PATCH 1/2] Prevent automatic replay of tag releases on fallback runners --- .github/workflows/ship_it.yml | 8 ++-- README.md | 5 ++- docs/release_recovery.md | 31 +++++++++++++ justfile | 6 ++- test/workflow_policy.rb | 85 +++++++++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 6 deletions(-) create mode 100644 docs/release_recovery.md create mode 100644 test/workflow_policy.rb diff --git a/.github/workflows/ship_it.yml b/.github/workflows/ship_it.yml index 8075df7..8fe1cc1 100644 --- a/.github/workflows/ship_it.yml +++ b/.github/workflows/ship_it.yml @@ -14,7 +14,7 @@ on: - ready_for_review workflow_dispatch: -# All jobs have the same outcome. We define multiple for resiliency reasons. +# Alternate runners provide resiliency for checks, not automatic release replay. jobs: # In thechangelog/pipely repository (a.k.a. upstream), # this is the preferred default (i.e. custom runners, faster than GitHub): @@ -23,11 +23,11 @@ jobs: uses: ./.github/workflows/_namespace.yml secrets: inherit - # Just in case Namespace.so misbehaves, we want a fallback. - # Always run two of everything™ + # Release failures may occur after publish/deploy/PURGE. Inspect before recovery; + # never replay the entire release automatically on another runner. on-github-fallback: needs: on-namespace - if: ${{ failure() }} + if: ${{ failure() && !(startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push') }} uses: ./.github/workflows/_github.yml secrets: inherit diff --git a/README.md b/README.md index 9be53a4..75a15a1 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,10 @@ While it's fun watching other people experiment with digital resin (varnish 😂 - 🐳 [Docker](https://docs.docker.com/engine/install/) - 🤖 [Just](https://github.com/casey/just?tab=readme-ov-file#installation) version `1.35.0` or higher -And that's about it. Everything else is containerized with Dagger. +The workflow scheduling checks in `just test` also require Ruby 2.6+ with its +standard-library YAML parser (no additional gems). The application tests and +build are containerized with Dagger. See [release recovery](docs/release_recovery.md) +for the automatic fallback boundary and its isolated tests. > [!NOTE] > **For Windows Developers:** diff --git a/docs/release_recovery.md b/docs/release_recovery.md new file mode 100644 index 0000000..1795e76 --- /dev/null +++ b/docs/release_recovery.md @@ -0,0 +1,31 @@ +# Release failure recovery + +Automatic GitHub-runner fallback remains available for non-release checks. +For a `v*` tag push, a failed Namespace run stops without automatic fallback. +The failure might happen after publication, deployment or production acceptance, +including during report upload. Repeating the whole workflow can repeat effects. + +Before an operator-approved recovery: + +1. Inspect the failed run and identify the last completed stage. A red workflow + does not imply deployment failed or that nothing changed. +2. Verify published image identity and actual deployed state read-only. Preserve + the run ID, commit, image digest and observed machines before deciding to act. +3. Select recovery for the incomplete stage. If deployment succeeded and only + reporting failed, do not redeploy merely to obtain a green run. +4. Obtain approval for the exact recovery effects, then verify the result. + +This guard does not make manual reruns, separate runs or tag pushes idempotent. +Do not use "Re-run all jobs" as an automatic recovery step. Production acceptance +also includes cache PURGE requests; it is not a read-only diagnostic. + +When Namespace is not configured, GitHub remains the primary runner and can +perform a normal tag release. The change only restricts automatic fallback. + +## Local scheduling-policy checks + +`just test-workflows` uses Ruby 2.6+ with its standard-library YAML parser; no +third-party gems, secrets, container build or provider calls are needed. It is +also included in `just test`. These tests evaluate actual workflow predicates +against synthetic events and stage failures; they do not emulate GitHub's full +scheduler or prove cross-run exactly-once delivery. diff --git a/justfile b/justfile index 0f1be91..0b9f491 100644 --- a/justfile +++ b/justfile @@ -47,7 +47,11 @@ docker-bash: @docker exec -it pipely.dev bash # Test VTC + acceptance locally -test: test-vtc test-acceptance-local +test: test-workflows test-vtc test-acceptance-local + +# Test workflow scheduling policy (Ruby standard library only) +test-workflows: + @ruby test/workflow_policy.rb # Test VCL config test-vtc: diff --git a/test/workflow_policy.rb b/test/workflow_policy.rb new file mode 100644 index 0000000..fe6e763 --- /dev/null +++ b/test/workflow_policy.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +# Scheduling-policy tests only: no jobs, provider calls or release commands run. +require 'yaml' + +root = File.expand_path('..', __dir__) +parent = YAML.safe_load(File.read(File.join(root, '.github/workflows/ship_it.yml'))) +children = %w[_namespace.yml _github.yml].map do |name| + YAML.safe_load(File.read(File.join(root, '.github/workflows', name))) +end +jobs = parent.fetch('jobs') + +# Evaluate only the boolean subset used by these workflow guards. Reject unknown +# syntax rather than silently modeling a changed policy as the old policy. +def evaluate(expression, event:, ref:, result:, runners:) + value = expression.delete_prefix('${{').delete_suffix('}}').strip + value = value.gsub('failure()', (result == 'failure').to_s) + value = value.gsub(/startsWith\(github.ref, '([^']*)'\)/) { ref.downcase.start_with?(Regexp.last_match(1).downcase).to_s } + value = value.gsub(/contains\(vars.RUNS_ON, '([^']*)'\)/) { runners.downcase.include?(Regexp.last_match(1).downcase).to_s } + value = value.gsub(/github.event_name == '([^']*)'/) { (event.downcase == Regexp.last_match(1).downcase).to_s } + raise "Unsupported guard: #{expression}" unless value.gsub(/true|false|[\s()!&|]/, '').empty? + + # Only boolean literals/operators remain, never arbitrary workflow code. + eval(value) # rubocop:disable Security/Eval +end + +checks = 0 +check = lambda do |condition, message| + raise message unless condition + + checks += 1 +end + +fallback = jobs.fetch('on-github-fallback') +check.call(fallback.fetch('needs') == 'on-namespace', 'Fallback must depend on Namespace') +check.call(fallback.fetch('uses') == './.github/workflows/_github.yml', 'Unexpected fallback child') + +contexts = [ + ['push', 'refs/tags/v1.3.0', true], + ['push', 'refs/tags/v', true], + ['push', 'refs/tags/vnext/test', true], + ['push', 'refs/tags/V1.3.0', true], + ['push', 'refs/heads/main', false], + ['push', 'refs/heads/vnext', false], + ['push', 'refs/tags/test', false], + ['pull_request', 'refs/pull/60/merge', false], + ['workflow_dispatch', 'refs/tags/v1.3.0', false] +] + +contexts.each do |event, ref, release| + %w[success failure skipped cancelled].each do |result| + context = { event: event, ref: ref, result: result, runners: 'namespace' } + actual = evaluate(fallback.fetch('if'), **context) + check.call(actual == (result == 'failure' && !release), "Unexpected fallback: #{context}") + end + + children.each do |child| + steps = child.fetch('jobs').fetch('run').fetch('steps') + effectful = steps.select { |step| step['run'].to_s.match?(/just (publish|deploy|test-acceptance-production)\b/) } + check.call(effectful.length == 2, 'Review policy coverage when release steps change') + effectful.each do |step| + context = { event: event, ref: ref, result: 'success', runners: '' } + check.call(evaluate(step.fetch('if'), **context) == release, "Child release predicate changed: #{step['name']}") + end + end +end + +# Inject failure at each boundary. Record only synthetic activation counts; +# acceptance includes PURGE in the real workflow, so it must never run here. +%w[before_build after_publish after_deploy production_acceptance report_upload].each do |stage| + context = { event: 'push', ref: 'refs/tags/v1.3.0', result: 'failure', runners: 'namespace' } + fallback_runs = evaluate(fallback.fetch('if'), **context) + already_deployed = %w[after_deploy production_acceptance report_upload].include?(stage) + activations = (already_deployed ? 1 : 0) + (fallback_runs ? 1 : 0) + check.call(!fallback_runs && activations <= 1, "Automatic release replay after #{stage}") +end + +['namespace', 'NAMESPACE', ''].each do |runners| + context = { event: 'push', ref: 'refs/tags/v1.3.0', result: 'success', runners: runners } + namespace = evaluate(jobs.fetch('on-namespace').fetch('if'), **context) + github = evaluate(jobs.fetch('on-github').fetch('if'), **context) + check.call(namespace == !runners.empty? && github == runners.empty?, 'Primary runner selection changed') +end + +puts "#{checks} workflow policy checks passed (no release commands executed)" From 258095e567e4816e3b60d82b1d28a3d235b9ebc7 Mon Sep 17 00:00:00 2001 From: adamstac Date: Tue, 8 Sep 2026 10:07:16 -0500 Subject: [PATCH 2/2] Reject unsupported operators in workflow policy tests --- README.md | 3 ++- justfile | 2 +- test/workflow_policy.rb | 21 ++++++++++++++++++++- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 75a15a1..6bd76a1 100644 --- a/README.md +++ b/README.md @@ -114,11 +114,12 @@ Available recipes: http-profile url="https://changelog.com/" # Observe all HTTP timings - https://blog.cloudflare.com/a-question-of-timing local-debug # Debug container locally local-run # Run container locally: available on http://localhost:9000 - test # Test VTC + acceptance locally + test # Test workflow policy, VTC and acceptance locally test-acceptance-local # Test acceptance local test-reports # Open test reports test-reports-rm # Clear test reports test-vtc # Test VCL config + test-workflows # Test workflow scheduling policy (Ruby standard library only) [team] envrc-secrets # Create .envrc.secrets with credentials from 1Password diff --git a/justfile b/justfile index 0b9f491..bd1d5ec 100644 --- a/justfile +++ b/justfile @@ -46,7 +46,7 @@ docker-run *ARGS: docker-bash: @docker exec -it pipely.dev bash -# Test VTC + acceptance locally +# Test workflow policy, VTC and acceptance locally test: test-workflows test-vtc test-acceptance-local # Test workflow scheduling policy (Ruby standard library only) diff --git a/test/workflow_policy.rb b/test/workflow_policy.rb index fe6e763..8150447 100644 --- a/test/workflow_policy.rb +++ b/test/workflow_policy.rb @@ -18,7 +18,7 @@ def evaluate(expression, event:, ref:, result:, runners:) value = value.gsub(/startsWith\(github.ref, '([^']*)'\)/) { ref.downcase.start_with?(Regexp.last_match(1).downcase).to_s } value = value.gsub(/contains\(vars.RUNS_ON, '([^']*)'\)/) { runners.downcase.include?(Regexp.last_match(1).downcase).to_s } value = value.gsub(/github.event_name == '([^']*)'/) { (event.downcase == Regexp.last_match(1).downcase).to_s } - raise "Unsupported guard: #{expression}" unless value.gsub(/true|false|[\s()!&|]/, '').empty? + raise "Unsupported guard: #{expression}" unless value.gsub(/true|false|&&|\|\||[\s()!]/, '').empty? # Only boolean literals/operators remain, never arbitrary workflow code. eval(value) # rubocop:disable Security/Eval @@ -31,6 +31,25 @@ def evaluate(expression, event:, ref:, result:, runners:) checks += 1 end +context = { event: 'push', ref: 'refs/heads/main', result: 'success', runners: '' } +{ + 'true && false' => false, + 'true || false' => true, + '!(true && false) && (false || true)' => true +}.each do |expression, expected| + check.call(evaluate(expression, **context) == expected, "Logical operator evaluation failed: #{expression}") +end + +['true & false', 'true | false', 'true &&& false', 'true ||| false'].each do |expression| + rejected = false + begin + evaluate(expression, **context) + rescue RuntimeError => error + rejected = error.message == "Unsupported guard: #{expression}" + end + check.call(rejected, "Unsupported operator accepted: #{expression}") +end + fallback = jobs.fetch('on-github-fallback') check.call(fallback.fetch('needs') == 'on-namespace', 'Fallback must depend on Namespace') check.call(fallback.fetch('uses') == './.github/workflows/_github.yml', 'Unexpected fallback child')