|
| 1 | +#!/usr/bin/env bash |
| 2 | +# Host a dev runner for the scenarios' Jobs, after every submit job has finished: |
| 3 | +# |
| 4 | +# runner.sh serve serve `test-spec` as `test-runner` for 30 minutes |
| 5 | +# runner.sh cancel serve `no-sandbox-spec` as `test-runner-no-sandbox`, and kill the runner |
| 6 | +# once the cancel scenario's Job is running, so the server cancels that Job |
| 7 | +# when its timeout and grace period pass |
| 8 | +set -euo pipefail |
| 9 | + |
| 10 | +now() { |
| 11 | + date -u +%Y-%m-%dT%H:%M:%SZ |
| 12 | +} |
| 13 | + |
| 14 | +rotate_key() { |
| 15 | + local key |
| 16 | + key="$(bencher runner key --token "$BENCHER_ADMIN_API_TOKEN" "$1" | jq -r '.key // empty')" |
| 17 | + if [ -z "$key" ]; then |
| 18 | + echo "::error::Rotating the key of $1 returned no key" |
| 19 | + return 1 |
| 20 | + fi |
| 21 | + echo "::add-mask::$key" |
| 22 | + export BENCHER_RUNNER_KEY="$key" |
| 23 | +} |
| 24 | + |
| 25 | +serve() { |
| 26 | + rotate_key test-runner |
| 27 | + echo "Runner up at $(now)" |
| 28 | + local status=0 |
| 29 | + timeout --kill-after 60s 30m runner up --runner test-runner --no-auto-update || status=$? |
| 30 | + # `timeout` exits 124 after it stops the runner, or 137 if the runner needed a SIGKILL. |
| 31 | + if [ "$status" -eq 124 ] || [ "$status" -eq 137 ]; then |
| 32 | + echo "Runner down at $(now)" |
| 33 | + return 0 |
| 34 | + fi |
| 35 | + return "$status" |
| 36 | +} |
| 37 | + |
| 38 | +cancel() { |
| 39 | + rotate_key test-runner-no-sandbox |
| 40 | + local jobs count job |
| 41 | + jobs="$(bencher job list "$PAID_PROJECT" --status pending --sort created --direction desc --per-page 255)" |
| 42 | + count="$(jq '[.[] | select(.spec.slug == "no-sandbox-spec")] | length' <<< "$jobs")" |
| 43 | + job="$(jq -r '[.[] | select(.spec.slug == "no-sandbox-spec")][0].uuid // empty' <<< "$jobs")" |
| 44 | + if [ -z "$job" ]; then |
| 45 | + echo "::error::There is no pending no-sandbox-spec Job in $PAID_PROJECT to cancel" |
| 46 | + return 1 |
| 47 | + fi |
| 48 | + if [ "$count" -gt 1 ]; then |
| 49 | + echo "::warning::$count no-sandbox-spec Jobs are pending in $PAID_PROJECT, and the runner may run older ones before $job" |
| 50 | + fi |
| 51 | + |
| 52 | + local log="$RUNNER_TEMP/runner.log" |
| 53 | + echo "Runner up at $(now), waiting for Job $job to start" |
| 54 | + runner up --runner test-runner-no-sandbox --danger-allow-no-sandbox --no-auto-update > "$log" 2>&1 & |
| 55 | + local pid=$! |
| 56 | + local deadline=$((SECONDS + 900)) |
| 57 | + until grep -qF "Starting iteration 1/1 for job $job" "$log"; do |
| 58 | + if ! kill -0 "$pid" 2> /dev/null; then |
| 59 | + cat "$log" |
| 60 | + echo "::error::The runner exited before Job $job started" |
| 61 | + return 1 |
| 62 | + fi |
| 63 | + if [ "$SECONDS" -ge "$deadline" ]; then |
| 64 | + kill -KILL "$pid" |
| 65 | + cat "$log" |
| 66 | + echo "::error::Job $job did not start within 15 minutes" |
| 67 | + return 1 |
| 68 | + fi |
| 69 | + sleep 1 |
| 70 | + done |
| 71 | + # A SIGTERM would let the runner finish the Job first, so the runner gets no chance to. |
| 72 | + kill -KILL "$pid" |
| 73 | + wait "$pid" || true |
| 74 | + cat "$log" |
| 75 | + echo "Killed the runner at $(now) while Job $job was running" |
| 76 | +} |
| 77 | + |
| 78 | +case "${1:-}" in |
| 79 | +serve) serve ;; |
| 80 | +cancel) cancel ;; |
| 81 | +*) |
| 82 | + echo "usage: runner.sh serve | cancel" >&2 |
| 83 | + exit 2 |
| 84 | + ;; |
| 85 | +esac |
0 commit comments