diff --git a/pkg/bundler/deployer/helm/helm_test.go b/pkg/bundler/deployer/helm/helm_test.go index 7299cf38e..98656df91 100644 --- a/pkg/bundler/deployer/helm/helm_test.go +++ b/pkg/bundler/deployer/helm/helm_test.go @@ -29,8 +29,10 @@ import ( "gopkg.in/yaml.v3" + "github.com/NVIDIA/aicr/pkg/bundler/config" "github.com/NVIDIA/aicr/pkg/bundler/deployer" "github.com/NVIDIA/aicr/pkg/bundler/deployer/localformat" + "github.com/NVIDIA/aicr/pkg/bundler/gatemanifest" "github.com/NVIDIA/aicr/pkg/component" "github.com/NVIDIA/aicr/pkg/recipe" ) @@ -1403,6 +1405,65 @@ func TestBundleGolden_OwnsCRDsChartOverride(t *testing.T) { assertBundleGolden(t, outDir, "testdata/owns_crds_chart_override") } +// TestBundleGolden_ReadinessGate pins the readiness folder a helm bundle +// ships, which until now had no golden at all. +// +// The absence was the bug's cover. gatemanifest.Render annotates the gate Job +// as a post-install,post-upgrade hook with +// hook-delete-policy: before-hook-creation, and both annotations are +// load-bearing under plain Helm: +// +// - the hook is what makes deploy.sh block. It passes --wait without +// --wait-for-jobs, which is correct only because --wait blocks on hook +// completion. A bare Job under --wait alone returns as soon as the object +// exists, so the "gate" would let dependents start against a cluster it +// has not finished checking. +// +// - before-hook-creation is what makes it re-run. A Job's spec.template is +// immutable, so an identical manifest is a no-op patch; without the +// delete-and-recreate the gate asserts once, at install, and every +// subsequent upgrade ships unverified. +// +// A golden here fails loudly if either annotation is stripped again. +func TestBundleGolden_ReadinessGate(t *testing.T) { + gate, err := gatemanifest.Render("foo", "nvcr.io/nvidia/aicr:v1.0.0", + []byte("apiVersion: chainsaw.kyverno.io/v1alpha1\nkind: Test\n"), + config.DeployerHelm) + if err != nil { + t.Fatalf("render gate manifest: %v", err) + } + + outDir := t.TempDir() + g := &Generator{ + RecipeResult: singleComponentRecipe( + "foo", "foo", "foo", "v1.0.0", "https://example.com/charts"), + ComponentValues: map[string]map[string]any{"foo": {}}, + ComponentReadiness: map[string]map[string][]byte{ + "foo": {"readiness.yaml": gate}, + }, + Version: "v1.0.0", + } + if _, genErr := g.Generate(context.Background(), outDir); genErr != nil { + t.Fatalf("Generate: %v", genErr) + } + assertBundleGolden(t, outDir, "testdata/readiness_gate") + + // Stated as an assertion as well as a golden: a golden diff shows that + // bytes moved, not which promise broke. + job, readErr := os.ReadFile(filepath.Join(outDir, "002-foo-readiness", "templates", "readiness.yaml")) + if readErr != nil { + t.Fatalf("read gate manifest from bundle: %v", readErr) + } + for _, want := range []string{ + "helm.sh/hook: post-install,post-upgrade", + "helm.sh/hook-delete-policy: before-hook-creation", + } { + if !strings.Contains(string(job), want) { + t.Errorf("the shipped gate lost %q:\n%s", want, job) + } + } +} + // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- diff --git a/pkg/bundler/deployer/helm/testdata/readiness_gate/001-foo/cluster-values.yaml b/pkg/bundler/deployer/helm/testdata/readiness_gate/001-foo/cluster-values.yaml new file mode 100644 index 000000000..3a4d1cfbb --- /dev/null +++ b/pkg/bundler/deployer/helm/testdata/readiness_gate/001-foo/cluster-values.yaml @@ -0,0 +1,2 @@ +# Generated by AICR +--- diff --git a/pkg/bundler/deployer/helm/testdata/readiness_gate/001-foo/install.sh b/pkg/bundler/deployer/helm/testdata/readiness_gate/001-foo/install.sh new file mode 100644 index 000000000..4d2fd0d52 --- /dev/null +++ b/pkg/bundler/deployer/helm/testdata/readiness_gate/001-foo/install.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "${SCRIPT_DIR}" +# shellcheck source=/dev/null +source ./upstream.env + +# Helm 4 uses server-side apply by default; --force-conflicts lets the +# upgrade overwrite fields that operators (cert-manager, gpu-operator, +# nvsentinel, ...) own on rotated webhook cert Secrets. Helm 3 uses +# client-side apply (no field-manager conflicts) and does not recognize +# the flag, so omit it on Helm 3. +HELM_MAJOR=$(helm version --template '{{.Version}}' 2>/dev/null | sed -nE 's/^v([0-9]+)\..*/\1/p') +FORCE_CONFLICTS_FLAG="" +if [[ "${HELM_MAJOR:-0}" -ge 4 ]]; then + FORCE_CONFLICTS_FLAG="--force-conflicts" +fi + +# CHART carries the full OCI URI for OCI charts and just the chart name for +# HTTP/HTTPS charts. REPO is non-empty only for HTTP/HTTPS charts; the +# ${REPO:+--repo "${REPO}"} expansion adds --repo iff REPO is set. +# When apply-crds.sh ran, it pulled the chart and read the CRDs it applied out +# of that one file. Installing from the same file keeps both phases bound to a +# single artifact; resolving CHART/VERSION again would be a second fetch that a +# mutable tag does not promise returns the same bytes. +CHART_REF="${CHART}" +CHART_VERSION_ARGS=(--version "${VERSION}") +# Under --dry-run, apply-crds.sh above never ran, so no pull happened this +# invocation: any archive present is leftover from an earlier real deploy and +# is stale by definition, not merely unverified. A dry-run install must preview +# what the next real run will actually fetch, not bytes that run never touched. +if [[ -z "${DRY_RUN_FLAG:-}" && -f "${SCRIPT_DIR}/.aicr-chart.tgz" ]]; then + CHART_REF="${SCRIPT_DIR}/.aicr-chart.tgz" + CHART_VERSION_ARGS=() + REPO="" +fi + +helm upgrade --install ${FORCE_CONFLICTS_FLAG} foo "${CHART_REF}" \ + ${REPO:+--repo "${REPO}"} "${CHART_VERSION_ARGS[@]}" \ + --namespace foo --create-namespace \ + -f values.yaml -f cluster-values.yaml \ + ${COMPONENT_WAIT_ARGS:-} ${DRY_RUN_FLAG:-} ${KUBECONFIG_FLAG:-} ${HELM_DEBUG_FLAG:-} diff --git a/pkg/bundler/deployer/helm/testdata/readiness_gate/001-foo/upstream.env b/pkg/bundler/deployer/helm/testdata/readiness_gate/001-foo/upstream.env new file mode 100644 index 000000000..07e54f4e6 --- /dev/null +++ b/pkg/bundler/deployer/helm/testdata/readiness_gate/001-foo/upstream.env @@ -0,0 +1,3 @@ +CHART='foo' +REPO='https://example.com/charts' +VERSION='v1.0.0' diff --git a/pkg/bundler/deployer/helm/testdata/readiness_gate/001-foo/values.yaml b/pkg/bundler/deployer/helm/testdata/readiness_gate/001-foo/values.yaml new file mode 100644 index 000000000..3a4d1cfbb --- /dev/null +++ b/pkg/bundler/deployer/helm/testdata/readiness_gate/001-foo/values.yaml @@ -0,0 +1,2 @@ +# Generated by AICR +--- diff --git a/pkg/bundler/deployer/helm/testdata/readiness_gate/002-foo-readiness/Chart.yaml b/pkg/bundler/deployer/helm/testdata/readiness_gate/002-foo-readiness/Chart.yaml new file mode 100644 index 000000000..fce6475d3 --- /dev/null +++ b/pkg/bundler/deployer/helm/testdata/readiness_gate/002-foo-readiness/Chart.yaml @@ -0,0 +1,23 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v2 +name: foo-readiness +description: Generated wrapper chart for foo local content. +type: application +version: "1.0.0" +appVersion: "v1.0.0" +annotations: + aicr.run/component-version: "v1.0.0" + aicr.run/generated-by: "1.0.0" diff --git a/pkg/bundler/deployer/helm/testdata/readiness_gate/002-foo-readiness/cluster-values.yaml b/pkg/bundler/deployer/helm/testdata/readiness_gate/002-foo-readiness/cluster-values.yaml new file mode 100644 index 000000000..3a4d1cfbb --- /dev/null +++ b/pkg/bundler/deployer/helm/testdata/readiness_gate/002-foo-readiness/cluster-values.yaml @@ -0,0 +1,2 @@ +# Generated by AICR +--- diff --git a/pkg/bundler/deployer/helm/testdata/readiness_gate/002-foo-readiness/install.sh b/pkg/bundler/deployer/helm/testdata/readiness_gate/002-foo-readiness/install.sh new file mode 100644 index 000000000..573b9f533 --- /dev/null +++ b/pkg/bundler/deployer/helm/testdata/readiness_gate/002-foo-readiness/install.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "${SCRIPT_DIR}" + +# Helm 4 uses server-side apply by default; --force-conflicts lets the +# upgrade overwrite fields that operators own on rotated webhook cert +# Secrets. Helm 3 uses client-side apply and does not recognize the flag. +HELM_MAJOR=$(helm version --template '{{.Version}}' 2>/dev/null | sed -nE 's/^v([0-9]+)\..*/\1/p') +FORCE_CONFLICTS_FLAG="" +if [[ "${HELM_MAJOR:-0}" -ge 4 ]]; then + FORCE_CONFLICTS_FLAG="--force-conflicts" +fi + +helm upgrade --install ${FORCE_CONFLICTS_FLAG} foo-readiness ./ \ + --namespace foo --create-namespace \ + -f values.yaml -f cluster-values.yaml \ + ${COMPONENT_WAIT_ARGS:-} ${DRY_RUN_FLAG:-} ${KUBECONFIG_FLAG:-} ${HELM_DEBUG_FLAG:-} diff --git a/pkg/bundler/deployer/helm/testdata/readiness_gate/002-foo-readiness/templates/readiness.yaml b/pkg/bundler/deployer/helm/testdata/readiness_gate/002-foo-readiness/templates/readiness.yaml new file mode 100644 index 000000000..7fd2a3d03 --- /dev/null +++ b/pkg/bundler/deployer/helm/testdata/readiness_gate/002-foo-readiness/templates/readiness.yaml @@ -0,0 +1,86 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: foo-readiness-gate + namespace: foo +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: foo-readiness-gate-foo +rules: + - apiGroups: [""] + resources: ["pods", "nodes", "namespaces", "services", "configmaps", "events"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: ["deployments", "daemonsets", "statefulsets", "replicasets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["batch"] + resources: ["jobs", "cronjobs"] + verbs: ["get", "list", "watch"] + - apiGroups: ["nvidia.com"] + resources: ["*"] + verbs: ["get", "list", "watch"] + - apiGroups: ["operators.coreos.com"] + resources: ["clusterserviceversions"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: foo-readiness-gate-foo +subjects: + - kind: ServiceAccount + name: foo-readiness-gate + namespace: foo +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: foo-readiness-gate-foo +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: foo-readiness-bundle + namespace: foo +data: + foo.yaml: | + apiVersion: chainsaw.kyverno.io/v1alpha1 + kind: Test +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: foo-readiness-gate + namespace: foo + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation +spec: + backoffLimit: 6 + template: + spec: + restartPolicy: Never + serviceAccountName: foo-readiness-gate + containers: + - name: gate + image: nvcr.io/nvidia/aicr:v1.0.0 + imagePullPolicy: IfNotPresent + args: + - --bundle-dir=/bundle + - --namespace=foo + - --timeout=2m0s + - --poll-interval=10s + - --stability-window=30s + - --max-wait=1h30m0s + volumeMounts: + - name: bundle + mountPath: /bundle + readOnly: true + volumes: + - name: bundle + configMap: + name: foo-readiness-bundle diff --git a/pkg/bundler/deployer/helm/testdata/readiness_gate/002-foo-readiness/values.yaml b/pkg/bundler/deployer/helm/testdata/readiness_gate/002-foo-readiness/values.yaml new file mode 100644 index 000000000..3a4d1cfbb --- /dev/null +++ b/pkg/bundler/deployer/helm/testdata/readiness_gate/002-foo-readiness/values.yaml @@ -0,0 +1,2 @@ +# Generated by AICR +--- diff --git a/pkg/bundler/deployer/helm/testdata/readiness_gate/README.md b/pkg/bundler/deployer/helm/testdata/readiness_gate/README.md new file mode 100644 index 000000000..2f1a05fdf --- /dev/null +++ b/pkg/bundler/deployer/helm/testdata/readiness_gate/README.md @@ -0,0 +1,150 @@ +# AI Cluster Runtime Deployment + +Recipe Version: v0.1.0 +Bundler Version: v1.0.0 + +Per-component bundle for deploying NVIDIA AI Cluster Runtime components +for GPU-accelerated Kubernetes workloads. + +## Configuration + + + +## Components + +The following components are included (deployed in order). Each component +lives in a numbered `NNN-/` folder and is installed as a Helm release +via its own `install.sh`: + +| Component | Version | Namespace | Source | +|-----------|---------|-----------|--------| +| foo | v1.0.0 | foo | foo (https://example.com/charts) | + + + + +## Quick Start + +Run the included deployment script: + +```bash +chmod +x deploy.sh +./deploy.sh +``` + +Use `--no-wait` to skip Helm chart-level waiting where AICR uses `--wait` (keeps `--timeout` for hooks): + +```bash +./deploy.sh --no-wait +``` + +> **Note:** The deploy script's final status reflects install/apply results. If `--best-effort` was used, one or more components may still have failed; check warning lines and logs. This does **not** guarantee the cluster is ready to schedule workloads — operator-driven cluster convergence (CRD reconciliation, node tuning, plugin registration, etc.) continues asynchronously after the script exits, in operator-specific ways. See the [AICR CLI Reference](https://github.com/NVIDIA/aicr/blob/main/docs/user/cli-reference.md#deploy-script-behavior-deploysh) for details. + +## Manual Installation + +Each component folder contains an `install.sh` that runs `helm upgrade --install` +with the right arguments baked in. To install a single component manually: + +```bash +cd NNN- +bash install.sh +``` + +> **Helm 4 vs Helm 3:** On Helm 4 (server-side apply by default), each +> `install.sh` automatically passes `--force-conflicts` so the upgrade can +> overwrite fields that operators (cert-manager, gpu-operator, nvsentinel, +> grove, ...) own on their rotated webhook cert Secrets — without it the +> upgrade fails on field-manager conflicts. On Helm 3 (client-side apply, +> no field-manager conflicts) the flag is omitted; the script detects the +> Helm major version at run time, so the same bundle works with either +> binary. + +## Customization + +Each component folder has its own `values.yaml` (static) and `cluster-values.yaml` +(dynamic, per-cluster). Edit either before deploying: + +```bash +vim NNN-/values.yaml +vim NNN-/cluster-values.yaml +``` + +## Upgrade + +Re-run the per-component install.sh to upgrade an already-installed release: + +```bash +cd NNN- +bash install.sh +``` + +> **CRDs on upgrade.** Helm installs a chart's `crds/` directory on first +> install and never touches it again, so a chart bump whose CRDs changed would +> otherwise run the new controller against the old schema. Folders that also +> contain an `apply-crds.sh` have `install.sh` run it first. It pulls the +> pinned chart once, reads the CRDs out of that archive, and creates or +> replaces each one, so a field the new chart removes actually disappears; +> server-side apply would leave fields Helm still owns in place. Only +> components audited as the sole owner of every CRD they ship get this, and +> only while the ref matches the registry's pinned source, chart, and version. +> Those folders need `kubectl` and `timeout` (GNU coreutils) on `$PATH` in +> addition to `helm`; the script refuses to run rather than run unbounded +> inside a deploy, so on macOS install coreutils or apply the CRDs by hand. +> Every helm and kubectl call it makes is bounded, 30s by default and +> overridable with `AICR_CRD_STEP_TIMEOUT`. The bound is per call and +> `deploy.sh` retries a failing component, so the budget it consumes is a +> multiple of that. `KUBECONFIG_FLAG` reaches this step as well, but only +> `--kube-context` and `--kubeconfig` are supported there; any other helm +> connection flag stops the step rather than apply CRDs to an unintended +> cluster. The error names the option only, never its argument, so a flag +> carrying a credential does not reach the log. The step is +> skipped when `DRY_RUN_FLAG` is set, and when +> neither the release nor any of the chart's CRDs exist yet, since only then +> does `helm install` create them. A release that was uninstalled leaves its +> CRDs behind, so a reinstall still applies them. Only components audited as +> the sole owner of every CRD they ship get the script; for the rest, replacing +> CRDs is unsafe because another component's release ships the same CRD with a +> different schema. + +## Uninstall + +Bundles do not ship an `undeploy.sh`. Uninstall releases in reverse +deployment order using `helm uninstall` directly — one command per +`NNN-/` folder the deploy script installs, including any +injected `*-pre` / `*-post` auxiliaries: + +```bash +helm uninstall foo-readiness -n foo +``` + +```bash +helm uninstall foo -n foo +``` + +CRDs installed by these charts are intentionally not deleted by Helm; remove +them only when you are sure no other release depends on them. See the +[deployer-native uninstall walkthrough](https://github.com/NVIDIA/aicr/blob/main/docs/user/cli-reference.md#bundle-uninstall) in the AICR CLI reference for details on +PVC handling, namespace teardown, and the equivalent paths for ArgoCD and +ArgoCD+Helm bundles. + +## Troubleshooting + +### Check deployment status + +```bash +kubectl get pods -A | grep -E 'foo' +``` + +### View component logs + +Inspect a single component's pods (replace `` and `` +with one of the entries from the table above): + +```bash +kubectl logs -n -l app.kubernetes.io/instance= +``` + + +## References + +- [AICR CLI Reference](https://github.com/NVIDIA/aicr/blob/main/docs/user/cli-reference.md) diff --git a/pkg/bundler/deployer/helm/testdata/readiness_gate/deploy.sh b/pkg/bundler/deployer/helm/testdata/readiness_gate/deploy.sh new file mode 100644 index 000000000..c5c2e7e8e --- /dev/null +++ b/pkg/bundler/deployer/helm/testdata/readiness_gate/deploy.sh @@ -0,0 +1,389 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +# AICR Deployment Script +# Generated by AICR Bundler v1.0.0 +# +# Usage: ./deploy.sh [--no-wait] [--best-effort] [--retries N] +# --no-wait Skip Helm chart-level wait where AICR uses --wait (keeps --timeout for hooks) +# --best-effort Continue past individual component failures (log warnings) +# --retries N Retry failed helm/kubectl operations N times with backoff (default: 5, 0 = fail-fast) +# +# This script is optional — each component subdirectory has its own install.sh +# with a single baked-in `helm upgrade --install` command. For detailed behavior +# docs (CRD ordering, async components, error handling), see the AICR CLI Reference: +# https://github.com/NVIDIA/aicr/blob/main/docs/user/cli-reference.md#deploy-script-behavior-deploysh + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Run helm commands from a temp directory to prevent local chart directories +# (e.g., bundle/001-nodewright-operator/) from shadowing remote chart references. +HELM_WORKDIR="$(mktemp -d)" +trap 'rm -rf "${HELM_WORKDIR}"; exit 130' INT TERM +trap 'rm -rf "${HELM_WORKDIR}"' EXIT + +HELM_TIMEOUT="10m" +NO_WAIT=false +BEST_EFFORT=false +FAILED_COMPONENTS="" +MAX_RETRIES=5 + +while [[ $# -gt 0 ]]; do + case "$1" in + --no-wait) NO_WAIT=true; shift ;; + --best-effort) BEST_EFFORT=true; shift ;; + --retries) + if [[ $# -lt 2 ]]; then echo "Error: --retries requires a value"; exit 1; fi + if ! [[ "$2" =~ ^[0-9]+$ ]]; then echo "Error: --retries requires a non-negative integer"; exit 1; fi + MAX_RETRIES="$2"; shift 2 ;; + *) echo "Error: unknown option: $1"; echo "Usage: ./deploy.sh [--no-wait] [--best-effort] [--retries N]"; exit 1 ;; + esac +done +# ============================================================================== +# Output helpers (respects NO_COLOR and non-TTY) +# ============================================================================== +if [[ -t 1 && -z "${NO_COLOR:-}" ]]; then + _G=$'\033[0;32m';_R=$'\033[0;31m';_Y=$'\033[1;33m' + _B=$'\033[1m';_D=$'\033[2m';_X=$'\033[0m' +else + _G='';_R='';_Y='';_B='';_D='';_X='' +fi + +function _ok() { printf '%s✓%s %s\n' "${_G}" "${_X}" "$*"; } +function _fail() { printf '%s✗%s %s\n' "${_R}" "${_X}" "$*"; } +function _warn_line(){ printf '%s⚠%s %s\n' "${_Y}" "${_X}" "$*"; } + +function _step_header() { + local manual_dir + printf -v manual_dir '%q' "$5" + printf '\n%s┌─ [%s/%s] %s → %s%s\n' "${_B}" "$1" "$2" "$3" "$4" "${_X}" + printf '%s│ Manual (approx, set KUBECONFIG_FLAG/DRY_RUN_FLAG/COMPONENT_WAIT_ARGS as needed): cd %s && bash install.sh%s\n' "${_D}" "${manual_dir}" "${_X}" +} + +function _step_ok() { + printf '%s└─ ✓%s %s installed\n' "${_G}" "${_X}" "$1" +} + +function _step_fail() { + printf '%s└─ ✗%s %s FAILED (after %s attempts)\n' "${_R}" "${_X}" "$1" "$2" +} + +function _step_retry() { + printf '%s ↺%s %s: attempt %s/%s failed, retrying in %ss...\n' "${_Y}" "${_X}" "$1" "$2" "$3" "$4" +} + +# Export env vars consumed by each folder's install.sh (rendered by localformat). +# DRY_RUN_FLAG / KUBECONFIG_FLAG / HELM_DEBUG_FLAG default to empty strings. +export DRY_RUN_FLAG="${DRY_RUN_FLAG:-}" +export KUBECONFIG_FLAG="${KUBECONFIG_FLAG:-}" +export HELM_DEBUG_FLAG="${HELM_DEBUG_FLAG:-}" + +function helm_failed() { + if [[ "${BEST_EFFORT}" == "true" ]]; then + _warn_line "$1 install failed, continuing (--best-effort)" + FAILED_COMPONENTS="${FAILED_COMPONENTS} $1" + else + exit 1 + fi +} + +# Compute backoff delay from attempt number (1-indexed). +# Examples: attempt 1→5s, 2→20s, 3→45s, 4→80s, 5→120s (cap) +function backoff_seconds() { + local attempt=$1 + local seconds=$(( attempt * attempt * 5 )) + if [[ ${seconds} -gt 120 ]]; then seconds=120; fi + echo "${seconds}" +} + +# Clean up stale Helm hook Jobs before retrying. When a hook Job (e.g., +# crd-upgrader) times out or fails, it stays in the namespace and blocks +# subsequent install attempts with "Job not ready" errors. +# Helm hooks are identified by the helm.sh/hook *annotation* (not a label), +# so we list all non-succeeded Jobs and check each individually via JSON. +function cleanup_helm_hooks() { + local namespace="$1" + local job_names + job_names=$(kubectl get jobs -n "${namespace}" \ + --field-selector=status.successful=0 \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' \ + 2>/dev/null || true) + if [[ -z "${job_names}" ]]; then + return + fi + while IFS= read -r name; do + [[ -z "${name}" ]] && continue + # Get the full Job JSON to reliably check annotations and status + local job_json + job_json=$(kubectl get job "${name}" -n "${namespace}" -o json 2>/dev/null || true) + [[ -z "${job_json}" ]] && continue + # Skip non-hook Jobs (no helm.sh/hook annotation) + local hook_val + hook_val=$(echo "${job_json}" | grep -o '"helm.sh/hook"' || true) + [[ -z "${hook_val}" ]] && continue + # Capture diagnostics before deleting. This helps diagnose transient hook + # failures (e.g., dynamo ssh-keygen) that are otherwise lost after cleanup. + echo " --- Failed hook Job ${name} diagnostics ---" + kubectl describe job "${name}" -n "${namespace}" 2>/dev/null | tail -50 || true + local pod_names + pod_names=$(kubectl get pods -n "${namespace}" -l "job-name=${name}" \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true) + for pod_name in ${pod_names}; do + echo " --- Hook pod ${pod_name} describe ---" + kubectl describe pod "${pod_name}" -n "${namespace}" 2>/dev/null | tail -50 || true + done + echo " --- End diagnostics for ${name} ---" + # Delete any non-succeeded hook Job. This function only runs after a Helm + # failure, so any hook Job without a successful completion is blocking the + # retry — whether it failed, is stuck Pending (timed out before the pod + # started), or is still active with a stuck container. + echo " Cleaning up stale Helm hook Job ${name} in ${namespace}..." + kubectl delete job "${name}" -n "${namespace}" --ignore-not-found 2>/dev/null || true + done <<< "${job_names}" +} + +function dump_kai_scheduler_helm_diagnostics() { + local namespace="$1" + if [[ "${namespace}" != "kai-scheduler" ]]; then + return + fi + + echo " --- ${namespace} diagnostics ---" + echo " Jobs:" + kubectl get jobs -n "${namespace}" 2>/dev/null || true + echo " Job descriptions:" + kubectl describe jobs -n "${namespace}" 2>/dev/null || true + echo " Pods:" + kubectl get pods -n "${namespace}" -o wide 2>/dev/null || true + echo " Pod descriptions:" + kubectl describe pods -n "${namespace}" 2>/dev/null || true + echo " Recent events:" + kubectl get events -n "${namespace}" --sort-by='.lastTimestamp' 2>/dev/null | tail -30 || true + echo " --- End ${namespace} diagnostics ---" +} + +# Components that use operator patterns with custom resources that reconcile +# asynchronously. Helm --wait may time out waiting for CR readiness even though +# all pods start successfully. These components are installed without --wait. +ASYNC_COMPONENTS="kai-scheduler" + +# ============================================================================== +# Pre-flight checks +# ============================================================================== +# Verify the cluster is clean before deploying. Stale webhooks, terminating +# namespaces, and orphaned API services from a previous install can block pod +# creation and namespace deletion, causing silent deployment failures. + +printf '\n%s══ Pre-flight checks ══════════════════════════════════════════════%s\n' "${_B}" "${_X}" + +preflight_failed=false + +# Bundle namespace list (deduplicated) +BUNDLE_NAMESPACES=$(echo "foo " | tr ' ' '\n' | sort -u | tr '\n' ' ') + +# Check for terminating namespaces that overlap with our components +for ns in ${BUNDLE_NAMESPACES}; do + phase=$(kubectl get ns "${ns}" -o jsonpath='{.status.phase}' 2>/dev/null || true) + if [[ "${phase}" == "Terminating" ]]; then + echo "ERROR: namespace '${ns}' is still terminating from a previous install." + echo " Wait for it to finish, or force-finalize with:" + echo " kubectl get ns ${ns} -o json | jq '.spec.finalizers=[]' | kubectl replace --raw /api/v1/namespaces/${ns}/finalize -f -" + preflight_failed=true + fi +done + +# Check for stale webhooks whose backing services no longer exist. +# Scoped to bundle namespaces only to avoid false positives from unrelated +# platform webhooks in shared clusters. +if command -v jq &>/dev/null; then + for kind in mutatingwebhookconfigurations validatingwebhookconfigurations; do + while IFS=$'\t' read -r wh_name svc_ns svc_name; do + # Only check webhooks pointing to our bundle namespaces + is_bundle_ns=false + for ns in ${BUNDLE_NAMESPACES}; do + [[ "${svc_ns}" == "${ns}" ]] && is_bundle_ns=true && break + done + [[ "${is_bundle_ns}" == "false" ]] && continue + + # Use explicit NotFound check to avoid false positives from transient errors + svc_check=$(kubectl get svc "${svc_name}" -n "${svc_ns}" 2>&1) || true + if echo "${svc_check}" | grep -q "NotFound\|not found"; then + echo "ERROR: ${kind} '${wh_name}' references non-existent service ${svc_ns}/${svc_name}." + echo " This will block pod/resource creation. Delete with: kubectl delete ${kind} ${wh_name}" + preflight_failed=true + fi + done < <(kubectl get "${kind}" -o json 2>/dev/null | \ + jq -r '.items[] | .metadata.name as $wh | .webhooks[]? | select(.clientConfig.service != null) | [$wh, .clientConfig.service.namespace, .clientConfig.service.name] | @tsv' 2>/dev/null || true) + done +else + echo "NOTE: jq not found — skipping webhook pre-flight checks. Install jq for full pre-flight validation." +fi + +# Check for stale API services (e.g., custom.metrics.k8s.io from prometheus-adapter) +if command -v jq &>/dev/null; then + for api_svc in $(kubectl get apiservices -o json 2>/dev/null | jq -r '.items[] | select(.status.conditions[]? | .type == "Available" and .status == "False") | .metadata.name' 2>/dev/null || true); do + echo "WARNING: API service '${api_svc}' is unavailable. This can block namespace deletion." + echo " Delete with: kubectl delete apiservice ${api_svc}" + # API service issues are warnings, not hard failures — they don't block deployment directly + done +else + echo "NOTE: jq not found — skipping API service pre-flight checks." +fi + +# Check for orphaned CRDs from previous deployments. +# Scoped to CRD groups belonging to components in this bundle to avoid +# false positives from unrelated platform installs on shared clusters. +ORPHANED_CRD_GROUPS="" +for group in ${ORPHANED_CRD_GROUPS}; do + orphaned=$(kubectl get crd -o name 2>/dev/null | grep "\.${group}$" || true) + if [[ -n "${orphaned}" ]]; then + echo "WARNING: orphaned CRDs from previous deployment: ${orphaned}" + echo " These may cause conflicts. Delete with: kubectl delete ${orphaned}" + fi +done + +# Check for stale nodewright node taints from a previous deployment. +# Only remove taints if nodewright-operator is NOT already running (i.e., fresh deploy). +# If the operator is running, taints are legitimate scheduling guards. + +if [[ "${preflight_failed}" == "true" ]]; then + echo "" + _fail "Pre-flight checks failed. Fix the issues above before deploying." + echo " To clean up partial state, run 'helm uninstall -n ' for each affected component, then retry." + exit 1 +fi + +_ok "Pre-flight checks passed" +printf '\n%s══ Deploying AICR components ══════════════════════════════════════%s\n' "${_B}" "${_X}" + +# ============================================================================== +# Install loop +# ============================================================================== +# Generic install loop. Each folder's install.sh is rendered by localformat +# with the right helm command baked in — deploy.sh has no per-component +# knowledge here. Per-component special-case logic (async wait, DRA plugin +# restart) runs in the post-install blocks below, matched by component name. +cd "${HELM_WORKDIR}" + +# Pre-count folders for accurate [N/M] progress display. +TOTAL_STEPS=0 +for _d in "${SCRIPT_DIR}"/[0-9][0-9][0-9]-*/; do + [[ -d "${_d}" ]] && TOTAL_STEPS=$((TOTAL_STEPS+1)) +done + +STEP=0 + +for dir in "${SCRIPT_DIR}"/[0-9][0-9][0-9]-*/; do + [[ -d "${dir}" ]] || continue + dir="${dir%/}" + base="${dir##*/}" + name="${base#[0-9][0-9][0-9]-}" + STEP=$((STEP+1)) + # Source the namespace from the folder's install.sh — not the folder + # basename — because the helm release name and its target namespace can + # differ (e.g. nodewright-operator → namespace skyhook; gpu-operator-post → + # namespace gpu-operator). cleanup_helm_hooks and the kai diagnostics + # both operate on the namespace. + namespace=$(awk '{ for (i=1;i