fix(bundler): the helm readiness gate blocks and re-runs on upgrade

pkg/bundler/deployer/helm/helm_test.go

@@ -29,8 +29,10 @@ import (
 
 	"gopkg.in/yaml.v3"
 
+	"github.com/NVIDIA/aicr/pkg/bundler/config"
 	"github.com/NVIDIA/aicr/pkg/bundler/deployer"
 	"github.com/NVIDIA/aicr/pkg/bundler/deployer/localformat"
+	"github.com/NVIDIA/aicr/pkg/bundler/gatemanifest"
 	"github.com/NVIDIA/aicr/pkg/component"
 	"github.com/NVIDIA/aicr/pkg/recipe"
 )
@@ -1403,6 +1405,65 @@ func TestBundleGolden_OwnsCRDsChartOverride(t *testing.T) {
 	assertBundleGolden(t, outDir, "testdata/owns_crds_chart_override")
 }
 
+// TestBundleGolden_ReadinessGate pins the readiness folder a helm bundle
+// ships, which until now had no golden at all.
+//
+// The absence was the bug's cover. gatemanifest.Render annotates the gate Job
+// as a post-install,post-upgrade hook with
+// hook-delete-policy: before-hook-creation, and both annotations are
+// load-bearing under plain Helm:
+//
+//   - the hook is what makes deploy.sh block. It passes --wait without
+//     --wait-for-jobs, which is correct only because --wait blocks on hook
+//     completion. A bare Job under --wait alone returns as soon as the object
+//     exists, so the "gate" would let dependents start against a cluster it
+//     has not finished checking.
+//
+//   - before-hook-creation is what makes it re-run. A Job's spec.template is
+//     immutable, so an identical manifest is a no-op patch; without the
+//     delete-and-recreate the gate asserts once, at install, and every
+//     subsequent upgrade ships unverified.
+//
+// A golden here fails loudly if either annotation is stripped again.
+func TestBundleGolden_ReadinessGate(t *testing.T) {
+	gate, err := gatemanifest.Render("foo", "nvcr.io/nvidia/aicr:v1.0.0",
+		[]byte("apiVersion: chainsaw.kyverno.io/v1alpha1\nkind: Test\n"),
+		config.DeployerHelm)
+	if err != nil {
+		t.Fatalf("render gate manifest: %v", err)
+	}
+
+	outDir := t.TempDir()
+	g := &Generator{
+		RecipeResult: singleComponentRecipe(
+			"foo", "foo", "foo", "v1.0.0", "https://example.com/charts"),
+		ComponentValues: map[string]map[string]any{"foo": {}},
+		ComponentReadiness: map[string]map[string][]byte{
+			"foo": {"readiness.yaml": gate},
+		},
+		Version: "v1.0.0",
+	}
+	if _, genErr := g.Generate(context.Background(), outDir); genErr != nil {
+		t.Fatalf("Generate: %v", genErr)
+	}
+	assertBundleGolden(t, outDir, "testdata/readiness_gate")
+
+	// Stated as an assertion as well as a golden: a golden diff shows that
+	// bytes moved, not which promise broke.
+	job, readErr := os.ReadFile(filepath.Join(outDir, "002-foo-readiness", "templates", "readiness.yaml"))
+	if readErr != nil {
+		t.Fatalf("read gate manifest from bundle: %v", readErr)
+	}
+	for _, want := range []string{
+		"helm.sh/hook: post-install,post-upgrade",
+		"helm.sh/hook-delete-policy: before-hook-creation",
+	} {
+		if !strings.Contains(string(job), want) {
+			t.Errorf("the shipped gate lost %q:\n%s", want, job)
+		}
+	}
+}
+
 // ---------------------------------------------------------------------------
 // Helpers
 // ---------------------------------------------------------------------------

pkg/bundler/deployer/helm/testdata/readiness_gate/001-foo/cluster-values.yaml

@@ -0,0 +1,2 @@
+# Generated by AICR
+---

pkg/bundler/deployer/helm/testdata/readiness_gate/001-foo/install.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "${SCRIPT_DIR}"
+# shellcheck source=/dev/null
+source ./upstream.env
+
+# Helm 4 uses server-side apply by default; --force-conflicts lets the
+# upgrade overwrite fields that operators (cert-manager, gpu-operator,
+# nvsentinel, ...) own on rotated webhook cert Secrets. Helm 3 uses
+# client-side apply (no field-manager conflicts) and does not recognize
+# the flag, so omit it on Helm 3.
+HELM_MAJOR=$(helm version --template '{{.Version}}' 2>/dev/null | sed -nE 's/^v([0-9]+)\..*/\1/p')
+FORCE_CONFLICTS_FLAG=""
+if [[ "${HELM_MAJOR:-0}" -ge 4 ]]; then
+  FORCE_CONFLICTS_FLAG="--force-conflicts"
+fi
+
+# CHART carries the full OCI URI for OCI charts and just the chart name for
+# HTTP/HTTPS charts. REPO is non-empty only for HTTP/HTTPS charts; the
+# ${REPO:+--repo "${REPO}"} expansion adds --repo iff REPO is set.
+# When apply-crds.sh ran, it pulled the chart and read the CRDs it applied out
+# of that one file. Installing from the same file keeps both phases bound to a
+# single artifact; resolving CHART/VERSION again would be a second fetch that a
+# mutable tag does not promise returns the same bytes.
+CHART_REF="${CHART}"
+CHART_VERSION_ARGS=(--version "${VERSION}")
+# Under --dry-run, apply-crds.sh above never ran, so no pull happened this
+# invocation: any archive present is leftover from an earlier real deploy and
+# is stale by definition, not merely unverified. A dry-run install must preview
+# what the next real run will actually fetch, not bytes that run never touched.
+if [[ -z "${DRY_RUN_FLAG:-}" && -f "${SCRIPT_DIR}/.aicr-chart.tgz" ]]; then
+  CHART_REF="${SCRIPT_DIR}/.aicr-chart.tgz"
+  CHART_VERSION_ARGS=()
+  REPO=""
+fi
+
+helm upgrade --install ${FORCE_CONFLICTS_FLAG} foo "${CHART_REF}" \
+  ${REPO:+--repo "${REPO}"} "${CHART_VERSION_ARGS[@]}" \
+  --namespace foo --create-namespace \
+  -f values.yaml -f cluster-values.yaml \
+  ${COMPONENT_WAIT_ARGS:-} ${DRY_RUN_FLAG:-} ${KUBECONFIG_FLAG:-} ${HELM_DEBUG_FLAG:-}

pkg/bundler/deployer/helm/testdata/readiness_gate/001-foo/upstream.env

@@ -0,0 +1,3 @@
+CHART='foo'
+REPO='https://example.com/charts'
+VERSION='v1.0.0'

pkg/bundler/deployer/helm/testdata/readiness_gate/001-foo/values.yaml

@@ -0,0 +1,2 @@
+# Generated by AICR
+---

pkg/bundler/deployer/helm/testdata/readiness_gate/002-foo-readiness/Chart.yaml

@@ -0,0 +1,23 @@
+# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v2
+name: foo-readiness
+description: Generated wrapper chart for foo local content.
+type: application
+version: "1.0.0"
+appVersion: "v1.0.0"
+annotations:
+  aicr.run/component-version: "v1.0.0"
+  aicr.run/generated-by: "1.0.0"

pkg/bundler/deployer/helm/testdata/readiness_gate/002-foo-readiness/cluster-values.yaml

@@ -0,0 +1,2 @@
+# Generated by AICR
+---

pkg/bundler/deployer/helm/testdata/readiness_gate/002-foo-readiness/install.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "${SCRIPT_DIR}"
+
+# Helm 4 uses server-side apply by default; --force-conflicts lets the
+# upgrade overwrite fields that operators own on rotated webhook cert
+# Secrets. Helm 3 uses client-side apply and does not recognize the flag.
+HELM_MAJOR=$(helm version --template '{{.Version}}' 2>/dev/null | sed -nE 's/^v([0-9]+)\..*/\1/p')
+FORCE_CONFLICTS_FLAG=""
+if [[ "${HELM_MAJOR:-0}" -ge 4 ]]; then
+  FORCE_CONFLICTS_FLAG="--force-conflicts"
+fi
+
+helm upgrade --install ${FORCE_CONFLICTS_FLAG} foo-readiness ./ \
+  --namespace foo --create-namespace \
+  -f values.yaml -f cluster-values.yaml \
+  ${COMPONENT_WAIT_ARGS:-} ${DRY_RUN_FLAG:-} ${KUBECONFIG_FLAG:-} ${HELM_DEBUG_FLAG:-}

pkg/bundler/deployer/helm/testdata/readiness_gate/002-foo-readiness/templates/readiness.yaml

@@ -0,0 +1,86 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: foo-readiness-gate
+  namespace: foo
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: foo-readiness-gate-foo
+rules:
+  - apiGroups: [""]
+    resources: ["pods", "nodes", "namespaces", "services", "configmaps", "events"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["apps"]
+    resources: ["deployments", "daemonsets", "statefulsets", "replicasets"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["batch"]
+    resources: ["jobs", "cronjobs"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["nvidia.com"]
+    resources: ["*"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["operators.coreos.com"]
+    resources: ["clusterserviceversions"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["apiextensions.k8s.io"]
+    resources: ["customresourcedefinitions"]
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: foo-readiness-gate-foo
+subjects:
+  - kind: ServiceAccount
+    name: foo-readiness-gate
+    namespace: foo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: foo-readiness-gate-foo
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: foo-readiness-bundle
+  namespace: foo
+data:
+  foo.yaml: |
+    apiVersion: chainsaw.kyverno.io/v1alpha1
+    kind: Test
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: foo-readiness-gate
+  namespace: foo
+  annotations:
+    helm.sh/hook: post-install,post-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation
+spec:
+  backoffLimit: 6
+  template:
+    spec:
+      restartPolicy: Never
+      serviceAccountName: foo-readiness-gate
+      containers:
+        - name: gate
+          image: nvcr.io/nvidia/aicr:v1.0.0
+          imagePullPolicy: IfNotPresent
+          args:
+            - --bundle-dir=/bundle
+            - --namespace=foo
+            - --timeout=2m0s
+            - --poll-interval=10s
+            - --stability-window=30s
+            - --max-wait=1h30m0s
+          volumeMounts:
+            - name: bundle
+              mountPath: /bundle
+              readOnly: true
+      volumes:
+        - name: bundle
+          configMap:
+            name: foo-readiness-bundle

pkg/bundler/deployer/helm/testdata/readiness_gate/002-foo-readiness/values.yaml

@@ -0,0 +1,2 @@
+# Generated by AICR
+---