diff --git a/Makefile b/Makefile index 361b50dc94..a06d87731c 100644 --- a/Makefile +++ b/Makefile @@ -111,8 +111,11 @@ UNBOUNDED_OPERATOR_API_SERVER_ENDPOINT ?= UNBOUNDED_OPERATOR_IMAGE_REGISTRY ?= $(CONTAINER_REGISTRY) UNBOUNDED_OPERATOR_REAP_LEGACY_RESOURCES ?= true export UNBOUNDED_OPERATOR_API_SERVER_ENDPOINT -UNBOUNDED_OPERATOR_MANIFEST_TEMPLATES_DIR := deploy/unbounded-operator UNBOUNDED_OPERATOR_MANIFEST_RENDERED_DIR := deploy/unbounded-operator/rendered +UNBOUNDED_OPERATOR_CHART_DIR := deploy/unbounded-operator/chart +UNBOUNDED_OPERATOR_RENDER_DIR := tmp/unbounded-operator-render +UNBOUNDED_OPERATOR_CHART_VERSION ?= 0.0.0-dev +UNBOUNDED_OPERATOR_CHART_APP_VERSION ?= $(VERSION_TAG) TOKEN_REFRESHER_NAMESPACE ?= $(UNBOUNDED_NAMESPACE) TOKEN_REFRESHER_MANIFEST_TEMPLATES_DIR := deploy/token-refresher @@ -235,7 +238,7 @@ NET_FRONTEND_CACHE_FILE := $(NET_FRONTEND_DIST_DIR)/.frontend-build-key REACT_DEV ?= false .PHONY: all help fmt lint lint-actions test build vulncheck check-deps kubectl-unbounded kubectl-unbounded-build install-tools install-protoc install-helm install-kind generate kubectl-unbounded forge relctl relctl-build agent-artifacts-builder agent-artifacts-builder-build orcadev unbounded-agent machina machina-build machina-oci machina-oci-push machina-manifests machine-ops-controller machine-ops-controller-build machine-ops-controller-oci machine-ops-controller-oci-push machine-ops-manifests metalman metalman-build metalman-oci metalman-oci-push unbounded-operator unbounded-operator-build unbounded-operator-manifests playpen-manifests e2e-gantry e2e-playpen e2e-racer gomod docs-serve unbounded-net-controller unbounded-net-controller-build unbounded-net-node unbounded-net-node-build unbounded-net-routeplan-debug unping unping-build unroute unroute-build license-check notice notice-check gantry gantry-build gantry-manifests inventory-manifests -.PHONY: net-frontend net-frontend-clean net-ebpf-build net-ebpf-generate net-ebpf-verify net-manifests gantry-chart-lint gantry-chart-package release-bom release-manifests unbounded-operator-release-manifest +.PHONY: net-frontend net-frontend-clean net-ebpf-build net-ebpf-generate net-ebpf-verify net-manifests gantry-chart-lint gantry-chart-package unbounded-operator-chart-lint unbounded-operator-chart-package release-bom release-manifests unbounded-operator-release-manifest .PHONY: image-machina-local image-token-refresher-local image-machine-ops-controller-local image-metalman-local image-unbounded-operator-local image-unbounded-operator-push image-playpen-local image-net-controller-local image-net-node-local image-gantry-local image-gantry-push images-local .PHONY: image-net-controller-push image-net-node-push images-net-all images-net-all-push @@ -349,6 +352,8 @@ help: ## Show this help @echo " unbounded-operator-manifests Render unbounded-operator manifests into deploy/unbounded-operator/rendered" @echo " gantry-chart-lint Validate the standalone Gantry Helm chart" @echo " gantry-chart-package Package the standalone Gantry Helm chart" + @echo " unbounded-operator-chart-lint Validate the unbounded-operator Helm chart" + @echo " unbounded-operator-chart-package Package the unbounded-operator Helm chart" @echo " unbounded-operator-release-manifest Build a versioned, directly applicable operator manifest under build/" @echo "" @echo "Net Kubernetes (apply to current kubectl context):" @@ -569,13 +574,13 @@ ifdef CI # In CI each job is independent; skip chained prerequisites. test: machina-manifests token-refresher-manifests machine-ops-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Run all tests with race detector - $(GOTEST) -race ./... + HELM=$(HELM) $(GOTEST) -race ./... else # Locally, chain test -> lint for convenience. test: lint machina-manifests token-refresher-manifests machine-ops-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Run all tests (implies lint) - $(GOTEST) ./... + HELM=$(HELM) $(GOTEST) ./... endif @@ -1226,19 +1231,37 @@ token-refresher-manifests: ## Render token-refresher manifests into deploy/token --set Image=$(TOKEN_REFRESHER_IMAGE) @echo "Rendered token-refresher manifests into $(TOKEN_REFRESHER_MANIFEST_RENDERED_DIR) (image: $(TOKEN_REFRESHER_IMAGE))" -unbounded-operator-manifests: ## Render unbounded-operator manifests into deploy/unbounded-operator/rendered +unbounded-operator-manifests: $(HELM) ## Render unbounded-operator manifests into deploy/unbounded-operator/rendered @mkdir -p $(UNBOUNDED_OPERATOR_MANIFEST_RENDERED_DIR) @find $(UNBOUNDED_OPERATOR_MANIFEST_RENDERED_DIR) -mindepth 1 -not -name .gitignore -delete - $(GOCMD) run ./hack/cmd/render-manifests \ - --templates-dir $(UNBOUNDED_OPERATOR_MANIFEST_TEMPLATES_DIR) \ - --output-dir $(UNBOUNDED_OPERATOR_MANIFEST_RENDERED_DIR) \ - --set Namespace=$(UNBOUNDED_OPERATOR_NAMESPACE) \ - --set OperatorImage=$(UNBOUNDED_OPERATOR_IMAGE) \ - --set ImageRegistry=$(UNBOUNDED_OPERATOR_IMAGE_REGISTRY) \ - --set "APIServerEndpoint=$${UNBOUNDED_OPERATOR_API_SERVER_ENDPOINT}" \ - --set ReapLegacyResources=$(UNBOUNDED_OPERATOR_REAP_LEGACY_RESOURCES) + @rm -rf $(UNBOUNDED_OPERATOR_RENDER_DIR) + @mkdir -p $(UNBOUNDED_OPERATOR_RENDER_DIR) + @# --set-file passes the endpoint verbatim; --set-string would split on commas. + @printf '%s' "$${UNBOUNDED_OPERATOR_API_SERVER_ENDPOINT}" > $(UNBOUNDED_OPERATOR_RENDER_DIR)/api-server-endpoint + $(HELM) template unbounded-operator $(UNBOUNDED_OPERATOR_CHART_DIR) \ + --namespace $(UNBOUNDED_OPERATOR_NAMESPACE) \ + --values $(UNBOUNDED_OPERATOR_CHART_DIR)/values-static.yaml \ + --skip-schema-validation \ + --set-string image.reference=$(UNBOUNDED_OPERATOR_IMAGE) \ + --set-string imageRegistry=$(UNBOUNDED_OPERATOR_IMAGE_REGISTRY) \ + --set-file apiServerEndpoint=$(UNBOUNDED_OPERATOR_RENDER_DIR)/api-server-endpoint \ + --set reapLegacyResources=$(UNBOUNDED_OPERATOR_REAP_LEGACY_RESOURCES) \ + --output-dir $(UNBOUNDED_OPERATOR_RENDER_DIR)/out + @cp $(UNBOUNDED_OPERATOR_RENDER_DIR)/out/unbounded-operator/templates/*.yaml $(UNBOUNDED_OPERATOR_MANIFEST_RENDERED_DIR)/ + @rm -rf $(UNBOUNDED_OPERATOR_RENDER_DIR) @echo "Rendered unbounded-operator manifests into $(UNBOUNDED_OPERATOR_MANIFEST_RENDERED_DIR) (image: $(UNBOUNDED_OPERATOR_IMAGE))" +unbounded-operator-chart-lint: $(HELM) ## Validate the unbounded-operator Helm chart + $(HELM) lint --strict $(UNBOUNDED_OPERATOR_CHART_DIR) --namespace $(UNBOUNDED_OPERATOR_NAMESPACE) + +unbounded-operator-chart-package: unbounded-operator-chart-lint ## Package the unbounded-operator Helm chart + @mkdir -p $(GANTRY_CHART_PACKAGE_DIR) + @rm -f $(GANTRY_CHART_PACKAGE_DIR)/unbounded-operator-$(UNBOUNDED_OPERATOR_CHART_VERSION).tgz + $(HELM) package $(UNBOUNDED_OPERATOR_CHART_DIR) \ + --version $(UNBOUNDED_OPERATOR_CHART_VERSION) \ + --app-version $(UNBOUNDED_OPERATOR_CHART_APP_VERSION) \ + --destination $(GANTRY_CHART_PACKAGE_DIR) + machine-ops-manifests: ## Render machine-ops-controller manifests into deploy/machine-ops/rendered @mkdir -p $(MACHINE_OPS_MANIFEST_RENDERED_DIR) @find $(MACHINE_OPS_MANIFEST_RENDERED_DIR) -mindepth 1 -not -name .gitignore -delete diff --git a/cmd/kubectl-unbounded/app/install.go b/cmd/kubectl-unbounded/app/install.go index 8f7a46e987..6e9af5c28a 100644 --- a/cmd/kubectl-unbounded/app/install.go +++ b/cmd/kubectl-unbounded/app/install.go @@ -146,6 +146,10 @@ func (h *installHandler) execute(ctx context.Context) error { } } + if err := h.refuseHelmManagedOperator(ctx); err != nil { + return err + } + if err := h.prepareOperatorConfig(ctx); err != nil { return err } @@ -175,6 +179,62 @@ func (h *installHandler) execute(ctx context.Context) error { return nil } +// helmReleaseAnnotation marks objects owned by a Helm release, including the +// unbounded-operator chart installed by Helm or an AKS cluster extension. +const helmReleaseAnnotation = "meta.helm.sh/release-name" + +// refuseHelmManagedOperator stops install from server-side applying over an +// operator that a Helm release owns. Applying with ForceOwnership would flip +// the operator image back and forth with every helm upgrade, and the shared +// ClusterRoleBinding would point at whichever operator was installed last, +// leaving the other one without RBAC. +func (h *installHandler) refuseHelmManagedOperator(ctx context.Context) error { + checks := []struct { + apiVersion, kind string + key client.ObjectKey + }{ + {"apps/v1", "Deployment", client.ObjectKey{Namespace: h.namespace, Name: "unbounded-operator"}}, + {"rbac.authorization.k8s.io/v1", "ClusterRoleBinding", client.ObjectKey{Name: "unbounded-operator"}}, + } + + for _, c := range checks { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion(c.apiVersion) + obj.SetKind(c.kind) + + if err := h.kubeResourcesCli.Get(ctx, c.key, obj); err != nil { + if apierrors.IsNotFound(err) { + continue + } + + return fmt.Errorf("inspect %s %s: %w", c.kind, c.key, err) + } + + release := obj.GetAnnotations()[helmReleaseAnnotation] + if release == "" { + continue + } + + where := obj.GetAnnotations()["meta.helm.sh/release-namespace"] + if where == "" { + where = obj.GetNamespace() + } + + ref := c.key.Name + if c.key.Namespace != "" { + ref = c.key.String() + } + + return fmt.Errorf( + "%s %s is managed by Helm release %q in namespace %q (for example the AKS flex operator extension); "+ + "upgrade or remove it with helm or the extension instead of kubectl unbounded install", + c.kind, ref, release, where, + ) + } + + return nil +} + func (h *installHandler) initializeClients() error { h.kubeconfigPath = getKubeconfigPath(h.kubeconfigPath) if !isReadableFile(h.kubeconfigPath) { diff --git a/cmd/kubectl-unbounded/app/install_test.go b/cmd/kubectl-unbounded/app/install_test.go index 79354a96a3..5919e3f59e 100644 --- a/cmd/kubectl-unbounded/app/install_test.go +++ b/cmd/kubectl-unbounded/app/install_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" + rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -66,6 +67,70 @@ func TestInstallRejectsLegacyNamespace(t *testing.T) { } } +func TestInstallRefusesHelmManagedOperator(t *testing.T) { + helmAnnotations := map[string]string{ + "meta.helm.sh/release-name": "flex-operator", + "meta.helm.sh/release-namespace": "flex-system", + } + + for name, existing := range map[string]client.Object{ + // Same namespace: install would overwrite the Helm release's Deployment. + "deployment in target namespace": &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{ + Name: "unbounded-operator", Namespace: "unbounded-system", Annotations: helmAnnotations, + }}, + // Different namespace: install would repoint the shared ClusterRoleBinding + // and strip the Helm-managed operator of its RBAC. + "cluster role binding": &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{ + Name: "unbounded-operator", Annotations: helmAnnotations, + }}, + } { + t.Run(name, func(t *testing.T) { + applied := 0 + cli := fakeclient.NewClientBuilder(). + WithObjects(existing). + WithInterceptorFuncs(interceptor.Funcs{ + Apply: func(context.Context, client.WithWatch, runtime.ApplyConfiguration, ...client.ApplyOption) error { + applied++ + return nil + }, + }). + Build() + + h := installHandler{ + namespace: "unbounded-system", + operatorImage: "unbounded-operator:test", + imageRegistry: "registry.example.com", + kubeResourcesCli: cli, + logger: discardLogger(), + } + + err := h.execute(context.Background()) + require.ErrorContains(t, err, `managed by Helm release "flex-operator"`) + require.Zero(t, applied, "install must not apply anything over a Helm-managed operator") + }) + } +} + +func TestInstallReappliesOwnInstall(t *testing.T) { + // A previous `kubectl unbounded install` has no Helm annotations, so a + // reinstall still goes through. + cli, captured := newCapturingInstallClient( + &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "unbounded-operator", Namespace: "unbounded-system"}}, + &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "unbounded-operator"}}, + ) + + h := installHandler{ + namespace: "unbounded-system", + operatorImage: "unbounded-operator:test", + imageRegistry: "registry.example.com", + kubeResourcesCli: cli, + logger: discardLogger(), + } + + require.NoError(t, h.execute(context.Background())) + require.NotNil(t, captured.deployment) +} + func TestInstallHandlerApplyBootstrapManifests(t *testing.T) { var ( mu sync.Mutex diff --git a/deploy/unbounded-operator/00-namespace.yaml.tmpl b/deploy/unbounded-operator/00-namespace.yaml.tmpl deleted file mode 100644 index 08464f6c01..0000000000 --- a/deploy/unbounded-operator/00-namespace.yaml.tmpl +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: Apache-2.0 - ---- -apiVersion: v1 -kind: Namespace -metadata: - name: {{ default "unbounded-system" .Namespace }} - labels: - app.kubernetes.io/name: unbounded-operator diff --git a/deploy/unbounded-operator/01-serviceaccount.yaml.tmpl b/deploy/unbounded-operator/01-serviceaccount.yaml.tmpl deleted file mode 100644 index 3bf7d57132..0000000000 --- a/deploy/unbounded-operator/01-serviceaccount.yaml.tmpl +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: Apache-2.0 - ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: unbounded-operator - namespace: {{ default "unbounded-system" .Namespace }} - labels: - app.kubernetes.io/name: unbounded-operator - app.kubernetes.io/component: controller diff --git a/deploy/unbounded-operator/chart/.helmignore b/deploy/unbounded-operator/chart/.helmignore new file mode 100644 index 0000000000..5811618d12 --- /dev/null +++ b/deploy/unbounded-operator/chart/.helmignore @@ -0,0 +1,2 @@ +# Only used by `make unbounded-operator-manifests`; not part of the packaged chart. +values-static.yaml diff --git a/deploy/unbounded-operator/chart/Chart.yaml b/deploy/unbounded-operator/chart/Chart.yaml new file mode 100644 index 0000000000..5478468a49 --- /dev/null +++ b/deploy/unbounded-operator/chart/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: unbounded-operator +description: Unbounded operator, which installs Unbounded CRDs and reconciles components from Sites +type: application +version: 0.1.0 +appVersion: dev diff --git a/deploy/unbounded-operator/chart/templates/00-namespace.yaml b/deploy/unbounded-operator/chart/templates/00-namespace.yaml new file mode 100644 index 0000000000..ff8f8f5829 --- /dev/null +++ b/deploy/unbounded-operator/chart/templates/00-namespace.yaml @@ -0,0 +1,12 @@ +{{- if .Values.namespace.create }} +# Only the static `kubectl apply` render creates the namespace (values-static.yaml, +# rendered with --skip-schema-validation). The schema pins namespace.create to +# false so a Helm or AKS extension install can't take ownership of the namespace +# that holds every operator-managed component. +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: unbounded-operator +{{- end }} diff --git a/deploy/unbounded-operator/chart/templates/01-serviceaccount.yaml b/deploy/unbounded-operator/chart/templates/01-serviceaccount.yaml new file mode 100644 index 0000000000..74160fbf98 --- /dev/null +++ b/deploy/unbounded-operator/chart/templates/01-serviceaccount.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: unbounded-operator + namespace: {{ .Release.Namespace }} + labels: + {{- include "unbounded-operator.labels" . | nindent 4 }} diff --git a/deploy/unbounded-operator/02-rbac.yaml.tmpl b/deploy/unbounded-operator/chart/templates/02-rbac.yaml similarity index 90% rename from deploy/unbounded-operator/02-rbac.yaml.tmpl rename to deploy/unbounded-operator/chart/templates/02-rbac.yaml index 794e30ec05..cc8754052b 100644 --- a/deploy/unbounded-operator/02-rbac.yaml.tmpl +++ b/deploy/unbounded-operator/chart/templates/02-rbac.yaml @@ -1,14 +1,9 @@ -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: Apache-2.0 - ---- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: unbounded-operator labels: - app.kubernetes.io/name: unbounded-operator - app.kubernetes.io/component: controller + {{- include "unbounded-operator.labels" . | nindent 4 }} rules: - apiGroups: ["racer.unbounded-cloud.io"] resources: ["clustercaches"] @@ -121,8 +116,7 @@ kind: ClusterRoleBinding metadata: name: unbounded-operator labels: - app.kubernetes.io/name: unbounded-operator - app.kubernetes.io/component: controller + {{- include "unbounded-operator.labels" . | nindent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -130,17 +124,16 @@ roleRef: subjects: - kind: ServiceAccount name: unbounded-operator - namespace: {{ default "unbounded-system" .Namespace }} + namespace: {{ .Release.Namespace }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: unbounded-operator-registry-auth-migration - namespace: {{ default "unbounded-system" .Namespace }} + namespace: {{ .Release.Namespace }} labels: - app.kubernetes.io/name: unbounded-operator - app.kubernetes.io/component: controller + {{- include "unbounded-operator.labels" . | nindent 4 }} rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] @@ -152,10 +145,9 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: unbounded-operator-registry-auth-migration - namespace: {{ default "unbounded-system" .Namespace }} + namespace: {{ .Release.Namespace }} labels: - app.kubernetes.io/name: unbounded-operator - app.kubernetes.io/component: controller + {{- include "unbounded-operator.labels" . | nindent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -163,4 +155,4 @@ roleRef: subjects: - kind: ServiceAccount name: unbounded-operator - namespace: {{ default "unbounded-system" .Namespace }} + namespace: {{ .Release.Namespace }} diff --git a/deploy/unbounded-operator/03-configmap.yaml.tmpl b/deploy/unbounded-operator/chart/templates/03-configmap.yaml similarity index 73% rename from deploy/unbounded-operator/03-configmap.yaml.tmpl rename to deploy/unbounded-operator/chart/templates/03-configmap.yaml index 497082f0c3..22e7407d1d 100644 --- a/deploy/unbounded-operator/03-configmap.yaml.tmpl +++ b/deploy/unbounded-operator/chart/templates/03-configmap.yaml @@ -1,7 +1,3 @@ -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: Apache-2.0 - ---- # unbounded-operator-config holds the environment-specific operator settings that # used to be passed as `kubectl unbounded install` flags. It is surfaced to the # operator Deployment via envFrom, and each operator flag defaults from the @@ -15,15 +11,14 @@ apiVersion: v1 kind: ConfigMap metadata: name: unbounded-operator-config - namespace: {{ default "unbounded-system" .Namespace }} + namespace: {{ .Release.Namespace }} labels: - app.kubernetes.io/name: unbounded-operator - app.kubernetes.io/component: controller + {{- include "unbounded-operator.labels" . | nindent 4 }} data: # Full image-repository prefix (registry host plus org/namespace) for all # operator-managed first-party component images. The flat component name and # operator version are appended, so ghcr.io/azure yields ghcr.io/azure/machina. - UNBOUNDED_IMAGE_REGISTRY: {{ default "ghcr.io/azure" .ImageRegistry | quote }} + UNBOUNDED_IMAGE_REGISTRY: {{ include "unbounded-operator.registry" . | quote }} # External Kubernetes API endpoint advertised to provisioned machines. Remote # nodes dial this to join, so it must be reachable from outside the cluster. # Leave empty to let the operator auto-discover it: it prefers the standard @@ -31,10 +26,10 @@ data: # (e.g. AKS), falls back to the KUBERNETES_SERVICE_HOST FQDN (the operator pod # carries kubernetes.azure.com/set-kube-service-host-fqdn so that env is the # public API FQDN). Set a value only to override discovery. Rendered from - # .APIServerEndpoint so make/GitOps deploys can override via - # --set APIServerEndpoint=; `kubectl unbounded install --api-server-endpoint` + # .Values.apiServerEndpoint so make/GitOps deploys can override via + # --set apiServerEndpoint=; `kubectl unbounded install --api-server-endpoint` # overrides it in place (an empty value is preserved across reinstalls). - UNBOUNDED_API_SERVER_ENDPOINT: {{ default "" .APIServerEndpoint | quote }} + UNBOUNDED_API_SERVER_ENDPOINT: {{ .Values.apiServerEndpoint | quote }} # Whether the operator translates legacy net-group Sites, migrates state into # this namespace, and reaps the pre-consolidation namespaces. - UNBOUNDED_REAP_LEGACY_RESOURCES: {{ default "true" .ReapLegacyResources | quote }} + UNBOUNDED_REAP_LEGACY_RESOURCES: {{ toString .Values.reapLegacyResources | quote }} diff --git a/deploy/unbounded-operator/04-deployment.yaml.tmpl b/deploy/unbounded-operator/chart/templates/04-deployment.yaml similarity index 74% rename from deploy/unbounded-operator/04-deployment.yaml.tmpl rename to deploy/unbounded-operator/chart/templates/04-deployment.yaml index baed30bfb2..fbad029406 100644 --- a/deploy/unbounded-operator/04-deployment.yaml.tmpl +++ b/deploy/unbounded-operator/chart/templates/04-deployment.yaml @@ -1,15 +1,11 @@ -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: Apache-2.0 - ---- +{{- include "unbounded-operator.ownershipCheck" . }} apiVersion: apps/v1 kind: Deployment metadata: name: unbounded-operator - namespace: {{ default "unbounded-system" .Namespace }} + namespace: {{ .Release.Namespace }} labels: - app.kubernetes.io/name: unbounded-operator - app.kubernetes.io/component: controller + {{- include "unbounded-operator.labels" . | nindent 4 }} spec: replicas: 1 strategy: @@ -21,8 +17,7 @@ spec: template: metadata: labels: - app.kubernetes.io/name: unbounded-operator - app.kubernetes.io/component: controller + {{- include "unbounded-operator.labels" . | nindent 8 }} annotations: # On AKS (which does not publish kube-public/cluster-info) this makes the # pod's KUBERNETES_SERVICE_HOST the public API FQDN, which the operator @@ -32,7 +27,7 @@ spec: kubernetes.azure.com/set-kube-service-host-fqdn: "true" # Rolls the operator when any ConfigMap data changes at render time. # Canonical JSON matches the `kubectl unbounded install` hash. - unbounded-cloud.io/operator-config-hash: {{ dict "UNBOUNDED_API_SERVER_ENDPOINT" (default "" .APIServerEndpoint) "UNBOUNDED_IMAGE_REGISTRY" (default "ghcr.io/azure" .ImageRegistry) "UNBOUNDED_REAP_LEGACY_RESOURCES" (default "true" .ReapLegacyResources) | toJson | sha256sum | quote }} + unbounded-cloud.io/operator-config-hash: {{ include "unbounded-operator.configHash" . | quote }} spec: # The operator installs the cluster CNI, so it must be able to reach the # API server before pod networking or cluster DNS is available. Default @@ -46,15 +41,15 @@ spec: effect: NoSchedule containers: - name: controller - image: {{ .OperatorImage }} - imagePullPolicy: Always + image: {{ include "unbounded-operator.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} args: - --leader-elect=true - - --leader-elect-namespace={{ default "unbounded-system" .Namespace }} - - --namespace={{ default "unbounded-system" .Namespace }} + - --leader-elect-namespace={{ .Release.Namespace }} + - --namespace={{ .Release.Namespace }} envFrom: # Environment-specific operator settings. See - # deploy/unbounded-operator/03-configmap.yaml.tmpl. + # templates/03-configmap.yaml. - configMapRef: name: unbounded-operator-config env: @@ -70,9 +65,7 @@ spec: containerPort: 8081 protocol: TCP resources: - requests: - cpu: 50m - memory: 64Mi + {{- toYaml .Values.resources | nindent 12 }} startupProbe: # The operator applies and establishes CRDs at startup # (operator.BootstrapCRDs, CRDBootstrapTimeout = 4m) before the diff --git a/deploy/unbounded-operator/chart/templates/NOTES.txt b/deploy/unbounded-operator/chart/templates/NOTES.txt new file mode 100644 index 0000000000..074e1b20e2 --- /dev/null +++ b/deploy/unbounded-operator/chart/templates/NOTES.txt @@ -0,0 +1,14 @@ +unbounded-operator {{ .Chart.AppVersion }} is installed in namespace {{ .Release.Namespace }}. + +Verify the rollout; the operator installs the Unbounded CRDs at startup: + kubectl -n {{ .Release.Namespace }} rollout status deployment/unbounded-operator + +Then create Sites, for example: + kubectl unbounded site init --name ... + +Manage this operator with Helm (or the AKS extension) from now on; +`kubectl unbounded install` refuses to apply over it. + +Uninstall removes the operator, its RBAC and ConfigMap. The release doesn't own +the namespace, so the CRDs, Sites and the components the operator reconciled +into it, including unbounded-net, are left in place. diff --git a/deploy/unbounded-operator/chart/templates/_helpers.tpl b/deploy/unbounded-operator/chart/templates/_helpers.tpl new file mode 100644 index 0000000000..4339c62545 --- /dev/null +++ b/deploy/unbounded-operator/chart/templates/_helpers.tpl @@ -0,0 +1,48 @@ +{{- define "unbounded-operator.labels" -}} +app.kubernetes.io/name: unbounded-operator +app.kubernetes.io/component: controller +{{- end }} + +{{- define "unbounded-operator.registry" -}} +{{- trimSuffix "/" .Values.imageRegistry -}} +{{- end }} + +{{- define "unbounded-operator.image" -}} +{{- if .Values.image.reference -}} +{{- .Values.image.reference -}} +{{- else -}} +{{- $repo := printf "%s/%s" (include "unbounded-operator.registry" .) .Values.image.name -}} +{{- if .Values.image.digest -}} +{{- printf "%s@%s" $repo .Values.image.digest -}} +{{- else -}} +{{- printf "%s:%s" $repo (default .Chart.AppVersion .Values.image.tag) -}} +{{- end -}} +{{- end -}} +{{- end }} + +{{/* +sha256 of the canonical (sorted, compact) JSON of the ConfigMap data. Must match +operatorConfigHash in cmd/kubectl-unbounded/app/install.go so a re-render or +`kubectl unbounded install` with changed settings rolls the operator. +*/}} +{{- define "unbounded-operator.configHash" -}} +{{- dict "UNBOUNDED_API_SERVER_ENDPOINT" .Values.apiServerEndpoint "UNBOUNDED_IMAGE_REGISTRY" (include "unbounded-operator.registry" .) "UNBOUNDED_REAP_LEGACY_RESOURCES" (toString .Values.reapLegacyResources) | toJson | sha256sum -}} +{{- end }} + +{{/* +Refuse to take over an operator Deployment this release doesn't own, such as +one created by `kubectl unbounded install`. lookup returns nothing during +`helm template` (which renders deploy/unbounded-operator/rendered), so this +only runs against a live cluster. +*/}} +{{- define "unbounded-operator.ownershipCheck" -}} +{{- $existing := lookup "apps/v1" "Deployment" .Release.Namespace "unbounded-operator" -}} +{{- if $existing -}} +{{- $annotations := default dict $existing.metadata.annotations -}} +{{- $releaseName := default "" (get $annotations "meta.helm.sh/release-name") -}} +{{- $releaseNamespace := default "" (get $annotations "meta.helm.sh/release-namespace") -}} +{{- if not (and (eq $releaseName .Release.Name) (eq $releaseNamespace .Release.Namespace)) -}} +{{- fail (printf "Deployment %s/unbounded-operator already exists and isn't managed by this Helm release; it was likely installed by `kubectl unbounded install`. Remove that installation before installing the chart." .Release.Namespace) -}} +{{- end -}} +{{- end -}} +{{- end }} diff --git a/deploy/unbounded-operator/chart/values-static.yaml b/deploy/unbounded-operator/chart/values-static.yaml new file mode 100644 index 0000000000..516f12ea45 --- /dev/null +++ b/deploy/unbounded-operator/chart/values-static.yaml @@ -0,0 +1,8 @@ +# Values for the static manifests in deploy/unbounded-operator/rendered, which +# `kubectl unbounded install` embeds and the release publishes as +# unbounded-operator-.yaml. `make unbounded-operator-manifests` supplies +# the image, registry, endpoint and reapLegacyResources, and renders with +# --skip-schema-validation because the schema forbids namespace.create for +# Helm installs. Excluded from the packaged chart by .helmignore. +namespace: + create: true diff --git a/deploy/unbounded-operator/chart/values.schema.json b/deploy/unbounded-operator/chart/values.schema.json new file mode 100644 index 0000000000..933bc61ff2 --- /dev/null +++ b/deploy/unbounded-operator/chart/values.schema.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$comment": "The root accepts extra keys: AKS cluster extensions inject platform values (Azure.Cluster.*, IdentityClientId, DataBoundary, scope, ...) that this chart ignores. The chart's own objects stay strict.", + "type": "object", + "required": ["imageRegistry", "image", "apiServerEndpoint", "reapLegacyResources", "resources", "namespace"], + "properties": { + "imageRegistry": {"type": "string", "minLength": 1}, + "image": { + "type": "object", + "additionalProperties": false, + "required": ["reference", "name", "tag", "digest", "pullPolicy"], + "properties": { + "reference": {"type": "string"}, + "name": {"type": "string", "pattern": "^[a-z0-9]+([._-][a-z0-9]+)*$"}, + "tag": {"type": "string"}, + "digest": {"type": "string", "pattern": "^(sha256:[0-9a-f]{64})?$"}, + "pullPolicy": {"enum": ["Always", "IfNotPresent", "Never"]} + } + }, + "apiServerEndpoint": {"type": "string"}, + "reapLegacyResources": {"type": "boolean"}, + "resources": {"type": "object"}, + "namespace": { + "type": "object", + "additionalProperties": false, + "required": ["create"], + "properties": {"create": {"const": false}} + } + } +} diff --git a/deploy/unbounded-operator/chart/values.yaml b/deploy/unbounded-operator/chart/values.yaml new file mode 100644 index 0000000000..f4ff79b455 --- /dev/null +++ b/deploy/unbounded-operator/chart/values.yaml @@ -0,0 +1,39 @@ +# Full image-repository prefix (registry host plus org/namespace) for the +# operator and every operator-managed first-party component image. The operator +# resolves components as /:, so the +# operator and its components always come from the same registry. +imageRegistry: ghcr.io/azure + +image: + # Full operator image reference; overrides name, tag and digest. Used by + # `make unbounded-operator-manifests`, whose UNBOUNDED_OPERATOR_IMAGE may be + # a locally built image outside imageRegistry. + reference: "" + # Operator image name without a registry; the registry is imageRegistry. + name: unbounded-operator + # Defaults to the chart appVersion. + tag: "" + # Optional digest (sha256:...). Takes precedence over tag. + digest: "" + pullPolicy: Always + +# External Kubernetes API endpoint advertised to provisioned machines. Empty +# lets the operator discover it from kube-public/cluster-info, or from the +# KUBERNETES_SERVICE_HOST FQDN on AKS. +apiServerEndpoint: "" + +# Translate legacy net-group Sites, migrate state into the release namespace, +# and reap the pre-consolidation namespaces. +reapLegacyResources: true + +resources: + requests: + cpu: 50m + memory: 64Mi + +namespace: + # Only `make unbounded-operator-manifests` sets this (values-static.yaml) to + # put a Namespace in the static `kubectl apply` manifests. The schema pins it + # to false for Helm and AKS extension installs: if a release owned the + # namespace, `helm uninstall` would delete every component in it. + create: false diff --git a/deploy/unbounded-operator/chart_test.go b/deploy/unbounded-operator/chart_test.go new file mode 100644 index 0000000000..242231db90 --- /dev/null +++ b/deploy/unbounded-operator/chart_test.go @@ -0,0 +1,287 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package unboundedoperator + +import ( + "bytes" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// staticRender mirrors the inputs of `make unbounded-operator-manifests`. +type staticRender struct { + namespace string + image string + registry string + endpoint string + reap string +} + +// renderStatic renders the chart exactly as `make unbounded-operator-manifests` +// does (values-static.yaml plus the make variables, --output-dir) and returns +// the directory holding the files `kubectl unbounded install` embeds. +func renderStatic(t *testing.T, r staticRender) string { + t.Helper() + + if r.namespace == "" { + r.namespace = "unbounded-system" + } + + if r.registry == "" { + r.registry = "ghcr.io/azure" + } + + if r.reap == "" { + r.reap = "true" + } + + chart := filepath.Join(repoRoot(t), "deploy", "unbounded-operator", "chart") + out := t.TempDir() + + // Like the Makefile, pass the endpoint with --set-file so commas and other + // --set syntax survive verbatim. + endpointFile := filepath.Join(out, "api-server-endpoint") + if err := os.WriteFile(endpointFile, []byte(r.endpoint), 0o600); err != nil { + t.Fatal(err) + } + + runHelm(t, + "template", "unbounded-operator", chart, + "--namespace", r.namespace, + "--values", filepath.Join(chart, "values-static.yaml"), + "--skip-schema-validation", + "--set-string", "image.reference="+r.image, + "--set-string", "imageRegistry="+r.registry, + "--set-file", "apiServerEndpoint="+endpointFile, + "--set", "reapLegacyResources="+r.reap, + "--output-dir", filepath.Join(out, "render"), + ) + + return filepath.Join(out, "render", "unbounded-operator", "templates") +} + +func runHelm(t *testing.T, args ...string) []byte { + t.Helper() + + out, err := exec.Command(helmBinary(t), args...).CombinedOutput() + if err != nil { + t.Fatalf("helm %s: %v\n%s", strings.Join(args, " "), err, out) + } + + return out +} + +// TestStaticAndHelmRenders pins the files and objects `kubectl unbounded +// install` and the release manifest rely on, and checks that a Helm or AKS +// cluster extension install renders the same objects minus the Namespace, +// which the release owns. +func TestStaticAndHelmRenders(t *testing.T) { + t.Parallel() + + layout := map[string][]string{ + "00-namespace.yaml": {"Namespace/"}, + "01-serviceaccount.yaml": {"ServiceAccount/unbounded-operator"}, + "02-rbac.yaml": { + "ClusterRole/unbounded-operator", + "ClusterRoleBinding/unbounded-operator", + "Role/unbounded-operator-registry-auth-migration", + "RoleBinding/unbounded-operator-registry-auth-migration", + }, + "03-configmap.yaml": {"ConfigMap/unbounded-operator-config"}, + "04-deployment.yaml": {"Deployment/unbounded-operator"}, + } + + for _, tc := range []staticRender{ + {namespace: "unbounded-system", registry: "ghcr.io/azure", reap: "true"}, + {namespace: "flex-system", registry: "registry.example.com/mirror", endpoint: "https://api.example.test:443", reap: "false"}, + } { + t.Run(tc.namespace, func(t *testing.T) { + t.Parallel() + + tc.image = tc.registry + "/unbounded-operator:v9.9.9" + dir := renderStatic(t, tc) + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + if len(entries) != len(layout) { + t.Fatalf("rendered %d files, want %d", len(entries), len(layout)) + } + + static := map[string]map[string]any{} + + for file, keys := range layout { + data, err := os.ReadFile(filepath.Join(dir, file)) + if err != nil { + t.Fatalf("read %s: %v", file, err) + } + + objs := decodeObjects(t, data) + + want := append([]string(nil), keys...) + + if file == "00-namespace.yaml" { + want = []string{"Namespace/" + tc.namespace} + } + + sort.Strings(want) + + if got := sortedKeys(objs); !reflect.DeepEqual(got, want) { + t.Errorf("%s objects = %v, want %v", file, got, want) + } + + for key, obj := range objs { + if !strings.HasPrefix(key, "Namespace/") { + static[key] = obj + } + } + } + + helm := decodeObjects(t, runHelm(t, + "template", "unbounded-operator", filepath.Join(repoRoot(t), "deploy", "unbounded-operator", "chart"), + "--namespace", tc.namespace, + "--set", "image.tag=v9.9.9", + "--set-string", "imageRegistry="+tc.registry, + "--set-string", "apiServerEndpoint="+tc.endpoint, + "--set", "reapLegacyResources="+tc.reap, + )) + + if !reflect.DeepEqual(sortedKeys(helm), sortedKeys(static)) { + t.Fatalf("helm install objects = %v, static objects = %v", sortedKeys(helm), sortedKeys(static)) + } + + for key, obj := range static { + if !reflect.DeepEqual(helm[key], obj) { + gy, _ := yaml.Marshal(helm[key]) + wy, _ := yaml.Marshal(obj) + t.Errorf("%s differs\n--- helm install\n%s--- static\n%s", key, gy, wy) + } + } + }) + } +} + +// TestChartRejectsInvalidValues checks the schema guards that keep the chart +// rendering the same images the operator expects. +func TestChartRejectsInvalidValues(t *testing.T) { + t.Parallel() + + helm := helmBinary(t) + chart := filepath.Join(repoRoot(t), "deploy", "unbounded-operator", "chart") + + for name, set := range map[string]string{ + "image name with registry": "image.name=ghcr.io/azure/unbounded-operator", + "malformed digest": "image.digest=latest", + "unknown image field": "image.repository=foo", + "chart-owned namespace": "namespace.create=true", + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + out, err := exec.Command(helm, "template", "x", chart, "--namespace", "unbounded-system", "--set", set).CombinedOutput() + if err == nil { + t.Fatalf("helm template --set %s succeeded, want a schema error\n%s", set, out) + } + }) + } +} + +// TestChartAcceptsExtensionManagerValues checks that the platform values an AKS +// cluster extension injects pass the schema. The keys are the ones Extension +// Manager sent on a real install; a strict root rejected them. +func TestChartAcceptsExtensionManagerValues(t *testing.T) { + t.Parallel() + + runHelm(t, + "template", "unbounded-operator", filepath.Join(repoRoot(t), "deploy", "unbounded-operator", "chart"), + "--namespace", "unbounded-system", + "--set-string", "Azure.Cluster.ResourceId=/subscriptions/0000/resourceGroups/rg/providers/Microsoft.ContainerService/managedClusters/c", + "--set-string", "Azure.Cluster.Region=eastus2euap", + "--set-string", "Azure.Cluster.Distribution=aks", + "--set-string", "IdentityClientId=00000000-0000-0000-0000-000000000000", + "--set-string", "IdentityPrincipalId=00000000-0000-0000-0000-000000000000", + "--set-string", "IdentityTenantId=00000000-0000-0000-0000-000000000000", + "--set-string", "IdentityType=SystemAssigned", + "--set", "IsWorkloadIdentityEnabled=false", + "--set-string", "DataBoundary=Global", + "--set-string", "scope=cluster", + ) +} + +func helmBinary(t *testing.T) string { + t.Helper() + + if p := os.Getenv("HELM"); p != "" { + return p + } + + if p := filepath.Join(repoRoot(t), "bin", "helm"); fileExists(p) { + return p + } + + p, err := exec.LookPath("helm") + if err != nil { + t.Fatal("helm not found; run make install-helm or set HELM") + } + + return p +} + +func fileExists(p string) bool { + _, err := os.Stat(p) + return err == nil +} + +func decodeObjects(t *testing.T, data []byte) map[string]map[string]any { + t.Helper() + + objects := map[string]map[string]any{} + dec := yaml.NewDecoder(bytes.NewReader(data)) + + for { + var obj map[string]any + + err := dec.Decode(&obj) + if errors.Is(err, io.EOF) { + break + } + + if err != nil { + t.Fatalf("decode YAML: %v", err) + } + + if obj == nil { + continue + } + + kind, _ := obj["kind"].(string) + meta, _ := obj["metadata"].(map[string]any) + name, _ := meta["name"].(string) + objects[kind+"/"+name] = obj + } + + return objects +} + +func sortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + + sort.Strings(keys) + + return keys +} diff --git a/deploy/unbounded-operator/examples/component-overrides.example.yaml b/deploy/unbounded-operator/examples/component-overrides.example.yaml index 8f4a75230f..a1d6f3771d 100644 --- a/deploy/unbounded-operator/examples/component-overrides.example.yaml +++ b/deploy/unbounded-operator/examples/component-overrides.example.yaml @@ -3,9 +3,9 @@ # Example component workload overrides. # -# This file is an example, not something the operator installs. It is a plain -# .yaml rather than a .yaml.tmpl so neither render-manifests nor go:embed picks -# it up. Copy what you need; do not apply it as-is. +# This file is an example, not something the operator installs. It lives +# outside chart/ so neither the operator manifests nor go:embed pick it up. +# Copy what you need; do not apply it as-is. # # SECURITY: write access to this ConfigMap is equivalent to root on every node # in every affected Site, and therefore to cluster-admin. The workloads it diff --git a/deploy/unbounded-operator/render_test.go b/deploy/unbounded-operator/render_test.go index d1cc56fc85..d92cc373e5 100644 --- a/deploy/unbounded-operator/render_test.go +++ b/deploy/unbounded-operator/render_test.go @@ -18,7 +18,6 @@ import ( "gopkg.in/yaml.v3" - "github.com/Azure/unbounded/hack/cmd/render-manifests/render" "github.com/Azure/unbounded/internal/operator" ) @@ -31,20 +30,13 @@ import ( func TestOperatorConfigEndpointAndHashRender(t *testing.T) { t.Parallel() - const endpoint = `https://api.example.test:6443/path?mode="strict"&ready=true` + const endpoint = `https://api.example.test:6443/path,tenant?mode="strict"&ready=true` - templatesDir := filepath.Join(repoRoot(t), "deploy", "unbounded-operator") - outputDir := t.TempDir() - - data := map[string]string{ - "Namespace": "unbounded-system", - "OperatorImage": "operator:test", - "ImageRegistry": "registry.example.com/mirror", - "APIServerEndpoint": endpoint, - } - if err := render.Render(templatesDir, outputDir, data); err != nil { - t.Fatalf("render.Render: %v", err) - } + outputDir := renderStatic(t, staticRender{ + image: "operator:test", + registry: "registry.example.com/mirror", + endpoint: endpoint, + }) // The ConfigMap carries the endpoint that the operator reads via envFrom. var cm struct { @@ -189,16 +181,12 @@ func TestOperatorConfigHashChangesWithEachConfigValue(t *testing.T) { renderHash := func(endpoint, registry, reapLegacyResources string) string { t.Helper() - outputDir := t.TempDir() - if err := render.Render(filepath.Join(repoRoot(t), "deploy", "unbounded-operator"), outputDir, map[string]string{ - "Namespace": "unbounded-system", - "OperatorImage": "operator:test", - "ImageRegistry": registry, - "APIServerEndpoint": endpoint, - "ReapLegacyResources": reapLegacyResources, - }); err != nil { - t.Fatalf("render.Render: %v", err) - } + outputDir := renderStatic(t, staticRender{ + image: "operator:test", + registry: registry, + endpoint: endpoint, + reap: reapLegacyResources, + }) var cm struct { Data map[string]string `yaml:"data"` @@ -257,14 +245,7 @@ func TestOperatorConfigHashChangesWithEachConfigValue(t *testing.T) { func TestOperatorRBACIncludesCachedReadKinds(t *testing.T) { t.Parallel() - outputDir := t.TempDir() - if err := render.Render(filepath.Join(repoRoot(t), "deploy", "unbounded-operator"), outputDir, map[string]string{ - "Namespace": "unbounded-system", - "OperatorImage": "operator:test", - "ImageRegistry": "ghcr.io", - }); err != nil { - t.Fatalf("render.Render: %v", err) - } + outputDir := renderStatic(t, staticRender{image: "operator:test", registry: "ghcr.io"}) var role struct { Rules []struct { @@ -324,15 +305,14 @@ func TestOperatorImageAndComponentRegistryShareRegistry(t *testing.T) { operatorImage = "ghcr.io/azure/unbounded-operator:" + tag ) - outputDir := t.TempDir() - // Omit ImageRegistry so the ConfigMap default is exercised; supply - // OperatorImage as the make/release render path does. - if err := render.Render(filepath.Join(repoRoot(t), "deploy", "unbounded-operator"), outputDir, map[string]string{ - "Namespace": "unbounded-system", - "OperatorImage": operatorImage, - }); err != nil { - t.Fatalf("render.Render: %v", err) - } + // Use the chart's default imageRegistry so its value is exercised; supply + // the operator image as the make/release render path does. + chartDefaults := struct { + ImageRegistry string `yaml:"imageRegistry"` + }{} + readYAML(t, filepath.Join(repoRoot(t), "deploy", "unbounded-operator", "chart", "values.yaml"), &chartDefaults) + + outputDir := renderStatic(t, staticRender{image: operatorImage, registry: chartDefaults.ImageRegistry}) var cm struct { Data map[string]string `yaml:"data"` @@ -389,12 +369,7 @@ func TestOperatorImageAndComponentRegistryShareRegistry(t *testing.T) { } func TestRegistryAuthMigrationLeaseDeletionIsNamespaceAndNameScoped(t *testing.T) { - outputDir := t.TempDir() - if err := render.Render(filepath.Join(repoRoot(t), "deploy", "unbounded-operator"), outputDir, map[string]string{ - "Namespace": "custom-system", "OperatorImage": "operator:test", "ImageRegistry": "ghcr.io", - }); err != nil { - t.Fatal(err) - } + outputDir := renderStatic(t, staticRender{namespace: "custom-system", image: "operator:test", registry: "ghcr.io"}) data, err := os.ReadFile(filepath.Join(outputDir, "02-rbac.yaml")) if err != nil { diff --git a/designs/component-workload-overrides.md b/designs/component-workload-overrides.md index f5a24b2fc8..d6cb85797e 100644 --- a/designs/component-workload-overrides.md +++ b/designs/component-workload-overrides.md @@ -201,7 +201,7 @@ YAML layer would not apply to `metalman`. `apiVersion` and `kind` (`env.go:241`) and performs no kind assertion. The operator's ClusterRole grants `escalate` and `bind` on `roles`, `rolebindings`, `clusterroles`, and `clusterrolebindings` -(`deploy/unbounded-operator/02-rbac.yaml.tmpl:60-66`), deliberately, so it can +(`deploy/unbounded-operator/chart/templates/02-rbac.yaml:85-92`), deliberately, so it can install component RBAC granting permissions it does not itself hold. Any mechanism that can influence the GVK of an applied object can therefore mint a cluster-admin binding. GVK containment is not optional. @@ -1133,7 +1133,7 @@ Dropping the operations instead leaves those workloads exactly as they are. The cluster holds the last good state because the operator does not write, which makes the behavior **restart-safe by construction**. This matters concretely: the operator runs `replicas: 1` with `strategy: Recreate` -(`deploy/unbounded-operator/04-deployment.yaml.tmpl:13-16`), so it restarts on +(`deploy/unbounded-operator/chart/templates/04-deployment.yaml:11-13`), so it restarts on every upgrade. An earlier revision cached last-known-good in memory, which that restart discards; the next apply would then have stripped every override. There is now no in-memory state to lose. @@ -1780,7 +1780,7 @@ images survive upgrades indefinitely and are the most likely cause of an install that behaves unlike its reported version. The operator restarts on every upgrade: it runs `replicas: 1` with -`strategy: Recreate` (`deploy/unbounded-operator/04-deployment.yaml.tmpl:13-16`). +`strategy: Recreate` (`deploy/unbounded-operator/chart/templates/04-deployment.yaml:11-13`). Because the failure model holds no in-memory state ([ยง9.3](#93-invalid-overrides-skip-they-do-not-revert)), that restart changes nothing about override behavior. diff --git a/docs/content/reference/cli.md b/docs/content/reference/cli.md index ecfa1a5b7b..30da88756f 100644 --- a/docs/content/reference/cli.md +++ b/docs/content/reference/cli.md @@ -98,6 +98,32 @@ change. A private or air-gapped install sets `--image-registry` (e.g. already), so a private or air-gapped setup runs `install --image-registry ...` first, then `site init`. +The operator objects are defined once, as the Helm chart in +`deploy/unbounded-operator/chart`. `install` applies manifests rendered from +that chart at build time (`make unbounded-operator-manifests`), and the same +render is published with each release as `unbounded-operator-.yaml`. Platforms +that manage cluster add-ons with Helm, such as AKS cluster extensions, can +install the chart directly; `imageRegistry`, `image.tag`, +`apiServerEndpoint` and `reapLegacyResources` correspond to the flags above. +Install it into the operator namespace: + +```bash +helm install unbounded-operator deploy/unbounded-operator/chart \ + --namespace unbounded-system --create-namespace \ + --set image.tag= +``` + +Use one method per cluster. Each refuses to take over the other: the chart +won't install over an operator that `install` created, and `install` won't +apply over an operator Deployment or ClusterRoleBinding owned by a Helm release +(including the AKS flex operator extension). To switch methods, remove the +existing install first. + +`helm uninstall` removes the operator, its RBAC and its ConfigMap. The chart +never creates the release namespace (`--create-namespace` does), so Helm leaves +the namespace alone, and the CRDs, Sites and the components the operator +reconciled into it, including unbounded-net, keep running. + > **Breaking change:** `--image-registry` is now a full image-repository prefix > (registry host plus org/namespace), not a bare host the operator appends > `/azure/` to. `--operator-image` is now a `name:tag` with no registry (the diff --git a/internal/operator/bootstrap.go b/internal/operator/bootstrap.go index 998f3ffc06..7a34effd67 100644 --- a/internal/operator/bootstrap.go +++ b/internal/operator/bootstrap.go @@ -40,7 +40,7 @@ const ( // CRDBootstrapTimeout bounds the complete CRD bootstrap, including manifest // applies and waiting for every CRD to be served by the apiserver. The operator // health server only binds after bootstrap, so the startupProbe budget in -// deploy/unbounded-operator/04-deployment.yaml.tmpl must exceed this timeout. +// deploy/unbounded-operator/chart/templates/04-deployment.yaml must exceed this timeout. const CRDBootstrapTimeout = 4 * time.Minute // RequiredCRDNames is the complete set of CRDs owned and bootstrapped by the diff --git a/internal/operator/rbac_test.go b/internal/operator/rbac_test.go index 8dc2fb35a7..e55b3dee49 100644 --- a/internal/operator/rbac_test.go +++ b/internal/operator/rbac_test.go @@ -36,10 +36,9 @@ var kindToResource = map[string]string{ "RoleBinding": "rolebindings", } -// loadOperatorClusterRole parses the operator ClusterRole from its template. The -// ClusterRole document contains no Go-template actions (only the accompanying -// ClusterRoleBinding does), so it parses as plain YAML directly from the source -// of truth without needing the manifests to be rendered. +// loadOperatorClusterRole parses the operator ClusterRole from the static +// manifests rendered from deploy/unbounded-operator/chart by +// `make unbounded-operator-manifests` (a prerequisite of `make test`). func loadOperatorClusterRole(t *testing.T) *rbacv1.ClusterRole { t.Helper() @@ -49,17 +48,16 @@ func loadOperatorClusterRole(t *testing.T) *rbacv1.ClusterRole { } path := filepath.Join(filepath.Dir(thisFile), "..", "..", - "deploy", "unbounded-operator", "02-rbac.yaml.tmpl") + "deploy", "unbounded-operator", "rendered", "02-rbac.yaml") data, err := os.ReadFile(path) if err != nil { - t.Fatalf("read operator rbac template: %v", err) + t.Fatalf("read rendered operator rbac (run `make unbounded-operator-manifests`): %v", err) } for _, doc := range strings.Split(string(data), "\n---") { doc = strings.TrimSpace(doc) - // Skip empty docs and the templated ClusterRoleBinding. - if doc == "" || strings.Contains(doc, "{{") { + if doc == "" { continue } diff --git a/internal/unbounded/namespace_drift_test.go b/internal/unbounded/namespace_drift_test.go index e8bb980b65..cae2ad91a7 100644 --- a/internal/unbounded/namespace_drift_test.go +++ b/internal/unbounded/namespace_drift_test.go @@ -39,7 +39,6 @@ func TestSystemNamespace_MatchesTemplateDefaults(t *testing.T) { {"machina", filepath.Join("deploy", "machina"), "01-namespace.yaml"}, {"machine-ops", filepath.Join("deploy", "machine-ops"), "00-namespace.yaml"}, {"orca", filepath.Join("deploy", "orca"), "01-namespace.yaml"}, - {"unbounded-operator", filepath.Join("deploy", "unbounded-operator"), "00-namespace.yaml"}, {"net", filepath.Join("deploy", "net"), "00-namespace.yaml"}, {"inventory", filepath.Join("deploy", "inventory"), filepath.Join("common", "01-namespace.yaml")}, } @@ -69,9 +68,9 @@ func TestSystemNamespace_MatchesTemplateDefaults(t *testing.T) { } } -// Gantry's chart uses Helm's release namespace rather than a hardcoded template -// fallback. The Makefile default passed to Helm is therefore its namespace -// source of truth. +// Gantry's and unbounded-operator's charts use Helm's release namespace rather +// than a hardcoded template fallback. The Makefile defaults passed to Helm are +// therefore their namespace source of truth. func TestSystemNamespace_MatchesGantryMakeDefault(t *testing.T) { root := repoRoot(t) @@ -83,9 +82,10 @@ func TestSystemNamespace_MatchesGantryMakeDefault(t *testing.T) { for _, assignment := range []string{ "UNBOUNDED_NAMESPACE ?= " + systemNamespace, "GANTRY_NAMESPACE ?= $(UNBOUNDED_NAMESPACE)", + "UNBOUNDED_OPERATOR_NAMESPACE ?= $(UNBOUNDED_NAMESPACE)", } { if !strings.Contains(string(raw), assignment) { - t.Fatalf("Makefile is missing %q; keep Gantry's render default aligned with systemNamespace %q", assignment, systemNamespace) + t.Fatalf("Makefile is missing %q; keep the Helm render defaults aligned with systemNamespace %q", assignment, systemNamespace) } } }