karpenter: build, package and deploy the unbounded-karpenter controller

.github/workflows/ci.yaml

@@ -162,8 +162,10 @@ jobs:
       - name: Test
         run: make test
 
+      # The plugin's API-server tests skip without envtest assets, so
+      # provision them here.
       - name: Test Karpenter plugin
-        run: make karpenter-test
+        run: make karpenter-test-ci
 
       - name: Upload coverage
         if: always()

AGENTS.md

@@ -28,6 +28,7 @@ unbounded-kube is organized into several directories:
 - `deploy/` - component manifests for deploying on a Kubernetes cluster.
   - `machina/` - machina controller manifest templates (*.yaml.tmpl) plus generated CRDs under `crd/`; rendered output lives under `machina/rendered/` (gitignored, produced by `make machina-manifests`).
   - `gantry/` - gantry DaemonSet, ConfigMap, and ServiceAccount manifests.
+  - `karpenter/` - rendered Karpenter plugin manifests under `karpenter/rendered/` (gitignored, produced by `make karpenter-manifests`); the template lives in the plugin module.
   - `net/` - unbounded-net controller and node manifest templates (*.yaml.tmpl); rendered output lives under `net/rendered/` (gitignored, produced by `make net-manifests`).
 - `designs/` - design documents, proposals, and internal planning documentation for the project.
 - `docs/` - public web site documentation only. Do not place design documents, plans, or ad-hoc internal docs here.
@@ -58,6 +59,10 @@ unbounded-kube is organized into several directories:
 - To build individual net binaries: `make unbounded-net-controller`, `make unbounded-net-node`, `make unbounded-net-routeplan-debug`, `make unping`, `make unroute`.
 - To build `gantry` use `make gantry` which runs tests and builds the binary.
 - To build `gantry` without lint/test use `make gantry-build` (used in Containerfiles).
+- The Karpenter plugin is a nested module, so the root `make fmt`, `make lint`, `make test` and `go build ./...` do not
+  cover it. Use `make karpenter` (lint, test, build), `make karpenter-fmt`, `make karpenter-test`,
+  `make karpenter-manifests` and `make image-karpenter-local`. Do not run `golangci-lint fmt` inside `plugins/`: it
+  resolves paths from the root config and reformats the whole repository.
 - Net-specific build tasks (container images, frontend, eBPF, render) are exposed via `net-` prefixed targets in the main `Makefile` (e.g., `make net-frontend`, `make net-ebpf-build`, `make net-ebpf-generate`, `make net-manifests`). Cluster deploy/undeploy targets live separately under `hack/net/` and are invoked via `make -C hack/net <target>` (e.g., `make -C hack/net deploy`). Run `make help` and `make -C hack/net help` for the full lists.
 - `make generate` runs `go generate ./...` to regenerate deepcopy, CRDs, and protobuf for all packages.
 - `make build` compiles all Go packages (`go build ./...`).

Makefile

@@ -92,7 +92,10 @@ MACHINE_OPS_CONTROLLER_IMAGE ?= $(CONTAINER_REGISTRY)/machine-ops-controller:$(V
 # The Unbounded Karpenter plugin is a nested Go module (plugins/karpenter),
 # because upstream Karpenter pins older Kubernetes libraries than this module.
 # Its targets delegate to plugins/karpenter/Makefile.
-KARPENTER_MAKE = $(MAKE) -C plugins/karpenter
+KARPENTER_IMAGE ?= $(CONTAINER_REGISTRY)/unbounded-karpenter:$(VERSION_TAG)
+KARPENTER_NAMESPACE ?= $(UNBOUNDED_NAMESPACE)
+KARPENTER_MAKE = $(MAKE) -C plugins/karpenter VERSION=$(VERSION) IMAGE=$(KARPENTER_IMAGE) \
+	NAMESPACE=$(KARPENTER_NAMESPACE) CONTAINER_ENGINE=$(CONTAINER_ENGINE)
 MACHINE_OPS_CONTROLLER_NAME ?= machine-ops-controller
 MACHINE_OPS_PROVIDER ?=
 MACHINE_OPS_SITE ?=
@@ -241,7 +244,7 @@ REACT_DEV ?= false
 
 .PHONY: all help fmt lint lint-actions test build vulncheck check-deps kubectl-unbounded kubectl-unbounded-build install-tools install-protoc install-helm install-kind generate kubectl-unbounded forge relctl relctl-build agent-artifacts-builder agent-artifacts-builder-build orcadev unbounded-agent machina machina-build machina-oci machina-oci-push machina-manifests machine-ops-controller machine-ops-controller-build machine-ops-controller-oci machine-ops-controller-oci-push machine-ops-manifests metalman metalman-build metalman-oci metalman-oci-push unbounded-operator unbounded-operator-build unbounded-operator-manifests playpen-manifests e2e-gantry e2e-playpen e2e-racer gomod docs-serve unbounded-net-controller unbounded-net-controller-build unbounded-net-node unbounded-net-node-build unbounded-net-routeplan-debug unping unping-build unroute unroute-build license-check notice notice-check gantry gantry-build gantry-manifests inventory-manifests
 .PHONY: net-frontend net-frontend-clean net-ebpf-build net-ebpf-generate net-ebpf-verify net-manifests gantry-chart-lint gantry-chart-package release-bom release-manifests unbounded-operator-release-manifest
-.PHONY: karpenter karpenter-fmt karpenter-lint karpenter-test
+.PHONY: karpenter karpenter-fmt karpenter-lint karpenter-test karpenter-test-ci karpenter-build karpenter-manifests image-karpenter-local
 .PHONY: image-machina-local image-token-refresher-local image-machine-ops-controller-local image-metalman-local image-unbounded-operator-local image-unbounded-operator-push image-playpen-local image-net-controller-local image-net-node-local image-gantry-local image-gantry-push images-local
 .PHONY: image-net-controller-push image-net-node-push images-net-all images-net-all-push
 
@@ -266,8 +269,10 @@ help: ## Show this help
 	@echo "  lint                             Run golangci-lint and actionlint"
 	@echo "  lint-actions                     Run actionlint over .github/workflows"
 	@echo "  test                             Run all tests"
-	@echo "  karpenter                        Lint and test the Karpenter plugin (nested module)"
-	@echo "  karpenter-fmt|lint|test          Individual Karpenter plugin steps"
+	@echo "  karpenter                        Lint, test and build the Karpenter plugin (nested module)"
+	@echo "  karpenter-fmt|lint|test|build    Individual Karpenter plugin steps"
+	@echo "  karpenter-manifests              Render the Karpenter plugin manifests"
+	@echo "  image-karpenter-local            Build the unbounded-karpenter image locally"
 	@echo "  build                            Compile all Go packages"
 	@echo "  generate                         Run go generate (deepcopy, CRDs, protobuf)"
 	@echo "  vulncheck                        Run govulncheck; fails only on vulnerabilities with final fixes"
@@ -991,7 +996,21 @@ karpenter-lint: ## Lint the Karpenter plugin module
 karpenter-test: ## Test the Karpenter plugin module
 	$(KARPENTER_MAKE) test
 
-karpenter: karpenter-lint karpenter-test ## Lint and test the Karpenter plugin
+karpenter-test-ci: $(SETUP_ENVTEST) ## Provision pinned API-server assets and test the Karpenter plugin module
+	@mkdir -p tmp/karpenter-envtest
+	@assets=$$(TMPDIR="$(CURDIR)/tmp/karpenter-envtest" timeout --signal=TERM --kill-after=10s 300s "$(SETUP_ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(CURDIR)/bin/envtest" -p path) && \
+		$(MAKE) karpenter-test KUBEBUILDER_ASSETS="$$assets"
+
+karpenter-build: ## Build the unbounded-karpenter binary (no lint/test)
+	$(KARPENTER_MAKE) build
+
+karpenter: karpenter-lint karpenter-test karpenter-build ## Build the Karpenter plugin (implies lint and test)
+
+karpenter-manifests: ## Render the Karpenter plugin manifests into deploy/karpenter/rendered
+	$(KARPENTER_MAKE) manifests
+
+image-karpenter-local: ## Build the unbounded-karpenter container image locally (single-arch)
+	$(KARPENTER_MAKE) image
 
 ##@ Gantry (peer-to-peer OCI distribution)
 

deploy/karpenter/rendered/.gitignore

@@ -0,0 +1,4 @@
+# `make karpenter-manifests` writes the rendered Karpenter plugin manifests
+# here; none of that output is tracked.
+*
+!.gitignore

images/karpenter/Containerfile

@@ -0,0 +1,48 @@
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# The Unbounded Karpenter plugin: upstream Karpenter's controllers compiled
+# with the Unbounded cloud provider. Build from the repository root:
+#   docker build -f images/karpenter/Containerfile .
+
+# Pin the builder to BUILDPLATFORM so the Go toolchain runs natively even when
+# targeting a foreign TARGETPLATFORM.
+FROM --platform=$BUILDPLATFORM docker.io/library/golang:1.27.3-trixie AS build
+
+ENV CGO_ENABLED=0 GOTOOLCHAIN=auto
+WORKDIR /src
+
+# The plugin is its own Go module under plugins/karpenter.
+COPY plugins/karpenter/go.mod plugins/karpenter/go.sum ./
+RUN go mod download
+
+COPY plugins/karpenter/ ./
+
+ARG TARGETOS=linux
+ARG TARGETARCH=amd64
+ARG VERSION=dev
+
+RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -trimpath \
+    -ldflags "-s -w -X sigs.k8s.io/karpenter/pkg/operator.Version=${VERSION}" \
+    -o /unbounded-karpenter ./cmd/unbounded-karpenter
+
+# Ship the license and notice files of the modules the binary is built
+# around, whatever versions go.mod pins.
+RUN set -eu; \
+    for module in sigs.k8s.io/karpenter github.com/awslabs/operatorpkg; do \
+        directory="$(go list -m -f '{{.Dir}}' "${module}")"; \
+        target="/licenses/$(basename "${module}")"; \
+        mkdir -p "${target}"; \
+        for file in LICENSE NOTICE; do \
+            if [ -f "${directory}/${file}" ]; then cp "${directory}/${file}" "${target}/"; fi; \
+        done; \
+    done
+
+FROM gcr.io/distroless/static-debian13:nonroot
+
+COPY --from=build /unbounded-karpenter /unbounded-karpenter
+COPY --from=build /licenses/ /unbounded/licenses/
+COPY LICENSE NOTICE /unbounded/licenses/
+
+USER 65532:65532
+ENTRYPOINT ["/unbounded-karpenter"]

plugins/karpenter/Makefile

@@ -3,13 +3,21 @@
 # targets from the repository root as `make karpenter-<target>`.
 
 ROOT := $(abspath ../..)
+BIN_DIR ?= $(ROOT)/bin
+RENDERED_DIR ?= $(ROOT)/deploy/karpenter/rendered
 GOLANGCI_LINT ?= golangci-lint
+CONTAINER_ENGINE ?= docker
+
+VERSION ?= dev
+NAMESPACE ?= unbounded-system
+IMAGE ?= ghcr.io/azure/unbounded-karpenter:$(VERSION)
+PRICE_CURRENCY ?= USD
 
 export CGO_ENABLED := 0
 
-.PHONY: all tidy fmt lint test
+.PHONY: all tidy fmt lint test build manifests image
 
-all: lint test
+all: lint test build
 
 tidy:
 	go mod tidy
@@ -18,11 +26,35 @@ tidy:
 # `golangci-lint fmt` with the root config would format the whole repository.
 # Format only this module, the way the root `make fmt` does.
 fmt:
-	gofumpt -w ./internal
+	gofumpt -w ./cmd ./internal
 	$(GOLANGCI_LINT) run --fix -c "$(ROOT)/.golangci.yaml" ./...
 
 lint:
 	$(GOLANGCI_LINT) run -c "$(ROOT)/.golangci.yaml" ./...
 
-test:
-	go test -count=1 ./...
+# The API-server tests in internal/integration skip unless KUBEBUILDER_ASSETS
+# is set. Use the repository's envtest assets when they are present, and give
+# the binary test the built binary.
+ENVTEST_ASSETS ?= $(firstword $(wildcard $(ROOT)/bin/envtest/k8s/*))
+
+test: build
+	KUBEBUILDER_ASSETS="$(or $(KUBEBUILDER_ASSETS),$(ENVTEST_ASSETS))" \
+	UNBOUNDED_KARPENTER_BINARY="$(BIN_DIR)/unbounded-karpenter" \
+		go test -count=1 ./...
+
+build:
+	go build -trimpath \
+		-ldflags "-X sigs.k8s.io/karpenter/pkg/operator.Version=$(VERSION)" \
+		-o "$(BIN_DIR)/unbounded-karpenter" ./cmd/unbounded-karpenter
+
+manifests:
+	@mkdir -p "$(RENDERED_DIR)"
+	go run ./cmd/render-manifests \
+		--namespace "$(NAMESPACE)" \
+		--image "$(IMAGE)" \
+		--price-currency "$(PRICE_CURRENCY)" \
+		--output "$(RENDERED_DIR)/karpenter.yaml"
+
+image:
+	$(CONTAINER_ENGINE) build --build-arg VERSION="$(VERSION)" \
+		-t "$(IMAGE)" -f "$(ROOT)/images/karpenter/Containerfile" "$(ROOT)"

plugins/karpenter/README.md

@@ -0,0 +1,55 @@
+# Unbounded Karpenter plugin
+
+`unbounded-karpenter` is upstream [Karpenter](https://github.com/kubernetes-sigs/karpenter)
+compiled with one cloud provider: Unbounded. It autoscales a cluster through
+two Unbounded resources and nothing else:
+
+- it reads **MachineClass** and presents its offerings as instance types;
+- it creates one **MachineAllocation** per NodeClaim and reports it back.
+
+The Unbounded provider controllers behind those resources launch and release
+the machines. The plugin has no cloud SDK, credentials, or per-provider code,
+so the same binary works with every provider that serves MachineClass and
+MachineAllocation.
+
+## Use
+
+1. Run a provider controller with capacity enabled, and create a MachineClass
+   for it. The class must be `Ready`.
+2. Install the plugin:
+
+   ```sh
+   make karpenter-manifests KARPENTER_IMAGE=<registry>/unbounded-karpenter:<tag>
+   kubectl apply --server-side -f deploy/karpenter/rendered/karpenter.yaml
+   ```
+
+3. Point a NodePool at the class. No NodeClass wrapper is needed:
+
+   ```yaml
+   apiVersion: karpenter.sh/v1
+   kind: NodePool
+   metadata:
+     name: workers
+   spec:
+     template:
+       spec:
+         nodeClassRef:
+           group: unbounded-cloud.io
+           kind: MachineClass
+           name: workers
+         requirements:
+           - key: karpenter.sh/capacity-type
+             operator: In
+             values: ["on-demand"]
+   ```
+
+Provider controllers need no Karpenter-specific flag, role, or start order.
+
+## What an offering needs
+
+Karpenter can use a MachineClass offering that has positive `cpu`, `memory`,
+and `pods` capacity and a `kubernetes.io/arch` label. The plugin defaults the
+rest: the zone to the class's Site name, the operating system to `linux`, and
+a missing price (or one in a currency other than `UNBOUNDED_PRICE_CURRENCY`,
+default `USD`) to an estimate from CPU and memory. An offering it cannot use
+is skipped and counted in `unbounded_karpenter_unusable_offerings`.

plugins/karpenter/cmd/render-manifests/main.go

@@ -0,0 +1,54 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+// Command render-manifests writes the plugin's deployment manifests,
+// including the Karpenter CRDs of the pinned upstream module.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/Azure/unbounded/plugins/karpenter/internal/manifests"
+)
+
+func main() {
+	var (
+		options manifests.Options
+		output  string
+	)
+
+	flag.StringVar(&options.Namespace, "namespace", "unbounded-system", "Controller namespace")
+	flag.StringVar(&options.Image, "image", "", "Explicit controller image reference")
+	flag.StringVar(&options.PriceCurrency, "price-currency", "USD", "ISO 4217 currency whose offering prices are used")
+	flag.StringVar(&output, "output", "-", "Output file, or - for standard output")
+	flag.Parse()
+
+	if err := run(options, output, os.Stdout); err != nil {
+		fmt.Fprintf(os.Stderr, "render manifests: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func run(options manifests.Options, output string, stdout io.Writer) error {
+	content, err := manifests.Render(options)
+	if err != nil {
+		return err
+	}
+
+	if output == "-" {
+		if _, err := stdout.Write(content); err != nil {
+			return fmt.Errorf("write manifests: %w", err)
+		}
+
+		return nil
+	}
+
+	if err := os.WriteFile(output, content, 0o600); err != nil {
+		return fmt.Errorf("write manifests: %w", err)
+	}
+
+	return nil
+}

plugins/karpenter/cmd/unbounded-karpenter/main.go

@@ -0,0 +1,52 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/karpenter/pkg/operator"
+
+	"github.com/Azure/unbounded/plugins/karpenter/internal/controller"
+	"github.com/Azure/unbounded/plugins/karpenter/internal/lifecycle"
+	"github.com/Azure/unbounded/plugins/karpenter/internal/nodeclass"
+	"github.com/Azure/unbounded/plugins/karpenter/internal/provider"
+)
+
+const (
+	// envPriceCurrency selects the ISO 4217 currency whose offering prices
+	// are used as Karpenter prices. Karpenter's operator owns the command
+	// line, so the plugin's settings are environment variables.
+	envPriceCurrency = "UNBOUNDED_PRICE_CURRENCY"
+)
+
+func main() {
+	nodeclass.AddToScheme(scheme.Scheme)
+
+	ctx, manager := operator.NewOperator()
+
+	if err := provider.RegisterIndexes(ctx, manager.GetFieldIndexer()); err != nil {
+		fmt.Fprintf(os.Stderr, "configure capacity indexes: %v\n", err)
+		os.Exit(1)
+	}
+
+	cloudProvider, err := provider.New(manager.GetCache(), manager.GetAPIReader(), manager.GetClient(), provider.Options{
+		PriceCurrency: os.Getenv(envPriceCurrency),
+	})
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "configure cloud provider: %v\n", err)
+		os.Exit(1)
+	}
+
+	allocations := &lifecycle.Controller{
+		Client:   manager.GetClient(),
+		Reader:   manager.GetAPIReader(),
+		Recorder: manager.GetEventRecorderFor("unbounded-karpenter"), //nolint:staticcheck // SA1019: the upstream operator uses the same recorder API.
+		Marker:   cloudProvider,
+	}
+
+	manager.WithControllers(ctx, controller.Controllers(ctx, manager, cloudProvider, allocations)...).Start(ctx)
+}

plugins/karpenter/go.mod

@@ -61,7 +61,6 @@ require (
 	github.com/spf13/cobra v1.10.2 // indirect
 	github.com/spf13/pflag v1.0.10 // indirect
 	github.com/stoewer/go-strcase v1.3.0 // indirect
-	github.com/stretchr/objx v0.5.3 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	go.opentelemetry.io/otel v1.44.0 // indirect
 	go.opentelemetry.io/otel/trace v1.44.0 // indirect
@@ -84,6 +83,7 @@ require (
 	gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	k8s.io/apiserver v0.36.0 // indirect
+	k8s.io/autoscaler/vertical-pod-autoscaler v1.7.0 // indirect
 	k8s.io/cloud-provider v0.35.0 // indirect
 	k8s.io/component-base v0.36.1 // indirect
 	k8s.io/component-helpers v0.35.0 // indirect