Start nspawn workers after reboot on RHEL and recover stopped workers

CHANGELOG.md

@@ -15,6 +15,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 - Refresh supplemental networking, offline-artifact, and NVIDIA GPU labs for AKS Flex Node `v0.3.0` and Unbounded `v0.21.0`, with recorded validation scopes. Correct offline agent staging, ARM egress, Gantry's loopback port conflict, and WireGuard host-port syntax.
 
+### Fixed
+
+- Start enabled nspawn workers after reboot by making the daemon unit pull in `machines.target`.
+- Handle a missing `machines.target` during preflight: warn for native online bootstrap, but fail for offline or containerized bootstrap; masked targets are always fatal.
+
 ## [0.3.0] - 2026-11-04
 
 ### Added

docs/usage/operations.md

@@ -62,6 +62,8 @@ aks-flex-node preflight \
 
 When `bootstrap.offlineArtifacts.source` is configured, missing host packages are fatal because offline bootstrap cannot rely on package installation during `start`.
 
+The `machines-target` check warns when `machines.target` is missing during native online bootstrap because bootstrap can install `systemd-container`. A missing target is fatal when offline artifacts are configured or the agent runs in a container, because neither mode can install host packages; provision `systemd-container` on the host first. A masked target, or a failure to inspect the target, is always fatal. A disabled target produces a warning: the installed daemon unit declares `Wants=machines.target`, so workers start at boot even on hosts whose systemd preset disables the target. Persistent linked targets are accepted; runtime-only links produce a warning because they may not survive reboot. Unrecognized unit-file states also produce a warning to verify that the target is available and persistent.
+
 ## Start
 
 Start installs host components, starts the nspawn-backed worker, installs the systemd unit, and starts the agent daemon.

pkg/cmd/preflight/preflight.go

@@ -83,7 +83,12 @@ func (h *handler) execute(ctx context.Context) error {
 		}
 	}
 
+	failMachinesTargetForMissing := daemon.Containerized() || strings.TrimSpace(cfg.Bootstrap.OfflineArtifacts.Source) != ""
 	checks := daemon.PreflightChecks(preflight.Flatten(
+		[]preflight.Checker{daemon.CheckMachinesTarget(
+			log,
+			failMachinesTargetForMissing,
+		)},
 		arc.Preflight(cfg, log),
 		host.Preflight(log, *agentCfg, gs),
 		nodeChecks,

pkg/daemon/assets/aks-flex-node-agent-container.service

@@ -3,7 +3,7 @@ Description=AKS Flex Node Agent
 # Arc's identity service, on a host with Arc, as a host's agent unit with Arc
 # enabled has: the agent's credentials may come from it. Without it, systemd
 # ignores both.
-Wants=network-online.target himdsd.service
+Wants=network-online.target machines.target himdsd.service
 # A restart after a failure waits for a recovery the failure started, so it runs
 # the image recovery restores rather than the one that failed: recovery starts a
 # container, which takes longer than RestartSec.

pkg/daemon/assets/aks-flex-node-agent.service

@@ -1,7 +1,7 @@
 [Unit]
 Description=AKS Flex Node Agent
 After=network-online.target systemd-udev-settle.service{{range .Dependencies}} {{.}}{{end}}
-Wants=network-online.target systemd-udev-settle.service{{range .Dependencies}} {{.}}{{end}}
+Wants=network-online.target systemd-udev-settle.service machines.target{{range .Dependencies}} {{.}}{{end}}
 OnFailure=aks-flex-node-agent-recovery.service
 # Trigger last-good recovery after the upgraded daemon repeatedly fails.
 StartLimitIntervalSec=300

pkg/daemon/preflight.go

@@ -0,0 +1,49 @@
+package daemon
+
+import (
+	"context"
+	"log/slog"
+	"strings"
+
+	"github.com/Azure/AKSFlexNode/pkg/utils/utilexec"
+	"github.com/Azure/unbounded/pkg/agent/preflight"
+)
+
+type machinesTargetChecker struct {
+	log            *slog.Logger
+	failForMissing bool
+	output         func(context.Context, *slog.Logger, string, ...string) (string, error)
+}
+
+// CheckMachinesTarget checks that systemd can start nspawn workers at boot.
+// TODO: Move this check to Unbounded's host preflight checks.
+func CheckMachinesTarget(log *slog.Logger, failForMissing bool) preflight.Checker {
+	return machinesTargetChecker{log: log, failForMissing: failForMissing, output: utilexec.OutputCmd}
+}
+
+func (machinesTargetChecker) Name() string { return "machines-target" }
+
+func (c machinesTargetChecker) Check(ctx context.Context) []preflight.Result {
+	const target = "machines.target"
+	state, err := c.output(ctx, c.log, "systemctl", "show", "--property=UnitFileState", "--value", target)
+	if err != nil {
+		return preflight.ResultsError(c.Name(), target, "failed to inspect machines.target")
+	}
+	switch strings.TrimSpace(state) {
+	case "enabled", "static", "linked":
+		return preflight.ResultsOK(c.Name(), target, "machines.target is available for worker startup")
+	case "disabled", "enabled-runtime":
+		return preflight.ResultsWarning(c.Name(), target, "machines.target is not enabled persistently; bootstrap installs a daemon Wants=machines.target dependency to start workers at boot")
+	case "linked-runtime":
+		return preflight.ResultsWarning(c.Name(), target, "machines.target is linked only at runtime; make the link persistent for workers to start at boot")
+	case "", "not-found":
+		if c.failForMissing {
+			return preflight.ResultsError(c.Name(), target, "machines.target is missing; install systemd-container on the host before offline or containerized bootstrap")
+		}
+		return preflight.ResultsWarning(c.Name(), target, "machines.target is missing and may be installed by online bootstrap with systemd-container")
+	case "masked", "masked-runtime":
+		return preflight.ResultsError(c.Name(), target, "machines.target is masked; unmask it before bootstrap")
+	default:
+		return preflight.ResultsWarning(c.Name(), target, "machines.target has unrecognized UnitFileState %q; verify that it is available and persistent", strings.TrimSpace(state))
+	}
+}

pkg/daemon/preflight_test.go

@@ -0,0 +1,136 @@
+package daemon
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"slices"
+	"strings"
+	"testing"
+
+	"github.com/Azure/unbounded/pkg/agent/preflight"
+)
+
+func TestCheckMachinesTarget(t *testing.T) {
+	t.Parallel()
+
+	tests := map[string]struct {
+		state          string
+		err            error
+		failForMissing bool
+		want           preflight.Severity
+		wantMessage    string
+	}{
+		"enabled":                 {state: "enabled\n", want: preflight.SeverityOK},
+		"static":                  {state: "static", want: preflight.SeverityOK},
+		"linked":                  {state: "linked", want: preflight.SeverityOK},
+		"disabled":                {state: "disabled", want: preflight.SeverityWarning},
+		"runtime only":            {state: "enabled-runtime", want: preflight.SeverityWarning},
+		"runtime linked":          {state: "linked-runtime", want: preflight.SeverityWarning, wantMessage: "make the link persistent"},
+		"masked":                  {state: "masked", want: preflight.SeverityError},
+		"runtime masked":          {state: "masked-runtime", want: preflight.SeverityError},
+		"masked strict":           {state: "masked", failForMissing: true, want: preflight.SeverityError},
+		"runtime masked strict":   {state: "masked-runtime", failForMissing: true, want: preflight.SeverityError},
+		"missing online":          {want: preflight.SeverityWarning, wantMessage: "may be installed by online bootstrap with systemd-container"},
+		"missing strict":          {failForMissing: true, want: preflight.SeverityError, wantMessage: "install systemd-container on the host before offline or containerized bootstrap"},
+		"not found online":        {state: "not-found", want: preflight.SeverityWarning},
+		"not found strict":        {state: "not-found", failForMissing: true, want: preflight.SeverityError, wantMessage: "install systemd-container on the host before offline or containerized bootstrap"},
+		"unrecognized unit state": {state: "future-state", want: preflight.SeverityWarning, wantMessage: "future-state"},
+		"inspect failure":         {err: errors.New("systemd unavailable"), want: preflight.SeverityError},
+	}
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			check := machinesTargetChecker{
+				log:            slog.Default(),
+				failForMissing: tt.failForMissing,
+				output: func(_ context.Context, _ *slog.Logger, command string, args ...string) (string, error) {
+					if command != "systemctl" || !slices.Equal(args, []string{"show", "--property=UnitFileState", "--value", "machines.target"}) {
+						t.Fatalf("unexpected command: %s %v", command, args)
+					}
+					return tt.state, tt.err
+				},
+			}
+			for range 2 {
+				results := check.Check(t.Context())
+				if len(results) != 1 || results[0].Severity != tt.want || results[0].Name != "machines-target" {
+					t.Fatalf("Check() = %#v, want severity %s", results, tt.want)
+				}
+				if !strings.Contains(results[0].Message, tt.wantMessage) {
+					t.Fatalf("Check() message = %q, want it to include %q", results[0].Message, tt.wantMessage)
+				}
+			}
+		})
+	}
+}
+
+func TestCheckMachinesTargetBootstrapModes(t *testing.T) {
+	t.Parallel()
+
+	modes := map[string]struct {
+		containerized bool
+		offline       bool
+		wantMissing   preflight.Severity
+	}{
+		"native online":         {wantMissing: preflight.SeverityWarning},
+		"native offline":        {offline: true, wantMissing: preflight.SeverityError},
+		"containerized online":  {containerized: true, wantMissing: preflight.SeverityError},
+		"containerized offline": {containerized: true, offline: true, wantMissing: preflight.SeverityError},
+	}
+	for name, mode := range modes {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			for _, state := range []string{"", "not-found", "masked", "masked-runtime"} {
+				t.Run("state="+state, func(t *testing.T) {
+					t.Parallel()
+
+					check, ok := CheckMachinesTarget(slog.Default(), mode.containerized || mode.offline).(machinesTargetChecker)
+					if !ok {
+						t.Fatal("CheckMachinesTarget() returned an unexpected checker type")
+					}
+					check.output = func(context.Context, *slog.Logger, string, ...string) (string, error) {
+						return state, nil
+					}
+					want := preflight.SeverityError
+					if state == "" || state == "not-found" {
+						want = mode.wantMissing
+					}
+					results := check.Check(t.Context())
+					if len(results) != 1 || results[0].Severity != want {
+						t.Fatalf("Check() = %#v, want severity %s", results, want)
+					}
+					if (state == "" || state == "not-found") && want == preflight.SeverityError &&
+						!strings.Contains(results[0].Message, "install systemd-container on the host") {
+						t.Fatalf("Check() message = %q, want host package provisioning guidance", results[0].Message)
+					}
+				})
+			}
+		})
+	}
+}
+
+func TestDaemonUnitsWantMachinesTarget(t *testing.T) {
+	t.Parallel()
+
+	hostUnit, err := renderAgentServiceUnit("/usr/local/bin/aks-flex-node", agentServiceOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for name, content := range map[string][]byte{
+		"host":      hostUnit,
+		"container": containerServiceUnitContent,
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			for _, line := range strings.Split(string(content), "\n") {
+				if wants, ok := strings.CutPrefix(line, "Wants="); ok && slices.Contains(strings.Fields(wants), "machines.target") {
+					return
+				}
+			}
+			t.Fatal("daemon unit must want machines.target so workers start even when the target is disabled")
+		})
+	}
+}

pkg/daemon/testdata/aks-flex-node-agent-arc.service.golden

@@ -1,7 +1,7 @@
 [Unit]
 Description=AKS Flex Node Agent
 After=network-online.target systemd-udev-settle.service himdsd.service
-Wants=network-online.target systemd-udev-settle.service himdsd.service
+Wants=network-online.target systemd-udev-settle.service machines.target himdsd.service
 OnFailure=aks-flex-node-agent-recovery.service
 # Trigger last-good recovery after the upgraded daemon repeatedly fails.
 StartLimitIntervalSec=300

pkg/daemon/testdata/aks-flex-node-agent.service.golden

@@ -1,7 +1,7 @@
 [Unit]
 Description=AKS Flex Node Agent
 After=network-online.target systemd-udev-settle.service
-Wants=network-online.target systemd-udev-settle.service
+Wants=network-online.target systemd-udev-settle.service machines.target
 OnFailure=aks-flex-node-agent-recovery.service
 # Trigger last-good recovery after the upgraded daemon repeatedly fails.
 StartLimitIntervalSec=300