feat(sidecar): build the node sidecar from this repo as its own module

.dockerignore

@@ -18,6 +18,13 @@
 # Re-include embedded shell scripts
 !**/*.sh
 
+# Re-include //go:embed targets that are not Go source. The build fails at
+# compile time when one is missing from the context — `pattern config.toml: no
+# matching files found` — so an addition here is required whenever a new asset
+# is embedded. Currently: sidecar/tasks/defaults/config.toml, the seid config
+# the sidecar writes on first start.
+!**/tasks/defaults/*.toml
+
 # Re-include the runner image's per-kind Go text templates. The runner
 # Dockerfile (runner/Dockerfile) bakes runner/templates/ into the image
 # at /templates/. Without this re-include the seitask-runner build fails

.github/workflows/ci.yml

@@ -6,6 +6,13 @@ on:
   pull_request:
     branches: ["*"]
 
+# `push` and `pull_request` both fire on every branch, so a PR commit runs this
+# workflow twice. Without a group, superseded runs keep going — and each one now
+# builds two container images.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   lint:
     runs-on: ubuntu-latest
@@ -15,19 +22,36 @@ jobs:
         # One job per Go module. golangci-lint resolves packages with Go
         # patterns, which stop at a nested module boundary — a root-only run
         # would never lint sidecarapi/ and would pass while it was broken.
-        module: [".", "sidecarapi"]
+        module: [".", "sidecarapi", "sidecar"]
     steps:
       - uses: actions/checkout@v4
+        with:
+          # only-new-issues below diffs against the merge base, which a shallow
+          # clone does not contain — the filter then silently passes and the
+          # module's whole pre-existing backlog is reported as new. A rebase or
+          # force-push is what exposes it.
+          fetch-depth: 0
       - uses: actions/setup-go@v5
         with:
           go-version-file: go.mod
+      # only-new-issues builds its filter from the pull-request patch, and on a
+      # diff this size that silently yields nothing — every pre-existing finding
+      # in an untouched file is then reported as new. --new-from-merge-base has
+      # git compute the same thing locally, so PR size cannot defeat it. The
+      # filter exists because of pre-existing debt from the v2.8.0 → v2.12.1
+      # bump, tracked in #163; remove it once that is paid down.
+      - name: Make the merge base available to the filter
+        run: |
+          git fetch --no-tags --quiet origin \
+            "+refs/heads/${BASE}:refs/remotes/origin/${BASE}"
+        env:
+          BASE: ${{ github.base_ref || github.event.repository.default_branch }}
       - uses: golangci/golangci-lint-action@v8
         with:
           version: v2.12.1
           working-directory: ${{ matrix.module }}
-          # Temporary override — pre-existing lint debt surfaced by the
-          # v2.8.0 → v2.12.1 bump. Tracked in #163; remove once paid down.
-          only-new-issues: true
+          args: >-
+            --new-from-merge-base=origin/${{ github.base_ref || github.event.repository.default_branch }}
 
   hygiene:
     runs-on: ubuntu-latest
@@ -44,6 +68,46 @@ jobs:
       # `go mod tidy` cannot resolve the graph at all.
       - run: make tidy-check
 
+  # Build both images without pushing. Nothing else in CI compiles a Dockerfile,
+  # and the publish workflow only runs on main — so a build-context problem
+  # reaches main unseen. That is not hypothetical: `.dockerignore` starts with
+  # `**`, and an embedded asset outside its re-include list
+  # (sidecar/tasks/defaults/config.toml, a //go:embed target) failed the sidecar
+  # build with `pattern config.toml: no matching files found` while every other
+  # check stayed green.
+  docker:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    # This job builds container images from Dockerfiles the branch controls, so
+    # it gets the narrowest token in the workflow. The repository default is
+    # write.
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
+      - name: Build controller image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: false
+          platforms: linux/amd64
+          cache-from: type=gha,scope=controller
+          cache-to: type=gha,scope=controller,mode=max
+      - name: Build sidecar image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: sidecar/Dockerfile
+          push: false
+          # amd64 only here — the publish job builds arm64 too, but doubling a
+          # sei-chain-sized build on every PR is not worth catching an
+          # arch-specific break that the Dockerfile's cross-compile makes
+          # unlikely.
+          platforms: linux/amd64
+          cache-from: type=gha,scope=sidecar
+          cache-to: type=gha,scope=sidecar,mode=max
+
   test:
     runs-on: ubuntu-latest
     steps:

.github/workflows/ecr.yml

@@ -56,3 +56,52 @@ jobs:
           # controller image's build.
           cache-from: type=registry,ref=${{ steps.ecr-login.outputs.registry }}/sei/build-cache:integration-harness
           cache-to: type=registry,ref=${{ steps.ecr-login.outputs.registry }}/sei/build-cache:integration-harness,mode=max
+
+  # A separate job, not a third step in `publish`. ECR sets
+  # image_tag_mutability = IMMUTABLE, so re-running this workflow on a sha whose
+  # controller image is already pushed fails that step and would abort every
+  # step after it — the sidecar would silently never publish. Independent jobs
+  # also mean a sidecar build failure does not block the controller image.
+  publish-sidecar:
+    runs-on: ubuntu-latest
+    # Bounded so a stuck build fails rather than burning a runner for six hours;
+    # generous because a cold-cache two-arch sei-chain compile is slow.
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::189176372795:role/common/gha
+          role-duration-seconds: 900
+          aws-region: us-east-2
+
+      - id: ecr-login
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - uses: docker/setup-buildx-action@v3
+
+      # The per-node sidecar. Built from the repo root so the sidecarapi module
+      # it resolves through a filesystem `replace` is in the context.
+      - name: Build and push sidecar image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: sidecar/Dockerfile
+          push: true
+          # amd64 only. The sei-node Karpenter pool pins
+          # kubernetes.io/arch In [amd64] (platform repo,
+          # clusters/prod/default/nodepool-default.yaml), so no node can schedule
+          # an arm64 sidecar. Publishing one would also make this repository the
+          # registry's first multi-arch image index, under an IMMUTABLE tag
+          # policy whose only lifecycle rule expires *untagged* manifests — the
+          # child manifests of an index — at 7 days. Untested, and a broken index
+          # cannot be replaced at the same tag.
+          platforms: linux/amd64
+          tags: ${{ steps.ecr-login.outputs.registry }}/sei/sei-sidecar:${{ inputs.tag || github.sha }}
+          # Dedicated cache ref, for the same reason the integration-harness has
+          # one — and more so here: this build pulls the whole sei-chain graph,
+          # and the image holds an operator keyring at runtime, so a poisoned
+          # layer must not be able to reach the controller image's build.
+          cache-from: type=registry,ref=${{ steps.ecr-login.outputs.registry }}/sei/build-cache:sei-sidecar
+          cache-to: type=registry,ref=${{ steps.ecr-login.outputs.registry }}/sei/build-cache:sei-sidecar,mode=max

.golangci.yml

@@ -64,22 +64,26 @@ linters:
         - omitzero
   exclusions:
     generated: lax
+    # `path` is a REGEXP, not a glob. An unanchored `sidecar/*` means "sidecar"
+    # followed by zero or more slashes, which also matches every sidecarapi/
+    # path — it silently turned goconst off for the contract module. Anchor with
+    # ^ so a rule applies to the tree it names.
     rules:
       - linters:
           - lll
-        path: api/*
+        path: ^api/
       - linters:
           - dupl
           - lll
-        path: internal/*
+        path: ^internal/
       - linters:
           - dupl
           - lll
-        path: sdk/*
+        path: ^sdk/
       - linters:
           - dupl
           - lll
-        path: sidecarapi/*
+        path: ^sidecarapi/
       # goconst flags the JSON keys inside the request-params maps — chainId,
       # keyName, fees, gas, title, initialDeposit, address. Those literals are
       # the wire contract: reading a params map and seeing the JSON it produces
@@ -88,7 +92,49 @@ linters:
       # DRY score. The remaining hits are test fixtures (1usei, config.toml).
       - linters:
           - goconst
-        path: sidecarapi/client/*
+        path: ^sidecarapi/client/
+      # sidecar/ arrived from a repo with no golangci-lint config, so none of it
+      # has ever been linted. Everything below is pre-existing debt made visible
+      # by the move, not new code — and only-new-issues cannot help, because every
+      # file is new to this repo. Scoped here rather than left red: a permanently
+      # failing check trains people to ignore it, and this way the debt is
+      # greppable and reviewable.
+      #
+      # lll and dupl match what internal/, api/, sdk/ and sidecarapi/ already get.
+      - linters: [dupl, lll]
+        path: ^sidecar/
+      # Test code carries different standards than the binary. bodyclose and
+      # staticcheck fire only in _test.go here, and errcheck almost entirely.
+      - linters: [bodyclose, errcheck, goconst, gocyclo, noctx, prealloc, staticcheck, unparam]
+        path: ^sidecar/.*_test\.go
+      # unparam flags cfg/cdc parameters that a few unexported methods do not
+      # read. They are there for symmetry across a family of sibling methods that
+      # do, so dropping them from some and not others makes the family harder to
+      # read — and changing signatures is churn inside a relocation. prealloc's
+      # one remaining hit is a json.Unmarshal destination, which Unmarshal
+      # allocates itself, so preallocating changes nothing.
+      - linters: [prealloc, unparam]
+        path: ^sidecar/
+      # goconst's remaining hits are wire and config literals: JSON field names
+      # in the EVM digest and shadow comparators (code, type, nonce, balance,
+      # gasUsed, storage), seid config.toml keys, which are hyphenated by
+      # convention and must read exactly as written (p2p, persistent-peers,
+      # statesync), and log attribute names. Same call as sidecarapi/client —
+      # reading the literal in place is what makes these checkable against the
+      # thing they mirror, and hoisting them trades that for a DRY score. One hit
+      # suggests reusing conditionHeight for a JSON key and a log attribute that
+      # merely share the spelling "height"; that constant names an
+      # await-condition kind, so reusing it would conflate two unrelated things.
+      - linters: [goconst]
+        path: ^sidecar/
+      # noctx wants the database/sql *Context variants throughout the store. That
+      # is a real improvement and a behaviour change — it makes queries
+      # cancellable in the component whose SetMaxOpenConns(1) and WAL-checkpoint
+      # invariants are the ones to be careful with — so it gets its own PR rather
+      # than riding along inside a 24,000-line relocation. Scoped to the two files
+      # that hold it so noctx stays live everywhere else in the module.
+      - linters: [noctx]
+        path: ^sidecar/engine/sqlite_(migrations|store)\.go
     paths:
       - third_party$
       - builtin$

CLAUDE.md

@@ -1,6 +1,6 @@
 # sei-k8s-controller
 
-Kubernetes operator for managing Sei blockchain nodes. Single binary, three controllers: `SeiNetwork` (genesis-ceremony orchestration: bootstraps a chain's genesis.json and founding validator set, owns the child SeiNodes), `SeiNode` (individual node lifecycle), and `SeiNodeTask` (sidecar-driven task execution).
+Kubernetes operator for managing Sei blockchain nodes, plus the per-node sidecar it drives. Two binaries across three Go modules. Three controllers: `SeiNetwork` (genesis-ceremony orchestration: bootstraps a chain's genesis.json and founding validator set, owns the child SeiNodes), `SeiNode` (individual node lifecycle), and `SeiNodeTask` (sidecar-driven task execution).
 
 ## Architecture
 
@@ -10,6 +10,35 @@ Kubernetes operator for managing Sei blockchain nodes. Single binary, three cont
 - **Entry point**: `cmd/main.go` — thin binary that creates a `manager.Manager` and registers both controllers
 - **Framework**: controller-runtime v0.23.1 / kubebuilder v4.12.0
 
+### Modules
+
+Three Go modules, wired by filesystem `replace` — **not** `go.work`. A workspace promotes a used module's `replace` directives to main-module status, and `sidecar/` pins `golang.org/x/crypto` and `google.golang.org/grpc` *down* from what the controller needs, so a workspace build would diverge from every `GOWORK=off` build (Docker, release CI, external consumers).
+
+| Module | Contains | Dependency profile |
+|---|---|---|
+| `.` (root) | the controller, `api/v1alpha1`, `sdk/sei` | controller-runtime + k8s. **No chain graph** |
+| `sidecarapi/` | `api/` (OpenAPI spec), `client/` (generated), `wire/` (contract types), `tomlpatch/` | light; ~10 modules. **No replace directives** — a dependency module's replaces are ignored, so any here would be a silent no-op |
+| `sidecar/` | the sidecar binary: `tasks/`, `engine/`, `server/`, `s3/`, `shadow/`, `rpc/`, `actions/` | the sei-chain graph; restates all 11 of sei-chain's replaces |
+
+Anything that walks packages must loop `MODULES` in the Makefile. Go package patterns stop at a nested module boundary, so `go list ./...` in the root does **not** see `sidecarapi/` or `sidecar/` — a root-only lint or test passes while a whole module goes uncompiled.
+
+Two checks keep the controller tidyable, both in `make ci`. The `depguard` rule `contract-stays-light` in `.golangci.yml` denies the chain graph to anything under `sidecarapi/`, `_test.go` files included — that is the import a test added once before, and it stopped every consumer's `go mod tidy` from working. `make tidy-check` runs `go mod tidy -diff` per module, which catches the unresolvable graph that import produces. Neither covers a third-party dependency that transitively reaches the chain graph while still resolving; that is a dependency-review question, not a lint one.
+
+### The sidecar binary
+
+`sidecar/main.go` → `sei-sidecar`, published to ECR as `sei/sei-sidecar`. The controller renders **no `Command`** for the sidecar container (`internal/noderesource/`, `internal/task/bootstrap_resources.go`), so the image's ENTRYPOINT — `sei-sidecar serve` — is what runs. The image owns its entrypoint; renaming the binary is an image-only change.
+
+**Changing the sidecar image is a coordinated deploy.** The image, `images.sidecar` in the platform app-config, and the controller ship together per cell. `images.sidecar` is read once at startup, so a config edit alone does nothing until the controller restarts — and the two halves failing apart is silent, not loud. A controller that renders no `Command` against an image whose entrypoint lacks the subcommand gets a container that prints help and exits 0, restarting forever under `restartPolicy: Always`, while seid blocks on a shell loop polling `/v0/healthz` behind a StartupProbe with `FailureThreshold: 86400` at 5s — about five days before Kubernetes calls it failed.
+
+The rollout is controller-driven, not Kubernetes-driven: StatefulSets use `UpdateStrategy: OnDelete`, so a template change never touches a live pod. Sidecar-image drift builds a NodeUpdate plan whose `replace-pod` task deletes pods at the old revision, for **every** node whose `status.currentSidecarImage` is set. Expect the whole cell to roll.
+
+`spec.sidecar.image` still overrides `images.sidecar` per node. Pin only images whose entrypoint matches what the controller renders.
+
+Two startup refusals in `sidecar/` are load-bearing; do not soften them into defaults:
+
+- `SEI_HOME` is **required**, with no fallback. It previously defaulted to `/sei` while the controller mounts the data PVC at `$HOME/.sei`, so a dropped value produced a running, probe-passing sidecar writing genesis and config into an empty directory.
+- A configured `SEI_KEYRING_BACKEND` with `SEI_SIDECAR_AUTHN_MODE` unauthenticated is refused (`checkKeyringNeedsAuthn`). Unauthenticated binds all interfaces and installs no middleware, so that combination exposes the sign-tx API — gov-vote included — to any pod in the cluster while the keyring is open.
+
 ## Subagents
 
 Always use the available subagents for relevant work:

Makefile

@@ -1,4 +1,7 @@
 IMG ?= sei-k8s-controller:latest
+# The sidecar publishes to ECR as sei/sei-sidecar; the tag is supplied by CI.
+SIDECAR_IMG ?= sei-sidecar:latest
+CONTAINER_TOOL ?= docker
 GOLANGCI_LINT ?= $(shell which golangci-lint 2>/dev/null || echo $(HOME)/go/bin/golangci-lint)
 
 # Pinned tool versions. Bump together: setup-envtest's release branch tracks
@@ -21,13 +24,26 @@ SETUP_ENVTEST ?= $(LOCALBIN)/setup-envtest
 # module boundary — `go list ./...` in the root does NOT descend into
 # sidecarapi/ — so anything that walks packages must loop this list or the
 # nested module goes unbuilt, unlinted and untested while CI stays green.
-MODULES ?= . sidecarapi
+MODULES ?= . sidecarapi sidecar
 
-.PHONY: build test test-modules test-integration test-all lint lint-modules tidy-check manifests generate verify-generated setup-envtest ci docker-build docker-push
+.PHONY: build build-sidecar docker-build-sidecar test test-modules test-integration test-all lint lint-modules tidy-check manifests generate verify-generated setup-envtest ci docker-build docker-push
 
 build: ## Build manager binary.
 	go build -o bin/manager ./cmd/
 
+build-sidecar: ## Build the sidecar binary.
+	@# GOWORK=off keeps this identical to the Docker and release builds: the
+	@# sidecar module pins x/crypto and grpc *down*, and a workspace would
+	@# promote those replaces into the root module's resolution.
+	cd sidecar && GOWORK=off go build -o ../bin/sei-sidecar .
+
+docker-build-sidecar: ## Build the sidecar container image.
+	@# Build context is the repo root — sidecar/ resolves sidecarapi/ through a
+	@# filesystem replace, so both module trees must be in the context.
+	@# --platform matches docker-build: without it this silently produces an
+	@# arm64 image on an Apple Silicon machine, which no cluster node runs.
+	$(CONTAINER_TOOL) build --platform linux/amd64 -f sidecar/Dockerfile -t $(SIDECAR_IMG) .
+
 test: test-modules ## Run tests (root module with coverage, then every other module).
 	go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out