Skip to content

Commit f5a5137

Browse files
authored
Merge pull request #6859 from Systemorph/fix/6825-deployment-bundles
Deployment record declares its bundle source; chart honours ingress.sessionAffinity
2 parents 7990ef7 + d018dcf commit f5a5137

12 files changed

Lines changed: 376 additions & 4 deletions

File tree

‎deploy/aks/scripts/check-chart-invariants.sh‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,12 @@ COMBOS=(
154154
# opt-out shape is here too, because "no issuer" must be a statement, never an omission.
155155
"a public host issued by cert-manager (fixture)|deploy/helm/values.yaml:deploy/aks/scripts/testdata/values.ingress-issuer.yaml"
156156
"a host whose TLS Secret is pre-provisioned (fixture)|deploy/helm/values.yaml:deploy/aks/scripts/testdata/values.ingress-preprovisioned.yaml"
157+
# 🚨 Cookie session affinity as the RECORD renders it (MeshWeaver#6825): HelmValues writes the typed
158+
# block under ingress.sessionAffinity.*, which the chart did not read, so a record-only render
159+
# dropped affinity from every multi-replica Blazor portal. The evidence check below the loop reads
160+
# both renders by name: enabled merges in order, disabled renders none of it.
161+
"cookie session affinity from the record's typed block (fixture)|deploy/helm/values.yaml:deploy/aks/scripts/testdata/values.ingress-session-affinity.yaml"
162+
"cookie session affinity switched off (fixture)|deploy/helm/values.yaml:deploy/aks/scripts/testdata/values.ingress-session-affinity-disabled.yaml"
157163
# 🚨 The instance's OWN database release (Doc/Architecture/InClusterDatabases): a CloudNativePG
158164
# Cluster beside the portal release, named by database.release. The only combination whose
159165
# connection strings are COMPOSED in the containers' env from a Secret the chart does not render.
@@ -336,6 +342,52 @@ else
336342
report "plugin source credentials: the fixture must render exactly Sources__1__TokenKey=\"PluginCatalog:SourceTokens:Extensions\" and Sources__2__InstallationOwner=\"fabrikam\" (trimmed), Sources__3__TokenKey=\"\" (a blank tokenKey is a DECLARATION the portal refuses — dropping it reads the repository with the instance's App or anonymously), NO credential key for the undeclared source 0 and the null tokenKey of source 4, and the chart defaults must render none"
337343
fi
338344

345+
# The session-affinity evidence (MeshWeaver#6825). Read by NAME: the record's typed block must reach
346+
# the portal Ingress — `affinity: cookie` and the cookie name derived from the typed fields,
347+
# nginxAnnotations over both, ingress.annotations over everything, the AGIC map NOT rendered — and a
348+
# disabled block must render none of it. The AKS overlay states the same block, so its render must
349+
# carry the affinity too. The plugin-registry identity rides the affinity fixture:
350+
# pluginCatalog.instanceId renders PluginCatalog__InstanceId and a HomeUrl derived from the host.
351+
affinity_annotations() {
352+
python3 - "$1" <<'PY'
353+
import sys, yaml, json
354+
for doc in yaml.safe_load_all(open(sys.argv[1])):
355+
if doc and doc.get("kind") == "Ingress" and doc["metadata"]["name"] == "memex-portal":
356+
print(json.dumps(doc["metadata"].get("annotations") or {}, sort_keys=True))
357+
break
358+
else:
359+
print("NO-INGRESS")
360+
PY
361+
}
362+
affinity_on="$(affinity_annotations "$(render_of "cookie session affinity from the record's typed block (fixture)")")"
363+
affinity_off="$(affinity_annotations "$(render_of "cookie session affinity switched off (fixture)")")"
364+
affinity_aks="$(affinity_annotations "$(render_of "AKS overlay (the layer every AKS install shares)")")"
365+
affinity_render="$(render_of "cookie session affinity from the record's typed block (fixture)")"
366+
if python3 - "$affinity_on" "$affinity_off" "$affinity_aks" <<'PY'
367+
import sys, json
368+
on, off, aks = (json.loads(a) for a in sys.argv[1:4])
369+
p = "nginx.ingress.kubernetes.io/"
370+
assert on.get(p + "affinity") == "cookie", on
371+
assert on.get(p + "session-cookie-name") == "MEMEX_AFFINITY", on
372+
assert on.get(p + "affinity-mode") == "persistent", on
373+
assert on.get(p + "session-cookie-max-age") == "3600", on # ingress.annotations wins
374+
assert on.get("cert-manager.io/cluster-issuer") == "letsencrypt-prod", on
375+
assert not any(k.startswith("appgw.") for k in on), on # AGIC is not rendered
376+
assert not any(k in off for k in (p + "affinity", p + "session-cookie-name", p + "affinity-mode")), off
377+
assert aks.get(p + "affinity") == "cookie" and aks.get(p + "session-cookie-name") == "MEMEX_AFFINITY", aks
378+
PY
379+
then
380+
if [ -f "$affinity_render" ] \
381+
&& grep -q '^ PluginCatalog__InstanceId: "instance-at-registry"$' "$affinity_render" \
382+
&& grep -q '^ PluginCatalog__HomeUrl: "https://instance.example.com"$' "$affinity_render"; then
383+
ok "cookie session affinity reaches the portal Ingress from ingress.sessionAffinity (typed, then nginxAnnotations, then ingress.annotations; AGIC not rendered; disabled renders none), and pluginCatalog.instanceId renders PluginCatalog__InstanceId and HomeUrl"
384+
else
385+
report "pluginCatalog.instanceId: the session-affinity fixture must render PluginCatalog__InstanceId=\"instance-at-registry\" and PluginCatalog__HomeUrl=\"https://instance.example.com\" — a record-driven deploy otherwise loses the deployment-config principal"
386+
fi
387+
else
388+
report "ingress.sessionAffinity: the portal Ingress must carry affinity=cookie and the cookie name when the block is enabled, nginxAnnotations over them, ingress.annotations over all, no AGIC annotation, and NONE of it when disabled — a record-only render otherwise drops cookie affinity (MeshWeaver#6825)"
389+
fi
390+
339391
# The bake-gate evidence (MeshWeaver#4588, #5544). Invariant 10b — an armed PreWarm__GateReadiness
340392
# is read by the readinessProbe on /ready, and the rollout deadline covers a cold bake — is
341393
# CONDITIONAL, so it is satisfied just as well by a set of renders where nothing ever arms the gate.
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# The negative control for values.ingress-session-affinity.yaml (MeshWeaver#6825): the same typed
2+
# block with `enabled: false`. A record that switches affinity OFF must render none of it — not the
3+
# derived `affinity: cookie`, not the cookie name, not its nginxAnnotations.
4+
ingress:
5+
enabled: true
6+
className: "webapprouting.kubernetes.azure.com"
7+
host: "instance.example.com"
8+
tlsSecret: "instance-tls"
9+
clusterIssuer: "letsencrypt-prod"
10+
sessionAffinity:
11+
enabled: false
12+
cookieName: "MEMEX_AFFINITY"
13+
nginxAnnotations:
14+
nginx.ingress.kubernetes.io/affinity-mode: "persistent"
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Cookie session affinity as the RECORD renders it (MeshWeaver#6825): MeshWeaver.Plugins
2+
# HelmValues.IngressValues writes the record's typed affinity under ingress.sessionAffinity.*, never
3+
# into ingress.annotations. Until the chart read that block, a record-only render dropped cookie
4+
# affinity from every multi-replica Blazor portal. The evidence check in check-chart-invariants.sh
5+
# reads this render by name and asserts the merge order: `affinity: cookie` and the cookie name
6+
# from the typed fields, nginxAnnotations over both, ingress.annotations over everything.
7+
#
8+
# Also the plugin-registry identity the record renders from registryInstanceId: the chart maps
9+
# pluginCatalog.instanceId to PluginCatalog__InstanceId and derives PluginCatalog__HomeUrl from
10+
# the host.
11+
ingress:
12+
enabled: true
13+
className: "webapprouting.kubernetes.azure.com"
14+
host: "instance.example.com"
15+
tlsSecret: "instance-tls"
16+
clusterIssuer: "letsencrypt-prod"
17+
annotations:
18+
# A hand-written annotation wins over the typed affinity block on a clash.
19+
nginx.ingress.kubernetes.io/session-cookie-max-age: "3600"
20+
sessionAffinity:
21+
enabled: true
22+
cookieName: "MEMEX_AFFINITY"
23+
nginxAnnotations:
24+
nginx.ingress.kubernetes.io/affinity-mode: "persistent"
25+
nginx.ingress.kubernetes.io/session-cookie-max-age: "172800"
26+
# Recorded intent for an Application Gateway controller; the chart's ingresses are nginx-shaped
27+
# and must NOT carry it.
28+
agicAnnotations:
29+
appgw.ingress.kubernetes.io/cookie-based-affinity: "true"
30+
pluginCatalog:
31+
instanceId: "instance-at-registry"

‎deploy/helm/templates/memex-portal/ingress.yaml‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,29 @@ metadata:
2828
# asked cert-manager for nothing and the controller served its fallback certificate.
2929
# A hand-written `cert-manager.io/cluster-issuer` in `annotations` still wins, so a record that
3030
# names its own issuer renders byte-identically to before.
31-
{{- $ann := .Values.ingress.annotations | default dict }}
31+
{{- /* ---- Sticky sessions (ingress.sessionAffinity, MeshWeaver#6825) ----------------------
32+
A Blazor circuit and its per-circuit hubs live on ONE pod, so a client bouncing between
33+
replicas loses its session. The record renders its typed affinity here
34+
(HelmValues.IngressValues → ingress.sessionAffinity.*) and values.aks.yaml states the same
35+
block — and until this merge the chart read NEITHER: a record-only render dropped cookie
36+
affinity from every multi-replica portal. Enabled ⇒ `affinity: cookie`, the cookie name
37+
when stated, then nginxAnnotations over both; ingress.annotations over ALL of it, so a
38+
hand-written annotation still wins. Disabled or absent renders nothing. The AGIC map is
39+
not merged: every ingress this chart renders is nginx-shaped. */}}
40+
{{- $affinity := .Values.ingress.sessionAffinity | default dict }}
41+
{{- $ann := dict }}
42+
{{- if $affinity.enabled }}
43+
{{- $_ := set $ann "nginx.ingress.kubernetes.io/affinity" "cookie" }}
44+
{{- with $affinity.cookieName }}
45+
{{- $_ := set $ann "nginx.ingress.kubernetes.io/session-cookie-name" . }}
46+
{{- end }}
47+
{{- range $k, $v := ($affinity.nginxAnnotations | default dict) }}
48+
{{- $_ := set $ann $k $v }}
49+
{{- end }}
50+
{{- end }}
51+
{{- range $k, $v := (.Values.ingress.annotations | default dict) }}
52+
{{- $_ := set $ann $k $v }}
53+
{{- end }}
3254
{{- if and $issuer (ne $issuer "none") (not $handWritten) }}
3355
cert-manager.io/cluster-issuer: {{ $issuer | quote }}
3456
{{- end }}

‎src/MeshWeaver.Deployment.Contract/DeploymentContent.cs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -581,6 +581,17 @@ public record DeploymentContent
581581
[Description("Container registry this instance hosts (the public instance only)")]
582582
public RegistrySpec? Registry { get; init; }
583583

584+
/// <summary>
585+
/// The registry this instance pulls its sealed NodeType publications from before the portal
586+
/// starts — the chart's <c>bundles</c> section (see <see cref="BundlesSpec"/>). Null, or a blank
587+
/// <see cref="BundlesSpec.Registry"/>, renders nothing and no bundle fetch runs. When stated, the
588+
/// pre-warm root (<c>PreWarm__PrebuiltBundleRoot</c>) renders from <see cref="BundlesSpec.Root"/>
589+
/// through <see cref="DeploymentPortalConfig.BundleRoot"/>, so the shelf the init container fills
590+
/// is the one the portal reads.
591+
/// </summary>
592+
[Description("Sealed NodeType bundles fetched from a registry before the portal starts")]
593+
public BundlesSpec? Bundles { get; init; }
594+
584595
/// <summary>OpenTelemetry export.</summary>
585596
[Description("Telemetry")]
586597
public TelemetrySpec? Telemetry { get; init; }

‎src/MeshWeaver.Deployment.Contract/DeploymentPortalConfig.cs‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -313,6 +313,35 @@ public static string OrleansClustering(DeploymentContent d) =>
313313
return host is null || string.Equals(host, DefaultSelfUpdateRegistry, StringComparison.OrdinalIgnoreCase) ? null : host;
314314
}
315315

316+
/// <summary>
317+
/// The bundle registry a record declares (<see cref="DeploymentContent.Bundles"/>), trimmed, or
318+
/// null when it declares none — the ONE test for "this record fetches bundles", shared by the
319+
/// pre-warm root below and the Helm renderer's <c>bundles</c> section. Pure.
320+
/// </summary>
321+
public static string? BundleRegistry(DeploymentContent d) =>
322+
string.IsNullOrWhiteSpace(d.Bundles?.Registry) ? null : d.Bundles!.Registry!.Trim();
323+
324+
/// <summary>
325+
/// The bundle sources a record declares, trimmed, blanks dropped, in the record's order. Empty
326+
/// when it declares none. Pure.
327+
/// </summary>
328+
public static ImmutableList<string> BundleSources(DeploymentContent d) =>
329+
(d.Bundles?.Sources ?? ImmutableList<string>.Empty)
330+
.Where(s => !string.IsNullOrWhiteSpace(s))
331+
.Select(s => s.Trim())
332+
.ToImmutableList();
333+
334+
/// <summary>
335+
/// <c>PreWarm__PrebuiltBundleRoot</c> for a record that fetches bundles: its
336+
/// <see cref="BundlesSpec.Root"/>, else <see cref="BundlesSpec.DefaultRoot"/>; null for a record
337+
/// that fetches none, so every other record renders as before. The chart's <c>bundles.root</c>
338+
/// defaults to this key, so the shelf the init container fills and the directory the pre-warm
339+
/// reads come from one value (MeshWeaver#6825). Pure.
340+
/// </summary>
341+
public static string? BundleRoot(DeploymentContent d) =>
342+
BundleRegistry(d) is null ? null
343+
: string.IsNullOrWhiteSpace(d.Bundles!.Root) ? BundlesSpec.DefaultRoot : d.Bundles.Root!.Trim();
344+
316345
/// <summary>
317346
/// <c>SelfUpdate__PortalRepository</c> for a record: the repository its pods pull
318347
/// (<see cref="DeploymentContent.ImageRepository"/>), or null when that IS the image's default —
@@ -877,6 +906,9 @@ public static SortedDictionary<string, string> PortalConfig(DeploymentContent d,
877906
Set("Storage__SourceType", s.ContentSourceType);
878907
Set("Storage__BasePath", s.ContentPath);
879908
Set("ClaudeCode__ConfigDirRoot", s.ClaudeCodeConfigDirRoot);
909+
// The bundle shelf (MeshWeaver#6825): rendered only for a record that fetches bundles, and
910+
// from the same value the chart's bundle-fetch init container fills — one source for both.
911+
Set("PreWarm__PrebuiltBundleRoot", BundleRoot(d));
880912

881913
Set("Deployment__Orleans__Clustering", OrleansClustering(d));
882914
Set("SelfUpdate__MinRollInterval", string.IsNullOrWhiteSpace(d.MinRollInterval) ? DefaultMinRollInterval : d.MinRollInterval!.Trim());

‎src/MeshWeaver.Deployment.Contract/DeploymentRecordExtensions.cs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -280,6 +280,27 @@ public static DeploymentContent DefaultGrants(this DeploymentContent d, params s
280280
/// <summary>The instance id at the plugin registry (blank = the deployment id).</summary>
281281
public static DeploymentContent WithRegistryInstanceId(this DeploymentContent d, string? instanceId) => d with { RegistryInstanceId = instanceId };
282282

283+
/// <summary>
284+
/// Pull the sealed NodeType publications of <paramref name="sources"/> from
285+
/// <paramref name="registry"/> before the portal starts (the chart's <c>bundles</c> section). The
286+
/// framework identity is read from the running image; <paramref name="identity"/> is only the
287+
/// fallback for an image that predates the identity file. <paramref name="root"/> blank →
288+
/// <see cref="BundlesSpec.DefaultRoot"/>. The pod's <see cref="DeploymentContent.ImagePullSecret"/>
289+
/// is the credential, so state it too.
290+
/// </summary>
291+
public static DeploymentContent WithBundles(
292+
this DeploymentContent d, string registry, IEnumerable<string> sources, string? identity = null, string? root = null) =>
293+
d with
294+
{
295+
Bundles = new BundlesSpec
296+
{
297+
Registry = registry,
298+
Sources = (sources ?? Enumerable.Empty<string>()).ToImmutableList(),
299+
Identity = identity,
300+
Root = root,
301+
},
302+
};
303+
283304
// ── modules ─────────────────────────────────────────────────────────────────────────────────
284305

285306
/// <summary>

‎src/MeshWeaver.Deployment.Contract/InstanceShape.cs‎

Lines changed: 56 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,53 @@ public sealed record RegistryKeyVaultSpec
135135
public string? NotificationSecretObject { get; init; }
136136
}
137137

138+
/// <summary>
139+
/// Where this instance pulls its SEALED NodeType publications from before the portal starts — the
140+
/// chart's <c>bundles</c> section, which renders the <c>bundle-identity</c> and <c>bundle-fetch</c>
141+
/// init containers (Doc/Architecture/PluginBundlesInTheRegistry). Without it every pod compiles
142+
/// the shipped NodeTypes itself, and a multi-pod instance can name an assembly in the shared
143+
/// records that only the compiling pod holds.
144+
///
145+
/// <para>Stated on the record so an estate outside the fleet's own subscription can declare its
146+
/// bundle source and still roll from the record alone: before this block existed the source lived
147+
/// only in a hand-written values layer, and a record-driven roll either dropped it or was refused
148+
/// for dropping it (MeshWeaver#6825).</para>
149+
///
150+
/// <para>The init container authenticates with the pod's <see cref="DeploymentContent.ImagePullSecret"/>:
151+
/// the key that pulls the platform image pulls the bundles, so a record that states a registry here
152+
/// states that Secret too. Names only — no credential.</para>
153+
/// </summary>
154+
public sealed record BundlesSpec
155+
{
156+
/// <summary>The fleet registry host the publications are pulled from (e.g. <c>cr.meshweaver.cloud</c>). Blank → no bundle fetch renders.</summary>
157+
[Description("Registry host the sealed bundles are pulled from (e.g. cr.meshweaver.cloud)")]
158+
public string? Registry { get; init; }
159+
160+
/// <summary>The source names to materialise, as the publisher named them (e.g. <c>plugins</c>, <c>reinsurance</c>). Required when <see cref="Registry"/> is set.</summary>
161+
[Description("Source names to materialise (e.g. plugins)")]
162+
public ImmutableList<string> Sources { get; init; } = ImmutableList<string>.Empty;
163+
164+
/// <summary>
165+
/// The framework identity to pull — the FALLBACK only. The identity is read from the running
166+
/// portal image (<c>meshweaver-framework.identity</c>, MeshWeaver#6052), so it follows every
167+
/// roll; this value is used only for an image that predates that file, and it does NOT follow a
168+
/// roll. Leave it blank on an image that carries the file.
169+
/// </summary>
170+
[Description("Fallback framework identity, used only for an image that predates the identity file")]
171+
public string? Identity { get; init; }
172+
173+
/// <summary>
174+
/// The directory the init container fills AND the pre-warm reads — rendered as
175+
/// <c>PreWarm__PrebuiltBundleRoot</c>, so the two cannot disagree. Blank →
176+
/// <see cref="DefaultRoot"/>.
177+
/// </summary>
178+
[Description("Directory the bundles land in and the pre-warm reads — blank means /data/prebuilt-bundles")]
179+
public string? Root { get; init; }
180+
181+
/// <summary>The fleet's bundle shelf: under the shared <c>/data</c> volume, where every AKS instance's pre-warm reads.</summary>
182+
public const string DefaultRoot = "/data/prebuilt-bundles";
183+
}
184+
138185
/// <summary>CPU and memory for one side of a resource envelope, as Kubernetes quantities.</summary>
139186
public sealed record ResourceQuantities
140187
{
@@ -267,8 +314,15 @@ public sealed record VolumeClaim
267314
/// Sticky sessions at the ingress. A Blazor circuit and its per-circuit hubs live on ONE pod
268315
/// (<c>[PreferLocalPlacement]</c>), so a client bouncing between pods loses its session — the
269316
/// cookie pins it. The nginx/AGIC annotation maps are the controller-specific spellings of the
270-
/// same intent; the future manifest generator derives them, the chart today reads only the
271-
/// flat <c>ingress.annotations</c>, so these are recorded intent until it does.
317+
/// same intent.
318+
///
319+
/// <para>The chart reads it (MeshWeaver#6825): when <see cref="Enabled"/>, the portal ingress
320+
/// carries <c>nginx.ingress.kubernetes.io/affinity: cookie</c>, the
321+
/// <c>session-cookie-name</c> from <see cref="CookieName"/> when stated, then
322+
/// <see cref="NginxAnnotations"/> over both, and the flat <c>ingress.annotations</c> over all of
323+
/// them — a hand-written annotation still wins. Disabled or absent renders none of it. The chart's
324+
/// ingresses are nginx-shaped (the gRPC and MCP ingresses carry nginx annotations only), so
325+
/// <see cref="AgicAnnotations"/> stays recorded intent and is not rendered.</para>
272326
/// </summary>
273327
public sealed record SessionAffinity
274328
{

0 commit comments

Comments
 (0)