fix(release): re-issue an Unavailable release re-cut on the same activation when the store answers again (#7072)

.github/scripts/shard-assign.sh

@@ -107,6 +107,13 @@
 #      428 s on each of the three green runs and 435 s on 37830774943; it now
 #      runs in two parts.
 #
+#      MeshWeaver.Compiler.Pipeline.Test was the one after that: listed at 227 s, it measured
+#      388 s solo on PR run 39964851393 (81% of the cap, already past the 288 s threshold) and was
+#      killed at 480 s exit=124 on PR run 39965744393 (#7087), still progressing through its
+#      classes, once #7072's regression class added three 36 s deliberate negative waits
+#      (`NotEmit(TestTimeouts.Quick)` is 36 s on CI). Measured total ~500 s; it runs in three
+#      parts since, so no every-Nth parity can stack its 36 s classes past the threshold.
+#
 # 🚨 The two triggers must BOTH be checked when re-measuring. A project that grows
 # past 288 s (60% of 480) needs splitting even while the LPT loop still reports
 # six balanced shards, which is precisely why the balance rule alone missed this.
@@ -158,7 +165,7 @@ WEIGHTS=$(cat <<'EOF'
 43 MeshWeaver.Layout.Test
 7 MeshWeaver.ContentCollections.Test
 37 MeshWeaver.Documentation.Test
-227 MeshWeaver.Compiler.Pipeline.Test
+500 MeshWeaver.Compiler.Pipeline.Test 3
 2 MeshWeaver.ContentCollections.Indexing.Test
 2 MeshWeaver.Markdown.Collaboration.Test
 2 MeshWeaver.Portal.E2E.Test

src/MeshWeaver.Compiler.Pipeline/LateReleaseAdoption.cs

@@ -73,6 +73,7 @@ public static class LateReleaseAdoption
             LatestReleasePath = stamped,
             UnreleasedBuildPath = null,
             UnreleasedBuildReason = null,
+            UnreleasedBuildFailureClass = null,
             ReleaseNotes = null,
         };
     }
@@ -219,6 +220,14 @@ internal static IObservable<string> CompleteInheritedRelease(
                     || InheritedObligation(definition, hubPath, NodeTypeCompilationHelpers.FrameworkVersion)
                         is not { } stamped)
                     return Observable.Empty<string>();
+                // 🚨 An inherited UNAVAILABLE stamp has exactly ONE owner on this activation: the
+                // recovery watch (ReissueWhenTheStoreAnswers), which counts the stamp itself as
+                // evidence and re-issues at once, and writes a verdict back to the record. Cutting it
+                // here as well ran two attempts at the same id concurrently, and a verdict this branch
+                // received was only logged — the watch then re-issued a create the store had already
+                // judged, from the stale class (review on #7087).
+                if (UnavailableObligation(definition, hubPath, NodeTypeCompilationHelpers.FrameworkVersion) is not null)
+                    return Observable.Empty<string>();
                 logger?.LogInformation(
                     "[ReleasePostCondition] {HubPath}: this activation inherits a build with no "
                     + "release — unreleasedBuildPath={ReleasePath} ({Reason}). Cutting it at that id "
@@ -254,6 +263,177 @@ internal static IObservable<string> CompleteInheritedRelease(
             });
     }
 
+    /// <summary>
+    /// 🚨 #7072 — the stamp THIS activation must re-issue when the store answers again, or <c>null</c>.
+    /// Pure.
+    ///
+    /// <para><b>The shape, as measured.</b> On memex on 2026-10-22 a managed database restart (#7068)
+    /// refused every write for about 40 seconds. Three Hosting NodeTypes settled inside it, and both
+    /// the release create and its same-id re-cut were answered <i>"could not be attempted: the data
+    /// store was unreachable … retrying the same request (with the same node id) is
+    /// meaningful"</i>. The settle stamped <see cref="NodeTypeDefinition.UnreleasedBuildPath"/>,
+    /// correctly, and then nothing on that pod ever made the create again: <see cref="Landings"/>
+    /// waits for a node nobody is writing, and <see cref="CompleteInheritedRelease"/> runs only on
+    /// the NEXT activation. The pod stayed up and ready all night, so the types advertised builds no
+    /// release named for 76 minutes (a different pod's boot cut two of them) and 3 hours (the next
+    /// compile superseded the third, whose stamped id was never created).</para>
+    ///
+    /// <para>So a stamp of class <see cref="ReleaseFailureClass.Unavailable"/> is an obligation of
+    /// the activation that HOLDS it, not only of the next one. The same guards as
+    /// <see cref="InheritedObligation"/> apply — a settled build of this framework whose recorded
+    /// bytes the stamped id names — so a re-issue can only ever mint the release for these bytes, at
+    /// the id already stamped. Every other class stands: a refusal re-issued would be refused again,
+    /// and an expiry's create may still land and is adopted by <see cref="Landings"/>.</para>
+    /// </summary>
+    /// <param name="definition">The definition as the owner currently holds it.</param>
+    /// <param name="hubPath">The NodeType's path.</param>
+    /// <param name="frameworkVersion">The live framework build identity.</param>
+    internal static string? UnavailableObligation(
+        NodeTypeDefinition? definition, string hubPath, string frameworkVersion)
+        => string.Equals(definition?.UnreleasedBuildFailureClass, ReleaseFailureClass.Unavailable,
+               StringComparison.Ordinal)
+            ? InheritedObligation(definition, hubPath, frameworkVersion)
+            : null;
+
+    /// <summary>
+    /// The record after a re-issue ended in a VERDICT rather than another Unavailable answer (review on
+    /// #7087), or <c>null</c> to leave it alone. Pure.
+    ///
+    /// <para>The stamp keeps its path — this build still has no release — but its reason and class
+    /// become the re-issue's own, so the record says what the store answered last and
+    /// <see cref="UnavailableObligation"/> no longer names it: a watch rebuilt from this record does not
+    /// re-issue a create the store has judged. Applied only while the record still holds the SAME
+    /// unavailable obligation; a later settle, an adoption or another writer that moved the stamp on
+    /// owns it now.</para>
+    /// </summary>
+    /// <param name="definition">The definition as the owner currently holds it.</param>
+    /// <param name="hubPath">The NodeType's path.</param>
+    /// <param name="stamped">The stamped path the re-issue was for.</param>
+    /// <param name="verdict">The re-issue's outcome — a failure that was not Unavailable.</param>
+    internal static NodeTypeDefinition? RecordReissueVerdict(
+        NodeTypeDefinition? definition, string hubPath, string stamped,
+        NodeTypeBuildState.ReleaseCreateOutcome verdict)
+    {
+        if (verdict.Succeeded || verdict.WasUnavailable
+            || !string.Equals(
+                UnavailableObligation(definition, hubPath, NodeTypeCompilationHelpers.FrameworkVersion),
+                stamped, StringComparison.Ordinal))
+            return null;
+        return definition! with
+        {
+            UnreleasedBuildReason = verdict.Failure is { Length: > 0 } reason
+                ? reason
+                : "the re-issue reported no reason, which is itself a defect in this pipeline",
+            UnreleasedBuildFailureClass = verdict.FailureClass ?? ReleaseFailureClass.Faulted,
+        };
+    }
+
+    /// <summary>
+    /// #7072 — the activation's half of <see cref="UnavailableObligation"/>: while the own record
+    /// carries such a stamp, re-issue the release create at the stamped id once the store is
+    /// evidenced to answer again (<see cref="StoreReachability.ReissueWhenStoreAnswers{T}"/> — a
+    /// change-feed gap, or a commit in this type's partition), and adopt it in one owner write when
+    /// it lands. Emits the adopted path.
+    ///
+    /// <para>🚨 No timer, no poll, no widened bound: the re-issue fires on a FACT that the store is
+    /// back, and while it is not, nothing happens. Idempotent by construction: the create is made at
+    /// the stamped id, so an earlier attempt that did land after all is met as
+    /// <c>NodeAlreadyExists</c> and adopted (<see cref="NodeTypeBuildState.Bounded"/>), and the
+    /// adoption re-checks the stamp in the owner write (<see cref="Adopt"/>). A new stamp supersedes
+    /// the watch (<c>Switch</c>); a cleared one ends it.</para>
+    /// </summary>
+    /// <param name="hub">The per-NodeType hub.</param>
+    /// <param name="workspace">Its workspace.</param>
+    /// <param name="accessService">Runs the create and the write as System.</param>
+    /// <param name="logger">Where the outcome is published.</param>
+    internal static IObservable<string> ReissueWhenTheStoreAnswers(
+        IMessageHub hub,
+        IWorkspace workspace,
+        AccessService? accessService,
+        ILogger? logger)
+    {
+        var hubPath = hub.Address.Path;
+        var options = hub.JsonSerializerOptions;
+        var feed = hub.ServiceProvider.GetService<IMeshInvalidationFeed>();
+        var partition = hubPath.Trim('/').Split('/', 2)[0];
+        return workspace.GetMeshNodeStream()
+            .Where(node => node is not null)
+            .Select(node => (Node: node!, Stamped: UnavailableObligation(
+                node!.ContentAs<NodeTypeDefinition>(options), hubPath,
+                NodeTypeCompilationHelpers.FrameworkVersion)))
+            .DistinctUntilChanged(x => x.Stamped)
+            .Select(x =>
+            {
+                if (x.Stamped is not { } stamped
+                    || x.Node.ContentAs<NodeTypeDefinition>(options) is not { } definition)
+                    return Observable.Empty<string>();
+                logger?.LogInformation(
+                    "[ReleasePostCondition] {HubPath}: the release at {ReleasePath} was not created "
+                    + "because the store could not be reached ({Reason}). It is re-issued at that id "
+                    + "now, since the store accepted this stamp after the failed create, and again on "
+                    + "each later piece of evidence that the store answers — a change-feed reconnect or a "
+                    + "commit in partition '{Partition}' (#7072).",
+                    hubPath, stamped, definition.UnreleasedBuildReason ?? "(no reason recorded)", partition);
+                var pending = x.Node with { Content = definition with { RequestedReleaseBy = null } };
+                var build = RecordedBuild(definition)!;
+                return feed
+                    .ReissueWhenStoreAnswers(
+                        partition,
+                        () => accessService.RunAsSystem(() => NodeTypeBuildState.TryCreateReleaseNode(
+                            hub, hubPath, build, pending, definition.LastCompilationActivityPath,
+                            logger, reusePath: stamped)),
+                        outcome => outcome.WasUnavailable,
+                        (outcome, commitsStillCount) => logger?.LogWarning(
+                            "[ReleasePostCondition] {HubPath}: re-issuing the release at {ReleasePath} "
+                            + "was answered Unavailable again{Because}. The next re-issue waits for {Next} (#7072).",
+                            hubPath, stamped, outcome.Because,
+                            commitsStillCount
+                                ? $"a change-feed reconnect or a commit in partition '{partition}'"
+                                : "a change-feed reconnect: this re-issue was started by a commit in the "
+                                  + "partition and the store still never saw this create"),
+                        // The stamp is itself a record the store accepted after the create failed, and
+                        // a reconnect or commit that preceded it is never replayed to this watch — an
+                        // idle partition would leave the release unpublished (review on #7087).
+                        storeAnsweredBeforeSubscription: true)
+                    .SelectMany(outcome =>
+                    {
+                        if (outcome.ReleasePath is not { } landed)
+                        {
+                            // A verdict, not an outage. The stamp stands and is the report — so it
+                            // must report THIS verdict (review on #7087): left at Unavailable with the
+                            // old reason, it would misstate the failure, and a rebuilt watch (a
+                            // transient fault on the own stream re-establishes it from the record)
+                            // would re-issue a create the store has now judged. The new class and
+                            // reason go to the record through the owner, and only while the record
+                            // still holds the SAME unavailable obligation; the next activation cuts
+                            // the stamp once more (#6056).
+                            logger?.LogWarning(
+                                "[ReleasePostCondition] {HubPath}: re-issuing the release at "
+                                + "{ReleasePath} after the store answered again did not land{Because}. "
+                                + "The stamp stands, now with this verdict's class {FailureClass} (#7072).",
+                                hubPath, stamped, outcome.Because,
+                                outcome.FailureClass ?? ReleaseFailureClass.Faulted);
+                            return accessService.RunAsSystem(() => workspace.GetMeshNodeStream()
+                                    .Update(current =>
+                                        RecordReissueVerdict(current.ContentAs<NodeTypeDefinition>(options),
+                                                hubPath, stamped, outcome) is { } recorded
+                                            ? current with { Content = recorded }
+                                            : current))
+                                .Take(1)
+                                .SelectMany(_ => Observable.Empty<string>());
+                        }
+                        return accessService.RunAsSystem(() => workspace.GetMeshNodeStream()
+                                .Update(current =>
+                                    Adopt(current.ContentAs<NodeTypeDefinition>(options), landed) is { } adopted
+                                        ? current with { Content = adopted }
+                                        : current))
+                            .Take(1)
+                            .Select(_ => landed);
+                    });
+            })
+            .Switch();
+    }
+
     /// <summary>
     /// Installs the watch on a per-NodeType hub, next to the compile and release-request watchers.
     /// Returns the subscription so the caller can register it for disposal with the hub.
@@ -309,6 +489,18 @@ public static IDisposable Install(IMessageHub hub, IWorkspace workspace)
                 hubPath, landed),
             logger,
             $"[LateReleaseAdoption] {hubPath}");
-        return new CompositeDisposable(completion, landings);
+        // #7072 — a stamp the store never saw is re-issued on THIS activation when the store is back.
+        var reissue = ActivityControlPlaneExtensions.SubscribeHubWatcher(
+            hub,
+            () => ReissueWhenTheStoreAnswers(hub, workspace, accessService, logger),
+            landed => logger?.LogInformation(
+                "[ReleasePostCondition] {HubPath}: the store answered again and the release was "
+                + "re-issued at {ReleasePath} and adopted: latestReleasePath names it and "
+                + "unreleasedBuildPath is cleared (#7072).",
+                hubPath, landed),
+            logger,
+            $"[ReleaseReissue] {hubPath}");
+
+        return new CompositeDisposable(completion, landings, reissue);
     }
 }