ci(module-pack): a caller can withhold packages from the hand-over (publish-withhold)

.github/scripts/module-pack-batch.py

@@ -677,6 +677,49 @@ def workflow_entry_handover_problems(workflow: str) -> list[str]:
     return problems
 
 
+WITHHOLD_PLAN = 'withheld=true; need_publish=false'
+WITHHOLD_MATCH = ('grep -qxF "$(bk get --module "$MODULE" package)" "$RUNNER_TEMP/publish-withhold.txt"')
+WITHHOLD_PUBLISHED = '[ "$NEWER" != true ] && [ "$WITHHELD" != true ]; then PUBLISHED=true; fi'
+WITHHOLD_RECEIPT = 'elif [ "$WITHHELD" = true ]; then PUBLICATION=withheld'
+WITHHOLD_STAGED = ('elif [ "$WITHHELD" = "true" ]; then\n'
+                   '                reason="no hand-over owed: the caller withholds package')
+
+
+def workflow_publish_withhold_problems(workflow: str) -> list[str]:
+    """`publish-withhold` holds a package back from the hand-over, and ONLY from the hand-over.
+
+    A caller whose publication is decided per package (MeshWeaver.Plugins: an unsettled lock or an
+    unstamped platform floor holds back its own dependency network, never the whole tree) names the
+    packages this run must not hand over. The lane's part is three lines, and each one failing
+    silently is a different defect: the plan must turn the hand-over off for the module (or a
+    package the caller withheld is published), it must match the entry's `package` EXACTLY (a
+    substring match withholds `AI.Codex` for `AI`, or publishes `Store` for `Stor`), and the receipt
+    must neither claim a publication that did not happen nor leave the reason to be inferred.
+    """
+    problems: list[str] = []
+    if len(re.findall(r"^      publish-withhold:$", workflow, re.M)) != 1:
+        problems.append("the lane declares no `publish-withhold` input (exactly one is expected) — a caller "
+                        "passing it would be refused as an unexpected input, or worse, silently ignored")
+    if workflow.count("PUBLISH_WITHHOLD: ${{ inputs.publish-withhold }}") != 1:
+        problems.append("the plan step does not read `inputs.publish-withhold` exactly once")
+    if workflow.count(WITHHOLD_MATCH) != 1:
+        problems.append("the plan step must match a withheld name against the entry's `package` as a WHOLE "
+                        "LINE (`grep -qxF`), exactly once — a looser match withholds or publishes a neighbour")
+    if workflow.count(WITHHOLD_PLAN) != 1:
+        problems.append("the plan step must turn the hand-over off for a withheld module "
+                        f"(`{WITHHOLD_PLAN}`), exactly once — otherwise the caller's withheld package is published")
+    if workflow.count(WITHHOLD_PUBLISHED) != 1:
+        problems.append("the receipt must not say `published: true` for a withheld module")
+    if workflow.count(WITHHOLD_RECEIPT) != 1:
+        problems.append("the receipt must name the reason (`publication: withheld`) — a withheld module must "
+                        "never read as `direct`, `staged` or `superseded`")
+    if workflow.count(WITHHOLD_STAGED) != 1:
+        problems.append("the STAGED record of a withheld module must state its own reason (the caller withholds "
+                        "the package) before the ledger branch — otherwise the publication lane prints 'the module "
+                        "build ledger already serves this key' for a key no registry serves")
+    return problems
+
+
 def self_test(workflow_path: Path | None = None) -> int:
     import subprocess
 
@@ -781,6 +824,29 @@ def entry(m: str, **kw) -> dict:
         handover_problems = workflow_entry_handover_problems(workflow)
         check("publish-newest-only reads the WHOLE matrix entry through `entry`, never `get`",
               not handover_problems, "; ".join(handover_problems))
+        # `publish-withhold` (per-package publication): the caller withholds packages; the lane
+        # must leave exactly their hand-over out and say so on the receipt.
+        withhold_problems = workflow_publish_withhold_problems(workflow)
+        check("publish-withhold turns the hand-over off for a withheld package, by exact name, and the receipt says so",
+              not withhold_problems, "; ".join(withhold_problems))
+        check("NEGATIVE CONTROL: a plan that keeps the hand-over ON for a withheld package is caught",
+              any("hand-over off" in w for w in workflow_publish_withhold_problems(
+                  workflow.replace(WITHHOLD_PLAN, "withheld=true", 1))))
+        check("NEGATIVE CONTROL: a substring match on the package name is caught",
+              any("WHOLE" in w for w in workflow_publish_withhold_problems(
+                  workflow.replace("grep -qxF \"$(bk get --module", "grep -qF \"$(bk get --module", 1))))
+        check("NEGATIVE CONTROL: a receipt that reports a withheld module as published is caught",
+              any("published: true" in w for w in workflow_publish_withhold_problems(
+                  workflow.replace(' && [ "$WITHHELD" != true ]; then PUBLISHED=true', '; then PUBLISHED=true', 1))))
+        check("NEGATIVE CONTROL: a receipt that does not name `withheld` is caught",
+              any("publication: withheld" in w for w in workflow_publish_withhold_problems(
+                  workflow.replace(WITHHOLD_RECEIPT, 'elif false; then :', 1))))
+        check("NEGATIVE CONTROL: a staged record that blames the ledger for a withheld module is caught",
+              any("STAGED record" in w for w in workflow_publish_withhold_problems(
+                  workflow.replace('elif [ "$WITHHELD" = "true" ]; then\n', 'elif false; then\n', 1))))
+        check("NEGATIVE CONTROL: a lane without the input is caught",
+              any("declares no" in w for w in workflow_publish_withhold_problems(
+                  workflow.replace("      publish-withhold:\n", "      publish-withheld:\n", 1))))
         regressed_handover = workflow.replace(
             'bk entry --module "$MODULE" --phase publish',
             "bk get --module \"$MODULE\" entry --default '{}'",

.github/scripts/module-publication.py

@@ -645,8 +645,10 @@ def publish(a: argparse.Namespace) -> int:
         print("nothing owes a hand-over in this call: "
               f"{len(mine)} staged record(s), {len(selected)} selected, 0 to publish.")
         summarise(["### Module publication", "",
-                   f"{len(mine)} staged record(s) accounted for; **none owed a hand-over** "
-                   "(the build ledger already serves these keys, or the selection was empty)."])
+                   f"{len(mine)} staged record(s) accounted for; **none owed a hand-over**.", ""]
+                  + ([f"- `{m}@{rec.get('version', '?')}`: {rec.get('reason') or 'no reason recorded'}"
+                      for m, rec in sorted(mine.items())]
+                     or ["The selection was empty."]))
         return 0
 
     token = os.environ.get(PUBLISH_TOKEN_ENV, "")

.github/scripts/node-repo-pack-verify.py

@@ -224,10 +224,18 @@ def verify(expected: list[str], receipts: dict[str, dict], broken: list[str],
         superseded = sorted(m for m, r in receipts.items() if r.get("publication") == "superseded")
         stood_down = (f"; {len(superseded)} stood down for a newer trunk commit that reaches them "
                       f"(its own run publishes them): " + ", ".join(superseded)) if superseded else ""
+        # `withheld` is a FIFTH (`publish-withhold`, per-package publication): the caller's
+        # publication leaves the module's package out of this run — its lock is not settled, its
+        # platform floor is not stamped, or it depends on a package in that state — so the leg
+        # built and tested it and handed nothing over. Named, for the same reason as the others.
+        withheld = sorted(m for m, r in receipts.items() if r.get("publication") == "withheld")
+        if withheld:
+            stood_down += (f"; {len(withheld)} WITHHELD by the caller's per-package publication "
+                           f"(built and tested, handed over by a later run): " + ", ".join(withheld))
         if published:
             notes.append(f"{len(want)} of {len(want)} selected module bundle(s) built; "
                          f"{len(published)} published to the registry{stood_down}")
-        elif superseded and not staged:
+        elif (superseded or withheld) and not staged:
             notes.append(f"{len(want)} of {len(want)} selected module bundle(s) built; none published "
                          f"from this run{stood_down}")
         elif staged:
@@ -863,6 +871,23 @@ def idr(identity: str) -> dict[str, dict]:
           code == 0 and any("none published from this run" in n for n in nts)
           and not any("not a trunk/release run" in n for n in nts), f"{errs} {nts}")
 
+    print("publish-withhold — a package the caller withholds is NAMED, never read as unpublished:")
+    code, errs, nts = verify(["A", "B"], {"A": {"module": "A", "published": True, "publication": "direct"},
+                                         "B": {"module": "B", "published": False, "publication": "withheld"}},
+                             [], "success", True)
+    check("one published + one withheld is green, and the note names the withheld module",
+          code == 0 and any("1 published" in n and "1 WITHHELD" in n and n.rstrip().endswith("B") for n in nts),
+          f"{errs} {nts}")
+    code, errs, nts = verify(["B"], {"B": {"module": "B", "published": False, "publication": "withheld"}},
+                             [], "success", True)
+    check("…and a run whose only leg was withheld says so — not 'not a trunk/release run'",
+          code == 0 and any("none published from this run" in n and "WITHHELD" in n for n in nts)
+          and not any("not a trunk/release run" in n for n in nts), f"{errs} {nts}")
+    code, errs, nts = verify(["A"], {"A": {"module": "A", "published": True, "publication": "direct"}},
+                             [], "success", True)
+    check("NEGATIVE CONTROL: with nothing withheld the note does not mention it",
+          code == 0 and not any("WITHHELD" in n for n in nts), f"{errs} {nts}")
+
     if failures:
         print(f"\n::error title=node-repo-pack-verify self-test failed::{len(failures)} case(s) — "
               "this gate is the only thing standing between a narrowed matrix and a silently "

.github/workflows/node-repo-module-pack.yml

@@ -469,6 +469,22 @@ on:
         type: boolean
         required: false
         default: false
+      publish-withhold:
+        description: >-
+          PACKAGES this call must NOT hand to the registry although `publish` is true — a
+          whitespace- or comma-separated list of package names (the `package` of a matrix entry),
+          empty by default. A withheld package's module is still built, inspected and tested, its
+          bundle artifact is still uploaded for the caller's gates, and its receipt says
+          `publication: withheld`; only the hand-over is left out, in BOTH publish modes. It exists
+          for a caller whose publication is decided PER PACKAGE (MeshWeaver.Plugins: a package whose
+          lock is not settled, or whose platform floor is not stamped for its sources, waits — and
+          every package that depends on one waits with it — while the others publish). The caller
+          owns that decision and its dependency closure; this lane only executes it. A name that
+          matches no selected entry is not an error: the list describes the caller's whole tree,
+          and this call may have selected a subset of it. Ignored when `publish` is false.
+        type: string
+        required: false
+        default: ''
       publish-mode:
         description: >
           WHERE the live-registry hand-over happens, when `publish` is true. Both values are
@@ -2397,6 +2413,7 @@ jobs:
         env:
           LEDGER: ${{ inputs.ledger }}
           PUBLISH: ${{ inputs.publish }}
+          PUBLISH_WITHHOLD: ${{ inputs.publish-withhold }}
           TEST_MODULES: ${{ inputs.test-modules }}
           MW_LEDGER_URL: ${{ inputs.registry-url }}
           MW_LEDGER_TOKEN: ${{ secrets.ledger-token }}
@@ -2416,6 +2433,9 @@ jobs:
             NARROWED=true
             printf '%s' "$TEST_MODULES" | tr ',' '\n' | tr ' ' '\n' | sed '/^$/d' | sort -u > "$RUNNER_TEMP/test-modules.txt"
           fi
+          # 🚨 `publish-withhold` — the PACKAGES the caller's publication leaves out of this run.
+          # One name per line, exact match on the entry's `package`; the decision is the caller's.
+          printf '%s' "$PUBLISH_WITHHOLD" | tr ',' '\n' | tr ' ' '\n' | sed '/^$/d' | sort -u > "$RUNNER_TEMP/publish-withhold.txt"
           mapfile -t modules < <(bk list --ok)
           for MODULE in "${modules[@]}"; do
             ENTRY_LEDGER="$(bk get --module "$MODULE" ledger --default null)"
@@ -2458,6 +2478,15 @@ jobs:
                 need_test=false
                 echo "plan: $MODULE is packed but NOT tested — test-modules does not name it (its suite is not owed by this change set)"
               fi
+              # A withheld package is built and tested like any other and is NOT handed over —
+              # whatever the ledger answered. Written down per module (`withheld`), so the receipt
+              # says WHY nothing was published rather than leaving it to be inferred.
+              withheld=false
+              if [ "$PUBLISH" = true ] && grep -qxF "$(bk get --module "$MODULE" package)" "$RUNNER_TEMP/publish-withhold.txt"; then
+                withheld=true; need_publish=false
+                echo "plan: $MODULE is built and tested but NOT published — the caller withholds its package '$(bk get --module "$MODULE" package)' from this run's publication (publish-withhold)"
+              fi
+              bk set --module "$MODULE" "withheld=$withheld"
               bk set --module "$MODULE" "decision=$decision" "key=$key" "need_test=$need_test" "need_publish=$need_publish" \
                 "art_run=$art_run" "art_name=$art_name" "sha=$sha" "holder=$holder" "store_loc=$store_loc"
             )
@@ -4050,6 +4079,9 @@ jobs:
             PACKAGE="$(bk get --module "$MODULE" package)"
             VERSION="$(bk get --module "$MODULE" version)"
             OWED="$(bk get --module "$MODULE" need_publish)"
+            # A withheld package is not owed for a DIFFERENT reason than a ledger-served key, and
+            # the staged record is what the publication lane prints: it must say which.
+            WITHHELD="$(bk get --module "$MODULE" withheld --default false)"
             DECISION="$(bk get --module "$MODULE" decision)"
             HOLDER="$(bk get --module "$MODULE" holder)"
             LEDGER_KEY="$(bk get --module "$MODULE" key)"
@@ -4072,6 +4104,8 @@ jobs:
               if [ "$OWED" = "true" ]; then
                 cp "$BUNDLE" "$RUNNER_TEMP/staged/$name"
                 echo "staged $MODULE@$VERSION ($name, sha256 $sha, built against $identity) — the publication lane hands it over once this run's validation is green"
+              elif [ "$WITHHELD" = "true" ]; then
+                reason="no hand-over owed: the caller withholds package '$PACKAGE' from this run's publication (publish-withhold) — the registry does NOT serve this key yet; a later run of the caller hands it over"
               else
                 reason="no hand-over owed: the module build ledger already serves this key (decision '$DECISION'${HOLDER:+, holder $HOLDER})"
                 echo "staged $MODULE@$VERSION as NOT OWED — $reason"
@@ -4150,16 +4184,20 @@ jobs:
             # `publish-mode: staged` nothing reached the registry from here, and a receipt saying
             # otherwise would make the lane's own report ("N published to the registry") a claim
             # about something that has not happened yet.
+            WITHHELD="$(bk get --module "$MODULE" withheld --default false)"
             PUBLISHED=false
-            if [ "$PUBLISH" = true ] && [ "$PUBLISH_MODE" = direct ] && [ "$NEWER" != true ]; then PUBLISHED=true; fi
+            if [ "$PUBLISH" = true ] && [ "$PUBLISH_MODE" = direct ] && [ "$NEWER" != true ] && [ "$WITHHELD" != true ]; then PUBLISHED=true; fi
             # 🚨 WHICH ARM OWNED THIS MODULE'S HAND-OVER, written down rather than inferred from an
             # `if:` nobody can read after the fact — the same discipline as `TESTS` below. `direct`:
             # this leg POSTed the bundle itself. `staged`: it staged the bytes for the caller's
             # publication lane, which hands over after the full source verdict (MeshWeaver#3878).
             # `superseded`: publish-newest-only stood it down for the trunk tip. `none`: this call
             # does not publish at all. Two mutually exclusive conditions can BOTH be false; this is
             # what makes that impossible to do quietly.
+            # `withheld`: the caller's publication leaves this package out of this run
+            # (`publish-withhold`) — built and tested, handed over by a later run of the caller.
             if [ "$PUBLISH" != true ]; then PUBLICATION=none
+            elif [ "$WITHHELD" = true ]; then PUBLICATION=withheld
             elif [ "$NEWER" = true ]; then PUBLICATION=superseded
             else PUBLICATION="$PUBLISH_MODE"; fi
             # publish-newest-only: the trunk tip whose own run publishes this module instead.

src/MeshWeaver.Documentation/Data/Architecture/ModuleBuildArchitecture.md

@@ -569,6 +569,15 @@ keeps the supersede model in the next section.
    `publication: superseded`, naming the tip): that commit's own never-cancelled run builds the
    module from a tree that contains this one. A module never goes backwards and never waits for an
    unrelated module.
+   **A caller may withhold packages (`publish-withhold`).** A caller whose publication is decided per
+   package passes the names of the packages this run must not hand over. Their modules are still
+   built, inspected and tested, and their bundle artifacts still reach the caller's gates; only the
+   hand-over is left out, and the receipt says `publication: withheld`, which the lane's verdict
+   names beside `superseded`. The lane executes the list and does not derive it: which packages
+   wait, and that everything depending on a waiting package waits with it, is the caller's
+   decision. MeshWeaver.Plugins uses it so that a package whose lock is not settled or whose
+   platform floor is not stamped for its sources holds back only its own dependency network,
+   instead of the whole tree (`Hosting/PackageFloors` in that repository).
 5. **The seal never goes backwards.** The same reordering reaches the NodeType bake. A bake whose
    commit is an ancestor of the commit already sealed answers `scope=none` at decision time
    (`bake-scope.sh`) and skips at write time (`publish-bake-bundles.sh`, through the compare API):