[CRCR] Add agentic triage verdict support to cross-repo CI relay

.github/actions/cross-repo-ci-relay-callback/action.yml

@@ -56,6 +56,48 @@ inputs:
       Example: [{"name":"test_foo","classname":"TestBar","message":"AssertionError"}]
     required: false
     default: ''
+  triage-verdict:
+    description: >
+      Optional JSON object with a triage verdict for this job, explaining *why*
+      it failed — most importantly whether the cause was the upstream
+      pytorch/pytorch change under test rather than the backend itself.  A repo
+      that runs its own log-triage step (agent-based or not) passes that step's
+      output here; leaving it unset keeps today's behaviour exactly.
+
+      Advisory only: the verdict never gates a merge and never replaces the raw
+      conclusion.  Today the relay validates it and forwards it to the HUD
+      backend; surfacing it on the HUD and in the on-call PR comment is being
+      added separately, so attaching one now is safe but not yet visible
+      anywhere.
+
+      Shape (schema_version 1) — `category`, `confidence` and `summary` are
+      required, everything else optional:
+        {
+          "schema_version": 1,
+          "category": "upstream",        // upstream | backend | infra | flake | unknown
+          "confidence": "high",          // high | medium | low
+          "summary": "aten::foo lost its out= overload in #194610.",
+          "suspected_upstream": {        // ONLY valid for category "upstream"
+            "pr": 194610, "commit": "0e797b5a6acf",
+            "reason": "signature change to aten::foo"
+          },
+          "evidence": [{"job": "build-npu", "test": "test_foo_out_variant",
+                        "log_url": "https://.../job/123#step:5:2007",
+                        "excerpt": "error: no matching function ..."}],
+          "reproduced_on_retry": false,
+          "analyzer": {"name": "ascend-ci-triage", "version": "0.3.1",
+                       "model": "claude-haiku-4-5", "prompt_version": "2026-08-20"},
+          "analyzed_at": "2026-08-24T14:22:10Z"
+        }
+
+      The relay enum-, type- and size-validates every field and drops the WHOLE
+      verdict on any structural violation (unknown enum value, wrong type,
+      unrecognized schema_version), falling back to today's plain result — so a
+      broken triage tool degrades to no verdict, never to a half-rendered one.
+      Oversized values are truncated rather than rejected: summary 1 KB,
+      evidence 10 entries with 1 KB excerpts, 16 KB for the object as a whole.
+    required: false
+    default: ''
   callback-url:
     description: >
       Base URL of the result callback server.
@@ -131,6 +173,7 @@ runs:
         WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         TEST_RESULTS: ${{ inputs.test-results }}
         FAILED_TESTS_DETAIL: ${{ inputs.failed-tests-detail }}
+        TRIAGE_VERDICT: ${{ inputs.triage-verdict }}
         CLIENT_PAYLOAD: ${{ toJson(github.event.client_payload) }}
         OIDC_TOKEN: ${{ steps.oidc.outputs.token }}
         CALLBACK_URL: ${{ inputs.callback-url }}

.github/actions/cross-repo-ci-relay-callback/report_callback.py

@@ -24,6 +24,7 @@
     EVENT_TYPE_OVERRIDE     "nightly" or "periodic" (self-report)
     CHECK_RUN_ID            GitHub check run ID (falls back to RUN_ID-RUN_ATTEMPT)
     TEST_RESULTS            JSON string with test result summary
+    TRIAGE_VERDICT          JSON object with a triage verdict for this job
     ARTIFACT_URL            URL to downstream artifacts
     MAX_TIME                curl --max-time (default 10)
 """
@@ -122,6 +123,16 @@ def build_payload() -> str:
         except json.JSONDecodeError as exc:
             sys.exit(f"Error: FAILED_TESTS_DETAIL is not valid JSON: {exc}")
 
+    triage_verdict = os.environ.get("TRIAGE_VERDICT", "").strip()
+    if triage_verdict:
+        try:
+            parsed = json.loads(triage_verdict)
+            if not isinstance(parsed, dict):
+                raise ValueError("TRIAGE_VERDICT must be a JSON object")
+            workflow["triage_verdict"] = parsed
+        except (json.JSONDecodeError, ValueError) as exc:
+            print(f"Warning: ignoring invalid TRIAGE_VERDICT: {exc}", file=sys.stderr)
+
     artifact_url = os.environ.get("ARTIFACT_URL", "").strip()
     if artifact_url:
         workflow["artifact_url"] = artifact_url

aws/lambda/cross_repo_ci_relay/README.md

@@ -114,7 +114,21 @@ The HUD request looks like (two top-level namespaces: `trusted` and `untrusted`)
         "started_at": "2026-05-04T20:48:28Z", // when status == in_progress, else None
         "completed_at": "2026-05-04T21:23:45Z", // when status == completed, else None
         "test_results": { "passed": 42, "failed": 3, "skipped": 5 },
-        "artifact_url": "https://github.com/org/repo/actions/runs/123/artifacts"
+        "artifact_url": "https://github.com/org/repo/actions/runs/123/artifacts",
+        "triage_verdict": {     // optional, validated and normalized by the relay
+          "schema_version": 1,
+          "category": "upstream",  // upstream | backend | infra | flake | unknown
+          "confidence": "high",    // high | medium | low
+          "summary": "aten::foo lost its out= overload in #194610.",
+          "suspected_upstream": { "pr": 194610, "commit": "0e797b5a6acf",
+                                  "reason": "signature change to aten::foo" },
+          "evidence": [{ "job": "build-npu", "test": "test_foo_out_variant",
+                         "log_url": "https://.../job/123#step:5:2007",
+                         "excerpt": "error: no matching function ..." }],
+          "reproduced_on_retry": false,
+          "analyzer": { "name": "ascend-ci-triage", "version": "0.3.1" },
+          "analyzed_at": "2026-08-24T14:22:10Z"
+        }
       }
     }
   }
@@ -131,6 +145,11 @@ Trust boundaries inside `untrusted.callback_payload`:
   trusted at dispatch time, but not re-verified on the callback.
 - `untrusted.callback_payload.workflow` is **self-reported by the downstream CI** and is not
   authenticated.  Only `verified_repo` carries a cryptographic identity.
+- `untrusted.callback_payload.workflow.triage_verdict` is the one self-reported field the relay
+  rewrites rather than forwarding verbatim: it is enum-, type- and size-validated
+  (`utils/triage_verdict.py`) and the *whole* object is dropped on any structural violation, so HUD
+  either sees a normalized verdict or none at all.  It is advisory — it never gates a merge, never
+  changes a conclusion, and never adjusts the raw pass rate.
 
 ### Error propagation back to the downstream workflow
 

aws/lambda/cross_repo_ci_relay/callback/callback_handler.py

@@ -19,13 +19,38 @@
     HTTPException,
 )
 from utils.redis_helper import check_rate_limit
+from utils.triage_verdict import validate_triage_verdict
 
 
 logger = logging.getLogger(__name__)
 
 _NIGHTLY_EVENT_TYPES = frozenset({"nightly", "periodic"})
 
 
+def _sanitize_triage_verdict(body: dict, verified_repo: str) -> None:
+    """Validate ``workflow.triage_verdict`` IN PLACE.
+
+    The callback body is forwarded to HUD verbatim as untrusted input, so the
+    raw verdict is *replaced* with the validated one (and deleted outright when
+    it does not validate).
+    """
+    workflow = body.get("workflow")
+    if not isinstance(workflow, dict) or "triage_verdict" not in workflow:
+        return
+
+    verdict = validate_triage_verdict(workflow.get("triage_verdict"))
+    if verdict is None:
+        # Advisory only: a bad verdict must never fail the callback.  Drop it
+        # and let everything downstream behave as if none had been attached.
+        logger.info(
+            "dropped invalid triage_verdict from repo=%s",
+            verified_repo,
+        )
+        workflow.pop("triage_verdict", None)
+    else:
+        workflow["triage_verdict"] = verdict
+
+
 def _build_trusted(
     verified_repo: str,
     repo_level: AllowlistLevel,
@@ -314,6 +339,8 @@ def _handle_nightly_callback(
             f"nightly/periodic callbacks must have status 'completed', got {status!r}",
         )
 
+    _sanitize_triage_verdict(body, verified_repo)
+
     trusted = _build_trusted(verified_repo, repo_level)
     untrusted = {"callback_payload": body}
 
@@ -377,6 +404,10 @@ def handle(config: RelayConfig, body: dict, verified_repo: str) -> dict:
         _parse_callback_body(body)
     )
 
+    # Before the body is cached or forwarded to HUD -- both consumers must see
+    # the validated verdict, never the raw one.
+    _sanitize_triage_verdict(body, verified_repo)
+
     dispatch_record = redis_helper.get_callback_state(
         config, delivery_id, verified_repo, DISPATCH_RUN_ID, DISPATCH_RUN_ATTEMPT
     )

aws/lambda/cross_repo_ci_relay/tests/test_callback_handler.py

@@ -600,6 +600,115 @@ def test_check_run_update_failure_does_not_break_response(self):
         self.assertEqual(result, {"ok": True, "status": "completed"})
 
 
+class TestCallbackTriageVerdict(unittest.TestCase):
+    """The optional triage verdict is validated before anything consumes it."""
+
+    def setUp(self):
+        self.patcher_allowlist = patch("callback.callback_handler.load_allowlist")
+        mock_map = MagicMock()
+        mock_map.get_repo_level.return_value = AllowlistLevel.L3
+        mock_map.needs_check_run.return_value = True
+        self.patcher_allowlist.start().return_value = mock_map
+
+        self.patcher_redis = patch("callback.callback_handler.redis_helper")
+        self.mock_redis = self.patcher_redis.start()
+        self.mock_redis.record_workflow_started.side_effect = lambda *a, **kw: (
+            a[5],
+            True,
+        )
+
+        def _get_state(
+            cfg,
+            delivery_id,
+            repo,
+            run_id_arg,
+            run_attempt_arg,
+            client=None,
+            job_name=None,
+        ):
+            if run_id_arg == DISPATCH_RUN_ID:
+                return CallbackStateRecord(CallbackState.DISPATCHED, 1000.0, {})
+            return CallbackStateRecord(CallbackState.IN_PROGRESS, 1030.0, {})
+
+        self.mock_redis.get_callback_state.side_effect = _get_state
+
+        self.patcher_rate = patch("callback.callback_handler.check_rate_limit")
+        self.patcher_rate.start().return_value = True
+
+        self.patcher_hud = patch("callback.callback_handler.forward_to_hud")
+        self.mock_hud = self.patcher_hud.start()
+
+        self.patcher_gh = patch("callback.callback_handler.gh_helper")
+        self.mock_gh = self.patcher_gh.start()
+        self.mock_gh.get_repo_access_token.return_value = "tok"
+        self.mock_gh.create_check_run.return_value = 888
+
+    def tearDown(self):
+        for patcher in (
+            self.patcher_allowlist,
+            self.patcher_redis,
+            self.patcher_rate,
+            self.patcher_hud,
+            self.patcher_gh,
+        ):
+            patcher.stop()
+
+    def _body_with(self, verdict):
+        body = _body(status="completed", job_name="build-npu")
+        body["workflow"]["triage_verdict"] = verdict
+        return body
+
+    def _valid(self, **overrides):
+        verdict = {
+            "schema_version": 1,
+            "category": "upstream",
+            "confidence": "high",
+            "summary": "aten::foo lost its out= overload.",
+        }
+        verdict.update(overrides)
+        return verdict
+
+    def _forwarded_workflow(self):
+        _, _, untrusted = self.mock_hud.call_args[0]
+        return untrusted["callback_payload"]["workflow"]
+
+    def test_valid_verdict_reaches_hud(self):
+        handle(_cfg(), self._body_with(self._valid()), verified_repo="org/repo")
+
+        self.assertEqual(self._forwarded_workflow()["triage_verdict"], self._valid())
+
+    def test_verdict_is_normalized_before_being_forwarded(self):
+        # HUD must never see the raw object: unknown keys and over-long values
+        # are what the size caps here exist to keep out of its records.
+        body = self._body_with(
+            self._valid(summary="s" * 5000, undeclared_field="x" * 5000)
+        )
+        handle(_cfg(), body, verified_repo="org/repo")
+
+        forwarded = self._forwarded_workflow()["triage_verdict"]
+        self.assertNotIn("undeclared_field", forwarded)
+        self.assertEqual(len(forwarded["summary"]), 1000)
+
+    def test_invalid_verdict_is_dropped_and_the_callback_still_succeeds(self):
+        result = handle(
+            _cfg(),
+            self._body_with(self._valid(category="cosmic-rays")),
+            verified_repo="org/repo",
+        )
+
+        self.assertEqual(result, {"ok": True, "status": "completed"})
+        self.assertNotIn("triage_verdict", self._forwarded_workflow())
+
+    def test_no_verdict_leaves_the_payload_untouched(self):
+        handle(
+            _cfg(),
+            _body(status="completed", job_name="build"),
+            verified_repo="org/repo",
+        )
+
+        self.assertNotIn("triage_verdict", self._forwarded_workflow())
+
+
 class TestNightlyCallback(unittest.TestCase):
     """Nightly/periodic callbacks bypass the state machine entirely."""
 
@@ -710,6 +819,30 @@ def test_nightly_missing_delivery_id_returns_400(self):
             handle(_cfg(), body, verified_repo="org/repo")
         self.assertEqual(ctx.exception.status_code, 400)
 
+    def test_nightly_verdict_is_validated_before_forwarding(self):
+        body = self._nightly_body(conclusion="failure")
+        body["workflow"]["triage_verdict"] = {
+            "schema_version": 1,
+            "category": "infra",
+            "confidence": "medium",
+            "summary": "device allocation timed out",
+            "undeclared_field": "x",
+        }
+        handle(_cfg(), body, verified_repo="org/repo")
+
+        _, _, untrusted = self.mock_hud.call_args[0]
+        forwarded = untrusted["callback_payload"]["workflow"]["triage_verdict"]
+        self.assertNotIn("undeclared_field", forwarded)
+        self.assertEqual(forwarded["category"], "infra")
+
+    def test_nightly_invalid_verdict_is_dropped(self):
+        body = self._nightly_body()
+        body["workflow"]["triage_verdict"] = {"category": "upstream"}
+        handle(_cfg(), body, verified_repo="org/repo")
+
+        _, _, untrusted = self.mock_hud.call_args[0]
+        self.assertNotIn("triage_verdict", untrusted["callback_payload"]["workflow"])
+
     def test_nightly_failure_conclusion_forwards(self):
         body = self._nightly_body()
         body["workflow"]["conclusion"] = "failure"