feat: standardize NeMo Fabric IDs on adapter Relay root scopes

adapters/python/common/README.md

@@ -148,3 +148,69 @@ This helper does not apply to adapters that send telemetry through an external
 Relay gateway or whose upstream integration creates an isolated scope context.
 Those adapters retain their native session correlation until their Relay
 boundary accepts a per-turn propagation context.
+
+### Correlation Metadata Keys
+
+Adapters that own their invocation-level Agent scope attach NeMo Fabric IDs to
+that root scope as Relay metadata. The following metadata keys are reserved for
+this purpose, and telemetry consumers match them literally:
+
+| Relay metadata key | `RuntimeContext` field | Meaning |
+| --- | --- | --- |
+| `nemo_fabric_request_id` | `request_id` | Correlates the caller's logical request. A caller can reuse the value across retries, so several invocations can share it. |
+| `nemo_fabric_invocation_id` | `invocation_id` | Identifies one invocation. NeMo Fabric assigns a new value to every invocation, so this key joins Relay telemetry to exactly one `RunResult`. |
+| `nemo_fabric_runtime_id` | `runtime_id` | Identifies the runtime that processed the invocation. One runtime can process multiple invocations. |
+
+Build the keys with
+`nemo_fabric_adapters.common.utils.relay_correlation_metadata()` and merge them
+into the metadata that `relay_request_context()` returns, as in the following
+example. Merging rather than replacing keeps any key that
+`relay_request_context()` adds, such as `nemo_fabric_session_root`.
+
+```python
+import nemo_fabric_adapters.common.utils as common_utils
+from nemo_relay import ScopeType
+from nemo_relay import scope
+
+request_context, metadata = common_utils.relay_request_context(
+    context.request_id, request.relay_session_root
+)
+metadata.update(common_utils.relay_correlation_metadata(context))
+with (
+    request_context,
+    scope.scope("my-adapter.request", ScopeType.Agent, metadata=metadata),
+):
+    ...
+```
+
+Nested scopes do not repeat the keys because the Relay scope hierarchy already
+links them to the root scope. The LangChain Deep Agents, mini-SWE-agent, and
+NOOA adapters attach all three keys. NOOA calls only
+`relay_correlation_metadata()`, so its request ID does not become the
+propagated Relay root.
+
+Claude, Codex, and Pi send telemetry through an external Relay gateway and do
+not yet carry per-invocation NeMo Fabric IDs into Relay. Hermes Agent emits
+Relay telemetry through its own plugin, so its adapter does not attach these
+keys either. Remote Agent forwards only `nemo_fabric_request_id`, in the remote
+API request body.
+
+ATOF records carry these keys in the root scope's event metadata. To export
+them as OpenTelemetry span attributes, add the `nemo_fabric_` prefix to
+`promote_metadata_prefixes` on the trace endpoint. NeMo Fabric passes the field
+to Relay unchanged:
+
+```python
+from nemo_fabric import RelayOpenTelemetryEndpointConfig
+
+endpoint = RelayOpenTelemetryEndpointConfig(
+    type="gen_ai",
+    endpoint="http://localhost:4318/v1/traces",
+    promote_metadata_prefixes=["nemo_fabric_"],
+)
+```
+
+Relay promotes the keys onto the invocation's root Agent span; child spans
+share that span's trace. Refer to "Event Metadata Promotion" in the
+[NeMo Relay OpenTelemetry documentation](https://github.com/NVIDIA/NeMo-Relay/blob/0.9.3/docs/configure-plugins/observability/opentelemetry.mdx)
+for prefix syntax and collision rules.

adapters/python/common/src/nemo_fabric_adapters/common/utils.py

@@ -16,10 +16,17 @@
 from typing import Any
 
 from nemo_fabric_adapter_contract.codec import JsonValue
+from nemo_fabric_adapter_contract.models import RuntimeContext
 
 
 _FIELD_NAME = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+")
 
+# Reserved Relay metadata keys that join Relay telemetry to NeMo Fabric IDs.
+# Consumers match these names literally, so treat them as a stable contract.
+RELAY_REQUEST_ID_METADATA_KEY = "nemo_fabric_request_id"
+RELAY_INVOCATION_ID_METADATA_KEY = "nemo_fabric_invocation_id"
+RELAY_RUNTIME_ID_METADATA_KEY = "nemo_fabric_runtime_id"
+
 
 def normalize_user_input(value: JsonValue) -> str:
     """Return user input as text, serializing structured values deterministically."""
@@ -278,7 +285,7 @@ def relay_request_context(
 ) -> tuple[Any, dict[str, str]]:
     """Root Relay at a usable session root, else a UUID request ID; preserve metadata."""
 
-    metadata = {"nemo_fabric_request_id": request_id}
+    metadata = {RELAY_REQUEST_ID_METADATA_KEY: request_id}
     request_uuid = _uuid_or_none(request_id)
     session_uuid = _uuid_or_none(session_root)
 
@@ -297,6 +304,16 @@ def relay_request_context(
     return use_scope_stack(stack), metadata
 
 
+def relay_correlation_metadata(context: RuntimeContext) -> dict[str, str]:
+    """Return the NeMo Fabric IDs to attach to an invocation's Relay root scope."""
+
+    return {
+        RELAY_REQUEST_ID_METADATA_KEY: context.request_id,
+        RELAY_INVOCATION_ID_METADATA_KEY: context.invocation_id,
+        RELAY_RUNTIME_ID_METADATA_KEY: context.runtime_id,
+    }
+
+
 def ambient_relay_plugin_config_paths() -> list[Path]:
     """Return ambient user or project Relay plugin configs visible to Python."""
 

adapters/python/deepagents/src/nemo_fabric_adapters/deepagents/adapter.py

@@ -724,7 +724,6 @@ async def invoke(
             )
 
         user_message = common_utils.normalize_user_input(request.input)
-        request_id = runtime_context.request_id
 
         resumed = self._completed_invocations > 0
         inherited_quarantine = self._telemetry_quarantine is not None
@@ -733,8 +732,7 @@ async def invoke(
         else:
             outcome = await self._invoke_with_telemetry(
                 user_message,
-                request_id,
-                runtime_context.invocation_id,
+                runtime_context,
                 request.relay_session_root,
             )
 
@@ -806,8 +804,7 @@ async def invoke(
     async def _invoke_with_telemetry(
         self,
         user_message: str,
-        request_id: str,
-        invocation_id: str,
+        runtime_context: RuntimeContext,
         session_root: str | None,
     ) -> TurnOutcome:
         """Run one turn inside the Relay plugin/scope, isolating telemetry faults.
@@ -838,9 +835,11 @@ async def _invoke_with_telemetry(
                 # turn, which would lose one of the two.
                 try:
                     request_context, metadata = common_utils.relay_request_context(
-                        request_id, session_root
+                        runtime_context.request_id, session_root
+                    )
+                    metadata.update(
+                        common_utils.relay_correlation_metadata(runtime_context)
                     )
-                    metadata["nemo_fabric_invocation_id"] = invocation_id
                     with request_context:
                         with self._relay_scope.scope(
                             "deepagents-request",

adapters/python/mini-swe-agent/README.md

@@ -153,20 +153,18 @@ hierarchy without requiring upstream changes.
 ### Correlation IDs
 
 The top-level `mini-swe-agent.request` Agent scope stores the NeMo Fabric
-request and invocation IDs, and a caller-supplied session root, as Relay
-metadata:
+request, invocation, and runtime IDs, and a caller-supplied session root, as
+Relay metadata:
 
 | NeMo Fabric ID | Relay metadata key | Meaning |
 | --- | --- | --- |
 | `request_id` | `nemo_fabric_request_id` | Correlates the caller's logical request. A caller can provide the same value when it wants to correlate retries or related processing. |
 | `invocation_id` | `nemo_fabric_invocation_id` | Identifies one concrete invocation attempt. NeMo Fabric assigns a new value to each invocation. |
+| `runtime_id` | `nemo_fabric_runtime_id` | Identifies the longer-lived retained runtime that can process multiple invocations. |
 | `RunRequest.relay_session_root` | `nemo_fabric_session_root` | Groups invocations into one Relay session. A UUID string Relay accepts becomes the propagated Relay root; a UUID `request_id` stays the parent, otherwise the session root is also the parent. Absent when the value is missing or unusable; the invocation proceeds without an error. |
 
 Nested step, LLM, and `bash` tool events are correlated through the Relay scope
-hierarchy; they do not repeat these metadata fields. The `runtime_id`
-identifies the longer-lived retained runtime that can process multiple
-invocations, but the mini-SWE-agent adapter does not currently add it to Relay
-scope metadata.
+hierarchy; they do not repeat these metadata fields.
 
 Install both the harness and Relay:
 

adapters/python/mini-swe-agent/src/nemo_fabric_adapters/mini_swe_agent/adapter.py

@@ -213,7 +213,7 @@ async def _run_with_relay(
                 request_context, metadata = common_utils.relay_request_context(
                     context.request_id, session_root
                 )
-                metadata["nemo_fabric_invocation_id"] = context.invocation_id
+                metadata.update(common_utils.relay_correlation_metadata(context))
                 with (
                     request_context,
                     self._relay_scope.scope(

adapters/python/nooa/src/nemo_fabric_adapters/nooa/telemetry.py

@@ -259,11 +259,7 @@ async def invoke(
                 common_utils.reject_inherited_relay_plugin_config(activation.report)
                 uninstall = install_nemo_relay_compat(agent.event_manager)
                 try:
-                    metadata = {
-                        "nemo_fabric_request_id": runtime_context.request_id,
-                        "nemo_fabric_invocation_id": runtime_context.invocation_id,
-                        "nemo_fabric_runtime_id": runtime_context.runtime_id,
-                    }
+                    metadata = common_utils.relay_correlation_metadata(runtime_context)
                     try:
                         with scope.scope(
                             self._scope_name,

examples/langgraph_custom_agent/adapter/telemetry.py

@@ -135,12 +135,7 @@ async def observe_invocation(
         request_context, metadata = common_utils.relay_request_context(
             context.request_id, session_root
         )
-        metadata.update(
-            {
-                "nemo_fabric_runtime_id": context.runtime_id,
-                "nemo_fabric_invocation_id": context.invocation_id,
-            }
-        )
+        metadata.update(common_utils.relay_correlation_metadata(context))
         with (
             request_context,
             scope.scope(

skills/nemo-fabric-build-adapter/SKILL.md

@@ -247,7 +247,13 @@ instead, a UUID request ID stays the parent (otherwise the session root is), and
 `nemo_fabric_session_root` is added to the metadata, so the caller's invocations
 share one Relay session. Context keys do not control propagation. An unusable
 session root falls back to a usable UUID request ID without raising; if neither
-value is usable, the helper returns `nullcontext()` without setting propagation. Do not apply this pattern to an external Relay gateway or an
+value is usable, the helper returns `nullcontext()` without setting propagation. Merge
+`nemo_fabric_adapters.common.utils.relay_correlation_metadata(context)` into
+that metadata with `metadata.update(...)` rather than replacing it, so keys
+from `relay_request_context()` reach the scope. `relay_correlation_metadata()`
+returns the reserved `nemo_fabric_request_id`, `nemo_fabric_invocation_id`, and
+`nemo_fabric_runtime_id` keys that consumers use to join Relay telemetry to a
+NeMo Fabric result. Do not apply this pattern to an external Relay gateway or an
 upstream integration that creates an isolated scope context unless its boundary
 accepts a per-turn propagation context.
 

tests/adapters/test_adapaters_common_utils.py

@@ -25,6 +25,7 @@
 from nemo_fabric import ModelConfig
 from nemo_fabric import RelayAtifConfig
 from nemo_fabric import RelayObservabilityConfig
+from nemo_fabric_adapter_contract.models import RuntimeContext
 
 
 ROOT = Path(__file__).resolve().parents[2]
@@ -187,6 +188,30 @@ def test_two_requests_share_one_session_root(monkeypatch: pytest.MonkeyPatch):
     ]
 
 
+def test_relay_correlation_metadata_returns_reserved_fabric_ids(tmp_path: Path):
+    context = RuntimeContext.from_mapping(
+        {
+            "runtime_id": "runtime-1",
+            "invocation_id": "invocation-1",
+            "request_id": "request-1",
+            "environment": {
+                "environment_id": "test-environment",
+                "provider": "test",
+                "control_location": "in_env_control",
+                "workspace": str(tmp_path),
+                "ownership": "caller_owned",
+            },
+            "artifacts": {},
+        }
+    )
+
+    assert common_utils.relay_correlation_metadata(context) == {
+        "nemo_fabric_request_id": "request-1",
+        "nemo_fabric_invocation_id": "invocation-1",
+        "nemo_fabric_runtime_id": "runtime-1",
+    }
+
+
 @pytest.mark.parametrize(
     ("value", "expected"),
     [

tests/adapters/test_deepagents.py

@@ -740,6 +740,7 @@ async def test_relay_telemetry_wraps_agent_and_reports_artifacts(
         {
             "nemo_fabric_request_id": "request-1",
             "nemo_fabric_invocation_id": "inv-1",
+            "nemo_fabric_runtime_id": "run-1",
         }
     ]
     # the Deep Agents callback handler is added to the LangGraph run config so
@@ -758,6 +759,7 @@ async def test_relay_telemetry_wraps_agent_and_reports_artifacts(
             {
                 "nemo_fabric_request_id": "018f47a4-3af7-7d94-8e61-9f0f89b5d312",
                 "nemo_fabric_invocation_id": "inv-1",
+                "nemo_fabric_runtime_id": "run-1",
             },
         ),
         (
@@ -766,6 +768,7 @@ async def test_relay_telemetry_wraps_agent_and_reports_artifacts(
             {
                 "nemo_fabric_request_id": "request-1",
                 "nemo_fabric_invocation_id": "inv-1",
+                "nemo_fabric_runtime_id": "run-1",
             },
         ),
     ],
@@ -1411,6 +1414,7 @@ async def test_native_telemetry_exports_without_artifacts(
         {
             "nemo_fabric_request_id": "request-1",
             "nemo_fabric_invocation_id": "inv-1",
+            "nemo_fabric_runtime_id": "run-1",
         }
     ]
     assert fake_relay["callback_handler"] in (fake_sdks["config"] or {}).get(
@@ -2216,10 +2220,12 @@ async def test_persistent_runtime_scopes_relay_per_invocation(
         {
             "nemo_fabric_request_id": "request-1",
             "nemo_fabric_invocation_id": "inv-1",
+            "nemo_fabric_runtime_id": "run-relay-persistent",
         },
         {
             "nemo_fabric_request_id": "request-1",
             "nemo_fabric_invocation_id": "inv-2",
+            "nemo_fabric_runtime_id": "run-relay-persistent",
         },
     ]
     assert fake_sdks["saver_exits"] == 1

tests/adapters/test_mini_swe_agent.py

@@ -422,6 +422,7 @@ async def test_relay_enabled_uses_instrumented_subclass_and_reports_artifacts(
     assert request_scope[2]["metadata"] == {
         "nemo_fabric_request_id": "mini-request",
         "nemo_fabric_invocation_id": "mini-invocation",
+        "nemo_fabric_runtime_id": "mini-runtime",
     }
     assert mock_relay["step_starts"][0][2]["handle"] is mock_relay["request_handle"]
     assert mock_relay["llm_starts"][0][2]["handle"] is mock_relay["step_handle"]
@@ -463,6 +464,7 @@ async def test_uuid_request_id_seeds_relay_propagation(
     assert mock_relay["request_scopes"][0][2]["metadata"] == {
         "nemo_fabric_request_id": request_id,
         "nemo_fabric_invocation_id": "mini-invocation",
+        "nemo_fabric_runtime_id": "mini-runtime",
     }
 
 
@@ -502,6 +504,7 @@ async def test_typed_session_root_roots_relay_propagation(
         "nemo_fabric_request_id": request_id,
         "nemo_fabric_session_root": SESSION_ROOT,
         "nemo_fabric_invocation_id": "mini-invocation",
+        "nemo_fabric_runtime_id": "mini-runtime",
     }