fix(telemetry): prevent deploy success with warning from recording as failure (#2769)

* chore(telemetry): model errors for legacy deploys

* fix(telemetry): prevent deploy success with warning from recording as failure in metrics

* fix(dev): continue harness session after legacy deploy warnings

Author: nborges-aws

src/errors/errors.tsx

@@ -77,6 +77,23 @@ export class AgentCoreCLIError extends Error {
 /** Base for CLI errors intentionally omitted from root stderr output. */
 export class SilentCLIError extends AgentCoreCLIError {}
 
+/** A v0 deploy completed, but its post-deploy steps reported warnings. */
+export class LegacyDeployWarningExit extends SilentCLIError {
+  constructor() {
+    super("v0 deploy completed with post-deploy warnings", { exitCode: 2 });
+  }
+}
+
+/** A v0 deploy failed after taking ownership of terminal output. */
+export class LegacyDeployError extends SilentCLIError {
+  constructor(exitCode: number) {
+    super(`v0 deploy exited unsuccessfully with code ${exitCode}`, {
+      exitCode,
+      meta: { legacyDeployExitCode: exitCode },
+    });
+  }
+}
+
 /** Error raised for invalid user input. */
 export class InputValidationError extends AgentCoreCLIError {
   constructor(message?: string, options?: Omit<AgentCoreCLIErrorOptions, "source">) {

src/errors/index.tsx

@@ -7,6 +7,8 @@ export {
   GatewayInvokeResponseError,
   InputValidationError,
   InvalidEnvironmentError,
+  LegacyDeployError,
+  LegacyDeployWarningExit,
   MalformedServiceResponseError,
   NestedProjectError,
   NetworkingError,

src/handlers/project/deploy/index.test.ts

@@ -1,7 +1,11 @@
 import { afterEach, describe, expect, test } from "bun:test";
 import { readFile, writeFile } from "node:fs/promises";
 import { join } from "node:path";
-import { UserCancellationError } from "../../../errors/errors";
+import {
+  LegacyDeployError,
+  LegacyDeployWarningExit,
+  UserCancellationError,
+} from "../../../errors/errors";
 import { createRootHandler } from "../../index";
 import { DEFAULT_GLOBAL_CONFIG } from "../../../globalConfig";
 import { ValueContext } from "../../../router";
@@ -344,7 +348,52 @@ describe("project deploy handler", () => {
     );
     const subject = await testDeployCommand({ outputs: {} }, [], { runV0Deploy: async () => 7 });
 
-    await expect(subject.run(["--yes"])).rejects.toMatchObject({ exitCode: 7 });
+    const error = await subject.run(["--yes"]).then(
+      () => undefined,
+      (caught: unknown) => caught,
+    );
+    expect(error).toBeInstanceOf(LegacyDeployError);
+    expect(error).toMatchObject({ exitCode: 7, meta: { legacyDeployExitCode: 7 } });
+    await assertMetricEmitted(subject.telemetryFilePath, "cli.command_run", {
+      command_path: "/agentcore/deploy",
+      exit_reason: "failure",
+      error_name: "LegacyDeployError",
+      legacy_deploy_exit_code: 7,
+    });
+    expect(subject.calls).toEqual([]);
+  });
+
+  test("preserves v0's warning exit code without reporting a failed deploy", async () => {
+    await inProjectWithTargets();
+    await writeFile(
+      join(process.cwd(), "agentcore", "agentcore.json"),
+      JSON.stringify({ name: "orders", version: 1 }),
+    );
+    const subject = await testDeployCommand({ outputs: {} }, [], { runV0Deploy: async () => 2 });
+
+    const error = await subject.run(["--yes"]).then(
+      () => undefined,
+      (caught: unknown) => caught,
+    );
+    expect(error).toBeInstanceOf(LegacyDeployWarningExit);
+    expect(error).toMatchObject({
+      name: "LegacyDeployWarningExit",
+      exitCode: 2,
+    });
+    await assertMetricEmitted(subject.telemetryFilePath, "cli.command_run", {
+      command_path: "/agentcore/deploy",
+      exit_reason: "success",
+      legacy_deploy_exit_code: 2,
+    });
+    const events = (await readFile(subject.telemetryFilePath, "utf8"))
+      .trim()
+      .split("\n")
+      .map((line) => JSON.parse(line));
+    const warningRun = events.find(
+      (event) =>
+        event.metricName === "cli.command_run" && event.attrs.legacy_deploy_exit_code === 2,
+    );
+    expect(warningRun?.attrs).not.toHaveProperty("error_name");
     expect(subject.calls).toEqual([]);
   });
 

src/handlers/project/deploy/index.ts

@@ -3,7 +3,12 @@ import z from "zod";
 import { enclosingProjectRoot, isV0Project } from "../../../core/project/fsUtils";
 import { runV0Deploy, type V0DeployRunner } from "../../../core/project/legacy/deploy";
 import { assertNoPendingMigration } from "../../../core/project/migrate/journal";
-import { ProjectStateError, SilentCLIError, UserCancellationError } from "../../../errors";
+import {
+  LegacyDeployError,
+  LegacyDeployWarningExit,
+  ProjectStateError,
+  UserCancellationError,
+} from "../../../errors";
 import type { AppIO } from "../../../io";
 import { DEFAULT_TARGET_NAME } from "../../../projectSchemas/aws-targets";
 import {
@@ -258,9 +263,9 @@ async function delegateV0Deploy(
     );
   }
   const exitCode = await (config.runV0Deploy ?? runV0Deploy)(input);
-  if (exitCode !== 0) {
-    throw new SilentCLIError("v0 deploy exited unsuccessfully", { exitCode });
-  }
+  ctx.value(CommandRunMetricEventKey)?.setAttributes({ legacy_deploy_exit_code: exitCode });
+  if (exitCode === 2) throw new LegacyDeployWarningExit();
+  if (exitCode !== 0) throw new LegacyDeployError(exitCode);
 }
 
 /**

src/handlers/project/dev/index.test.ts

@@ -24,6 +24,7 @@ import { JsonRendererKey } from "../../../tui";
 import { JsonKey, RegionKey } from "../../keys";
 import type { DeployProjectInput, Project, ProjectManager } from "../types";
 import type { V0DeployInput } from "../../../core/project/legacy/deploy";
+import type { V0DeployRunner } from "../../../core/project/legacy/deploy";
 import { ProjectSpecSchema } from "../../../projectSchemas/project";
 import {
   BMA_POLICY_FILE,
@@ -125,6 +126,7 @@ type HarnessOptions = {
   loadEnvironment?: DevProjectHandlerConfig["loadDevEnvironment"];
   /** Deploy through the real project manager instead of recording deployments. */
   realDeploy?: boolean;
+  runV0Deploy?: V0DeployRunner;
   /**
    * Replaces the build-settings sync. By default it passes the project through,
    * since most projects here are in-memory; `"real"` uses the project manager's.
@@ -209,7 +211,7 @@ function harness(options: HarnessOptions = {}) {
     projectManager,
     runV0Deploy: async (input) => {
       legacyDeployments.push(input);
-      return 0;
+      return options.runV0Deploy ? options.runV0Deploy(input) : 0;
     },
     waitReady: async () => {
       await Bun.sleep(5);
@@ -877,6 +879,25 @@ describe("project dev harnesses", () => {
     );
   });
 
+  test("v0 post-deploy warnings still allow harness Inspector to start", async () => {
+    const configuredProject = harnessProject();
+    configuredProject.legacy = { format: "v0", v0Spec: {} as never };
+    const subject = harness({ project: configuredProject, runV0Deploy: async () => 2 });
+    const pending = subject.route(["--mode", "browser", "--target", "staging"]);
+    pending.catch(() => undefined);
+    await Bun.sleep(10);
+
+    try {
+      expect(subject.legacyDeployments).toHaveLength(1);
+      expect(subject.ui.starts).toEqual([{ port: 8081 }]);
+      expect(subject.io.stderr()).toContain("Agent Inspector running at http://127.0.0.1:8081");
+    } finally {
+      process.emit("SIGTERM", "SIGTERM");
+      await expect(pending).rejects.toBeInstanceOf(UserCancellationError);
+    }
+    expect(subject.ui.closed).toBe(1);
+  });
+
   test("harness dev's deploy refuses a target a migration hasn't finished", async () => {
     const root = await mkdtemp(join(tmpdir(), "agentcore-dev-migrate-"));
     try {

src/handlers/project/dev/index.ts

@@ -13,6 +13,7 @@ import {
   ERROR_SOURCE,
   InputValidationError,
   DeserializationError,
+  LegacyDeployWarningExit,
   NotImplementedError,
   ResourceNotFoundError,
   SilentCLIError,
@@ -257,11 +258,16 @@ async function runHarnessDev({
 }): Promise<void> {
   const { harnesses, deploymentTarget } = input;
   if (!input.skipDeploy) {
-    await createDeployProjectHandler(config).handle(
-      ctx,
-      { target: deploymentTarget, yes: input.yes },
-      {},
-    );
+    try {
+      await createDeployProjectHandler(config).handle(
+        ctx,
+        { target: deploymentTarget, yes: input.yes },
+        {},
+      );
+    } catch (error) {
+      // v0 deploy completed; its post-deploy warnings must not prevent harness dev from starting.
+      if (!(error instanceof LegacyDeployWarningExit)) throw error;
+    }
   }
   if (input.mode === "headless" && (input.skipDeploy || ctx.require(JsonKey))) {
     config.io.stderr.write(`next step:\n  ${DEPLOY_NEXT_STEP}\n`);

src/index.ts

@@ -20,6 +20,7 @@ import { runWithExitCode } from "./runnable";
 import { DefaultGlobalConfigAccessor } from "./globalConfig";
 import { DefaultTelemetryClient, printFirstRunNotice } from "./telemetry";
 import { AgentCoreCLIError } from "./errors";
+import { shouldRecordCommandFailure } from "./telemetry/commandOutcome";
 import { PACKAGE_VERSION } from "./constants";
 import { CommandRunMetricEventKey, ValueContext } from "./router";
 import { NpmCliVersionManager, printUpdateNotice } from "./cliVersionManager";
@@ -116,7 +117,7 @@ process.exit(
       await rootHandler.route(argv, context);
     } catch (e) {
       const error = AgentCoreCLIError.fromError(e);
-      if (error.exitCode !== 0) {
+      if (shouldRecordCommandFailure(error)) {
         rootLogger.child({ error: error.json() }).error();
         commandRunMetricEvent.setAttributes({
           exit_reason: "failure",

src/telemetry/commandOutcome.ts

@@ -0,0 +1,6 @@
+import { AgentCoreCLIError, LegacyDeployWarningExit } from "../errors";
+
+/** The v0 deployer uses exit 2 after a successful deploy with post-deploy warnings. */
+export function shouldRecordCommandFailure(error: AgentCoreCLIError): boolean {
+  return error.exitCode !== 0 && !(error instanceof LegacyDeployWarningExit);
+}

src/telemetry/shapes.tsx

@@ -64,6 +64,7 @@ export const commandRunSchema = safeSchema({
   error_source: z.enum(Object.values(ERROR_SOURCE)).optional(),
   is_tui: z.boolean().default(false),
   project_format: z.enum(["v0", "v1", "unknown"]).optional(),
+  legacy_deploy_exit_code: z.number().int().min(0).max(255).optional(),
   project_runtime_count: z.number().int().nonnegative().optional(),
   project_memory_count: z.number().int().nonnegative().optional(),
   project_knowledge_base_count: z.number().int().nonnegative().optional(),

src/testing/telemetry.ts

@@ -6,6 +6,7 @@ import { PACKAGE_VERSION } from "../constants";
 import { AgentCoreCLIError } from "../errors/errors";
 import { CommandRunMetricEventKey, type Context } from "../router";
 import { DefaultTelemetryClient } from "../telemetry";
+import { shouldRecordCommandFailure } from "../telemetry/commandOutcome";
 import { FileSystemSink } from "../telemetry/fileSystemSink";
 import type { AttributesOf, MetricName, TelemetryClient } from "../telemetry/types";
 import { TestGlobalConfigAccessor } from "./globalConfig";
@@ -77,7 +78,7 @@ export async function withTelemetry<T>(
     return await command(ctx.withValue(CommandRunMetricEventKey, event));
   } catch (cause) {
     const error = AgentCoreCLIError.fromError(cause);
-    if (error.exitCode !== 0) {
+    if (shouldRecordCommandFailure(error)) {
       event.setAttributes({
         exit_reason: "failure",
         error_name: error.name,