feat(harbor): task.yaml/experiment.yaml → Harbor export + coder-eval as a Harbor agent

.github/scripts/harbor_e2e.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+"""Harbor end-to-end smoke test.
+
+Exports a handful of coder-eval tasks to Harbor (`coder-eval export --format
+harbor`), runs each with `hb run -a coder_eval.harbor.agent:CoderEvalAgent`
+against real Docker, and asserts that:
+
+  - the verifier phase wrote `reward.json` with `reward == 1.0`
+  - the agent phase wrote a real ATIF `trajectory.json`
+  - both the agent and verifier phases left a `task.json` behind
+
+Invoked by `.github/workflows/harbor-e2e.yml` -- deliberately NOT part of
+`make test`: it shells out to a real `hb` CLI, real Docker builds, and (for
+the llm_judge scenario) a real model call, none of which belong in the fast
+unit-test suite.
+"""
+
+from __future__ import annotations
+
+import json
+import shutil
+import subprocess
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+WORK_DIR = REPO_ROOT / "tmp" / "harbor_e2e"
+AGENT_IMPORT_PATH = "coder_eval.harbor.agent:CoderEvalAgent"
+
+
+@dataclass(frozen=True)
+class Scenario:
+    """One (task.yaml, export flags) pair to round-trip through Harbor."""
+
+    name: str
+    task_file: Path
+    allow_credentials: bool = False
+
+
+SCENARIOS: list[Scenario] = [
+    # "tmpdir" baseline: plain docker driver, default coder-eval-agent image,
+    # no dockerfile_path / template_sources / llm_judge -- exercises the bare
+    # export -> CoderEvalAgent -> --workspace-dir -> verifier round trip.
+    Scenario("baseline", REPO_ROOT / "tests/harbor_e2e/fixtures/docker_baseline.yaml"),
+    # llm_judge: real model call inside the VERIFIER phase, not just the agent.
+    Scenario("llm_judge", REPO_ROOT / "tests/harbor_e2e/fixtures/llm_judge.yaml", allow_credentials=True),
+    # Custom (BYOD) Docker image via dockerfile_path -- reuses the in-tree
+    # byod_smoke_test task/image rather than duplicating it.
+    Scenario("docker_custom_image", REPO_ROOT / "tasks/byod_smoke_test.yaml"),
+    # template_sources: TemplateDirSource copy-in + rewritten path, plus
+    # sandbox.python.env_packages surviving the agent-phase task.yaml merge.
+    Scenario("template_sources", REPO_ROOT / "tests/harbor_e2e/fixtures/template_sources.yaml"),
+]
+
+
+def _run(cmd: list[str]) -> subprocess.CompletedProcess[str]:
+    print(f"+ {' '.join(cmd)}", flush=True)
+    return subprocess.run(cmd, check=False, text=True, capture_output=True)
+
+
+def export_task(scenario: Scenario, out_dir: Path) -> None:
+    if out_dir.exists():
+        shutil.rmtree(out_dir)
+    cmd = ["coder-eval", "export", str(scenario.task_file), "-o", str(out_dir)]
+    if scenario.allow_credentials:
+        cmd.append("--allow-credentials")
+    result = _run(cmd)
+    print(result.stdout)
+    print(result.stderr, file=sys.stderr)
+    if result.returncode != 0:
+        raise RuntimeError(f"[{scenario.name}] `coder-eval export` failed (exit {result.returncode})")
+
+
+def run_harbor(scenario: Scenario, export_dir: Path, jobs_dir: Path) -> Path:
+    if jobs_dir.exists():
+        shutil.rmtree(jobs_dir)
+    cmd = [
+        "hb",
+        "run",
+        "-p",
+        str(export_dir),
+        "-a",
+        AGENT_IMPORT_PATH,
+        "--jobs-dir",
+        str(jobs_dir),
+        "-n",
+        "1",
+        "-y",
+    ]
+    result = _run(cmd)
+    print(result.stdout)
+    print(result.stderr, file=sys.stderr)
+    if result.returncode != 0:
+        raise RuntimeError(f"[{scenario.name}] `hb run` failed (exit {result.returncode})")
+
+    # <jobs_dir>/<job_timestamp>/<trial_name>/{agent,verifier}/... -- glob for
+    # the one directory two levels down that actually holds a verifier/ output,
+    # rather than assuming a fixed trial-name shape Harbor doesn't guarantee.
+    trial_dirs = [d for d in jobs_dir.glob("*/*/") if (d / "verifier").is_dir()]
+    if len(trial_dirs) != 1:
+        raise RuntimeError(
+            f"[{scenario.name}] expected exactly one trial directory under {jobs_dir}, found {len(trial_dirs)}"
+        )
+    return trial_dirs[0]
+
+
+def assert_scenario_artifacts(scenario: Scenario, trial_dir: Path) -> None:
+    reward_path = trial_dir / "verifier" / "reward.json"
+    if not reward_path.is_file():
+        raise RuntimeError(f"[{scenario.name}] missing {reward_path}")
+    reward = json.loads(reward_path.read_text(encoding="utf-8"))
+    if reward.get("reward") != 1.0:
+        raise RuntimeError(f"[{scenario.name}] expected reward 1.0, got {reward!r} ({reward_path})")
+
+    trajectory_path = trial_dir / "agent" / "trajectory.json"
+    if not trajectory_path.is_file():
+        raise RuntimeError(f"[{scenario.name}] missing {trajectory_path}")
+    trajectory = json.loads(trajectory_path.read_text(encoding="utf-8"))
+    if "schema_version" not in trajectory:
+        raise RuntimeError(f"[{scenario.name}] {trajectory_path} is missing 'schema_version'")
+
+    agent_task_jsons = list((trial_dir / "agent").glob("**/task.json"))
+    if not agent_task_jsons:
+        raise RuntimeError(f"[{scenario.name}] no task.json found under {trial_dir / 'agent'}")
+    verifier_task_json = trial_dir / "verifier" / "task.json"
+    if not verifier_task_json.is_file():
+        raise RuntimeError(f"[{scenario.name}] missing {verifier_task_json}")
+
+    print(
+        f"[{scenario.name}] OK: reward=1.0, trajectory.json present, "
+        + f"{len(agent_task_jsons)} agent task.json + verifier/task.json present"
+    )
+
+
+def main() -> int:
+    WORK_DIR.mkdir(parents=True, exist_ok=True)
+    failures: list[str] = []
+    for scenario in SCENARIOS:
+        export_dir = WORK_DIR / scenario.name / "export"
+        jobs_dir = WORK_DIR / scenario.name / "jobs"
+        print(f"\n=== {scenario.name} ===", flush=True)
+        try:
+            export_task(scenario, export_dir)
+            trial_dir = run_harbor(scenario, export_dir, jobs_dir)
+            assert_scenario_artifacts(scenario, trial_dir)
+        except Exception as exc:
+            print(f"[{scenario.name}] FAILED: {exc}", file=sys.stderr)
+            failures.append(scenario.name)
+
+    print("\n=== Summary ===")
+    for scenario in SCENARIOS:
+        print(f"  {scenario.name}: {'FAILED' if scenario.name in failures else 'OK'}")
+
+    if failures:
+        print(f"\n{len(failures)}/{len(SCENARIOS)} scenario(s) failed: {', '.join(failures)}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/workflows/harbor-e2e.yml

@@ -0,0 +1,91 @@
+name: Harbor E2E
+
+# Deliberately NOT triggered on pull_request: this exercises real Docker
+# builds, a real `harbor` install, and (for the llm_judge scenario) a real
+# model call, so it is informational rather than a required PR check for now
+# (see .github/scripts/harbor_e2e.py's module docstring). workflow_dispatch
+# lets a maintainer run it on demand; the nightly schedule catches drift
+# between coder-eval's own release and Harbor's own upstream releases without
+# blocking anyone's PR.
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "17 5 * * *" # nightly, off the hour to avoid GitHub's peak-load pile-up
+  push:
+    branches: [main]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+env:
+  TELEMETRY_ENABLED: "false"
+  # Route through Bedrock, mirroring pr-checks.yml's smoke-pass job -- keeps
+  # Anthropic-credit spend off this path; DirectRoute is exercised elsewhere.
+  API_BACKEND: "bedrock"
+  CLAUDE_CODE_USE_BEDROCK: "1"
+  AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
+  AWS_REGION: ${{ secrets.AWS_REGION }}
+  BEDROCK_MODEL: ${{ secrets.BEDROCK_MODEL }}
+
+jobs:
+  harbor-e2e:
+    name: Harbor export + CoderEvalAgent round trip
+    runs-on: uipath-ubuntu-latest
+    timeout-minutes: 20
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+
+      - name: Set up Python 3.13
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
+        with:
+          python-version: "3.13"
+
+      - name: Set up Node.js 20
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020  # v4.4.0
+        with:
+          node-version: "20"
+
+      - name: Install Claude CLI
+        run: npm install -g @anthropic-ai/claude-code
+
+      - name: Cache dependencies
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5.0.5
+        with:
+          path: |
+            ~/.cache/uv
+            ~/.cache/pip
+          key: ${{ runner.os }}-py3.13-harbor-e2e-${{ hashFiles('pyproject.toml', 'uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-py3.13-harbor-e2e-
+
+      - name: Install uv
+        run: |
+          python -m pip install --upgrade "pip>=26.2"
+          pip install uv
+
+      - name: Install project dependencies (hash-verified from uv.lock)
+        # --extra harbor installs `harbor` into the SAME venv as coder-eval:
+        # CoderEvalAgent is resolved by `hb run -a
+        # coder_eval.harbor.agent:CoderEvalAgent` on the HOST process, so
+        # `harbor` and `coder_eval` must be importable from the same
+        # interpreter (see harbor/agent.py's module docstring). The version is
+        # pinned once, in pyproject.toml's `harbor` extra -- bump it there.
+        run: uv sync --frozen --extra dev --extra harbor
+
+      - name: Put the project venv on PATH
+        run: echo "${{ github.workspace }}/.venv/bin" >> "$GITHUB_PATH"
+
+      - name: Build coder-eval-agent base Docker image
+        run: make docker-image
+
+      - name: Build BYOD template Docker image
+        run: docker build -t byod-custom-image:0.1.0 templates/byod_smoke_test/
+
+      - name: Run Harbor E2E scenarios
+        run: python .github/scripts/harbor_e2e.py

.github/workflows/pr-checks.yml

@@ -80,7 +80,9 @@ jobs:
           pip install uv
 
       - name: Install project dependencies (hash-verified from uv.lock)
-        run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm
+        # --extra harbor: pyright below type-checks src/coder_eval/harbor/agent.py
+        # against harbor's real types, not a scoped ignore.
+        run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm --extra harbor
 
       # PHASE 1: Fast checks (fail early)
       - name: Check code formatting (ruff format)
@@ -385,7 +387,9 @@ jobs:
           pip install uv
 
       - name: Install project dependencies (hash-verified from uv.lock)
-        run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm
+        # --extra harbor: pyright below type-checks src/coder_eval/harbor/agent.py
+        # against harbor's real types, not a scoped ignore.
+        run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm --extra harbor
 
       - name: Check code formatting (ruff format)
         run: .venv/Scripts/ruff format --check src/ tests/

docker/Dockerfile

@@ -54,15 +54,19 @@ COPY src/ ./src/
 # experiments/default.yaml is force-included by hatchling per pyproject.toml.
 COPY experiments/default.yaml ./experiments/default.yaml
 
-# Codex, Antigravity, and litellm are always baked into the image -- codex/
-# antigravity are peers to the claude-code agent installed above; litellm backs
-# the `checker_context.api_route.route: litellm` judge dispatch (see
-# pyproject.toml's `litellm` extra), which every DockerRunner-isolated task
-# needs available IN-container since the checker runs there too, not just on the
-# host. `openai-codex` (+ its pinned cli-bin), `google-antigravity` (which
-# bundles its `localharness` binary as a manylinux wheel), and `litellm` all
+# Codex, Antigravity, litellm, and harbor are always baked into the image --
+# codex/antigravity are peers to the claude-code agent installed above;
+# litellm backs the `checker_context.api_route.route: litellm` judge dispatch
+# (see pyproject.toml's `litellm` extra), which every DockerRunner-isolated
+# task needs available IN-container since the checker runs there too, not
+# just on the host; harbor lets this same image double as a Harbor agent
+# image (`coder_eval.harbor.agent:CoderEvalAgent`, a
+# `harbor.agents.installed.base.BaseInstalledAgent` subclass -- see
+# src/coder_eval/harbor/agent.py) without a separate build. `openai-codex`
+# (+ its pinned cli-bin), `google-antigravity` (which bundles its
+# `localharness` binary as a manylinux wheel), `litellm`, and `harbor` all
 # come from public PyPI, so this needs no private-index credentials. The RUN
-# below always passes `--extra codex --extra antigravity --extra litellm`.
+# below always passes `--extra codex --extra antigravity --extra litellm --extra harbor`.
 #
 # `pi` IS baked above (pinned PI_VERSION) and its OPENROUTER_API_KEY provider
 # credential is in DockerDriverConfig.env_passthrough, so `--driver docker --type pi`
@@ -90,7 +94,7 @@ ARG SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS="openai-codex-cli-bin,openai-codex
 # All extras (codex, antigravity, litellm, and the opt-in uipath) resolve from
 # public PyPI per uv.lock, so the build needs no private-index credentials.
 RUN export SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS="${SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS}" && \
-    uv export --frozen --extra codex --extra antigravity --extra litellm ${CODER_EVAL_UV_EXTRAS} | uv pip install --system -r /dev/stdin
+    uv export --frozen --extra codex --extra antigravity --extra litellm --extra harbor ${CODER_EVAL_UV_EXTRAS} | uv pip install --system -r /dev/stdin
 
 # Sanity check: the in-container entrypoint subcommand must be wired up.
 RUN coder-eval _run-task-internal --help > /dev/null

pyproject.toml

@@ -30,6 +30,7 @@ dependencies = [
     "pydantic>=2.12.5",
     "pydantic-settings>=2.14.2",
     "pyyaml>=6.0.3",
+    "tomli-w>=1.0.0",
     "typer>=0.24.1",
     "click>=8.3.3",
     "rich>=14.3.3",
@@ -160,6 +161,23 @@ opencode = []
 # the CLI on PATH the framework still installs and runs; Pi tasks fail at start()
 # with a clear hint pointing here. See docs/agents/PI.md.
 pi = []
+# Optional extra for the Harbor-interop direction: coder-eval AS a Harbor agent
+# (`coder_eval.harbor.agent:CoderEvalAgent`, a `harbor.agents.installed.base.BaseInstalledAgent`
+# subclass) run via `harbor run -a coder_eval.harbor.agent:CoderEvalAgent`. `harbor` itself
+# is never imported by anything OUTSIDE `coder_eval.harbor` (the export/packager side of
+# the integration reads/writes Harbor's task.toml directly and needs no `harbor` import at
+# all), so this extra exists only to satisfy that one module -- a task-image author (or a CI
+# job driving `hb run`) installs it with `pip install 'coder-eval[harbor]'` into the SAME
+# interpreter that also has `coder_eval`, since Harbor resolves the agent class on the host
+# process (see harbor/agent.py's module docstring). Pinned exactly, mirroring the other
+# harness pins: Harbor's own trial/environment/agent-context models are a load-bearing
+# runtime contract for this class, not a loose API. Without this extra the framework still
+# installs and runs; `coder_eval.harbor.agent` fails to import with a clear hint pointing
+# back here (it is not needed to EXPORT a task to Harbor format, only to run coder-eval
+# itself as Harbor's agent).
+harbor = [
+    "harbor==0.22.0",
+]
 
 [project.scripts]
 coder-eval = "coder_eval.cli:app"

src/coder_eval/cli/__init__.py

@@ -8,6 +8,8 @@
 from .console import console
 from .evaluate_command import evaluate_command
 from .execute_command import execute_command
+from .export_command import export_command
+from .harbor_command import harbor_app, reward_command
 from .plan_command import plan_command
 from .report_command import report_command
 from .run_command import run_command
@@ -82,6 +84,9 @@ def main(
 app.command(name="evaluate")(track_command("evaluate")(evaluate_command))
 app.command(name="report")(track_command("report")(report_command))
 app.command(name="aggregate")(track_command("aggregate")(aggregate_command))
+app.command(name="export")(track_command("export")(export_command))
+harbor_app.command(name="reward")(track_command("harbor-reward")(reward_command))
+app.add_typer(harbor_app, name="harbor")
 # Hidden internal command invoked inside the Docker container only — UNWRAPPED
 # (it runs inside the run-task subprocess and would double-count / pollute events).
 app.command(name="_run-task-internal", hidden=True)(run_task_internal_command)