diff --git a/.github/scripts/harbor_e2e.py b/.github/scripts/harbor_e2e.py new file mode 100644 index 00000000..2dd436a2 --- /dev/null +++ b/.github/scripts/harbor_e2e.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""Harbor end-to-end smoke test. + +Exports a handful of coder-eval tasks to Harbor (`coder-eval export --format +harbor`), runs each with `hb run -a coder_eval.harbor.agent:CoderEvalAgent` +against real Docker, and asserts that: + + - the verifier phase wrote `reward.json` with `reward == 1.0` + - the agent phase wrote a real ATIF `trajectory.json` + - both the agent and verifier phases left a `task.json` behind + +Invoked by `.github/workflows/harbor-e2e.yml` -- deliberately NOT part of +`make test`: it shells out to a real `hb` CLI, real Docker builds, and (for +the llm_judge scenario) a real model call, none of which belong in the fast +unit-test suite. +""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +WORK_DIR = REPO_ROOT / "tmp" / "harbor_e2e" +AGENT_IMPORT_PATH = "coder_eval.harbor.agent:CoderEvalAgent" + + +@dataclass(frozen=True) +class Scenario: + """One (task.yaml, export flags) pair to round-trip through Harbor.""" + + name: str + task_file: Path + allow_credentials: bool = False + + +SCENARIOS: list[Scenario] = [ + # "tmpdir" baseline: plain docker driver, default coder-eval-agent image, + # no dockerfile_path / template_sources / llm_judge -- exercises the bare + # export -> CoderEvalAgent -> --workspace-dir -> verifier round trip. + Scenario("baseline", REPO_ROOT / "tests/harbor_e2e/fixtures/docker_baseline.yaml"), + # llm_judge: real model call inside the VERIFIER phase, not just the agent. + Scenario("llm_judge", REPO_ROOT / "tests/harbor_e2e/fixtures/llm_judge.yaml", allow_credentials=True), + # Custom (BYOD) Docker image via dockerfile_path -- reuses the in-tree + # byod_smoke_test task/image rather than duplicating it. + Scenario("docker_custom_image", REPO_ROOT / "tasks/byod_smoke_test.yaml"), + # template_sources: TemplateDirSource copy-in + rewritten path, plus + # sandbox.python.env_packages surviving the agent-phase task.yaml merge. + Scenario("template_sources", REPO_ROOT / "tests/harbor_e2e/fixtures/template_sources.yaml"), +] + + +def _run(cmd: list[str]) -> subprocess.CompletedProcess[str]: + print(f"+ {' '.join(cmd)}", flush=True) + return subprocess.run(cmd, check=False, text=True, capture_output=True) + + +def export_task(scenario: Scenario, out_dir: Path) -> None: + if out_dir.exists(): + shutil.rmtree(out_dir) + cmd = ["coder-eval", "export", str(scenario.task_file), "-o", str(out_dir)] + if scenario.allow_credentials: + cmd.append("--allow-credentials") + result = _run(cmd) + print(result.stdout) + print(result.stderr, file=sys.stderr) + if result.returncode != 0: + raise RuntimeError(f"[{scenario.name}] `coder-eval export` failed (exit {result.returncode})") + + +def run_harbor(scenario: Scenario, export_dir: Path, jobs_dir: Path) -> Path: + if jobs_dir.exists(): + shutil.rmtree(jobs_dir) + cmd = [ + "hb", + "run", + "-p", + str(export_dir), + "-a", + AGENT_IMPORT_PATH, + "--jobs-dir", + str(jobs_dir), + "-n", + "1", + "-y", + ] + result = _run(cmd) + print(result.stdout) + print(result.stderr, file=sys.stderr) + if result.returncode != 0: + raise RuntimeError(f"[{scenario.name}] `hb run` failed (exit {result.returncode})") + + # ///{agent,verifier}/... -- glob for + # the one directory two levels down that actually holds a verifier/ output, + # rather than assuming a fixed trial-name shape Harbor doesn't guarantee. + trial_dirs = [d for d in jobs_dir.glob("*/*/") if (d / "verifier").is_dir()] + if len(trial_dirs) != 1: + raise RuntimeError( + f"[{scenario.name}] expected exactly one trial directory under {jobs_dir}, found {len(trial_dirs)}" + ) + return trial_dirs[0] + + +def assert_scenario_artifacts(scenario: Scenario, trial_dir: Path) -> None: + reward_path = trial_dir / "verifier" / "reward.json" + if not reward_path.is_file(): + raise RuntimeError(f"[{scenario.name}] missing {reward_path}") + reward = json.loads(reward_path.read_text(encoding="utf-8")) + if reward.get("reward") != 1.0: + raise RuntimeError(f"[{scenario.name}] expected reward 1.0, got {reward!r} ({reward_path})") + + trajectory_path = trial_dir / "agent" / "trajectory.json" + if not trajectory_path.is_file(): + raise RuntimeError(f"[{scenario.name}] missing {trajectory_path}") + trajectory = json.loads(trajectory_path.read_text(encoding="utf-8")) + if "schema_version" not in trajectory: + raise RuntimeError(f"[{scenario.name}] {trajectory_path} is missing 'schema_version'") + + agent_task_jsons = list((trial_dir / "agent").glob("**/task.json")) + if not agent_task_jsons: + raise RuntimeError(f"[{scenario.name}] no task.json found under {trial_dir / 'agent'}") + verifier_task_json = trial_dir / "verifier" / "task.json" + if not verifier_task_json.is_file(): + raise RuntimeError(f"[{scenario.name}] missing {verifier_task_json}") + + print( + f"[{scenario.name}] OK: reward=1.0, trajectory.json present, " + + f"{len(agent_task_jsons)} agent task.json + verifier/task.json present" + ) + + +def main() -> int: + WORK_DIR.mkdir(parents=True, exist_ok=True) + failures: list[str] = [] + for scenario in SCENARIOS: + export_dir = WORK_DIR / scenario.name / "export" + jobs_dir = WORK_DIR / scenario.name / "jobs" + print(f"\n=== {scenario.name} ===", flush=True) + try: + export_task(scenario, export_dir) + trial_dir = run_harbor(scenario, export_dir, jobs_dir) + assert_scenario_artifacts(scenario, trial_dir) + except Exception as exc: + print(f"[{scenario.name}] FAILED: {exc}", file=sys.stderr) + failures.append(scenario.name) + + print("\n=== Summary ===") + for scenario in SCENARIOS: + print(f" {scenario.name}: {'FAILED' if scenario.name in failures else 'OK'}") + + if failures: + print(f"\n{len(failures)}/{len(SCENARIOS)} scenario(s) failed: {', '.join(failures)}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/harbor-e2e.yml b/.github/workflows/harbor-e2e.yml new file mode 100644 index 00000000..d11933d3 --- /dev/null +++ b/.github/workflows/harbor-e2e.yml @@ -0,0 +1,91 @@ +name: Harbor E2E + +# Deliberately NOT triggered on pull_request: this exercises real Docker +# builds, a real `harbor` install, and (for the llm_judge scenario) a real +# model call, so it is informational rather than a required PR check for now +# (see .github/scripts/harbor_e2e.py's module docstring). workflow_dispatch +# lets a maintainer run it on demand; the nightly schedule catches drift +# between coder-eval's own release and Harbor's own upstream releases without +# blocking anyone's PR. +on: + workflow_dispatch: + schedule: + - cron: "17 5 * * *" # nightly, off the hour to avoid GitHub's peak-load pile-up + push: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + TELEMETRY_ENABLED: "false" + # Route through Bedrock, mirroring pr-checks.yml's smoke-pass job -- keeps + # Anthropic-credit spend off this path; DirectRoute is exercised elsewhere. + API_BACKEND: "bedrock" + CLAUDE_CODE_USE_BEDROCK: "1" + AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }} + AWS_REGION: ${{ secrets.AWS_REGION }} + BEDROCK_MODEL: ${{ secrets.BEDROCK_MODEL }} + +jobs: + harbor-e2e: + name: Harbor export + CoderEvalAgent round trip + runs-on: uipath-ubuntu-latest + timeout-minutes: 20 + + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Python 3.13 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.13" + + - name: Set up Node.js 20 + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "20" + + - name: Install Claude CLI + run: npm install -g @anthropic-ai/claude-code + + - name: Cache dependencies + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: | + ~/.cache/uv + ~/.cache/pip + key: ${{ runner.os }}-py3.13-harbor-e2e-${{ hashFiles('pyproject.toml', 'uv.lock') }} + restore-keys: | + ${{ runner.os }}-py3.13-harbor-e2e- + + - name: Install uv + run: | + python -m pip install --upgrade "pip>=26.2" + pip install uv + + - name: Install project dependencies (hash-verified from uv.lock) + # --extra harbor installs `harbor` into the SAME venv as coder-eval: + # CoderEvalAgent is resolved by `hb run -a + # coder_eval.harbor.agent:CoderEvalAgent` on the HOST process, so + # `harbor` and `coder_eval` must be importable from the same + # interpreter (see harbor/agent.py's module docstring). The version is + # pinned once, in pyproject.toml's `harbor` extra -- bump it there. + run: uv sync --frozen --extra dev --extra harbor + + - name: Put the project venv on PATH + run: echo "${{ github.workspace }}/.venv/bin" >> "$GITHUB_PATH" + + - name: Build coder-eval-agent base Docker image + run: make docker-image + + - name: Build BYOD template Docker image + run: docker build -t byod-custom-image:0.1.0 templates/byod_smoke_test/ + + - name: Run Harbor E2E scenarios + run: python .github/scripts/harbor_e2e.py diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 58c450cd..cd0150be 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -80,7 +80,9 @@ jobs: pip install uv - name: Install project dependencies (hash-verified from uv.lock) - run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm + # --extra harbor: pyright below type-checks src/coder_eval/harbor/agent.py + # against harbor's real types, not a scoped ignore. + run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm --extra harbor # PHASE 1: Fast checks (fail early) - name: Check code formatting (ruff format) @@ -385,7 +387,9 @@ jobs: pip install uv - name: Install project dependencies (hash-verified from uv.lock) - run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm + # --extra harbor: pyright below type-checks src/coder_eval/harbor/agent.py + # against harbor's real types, not a scoped ignore. + run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm --extra harbor - name: Check code formatting (ruff format) run: .venv/Scripts/ruff format --check src/ tests/ diff --git a/docker/Dockerfile b/docker/Dockerfile index 99467fac..e3ff04ab 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -54,15 +54,19 @@ COPY src/ ./src/ # experiments/default.yaml is force-included by hatchling per pyproject.toml. COPY experiments/default.yaml ./experiments/default.yaml -# Codex, Antigravity, and litellm are always baked into the image -- codex/ -# antigravity are peers to the claude-code agent installed above; litellm backs -# the `checker_context.api_route.route: litellm` judge dispatch (see -# pyproject.toml's `litellm` extra), which every DockerRunner-isolated task -# needs available IN-container since the checker runs there too, not just on the -# host. `openai-codex` (+ its pinned cli-bin), `google-antigravity` (which -# bundles its `localharness` binary as a manylinux wheel), and `litellm` all +# Codex, Antigravity, litellm, and harbor are always baked into the image -- +# codex/antigravity are peers to the claude-code agent installed above; +# litellm backs the `checker_context.api_route.route: litellm` judge dispatch +# (see pyproject.toml's `litellm` extra), which every DockerRunner-isolated +# task needs available IN-container since the checker runs there too, not +# just on the host; harbor lets this same image double as a Harbor agent +# image (`coder_eval.harbor.agent:CoderEvalAgent`, a +# `harbor.agents.installed.base.BaseInstalledAgent` subclass -- see +# src/coder_eval/harbor/agent.py) without a separate build. `openai-codex` +# (+ its pinned cli-bin), `google-antigravity` (which bundles its +# `localharness` binary as a manylinux wheel), `litellm`, and `harbor` all # come from public PyPI, so this needs no private-index credentials. The RUN -# below always passes `--extra codex --extra antigravity --extra litellm`. +# below always passes `--extra codex --extra antigravity --extra litellm --extra harbor`. # # `pi` IS baked above (pinned PI_VERSION) and its OPENROUTER_API_KEY provider # credential is in DockerDriverConfig.env_passthrough, so `--driver docker --type pi` @@ -90,7 +94,7 @@ ARG SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS="openai-codex-cli-bin,openai-codex # All extras (codex, antigravity, litellm, and the opt-in uipath) resolve from # public PyPI per uv.lock, so the build needs no private-index credentials. RUN export SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS="${SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS}" && \ - uv export --frozen --extra codex --extra antigravity --extra litellm ${CODER_EVAL_UV_EXTRAS} | uv pip install --system -r /dev/stdin + uv export --frozen --extra codex --extra antigravity --extra litellm --extra harbor ${CODER_EVAL_UV_EXTRAS} | uv pip install --system -r /dev/stdin # Sanity check: the in-container entrypoint subcommand must be wired up. RUN coder-eval _run-task-internal --help > /dev/null diff --git a/pyproject.toml b/pyproject.toml index b1e08974..e2253204 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ "pydantic>=2.12.5", "pydantic-settings>=2.14.2", "pyyaml>=6.0.3", + "tomli-w>=1.0.0", "typer>=0.24.1", "click>=8.3.3", "rich>=14.3.3", @@ -160,6 +161,23 @@ opencode = [] # the CLI on PATH the framework still installs and runs; Pi tasks fail at start() # with a clear hint pointing here. See docs/agents/PI.md. pi = [] +# Optional extra for the Harbor-interop direction: coder-eval AS a Harbor agent +# (`coder_eval.harbor.agent:CoderEvalAgent`, a `harbor.agents.installed.base.BaseInstalledAgent` +# subclass) run via `harbor run -a coder_eval.harbor.agent:CoderEvalAgent`. `harbor` itself +# is never imported by anything OUTSIDE `coder_eval.harbor` (the export/packager side of +# the integration reads/writes Harbor's task.toml directly and needs no `harbor` import at +# all), so this extra exists only to satisfy that one module -- a task-image author (or a CI +# job driving `hb run`) installs it with `pip install 'coder-eval[harbor]'` into the SAME +# interpreter that also has `coder_eval`, since Harbor resolves the agent class on the host +# process (see harbor/agent.py's module docstring). Pinned exactly, mirroring the other +# harness pins: Harbor's own trial/environment/agent-context models are a load-bearing +# runtime contract for this class, not a loose API. Without this extra the framework still +# installs and runs; `coder_eval.harbor.agent` fails to import with a clear hint pointing +# back here (it is not needed to EXPORT a task to Harbor format, only to run coder-eval +# itself as Harbor's agent). +harbor = [ + "harbor==0.22.0", +] [project.scripts] coder-eval = "coder_eval.cli:app" diff --git a/src/coder_eval/cli/__init__.py b/src/coder_eval/cli/__init__.py index 66eaddae..62eda48a 100644 --- a/src/coder_eval/cli/__init__.py +++ b/src/coder_eval/cli/__init__.py @@ -8,6 +8,8 @@ from .console import console from .evaluate_command import evaluate_command from .execute_command import execute_command +from .export_command import export_command +from .harbor_command import harbor_app, reward_command from .plan_command import plan_command from .report_command import report_command from .run_command import run_command @@ -82,6 +84,9 @@ def main( app.command(name="evaluate")(track_command("evaluate")(evaluate_command)) app.command(name="report")(track_command("report")(report_command)) app.command(name="aggregate")(track_command("aggregate")(aggregate_command)) +app.command(name="export")(track_command("export")(export_command)) +harbor_app.command(name="reward")(track_command("harbor-reward")(reward_command)) +app.add_typer(harbor_app, name="harbor") # Hidden internal command invoked inside the Docker container only — UNWRAPPED # (it runs inside the run-task subprocess and would double-count / pollute events). app.command(name="_run-task-internal", hidden=True)(run_task_internal_command) diff --git a/src/coder_eval/cli/evaluate_command.py b/src/coder_eval/cli/evaluate_command.py index f8ba3f45..d8f47ad5 100644 --- a/src/coder_eval/cli/evaluate_command.py +++ b/src/coder_eval/cli/evaluate_command.py @@ -11,6 +11,8 @@ from rich.markup import escape from ..evaluation.judge_persistence import TASK_JSON_TRANSCRIPT_EXCLUDE +from ..harbor.atif_hydrate import seed_from_atif_trajectory +from ..harbor.atif_models import Trajectory from ..logging_config import setup_logging from ..models import ( AgentKind, @@ -311,6 +313,23 @@ def evaluate_command( "--run-dir", help="Where the graded task.json lands (default: auto-generated timestamped directory in runs/)", ), + format: str | None = typer.Option( + None, + "--format", + help=( + "Only 'harbor' is supported: grade a directory whose agent phase ran OUTSIDE this " + "process (a Harbor agent's `coder-eval execute --format harbor`) by hydrating trajectory " + "context from an ATIF trajectory.json instead of a run directory's task.json. Requires " + "--trajectory and the two-argument `TASK_FILE WORK_DIR` form." + ), + ), + trajectory: Path | None = typer.Option( # noqa: B008 + None, + "--trajectory", + help="Path to an ATIF trajectory.json to hydrate trajectory context from. Required with --format harbor.", + exists=True, + dir_okay=False, + ), ) -> None: """Evaluate criteria against a directory, or re-grade a finished run. @@ -343,6 +362,8 @@ def evaluate_command( allow_recorded_commands=allow_recorded_commands, allow_host_grading=allow_host_grading, run_dir=run_dir, + format=format, + trajectory=trajectory, ) @@ -357,6 +378,8 @@ def run_evaluation( allow_recorded_commands: bool = False, allow_host_grading: bool = False, run_dir: Path | None = None, + format: str | None = None, + trajectory: Path | None = None, ) -> None: """The body of ``coder-eval evaluate``, with real Python defaults. @@ -384,6 +407,43 @@ def run_evaluation( prior = inputs.prior target = inputs.target + if format is not None and format != "harbor": + console.print(f"[red]✗ Unsupported --format {format!r}. Supported: harbor.[/red]") + raise typer.Exit(1) + if format == "harbor": + if target.mode is not EvaluateMode.WORK_DIR: + console.print( + "[red]✗ --format harbor only applies to the two-argument `TASK_FILE WORK_DIR` form — " + + "a run directory already carries its own trajectory in task.json.[/red]" + ) + raise typer.Exit(1) + if trajectory is None: + console.print("[red]✗ --format harbor requires --trajectory .[/red]") + raise typer.Exit(1) + try: + atif_trajectory = Trajectory.model_validate_json(trajectory.read_text(encoding="utf-8")) + except (OSError, ValueError) as e: + console.print(f"[red]✗ Could not read {trajectory} as an ATIF trajectory:[/red] {escape(str(e))}") + raise typer.Exit(1) from e + prior = seed_from_atif_trajectory( + atif_trajectory, + task_id=task.task_id, + task_description=task.description, + ) + + if not task.success_criteria: + # `evaluate` always grades -- unlike `execute`, there is no legal reason + # for a zero-criteria task to reach here. `regrade_in_place` guards its + # own delegating branch; this guard covers the sibling orchestrator-direct + # branch below (fresh work-dir grading, or `--copy`), which never calls + # `regrade_in_place` and would otherwise finalize a criteria-free task as + # SUCCESS at weighted_score 0.0. + console.print( + f"[red]✗ Task {task.task_id!r} has no `success_criteria` and cannot be graded " + + "(it would silently score SUCCESS at weighted_score 0.0). Add at least one criterion.[/red]" + ) + raise typer.Exit(1) + grade_in_place = resolve_grade_in_place(target, in_place) try: @@ -565,7 +625,15 @@ def _report_and_exit( if result.sandbox_path: console.print(f"[dim]Artifacts: {result.sandbox_path}[/dim]") - if prior is not None: + # `prior is not None` alone is not enough: `--format harbor` seeds a + # SYNTHETIC prior on the WORK_DIR shape (from the supplied + # `--trajectory`), which is not a run directory and carries no + # `task.execute.json` sibling to preserve. `_write_back` is documented as + # "replace the graded RUN's task.json" and writes into `target.target`, + # which in WORK_DIR mode is the directory being graded, not a run dir -- + # writing there planted a spurious task.json into the Harbor-synced + # workdir and wedged a later `evaluate` on it into RUN_DIR mode. + if prior is not None and target.mode is EvaluateMode.RUN_DIR: console.print( f"[dim]Re-graded {prior.final_status.value} → {result.final_status.value} " + f"over {len(result.iterations)} recorded turn(s).[/dim]" diff --git a/src/coder_eval/cli/execute_command.py b/src/coder_eval/cli/execute_command.py index 15bbdc2a..8047dcdb 100644 --- a/src/coder_eval/cli/execute_command.py +++ b/src/coder_eval/cli/execute_command.py @@ -181,6 +181,29 @@ def execute_command( "-D sandbox.driver.)" ), ), + format: str | None = typer.Option( + None, + "--format", + help=( + "Emit an additional interchange trajectory alongside task.json. Only 'harbor' is " + "supported: writes a sibling trajectory.json (ATIF format) for every task, so an " + "external `coder-eval evaluate --format harbor` invocation can grade the trajectory " + "without access to this process's task.json. This is the flag a Harbor agent's " + "`coder-eval execute --format harbor --run-dir ` invocation passes." + ), + ), + workspace_dir: Path | None = typer.Option( # noqa: B008 + None, + "--workspace-dir", + help=( + "Run the single resolved task's agent in-place at this absolute path instead of the " + "standard run_dir/artifacts workspace (copied out to run_dir/artifacts/ at " + "cleanup). Requires exactly one resolved task; refused for sandbox.driver: docker " + "(the docker driver already aligns automatically via sandbox.docker.working_dir). " + "Meant for a Harbor `CoderEvalAgent` invocation, so the agent's writes land at the " + "container's own WORKDIR, where Harbor's verifier phase looks for them." + ), + ), ) -> None: """Run evaluation tasks WITHOUT checking their success criteria. @@ -233,4 +256,6 @@ def execute_command( repeats=repeats, driver=driver, set_overrides=set_overrides, + format=format, + workspace_dir=workspace_dir, ) diff --git a/src/coder_eval/cli/export_command.py b/src/coder_eval/cli/export_command.py new file mode 100644 index 00000000..5afd69de --- /dev/null +++ b/src/coder_eval/cli/export_command.py @@ -0,0 +1,126 @@ +"""``coder-eval export`` — emit a task in another framework's native format. + +Currently one target: ``--format harbor`` (C2). This is a *writer* only — +``--format`` is reserved so the same flag can later grow a *reader* +(``coder-eval run ``, demoted — see ``tmp/harborframework.partB.md``) +without a naming collision. + +Two modes, told apart by whether ``-e/--experiment`` is passed: + +- Single task.yaml → one Harbor task directory at ``-o``, unchanged from C2. +- One or more task.yaml files + ``-e experiment.yaml`` → the experiment's own + ``resolve_all_tasks`` pipeline resolves every (task, variant, replicate[, + dataset row]) combination, and each is written to its own subdirectory under + ``-o`` (``//[/]rep/``). See + ``tmp/harborframework_conversion.md`` for what an experiment-introduced + override this export cannot honor looks like, and why. +""" + +from __future__ import annotations + +from pathlib import Path + +import typer + +from ..harbor.experiment_packager import export_experiment +from ..harbor.packager import CriteriaNotExportableError, TaskNotExportableError, export_task +from .console import console +from .run_helpers import expand_task_files + + +_SUPPORTED_FORMATS = ("harbor",) + + +def export_command( + task_files: list[Path] = typer.Argument( # noqa: B008 + ..., + help="The coder-eval task YAML(s) to export (glob patterns allowed with --experiment).", + ), + output_dir: Path = typer.Option( # noqa: B008 + ..., + "--output", + "-o", + help="Directory to write the exported task(s) into (created if missing).", + file_okay=False, + ), + experiment: Path | None = typer.Option( # noqa: B008 + None, + "--experiment", + "-e", + help=( + "Experiment definition YAML. Resolves every (task, variant, replicate[, dataset row]) " + "combination via the same pipeline `coder-eval run -e` uses, and exports each as its own " + "Harbor directory under --output." + ), + exists=True, + dir_okay=False, + ), + format: str = typer.Option( + "harbor", + "--format", + help=f"Target format. Supported: {', '.join(_SUPPORTED_FORMATS)}.", + ), + allow_credentials: bool = typer.Option( + False, + "--allow-credentials", + help=( + "Export criteria that need model credentials/network inside the verifier " + "(llm_judge / agent_judge / uipath_eval) anyway. Only pass this if you have " + "already provisioned that access yourself — the export does not do it for you." + ), + ), +) -> None: + """Export a coder-eval task (or task x experiment.yaml variants) to another framework's directory format. + + Examples: + coder-eval export tasks/my_task.yaml -o dist/harbor/my_task --format harbor + coder-eval export tasks/my_task.yaml -e experiments/model-comparison.yaml -o dist/harbor/my_experiment + """ + if format not in _SUPPORTED_FORMATS: + console.print(f"[red]✗[/] Unsupported --format {format!r}. Supported: {', '.join(_SUPPORTED_FORMATS)}.") + raise typer.Exit(1) + + if experiment is None: + if len(task_files) != 1: + console.print("[red]✗[/] Without --experiment, pass exactly one task YAML.") + raise typer.Exit(1) + try: + result = export_task(task_files[0], output_dir, allow_credentials=allow_credentials) + except (TaskNotExportableError, CriteriaNotExportableError) as e: + console.print(f"[red]✗[/] {e}") + raise typer.Exit(1) from e + console.print(f"[green]✓[/] Exported {task_files[0]} → {result.out_dir} (format: {format})") + for warning in result.warnings: + console.print(f"[yellow]⚠[/] {warning}") + return + + all_task_files = expand_task_files(task_files) + exp_result = export_experiment( + all_task_files, + experiment, + output_dir, + allow_credentials=allow_credentials, + ) + + for exported in exp_result.exported: + console.print(f"[green]✓[/] Exported → {exported.out_dir} (format: {format})") + for warning in exported.warnings: + console.print(f"[yellow]⚠[/] {warning}") + for skip in exp_result.skipped: + console.print( + f"[yellow]⚠[/] Skipped variant={skip.variant_id!r} task={skip.task_id!r} " + + f"rep={skip.replicate_index}: {skip.reason}" + ) + for load_skip in exp_result.load_skipped: + console.print(f"[yellow]⚠[/] Skipped task load: {load_skip}") + + console.print( + f"[bold]{len(exp_result.exported)}[/] directories exported, " + + f"[bold]{len(exp_result.skipped)}[/] variant(s) skipped, " + + f"[bold]{len(exp_result.load_skipped)}[/] task file(s) skipped at load." + ) + if not exp_result.exported: + raise typer.Exit(1) + + +__all__ = ["export_command"] diff --git a/src/coder_eval/cli/harbor_command.py b/src/coder_eval/cli/harbor_command.py new file mode 100644 index 00000000..17c68ffd --- /dev/null +++ b/src/coder_eval/cli/harbor_command.py @@ -0,0 +1,65 @@ +"""``coder-eval harbor`` — Harbor framework adherence commands. + +Distinct from task *definition* (``coder-eval export --format harbor``, +tracked separately): this namespace is for what an outer harness's runtime +contract expects — right now, the verifier's reward file. See +``coder_eval.harbor`` for the shared logic and ``tmp/harborframework.md`` for +the design this implements (C1.1). +""" + +from __future__ import annotations + +from pathlib import Path + +import typer + +from ..harbor.reward import RegradeError, RewardWriteSkippedError, write_reward +from .console import console + + +harbor_app = typer.Typer( + name="harbor", + help="Harbor framework adherence commands (reward reporting, log-layout shims).", + add_completion=False, +) + + +def reward_command( + run_dir: Path = typer.Argument( # noqa: B008 + ..., + help="A graded coder-eval run directory (holds task.json) — typically the " + + "--run-dir passed to the preceding `coder-eval evaluate`.", + exists=True, + file_okay=False, + ), + out: Path = typer.Option( # noqa: B008 + ..., + "--out", + help="Where to write Harbor's reward file, e.g. /logs/verifier/reward.json.", + ), +) -> None: + """Translate a graded run's ``task.json`` into Harbor's ``reward.json`` contract. + + Writes NOTHING and exits non-zero when the row carries no measured verdict + (``weighted_score`` is ``None`` — an ungraded or crashed row) or when + ``task.json`` itself is missing/unparseable. That is deliberate: Harbor's + own verifier already treats a missing reward file as an infrastructure + failure distinct from a measured zero (see ``coder_eval.harbor.reward``), + so writing ``{"reward": 0.0}`` here would silently convert "never + measured" into "measured and scored zero" — the exact defect CE049 guards + against one layer up, in this same shape at the artifact boundary. + + Typical use, from a Harbor task's ``tests/test.sh``: + + coder-eval evaluate tests/task.yaml "$WORKDIR" --in-place --run-dir /logs/verifier + coder-eval harbor reward /logs/verifier --out /logs/verifier/reward.json + """ + try: + rewards = write_reward(run_dir, out) + except (RewardWriteSkippedError, RegradeError) as e: + console.print(f"[yellow]⚠[/] {e}") + raise typer.Exit(code=1) from e + console.print(f"[green]✓[/] wrote {out}: {rewards}") + + +__all__ = ["harbor_app", "reward_command"] diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py index d6994a1e..8e234de7 100644 --- a/src/coder_eval/cli/run_command.py +++ b/src/coder_eval/cli/run_command.py @@ -3,6 +3,7 @@ import asyncio import logging import os +import shutil import sys import urllib.error import urllib.parse @@ -354,6 +355,29 @@ def run_command( "-D sandbox.driver.)" ), ), + format: str | None = typer.Option( + None, + "--format", + help=( + "Emit an additional interchange trajectory alongside task.json. Only 'harbor' is " + "supported: writes a sibling trajectory.json (ATIF format) for every task, so an " + "external `coder-eval evaluate --format harbor` invocation can grade the trajectory " + "without access to this process's task.json. Meant for `coder-eval execute --format " + "harbor --run-dir `, invoked by a Harbor agent." + ), + ), + workspace_dir: Path | None = typer.Option( # noqa: B008 + None, + "--workspace-dir", + help=( + "Run the single resolved task's agent in-place at this absolute path instead of the " + "standard run_dir/artifacts workspace (copied out to run_dir/artifacts/ at " + "cleanup). Requires exactly one resolved task; refused for sandbox.driver: docker " + "(the docker driver already aligns automatically via sandbox.docker.working_dir). " + "Meant for a Harbor `CoderEvalAgent` invocation, so the agent's writes land at the " + "container's own WORKDIR, where Harbor's verifier phase looks for them." + ), + ), ) -> None: """Run evaluation tasks (optionally in parallel). @@ -404,6 +428,8 @@ def run_command( repeats=repeats, driver=driver, set_overrides=set_overrides, + format=format, + workspace_dir=workspace_dir, ) @@ -432,6 +458,8 @@ def run_pipeline( repeats: int | None, driver: str | None, set_overrides: list[str], + format: str | None = None, + workspace_dir: Path | None = None, ) -> None: """The shared body of ``coder-eval run`` and ``coder-eval execute``. @@ -441,6 +469,8 @@ def run_pipeline( commands are pure flag-parsing wrappers over this function, so a behavior change can never apply to one and miss the other. """ + if format is not None and format != "harbor": + raise typer.BadParameter(f"Unsupported --format {format!r}. Supported: harbor.") # --resume needs an explicit run dir to resume into (auto-generated dirs are always fresh). if resume and run_dir is None: raise typer.BadParameter("--resume requires --run-dir pointing at the run to continue.") @@ -513,6 +543,8 @@ def run_pipeline( include_skipped=include_skipped, junit_xml=junit_xml, grade=grade, + format=format, + workspace_dir=workspace_dir, ) ) except KeyboardInterrupt: @@ -540,6 +572,8 @@ async def _run_all_tasks( include_skipped: bool = False, junit_xml: Path | None = None, grade: bool = True, + format: str | None = None, + workspace_dir: Path | None = None, ) -> None: """Async entry point for running all tasks (optionally in parallel). @@ -561,6 +595,22 @@ async def _run_all_tasks( junit_xml: Optional path to write a JUnit XML report to, after the run summary is persisted and before the failure exit-code gate. grade: False for `coder-eval execute` — run and capture, score nothing. + format: 'harbor' writes a trajectory.json (ATIF) sibling for every + task.json once the run finishes — see `harbor.atif_emit.emit_trajectories_for_run`. + When the run wrote exactly ONE trajectory (the shape a `CoderEvalAgent` + Harbor agent invocation always produces — one fixed-path agent-phase + task.yaml, no dataset/experiment fan-out), it is additionally copied to + `/trajectory.json` so a caller that pointed `--run-dir` at a + fixed discovery path (e.g. Harbor's `self.logs_dir`) can find it there + without knowing coder-eval's internal `///` + nesting. Multi-task runs are left nested only — there is no single + trajectory to promote. + workspace_dir: Run the single resolved task's agent in-place at this path + instead of run_dir/artifacts (see `BatchRunConfig.workspace_dir` and + `Orchestrator.workspace_dir`). Meant for a `CoderEvalAgent` invocation + inside a container someone else already built (Harbor's), so the + agent's writes land where that container's own verifier looks for + them, rather than in a throwaway tempdir the verifier never sees. """ # Prepare run directory run_dir = prepare_run_directory(run_dir) @@ -587,6 +637,7 @@ async def _run_all_tasks( verbose=verbose, include_skipped=include_skipped, grade=grade, + workspace_dir=workspace_dir, ) from ..telemetry import flush_telemetry, track_event @@ -632,6 +683,17 @@ async def _run_all_tasks( aggregate_task_logs(run_dir) + if format == "harbor": + from ..harbor.atif_emit import emit_trajectories_for_run + + written = emit_trajectories_for_run(run_dir) + console.print(f"[dim]Wrote {len(written)} trajectory.json (ATIF) file(s) under {run_dir}[/dim]") + + flat_trajectory_path = run_dir / "trajectory.json" + if len(written) == 1 and written[0] != flat_trajectory_path: + await asyncio.to_thread(shutil.copy2, written[0], flat_trajectory_path) + console.print(f"[dim]Copied the single trajectory to {flat_trajectory_path}[/dim]") + # Print execution summary print_execution_summary(run_dir, summary) @@ -880,6 +942,13 @@ async def _apply_resume( # to_grade is deliberately NOT cleared: its artifacts are the run's output # and the very thing being graded. cleared = clear_rerun_artifacts(part.to_run) + # `to_grade` rows are about to be graded by `_grade_resumed_tasks` below + # (which delegates to `regrade_in_place`), so the same refusal `to_run` + # gets via `_reject_empty_criteria_under_grade` applies here too -- checked + # explicitly rather than relying solely on `regrade_in_place`'s own guard + # so the whole batch is refused up front (exit 2) instead of one row at a + # time turning into a per-task "could not grade" warning mid-resume. + _reject_empty_criteria_under_grade(part.to_grade, grade=grade) console.print( f"[cyan]↻ Resume:[/] {len(prior_results)} task(s) already complete, " + f"running {len(part.to_run)} remaining" @@ -918,6 +987,40 @@ def _reject_simulation_under_execute(resolved: list[ResolvedTask], *, grade: boo ) +def _reject_empty_criteria_under_grade(resolved: list[ResolvedTask], *, grade: bool) -> None: + """Refuse a task with zero ``success_criteria`` under ``run``/``evaluate`` rather than scoring it. + + ``TaskDefinition.success_criteria`` accepts an empty list at the model level + (needed so the Harbor agent-phase ``task.yaml`` -- criteria-free by design, + see ``harbor/packager.py::_write_agent_phase_task_yaml`` -- can round-trip + through ``coder-eval execute``, which never grades). But `EvaluationResult`'s + scoring is vacuous over an empty list: `all_criteria_passed` returns `True` + and `calculate_weighted_score` returns `0.0`, so a criteria-free task graded + under `run` would silently finalize as `FinalStatus.SUCCESS` with + `weighted_score: 0.0` -- an internally contradictory "successful" result for + what is actually a misconfigured task (a typo, a bad merge, a `-D` override + that cleared the list). `execute` (`grade=False`) is exactly the case this + is legal for, so the check is scoped to `grade` the same way + ``_reject_simulation_under_execute`` scopes its own check. + + Callers must pass the POST-`--resume` set (``to_run``, not the full + ``resolved``): a resumed, already-finalized row is folded back from + ``prior_results`` and never re-executed or re-graded, so its own + (possibly empty) criteria are moot to this run and must not block one + that is not actually going to grade it. + """ + if not grade: + return + empty = sorted(rt.task.task_id for rt in resolved if not rt.task.success_criteria) + if empty: + raise typer.BadParameter( + "task(s) with no `success_criteria` cannot be graded (they would silently score " + + "SUCCESS at weighted_score 0.0): " + + ", ".join(empty) + + ". Add at least one criterion, or use `coder-eval execute` to run without grading." + ) + + async def _run_with_experiment( all_task_files: list[Path], config: BatchRunConfig, @@ -1036,21 +1139,35 @@ async def _run_with_experiment( resolved, grade=grade, allow_host_grading=allow_host_grading ) + # Checked against `to_run`, not `resolved`: a `--resume` peels off tasks + # already finalized (folded back from `prior_results`, never re-executed + # or re-graded), so an already-finalized row with empty success_criteria + # (e.g. it was originally run via `execute`) must not block a `run + # --resume` that isn't actually going to grade it. + _reject_empty_criteria_under_grade(to_run, grade=grade) + # Print execution mode print_execution_mode(len(to_run), max_parallel) - summary, task_results = await _run_with_callbacks( - execute_fn=lambda **kwargs: run_batch( - resolved_tasks=to_run, - config=config, - skipped_tasks=skipped, - prior_results=prior_results, - prior_resolved=prior_resolved, - **kwargs, - ), - task_count=len(to_run), - stream_mode=stream_mode, - ) + try: + summary, task_results = await _run_with_callbacks( + execute_fn=lambda **kwargs: run_batch( + resolved_tasks=to_run, + config=config, + skipped_tasks=skipped, + prior_results=prior_results, + prior_resolved=prior_resolved, + **kwargs, + ), + task_count=len(to_run), + stream_mode=stream_mode, + ) + except ValueError as e: + # run_batch's own resolution-time guards (e.g. --workspace-dir requiring + # exactly one non-docker task) raise a plain ValueError -- convert it to + # the same clean CLI error every other resolution-time refusal in this + # function gets, instead of an unhandled traceback. + raise typer.BadParameter(str(e)) from e # Generate experiment reports experiment_result = aggregate_results( diff --git a/src/coder_eval/harbor/__init__.py b/src/coder_eval/harbor/__init__.py new file mode 100644 index 00000000..c726b0f6 --- /dev/null +++ b/src/coder_eval/harbor/__init__.py @@ -0,0 +1,22 @@ +"""``coder_eval.harbor`` — the Harbor framework adherence layer. + +Harbor (Laude Institute / Terminal-Bench 2.0, harborframework.com) is an outer +harness with its own runtime contract: a fixed reward-file convention the +verifier phase must satisfy, a fixed log layout, a trajectory format. This +package is deliberately narrow — it translates coder-eval's own artifacts +(``task.json``, ``weighted_score``) into that contract. It does not know how a +task is *defined*; that is the export/packager concern (``coder-eval export +--format harbor``), tracked separately. + +Direction-agnostic by design: the same shim serves a coder-eval task exported +to run under Harbor (coder-eval as the grader) and a coder-eval agent embedded +inside a Harbor-authored task (Harbor's own ``tests/test.sh`` as the grader, +Part A step 8 — coder-eval is purely the agent there and this package is +unused on that path). + +This package is a core layer like ``orchestration/`` — it must not import +``coder_eval.cli`` (CE004). Raise plain exceptions and let the CLI wrap them, +exactly as ``orchestration/regrade.py`` does. +""" + +from __future__ import annotations diff --git a/src/coder_eval/harbor/agent.py b/src/coder_eval/harbor/agent.py new file mode 100644 index 00000000..bccfcab0 --- /dev/null +++ b/src/coder_eval/harbor/agent.py @@ -0,0 +1,160 @@ +"""``CoderEvalAgent`` — coder-eval as a Harbor agent (C1.2). + +The mirror image of Part C's packager: instead of coder-eval grading a +Harbor-authored task (coder-eval as the verifier), this makes coder-eval +Harbor's AGENT — ``harbor run -a coder_eval.harbor.agent:CoderEvalAgent`` +invokes ``coder-eval execute --format harbor`` inside Harbor's own container +against the fixed-path agent-phase task.yaml the packager bakes in (see +``agent_paths.py``), then Harbor picks up the resulting ``trajectory.json`` +from ``self.logs_dir`` exactly as it does for its own ``ClaudeCode`` agent. + +Design, per ``tmp/harborframework.md``'s "Scoping note — Part A revisited": + +- The agent-phase task.yaml is at :data:`AGENT_TASK_YAML_PATH`, criteria-free + (see ``packager.py``'s ``_write_agent_phase_task_yaml``) — this agent never + sees ``success_criteria``, only the real ``agent``/prompt/sandbox config. +- ``coder-eval execute --format harbor --run-dir `` + writes ``task.json`` AND a ``trajectory.json`` (ATIF) sibling directly into + the container's ``/logs/agent/`` (``environment_logs_dir``), which Harbor + bind-mounts from ``self.logs_dir`` on the host — the same path Harbor's own + ``ClaudeCode`` agent writes its trajectory to (verified against the + installed ``harbor`` package's ``populate_context_post_run``: it writes to + ``self.logs_dir / "trajectory.json"`` on the HOST side, after the container + syncs back). This class instead has coder-eval write it directly inside the + container at the mirrored path. +- ``--workspace-dir "$(pwd)"`` (Gap 2's real fix, not a workaround): without + it, ``coder-eval execute``'s own ``tempdir`` sandbox writes the agent's + workspace to a throwaway ``mkdtemp()`` elsewhere in the container, never + where Harbor's verifier phase (``tests/test.sh``) looks (the container's + ``WORKDIR``) — confirmed live, agent output was real but every criterion + scored 0 as "file does not exist". See ``run()``'s docstring. + +Verified against a real ``harbor==0.22.0`` install (``tmp/harbor-venv``): +``BaseAgent.name()`` is a ``@staticmethod``; ``run()`` returns ``None`` and +must not return the trajectory itself (Harbor discovers it by reading +``self.logs_dir / "trajectory.json"`` after the container syncs back, the same +way ``ClaudeCode.populate_context_post_run`` does) — so this class overrides +``populate_context_post_run`` to parse that file and fill in +``AgentContext``'s token/cost fields, matching ``ClaudeCode``'s own pattern +exactly. ``environment.exec()`` takes a single shell command STRING (not an +argv list). +""" + +from __future__ import annotations + +import shlex +from typing import TYPE_CHECKING + +from coder_eval.harbor.agent_paths import AGENT_TASK_YAML_PATH + + +if TYPE_CHECKING: + from harbor.environments.base import BaseEnvironment + from harbor.models.agent.context import AgentContext + +try: + from harbor.agents.installed.base import BaseInstalledAgent +except ImportError as e: # pragma: no cover - exercised only where `harbor` is installed + raise ImportError( + "coder_eval.harbor.agent requires the `harbor` package (not installed here). This module is " + + "meant to run INSIDE a Harbor trial container, where `harbor` is already present -- it is not " + + "a coder-eval runtime dependency. Install it with `pip install 'coder-eval[harbor]'` into the " + + "same interpreter as coder_eval (the image the packager builds, or a `hb run` host)." + ) from e + +try: + from coder_eval import __version__ +except ImportError: # pragma: no cover - defensive; coder_eval always defines this + __version__ = "0.0.0" + + +class CoderEvalAgent(BaseInstalledAgent): + """Runs coder-eval's own agent loop as a Harbor agent. + + ``run()`` shells out to ``coder-eval execute --format harbor`` against the + baked-in agent-phase task.yaml rather than importing coder-eval's + orchestrator in-process — the whole point is that this class lives inside + the SAME container image the packager built, so ``coder-eval`` is already + on PATH there exactly as ``tests/test.sh`` assumes it is for grading. + """ + + SUPPORTS_ATIF: bool = True + + @staticmethod + def name() -> str: + return "coder-eval" + + def version(self) -> str: + return __version__ + + async def install(self, environment: BaseEnvironment) -> None: + """No-op: the packager's exported image is expected to already have `coder-eval` installed. + + Unlike an agent whose CLI is fetched at trial time (npm/pip install + inside ``install()``), coder-eval is baked into the image at export + time -- re-installing it here would fight whatever version the image + pins. + """ + + async def run(self, instruction: str, environment: BaseEnvironment, context: AgentContext) -> None: + """Run ``coder-eval execute --format harbor`` inside the environment. + + ``instruction`` (Harbor's resolved ``instruction.md`` text) is NOT + forwarded — the agent-phase task.yaml at :data:`AGENT_TASK_YAML_PATH` + already carries the identical resolved prompt (``packager.py`` writes + both from the same source), so there is nothing to forward. Token/cost + totals are filled in afterward by ``populate_context_post_run``, not + here, matching every other installed agent's convention. + + ``--workspace-dir "$(pwd)"``: without it, ``coder-eval execute``'s own + ``tempdir`` sandbox (the agent-phase task.yaml forces ``driver: + tempdir`` — see ``packager._write_agent_phase_task_yaml``) writes the + agent's workspace to a fresh ``mkdtemp()`` elsewhere in the container, + NOT at the image's ``WORKDIR`` -- which is exactly where Harbor's own + verifier phase (``tests/test.sh``) looks for the agent's output. + ``$(pwd)`` is resolved by the container's shell at exec time, not by + this Python process, and equals the WORKDIR because ``environment.exec`` + is not given an explicit ``cwd`` (Docker execs default to the image's + configured WORKDIR). Confirmed live: without this flag the agent wrote + real output but the verifier scored every criterion 0 with "file does + not exist", because it never left the tempdir. + """ + del instruction, context # nothing to forward; context is populated post-run + run_dir = self.environment_logs_dir.as_posix() + command = ( + f"coder-eval execute {shlex.quote(AGENT_TASK_YAML_PATH)} --format harbor " + f'--run-dir {shlex.quote(run_dir)} --workspace-dir "$(pwd)"' + ) + await self._exec(environment, command) + + def populate_context_post_run(self, context: AgentContext) -> None: + """Parse the ``trajectory.json`` coder-eval wrote and fill in token/cost totals. + + Mirrors ``ClaudeCode.populate_context_post_run`` exactly: by the time + this runs, Harbor has synced the container's ``environment_logs_dir`` + back to the host at ``self.logs_dir``, so the file coder-eval wrote + during ``run()`` is now readable here. + """ + trajectory_path = self.logs_dir / "trajectory.json" + if not trajectory_path.is_file(): + self.logger.debug(f"No trajectory.json at {trajectory_path}; coder-eval execute may have failed") + return + + from coder_eval.harbor.atif_models import Trajectory + + try: + trajectory = Trajectory.model_validate_json(trajectory_path.read_text(encoding="utf-8")) + except Exception as exc: # best-effort context enrichment, never fatal to the trial + self.logger.debug(f"Failed to parse {trajectory_path}: {exc}") + return + + if trajectory.final_metrics is None: + return + metrics = trajectory.final_metrics + context.cost_usd = metrics.total_cost_usd + context.n_input_tokens = metrics.total_prompt_tokens or 0 + context.n_cache_tokens = metrics.total_cached_tokens or 0 + context.n_output_tokens = metrics.total_completion_tokens or 0 + + +__all__ = ["CoderEvalAgent"] diff --git a/src/coder_eval/harbor/agent_paths.py b/src/coder_eval/harbor/agent_paths.py new file mode 100644 index 00000000..22d53744 --- /dev/null +++ b/src/coder_eval/harbor/agent_paths.py @@ -0,0 +1,29 @@ +"""Fixed in-container path for the agent-phase task.yaml a Harbor export bakes into its image. + +Shared by ``packager.py`` (the writer — bakes a criteria-free copy of the task +into ``environment/task.yaml`` and ``COPY``s it here) and ``agent.py``'s +``CoderEvalAgent`` (the reader — the Harbor agent that runs +``coder-eval execute --format harbor`` against this exact path), so the two +sides can never independently drift on where the file lives. Fixed rather than +discovered: a Harbor agent has no way to ask the export what path it chose, so +the path itself is the contract (tmp/harborframework.md's "Gap 1" resolution — +the agent can always execute a task.yaml at a fixed path; it's up to the +Dockerfile to put it there). +""" + +from __future__ import annotations + + +AGENT_TASK_YAML_PATH = "/opt/coder-eval-task/task.yaml" + +AGENT_TASK_TEMPLATES_DIR = "/opt/coder-eval-task/templates" +"""Sibling of :data:`AGENT_TASK_YAML_PATH` for ``TemplateDirSource`` copies. A +``TemplateDirSource.path`` is resolved to an absolute HOST path at task-load +time (``task_loader.resolve_template_source_paths``) — that path does not +exist inside the container, so ``packager.py`` copies each source's directory +under here (``environment/templates/-/`` on the export side, ``COPY`` +'d into the image at build time) and rewrites ``environment/task.yaml``'s +``template_sources[].path`` to point at the in-container copy instead. +""" + +__all__ = ["AGENT_TASK_TEMPLATES_DIR", "AGENT_TASK_YAML_PATH"] diff --git a/src/coder_eval/harbor/atif_emit.py b/src/coder_eval/harbor/atif_emit.py new file mode 100644 index 00000000..99303ce3 --- /dev/null +++ b/src/coder_eval/harbor/atif_emit.py @@ -0,0 +1,422 @@ +"""EvaluationResult → ATIF Trajectory converter (the emit direction). + +Maps coder_eval's persisted trajectory (``EvaluationResult.iterations`` — the +``TurnRecord`` envelope over the per-generation ``messages`` stream) onto the +vendored ATIF models, so every run can be consumed by ``harbor view``, Harbor +Hub, and ATIF-based SFT/RL pipelines. + +Mapping highlights: + +- ``UserMessage`` → ``Step(source="user")``; ``AssistantMessage`` (one per LLM + generation) → ``Step(source="agent")`` with per-generation ``Metrics``. +- ``CommandTelemetry`` joins its generation via ``assistant_turn_index`` and + becomes that step's ``tool_calls`` + ``observation``. +- Sub-agent generations (``parent_tool_use_id`` set) are NESTED into embedded + ``subagent_trajectories`` — flattening them into the main thread would + corrupt SFT data derived from the trajectory. +- ``ReconciliationMessage`` entries never become steps: their residuals are + recorded in ``Trajectory.extra["reconciliation"]`` and are already included + in the authoritative ``FinalMetrics`` totals (``total_token_usage``). +- Turns with no message stream (legacy task.json, minimal agents) degrade to + one synthetic user step + one agent step carrying all the turn's commands. + +The converter is a PURE function of the models: no I/O, no agent-type +branching, no mutation of the input result. +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Any + +from coder_eval.harbor.atif_models import ( + AtifAgent, + FinalMetrics, + Metrics, + Observation, + ObservationResult, + Step, + SubagentTrajectoryRef, + ToolCall, + Trajectory, +) +from coder_eval.models import ( + AssistantMessage, + CommandTelemetry, + EvaluationResult, + ReconciliationMessage, + TokenUsage, + TurnRecord, + UserMessage, +) +from coder_eval.path_utils import TASK_JSON_FILENAME, write_text_atomic + + +logger = logging.getLogger(__name__) + + +def _message_text(msg: AssistantMessage) -> str: + """Concatenate the message's text blocks in emission order.""" + parts = [b.text for b in msg.content_blocks if b.block_type == "text" and b.text] + return "\n".join(parts) + + +def _reasoning_text(msg: AssistantMessage) -> str | None: + """Concatenate the message's thinking blocks, or None when there are none.""" + parts = [b.thinking for b in msg.content_blocks if b.block_type == "thinking" and b.thinking] + return "\n".join(parts) if parts else None + + +def _metrics_for(msg: AssistantMessage) -> Metrics | None: + """Per-generation Metrics; None when the message carries no token buckets.""" + if not any((msg.input_tokens, msg.output_tokens, msg.cache_creation_tokens, msg.cache_read_tokens)): + return None + # SSOT: the "three prompt buckets sum to the full prompt" rule lives in + # TokenUsage.input_tokens (a computed field) — do not hand-add them here. + usage = TokenUsage( + uncached_input_tokens=msg.input_tokens, + cache_creation_input_tokens=msg.cache_creation_tokens, + cache_read_input_tokens=msg.cache_read_tokens, + output_tokens=msg.output_tokens, + ) + return Metrics( + prompt_tokens=usage.input_tokens, + completion_tokens=msg.output_tokens, + cached_tokens=msg.cache_read_tokens, + ) + + +def _assistant_extra(msg: AssistantMessage, iteration: int) -> dict[str, Any]: + """Step.extra for one generation: iteration tag + fidelity ATIF has no slot for.""" + extra: dict[str, Any] = {"iteration": iteration} + if msg.cache_creation_tokens: + extra["cache_creation_tokens"] = msg.cache_creation_tokens + if msg.reasoning_tokens: + extra["reasoning_tokens"] = msg.reasoning_tokens + if msg.message_id is not None: + extra["message_id"] = msg.message_id + if msg.stop_reason is not None: + extra["stop_reason"] = msg.stop_reason + return extra + + +def _tool_calls_for(commands: list[CommandTelemetry]) -> tuple[list[ToolCall], Observation] | tuple[None, None]: + """Build a step's tool_calls + observation from its commands (None when empty).""" + if not commands: + return None, None + calls: list[ToolCall] = [] + results: list[ObservationResult] = [] + for cmd in commands: + call_extra: dict[str, Any] = {} + if cmd.result_status is not None: + call_extra["result_status"] = cmd.result_status + calls.append( + ToolCall( + tool_call_id=cmd.tool_id, + function_name=cmd.tool_name, + arguments=cmd.parameters, + extra=call_extra or None, + ) + ) + result_extra: dict[str, Any] = {} + if cmd.duration_ms is not None: + result_extra["duration_ms"] = cmd.duration_ms + results.append( + ObservationResult( + source_call_id=cmd.tool_id, + content=cmd.result_summary, + extra=result_extra or None, + ) + ) + return calls, Observation(results=results) + + +class _StepBuilder: + """Accumulates main-thread and per-sub-agent step lists with sequential ids.""" + + def __init__(self) -> None: + self.main: list[Step] = [] + # parent_tool_use_id -> child steps, insertion-ordered. + self.subagents: dict[str, list[Step]] = {} + + def append(self, parent_tool_use_id: str | None, **step_fields: Any) -> Step: + target = self.main if parent_tool_use_id is None else self.subagents.setdefault(parent_tool_use_id, []) + step = Step(step_id=len(target) + 1, **step_fields) + target.append(step) + return step + + def last_main_agent_step(self, start: int = 0) -> Step | None: + """Most recent main-thread agent step at or after index ``start``. + + ``start`` scopes the search to the current turn (leftover-command + attribution must never cross turn boundaries — a command attached to a + previous iteration's step would mislabel its ``extra["iteration"]``). + """ + for step in reversed(self.main[start:]): + if step.source == "agent": + return step + return None + + +def _extend_step_commands(step: Step, commands: list[CommandTelemetry]) -> None: + """Attach extra commands to an already-built step (fallback attribution).""" + calls, observation = _tool_calls_for(commands) + if calls is None or observation is None: + return + step.tool_calls = [*(step.tool_calls or []), *calls] + existing = step.observation.results if step.observation is not None else [] + step.observation = Observation(results=[*existing, *observation.results]) + + +def _emit_turn(builder: _StepBuilder, turn: TurnRecord, reconciliation: list[dict[str, Any]]) -> None: + """Emit one TurnRecord's steps into the builder (main thread + sub-agent groups).""" + turn_start = len(builder.main) + turn_extra: dict[str, Any] = {"iteration": turn.iteration} + if turn.crashed: + turn_extra["crashed"] = True + + assistant_msgs = [m for m in turn.messages if isinstance(m, AssistantMessage)] + + # Group commands by their generation; collect the unattributable ones. + commands_by_index: dict[int, list[CommandTelemetry]] = {} + leftover_commands: list[CommandTelemetry] = [] + for cmd in turn.commands: + idx = cmd.assistant_turn_index + if idx is not None and 0 <= idx < len(assistant_msgs): + commands_by_index.setdefault(idx, []).append(cmd) + else: + if idx is not None: + logger.debug("assistant_turn_index %s out of range for turn %s", idx, turn.iteration) + leftover_commands.append(cmd) + + first_step_of_turn = True + + def _step_extra(base: dict[str, Any]) -> dict[str, Any]: + nonlocal first_step_of_turn + extra = {**turn_extra, **base} + if first_step_of_turn and turn.crashed and turn.crash_reason is not None: + extra["crash_reason"] = turn.crash_reason + first_step_of_turn = False + return extra + + # Single-shot runs carry no UserMessage in the stream — synthesize the + # iteration's user step from user_input (exactly one per iteration). + if not any(isinstance(m, UserMessage) for m in turn.messages): + builder.append(None, source="user", message=turn.user_input, extra=_step_extra({})) + + if not turn.messages: + # Legacy / minimal-agent turn (EMPTY message stream — a stream that has + # user/reconciliation entries but no generations takes the normal path + # below so those entries survive): one agent step from agent_output + # carrying ALL the turn's commands and the turn-level token usage. + metrics = None + if turn.token_usage is not None and not turn.token_usage.is_empty(): + metrics = Metrics( + prompt_tokens=turn.token_usage.input_tokens, + completion_tokens=turn.token_usage.output_tokens, + cached_tokens=turn.token_usage.cache_read_input_tokens, + ) + calls, observation = _tool_calls_for(turn.commands) + builder.append( + None, + source="agent", + message=turn.agent_output, + model_name=turn.model_used, + metrics=metrics, + tool_calls=calls, + observation=observation, + extra=_step_extra({}), + ) + return + + assistant_index = -1 + for msg in turn.messages: + if isinstance(msg, ReconciliationMessage): + reconciliation.append( + { + "iteration": turn.iteration, + "input_tokens": msg.input_tokens, + "output_tokens": msg.output_tokens, + "cache_creation_tokens": msg.cache_creation_tokens, + "cache_read_tokens": msg.cache_read_tokens, + "note": msg.note, + } + ) + continue + if isinstance(msg, UserMessage): + # Every UserMessage is a genuine user utterance today (constructed + # only for simulator turns / pinned openers). If a tool-result + # UserMessage variant ever gains a producing code path (its + # docstring reserves one), the converter must learn to SKIP those + # here — tool results already live in step observations. + builder.append( + None, + source="user", + message=msg.text, + timestamp=msg.completed_at.isoformat() if msg.completed_at is not None else None, + extra=_step_extra({}), + ) + continue + assistant_index += 1 + calls, observation = _tool_calls_for(commands_by_index.get(assistant_index, [])) + builder.append( + msg.parent_tool_use_id, + source="agent", + message=_message_text(msg), + reasoning_content=_reasoning_text(msg), + model_name=msg.model, + timestamp=msg.completed_at.isoformat(), + metrics=_metrics_for(msg), + tool_calls=calls, + observation=observation, + extra=_step_extra(_assistant_extra(msg, turn.iteration)), + ) + + if leftover_commands: + # Documented fallback: unattributable commands ride the turn's last + # main-thread agent step; synthesize one from agent_output if none exists. + target = builder.last_main_agent_step(start=turn_start) + if target is None: + target = builder.append(None, source="agent", message=turn.agent_output, extra=_step_extra({})) + _extend_step_commands(target, leftover_commands) + + +def _attach_subagent_refs(main_steps: list[Step], parent_ids: list[str]) -> None: + """Point each spawning tool call's observation at its embedded child trajectory. + + Only when a main-thread ToolCall with ``tool_call_id == parent_id`` exists — + never fabricate a tool call for an orphaned sub-agent group. + """ + for parent_id in parent_ids: + for step in main_steps: + if not step.tool_calls or all(tc.tool_call_id != parent_id for tc in step.tool_calls): + continue + ref = SubagentTrajectoryRef(trajectory_id=parent_id) + results = list(step.observation.results) if step.observation is not None else [] + for result in results: + if result.source_call_id == parent_id: + result.subagent_trajectory_ref = [ref] + break + else: + results.append(ObservationResult(source_call_id=parent_id, subagent_trajectory_ref=[ref])) + step.observation = Observation(results=results) + break + + +def _total_usage(result: EvaluationResult) -> TokenUsage | None: + """The authoritative run total, falling back to summing per-turn usage.""" + if result.total_token_usage is not None: + return result.total_token_usage + usages = [t.token_usage for t in result.iterations if t.token_usage is not None] + if not usages: + return None + total = usages[0] + for usage in usages[1:]: + total = total + usage + return total + + +def evaluation_result_to_trajectory(result: EvaluationResult) -> Trajectory | None: + """Convert a finished EvaluationResult into an ATIF Trajectory. + + Returns None when no steps would result (e.g. evaluate-only runs) — ATIF + requires at least one step. + """ + from coder_eval import __version__ + + builder = _StepBuilder() + reconciliation: list[dict[str, Any]] = [] + for turn in result.iterations: + _emit_turn(builder, turn, reconciliation) + + if not builder.main: + return None + + _attach_subagent_refs(builder.main, list(builder.subagents)) + + agent = AtifAgent(name=result.agent_type, version=__version__, model_name=result.model_used) + session_id = f"{result.task_id}/{result.variant_id}" + subagent_trajectories = [ + Trajectory(session_id=session_id, trajectory_id=parent_id, agent=agent, steps=steps) + for parent_id, steps in builder.subagents.items() + ] + + final_metrics: FinalMetrics | None = None + total = _total_usage(result) + if total is not None: + final_metrics = FinalMetrics( + total_prompt_tokens=total.input_tokens, + total_completion_tokens=total.output_tokens, + total_cached_tokens=total.cache_read_input_tokens, + total_cost_usd=total.total_cost_usd, + total_steps=len(builder.main), + ) + + return Trajectory( + session_id=session_id, + agent=agent, + steps=builder.main, + final_metrics=final_metrics, + extra={"reconciliation": reconciliation} if reconciliation else None, + subagent_trajectories=subagent_trajectories or None, + ) + + +def write_trajectory_json_strict(result: EvaluationResult, path: Path) -> Path | None: + """Convert and atomically write ``trajectory.json``; RAISES on failure. + + Returns the written path, or None ONLY for the legitimate zero-step skip + (ATIF requires >= 1 step; e.g. evaluate-only runs). Conversion or write + errors propagate — for callers that must distinguish "nothing to emit" + from "emission failed" (the report backfill). + """ + trajectory = evaluation_result_to_trajectory(result) + if trajectory is None: + logger.debug("No trajectory steps for %s — skipping trajectory.json", result.task_id) + return None + write_text_atomic(path, trajectory.model_dump_json(indent=2, exclude_none=True) + "\n") + return path + + +def write_trajectory_json(result: EvaluationResult, path: Path) -> Path | None: + """Convert and atomically write ``trajectory.json``; never raises. + + Returns the written path, or None when the result yields no steps or the + conversion/write fails (logged at WARNING — a trajectory failure must + never mask the run outcome). Used by :func:`emit_trajectories_for_run` + (``coder-eval execute --format harbor``); see + :func:`write_trajectory_json_strict` for the raising one. + """ + try: + return write_trajectory_json_strict(result, path) + except Exception: + logger.warning("Failed to write trajectory.json for %s", result.task_id, exc_info=True) + return None + + +def emit_trajectories_for_run(run_dir: Path) -> list[Path]: + """Write a ``trajectory.json`` sibling for every ``task.json`` under ``run_dir``. + + The ``--format harbor`` post-pass for ``coder-eval execute``: ATIF emission + is opt-in (unlike the old always-on design this module's predecessor + shipped), so a plain ``run``/``execute`` never gains a new output file. + Walks the run directory rather than hooking the orchestrator's finalize + path, keeping this package's "translate coder-eval's own artifacts" + scope (see ``coder_eval.harbor``'s module docstring) — it needs no access + to orchestrator internals, only the ``task.json`` files a run already + wrote. Per-task failures are logged and skipped (see + :func:`write_trajectory_json`), never aborting the rest of the run's export. + """ + written: list[Path] = [] + for task_json in sorted(run_dir.glob(f"**/{TASK_JSON_FILENAME}")): + try: + result = EvaluationResult.model_validate_json(task_json.read_text(encoding="utf-8")) + except (OSError, ValueError): + logger.warning("Could not read %s as an EvaluationResult — skipping ATIF emission", task_json) + continue + trajectory_path = task_json.with_name("trajectory.json") + out = write_trajectory_json(result, trajectory_path) + if out is not None: + written.append(out) + return written diff --git a/src/coder_eval/harbor/atif_hydrate.py b/src/coder_eval/harbor/atif_hydrate.py new file mode 100644 index 00000000..729fed17 --- /dev/null +++ b/src/coder_eval/harbor/atif_hydrate.py @@ -0,0 +1,180 @@ +"""ATIF Trajectory -> coder-eval ``TurnRecord`` list (the hydrate direction). + +The reverse of ``atif_emit``: given a trajectory that was produced OUTSIDE this +process (a Harbor agent ran ``coder-eval execute --format harbor``, which wrote +``trajectory.json`` next to ``task.json``), reconstruct enough of coder-eval's +own trajectory shape to let the criteria checkers that read it +(``command_executed``, ``cli_called``, ``commands_efficiency``, +``skill_triggered``, ``llm_judge``'s transcript) work against it during a +separate ``coder-eval evaluate --format harbor`` invocation. + +Every criterion checker receives ``turn_records: list[TurnRecord] | None`` — +in a normal run this is ``EvaluationResult.iterations`` — and reads only +``TurnRecord.commands`` (tool calls) and ``TurnRecord.messages`` (for judge +transcripts); see ``criteria/command_executed.py``, ``criteria/skill_triggered.py``, +``criteria/commands_efficiency.py``. Those two fields are what this module +reconstructs. It does NOT attempt a lossless round-trip of ``atif_emit``'s +mapping: + +- Per-generation token buckets (``AssistantMessage.input_tokens`` etc.) are + NOT recovered from ``Step.metrics`` — ``TurnRecord.token_usage`` is left + unset. Cost/token reporting for a hydrated result is therefore incomplete; + only trajectory-shaped criteria are the target here. +- Sub-agent nesting is flattened: ``subagent_trajectories`` steps are appended + to the parent turn's commands (via their tool_calls) rather than + reconstructing a nested ``parent_tool_use_id`` relationship. +- Turn boundaries are recovered by splitting on ``source="user"`` steps + (mirroring ``atif_emit``'s "one synthetic user step per turn" convention), + not by any explicit iteration marker ATIF carries. +""" + +from __future__ import annotations + +from datetime import UTC, datetime + +from coder_eval.harbor.atif_models import ObservationResult, Step, ToolCall, Trajectory +from coder_eval.models import CommandTelemetry, EvaluationResult, FinalStatus, TurnRecord + + +def _tool_call_to_command( + call: ToolCall, + results_by_id: dict[str, ObservationResult], + *, + assistant_turn_index: int, +) -> CommandTelemetry: + result = results_by_id.get(call.tool_call_id) + content = result.content if result is not None else None + result_summary = content if isinstance(content, str) else (None if content is None else str(content)) + # The mirror of atif_emit's `_tool_calls_for`: `result_status` rides on the + # ToolCall's own `extra` (call_extra) and `duration_ms` on its matching + # ObservationResult's `extra` (result_extra) -- read both back, or + # `command_executed(require_success: true)` silently scores a successful + # command 0.0 on every hydrated trajectory (result_status defaults to None, + # not "success"). + call_extra = call.extra or {} + result_extra = (result.extra or {}) if result is not None else {} + return CommandTelemetry( + tool_name=call.function_name, + tool_id=call.tool_call_id, + timestamp=datetime.now(UTC), + parameters=call.arguments, + result_summary=result_summary, + result_status=call_extra.get("result_status"), + duration_ms=result_extra.get("duration_ms"), + assistant_turn_index=assistant_turn_index, + sequence_number=assistant_turn_index, + ) + + +def _step_text(step: Step) -> str: + if isinstance(step.message, str): + return step.message + return "\n".join(part.text for part in step.message if part.type == "text" and part.text) + + +def _commands_for_step(step: Step, assistant_turn_index: int) -> list[CommandTelemetry]: + if not step.tool_calls: + return [] + results_by_id = { + r.source_call_id: r + for r in (step.observation.results if step.observation is not None else []) + if r.source_call_id + } + return [ + _tool_call_to_command(call, results_by_id, assistant_turn_index=assistant_turn_index) + for call in step.tool_calls + ] + + +def _turn_from_steps(iteration: int, steps: list[Step]) -> TurnRecord: + """Build one TurnRecord from a contiguous run of steps starting at a user step.""" + user_step = steps[0] if steps and steps[0].source == "user" else None + agent_steps = [s for s in steps if s.source == "agent"] + + commands: list[CommandTelemetry] = [] + assistant_index = -1 + for step in steps: + if step.source != "agent": + continue + assistant_index += 1 + commands.extend(_commands_for_step(step, assistant_index)) + + return TurnRecord( + iteration=iteration, + user_input=_step_text(user_step) if user_step is not None else "", + agent_output=_step_text(agent_steps[-1]) if agent_steps else "", + commands=commands, + ) + + +def trajectory_to_turn_records(trajectory: Trajectory) -> list[TurnRecord]: + """Split ``trajectory.steps`` into per-turn groups and reconstruct ``TurnRecord``s. + + Sub-agent trajectories are flattened in: each embedded child's tool calls + are appended onto whichever main-thread turn contains the spawning tool + call (matched by ``tool_call_id``), falling back to the last turn when no + spawning call is found (an orphaned embed, mirroring ``atif_emit``'s own + orphan-tolerant behavior on the way out). + """ + groups: list[list[Step]] = [] + for step in trajectory.steps: + if step.source == "user" or not groups: + groups.append([step]) + else: + groups[-1].append(step) + + turns = [_turn_from_steps(i + 1, steps) for i, steps in enumerate(groups)] + + for sub in trajectory.subagent_trajectories or []: + sub_commands: list[CommandTelemetry] = [] + assistant_index = -1 + for step in sub.steps: + if step.source != "agent": + continue + assistant_index += 1 + sub_commands.extend(_commands_for_step(step, assistant_index)) + if not sub_commands: + continue + target = turns[-1] if turns else None + for turn, steps in zip(turns, groups, strict=True): + if any(sub.trajectory_id in {tc.tool_call_id for tc in (s.tool_calls or [])} for s in steps): + target = turn + break + if target is not None: + target.commands.extend(sub_commands) + + return turns + + +def seed_from_atif_trajectory( + trajectory: Trajectory, + *, + task_id: str, + task_description: str = "", + variant_id: str = "harbor", +) -> EvaluationResult: + """Build a minimal ``EvaluationResult`` from an ATIF trajectory for detached grading. + + Only the fields ``evaluate --format harbor``'s grading path actually reads + are populated meaningfully: ``iterations`` (via + :func:`trajectory_to_turn_records`) and ``iteration_count``. + ``final_status``/``weighted_score`` are placeholders — grading recomputes + them; this object exists only to carry trajectory context into + ``Orchestrator(prior_result=...)``, exactly as a re-graded run directory's + own ``task.json`` does. + """ + turns = trajectory_to_turn_records(trajectory) + return EvaluationResult( + task_id=task_id, + task_description=task_description, + variant_id=variant_id, + agent_type=trajectory.agent.name, + model_used=trajectory.agent.model_name, + started_at=datetime.now(UTC), + final_status=FinalStatus.NOT_GRADED, + iteration_count=len(turns), + iterations=turns, + ) + + +__all__ = ["seed_from_atif_trajectory", "trajectory_to_turn_records"] diff --git a/src/coder_eval/harbor/atif_models.py b/src/coder_eval/harbor/atif_models.py new file mode 100644 index 00000000..22ab153b --- /dev/null +++ b/src/coder_eval/harbor/atif_models.py @@ -0,0 +1,290 @@ +"""Vendored ATIF (Agent Trajectory Interchange Format) models, v1.7. + +Mirrors the schema in ``harbor.models.trajectories`` (harbor 0.22.0) so +coder_eval can emit and parse ATIF trajectories with ZERO runtime dependency +on the ``harbor`` pip package. Fidelity is guarded by a frozen fixture in +``tests/fixtures/atif/`` that was validated once against the real harbor +models (see ``tests/test_atif_models.py`` for the reproducible procedure). + +Deliberate deviations from harbor's models: + +- ``schema_version`` is a pattern-validated ``str`` (``^ATIF-v1\\.\\d+$``) + instead of a closed Literal, so trajectories written by a FUTURE harbor + minor version (e.g. ``ATIF-v1.9``) still parse — harbor 0.22.0 itself + would reject them. Major-version bumps (``ATIF-v2.0``) are rejected. +- harbor's ``Agent`` model is named :class:`AtifAgent` here to avoid clashing + with ``coder_eval.agent.Agent``. +- ``ContentPart.source`` (image payloads) is an untyped dict — coder_eval + emits text-only content and only needs to *tolerate* image parts on read. + +Deliberate deviation from the repo convention "all models importable from +``coder_eval.models``": these are interchange-format models for Harbor +interop, not evaluation models — they are exported from ``coder_eval.harbor`` +to keep the core model namespace clean. +""" + +from __future__ import annotations + +from typing import Any, Literal, Self + +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +ATIF_SCHEMA_VERSION = "ATIF-v1.7" +"""The ATIF version coder_eval emits (the version the vendored schema mirrors).""" + + +class ContentPart(BaseModel): + """One part of a multimodal message (text or image). + + coder_eval emits text-only content; the image variant exists so ATIF + documents produced by other agents still parse. + """ + + model_config = ConfigDict(extra="forbid") + + type: Literal["text", "image"] = Field(description="Content part kind.") + text: str | None = Field(default=None, description="Text content. Required when type='text'.") + source: dict[str, Any] | None = Field( + default=None, + description="Image payload (media type + data). Only meaningful when type='image'; untyped by design.", + ) + + @model_validator(mode="after") + def _validate_by_type(self) -> Self: + if self.type == "text": + if self.text is None: + raise ValueError("'text' field is required when type='text'") + if self.source is not None: + raise ValueError("'source' field is not allowed when type='text'") + return self + + +class ToolCall(BaseModel): + """A tool call within a step.""" + + model_config = ConfigDict(extra="forbid") + + tool_call_id: str = Field(description="Unique identifier for this specific tool call.") + function_name: str = Field(description="The name of the function or tool being invoked.") + arguments: dict[str, Any] = Field(description="Arguments passed to the function (can be empty dict).") + extra: dict[str, Any] | None = Field(default=None, description="Custom tool-call-level metadata.") + + +class SubagentTrajectoryRef(BaseModel): + """Reference to a delegated subagent trajectory. + + All four fields optional, mirroring ``harbor==0.22.0``'s real shape + (verified by installing it and diffing ``model_fields``): a document may + carry ``session_id`` alone (no ``trajectory_id``) or ``trajectory_path`` + alone (the spec's file-ref form), and ``extra`` is a documented carry-all. + The earlier version made ``trajectory_id`` required and omitted + ``session_id``/``extra`` entirely, which rejected both of those valid + shapes under ``extra="forbid"``. + """ + + model_config = ConfigDict(extra="forbid") + + trajectory_id: str | None = Field(default=None, description="trajectory_id of the referenced subagent trajectory.") + session_id: str | None = Field( + default=None, description="session_id of the referenced subagent trajectory, when addressed that way." + ) + trajectory_path: str | None = Field( + default=None, + description=( + "Path to an external trajectory file. Null means embedded resolution: the id matches an " + "entry in the root trajectory's subagent_trajectories array." + ), + ) + extra: dict[str, Any] | None = Field(default=None, description="Custom reference-level metadata.") + + +class ObservationResult(BaseModel): + """The result of one tool call or action within a step's observation.""" + + model_config = ConfigDict(extra="forbid") + + source_call_id: str | None = Field( + default=None, + description=( + "The tool_call_id from this step's tool_calls that this result corresponds to. Null for " + "results from actions that don't use the standard tool-calling format." + ), + ) + content: str | list[ContentPart] | None = Field( + default=None, + description="The output from the tool execution (string, or ContentPart list for multimodal).", + ) + subagent_trajectory_ref: list[SubagentTrajectoryRef] | None = Field( + default=None, + description="References to delegated subagent trajectories spawned by this call.", + ) + extra: dict[str, Any] | None = Field(default=None, description="Custom observation-result-level metadata.") + + +class Observation(BaseModel): + """The environment feedback for one step.""" + + model_config = ConfigDict(extra="forbid") + + results: list[ObservationResult] = Field(description="Result objects from tool calls or actions.") + + +class Metrics(BaseModel): + """Per-step LLM metrics.""" + + model_config = ConfigDict(extra="forbid") + + prompt_tokens: int | None = Field(default=None, description="Prompt tokens for this LLM call (full prompt).") + completion_tokens: int | None = Field(default=None, description="Completion tokens generated by this call.") + cached_tokens: int | None = Field(default=None, description="Prompt tokens served from cache.") + cost_usd: float | None = Field(default=None, description="Cost of this call in USD.") + prompt_token_ids: list[int] | None = Field(default=None, description="Token ids of the prompt (RL pipelines).") + completion_token_ids: list[int] | None = Field( + default=None, description="Token ids of the completion (RL pipelines)." + ) + logprobs: list[float] | None = Field(default=None, description="Per-token logprobs of the completion.") + extra: dict[str, Any] | None = Field(default=None, description="Custom metrics-level metadata.") + + +class FinalMetrics(BaseModel): + """Summary metrics for the entire trajectory.""" + + model_config = ConfigDict(extra="forbid") + + total_prompt_tokens: int | None = Field(default=None, description="Total prompt tokens across the trajectory.") + total_completion_tokens: int | None = Field(default=None, description="Total completion tokens.") + total_cached_tokens: int | None = Field(default=None, description="Total cached prompt tokens.") + total_cost_usd: float | None = Field(default=None, description="Total cost in USD.") + total_steps: int | None = Field(default=None, description="Number of steps in the trajectory.") + extra: dict[str, Any] | None = Field(default=None, description="Custom final-metrics metadata.") + + +class AtifAgent(BaseModel): + """The agent configuration that produced the trajectory. + + harbor names this model ``Agent``; renamed here to avoid clashing with + ``coder_eval.agent.Agent``. + """ + + model_config = ConfigDict(extra="forbid") + + name: str = Field(description="Agent name (e.g. the registered agent kind).") + version: str = Field(description="Agent version string (REQUIRED by the ATIF spec).") + model_name: str | None = Field(default=None, description="Default model for the trajectory's steps.") + tool_definitions: list[dict[str, Any]] | None = Field( + default=None, + description="Tool/function definitions available to the agent (OpenAI function-calling schema).", + ) + extra: dict[str, Any] | None = Field(default=None, description="Custom agent-level metadata.") + + +class Step(BaseModel): + """A single step in the trajectory (one message event).""" + + model_config = ConfigDict(extra="forbid") + + step_id: int = Field(ge=1, description="Ordinal index of the step (sequential, starting from 1).") + timestamp: str | None = Field(default=None, description="ISO 8601 timestamp of when this step occurred.") + source: Literal["system", "user", "agent"] = Field(description="The originator of this step.") + model_name: str | None = Field( + default=None, + description="LLM model used for this step. Omission implies the root agent config's model.", + ) + reasoning_effort: str | float | None = Field( + default=None, description="Qualitative or quantitative measure of effort." + ) + message: str | list[ContentPart] = Field( + description="The dialogue message (string, or ContentPart list for multimodal)." + ) + reasoning_content: str | None = Field(default=None, description="The agent's explicit internal reasoning.") + tool_calls: list[ToolCall] | None = Field(default=None, description="Tool calls issued in this step.") + observation: Observation | None = Field(default=None, description="Environment feedback for this step.") + metrics: Metrics | None = Field(default=None, description="LLM metrics for this step.") + is_copied_context: bool | None = Field( + default=None, description="True when the step's content was copied from a prior context (compaction)." + ) + llm_call_count: int | None = Field(default=None, description="Number of LLM calls this step represents.") + extra: dict[str, Any] | None = Field(default=None, description="Custom step-level metadata.") + + +class Trajectory(BaseModel): + """Agent Trajectory in ATIF (Agent Trajectory Interchange Format).""" + + model_config = ConfigDict(extra="forbid") + + schema_version: str = Field( + default=ATIF_SCHEMA_VERSION, + pattern=r"^ATIF-v1\.\d+$", + description=( + "ATIF compatibility version. Any v1.x parses (forward-tolerant read, unlike harbor's closed " + "Literal); v2+ is rejected." + ), + ) + session_id: str | None = Field( + default=None, + description="Run-scoped identifier; may be shared by a parent trajectory and its embedded subagents.", + ) + trajectory_id: str | None = Field( + default=None, + description=( + "Per-trajectory-document unique identifier. Optional on standalone trajectories; REQUIRED and " + "unique on trajectories embedded in a parent's subagent_trajectories array." + ), + ) + agent: AtifAgent = Field(description="The agent configuration that produced this trajectory.") + steps: list[Step] = Field(min_length=1, description="The complete interaction history.") + notes: str | None = Field(default=None, description="Custom information, design notes, or explanations.") + final_metrics: FinalMetrics | None = Field(default=None, description="Summary metrics for the trajectory.") + continued_trajectory_ref: str | None = Field( + default=None, description="Reference to the continuation trajectory file, if continued elsewhere." + ) + extra: dict[str, Any] | None = Field(default=None, description="Custom root-level metadata.") + subagent_trajectories: list[Trajectory] | None = Field( + default=None, + description="Embedded subagent trajectories; each must carry a unique, non-null trajectory_id.", + ) + + @model_validator(mode="after") + def validate_step_ids(self) -> Self: + """step_ids must be sequential starting from 1 (mirrors harbor's validator).""" + for i, step in enumerate(self.steps): + expected = i + 1 + if step.step_id != expected: + raise ValueError(f"steps[{i}].step_id: expected {expected} (sequential from 1), got {step.step_id}") + return self + + @model_validator(mode="after") + def validate_tool_call_references(self) -> Self: + """Every observation source_call_id must reference a tool_call_id in the SAME step.""" + for step in self.steps: + if step.observation is None: + continue + tool_call_ids = {tc.tool_call_id for tc in step.tool_calls} if step.tool_calls else set() + for result in step.observation.results: + if result.source_call_id is not None and result.source_call_id not in tool_call_ids: + raise ValueError( + f"Observation result references source_call_id '{result.source_call_id}' " + + f"which is not found in step {step.step_id}'s tool_calls" + ) + return self + + @model_validator(mode="after") + def validate_embedded_subagent_trajectory_ids(self) -> Self: + """Embedded subagents must carry a unique, non-null trajectory_id (resolution key).""" + if not self.subagent_trajectories: + return self + seen: set[str] = set() + for i, sub in enumerate(self.subagent_trajectories): + if sub.trajectory_id is None: + raise ValueError( + f"subagent_trajectories[{i}].trajectory_id is required for embedded subagents " + + f"(agent.name={sub.agent.name!r}, session_id={sub.session_id!r})" + ) + if sub.trajectory_id in seen: + raise ValueError( + f"subagent_trajectories[{i}].trajectory_id {sub.trajectory_id!r} is not unique " + + "within subagent_trajectories" + ) + seen.add(sub.trajectory_id) + return self diff --git a/src/coder_eval/harbor/experiment_packager.py b/src/coder_eval/harbor/experiment_packager.py new file mode 100644 index 00000000..a8e3e614 --- /dev/null +++ b/src/coder_eval/harbor/experiment_packager.py @@ -0,0 +1,211 @@ +"""Experiment export — ``coder-eval export --format harbor ... -e -o ``. + +Extends the single-task Harbor packager (``packager.py``) to ``experiment.yaml`` +variants. Harbor's ``task.toml`` has no concept of a "variant" — an experiment +comparing N variants over M task files becomes N*M (times replicates, times +dataset rows) independent Harbor task directories, one per resolved +(task, variant, replicate[, dataset row]) combination. + +Resolution reuses ``orchestration/experiment.py::resolve_all_tasks`` — the exact +machinery ``coder-eval run -e`` uses — so the export sees identical resolved +configs to a real experiment run, with zero duplicated merge logic. Each +resolved task is then written out via ``packager.export_resolved_task``, the +same writer the single-task path uses. + +See ``tmp/harborframework_conversion.md`` for the full table of what does and +does not survive this translation. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + +from coder_eval.harbor.packager import ( + CriteriaNotExportableError, + ExportResult, + TaskNotExportableError, + export_resolved_task, +) +from coder_eval.models import ResolvedTask, SkippedTask +from coder_eval.orchestration.config import BatchRunConfig +from coder_eval.orchestration.experiment import DEFAULT_EXPERIMENT_PATH, load_experiment, resolve_all_tasks + + +# Sources a resolved field's ConfigLineageEntry can carry (see +# models/results.py::ConfigLineageEntry). Only these two mean "the experiment +# introduced this, the task's own YAML did not." +_EXPERIMENT_INTRODUCED_SOURCES = {"variant", "experiment-defaults"} + + +@dataclass(frozen=True) +class SkippedVariantExport: + """One resolved (task, variant, replicate) combination that was not exported, and why.""" + + variant_id: str + task_id: str + replicate_index: int + reason: str + + +@dataclass(frozen=True) +class ExperimentExportResult: + """What ``export_experiment`` produced, for the CLI to report.""" + + exported: list[ExportResult] = field(default_factory=list) + skipped: list[SkippedVariantExport] = field(default_factory=list) + load_skipped: list[SkippedTask] = field(default_factory=list) + + +def _unhonorable_override_reason(resolved: ResolvedTask) -> str | None: + """Why this resolved task's Harbor export can't honor an experiment-introduced override. + + An agent override (model/plugins/system_prompt/tools/type) IS honorable + now: ``packager.py::_write_agent_phase_task_yaml`` carries ``task.agent`` + verbatim into ``environment/task.yaml``, which ``CoderEvalAgent.run()`` + (``harbor/agent.py``, C1.2) executes via ``coder-eval execute`` — so a + variant that only touches ``agent.*`` exports its own distinct directory + and is NOT skipped. ``simulation`` is the one override that remains + unhonorable: Harbor's ``Trial`` model has no multi-turn user-simulator + concept, and the dialog's turn-continuation logic reads coder-eval's own + criteria results mid-run, which a single-shot verifier export cannot + represent. Detected via ``ResolvedTask.config_lineage`` — a field counts + only when its most-specific source is the variant or the experiment + defaults, never the task's own YAML or the baseline defaults. + """ + sim_entry = resolved.config_lineage.get("simulation") + if sim_entry is not None and sim_entry.source in _EXPERIMENT_INTRODUCED_SOURCES: + return ( + f"variant {resolved.variant_id!r} sets a simulation override that Harbor's single-shot verifier " + "export cannot represent -- Harbor's Trial model has no multi-turn user-simulator concept. Skipped." + ) + return None + + +class UnsafeExportPathError(ValueError): + """A resolved ``variant_id``/``task_id``/``row_id`` would export outside ``out_dir``.""" + + +def _out_subdir(out_dir: Path, resolved: ResolvedTask, *, needs_replicate_segment: bool) -> Path: + """``///[/]rep/`` — row/replicate segments only when they fan out. + + ``variant_id``/``task_id``/``row_id`` are free-form, author-controlled + strings (``row_id`` in particular comes straight from a dataset row's + ``id_field``, which may be sourced from an external CSV/JSONL — less + trusted than the task YAML itself). None of them are validated as + filesystem-safe elsewhere, so a value like ``"../../../etc"`` would + otherwise let a crafted experiment/dataset write Harbor's exported + directory (including an executable ``tests/test.sh``) anywhere the + invoking user can write. Resolve the computed destination and refuse it + outright if it would land outside ``out_dir``, rather than trusting the + segments to be well-formed. + """ + out_dir_resolved = out_dir.resolve() + dest = out_dir / resolved.variant_id / resolved.task.task_id + if resolved.task.row_id: + dest = dest / resolved.task.row_id + if needs_replicate_segment: + dest = dest / f"rep{resolved.replicate_index:02d}" + # `resolve()` on a path that doesn't exist yet still normalizes `..` + # segments against its (existing) parents, so this catches traversal + # without requiring `dest` to already exist. + if not dest.resolve().is_relative_to(out_dir_resolved): + raise UnsafeExportPathError( + f"resolved export path for variant {resolved.variant_id!r}, task {resolved.task.task_id!r} " + + f"would land outside the output directory ({dest.resolve()} is not under {out_dir_resolved}) -- " + + "check variant_id/task_id/row_id for path-traversal sequences." + ) + return dest + + +def export_experiment( + task_files: list[Path], + experiment_file: Path, + out_dir: Path, + *, + allow_credentials: bool = False, +) -> ExperimentExportResult: + """Export every (task x variant x replicate[ x dataset row]) combination to Harbor directories. + + Resolves ``task_files`` against ``experiment_file`` through the same + ``resolve_all_tasks`` pipeline ``coder-eval run -e`` uses (including + ``experiments/default.yaml`` as the baseline layer, when present), then + writes one Harbor task directory per resolved task at + ``///[/]rep/``. A resolved task + is SKIPPED (not fatal to the rest of the export) rather than written when: + it carries an experiment-introduced ``agent``/``simulation`` override + Harbor's export cannot honor (see ``_unhonorable_override_reason``), its + sandbox driver isn't ``docker``, or its criteria fail C1.4's portability + audit -- each mirrors an existing per-task refusal in ``packager.py``, + demoted from a raised exception to a collected skip so one bad variant + does not abort the whole experiment's export. + """ + experiment = load_experiment(experiment_file) + if experiment_file == DEFAULT_EXPERIMENT_PATH: + default_experiment = experiment + elif DEFAULT_EXPERIMENT_PATH.exists(): + default_experiment = load_experiment(DEFAULT_EXPERIMENT_PATH) + else: + default_experiment = experiment + + config = BatchRunConfig(run_dir=out_dir) + resolved_tasks, load_skipped = resolve_all_tasks( + task_files=task_files, + experiment=experiment, + default_experiment=default_experiment, + config=config, + experiment_file=experiment_file, + ) + + # How many replicates each (variant, task) group actually fanned out to, + # so a single-replicate group's directory omits the `rep00` segment. + replicate_counts: dict[tuple[str, str], int] = {} + for r in resolved_tasks: + key = (r.variant_id, r.task.task_id) + replicate_counts[key] = max(replicate_counts.get(key, 0), r.replicate_index + 1) + + exported: list[ExportResult] = [] + skipped: list[SkippedVariantExport] = [] + for resolved in resolved_tasks: + reason = _unhonorable_override_reason(resolved) + if reason is not None: + skipped.append( + SkippedVariantExport( + variant_id=resolved.variant_id, + task_id=resolved.task.task_id, + replicate_index=resolved.replicate_index, + reason=reason, + ) + ) + continue + + needs_replicate_segment = replicate_counts[(resolved.variant_id, resolved.task.task_id)] > 1 + dest = _out_subdir(out_dir, resolved, needs_replicate_segment=needs_replicate_segment) + try: + result = export_resolved_task( + resolved.task, + resolved.task_file, + dest, + allow_credentials=allow_credentials, + ) + except (TaskNotExportableError, CriteriaNotExportableError) as e: + skipped.append( + SkippedVariantExport( + variant_id=resolved.variant_id, + task_id=resolved.task.task_id, + replicate_index=resolved.replicate_index, + reason=str(e), + ) + ) + continue + exported.append(result) + + return ExperimentExportResult(exported=exported, skipped=skipped, load_skipped=load_skipped) + + +__all__ = [ + "ExperimentExportResult", + "SkippedVariantExport", + "UnsafeExportPathError", + "export_experiment", +] diff --git a/src/coder_eval/harbor/packager.py b/src/coder_eval/harbor/packager.py new file mode 100644 index 00000000..2624534b --- /dev/null +++ b/src/coder_eval/harbor/packager.py @@ -0,0 +1,701 @@ +"""C2 — the packager. ``coder-eval export --format harbor -o ``. + +Emits a Harbor task directory from a coder-eval task, with coder-eval's own +criteria as the grader (C1.1's verifier shim). Task *definition* only; the +runtime contract (C1.1's reward writer) is what makes the emitted +``tests/test.sh`` actually work once Harbor runs it. + +Emitted layout (see ``tmp/harborframework.md`` § C2 for the full mapping +table): + + / + ├── task.toml + ├── instruction.md # fixed placeholder -- the real prompt is in environment/task.yaml + ├── environment/ + │ ├── Dockerfile # copied from dockerfile_path, or synthesized from + │ │ # sandbox.docker.image -- always written, WORKDIR pinned + │ ├── task.yaml # criteria-free copy for the CoderEvalAgent embed + │ └── templates/ # sandbox.template_sources' TemplateDirSource dirs, + │ # present only when the task has any (see agent_paths.py) + └── tests/ + ├── test.sh # C1.1's two-line shim + ├── task.yaml # the criteria, as authored + └── reference/ # task.reference, verifier-side only + +One design point worth stating explicitly because it is not obvious from +either side's docs: ``tests/task.yaml`` (uploaded whole into the container at +``/tests/`` by Harbor's own verifier — see ``_TEST_SH_TEMPLATE``, which +therefore references it as ``/tests/task.yaml``, not a cwd-relative path; +confirmed live against real docker, not assumed) must NOT set +``agent: {type: none}`` +even though C1.1's own docstring describes the verifier phase as exactly +that. coder-eval's own ``check_none_agent`` validator rejects any criterion +with ``requires_agent=True`` — which includes ``reference_comparison``, a +criterion this module treats as portable (C1.4) — the moment ``agent.type`` +is literally ``none``, regardless of whether an agent actually runs. +``coder-eval evaluate `` (the bare-task/work-dir path +C1.1's shim calls) never instantiates an agent no matter what ``agent.type`` +says (see ``evaluate_command.py``'s own comment: "no agent is created" is +true unconditionally on that path) — so a placeholder real agent type plus a +placeholder prompt satisfies coder-eval's validators without changing +behaviour, and unblocks ``reference_comparison``. Verified directly (not +inferred) before writing this module. +""" + +from __future__ import annotations + +import shlex +import shutil +import subprocess +from dataclasses import dataclass, field +from pathlib import Path + +import tomli_w +import yaml + +from coder_eval.harbor.agent_paths import AGENT_TASK_TEMPLATES_DIR, AGENT_TASK_YAML_PATH +from coder_eval.harbor.portability import PortabilityIssue, audit_criteria +from coder_eval.models import TaskDefinition, TemplateDirSource +from coder_eval.orchestration.task_loader import load_task +from coder_eval.path_utils import REFERENCE_COPY_IGNORE, ignore_patterns_and_symlinks + + +DEFAULT_WORKDIR = "/app" +_HARBOR_SCHEMA_VERSION = "1.4" # pinned to the harbor 0.22.0 findings in tmp/harborframework.md § C0 + +# The task.yaml this module emits is graded via `coder-eval evaluate`, which +# never instantiates an agent on the bare-task/work-dir path regardless of +# `agent.type` (see module docstring) — this prompt is never read. +_VERIFIER_PLACEHOLDER_PROMPT = ( + "(unused placeholder — this file is graded via `coder-eval evaluate`, which does not invoke an agent on this path)" +) + + +_TEST_SH_TEMPLATE = """#!/bin/sh +# Generated by `coder-eval export --format harbor` — do not hand-edit; a +# regenerated export will overwrite it. See tmp/harborframework.md § C1.1. +# +# NOT `set -e`: `coder-eval evaluate` exits non-zero whenever any criterion +# fails its gate -- a genuine MEASURED verdict, not an infrastructure +# failure. `set -e` here would abort before `coder-eval harbor reward` ever +# ran, turning every real failing score into a masked (unmeasured) trial -- +# confirmed live: a solution that runs but produces the wrong content +# scored weighted_score=0.500 in task.json, yet `set -e` discarded it and +# Harbor reported RewardFileNotFoundError instead of reward=0.5. The reward +# writer is what decides infra-vs-policy (None -> no file); this script must +# always reach it. +set -u + +coder-eval evaluate /tests/task.yaml "{workdir}" --in-place --run-dir /logs/verifier || true +coder-eval harbor reward /logs/verifier --out /logs/verifier/reward.json +""" + + +class TaskNotExportableError(Exception): + """This task cannot export to Harbor v1 as specified — a structural refusal, not a criteria issue.""" + + +class CriteriaNotExportableError(Exception): + """The task's criteria include one or more Harbor cannot grade in v1 (see C1.4's audit).""" + + def __init__(self, issues: list[PortabilityIssue]) -> None: + self.issues = issues + lines = "\n".join( + f" - {i.criterion_description!r} ({i.criterion_type}): {i.portability.value}" for i in issues + ) + super().__init__( + f"{len(issues)} criterion/criteria cannot export to Harbor v1:\n{lines}\n" + + "Remove them, or pass allow_credentials=True (--allow-credentials on the CLI) for the " + + "NEEDS_CREDENTIALS ones if you have already provisioned model access inside the verifier " + + "container yourself." + ) + + +@dataclass(frozen=True) +class ExportResult: + """What ``export_task`` produced, for the CLI to report.""" + + out_dir: Path + workdir: str + warnings: list[str] = field(default_factory=list) + + +def export_task( + task_file: Path, + out_dir: Path, + *, + allow_credentials: bool = False, +) -> ExportResult: + """Emit a Harbor task directory at ``out_dir`` from the coder-eval task at ``task_file``. + + Raises: + TaskNotExportableError: the task's sandbox isn't ``driver: docker`` — + Harbor always builds a container per task, so there is nothing to + derive ``environment/`` from otherwise (v1; no host-execution + equivalent is attempted). + CriteriaNotExportableError: one or more criteria fail C1.4's + portability audit. + """ + task, _raw_yaml = load_task(task_file) + return export_resolved_task(task, task_file, out_dir, allow_credentials=allow_credentials) + + +def export_resolved_task( + task: TaskDefinition, + task_file: Path, + out_dir: Path, + *, + allow_credentials: bool = False, +) -> ExportResult: + """Emit a Harbor task directory from an already-loaded/resolved ``TaskDefinition``. + + Shared by ``export_task`` (single task.yaml) and + ``harbor.experiment_packager.export_experiment`` (task.yaml + experiment.yaml, + one resolved task per variant/replicate/dataset row) — one writer, so the + two paths cannot silently drift. ``task_file`` is the ORIGINAL task YAML + path (not a synthetic one) — it is only used to resolve ``task.reference``, + which ``load_task`` deliberately leaves relative (see ``_write_reference``); + every other path-shaped field is already absolute by the time a + ``TaskDefinition`` reaches this function (``load_task`` resolves + ``dockerfile_path``/``initial_prompt_file``/``system_prompt_file`` inline). + + Raises the same two errors as ``export_task``, for the same reasons. + """ + issues = audit_criteria(task.success_criteria, allow_credentials=allow_credentials) + if issues: + raise CriteriaNotExportableError(issues) + + if task.sandbox.driver != "docker": + raise TaskNotExportableError( + f"Task {task.task_id!r} uses sandbox.driver={task.sandbox.driver!r}, but Harbor always builds a " + + "container per task and v1 export has no host-execution equivalent to derive environment/ from. " + + "Set sandbox.driver: docker (with dockerfile_path or a custom image) to export this task." + ) + + if task.dataset is not None: + # `export_task`/`export_resolved_task` calls `load_task`, which does + # NOT run `expand_dataset` -- fan-out happens later, in the experiment + # pipeline (`export_experiment` resolves it per row before reaching + # here). Exporting a raw dataset-backed task would silently emit ONE + # Harbor task whose prompt and criteria still contain literal + # `${row.}` placeholders: never expressible, but scored anyway. + raise TaskNotExportableError( + f"Task {task.task_id!r} has a `dataset:` block, which this function does not expand -- its " + + "`${row.*}` placeholders would export unsubstituted. Export via `coder-eval export ... " + + "-e ` (harbor.experiment_packager.export_experiment), which resolves one " + + "Harbor task directory per dataset row." + ) + + if task.simulation is not None and task.simulation.enabled: + raise TaskNotExportableError( + f"Task {task.task_id!r} has an enabled `simulation:` block -- its turn-continuation logic reads " + + "coder-eval's own criteria results mid-dialog, which has no Harbor equivalent. Simulation tasks " + + "cannot be exported." + ) + + warnings: list[str] = [] + if task.pre_run: + warnings.append( + f"{len(task.pre_run)} pre_run command(s) were NOT translated — they run against a live sandbox " + + "with template files already staged, which has no Dockerfile-build-time equivalent. Fold their " + + "effect into environment/Dockerfile by hand if the exported task needs it." + ) + if task.post_run: + warnings.append( + f"{len(task.post_run)} post_run command(s) were NOT translated — they run after the verdict is " + + "computed and coder-eval's own docs mark this mapping as deferred (see C2's table). Add them to " + + "tests/test.sh by hand, after the reward is written, if the exported task needs them." + ) + + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / "tests").mkdir(parents=True, exist_ok=True) + + workdir = _write_environment(task, task_file, out_dir, warnings) + _write_instruction(out_dir) + _write_verifier_task_yaml(task, out_dir) + _write_test_sh(out_dir, workdir=workdir) + _write_reference(task, task_file, out_dir) + _write_task_toml(task, out_dir, workdir=workdir) + + return ExportResult(out_dir=out_dir, workdir=workdir, warnings=warnings) + + +def _resolve_prompt_text(task: TaskDefinition, task_file: Path) -> str: + """The agent's actual instruction text, resolved from whichever of the two prompt fields is set. + + Feeds ``environment/task.yaml``'s ``initial_prompt`` -- the ``CoderEvalAgent`` + embed's real prompt (``instruction.md`` is a fixed placeholder, not this + text; see ``_write_instruction``). + """ + if task.initial_prompt is not None: + return task.initial_prompt + if task.initial_prompt_file is not None: + return (task_file.parent / task.initial_prompt_file).read_text(encoding="utf-8") + # is_none_agent or simulation-only tasks carry no prompt at all — fall back + # to the description so `environment/task.yaml`'s initial_prompt is never empty. + return task.description + + +_INSTRUCTION_MD_CONTENT = ( + "This is a coder-eval task, exported to Harbor.\n" + "\n" + "This file is autogenerated -- do not edit it directly. The task's real\n" + "instruction and agent configuration live in environment/task.yaml, and\n" + "must be run with the CoderEvalAgent Harbor agent:\n" + "\n" + " harbor run -a coder_eval.harbor.agent:CoderEvalAgent -p \n" + "\n" + "See https://coder-eval.com for details on the coder-eval <-> Harbor integration.\n" +) + + +def _write_instruction(out_dir: Path) -> None: + """``instruction.md`` — a fixed placeholder, deliberately NOT the real prompt. + + Harbor's own stock agents (``ClaudeCode`` etc.) read this file as their + actual task instruction. This export is meant to run ONLY via the + ``CoderEvalAgent`` embed (which ignores ``instruction.md`` entirely and + instead runs ``coder-eval execute`` against ``environment/task.yaml`` -- + see ``harbor/agent.py``'s ``run()``) -- so putting the real prompt here + would silently let a stock Harbor agent run the task with a DIFFERENT + (and unmonitored) agent loop than the one the export was built for. + """ + (out_dir / "instruction.md").write_text(_INSTRUCTION_MD_CONTENT, encoding="utf-8") + + +def _write_environment( + task: TaskDefinition, + task_file: Path, + out_dir: Path, + warnings: list[str], +) -> str: + """Copy/derive ``environment/`` and return the WORKDIR both it and test.sh must agree on. + + Per C0 § 5, Harbor has no fixed workspace path — the verifier (default + SHARED mode) runs at whatever the container's own WORKDIR is. This + function is the one place that decides it, so nothing downstream can + silently disagree. + + A ``Dockerfile`` is ALWAYS written here (never left to a pre-built + ``docker_image`` reference in ``task.toml``) — the whole point of + ``environment/task.yaml`` is to be baked in at :data:`AGENT_TASK_YAML_PATH` + for the ``CoderEvalAgent`` Harbor-agent embed, and a task exported without + a Dockerfile has no `COPY` step to put it there. Two shapes, both ending + in the same `COPY task.yaml ...` line: + + - ``dockerfile_path`` set: copy the user's Dockerfile as the base, appending + a `WORKDIR` (if it declared none) and the `COPY` line. + - unset: synthesize a minimal one (`FROM ` + `WORKDIR` + + `COPY`) — ``docker_cfg.image`` always has a value (default_factory= + ``get_default_docker_image_tag``), so this is a real choice, not a + null-vs-set distinction. + """ + env_dir = out_dir / "environment" + env_dir.mkdir(parents=True, exist_ok=True) + docker_cfg = task.sandbox.docker + has_templates = _write_agent_phase_task_yaml( + task, env_dir, initial_prompt=_resolve_prompt_text(task, task_file), warnings=warnings + ) + templates_copy_line = f"COPY templates/ {AGENT_TASK_TEMPLATES_DIR}/\n" if has_templates else "" + dest_dockerfile = env_dir / "Dockerfile" + + if docker_cfg.dockerfile_path is not None: + source_dockerfile = Path(docker_cfg.dockerfile_path) # already absolute — load_task resolves it + shutil.copy2(source_dockerfile, dest_dockerfile) + workdir = docker_cfg.working_dir or _find_workdir(dest_dockerfile) or DEFAULT_WORKDIR + if _find_workdir(dest_dockerfile) is None: + with dest_dockerfile.open("a", encoding="utf-8") as fh: + fh.write(f"\nWORKDIR {workdir}\n") + warnings.append( + f"environment/Dockerfile declared no WORKDIR; appended `WORKDIR {workdir}` so the " + + "verifier and agent phases agree on a path." + ) + with dest_dockerfile.open("a", encoding="utf-8") as fh: + fh.write(f"\nCOPY task.yaml {AGENT_TASK_YAML_PATH}\n{templates_copy_line}") + if not _from_line_mentions_coder_eval_agent(dest_dockerfile): + warnings.append(_MISSING_CODER_EVAL_WARNING) + # Non-Dockerfile build context (COPY sources etc.) is not carried over in + # v1 — a dockerfile_path build context beyond the Dockerfile itself needs + # its own decision (open question) before this is safe to widen. + if source_dockerfile.parent != task_file.parent: + other_files = [p for p in source_dockerfile.parent.iterdir() if p != source_dockerfile] + if other_files: + warnings.append( + f"{source_dockerfile.parent} holds {len(other_files)} other file(s) alongside the " + + "Dockerfile (build context) that were NOT copied — v1 only copies the Dockerfile itself." + ) + return workdir + + # No dockerfile_path — synthesize a minimal Dockerfile on top of the + # pre-built image so the `CoderEvalAgent` embed always has somewhere to + # `COPY task.yaml` into. + if "coder-eval-agent" not in docker_cfg.image: + warnings.append(_MISSING_CODER_EVAL_WARNING) + if docker_cfg.working_dir is not None: + workdir = docker_cfg.working_dir + else: + inspected = _inspect_image_workdir(docker_cfg.image) + if inspected is not None: + workdir = inspected + else: + workdir = DEFAULT_WORKDIR + warnings.append( + f"Could not determine {docker_cfg.image}'s own WORKDIR (image not present locally, or " + + f"docker unavailable at export time) -- defaulting to `{DEFAULT_WORKDIR}`. Harbor's " + + "`docker exec -w` hard-fails if that path does not already exist in the image (unlike " + + "`docker run -w`, it will not create it); set `sandbox.docker.working_dir` explicitly " + + "to the image's real WORKDIR to avoid a verify-time exit 127." + ) + dest_dockerfile.write_text( + f"FROM {docker_cfg.image}\nWORKDIR {workdir}\n\nCOPY task.yaml {AGENT_TASK_YAML_PATH}\n{templates_copy_line}", + encoding="utf-8", + ) + return workdir + + +def _inspect_image_workdir(image: str) -> str | None: + """Best-effort ``docker image inspect`` for a pre-built image's own ``WORKDIR``. + + A pre-built image has no Dockerfile for ``_find_workdir`` to read, so this + is the only way to avoid guessing a path that doesn't exist in it (Harbor's + ``docker exec -w`` -- unlike ``docker run -w`` -- fails outright if the + directory isn't already there; confirmed live). Returns ``None`` (never + raises) whenever docker isn't available, the image isn't present locally, + or it declares no WORKDIR -- callers fall back to ``DEFAULT_WORKDIR`` and + warn. + """ + try: + result = subprocess.run( + # `--` before `image` (task-YAML-controlled) stops it from being + # parsed as an option if it happens to start with "-". + ["docker", "image", "inspect", "--format", "{{.Config.WorkingDir}}", "--", image], + capture_output=True, + text=True, + encoding="utf-8", + timeout=30, + check=False, + ) + except (OSError, subprocess.SubprocessError): + return None + if result.returncode != 0: + return None + workdir = result.stdout.strip() + return workdir or None + + +_MISSING_CODER_EVAL_WARNING = ( + "environment/ does not appear to be based on a coder-eval-agent image. `tests/test.sh` calls " + "`coder-eval` (for the reward writer), which must be installed inside the container -- v1 assumes " + "`FROM coder-eval-agent:` (or a custom image that already extends it). A base-image-agnostic " + "fix (grafting the relocatable /opt/coder-eval kit from `make coder-eval-runtime` into any " + "Dockerfile) is tracked as follow-up in tmp/harborframework.md, not yet built." +) + + +def _from_line_mentions_coder_eval_agent(dockerfile: Path) -> bool: + """Heuristic only: does any ``FROM`` line in this Dockerfile name a coder-eval-agent image? + + A substring match, not an image-history inspection -- a multi-stage build + whose FINAL stage derives from coder-eval-agent under a different alias, + or an image that installs coder-eval by some other means entirely, both + read as a false warning here. That is the intended failure direction: a + missed real positive is silent breakage at verify time, so this errs + toward warning more often, not less. + """ + for line in dockerfile.read_text(encoding="utf-8").splitlines(): + if line.strip().upper().startswith("FROM ") and "coder-eval-agent" in line: + return True + return False + + +def _find_workdir(dockerfile: Path) -> str | None: + """The last ``WORKDIR`` directive in a Dockerfile, matching Docker's own last-wins semantics.""" + workdir: str | None = None + for line in dockerfile.read_text(encoding="utf-8").splitlines(): + stripped = line.strip() + if stripped.upper().startswith("WORKDIR "): + workdir = stripped.split(None, 1)[1].strip() + return workdir + + +def _write_verifier_task_yaml(task: TaskDefinition, out_dir: Path) -> None: + """``tests/task.yaml`` — the criteria, as authored. See the module docstring for the agent-type caveat.""" + payload: dict[str, object] = { + "task_id": task.task_id, + "description": task.description, + "agent": {"type": "claude-code"}, # placeholder; never instantiated (see module docstring) + "initial_prompt": _VERIFIER_PLACEHOLDER_PROMPT, + "success_criteria": [c.model_dump(mode="json", exclude_none=True) for c in task.success_criteria], + } + if task.reference is not None: + payload["reference"] = {"directory": "reference"} + if task.run_limits is not None: + # Carried through so `coder-eval evaluate` (run by tests/test.sh inside + # Harbor's verifier container) sees the same turn/token/USD caps the + # task author declared, rather than silently falling back to the + # packaged default experiment's -- grading itself has no agent loop to + # cap, but `run_limits.task_timeout` still bounds the verifier + # invocation, and a future criterion or judge call reading `run_limits` + # off the resolved task should see the real value, not the default. + payload["run_limits"] = task.run_limits.model_dump(mode="json", exclude_none=True) + if task.checker_context is not None: + # The judge-route override (`checker_context.api_route`) determines + # which backend an `llm_judge`/`agent_judge` criterion dispatches + # through at verify time -- dropping it silently replaces a pinned + # route with the verifier environment's own default. + payload["checker_context"] = task.checker_context.model_dump(mode="json", exclude_none=True) + (out_dir / "tests" / "task.yaml").write_text( + yaml.safe_dump(payload, sort_keys=False, allow_unicode=True), encoding="utf-8" + ) + + +def _copy_template_sources(task: TaskDefinition, env_dir: Path, warnings: list[str]) -> list[dict[str, object]] | None: + """Copy each ``TemplateDirSource``'s directory into ``environment/templates/-/`` + and return a rewritten ``template_sources`` list pointing at the in-container copy + (:data:`AGENT_TASK_TEMPLATES_DIR`), or ``None`` if the task has no template sources. + + ``TemplateDirSource.path`` is resolved to an absolute HOST path at task-load time + (``task_loader.resolve_template_source_paths``) -- baking that path verbatim into + ``environment/task.yaml`` would point the in-container agent at a directory that + doesn't exist there. Only ``TemplateDirSource`` is copied: ``RepoSource`` (clones at + runtime) and ``StarterFilesSource`` (inline file content) resolve entirely inside the + container already and are carried over unchanged. + """ + sources = task.sandbox.template_sources + if not sources: + return None + templates_dir = env_dir / "templates" + rewritten: list[dict[str, object]] = [] + for i, source in enumerate(sources): + dumped = source.model_dump(mode="json", exclude_none=True) + if isinstance(source, TemplateDirSource): + source_path = Path(source.path) # already absolute — load_task resolved it + dest_name = f"{i:02d}-{source_path.name}" + dest = templates_dir / dest_name + if not source_path.is_dir(): + # A hard failure, not a warning: the agent-phase task.yaml + # still references this template (starter files, or a pytest + # suite the prompt expects), so a silently-skipped copy ships + # an export whose agent has no starter code -- every criterion + # then reads "file does not exist" indistinguishable from a + # real agent failure (the exact CE039 anti-pattern, one layer + # up at the export boundary instead of the grading boundary). + raise TaskNotExportableError( + f"template_sources[{i}].path {source_path} is not a directory -- cannot copy it into the " + + "export. Fix the task's template_sources entry before exporting." + ) + if dest.exists(): + shutil.rmtree(dest) + # Symlinks dereferenced by default (`shutil.copytree`'s default + # `symlinks=False`) would write a symlink TARGET's content into the + # distributable export -- e.g. a `creds -> /root/.aws/credentials` + # plant. Drop symlinks outright rather than following them, same + # rule as the sibling reference copy below and every other + # task-authored-tree copy in `src/` (`orchestration/evaluation.py`, + # `isolation/docker_runner.py`, `evaluation/sub_agent.py`). + shutil.copytree(source_path, dest, ignore=ignore_patterns_and_symlinks(REFERENCE_COPY_IGNORE)) + dumped["path"] = f"{AGENT_TASK_TEMPLATES_DIR}/{dest_name}" + else: + warnings.append( + f"template_sources[{i}] ({type(source).__name__}) is not a TemplateDirSource -- carried over " + + "unchanged; only TemplateDirSource directories are copied into the export." + ) + rewritten.append(dumped) + return rewritten + + +def _write_agent_phase_task_yaml( + task: TaskDefinition, env_dir: Path, *, initial_prompt: str, warnings: list[str] +) -> bool: + """``environment/task.yaml`` — the REAL agent config, but criteria-free. + + Baked into the image at :data:`AGENT_TASK_YAML_PATH` (see the Dockerfile + ``COPY`` line in ``_write_environment``) for a ``CoderEvalAgent`` Harbor + agent embed (``harbor/agent.py``) to run via ``coder-eval execute --format + harbor``. Unlike ``tests/task.yaml`` (the verifier's placeholder-agent, + real-criteria file), this is the mirror image: ``task.agent`` and the + resolved prompt are carried over VERBATIM (the whole point is running the + task's actual configured agent), but ``success_criteria`` is forced to + ``[]`` — never leaked into the agent-visible image, and never read either + (`coder-eval execute` never grades). ``TaskDefinition`` no longer requires + at least one criterion, so this no longer needs a placeholder. + + ``sandbox`` is the original task's ``sandbox`` block, field-merged with + ``driver: tempdir`` and (when present) a rewritten ``template_sources`` — + everything else (``python.env_packages``, ``limits``, ...) is preserved, + not blanked. ``driver`` must be forced regardless of the original task's + driver: this file runs INSIDE the container Harbor already built, so + re-declaring ``driver: docker`` here would have ``coder-eval execute`` try + to launch a second, nested container rather than just using its own + in-process sandbox at the container's current workdir. ``docker`` config + is dropped along with it — moot once ``driver`` is forced to ``tempdir``. + + ``initial_prompt`` is omitted entirely for a ``type: none`` (agentless) + task: coder-eval's own schema forbids a no-op agent from setting a prompt + (no agent ever runs to read it) and raises at load time otherwise -- + verified live via ``harbor run`` against a real ``harbor`` install, which + surfaced exactly this ``TaskDefinition`` validation error before this + guard was added. + + Returns whether any template directory was copied into ``environment/templates/``, + so ``_write_environment`` knows whether to add the corresponding Dockerfile ``COPY``. + """ + is_agentless = task.agent is not None and task.agent.type == "none" + rewritten_template_sources = _copy_template_sources(task, env_dir, warnings) + sandbox_dict = task.sandbox.model_dump(mode="json", exclude_none=True) + sandbox_dict["driver"] = "tempdir" + sandbox_dict.pop("docker", None) + if rewritten_template_sources is not None: + sandbox_dict["template_sources"] = rewritten_template_sources + payload: dict[str, object] = { + "task_id": task.task_id, + "description": task.description, + "agent": ( + task.agent.model_dump(mode="json", exclude_none=True) if task.agent is not None else {"type": "claude-code"} + ), + "sandbox": sandbox_dict, + "success_criteria": [], + } + if not is_agentless: + payload["initial_prompt"] = initial_prompt + if task.run_limits is not None: + # `CoderEvalAgent.run()` invokes `coder-eval execute` against this + # file, which enforces `max_turns`/`turn_timeout`/`task_timeout`/the + # token+USD budget caps during the agent phase itself -- dropping this + # silently replaced a declared cap with the packaged default + # experiment's (`max_turns: 100`, `turn_timeout: 300`, no `max_usd` / + # token ceiling at all). + payload["run_limits"] = task.run_limits.model_dump(mode="json", exclude_none=True) + (env_dir / "task.yaml").write_text(yaml.safe_dump(payload, sort_keys=False, allow_unicode=True), encoding="utf-8") + return (env_dir / "templates").is_dir() + + +def _write_test_sh(out_dir: Path, *, workdir: str) -> None: + path = out_dir / "tests" / "test.sh" + # `workdir` comes from task-YAML-controlled `sandbox.docker.working_dir` + # (or a derived default), and the only validator on that field checks for a + # leading "/" -- it does not reject quotes, `$(...)`, backticks or + # newlines. shlex.quote it before interpolating into the generated /bin/sh + # script so a crafted working_dir can't break out of the argument it's + # meant to be. + path.write_text(_TEST_SH_TEMPLATE.format(workdir=shlex.quote(workdir)), encoding="utf-8") + path.chmod(0o755) + + +def _write_reference(task: TaskDefinition, task_file: Path, out_dir: Path) -> None: + if task.reference is None: + return + source = task_file.parent / task.reference.directory + dest = out_dir / "tests" / "reference" + if dest.exists(): + shutil.rmtree(dest) + # Same rule as the template copy above: drop symlinks rather than + # dereferencing them into the distributable export. Wrapped in + # TaskNotExportableError rather than left to raise a bare OSError: the + # `export` CLI only catches `(TaskNotExportableError, + # CriteriaNotExportableError)`, so an unreadable/missing reference tree + # would otherwise surface as an uncaught traceback (single-task path) or + # abort a whole experiment export the docstring promises it won't abort. + try: + shutil.copytree(source, dest, ignore=ignore_patterns_and_symlinks(REFERENCE_COPY_IGNORE)) + except OSError as e: + raise TaskNotExportableError( + f"Task {task.task_id!r}'s reference directory {source} could not be copied into the export: {e}" + ) from e + + +def _write_task_toml(task: TaskDefinition, out_dir: Path, *, workdir: str) -> None: + verifier_section: dict[str, object] = {} + env_names = _env_passthrough_names(task) + if env_names: + verifier_section["env"] = _env_template_dict(env_names) + if task.run_limits is not None and task.run_limits.task_timeout is not None: + timeout = float(task.run_limits.task_timeout) + verifier_section["timeout_sec"] = timeout + + doc: dict[str, object] = { + "schema_version": _HARBOR_SCHEMA_VERSION, + "task": { + "name": f"coder-eval/{task.task_id}", + "description": task.description, + "keywords": list(task.tags), + }, + "environment": _build_environment_section(task, workdir=workdir), + } + if verifier_section: + doc["verifier"] = verifier_section + if task.run_limits is not None and task.run_limits.task_timeout is not None: + doc["agent"] = {"timeout_sec": float(task.run_limits.task_timeout)} + (out_dir / "task.toml").write_bytes(tomli_w.dumps(doc).encode("utf-8")) + + +_HARBOR_ENV_PASSTHROUGH_EXCLUDE = { + # `HOME` is intentional in `env_passthrough`'s default ONLY because + # `docker_runner.py` also bind-mounts the host's `~/.claude` into the + # container at that same path, so the host `HOME` value still resolves to + # a real, populated directory there (see docs/DOCKER_ISOLATION.md). Harbor + # builds its own container with no such mount, so forwarding the host's + # literal `HOME` (e.g. `/Users/alice`) would point the container at a + # directory that doesn't exist in it -- a real regression, not a no-op. + "HOME", +} + + +def _env_passthrough_names(task: TaskDefinition) -> list[str]: + """The env var NAMES this task's docker driver would forward on the host (``docker_runner.py``'s + ``merged_allowlist``), reused here as the SSOT for which credentials a Harbor export's + agent/verifier containers need declared -- rather than a second hardcoded list drifting + from that one. Minus :data:`_HARBOR_ENV_PASSTHROUGH_EXCLUDE`, entries whose coder-eval-driver + behavior does not carry over to a Harbor-built container. + """ + docker_cfg = task.sandbox.docker + names = set(docker_cfg.env_passthrough) | set(docker_cfg.env_passthrough_extra) + return sorted(names - _HARBOR_ENV_PASSTHROUGH_EXCLUDE) + + +def _env_template_dict(names: list[str]) -> dict[str, str]: + """``{NAME: "${NAME:-}"}`` -- Harbor's ``${VAR:-default}`` host-env-passthrough template + syntax (see ``harbor.utils.env.resolve_env_vars``): a NAME only, never a value, so no + secret is ever written into the exported ``task.toml``. + + The ``:-`` (empty default) is load-bearing, not cosmetic: a bare ``${VAR}`` makes Harbor + treat the var as REQUIRED (``get_required_host_vars`` -> ``_confirm_host_env_access`` + hard-exits with "Missing Environment Variables" for every name absent from the invoking + shell, confirmed live) -- and `env_passthrough`'s default list names ~24 vars spanning + every backend/agent this task might not even use (Codex, Antigravity, Pi, LiteLLM, UiPath + ...). Requiring an operator to export all of them just to run a Bedrock-only task is + exactly the friction this fixes; `${VAR:-}` resolves to an empty string when unset instead, + so Harbor only forwards what's actually set. + """ + return {name: f"${{{name}:-}}" for name in names} + + +def _build_environment_section(task: TaskDefinition, *, workdir: str) -> dict[str, object]: + """No ``docker_image`` key: ``_write_environment`` always writes ``environment/Dockerfile`` + + now (either copied from ``dockerfile_path`` or synthesized from + ``docker_cfg.image``), so Harbor always builds from that file rather than + pulling a bare image reference named in ``task.toml``. + """ + docker_cfg = task.sandbox.docker + section: dict[str, object] = {"workdir": workdir} + limits = task.sandbox.limits + if limits.max_memory_mb is not None: + section["memory_mb"] = limits.max_memory_mb + if limits.max_cpus is not None: + section["cpus"] = max(1, round(limits.max_cpus)) + section["network_mode"] = "no-network" if docker_cfg.network == "none" else "public" + env_names = _env_passthrough_names(task) + if env_names: + section["env"] = _env_template_dict(env_names) + return section + + +__all__ = [ + "DEFAULT_WORKDIR", + "CriteriaNotExportableError", + "ExportResult", + "TaskNotExportableError", + "export_resolved_task", + "export_task", +] diff --git a/src/coder_eval/harbor/portability.py b/src/coder_eval/harbor/portability.py new file mode 100644 index 00000000..41896629 --- /dev/null +++ b/src/coder_eval/harbor/portability.py @@ -0,0 +1,149 @@ +"""C1.4 — criteria portability audit for the Harbor export direction. + +Not every criterion type can grade truthfully inside a verifier container +that another harness built. This module classifies each of the 15 criterion +types and lets the packager (C2) refuse an unsupported task **at export +time**, where the operator sees why, rather than at verify time, where the +failure is an unexplained low reward with no obvious cause. + +Classification, v1: + +- ``PORTABLE`` — filesystem/exit-code checks. Nothing about the verifier + container changes what these need: ``file_exists``, ``file_contains``, + ``file_matches_regex``, ``json_check``, ``file_check``, ``run_command``, + ``classification_match``. +- ``NEEDS_REFERENCE`` — ``reference_comparison``. Needs the reference tree, + which C2 places under ``tests/reference/`` (verifier-side only, never in + the agent's view — see C2's mapping table). +- ``NEEDS_TRAJECTORY`` — ``command_executed``, ``commands_efficiency``, + ``skill_triggered``. These read coder-eval's own ``TurnRecord`` iterations. + In the export direction the verifier is a separate process from the agent + phase, and the agent may not even be coder-eval (Harbor natively supports + many agents) — so there is no ``iterations`` list to read from without + C1.3 (ATIF ingestion + ``evaluate --trajectory``), which is not built yet. + Hard-error until it is. +- ``NEEDS_CLI_RECORDER`` — ``cli_called``. Reads coder-eval's own JSON Lines + invocation log (``invocation_log.py``), written by a recorder shim + coder-eval's OWN sandbox setup installs into `PATH` + (``_generate_cli_recorders``). The exported Dockerfile does not provision + that shim. Hard-error until C2 learns to bake it in. +- ``NEEDS_CREDENTIALS`` — ``llm_judge``, ``agent_judge``, ``uipath_eval``. + Need model credentials and network reachable from inside the verifier + container, and the judge must not follow the agent's own route (the old + note's blocking issues). Hard-error in v1, with an explicit opt-in escape + hatch for an operator who has already provisioned that themselves. +""" + +from __future__ import annotations + +import enum +from dataclasses import dataclass + +from coder_eval.models import SuccessCriterion + + +class CriterionPortability(enum.Enum): + """How (or whether) a criterion type can grade inside a Harbor verifier container.""" + + PORTABLE = "portable" + NEEDS_REFERENCE = "needs_reference" + NEEDS_TRAJECTORY = "needs_trajectory" + NEEDS_CLI_RECORDER = "needs_cli_recorder" + NEEDS_CREDENTIALS = "needs_credentials" + + +_PORTABILITY_BY_TYPE: dict[str, CriterionPortability] = { + "file_exists": CriterionPortability.PORTABLE, + "file_contains": CriterionPortability.PORTABLE, + "file_matches_regex": CriterionPortability.PORTABLE, + "json_check": CriterionPortability.PORTABLE, + "file_check": CriterionPortability.PORTABLE, + "run_command": CriterionPortability.PORTABLE, + "classification_match": CriterionPortability.PORTABLE, + "reference_comparison": CriterionPortability.NEEDS_REFERENCE, + "command_executed": CriterionPortability.NEEDS_TRAJECTORY, + "commands_efficiency": CriterionPortability.NEEDS_TRAJECTORY, + "skill_triggered": CriterionPortability.NEEDS_TRAJECTORY, + "cli_called": CriterionPortability.NEEDS_CLI_RECORDER, + "llm_judge": CriterionPortability.NEEDS_CREDENTIALS, + "agent_judge": CriterionPortability.NEEDS_CREDENTIALS, + "uipath_eval": CriterionPortability.NEEDS_CREDENTIALS, +} + +# Registry-derived coverage: fails closed on a 16th criterion type added to the +# union without a portability classification, rather than silently exporting +# it as if it were PORTABLE. +_KNOWN_CRITERION_TYPES = frozenset(_PORTABILITY_BY_TYPE) + +# Which non-PORTABLE classes v1 refuses to export outright (vs. tolerating +# with a caveat, like NEEDS_REFERENCE — C2 always emits tests/reference/ when +# task.reference is set, so that class is never actually blocking). +_BLOCKING_IN_V1 = frozenset( + { + CriterionPortability.NEEDS_TRAJECTORY, + CriterionPortability.NEEDS_CLI_RECORDER, + CriterionPortability.NEEDS_CREDENTIALS, + } +) + + +class UnknownCriterionTypeError(Exception): + """A criterion type has no portability classification — fail closed, not open.""" + + +@dataclass(frozen=True) +class PortabilityIssue: + """One criterion this export cannot grade truthfully inside Harbor's verifier.""" + + criterion_description: str + criterion_type: str + portability: CriterionPortability + + +def classify(criterion_type: str) -> CriterionPortability: + """Look up a single criterion type's portability, raising on an unclassified type.""" + try: + return _PORTABILITY_BY_TYPE[criterion_type] + except KeyError: + raise UnknownCriterionTypeError( + f"Criterion type {criterion_type!r} has no Harbor-export portability classification " + + f"(known types: {sorted(_KNOWN_CRITERION_TYPES)}). Classify it in " + + "coder_eval.harbor.portability before exporting a task that uses it." + ) from None + + +def audit_criteria( + criteria: list[SuccessCriterion], + *, + allow_credentials: bool = False, +) -> list[PortabilityIssue]: + """Return every criterion this v1 export would refuse, or ``[]`` if the task exports cleanly. + + ``allow_credentials`` is the escape hatch for ``NEEDS_CREDENTIALS`` criteria + (``llm_judge`` / ``agent_judge`` / ``uipath_eval``) — an operator who has + already provisioned model credentials and network access inside the + verifier container may pass it to export anyway. It does not affect + ``NEEDS_TRAJECTORY`` or ``NEEDS_CLI_RECORDER``, which are missing + functionality (C1.3, a recorder-baking step in C2), not a missing + permission — no flag can supply what does not exist yet. + """ + issues: list[PortabilityIssue] = [] + for c in criteria: + portability = classify(c.type) + if portability not in _BLOCKING_IN_V1: + continue + if portability is CriterionPortability.NEEDS_CREDENTIALS and allow_credentials: + continue + issues.append( + PortabilityIssue(criterion_description=c.description, criterion_type=c.type, portability=portability) + ) + return issues + + +__all__ = [ + "CriterionPortability", + "PortabilityIssue", + "UnknownCriterionTypeError", + "audit_criteria", + "classify", +] diff --git a/src/coder_eval/harbor/reward.py b/src/coder_eval/harbor/reward.py new file mode 100644 index 00000000..265028ca --- /dev/null +++ b/src/coder_eval/harbor/reward.py @@ -0,0 +1,102 @@ +"""Translate a graded coder-eval run into Harbor's reward-file contract. + +Harbor's verifier (``harbor 0.22.0``, verified against source — see +``tmp/harborframework.md`` § C0) reads ``/logs/verifier/reward.json`` (a flat +``dict[str, float]``) or falls back to ``/logs/verifier/reward.txt`` (a bare +float, synthesized into the single key ``"reward"``). It does NOT inspect the +verifier script's exit code — only whether the reward file exists, is +non-empty, and parses. A missing/empty/malformed file raises inside Harbor's +own ``Verifier.verify()`` (``RewardFileNotFoundError`` / +``RewardFileEmptyError`` / ``VerifierOutputParseError``), which Harbor's +``Trial.run()`` catches: it records the exception on ``TrialResult`` and +leaves ``TrialResult.verifier_result`` at its default ``None`` — never +coalesced to a zero-reward object. That is Harbor's own infra-vs-policy split, +already built. + +This module's whole job, therefore, is: **write the file, or don't.** + +The "don't" case is not a degenerate corner — it is the load-bearing one. An +unmeasured row (``weighted_score is None`` — an ungraded row, or a task.json +that failed to load at all) must not become ``reward=0.0``: that would train +"the agent's behaviour was bad" from a measurement that never happened. Not +writing the file lets Harbor's own missing-reward path mask the trial +instead — the same principle as CE049 (never coalesce a possibly-unmeasured +score to a numeric literal), one level up, at the artifact-writing boundary +rather than the in-process one. + +A grading-time INFRASTRUCTURE failure is the same case in disguise: +``EvaluationResult.calculate_weighted_score`` (``models/results.py``) +short-circuits an empty ``success_criteria_results`` list to a hard ``0.0``, +not ``None`` -- so a checker that raises ``JudgeInfrastructureError`` / +``CheckerMisuseError`` / ``ReferenceTamperedError`` (escalating exceptions +that deliberately propagate out of grading rather than being captured into a +scored-0.0 result) finalizes the row ``FinalStatus.ERROR`` with +``weighted_score == 0.0``, not ``None``. That is not a measurement either, so +it gets the same "write nothing" treatment via ``final_status.category == +"error"``. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from coder_eval.orchestration.regrade import RegradeError, load_prior_result +from coder_eval.path_utils import write_text_atomic + + +class RewardWriteSkippedError(Exception): + """The run carries no measured verdict — the caller must write no reward file. + + Raised for an ungraded row (``weighted_score is None``, e.g. ``execute`` + left it ``NOT_GRADED``, or the row crashed before grading). Distinct from + ``RegradeError`` (also propagated by this module) only in *why* nothing + can be written — both are infrastructure failures with the identical + "write nothing" contract; keeping them separate types documents which + failure mode actually occurred without changing what the caller must do. + """ + + +def compute_reward(run_dir: Path) -> dict[str, float]: + """Read a graded run's ``task.json`` and derive Harbor's reward dict. + + Reuses ``load_prior_result`` (the same reader ``evaluate `` / + `run --resume`` use) rather than re-implementing ``task.json`` loading — + a second reader is how two copies of "what does this run's outcome mean" + drift into two different verdicts for the same run. + + Raises: + RegradeError: ``task.json`` is missing or does not parse as an + ``EvaluationResult`` (propagated from ``load_prior_result`` — an + infrastructure failure, same "write nothing" contract as below). + RewardWriteSkippedError: the row's ``weighted_score`` is ``None`` — an + unmeasured row, never to be coalesced to ``0.0`` — or its + ``final_status.category`` is ``"error"``, a grading-time + infrastructure crash that also carries no real measurement despite + ``calculate_weighted_score`` writing a literal ``0.0`` for it. + """ + result = load_prior_result(run_dir) + if result.weighted_score is None or result.final_status.category == "error": + raise RewardWriteSkippedError( + f"{run_dir} carries no measured verdict (final_status={result.final_status.value!r}, " + + f"weighted_score={result.weighted_score!r}); writing no reward file so Harbor's own " + + "missing-reward path masks the trial instead of scoring it 0.0" + ) + return {"reward": result.weighted_score} + + +def write_reward(run_dir: Path, out_path: Path) -> dict[str, float]: + """Write Harbor's ``reward.json`` at ``out_path``. + + Must never be called when ``compute_reward`` would raise — the caller + (the ``coder-eval harbor reward`` CLI command) is expected to let + ``RewardWriteSkippedError`` / ``RegradeError`` propagate uncaught rather than + catching them here and writing a file anyway. + """ + rewards = compute_reward(run_dir) + out_path.parent.mkdir(parents=True, exist_ok=True) + write_text_atomic(out_path, json.dumps(rewards, indent=2) + "\n") + return rewards + + +__all__ = ["RegradeError", "RewardWriteSkippedError", "compute_reward", "write_reward"] diff --git a/src/coder_eval/models/tasks.py b/src/coder_eval/models/tasks.py index bb60e743..f1f36cf1 100644 --- a/src/coder_eval/models/tasks.py +++ b/src/coder_eval/models/tasks.py @@ -792,11 +792,3 @@ def check_removed_criteria_types(cls, v: Any) -> Any: normalized.append(item) return normalized return v - - @field_validator("success_criteria") - @classmethod - def validate_success_criteria(cls, v: Any) -> Any: - """Ensure at least one success criterion is defined.""" - if not v: - raise ValueError("At least one success criterion must be defined") - return v diff --git a/src/coder_eval/orchestration/batch.py b/src/coder_eval/orchestration/batch.py index 636156bb..2dc2379d 100644 --- a/src/coder_eval/orchestration/batch.py +++ b/src/coder_eval/orchestration/batch.py @@ -117,6 +117,19 @@ async def run_batch( start_time = datetime.now() + if config.workspace_dir is not None: + if len(resolved_tasks) != 1: + raise ValueError( + f"--workspace-dir requires exactly one resolved task; got {len(resolved_tasks)}. " + + "It names a single in-place directory every task would otherwise collide on." + ) + (_workspace_dir_task,) = resolved_tasks + if _workspace_dir_task.task.sandbox.driver == "docker": + raise ValueError( + "--workspace-dir is not for sandbox.driver: docker tasks -- the docker driver already " + + "aligns automatically via sandbox.docker.working_dir (see DockerRunner)." + ) + check_pricing_coverage(resolved_tasks) if on_batch_start is not None: @@ -187,6 +200,7 @@ async def run_single(rt: ResolvedTask) -> TaskResult: config_lineage=rt.config_lineage, replicate_index=rt.replicate_index, grade=config.grade, + workspace_dir=config.workspace_dir, ) result = await orchestrator.run() tr = TaskResult( diff --git a/src/coder_eval/orchestration/config.py b/src/coder_eval/orchestration/config.py index 80a3d812..0a287fbd 100644 --- a/src/coder_eval/orchestration/config.py +++ b/src/coder_eval/orchestration/config.py @@ -111,6 +111,24 @@ class BatchRunConfig(BaseModel): # Logging verbose: bool = Field(default=False, description="Enable verbose (DEBUG level) logging for Docker output") + # Docker WORKDIR alignment for the non-docker-driver dispatch path (host + # process, or already inside a container someone else built — e.g. a Harbor + # trial container running `coder-eval execute` as its agent). Mirrors what + # `DockerRunner`/`_run-task-internal` already do for `sandbox.driver: docker` + # (see `Orchestrator.workspace_dir`'s docstring); this is the same mechanism, + # exposed publicly for the case where coder-eval's OWN docker driver isn't + # the one building the container. Only meaningful for a single resolved task + # — `run_batch` raises if more than one task would collide on it. + workspace_dir: Path | None = Field( + default=None, + description=( + "Run the agent in-place at this absolute path instead of the standard " + "run_dir/artifacts workspace, copying it out to run_dir/artifacts/ at " + "cleanup. For a single task only. Not for sandbox.driver: docker tasks — " + "the docker driver already aligns automatically via sandbox.docker.working_dir." + ), + ) + # TODO(container-death-diagnostics): consider a run-level default resource # cap. Containers run uncapped today (sandbox.limits.{max_memory_mb, # max_cpus,max_pids} default to None -> _build_argv emits no --memory/ diff --git a/src/coder_eval/orchestration/regrade.py b/src/coder_eval/orchestration/regrade.py index 70526d3f..ae4b19aa 100644 --- a/src/coder_eval/orchestration/regrade.py +++ b/src/coder_eval/orchestration/regrade.py @@ -1128,6 +1128,21 @@ async def regrade_in_place( """ from coder_eval.orchestrator import Orchestrator + # Every path through this function grades, so an empty `success_criteria` + # is never legal here the way it is for `execute` (which never calls this + # function at all). Checked once, at the single choke point every re-grade + # entry point (`evaluate `, `run --resume`'s `to_grade` set, and + # the container-dispatch branch below) shares -- a criteria-free task would + # otherwise finalize as `FinalStatus.SUCCESS` at `weighted_score: 0.0` + # (`all_criteria_passed([])` is vacuously `True`, + # `calculate_weighted_score([])` writes `0.0`), an internally contradictory + # "successful" result for what is actually a misconfigured task. + if not task.success_criteria: + raise RegradeError( + f"task {task.task_id!r} has no `success_criteria` and cannot be graded (it would silently " + + "score SUCCESS at weighted_score 0.0). Add at least one criterion before re-grading it." + ) + # A `driver: docker` row is graded INSIDE a container of the same image, # which is the only place its criteria mean what they meant during the run. # Dispatched before anything else here, including the reference check, so the diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index 17f969bd..cf02242e 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -35,6 +35,7 @@ from .models import ( CONTAINER_REFERENCE_DIR, DEFAULT_STOP_EARLY_GATE_THRESHOLD, + IN_CONTAINER_ENV, ROUTE_NAMES, AgentJudgeCriterion, AgentKind, @@ -1650,11 +1651,19 @@ async def _setup(self) -> None: # DIRECT_WRITE deliberately does NOT clear the target dir, so a reused # --run-dir (or --resume) can leave a prior run's files alongside this # run's outputs and silently perturb file-based criteria. Surface it. - # Skipped in workspace_dir mode: a WORKDIR (/root, /app) legitimately holds - # the image's baked inputs — not stale prior-run files — so the warning - # would fire on essentially every run and cry wolf. + # Suppressed in workspace_dir mode ONLY inside a container + # (IN_CONTAINER_ENV, per CE056 -- never on the field itself): the + # original writer was exclusively `run_task_internal_command`, where + # the WORKDIR is a fresh container filesystem every run, so a WORKDIR + # (/root, /app) legitimately holds the image's baked inputs there, not + # stale prior-run files. `--workspace-dir` is now also a host-reachable + # CLI flag on `run`/`execute`, where the named directory persists + # across invocations exactly like DIRECT_WRITE's own target -- keying + # the suppression on `workspace_dir is None` silently disabled the + # warning on precisely the new path where it is needed. + in_container = os.environ.get(IN_CONTAINER_ENV) == "1" if ( - self.workspace_dir is None + not (self.workspace_dir is not None and in_container) and direct_target is not None and direct_target.exists() and any(direct_target.iterdir()) @@ -2569,13 +2578,13 @@ async def _run_dialog_criteria_check( # RAISES rather than returning an empty list, matching the evaluate-only # path's refusal. The empty-list version described itself as a # "defensive no-op so the gate holds", and it was neither: both callers - # go straight on to `all_criteria_passed`, whose first act is a - # length pre-check that raises on a mismatch — and an empty criteria - # list is forbidden by `TaskDefinition.validate_success_criteria`, so - # the mismatch was guaranteed. If the simulation restriction ever lifts, - # that "no-op" turns every ungraded dialog into FinalStatus.ERROR. A - # loud refusal here is honest about the fact that this path has no - # ungraded semantics yet. + # go straight on to `all_criteria_passed`/`calculate_weighted_score`, + # which treat an empty criteria list as a vacuous pass/0.0 rather than + # raising, so a silent no-op here would produce a criteria-free dialog + # that scores as though nothing had been asked of the agent. If the + # simulation restriction ever lifts, that "no-op" turns every ungraded + # dialog into a silently-passing one. A loud refusal here is honest + # about the fact that this path has no ungraded semantics yet. if not self.grade: raise ValueError( "Grading is disabled but the simulation dialog path requires criteria results to " diff --git a/tests/_fixtures/harbor_export_golden/expected/environment/Dockerfile b/tests/_fixtures/harbor_export_golden/expected/environment/Dockerfile new file mode 100644 index 00000000..5a55d1ce --- /dev/null +++ b/tests/_fixtures/harbor_export_golden/expected/environment/Dockerfile @@ -0,0 +1,6 @@ +FROM ubuntu:24.04 +RUN apt-get update && apt-get install -y --no-install-recommends coreutils && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY task.yaml /opt/coder-eval-task/task.yaml diff --git a/tests/_fixtures/harbor_export_golden/expected/environment/task.yaml b/tests/_fixtures/harbor_export_golden/expected/environment/task.yaml new file mode 100644 index 00000000..d45b9d7d --- /dev/null +++ b/tests/_fixtures/harbor_export_golden/expected/environment/task.yaml @@ -0,0 +1,25 @@ +task_id: golden_export_example +description: A canonical task exercising every C2 mapping row at once (golden fixture + — do not edit casually). +agent: + type: claude-code + permission_mode: acceptEdits + ignore_patterns: [] + system_prompt_mode: append + sdk_options: {} +sandbox: + driver: tempdir + python: + env_packages: [] + limits: + timeout: 300 + max_memory_mb: 1024 + max_cpus: 1.5 + ignore_patterns: [] +success_criteria: [] +initial_prompt: Write 'hello world' to greeting.txt, matching reference/greeting.txt. +run_limits: + task_timeout: 300 + count_cached_input: false + count_cache_creation: false + stop_early_gate_threshold: 1.0 diff --git a/tests/_fixtures/harbor_export_golden/expected/instruction.md b/tests/_fixtures/harbor_export_golden/expected/instruction.md new file mode 100644 index 00000000..89318b6c --- /dev/null +++ b/tests/_fixtures/harbor_export_golden/expected/instruction.md @@ -0,0 +1,9 @@ +This is a coder-eval task, exported to Harbor. + +This file is autogenerated -- do not edit it directly. The task's real +instruction and agent configuration live in environment/task.yaml, and +must be run with the CoderEvalAgent Harbor agent: + + harbor run -a coder_eval.harbor.agent:CoderEvalAgent -p + +See https://coder-eval.com for details on the coder-eval <-> Harbor integration. diff --git a/tests/_fixtures/harbor_export_golden/expected/task.toml b/tests/_fixtures/harbor_export_golden/expected/task.toml new file mode 100644 index 00000000..1ae65469 --- /dev/null +++ b/tests/_fixtures/harbor_export_golden/expected/task.toml @@ -0,0 +1,73 @@ +schema_version = "1.4" + +[task] +name = "coder-eval/golden_export_example" +description = "A canonical task exercising every C2 mapping row at once (golden fixture — do not edit casually)." +keywords = [ + "golden", + "harbor-export", +] + +[environment] +workdir = "/app" +memory_mb = 1024 +cpus = 2 +network_mode = "no-network" + +[environment.env] +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY:-}" +ANTHROPIC_MODEL = "${ANTHROPIC_MODEL:-}" +ANTIGRAVITY_MODEL = "${ANTIGRAVITY_MODEL:-}" +API_BACKEND = "${API_BACKEND:-}" +AWS_BEARER_TOKEN_BEDROCK = "${AWS_BEARER_TOKEN_BEDROCK:-}" +AWS_REGION = "${AWS_REGION:-}" +BEDROCK_MODEL = "${BEDROCK_MODEL:-}" +CLAUDE_CODE_USE_BEDROCK = "${CLAUDE_CODE_USE_BEDROCK:-}" +CODEX_API_KEY = "${CODEX_API_KEY:-}" +CODEX_BASE_URL = "${CODEX_BASE_URL:-}" +CODEX_MODEL = "${CODEX_MODEL:-}" +GEMINI_API_KEY = "${GEMINI_API_KEY:-}" +LITELLM_AUTH_TOKEN = "${LITELLM_AUTH_TOKEN:-}" +LITELLM_BASE_URL = "${LITELLM_BASE_URL:-}" +LITELLM_COST_LOG = "${LITELLM_COST_LOG:-}" +LITELLM_MODEL = "${LITELLM_MODEL:-}" +LITELLM_SMALL_MODEL = "${LITELLM_SMALL_MODEL:-}" +OPENROUTER_API_KEY = "${OPENROUTER_API_KEY:-}" +UIPATH_ACCESS_TOKEN = "${UIPATH_ACCESS_TOKEN:-}" +UIPATH_CLI_DISABLE_VERSION_SYNC = "${UIPATH_CLI_DISABLE_VERSION_SYNC:-}" +UIPATH_LLM_BACKEND = "${UIPATH_LLM_BACKEND:-}" +UIPATH_ORGANIZATION_ID = "${UIPATH_ORGANIZATION_ID:-}" +UIPATH_TENANT_ID = "${UIPATH_TENANT_ID:-}" +UIPATH_URL = "${UIPATH_URL:-}" + +[verifier] +timeout_sec = 300.0 + +[verifier.env] +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY:-}" +ANTHROPIC_MODEL = "${ANTHROPIC_MODEL:-}" +ANTIGRAVITY_MODEL = "${ANTIGRAVITY_MODEL:-}" +API_BACKEND = "${API_BACKEND:-}" +AWS_BEARER_TOKEN_BEDROCK = "${AWS_BEARER_TOKEN_BEDROCK:-}" +AWS_REGION = "${AWS_REGION:-}" +BEDROCK_MODEL = "${BEDROCK_MODEL:-}" +CLAUDE_CODE_USE_BEDROCK = "${CLAUDE_CODE_USE_BEDROCK:-}" +CODEX_API_KEY = "${CODEX_API_KEY:-}" +CODEX_BASE_URL = "${CODEX_BASE_URL:-}" +CODEX_MODEL = "${CODEX_MODEL:-}" +GEMINI_API_KEY = "${GEMINI_API_KEY:-}" +LITELLM_AUTH_TOKEN = "${LITELLM_AUTH_TOKEN:-}" +LITELLM_BASE_URL = "${LITELLM_BASE_URL:-}" +LITELLM_COST_LOG = "${LITELLM_COST_LOG:-}" +LITELLM_MODEL = "${LITELLM_MODEL:-}" +LITELLM_SMALL_MODEL = "${LITELLM_SMALL_MODEL:-}" +OPENROUTER_API_KEY = "${OPENROUTER_API_KEY:-}" +UIPATH_ACCESS_TOKEN = "${UIPATH_ACCESS_TOKEN:-}" +UIPATH_CLI_DISABLE_VERSION_SYNC = "${UIPATH_CLI_DISABLE_VERSION_SYNC:-}" +UIPATH_LLM_BACKEND = "${UIPATH_LLM_BACKEND:-}" +UIPATH_ORGANIZATION_ID = "${UIPATH_ORGANIZATION_ID:-}" +UIPATH_TENANT_ID = "${UIPATH_TENANT_ID:-}" +UIPATH_URL = "${UIPATH_URL:-}" + +[agent] +timeout_sec = 300.0 diff --git a/tests/_fixtures/harbor_export_golden/expected/tests/reference/greeting.txt b/tests/_fixtures/harbor_export_golden/expected/tests/reference/greeting.txt new file mode 100644 index 00000000..3b18e512 --- /dev/null +++ b/tests/_fixtures/harbor_export_golden/expected/tests/reference/greeting.txt @@ -0,0 +1 @@ +hello world diff --git a/tests/_fixtures/harbor_export_golden/expected/tests/task.yaml b/tests/_fixtures/harbor_export_golden/expected/tests/task.yaml new file mode 100644 index 00000000..2cb1dca6 --- /dev/null +++ b/tests/_fixtures/harbor_export_golden/expected/tests/task.yaml @@ -0,0 +1,28 @@ +task_id: golden_export_example +description: A canonical task exercising every C2 mapping row at once (golden fixture + — do not edit casually). +agent: + type: claude-code +initial_prompt: (unused placeholder — this file is graded via `coder-eval evaluate`, + which does not invoke an agent on this path) +success_criteria: +- type: file_exists + description: greeting.txt exists + weight: 1.0 + pass_threshold: 0.9 + path: greeting.txt +- type: reference_comparison + description: matches the reference greeting + weight: 1.0 + pass_threshold: 0.8 + agent_file: greeting.txt + reference_file: greeting.txt + comparison_method: ast + similarity_threshold: 0.8 +reference: + directory: reference +run_limits: + task_timeout: 300 + count_cached_input: false + count_cache_creation: false + stop_early_gate_threshold: 1.0 diff --git a/tests/_fixtures/harbor_export_golden/expected/tests/test.sh b/tests/_fixtures/harbor_export_golden/expected/tests/test.sh new file mode 100755 index 00000000..f5aa589c --- /dev/null +++ b/tests/_fixtures/harbor_export_golden/expected/tests/test.sh @@ -0,0 +1,17 @@ +#!/bin/sh +# Generated by `coder-eval export --format harbor` — do not hand-edit; a +# regenerated export will overwrite it. See tmp/harborframework.md § C1.1. +# +# NOT `set -e`: `coder-eval evaluate` exits non-zero whenever any criterion +# fails its gate -- a genuine MEASURED verdict, not an infrastructure +# failure. `set -e` here would abort before `coder-eval harbor reward` ever +# ran, turning every real failing score into a masked (unmeasured) trial -- +# confirmed live: a solution that runs but produces the wrong content +# scored weighted_score=0.500 in task.json, yet `set -e` discarded it and +# Harbor reported RewardFileNotFoundError instead of reward=0.5. The reward +# writer is what decides infra-vs-policy (None -> no file); this script must +# always reach it. +set -u + +coder-eval evaluate /tests/task.yaml "/app" --in-place --run-dir /logs/verifier || true +coder-eval harbor reward /logs/verifier --out /logs/verifier/reward.json diff --git a/tests/_fixtures/harbor_export_golden/source_task/environment/Dockerfile b/tests/_fixtures/harbor_export_golden/source_task/environment/Dockerfile new file mode 100644 index 00000000..5a792680 --- /dev/null +++ b/tests/_fixtures/harbor_export_golden/source_task/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM ubuntu:24.04 +RUN apt-get update && apt-get install -y --no-install-recommends coreutils && rm -rf /var/lib/apt/lists/* diff --git a/tests/_fixtures/harbor_export_golden/source_task/reference/greeting.txt b/tests/_fixtures/harbor_export_golden/source_task/reference/greeting.txt new file mode 100644 index 00000000..3b18e512 --- /dev/null +++ b/tests/_fixtures/harbor_export_golden/source_task/reference/greeting.txt @@ -0,0 +1 @@ +hello world diff --git a/tests/_fixtures/harbor_export_golden/source_task/task.yaml b/tests/_fixtures/harbor_export_golden/source_task/task.yaml new file mode 100644 index 00000000..b45f53cf --- /dev/null +++ b/tests/_fixtures/harbor_export_golden/source_task/task.yaml @@ -0,0 +1,33 @@ +task_id: golden_export_example +description: A canonical task exercising every C2 mapping row at once (golden fixture — do not edit casually). +tags: [golden, harbor-export] + +agent: + type: claude-code + +sandbox: + driver: docker + docker: + dockerfile_path: environment/Dockerfile + network: none + limits: + max_memory_mb: 1024 + max_cpus: 1.5 + +initial_prompt: Write 'hello world' to greeting.txt, matching reference/greeting.txt. + +reference: + directory: reference + +run_limits: + task_timeout: 300 + +success_criteria: + - type: file_exists + path: greeting.txt + description: greeting.txt exists + + - type: reference_comparison + agent_file: greeting.txt + reference_file: greeting.txt + description: matches the reference greeting diff --git a/tests/fixtures/atif/known_good_trajectory.json b/tests/fixtures/atif/known_good_trajectory.json new file mode 100644 index 00000000..6a7b013f --- /dev/null +++ b/tests/fixtures/atif/known_good_trajectory.json @@ -0,0 +1,125 @@ +{ + "schema_version": "ATIF-v1.7", + "session_id": "hello_date_smoke_test/default", + "agent": { + "name": "claude-code", + "version": "0.8.4", + "model_name": "claude-sonnet-4-6" + }, + "steps": [ + { + "step_id": 1, + "timestamp": "2026-07-20T10:00:00+00:00", + "source": "user", + "message": "Create a Python file named app.py that prints hello.", + "extra": {"iteration": 1} + }, + { + "step_id": 2, + "timestamp": "2026-07-20T10:00:05+00:00", + "source": "agent", + "model_name": "claude-sonnet-4-6", + "message": "I'll create app.py now.", + "reasoning_content": "The user wants a simple script; Write then run it.", + "tool_calls": [ + { + "tool_call_id": "toolu_01", + "function_name": "Write", + "arguments": {"file_path": "app.py", "content": "print('hello')"} + } + ], + "observation": { + "results": [ + { + "source_call_id": "toolu_01", + "content": "File created successfully at: app.py", + "extra": {"result_status": "success", "duration_ms": 12.5} + } + ] + }, + "metrics": { + "prompt_tokens": 2048, + "completion_tokens": 150, + "cached_tokens": 1024 + }, + "extra": {"iteration": 1, "cache_creation_tokens": 512, "reasoning_tokens": 40} + }, + { + "step_id": 3, + "timestamp": "2026-07-20T10:00:09+00:00", + "source": "agent", + "model_name": "claude-sonnet-4-6", + "message": [ + {"type": "text", "text": "Delegating verification to a sub-agent."} + ], + "tool_calls": [ + { + "tool_call_id": "toolu_02", + "function_name": "Agent", + "arguments": {"prompt": "Verify app.py runs"} + } + ], + "observation": { + "results": [ + { + "source_call_id": "toolu_02", + "content": "Verified: app.py prints hello", + "subagent_trajectory_ref": [ + {"trajectory_id": "toolu_02"} + ] + } + ] + }, + "metrics": { + "prompt_tokens": 2300, + "completion_tokens": 60, + "cached_tokens": 2048 + }, + "extra": {"iteration": 1} + } + ], + "final_metrics": { + "total_prompt_tokens": 4948, + "total_completion_tokens": 260, + "total_cached_tokens": 3472, + "total_cost_usd": 0.0421, + "total_steps": 3 + }, + "extra": { + "reconciliation": [ + {"iteration": 1, "input_tokens": 512, "output_tokens": 0, "cache_creation_tokens": 0, "cache_read_tokens": 400, "note": "prompt slice billed on no streamed message"} + ] + }, + "subagent_trajectories": [ + { + "schema_version": "ATIF-v1.7", + "trajectory_id": "toolu_02", + "agent": { + "name": "claude-code", + "version": "0.8.4", + "model_name": "claude-haiku-4-5" + }, + "steps": [ + { + "step_id": 1, + "source": "agent", + "model_name": "claude-haiku-4-5", + "message": "Running app.py to verify.", + "tool_calls": [ + { + "tool_call_id": "toolu_sub_01", + "function_name": "Bash", + "arguments": {"command": "python app.py"} + } + ], + "observation": { + "results": [ + {"source_call_id": "toolu_sub_01", "content": "hello"} + ] + }, + "metrics": {"prompt_tokens": 900, "completion_tokens": 50, "cached_tokens": 0} + } + ] + } + ] +} diff --git a/tests/harbor_e2e/fixtures/docker_baseline.yaml b/tests/harbor_e2e/fixtures/docker_baseline.yaml new file mode 100644 index 00000000..a078b5b2 --- /dev/null +++ b/tests/harbor_e2e/fixtures/docker_baseline.yaml @@ -0,0 +1,35 @@ +task_id: "harbor_e2e_baseline" +description: > + Harbor E2E baseline: a real claude-code agent turn against the default + coder-eval-agent image (no dockerfile_path, no template_sources, no + llm_judge). Exercises the plain export -> CoderEvalAgent -> --workspace-dir + -> verifier round trip with nothing else layered on top. + +agent: + type: "claude-code" + permission_mode: "acceptEdits" + allowed_tools: ["Read", "Write", "Bash"] + +sandbox: + driver: docker + docker: + image: coder-eval-agent:latest + +initial_prompt: > + Create a Python file named app.py in the current working directory that + prints 'Hello, Claude!' on one line, and today's date in YYYY-MM-DD format + on the next line. Use the datetime module. Then run the script with: + python app.py + +success_criteria: + - type: "file_exists" + path: "app.py" + description: "The file app.py must be created." + - type: "file_contains" + path: "app.py" + includes: ["Hello, Claude!", "datetime"] + description: "The script must contain the required string and import." + - type: "run_command" + command: "python app.py" + timeout: 10 + description: "The script must execute successfully." diff --git a/tests/harbor_e2e/fixtures/llm_judge.yaml b/tests/harbor_e2e/fixtures/llm_judge.yaml new file mode 100644 index 00000000..d8f1cbb7 --- /dev/null +++ b/tests/harbor_e2e/fixtures/llm_judge.yaml @@ -0,0 +1,44 @@ +task_id: "harbor_e2e_llm_judge" +description: > + Harbor E2E llm_judge case: a real claude-code agent turn, graded by a real + llm_judge call inside the Harbor verifier phase (--allow-credentials). + Catches regressions in judge-prompt construction / route resolution / JSON + verdict parsing when run through a Harbor-exported task, not just through + coder-eval's own `run`/`evaluate`. + +run_limits: + max_turns: 2 + +agent: + type: "claude-code" + permission_mode: "acceptEdits" + allowed_tools: ["Write"] + +sandbox: + driver: docker + docker: + image: coder-eval-agent:latest + +initial_prompt: > + Use the Write tool to create poem.txt containing a 2-line poem about the + Python programming language. The poem must mention 'Python' explicitly and + must be exactly 2 lines (one newline between them, no trailing blank line). + +success_criteria: + - type: "file_exists" + path: "poem.txt" + description: "Sanity check: agent must have written the file." + - type: "llm_judge" + files: ["poem.txt"] + max_tokens: 200 + pass_threshold: 0.8 + prompt: > + You are grading a 2-line poem about Python. The agent was instructed to + produce exactly 2 lines and explicitly mention 'Python'. + + Score 1.0 if the file contains exactly 2 non-empty lines AND the word + 'Python' (case-insensitive) appears at least once. Score 0.0 otherwise. + Do not penalize style, rhyme, or meter; this is a structural check. + + Return JSON: {"score": <0.0|1.0>, "rationale": ""} + description: "LLM-graded structural check on the poem output." diff --git a/tests/harbor_e2e/fixtures/template_sources.yaml b/tests/harbor_e2e/fixtures/template_sources.yaml new file mode 100644 index 00000000..5ada088e --- /dev/null +++ b/tests/harbor_e2e/fixtures/template_sources.yaml @@ -0,0 +1,38 @@ +task_id: "harbor_e2e_template_sources" +description: > + Harbor E2E template_sources case: a real claude-code agent turn against a + task using sandbox.template_sources + sandbox.python.env_packages, on + driver: docker. Exercises the packager copying the TemplateDirSource + directory into environment/templates/ and rewriting its path, plus + environment/task.yaml's sandbox block being field-merged (not replaced). + +run_limits: + expected_turns: 3 + +agent: + type: "claude-code" + permission_mode: "acceptEdits" + allowed_tools: ["Bash"] + +sandbox: + driver: docker + docker: + image: coder-eval-agent:latest + python: + env_packages: + - pytest + template_sources: + - type: template_dir + path: "../../../templates/fibonacci-starter" + +initial_prompt: > + Implement the fibonacci function in src/main.py. + The file has a stub function already defined with a docstring. + The tests in tests/test_main.py define the expected behavior. + Make all tests pass. + +success_criteria: + - type: run_command + description: "All tests pass" + command: "python -m pytest tests/" + timeout: 60 diff --git a/tests/lint/rules/no_cli_imports_in_core.py b/tests/lint/rules/no_cli_imports_in_core.py index cea8d840..f09f4018 100644 --- a/tests/lint/rules/no_cli_imports_in_core.py +++ b/tests/lint/rules/no_cli_imports_in_core.py @@ -2,9 +2,14 @@ The "core" layer comprises every package that should be usable without the CLI: criteria/, evaluation/, models/, simulation/, scoring/, streaming/, -errors/, orchestration/, agents/. Importing from coder_eval.cli +errors/, orchestration/, agents/, harbor/. Importing from coder_eval.cli creates an upward dependency that breaks testability in isolation. +``harbor/`` joined this list for the same reason ``orchestration/`` is on it: +its reward writer wants to raise a plain exception (``RewardWriteSkippedError``, +or the re-exported ``RegradeError``) and let the CLI wrap it into an exit +code — exactly the ``orchestration/regrade.py`` -> ``evaluate`` shape. + Note: this is a single, narrow rule (no upward imports into cli). For a fully layered import graph (no upward imports between any layers), evaluate import-linter / grimp — purpose-built for that. CE004 is the cheap version @@ -18,7 +23,7 @@ _CORE_DIRS = re.compile( - r"[/\\](criteria|evaluation|models|simulation|scoring|streaming|errors|orchestration|agents)[/\\]" + r"[/\\](criteria|evaluation|models|simulation|scoring|streaming|errors|orchestration|agents|harbor)[/\\]" ) _BANNED = re.compile(r"^coder_eval\.cli") diff --git a/tests/test_atif_emit.py b/tests/test_atif_emit.py new file mode 100644 index 00000000..a0b60769 --- /dev/null +++ b/tests/test_atif_emit.py @@ -0,0 +1,365 @@ +"""Tests for the EvaluationResult → ATIF Trajectory converter (atif_emit).""" + +import json +from datetime import UTC, datetime + +from coder_eval.harbor import atif_emit +from coder_eval.harbor.atif_emit import evaluation_result_to_trajectory, write_trajectory_json +from coder_eval.harbor.atif_models import Trajectory +from coder_eval.models import ( + AssistantMessage, + CommandTelemetry, + ContentBlock, + EvaluationResult, + FinalStatus, + ReconciliationMessage, + TokenUsage, + TurnRecord, + UserMessage, +) +from coder_eval.path_utils import write_text_atomic + + +T0 = datetime(2026, 7, 20, 10, 0, 0, tzinfo=UTC) + + +def _assistant( + text: str = "working on it", + *, + thinking: str | None = None, + parent_tool_use_id: str | None = None, + input_tokens: int = 100, + output_tokens: int = 50, + cache_creation: int = 20, + cache_read: int = 30, + model: str = "claude-sonnet-4-6", +) -> AssistantMessage: + blocks = [ContentBlock(block_type="text", sequence=0, text=text)] + if thinking is not None: + blocks.insert(0, ContentBlock(block_type="thinking", sequence=0, thinking=thinking)) + return AssistantMessage( + started_at=T0, + completed_at=T0, + generation_duration_ms=1000.0, + content_blocks=blocks, + input_tokens=input_tokens, + output_tokens=output_tokens, + cache_creation_tokens=cache_creation, + cache_read_tokens=cache_read, + model=model, + parent_tool_use_id=parent_tool_use_id, + ) + + +def _cmd(tool_id: str, *, index: int | None, tool_name: str = "Bash", command: str = "ls") -> CommandTelemetry: + return CommandTelemetry( + tool_name=tool_name, + tool_id=tool_id, + timestamp=T0, + parameters={"command": command}, + result_status="success", + result_summary=f"output of {tool_id}", + assistant_turn_index=index, + sequence_number=0, + ) + + +def _turn( + *, + iteration: int = 1, + user_input: str = "do the task", + agent_output: str = "done", + messages: list | None = None, + commands: list[CommandTelemetry] | None = None, + token_usage: TokenUsage | None = None, + crashed: bool = False, + crash_reason: str | None = None, +) -> TurnRecord: + return TurnRecord( + iteration=iteration, + user_input=user_input, + agent_output=agent_output, + messages=messages or [], + commands=commands or [], + token_usage=token_usage, + crashed=crashed, + crash_reason=crash_reason, + ) + + +def _result(turns: list[TurnRecord], *, total_token_usage: TokenUsage | None = None) -> EvaluationResult: + return EvaluationResult( + task_id="atif_emit_test", + task_description="converter test", + variant_id="default", + agent_type="claude-code", + model_used="claude-sonnet-4-6", + started_at=T0, + final_status=FinalStatus.SUCCESS, + iteration_count=len(turns), + iterations=turns, + total_token_usage=total_token_usage, + ) + + +class TestHappyPath: + def test_user_step_then_agent_steps_with_tool_attribution(self): + messages = [_assistant("first gen"), _assistant("second gen")] + commands = [_cmd("toolu_a", index=0), _cmd("toolu_b", index=1), _cmd("toolu_c", index=1)] + result = _result([_turn(messages=messages, commands=commands)]) + + t = evaluation_result_to_trajectory(result) + assert t is not None + # Synthetic user step (no UserMessage in stream) + 2 agent steps. + assert [s.source for s in t.steps] == ["user", "agent", "agent"] + assert [s.step_id for s in t.steps] == [1, 2, 3] + assert t.steps[0].message == "do the task" + # Commands attached to their own generation. + assert [tc.tool_call_id for tc in t.steps[1].tool_calls] == ["toolu_a"] + assert [tc.tool_call_id for tc in t.steps[2].tool_calls] == ["toolu_b", "toolu_c"] + # Observation joins by source_call_id within the same step (validator-checked). + assert t.steps[2].observation.results[0].source_call_id == "toolu_b" + assert t.steps[2].observation.results[0].content == "output of toolu_b" + assert t.session_id == "atif_emit_test/default" + assert t.agent.name == "claude-code" + assert t.agent.version # coder_eval __version__, non-empty + + def test_metrics_mapping_uses_token_usage_derivation(self): + messages = [_assistant(input_tokens=100, output_tokens=50, cache_creation=20, cache_read=30)] + t = evaluation_result_to_trajectory(_result([_turn(messages=messages)])) + m = t.steps[1].metrics + # prompt = uncached(100) + cache_creation(20) + cache_read(30) — via TokenUsage.input_tokens. + assert m.prompt_tokens == 150 + assert m.completion_tokens == 50 + assert m.cached_tokens == 30 + # Finer split preserved in extra. + assert t.steps[1].extra["cache_creation_tokens"] == 20 + + def test_reasoning_content_from_thinking_blocks(self): + messages = [_assistant("answer", thinking="let me think")] + t = evaluation_result_to_trajectory(_result([_turn(messages=messages)])) + assert t.steps[1].reasoning_content == "let me think" + assert t.steps[1].message == "answer" + + def test_explicit_user_message_becomes_user_step(self): + messages = [UserMessage(text="simulated utterance", completed_at=T0), _assistant("reply")] + t = evaluation_result_to_trajectory(_result([_turn(messages=messages)])) + # Explicit UserMessage → no synthetic user step is added. + assert [s.source for s in t.steps] == ["user", "agent"] + assert t.steps[0].message == "simulated utterance" + assert t.steps[0].timestamp == T0.isoformat() + + +class TestSubagentNesting: + def test_subagent_generations_nest_not_flatten(self): + spawn = _cmd("toolu_task", index=0, tool_name="Agent", command="") + messages = [ + _assistant("spawning a sub-agent"), + _assistant("sub work 1", parent_tool_use_id="toolu_task"), + _assistant("sub work 2", parent_tool_use_id="toolu_task"), + _assistant("main continues"), + ] + sub_cmd = _cmd("toolu_sub", index=1) # index 1 = first sub-agent generation + result = _result([_turn(messages=messages, commands=[spawn, sub_cmd])]) + + t = evaluation_result_to_trajectory(result) + # Main thread: user + 2 main agent steps only. + assert [s.source for s in t.steps] == ["user", "agent", "agent"] + assert t.steps[1].message == "spawning a sub-agent" + assert t.steps[2].message == "main continues" + # One embedded child, re-indexed from 1, carrying the sub-agent's command. + assert len(t.subagent_trajectories) == 1 + child = t.subagent_trajectories[0] + assert child.trajectory_id == "toolu_task" + assert [s.step_id for s in child.steps] == [1, 2] + assert child.steps[0].tool_calls[0].tool_call_id == "toolu_sub" + assert child.agent.version == t.agent.version + # The spawning call's observation references the child. + spawn_result = next(r for r in t.steps[1].observation.results if r.source_call_id == "toolu_task") + assert spawn_result.subagent_trajectory_ref[0].trajectory_id == "toolu_task" + + def test_orphan_subagent_group_embeds_without_ref(self): + # Sub-agent messages whose spawning tool call is not in the telemetry. + messages = [_assistant("main"), _assistant("orphan sub", parent_tool_use_id="toolu_ghost")] + t = evaluation_result_to_trajectory(_result([_turn(messages=messages)])) + assert t.subagent_trajectories[0].trajectory_id == "toolu_ghost" + # No fabricated tool call / ref on the main thread. + for step in t.steps: + assert not step.tool_calls or all(tc.tool_call_id != "toolu_ghost" for tc in step.tool_calls) + + +class TestGenerationlessTurn: + def test_user_message_and_reconciliation_survive_without_generations(self): + """A turn whose stream has user/reconciliation entries but NO assistant + generations (e.g. a simulation turn that crashed before the first + generation) must not fall into the legacy path — the UserMessage still + becomes a user step and the residual is still recorded.""" + messages = [ + UserMessage(text="simulated ask", completed_at=T0), + ReconciliationMessage(input_tokens=42, note="billed, never streamed"), + ] + t = evaluation_result_to_trajectory(_result([_turn(messages=messages, agent_output="")])) + assert [s.source for s in t.steps] == ["user"] + assert t.steps[0].message == "simulated ask" + assert t.extra["reconciliation"][0]["input_tokens"] == 42 + + +class TestReconciliation: + def test_reconciliation_never_a_step_and_totals_match(self): + total = TokenUsage( + uncached_input_tokens=612, + cache_creation_input_tokens=20, + cache_read_input_tokens=430, + output_tokens=50, + total_cost_usd=0.05, + ) + messages = [ + _assistant(input_tokens=100, output_tokens=50, cache_creation=20, cache_read=30), + ReconciliationMessage(input_tokens=512, cache_read_tokens=400, note="prompt slice"), + ] + result = _result( + [_turn(messages=messages, token_usage=total)], + total_token_usage=total, + ) + t = evaluation_result_to_trajectory(result) + assert all(s.source != "reconciliation" for s in t.steps) # not representable anyway + assert len(t.steps) == 2 # user + one generation + # Residual recorded at root. + recon = t.extra["reconciliation"] + assert recon == [ + { + "iteration": 1, + "input_tokens": 512, + "output_tokens": 0, + "cache_creation_tokens": 0, + "cache_read_tokens": 400, + "note": "prompt slice", + } + ] + # FinalMetrics ≡ authoritative total (which already includes the residual). + fm = t.final_metrics + assert fm.total_prompt_tokens == total.input_tokens == 1062 + assert fm.total_completion_tokens == 50 + assert fm.total_cached_tokens == 430 + assert fm.total_cost_usd == 0.05 + assert fm.total_steps == 2 + + def test_final_metrics_falls_back_to_summing_turns(self): + u1 = TokenUsage(uncached_input_tokens=10, output_tokens=5) + u2 = TokenUsage(uncached_input_tokens=20, output_tokens=15) + result = _result( + [ + _turn(iteration=1, messages=[_assistant()], token_usage=u1), + _turn(iteration=2, messages=[_assistant()], token_usage=u2), + ], + ) + t = evaluation_result_to_trajectory(result) + assert t.final_metrics.total_prompt_tokens == 30 + assert t.final_metrics.total_completion_tokens == 20 + + def test_no_usage_anywhere_omits_final_metrics(self): + t = evaluation_result_to_trajectory(_result([_turn(messages=[_assistant()])])) + # per-message metrics exist but no turn/total usage → final_metrics omitted. + assert t.final_metrics is None + + +class TestLegacyAndEdgeCases: + def test_legacy_turn_without_messages(self): + usage = TokenUsage(uncached_input_tokens=100, cache_read_input_tokens=40, output_tokens=25) + commands = [_cmd("toolu_x", index=None)] + result = _result([_turn(messages=[], commands=commands, token_usage=usage)]) + t = evaluation_result_to_trajectory(result) + assert [s.source for s in t.steps] == ["user", "agent"] + agent_step = t.steps[1] + assert agent_step.message == "done" + assert agent_step.metrics.prompt_tokens == 140 + assert agent_step.tool_calls[0].tool_call_id == "toolu_x" + + def test_unattributed_command_rides_last_main_agent_step(self): + messages = [_assistant("gen 1"), _assistant("gen 2")] + commands = [_cmd("toolu_none", index=None), _cmd("toolu_oob", index=99)] + t = evaluation_result_to_trajectory(_result([_turn(messages=messages, commands=commands)])) + last_agent = t.steps[-1] + ids = [tc.tool_call_id for tc in last_agent.tool_calls] + assert ids == ["toolu_none", "toolu_oob"] + + def test_leftover_commands_stay_within_their_turn(self): + """A turn with a stream but no main-thread agent step must synthesize its + own agent step for leftover commands — never attach them to a previous + iteration's step (which would mislabel the command's iteration).""" + turn1 = _turn(iteration=1, messages=[_assistant("gen 1")]) + turn2 = _turn( + iteration=2, + user_input="follow-up", + agent_output="crashed early", + messages=[UserMessage(text="follow-up", completed_at=T0)], + commands=[_cmd("toolu_orphan", index=None)], + ) + t = evaluation_result_to_trajectory(_result([turn1, turn2])) + # Turn 1's agent step must NOT have absorbed turn 2's command. + turn1_agent = next(s for s in t.steps if s.source == "agent" and s.extra["iteration"] == 1) + assert not turn1_agent.tool_calls + # Turn 2 synthesized its own agent step carrying the command. + turn2_agent = next(s for s in t.steps if s.source == "agent" and s.extra["iteration"] == 2) + assert [tc.tool_call_id for tc in turn2_agent.tool_calls] == ["toolu_orphan"] + assert turn2_agent.message == "crashed early" + + def test_multi_iteration_sequential_step_ids_and_user_steps(self): + result = _result( + [ + _turn(iteration=1, user_input="first ask", messages=[_assistant("a")]), + _turn(iteration=2, user_input="feedback", messages=[_assistant("b")]), + ] + ) + t = evaluation_result_to_trajectory(result) + assert [s.step_id for s in t.steps] == [1, 2, 3, 4] + assert [s.source for s in t.steps] == ["user", "agent", "user", "agent"] + assert t.steps[2].message == "feedback" + assert t.steps[2].extra["iteration"] == 2 + + def test_crashed_turn_marks_steps(self): + result = _result([_turn(messages=[_assistant("partial work")], crashed=True, crash_reason="timeout")]) + t = evaluation_result_to_trajectory(result) + assert t.steps[0].extra["crashed"] is True + assert t.steps[0].extra["crash_reason"] == "timeout" # first step of the turn only + assert t.steps[1].extra["crashed"] is True + assert "crash_reason" not in t.steps[1].extra + + def test_empty_result_returns_none(self): + assert evaluation_result_to_trajectory(_result([])) is None + + def test_converter_does_not_mutate_input(self): + result = _result([_turn(messages=[_assistant()], commands=[_cmd("toolu_a", index=0)])]) + before = result.model_dump_json() + evaluation_result_to_trajectory(result) + assert result.model_dump_json() == before + + +class TestWriteTrajectoryJson: + def test_writes_valid_atif_json(self, tmp_path): + result = _result([_turn(messages=[_assistant()])]) + path = tmp_path / "trajectory.json" + written = write_trajectory_json(result, path) + assert written == path + parsed = Trajectory.model_validate(json.loads(path.read_text(encoding="utf-8"))) + assert parsed.session_id == "atif_emit_test/default" + assert not path.with_suffix(".json.tmp").exists() + + def test_zero_step_result_writes_nothing(self, tmp_path): + path = tmp_path / "trajectory.json" + assert write_trajectory_json(_result([]), path) is None + assert not path.exists() + + def test_converter_exception_swallowed(self, tmp_path, monkeypatch): + monkeypatch.setattr(atif_emit, "evaluation_result_to_trajectory", lambda _: 1 / 0) + path = tmp_path / "trajectory.json" + assert atif_emit.write_trajectory_json(_result([_turn()]), path) is None + assert not path.exists() + + +class TestAtomicWriteText: + def test_writes_and_leaves_no_tmp(self, tmp_path): + path = tmp_path / "out.json" + write_text_atomic(path, '{"a": 1}') + assert path.read_text(encoding="utf-8") == '{"a": 1}' + assert list(tmp_path.iterdir()) == [path] diff --git a/tests/test_atif_hydrate.py b/tests/test_atif_hydrate.py new file mode 100644 index 00000000..090e196a --- /dev/null +++ b/tests/test_atif_hydrate.py @@ -0,0 +1,138 @@ +"""Round-trip tests: EvaluationResult -> ATIF Trajectory -> TurnRecords -> criteria. + +Confirms a trajectory produced OUTSIDE this process (a Harbor agent's +``coder-eval execute --format harbor``) can still be graded by +``coder-eval evaluate --format harbor`` — the criteria that key off +``turn_records`` (``command_executed``, ``cli_called``, ``commands_efficiency``, +``skill_triggered``) must see the same commands after the ATIF round-trip. +""" + +import asyncio +from datetime import UTC, datetime + +from coder_eval.criteria.base import CheckContext +from coder_eval.criteria.command_executed import CommandExecutedChecker +from coder_eval.criteria.commands_efficiency import CommandsEfficiencyChecker +from coder_eval.criteria.skill_triggered import SkillTriggeredChecker +from coder_eval.harbor.atif_emit import evaluation_result_to_trajectory +from coder_eval.harbor.atif_hydrate import seed_from_atif_trajectory, trajectory_to_turn_records +from coder_eval.models import ( + AssistantMessage, + CommandExecutedCriterion, + CommandsEfficiencyCriterion, + CommandTelemetry, + ContentBlock, + EvaluationResult, + FinalStatus, + SkillTriggeredCriterion, + TurnRecord, +) + + +T0 = datetime(2026, 9, 10, 12, 0, 0, tzinfo=UTC) + + +def _assistant(text: str = "working", **kwargs) -> AssistantMessage: + return AssistantMessage( + started_at=T0, + completed_at=T0, + generation_duration_ms=100.0, + content_blocks=[ContentBlock(block_type="text", sequence=0, text=text)], + **kwargs, + ) + + +def _cmd( + tool_id: str, tool_name: str, *, index: int, parameters: dict | None = None, command: str = "ls" +) -> CommandTelemetry: + return CommandTelemetry( + tool_name=tool_name, + tool_id=tool_id, + timestamp=T0, + parameters=parameters if parameters is not None else {"command": command}, + result_status="success", + result_summary=f"ran {command}", + assistant_turn_index=index, + sequence_number=index, + ) + + +def _original_result() -> EvaluationResult: + turn = TurnRecord( + iteration=1, + user_input="do the task", + agent_output="done", + messages=[_assistant("running Bash"), _assistant("running Skill")], + commands=[ + _cmd("toolu_a", "Bash", index=0, command="uv run pytest"), + _cmd("toolu_b", "Skill", index=1, parameters={"skill": "my-skill"}), + ], + ) + return EvaluationResult( + task_id="atif_hydrate_test", + task_description="round-trip test", + variant_id="default", + agent_type="claude-code", + model_used="claude-sonnet-5", + started_at=T0, + final_status=FinalStatus.SUCCESS, + iteration_count=1, + iterations=[turn], + ) + + +class TestRoundTrip: + def test_commands_survive_emit_then_hydrate(self): + original = _original_result() + trajectory = evaluation_result_to_trajectory(original) + assert trajectory is not None + + turns = trajectory_to_turn_records(trajectory) + all_commands = [cmd for turn in turns for cmd in turn.commands] + assert {cmd.tool_id for cmd in all_commands} == {"toolu_a", "toolu_b"} + assert {cmd.tool_name for cmd in all_commands} == {"Bash", "Skill"} + + def test_seed_from_atif_trajectory_builds_valid_result(self): + trajectory = evaluation_result_to_trajectory(_original_result()) + seeded = seed_from_atif_trajectory(trajectory, task_id="atif_hydrate_test") + assert seeded.iteration_count == len(seeded.iterations) + assert seeded.agent_type == "claude-code" + + +class TestCriteriaAgainstHydratedTrajectory: + def _turn_records(self) -> list[TurnRecord]: + trajectory = evaluation_result_to_trajectory(_original_result()) + assert trajectory is not None + return trajectory_to_turn_records(trajectory) + + def test_command_executed_matches_hydrated_bash_call(self): + turn_records = self._turn_records() + checker = CommandExecutedChecker() + criterion = CommandExecutedCriterion( + description="ran pytest", + tool_name="Bash", + command_pattern="pytest", + min_count=1, + ) + result = asyncio.run( + checker.check_async(criterion, sandbox=None, turn_records=turn_records, context=CheckContext()) + ) + assert result.score == 1.0 + + def test_commands_efficiency_counts_hydrated_commands(self): + turn_records = self._turn_records() + checker = CommandsEfficiencyChecker() + criterion = CommandsEfficiencyCriterion(description="efficiency", expected_commands=2) + result = asyncio.run( + checker.check_async(criterion, sandbox=None, turn_records=turn_records, context=CheckContext()) + ) + assert result.score == 1.0 + + def test_skill_triggered_sees_hydrated_skill_call(self): + turn_records = self._turn_records() + checker = SkillTriggeredChecker() + criterion = SkillTriggeredCriterion(description="skill", skill_name="my-skill", expected_skill="my-skill") + result = asyncio.run( + checker.check_async(criterion, sandbox=None, turn_records=turn_records, context=CheckContext()) + ) + assert result.score == 1.0 diff --git a/tests/test_atif_models.py b/tests/test_atif_models.py new file mode 100644 index 00000000..550517e8 --- /dev/null +++ b/tests/test_atif_models.py @@ -0,0 +1,181 @@ +"""Tests for the vendored ATIF models (coder_eval.harbor.atif_models). + +Fixture compatibility guarantee: ``tests/fixtures/atif/known_good_trajectory.json`` +was originally validated against harbor==0.20.0. ``harbor`` is now a real, +installed optional extra (``pyproject.toml``'s ``harbor`` extra, pinned to +0.22.0 — see ``pyproject.toml`` and both pyright CI jobs), so this fixture and +the vendored models above can be re-verified directly against the installed +package: + + uv run python -c "from harbor.models.trajectories.trajectory import Trajectory; \\ + import json; Trajectory.model_validate(json.load(open('tests/fixtures/atif/known_good_trajectory.json'))); \\ + print('OK')" + +Last validated: harbor 0.22.0 (2026-09-11). If the vendored models and this +fixture ever disagree with harbor, re-run the procedure and reconcile. +""" + +import json +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from coder_eval.harbor.atif_models import ( + AtifAgent, + ContentPart, + Observation, + ObservationResult, + Step, + SubagentTrajectoryRef, + ToolCall, + Trajectory, +) + + +FIXTURE = Path(__file__).parent / "fixtures" / "atif" / "known_good_trajectory.json" + + +def _agent() -> AtifAgent: + return AtifAgent(name="claude-code", version="1.0.0") + + +def _step(step_id: int, source: str = "agent", **kwargs) -> Step: + return Step(step_id=step_id, source=source, message=f"step {step_id}", **kwargs) + + +def _tool_step(step_id: int, tool_call_id: str = "toolu_01", source_call_id: str | None = "toolu_01") -> Step: + return Step( + step_id=step_id, + source="agent", + message="calling a tool", + tool_calls=[ToolCall(tool_call_id=tool_call_id, function_name="Bash", arguments={"command": "ls"})], + observation=Observation(results=[ObservationResult(source_call_id=source_call_id, content="ok")]), + ) + + +class TestHappyPath: + def test_two_step_trajectory_with_tool_call_round_trips(self): + t = Trajectory(agent=_agent(), steps=[_step(1, source="user"), _tool_step(2)]) + dumped = t.model_dump(exclude_none=True) + # exclude_none is how harbor serializes (Trajectory.to_json_dict). + reparsed = Trajectory.model_validate(dumped) + assert reparsed == t + + def test_defaults(self): + t = Trajectory(agent=_agent(), steps=[_step(1)]) + assert t.schema_version == "ATIF-v1.7" + assert t.trajectory_id is None + assert t.final_metrics is None + + +class TestValidators: + def test_empty_steps_rejected(self): + with pytest.raises(ValidationError, match="at least 1"): + Trajectory(agent=_agent(), steps=[]) + + def test_non_sequential_step_ids_rejected(self): + with pytest.raises(ValidationError, match=r"expected 2 \(sequential from 1\), got 3"): + Trajectory(agent=_agent(), steps=[_step(1), _step(3)]) + + def test_step_ids_must_start_at_one(self): + with pytest.raises(ValidationError, match=r"expected 1 \(sequential from 1\), got 2"): + Trajectory(agent=_agent(), steps=[_step(2)]) + + def test_cross_step_source_call_id_rejected(self): + # Step 2's observation references step 1's tool_call_id — invalid. + bad = _tool_step(2, tool_call_id="toolu_other", source_call_id="toolu_01") + with pytest.raises(ValidationError, match="source_call_id 'toolu_01'"): + Trajectory(agent=_agent(), steps=[_step(1, source="user"), bad]) + + def test_null_source_call_id_allowed(self): + step = _tool_step(1, source_call_id=None) + Trajectory(agent=_agent(), steps=[step]) # does not raise + + def test_subagent_without_trajectory_id_rejected(self): + sub = Trajectory(agent=_agent(), steps=[_step(1)]) + with pytest.raises(ValidationError, match="trajectory_id is required"): + Trajectory(agent=_agent(), steps=[_step(1)], subagent_trajectories=[sub]) + + def test_duplicate_subagent_trajectory_ids_rejected(self): + sub1 = Trajectory(agent=_agent(), steps=[_step(1)], trajectory_id="dup") + sub2 = Trajectory(agent=_agent(), steps=[_step(1)], trajectory_id="dup") + with pytest.raises(ValidationError, match="not unique"): + Trajectory(agent=_agent(), steps=[_step(1)], subagent_trajectories=[sub1, sub2]) + + def test_agent_version_required(self): + # Verified against harbor 0.22.0: Agent.version is REQUIRED, not optional. + with pytest.raises(ValidationError, match="version"): + AtifAgent(name="claude-code") # type: ignore[call-arg] + + def test_content_part_text_requires_text(self): + with pytest.raises(ValidationError, match="'text' field is required"): + ContentPart(type="text") + + def test_content_part_text_forbids_source(self): + with pytest.raises(ValidationError, match="'source' field is not allowed"): + ContentPart(type="text", text="hi", source={"media_type": "image/png"}) + + +class TestVersionTolerance: + @pytest.mark.parametrize("version", ["ATIF-v1.0", "ATIF-v1.7", "ATIF-v1.9", "ATIF-v1.42"]) + def test_any_v1_minor_accepted(self, version): + t = Trajectory(schema_version=version, agent=_agent(), steps=[_step(1)]) + assert t.schema_version == version + + @pytest.mark.parametrize("version", ["ATIF-v2.0", "ATIF-2.0", "v1.7", "garbage", ""]) + def test_non_v1_rejected(self, version): + with pytest.raises(ValidationError, match="schema_version"): + Trajectory(schema_version=version, agent=_agent(), steps=[_step(1)]) + + +class TestFrozenFixture: + def test_known_good_fixture_parses(self): + """The harbor-0.20.0-validated fixture must parse with the vendored models.""" + t = Trajectory.model_validate(json.loads(FIXTURE.read_text(encoding="utf-8"))) + assert t.schema_version == "ATIF-v1.7" + assert len(t.steps) == 3 + assert t.subagent_trajectories is not None + assert t.subagent_trajectories[0].trajectory_id == "toolu_02" + # Multimodal message variant (list of ContentPart) survives. + assert isinstance(t.steps[2].message, list) + # Sub-agent ref resolves against the embedded array. + ref = t.steps[2].observation.results[0].subagent_trajectory_ref + assert ref is not None and ref[0].trajectory_id == "toolu_02" + + def test_fixture_round_trips_through_vendored_models(self): + raw = json.loads(FIXTURE.read_text(encoding="utf-8")) + t = Trajectory.model_validate(raw) + assert Trajectory.model_validate(t.model_dump(exclude_none=True)) == t + + +class TestExtraForbid: + def test_unknown_field_rejected(self): + with pytest.raises(ValidationError): + Trajectory(agent=_agent(), steps=[_step(1)], not_a_field=1) # type: ignore[call-arg] + + def test_unknown_step_field_rejected(self): + with pytest.raises(ValidationError): + Step(step_id=1, source="agent", message="x", bogus=True) # type: ignore[call-arg] + + def test_subagent_ref_rejects_unknown_field(self): + # All four real fields (trajectory_id, session_id, trajectory_path, + # extra) are optional in harbor==0.22.0 -- a bare SubagentTrajectoryRef() + # is a valid document (e.g. a session_id-only or trajectory_path-only + # reference need not repeat the others), so only an actually-unknown + # field should be rejected under extra="forbid". + SubagentTrajectoryRef() + with pytest.raises(ValidationError): + SubagentTrajectoryRef(bogus=1) # type: ignore[call-arg] + + def test_subagent_ref_accepts_session_id_only_form(self): + """harbor==0.22.0's session_id-addressed shape, verified against a real install.""" + ref = SubagentTrajectoryRef(session_id="sess-123") + assert ref.trajectory_id is None + assert ref.session_id == "sess-123" + + def test_subagent_ref_accepts_trajectory_path_only_form(self): + """The spec's file-ref form: trajectory_path alone, no trajectory_id.""" + ref = SubagentTrajectoryRef(trajectory_path="subagents/child.json") + assert ref.trajectory_id is None + assert ref.trajectory_path == "subagents/child.json" diff --git a/tests/test_detached_grading_guards.py b/tests/test_detached_grading_guards.py index 2a1c5a28..f3939774 100644 --- a/tests/test_detached_grading_guards.py +++ b/tests/test_detached_grading_guards.py @@ -12,6 +12,7 @@ from pathlib import Path from unittest.mock import AsyncMock, MagicMock, patch +import click import pytest from typer.testing import CliRunner @@ -91,6 +92,45 @@ def test_run_still_accepts_the_same_simulation_task(tmp_path: Path) -> None: assert "does not support simulation" not in result.output +# -------------------------------------------------------------------------- +# `run` refuses a task with zero success_criteria; `execute` still accepts it +# -------------------------------------------------------------------------- + +_NO_CRITERIA = """task_id: t +description: d +agent: + type: none +success_criteria: [] +""" + + +def test_run_refuses_a_task_with_no_success_criteria(tmp_path: Path) -> None: + """An empty `success_criteria:` scores vacuously (all_criteria_passed([]) + is True, calculate_weighted_score([]) is 0.0), so a graded run of such a + task would silently finalize as SUCCESS at weighted_score 0.0 -- a + misconfigured task, not a real result. `run` must refuse it by name.""" + path = tmp_path / "t.yaml" + path.write_text(_NO_CRITERIA, encoding="utf-8") + + result = runner.invoke(app, ["run", str(path), "--run-dir", str(tmp_path / "r")]) + + assert result.exit_code != 0 + assert "success_criteria" in result.output + assert "t" in result.output + + +def test_execute_still_accepts_a_task_with_no_success_criteria(tmp_path: Path) -> None: + """The control: `execute` never grades, so a criteria-free task.yaml (the + shape the Harbor agent-phase export deliberately produces) is legal there.""" + path = tmp_path / "t.yaml" + path.write_text(_NO_CRITERIA, encoding="utf-8") + + with patch("coder_eval.cli.run_command._run_with_experiment", new=AsyncMock(return_value=(MagicMock(), 0))): + result = runner.invoke(app, ["execute", str(path), "--run-dir", str(tmp_path / "r")]) + + assert "success_criteria" not in result.output + + # -------------------------------------------------------------------------- # The evaluate-only path refuses grade=False # -------------------------------------------------------------------------- @@ -360,3 +400,80 @@ def test_the_recorded_task_defaults_to_the_task_being_run(tmp_path: Path) -> Non ) orch = Orchestrator(task=task, run_dir=tmp_path, variant_id="v") assert orch.recorded_task is task + + +# -------------------------------------------------------------------------- +# `--workspace-dir` misuse is a clean CLI error, not a traceback +# -------------------------------------------------------------------------- + +_DOCKER_TASK = """task_id: t +description: d +initial_prompt: p +agent: + type: claude-code +sandbox: + driver: docker + docker: + image: some-image:latest +success_criteria: + - type: file_exists + path: proof.txt + description: x +""" + + +def test_workspace_dir_with_docker_driver_is_a_clean_cli_error(tmp_path: Path) -> None: + """run_batch's own guard raises a plain ValueError; the CLI must convert it + to typer.BadParameter (exit 2, clean message) instead of an unhandled + traceback -- --workspace-dir is not for sandbox.driver: docker tasks.""" + path = tmp_path / "t.yaml" + path.write_text(_DOCKER_TASK, encoding="utf-8") + + result = runner.invoke( + app, ["run", str(path), "--run-dir", str(tmp_path / "r"), "--workspace-dir", str(tmp_path / "ws")] + ) + + assert result.exit_code != 0 + # click.unstyle strips ANSI color codes -- CI renders Click's error box + # with color (option names highlighted char-by-char), which would + # otherwise split "--workspace-dir" across escape sequences and silently + # break this check (see test_execute_format_harbor.py's equivalent). + assert "--workspace-dir" in click.unstyle(result.output) + assert "Traceback" not in result.output + + +# -------------------------------------------------------------------------- +# The empty-criteria guard checks post-`--resume` `to_run`, not the full +# `resolved` set -- an already-finalized row folded back from prior_results +# is never re-graded, so its own (possibly empty) criteria are moot. +# -------------------------------------------------------------------------- + + +def test_empty_criteria_guard_ignores_an_already_finalized_resumed_row(tmp_path: Path) -> None: + import typer + + from coder_eval.cli.run_command import _reject_empty_criteria_under_grade + from coder_eval.models import ResolvedTask, TaskDefinition + + finalized_but_empty = ResolvedTask( + task=TaskDefinition( + task_id="already-done", + description="d", + agent=parse_agent_config(type=AgentKind.NONE), + success_criteria=[], + ), + task_file=tmp_path / "t.yaml", + run_dir=tmp_path, + variant_id="v", + ) + + # The full `resolved` set (pre-resume) still refuses when actually graded + # -- this is the control, proving the guard is not simply disabled. + with pytest.raises(typer.BadParameter): + _reject_empty_criteria_under_grade([finalized_but_empty], grade=True) + + # But `to_run` (post-resume) is what a real call site must pass: an + # already-finalized row is peeled off by `_apply_resume` and folded back + # from `prior_results` without being re-graded, so it is NOT in `to_run` + # -- the empty `to_run` a resumed run would actually see must not raise. + _reject_empty_criteria_under_grade([], grade=True) diff --git a/tests/test_evaluate_format_harbor.py b/tests/test_evaluate_format_harbor.py new file mode 100644 index 00000000..298129cf --- /dev/null +++ b/tests/test_evaluate_format_harbor.py @@ -0,0 +1,126 @@ +"""``coder-eval evaluate --format harbor`` — grade a workdir using ATIF trajectory context. + +End-to-end: build an ATIF trajectory.json by hand (the shape a Harbor agent's +``coder-eval execute --format harbor`` would produce), drop a file in a plain +workdir (standing in for what the Harbor agent's container left behind), and +grade it with both a workdir-only criterion (``file_exists``) and a +trajectory-dependent one (``command_executed``) to prove hydration actually +feeds the checker. +""" + +import json + +import click +from typer.testing import CliRunner + +from coder_eval.cli import app +from coder_eval.harbor.atif_emit import evaluation_result_to_trajectory +from coder_eval.models import CommandTelemetry, EvaluationResult, FinalStatus, TurnRecord + + +runner = CliRunner() + +_TASK_YAML = """ +task_id: "atif_format_harbor_evaluate_test" +description: "evaluate --format harbor smoke test" +initial_prompt: "unused" +agent: + type: "claude-code" + +sandbox: + driver: "tempdir" + python: null + +success_criteria: + - type: "file_exists" + path: "test.txt" + description: "Test file must exist" + - type: "command_executed" + tool_name: "Bash" + command_pattern: "touch test.txt" + min_count: 1 + description: "The agent must have run touch" +""" + + +def _write_trajectory(tmp_path): + turn = TurnRecord( + iteration=1, + user_input="create test.txt", + agent_output="done", + commands=[ + CommandTelemetry( + tool_name="Bash", + tool_id="toolu_1", + timestamp="2026-09-10T00:00:00Z", + parameters={"command": "touch test.txt"}, + result_status="success", + result_summary="", + assistant_turn_index=0, + sequence_number=0, + ) + ], + ) + result = EvaluationResult( + task_id="atif_format_harbor_evaluate_test", + task_description="d", + agent_type="harbor-agent", + started_at="2026-09-10T00:00:00Z", + final_status=FinalStatus.NOT_GRADED, + iteration_count=1, + iterations=[turn], + ) + trajectory = evaluation_result_to_trajectory(result) + assert trajectory is not None + path = tmp_path / "trajectory.json" + path.write_text(trajectory.model_dump_json(exclude_none=True), encoding="utf-8") + return path + + +def test_evaluate_format_harbor_grades_with_hydrated_trajectory(tmp_path): + task_file = tmp_path / "task.yaml" + task_file.write_text(_TASK_YAML, encoding="utf-8") + + work_dir = tmp_path / "workdir" + work_dir.mkdir() + (work_dir / "test.txt").write_text("hi", encoding="utf-8") + + trajectory_path = _write_trajectory(tmp_path) + run_dir = tmp_path / "run" + + result = runner.invoke( + app, + [ + "evaluate", + str(task_file), + str(work_dir), + "--format", + "harbor", + "--trajectory", + str(trajectory_path), + "--run-dir", + str(run_dir), + ], + ) + assert result.exit_code == 0, result.output + + task_jsons = sorted(run_dir.glob("**/task.json")) + assert len(task_jsons) == 1 + graded = json.loads(task_jsons[0].read_text(encoding="utf-8")) + assert graded["weighted_score"] == 1.0 + scores = {r["criterion_type"]: r["score"] for r in graded["success_criteria_results"]} + assert scores["file_exists"] == 1.0 + assert scores["command_executed"] == 1.0 + + +def test_evaluate_format_harbor_requires_trajectory(tmp_path): + task_file = tmp_path / "task.yaml" + task_file.write_text(_TASK_YAML, encoding="utf-8") + work_dir = tmp_path / "workdir" + work_dir.mkdir() + + result = runner.invoke(app, ["evaluate", str(task_file), str(work_dir), "--format", "harbor"]) + assert result.exit_code != 0 + # click.unstyle strips ANSI color codes -- see test_execute_format_harbor.py's + # equivalent assertion for why a plain substring check is not portable here. + assert "--trajectory" in click.unstyle(result.output) diff --git a/tests/test_execute_format_harbor.py b/tests/test_execute_format_harbor.py new file mode 100644 index 00000000..55e326ab --- /dev/null +++ b/tests/test_execute_format_harbor.py @@ -0,0 +1,87 @@ +"""``coder-eval execute --format harbor`` — writes a trajectory.json (ATIF) sibling for task.json.""" + +import json + +import click +import pytest +from typer.testing import CliRunner + +from coder_eval.cli import app +from coder_eval.orchestrator import Orchestrator +from tests.fixtures.mock_agent import MockAgent + + +runner = CliRunner() + + +@pytest.fixture +def success_task(tmp_path): + task_content = """ +task_id: "atif_format_harbor_test" +description: "execute --format harbor smoke test" +initial_prompt: "Create a file named 'test.txt'" +agent: + type: "claude-code" + permission_mode: "acceptEdits" + +sandbox: + driver: "tempdir" + python: null + +success_criteria: + - type: "file_exists" + path: "test.txt" + description: "Test file must exist" +""" + task_file = tmp_path / "task.yaml" + task_file.write_text(task_content) + return task_file + + +@pytest.fixture +def mock_agent(monkeypatch): + async def _mock_create_agent(self): + return MockAgent(self.task, scenario="success") + + monkeypatch.setattr(Orchestrator, "_create_agent", _mock_create_agent) + + +def test_execute_format_harbor_writes_trajectory_json(tmp_path, success_task, mock_agent): + run_dir = tmp_path / "run" + result = runner.invoke( + app, + ["execute", str(success_task), "--run-dir", str(run_dir), "--format", "harbor"], + ) + assert result.exit_code == 0, result.output + + task_jsons = sorted(run_dir.glob("**/task.json")) + assert len(task_jsons) == 1 + trajectory_path = task_jsons[0].with_name("trajectory.json") + assert trajectory_path.exists() + + trajectory = json.loads(trajectory_path.read_text(encoding="utf-8")) + assert trajectory["schema_version"].startswith("ATIF-v1.") + assert len(trajectory["steps"]) >= 1 + + +def test_execute_without_format_does_not_write_trajectory_json(tmp_path, success_task, mock_agent): + run_dir = tmp_path / "run" + result = runner.invoke(app, ["execute", str(success_task), "--run-dir", str(run_dir)]) + assert result.exit_code == 0, result.output + + task_jsons = sorted(run_dir.glob("**/task.json")) + assert len(task_jsons) == 1 + assert not task_jsons[0].with_name("trajectory.json").exists() + + +def test_unknown_format_value_errors_cleanly(tmp_path, success_task): + run_dir = tmp_path / "run" + result = runner.invoke( + app, + ["execute", str(success_task), "--run-dir", str(run_dir), "--format", "nonsense"], + ) + assert result.exit_code != 0 + # click.unstyle strips ANSI color codes -- CI renders Click's own error box + # with color (option names highlighted char-by-char), which would otherwise + # split "--format" across escape sequences and silently break this check. + assert "Unsupported --format" in click.unstyle(result.output) diff --git a/tests/test_harbor_agent.py b/tests/test_harbor_agent.py new file mode 100644 index 00000000..d012533b --- /dev/null +++ b/tests/test_harbor_agent.py @@ -0,0 +1,171 @@ +"""``coder_eval.harbor.agent.CoderEvalAgent`` — the Harbor-agent extension point. + +``harbor`` is not a project dependency (it only needs to be present INSIDE a +Harbor trial container, see ``harbor/agent.py``'s module docstring), so this +module cannot simply be imported in a normal test run. Rather than skip it +entirely (leaving `run()`'s command construction and +`populate_context_post_run()`'s trajectory parsing with zero coverage), stub +just enough of the `harbor` package tree to satisfy the one import +(`harbor.agents.installed.base.BaseInstalledAgent`) and import the real module +against that stub. +""" + +from __future__ import annotations + +import sys +import types +from pathlib import Path + +import pytest + +from coder_eval.harbor.atif_models import AtifAgent, FinalMetrics, Step, Trajectory + + +class _FakeBaseInstalledAgent: + """Just enough of Harbor's `BaseInstalledAgent` for `CoderEvalAgent` to subclass.""" + + +@pytest.fixture +def coder_eval_agent_module(monkeypatch: pytest.MonkeyPatch): + """Import `coder_eval.harbor.agent` against a stubbed `harbor` package tree.""" + harbor_mod = types.ModuleType("harbor") + agents_mod = types.ModuleType("harbor.agents") + installed_mod = types.ModuleType("harbor.agents.installed") + base_mod = types.ModuleType("harbor.agents.installed.base") + base_mod.BaseInstalledAgent = _FakeBaseInstalledAgent # type: ignore[attr-defined] + + for name, mod in ( + ("harbor", harbor_mod), + ("harbor.agents", agents_mod), + ("harbor.agents.installed", installed_mod), + ("harbor.agents.installed.base", base_mod), + ): + monkeypatch.setitem(sys.modules, name, mod) + + # `coder_eval.harbor.agent` may already be cached (unlikely -- nothing else + # imports it), but force a fresh import against the stub either way. + monkeypatch.delitem(sys.modules, "coder_eval.harbor.agent", raising=False) + import importlib + + return importlib.import_module("coder_eval.harbor.agent") + + +def _make_agent(module, *, logs_dir: Path): + """Build a `CoderEvalAgent` instance without running Harbor's real `__init__`.""" + agent = object.__new__(module.CoderEvalAgent) + agent.logs_dir = logs_dir + agent.environment_logs_dir = logs_dir + import logging + + agent.logger = logging.getLogger("test-coder-eval-agent") + return agent + + +class TestRunCommandConstruction: + async def test_run_shells_out_to_execute_with_workspace_dir_and_run_dir(self, coder_eval_agent_module, tmp_path): + """`--workspace-dir "$(pwd)"` is Gap 2's real fix: without it the agent's + tempdir sandbox writes outside the container's WORKDIR, where Harbor's + verifier phase looks. `--run-dir` must point at `environment_logs_dir` + (bind-mounted from Harbor's `self.logs_dir` on the host).""" + agent = _make_agent(coder_eval_agent_module, logs_dir=tmp_path) + + captured: dict[str, object] = {} + + async def _fake_exec(environment, command): + captured["environment"] = environment + captured["command"] = command + + agent._exec = _fake_exec # type: ignore[method-assign] + + fake_environment = object() + await agent.run("unused instruction", fake_environment, context=None) + + assert captured["environment"] is fake_environment + command = captured["command"] + assert isinstance(command, str) + assert command.startswith("coder-eval execute ") + assert "--format harbor" in command + assert f"--run-dir {tmp_path.as_posix()}" in command + assert '--workspace-dir "$(pwd)"' in command + + +class TestPopulateContextPostRun: + def _write_trajectory(self, path: Path, *, final_metrics: FinalMetrics | None) -> None: + trajectory = Trajectory( + agent=AtifAgent(name="coder-eval", version="0.0.0"), + steps=[Step(step_id=1, source="agent", message="did the thing")], + final_metrics=final_metrics, + ) + path.write_text(trajectory.model_dump_json(exclude_none=True), encoding="utf-8") + + def test_fills_cost_and_token_fields_from_trajectory_json(self, coder_eval_agent_module, tmp_path): + agent = _make_agent(coder_eval_agent_module, logs_dir=tmp_path) + self._write_trajectory( + tmp_path / "trajectory.json", + final_metrics=FinalMetrics( + total_prompt_tokens=100, + total_completion_tokens=50, + total_cached_tokens=10, + total_cost_usd=0.25, + ), + ) + + class _Context: + cost_usd = None + n_input_tokens = None + n_cache_tokens = None + n_output_tokens = None + + context = _Context() + agent.populate_context_post_run(context) + + assert context.cost_usd == 0.25 + assert context.n_input_tokens == 100 + assert context.n_cache_tokens == 10 + assert context.n_output_tokens == 50 + + def test_no_trajectory_json_is_a_silent_noop(self, coder_eval_agent_module, tmp_path): + """coder-eval execute may have crashed before writing anything -- this + must not raise, since it runs on every trial regardless.""" + agent = _make_agent(coder_eval_agent_module, logs_dir=tmp_path) + + class _Context: + cost_usd = "untouched" + + context = _Context() + agent.populate_context_post_run(context) + + assert context.cost_usd == "untouched" + + def test_trajectory_with_no_final_metrics_is_a_silent_noop(self, coder_eval_agent_module, tmp_path): + agent = _make_agent(coder_eval_agent_module, logs_dir=tmp_path) + self._write_trajectory(tmp_path / "trajectory.json", final_metrics=None) + + class _Context: + cost_usd = "untouched" + + context = _Context() + agent.populate_context_post_run(context) + + assert context.cost_usd == "untouched" + + def test_malformed_trajectory_json_is_a_silent_noop(self, coder_eval_agent_module, tmp_path): + """Best-effort context enrichment: a parse failure must never fail the trial.""" + agent = _make_agent(coder_eval_agent_module, logs_dir=tmp_path) + (tmp_path / "trajectory.json").write_text("not json", encoding="utf-8") + + class _Context: + cost_usd = "untouched" + + context = _Context() + agent.populate_context_post_run(context) + + assert context.cost_usd == "untouched" + + +def test_name_and_version(coder_eval_agent_module): + from coder_eval import __version__ + + assert coder_eval_agent_module.CoderEvalAgent.name() == "coder-eval" + agent = object.__new__(coder_eval_agent_module.CoderEvalAgent) + assert agent.version() == __version__ diff --git a/tests/test_harbor_experiment_packager.py b/tests/test_harbor_experiment_packager.py new file mode 100644 index 00000000..2be63d64 --- /dev/null +++ b/tests/test_harbor_experiment_packager.py @@ -0,0 +1,308 @@ +"""``coder_eval.harbor.experiment_packager`` — task.yaml + experiment.yaml -> Harbor directories. + +Builds real task/experiment YAML files on disk and drives them through +``export_experiment`` exactly as `coder-eval export ... -e ...` does, so these +tests exercise the same resolution path (``orchestration.experiment.resolve_all_tasks``) +a real invocation runs through. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +from coder_eval.harbor import packager +from coder_eval.harbor.experiment_packager import export_experiment + + +_BASE_TASK: dict[str, object] = { + "task_id": "greet", + "description": "Write a greeting to greeting.txt.", + "agent": {"type": "claude-code"}, + "initial_prompt": "Write 'hello' to greeting.txt.", + "sandbox": { + "driver": "docker", + "docker": {"image": "byod-custom-image:0.1.0", "network": "none"}, + }, + "success_criteria": [ + {"type": "file_exists", "path": "greeting.txt", "description": "exists"}, + ], +} + + +@pytest.fixture(autouse=True) +def _no_real_docker_inspection(monkeypatch: pytest.MonkeyPatch) -> None: + """Same hermeticity guard as test_harbor_packager.py -- never shell out to real docker.""" + monkeypatch.setattr(packager, "_inspect_image_workdir", lambda image: None) + + +def _write_task(tmp_path: Path, overrides: dict[str, object] | None = None, name: str = "task.yaml") -> Path: + payload = {**_BASE_TASK, **(overrides or {})} + task_file = tmp_path / name + task_file.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") + return task_file + + +def _write_experiment(tmp_path: Path, definition: dict[str, object], name: str = "experiment.yaml") -> Path: + exp_file = tmp_path / name + exp_file.write_text(yaml.safe_dump(definition, sort_keys=False), encoding="utf-8") + return exp_file + + +class TestRunLimitOnlyVariants: + """Variants that only touch run_limits/sandbox/prompt -- fully honorable, should export.""" + + def test_two_variants_export_two_directories_with_distinct_timeouts(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + exp_file = _write_experiment( + tmp_path, + { + "experiment_id": "run-limit-ab", + "variants": [ + {"variant_id": "fast", "run_limits": {"task_timeout": 60}}, + {"variant_id": "slow", "run_limits": {"task_timeout": 900}}, + ], + }, + ) + out_dir = tmp_path / "out" + + result = export_experiment([task_file], exp_file, out_dir) + + assert len(result.exported) == 2 + assert result.skipped == [] + assert result.load_skipped == [] + fast_dir = out_dir / "fast" / "greet" + slow_dir = out_dir / "slow" / "greet" + assert (fast_dir / "task.toml").exists() + assert (slow_dir / "task.toml").exists() + assert "timeout_sec = 60" in (fast_dir / "task.toml").read_text(encoding="utf-8") + assert "timeout_sec = 900" in (slow_dir / "task.toml").read_text(encoding="utf-8") + + def test_variant_prompt_override_lands_in_environment_task_yaml(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + exp_file = _write_experiment( + tmp_path, + { + "experiment_id": "prompt-ab", + "variants": [ + {"variant_id": "rewritten", "initial_prompt": "Write 'howdy' to greeting.txt."}, + ], + }, + ) + out_dir = tmp_path / "out" + + result = export_experiment([task_file], exp_file, out_dir) + + assert len(result.exported) == 1 + # instruction.md is a fixed placeholder now -- the real (overridden) prompt + # lands in environment/task.yaml instead. + instruction = (out_dir / "rewritten" / "greet" / "instruction.md").read_text(encoding="utf-8") + assert "howdy" not in instruction + emitted = yaml.safe_load( + (out_dir / "rewritten" / "greet" / "environment" / "task.yaml").read_text(encoding="utf-8") + ) + assert emitted["initial_prompt"] == "Write 'howdy' to greeting.txt." + + +class TestAgentOverridesAreHonored: + """Agent overrides ARE honorable now: CoderEvalAgent (C1.2) carries `task.agent` + verbatim into `environment/task.yaml` and executes it, so each variant exports + its own distinct directory rather than being skipped as indistinguishable.""" + + def test_variant_agent_model_override_is_exported_not_skipped(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + exp_file = _write_experiment( + tmp_path, + { + "experiment_id": "model-ab", + "variants": [ + {"variant_id": "sonnet", "agent": {"model": "claude-sonnet-5"}}, + {"variant_id": "opus", "agent": {"model": "claude-opus-5"}}, + ], + }, + ) + out_dir = tmp_path / "out" + + result = export_experiment([task_file], exp_file, out_dir) + + assert {r.out_dir for r in result.exported} == {out_dir / "sonnet" / "greet", out_dir / "opus" / "greet"} + assert result.skipped == [] + sonnet_yaml = (out_dir / "sonnet" / "greet" / "environment" / "task.yaml").read_text(encoding="utf-8") + opus_yaml = (out_dir / "opus" / "greet" / "environment" / "task.yaml").read_text(encoding="utf-8") + assert "claude-sonnet-5" in sonnet_yaml + assert "claude-opus-5" in opus_yaml + + def test_experiment_defaults_agent_override_is_exported(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + exp_file = _write_experiment( + tmp_path, + { + "experiment_id": "defaults-agent", + "defaults": {"agent": {"permission_mode": "bypassPermissions"}}, + "variants": [{"variant_id": "only"}], + }, + ) + out_dir = tmp_path / "out" + + result = export_experiment([task_file], exp_file, out_dir) + + assert len(result.exported) == 1 + assert result.skipped == [] + + def test_task_level_agent_config_alone_does_not_trip_the_skip(self, tmp_path: Path) -> None: + """The task's OWN agent.type (source='task') must not be mistaken for an experiment override.""" + task_file = _write_task(tmp_path) # agent: {type: claude-code} is the task's own, not the experiment's + exp_file = _write_experiment( + tmp_path, + { + "experiment_id": "no-override", + "variants": [{"variant_id": "only"}], + }, + ) + out_dir = tmp_path / "out" + + result = export_experiment([task_file], exp_file, out_dir) + + assert len(result.exported) == 1 + assert result.skipped == [] + + +class TestUnhonorableOverridesAreSkipped: + """A simulation override is the one thing an exported Harbor directory still cannot express.""" + + def test_variant_simulation_override_is_skipped_not_exported(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + exp_file = _write_experiment( + tmp_path, + { + "experiment_id": "sim-ab", + "variants": [ + { + "variant_id": "dialog", + "simulation": { + "enabled": True, + "persona": "a confused new user", + "goal": "get the greeting written", + "max_turns": 3, + }, + }, + ], + }, + ) + out_dir = tmp_path / "out" + + result = export_experiment([task_file], exp_file, out_dir) + + assert result.exported == [] + assert [s.variant_id for s in result.skipped] == ["dialog"] + assert "simulation" in result.skipped[0].reason + + def test_variant_with_no_simulation_override_still_exports_alongside_a_skipped_one(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + exp_file = _write_experiment( + tmp_path, + { + "experiment_id": "mixed", + "variants": [ + {"variant_id": "unmodified"}, + { + "variant_id": "dialog", + "simulation": { + "enabled": True, + "persona": "a confused new user", + "goal": "get the greeting written", + "max_turns": 3, + }, + }, + ], + }, + ) + out_dir = tmp_path / "out" + + result = export_experiment([task_file], exp_file, out_dir) + + assert len(result.exported) == 1 + assert result.exported[0].out_dir == out_dir / "unmodified" / "greet" + assert [s.variant_id for s in result.skipped] == ["dialog"] + + +class TestReplicateFanOut: + def test_repeats_produce_rep_subdirectories(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + exp_file = _write_experiment( + tmp_path, + { + "experiment_id": "reps", + "variants": [{"variant_id": "baseline", "repeats": 2}], + }, + ) + out_dir = tmp_path / "out" + + result = export_experiment([task_file], exp_file, out_dir) + + assert len(result.exported) == 2 + dests = sorted(str(r.out_dir.relative_to(out_dir)) for r in result.exported) + assert dests == [ + str(Path("baseline") / "greet" / "rep00"), + str(Path("baseline") / "greet" / "rep01"), + ] + + def test_single_replicate_omits_the_rep_segment(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + exp_file = _write_experiment( + tmp_path, + {"experiment_id": "no-reps", "variants": [{"variant_id": "only"}]}, + ) + out_dir = tmp_path / "out" + + result = export_experiment([task_file], exp_file, out_dir) + + assert result.exported[0].out_dir == out_dir / "only" / "greet" + + +class TestStructuralRefusalsPerVariant: + def test_non_docker_driver_variant_is_skipped_not_fatal(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path, {"sandbox": {"driver": "tempdir"}}) + exp_file = _write_experiment( + tmp_path, + {"experiment_id": "bad-driver", "variants": [{"variant_id": "only"}]}, + ) + out_dir = tmp_path / "out" + + result = export_experiment([task_file], exp_file, out_dir) + + assert result.exported == [] + assert len(result.skipped) == 1 + assert "sandbox.driver" in result.skipped[0].reason + + +class TestExportPathContainment: + """A crafted `variant_id`/`task_id`/`row_id` must not escape `out_dir`. + + None of these identifiers are validated as filesystem-safe anywhere in the + resolution chain (`variant_id` is a free string on `ExperimentVariant`), + and `row_id` in particular can come from an externally-sourced dataset + row. `_out_subdir` must refuse rather than silently write outside + `out_dir`. + """ + + def test_traversal_variant_id_is_refused(self, tmp_path: Path) -> None: + from coder_eval.harbor.experiment_packager import UnsafeExportPathError + + task_file = _write_task(tmp_path) + exp_file = _write_experiment( + tmp_path, + { + "experiment_id": "evil", + "variants": [{"variant_id": "../../../../tmp/pwned"}], + }, + ) + out_dir = tmp_path / "out" + + with pytest.raises(UnsafeExportPathError): + export_experiment([task_file], exp_file, out_dir) + + # Nothing should have been written outside out_dir. + assert not (tmp_path.parent / "tmp" / "pwned").exists() diff --git a/tests/test_harbor_export_golden.py b/tests/test_harbor_export_golden.py new file mode 100644 index 00000000..f543ba9b --- /dev/null +++ b/tests/test_harbor_export_golden.py @@ -0,0 +1,80 @@ +"""Golden-master test for the C2 packager's emitted directory. + +The regression this test is FOR: a transpiler's characteristic failure mode +is plausible-looking drift — a field silently starts mapping to the wrong +TOML key, a template loses a line — that no unit assertion happens to probe. +Diffing the whole emitted tree against a committed snapshot catches that +class of bug directly, cheaper than enumerating every field in prose. + +The source fixture (``tests/_fixtures/harbor_export_golden/source_task/``) +deliberately exercises every C2 mapping row at once: ``dockerfile_path`` with +no ``WORKDIR`` (exercises the append-a-WORKDIR path), ``reference:``, +``run_limits.task_timeout``, resource limits, ``network: none``, and both a +filesystem criterion and a ``reference_comparison`` criterion (proving the +placeholder-agent design from the packager's own module docstring survives +end to end, not just in an isolated unit test). + +Regenerate after an INTENTIONAL mapping change with:: + + GOLDEN_REGEN=1 uv run pytest tests/test_harbor_export_golden.py + +and review the resulting diff before committing. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from coder_eval.harbor.packager import export_task + + +_FIXTURE_ROOT = Path(__file__).parent / "_fixtures" / "harbor_export_golden" +_SOURCE_TASK = _FIXTURE_ROOT / "source_task" / "task.yaml" +_EXPECTED_DIR = _FIXTURE_ROOT / "expected" +_REGEN = os.environ.get("GOLDEN_REGEN", "").strip().lower() in {"1", "true", "yes", "on"} + + +def _relative_files(root: Path) -> dict[str, str]: + """Every file under root, as {posix-relative-path: text-content}.""" + return { + p.relative_to(root).as_posix(): p.read_text(encoding="utf-8") for p in sorted(root.rglob("*")) if p.is_file() + } + + +def test_export_matches_the_committed_golden_tree(tmp_path: Path) -> None: + out_dir = tmp_path / "out" + export_task(_SOURCE_TASK, out_dir) + actual = _relative_files(out_dir) + + if _REGEN: + for rel_path, content in actual.items(): + dest = _EXPECTED_DIR / rel_path + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(content, encoding="utf-8") + # test.sh's executable bit is part of the contract (C1.1) but git/the + # filesystem, not file content, carries it -- regenerate it too so a + # fresh checkout of the golden dir stays runnable if ever copied out. + (_EXPECTED_DIR / "tests" / "test.sh").chmod(0o755) + return + + assert _EXPECTED_DIR.is_dir(), "no committed golden tree yet -- run with GOLDEN_REGEN=1 first" + expected = _relative_files(_EXPECTED_DIR) + + assert set(actual) == set(expected), ( + f"emitted file set drifted from the golden tree.\n" + f" only in export: {sorted(set(actual) - set(expected))}\n" + f" only in golden: {sorted(set(expected) - set(actual))}" + ) + for rel_path in expected: + assert actual[rel_path] == expected[rel_path], f"content drifted for {rel_path}" + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX-only: NTFS has no chmod executable bit") +def test_test_sh_is_executable_in_the_golden_tree() -> None: + """The executable bit is part of C1.1's contract and isn't captured by file content.""" + test_sh = _EXPECTED_DIR / "tests" / "test.sh" + assert test_sh.is_file() + assert test_sh.stat().st_mode & 0o111 diff --git a/tests/test_harbor_packager.py b/tests/test_harbor_packager.py new file mode 100644 index 00000000..7299f2a9 --- /dev/null +++ b/tests/test_harbor_packager.py @@ -0,0 +1,659 @@ +"""``coder_eval.harbor.packager`` — the C2 export packager. + +Builds real task YAML files on disk (via ``load_task``, exactly the path a +user's ``coder-eval export`` invocation takes) rather than constructing +``TaskDefinition`` objects directly, so these tests exercise the same load +path C2 actually runs through. +""" + +from __future__ import annotations + +import os +import tomllib +from pathlib import Path + +import pytest +import yaml + +from coder_eval.harbor import packager +from coder_eval.harbor.packager import ( + DEFAULT_WORKDIR, + CriteriaNotExportableError, + TaskNotExportableError, + export_task, +) +from coder_eval.models import TaskDefinition + + +_REAL_INSPECT_IMAGE_WORKDIR = packager._inspect_image_workdir + + +@pytest.fixture(autouse=True) +def _no_real_docker_inspection(monkeypatch: pytest.MonkeyPatch) -> None: + """Keep this module hermetic: never let ``export_task`` shell out to a real + ``docker image inspect``, whose answer depends on what happens to be + cached on the machine running the tests (an image literally named + ``byod-custom-image:0.1.0`` -- this file's own placeholder BYOD image + name -- built by an unrelated docker-integration test elsewhere in the + suite answered ``/work`` here once, silently flipping this file's + DEFAULT_WORKDIR assertions). Tests that care about the inspection path + itself override this via monkeypatch locally. + """ + monkeypatch.setattr(packager, "_inspect_image_workdir", lambda image: None) + + +_BASE_TASK: dict[str, object] = { + "task_id": "greet", + "description": "Write a greeting to greeting.txt.", + "tags": ["smoke"], + "agent": {"type": "claude-code"}, + "initial_prompt": "Write 'hello' to greeting.txt.", + "sandbox": { + "driver": "docker", + "docker": {"image": "byod-custom-image:0.1.0", "network": "none"}, + "limits": {"max_memory_mb": 2048, "max_cpus": 2}, + }, + "success_criteria": [ + {"type": "file_exists", "path": "greeting.txt", "description": "exists"}, + {"type": "file_contains", "path": "greeting.txt", "includes": ["hello"], "description": "content"}, + ], +} + + +def _write_task(tmp_path: Path, overrides: dict[str, object] | None = None, name: str = "task.yaml") -> Path: + payload = {**_BASE_TASK, **(overrides or {})} + task_file = tmp_path / name + task_file.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") + return task_file + + +class TestStructuralRefusals: + def test_tempdir_driver_is_refused(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path, {"sandbox": {"driver": "tempdir"}}) + with pytest.raises(TaskNotExportableError): + export_task(task_file, tmp_path / "out") + + def test_unsupported_criteria_are_refused_before_any_file_is_written(self, tmp_path: Path) -> None: + task_file = _write_task( + tmp_path, + { + "success_criteria": [ + {"type": "skill_triggered", "expected_skill": "s", "skill_name": "s", "description": "d"}, + ] + }, + ) + out_dir = tmp_path / "out" + with pytest.raises(CriteriaNotExportableError): + export_task(task_file, out_dir) + assert not out_dir.exists(), "a refused export must not leave a partial directory behind" + + def test_credentials_criteria_can_be_allowed_explicitly(self, tmp_path: Path) -> None: + task_file = _write_task( + tmp_path, + {"success_criteria": [{"type": "llm_judge", "prompt": "grade it", "description": "d"}]}, + ) + result = export_task(task_file, tmp_path / "out", allow_credentials=True) + assert result.out_dir.exists() + + +class TestEmittedDirectoryStructure: + def test_full_export_produces_the_documented_layout(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + out_dir = tmp_path / "out" + + result = export_task(task_file, out_dir) + + assert result.out_dir == out_dir + assert (out_dir / "task.toml").is_file() + assert (out_dir / "instruction.md").is_file() + assert (out_dir / "environment").is_dir() + assert (out_dir / "tests" / "test.sh").is_file() + assert (out_dir / "tests" / "task.yaml").is_file() + + def test_instruction_md_is_a_fixed_placeholder_not_the_real_prompt(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) # _BASE_TASK's initial_prompt is "Write 'hello' to greeting.txt." + out_dir = tmp_path / "out" + export_task(task_file, out_dir) + text = (out_dir / "instruction.md").read_text(encoding="utf-8") + assert "greeting.txt" not in text # the real prompt must never land here + assert "autogenerated" in text.lower() + assert "environment/task.yaml" in text + assert "CoderEvalAgent" in text + + emitted = yaml.safe_load((out_dir / "environment" / "task.yaml").read_text(encoding="utf-8")) + assert emitted["initial_prompt"] == "Write 'hello' to greeting.txt." # the real prompt lives here instead + + def test_test_sh_is_executable_and_references_the_resolved_workdir(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + out_dir = tmp_path / "out" + + result = export_task(task_file, out_dir) + + test_sh = out_dir / "tests" / "test.sh" + if os.name != "nt": # NTFS has no chmod executable bit + assert test_sh.stat().st_mode & 0o111, "test.sh must be executable" + content = test_sh.read_text(encoding="utf-8") + assert f'coder-eval evaluate /tests/task.yaml "{result.workdir}"' in content + assert "coder-eval harbor reward /logs/verifier --out /logs/verifier/reward.json" in content + + def test_task_toml_parses_and_carries_the_mapped_fields(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + doc = tomllib.loads((out_dir / "task.toml").read_text(encoding="utf-8")) + assert doc["task"]["name"] == "coder-eval/greet" + assert doc["task"]["keywords"] == ["smoke"] + assert "docker_image" not in doc["environment"] # always built from environment/Dockerfile now + assert doc["environment"]["memory_mb"] == 2048 + assert doc["environment"]["cpus"] == 2 + assert doc["environment"]["network_mode"] == "no-network" + assert doc["environment"]["workdir"] == DEFAULT_WORKDIR + + def test_network_bridge_maps_to_public(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path, {"sandbox": {"driver": "docker", "docker": {"network": "bridge"}}}) + out_dir = tmp_path / "out" + export_task(task_file, out_dir) + doc = tomllib.loads((out_dir / "task.toml").read_text(encoding="utf-8")) + assert doc["environment"]["network_mode"] == "public" + + +class TestVerifierTaskYaml: + """tests/task.yaml must never set agent: {type: none} — see packager.py's module docstring.""" + + def test_never_sets_agent_type_none(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + out_dir = tmp_path / "out" + export_task(task_file, out_dir) + + emitted = yaml.safe_load((out_dir / "tests" / "task.yaml").read_text(encoding="utf-8")) + assert emitted["agent"]["type"] != "none" + + def test_reloads_as_a_valid_task_definition(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + out_dir = tmp_path / "out" + export_task(task_file, out_dir) + + emitted = yaml.safe_load((out_dir / "tests" / "task.yaml").read_text(encoding="utf-8")) + reloaded = TaskDefinition.model_validate(emitted) + assert reloaded.task_id == "greet" + assert len(reloaded.success_criteria) == 2 + + def test_reference_comparison_survives_the_none_agent_trap(self, tmp_path: Path) -> None: + """The corrected design: a placeholder real agent type unblocks reference_comparison. + + If tests/task.yaml set agent: {type: none} instead, this would raise at + TaskDefinition.model_validate — see the module docstring for why. + """ + task_file = _write_task( + tmp_path, + { + "reference": {"directory": "reference"}, + "success_criteria": [ + { + "type": "reference_comparison", + "agent_file": "greeting.txt", + "reference_file": "greeting.txt", + "description": "matches reference", + } + ], + }, + ) + (tmp_path / "reference").mkdir() + (tmp_path / "reference" / "greeting.txt").write_text("hello", encoding="utf-8") + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + emitted = yaml.safe_load((out_dir / "tests" / "task.yaml").read_text(encoding="utf-8")) + reloaded = TaskDefinition.model_validate(emitted) # must not raise + assert reloaded.reference is not None and reloaded.reference.directory == "reference" + assert (out_dir / "tests" / "reference" / "greeting.txt").read_text(encoding="utf-8") == "hello" + + +class TestAgentPhaseTaskYaml: + """environment/task.yaml — the CoderEvalAgent embed's criteria-free real-agent config.""" + + def test_baked_at_fixed_path_via_dockerfile_copy(self, tmp_path: Path) -> None: + env_dir = tmp_path / "environment" + env_dir.mkdir() + (env_dir / "Dockerfile").write_text("FROM ubuntu:24.04\nWORKDIR /app\n", encoding="utf-8") + task_file = _write_task( + tmp_path, + {"sandbox": {"driver": "docker", "docker": {"dockerfile_path": "environment/Dockerfile"}}}, + ) + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + assert (out_dir / "environment" / "task.yaml").exists() + dockerfile_text = (out_dir / "environment" / "Dockerfile").read_text(encoding="utf-8") + assert "COPY task.yaml /opt/coder-eval-task/task.yaml" in dockerfile_text + + def test_carries_the_real_agent_config_but_no_real_criteria(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path, {"agent": {"type": "claude-code", "model": "claude-opus-5"}}) + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + emitted = yaml.safe_load((out_dir / "environment" / "task.yaml").read_text(encoding="utf-8")) + assert emitted["agent"]["type"] == "claude-code" + assert emitted["agent"]["model"] == "claude-opus-5" + assert emitted["success_criteria"] == [] # never the real ones + # The real task's success_criteria (2 entries in _BASE_TASK) must never leak in. + assert "reference_comparison" not in str(emitted["success_criteria"]) + + def test_sandbox_driver_forced_to_tempdir_no_nested_docker(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) # _BASE_TASK uses sandbox.driver: docker + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + emitted = yaml.safe_load((out_dir / "environment" / "task.yaml").read_text(encoding="utf-8")) + assert emitted["sandbox"]["driver"] == "tempdir" + + def test_reloads_as_a_valid_task_definition(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + emitted = yaml.safe_load((out_dir / "environment" / "task.yaml").read_text(encoding="utf-8")) + reloaded = TaskDefinition.model_validate(emitted) # must not raise + assert reloaded.task_id == "greet" + + def test_prebuilt_image_with_no_dockerfile_path_gets_one_synthesized(self, tmp_path: Path) -> None: + task_file = _write_task( + tmp_path, {"sandbox": {"driver": "docker", "docker": {"image": "byod-custom-image:0.1.0"}}} + ) + out_dir = tmp_path / "out" + + result = export_task(task_file, out_dir) + + assert (out_dir / "environment" / "task.yaml").exists() + dockerfile_text = (out_dir / "environment" / "Dockerfile").read_text(encoding="utf-8") + assert dockerfile_text.startswith("FROM byod-custom-image:0.1.0\n") + assert "COPY task.yaml /opt/coder-eval-task/task.yaml" in dockerfile_text + assert not any("No Dockerfile to bake" in w for w in result.warnings) + + +class TestDockerfileWorkdirResolution: + def test_dockerfile_with_no_workdir_gets_one_appended_and_warned(self, tmp_path: Path) -> None: + env_dir = tmp_path / "environment" + env_dir.mkdir() + (env_dir / "Dockerfile").write_text("FROM ubuntu:24.04\nRUN apt-get update\n", encoding="utf-8") + task_file = _write_task( + tmp_path, + {"sandbox": {"driver": "docker", "docker": {"dockerfile_path": "environment/Dockerfile"}}}, + ) + out_dir = tmp_path / "out" + + result = export_task(task_file, out_dir) + + assert result.workdir == DEFAULT_WORKDIR + dockerfile_text = (out_dir / "environment" / "Dockerfile").read_text(encoding="utf-8") + assert f"WORKDIR {DEFAULT_WORKDIR}" in dockerfile_text + assert any("declared no WORKDIR" in w for w in result.warnings) + + def test_dockerfile_with_an_existing_workdir_is_respected_and_not_touched(self, tmp_path: Path) -> None: + env_dir = tmp_path / "environment" + env_dir.mkdir() + original = "FROM ubuntu:24.04\nWORKDIR /workspace\nRUN apt-get update\n" + (env_dir / "Dockerfile").write_text(original, encoding="utf-8") + task_file = _write_task( + tmp_path, + {"sandbox": {"driver": "docker", "docker": {"dockerfile_path": "environment/Dockerfile"}}}, + ) + out_dir = tmp_path / "out" + + result = export_task(task_file, out_dir) + + assert result.workdir == "/workspace" + dockerfile_text = (out_dir / "environment" / "Dockerfile").read_text(encoding="utf-8") + # The WORKDIR-bearing content is untouched; only the task.yaml COPY + # line (baked in for a CoderEvalAgent embed, see agent_paths.py) is appended. + assert dockerfile_text.startswith(original) + assert "COPY task.yaml /opt/coder-eval-task/task.yaml" in dockerfile_text + assert not any("declared no WORKDIR" in w for w in result.warnings) + # test.sh and task.toml must agree with the same resolved workdir. + assert '"/workspace"' in (out_dir / "tests" / "test.sh").read_text(encoding="utf-8") + doc = tomllib.loads((out_dir / "task.toml").read_text(encoding="utf-8")) + assert doc["environment"]["workdir"] == "/workspace" + + def test_docker_working_dir_override_wins_over_the_dockerfile(self, tmp_path: Path) -> None: + env_dir = tmp_path / "environment" + env_dir.mkdir() + (env_dir / "Dockerfile").write_text("FROM ubuntu:24.04\nWORKDIR /workspace\n", encoding="utf-8") + task_file = _write_task( + tmp_path, + { + "sandbox": { + "driver": "docker", + "docker": {"dockerfile_path": "environment/Dockerfile", "working_dir": "/custom"}, + } + }, + ) + out_dir = tmp_path / "out" + + result = export_task(task_file, out_dir) + + assert result.workdir == "/custom" + + +class TestCoderEvalAgentBaseImageWarning: + """v1 assumes the exported image already has coder-eval installed (`FROM coder-eval-agent:`).""" + + def test_warns_when_dockerfile_does_not_from_coder_eval_agent(self, tmp_path: Path) -> None: + env_dir = tmp_path / "environment" + env_dir.mkdir() + (env_dir / "Dockerfile").write_text("FROM ubuntu:24.04\n", encoding="utf-8") + task_file = _write_task( + tmp_path, + {"sandbox": {"driver": "docker", "docker": {"dockerfile_path": "environment/Dockerfile"}}}, + ) + + result = export_task(task_file, tmp_path / "out") + + assert any("coder-eval-agent" in w for w in result.warnings) + + def test_no_warning_when_dockerfile_froms_coder_eval_agent(self, tmp_path: Path) -> None: + env_dir = tmp_path / "environment" + env_dir.mkdir() + (env_dir / "Dockerfile").write_text("FROM coder-eval-agent:latest\n", encoding="utf-8") + task_file = _write_task( + tmp_path, + {"sandbox": {"driver": "docker", "docker": {"dockerfile_path": "environment/Dockerfile"}}}, + ) + + result = export_task(task_file, tmp_path / "out") + + assert not any("coder-eval-agent" in w for w in result.warnings) + + def test_warns_when_prebuilt_image_does_not_name_coder_eval_agent(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) # _BASE_TASK's image is byod-custom-image:0.1.0 + result = export_task(task_file, tmp_path / "out") + assert any("coder-eval-agent" in w for w in result.warnings) + + def test_no_warning_when_prebuilt_image_names_coder_eval_agent( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + # Give inspection a real-looking answer -- a None here would ALSO warn + # ("Could not determine ...'s own WORKDIR"), whose text incidentally + # contains "coder-eval-agent" (the image name), which is not the + # warning this test is checking for. + monkeypatch.setattr(packager, "_inspect_image_workdir", lambda image: "/work") + task_file = _write_task( + tmp_path, {"sandbox": {"driver": "docker", "docker": {"image": "coder-eval-agent:0.12.0"}}} + ) + result = export_task(task_file, tmp_path / "out") + assert not any("coder-eval-agent" in w for w in result.warnings) + + +class TestPrePostRunWarnings: + def test_pre_run_and_post_run_are_warned_not_silently_dropped(self, tmp_path: Path) -> None: + task_file = _write_task( + tmp_path, + { + "pre_run": [{"command": "echo setup"}], + "post_run": [{"command": "echo cleanup"}], + }, + ) + out_dir = tmp_path / "out" + + result = export_task(task_file, out_dir) + + assert any("pre_run" in w for w in result.warnings) + assert any("post_run" in w for w in result.warnings) + + def test_no_pre_or_post_run_produces_no_such_warnings(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + result = export_task(task_file, tmp_path / "out") + assert not any("pre_run" in w or "post_run" in w for w in result.warnings) + + +class TestPrebuiltImageWorkdirInspection: + """A pre-built (no ``dockerfile_path``) image has no Dockerfile ``WORKDIR`` line + to read, so the packager shells out to ``docker image inspect`` for it -- a + real bug this closes: defaulting to ``/app`` unconditionally exported a task + that failed at Harbor verify time with exit 127, because Harbor's + ``docker exec -w`` (unlike ``docker run -w``) refuses to chdir into a path + that doesn't already exist in the image (confirmed live against + ``coder-eval-agent:latest``, whose real WORKDIR is ``/work``). + """ + + def test_uses_the_inspected_workdir_when_docker_reports_one( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(packager, "_inspect_image_workdir", lambda image: "/work") + task_file = _write_task(tmp_path) # _BASE_TASK's image is byod-custom-image:0.1.0 + + result = export_task(task_file, tmp_path / "out") + + assert result.workdir == "/work" + assert not any("Could not determine" in w for w in result.warnings) + + def test_falls_back_to_default_and_warns_when_inspection_fails( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(packager, "_inspect_image_workdir", lambda image: None) + task_file = _write_task(tmp_path) + + result = export_task(task_file, tmp_path / "out") + + assert result.workdir == DEFAULT_WORKDIR + assert any("Could not determine" in w and "byod-custom-image:0.1.0" in w for w in result.warnings) + + def test_explicit_working_dir_wins_over_inspection(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(packager, "_inspect_image_workdir", lambda image: "/from-inspection") + task_file = _write_task(tmp_path, {"sandbox": {"driver": "docker", "docker": {"working_dir": "/explicit"}}}) + + result = export_task(task_file, tmp_path / "out") + + assert result.workdir == "/explicit" + + def test_inspect_image_workdir_parses_real_subprocess_output(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Exercise the real (un-mocked) helper against a stubbed ``subprocess.run``. + + The module-wide autouse fixture stubs out ``_inspect_image_workdir`` + itself for hermeticity, so this restores the real function first -- + it is the one thing here under test. + """ + monkeypatch.setattr(packager, "_inspect_image_workdir", _REAL_INSPECT_IMAGE_WORKDIR) + + class _FakeResult: + returncode = 0 + stdout = "/work\n" + + def _fake_run(cmd, **kwargs): + assert cmd[:3] == ["docker", "image", "inspect"] + return _FakeResult() + + monkeypatch.setattr(packager.subprocess, "run", _fake_run) + assert packager._inspect_image_workdir("some-image:tag") == "/work" + + def test_inspect_image_workdir_returns_none_when_docker_binary_is_missing( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(packager, "_inspect_image_workdir", _REAL_INSPECT_IMAGE_WORKDIR) + + def _raise(cmd, **kwargs): + raise FileNotFoundError("docker not found") + + monkeypatch.setattr(packager.subprocess, "run", _raise) + assert packager._inspect_image_workdir("some-image:tag") is None + + +class TestEnvPassthroughSections: + """``task.toml``'s ``[environment.env]``/``[verifier.env]`` -- the SSOT-reuse fix: these + are derived from the resolved task's OWN ``sandbox.docker.env_passthrough`` (the same + allowlist coder-eval's own docker driver already uses), not a second hardcoded list. + """ + + def test_default_env_passthrough_names_land_in_both_sections_as_templates(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + doc = tomllib.loads((out_dir / "task.toml").read_text(encoding="utf-8")) + assert doc["environment"]["env"]["ANTHROPIC_API_KEY"] == "${ANTHROPIC_API_KEY:-}" + assert doc["environment"]["env"]["AWS_BEARER_TOKEN_BEDROCK"] == "${AWS_BEARER_TOKEN_BEDROCK:-}" + assert doc["verifier"]["env"]["ANTHROPIC_API_KEY"] == "${ANTHROPIC_API_KEY:-}" + # No literal value ever appears -- only the `${NAME:-}` template (empty default, never required). + assert "$" in doc["environment"]["env"]["ANTHROPIC_API_KEY"] + + def test_home_is_excluded_even_though_it_is_in_the_default_allowlist(self, tmp_path: Path) -> None: + # HOME is intentional in coder-eval's OWN docker driver only because it also + # bind-mounts ~/.claude; Harbor's container has no such mount, so forwarding + # the host's literal HOME would point the container at a directory that + # doesn't exist in it. + task_file = _write_task(tmp_path) + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + doc = tomllib.loads((out_dir / "task.toml").read_text(encoding="utf-8")) + assert "HOME" not in doc["environment"]["env"] + assert "HOME" not in doc["verifier"]["env"] + + def test_env_passthrough_extra_is_included_too(self, tmp_path: Path) -> None: + task_file = _write_task( + tmp_path, + {"sandbox": {"driver": "docker", "docker": {"env_passthrough_extra": ["MY_CUSTOM_TOKEN"]}}}, + ) + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + doc = tomllib.loads((out_dir / "task.toml").read_text(encoding="utf-8")) + assert doc["environment"]["env"]["MY_CUSTOM_TOKEN"] == "${MY_CUSTOM_TOKEN:-}" + + def test_no_env_section_when_the_resolved_allowlist_is_empty(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path, {"sandbox": {"driver": "docker", "docker": {"env_passthrough": []}}}) + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + doc = tomllib.loads((out_dir / "task.toml").read_text(encoding="utf-8")) + assert "env" not in doc["environment"] + assert "verifier" not in doc # no timeout_sec either in this fixture -- section fully omitted + + +class TestTemplateSourcesCopy: + """``TemplateDirSource`` directories must be copied into the export -- otherwise + ``environment/task.yaml`` would name an absolute HOST path (see + ``task_loader.resolve_template_source_paths``) that does not exist inside the + container the ``CoderEvalAgent`` embed actually runs in. + """ + + def _write_template_dir(self, tmp_path: Path, name: str = "starter") -> Path: + template_dir = tmp_path / name + template_dir.mkdir() + (template_dir / "main.py").write_text("def stub(): ...\n", encoding="utf-8") + return template_dir + + def test_template_dir_is_copied_and_path_rewritten(self, tmp_path: Path) -> None: + template_dir = self._write_template_dir(tmp_path) + task_file = _write_task( + tmp_path, + { + "sandbox": { + "driver": "docker", + "docker": {"image": "byod-custom-image:0.1.0", "network": "none"}, + "template_sources": [{"type": "template_dir", "path": str(template_dir)}], + } + }, + ) + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + copied = out_dir / "environment" / "templates" / "00-starter" / "main.py" + assert copied.read_text(encoding="utf-8") == "def stub(): ...\n" + + emitted = yaml.safe_load((out_dir / "environment" / "task.yaml").read_text(encoding="utf-8")) + assert emitted["sandbox"]["template_sources"][0]["path"] == "/opt/coder-eval-task/templates/00-starter" + + dockerfile_text = (out_dir / "environment" / "Dockerfile").read_text(encoding="utf-8") + assert "COPY templates/ /opt/coder-eval-task/templates/" in dockerfile_text + + def test_agent_phase_sandbox_preserves_python_and_limits(self, tmp_path: Path) -> None: + task_file = _write_task( + tmp_path, + { + "sandbox": { + "driver": "docker", + "docker": {"image": "byod-custom-image:0.1.0", "network": "none"}, + "python": {"env_packages": ["pytest"]}, + } + }, + ) + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + emitted = yaml.safe_load((out_dir / "environment" / "task.yaml").read_text(encoding="utf-8")) + assert emitted["sandbox"]["driver"] == "tempdir" + assert emitted["sandbox"]["python"]["env_packages"] == ["pytest"] + assert "docker" not in emitted["sandbox"] + + def test_no_templates_dir_or_copy_line_when_the_task_has_no_template_sources(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + assert not (out_dir / "environment" / "templates").exists() + dockerfile_text = (out_dir / "environment" / "Dockerfile").read_text(encoding="utf-8") + assert "templates" not in dockerfile_text + + def test_nonexistent_template_dir_is_a_hard_export_failure(self, tmp_path: Path) -> None: + """A missing template dir is NOT downgraded to a warning: the agent-phase + task.yaml still references it, so a silently-skipped copy would ship an + export whose agent has no starter code -- every criterion then reads + "file does not exist" indistinguishable from a real agent failure.""" + missing = tmp_path / "does-not-exist" + task_file = _write_task( + tmp_path, + { + "sandbox": { + "driver": "docker", + "docker": {"image": "byod-custom-image:0.1.0", "network": "none"}, + "template_sources": [{"type": "template_dir", "path": str(missing)}], + } + }, + ) + out_dir = tmp_path / "out" + + with pytest.raises(packager.TaskNotExportableError, match="is not a directory"): + export_task(task_file, out_dir) + + assert not (out_dir / "environment" / "templates").exists() + + def test_non_template_dir_source_is_carried_over_unchanged_with_a_warning(self, tmp_path: Path) -> None: + """A `RepoSource`/`StarterFilesSource` template_sources entry resolves + entirely inside the container already (clone at runtime / inline file + content) -- it is legitimately not locally copyable, so it warns and is + carried over unchanged rather than failing the export.""" + task_file = _write_task( + tmp_path, + { + "sandbox": { + "driver": "docker", + "docker": {"image": "byod-custom-image:0.1.0", "network": "none"}, + "template_sources": [ + {"type": "starter_files", "files": [{"path": "README.md", "content": "hello"}]}, + ], + } + }, + ) + out_dir = tmp_path / "out" + + result = export_task(task_file, out_dir) + + assert any("StarterFilesSource" in w and "not a TemplateDirSource" in w for w in result.warnings) + emitted = yaml.safe_load((out_dir / "environment" / "task.yaml").read_text(encoding="utf-8")) + template_sources = emitted["sandbox"]["template_sources"] + assert len(template_sources) == 1 + assert template_sources[0]["files"] == [{"path": "README.md", "content": "hello"}] diff --git a/tests/test_harbor_portability.py b/tests/test_harbor_portability.py new file mode 100644 index 00000000..98a6dd08 --- /dev/null +++ b/tests/test_harbor_portability.py @@ -0,0 +1,135 @@ +"""``coder_eval.harbor.portability`` — the C1.4 criteria portability audit. + +Registry-derived coverage (the same shape CE036 uses for +``live_decidable_polarities``): every criterion type in the real +``SuccessCriterion`` union must have a portability classification, so a 16th +criterion type added later fails this test instead of silently exporting as +if it were ``PORTABLE``. +""" + +from __future__ import annotations + +import typing + +import pytest + +from coder_eval.harbor.portability import ( + _PORTABILITY_BY_TYPE, + CriterionPortability, + UnknownCriterionTypeError, + audit_criteria, + classify, +) +from coder_eval.models import SuccessCriterion +from coder_eval.models.criteria import ( + AgentJudgeCriterion, + ClassificationMatchCriterion, + CliCalledCriterion, + CommandExecutedCriterion, + CommandsEfficiencyCriterion, + FileCheckCriterion, + FileContainsCriterion, + FileExistsCriterion, + FileMatchesRegexCriterion, + JsonCheckCriterion, + LLMJudgeCriterion, + ReferenceComparisonCriterion, + RunCommandCriterion, + SkillTriggeredCriterion, + UiPathEvalCriterion, +) + + +def _union_member_type_tags() -> set[str]: + """Every ``type:`` literal tag actually reachable through the real union.""" + # SuccessCriterion is `Annotated[X | Y | ..., Field(discriminator="type")]`. + union = typing.get_args(SuccessCriterion)[0] + tags: set[str] = set() + for member in typing.get_args(union): + default = member.model_fields["type"].default + assert isinstance(default, str) + tags.add(default) + return tags + + +def test_every_real_criterion_type_is_classified() -> None: + """Fails closed: a new criterion type added to the union with no line here is a bug, not a PORTABLE default.""" + assert _union_member_type_tags() == set(_PORTABILITY_BY_TYPE) + + +def test_unknown_type_raises_rather_than_defaulting_portable() -> None: + with pytest.raises(UnknownCriterionTypeError): + classify("not_a_real_criterion_type") + + +@pytest.mark.parametrize( + "criterion", + [ + FileExistsCriterion(description="d", path="p"), + FileContainsCriterion(description="d", path="p", includes=["t"]), + FileMatchesRegexCriterion(description="d", path="p", pattern="."), + JsonCheckCriterion(description="d", path="p"), + FileCheckCriterion(description="d", path="p"), + RunCommandCriterion(description="d", command="true"), + ClassificationMatchCriterion(description="d", path="p", expected_label="x", allowed_labels=["x", "y"]), + ], +) +def test_portable_criteria_never_block_export(criterion: object) -> None: + assert audit_criteria([criterion]) == [] # type: ignore[list-item] + + +def test_reference_comparison_does_not_block_export() -> None: + """NEEDS_REFERENCE is not in _BLOCKING_IN_V1 — C2 always emits tests/reference/.""" + criterion = ReferenceComparisonCriterion(description="d", agent_file="p", reference_file="r") + assert audit_criteria([criterion]) == [] + + +@pytest.mark.parametrize( + "criterion", + [ + CommandExecutedCriterion(description="d", command_pattern="."), + CommandsEfficiencyCriterion(description="d", expected_commands=3), + SkillTriggeredCriterion(description="d", expected_skill="s", skill_name="s"), + CliCalledCriterion(description="d", verb="v"), + ], +) +def test_missing_functionality_criteria_always_block_export(criterion: object) -> None: + """No flag can supply what does not exist yet (C1.3 / a recorder-baking step).""" + issues = audit_criteria([criterion], allow_credentials=True) # type: ignore[list-item] + assert len(issues) == 1 + assert issues[0].criterion_type == criterion.type # type: ignore[attr-defined] + + +@pytest.mark.parametrize( + "criterion", + [ + LLMJudgeCriterion(description="d", prompt="p"), + AgentJudgeCriterion(description="d", prompt="p"), + UiPathEvalCriterion(description="d", agent_name="a", eval_set="p", thresholds={}), + ], +) +def test_credentials_criteria_block_by_default_but_have_an_escape_hatch(criterion: object) -> None: + assert len(audit_criteria([criterion])) == 1 # type: ignore[list-item] + assert audit_criteria([criterion], allow_credentials=True) == [] # type: ignore[list-item] + + +def test_a_clean_multi_criterion_task_reports_no_issues() -> None: + criteria = [ + FileExistsCriterion(description="d1", path="p1"), + ReferenceComparisonCriterion(description="d2", agent_file="p2", reference_file="r"), + ] + assert audit_criteria(criteria) == [] + + +def test_a_mixed_task_reports_only_the_blocking_criteria() -> None: + criteria = [ + FileExistsCriterion(description="portable", path="p1"), + SkillTriggeredCriterion(description="needs trajectory", expected_skill="s", skill_name="s"), + LLMJudgeCriterion(description="needs credentials", prompt="p"), + ] + issues = audit_criteria(criteria) + assert {i.criterion_description for i in issues} == {"needs trajectory", "needs credentials"} + assert {i.portability for i in issues} == { + CriterionPortability.NEEDS_TRAJECTORY, + CriterionPortability.NEEDS_CREDENTIALS, + } diff --git a/tests/test_harbor_reward.py b/tests/test_harbor_reward.py new file mode 100644 index 00000000..3b6652c1 --- /dev/null +++ b/tests/test_harbor_reward.py @@ -0,0 +1,123 @@ +"""``coder_eval.harbor.reward`` — the C1.1 verifier shim's reward writer. + +Table test over the three no-file exclusion cases the design doc (C1.1 / +C3) calls out: an ungraded row, a missing ``task.json``, and a malformed +``task.json`` must all write NO reward file and raise, distinctly from a +measured row (including a genuine zero) writing the file. The whole point of +this module is that those two outcomes are never confusable. +""" + +from __future__ import annotations + +import json +from datetime import datetime +from pathlib import Path + +import pytest + +from coder_eval.harbor.reward import RegradeError, RewardWriteSkippedError, compute_reward, write_reward +from coder_eval.models import AgentKind, EvaluationResult, FinalStatus +from coder_eval.path_utils import TASK_JSON_FILENAME + + +def _write_task_json(run_dir: Path, **overrides: object) -> None: + """Write a run_dir/task.json from an EvaluationResult, with field overrides.""" + fields: dict[str, object] = { + "task_id": "t", + "task_description": "d", + "variant_id": "default", + "agent_type": AgentKind.CLAUDE_CODE, + "started_at": datetime(2020, 1, 1), + "final_status": FinalStatus.SUCCESS, + "iteration_count": 1, + } + fields.update(overrides) + result = EvaluationResult(**fields) # type: ignore[arg-type] + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / TASK_JSON_FILENAME).write_text(result.model_dump_json(), encoding="utf-8") + + +class TestComputeReward: + """Every path through compute_reward: measured (incl. zero) vs. the three unmeasured cases.""" + + def test_measured_row_returns_weighted_score(self, tmp_path: Path) -> None: + run_dir = tmp_path / "run" + _write_task_json(run_dir, final_status=FinalStatus.SUCCESS, weighted_score=0.75) + assert compute_reward(run_dir) == {"reward": 0.75} + + def test_measured_zero_is_not_confused_with_unmeasured(self, tmp_path: Path) -> None: + """A criterion suite that genuinely scored 0.0 must still write the file. + + This is the case the whole design guards: `weighted_score or 0.0` would + pass this test AND the ungraded test below identically, which is exactly + the CE049 defect. `weighted_score is not None` is the only correct test. + """ + run_dir = tmp_path / "run" + _write_task_json(run_dir, final_status=FinalStatus.FAILURE, weighted_score=0.0) + assert compute_reward(run_dir) == {"reward": 0.0} + + def test_ungraded_row_raises_reward_write_skipped(self, tmp_path: Path) -> None: + run_dir = tmp_path / "run" + _write_task_json(run_dir, final_status=FinalStatus.NOT_GRADED, weighted_score=None) + with pytest.raises(RewardWriteSkippedError): + compute_reward(run_dir) + + def test_missing_task_json_raises_regrade_error(self, tmp_path: Path) -> None: + run_dir = tmp_path / "run" + run_dir.mkdir() + with pytest.raises(RegradeError): + compute_reward(run_dir) + + def test_malformed_task_json_raises_regrade_error(self, tmp_path: Path) -> None: + run_dir = tmp_path / "run" + run_dir.mkdir() + (run_dir / TASK_JSON_FILENAME).write_text("{not json", encoding="utf-8") + with pytest.raises(RegradeError): + compute_reward(run_dir) + + +class TestWriteReward: + """The file-writing half: confirms the no-file contract, not just the exception.""" + + def test_writes_reward_json_for_a_measured_row(self, tmp_path: Path) -> None: + run_dir = tmp_path / "run" + _write_task_json(run_dir, final_status=FinalStatus.SUCCESS, weighted_score=0.5) + out = tmp_path / "verifier" / "reward.json" + + rewards = write_reward(run_dir, out) + + assert rewards == {"reward": 0.5} + assert json.loads(out.read_text(encoding="utf-8")) == {"reward": 0.5} + + def test_writes_no_file_for_an_ungraded_row(self, tmp_path: Path) -> None: + run_dir = tmp_path / "run" + _write_task_json(run_dir, final_status=FinalStatus.NOT_GRADED, weighted_score=None) + out = tmp_path / "verifier" / "reward.json" + + with pytest.raises(RewardWriteSkippedError): + write_reward(run_dir, out) + + assert not out.exists() + assert not out.parent.exists(), "must not even create the destination dir on a skipped write" + + def test_writes_no_file_when_task_json_is_missing(self, tmp_path: Path) -> None: + run_dir = tmp_path / "run" + run_dir.mkdir() + out = tmp_path / "verifier" / "reward.json" + + with pytest.raises(RegradeError): + write_reward(run_dir, out) + + assert not out.exists() + + def test_reward_json_is_a_flat_dict_per_harbor_contract(self, tmp_path: Path) -> None: + """Harbor's VerifierResult.rewards is dict[str, float | int] | None, never a bare scalar.""" + run_dir = tmp_path / "run" + _write_task_json(run_dir, final_status=FinalStatus.SUCCESS, weighted_score=1.0) + out = tmp_path / "verifier" / "reward.json" + + write_reward(run_dir, out) + + parsed = json.loads(out.read_text(encoding="utf-8")) + assert isinstance(parsed, dict) + assert parsed == {"reward": 1.0} diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index a1fbeed1..b894ac6f 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -878,6 +878,79 @@ async def create_dummy_agent(_self): await orchestrator._cleanup() +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("in_container", "expect_warning"), + [(False, True), (True, False)], + ids=["host_reachable_warns", "in_container_silent"], +) +async def test_workspace_dir_staleness_warning_keys_on_in_container_not_field( + tmp_path, monkeypatch, caplog, in_container, expect_warning +): + """`--workspace-dir` on the host must warn on a pre-populated dir exactly like + DIRECT_WRITE does; only the in-container writer (a fresh container filesystem + every run) legitimately suppresses it. Regression for the bug where the + suppression keyed on `workspace_dir is None` instead of `IN_CONTAINER_ENV`.""" + import logging + from datetime import datetime + + from coder_eval import orchestrator as orchestrator_module + from coder_eval.models import IN_CONTAINER_ENV, ApiBackend, DirectRoute, EvaluationResult + + class DummyAgent: + async def start(self, working_directory, *, env_path_prepend=None, plugin_tools_dir=None): + self.working_directory = working_directory + + def get_sdk_options(self): + return {"env": {"PATH": os.environ.get("PATH", "")}} + + def get_environment_info(self): + return {} + + async def create_dummy_agent(_self): + return DummyAgent() + + task_file = Path("tasks/hello_date.yaml") + task, _ = load_task(task_file) + task.sandbox.python = None + + run_dir = tmp_path / "test_run" / "hello_date" + ws = tmp_path / "ws" + ws.mkdir(parents=True) + (ws / "stale.txt").write_text("from a prior run") + + orchestrator = Orchestrator(task=task, run_dir=run_dir, variant_id="test-variant", workspace_dir=ws) + orchestrator.task_file = task_file + orchestrator.result = EvaluationResult( + task_id=task.task_id, + task_description=task.description, + variant_id="test-variant", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime.now(), + final_status="FAILURE", + iteration_count=0, + environment_info={}, + ) + + if in_container: + monkeypatch.setenv(IN_CONTAINER_ENV, "1") + else: + monkeypatch.delenv(IN_CONTAINER_ENV, raising=False) + + monkeypatch.setattr(orchestrator_module.settings, "api_backend", ApiBackend.DIRECT) + monkeypatch.setattr(type(orchestrator_module.settings), "validate_api_keys", lambda _self, _agent_type: None) + monkeypatch.setattr(orchestrator_module, "resolve_route", lambda _settings: DirectRoute(judge_transport=None)) + monkeypatch.setattr(Orchestrator, "_create_agent", create_dummy_agent) + + with caplog.at_level(logging.WARNING, logger="coder_eval.orchestrator"): + await orchestrator._setup() + + warned = any("already exists and is non-empty" in r.message for r in caplog.records) + assert warned is expect_warning + + await orchestrator._cleanup() + + @pytest.mark.asyncio async def test_orchestrator_cleanup_persistent_sandbox(tmp_path): """DIRECT_WRITE: sandbox already lives in artifacts; _cleanup keeps it in place (no move).""" diff --git a/uv.lock b/uv.lock index 28bca6f6..990baa92 100644 --- a/uv.lock +++ b/uv.lock @@ -495,6 +495,7 @@ dependencies = [ { name = "radon" }, { name = "rich" }, { name = "starlette" }, + { name = "tomli-w" }, { name = "tqdm" }, { name = "typer" }, ] @@ -520,6 +521,9 @@ dev = [ { name = "pytest-xdist", extra = ["psutil"] }, { name = "ruff" }, ] +harbor = [ + { name = "harbor" }, +] litellm = [ { name = "litellm" }, ] @@ -537,6 +541,7 @@ requires-dist = [ { name = "click", specifier = ">=8.3.3" }, { name = "defusedxml", marker = "extra == 'dev'", specifier = ">=0.7.1" }, { name = "google-antigravity", marker = "extra == 'antigravity'", specifier = "==0.1.8" }, + { name = "harbor", marker = "extra == 'harbor'", specifier = "==0.22.0" }, { name = "httpx2", specifier = ">=2.12.0,<3.0.0" }, { name = "jmespath", specifier = ">=1.1.0" }, { name = "jsonschema", specifier = ">=4.26.0" }, @@ -560,11 +565,12 @@ requires-dist = [ { name = "rich", specifier = ">=14.3.3" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.15.7" }, { name = "starlette", specifier = ">=1.3.1" }, + { name = "tomli-w", specifier = ">=1.0.0" }, { name = "tqdm", specifier = ">=4.67.3" }, { name = "typer", specifier = ">=0.24.1" }, { name = "uipath", marker = "extra == 'uipath'", specifier = ">=2.10.31" }, ] -provides-extras = ["dev", "uipath", "litellm", "codex", "antigravity", "opencode", "pi"] +provides-extras = ["dev", "uipath", "litellm", "codex", "antigravity", "opencode", "pi", "harbor"] [[package]] name = "colorama" @@ -718,6 +724,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, ] +[[package]] +name = "deprecation" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/d3/8ae2869247df154b64c1884d7346d412fed0c49df84db635aab2d1c40e62/deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff", size = 173788, upload-time = "2020-04-20T14:23:38.738Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/c3/253a89ee03fc9b9682f1541728eb66db7db22148cd94f89ab22528cd1e1b/deprecation-2.1.0-py2.py3-none-any.whl", hash = "sha256:a10811591210e1fb0e768a8c25517cabeabcba6f0bf96564f8ff45189f90b14a", size = 11178, upload-time = "2020-04-20T14:23:36.581Z" }, +] + +[[package]] +name = "dirhash" +version = "0.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "scantree" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1d/70/49f93897f3a4f7ab5f20a854ebc91aad47854e9fb2cd169e3a4452fa3f5e/dirhash-0.5.0.tar.gz", hash = "sha256:e60760f0ab2e935d8cb088923ea2c6492398dca42cec785df778985fd4cd5386", size = 21377, upload-time = "2024-08-03T22:14:13.322Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/1f/c8bf92552b7f0a13b9f12b85e3de8df6d9814240e0f8ce8f37433df028b3/dirhash-0.5.0-py3-none-any.whl", hash = "sha256:523dfd6b058c64f45b31604376926c6e2bd2ea301d0df23095d4055674e38b09", size = 13119, upload-time = "2024-08-03T22:14:11.688Z" }, +] + [[package]] name = "distlib" version = "0.4.0" @@ -754,6 +784,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, ] +[[package]] +name = "fastapi" +version = "0.141.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8a/02/91e3416a8fdd715abb903a952a6bec7cdd8d14eed55d415fc8595524c319/fastapi-0.141.1.tar.gz", hash = "sha256:e8822fc40db1e1858054d7a949a888695bc9bdce70139178e33bd2871a453ca1", size = 425799, upload-time = "2026-07-29T17:18:05.568Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/03/10388a42375ee7e4ac9b94eb2c5c569c8b5795e377e701c9ac3ad63de890/fastapi-0.141.1-py3-none-any.whl", hash = "sha256:bfb91aa2d334c61cb35ba9a116fc123b3d3df31640b801cf57a7a78ec3f603b3", size = 131954, upload-time = "2026-07-29T17:18:04.364Z" }, +] + [[package]] name = "fastuuid" version = "0.14.0" @@ -786,11 +832,11 @@ wheels = [ [[package]] name = "filelock" -version = "3.25.2" +version = "3.32.6" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/94/b8/00651a0f559862f3bb7d6f7477b192afe3f583cc5e26403b44e59a55ab34/filelock-3.25.2.tar.gz", hash = "sha256:b64ece2b38f4ca29dd3e810287aa8c48182bbecd1ae6e9ae126c9b35f1382694", size = 40480, upload-time = "2026-03-11T20:45:38.487Z" } +sdist = { url = "https://files.pythonhosted.org/packages/38/46/126b1831dca12060d4a8296bf9c4fe5c93c4f22197fa239cb0cc82042bba/filelock-3.32.6.tar.gz", hash = "sha256:a3f55a18af3652a94d8f47d6055df434f254ca1d02ef2524850c6d249ca2512c", size = 225172, upload-time = "2026-09-08T22:57:11.528Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a4/a5/842ae8f0c08b61d6484b52f99a03510a3a72d23141942d216ebe81fefbce/filelock-3.25.2-py3-none-any.whl", hash = "sha256:ca8afb0da15f229774c9ad1b455ed96e85a81373065fb10446672f64444ddf70", size = 26759, upload-time = "2026-03-11T20:45:37.437Z" }, + { url = "https://files.pythonhosted.org/packages/cc/06/4f138f618dbea66803291274f228f01daf29f306fe8b96bc30dab765df75/filelock-3.32.6-py3-none-any.whl", hash = "sha256:3f16ecd0117feae0dfc147e8c62eb5daeccd8bd800378c3ddf416de9b4feb6b1", size = 100189, upload-time = "2026-09-08T22:57:10.182Z" }, ] [[package]] @@ -953,6 +999,51 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] +[[package]] +name = "h2" +version = "4.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "hpack" }, + { name = "hyperframe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e7/85/7c366e69d84c17bb778fe41419e1fbcce3033d5b7ce29bbffff0a98b859f/h2-4.4.1.tar.gz", hash = "sha256:4e866ffb1a869ae14dd9b5e6beb5c24a13da0495ad72b65925ded182521c1516", size = 2157281, upload-time = "2026-08-03T11:45:09.509Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/22/e85faf23bd72a92d1921e37d674ca56eb298a3c8be31fdecef0ff2b3aaac/h2-4.4.1-py3-none-any.whl", hash = "sha256:0e25f1462b23c9cb82d9eb02e28bc706dac2a68cb457c6a0d74d63c8a2a5d0e6", size = 62636, upload-time = "2026-08-03T11:44:59.164Z" }, +] + +[[package]] +name = "harbor" +version = "0.22.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dirhash" }, + { name = "fastapi" }, + { name = "filelock" }, + { name = "httpx" }, + { name = "jinja2" }, + { name = "litellm" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "platformdirs" }, + { name = "pydantic" }, + { name = "pyjwt" }, + { name = "python-dotenv" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "rich" }, + { name = "shortuuid" }, + { name = "supabase" }, + { name = "tenacity" }, + { name = "toml" }, + { name = "typer" }, + { name = "uvicorn" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/12/a0/a37532fb647b93ef709b63a7fbdf613a84837a569fdd88f69dd7105bf093/harbor-0.22.0.tar.gz", hash = "sha256:becf0ce354026cc37899855e0a0d2687cd5188034a43635849245069aad0938b", size = 1832554, upload-time = "2026-08-22T03:22:39.798Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/12/d517f1ca18738f7be78a210b13264c64bace531fdf644632128966aca530/harbor-0.22.0-py3-none-any.whl", hash = "sha256:4c4c6571b3d160ed0cb45b82918136751fb08e7b8596412723ac00dde12eeabb", size = 2058718, upload-time = "2026-08-22T03:22:38.071Z" }, +] + [[package]] name = "hf-xet" version = "1.6.0" @@ -977,6 +1068,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/63/ca511b6f802f28cf3489b280fe77475bcca8de85e81a6299d7916b5b5555/hf_xet-1.6.0-cp38-abi3-win_arm64.whl", hash = "sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3", size = 3859359, upload-time = "2026-08-03T22:33:11.725Z" }, ] +[[package]] +name = "hpack" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/26/5b/fcabf6028144a8723726318b07a32c2f3314acdff6265743cf08a344b18e/hpack-4.2.0.tar.gz", hash = "sha256:0895cfa3b5531fc65fe439c05eb65144f123bf7a394fcaa56aa423548d8e45c0", size = 51300, upload-time = "2026-06-23T18:34:46.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/b4/4a9fcfb2aef6ba44d9073ecd301443aa00b3dac95de5619f2a7de7ec8a91/hpack-4.2.0-py3-none-any.whl", hash = "sha256:858ac0b02280fa582b5080d68db0899c62a80375e0e5413a74970c5e518b6986", size = 34246, upload-time = "2026-06-23T18:34:45.472Z" }, +] + [[package]] name = "httpcore" version = "1.0.9" @@ -1018,6 +1118,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[package.optional-dependencies] +http2 = [ + { name = "h2" }, +] + [[package]] name = "httpx-sse" version = "0.4.3" @@ -1072,6 +1177,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/0e/eafef18f1a75e125e68395db21131db0cf868a128ecd2fce69b4df6c584b/huggingface_hub-1.28.0-py3-none-any.whl", hash = "sha256:58a8bacb03072edfc38067065e9dc24bbb34805410fcd36a1632de0b329660bb", size = 793202, upload-time = "2026-08-18T12:27:12.719Z" }, ] +[[package]] +name = "hyperframe" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/02/e7/94f8232d4a74cc99514c13a9f995811485a6903d48e5d952771ef6322e30/hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08", size = 26566, upload-time = "2025-01-22T21:41:49.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, +] + [[package]] name = "identify" version = "2.6.18" @@ -1739,6 +1853,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/f9/690a8600b93c332de3ab4a344a4ac34f00c8f104917061f779db6a918ed6/pathlib-1.0.1-py3-none-any.whl", hash = "sha256:f35f95ab8b0f59e6d354090350b44a80a80635d22efdedfa84c7ad1cf0a74147", size = 14363, upload-time = "2022-05-04T13:37:20.585Z" }, ] +[[package]] +name = "pathspec" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, +] + [[package]] name = "pip" version = "26.2.1" @@ -1796,11 +1919,11 @@ wheels = [ [[package]] name = "platformdirs" -version = "4.9.4" +version = "4.11.8" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/19/56/8d4c30c8a1d07013911a8fdbd8f89440ef9f08d07a1b50ab8ca8be5a20f9/platformdirs-4.9.4.tar.gz", hash = "sha256:1ec356301b7dc906d83f371c8f487070e99d3ccf9e501686456394622a01a934", size = 28737, upload-time = "2026-03-05T18:34:13.271Z" } +sdist = { url = "https://files.pythonhosted.org/packages/53/18/f3bb8ef0d3b930692343da8aa4d3cbcd6749477c053959395ac81965a6e9/platformdirs-4.11.8.tar.gz", hash = "sha256:f23abafea7dd4276d1f29104b83598d7dcc567cafd07c9c951e66665645437fc", size = 37182, upload-time = "2026-09-08T22:20:42.866Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/63/d7/97f7e3a6abb67d8080dd406fd4df842c2be0efaf712d1c899c32a075027c/platformdirs-4.9.4-py3-none-any.whl", hash = "sha256:68a9a4619a666ea6439f2ff250c12a853cd1cbd5158d258bd824a7df6be2f868", size = 21216, upload-time = "2026-03-05T18:34:12.172Z" }, + { url = "https://files.pythonhosted.org/packages/f4/e1/5b7b8bbb55084d1425bcb9bc823ff519e1b2be05f6ebb0089e2eacc38413/platformdirs-4.11.8-py3-none-any.whl", hash = "sha256:52f2f181bbfde907966932cc8312d967d02976422d66d537ea16092b8e291081", size = 24027, upload-time = "2026-09-08T22:20:41.537Z" }, ] [[package]] @@ -1812,6 +1935,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "postgrest" +version = "2.31.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "deprecation" }, + { name = "httpx", extra = ["http2"] }, + { name = "pydantic" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e2/22/88c470d8838d2678a44e0172d061630b8837cba3fb7fb492e28f6578c309/postgrest-2.31.0.tar.gz", hash = "sha256:2f395d84b2ee34fc57622ff2f711df603e2ede625f98e5015240741888f7bd0c", size = 14419, upload-time = "2026-06-04T13:37:20.474Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/3e/41909586cb148db0259e0208310067afda4cc097f4c7a779e829c1e68c46/postgrest-2.31.0-py3-none-any.whl", hash = "sha256:c2fd47c94e13ee8335111c4f03c9a24ea9766ce9d35fc3cd7330057c9e7ea0c3", size = 23098, upload-time = "2026-06-04T13:37:19.452Z" }, +] + [[package]] name = "pre-commit" version = "4.5.1" @@ -2338,6 +2476,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/93/f7/d00d9b4a0313a6be3a3e0818e6375e15da6d7076f4ae47d1324e7ca986a1/radon-6.0.1-py2.py3-none-any.whl", hash = "sha256:632cc032364a6f8bb1010a2f6a12d0f14bc7e5ede76585ef29dc0cecf4cd8859", size = 52784, upload-time = "2023-03-26T06:24:33.949Z" }, ] +[[package]] +name = "realtime" +version = "2.31.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "typing-extensions" }, + { name = "websockets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/34/54a1eaaefa24db5cb12596fd74792e08efa53ed30dc5bce2c0a68ded6146/realtime-2.31.0.tar.gz", hash = "sha256:9e641cb4d77ca0fe768515f8cf9f83550c79f49ce1550a95afc2dc0e252be8c9", size = 18716, upload-time = "2026-06-04T13:37:22.089Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/60/164246615e8b059f6d53d34648a0784260421ec98a07eb1e45160f063221/realtime-2.31.0-py3-none-any.whl", hash = "sha256:f6e494b53d6a6e80b6efcee6711c8dd40413a52e766271de1bce8ced6c36cc1d", size = 22374, upload-time = "2026-06-04T13:37:21.162Z" }, +] + [[package]] name = "referencing" version = "0.37.0" @@ -2567,6 +2719,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/e7/5c595c75e9f41a44f30e526eda465ea0b4eec93470e074e4a111b253f13a/s3transfer-0.19.2-py3-none-any.whl", hash = "sha256:d8168eccca828cbb2cd573675333f3bddd254313a9c42494b84c76b539e8ba25", size = 90216, upload-time = "2026-07-22T19:30:43.251Z" }, ] +[[package]] +name = "scantree" +version = "0.0.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "pathspec" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/e4/40998faefc72ba1ddeb640a44fba92935353525dba110488806da8339c0b/scantree-0.0.4.tar.gz", hash = "sha256:15bd5cb24483b04db2c70653604e8ea3522e98087db7e38ab8482f053984c0ac", size = 24643, upload-time = "2024-08-03T20:08:59.413Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/ce/828467ddfa0d2fe473673026442d2032d552a168e42cfbf25fd0e5264e0c/scantree-0.0.4-py3-none-any.whl", hash = "sha256:7616ab65aa6b7f16fcf8e6fa1d9afaa99a27ab72bba05c61b691853b96763174", size = 20690, upload-time = "2024-08-03T20:08:58.137Z" }, +] + [[package]] name = "shellingham" version = "1.5.4" @@ -2576,6 +2741,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, ] +[[package]] +name = "shortuuid" +version = "1.0.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/e2/bcf761f3bff95856203f9559baf3741c416071dd200c0fc19fad7f078f86/shortuuid-1.0.13.tar.gz", hash = "sha256:3bb9cf07f606260584b1df46399c0b87dd84773e7b25912b7e391e30797c5e72", size = 9662, upload-time = "2024-03-11T20:11:06.879Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/44/21d6bf170bf40b41396480d8d49ad640bca3f2b02139cd52aa1e272830a5/shortuuid-1.0.13-py3-none-any.whl", hash = "sha256:a482a497300b49b4953e15108a7913244e1bb0d41f9d332f5e9925dba33a3c5a", size = 10529, upload-time = "2024-03-11T20:11:04.807Z" }, +] + [[package]] name = "simple-websocket" version = "1.1.0" @@ -2658,6 +2832,76 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/69/06/36d260a695f383345ab5bbc3fd447249594ae2fa8dfd19c533d5ae23f46b/stevedore-5.7.0-py3-none-any.whl", hash = "sha256:fd25efbb32f1abb4c9e502f385f0018632baac11f9ee5d1b70f88cc5e22ad4ed", size = 54483, upload-time = "2026-02-20T13:27:05.561Z" }, ] +[[package]] +name = "storage3" +version = "2.31.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "deprecation" }, + { name = "httpx", extra = ["http2"] }, + { name = "pydantic" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/13/30/fee43d523d3f680a833a4aae5bf8094de0b9031b0c2bddb3e0bc6e829e1b/storage3-2.31.0.tar.gz", hash = "sha256:d2161e2ea650dc115a1787c30e09b118365589ac772f4dd8643e3a503ecfc667", size = 20348, upload-time = "2026-06-04T13:37:23.703Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/b2/60d86a3a99ae743e8a00a6df912f85269b3bc882ba217b8ce1690881c669/storage3-2.31.0-py3-none-any.whl", hash = "sha256:4bf46e8bea320743179a6beafdc7531c5242495e00e0cc22af7c7a9d69d4ed84", size = 28492, upload-time = "2026-06-04T13:37:22.792Z" }, +] + +[[package]] +name = "strenum" +version = "0.4.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/ad/430fb60d90e1d112a62ff57bdd1f286ec73a2a0331272febfddd21f330e1/StrEnum-0.4.15.tar.gz", hash = "sha256:878fb5ab705442070e4dd1929bb5e2249511c0bcf2b0eeacf3bcd80875c82eff", size = 23384, upload-time = "2023-06-29T22:02:58.399Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/69/297302c5f5f59c862faa31e6cb9a4cd74721cd1e052b38e464c5b402df8b/StrEnum-0.4.15-py3-none-any.whl", hash = "sha256:a30cda4af7cc6b5bf52c8055bc4bf4b2b6b14a93b574626da33df53cf7740659", size = 8851, upload-time = "2023-06-29T22:02:56.947Z" }, +] + +[[package]] +name = "supabase" +version = "2.31.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx" }, + { name = "postgrest" }, + { name = "realtime" }, + { name = "storage3" }, + { name = "supabase-auth" }, + { name = "supabase-functions" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fe/8e/54a2f950629689b1613434a61fc3bff5f92f84ba6b20213f5b2add05c1bb/supabase-2.31.0.tar.gz", hash = "sha256:3467b09d00482b9a0138235bdbde7a350426f93cf2a1342372eaddfc669f1206", size = 9805, upload-time = "2026-06-04T13:37:25.22Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/06/5e6f4bf89dedadf81f893832115c1866da1aa093142c9989c2818780dae3/supabase-2.31.0-py3-none-any.whl", hash = "sha256:25f2a99207a75f2d9377e2332783b4389cf56b02cbebdaf0c1743112dcbb704e", size = 16728, upload-time = "2026-06-04T13:37:24.278Z" }, +] + +[[package]] +name = "supabase-auth" +version = "2.31.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx", extra = ["http2"] }, + { name = "pydantic" }, + { name = "pyjwt", extra = ["crypto"] }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/8a/408689cf39820f0d46d2731d6747ff94dbefc87ae977b4b5c4066da5b070/supabase_auth-2.31.0.tar.gz", hash = "sha256:0945b33fa96239c76dc8eaf96d7d2c94991950d24b4cfe4a5c2da9aa5e909663", size = 39151, upload-time = "2026-06-04T13:37:27.375Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/5e/22f3b0546bb1f0985f06fb1ebf5f6405a3b1bb7a5db01142c26cf148e988/supabase_auth-2.31.0-py3-none-any.whl", hash = "sha256:5e9c8b4ecdee6af04dbcb06455ce78cb15674806fcb6b425170455307d70b0ee", size = 48363, upload-time = "2026-06-04T13:37:26.26Z" }, +] + +[[package]] +name = "supabase-functions" +version = "2.31.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx", extra = ["http2"] }, + { name = "strenum" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/84/5d/61c2446ed26a57fa5543f9c270a731320911569202340d15341f72cdba7c/supabase_functions-2.31.0.tar.gz", hash = "sha256:4ad027b3ae3bd28b31233339f4db1da6965affd3546f655b421baf40cee2690f", size = 4683, upload-time = "2026-06-04T13:37:28.878Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/79/1a8162ce7d705381a4f2668c0da68e097b103c1aa701418a31da52905c7b/supabase_functions-2.31.0-py3-none-any.whl", hash = "sha256:3fdc4c4766152bfda63bdd0e286fc8a06f50e1280711fae4a1dfc9b7e9ebabc6", size = 8794, upload-time = "2026-06-04T13:37:28.022Z" }, +] + [[package]] name = "tenacity" version = "9.1.4" @@ -2741,6 +2985,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cd/2b/2be299bab55fc595e3d38567edb1a87f86e594842968fa9515a07bdcf422/tokenizers-0.23.1-cp310-abi3-win_arm64.whl", hash = "sha256:a26197957d8e4425dfba746315f3c425ea00cfa8367c5fbc4ec73447893dcea9", size = 2664127, upload-time = "2026-04-27T14:43:26.949Z" }, ] +[[package]] +name = "toml" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f", size = 22253, upload-time = "2020-11-01T01:40:22.204Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588, upload-time = "2020-11-01T01:40:20.672Z" }, +] + [[package]] name = "tomli" version = "2.4.0"