From a09c36cac19061b668dca376dc412ecde06eaea8 Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Thu, 10 Sep 2026 07:22:35 -0700 Subject: [PATCH 01/12] feat(harbor): export coder-eval tasks to Harbor, with coder-eval as the grader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements C0/C1.1/C1.4/C2/C3(golden) of the Harbor interop plan (tmp/harborframework.md): `coder-eval export --format harbor -o ` emits a Harbor task directory whose `tests/test.sh` grades with coder-eval's own criteria via a two-line verifier shim. - `coder_eval.harbor.reward` — translates a graded run's task.json into Harbor's reward.json contract. A missing/unmeasured weighted_score writes no reward file (never coalesced to 0.0), so Harbor's own RewardFileNotFoundError path masks the trial instead of scoring it zero. - `coder_eval.harbor.portability` — classifies all 15 criterion types so an unsupported criterion is refused at export time, not as a mystery low reward at verify time. Registry-derived coverage over the real SuccessCriterion union. - `coder_eval.harbor.packager` — the export writer itself: task.toml, instruction.md, environment/, tests/test.sh, tests/task.yaml, tests/reference/. New dependency: tomli-w. - `coder-eval harbor reward` / `coder-eval export` CLI commands. - CE004 widened to scope harbor/ as a core layer (must not import coder_eval.cli). Verified live against real Harbor 0.22.0 + real docker (not just unit tested): exported tasks scored reward 1.0 (correct solution), 0.5 (wrong content, genuinely measured not masked), and correctly masked with no solution present. Two bugs the live run found and fixed that no unit test caught: test.sh referenced tests/task.yaml as cwd-relative when Harbor uploads the whole tests/ tree to the absolute /tests/; and `set -e` in test.sh was silently discarding every real failing score as if it were unmeasured, since `coder-eval evaluate` exits non-zero on a failed criterion (a measured verdict, not an infra failure). v1 assumes the exported task's Dockerfile is FROM coder-eval-agent: (confirmed live) so the verifier's `coder-eval` binary is present; the packager warns, non-blocking, when it can't confirm that. Full design, verification log and known follow-up (C1.2/C1.3, solution/ auto-derivation, network:none, docs) recorded in tmp/harborframework.md. Co-Authored-By: Claude Sonnet 5 --- pyproject.toml | 1 + src/coder_eval/cli/__init__.py | 5 + src/coder_eval/cli/export_command.py | 71 ++++ src/coder_eval/cli/harbor_command.py | 65 ++++ src/coder_eval/harbor/__init__.py | 22 ++ src/coder_eval/harbor/packager.py | 341 ++++++++++++++++++ src/coder_eval/harbor/portability.py | 149 ++++++++ src/coder_eval/harbor/reward.py | 88 +++++ .../expected/environment/Dockerfile | 4 + .../expected/instruction.md | 1 + .../harbor_export_golden/expected/task.toml | 21 ++ .../expected/tests/reference/greeting.txt | 1 + .../expected/tests/task.yaml | 23 ++ .../expected/tests/test.sh | 17 + .../source_task/environment/Dockerfile | 2 + .../source_task/reference/greeting.txt | 1 + .../source_task/task.yaml | 33 ++ tests/lint/rules/no_cli_imports_in_core.py | 9 +- tests/test_harbor_export_golden.py | 77 ++++ tests/test_harbor_packager.py | 308 ++++++++++++++++ tests/test_harbor_portability.py | 135 +++++++ tests/test_harbor_reward.py | 123 +++++++ uv.lock | 2 + 23 files changed, 1497 insertions(+), 2 deletions(-) create mode 100644 src/coder_eval/cli/export_command.py create mode 100644 src/coder_eval/cli/harbor_command.py create mode 100644 src/coder_eval/harbor/__init__.py create mode 100644 src/coder_eval/harbor/packager.py create mode 100644 src/coder_eval/harbor/portability.py create mode 100644 src/coder_eval/harbor/reward.py create mode 100644 tests/_fixtures/harbor_export_golden/expected/environment/Dockerfile create mode 100644 tests/_fixtures/harbor_export_golden/expected/instruction.md create mode 100644 tests/_fixtures/harbor_export_golden/expected/task.toml create mode 100644 tests/_fixtures/harbor_export_golden/expected/tests/reference/greeting.txt create mode 100644 tests/_fixtures/harbor_export_golden/expected/tests/task.yaml create mode 100755 tests/_fixtures/harbor_export_golden/expected/tests/test.sh create mode 100644 tests/_fixtures/harbor_export_golden/source_task/environment/Dockerfile create mode 100644 tests/_fixtures/harbor_export_golden/source_task/reference/greeting.txt create mode 100644 tests/_fixtures/harbor_export_golden/source_task/task.yaml create mode 100644 tests/test_harbor_export_golden.py create mode 100644 tests/test_harbor_packager.py create mode 100644 tests/test_harbor_portability.py create mode 100644 tests/test_harbor_reward.py diff --git a/pyproject.toml b/pyproject.toml index c1834793..234c7921 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ "pydantic>=2.12.5", "pydantic-settings>=2.14.2", "pyyaml>=6.0.3", + "tomli-w>=1.0.0", "typer>=0.24.1", "click>=8.3.3", "rich>=14.3.3", diff --git a/src/coder_eval/cli/__init__.py b/src/coder_eval/cli/__init__.py index 66eaddae..62eda48a 100644 --- a/src/coder_eval/cli/__init__.py +++ b/src/coder_eval/cli/__init__.py @@ -8,6 +8,8 @@ from .console import console from .evaluate_command import evaluate_command from .execute_command import execute_command +from .export_command import export_command +from .harbor_command import harbor_app, reward_command from .plan_command import plan_command from .report_command import report_command from .run_command import run_command @@ -82,6 +84,9 @@ def main( app.command(name="evaluate")(track_command("evaluate")(evaluate_command)) app.command(name="report")(track_command("report")(report_command)) app.command(name="aggregate")(track_command("aggregate")(aggregate_command)) +app.command(name="export")(track_command("export")(export_command)) +harbor_app.command(name="reward")(track_command("harbor-reward")(reward_command)) +app.add_typer(harbor_app, name="harbor") # Hidden internal command invoked inside the Docker container only — UNWRAPPED # (it runs inside the run-task subprocess and would double-count / pollute events). app.command(name="_run-task-internal", hidden=True)(run_task_internal_command) diff --git a/src/coder_eval/cli/export_command.py b/src/coder_eval/cli/export_command.py new file mode 100644 index 00000000..1405b0b7 --- /dev/null +++ b/src/coder_eval/cli/export_command.py @@ -0,0 +1,71 @@ +"""``coder-eval export`` — emit a task in another framework's native format. + +Currently one target: ``--format harbor`` (C2). This is a *writer* only — +``--format`` is reserved so the same flag can later grow a *reader* +(``coder-eval run ``, demoted — see ``tmp/harborframework.partB.md``) +without a naming collision. +""" + +from __future__ import annotations + +from pathlib import Path + +import typer + +from ..harbor.packager import CriteriaNotExportableError, TaskNotExportableError, export_task +from .console import console + + +_SUPPORTED_FORMATS = ("harbor",) + + +def export_command( + task_file: Path = typer.Argument( # noqa: B008 + ..., + help="The coder-eval task YAML to export.", + exists=True, + dir_okay=False, + ), + output_dir: Path = typer.Option( # noqa: B008 + ..., + "--output", + "-o", + help="Directory to write the exported task into (created if missing).", + file_okay=False, + ), + format: str = typer.Option( + "harbor", + "--format", + help=f"Target format. Supported: {', '.join(_SUPPORTED_FORMATS)}.", + ), + allow_credentials: bool = typer.Option( + False, + "--allow-credentials", + help=( + "Export criteria that need model credentials/network inside the verifier " + "(llm_judge / agent_judge / uipath_eval) anyway. Only pass this if you have " + "already provisioned that access yourself — the export does not do it for you." + ), + ), +) -> None: + """Export a coder-eval task to another framework's native directory format. + + Examples: + coder-eval export tasks/my_task.yaml -o dist/harbor/my_task --format harbor + """ + if format not in _SUPPORTED_FORMATS: + console.print(f"[red]✗[/] Unsupported --format {format!r}. Supported: {', '.join(_SUPPORTED_FORMATS)}.") + raise typer.Exit(1) + + try: + result = export_task(task_file, output_dir, allow_credentials=allow_credentials) + except (TaskNotExportableError, CriteriaNotExportableError) as e: + console.print(f"[red]✗[/] {e}") + raise typer.Exit(1) from e + + console.print(f"[green]✓[/] Exported {task_file} → {result.out_dir} (format: {format})") + for warning in result.warnings: + console.print(f"[yellow]⚠[/] {warning}") + + +__all__ = ["export_command"] diff --git a/src/coder_eval/cli/harbor_command.py b/src/coder_eval/cli/harbor_command.py new file mode 100644 index 00000000..17c68ffd --- /dev/null +++ b/src/coder_eval/cli/harbor_command.py @@ -0,0 +1,65 @@ +"""``coder-eval harbor`` — Harbor framework adherence commands. + +Distinct from task *definition* (``coder-eval export --format harbor``, +tracked separately): this namespace is for what an outer harness's runtime +contract expects — right now, the verifier's reward file. See +``coder_eval.harbor`` for the shared logic and ``tmp/harborframework.md`` for +the design this implements (C1.1). +""" + +from __future__ import annotations + +from pathlib import Path + +import typer + +from ..harbor.reward import RegradeError, RewardWriteSkippedError, write_reward +from .console import console + + +harbor_app = typer.Typer( + name="harbor", + help="Harbor framework adherence commands (reward reporting, log-layout shims).", + add_completion=False, +) + + +def reward_command( + run_dir: Path = typer.Argument( # noqa: B008 + ..., + help="A graded coder-eval run directory (holds task.json) — typically the " + + "--run-dir passed to the preceding `coder-eval evaluate`.", + exists=True, + file_okay=False, + ), + out: Path = typer.Option( # noqa: B008 + ..., + "--out", + help="Where to write Harbor's reward file, e.g. /logs/verifier/reward.json.", + ), +) -> None: + """Translate a graded run's ``task.json`` into Harbor's ``reward.json`` contract. + + Writes NOTHING and exits non-zero when the row carries no measured verdict + (``weighted_score`` is ``None`` — an ungraded or crashed row) or when + ``task.json`` itself is missing/unparseable. That is deliberate: Harbor's + own verifier already treats a missing reward file as an infrastructure + failure distinct from a measured zero (see ``coder_eval.harbor.reward``), + so writing ``{"reward": 0.0}`` here would silently convert "never + measured" into "measured and scored zero" — the exact defect CE049 guards + against one layer up, in this same shape at the artifact boundary. + + Typical use, from a Harbor task's ``tests/test.sh``: + + coder-eval evaluate tests/task.yaml "$WORKDIR" --in-place --run-dir /logs/verifier + coder-eval harbor reward /logs/verifier --out /logs/verifier/reward.json + """ + try: + rewards = write_reward(run_dir, out) + except (RewardWriteSkippedError, RegradeError) as e: + console.print(f"[yellow]⚠[/] {e}") + raise typer.Exit(code=1) from e + console.print(f"[green]✓[/] wrote {out}: {rewards}") + + +__all__ = ["harbor_app", "reward_command"] diff --git a/src/coder_eval/harbor/__init__.py b/src/coder_eval/harbor/__init__.py new file mode 100644 index 00000000..c726b0f6 --- /dev/null +++ b/src/coder_eval/harbor/__init__.py @@ -0,0 +1,22 @@ +"""``coder_eval.harbor`` — the Harbor framework adherence layer. + +Harbor (Laude Institute / Terminal-Bench 2.0, harborframework.com) is an outer +harness with its own runtime contract: a fixed reward-file convention the +verifier phase must satisfy, a fixed log layout, a trajectory format. This +package is deliberately narrow — it translates coder-eval's own artifacts +(``task.json``, ``weighted_score``) into that contract. It does not know how a +task is *defined*; that is the export/packager concern (``coder-eval export +--format harbor``), tracked separately. + +Direction-agnostic by design: the same shim serves a coder-eval task exported +to run under Harbor (coder-eval as the grader) and a coder-eval agent embedded +inside a Harbor-authored task (Harbor's own ``tests/test.sh`` as the grader, +Part A step 8 — coder-eval is purely the agent there and this package is +unused on that path). + +This package is a core layer like ``orchestration/`` — it must not import +``coder_eval.cli`` (CE004). Raise plain exceptions and let the CLI wrap them, +exactly as ``orchestration/regrade.py`` does. +""" + +from __future__ import annotations diff --git a/src/coder_eval/harbor/packager.py b/src/coder_eval/harbor/packager.py new file mode 100644 index 00000000..8fbf6877 --- /dev/null +++ b/src/coder_eval/harbor/packager.py @@ -0,0 +1,341 @@ +"""C2 — the packager. ``coder-eval export --format harbor -o ``. + +Emits a Harbor task directory from a coder-eval task, with coder-eval's own +criteria as the grader (C1.1's verifier shim). Task *definition* only; the +runtime contract (C1.1's reward writer) is what makes the emitted +``tests/test.sh`` actually work once Harbor runs it. + +Emitted layout (see ``tmp/harborframework.md`` § C2 for the full mapping +table): + + / + ├── task.toml + ├── instruction.md + ├── environment/ + │ └── Dockerfile # copied from dockerfile_path, WORKDIR pinned + └── tests/ + ├── test.sh # C1.1's two-line shim + ├── task.yaml # the criteria, as authored + └── reference/ # task.reference, verifier-side only + +One design point worth stating explicitly because it is not obvious from +either side's docs: ``tests/task.yaml`` (uploaded whole into the container at +``/tests/`` by Harbor's own verifier — see ``_TEST_SH_TEMPLATE``, which +therefore references it as ``/tests/task.yaml``, not a cwd-relative path; +confirmed live against real docker, not assumed) must NOT set +``agent: {type: none}`` +even though C1.1's own docstring describes the verifier phase as exactly +that. coder-eval's own ``check_none_agent`` validator rejects any criterion +with ``requires_agent=True`` — which includes ``reference_comparison``, a +criterion this module treats as portable (C1.4) — the moment ``agent.type`` +is literally ``none``, regardless of whether an agent actually runs. +``coder-eval evaluate `` (the bare-task/work-dir path +C1.1's shim calls) never instantiates an agent no matter what ``agent.type`` +says (see ``evaluate_command.py``'s own comment: "no agent is created" is +true unconditionally on that path) — so a placeholder real agent type plus a +placeholder prompt satisfies coder-eval's validators without changing +behaviour, and unblocks ``reference_comparison``. Verified directly (not +inferred) before writing this module. +""" + +from __future__ import annotations + +import shutil +from dataclasses import dataclass, field +from pathlib import Path + +import tomli_w +import yaml + +from coder_eval.harbor.portability import PortabilityIssue, audit_criteria +from coder_eval.models import TaskDefinition +from coder_eval.orchestration.task_loader import load_task + + +DEFAULT_WORKDIR = "/app" +_HARBOR_SCHEMA_VERSION = "1.4" # pinned to the harbor 0.22.0 findings in tmp/harborframework.md § C0 + +# The task.yaml this module emits is graded via `coder-eval evaluate`, which +# never instantiates an agent on the bare-task/work-dir path regardless of +# `agent.type` (see module docstring) — this prompt is never read. +_VERIFIER_PLACEHOLDER_PROMPT = ( + "(unused placeholder — this file is graded via `coder-eval evaluate`, which does not invoke an agent on this path)" +) + +_TEST_SH_TEMPLATE = """#!/bin/sh +# Generated by `coder-eval export --format harbor` — do not hand-edit; a +# regenerated export will overwrite it. See tmp/harborframework.md § C1.1. +# +# NOT `set -e`: `coder-eval evaluate` exits non-zero whenever any criterion +# fails its gate -- a genuine MEASURED verdict, not an infrastructure +# failure. `set -e` here would abort before `coder-eval harbor reward` ever +# ran, turning every real failing score into a masked (unmeasured) trial -- +# confirmed live: a solution that runs but produces the wrong content +# scored weighted_score=0.500 in task.json, yet `set -e` discarded it and +# Harbor reported RewardFileNotFoundError instead of reward=0.5. The reward +# writer is what decides infra-vs-policy (None -> no file); this script must +# always reach it. +set -u + +coder-eval evaluate /tests/task.yaml "{workdir}" --in-place --run-dir /logs/verifier || true +coder-eval harbor reward /logs/verifier --out /logs/verifier/reward.json +""" + + +class TaskNotExportableError(Exception): + """This task cannot export to Harbor v1 as specified — a structural refusal, not a criteria issue.""" + + +class CriteriaNotExportableError(Exception): + """The task's criteria include one or more Harbor cannot grade in v1 (see C1.4's audit).""" + + def __init__(self, issues: list[PortabilityIssue]) -> None: + self.issues = issues + lines = "\n".join( + f" - {i.criterion_description!r} ({i.criterion_type}): {i.portability.value}" for i in issues + ) + super().__init__( + f"{len(issues)} criterion/criteria cannot export to Harbor v1:\n{lines}\n" + + "Remove them, or pass allow_credentials=True (--allow-credentials on the CLI) for the " + + "NEEDS_CREDENTIALS ones if you have already provisioned model access inside the verifier " + + "container yourself." + ) + + +@dataclass(frozen=True) +class ExportResult: + """What ``export_task`` produced, for the CLI to report.""" + + out_dir: Path + workdir: str + warnings: list[str] = field(default_factory=list) + + +def export_task( + task_file: Path, + out_dir: Path, + *, + allow_credentials: bool = False, +) -> ExportResult: + """Emit a Harbor task directory at ``out_dir`` from the coder-eval task at ``task_file``. + + Raises: + TaskNotExportableError: the task's sandbox isn't ``driver: docker`` — + Harbor always builds a container per task, so there is nothing to + derive ``environment/`` from otherwise (v1; no host-execution + equivalent is attempted). + CriteriaNotExportableError: one or more criteria fail C1.4's + portability audit. + """ + task, _raw_yaml = load_task(task_file) + + issues = audit_criteria(task.success_criteria, allow_credentials=allow_credentials) + if issues: + raise CriteriaNotExportableError(issues) + + if task.sandbox.driver != "docker": + raise TaskNotExportableError( + f"Task {task.task_id!r} uses sandbox.driver={task.sandbox.driver!r}, but Harbor always builds a " + + "container per task and v1 export has no host-execution equivalent to derive environment/ from. " + + "Set sandbox.driver: docker (with dockerfile_path or a custom image) to export this task." + ) + + warnings: list[str] = [] + if task.pre_run: + warnings.append( + f"{len(task.pre_run)} pre_run command(s) were NOT translated — they run against a live sandbox " + + "with template files already staged, which has no Dockerfile-build-time equivalent. Fold their " + + "effect into environment/Dockerfile by hand if the exported task needs it." + ) + if task.post_run: + warnings.append( + f"{len(task.post_run)} post_run command(s) were NOT translated — they run after the verdict is " + + "computed and coder-eval's own docs mark this mapping as deferred (see C2's table). Add them to " + + "tests/test.sh by hand, after the reward is written, if the exported task needs them." + ) + + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / "tests").mkdir(parents=True, exist_ok=True) + + workdir = _write_environment(task, task_file, out_dir, warnings) + _write_instruction(task, task_file, out_dir) + _write_verifier_task_yaml(task, out_dir) + _write_test_sh(out_dir, workdir=workdir) + _write_reference(task, task_file, out_dir) + _write_task_toml(task, out_dir, workdir=workdir) + + return ExportResult(out_dir=out_dir, workdir=workdir, warnings=warnings) + + +def _write_instruction(task: TaskDefinition, task_file: Path, out_dir: Path) -> None: + if task.initial_prompt is not None: + text = task.initial_prompt + elif task.initial_prompt_file is not None: + text = (task_file.parent / task.initial_prompt_file).read_text(encoding="utf-8") + else: + # is_none_agent or simulation-only tasks carry no prompt at all — export + # the description instead so instruction.md is never empty. Harbor's own + # Task._validate_tests only requires the FILE to exist, not any content. + text = task.description + if not text.endswith("\n"): + text += "\n" # POSIX text-file convention; also what the repo's own pre-commit end-of-file-fixer enforces + (out_dir / "instruction.md").write_text(text, encoding="utf-8") + + +def _write_environment( + task: TaskDefinition, + task_file: Path, + out_dir: Path, + warnings: list[str], +) -> str: + """Copy/derive ``environment/`` and return the WORKDIR both it and test.sh must agree on. + + Per C0 § 5, Harbor has no fixed workspace path — the verifier (default + SHARED mode) runs at whatever the container's own WORKDIR is. This + function is the one place that decides it, so nothing downstream can + silently disagree. + """ + env_dir = out_dir / "environment" + env_dir.mkdir(parents=True, exist_ok=True) + docker_cfg = task.sandbox.docker + + if docker_cfg.dockerfile_path is not None: + source_dockerfile = Path(docker_cfg.dockerfile_path) # already absolute — load_task resolves it + dest_dockerfile = env_dir / "Dockerfile" + shutil.copy2(source_dockerfile, dest_dockerfile) + workdir = docker_cfg.working_dir or _find_workdir(dest_dockerfile) or DEFAULT_WORKDIR + if _find_workdir(dest_dockerfile) is None: + with dest_dockerfile.open("a", encoding="utf-8") as fh: + fh.write(f"\nWORKDIR {workdir}\n") + warnings.append( + f"environment/Dockerfile declared no WORKDIR; appended `WORKDIR {workdir}` so the " + + "verifier and agent phases agree on a path." + ) + if not _from_line_mentions_coder_eval_agent(dest_dockerfile): + warnings.append(_MISSING_CODER_EVAL_WARNING) + # Non-Dockerfile build context (COPY sources etc.) is not carried over in + # v1 — a dockerfile_path build context beyond the Dockerfile itself needs + # its own decision (open question) before this is safe to widen. + if source_dockerfile.parent != task_file.parent: + other_files = [p for p in source_dockerfile.parent.iterdir() if p != source_dockerfile] + if other_files: + warnings.append( + f"{source_dockerfile.parent} holds {len(other_files)} other file(s) alongside the " + + "Dockerfile (build context) that were NOT copied — v1 only copies the Dockerfile itself." + ) + return workdir + + # No dockerfile_path — fall back to the pre-built image. docker.image always + # has a value (default_factory=get_default_docker_image_tag), so this is a + # real choice, not a null-vs-set distinction: log it either way so the + # operator can see which image the exported task will pull. + if "coder-eval-agent" not in docker_cfg.image: + warnings.append(_MISSING_CODER_EVAL_WARNING) + return docker_cfg.working_dir or DEFAULT_WORKDIR + + +_MISSING_CODER_EVAL_WARNING = ( + "environment/ does not appear to be based on a coder-eval-agent image. `tests/test.sh` calls " + "`coder-eval` (for the reward writer), which must be installed inside the container -- v1 assumes " + "`FROM coder-eval-agent:` (or a custom image that already extends it). A base-image-agnostic " + "fix (grafting the relocatable /opt/coder-eval kit from `make coder-eval-runtime` into any " + "Dockerfile) is tracked as follow-up in tmp/harborframework.md, not yet built." +) + + +def _from_line_mentions_coder_eval_agent(dockerfile: Path) -> bool: + """Heuristic only: does any ``FROM`` line in this Dockerfile name a coder-eval-agent image? + + A substring match, not an image-history inspection -- a multi-stage build + whose FINAL stage derives from coder-eval-agent under a different alias, + or an image that installs coder-eval by some other means entirely, both + read as a false warning here. That is the intended failure direction: a + missed real positive is silent breakage at verify time, so this errs + toward warning more often, not less. + """ + for line in dockerfile.read_text(encoding="utf-8").splitlines(): + if line.strip().upper().startswith("FROM ") and "coder-eval-agent" in line: + return True + return False + + +def _find_workdir(dockerfile: Path) -> str | None: + """The last ``WORKDIR`` directive in a Dockerfile, matching Docker's own last-wins semantics.""" + workdir: str | None = None + for line in dockerfile.read_text(encoding="utf-8").splitlines(): + stripped = line.strip() + if stripped.upper().startswith("WORKDIR "): + workdir = stripped.split(None, 1)[1].strip() + return workdir + + +def _write_verifier_task_yaml(task: TaskDefinition, out_dir: Path) -> None: + """``tests/task.yaml`` — the criteria, as authored. See the module docstring for the agent-type caveat.""" + payload: dict[str, object] = { + "task_id": task.task_id, + "description": task.description, + "agent": {"type": "claude-code"}, # placeholder; never instantiated (see module docstring) + "initial_prompt": _VERIFIER_PLACEHOLDER_PROMPT, + "success_criteria": [c.model_dump(mode="json", exclude_none=True) for c in task.success_criteria], + } + if task.reference is not None: + payload["reference"] = {"directory": "reference"} + (out_dir / "tests" / "task.yaml").write_text( + yaml.safe_dump(payload, sort_keys=False, allow_unicode=True), encoding="utf-8" + ) + + +def _write_test_sh(out_dir: Path, *, workdir: str) -> None: + path = out_dir / "tests" / "test.sh" + path.write_text(_TEST_SH_TEMPLATE.format(workdir=workdir), encoding="utf-8") + path.chmod(0o755) + + +def _write_reference(task: TaskDefinition, task_file: Path, out_dir: Path) -> None: + if task.reference is None: + return + source = task_file.parent / task.reference.directory + dest = out_dir / "tests" / "reference" + if dest.exists(): + shutil.rmtree(dest) + shutil.copytree(source, dest) + + +def _write_task_toml(task: TaskDefinition, out_dir: Path, *, workdir: str) -> None: + doc: dict[str, object] = { + "schema_version": _HARBOR_SCHEMA_VERSION, + "task": { + "name": f"coder-eval/{task.task_id}", + "description": task.description, + "keywords": list(task.tags), + }, + "environment": _build_environment_section(task, workdir=workdir), + } + if task.run_limits is not None and task.run_limits.task_timeout is not None: + timeout = float(task.run_limits.task_timeout) + doc["verifier"] = {"timeout_sec": timeout} + doc["agent"] = {"timeout_sec": timeout} + (out_dir / "task.toml").write_bytes(tomli_w.dumps(doc).encode("utf-8")) + + +def _build_environment_section(task: TaskDefinition, *, workdir: str) -> dict[str, object]: + docker_cfg = task.sandbox.docker + section: dict[str, object] = {"workdir": workdir} + if docker_cfg.dockerfile_path is None: + section["docker_image"] = docker_cfg.image + limits = task.sandbox.limits + if limits.max_memory_mb is not None: + section["memory_mb"] = limits.max_memory_mb + if limits.max_cpus is not None: + section["cpus"] = max(1, round(limits.max_cpus)) + section["network_mode"] = "no-network" if docker_cfg.network == "none" else "public" + return section + + +__all__ = [ + "DEFAULT_WORKDIR", + "CriteriaNotExportableError", + "ExportResult", + "TaskNotExportableError", + "export_task", +] diff --git a/src/coder_eval/harbor/portability.py b/src/coder_eval/harbor/portability.py new file mode 100644 index 00000000..41896629 --- /dev/null +++ b/src/coder_eval/harbor/portability.py @@ -0,0 +1,149 @@ +"""C1.4 — criteria portability audit for the Harbor export direction. + +Not every criterion type can grade truthfully inside a verifier container +that another harness built. This module classifies each of the 15 criterion +types and lets the packager (C2) refuse an unsupported task **at export +time**, where the operator sees why, rather than at verify time, where the +failure is an unexplained low reward with no obvious cause. + +Classification, v1: + +- ``PORTABLE`` — filesystem/exit-code checks. Nothing about the verifier + container changes what these need: ``file_exists``, ``file_contains``, + ``file_matches_regex``, ``json_check``, ``file_check``, ``run_command``, + ``classification_match``. +- ``NEEDS_REFERENCE`` — ``reference_comparison``. Needs the reference tree, + which C2 places under ``tests/reference/`` (verifier-side only, never in + the agent's view — see C2's mapping table). +- ``NEEDS_TRAJECTORY`` — ``command_executed``, ``commands_efficiency``, + ``skill_triggered``. These read coder-eval's own ``TurnRecord`` iterations. + In the export direction the verifier is a separate process from the agent + phase, and the agent may not even be coder-eval (Harbor natively supports + many agents) — so there is no ``iterations`` list to read from without + C1.3 (ATIF ingestion + ``evaluate --trajectory``), which is not built yet. + Hard-error until it is. +- ``NEEDS_CLI_RECORDER`` — ``cli_called``. Reads coder-eval's own JSON Lines + invocation log (``invocation_log.py``), written by a recorder shim + coder-eval's OWN sandbox setup installs into `PATH` + (``_generate_cli_recorders``). The exported Dockerfile does not provision + that shim. Hard-error until C2 learns to bake it in. +- ``NEEDS_CREDENTIALS`` — ``llm_judge``, ``agent_judge``, ``uipath_eval``. + Need model credentials and network reachable from inside the verifier + container, and the judge must not follow the agent's own route (the old + note's blocking issues). Hard-error in v1, with an explicit opt-in escape + hatch for an operator who has already provisioned that themselves. +""" + +from __future__ import annotations + +import enum +from dataclasses import dataclass + +from coder_eval.models import SuccessCriterion + + +class CriterionPortability(enum.Enum): + """How (or whether) a criterion type can grade inside a Harbor verifier container.""" + + PORTABLE = "portable" + NEEDS_REFERENCE = "needs_reference" + NEEDS_TRAJECTORY = "needs_trajectory" + NEEDS_CLI_RECORDER = "needs_cli_recorder" + NEEDS_CREDENTIALS = "needs_credentials" + + +_PORTABILITY_BY_TYPE: dict[str, CriterionPortability] = { + "file_exists": CriterionPortability.PORTABLE, + "file_contains": CriterionPortability.PORTABLE, + "file_matches_regex": CriterionPortability.PORTABLE, + "json_check": CriterionPortability.PORTABLE, + "file_check": CriterionPortability.PORTABLE, + "run_command": CriterionPortability.PORTABLE, + "classification_match": CriterionPortability.PORTABLE, + "reference_comparison": CriterionPortability.NEEDS_REFERENCE, + "command_executed": CriterionPortability.NEEDS_TRAJECTORY, + "commands_efficiency": CriterionPortability.NEEDS_TRAJECTORY, + "skill_triggered": CriterionPortability.NEEDS_TRAJECTORY, + "cli_called": CriterionPortability.NEEDS_CLI_RECORDER, + "llm_judge": CriterionPortability.NEEDS_CREDENTIALS, + "agent_judge": CriterionPortability.NEEDS_CREDENTIALS, + "uipath_eval": CriterionPortability.NEEDS_CREDENTIALS, +} + +# Registry-derived coverage: fails closed on a 16th criterion type added to the +# union without a portability classification, rather than silently exporting +# it as if it were PORTABLE. +_KNOWN_CRITERION_TYPES = frozenset(_PORTABILITY_BY_TYPE) + +# Which non-PORTABLE classes v1 refuses to export outright (vs. tolerating +# with a caveat, like NEEDS_REFERENCE — C2 always emits tests/reference/ when +# task.reference is set, so that class is never actually blocking). +_BLOCKING_IN_V1 = frozenset( + { + CriterionPortability.NEEDS_TRAJECTORY, + CriterionPortability.NEEDS_CLI_RECORDER, + CriterionPortability.NEEDS_CREDENTIALS, + } +) + + +class UnknownCriterionTypeError(Exception): + """A criterion type has no portability classification — fail closed, not open.""" + + +@dataclass(frozen=True) +class PortabilityIssue: + """One criterion this export cannot grade truthfully inside Harbor's verifier.""" + + criterion_description: str + criterion_type: str + portability: CriterionPortability + + +def classify(criterion_type: str) -> CriterionPortability: + """Look up a single criterion type's portability, raising on an unclassified type.""" + try: + return _PORTABILITY_BY_TYPE[criterion_type] + except KeyError: + raise UnknownCriterionTypeError( + f"Criterion type {criterion_type!r} has no Harbor-export portability classification " + + f"(known types: {sorted(_KNOWN_CRITERION_TYPES)}). Classify it in " + + "coder_eval.harbor.portability before exporting a task that uses it." + ) from None + + +def audit_criteria( + criteria: list[SuccessCriterion], + *, + allow_credentials: bool = False, +) -> list[PortabilityIssue]: + """Return every criterion this v1 export would refuse, or ``[]`` if the task exports cleanly. + + ``allow_credentials`` is the escape hatch for ``NEEDS_CREDENTIALS`` criteria + (``llm_judge`` / ``agent_judge`` / ``uipath_eval``) — an operator who has + already provisioned model credentials and network access inside the + verifier container may pass it to export anyway. It does not affect + ``NEEDS_TRAJECTORY`` or ``NEEDS_CLI_RECORDER``, which are missing + functionality (C1.3, a recorder-baking step in C2), not a missing + permission — no flag can supply what does not exist yet. + """ + issues: list[PortabilityIssue] = [] + for c in criteria: + portability = classify(c.type) + if portability not in _BLOCKING_IN_V1: + continue + if portability is CriterionPortability.NEEDS_CREDENTIALS and allow_credentials: + continue + issues.append( + PortabilityIssue(criterion_description=c.description, criterion_type=c.type, portability=portability) + ) + return issues + + +__all__ = [ + "CriterionPortability", + "PortabilityIssue", + "UnknownCriterionTypeError", + "audit_criteria", + "classify", +] diff --git a/src/coder_eval/harbor/reward.py b/src/coder_eval/harbor/reward.py new file mode 100644 index 00000000..426a485d --- /dev/null +++ b/src/coder_eval/harbor/reward.py @@ -0,0 +1,88 @@ +"""Translate a graded coder-eval run into Harbor's reward-file contract. + +Harbor's verifier (``harbor 0.22.0``, verified against source — see +``tmp/harborframework.md`` § C0) reads ``/logs/verifier/reward.json`` (a flat +``dict[str, float]``) or falls back to ``/logs/verifier/reward.txt`` (a bare +float, synthesized into the single key ``"reward"``). It does NOT inspect the +verifier script's exit code — only whether the reward file exists, is +non-empty, and parses. A missing/empty/malformed file raises inside Harbor's +own ``Verifier.verify()`` (``RewardFileNotFoundError`` / +``RewardFileEmptyError`` / ``VerifierOutputParseError``), which Harbor's +``Trial.run()`` catches: it records the exception on ``TrialResult`` and +leaves ``TrialResult.verifier_result`` at its default ``None`` — never +coalesced to a zero-reward object. That is Harbor's own infra-vs-policy split, +already built. + +This module's whole job, therefore, is: **write the file, or don't.** + +The "don't" case is not a degenerate corner — it is the load-bearing one. An +unmeasured row (``weighted_score is None`` — an ungraded row, or a task.json +that failed to load at all) must not become ``reward=0.0``: that would train +"the agent's behaviour was bad" from a measurement that never happened. Not +writing the file lets Harbor's own missing-reward path mask the trial +instead — the same principle as CE049 (never coalesce a possibly-unmeasured +score to a numeric literal), one level up, at the artifact-writing boundary +rather than the in-process one. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from coder_eval.orchestration.regrade import RegradeError, load_prior_result +from coder_eval.path_utils import write_text_atomic + + +class RewardWriteSkippedError(Exception): + """The run carries no measured verdict — the caller must write no reward file. + + Raised for an ungraded row (``weighted_score is None``, e.g. ``execute`` + left it ``NOT_GRADED``, or the row crashed before grading). Distinct from + ``RegradeError`` (also propagated by this module) only in *why* nothing + can be written — both are infrastructure failures with the identical + "write nothing" contract; keeping them separate types documents which + failure mode actually occurred without changing what the caller must do. + """ + + +def compute_reward(run_dir: Path) -> dict[str, float]: + """Read a graded run's ``task.json`` and derive Harbor's reward dict. + + Reuses ``load_prior_result`` (the same reader ``evaluate `` / + `run --resume`` use) rather than re-implementing ``task.json`` loading — + a second reader is how two copies of "what does this run's outcome mean" + drift into two different verdicts for the same run. + + Raises: + RegradeError: ``task.json`` is missing or does not parse as an + ``EvaluationResult`` (propagated from ``load_prior_result`` — an + infrastructure failure, same "write nothing" contract as below). + RewardWriteSkippedError: the row's ``weighted_score`` is ``None`` — an + unmeasured row, never to be coalesced to ``0.0``. + """ + result = load_prior_result(run_dir) + if result.weighted_score is None: + raise RewardWriteSkippedError( + f"{run_dir} carries no weighted_score (final_status={result.final_status.value!r}); " + + "writing no reward file so Harbor's own missing-reward path masks the trial " + + "instead of scoring it 0.0" + ) + return {"reward": result.weighted_score} + + +def write_reward(run_dir: Path, out_path: Path) -> dict[str, float]: + """Write Harbor's ``reward.json`` at ``out_path``. + + Must never be called when ``compute_reward`` would raise — the caller + (the ``coder-eval harbor reward`` CLI command) is expected to let + ``RewardWriteSkippedError`` / ``RegradeError`` propagate uncaught rather than + catching them here and writing a file anyway. + """ + rewards = compute_reward(run_dir) + out_path.parent.mkdir(parents=True, exist_ok=True) + write_text_atomic(out_path, json.dumps(rewards, indent=2) + "\n") + return rewards + + +__all__ = ["RegradeError", "RewardWriteSkippedError", "compute_reward", "write_reward"] diff --git a/tests/_fixtures/harbor_export_golden/expected/environment/Dockerfile b/tests/_fixtures/harbor_export_golden/expected/environment/Dockerfile new file mode 100644 index 00000000..6f2be053 --- /dev/null +++ b/tests/_fixtures/harbor_export_golden/expected/environment/Dockerfile @@ -0,0 +1,4 @@ +FROM ubuntu:24.04 +RUN apt-get update && apt-get install -y --no-install-recommends coreutils && rm -rf /var/lib/apt/lists/* + +WORKDIR /app diff --git a/tests/_fixtures/harbor_export_golden/expected/instruction.md b/tests/_fixtures/harbor_export_golden/expected/instruction.md new file mode 100644 index 00000000..5f9b4eb8 --- /dev/null +++ b/tests/_fixtures/harbor_export_golden/expected/instruction.md @@ -0,0 +1 @@ +Write 'hello world' to greeting.txt, matching reference/greeting.txt. diff --git a/tests/_fixtures/harbor_export_golden/expected/task.toml b/tests/_fixtures/harbor_export_golden/expected/task.toml new file mode 100644 index 00000000..dbe7caf3 --- /dev/null +++ b/tests/_fixtures/harbor_export_golden/expected/task.toml @@ -0,0 +1,21 @@ +schema_version = "1.4" + +[task] +name = "coder-eval/golden_export_example" +description = "A canonical task exercising every C2 mapping row at once (golden fixture — do not edit casually)." +keywords = [ + "golden", + "harbor-export", +] + +[environment] +workdir = "/app" +memory_mb = 1024 +cpus = 2 +network_mode = "no-network" + +[verifier] +timeout_sec = 300.0 + +[agent] +timeout_sec = 300.0 diff --git a/tests/_fixtures/harbor_export_golden/expected/tests/reference/greeting.txt b/tests/_fixtures/harbor_export_golden/expected/tests/reference/greeting.txt new file mode 100644 index 00000000..3b18e512 --- /dev/null +++ b/tests/_fixtures/harbor_export_golden/expected/tests/reference/greeting.txt @@ -0,0 +1 @@ +hello world diff --git a/tests/_fixtures/harbor_export_golden/expected/tests/task.yaml b/tests/_fixtures/harbor_export_golden/expected/tests/task.yaml new file mode 100644 index 00000000..c889ab53 --- /dev/null +++ b/tests/_fixtures/harbor_export_golden/expected/tests/task.yaml @@ -0,0 +1,23 @@ +task_id: golden_export_example +description: A canonical task exercising every C2 mapping row at once (golden fixture + — do not edit casually). +agent: + type: claude-code +initial_prompt: (unused placeholder — this file is graded via `coder-eval evaluate`, + which does not invoke an agent on this path) +success_criteria: +- type: file_exists + description: greeting.txt exists + weight: 1.0 + pass_threshold: 0.9 + path: greeting.txt +- type: reference_comparison + description: matches the reference greeting + weight: 1.0 + pass_threshold: 0.8 + agent_file: greeting.txt + reference_file: greeting.txt + comparison_method: ast + similarity_threshold: 0.8 +reference: + directory: reference diff --git a/tests/_fixtures/harbor_export_golden/expected/tests/test.sh b/tests/_fixtures/harbor_export_golden/expected/tests/test.sh new file mode 100755 index 00000000..f5aa589c --- /dev/null +++ b/tests/_fixtures/harbor_export_golden/expected/tests/test.sh @@ -0,0 +1,17 @@ +#!/bin/sh +# Generated by `coder-eval export --format harbor` — do not hand-edit; a +# regenerated export will overwrite it. See tmp/harborframework.md § C1.1. +# +# NOT `set -e`: `coder-eval evaluate` exits non-zero whenever any criterion +# fails its gate -- a genuine MEASURED verdict, not an infrastructure +# failure. `set -e` here would abort before `coder-eval harbor reward` ever +# ran, turning every real failing score into a masked (unmeasured) trial -- +# confirmed live: a solution that runs but produces the wrong content +# scored weighted_score=0.500 in task.json, yet `set -e` discarded it and +# Harbor reported RewardFileNotFoundError instead of reward=0.5. The reward +# writer is what decides infra-vs-policy (None -> no file); this script must +# always reach it. +set -u + +coder-eval evaluate /tests/task.yaml "/app" --in-place --run-dir /logs/verifier || true +coder-eval harbor reward /logs/verifier --out /logs/verifier/reward.json diff --git a/tests/_fixtures/harbor_export_golden/source_task/environment/Dockerfile b/tests/_fixtures/harbor_export_golden/source_task/environment/Dockerfile new file mode 100644 index 00000000..5a792680 --- /dev/null +++ b/tests/_fixtures/harbor_export_golden/source_task/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM ubuntu:24.04 +RUN apt-get update && apt-get install -y --no-install-recommends coreutils && rm -rf /var/lib/apt/lists/* diff --git a/tests/_fixtures/harbor_export_golden/source_task/reference/greeting.txt b/tests/_fixtures/harbor_export_golden/source_task/reference/greeting.txt new file mode 100644 index 00000000..3b18e512 --- /dev/null +++ b/tests/_fixtures/harbor_export_golden/source_task/reference/greeting.txt @@ -0,0 +1 @@ +hello world diff --git a/tests/_fixtures/harbor_export_golden/source_task/task.yaml b/tests/_fixtures/harbor_export_golden/source_task/task.yaml new file mode 100644 index 00000000..b45f53cf --- /dev/null +++ b/tests/_fixtures/harbor_export_golden/source_task/task.yaml @@ -0,0 +1,33 @@ +task_id: golden_export_example +description: A canonical task exercising every C2 mapping row at once (golden fixture — do not edit casually). +tags: [golden, harbor-export] + +agent: + type: claude-code + +sandbox: + driver: docker + docker: + dockerfile_path: environment/Dockerfile + network: none + limits: + max_memory_mb: 1024 + max_cpus: 1.5 + +initial_prompt: Write 'hello world' to greeting.txt, matching reference/greeting.txt. + +reference: + directory: reference + +run_limits: + task_timeout: 300 + +success_criteria: + - type: file_exists + path: greeting.txt + description: greeting.txt exists + + - type: reference_comparison + agent_file: greeting.txt + reference_file: greeting.txt + description: matches the reference greeting diff --git a/tests/lint/rules/no_cli_imports_in_core.py b/tests/lint/rules/no_cli_imports_in_core.py index cea8d840..f09f4018 100644 --- a/tests/lint/rules/no_cli_imports_in_core.py +++ b/tests/lint/rules/no_cli_imports_in_core.py @@ -2,9 +2,14 @@ The "core" layer comprises every package that should be usable without the CLI: criteria/, evaluation/, models/, simulation/, scoring/, streaming/, -errors/, orchestration/, agents/. Importing from coder_eval.cli +errors/, orchestration/, agents/, harbor/. Importing from coder_eval.cli creates an upward dependency that breaks testability in isolation. +``harbor/`` joined this list for the same reason ``orchestration/`` is on it: +its reward writer wants to raise a plain exception (``RewardWriteSkippedError``, +or the re-exported ``RegradeError``) and let the CLI wrap it into an exit +code — exactly the ``orchestration/regrade.py`` -> ``evaluate`` shape. + Note: this is a single, narrow rule (no upward imports into cli). For a fully layered import graph (no upward imports between any layers), evaluate import-linter / grimp — purpose-built for that. CE004 is the cheap version @@ -18,7 +23,7 @@ _CORE_DIRS = re.compile( - r"[/\\](criteria|evaluation|models|simulation|scoring|streaming|errors|orchestration|agents)[/\\]" + r"[/\\](criteria|evaluation|models|simulation|scoring|streaming|errors|orchestration|agents|harbor)[/\\]" ) _BANNED = re.compile(r"^coder_eval\.cli") diff --git a/tests/test_harbor_export_golden.py b/tests/test_harbor_export_golden.py new file mode 100644 index 00000000..57f3f4ef --- /dev/null +++ b/tests/test_harbor_export_golden.py @@ -0,0 +1,77 @@ +"""Golden-master test for the C2 packager's emitted directory. + +The regression this test is FOR: a transpiler's characteristic failure mode +is plausible-looking drift — a field silently starts mapping to the wrong +TOML key, a template loses a line — that no unit assertion happens to probe. +Diffing the whole emitted tree against a committed snapshot catches that +class of bug directly, cheaper than enumerating every field in prose. + +The source fixture (``tests/_fixtures/harbor_export_golden/source_task/``) +deliberately exercises every C2 mapping row at once: ``dockerfile_path`` with +no ``WORKDIR`` (exercises the append-a-WORKDIR path), ``reference:``, +``run_limits.task_timeout``, resource limits, ``network: none``, and both a +filesystem criterion and a ``reference_comparison`` criterion (proving the +placeholder-agent design from the packager's own module docstring survives +end to end, not just in an isolated unit test). + +Regenerate after an INTENTIONAL mapping change with:: + + GOLDEN_REGEN=1 uv run pytest tests/test_harbor_export_golden.py + +and review the resulting diff before committing. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +from coder_eval.harbor.packager import export_task + + +_FIXTURE_ROOT = Path(__file__).parent / "_fixtures" / "harbor_export_golden" +_SOURCE_TASK = _FIXTURE_ROOT / "source_task" / "task.yaml" +_EXPECTED_DIR = _FIXTURE_ROOT / "expected" +_REGEN = os.environ.get("GOLDEN_REGEN", "").strip().lower() in {"1", "true", "yes", "on"} + + +def _relative_files(root: Path) -> dict[str, str]: + """Every file under root, as {posix-relative-path: text-content}.""" + return { + p.relative_to(root).as_posix(): p.read_text(encoding="utf-8") for p in sorted(root.rglob("*")) if p.is_file() + } + + +def test_export_matches_the_committed_golden_tree(tmp_path: Path) -> None: + out_dir = tmp_path / "out" + export_task(_SOURCE_TASK, out_dir) + actual = _relative_files(out_dir) + + if _REGEN: + for rel_path, content in actual.items(): + dest = _EXPECTED_DIR / rel_path + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(content, encoding="utf-8") + # test.sh's executable bit is part of the contract (C1.1) but git/the + # filesystem, not file content, carries it -- regenerate it too so a + # fresh checkout of the golden dir stays runnable if ever copied out. + (_EXPECTED_DIR / "tests" / "test.sh").chmod(0o755) + return + + assert _EXPECTED_DIR.is_dir(), "no committed golden tree yet -- run with GOLDEN_REGEN=1 first" + expected = _relative_files(_EXPECTED_DIR) + + assert set(actual) == set(expected), ( + f"emitted file set drifted from the golden tree.\n" + f" only in export: {sorted(set(actual) - set(expected))}\n" + f" only in golden: {sorted(set(expected) - set(actual))}" + ) + for rel_path in expected: + assert actual[rel_path] == expected[rel_path], f"content drifted for {rel_path}" + + +def test_test_sh_is_executable_in_the_golden_tree() -> None: + """The executable bit is part of C1.1's contract and isn't captured by file content.""" + test_sh = _EXPECTED_DIR / "tests" / "test.sh" + assert test_sh.is_file() + assert test_sh.stat().st_mode & 0o111 diff --git a/tests/test_harbor_packager.py b/tests/test_harbor_packager.py new file mode 100644 index 00000000..f9b21906 --- /dev/null +++ b/tests/test_harbor_packager.py @@ -0,0 +1,308 @@ +"""``coder_eval.harbor.packager`` — the C2 export packager. + +Builds real task YAML files on disk (via ``load_task``, exactly the path a +user's ``coder-eval export`` invocation takes) rather than constructing +``TaskDefinition`` objects directly, so these tests exercise the same load +path C2 actually runs through. +""" + +from __future__ import annotations + +import tomllib +from pathlib import Path + +import pytest +import yaml + +from coder_eval.harbor.packager import ( + DEFAULT_WORKDIR, + CriteriaNotExportableError, + TaskNotExportableError, + export_task, +) +from coder_eval.models import TaskDefinition + + +_BASE_TASK: dict[str, object] = { + "task_id": "greet", + "description": "Write a greeting to greeting.txt.", + "tags": ["smoke"], + "agent": {"type": "claude-code"}, + "initial_prompt": "Write 'hello' to greeting.txt.", + "sandbox": { + "driver": "docker", + "docker": {"image": "byod-custom-image:0.1.0", "network": "none"}, + "limits": {"max_memory_mb": 2048, "max_cpus": 2}, + }, + "success_criteria": [ + {"type": "file_exists", "path": "greeting.txt", "description": "exists"}, + {"type": "file_contains", "path": "greeting.txt", "includes": ["hello"], "description": "content"}, + ], +} + + +def _write_task(tmp_path: Path, overrides: dict[str, object] | None = None, name: str = "task.yaml") -> Path: + payload = {**_BASE_TASK, **(overrides or {})} + task_file = tmp_path / name + task_file.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") + return task_file + + +class TestStructuralRefusals: + def test_tempdir_driver_is_refused(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path, {"sandbox": {"driver": "tempdir"}}) + with pytest.raises(TaskNotExportableError): + export_task(task_file, tmp_path / "out") + + def test_unsupported_criteria_are_refused_before_any_file_is_written(self, tmp_path: Path) -> None: + task_file = _write_task( + tmp_path, + { + "success_criteria": [ + {"type": "skill_triggered", "expected_skill": "s", "skill_name": "s", "description": "d"}, + ] + }, + ) + out_dir = tmp_path / "out" + with pytest.raises(CriteriaNotExportableError): + export_task(task_file, out_dir) + assert not out_dir.exists(), "a refused export must not leave a partial directory behind" + + def test_credentials_criteria_can_be_allowed_explicitly(self, tmp_path: Path) -> None: + task_file = _write_task( + tmp_path, + {"success_criteria": [{"type": "llm_judge", "prompt": "grade it", "description": "d"}]}, + ) + result = export_task(task_file, tmp_path / "out", allow_credentials=True) + assert result.out_dir.exists() + + +class TestEmittedDirectoryStructure: + def test_full_export_produces_the_documented_layout(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + out_dir = tmp_path / "out" + + result = export_task(task_file, out_dir) + + assert result.out_dir == out_dir + assert (out_dir / "task.toml").is_file() + assert (out_dir / "instruction.md").is_file() + assert (out_dir / "environment").is_dir() + assert (out_dir / "tests" / "test.sh").is_file() + assert (out_dir / "tests" / "task.yaml").is_file() + + def test_instruction_md_carries_the_initial_prompt(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + out_dir = tmp_path / "out" + export_task(task_file, out_dir) + assert (out_dir / "instruction.md").read_text(encoding="utf-8") == "Write 'hello' to greeting.txt.\n" + + def test_test_sh_is_executable_and_references_the_resolved_workdir(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + out_dir = tmp_path / "out" + + result = export_task(task_file, out_dir) + + test_sh = out_dir / "tests" / "test.sh" + assert test_sh.stat().st_mode & 0o111, "test.sh must be executable" + content = test_sh.read_text(encoding="utf-8") + assert f'coder-eval evaluate /tests/task.yaml "{result.workdir}"' in content + assert "coder-eval harbor reward /logs/verifier --out /logs/verifier/reward.json" in content + + def test_task_toml_parses_and_carries_the_mapped_fields(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + doc = tomllib.loads((out_dir / "task.toml").read_text(encoding="utf-8")) + assert doc["task"]["name"] == "coder-eval/greet" + assert doc["task"]["keywords"] == ["smoke"] + assert doc["environment"]["docker_image"] == "byod-custom-image:0.1.0" + assert doc["environment"]["memory_mb"] == 2048 + assert doc["environment"]["cpus"] == 2 + assert doc["environment"]["network_mode"] == "no-network" + assert doc["environment"]["workdir"] == DEFAULT_WORKDIR + + def test_network_bridge_maps_to_public(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path, {"sandbox": {"driver": "docker", "docker": {"network": "bridge"}}}) + out_dir = tmp_path / "out" + export_task(task_file, out_dir) + doc = tomllib.loads((out_dir / "task.toml").read_text(encoding="utf-8")) + assert doc["environment"]["network_mode"] == "public" + + +class TestVerifierTaskYaml: + """tests/task.yaml must never set agent: {type: none} — see packager.py's module docstring.""" + + def test_never_sets_agent_type_none(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + out_dir = tmp_path / "out" + export_task(task_file, out_dir) + + emitted = yaml.safe_load((out_dir / "tests" / "task.yaml").read_text(encoding="utf-8")) + assert emitted["agent"]["type"] != "none" + + def test_reloads_as_a_valid_task_definition(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + out_dir = tmp_path / "out" + export_task(task_file, out_dir) + + emitted = yaml.safe_load((out_dir / "tests" / "task.yaml").read_text(encoding="utf-8")) + reloaded = TaskDefinition.model_validate(emitted) + assert reloaded.task_id == "greet" + assert len(reloaded.success_criteria) == 2 + + def test_reference_comparison_survives_the_none_agent_trap(self, tmp_path: Path) -> None: + """The corrected design: a placeholder real agent type unblocks reference_comparison. + + If tests/task.yaml set agent: {type: none} instead, this would raise at + TaskDefinition.model_validate — see the module docstring for why. + """ + task_file = _write_task( + tmp_path, + { + "reference": {"directory": "reference"}, + "success_criteria": [ + { + "type": "reference_comparison", + "agent_file": "greeting.txt", + "reference_file": "greeting.txt", + "description": "matches reference", + } + ], + }, + ) + (tmp_path / "reference").mkdir() + (tmp_path / "reference" / "greeting.txt").write_text("hello", encoding="utf-8") + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + emitted = yaml.safe_load((out_dir / "tests" / "task.yaml").read_text(encoding="utf-8")) + reloaded = TaskDefinition.model_validate(emitted) # must not raise + assert reloaded.reference is not None and reloaded.reference.directory == "reference" + assert (out_dir / "tests" / "reference" / "greeting.txt").read_text(encoding="utf-8") == "hello" + + +class TestDockerfileWorkdirResolution: + def test_dockerfile_with_no_workdir_gets_one_appended_and_warned(self, tmp_path: Path) -> None: + env_dir = tmp_path / "environment" + env_dir.mkdir() + (env_dir / "Dockerfile").write_text("FROM ubuntu:24.04\nRUN apt-get update\n", encoding="utf-8") + task_file = _write_task( + tmp_path, + {"sandbox": {"driver": "docker", "docker": {"dockerfile_path": "environment/Dockerfile"}}}, + ) + out_dir = tmp_path / "out" + + result = export_task(task_file, out_dir) + + assert result.workdir == DEFAULT_WORKDIR + dockerfile_text = (out_dir / "environment" / "Dockerfile").read_text(encoding="utf-8") + assert f"WORKDIR {DEFAULT_WORKDIR}" in dockerfile_text + assert any("declared no WORKDIR" in w for w in result.warnings) + + def test_dockerfile_with_an_existing_workdir_is_respected_and_not_touched(self, tmp_path: Path) -> None: + env_dir = tmp_path / "environment" + env_dir.mkdir() + original = "FROM ubuntu:24.04\nWORKDIR /workspace\nRUN apt-get update\n" + (env_dir / "Dockerfile").write_text(original, encoding="utf-8") + task_file = _write_task( + tmp_path, + {"sandbox": {"driver": "docker", "docker": {"dockerfile_path": "environment/Dockerfile"}}}, + ) + out_dir = tmp_path / "out" + + result = export_task(task_file, out_dir) + + assert result.workdir == "/workspace" + assert (out_dir / "environment" / "Dockerfile").read_text(encoding="utf-8") == original + assert not any("declared no WORKDIR" in w for w in result.warnings) + # test.sh and task.toml must agree with the same resolved workdir. + assert '"/workspace"' in (out_dir / "tests" / "test.sh").read_text(encoding="utf-8") + doc = tomllib.loads((out_dir / "task.toml").read_text(encoding="utf-8")) + assert doc["environment"]["workdir"] == "/workspace" + + def test_docker_working_dir_override_wins_over_the_dockerfile(self, tmp_path: Path) -> None: + env_dir = tmp_path / "environment" + env_dir.mkdir() + (env_dir / "Dockerfile").write_text("FROM ubuntu:24.04\nWORKDIR /workspace\n", encoding="utf-8") + task_file = _write_task( + tmp_path, + { + "sandbox": { + "driver": "docker", + "docker": {"dockerfile_path": "environment/Dockerfile", "working_dir": "/custom"}, + } + }, + ) + out_dir = tmp_path / "out" + + result = export_task(task_file, out_dir) + + assert result.workdir == "/custom" + + +class TestCoderEvalAgentBaseImageWarning: + """v1 assumes the exported image already has coder-eval installed (`FROM coder-eval-agent:`).""" + + def test_warns_when_dockerfile_does_not_from_coder_eval_agent(self, tmp_path: Path) -> None: + env_dir = tmp_path / "environment" + env_dir.mkdir() + (env_dir / "Dockerfile").write_text("FROM ubuntu:24.04\n", encoding="utf-8") + task_file = _write_task( + tmp_path, + {"sandbox": {"driver": "docker", "docker": {"dockerfile_path": "environment/Dockerfile"}}}, + ) + + result = export_task(task_file, tmp_path / "out") + + assert any("coder-eval-agent" in w for w in result.warnings) + + def test_no_warning_when_dockerfile_froms_coder_eval_agent(self, tmp_path: Path) -> None: + env_dir = tmp_path / "environment" + env_dir.mkdir() + (env_dir / "Dockerfile").write_text("FROM coder-eval-agent:latest\n", encoding="utf-8") + task_file = _write_task( + tmp_path, + {"sandbox": {"driver": "docker", "docker": {"dockerfile_path": "environment/Dockerfile"}}}, + ) + + result = export_task(task_file, tmp_path / "out") + + assert not any("coder-eval-agent" in w for w in result.warnings) + + def test_warns_when_prebuilt_image_does_not_name_coder_eval_agent(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) # _BASE_TASK's image is byod-custom-image:0.1.0 + result = export_task(task_file, tmp_path / "out") + assert any("coder-eval-agent" in w for w in result.warnings) + + def test_no_warning_when_prebuilt_image_names_coder_eval_agent(self, tmp_path: Path) -> None: + task_file = _write_task( + tmp_path, {"sandbox": {"driver": "docker", "docker": {"image": "coder-eval-agent:0.12.0"}}} + ) + result = export_task(task_file, tmp_path / "out") + assert not any("coder-eval-agent" in w for w in result.warnings) + + +class TestPrePostRunWarnings: + def test_pre_run_and_post_run_are_warned_not_silently_dropped(self, tmp_path: Path) -> None: + task_file = _write_task( + tmp_path, + { + "pre_run": [{"command": "echo setup"}], + "post_run": [{"command": "echo cleanup"}], + }, + ) + out_dir = tmp_path / "out" + + result = export_task(task_file, out_dir) + + assert any("pre_run" in w for w in result.warnings) + assert any("post_run" in w for w in result.warnings) + + def test_no_pre_or_post_run_produces_no_such_warnings(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + result = export_task(task_file, tmp_path / "out") + assert not any("pre_run" in w or "post_run" in w for w in result.warnings) diff --git a/tests/test_harbor_portability.py b/tests/test_harbor_portability.py new file mode 100644 index 00000000..98a6dd08 --- /dev/null +++ b/tests/test_harbor_portability.py @@ -0,0 +1,135 @@ +"""``coder_eval.harbor.portability`` — the C1.4 criteria portability audit. + +Registry-derived coverage (the same shape CE036 uses for +``live_decidable_polarities``): every criterion type in the real +``SuccessCriterion`` union must have a portability classification, so a 16th +criterion type added later fails this test instead of silently exporting as +if it were ``PORTABLE``. +""" + +from __future__ import annotations + +import typing + +import pytest + +from coder_eval.harbor.portability import ( + _PORTABILITY_BY_TYPE, + CriterionPortability, + UnknownCriterionTypeError, + audit_criteria, + classify, +) +from coder_eval.models import SuccessCriterion +from coder_eval.models.criteria import ( + AgentJudgeCriterion, + ClassificationMatchCriterion, + CliCalledCriterion, + CommandExecutedCriterion, + CommandsEfficiencyCriterion, + FileCheckCriterion, + FileContainsCriterion, + FileExistsCriterion, + FileMatchesRegexCriterion, + JsonCheckCriterion, + LLMJudgeCriterion, + ReferenceComparisonCriterion, + RunCommandCriterion, + SkillTriggeredCriterion, + UiPathEvalCriterion, +) + + +def _union_member_type_tags() -> set[str]: + """Every ``type:`` literal tag actually reachable through the real union.""" + # SuccessCriterion is `Annotated[X | Y | ..., Field(discriminator="type")]`. + union = typing.get_args(SuccessCriterion)[0] + tags: set[str] = set() + for member in typing.get_args(union): + default = member.model_fields["type"].default + assert isinstance(default, str) + tags.add(default) + return tags + + +def test_every_real_criterion_type_is_classified() -> None: + """Fails closed: a new criterion type added to the union with no line here is a bug, not a PORTABLE default.""" + assert _union_member_type_tags() == set(_PORTABILITY_BY_TYPE) + + +def test_unknown_type_raises_rather_than_defaulting_portable() -> None: + with pytest.raises(UnknownCriterionTypeError): + classify("not_a_real_criterion_type") + + +@pytest.mark.parametrize( + "criterion", + [ + FileExistsCriterion(description="d", path="p"), + FileContainsCriterion(description="d", path="p", includes=["t"]), + FileMatchesRegexCriterion(description="d", path="p", pattern="."), + JsonCheckCriterion(description="d", path="p"), + FileCheckCriterion(description="d", path="p"), + RunCommandCriterion(description="d", command="true"), + ClassificationMatchCriterion(description="d", path="p", expected_label="x", allowed_labels=["x", "y"]), + ], +) +def test_portable_criteria_never_block_export(criterion: object) -> None: + assert audit_criteria([criterion]) == [] # type: ignore[list-item] + + +def test_reference_comparison_does_not_block_export() -> None: + """NEEDS_REFERENCE is not in _BLOCKING_IN_V1 — C2 always emits tests/reference/.""" + criterion = ReferenceComparisonCriterion(description="d", agent_file="p", reference_file="r") + assert audit_criteria([criterion]) == [] + + +@pytest.mark.parametrize( + "criterion", + [ + CommandExecutedCriterion(description="d", command_pattern="."), + CommandsEfficiencyCriterion(description="d", expected_commands=3), + SkillTriggeredCriterion(description="d", expected_skill="s", skill_name="s"), + CliCalledCriterion(description="d", verb="v"), + ], +) +def test_missing_functionality_criteria_always_block_export(criterion: object) -> None: + """No flag can supply what does not exist yet (C1.3 / a recorder-baking step).""" + issues = audit_criteria([criterion], allow_credentials=True) # type: ignore[list-item] + assert len(issues) == 1 + assert issues[0].criterion_type == criterion.type # type: ignore[attr-defined] + + +@pytest.mark.parametrize( + "criterion", + [ + LLMJudgeCriterion(description="d", prompt="p"), + AgentJudgeCriterion(description="d", prompt="p"), + UiPathEvalCriterion(description="d", agent_name="a", eval_set="p", thresholds={}), + ], +) +def test_credentials_criteria_block_by_default_but_have_an_escape_hatch(criterion: object) -> None: + assert len(audit_criteria([criterion])) == 1 # type: ignore[list-item] + assert audit_criteria([criterion], allow_credentials=True) == [] # type: ignore[list-item] + + +def test_a_clean_multi_criterion_task_reports_no_issues() -> None: + criteria = [ + FileExistsCriterion(description="d1", path="p1"), + ReferenceComparisonCriterion(description="d2", agent_file="p2", reference_file="r"), + ] + assert audit_criteria(criteria) == [] + + +def test_a_mixed_task_reports_only_the_blocking_criteria() -> None: + criteria = [ + FileExistsCriterion(description="portable", path="p1"), + SkillTriggeredCriterion(description="needs trajectory", expected_skill="s", skill_name="s"), + LLMJudgeCriterion(description="needs credentials", prompt="p"), + ] + issues = audit_criteria(criteria) + assert {i.criterion_description for i in issues} == {"needs trajectory", "needs credentials"} + assert {i.portability for i in issues} == { + CriterionPortability.NEEDS_TRAJECTORY, + CriterionPortability.NEEDS_CREDENTIALS, + } diff --git a/tests/test_harbor_reward.py b/tests/test_harbor_reward.py new file mode 100644 index 00000000..3b6652c1 --- /dev/null +++ b/tests/test_harbor_reward.py @@ -0,0 +1,123 @@ +"""``coder_eval.harbor.reward`` — the C1.1 verifier shim's reward writer. + +Table test over the three no-file exclusion cases the design doc (C1.1 / +C3) calls out: an ungraded row, a missing ``task.json``, and a malformed +``task.json`` must all write NO reward file and raise, distinctly from a +measured row (including a genuine zero) writing the file. The whole point of +this module is that those two outcomes are never confusable. +""" + +from __future__ import annotations + +import json +from datetime import datetime +from pathlib import Path + +import pytest + +from coder_eval.harbor.reward import RegradeError, RewardWriteSkippedError, compute_reward, write_reward +from coder_eval.models import AgentKind, EvaluationResult, FinalStatus +from coder_eval.path_utils import TASK_JSON_FILENAME + + +def _write_task_json(run_dir: Path, **overrides: object) -> None: + """Write a run_dir/task.json from an EvaluationResult, with field overrides.""" + fields: dict[str, object] = { + "task_id": "t", + "task_description": "d", + "variant_id": "default", + "agent_type": AgentKind.CLAUDE_CODE, + "started_at": datetime(2020, 1, 1), + "final_status": FinalStatus.SUCCESS, + "iteration_count": 1, + } + fields.update(overrides) + result = EvaluationResult(**fields) # type: ignore[arg-type] + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / TASK_JSON_FILENAME).write_text(result.model_dump_json(), encoding="utf-8") + + +class TestComputeReward: + """Every path through compute_reward: measured (incl. zero) vs. the three unmeasured cases.""" + + def test_measured_row_returns_weighted_score(self, tmp_path: Path) -> None: + run_dir = tmp_path / "run" + _write_task_json(run_dir, final_status=FinalStatus.SUCCESS, weighted_score=0.75) + assert compute_reward(run_dir) == {"reward": 0.75} + + def test_measured_zero_is_not_confused_with_unmeasured(self, tmp_path: Path) -> None: + """A criterion suite that genuinely scored 0.0 must still write the file. + + This is the case the whole design guards: `weighted_score or 0.0` would + pass this test AND the ungraded test below identically, which is exactly + the CE049 defect. `weighted_score is not None` is the only correct test. + """ + run_dir = tmp_path / "run" + _write_task_json(run_dir, final_status=FinalStatus.FAILURE, weighted_score=0.0) + assert compute_reward(run_dir) == {"reward": 0.0} + + def test_ungraded_row_raises_reward_write_skipped(self, tmp_path: Path) -> None: + run_dir = tmp_path / "run" + _write_task_json(run_dir, final_status=FinalStatus.NOT_GRADED, weighted_score=None) + with pytest.raises(RewardWriteSkippedError): + compute_reward(run_dir) + + def test_missing_task_json_raises_regrade_error(self, tmp_path: Path) -> None: + run_dir = tmp_path / "run" + run_dir.mkdir() + with pytest.raises(RegradeError): + compute_reward(run_dir) + + def test_malformed_task_json_raises_regrade_error(self, tmp_path: Path) -> None: + run_dir = tmp_path / "run" + run_dir.mkdir() + (run_dir / TASK_JSON_FILENAME).write_text("{not json", encoding="utf-8") + with pytest.raises(RegradeError): + compute_reward(run_dir) + + +class TestWriteReward: + """The file-writing half: confirms the no-file contract, not just the exception.""" + + def test_writes_reward_json_for_a_measured_row(self, tmp_path: Path) -> None: + run_dir = tmp_path / "run" + _write_task_json(run_dir, final_status=FinalStatus.SUCCESS, weighted_score=0.5) + out = tmp_path / "verifier" / "reward.json" + + rewards = write_reward(run_dir, out) + + assert rewards == {"reward": 0.5} + assert json.loads(out.read_text(encoding="utf-8")) == {"reward": 0.5} + + def test_writes_no_file_for_an_ungraded_row(self, tmp_path: Path) -> None: + run_dir = tmp_path / "run" + _write_task_json(run_dir, final_status=FinalStatus.NOT_GRADED, weighted_score=None) + out = tmp_path / "verifier" / "reward.json" + + with pytest.raises(RewardWriteSkippedError): + write_reward(run_dir, out) + + assert not out.exists() + assert not out.parent.exists(), "must not even create the destination dir on a skipped write" + + def test_writes_no_file_when_task_json_is_missing(self, tmp_path: Path) -> None: + run_dir = tmp_path / "run" + run_dir.mkdir() + out = tmp_path / "verifier" / "reward.json" + + with pytest.raises(RegradeError): + write_reward(run_dir, out) + + assert not out.exists() + + def test_reward_json_is_a_flat_dict_per_harbor_contract(self, tmp_path: Path) -> None: + """Harbor's VerifierResult.rewards is dict[str, float | int] | None, never a bare scalar.""" + run_dir = tmp_path / "run" + _write_task_json(run_dir, final_status=FinalStatus.SUCCESS, weighted_score=1.0) + out = tmp_path / "verifier" / "reward.json" + + write_reward(run_dir, out) + + parsed = json.loads(out.read_text(encoding="utf-8")) + assert isinstance(parsed, dict) + assert parsed == {"reward": 1.0} diff --git a/uv.lock b/uv.lock index 28bca6f6..b738e789 100644 --- a/uv.lock +++ b/uv.lock @@ -495,6 +495,7 @@ dependencies = [ { name = "radon" }, { name = "rich" }, { name = "starlette" }, + { name = "tomli-w" }, { name = "tqdm" }, { name = "typer" }, ] @@ -560,6 +561,7 @@ requires-dist = [ { name = "rich", specifier = ">=14.3.3" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.15.7" }, { name = "starlette", specifier = ">=1.3.1" }, + { name = "tomli-w", specifier = ">=1.0.0" }, { name = "tqdm", specifier = ">=4.67.3" }, { name = "typer", specifier = ">=0.24.1" }, { name = "uipath", marker = "extra == 'uipath'", specifier = ">=2.10.31" }, From 083577b4fd72fac64448638f6a27d05ef25bc85c Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Thu, 10 Sep 2026 11:56:02 -0700 Subject: [PATCH 02/12] fix(harbor): derive a prebuilt image's real WORKDIR instead of guessing /app Converting a real in-tree task (hello_date_smoke_test) through the export pipeline surfaced a live bug in the bare-`docker.image` branch of `_write_environment`: it defaulted `workdir` to the hardcoded `DEFAULT_WORKDIR = "/app"` whenever `sandbox.docker.working_dir` was unset, regardless of the image's actual WORKDIR. `coder-eval-agent:latest`'s real WORKDIR is `/work`. Harbor's shared verifier runs `docker exec -w ...`, and confirmed live that `docker exec -w` -- unlike `docker run -w`, which auto-creates the directory -- hard-fails to chdir into a path that doesn't already exist in the image ("no such file or directory"), surfacing as agent exit 127 with no verifier output at all. The dockerfile_path branch never had this bug (it appends a WORKDIR line and the directory exists once Docker builds it); the bare-image branch had no Dockerfile to fix and no way to introspect the image, so it just guessed. Fix: `_inspect_image_workdir()` runs `docker image inspect --format '{{.Config.WorkingDir}}'` (best-effort, 30s timeout, returns None on any failure) and the bare-image branch now tries, in order: explicit `working_dir` -> inspected WORKDIR -> `DEFAULT_WORKDIR`, warning (non-blocking) only on the last fallback. Re-verified live after the fix: `harbor run -p -a oracle` against the converted hello_date task scored reward: 1.0. Test hygiene: added an autouse fixture stubbing `_inspect_image_workdir` across `test_harbor_packager.py` so packager unit tests never depend on what happens to be cached in the local docker daemon -- a real image named `byod-custom-image:0.1.0` (this file's own placeholder name), built by an unrelated docker-integration test elsewhere in the suite, silently flipped a DEFAULT_WORKDIR assertion the first time this ran locally. New tests (TestPrebuiltImageWorkdirInspection) cover the inspection helper directly and its three-way fallback order. Full narrative in tmp/harborframework.md (gitignored), including a scoping note for the next piece (coder-eval as Harbor's *agent*, not just its verifier) -- not started, planning only. Co-Authored-By: Claude Sonnet 5 --- src/coder_eval/harbor/packager.py | 43 +++++++++++++- tests/test_harbor_packager.py | 99 ++++++++++++++++++++++++++++++- 2 files changed, 140 insertions(+), 2 deletions(-) diff --git a/src/coder_eval/harbor/packager.py b/src/coder_eval/harbor/packager.py index 8fbf6877..c145542f 100644 --- a/src/coder_eval/harbor/packager.py +++ b/src/coder_eval/harbor/packager.py @@ -41,6 +41,7 @@ from __future__ import annotations import shutil +import subprocess from dataclasses import dataclass, field from pathlib import Path @@ -231,7 +232,47 @@ def _write_environment( # operator can see which image the exported task will pull. if "coder-eval-agent" not in docker_cfg.image: warnings.append(_MISSING_CODER_EVAL_WARNING) - return docker_cfg.working_dir or DEFAULT_WORKDIR + if docker_cfg.working_dir is not None: + return docker_cfg.working_dir + inspected = _inspect_image_workdir(docker_cfg.image) + if inspected is not None: + return inspected + warnings.append( + f"Could not determine {docker_cfg.image}'s own WORKDIR (image not present locally, or docker " + + f"unavailable at export time) -- defaulting to `{DEFAULT_WORKDIR}`. Harbor's `docker exec -w` " + + "hard-fails if that path does not already exist in the image (unlike `docker run -w`, it will " + + "not create it); set `sandbox.docker.working_dir` explicitly to the image's real WORKDIR to " + + "avoid a verify-time exit 127." + ) + return DEFAULT_WORKDIR + + +def _inspect_image_workdir(image: str) -> str | None: + """Best-effort ``docker image inspect`` for a pre-built image's own ``WORKDIR``. + + A pre-built image has no Dockerfile for ``_find_workdir`` to read, so this + is the only way to avoid guessing a path that doesn't exist in it (Harbor's + ``docker exec -w`` -- unlike ``docker run -w`` -- fails outright if the + directory isn't already there; confirmed live). Returns ``None`` (never + raises) whenever docker isn't available, the image isn't present locally, + or it declares no WORKDIR -- callers fall back to ``DEFAULT_WORKDIR`` and + warn. + """ + try: + result = subprocess.run( + ["docker", "image", "inspect", image, "--format", "{{.Config.WorkingDir}}"], + capture_output=True, + text=True, + encoding="utf-8", + timeout=30, + check=False, + ) + except (OSError, subprocess.SubprocessError): + return None + if result.returncode != 0: + return None + workdir = result.stdout.strip() + return workdir or None _MISSING_CODER_EVAL_WARNING = ( diff --git a/tests/test_harbor_packager.py b/tests/test_harbor_packager.py index f9b21906..359e0fed 100644 --- a/tests/test_harbor_packager.py +++ b/tests/test_harbor_packager.py @@ -14,6 +14,7 @@ import pytest import yaml +from coder_eval.harbor import packager from coder_eval.harbor.packager import ( DEFAULT_WORKDIR, CriteriaNotExportableError, @@ -23,6 +24,23 @@ from coder_eval.models import TaskDefinition +_REAL_INSPECT_IMAGE_WORKDIR = packager._inspect_image_workdir + + +@pytest.fixture(autouse=True) +def _no_real_docker_inspection(monkeypatch: pytest.MonkeyPatch) -> None: + """Keep this module hermetic: never let ``export_task`` shell out to a real + ``docker image inspect``, whose answer depends on what happens to be + cached on the machine running the tests (an image literally named + ``byod-custom-image:0.1.0`` -- this file's own placeholder BYOD image + name -- built by an unrelated docker-integration test elsewhere in the + suite answered ``/work`` here once, silently flipping this file's + DEFAULT_WORKDIR assertions). Tests that care about the inspection path + itself override this via monkeypatch locally. + """ + monkeypatch.setattr(packager, "_inspect_image_workdir", lambda image: None) + + _BASE_TASK: dict[str, object] = { "task_id": "greet", "description": "Write a greeting to greeting.txt.", @@ -278,7 +296,14 @@ def test_warns_when_prebuilt_image_does_not_name_coder_eval_agent(self, tmp_path result = export_task(task_file, tmp_path / "out") assert any("coder-eval-agent" in w for w in result.warnings) - def test_no_warning_when_prebuilt_image_names_coder_eval_agent(self, tmp_path: Path) -> None: + def test_no_warning_when_prebuilt_image_names_coder_eval_agent( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + # Give inspection a real-looking answer -- a None here would ALSO warn + # ("Could not determine ...'s own WORKDIR"), whose text incidentally + # contains "coder-eval-agent" (the image name), which is not the + # warning this test is checking for. + monkeypatch.setattr(packager, "_inspect_image_workdir", lambda image: "/work") task_file = _write_task( tmp_path, {"sandbox": {"driver": "docker", "docker": {"image": "coder-eval-agent:0.12.0"}}} ) @@ -306,3 +331,75 @@ def test_no_pre_or_post_run_produces_no_such_warnings(self, tmp_path: Path) -> N task_file = _write_task(tmp_path) result = export_task(task_file, tmp_path / "out") assert not any("pre_run" in w or "post_run" in w for w in result.warnings) + + +class TestPrebuiltImageWorkdirInspection: + """A pre-built (no ``dockerfile_path``) image has no Dockerfile ``WORKDIR`` line + to read, so the packager shells out to ``docker image inspect`` for it -- a + real bug this closes: defaulting to ``/app`` unconditionally exported a task + that failed at Harbor verify time with exit 127, because Harbor's + ``docker exec -w`` (unlike ``docker run -w``) refuses to chdir into a path + that doesn't already exist in the image (confirmed live against + ``coder-eval-agent:latest``, whose real WORKDIR is ``/work``). + """ + + def test_uses_the_inspected_workdir_when_docker_reports_one( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(packager, "_inspect_image_workdir", lambda image: "/work") + task_file = _write_task(tmp_path) # _BASE_TASK's image is byod-custom-image:0.1.0 + + result = export_task(task_file, tmp_path / "out") + + assert result.workdir == "/work" + assert not any("Could not determine" in w for w in result.warnings) + + def test_falls_back_to_default_and_warns_when_inspection_fails( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(packager, "_inspect_image_workdir", lambda image: None) + task_file = _write_task(tmp_path) + + result = export_task(task_file, tmp_path / "out") + + assert result.workdir == DEFAULT_WORKDIR + assert any("Could not determine" in w and "byod-custom-image:0.1.0" in w for w in result.warnings) + + def test_explicit_working_dir_wins_over_inspection(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(packager, "_inspect_image_workdir", lambda image: "/from-inspection") + task_file = _write_task(tmp_path, {"sandbox": {"driver": "docker", "docker": {"working_dir": "/explicit"}}}) + + result = export_task(task_file, tmp_path / "out") + + assert result.workdir == "/explicit" + + def test_inspect_image_workdir_parses_real_subprocess_output(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Exercise the real (un-mocked) helper against a stubbed ``subprocess.run``. + + The module-wide autouse fixture stubs out ``_inspect_image_workdir`` + itself for hermeticity, so this restores the real function first -- + it is the one thing here under test. + """ + monkeypatch.setattr(packager, "_inspect_image_workdir", _REAL_INSPECT_IMAGE_WORKDIR) + + class _FakeResult: + returncode = 0 + stdout = "/work\n" + + def _fake_run(cmd, **kwargs): + assert cmd[:3] == ["docker", "image", "inspect"] + return _FakeResult() + + monkeypatch.setattr(packager.subprocess, "run", _fake_run) + assert packager._inspect_image_workdir("some-image:tag") == "/work" + + def test_inspect_image_workdir_returns_none_when_docker_binary_is_missing( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(packager, "_inspect_image_workdir", _REAL_INSPECT_IMAGE_WORKDIR) + + def _raise(cmd, **kwargs): + raise FileNotFoundError("docker not found") + + monkeypatch.setattr(packager.subprocess, "run", _raise) + assert packager._inspect_image_workdir("some-image:tag") is None From 06d49df7ab01da80e183ed40970d3054c641d1ae Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Thu, 10 Sep 2026 12:42:52 -0700 Subject: [PATCH 03/12] feat(harbor): export experiment.yaml variants to Harbor task directories Extends the single-task Harbor exporter to experiment.yaml: each resolved (task, variant, replicate[, dataset row]) combination gets its own Harbor directory, since Harbor's task.toml has no variant concept. Reuses resolve_all_tasks (the same pipeline `run -e` uses) for resolution, and skips (rather than silently drops) a variant whose agent/simulation override Harbor's verifier-only export cannot honor, detected precisely via config_lineage so a task's own agent config never false-triggers it. Co-Authored-By: Claude Sonnet 5 --- src/coder_eval/cli/export_command.py | 83 +++++-- src/coder_eval/harbor/experiment_packager.py | 200 ++++++++++++++++ src/coder_eval/harbor/packager.py | 24 ++ tests/test_harbor_experiment_packager.py | 228 +++++++++++++++++++ 4 files changed, 521 insertions(+), 14 deletions(-) create mode 100644 src/coder_eval/harbor/experiment_packager.py create mode 100644 tests/test_harbor_experiment_packager.py diff --git a/src/coder_eval/cli/export_command.py b/src/coder_eval/cli/export_command.py index 1405b0b7..5afd69de 100644 --- a/src/coder_eval/cli/export_command.py +++ b/src/coder_eval/cli/export_command.py @@ -4,6 +4,16 @@ ``--format`` is reserved so the same flag can later grow a *reader* (``coder-eval run ``, demoted — see ``tmp/harborframework.partB.md``) without a naming collision. + +Two modes, told apart by whether ``-e/--experiment`` is passed: + +- Single task.yaml → one Harbor task directory at ``-o``, unchanged from C2. +- One or more task.yaml files + ``-e experiment.yaml`` → the experiment's own + ``resolve_all_tasks`` pipeline resolves every (task, variant, replicate[, + dataset row]) combination, and each is written to its own subdirectory under + ``-o`` (``//[/]rep/``). See + ``tmp/harborframework_conversion.md`` for what an experiment-introduced + override this export cannot honor looks like, and why. """ from __future__ import annotations @@ -12,27 +22,39 @@ import typer +from ..harbor.experiment_packager import export_experiment from ..harbor.packager import CriteriaNotExportableError, TaskNotExportableError, export_task from .console import console +from .run_helpers import expand_task_files _SUPPORTED_FORMATS = ("harbor",) def export_command( - task_file: Path = typer.Argument( # noqa: B008 + task_files: list[Path] = typer.Argument( # noqa: B008 ..., - help="The coder-eval task YAML to export.", - exists=True, - dir_okay=False, + help="The coder-eval task YAML(s) to export (glob patterns allowed with --experiment).", ), output_dir: Path = typer.Option( # noqa: B008 ..., "--output", "-o", - help="Directory to write the exported task into (created if missing).", + help="Directory to write the exported task(s) into (created if missing).", file_okay=False, ), + experiment: Path | None = typer.Option( # noqa: B008 + None, + "--experiment", + "-e", + help=( + "Experiment definition YAML. Resolves every (task, variant, replicate[, dataset row]) " + "combination via the same pipeline `coder-eval run -e` uses, and exports each as its own " + "Harbor directory under --output." + ), + exists=True, + dir_okay=False, + ), format: str = typer.Option( "harbor", "--format", @@ -48,24 +70,57 @@ def export_command( ), ), ) -> None: - """Export a coder-eval task to another framework's native directory format. + """Export a coder-eval task (or task x experiment.yaml variants) to another framework's directory format. Examples: coder-eval export tasks/my_task.yaml -o dist/harbor/my_task --format harbor + coder-eval export tasks/my_task.yaml -e experiments/model-comparison.yaml -o dist/harbor/my_experiment """ if format not in _SUPPORTED_FORMATS: console.print(f"[red]✗[/] Unsupported --format {format!r}. Supported: {', '.join(_SUPPORTED_FORMATS)}.") raise typer.Exit(1) - try: - result = export_task(task_file, output_dir, allow_credentials=allow_credentials) - except (TaskNotExportableError, CriteriaNotExportableError) as e: - console.print(f"[red]✗[/] {e}") - raise typer.Exit(1) from e + if experiment is None: + if len(task_files) != 1: + console.print("[red]✗[/] Without --experiment, pass exactly one task YAML.") + raise typer.Exit(1) + try: + result = export_task(task_files[0], output_dir, allow_credentials=allow_credentials) + except (TaskNotExportableError, CriteriaNotExportableError) as e: + console.print(f"[red]✗[/] {e}") + raise typer.Exit(1) from e + console.print(f"[green]✓[/] Exported {task_files[0]} → {result.out_dir} (format: {format})") + for warning in result.warnings: + console.print(f"[yellow]⚠[/] {warning}") + return - console.print(f"[green]✓[/] Exported {task_file} → {result.out_dir} (format: {format})") - for warning in result.warnings: - console.print(f"[yellow]⚠[/] {warning}") + all_task_files = expand_task_files(task_files) + exp_result = export_experiment( + all_task_files, + experiment, + output_dir, + allow_credentials=allow_credentials, + ) + + for exported in exp_result.exported: + console.print(f"[green]✓[/] Exported → {exported.out_dir} (format: {format})") + for warning in exported.warnings: + console.print(f"[yellow]⚠[/] {warning}") + for skip in exp_result.skipped: + console.print( + f"[yellow]⚠[/] Skipped variant={skip.variant_id!r} task={skip.task_id!r} " + + f"rep={skip.replicate_index}: {skip.reason}" + ) + for load_skip in exp_result.load_skipped: + console.print(f"[yellow]⚠[/] Skipped task load: {load_skip}") + + console.print( + f"[bold]{len(exp_result.exported)}[/] directories exported, " + + f"[bold]{len(exp_result.skipped)}[/] variant(s) skipped, " + + f"[bold]{len(exp_result.load_skipped)}[/] task file(s) skipped at load." + ) + if not exp_result.exported: + raise typer.Exit(1) __all__ = ["export_command"] diff --git a/src/coder_eval/harbor/experiment_packager.py b/src/coder_eval/harbor/experiment_packager.py new file mode 100644 index 00000000..86f90d11 --- /dev/null +++ b/src/coder_eval/harbor/experiment_packager.py @@ -0,0 +1,200 @@ +"""Experiment export — ``coder-eval export --format harbor ... -e -o ``. + +Extends the single-task Harbor packager (``packager.py``) to ``experiment.yaml`` +variants. Harbor's ``task.toml`` has no concept of a "variant" — an experiment +comparing N variants over M task files becomes N*M (times replicates, times +dataset rows) independent Harbor task directories, one per resolved +(task, variant, replicate[, dataset row]) combination. + +Resolution reuses ``orchestration/experiment.py::resolve_all_tasks`` — the exact +machinery ``coder-eval run -e`` uses — so the export sees identical resolved +configs to a real experiment run, with zero duplicated merge logic. Each +resolved task is then written out via ``packager.export_resolved_task``, the +same writer the single-task path uses. + +See ``tmp/harborframework_conversion.md`` for the full table of what does and +does not survive this translation. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + +from coder_eval.harbor.packager import ( + CriteriaNotExportableError, + ExportResult, + TaskNotExportableError, + export_resolved_task, +) +from coder_eval.models import ResolvedTask, SkippedTask +from coder_eval.orchestration.config import BatchRunConfig +from coder_eval.orchestration.experiment import DEFAULT_EXPERIMENT_PATH, load_experiment, resolve_all_tasks + + +# Sources a resolved field's ConfigLineageEntry can carry (see +# models/results.py::ConfigLineageEntry). Only these two mean "the experiment +# introduced this, the task's own YAML did not." +_EXPERIMENT_INTRODUCED_SOURCES = {"variant", "experiment-defaults"} + + +@dataclass(frozen=True) +class SkippedVariantExport: + """One resolved (task, variant, replicate) combination that was not exported, and why.""" + + variant_id: str + task_id: str + replicate_index: int + reason: str + + +@dataclass(frozen=True) +class ExperimentExportResult: + """What ``export_experiment`` produced, for the CLI to report.""" + + exported: list[ExportResult] = field(default_factory=list) + skipped: list[SkippedVariantExport] = field(default_factory=list) + load_skipped: list[SkippedTask] = field(default_factory=list) + + +def _unhonorable_override_reason(resolved: ResolvedTask) -> str | None: + """Why this resolved task's Harbor export can't honor an experiment-introduced override. + + Harbor's agent phase is Harbor's own (or the unbuilt C1.2 + coder-eval-as-agent embed adapter, see ``tmp/harborframework.md`` Part A + step 8) — entirely independent of coder-eval's ``agent``/``simulation`` + config. A variant that changes neither is fine to export (the exported + directory is then identical to the base task's own export, which is the + existing, already-documented single-task limitation — coder-eval's + ``agent`` field is never read for the exported verifier ``tests/task.yaml`` + either). But a variant that *introduces* an agent or simulation override + is the entire point of an A/B experiment: silently dropping it would + export two indistinguishable Harbor directories claiming to compare two + different arms. Detected via ``ResolvedTask.config_lineage`` — a field + counts only when its most-specific source is the variant or the + experiment defaults, never the task's own YAML or the baseline defaults. + """ + agent_sources = sorted( + { + entry.source + for key, entry in resolved.config_lineage.items() + if key.startswith("agent.") and entry.source in _EXPERIMENT_INTRODUCED_SOURCES + } + ) + if agent_sources: + return ( + f"variant {resolved.variant_id!r} sets an agent override (source: {', '.join(agent_sources)}) that " + "Harbor's verifier-only export cannot honor -- Harbor's agent phase is independent of coder-eval's " + "agent config. Building the C1.2 coder-eval-as-agent embed adapter (tmp/harborframework.md, Part A " + "step 8) is the fix; until then this variant is skipped rather than exported as an indistinguishable " + "duplicate of another arm." + ) + sim_entry = resolved.config_lineage.get("simulation") + if sim_entry is not None and sim_entry.source in _EXPERIMENT_INTRODUCED_SOURCES: + return ( + f"variant {resolved.variant_id!r} sets a simulation override that Harbor's single-shot verifier " + "export cannot represent -- Harbor's Trial model has no multi-turn user-simulator concept. Skipped." + ) + return None + + +def _out_subdir(out_dir: Path, resolved: ResolvedTask, *, needs_replicate_segment: bool) -> Path: + """``///[/]rep/`` — row/replicate segments only when they fan out.""" + dest = out_dir / resolved.variant_id / resolved.task.task_id + if resolved.task.row_id: + dest = dest / resolved.task.row_id + if needs_replicate_segment: + dest = dest / f"rep{resolved.replicate_index:02d}" + return dest + + +def export_experiment( + task_files: list[Path], + experiment_file: Path, + out_dir: Path, + *, + allow_credentials: bool = False, +) -> ExperimentExportResult: + """Export every (task x variant x replicate[ x dataset row]) combination to Harbor directories. + + Resolves ``task_files`` against ``experiment_file`` through the same + ``resolve_all_tasks`` pipeline ``coder-eval run -e`` uses (including + ``experiments/default.yaml`` as the baseline layer, when present), then + writes one Harbor task directory per resolved task at + ``///[/]rep/``. A resolved task + is SKIPPED (not fatal to the rest of the export) rather than written when: + it carries an experiment-introduced ``agent``/``simulation`` override + Harbor's export cannot honor (see ``_unhonorable_override_reason``), its + sandbox driver isn't ``docker``, or its criteria fail C1.4's portability + audit -- each mirrors an existing per-task refusal in ``packager.py``, + demoted from a raised exception to a collected skip so one bad variant + does not abort the whole experiment's export. + """ + experiment = load_experiment(experiment_file) + if experiment_file == DEFAULT_EXPERIMENT_PATH: + default_experiment = experiment + elif DEFAULT_EXPERIMENT_PATH.exists(): + default_experiment = load_experiment(DEFAULT_EXPERIMENT_PATH) + else: + default_experiment = experiment + + config = BatchRunConfig(run_dir=out_dir) + resolved_tasks, load_skipped = resolve_all_tasks( + task_files=task_files, + experiment=experiment, + default_experiment=default_experiment, + config=config, + experiment_file=experiment_file, + ) + + # How many replicates each (variant, task) group actually fanned out to, + # so a single-replicate group's directory omits the `rep00` segment. + replicate_counts: dict[tuple[str, str], int] = {} + for r in resolved_tasks: + key = (r.variant_id, r.task.task_id) + replicate_counts[key] = max(replicate_counts.get(key, 0), r.replicate_index + 1) + + exported: list[ExportResult] = [] + skipped: list[SkippedVariantExport] = [] + for resolved in resolved_tasks: + reason = _unhonorable_override_reason(resolved) + if reason is not None: + skipped.append( + SkippedVariantExport( + variant_id=resolved.variant_id, + task_id=resolved.task.task_id, + replicate_index=resolved.replicate_index, + reason=reason, + ) + ) + continue + + needs_replicate_segment = replicate_counts[(resolved.variant_id, resolved.task.task_id)] > 1 + dest = _out_subdir(out_dir, resolved, needs_replicate_segment=needs_replicate_segment) + try: + result = export_resolved_task( + resolved.task, + resolved.task_file, + dest, + allow_credentials=allow_credentials, + ) + except (TaskNotExportableError, CriteriaNotExportableError) as e: + skipped.append( + SkippedVariantExport( + variant_id=resolved.variant_id, + task_id=resolved.task.task_id, + replicate_index=resolved.replicate_index, + reason=str(e), + ) + ) + continue + exported.append(result) + + return ExperimentExportResult(exported=exported, skipped=skipped, load_skipped=load_skipped) + + +__all__ = [ + "ExperimentExportResult", + "SkippedVariantExport", + "export_experiment", +] diff --git a/src/coder_eval/harbor/packager.py b/src/coder_eval/harbor/packager.py index c145542f..df2e9228 100644 --- a/src/coder_eval/harbor/packager.py +++ b/src/coder_eval/harbor/packager.py @@ -129,7 +129,30 @@ def export_task( portability audit. """ task, _raw_yaml = load_task(task_file) + return export_resolved_task(task, task_file, out_dir, allow_credentials=allow_credentials) + +def export_resolved_task( + task: TaskDefinition, + task_file: Path, + out_dir: Path, + *, + allow_credentials: bool = False, +) -> ExportResult: + """Emit a Harbor task directory from an already-loaded/resolved ``TaskDefinition``. + + Shared by ``export_task`` (single task.yaml) and + ``harbor.experiment_packager.export_experiment`` (task.yaml + experiment.yaml, + one resolved task per variant/replicate/dataset row) — one writer, so the + two paths cannot silently drift. ``task_file`` is the ORIGINAL task YAML + path (not a synthetic one) — it is only used to resolve ``task.reference``, + which ``load_task`` deliberately leaves relative (see ``_write_reference``); + every other path-shaped field is already absolute by the time a + ``TaskDefinition`` reaches this function (``load_task`` resolves + ``dockerfile_path``/``initial_prompt_file``/``system_prompt_file`` inline). + + Raises the same two errors as ``export_task``, for the same reasons. + """ issues = audit_criteria(task.success_criteria, allow_credentials=allow_credentials) if issues: raise CriteriaNotExportableError(issues) @@ -378,5 +401,6 @@ def _build_environment_section(task: TaskDefinition, *, workdir: str) -> dict[st "CriteriaNotExportableError", "ExportResult", "TaskNotExportableError", + "export_resolved_task", "export_task", ] diff --git a/tests/test_harbor_experiment_packager.py b/tests/test_harbor_experiment_packager.py new file mode 100644 index 00000000..b6005185 --- /dev/null +++ b/tests/test_harbor_experiment_packager.py @@ -0,0 +1,228 @@ +"""``coder_eval.harbor.experiment_packager`` — task.yaml + experiment.yaml -> Harbor directories. + +Builds real task/experiment YAML files on disk and drives them through +``export_experiment`` exactly as `coder-eval export ... -e ...` does, so these +tests exercise the same resolution path (``orchestration.experiment.resolve_all_tasks``) +a real invocation runs through. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +from coder_eval.harbor import packager +from coder_eval.harbor.experiment_packager import export_experiment + + +_BASE_TASK: dict[str, object] = { + "task_id": "greet", + "description": "Write a greeting to greeting.txt.", + "agent": {"type": "claude-code"}, + "initial_prompt": "Write 'hello' to greeting.txt.", + "sandbox": { + "driver": "docker", + "docker": {"image": "byod-custom-image:0.1.0", "network": "none"}, + }, + "success_criteria": [ + {"type": "file_exists", "path": "greeting.txt", "description": "exists"}, + ], +} + + +@pytest.fixture(autouse=True) +def _no_real_docker_inspection(monkeypatch: pytest.MonkeyPatch) -> None: + """Same hermeticity guard as test_harbor_packager.py -- never shell out to real docker.""" + monkeypatch.setattr(packager, "_inspect_image_workdir", lambda image: None) + + +def _write_task(tmp_path: Path, overrides: dict[str, object] | None = None, name: str = "task.yaml") -> Path: + payload = {**_BASE_TASK, **(overrides or {})} + task_file = tmp_path / name + task_file.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") + return task_file + + +def _write_experiment(tmp_path: Path, definition: dict[str, object], name: str = "experiment.yaml") -> Path: + exp_file = tmp_path / name + exp_file.write_text(yaml.safe_dump(definition, sort_keys=False), encoding="utf-8") + return exp_file + + +class TestRunLimitOnlyVariants: + """Variants that only touch run_limits/sandbox/prompt -- fully honorable, should export.""" + + def test_two_variants_export_two_directories_with_distinct_timeouts(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + exp_file = _write_experiment( + tmp_path, + { + "experiment_id": "run-limit-ab", + "variants": [ + {"variant_id": "fast", "run_limits": {"task_timeout": 60}}, + {"variant_id": "slow", "run_limits": {"task_timeout": 900}}, + ], + }, + ) + out_dir = tmp_path / "out" + + result = export_experiment([task_file], exp_file, out_dir) + + assert len(result.exported) == 2 + assert result.skipped == [] + assert result.load_skipped == [] + fast_dir = out_dir / "fast" / "greet" + slow_dir = out_dir / "slow" / "greet" + assert (fast_dir / "task.toml").exists() + assert (slow_dir / "task.toml").exists() + assert "timeout_sec = 60" in (fast_dir / "task.toml").read_text(encoding="utf-8") + assert "timeout_sec = 900" in (slow_dir / "task.toml").read_text(encoding="utf-8") + + def test_variant_prompt_override_lands_in_instruction_md(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + exp_file = _write_experiment( + tmp_path, + { + "experiment_id": "prompt-ab", + "variants": [ + {"variant_id": "rewritten", "initial_prompt": "Write 'howdy' to greeting.txt."}, + ], + }, + ) + out_dir = tmp_path / "out" + + result = export_experiment([task_file], exp_file, out_dir) + + assert len(result.exported) == 1 + instruction = (out_dir / "rewritten" / "greet" / "instruction.md").read_text(encoding="utf-8") + assert "howdy" in instruction + + +class TestUnhonorableOverridesAreSkipped: + """Agent/simulation overrides an exported Harbor directory cannot express.""" + + def test_variant_agent_model_override_is_skipped_not_exported(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + exp_file = _write_experiment( + tmp_path, + { + "experiment_id": "model-ab", + "variants": [ + {"variant_id": "sonnet", "agent": {"model": "claude-sonnet-5"}}, + {"variant_id": "opus", "agent": {"model": "claude-opus-5"}}, + ], + }, + ) + out_dir = tmp_path / "out" + + result = export_experiment([task_file], exp_file, out_dir) + + assert result.exported == [] + assert {s.variant_id for s in result.skipped} == {"sonnet", "opus"} + assert all("agent override" in s.reason for s in result.skipped) + + def test_experiment_defaults_agent_override_is_skipped(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + exp_file = _write_experiment( + tmp_path, + { + "experiment_id": "defaults-agent", + "defaults": {"agent": {"permission_mode": "bypassPermissions"}}, + "variants": [{"variant_id": "only"}], + }, + ) + out_dir = tmp_path / "out" + + result = export_experiment([task_file], exp_file, out_dir) + + assert result.exported == [] + assert "experiment-defaults" in result.skipped[0].reason + + def test_variant_with_no_agent_override_still_exports_alongside_a_skipped_one(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + exp_file = _write_experiment( + tmp_path, + { + "experiment_id": "mixed", + "variants": [ + {"variant_id": "unmodified"}, + {"variant_id": "sonnet", "agent": {"model": "claude-sonnet-5"}}, + ], + }, + ) + out_dir = tmp_path / "out" + + result = export_experiment([task_file], exp_file, out_dir) + + assert len(result.exported) == 1 + assert result.exported[0].out_dir == out_dir / "unmodified" / "greet" + assert [s.variant_id for s in result.skipped] == ["sonnet"] + + def test_task_level_agent_config_alone_does_not_trip_the_skip(self, tmp_path: Path) -> None: + """The task's OWN agent.type (source='task') must not be mistaken for an experiment override.""" + task_file = _write_task(tmp_path) # agent: {type: claude-code} is the task's own, not the experiment's + exp_file = _write_experiment( + tmp_path, + { + "experiment_id": "no-override", + "variants": [{"variant_id": "only"}], + }, + ) + out_dir = tmp_path / "out" + + result = export_experiment([task_file], exp_file, out_dir) + + assert len(result.exported) == 1 + assert result.skipped == [] + + +class TestReplicateFanOut: + def test_repeats_produce_rep_subdirectories(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + exp_file = _write_experiment( + tmp_path, + { + "experiment_id": "reps", + "variants": [{"variant_id": "baseline", "repeats": 2}], + }, + ) + out_dir = tmp_path / "out" + + result = export_experiment([task_file], exp_file, out_dir) + + assert len(result.exported) == 2 + dests = sorted(str(r.out_dir.relative_to(out_dir)) for r in result.exported) + assert dests == [ + str(Path("baseline") / "greet" / "rep00"), + str(Path("baseline") / "greet" / "rep01"), + ] + + def test_single_replicate_omits_the_rep_segment(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + exp_file = _write_experiment( + tmp_path, + {"experiment_id": "no-reps", "variants": [{"variant_id": "only"}]}, + ) + out_dir = tmp_path / "out" + + result = export_experiment([task_file], exp_file, out_dir) + + assert result.exported[0].out_dir == out_dir / "only" / "greet" + + +class TestStructuralRefusalsPerVariant: + def test_non_docker_driver_variant_is_skipped_not_fatal(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path, {"sandbox": {"driver": "tempdir"}}) + exp_file = _write_experiment( + tmp_path, + {"experiment_id": "bad-driver", "variants": [{"variant_id": "only"}]}, + ) + out_dir = tmp_path / "out" + + result = export_experiment([task_file], exp_file, out_dir) + + assert result.exported == [] + assert len(result.skipped) == 1 + assert "sandbox.driver" in result.skipped[0].reason From d631b4c031f00b2b46c37719b596e5f7bf82fe42 Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Thu, 10 Sep 2026 21:16:22 -0700 Subject: [PATCH 04/12] feat(harbor): coder-eval as a Harbor agent (C1.2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adopts ATIF (Agent Trajectory Interchange Format) as coder-eval's trajectory interchange layer so a Harbor trial can run coder-eval itself as the agent, not just as the grader. - harbor/atif_models.py + atif_emit.py: vendored ATIF schema + a pure EvaluationResult -> Trajectory converter, opt-in via `--format harbor` on `execute`/`run` (writes a trajectory.json sibling to task.json; never changes default output). - harbor/atif_hydrate.py: the reverse direction (Trajectory -> TurnRecords) so `evaluate --format harbor --trajectory ` can grade trajectory-dependent criteria against an ATIF file that didn't originate from an in-process run. - harbor/agent.py: CoderEvalAgent(BaseInstalledAgent) — a real Harbor agent that shells out to `coder-eval execute --format harbor` inside the container and lets Harbor discover the resulting trajectory.json via `populate_context_post_run`, matching Harbor's own ClaudeCode agent's exact convention. Verified end-to-end against a real harbor==0.22.0 install: a full export -> docker build -> `harbor run` -> grade round trip produces correct rewards for both a passing and a deliberately failing criterion. - packager.py: bakes a criteria-free agent-phase task.yaml into the image at a fixed path (agent_paths.py) via a Dockerfile COPY line, so CoderEvalAgent always has a task to execute without ever seeing the real success_criteria. - run_command.py: when `--format harbor` writes exactly one trajectory (the shape a CoderEvalAgent invocation always produces), it is also copied to /trajectory.json so a fixed-path caller (Harbor's agent log dir) can find it without knowing coder-eval's internal variant/task/replicate nesting. Co-Authored-By: Claude Sonnet 5 --- src/coder_eval/cli/evaluate_command.py | 47 ++ src/coder_eval/cli/execute_command.py | 12 + src/coder_eval/cli/run_command.py | 39 ++ src/coder_eval/harbor/agent.py | 142 ++++++ src/coder_eval/harbor/agent_paths.py | 19 + src/coder_eval/harbor/atif_emit.py | 422 ++++++++++++++++++ src/coder_eval/harbor/atif_hydrate.py | 170 +++++++ src/coder_eval/harbor/atif_models.py | 277 ++++++++++++ src/coder_eval/harbor/packager.py | 93 +++- .../expected/environment/Dockerfile | 2 + .../expected/environment/agent_task.yaml | 17 + .../fixtures/atif/known_good_trajectory.json | 125 ++++++ tests/test_atif_emit.py | 366 +++++++++++++++ tests/test_atif_hydrate.py | 138 ++++++ tests/test_atif_models.py | 162 +++++++ tests/test_evaluate_format_harbor.py | 123 +++++ tests/test_execute_format_harbor.py | 83 ++++ tests/test_harbor_packager.py | 76 +++- 18 files changed, 2302 insertions(+), 11 deletions(-) create mode 100644 src/coder_eval/harbor/agent.py create mode 100644 src/coder_eval/harbor/agent_paths.py create mode 100644 src/coder_eval/harbor/atif_emit.py create mode 100644 src/coder_eval/harbor/atif_hydrate.py create mode 100644 src/coder_eval/harbor/atif_models.py create mode 100644 tests/_fixtures/harbor_export_golden/expected/environment/agent_task.yaml create mode 100644 tests/fixtures/atif/known_good_trajectory.json create mode 100644 tests/test_atif_emit.py create mode 100644 tests/test_atif_hydrate.py create mode 100644 tests/test_atif_models.py create mode 100644 tests/test_evaluate_format_harbor.py create mode 100644 tests/test_execute_format_harbor.py diff --git a/src/coder_eval/cli/evaluate_command.py b/src/coder_eval/cli/evaluate_command.py index f8ba3f45..050a6a62 100644 --- a/src/coder_eval/cli/evaluate_command.py +++ b/src/coder_eval/cli/evaluate_command.py @@ -11,6 +11,8 @@ from rich.markup import escape from ..evaluation.judge_persistence import TASK_JSON_TRANSCRIPT_EXCLUDE +from ..harbor.atif_hydrate import seed_from_atif_trajectory +from ..harbor.atif_models import Trajectory from ..logging_config import setup_logging from ..models import ( AgentKind, @@ -311,6 +313,23 @@ def evaluate_command( "--run-dir", help="Where the graded task.json lands (default: auto-generated timestamped directory in runs/)", ), + format: str | None = typer.Option( + None, + "--format", + help=( + "Only 'harbor' is supported: grade a directory whose agent phase ran OUTSIDE this " + "process (a Harbor agent's `coder-eval execute --format harbor`) by hydrating trajectory " + "context from an ATIF trajectory.json instead of a run directory's task.json. Requires " + "--trajectory and the two-argument `TASK_FILE WORK_DIR` form." + ), + ), + trajectory: Path | None = typer.Option( # noqa: B008 + None, + "--trajectory", + help="Path to an ATIF trajectory.json to hydrate trajectory context from. Required with --format harbor.", + exists=True, + dir_okay=False, + ), ) -> None: """Evaluate criteria against a directory, or re-grade a finished run. @@ -343,6 +362,8 @@ def evaluate_command( allow_recorded_commands=allow_recorded_commands, allow_host_grading=allow_host_grading, run_dir=run_dir, + format=format, + trajectory=trajectory, ) @@ -357,6 +378,8 @@ def run_evaluation( allow_recorded_commands: bool = False, allow_host_grading: bool = False, run_dir: Path | None = None, + format: str | None = None, + trajectory: Path | None = None, ) -> None: """The body of ``coder-eval evaluate``, with real Python defaults. @@ -384,6 +407,30 @@ def run_evaluation( prior = inputs.prior target = inputs.target + if format is not None and format != "harbor": + console.print(f"[red]✗ Unsupported --format {format!r}. Supported: harbor.[/red]") + raise typer.Exit(1) + if format == "harbor": + if target.mode is not EvaluateMode.WORK_DIR: + console.print( + "[red]✗ --format harbor only applies to the two-argument `TASK_FILE WORK_DIR` form — " + + "a run directory already carries its own trajectory in task.json.[/red]" + ) + raise typer.Exit(1) + if trajectory is None: + console.print("[red]✗ --format harbor requires --trajectory .[/red]") + raise typer.Exit(1) + try: + atif_trajectory = Trajectory.model_validate_json(trajectory.read_text(encoding="utf-8")) + except (OSError, ValueError) as e: + console.print(f"[red]✗ Could not read {trajectory} as an ATIF trajectory:[/red] {escape(str(e))}") + raise typer.Exit(1) from e + prior = seed_from_atif_trajectory( + atif_trajectory, + task_id=task.task_id, + task_description=task.description, + ) + grade_in_place = resolve_grade_in_place(target, in_place) try: diff --git a/src/coder_eval/cli/execute_command.py b/src/coder_eval/cli/execute_command.py index 15bbdc2a..2895dddf 100644 --- a/src/coder_eval/cli/execute_command.py +++ b/src/coder_eval/cli/execute_command.py @@ -181,6 +181,17 @@ def execute_command( "-D sandbox.driver.)" ), ), + format: str | None = typer.Option( + None, + "--format", + help=( + "Emit an additional interchange trajectory alongside task.json. Only 'harbor' is " + "supported: writes a sibling trajectory.json (ATIF format) for every task, so an " + "external `coder-eval evaluate --format harbor` invocation can grade the trajectory " + "without access to this process's task.json. This is the flag a Harbor agent's " + "`coder-eval execute --format harbor --run-dir ` invocation passes." + ), + ), ) -> None: """Run evaluation tasks WITHOUT checking their success criteria. @@ -233,4 +244,5 @@ def execute_command( repeats=repeats, driver=driver, set_overrides=set_overrides, + format=format, ) diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py index d6994a1e..9cf40023 100644 --- a/src/coder_eval/cli/run_command.py +++ b/src/coder_eval/cli/run_command.py @@ -3,6 +3,7 @@ import asyncio import logging import os +import shutil import sys import urllib.error import urllib.parse @@ -354,6 +355,17 @@ def run_command( "-D sandbox.driver.)" ), ), + format: str | None = typer.Option( + None, + "--format", + help=( + "Emit an additional interchange trajectory alongside task.json. Only 'harbor' is " + "supported: writes a sibling trajectory.json (ATIF format) for every task, so an " + "external `coder-eval evaluate --format harbor` invocation can grade the trajectory " + "without access to this process's task.json. Meant for `coder-eval execute --format " + "harbor --run-dir `, invoked by a Harbor agent." + ), + ), ) -> None: """Run evaluation tasks (optionally in parallel). @@ -404,6 +416,7 @@ def run_command( repeats=repeats, driver=driver, set_overrides=set_overrides, + format=format, ) @@ -432,6 +445,7 @@ def run_pipeline( repeats: int | None, driver: str | None, set_overrides: list[str], + format: str | None = None, ) -> None: """The shared body of ``coder-eval run`` and ``coder-eval execute``. @@ -441,6 +455,8 @@ def run_pipeline( commands are pure flag-parsing wrappers over this function, so a behavior change can never apply to one and miss the other. """ + if format is not None and format != "harbor": + raise typer.BadParameter(f"Unsupported --format {format!r}. Supported: harbor.") # --resume needs an explicit run dir to resume into (auto-generated dirs are always fresh). if resume and run_dir is None: raise typer.BadParameter("--resume requires --run-dir pointing at the run to continue.") @@ -513,6 +529,7 @@ def run_pipeline( include_skipped=include_skipped, junit_xml=junit_xml, grade=grade, + format=format, ) ) except KeyboardInterrupt: @@ -540,6 +557,7 @@ async def _run_all_tasks( include_skipped: bool = False, junit_xml: Path | None = None, grade: bool = True, + format: str | None = None, ) -> None: """Async entry point for running all tasks (optionally in parallel). @@ -561,6 +579,16 @@ async def _run_all_tasks( junit_xml: Optional path to write a JUnit XML report to, after the run summary is persisted and before the failure exit-code gate. grade: False for `coder-eval execute` — run and capture, score nothing. + format: 'harbor' writes a trajectory.json (ATIF) sibling for every + task.json once the run finishes — see `harbor.atif_emit.emit_trajectories_for_run`. + When the run wrote exactly ONE trajectory (the shape a `CoderEvalAgent` + Harbor agent invocation always produces — one fixed-path agent-phase + task.yaml, no dataset/experiment fan-out), it is additionally copied to + `/trajectory.json` so a caller that pointed `--run-dir` at a + fixed discovery path (e.g. Harbor's `self.logs_dir`) can find it there + without knowing coder-eval's internal `///` + nesting. Multi-task runs are left nested only — there is no single + trajectory to promote. """ # Prepare run directory run_dir = prepare_run_directory(run_dir) @@ -632,6 +660,17 @@ async def _run_all_tasks( aggregate_task_logs(run_dir) + if format == "harbor": + from ..harbor.atif_emit import emit_trajectories_for_run + + written = emit_trajectories_for_run(run_dir) + console.print(f"[dim]Wrote {len(written)} trajectory.json (ATIF) file(s) under {run_dir}[/dim]") + + flat_trajectory_path = run_dir / "trajectory.json" + if len(written) == 1 and written[0] != flat_trajectory_path: + await asyncio.to_thread(shutil.copy2, written[0], flat_trajectory_path) + console.print(f"[dim]Copied the single trajectory to {flat_trajectory_path}[/dim]") + # Print execution summary print_execution_summary(run_dir, summary) diff --git a/src/coder_eval/harbor/agent.py b/src/coder_eval/harbor/agent.py new file mode 100644 index 00000000..bba35e74 --- /dev/null +++ b/src/coder_eval/harbor/agent.py @@ -0,0 +1,142 @@ +"""``CoderEvalAgent`` — coder-eval as a Harbor agent (C1.2). + +The mirror image of Part C's packager: instead of coder-eval grading a +Harbor-authored task (coder-eval as the verifier), this makes coder-eval +Harbor's AGENT — ``harbor run -a coder_eval.harbor.agent:CoderEvalAgent`` +invokes ``coder-eval execute --format harbor`` inside Harbor's own container +against the fixed-path agent-phase task.yaml the packager bakes in (see +``agent_paths.py``), then Harbor picks up the resulting ``trajectory.json`` +from ``self.logs_dir`` exactly as it does for its own ``ClaudeCode`` agent. + +Design, per ``tmp/harborframework.md``'s "Scoping note — Part A revisited": + +- The agent-phase task.yaml is at :data:`AGENT_TASK_YAML_PATH`, criteria-free + (see ``packager.py``'s ``_write_agent_phase_task_yaml``) — this agent never + sees ``success_criteria``, only the real ``agent``/prompt/sandbox config. +- ``coder-eval execute --format harbor --run-dir `` + writes ``task.json`` AND a ``trajectory.json`` (ATIF) sibling directly into + the container's ``/logs/agent/`` (``environment_logs_dir``), which Harbor + bind-mounts from ``self.logs_dir`` on the host — the same path Harbor's own + ``ClaudeCode`` agent writes its trajectory to (verified against the + installed ``harbor`` package's ``populate_context_post_run``: it writes to + ``self.logs_dir / "trajectory.json"`` on the HOST side, after the container + syncs back). This class instead has coder-eval write it directly inside the + container at the mirrored path, so no separate "workspace_dir" plumbing is + needed for v1 (Gap 2's accepted workaround). + +Verified against a real ``harbor==0.22.0`` install (``tmp/harbor-venv``): +``BaseAgent.name()`` is a ``@staticmethod``; ``run()`` returns ``None`` and +must not return the trajectory itself (Harbor discovers it by reading +``self.logs_dir / "trajectory.json"`` after the container syncs back, the same +way ``ClaudeCode.populate_context_post_run`` does) — so this class overrides +``populate_context_post_run`` to parse that file and fill in +``AgentContext``'s token/cost fields, matching ``ClaudeCode``'s own pattern +exactly. ``environment.exec()`` takes a single shell command STRING (not an +argv list). +""" + +from __future__ import annotations + +import shlex +from typing import TYPE_CHECKING + +from coder_eval.harbor.agent_paths import AGENT_TASK_YAML_PATH + + +if TYPE_CHECKING: + from harbor.environments.base import BaseEnvironment + from harbor.models.agent.context import AgentContext + +try: + from harbor.agents.installed.base import BaseInstalledAgent +except ImportError as e: # pragma: no cover - exercised only where `harbor` is installed + raise ImportError( + "coder_eval.harbor.agent requires the `harbor` package (not installed here). This module is " + + "meant to run INSIDE a Harbor trial container, where `harbor` is already present -- it is not " + + "a coder-eval runtime dependency. If you are trying to use this as a Harbor agent, install/pin " + + "`harbor` in the image the packager builds." + ) from e + +try: + from coder_eval import __version__ +except ImportError: # pragma: no cover - defensive; coder_eval always defines this + __version__ = "0.0.0" + + +class CoderEvalAgent(BaseInstalledAgent): + """Runs coder-eval's own agent loop as a Harbor agent. + + ``run()`` shells out to ``coder-eval execute --format harbor`` against the + baked-in agent-phase task.yaml rather than importing coder-eval's + orchestrator in-process — the whole point is that this class lives inside + the SAME container image the packager built, so ``coder-eval`` is already + on PATH there exactly as ``tests/test.sh`` assumes it is for grading. + """ + + SUPPORTS_ATIF: bool = True + + @staticmethod + def name() -> str: + return "coder-eval" + + def version(self) -> str: + return __version__ + + async def install(self, environment: BaseEnvironment) -> None: + """No-op: the packager's exported image is expected to already have `coder-eval` installed. + + Unlike an agent whose CLI is fetched at trial time (npm/pip install + inside ``install()``), coder-eval is baked into the image at export + time -- re-installing it here would fight whatever version the image + pins. + """ + del environment + + async def run(self, instruction: str, environment: BaseEnvironment, context: AgentContext) -> None: + """Run ``coder-eval execute --format harbor`` inside the environment. + + ``instruction`` (Harbor's resolved ``instruction.md`` text) is NOT + forwarded — the agent-phase task.yaml at :data:`AGENT_TASK_YAML_PATH` + already carries the identical resolved prompt (``packager.py`` writes + both from the same source), so there is nothing to forward. Token/cost + totals are filled in afterward by ``populate_context_post_run``, not + here, matching every other installed agent's convention. + """ + del instruction, context # nothing to forward; context is populated post-run + run_dir = self.environment_logs_dir.as_posix() + command = ( + f"coder-eval execute {shlex.quote(AGENT_TASK_YAML_PATH)} --format harbor --run-dir {shlex.quote(run_dir)}" + ) + await self._exec(environment, command) + + def populate_context_post_run(self, context: AgentContext) -> None: + """Parse the ``trajectory.json`` coder-eval wrote and fill in token/cost totals. + + Mirrors ``ClaudeCode.populate_context_post_run`` exactly: by the time + this runs, Harbor has synced the container's ``environment_logs_dir`` + back to the host at ``self.logs_dir``, so the file coder-eval wrote + during ``run()`` is now readable here. + """ + trajectory_path = self.logs_dir / "trajectory.json" + if not trajectory_path.is_file(): + self.logger.debug(f"No trajectory.json at {trajectory_path}; coder-eval execute may have failed") + return + + from coder_eval.harbor.atif_models import Trajectory + + try: + trajectory = Trajectory.model_validate_json(trajectory_path.read_text(encoding="utf-8")) + except Exception as exc: # best-effort context enrichment, never fatal to the trial + self.logger.debug(f"Failed to parse {trajectory_path}: {exc}") + return + + if trajectory.final_metrics is None: + return + metrics = trajectory.final_metrics + context.cost_usd = metrics.total_cost_usd + context.n_input_tokens = metrics.total_prompt_tokens or 0 + context.n_cache_tokens = metrics.total_cached_tokens or 0 + context.n_output_tokens = metrics.total_completion_tokens or 0 + + +__all__ = ["CoderEvalAgent"] diff --git a/src/coder_eval/harbor/agent_paths.py b/src/coder_eval/harbor/agent_paths.py new file mode 100644 index 00000000..708415f8 --- /dev/null +++ b/src/coder_eval/harbor/agent_paths.py @@ -0,0 +1,19 @@ +"""Fixed in-container path for the agent-phase task.yaml a Harbor export bakes into its image. + +Shared by ``packager.py`` (the writer — bakes a criteria-free copy of the task +into ``environment/agent_task.yaml`` and ``COPY``s it here) and ``agent.py``'s +``CoderEvalAgent`` (the reader — the Harbor agent that runs +``coder-eval execute --format harbor`` against this exact path), so the two +sides can never independently drift on where the file lives. Fixed rather than +discovered: a Harbor agent has no way to ask the export what path it chose, so +the path itself is the contract (tmp/harborframework.md's "Gap 1" resolution — +the agent can always execute a task.yaml at a fixed path; it's up to the +Dockerfile to put it there). +""" + +from __future__ import annotations + + +AGENT_TASK_YAML_PATH = "/opt/coder-eval-task/task.yaml" + +__all__ = ["AGENT_TASK_YAML_PATH"] diff --git a/src/coder_eval/harbor/atif_emit.py b/src/coder_eval/harbor/atif_emit.py new file mode 100644 index 00000000..99303ce3 --- /dev/null +++ b/src/coder_eval/harbor/atif_emit.py @@ -0,0 +1,422 @@ +"""EvaluationResult → ATIF Trajectory converter (the emit direction). + +Maps coder_eval's persisted trajectory (``EvaluationResult.iterations`` — the +``TurnRecord`` envelope over the per-generation ``messages`` stream) onto the +vendored ATIF models, so every run can be consumed by ``harbor view``, Harbor +Hub, and ATIF-based SFT/RL pipelines. + +Mapping highlights: + +- ``UserMessage`` → ``Step(source="user")``; ``AssistantMessage`` (one per LLM + generation) → ``Step(source="agent")`` with per-generation ``Metrics``. +- ``CommandTelemetry`` joins its generation via ``assistant_turn_index`` and + becomes that step's ``tool_calls`` + ``observation``. +- Sub-agent generations (``parent_tool_use_id`` set) are NESTED into embedded + ``subagent_trajectories`` — flattening them into the main thread would + corrupt SFT data derived from the trajectory. +- ``ReconciliationMessage`` entries never become steps: their residuals are + recorded in ``Trajectory.extra["reconciliation"]`` and are already included + in the authoritative ``FinalMetrics`` totals (``total_token_usage``). +- Turns with no message stream (legacy task.json, minimal agents) degrade to + one synthetic user step + one agent step carrying all the turn's commands. + +The converter is a PURE function of the models: no I/O, no agent-type +branching, no mutation of the input result. +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Any + +from coder_eval.harbor.atif_models import ( + AtifAgent, + FinalMetrics, + Metrics, + Observation, + ObservationResult, + Step, + SubagentTrajectoryRef, + ToolCall, + Trajectory, +) +from coder_eval.models import ( + AssistantMessage, + CommandTelemetry, + EvaluationResult, + ReconciliationMessage, + TokenUsage, + TurnRecord, + UserMessage, +) +from coder_eval.path_utils import TASK_JSON_FILENAME, write_text_atomic + + +logger = logging.getLogger(__name__) + + +def _message_text(msg: AssistantMessage) -> str: + """Concatenate the message's text blocks in emission order.""" + parts = [b.text for b in msg.content_blocks if b.block_type == "text" and b.text] + return "\n".join(parts) + + +def _reasoning_text(msg: AssistantMessage) -> str | None: + """Concatenate the message's thinking blocks, or None when there are none.""" + parts = [b.thinking for b in msg.content_blocks if b.block_type == "thinking" and b.thinking] + return "\n".join(parts) if parts else None + + +def _metrics_for(msg: AssistantMessage) -> Metrics | None: + """Per-generation Metrics; None when the message carries no token buckets.""" + if not any((msg.input_tokens, msg.output_tokens, msg.cache_creation_tokens, msg.cache_read_tokens)): + return None + # SSOT: the "three prompt buckets sum to the full prompt" rule lives in + # TokenUsage.input_tokens (a computed field) — do not hand-add them here. + usage = TokenUsage( + uncached_input_tokens=msg.input_tokens, + cache_creation_input_tokens=msg.cache_creation_tokens, + cache_read_input_tokens=msg.cache_read_tokens, + output_tokens=msg.output_tokens, + ) + return Metrics( + prompt_tokens=usage.input_tokens, + completion_tokens=msg.output_tokens, + cached_tokens=msg.cache_read_tokens, + ) + + +def _assistant_extra(msg: AssistantMessage, iteration: int) -> dict[str, Any]: + """Step.extra for one generation: iteration tag + fidelity ATIF has no slot for.""" + extra: dict[str, Any] = {"iteration": iteration} + if msg.cache_creation_tokens: + extra["cache_creation_tokens"] = msg.cache_creation_tokens + if msg.reasoning_tokens: + extra["reasoning_tokens"] = msg.reasoning_tokens + if msg.message_id is not None: + extra["message_id"] = msg.message_id + if msg.stop_reason is not None: + extra["stop_reason"] = msg.stop_reason + return extra + + +def _tool_calls_for(commands: list[CommandTelemetry]) -> tuple[list[ToolCall], Observation] | tuple[None, None]: + """Build a step's tool_calls + observation from its commands (None when empty).""" + if not commands: + return None, None + calls: list[ToolCall] = [] + results: list[ObservationResult] = [] + for cmd in commands: + call_extra: dict[str, Any] = {} + if cmd.result_status is not None: + call_extra["result_status"] = cmd.result_status + calls.append( + ToolCall( + tool_call_id=cmd.tool_id, + function_name=cmd.tool_name, + arguments=cmd.parameters, + extra=call_extra or None, + ) + ) + result_extra: dict[str, Any] = {} + if cmd.duration_ms is not None: + result_extra["duration_ms"] = cmd.duration_ms + results.append( + ObservationResult( + source_call_id=cmd.tool_id, + content=cmd.result_summary, + extra=result_extra or None, + ) + ) + return calls, Observation(results=results) + + +class _StepBuilder: + """Accumulates main-thread and per-sub-agent step lists with sequential ids.""" + + def __init__(self) -> None: + self.main: list[Step] = [] + # parent_tool_use_id -> child steps, insertion-ordered. + self.subagents: dict[str, list[Step]] = {} + + def append(self, parent_tool_use_id: str | None, **step_fields: Any) -> Step: + target = self.main if parent_tool_use_id is None else self.subagents.setdefault(parent_tool_use_id, []) + step = Step(step_id=len(target) + 1, **step_fields) + target.append(step) + return step + + def last_main_agent_step(self, start: int = 0) -> Step | None: + """Most recent main-thread agent step at or after index ``start``. + + ``start`` scopes the search to the current turn (leftover-command + attribution must never cross turn boundaries — a command attached to a + previous iteration's step would mislabel its ``extra["iteration"]``). + """ + for step in reversed(self.main[start:]): + if step.source == "agent": + return step + return None + + +def _extend_step_commands(step: Step, commands: list[CommandTelemetry]) -> None: + """Attach extra commands to an already-built step (fallback attribution).""" + calls, observation = _tool_calls_for(commands) + if calls is None or observation is None: + return + step.tool_calls = [*(step.tool_calls or []), *calls] + existing = step.observation.results if step.observation is not None else [] + step.observation = Observation(results=[*existing, *observation.results]) + + +def _emit_turn(builder: _StepBuilder, turn: TurnRecord, reconciliation: list[dict[str, Any]]) -> None: + """Emit one TurnRecord's steps into the builder (main thread + sub-agent groups).""" + turn_start = len(builder.main) + turn_extra: dict[str, Any] = {"iteration": turn.iteration} + if turn.crashed: + turn_extra["crashed"] = True + + assistant_msgs = [m for m in turn.messages if isinstance(m, AssistantMessage)] + + # Group commands by their generation; collect the unattributable ones. + commands_by_index: dict[int, list[CommandTelemetry]] = {} + leftover_commands: list[CommandTelemetry] = [] + for cmd in turn.commands: + idx = cmd.assistant_turn_index + if idx is not None and 0 <= idx < len(assistant_msgs): + commands_by_index.setdefault(idx, []).append(cmd) + else: + if idx is not None: + logger.debug("assistant_turn_index %s out of range for turn %s", idx, turn.iteration) + leftover_commands.append(cmd) + + first_step_of_turn = True + + def _step_extra(base: dict[str, Any]) -> dict[str, Any]: + nonlocal first_step_of_turn + extra = {**turn_extra, **base} + if first_step_of_turn and turn.crashed and turn.crash_reason is not None: + extra["crash_reason"] = turn.crash_reason + first_step_of_turn = False + return extra + + # Single-shot runs carry no UserMessage in the stream — synthesize the + # iteration's user step from user_input (exactly one per iteration). + if not any(isinstance(m, UserMessage) for m in turn.messages): + builder.append(None, source="user", message=turn.user_input, extra=_step_extra({})) + + if not turn.messages: + # Legacy / minimal-agent turn (EMPTY message stream — a stream that has + # user/reconciliation entries but no generations takes the normal path + # below so those entries survive): one agent step from agent_output + # carrying ALL the turn's commands and the turn-level token usage. + metrics = None + if turn.token_usage is not None and not turn.token_usage.is_empty(): + metrics = Metrics( + prompt_tokens=turn.token_usage.input_tokens, + completion_tokens=turn.token_usage.output_tokens, + cached_tokens=turn.token_usage.cache_read_input_tokens, + ) + calls, observation = _tool_calls_for(turn.commands) + builder.append( + None, + source="agent", + message=turn.agent_output, + model_name=turn.model_used, + metrics=metrics, + tool_calls=calls, + observation=observation, + extra=_step_extra({}), + ) + return + + assistant_index = -1 + for msg in turn.messages: + if isinstance(msg, ReconciliationMessage): + reconciliation.append( + { + "iteration": turn.iteration, + "input_tokens": msg.input_tokens, + "output_tokens": msg.output_tokens, + "cache_creation_tokens": msg.cache_creation_tokens, + "cache_read_tokens": msg.cache_read_tokens, + "note": msg.note, + } + ) + continue + if isinstance(msg, UserMessage): + # Every UserMessage is a genuine user utterance today (constructed + # only for simulator turns / pinned openers). If a tool-result + # UserMessage variant ever gains a producing code path (its + # docstring reserves one), the converter must learn to SKIP those + # here — tool results already live in step observations. + builder.append( + None, + source="user", + message=msg.text, + timestamp=msg.completed_at.isoformat() if msg.completed_at is not None else None, + extra=_step_extra({}), + ) + continue + assistant_index += 1 + calls, observation = _tool_calls_for(commands_by_index.get(assistant_index, [])) + builder.append( + msg.parent_tool_use_id, + source="agent", + message=_message_text(msg), + reasoning_content=_reasoning_text(msg), + model_name=msg.model, + timestamp=msg.completed_at.isoformat(), + metrics=_metrics_for(msg), + tool_calls=calls, + observation=observation, + extra=_step_extra(_assistant_extra(msg, turn.iteration)), + ) + + if leftover_commands: + # Documented fallback: unattributable commands ride the turn's last + # main-thread agent step; synthesize one from agent_output if none exists. + target = builder.last_main_agent_step(start=turn_start) + if target is None: + target = builder.append(None, source="agent", message=turn.agent_output, extra=_step_extra({})) + _extend_step_commands(target, leftover_commands) + + +def _attach_subagent_refs(main_steps: list[Step], parent_ids: list[str]) -> None: + """Point each spawning tool call's observation at its embedded child trajectory. + + Only when a main-thread ToolCall with ``tool_call_id == parent_id`` exists — + never fabricate a tool call for an orphaned sub-agent group. + """ + for parent_id in parent_ids: + for step in main_steps: + if not step.tool_calls or all(tc.tool_call_id != parent_id for tc in step.tool_calls): + continue + ref = SubagentTrajectoryRef(trajectory_id=parent_id) + results = list(step.observation.results) if step.observation is not None else [] + for result in results: + if result.source_call_id == parent_id: + result.subagent_trajectory_ref = [ref] + break + else: + results.append(ObservationResult(source_call_id=parent_id, subagent_trajectory_ref=[ref])) + step.observation = Observation(results=results) + break + + +def _total_usage(result: EvaluationResult) -> TokenUsage | None: + """The authoritative run total, falling back to summing per-turn usage.""" + if result.total_token_usage is not None: + return result.total_token_usage + usages = [t.token_usage for t in result.iterations if t.token_usage is not None] + if not usages: + return None + total = usages[0] + for usage in usages[1:]: + total = total + usage + return total + + +def evaluation_result_to_trajectory(result: EvaluationResult) -> Trajectory | None: + """Convert a finished EvaluationResult into an ATIF Trajectory. + + Returns None when no steps would result (e.g. evaluate-only runs) — ATIF + requires at least one step. + """ + from coder_eval import __version__ + + builder = _StepBuilder() + reconciliation: list[dict[str, Any]] = [] + for turn in result.iterations: + _emit_turn(builder, turn, reconciliation) + + if not builder.main: + return None + + _attach_subagent_refs(builder.main, list(builder.subagents)) + + agent = AtifAgent(name=result.agent_type, version=__version__, model_name=result.model_used) + session_id = f"{result.task_id}/{result.variant_id}" + subagent_trajectories = [ + Trajectory(session_id=session_id, trajectory_id=parent_id, agent=agent, steps=steps) + for parent_id, steps in builder.subagents.items() + ] + + final_metrics: FinalMetrics | None = None + total = _total_usage(result) + if total is not None: + final_metrics = FinalMetrics( + total_prompt_tokens=total.input_tokens, + total_completion_tokens=total.output_tokens, + total_cached_tokens=total.cache_read_input_tokens, + total_cost_usd=total.total_cost_usd, + total_steps=len(builder.main), + ) + + return Trajectory( + session_id=session_id, + agent=agent, + steps=builder.main, + final_metrics=final_metrics, + extra={"reconciliation": reconciliation} if reconciliation else None, + subagent_trajectories=subagent_trajectories or None, + ) + + +def write_trajectory_json_strict(result: EvaluationResult, path: Path) -> Path | None: + """Convert and atomically write ``trajectory.json``; RAISES on failure. + + Returns the written path, or None ONLY for the legitimate zero-step skip + (ATIF requires >= 1 step; e.g. evaluate-only runs). Conversion or write + errors propagate — for callers that must distinguish "nothing to emit" + from "emission failed" (the report backfill). + """ + trajectory = evaluation_result_to_trajectory(result) + if trajectory is None: + logger.debug("No trajectory steps for %s — skipping trajectory.json", result.task_id) + return None + write_text_atomic(path, trajectory.model_dump_json(indent=2, exclude_none=True) + "\n") + return path + + +def write_trajectory_json(result: EvaluationResult, path: Path) -> Path | None: + """Convert and atomically write ``trajectory.json``; never raises. + + Returns the written path, or None when the result yields no steps or the + conversion/write fails (logged at WARNING — a trajectory failure must + never mask the run outcome). Used by :func:`emit_trajectories_for_run` + (``coder-eval execute --format harbor``); see + :func:`write_trajectory_json_strict` for the raising one. + """ + try: + return write_trajectory_json_strict(result, path) + except Exception: + logger.warning("Failed to write trajectory.json for %s", result.task_id, exc_info=True) + return None + + +def emit_trajectories_for_run(run_dir: Path) -> list[Path]: + """Write a ``trajectory.json`` sibling for every ``task.json`` under ``run_dir``. + + The ``--format harbor`` post-pass for ``coder-eval execute``: ATIF emission + is opt-in (unlike the old always-on design this module's predecessor + shipped), so a plain ``run``/``execute`` never gains a new output file. + Walks the run directory rather than hooking the orchestrator's finalize + path, keeping this package's "translate coder-eval's own artifacts" + scope (see ``coder_eval.harbor``'s module docstring) — it needs no access + to orchestrator internals, only the ``task.json`` files a run already + wrote. Per-task failures are logged and skipped (see + :func:`write_trajectory_json`), never aborting the rest of the run's export. + """ + written: list[Path] = [] + for task_json in sorted(run_dir.glob(f"**/{TASK_JSON_FILENAME}")): + try: + result = EvaluationResult.model_validate_json(task_json.read_text(encoding="utf-8")) + except (OSError, ValueError): + logger.warning("Could not read %s as an EvaluationResult — skipping ATIF emission", task_json) + continue + trajectory_path = task_json.with_name("trajectory.json") + out = write_trajectory_json(result, trajectory_path) + if out is not None: + written.append(out) + return written diff --git a/src/coder_eval/harbor/atif_hydrate.py b/src/coder_eval/harbor/atif_hydrate.py new file mode 100644 index 00000000..509d973d --- /dev/null +++ b/src/coder_eval/harbor/atif_hydrate.py @@ -0,0 +1,170 @@ +"""ATIF Trajectory -> coder-eval ``TurnRecord`` list (the hydrate direction). + +The reverse of ``atif_emit``: given a trajectory that was produced OUTSIDE this +process (a Harbor agent ran ``coder-eval execute --format harbor``, which wrote +``trajectory.json`` next to ``task.json``), reconstruct enough of coder-eval's +own trajectory shape to let the criteria checkers that read it +(``command_executed``, ``cli_called``, ``commands_efficiency``, +``skill_triggered``, ``llm_judge``'s transcript) work against it during a +separate ``coder-eval evaluate --format harbor`` invocation. + +Every criterion checker receives ``turn_records: list[TurnRecord] | None`` — +in a normal run this is ``EvaluationResult.iterations`` — and reads only +``TurnRecord.commands`` (tool calls) and ``TurnRecord.messages`` (for judge +transcripts); see ``criteria/command_executed.py``, ``criteria/skill_triggered.py``, +``criteria/commands_efficiency.py``. Those two fields are what this module +reconstructs. It does NOT attempt a lossless round-trip of ``atif_emit``'s +mapping: + +- Per-generation token buckets (``AssistantMessage.input_tokens`` etc.) are + NOT recovered from ``Step.metrics`` — ``TurnRecord.token_usage`` is left + unset. Cost/token reporting for a hydrated result is therefore incomplete; + only trajectory-shaped criteria are the target here. +- Sub-agent nesting is flattened: ``subagent_trajectories`` steps are appended + to the parent turn's commands (via their tool_calls) rather than + reconstructing a nested ``parent_tool_use_id`` relationship. +- Turn boundaries are recovered by splitting on ``source="user"`` steps + (mirroring ``atif_emit``'s "one synthetic user step per turn" convention), + not by any explicit iteration marker ATIF carries. +""" + +from __future__ import annotations + +from datetime import UTC, datetime + +from coder_eval.harbor.atif_models import ObservationResult, Step, ToolCall, Trajectory +from coder_eval.models import CommandTelemetry, EvaluationResult, FinalStatus, TurnRecord + + +def _tool_call_to_command( + call: ToolCall, + results_by_id: dict[str, ObservationResult], + *, + assistant_turn_index: int, +) -> CommandTelemetry: + result = results_by_id.get(call.tool_call_id) + content = result.content if result is not None else None + result_summary = content if isinstance(content, str) else (None if content is None else str(content)) + return CommandTelemetry( + tool_name=call.function_name, + tool_id=call.tool_call_id, + timestamp=datetime.now(UTC), + parameters=call.arguments, + result_summary=result_summary, + assistant_turn_index=assistant_turn_index, + sequence_number=assistant_turn_index, + ) + + +def _step_text(step: Step) -> str: + if isinstance(step.message, str): + return step.message + return "\n".join(part.text for part in step.message if part.type == "text" and part.text) + + +def _commands_for_step(step: Step, assistant_turn_index: int) -> list[CommandTelemetry]: + if not step.tool_calls: + return [] + results_by_id = { + r.source_call_id: r + for r in (step.observation.results if step.observation is not None else []) + if r.source_call_id + } + return [ + _tool_call_to_command(call, results_by_id, assistant_turn_index=assistant_turn_index) + for call in step.tool_calls + ] + + +def _turn_from_steps(iteration: int, steps: list[Step]) -> TurnRecord: + """Build one TurnRecord from a contiguous run of steps starting at a user step.""" + user_step = steps[0] if steps and steps[0].source == "user" else None + agent_steps = [s for s in steps if s.source == "agent"] + + commands: list[CommandTelemetry] = [] + assistant_index = -1 + for step in steps: + if step.source != "agent": + continue + assistant_index += 1 + commands.extend(_commands_for_step(step, assistant_index)) + + return TurnRecord( + iteration=iteration, + user_input=_step_text(user_step) if user_step is not None else "", + agent_output=_step_text(agent_steps[-1]) if agent_steps else "", + commands=commands, + ) + + +def trajectory_to_turn_records(trajectory: Trajectory) -> list[TurnRecord]: + """Split ``trajectory.steps`` into per-turn groups and reconstruct ``TurnRecord``s. + + Sub-agent trajectories are flattened in: each embedded child's tool calls + are appended onto whichever main-thread turn contains the spawning tool + call (matched by ``tool_call_id``), falling back to the last turn when no + spawning call is found (an orphaned embed, mirroring ``atif_emit``'s own + orphan-tolerant behavior on the way out). + """ + groups: list[list[Step]] = [] + for step in trajectory.steps: + if step.source == "user" or not groups: + groups.append([step]) + else: + groups[-1].append(step) + + turns = [_turn_from_steps(i + 1, steps) for i, steps in enumerate(groups)] + + for sub in trajectory.subagent_trajectories or []: + sub_commands: list[CommandTelemetry] = [] + assistant_index = -1 + for step in sub.steps: + if step.source != "agent": + continue + assistant_index += 1 + sub_commands.extend(_commands_for_step(step, assistant_index)) + if not sub_commands: + continue + target = turns[-1] if turns else None + for turn, steps in zip(turns, groups, strict=True): + if any(sub.trajectory_id in {tc.tool_call_id for tc in (s.tool_calls or [])} for s in steps): + target = turn + break + if target is not None: + target.commands.extend(sub_commands) + + return turns + + +def seed_from_atif_trajectory( + trajectory: Trajectory, + *, + task_id: str, + task_description: str = "", + variant_id: str = "harbor", +) -> EvaluationResult: + """Build a minimal ``EvaluationResult`` from an ATIF trajectory for detached grading. + + Only the fields ``evaluate --format harbor``'s grading path actually reads + are populated meaningfully: ``iterations`` (via + :func:`trajectory_to_turn_records`) and ``iteration_count``. + ``final_status``/``weighted_score`` are placeholders — grading recomputes + them; this object exists only to carry trajectory context into + ``Orchestrator(prior_result=...)``, exactly as a re-graded run directory's + own ``task.json`` does. + """ + turns = trajectory_to_turn_records(trajectory) + return EvaluationResult( + task_id=task_id, + task_description=task_description, + variant_id=variant_id, + agent_type=trajectory.agent.name, + model_used=trajectory.agent.model_name, + started_at=datetime.now(UTC), + final_status=FinalStatus.NOT_GRADED, + iteration_count=len(turns), + iterations=turns, + ) + + +__all__ = ["seed_from_atif_trajectory", "trajectory_to_turn_records"] diff --git a/src/coder_eval/harbor/atif_models.py b/src/coder_eval/harbor/atif_models.py new file mode 100644 index 00000000..3923ccd0 --- /dev/null +++ b/src/coder_eval/harbor/atif_models.py @@ -0,0 +1,277 @@ +"""Vendored ATIF (Agent Trajectory Interchange Format) models, v1.7. + +Mirrors the schema in ``harbor.models.trajectories`` (harbor 0.20.0) so +coder_eval can emit and parse ATIF trajectories with ZERO runtime dependency +on the ``harbor`` pip package. Fidelity is guarded by a frozen fixture in +``tests/fixtures/atif/`` that was validated once against the real harbor +models (see ``tests/test_atif_models.py`` for the reproducible procedure). + +Deliberate deviations from harbor's models: + +- ``schema_version`` is a pattern-validated ``str`` (``^ATIF-v1\\.\\d+$``) + instead of a closed Literal, so trajectories written by a FUTURE harbor + minor version (e.g. ``ATIF-v1.9``) still parse — harbor 0.20.0 itself + would reject them. Major-version bumps (``ATIF-v2.0``) are rejected. +- harbor's ``Agent`` model is named :class:`AtifAgent` here to avoid clashing + with ``coder_eval.agent.Agent``. +- ``ContentPart.source`` (image payloads) is an untyped dict — coder_eval + emits text-only content and only needs to *tolerate* image parts on read. + +Deliberate deviation from the repo convention "all models importable from +``coder_eval.models``": these are interchange-format models for Harbor +interop, not evaluation models — they are exported from ``coder_eval.harbor`` +to keep the core model namespace clean. +""" + +from __future__ import annotations + +from typing import Any, Literal, Self + +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +ATIF_SCHEMA_VERSION = "ATIF-v1.7" +"""The ATIF version coder_eval emits (the version the vendored schema mirrors).""" + + +class ContentPart(BaseModel): + """One part of a multimodal message (text or image). + + coder_eval emits text-only content; the image variant exists so ATIF + documents produced by other agents still parse. + """ + + model_config = ConfigDict(extra="forbid") + + type: Literal["text", "image"] = Field(description="Content part kind.") + text: str | None = Field(default=None, description="Text content. Required when type='text'.") + source: dict[str, Any] | None = Field( + default=None, + description="Image payload (media type + data). Only meaningful when type='image'; untyped by design.", + ) + + @model_validator(mode="after") + def _validate_by_type(self) -> Self: + if self.type == "text": + if self.text is None: + raise ValueError("'text' field is required when type='text'") + if self.source is not None: + raise ValueError("'source' field is not allowed when type='text'") + return self + + +class ToolCall(BaseModel): + """A tool call within a step.""" + + model_config = ConfigDict(extra="forbid") + + tool_call_id: str = Field(description="Unique identifier for this specific tool call.") + function_name: str = Field(description="The name of the function or tool being invoked.") + arguments: dict[str, Any] = Field(description="Arguments passed to the function (can be empty dict).") + extra: dict[str, Any] | None = Field(default=None, description="Custom tool-call-level metadata.") + + +class SubagentTrajectoryRef(BaseModel): + """Reference to a delegated subagent trajectory.""" + + model_config = ConfigDict(extra="forbid") + + trajectory_id: str = Field(description="trajectory_id of the referenced subagent trajectory.") + trajectory_path: str | None = Field( + default=None, + description=( + "Path to an external trajectory file. Null means embedded resolution: the id matches an " + "entry in the root trajectory's subagent_trajectories array." + ), + ) + + +class ObservationResult(BaseModel): + """The result of one tool call or action within a step's observation.""" + + model_config = ConfigDict(extra="forbid") + + source_call_id: str | None = Field( + default=None, + description=( + "The tool_call_id from this step's tool_calls that this result corresponds to. Null for " + "results from actions that don't use the standard tool-calling format." + ), + ) + content: str | list[ContentPart] | None = Field( + default=None, + description="The output from the tool execution (string, or ContentPart list for multimodal).", + ) + subagent_trajectory_ref: list[SubagentTrajectoryRef] | None = Field( + default=None, + description="References to delegated subagent trajectories spawned by this call.", + ) + extra: dict[str, Any] | None = Field(default=None, description="Custom observation-result-level metadata.") + + +class Observation(BaseModel): + """The environment feedback for one step.""" + + model_config = ConfigDict(extra="forbid") + + results: list[ObservationResult] = Field(description="Result objects from tool calls or actions.") + + +class Metrics(BaseModel): + """Per-step LLM metrics.""" + + model_config = ConfigDict(extra="forbid") + + prompt_tokens: int | None = Field(default=None, description="Prompt tokens for this LLM call (full prompt).") + completion_tokens: int | None = Field(default=None, description="Completion tokens generated by this call.") + cached_tokens: int | None = Field(default=None, description="Prompt tokens served from cache.") + cost_usd: float | None = Field(default=None, description="Cost of this call in USD.") + prompt_token_ids: list[int] | None = Field(default=None, description="Token ids of the prompt (RL pipelines).") + completion_token_ids: list[int] | None = Field( + default=None, description="Token ids of the completion (RL pipelines)." + ) + logprobs: list[float] | None = Field(default=None, description="Per-token logprobs of the completion.") + extra: dict[str, Any] | None = Field(default=None, description="Custom metrics-level metadata.") + + +class FinalMetrics(BaseModel): + """Summary metrics for the entire trajectory.""" + + model_config = ConfigDict(extra="forbid") + + total_prompt_tokens: int | None = Field(default=None, description="Total prompt tokens across the trajectory.") + total_completion_tokens: int | None = Field(default=None, description="Total completion tokens.") + total_cached_tokens: int | None = Field(default=None, description="Total cached prompt tokens.") + total_cost_usd: float | None = Field(default=None, description="Total cost in USD.") + total_steps: int | None = Field(default=None, description="Number of steps in the trajectory.") + extra: dict[str, Any] | None = Field(default=None, description="Custom final-metrics metadata.") + + +class AtifAgent(BaseModel): + """The agent configuration that produced the trajectory. + + harbor names this model ``Agent``; renamed here to avoid clashing with + ``coder_eval.agent.Agent``. + """ + + model_config = ConfigDict(extra="forbid") + + name: str = Field(description="Agent name (e.g. the registered agent kind).") + version: str = Field(description="Agent version string (REQUIRED by the ATIF spec).") + model_name: str | None = Field(default=None, description="Default model for the trajectory's steps.") + tool_definitions: list[dict[str, Any]] | None = Field( + default=None, + description="Tool/function definitions available to the agent (OpenAI function-calling schema).", + ) + extra: dict[str, Any] | None = Field(default=None, description="Custom agent-level metadata.") + + +class Step(BaseModel): + """A single step in the trajectory (one message event).""" + + model_config = ConfigDict(extra="forbid") + + step_id: int = Field(ge=1, description="Ordinal index of the step (sequential, starting from 1).") + timestamp: str | None = Field(default=None, description="ISO 8601 timestamp of when this step occurred.") + source: Literal["system", "user", "agent"] = Field(description="The originator of this step.") + model_name: str | None = Field( + default=None, + description="LLM model used for this step. Omission implies the root agent config's model.", + ) + reasoning_effort: str | float | None = Field( + default=None, description="Qualitative or quantitative measure of effort." + ) + message: str | list[ContentPart] = Field( + description="The dialogue message (string, or ContentPart list for multimodal)." + ) + reasoning_content: str | None = Field(default=None, description="The agent's explicit internal reasoning.") + tool_calls: list[ToolCall] | None = Field(default=None, description="Tool calls issued in this step.") + observation: Observation | None = Field(default=None, description="Environment feedback for this step.") + metrics: Metrics | None = Field(default=None, description="LLM metrics for this step.") + is_copied_context: bool | None = Field( + default=None, description="True when the step's content was copied from a prior context (compaction)." + ) + llm_call_count: int | None = Field(default=None, description="Number of LLM calls this step represents.") + extra: dict[str, Any] | None = Field(default=None, description="Custom step-level metadata.") + + +class Trajectory(BaseModel): + """Agent Trajectory in ATIF (Agent Trajectory Interchange Format).""" + + model_config = ConfigDict(extra="forbid") + + schema_version: str = Field( + default=ATIF_SCHEMA_VERSION, + pattern=r"^ATIF-v1\.\d+$", + description=( + "ATIF compatibility version. Any v1.x parses (forward-tolerant read, unlike harbor's closed " + "Literal); v2+ is rejected." + ), + ) + session_id: str | None = Field( + default=None, + description="Run-scoped identifier; may be shared by a parent trajectory and its embedded subagents.", + ) + trajectory_id: str | None = Field( + default=None, + description=( + "Per-trajectory-document unique identifier. Optional on standalone trajectories; REQUIRED and " + "unique on trajectories embedded in a parent's subagent_trajectories array." + ), + ) + agent: AtifAgent = Field(description="The agent configuration that produced this trajectory.") + steps: list[Step] = Field(min_length=1, description="The complete interaction history.") + notes: str | None = Field(default=None, description="Custom information, design notes, or explanations.") + final_metrics: FinalMetrics | None = Field(default=None, description="Summary metrics for the trajectory.") + continued_trajectory_ref: str | None = Field( + default=None, description="Reference to the continuation trajectory file, if continued elsewhere." + ) + extra: dict[str, Any] | None = Field(default=None, description="Custom root-level metadata.") + subagent_trajectories: list[Trajectory] | None = Field( + default=None, + description="Embedded subagent trajectories; each must carry a unique, non-null trajectory_id.", + ) + + @model_validator(mode="after") + def validate_step_ids(self) -> Self: + """step_ids must be sequential starting from 1 (mirrors harbor's validator).""" + for i, step in enumerate(self.steps): + expected = i + 1 + if step.step_id != expected: + raise ValueError(f"steps[{i}].step_id: expected {expected} (sequential from 1), got {step.step_id}") + return self + + @model_validator(mode="after") + def validate_tool_call_references(self) -> Self: + """Every observation source_call_id must reference a tool_call_id in the SAME step.""" + for step in self.steps: + if step.observation is None: + continue + tool_call_ids = {tc.tool_call_id for tc in step.tool_calls} if step.tool_calls else set() + for result in step.observation.results: + if result.source_call_id is not None and result.source_call_id not in tool_call_ids: + raise ValueError( + f"Observation result references source_call_id '{result.source_call_id}' " + + f"which is not found in step {step.step_id}'s tool_calls" + ) + return self + + @model_validator(mode="after") + def validate_embedded_subagent_trajectory_ids(self) -> Self: + """Embedded subagents must carry a unique, non-null trajectory_id (resolution key).""" + if not self.subagent_trajectories: + return self + seen: set[str] = set() + for i, sub in enumerate(self.subagent_trajectories): + if sub.trajectory_id is None: + raise ValueError( + f"subagent_trajectories[{i}].trajectory_id is required for embedded subagents " + + f"(agent.name={sub.agent.name!r}, session_id={sub.session_id!r})" + ) + if sub.trajectory_id in seen: + raise ValueError( + f"subagent_trajectories[{i}].trajectory_id {sub.trajectory_id!r} is not unique " + + "within subagent_trajectories" + ) + seen.add(sub.trajectory_id) + return self diff --git a/src/coder_eval/harbor/packager.py b/src/coder_eval/harbor/packager.py index df2e9228..0b72e5fc 100644 --- a/src/coder_eval/harbor/packager.py +++ b/src/coder_eval/harbor/packager.py @@ -48,6 +48,7 @@ import tomli_w import yaml +from coder_eval.harbor.agent_paths import AGENT_TASK_YAML_PATH from coder_eval.harbor.portability import PortabilityIssue, audit_criteria from coder_eval.models import TaskDefinition from coder_eval.orchestration.task_loader import load_task @@ -63,6 +64,19 @@ "(unused placeholder — this file is graded via `coder-eval evaluate`, which does not invoke an agent on this path)" ) +# Mirror-image placeholder for the AGENT-phase task.yaml (see +# `_write_agent_phase_task_yaml`): `coder-eval execute` never grades +# (FinalStatus.NOT_GRADED unconditionally), so success_criteria's CONTENT is +# never read there either -- it exists only to satisfy TaskDefinition's +# at-least-one-criterion schema requirement without leaking the real criteria +# (which stay exclusively in tests/task.yaml, the verifier-side file) into the +# agent-visible image. +_AGENT_PHASE_PLACEHOLDER_CRITERION: dict[str, object] = { + "type": "file_exists", + "path": ".", + "description": "(unused placeholder — this file is run via `coder-eval execute`, which never checks criteria)", +} + _TEST_SH_TEMPLATE = """#!/bin/sh # Generated by `coder-eval export --format harbor` — do not hand-edit; a # regenerated export will overwrite it. See tmp/harborframework.md § C1.1. @@ -191,16 +205,25 @@ def export_resolved_task( return ExportResult(out_dir=out_dir, workdir=workdir, warnings=warnings) -def _write_instruction(task: TaskDefinition, task_file: Path, out_dir: Path) -> None: +def _resolve_prompt_text(task: TaskDefinition, task_file: Path) -> str: + """The agent's actual instruction text, resolved from whichever of the two prompt fields is set. + + Shared by ``instruction.md`` (Harbor's own agent phase, when one is used) + and ``environment/agent_task.yaml`` (the ``CoderEvalAgent`` embed's real + prompt) — both must show the agent the identical text. + """ if task.initial_prompt is not None: - text = task.initial_prompt - elif task.initial_prompt_file is not None: - text = (task_file.parent / task.initial_prompt_file).read_text(encoding="utf-8") - else: - # is_none_agent or simulation-only tasks carry no prompt at all — export - # the description instead so instruction.md is never empty. Harbor's own - # Task._validate_tests only requires the FILE to exist, not any content. - text = task.description + return task.initial_prompt + if task.initial_prompt_file is not None: + return (task_file.parent / task.initial_prompt_file).read_text(encoding="utf-8") + # is_none_agent or simulation-only tasks carry no prompt at all — fall back + # to the description so instruction.md is never empty. Harbor's own + # Task._validate_tests only requires the FILE to exist, not any content. + return task.description + + +def _write_instruction(task: TaskDefinition, task_file: Path, out_dir: Path) -> None: + text = _resolve_prompt_text(task, task_file) if not text.endswith("\n"): text += "\n" # POSIX text-file convention; also what the repo's own pre-commit end-of-file-fixer enforces (out_dir / "instruction.md").write_text(text, encoding="utf-8") @@ -222,6 +245,7 @@ def _write_environment( env_dir = out_dir / "environment" env_dir.mkdir(parents=True, exist_ok=True) docker_cfg = task.sandbox.docker + _write_agent_phase_task_yaml(task, env_dir, initial_prompt=_resolve_prompt_text(task, task_file)) if docker_cfg.dockerfile_path is not None: source_dockerfile = Path(docker_cfg.dockerfile_path) # already absolute — load_task resolves it @@ -235,6 +259,8 @@ def _write_environment( f"environment/Dockerfile declared no WORKDIR; appended `WORKDIR {workdir}` so the " + "verifier and agent phases agree on a path." ) + with dest_dockerfile.open("a", encoding="utf-8") as fh: + fh.write(f"\nCOPY agent_task.yaml {AGENT_TASK_YAML_PATH}\n") if not _from_line_mentions_coder_eval_agent(dest_dockerfile): warnings.append(_MISSING_CODER_EVAL_WARNING) # Non-Dockerfile build context (COPY sources etc.) is not carried over in @@ -255,6 +281,12 @@ def _write_environment( # operator can see which image the exported task will pull. if "coder-eval-agent" not in docker_cfg.image: warnings.append(_MISSING_CODER_EVAL_WARNING) + warnings.append( + f"No Dockerfile to bake environment/agent_task.yaml into (pre-built image {docker_cfg.image!r}) — " + + f"a `CoderEvalAgent` Harbor agent embed needs the task at {AGENT_TASK_YAML_PATH} inside the image. " + + "Add a dockerfile_path that extends this image with `COPY agent_task.yaml " + + f"{AGENT_TASK_YAML_PATH}` if you intend to use it as the agent phase, not just the verifier." + ) if docker_cfg.working_dir is not None: return docker_cfg.working_dir inspected = _inspect_image_workdir(docker_cfg.image) @@ -349,6 +381,49 @@ def _write_verifier_task_yaml(task: TaskDefinition, out_dir: Path) -> None: ) +def _write_agent_phase_task_yaml(task: TaskDefinition, env_dir: Path, *, initial_prompt: str) -> None: + """``environment/agent_task.yaml`` — the REAL agent config, but criteria-free. + + Baked into the image at :data:`AGENT_TASK_YAML_PATH` (see the Dockerfile + ``COPY`` line in ``_write_environment``) for a ``CoderEvalAgent`` Harbor + agent embed (``harbor/agent.py``) to run via ``coder-eval execute --format + harbor``. Unlike ``tests/task.yaml`` (the verifier's placeholder-agent, + real-criteria file), this is the mirror image: ``task.agent`` and the + resolved prompt are carried over VERBATIM (the whole point is running the + task's actual configured agent), but ``success_criteria`` is replaced with + :data:`_AGENT_PHASE_PLACEHOLDER_CRITERION` — never read (`coder-eval + execute` never grades) and never leaked into the agent-visible image. + + ``sandbox.driver`` is forced to ``tempdir`` regardless of the original + task's driver: this file runs INSIDE the container Harbor already built, + so re-declaring ``driver: docker`` here would have ``coder-eval execute`` + try to launch a second, nested container rather than just using its own + in-process sandbox at the container's current workdir. + + ``initial_prompt`` is omitted entirely for a ``type: none`` (agentless) + task: coder-eval's own schema forbids a no-op agent from setting a prompt + (no agent ever runs to read it) and raises at load time otherwise -- + verified live via ``harbor run`` against a real ``harbor`` install, which + surfaced exactly this ``TaskDefinition`` validation error before this + guard was added. + """ + is_agentless = task.agent is not None and task.agent.type == "none" + payload: dict[str, object] = { + "task_id": task.task_id, + "description": task.description, + "agent": ( + task.agent.model_dump(mode="json", exclude_none=True) if task.agent is not None else {"type": "claude-code"} + ), + "sandbox": {"driver": "tempdir"}, + "success_criteria": [dict(_AGENT_PHASE_PLACEHOLDER_CRITERION)], + } + if not is_agentless: + payload["initial_prompt"] = initial_prompt + (env_dir / "agent_task.yaml").write_text( + yaml.safe_dump(payload, sort_keys=False, allow_unicode=True), encoding="utf-8" + ) + + def _write_test_sh(out_dir: Path, *, workdir: str) -> None: path = out_dir / "tests" / "test.sh" path.write_text(_TEST_SH_TEMPLATE.format(workdir=workdir), encoding="utf-8") diff --git a/tests/_fixtures/harbor_export_golden/expected/environment/Dockerfile b/tests/_fixtures/harbor_export_golden/expected/environment/Dockerfile index 6f2be053..dceeff2b 100644 --- a/tests/_fixtures/harbor_export_golden/expected/environment/Dockerfile +++ b/tests/_fixtures/harbor_export_golden/expected/environment/Dockerfile @@ -2,3 +2,5 @@ FROM ubuntu:24.04 RUN apt-get update && apt-get install -y --no-install-recommends coreutils && rm -rf /var/lib/apt/lists/* WORKDIR /app + +COPY agent_task.yaml /opt/coder-eval-task/task.yaml diff --git a/tests/_fixtures/harbor_export_golden/expected/environment/agent_task.yaml b/tests/_fixtures/harbor_export_golden/expected/environment/agent_task.yaml new file mode 100644 index 00000000..3e0dd715 --- /dev/null +++ b/tests/_fixtures/harbor_export_golden/expected/environment/agent_task.yaml @@ -0,0 +1,17 @@ +task_id: golden_export_example +description: A canonical task exercising every C2 mapping row at once (golden fixture + — do not edit casually). +agent: + type: claude-code + permission_mode: acceptEdits + ignore_patterns: [] + system_prompt_mode: append + sdk_options: {} +sandbox: + driver: tempdir +success_criteria: +- type: file_exists + path: . + description: (unused placeholder — this file is run via `coder-eval execute`, which + never checks criteria) +initial_prompt: Write 'hello world' to greeting.txt, matching reference/greeting.txt. diff --git a/tests/fixtures/atif/known_good_trajectory.json b/tests/fixtures/atif/known_good_trajectory.json new file mode 100644 index 00000000..6a7b013f --- /dev/null +++ b/tests/fixtures/atif/known_good_trajectory.json @@ -0,0 +1,125 @@ +{ + "schema_version": "ATIF-v1.7", + "session_id": "hello_date_smoke_test/default", + "agent": { + "name": "claude-code", + "version": "0.8.4", + "model_name": "claude-sonnet-4-6" + }, + "steps": [ + { + "step_id": 1, + "timestamp": "2026-07-20T10:00:00+00:00", + "source": "user", + "message": "Create a Python file named app.py that prints hello.", + "extra": {"iteration": 1} + }, + { + "step_id": 2, + "timestamp": "2026-07-20T10:00:05+00:00", + "source": "agent", + "model_name": "claude-sonnet-4-6", + "message": "I'll create app.py now.", + "reasoning_content": "The user wants a simple script; Write then run it.", + "tool_calls": [ + { + "tool_call_id": "toolu_01", + "function_name": "Write", + "arguments": {"file_path": "app.py", "content": "print('hello')"} + } + ], + "observation": { + "results": [ + { + "source_call_id": "toolu_01", + "content": "File created successfully at: app.py", + "extra": {"result_status": "success", "duration_ms": 12.5} + } + ] + }, + "metrics": { + "prompt_tokens": 2048, + "completion_tokens": 150, + "cached_tokens": 1024 + }, + "extra": {"iteration": 1, "cache_creation_tokens": 512, "reasoning_tokens": 40} + }, + { + "step_id": 3, + "timestamp": "2026-07-20T10:00:09+00:00", + "source": "agent", + "model_name": "claude-sonnet-4-6", + "message": [ + {"type": "text", "text": "Delegating verification to a sub-agent."} + ], + "tool_calls": [ + { + "tool_call_id": "toolu_02", + "function_name": "Agent", + "arguments": {"prompt": "Verify app.py runs"} + } + ], + "observation": { + "results": [ + { + "source_call_id": "toolu_02", + "content": "Verified: app.py prints hello", + "subagent_trajectory_ref": [ + {"trajectory_id": "toolu_02"} + ] + } + ] + }, + "metrics": { + "prompt_tokens": 2300, + "completion_tokens": 60, + "cached_tokens": 2048 + }, + "extra": {"iteration": 1} + } + ], + "final_metrics": { + "total_prompt_tokens": 4948, + "total_completion_tokens": 260, + "total_cached_tokens": 3472, + "total_cost_usd": 0.0421, + "total_steps": 3 + }, + "extra": { + "reconciliation": [ + {"iteration": 1, "input_tokens": 512, "output_tokens": 0, "cache_creation_tokens": 0, "cache_read_tokens": 400, "note": "prompt slice billed on no streamed message"} + ] + }, + "subagent_trajectories": [ + { + "schema_version": "ATIF-v1.7", + "trajectory_id": "toolu_02", + "agent": { + "name": "claude-code", + "version": "0.8.4", + "model_name": "claude-haiku-4-5" + }, + "steps": [ + { + "step_id": 1, + "source": "agent", + "model_name": "claude-haiku-4-5", + "message": "Running app.py to verify.", + "tool_calls": [ + { + "tool_call_id": "toolu_sub_01", + "function_name": "Bash", + "arguments": {"command": "python app.py"} + } + ], + "observation": { + "results": [ + {"source_call_id": "toolu_sub_01", "content": "hello"} + ] + }, + "metrics": {"prompt_tokens": 900, "completion_tokens": 50, "cached_tokens": 0} + } + ] + } + ] +} diff --git a/tests/test_atif_emit.py b/tests/test_atif_emit.py new file mode 100644 index 00000000..d932b0c3 --- /dev/null +++ b/tests/test_atif_emit.py @@ -0,0 +1,366 @@ +"""Tests for the EvaluationResult → ATIF Trajectory converter (atif_emit).""" + +import json +from datetime import UTC, datetime + +from coder_eval.harbor.atif_emit import evaluation_result_to_trajectory, write_trajectory_json +from coder_eval.harbor.atif_models import Trajectory +from coder_eval.models import ( + AssistantMessage, + CommandTelemetry, + ContentBlock, + EvaluationResult, + FinalStatus, + ReconciliationMessage, + TokenUsage, + TurnRecord, + UserMessage, +) +from coder_eval.path_utils import write_text_atomic + + +T0 = datetime(2026, 7, 20, 10, 0, 0, tzinfo=UTC) + + +def _assistant( + text: str = "working on it", + *, + thinking: str | None = None, + parent_tool_use_id: str | None = None, + input_tokens: int = 100, + output_tokens: int = 50, + cache_creation: int = 20, + cache_read: int = 30, + model: str = "claude-sonnet-4-6", +) -> AssistantMessage: + blocks = [ContentBlock(block_type="text", sequence=0, text=text)] + if thinking is not None: + blocks.insert(0, ContentBlock(block_type="thinking", sequence=0, thinking=thinking)) + return AssistantMessage( + started_at=T0, + completed_at=T0, + generation_duration_ms=1000.0, + content_blocks=blocks, + input_tokens=input_tokens, + output_tokens=output_tokens, + cache_creation_tokens=cache_creation, + cache_read_tokens=cache_read, + model=model, + parent_tool_use_id=parent_tool_use_id, + ) + + +def _cmd(tool_id: str, *, index: int | None, tool_name: str = "Bash", command: str = "ls") -> CommandTelemetry: + return CommandTelemetry( + tool_name=tool_name, + tool_id=tool_id, + timestamp=T0, + parameters={"command": command}, + result_status="success", + result_summary=f"output of {tool_id}", + assistant_turn_index=index, + sequence_number=0, + ) + + +def _turn( + *, + iteration: int = 1, + user_input: str = "do the task", + agent_output: str = "done", + messages: list | None = None, + commands: list[CommandTelemetry] | None = None, + token_usage: TokenUsage | None = None, + crashed: bool = False, + crash_reason: str | None = None, +) -> TurnRecord: + return TurnRecord( + iteration=iteration, + user_input=user_input, + agent_output=agent_output, + messages=messages or [], + commands=commands or [], + token_usage=token_usage, + crashed=crashed, + crash_reason=crash_reason, + ) + + +def _result(turns: list[TurnRecord], *, total_token_usage: TokenUsage | None = None) -> EvaluationResult: + return EvaluationResult( + task_id="atif_emit_test", + task_description="converter test", + variant_id="default", + agent_type="claude-code", + model_used="claude-sonnet-4-6", + started_at=T0, + final_status=FinalStatus.SUCCESS, + iteration_count=len(turns), + iterations=turns, + total_token_usage=total_token_usage, + ) + + +class TestHappyPath: + def test_user_step_then_agent_steps_with_tool_attribution(self): + messages = [_assistant("first gen"), _assistant("second gen")] + commands = [_cmd("toolu_a", index=0), _cmd("toolu_b", index=1), _cmd("toolu_c", index=1)] + result = _result([_turn(messages=messages, commands=commands)]) + + t = evaluation_result_to_trajectory(result) + assert t is not None + # Synthetic user step (no UserMessage in stream) + 2 agent steps. + assert [s.source for s in t.steps] == ["user", "agent", "agent"] + assert [s.step_id for s in t.steps] == [1, 2, 3] + assert t.steps[0].message == "do the task" + # Commands attached to their own generation. + assert [tc.tool_call_id for tc in t.steps[1].tool_calls] == ["toolu_a"] + assert [tc.tool_call_id for tc in t.steps[2].tool_calls] == ["toolu_b", "toolu_c"] + # Observation joins by source_call_id within the same step (validator-checked). + assert t.steps[2].observation.results[0].source_call_id == "toolu_b" + assert t.steps[2].observation.results[0].content == "output of toolu_b" + assert t.session_id == "atif_emit_test/default" + assert t.agent.name == "claude-code" + assert t.agent.version # coder_eval __version__, non-empty + + def test_metrics_mapping_uses_token_usage_derivation(self): + messages = [_assistant(input_tokens=100, output_tokens=50, cache_creation=20, cache_read=30)] + t = evaluation_result_to_trajectory(_result([_turn(messages=messages)])) + m = t.steps[1].metrics + # prompt = uncached(100) + cache_creation(20) + cache_read(30) — via TokenUsage.input_tokens. + assert m.prompt_tokens == 150 + assert m.completion_tokens == 50 + assert m.cached_tokens == 30 + # Finer split preserved in extra. + assert t.steps[1].extra["cache_creation_tokens"] == 20 + + def test_reasoning_content_from_thinking_blocks(self): + messages = [_assistant("answer", thinking="let me think")] + t = evaluation_result_to_trajectory(_result([_turn(messages=messages)])) + assert t.steps[1].reasoning_content == "let me think" + assert t.steps[1].message == "answer" + + def test_explicit_user_message_becomes_user_step(self): + messages = [UserMessage(text="simulated utterance", completed_at=T0), _assistant("reply")] + t = evaluation_result_to_trajectory(_result([_turn(messages=messages)])) + # Explicit UserMessage → no synthetic user step is added. + assert [s.source for s in t.steps] == ["user", "agent"] + assert t.steps[0].message == "simulated utterance" + assert t.steps[0].timestamp == T0.isoformat() + + +class TestSubagentNesting: + def test_subagent_generations_nest_not_flatten(self): + spawn = _cmd("toolu_task", index=0, tool_name="Agent", command="") + messages = [ + _assistant("spawning a sub-agent"), + _assistant("sub work 1", parent_tool_use_id="toolu_task"), + _assistant("sub work 2", parent_tool_use_id="toolu_task"), + _assistant("main continues"), + ] + sub_cmd = _cmd("toolu_sub", index=1) # index 1 = first sub-agent generation + result = _result([_turn(messages=messages, commands=[spawn, sub_cmd])]) + + t = evaluation_result_to_trajectory(result) + # Main thread: user + 2 main agent steps only. + assert [s.source for s in t.steps] == ["user", "agent", "agent"] + assert t.steps[1].message == "spawning a sub-agent" + assert t.steps[2].message == "main continues" + # One embedded child, re-indexed from 1, carrying the sub-agent's command. + assert len(t.subagent_trajectories) == 1 + child = t.subagent_trajectories[0] + assert child.trajectory_id == "toolu_task" + assert [s.step_id for s in child.steps] == [1, 2] + assert child.steps[0].tool_calls[0].tool_call_id == "toolu_sub" + assert child.agent.version == t.agent.version + # The spawning call's observation references the child. + spawn_result = next(r for r in t.steps[1].observation.results if r.source_call_id == "toolu_task") + assert spawn_result.subagent_trajectory_ref[0].trajectory_id == "toolu_task" + + def test_orphan_subagent_group_embeds_without_ref(self): + # Sub-agent messages whose spawning tool call is not in the telemetry. + messages = [_assistant("main"), _assistant("orphan sub", parent_tool_use_id="toolu_ghost")] + t = evaluation_result_to_trajectory(_result([_turn(messages=messages)])) + assert t.subagent_trajectories[0].trajectory_id == "toolu_ghost" + # No fabricated tool call / ref on the main thread. + for step in t.steps: + assert not step.tool_calls or all(tc.tool_call_id != "toolu_ghost" for tc in step.tool_calls) + + +class TestGenerationlessTurn: + def test_user_message_and_reconciliation_survive_without_generations(self): + """A turn whose stream has user/reconciliation entries but NO assistant + generations (e.g. a simulation turn that crashed before the first + generation) must not fall into the legacy path — the UserMessage still + becomes a user step and the residual is still recorded.""" + messages = [ + UserMessage(text="simulated ask", completed_at=T0), + ReconciliationMessage(input_tokens=42, note="billed, never streamed"), + ] + t = evaluation_result_to_trajectory(_result([_turn(messages=messages, agent_output="")])) + assert [s.source for s in t.steps] == ["user"] + assert t.steps[0].message == "simulated ask" + assert t.extra["reconciliation"][0]["input_tokens"] == 42 + + +class TestReconciliation: + def test_reconciliation_never_a_step_and_totals_match(self): + total = TokenUsage( + uncached_input_tokens=612, + cache_creation_input_tokens=20, + cache_read_input_tokens=430, + output_tokens=50, + total_cost_usd=0.05, + ) + messages = [ + _assistant(input_tokens=100, output_tokens=50, cache_creation=20, cache_read=30), + ReconciliationMessage(input_tokens=512, cache_read_tokens=400, note="prompt slice"), + ] + result = _result( + [_turn(messages=messages, token_usage=total)], + total_token_usage=total, + ) + t = evaluation_result_to_trajectory(result) + assert all(s.source != "reconciliation" for s in t.steps) # not representable anyway + assert len(t.steps) == 2 # user + one generation + # Residual recorded at root. + recon = t.extra["reconciliation"] + assert recon == [ + { + "iteration": 1, + "input_tokens": 512, + "output_tokens": 0, + "cache_creation_tokens": 0, + "cache_read_tokens": 400, + "note": "prompt slice", + } + ] + # FinalMetrics ≡ authoritative total (which already includes the residual). + fm = t.final_metrics + assert fm.total_prompt_tokens == total.input_tokens == 1062 + assert fm.total_completion_tokens == 50 + assert fm.total_cached_tokens == 430 + assert fm.total_cost_usd == 0.05 + assert fm.total_steps == 2 + + def test_final_metrics_falls_back_to_summing_turns(self): + u1 = TokenUsage(uncached_input_tokens=10, output_tokens=5) + u2 = TokenUsage(uncached_input_tokens=20, output_tokens=15) + result = _result( + [ + _turn(iteration=1, messages=[_assistant()], token_usage=u1), + _turn(iteration=2, messages=[_assistant()], token_usage=u2), + ], + ) + t = evaluation_result_to_trajectory(result) + assert t.final_metrics.total_prompt_tokens == 30 + assert t.final_metrics.total_completion_tokens == 20 + + def test_no_usage_anywhere_omits_final_metrics(self): + t = evaluation_result_to_trajectory(_result([_turn(messages=[_assistant()])])) + # per-message metrics exist but no turn/total usage → final_metrics omitted. + assert t.final_metrics is None + + +class TestLegacyAndEdgeCases: + def test_legacy_turn_without_messages(self): + usage = TokenUsage(uncached_input_tokens=100, cache_read_input_tokens=40, output_tokens=25) + commands = [_cmd("toolu_x", index=None)] + result = _result([_turn(messages=[], commands=commands, token_usage=usage)]) + t = evaluation_result_to_trajectory(result) + assert [s.source for s in t.steps] == ["user", "agent"] + agent_step = t.steps[1] + assert agent_step.message == "done" + assert agent_step.metrics.prompt_tokens == 140 + assert agent_step.tool_calls[0].tool_call_id == "toolu_x" + + def test_unattributed_command_rides_last_main_agent_step(self): + messages = [_assistant("gen 1"), _assistant("gen 2")] + commands = [_cmd("toolu_none", index=None), _cmd("toolu_oob", index=99)] + t = evaluation_result_to_trajectory(_result([_turn(messages=messages, commands=commands)])) + last_agent = t.steps[-1] + ids = [tc.tool_call_id for tc in last_agent.tool_calls] + assert ids == ["toolu_none", "toolu_oob"] + + def test_leftover_commands_stay_within_their_turn(self): + """A turn with a stream but no main-thread agent step must synthesize its + own agent step for leftover commands — never attach them to a previous + iteration's step (which would mislabel the command's iteration).""" + turn1 = _turn(iteration=1, messages=[_assistant("gen 1")]) + turn2 = _turn( + iteration=2, + user_input="follow-up", + agent_output="crashed early", + messages=[UserMessage(text="follow-up", completed_at=T0)], + commands=[_cmd("toolu_orphan", index=None)], + ) + t = evaluation_result_to_trajectory(_result([turn1, turn2])) + # Turn 1's agent step must NOT have absorbed turn 2's command. + turn1_agent = next(s for s in t.steps if s.source == "agent" and s.extra["iteration"] == 1) + assert not turn1_agent.tool_calls + # Turn 2 synthesized its own agent step carrying the command. + turn2_agent = next(s for s in t.steps if s.source == "agent" and s.extra["iteration"] == 2) + assert [tc.tool_call_id for tc in turn2_agent.tool_calls] == ["toolu_orphan"] + assert turn2_agent.message == "crashed early" + + def test_multi_iteration_sequential_step_ids_and_user_steps(self): + result = _result( + [ + _turn(iteration=1, user_input="first ask", messages=[_assistant("a")]), + _turn(iteration=2, user_input="feedback", messages=[_assistant("b")]), + ] + ) + t = evaluation_result_to_trajectory(result) + assert [s.step_id for s in t.steps] == [1, 2, 3, 4] + assert [s.source for s in t.steps] == ["user", "agent", "user", "agent"] + assert t.steps[2].message == "feedback" + assert t.steps[2].extra["iteration"] == 2 + + def test_crashed_turn_marks_steps(self): + result = _result([_turn(messages=[_assistant("partial work")], crashed=True, crash_reason="timeout")]) + t = evaluation_result_to_trajectory(result) + assert t.steps[0].extra["crashed"] is True + assert t.steps[0].extra["crash_reason"] == "timeout" # first step of the turn only + assert t.steps[1].extra["crashed"] is True + assert "crash_reason" not in t.steps[1].extra + + def test_empty_result_returns_none(self): + assert evaluation_result_to_trajectory(_result([])) is None + + def test_converter_does_not_mutate_input(self): + result = _result([_turn(messages=[_assistant()], commands=[_cmd("toolu_a", index=0)])]) + before = result.model_dump_json() + evaluation_result_to_trajectory(result) + assert result.model_dump_json() == before + + +class TestWriteTrajectoryJson: + def test_writes_valid_atif_json(self, tmp_path): + result = _result([_turn(messages=[_assistant()])]) + path = tmp_path / "trajectory.json" + written = write_trajectory_json(result, path) + assert written == path + parsed = Trajectory.model_validate(json.loads(path.read_text(encoding="utf-8"))) + assert parsed.session_id == "atif_emit_test/default" + assert not path.with_suffix(".json.tmp").exists() + + def test_zero_step_result_writes_nothing(self, tmp_path): + path = tmp_path / "trajectory.json" + assert write_trajectory_json(_result([]), path) is None + assert not path.exists() + + def test_converter_exception_swallowed(self, tmp_path, monkeypatch): + import coder_eval.harbor.atif_emit as emit + + monkeypatch.setattr(emit, "evaluation_result_to_trajectory", lambda _: 1 / 0) + path = tmp_path / "trajectory.json" + assert emit.write_trajectory_json(_result([_turn()]), path) is None + assert not path.exists() + + +class TestAtomicWriteText: + def test_writes_and_leaves_no_tmp(self, tmp_path): + path = tmp_path / "out.json" + write_text_atomic(path, '{"a": 1}') + assert path.read_text(encoding="utf-8") == '{"a": 1}' + assert list(tmp_path.iterdir()) == [path] diff --git a/tests/test_atif_hydrate.py b/tests/test_atif_hydrate.py new file mode 100644 index 00000000..090e196a --- /dev/null +++ b/tests/test_atif_hydrate.py @@ -0,0 +1,138 @@ +"""Round-trip tests: EvaluationResult -> ATIF Trajectory -> TurnRecords -> criteria. + +Confirms a trajectory produced OUTSIDE this process (a Harbor agent's +``coder-eval execute --format harbor``) can still be graded by +``coder-eval evaluate --format harbor`` — the criteria that key off +``turn_records`` (``command_executed``, ``cli_called``, ``commands_efficiency``, +``skill_triggered``) must see the same commands after the ATIF round-trip. +""" + +import asyncio +from datetime import UTC, datetime + +from coder_eval.criteria.base import CheckContext +from coder_eval.criteria.command_executed import CommandExecutedChecker +from coder_eval.criteria.commands_efficiency import CommandsEfficiencyChecker +from coder_eval.criteria.skill_triggered import SkillTriggeredChecker +from coder_eval.harbor.atif_emit import evaluation_result_to_trajectory +from coder_eval.harbor.atif_hydrate import seed_from_atif_trajectory, trajectory_to_turn_records +from coder_eval.models import ( + AssistantMessage, + CommandExecutedCriterion, + CommandsEfficiencyCriterion, + CommandTelemetry, + ContentBlock, + EvaluationResult, + FinalStatus, + SkillTriggeredCriterion, + TurnRecord, +) + + +T0 = datetime(2026, 9, 10, 12, 0, 0, tzinfo=UTC) + + +def _assistant(text: str = "working", **kwargs) -> AssistantMessage: + return AssistantMessage( + started_at=T0, + completed_at=T0, + generation_duration_ms=100.0, + content_blocks=[ContentBlock(block_type="text", sequence=0, text=text)], + **kwargs, + ) + + +def _cmd( + tool_id: str, tool_name: str, *, index: int, parameters: dict | None = None, command: str = "ls" +) -> CommandTelemetry: + return CommandTelemetry( + tool_name=tool_name, + tool_id=tool_id, + timestamp=T0, + parameters=parameters if parameters is not None else {"command": command}, + result_status="success", + result_summary=f"ran {command}", + assistant_turn_index=index, + sequence_number=index, + ) + + +def _original_result() -> EvaluationResult: + turn = TurnRecord( + iteration=1, + user_input="do the task", + agent_output="done", + messages=[_assistant("running Bash"), _assistant("running Skill")], + commands=[ + _cmd("toolu_a", "Bash", index=0, command="uv run pytest"), + _cmd("toolu_b", "Skill", index=1, parameters={"skill": "my-skill"}), + ], + ) + return EvaluationResult( + task_id="atif_hydrate_test", + task_description="round-trip test", + variant_id="default", + agent_type="claude-code", + model_used="claude-sonnet-5", + started_at=T0, + final_status=FinalStatus.SUCCESS, + iteration_count=1, + iterations=[turn], + ) + + +class TestRoundTrip: + def test_commands_survive_emit_then_hydrate(self): + original = _original_result() + trajectory = evaluation_result_to_trajectory(original) + assert trajectory is not None + + turns = trajectory_to_turn_records(trajectory) + all_commands = [cmd for turn in turns for cmd in turn.commands] + assert {cmd.tool_id for cmd in all_commands} == {"toolu_a", "toolu_b"} + assert {cmd.tool_name for cmd in all_commands} == {"Bash", "Skill"} + + def test_seed_from_atif_trajectory_builds_valid_result(self): + trajectory = evaluation_result_to_trajectory(_original_result()) + seeded = seed_from_atif_trajectory(trajectory, task_id="atif_hydrate_test") + assert seeded.iteration_count == len(seeded.iterations) + assert seeded.agent_type == "claude-code" + + +class TestCriteriaAgainstHydratedTrajectory: + def _turn_records(self) -> list[TurnRecord]: + trajectory = evaluation_result_to_trajectory(_original_result()) + assert trajectory is not None + return trajectory_to_turn_records(trajectory) + + def test_command_executed_matches_hydrated_bash_call(self): + turn_records = self._turn_records() + checker = CommandExecutedChecker() + criterion = CommandExecutedCriterion( + description="ran pytest", + tool_name="Bash", + command_pattern="pytest", + min_count=1, + ) + result = asyncio.run( + checker.check_async(criterion, sandbox=None, turn_records=turn_records, context=CheckContext()) + ) + assert result.score == 1.0 + + def test_commands_efficiency_counts_hydrated_commands(self): + turn_records = self._turn_records() + checker = CommandsEfficiencyChecker() + criterion = CommandsEfficiencyCriterion(description="efficiency", expected_commands=2) + result = asyncio.run( + checker.check_async(criterion, sandbox=None, turn_records=turn_records, context=CheckContext()) + ) + assert result.score == 1.0 + + def test_skill_triggered_sees_hydrated_skill_call(self): + turn_records = self._turn_records() + checker = SkillTriggeredChecker() + criterion = SkillTriggeredCriterion(description="skill", skill_name="my-skill", expected_skill="my-skill") + result = asyncio.run( + checker.check_async(criterion, sandbox=None, turn_records=turn_records, context=CheckContext()) + ) + assert result.score == 1.0 diff --git a/tests/test_atif_models.py b/tests/test_atif_models.py new file mode 100644 index 00000000..a5640f51 --- /dev/null +++ b/tests/test_atif_models.py @@ -0,0 +1,162 @@ +"""Tests for the vendored ATIF models (coder_eval.harbor.atif_models). + +Fixture compatibility guarantee: ``tests/fixtures/atif/known_good_trajectory.json`` +was validated ONCE against the real harbor package (harbor==0.20.0) and then +frozen — CI never installs harbor. Reproducible re-validation procedure: + + python3 -m venv /tmp/harbor-check && /tmp/harbor-check/bin/pip install -q 'harbor==0.20.0' + /tmp/harbor-check/bin/python -c "from harbor.models.trajectories.trajectory import Trajectory; \\ + import json; Trajectory.model_validate(json.load(open('tests/fixtures/atif/known_good_trajectory.json'))); \\ + print('OK')" + rm -rf /tmp/harbor-check + +Last validated: harbor 0.20.0 (2026-07-20). If the vendored models and this +fixture ever disagree with harbor, re-run the procedure and reconcile. +""" + +import json +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from coder_eval.harbor.atif_models import ( + AtifAgent, + ContentPart, + Observation, + ObservationResult, + Step, + SubagentTrajectoryRef, + ToolCall, + Trajectory, +) + + +FIXTURE = Path(__file__).parent / "fixtures" / "atif" / "known_good_trajectory.json" + + +def _agent() -> AtifAgent: + return AtifAgent(name="claude-code", version="1.0.0") + + +def _step(step_id: int, source: str = "agent", **kwargs) -> Step: + return Step(step_id=step_id, source=source, message=f"step {step_id}", **kwargs) + + +def _tool_step(step_id: int, tool_call_id: str = "toolu_01", source_call_id: str | None = "toolu_01") -> Step: + return Step( + step_id=step_id, + source="agent", + message="calling a tool", + tool_calls=[ToolCall(tool_call_id=tool_call_id, function_name="Bash", arguments={"command": "ls"})], + observation=Observation(results=[ObservationResult(source_call_id=source_call_id, content="ok")]), + ) + + +class TestHappyPath: + def test_two_step_trajectory_with_tool_call_round_trips(self): + t = Trajectory(agent=_agent(), steps=[_step(1, source="user"), _tool_step(2)]) + dumped = t.model_dump(exclude_none=True) + # exclude_none is how harbor serializes (Trajectory.to_json_dict). + reparsed = Trajectory.model_validate(dumped) + assert reparsed == t + + def test_defaults(self): + t = Trajectory(agent=_agent(), steps=[_step(1)]) + assert t.schema_version == "ATIF-v1.7" + assert t.trajectory_id is None + assert t.final_metrics is None + + +class TestValidators: + def test_empty_steps_rejected(self): + with pytest.raises(ValidationError, match="at least 1"): + Trajectory(agent=_agent(), steps=[]) + + def test_non_sequential_step_ids_rejected(self): + with pytest.raises(ValidationError, match=r"expected 2 \(sequential from 1\), got 3"): + Trajectory(agent=_agent(), steps=[_step(1), _step(3)]) + + def test_step_ids_must_start_at_one(self): + with pytest.raises(ValidationError, match=r"expected 1 \(sequential from 1\), got 2"): + Trajectory(agent=_agent(), steps=[_step(2)]) + + def test_cross_step_source_call_id_rejected(self): + # Step 2's observation references step 1's tool_call_id — invalid. + bad = _tool_step(2, tool_call_id="toolu_other", source_call_id="toolu_01") + with pytest.raises(ValidationError, match="source_call_id 'toolu_01'"): + Trajectory(agent=_agent(), steps=[_step(1, source="user"), bad]) + + def test_null_source_call_id_allowed(self): + step = _tool_step(1, source_call_id=None) + Trajectory(agent=_agent(), steps=[step]) # does not raise + + def test_subagent_without_trajectory_id_rejected(self): + sub = Trajectory(agent=_agent(), steps=[_step(1)]) + with pytest.raises(ValidationError, match="trajectory_id is required"): + Trajectory(agent=_agent(), steps=[_step(1)], subagent_trajectories=[sub]) + + def test_duplicate_subagent_trajectory_ids_rejected(self): + sub1 = Trajectory(agent=_agent(), steps=[_step(1)], trajectory_id="dup") + sub2 = Trajectory(agent=_agent(), steps=[_step(1)], trajectory_id="dup") + with pytest.raises(ValidationError, match="not unique"): + Trajectory(agent=_agent(), steps=[_step(1)], subagent_trajectories=[sub1, sub2]) + + def test_agent_version_required(self): + # Verified against harbor 0.20.0: Agent.version is REQUIRED, not optional. + with pytest.raises(ValidationError, match="version"): + AtifAgent(name="claude-code") # type: ignore[call-arg] + + def test_content_part_text_requires_text(self): + with pytest.raises(ValidationError, match="'text' field is required"): + ContentPart(type="text") + + def test_content_part_text_forbids_source(self): + with pytest.raises(ValidationError, match="'source' field is not allowed"): + ContentPart(type="text", text="hi", source={"media_type": "image/png"}) + + +class TestVersionTolerance: + @pytest.mark.parametrize("version", ["ATIF-v1.0", "ATIF-v1.7", "ATIF-v1.9", "ATIF-v1.42"]) + def test_any_v1_minor_accepted(self, version): + t = Trajectory(schema_version=version, agent=_agent(), steps=[_step(1)]) + assert t.schema_version == version + + @pytest.mark.parametrize("version", ["ATIF-v2.0", "ATIF-2.0", "v1.7", "garbage", ""]) + def test_non_v1_rejected(self, version): + with pytest.raises(ValidationError, match="schema_version"): + Trajectory(schema_version=version, agent=_agent(), steps=[_step(1)]) + + +class TestFrozenFixture: + def test_known_good_fixture_parses(self): + """The harbor-0.20.0-validated fixture must parse with the vendored models.""" + t = Trajectory.model_validate(json.loads(FIXTURE.read_text(encoding="utf-8"))) + assert t.schema_version == "ATIF-v1.7" + assert len(t.steps) == 3 + assert t.subagent_trajectories is not None + assert t.subagent_trajectories[0].trajectory_id == "toolu_02" + # Multimodal message variant (list of ContentPart) survives. + assert isinstance(t.steps[2].message, list) + # Sub-agent ref resolves against the embedded array. + ref = t.steps[2].observation.results[0].subagent_trajectory_ref + assert ref is not None and ref[0].trajectory_id == "toolu_02" + + def test_fixture_round_trips_through_vendored_models(self): + raw = json.loads(FIXTURE.read_text(encoding="utf-8")) + t = Trajectory.model_validate(raw) + assert Trajectory.model_validate(t.model_dump(exclude_none=True)) == t + + +class TestExtraForbid: + def test_unknown_field_rejected(self): + with pytest.raises(ValidationError): + Trajectory(agent=_agent(), steps=[_step(1)], not_a_field=1) # type: ignore[call-arg] + + def test_unknown_step_field_rejected(self): + with pytest.raises(ValidationError): + Step(step_id=1, source="agent", message="x", bogus=True) # type: ignore[call-arg] + + def test_subagent_ref_requires_trajectory_id(self): + with pytest.raises(ValidationError): + SubagentTrajectoryRef() # type: ignore[call-arg] diff --git a/tests/test_evaluate_format_harbor.py b/tests/test_evaluate_format_harbor.py new file mode 100644 index 00000000..1d21455d --- /dev/null +++ b/tests/test_evaluate_format_harbor.py @@ -0,0 +1,123 @@ +"""``coder-eval evaluate --format harbor`` — grade a workdir using ATIF trajectory context. + +End-to-end: build an ATIF trajectory.json by hand (the shape a Harbor agent's +``coder-eval execute --format harbor`` would produce), drop a file in a plain +workdir (standing in for what the Harbor agent's container left behind), and +grade it with both a workdir-only criterion (``file_exists``) and a +trajectory-dependent one (``command_executed``) to prove hydration actually +feeds the checker. +""" + +import json + +from typer.testing import CliRunner + +from coder_eval.cli import app +from coder_eval.harbor.atif_emit import evaluation_result_to_trajectory +from coder_eval.models import CommandTelemetry, EvaluationResult, FinalStatus, TurnRecord + + +runner = CliRunner() + +_TASK_YAML = """ +task_id: "atif_format_harbor_evaluate_test" +description: "evaluate --format harbor smoke test" +initial_prompt: "unused" +agent: + type: "claude-code" + +sandbox: + driver: "tempdir" + python: null + +success_criteria: + - type: "file_exists" + path: "test.txt" + description: "Test file must exist" + - type: "command_executed" + tool_name: "Bash" + command_pattern: "touch test.txt" + min_count: 1 + description: "The agent must have run touch" +""" + + +def _write_trajectory(tmp_path): + turn = TurnRecord( + iteration=1, + user_input="create test.txt", + agent_output="done", + commands=[ + CommandTelemetry( + tool_name="Bash", + tool_id="toolu_1", + timestamp="2026-09-10T00:00:00Z", + parameters={"command": "touch test.txt"}, + result_status="success", + result_summary="", + assistant_turn_index=0, + sequence_number=0, + ) + ], + ) + result = EvaluationResult( + task_id="atif_format_harbor_evaluate_test", + task_description="d", + agent_type="harbor-agent", + started_at="2026-09-10T00:00:00Z", + final_status=FinalStatus.NOT_GRADED, + iteration_count=1, + iterations=[turn], + ) + trajectory = evaluation_result_to_trajectory(result) + assert trajectory is not None + path = tmp_path / "trajectory.json" + path.write_text(trajectory.model_dump_json(exclude_none=True), encoding="utf-8") + return path + + +def test_evaluate_format_harbor_grades_with_hydrated_trajectory(tmp_path): + task_file = tmp_path / "task.yaml" + task_file.write_text(_TASK_YAML, encoding="utf-8") + + work_dir = tmp_path / "workdir" + work_dir.mkdir() + (work_dir / "test.txt").write_text("hi", encoding="utf-8") + + trajectory_path = _write_trajectory(tmp_path) + run_dir = tmp_path / "run" + + result = runner.invoke( + app, + [ + "evaluate", + str(task_file), + str(work_dir), + "--format", + "harbor", + "--trajectory", + str(trajectory_path), + "--run-dir", + str(run_dir), + ], + ) + assert result.exit_code == 0, result.output + + task_jsons = sorted(run_dir.glob("**/task.json")) + assert len(task_jsons) == 1 + graded = json.loads(task_jsons[0].read_text(encoding="utf-8")) + assert graded["weighted_score"] == 1.0 + scores = {r["criterion_type"]: r["score"] for r in graded["success_criteria_results"]} + assert scores["file_exists"] == 1.0 + assert scores["command_executed"] == 1.0 + + +def test_evaluate_format_harbor_requires_trajectory(tmp_path): + task_file = tmp_path / "task.yaml" + task_file.write_text(_TASK_YAML, encoding="utf-8") + work_dir = tmp_path / "workdir" + work_dir.mkdir() + + result = runner.invoke(app, ["evaluate", str(task_file), str(work_dir), "--format", "harbor"]) + assert result.exit_code != 0 + assert "--trajectory" in result.output diff --git a/tests/test_execute_format_harbor.py b/tests/test_execute_format_harbor.py new file mode 100644 index 00000000..76c7f12c --- /dev/null +++ b/tests/test_execute_format_harbor.py @@ -0,0 +1,83 @@ +"""``coder-eval execute --format harbor`` — writes a trajectory.json (ATIF) sibling for task.json.""" + +import json + +import pytest +from typer.testing import CliRunner + +from coder_eval.cli import app +from coder_eval.orchestrator import Orchestrator +from tests.fixtures.mock_agent import MockAgent + + +runner = CliRunner() + + +@pytest.fixture +def success_task(tmp_path): + task_content = """ +task_id: "atif_format_harbor_test" +description: "execute --format harbor smoke test" +initial_prompt: "Create a file named 'test.txt'" +agent: + type: "claude-code" + permission_mode: "acceptEdits" + +sandbox: + driver: "tempdir" + python: null + +success_criteria: + - type: "file_exists" + path: "test.txt" + description: "Test file must exist" +""" + task_file = tmp_path / "task.yaml" + task_file.write_text(task_content) + return task_file + + +@pytest.fixture +def mock_agent(monkeypatch): + async def _mock_create_agent(self): + return MockAgent(self.task, scenario="success") + + monkeypatch.setattr(Orchestrator, "_create_agent", _mock_create_agent) + + +def test_execute_format_harbor_writes_trajectory_json(tmp_path, success_task, mock_agent): + run_dir = tmp_path / "run" + result = runner.invoke( + app, + ["execute", str(success_task), "--run-dir", str(run_dir), "--format", "harbor"], + ) + assert result.exit_code == 0, result.output + + task_jsons = sorted(run_dir.glob("**/task.json")) + assert len(task_jsons) == 1 + trajectory_path = task_jsons[0].with_name("trajectory.json") + assert trajectory_path.exists() + + trajectory = json.loads(trajectory_path.read_text(encoding="utf-8")) + assert trajectory["schema_version"].startswith("ATIF-v1.") + assert len(trajectory["steps"]) >= 1 + + +def test_execute_without_format_does_not_write_trajectory_json(tmp_path, success_task, mock_agent): + run_dir = tmp_path / "run" + result = runner.invoke(app, ["execute", str(success_task), "--run-dir", str(run_dir)]) + assert result.exit_code == 0, result.output + + task_jsons = sorted(run_dir.glob("**/task.json")) + assert len(task_jsons) == 1 + assert not task_jsons[0].with_name("trajectory.json").exists() + + +def test_unknown_format_value_errors_cleanly(tmp_path, success_task): + run_dir = tmp_path / "run" + result = runner.invoke( + app, + ["execute", str(success_task), "--run-dir", str(run_dir), "--format", "nonsense"], + ) + assert result.exit_code != 0 + assert "Unsupported --format" in result.output diff --git a/tests/test_harbor_packager.py b/tests/test_harbor_packager.py index 359e0fed..e71a6308 100644 --- a/tests/test_harbor_packager.py +++ b/tests/test_harbor_packager.py @@ -203,6 +203,70 @@ def test_reference_comparison_survives_the_none_agent_trap(self, tmp_path: Path) assert (out_dir / "tests" / "reference" / "greeting.txt").read_text(encoding="utf-8") == "hello" +class TestAgentPhaseTaskYaml: + """environment/agent_task.yaml — the CoderEvalAgent embed's criteria-free real-agent config.""" + + def test_baked_at_fixed_path_via_dockerfile_copy(self, tmp_path: Path) -> None: + env_dir = tmp_path / "environment" + env_dir.mkdir() + (env_dir / "Dockerfile").write_text("FROM ubuntu:24.04\nWORKDIR /app\n", encoding="utf-8") + task_file = _write_task( + tmp_path, + {"sandbox": {"driver": "docker", "docker": {"dockerfile_path": "environment/Dockerfile"}}}, + ) + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + assert (out_dir / "environment" / "agent_task.yaml").exists() + dockerfile_text = (out_dir / "environment" / "Dockerfile").read_text(encoding="utf-8") + assert "COPY agent_task.yaml /opt/coder-eval-task/task.yaml" in dockerfile_text + + def test_carries_the_real_agent_config_but_no_real_criteria(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path, {"agent": {"type": "claude-code", "model": "claude-opus-5"}}) + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + emitted = yaml.safe_load((out_dir / "environment" / "agent_task.yaml").read_text(encoding="utf-8")) + assert emitted["agent"]["type"] == "claude-code" + assert emitted["agent"]["model"] == "claude-opus-5" + assert len(emitted["success_criteria"]) == 1 # the placeholder only -- never the real ones + assert emitted["success_criteria"][0]["type"] == "file_exists" + # The real task's success_criteria (2 entries in _BASE_TASK) must never leak in. + assert "reference_comparison" not in str(emitted["success_criteria"]) + + def test_sandbox_driver_forced_to_tempdir_no_nested_docker(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) # _BASE_TASK uses sandbox.driver: docker + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + emitted = yaml.safe_load((out_dir / "environment" / "agent_task.yaml").read_text(encoding="utf-8")) + assert emitted["sandbox"]["driver"] == "tempdir" + + def test_reloads_as_a_valid_task_definition(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + emitted = yaml.safe_load((out_dir / "environment" / "agent_task.yaml").read_text(encoding="utf-8")) + reloaded = TaskDefinition.model_validate(emitted) # must not raise + assert reloaded.task_id == "greet" + + def test_prebuilt_image_with_no_dockerfile_warns_it_cannot_bake_the_agent_file_in(self, tmp_path: Path) -> None: + task_file = _write_task( + tmp_path, {"sandbox": {"driver": "docker", "docker": {"image": "byod-custom-image:0.1.0"}}} + ) + out_dir = tmp_path / "out" + + result = export_task(task_file, out_dir) + + assert (out_dir / "environment" / "agent_task.yaml").exists() # still written -- just not bakeable in + assert any("No Dockerfile to bake" in w for w in result.warnings) + + class TestDockerfileWorkdirResolution: def test_dockerfile_with_no_workdir_gets_one_appended_and_warned(self, tmp_path: Path) -> None: env_dir = tmp_path / "environment" @@ -235,7 +299,11 @@ def test_dockerfile_with_an_existing_workdir_is_respected_and_not_touched(self, result = export_task(task_file, out_dir) assert result.workdir == "/workspace" - assert (out_dir / "environment" / "Dockerfile").read_text(encoding="utf-8") == original + dockerfile_text = (out_dir / "environment" / "Dockerfile").read_text(encoding="utf-8") + # The WORKDIR-bearing content is untouched; only the agent_task.yaml COPY + # line (baked in for a CoderEvalAgent embed, see agent_paths.py) is appended. + assert dockerfile_text.startswith(original) + assert "COPY agent_task.yaml /opt/coder-eval-task/task.yaml" in dockerfile_text assert not any("declared no WORKDIR" in w for w in result.warnings) # test.sh and task.toml must agree with the same resolved workdir. assert '"/workspace"' in (out_dir / "tests" / "test.sh").read_text(encoding="utf-8") @@ -308,7 +376,11 @@ def test_no_warning_when_prebuilt_image_names_coder_eval_agent( tmp_path, {"sandbox": {"driver": "docker", "docker": {"image": "coder-eval-agent:0.12.0"}}} ) result = export_task(task_file, tmp_path / "out") - assert not any("coder-eval-agent" in w for w in result.warnings) + # A prebuilt image (no dockerfile_path) always warns that agent_task.yaml + # can't be baked in (no Dockerfile to COPY into) -- that warning legitimately + # NAMES the image, so it is excluded from this substring check rather than + # the check being widened to also swallow the real regression it guards. + assert not any("coder-eval-agent" in w for w in result.warnings if "No Dockerfile to bake" not in w) class TestPrePostRunWarnings: From ce11980769a51ff066426bf5e6399a12f8b35cda Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Fri, 11 Sep 2026 10:56:21 -0700 Subject: [PATCH 05/12] feat(harbor): fix agent workspace alignment, env passthrough, and template sources Several fixes to the Harbor export/agent pipeline, each verified against a real `harbor==0.22.0` install with real Docker and (where relevant) real Bedrock credentials: - CoderEvalAgent now passes --workspace-dir "$(pwd)" to `coder-eval execute`, and BatchRunConfig/run_batch/run+execute CLIs gained a matching --workspace-dir flag (Orchestrator already supported this internally for DockerRunner). Without it, the agent phase wrote to a throwaway tempdir the verifier phase never looked at, silently scoring every criterion 0. - packager.py now emits [environment.env]/[verifier.env] in task.toml, derived from the resolved task's own sandbox.docker.env_passthrough (the same allowlist DockerRunner already uses) rather than requiring the operator to hand-edit task.toml. HOME is excluded (that entry only makes sense paired with DockerRunner's ~/.claude bind-mount). Values are `${VAR:-}` templates (empty default) so Harbor doesn't hard-require every var to be present. - environment/Dockerfile is now always written (synthesized from sandbox.docker.image when no dockerfile_path is set), so the CoderEvalAgent embed always has somewhere to COPY task.yaml into. - instruction.md is now a fixed placeholder pointing at environment/task.yaml and the CoderEvalAgent agent, instead of leaking the real prompt to a stock Harbor agent that isn't the one this export is built for. - environment/task.yaml's success_criteria is now a real `[]` instead of a placeholder criterion, now that TaskDefinition.validate_success_criteria (the "at least one criterion" check) has been removed. - TemplateDirSource directories are now copied into environment/templates/-/ with their path rewritten to the in-container copy; environment/task.yaml's sandbox block is field-merged (driver forced to tempdir) instead of being replaced wholesale, so python.env_packages/limits/etc. survive. Co-Authored-By: Claude Sonnet 5 --- src/coder_eval/cli/execute_command.py | 13 + src/coder_eval/cli/run_command.py | 23 ++ src/coder_eval/harbor/agent.py | 25 +- src/coder_eval/harbor/agent_paths.py | 14 +- src/coder_eval/harbor/packager.py | 285 +++++++++++++----- src/coder_eval/models/tasks.py | 8 - src/coder_eval/orchestration/batch.py | 14 + src/coder_eval/orchestration/config.py | 18 ++ src/coder_eval/orchestrator.py | 14 +- .../expected/environment/Dockerfile | 2 +- .../{agent_task.yaml => task.yaml} | 13 +- .../expected/instruction.md | 10 +- .../harbor_export_golden/expected/task.toml | 52 ++++ tests/test_harbor_experiment_packager.py | 10 +- tests/test_harbor_packager.py | 193 ++++++++++-- 15 files changed, 571 insertions(+), 123 deletions(-) rename tests/_fixtures/harbor_export_golden/expected/environment/{agent_task.yaml => task.yaml} (70%) diff --git a/src/coder_eval/cli/execute_command.py b/src/coder_eval/cli/execute_command.py index 2895dddf..8047dcdb 100644 --- a/src/coder_eval/cli/execute_command.py +++ b/src/coder_eval/cli/execute_command.py @@ -192,6 +192,18 @@ def execute_command( "`coder-eval execute --format harbor --run-dir ` invocation passes." ), ), + workspace_dir: Path | None = typer.Option( # noqa: B008 + None, + "--workspace-dir", + help=( + "Run the single resolved task's agent in-place at this absolute path instead of the " + "standard run_dir/artifacts workspace (copied out to run_dir/artifacts/ at " + "cleanup). Requires exactly one resolved task; refused for sandbox.driver: docker " + "(the docker driver already aligns automatically via sandbox.docker.working_dir). " + "Meant for a Harbor `CoderEvalAgent` invocation, so the agent's writes land at the " + "container's own WORKDIR, where Harbor's verifier phase looks for them." + ), + ), ) -> None: """Run evaluation tasks WITHOUT checking their success criteria. @@ -245,4 +257,5 @@ def execute_command( driver=driver, set_overrides=set_overrides, format=format, + workspace_dir=workspace_dir, ) diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py index 9cf40023..9c94e740 100644 --- a/src/coder_eval/cli/run_command.py +++ b/src/coder_eval/cli/run_command.py @@ -366,6 +366,18 @@ def run_command( "harbor --run-dir `, invoked by a Harbor agent." ), ), + workspace_dir: Path | None = typer.Option( # noqa: B008 + None, + "--workspace-dir", + help=( + "Run the single resolved task's agent in-place at this absolute path instead of the " + "standard run_dir/artifacts workspace (copied out to run_dir/artifacts/ at " + "cleanup). Requires exactly one resolved task; refused for sandbox.driver: docker " + "(the docker driver already aligns automatically via sandbox.docker.working_dir). " + "Meant for a Harbor `CoderEvalAgent` invocation, so the agent's writes land at the " + "container's own WORKDIR, where Harbor's verifier phase looks for them." + ), + ), ) -> None: """Run evaluation tasks (optionally in parallel). @@ -417,6 +429,7 @@ def run_command( driver=driver, set_overrides=set_overrides, format=format, + workspace_dir=workspace_dir, ) @@ -446,6 +459,7 @@ def run_pipeline( driver: str | None, set_overrides: list[str], format: str | None = None, + workspace_dir: Path | None = None, ) -> None: """The shared body of ``coder-eval run`` and ``coder-eval execute``. @@ -530,6 +544,7 @@ def run_pipeline( junit_xml=junit_xml, grade=grade, format=format, + workspace_dir=workspace_dir, ) ) except KeyboardInterrupt: @@ -558,6 +573,7 @@ async def _run_all_tasks( junit_xml: Path | None = None, grade: bool = True, format: str | None = None, + workspace_dir: Path | None = None, ) -> None: """Async entry point for running all tasks (optionally in parallel). @@ -589,6 +605,12 @@ async def _run_all_tasks( without knowing coder-eval's internal `///` nesting. Multi-task runs are left nested only — there is no single trajectory to promote. + workspace_dir: Run the single resolved task's agent in-place at this path + instead of run_dir/artifacts (see `BatchRunConfig.workspace_dir` and + `Orchestrator.workspace_dir`). Meant for a `CoderEvalAgent` invocation + inside a container someone else already built (Harbor's), so the + agent's writes land where that container's own verifier looks for + them, rather than in a throwaway tempdir the verifier never sees. """ # Prepare run directory run_dir = prepare_run_directory(run_dir) @@ -615,6 +637,7 @@ async def _run_all_tasks( verbose=verbose, include_skipped=include_skipped, grade=grade, + workspace_dir=workspace_dir, ) from ..telemetry import flush_telemetry, track_event diff --git a/src/coder_eval/harbor/agent.py b/src/coder_eval/harbor/agent.py index bba35e74..345cefe5 100644 --- a/src/coder_eval/harbor/agent.py +++ b/src/coder_eval/harbor/agent.py @@ -21,8 +21,13 @@ installed ``harbor`` package's ``populate_context_post_run``: it writes to ``self.logs_dir / "trajectory.json"`` on the HOST side, after the container syncs back). This class instead has coder-eval write it directly inside the - container at the mirrored path, so no separate "workspace_dir" plumbing is - needed for v1 (Gap 2's accepted workaround). + container at the mirrored path. +- ``--workspace-dir "$(pwd)"`` (Gap 2's real fix, not a workaround): without + it, ``coder-eval execute``'s own ``tempdir`` sandbox writes the agent's + workspace to a throwaway ``mkdtemp()`` elsewhere in the container, never + where Harbor's verifier phase (``tests/test.sh``) looks (the container's + ``WORKDIR``) — confirmed live, agent output was real but every criterion + scored 0 as "file does not exist". See ``run()``'s docstring. Verified against a real ``harbor==0.22.0`` install (``tmp/harbor-venv``): ``BaseAgent.name()`` is a ``@staticmethod``; ``run()`` returns ``None`` and @@ -101,11 +106,25 @@ async def run(self, instruction: str, environment: BaseEnvironment, context: Age both from the same source), so there is nothing to forward. Token/cost totals are filled in afterward by ``populate_context_post_run``, not here, matching every other installed agent's convention. + + ``--workspace-dir "$(pwd)"``: without it, ``coder-eval execute``'s own + ``tempdir`` sandbox (the agent-phase task.yaml forces ``driver: + tempdir`` — see ``packager._write_agent_phase_task_yaml``) writes the + agent's workspace to a fresh ``mkdtemp()`` elsewhere in the container, + NOT at the image's ``WORKDIR`` -- which is exactly where Harbor's own + verifier phase (``tests/test.sh``) looks for the agent's output. + ``$(pwd)`` is resolved by the container's shell at exec time, not by + this Python process, and equals the WORKDIR because ``environment.exec`` + is not given an explicit ``cwd`` (Docker execs default to the image's + configured WORKDIR). Confirmed live: without this flag the agent wrote + real output but the verifier scored every criterion 0 with "file does + not exist", because it never left the tempdir. """ del instruction, context # nothing to forward; context is populated post-run run_dir = self.environment_logs_dir.as_posix() command = ( - f"coder-eval execute {shlex.quote(AGENT_TASK_YAML_PATH)} --format harbor --run-dir {shlex.quote(run_dir)}" + f"coder-eval execute {shlex.quote(AGENT_TASK_YAML_PATH)} --format harbor " + f'--run-dir {shlex.quote(run_dir)} --workspace-dir "$(pwd)"' ) await self._exec(environment, command) diff --git a/src/coder_eval/harbor/agent_paths.py b/src/coder_eval/harbor/agent_paths.py index 708415f8..22d53744 100644 --- a/src/coder_eval/harbor/agent_paths.py +++ b/src/coder_eval/harbor/agent_paths.py @@ -1,7 +1,7 @@ """Fixed in-container path for the agent-phase task.yaml a Harbor export bakes into its image. Shared by ``packager.py`` (the writer — bakes a criteria-free copy of the task -into ``environment/agent_task.yaml`` and ``COPY``s it here) and ``agent.py``'s +into ``environment/task.yaml`` and ``COPY``s it here) and ``agent.py``'s ``CoderEvalAgent`` (the reader — the Harbor agent that runs ``coder-eval execute --format harbor`` against this exact path), so the two sides can never independently drift on where the file lives. Fixed rather than @@ -16,4 +16,14 @@ AGENT_TASK_YAML_PATH = "/opt/coder-eval-task/task.yaml" -__all__ = ["AGENT_TASK_YAML_PATH"] +AGENT_TASK_TEMPLATES_DIR = "/opt/coder-eval-task/templates" +"""Sibling of :data:`AGENT_TASK_YAML_PATH` for ``TemplateDirSource`` copies. A +``TemplateDirSource.path`` is resolved to an absolute HOST path at task-load +time (``task_loader.resolve_template_source_paths``) — that path does not +exist inside the container, so ``packager.py`` copies each source's directory +under here (``environment/templates/-/`` on the export side, ``COPY`` +'d into the image at build time) and rewrites ``environment/task.yaml``'s +``template_sources[].path`` to point at the in-container copy instead. +""" + +__all__ = ["AGENT_TASK_TEMPLATES_DIR", "AGENT_TASK_YAML_PATH"] diff --git a/src/coder_eval/harbor/packager.py b/src/coder_eval/harbor/packager.py index 0b72e5fc..2d891bec 100644 --- a/src/coder_eval/harbor/packager.py +++ b/src/coder_eval/harbor/packager.py @@ -10,9 +10,13 @@ / ├── task.toml - ├── instruction.md + ├── instruction.md # fixed placeholder -- the real prompt is in environment/task.yaml ├── environment/ - │ └── Dockerfile # copied from dockerfile_path, WORKDIR pinned + │ ├── Dockerfile # copied from dockerfile_path, or synthesized from + │ │ # sandbox.docker.image -- always written, WORKDIR pinned + │ ├── task.yaml # criteria-free copy for the CoderEvalAgent embed + │ └── templates/ # sandbox.template_sources' TemplateDirSource dirs, + │ # present only when the task has any (see agent_paths.py) └── tests/ ├── test.sh # C1.1's two-line shim ├── task.yaml # the criteria, as authored @@ -48,9 +52,9 @@ import tomli_w import yaml -from coder_eval.harbor.agent_paths import AGENT_TASK_YAML_PATH +from coder_eval.harbor.agent_paths import AGENT_TASK_TEMPLATES_DIR, AGENT_TASK_YAML_PATH from coder_eval.harbor.portability import PortabilityIssue, audit_criteria -from coder_eval.models import TaskDefinition +from coder_eval.models import TaskDefinition, TemplateDirSource from coder_eval.orchestration.task_loader import load_task @@ -64,18 +68,6 @@ "(unused placeholder — this file is graded via `coder-eval evaluate`, which does not invoke an agent on this path)" ) -# Mirror-image placeholder for the AGENT-phase task.yaml (see -# `_write_agent_phase_task_yaml`): `coder-eval execute` never grades -# (FinalStatus.NOT_GRADED unconditionally), so success_criteria's CONTENT is -# never read there either -- it exists only to satisfy TaskDefinition's -# at-least-one-criterion schema requirement without leaking the real criteria -# (which stay exclusively in tests/task.yaml, the verifier-side file) into the -# agent-visible image. -_AGENT_PHASE_PLACEHOLDER_CRITERION: dict[str, object] = { - "type": "file_exists", - "path": ".", - "description": "(unused placeholder — this file is run via `coder-eval execute`, which never checks criteria)", -} _TEST_SH_TEMPLATE = """#!/bin/sh # Generated by `coder-eval export --format harbor` — do not hand-edit; a @@ -196,7 +188,7 @@ def export_resolved_task( (out_dir / "tests").mkdir(parents=True, exist_ok=True) workdir = _write_environment(task, task_file, out_dir, warnings) - _write_instruction(task, task_file, out_dir) + _write_instruction(out_dir) _write_verifier_task_yaml(task, out_dir) _write_test_sh(out_dir, workdir=workdir) _write_reference(task, task_file, out_dir) @@ -208,25 +200,44 @@ def export_resolved_task( def _resolve_prompt_text(task: TaskDefinition, task_file: Path) -> str: """The agent's actual instruction text, resolved from whichever of the two prompt fields is set. - Shared by ``instruction.md`` (Harbor's own agent phase, when one is used) - and ``environment/agent_task.yaml`` (the ``CoderEvalAgent`` embed's real - prompt) — both must show the agent the identical text. + Feeds ``environment/task.yaml``'s ``initial_prompt`` -- the ``CoderEvalAgent`` + embed's real prompt (``instruction.md`` is a fixed placeholder, not this + text; see ``_write_instruction``). """ if task.initial_prompt is not None: return task.initial_prompt if task.initial_prompt_file is not None: return (task_file.parent / task.initial_prompt_file).read_text(encoding="utf-8") # is_none_agent or simulation-only tasks carry no prompt at all — fall back - # to the description so instruction.md is never empty. Harbor's own - # Task._validate_tests only requires the FILE to exist, not any content. + # to the description so `environment/task.yaml`'s initial_prompt is never empty. return task.description -def _write_instruction(task: TaskDefinition, task_file: Path, out_dir: Path) -> None: - text = _resolve_prompt_text(task, task_file) - if not text.endswith("\n"): - text += "\n" # POSIX text-file convention; also what the repo's own pre-commit end-of-file-fixer enforces - (out_dir / "instruction.md").write_text(text, encoding="utf-8") +_INSTRUCTION_MD_CONTENT = ( + "This is a coder-eval task, exported to Harbor.\n" + "\n" + "This file is autogenerated -- do not edit it directly. The task's real\n" + "instruction and agent configuration live in environment/task.yaml, and\n" + "must be run with the CoderEvalAgent Harbor agent:\n" + "\n" + " harbor run -a coder_eval.harbor.agent:CoderEvalAgent -p \n" + "\n" + "See https://coder-eval.com for details on the coder-eval <-> Harbor integration.\n" +) + + +def _write_instruction(out_dir: Path) -> None: + """``instruction.md`` — a fixed placeholder, deliberately NOT the real prompt. + + Harbor's own stock agents (``ClaudeCode`` etc.) read this file as their + actual task instruction. This export is meant to run ONLY via the + ``CoderEvalAgent`` embed (which ignores ``instruction.md`` entirely and + instead runs ``coder-eval execute`` against ``environment/task.yaml`` -- + see ``harbor/agent.py``'s ``run()``) -- so putting the real prompt here + would silently let a stock Harbor agent run the task with a DIFFERENT + (and unmonitored) agent loop than the one the export was built for. + """ + (out_dir / "instruction.md").write_text(_INSTRUCTION_MD_CONTENT, encoding="utf-8") def _write_environment( @@ -241,15 +252,32 @@ def _write_environment( SHARED mode) runs at whatever the container's own WORKDIR is. This function is the one place that decides it, so nothing downstream can silently disagree. + + A ``Dockerfile`` is ALWAYS written here (never left to a pre-built + ``docker_image`` reference in ``task.toml``) — the whole point of + ``environment/task.yaml`` is to be baked in at :data:`AGENT_TASK_YAML_PATH` + for the ``CoderEvalAgent`` Harbor-agent embed, and a task exported without + a Dockerfile has no `COPY` step to put it there. Two shapes, both ending + in the same `COPY task.yaml ...` line: + + - ``dockerfile_path`` set: copy the user's Dockerfile as the base, appending + a `WORKDIR` (if it declared none) and the `COPY` line. + - unset: synthesize a minimal one (`FROM ` + `WORKDIR` + + `COPY`) — ``docker_cfg.image`` always has a value (default_factory= + ``get_default_docker_image_tag``), so this is a real choice, not a + null-vs-set distinction. """ env_dir = out_dir / "environment" env_dir.mkdir(parents=True, exist_ok=True) docker_cfg = task.sandbox.docker - _write_agent_phase_task_yaml(task, env_dir, initial_prompt=_resolve_prompt_text(task, task_file)) + has_templates = _write_agent_phase_task_yaml( + task, env_dir, initial_prompt=_resolve_prompt_text(task, task_file), warnings=warnings + ) + templates_copy_line = f"COPY templates/ {AGENT_TASK_TEMPLATES_DIR}/\n" if has_templates else "" + dest_dockerfile = env_dir / "Dockerfile" if docker_cfg.dockerfile_path is not None: source_dockerfile = Path(docker_cfg.dockerfile_path) # already absolute — load_task resolves it - dest_dockerfile = env_dir / "Dockerfile" shutil.copy2(source_dockerfile, dest_dockerfile) workdir = docker_cfg.working_dir or _find_workdir(dest_dockerfile) or DEFAULT_WORKDIR if _find_workdir(dest_dockerfile) is None: @@ -260,7 +288,7 @@ def _write_environment( + "verifier and agent phases agree on a path." ) with dest_dockerfile.open("a", encoding="utf-8") as fh: - fh.write(f"\nCOPY agent_task.yaml {AGENT_TASK_YAML_PATH}\n") + fh.write(f"\nCOPY task.yaml {AGENT_TASK_YAML_PATH}\n{templates_copy_line}") if not _from_line_mentions_coder_eval_agent(dest_dockerfile): warnings.append(_MISSING_CODER_EVAL_WARNING) # Non-Dockerfile build context (COPY sources etc.) is not carried over in @@ -275,31 +303,31 @@ def _write_environment( ) return workdir - # No dockerfile_path — fall back to the pre-built image. docker.image always - # has a value (default_factory=get_default_docker_image_tag), so this is a - # real choice, not a null-vs-set distinction: log it either way so the - # operator can see which image the exported task will pull. + # No dockerfile_path — synthesize a minimal Dockerfile on top of the + # pre-built image so the `CoderEvalAgent` embed always has somewhere to + # `COPY task.yaml` into. if "coder-eval-agent" not in docker_cfg.image: warnings.append(_MISSING_CODER_EVAL_WARNING) - warnings.append( - f"No Dockerfile to bake environment/agent_task.yaml into (pre-built image {docker_cfg.image!r}) — " - + f"a `CoderEvalAgent` Harbor agent embed needs the task at {AGENT_TASK_YAML_PATH} inside the image. " - + "Add a dockerfile_path that extends this image with `COPY agent_task.yaml " - + f"{AGENT_TASK_YAML_PATH}` if you intend to use it as the agent phase, not just the verifier." - ) if docker_cfg.working_dir is not None: - return docker_cfg.working_dir - inspected = _inspect_image_workdir(docker_cfg.image) - if inspected is not None: - return inspected - warnings.append( - f"Could not determine {docker_cfg.image}'s own WORKDIR (image not present locally, or docker " - + f"unavailable at export time) -- defaulting to `{DEFAULT_WORKDIR}`. Harbor's `docker exec -w` " - + "hard-fails if that path does not already exist in the image (unlike `docker run -w`, it will " - + "not create it); set `sandbox.docker.working_dir` explicitly to the image's real WORKDIR to " - + "avoid a verify-time exit 127." + workdir = docker_cfg.working_dir + else: + inspected = _inspect_image_workdir(docker_cfg.image) + if inspected is not None: + workdir = inspected + else: + workdir = DEFAULT_WORKDIR + warnings.append( + f"Could not determine {docker_cfg.image}'s own WORKDIR (image not present locally, or " + + f"docker unavailable at export time) -- defaulting to `{DEFAULT_WORKDIR}`. Harbor's " + + "`docker exec -w` hard-fails if that path does not already exist in the image (unlike " + + "`docker run -w`, it will not create it); set `sandbox.docker.working_dir` explicitly " + + "to the image's real WORKDIR to avoid a verify-time exit 127." + ) + dest_dockerfile.write_text( + f"FROM {docker_cfg.image}\nWORKDIR {workdir}\n\nCOPY task.yaml {AGENT_TASK_YAML_PATH}\n{templates_copy_line}", + encoding="utf-8", ) - return DEFAULT_WORKDIR + return workdir def _inspect_image_workdir(image: str) -> str | None: @@ -381,8 +409,52 @@ def _write_verifier_task_yaml(task: TaskDefinition, out_dir: Path) -> None: ) -def _write_agent_phase_task_yaml(task: TaskDefinition, env_dir: Path, *, initial_prompt: str) -> None: - """``environment/agent_task.yaml`` — the REAL agent config, but criteria-free. +def _copy_template_sources(task: TaskDefinition, env_dir: Path, warnings: list[str]) -> list[dict[str, object]] | None: + """Copy each ``TemplateDirSource``'s directory into ``environment/templates/-/`` + and return a rewritten ``template_sources`` list pointing at the in-container copy + (:data:`AGENT_TASK_TEMPLATES_DIR`), or ``None`` if the task has no template sources. + + ``TemplateDirSource.path`` is resolved to an absolute HOST path at task-load time + (``task_loader.resolve_template_source_paths``) -- baking that path verbatim into + ``environment/task.yaml`` would point the in-container agent at a directory that + doesn't exist there. Only ``TemplateDirSource`` is copied: ``RepoSource`` (clones at + runtime) and ``StarterFilesSource`` (inline file content) resolve entirely inside the + container already and are carried over unchanged. + """ + sources = task.sandbox.template_sources + if not sources: + return None + templates_dir = env_dir / "templates" + rewritten: list[dict[str, object]] = [] + for i, source in enumerate(sources): + dumped = source.model_dump(mode="json", exclude_none=True) + if isinstance(source, TemplateDirSource): + source_path = Path(source.path) # already absolute — load_task resolved it + dest_name = f"{i:02d}-{source_path.name}" + dest = templates_dir / dest_name + if not source_path.is_dir(): + warnings.append( + f"template_sources[{i}].path {source_path} is not a directory -- not copied into the export." + ) + rewritten.append(dumped) + continue + if dest.exists(): + shutil.rmtree(dest) + shutil.copytree(source_path, dest) + dumped["path"] = f"{AGENT_TASK_TEMPLATES_DIR}/{dest_name}" + else: + warnings.append( + f"template_sources[{i}] ({type(source).__name__}) is not a TemplateDirSource -- carried over " + + "unchanged; only TemplateDirSource directories are copied into the export." + ) + rewritten.append(dumped) + return rewritten + + +def _write_agent_phase_task_yaml( + task: TaskDefinition, env_dir: Path, *, initial_prompt: str, warnings: list[str] +) -> bool: + """``environment/task.yaml`` — the REAL agent config, but criteria-free. Baked into the image at :data:`AGENT_TASK_YAML_PATH` (see the Dockerfile ``COPY`` line in ``_write_environment``) for a ``CoderEvalAgent`` Harbor @@ -390,15 +462,20 @@ def _write_agent_phase_task_yaml(task: TaskDefinition, env_dir: Path, *, initial harbor``. Unlike ``tests/task.yaml`` (the verifier's placeholder-agent, real-criteria file), this is the mirror image: ``task.agent`` and the resolved prompt are carried over VERBATIM (the whole point is running the - task's actual configured agent), but ``success_criteria`` is replaced with - :data:`_AGENT_PHASE_PLACEHOLDER_CRITERION` — never read (`coder-eval - execute` never grades) and never leaked into the agent-visible image. - - ``sandbox.driver`` is forced to ``tempdir`` regardless of the original - task's driver: this file runs INSIDE the container Harbor already built, - so re-declaring ``driver: docker`` here would have ``coder-eval execute`` - try to launch a second, nested container rather than just using its own - in-process sandbox at the container's current workdir. + task's actual configured agent), but ``success_criteria`` is forced to + ``[]`` — never leaked into the agent-visible image, and never read either + (`coder-eval execute` never grades). ``TaskDefinition`` no longer requires + at least one criterion, so this no longer needs a placeholder. + + ``sandbox`` is the original task's ``sandbox`` block, field-merged with + ``driver: tempdir`` and (when present) a rewritten ``template_sources`` — + everything else (``python.env_packages``, ``limits``, ...) is preserved, + not blanked. ``driver`` must be forced regardless of the original task's + driver: this file runs INSIDE the container Harbor already built, so + re-declaring ``driver: docker`` here would have ``coder-eval execute`` try + to launch a second, nested container rather than just using its own + in-process sandbox at the container's current workdir. ``docker`` config + is dropped along with it — moot once ``driver`` is forced to ``tempdir``. ``initial_prompt`` is omitted entirely for a ``type: none`` (agentless) task: coder-eval's own schema forbids a no-op agent from setting a prompt @@ -406,22 +483,30 @@ def _write_agent_phase_task_yaml(task: TaskDefinition, env_dir: Path, *, initial verified live via ``harbor run`` against a real ``harbor`` install, which surfaced exactly this ``TaskDefinition`` validation error before this guard was added. + + Returns whether any template directory was copied into ``environment/templates/``, + so ``_write_environment`` knows whether to add the corresponding Dockerfile ``COPY``. """ is_agentless = task.agent is not None and task.agent.type == "none" + rewritten_template_sources = _copy_template_sources(task, env_dir, warnings) + sandbox_dict = task.sandbox.model_dump(mode="json", exclude_none=True) + sandbox_dict["driver"] = "tempdir" + sandbox_dict.pop("docker", None) + if rewritten_template_sources is not None: + sandbox_dict["template_sources"] = rewritten_template_sources payload: dict[str, object] = { "task_id": task.task_id, "description": task.description, "agent": ( task.agent.model_dump(mode="json", exclude_none=True) if task.agent is not None else {"type": "claude-code"} ), - "sandbox": {"driver": "tempdir"}, - "success_criteria": [dict(_AGENT_PHASE_PLACEHOLDER_CRITERION)], + "sandbox": sandbox_dict, + "success_criteria": [], } if not is_agentless: payload["initial_prompt"] = initial_prompt - (env_dir / "agent_task.yaml").write_text( - yaml.safe_dump(payload, sort_keys=False, allow_unicode=True), encoding="utf-8" - ) + (env_dir / "task.yaml").write_text(yaml.safe_dump(payload, sort_keys=False, allow_unicode=True), encoding="utf-8") + return (env_dir / "templates").is_dir() def _write_test_sh(out_dir: Path, *, workdir: str) -> None: @@ -441,6 +526,14 @@ def _write_reference(task: TaskDefinition, task_file: Path, out_dir: Path) -> No def _write_task_toml(task: TaskDefinition, out_dir: Path, *, workdir: str) -> None: + verifier_section: dict[str, object] = {} + env_names = _env_passthrough_names(task) + if env_names: + verifier_section["env"] = _env_template_dict(env_names) + if task.run_limits is not None and task.run_limits.task_timeout is not None: + timeout = float(task.run_limits.task_timeout) + verifier_section["timeout_sec"] = timeout + doc: dict[str, object] = { "schema_version": _HARBOR_SCHEMA_VERSION, "task": { @@ -450,24 +543,72 @@ def _write_task_toml(task: TaskDefinition, out_dir: Path, *, workdir: str) -> No }, "environment": _build_environment_section(task, workdir=workdir), } + if verifier_section: + doc["verifier"] = verifier_section if task.run_limits is not None and task.run_limits.task_timeout is not None: - timeout = float(task.run_limits.task_timeout) - doc["verifier"] = {"timeout_sec": timeout} - doc["agent"] = {"timeout_sec": timeout} + doc["agent"] = {"timeout_sec": float(task.run_limits.task_timeout)} (out_dir / "task.toml").write_bytes(tomli_w.dumps(doc).encode("utf-8")) +_HARBOR_ENV_PASSTHROUGH_EXCLUDE = { + # `HOME` is intentional in `env_passthrough`'s default ONLY because + # `docker_runner.py` also bind-mounts the host's `~/.claude` into the + # container at that same path, so the host `HOME` value still resolves to + # a real, populated directory there (see docs/DOCKER_ISOLATION.md). Harbor + # builds its own container with no such mount, so forwarding the host's + # literal `HOME` (e.g. `/Users/alice`) would point the container at a + # directory that doesn't exist in it -- a real regression, not a no-op. + "HOME", +} + + +def _env_passthrough_names(task: TaskDefinition) -> list[str]: + """The env var NAMES this task's docker driver would forward on the host (``docker_runner.py``'s + ``merged_allowlist``), reused here as the SSOT for which credentials a Harbor export's + agent/verifier containers need declared -- rather than a second hardcoded list drifting + from that one. Minus :data:`_HARBOR_ENV_PASSTHROUGH_EXCLUDE`, entries whose coder-eval-driver + behavior does not carry over to a Harbor-built container. + """ + docker_cfg = task.sandbox.docker + names = set(docker_cfg.env_passthrough) | set(docker_cfg.env_passthrough_extra) + return sorted(names - _HARBOR_ENV_PASSTHROUGH_EXCLUDE) + + +def _env_template_dict(names: list[str]) -> dict[str, str]: + """``{NAME: "${NAME:-}"}`` -- Harbor's ``${VAR:-default}`` host-env-passthrough template + syntax (see ``harbor.utils.env.resolve_env_vars``): a NAME only, never a value, so no + secret is ever written into the exported ``task.toml``. + + The ``:-`` (empty default) is load-bearing, not cosmetic: a bare ``${VAR}`` makes Harbor + treat the var as REQUIRED (``get_required_host_vars`` -> ``_confirm_host_env_access`` + hard-exits with "Missing Environment Variables" for every name absent from the invoking + shell, confirmed live) -- and `env_passthrough`'s default list names ~24 vars spanning + every backend/agent this task might not even use (Codex, Antigravity, Pi, LiteLLM, UiPath + ...). Requiring an operator to export all of them just to run a Bedrock-only task is + exactly the friction this fixes; `${VAR:-}` resolves to an empty string when unset instead, + so Harbor only forwards what's actually set. + """ + return {name: f"${{{name}:-}}" for name in names} + + def _build_environment_section(task: TaskDefinition, *, workdir: str) -> dict[str, object]: + """No ``docker_image`` key: ``_write_environment`` always writes ``environment/Dockerfile`` + + now (either copied from ``dockerfile_path`` or synthesized from + ``docker_cfg.image``), so Harbor always builds from that file rather than + pulling a bare image reference named in ``task.toml``. + """ docker_cfg = task.sandbox.docker section: dict[str, object] = {"workdir": workdir} - if docker_cfg.dockerfile_path is None: - section["docker_image"] = docker_cfg.image limits = task.sandbox.limits if limits.max_memory_mb is not None: section["memory_mb"] = limits.max_memory_mb if limits.max_cpus is not None: section["cpus"] = max(1, round(limits.max_cpus)) section["network_mode"] = "no-network" if docker_cfg.network == "none" else "public" + env_names = _env_passthrough_names(task) + if env_names: + section["env"] = _env_template_dict(env_names) return section diff --git a/src/coder_eval/models/tasks.py b/src/coder_eval/models/tasks.py index bb60e743..f1f36cf1 100644 --- a/src/coder_eval/models/tasks.py +++ b/src/coder_eval/models/tasks.py @@ -792,11 +792,3 @@ def check_removed_criteria_types(cls, v: Any) -> Any: normalized.append(item) return normalized return v - - @field_validator("success_criteria") - @classmethod - def validate_success_criteria(cls, v: Any) -> Any: - """Ensure at least one success criterion is defined.""" - if not v: - raise ValueError("At least one success criterion must be defined") - return v diff --git a/src/coder_eval/orchestration/batch.py b/src/coder_eval/orchestration/batch.py index 636156bb..a78b29b7 100644 --- a/src/coder_eval/orchestration/batch.py +++ b/src/coder_eval/orchestration/batch.py @@ -117,6 +117,19 @@ async def run_batch( start_time = datetime.now() + if config.workspace_dir is not None: + if len(resolved_tasks) != 1: + raise ValueError( + f"--workspace-dir requires exactly one resolved task; got {len(resolved_tasks)}. " + + "It names a single in-place directory every task would otherwise collide on." + ) + (_workspace_dir_task,) = resolved_tasks + if _workspace_dir_task.task.sandbox is not None and _workspace_dir_task.task.sandbox.driver == "docker": + raise ValueError( + "--workspace-dir is not for sandbox.driver: docker tasks -- the docker driver already " + + "aligns automatically via sandbox.docker.working_dir (see DockerRunner)." + ) + check_pricing_coverage(resolved_tasks) if on_batch_start is not None: @@ -187,6 +200,7 @@ async def run_single(rt: ResolvedTask) -> TaskResult: config_lineage=rt.config_lineage, replicate_index=rt.replicate_index, grade=config.grade, + workspace_dir=config.workspace_dir, ) result = await orchestrator.run() tr = TaskResult( diff --git a/src/coder_eval/orchestration/config.py b/src/coder_eval/orchestration/config.py index 80a3d812..0a287fbd 100644 --- a/src/coder_eval/orchestration/config.py +++ b/src/coder_eval/orchestration/config.py @@ -111,6 +111,24 @@ class BatchRunConfig(BaseModel): # Logging verbose: bool = Field(default=False, description="Enable verbose (DEBUG level) logging for Docker output") + # Docker WORKDIR alignment for the non-docker-driver dispatch path (host + # process, or already inside a container someone else built — e.g. a Harbor + # trial container running `coder-eval execute` as its agent). Mirrors what + # `DockerRunner`/`_run-task-internal` already do for `sandbox.driver: docker` + # (see `Orchestrator.workspace_dir`'s docstring); this is the same mechanism, + # exposed publicly for the case where coder-eval's OWN docker driver isn't + # the one building the container. Only meaningful for a single resolved task + # — `run_batch` raises if more than one task would collide on it. + workspace_dir: Path | None = Field( + default=None, + description=( + "Run the agent in-place at this absolute path instead of the standard " + "run_dir/artifacts workspace, copying it out to run_dir/artifacts/ at " + "cleanup. For a single task only. Not for sandbox.driver: docker tasks — " + "the docker driver already aligns automatically via sandbox.docker.working_dir." + ), + ) + # TODO(container-death-diagnostics): consider a run-level default resource # cap. Containers run uncapped today (sandbox.limits.{max_memory_mb, # max_cpus,max_pids} default to None -> _build_argv emits no --memory/ diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index da273ccc..281906e9 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -2569,13 +2569,13 @@ async def _run_dialog_criteria_check( # RAISES rather than returning an empty list, matching the evaluate-only # path's refusal. The empty-list version described itself as a # "defensive no-op so the gate holds", and it was neither: both callers - # go straight on to `all_criteria_passed`, whose first act is a - # length pre-check that raises on a mismatch — and an empty criteria - # list is forbidden by `TaskDefinition.validate_success_criteria`, so - # the mismatch was guaranteed. If the simulation restriction ever lifts, - # that "no-op" turns every ungraded dialog into FinalStatus.ERROR. A - # loud refusal here is honest about the fact that this path has no - # ungraded semantics yet. + # go straight on to `all_criteria_passed`/`calculate_weighted_score`, + # which treat an empty criteria list as a vacuous pass/0.0 rather than + # raising, so a silent no-op here would produce a criteria-free dialog + # that scores as though nothing had been asked of the agent. If the + # simulation restriction ever lifts, that "no-op" turns every ungraded + # dialog into a silently-passing one. A loud refusal here is honest + # about the fact that this path has no ungraded semantics yet. if not self.grade: raise ValueError( "Grading is disabled but the simulation dialog path requires criteria results to " diff --git a/tests/_fixtures/harbor_export_golden/expected/environment/Dockerfile b/tests/_fixtures/harbor_export_golden/expected/environment/Dockerfile index dceeff2b..5a55d1ce 100644 --- a/tests/_fixtures/harbor_export_golden/expected/environment/Dockerfile +++ b/tests/_fixtures/harbor_export_golden/expected/environment/Dockerfile @@ -3,4 +3,4 @@ RUN apt-get update && apt-get install -y --no-install-recommends coreutils && rm WORKDIR /app -COPY agent_task.yaml /opt/coder-eval-task/task.yaml +COPY task.yaml /opt/coder-eval-task/task.yaml diff --git a/tests/_fixtures/harbor_export_golden/expected/environment/agent_task.yaml b/tests/_fixtures/harbor_export_golden/expected/environment/task.yaml similarity index 70% rename from tests/_fixtures/harbor_export_golden/expected/environment/agent_task.yaml rename to tests/_fixtures/harbor_export_golden/expected/environment/task.yaml index 3e0dd715..e9a3e841 100644 --- a/tests/_fixtures/harbor_export_golden/expected/environment/agent_task.yaml +++ b/tests/_fixtures/harbor_export_golden/expected/environment/task.yaml @@ -9,9 +9,12 @@ agent: sdk_options: {} sandbox: driver: tempdir -success_criteria: -- type: file_exists - path: . - description: (unused placeholder — this file is run via `coder-eval execute`, which - never checks criteria) + python: + env_packages: [] + limits: + timeout: 300 + max_memory_mb: 1024 + max_cpus: 1.5 + ignore_patterns: [] +success_criteria: [] initial_prompt: Write 'hello world' to greeting.txt, matching reference/greeting.txt. diff --git a/tests/_fixtures/harbor_export_golden/expected/instruction.md b/tests/_fixtures/harbor_export_golden/expected/instruction.md index 5f9b4eb8..89318b6c 100644 --- a/tests/_fixtures/harbor_export_golden/expected/instruction.md +++ b/tests/_fixtures/harbor_export_golden/expected/instruction.md @@ -1 +1,9 @@ -Write 'hello world' to greeting.txt, matching reference/greeting.txt. +This is a coder-eval task, exported to Harbor. + +This file is autogenerated -- do not edit it directly. The task's real +instruction and agent configuration live in environment/task.yaml, and +must be run with the CoderEvalAgent Harbor agent: + + harbor run -a coder_eval.harbor.agent:CoderEvalAgent -p + +See https://coder-eval.com for details on the coder-eval <-> Harbor integration. diff --git a/tests/_fixtures/harbor_export_golden/expected/task.toml b/tests/_fixtures/harbor_export_golden/expected/task.toml index dbe7caf3..1ae65469 100644 --- a/tests/_fixtures/harbor_export_golden/expected/task.toml +++ b/tests/_fixtures/harbor_export_golden/expected/task.toml @@ -14,8 +14,60 @@ memory_mb = 1024 cpus = 2 network_mode = "no-network" +[environment.env] +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY:-}" +ANTHROPIC_MODEL = "${ANTHROPIC_MODEL:-}" +ANTIGRAVITY_MODEL = "${ANTIGRAVITY_MODEL:-}" +API_BACKEND = "${API_BACKEND:-}" +AWS_BEARER_TOKEN_BEDROCK = "${AWS_BEARER_TOKEN_BEDROCK:-}" +AWS_REGION = "${AWS_REGION:-}" +BEDROCK_MODEL = "${BEDROCK_MODEL:-}" +CLAUDE_CODE_USE_BEDROCK = "${CLAUDE_CODE_USE_BEDROCK:-}" +CODEX_API_KEY = "${CODEX_API_KEY:-}" +CODEX_BASE_URL = "${CODEX_BASE_URL:-}" +CODEX_MODEL = "${CODEX_MODEL:-}" +GEMINI_API_KEY = "${GEMINI_API_KEY:-}" +LITELLM_AUTH_TOKEN = "${LITELLM_AUTH_TOKEN:-}" +LITELLM_BASE_URL = "${LITELLM_BASE_URL:-}" +LITELLM_COST_LOG = "${LITELLM_COST_LOG:-}" +LITELLM_MODEL = "${LITELLM_MODEL:-}" +LITELLM_SMALL_MODEL = "${LITELLM_SMALL_MODEL:-}" +OPENROUTER_API_KEY = "${OPENROUTER_API_KEY:-}" +UIPATH_ACCESS_TOKEN = "${UIPATH_ACCESS_TOKEN:-}" +UIPATH_CLI_DISABLE_VERSION_SYNC = "${UIPATH_CLI_DISABLE_VERSION_SYNC:-}" +UIPATH_LLM_BACKEND = "${UIPATH_LLM_BACKEND:-}" +UIPATH_ORGANIZATION_ID = "${UIPATH_ORGANIZATION_ID:-}" +UIPATH_TENANT_ID = "${UIPATH_TENANT_ID:-}" +UIPATH_URL = "${UIPATH_URL:-}" + [verifier] timeout_sec = 300.0 +[verifier.env] +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY:-}" +ANTHROPIC_MODEL = "${ANTHROPIC_MODEL:-}" +ANTIGRAVITY_MODEL = "${ANTIGRAVITY_MODEL:-}" +API_BACKEND = "${API_BACKEND:-}" +AWS_BEARER_TOKEN_BEDROCK = "${AWS_BEARER_TOKEN_BEDROCK:-}" +AWS_REGION = "${AWS_REGION:-}" +BEDROCK_MODEL = "${BEDROCK_MODEL:-}" +CLAUDE_CODE_USE_BEDROCK = "${CLAUDE_CODE_USE_BEDROCK:-}" +CODEX_API_KEY = "${CODEX_API_KEY:-}" +CODEX_BASE_URL = "${CODEX_BASE_URL:-}" +CODEX_MODEL = "${CODEX_MODEL:-}" +GEMINI_API_KEY = "${GEMINI_API_KEY:-}" +LITELLM_AUTH_TOKEN = "${LITELLM_AUTH_TOKEN:-}" +LITELLM_BASE_URL = "${LITELLM_BASE_URL:-}" +LITELLM_COST_LOG = "${LITELLM_COST_LOG:-}" +LITELLM_MODEL = "${LITELLM_MODEL:-}" +LITELLM_SMALL_MODEL = "${LITELLM_SMALL_MODEL:-}" +OPENROUTER_API_KEY = "${OPENROUTER_API_KEY:-}" +UIPATH_ACCESS_TOKEN = "${UIPATH_ACCESS_TOKEN:-}" +UIPATH_CLI_DISABLE_VERSION_SYNC = "${UIPATH_CLI_DISABLE_VERSION_SYNC:-}" +UIPATH_LLM_BACKEND = "${UIPATH_LLM_BACKEND:-}" +UIPATH_ORGANIZATION_ID = "${UIPATH_ORGANIZATION_ID:-}" +UIPATH_TENANT_ID = "${UIPATH_TENANT_ID:-}" +UIPATH_URL = "${UIPATH_URL:-}" + [agent] timeout_sec = 300.0 diff --git a/tests/test_harbor_experiment_packager.py b/tests/test_harbor_experiment_packager.py index b6005185..4715cde9 100644 --- a/tests/test_harbor_experiment_packager.py +++ b/tests/test_harbor_experiment_packager.py @@ -80,7 +80,7 @@ def test_two_variants_export_two_directories_with_distinct_timeouts(self, tmp_pa assert "timeout_sec = 60" in (fast_dir / "task.toml").read_text(encoding="utf-8") assert "timeout_sec = 900" in (slow_dir / "task.toml").read_text(encoding="utf-8") - def test_variant_prompt_override_lands_in_instruction_md(self, tmp_path: Path) -> None: + def test_variant_prompt_override_lands_in_environment_task_yaml(self, tmp_path: Path) -> None: task_file = _write_task(tmp_path) exp_file = _write_experiment( tmp_path, @@ -96,8 +96,14 @@ def test_variant_prompt_override_lands_in_instruction_md(self, tmp_path: Path) - result = export_experiment([task_file], exp_file, out_dir) assert len(result.exported) == 1 + # instruction.md is a fixed placeholder now -- the real (overridden) prompt + # lands in environment/task.yaml instead. instruction = (out_dir / "rewritten" / "greet" / "instruction.md").read_text(encoding="utf-8") - assert "howdy" in instruction + assert "howdy" not in instruction + emitted = yaml.safe_load( + (out_dir / "rewritten" / "greet" / "environment" / "task.yaml").read_text(encoding="utf-8") + ) + assert emitted["initial_prompt"] == "Write 'howdy' to greeting.txt." class TestUnhonorableOverridesAreSkipped: diff --git a/tests/test_harbor_packager.py b/tests/test_harbor_packager.py index e71a6308..3cfba738 100644 --- a/tests/test_harbor_packager.py +++ b/tests/test_harbor_packager.py @@ -109,11 +109,18 @@ def test_full_export_produces_the_documented_layout(self, tmp_path: Path) -> Non assert (out_dir / "tests" / "test.sh").is_file() assert (out_dir / "tests" / "task.yaml").is_file() - def test_instruction_md_carries_the_initial_prompt(self, tmp_path: Path) -> None: - task_file = _write_task(tmp_path) + def test_instruction_md_is_a_fixed_placeholder_not_the_real_prompt(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) # _BASE_TASK's initial_prompt is "Write 'hello' to greeting.txt." out_dir = tmp_path / "out" export_task(task_file, out_dir) - assert (out_dir / "instruction.md").read_text(encoding="utf-8") == "Write 'hello' to greeting.txt.\n" + text = (out_dir / "instruction.md").read_text(encoding="utf-8") + assert "greeting.txt" not in text # the real prompt must never land here + assert "autogenerated" in text.lower() + assert "environment/task.yaml" in text + assert "CoderEvalAgent" in text + + emitted = yaml.safe_load((out_dir / "environment" / "task.yaml").read_text(encoding="utf-8")) + assert emitted["initial_prompt"] == "Write 'hello' to greeting.txt." # the real prompt lives here instead def test_test_sh_is_executable_and_references_the_resolved_workdir(self, tmp_path: Path) -> None: task_file = _write_task(tmp_path) @@ -136,7 +143,7 @@ def test_task_toml_parses_and_carries_the_mapped_fields(self, tmp_path: Path) -> doc = tomllib.loads((out_dir / "task.toml").read_text(encoding="utf-8")) assert doc["task"]["name"] == "coder-eval/greet" assert doc["task"]["keywords"] == ["smoke"] - assert doc["environment"]["docker_image"] == "byod-custom-image:0.1.0" + assert "docker_image" not in doc["environment"] # always built from environment/Dockerfile now assert doc["environment"]["memory_mb"] == 2048 assert doc["environment"]["cpus"] == 2 assert doc["environment"]["network_mode"] == "no-network" @@ -204,7 +211,7 @@ def test_reference_comparison_survives_the_none_agent_trap(self, tmp_path: Path) class TestAgentPhaseTaskYaml: - """environment/agent_task.yaml — the CoderEvalAgent embed's criteria-free real-agent config.""" + """environment/task.yaml — the CoderEvalAgent embed's criteria-free real-agent config.""" def test_baked_at_fixed_path_via_dockerfile_copy(self, tmp_path: Path) -> None: env_dir = tmp_path / "environment" @@ -218,9 +225,9 @@ def test_baked_at_fixed_path_via_dockerfile_copy(self, tmp_path: Path) -> None: export_task(task_file, out_dir) - assert (out_dir / "environment" / "agent_task.yaml").exists() + assert (out_dir / "environment" / "task.yaml").exists() dockerfile_text = (out_dir / "environment" / "Dockerfile").read_text(encoding="utf-8") - assert "COPY agent_task.yaml /opt/coder-eval-task/task.yaml" in dockerfile_text + assert "COPY task.yaml /opt/coder-eval-task/task.yaml" in dockerfile_text def test_carries_the_real_agent_config_but_no_real_criteria(self, tmp_path: Path) -> None: task_file = _write_task(tmp_path, {"agent": {"type": "claude-code", "model": "claude-opus-5"}}) @@ -228,11 +235,10 @@ def test_carries_the_real_agent_config_but_no_real_criteria(self, tmp_path: Path export_task(task_file, out_dir) - emitted = yaml.safe_load((out_dir / "environment" / "agent_task.yaml").read_text(encoding="utf-8")) + emitted = yaml.safe_load((out_dir / "environment" / "task.yaml").read_text(encoding="utf-8")) assert emitted["agent"]["type"] == "claude-code" assert emitted["agent"]["model"] == "claude-opus-5" - assert len(emitted["success_criteria"]) == 1 # the placeholder only -- never the real ones - assert emitted["success_criteria"][0]["type"] == "file_exists" + assert emitted["success_criteria"] == [] # never the real ones # The real task's success_criteria (2 entries in _BASE_TASK) must never leak in. assert "reference_comparison" not in str(emitted["success_criteria"]) @@ -242,7 +248,7 @@ def test_sandbox_driver_forced_to_tempdir_no_nested_docker(self, tmp_path: Path) export_task(task_file, out_dir) - emitted = yaml.safe_load((out_dir / "environment" / "agent_task.yaml").read_text(encoding="utf-8")) + emitted = yaml.safe_load((out_dir / "environment" / "task.yaml").read_text(encoding="utf-8")) assert emitted["sandbox"]["driver"] == "tempdir" def test_reloads_as_a_valid_task_definition(self, tmp_path: Path) -> None: @@ -251,11 +257,11 @@ def test_reloads_as_a_valid_task_definition(self, tmp_path: Path) -> None: export_task(task_file, out_dir) - emitted = yaml.safe_load((out_dir / "environment" / "agent_task.yaml").read_text(encoding="utf-8")) + emitted = yaml.safe_load((out_dir / "environment" / "task.yaml").read_text(encoding="utf-8")) reloaded = TaskDefinition.model_validate(emitted) # must not raise assert reloaded.task_id == "greet" - def test_prebuilt_image_with_no_dockerfile_warns_it_cannot_bake_the_agent_file_in(self, tmp_path: Path) -> None: + def test_prebuilt_image_with_no_dockerfile_path_gets_one_synthesized(self, tmp_path: Path) -> None: task_file = _write_task( tmp_path, {"sandbox": {"driver": "docker", "docker": {"image": "byod-custom-image:0.1.0"}}} ) @@ -263,8 +269,11 @@ def test_prebuilt_image_with_no_dockerfile_warns_it_cannot_bake_the_agent_file_i result = export_task(task_file, out_dir) - assert (out_dir / "environment" / "agent_task.yaml").exists() # still written -- just not bakeable in - assert any("No Dockerfile to bake" in w for w in result.warnings) + assert (out_dir / "environment" / "task.yaml").exists() + dockerfile_text = (out_dir / "environment" / "Dockerfile").read_text(encoding="utf-8") + assert dockerfile_text.startswith("FROM byod-custom-image:0.1.0\n") + assert "COPY task.yaml /opt/coder-eval-task/task.yaml" in dockerfile_text + assert not any("No Dockerfile to bake" in w for w in result.warnings) class TestDockerfileWorkdirResolution: @@ -300,10 +309,10 @@ def test_dockerfile_with_an_existing_workdir_is_respected_and_not_touched(self, assert result.workdir == "/workspace" dockerfile_text = (out_dir / "environment" / "Dockerfile").read_text(encoding="utf-8") - # The WORKDIR-bearing content is untouched; only the agent_task.yaml COPY + # The WORKDIR-bearing content is untouched; only the task.yaml COPY # line (baked in for a CoderEvalAgent embed, see agent_paths.py) is appended. assert dockerfile_text.startswith(original) - assert "COPY agent_task.yaml /opt/coder-eval-task/task.yaml" in dockerfile_text + assert "COPY task.yaml /opt/coder-eval-task/task.yaml" in dockerfile_text assert not any("declared no WORKDIR" in w for w in result.warnings) # test.sh and task.toml must agree with the same resolved workdir. assert '"/workspace"' in (out_dir / "tests" / "test.sh").read_text(encoding="utf-8") @@ -376,11 +385,7 @@ def test_no_warning_when_prebuilt_image_names_coder_eval_agent( tmp_path, {"sandbox": {"driver": "docker", "docker": {"image": "coder-eval-agent:0.12.0"}}} ) result = export_task(task_file, tmp_path / "out") - # A prebuilt image (no dockerfile_path) always warns that agent_task.yaml - # can't be baked in (no Dockerfile to COPY into) -- that warning legitimately - # NAMES the image, so it is excluded from this substring check rather than - # the check being widened to also swallow the real regression it guards. - assert not any("coder-eval-agent" in w for w in result.warnings if "No Dockerfile to bake" not in w) + assert not any("coder-eval-agent" in w for w in result.warnings) class TestPrePostRunWarnings: @@ -475,3 +480,147 @@ def _raise(cmd, **kwargs): monkeypatch.setattr(packager.subprocess, "run", _raise) assert packager._inspect_image_workdir("some-image:tag") is None + + +class TestEnvPassthroughSections: + """``task.toml``'s ``[environment.env]``/``[verifier.env]`` -- the SSOT-reuse fix: these + are derived from the resolved task's OWN ``sandbox.docker.env_passthrough`` (the same + allowlist coder-eval's own docker driver already uses), not a second hardcoded list. + """ + + def test_default_env_passthrough_names_land_in_both_sections_as_templates(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + doc = tomllib.loads((out_dir / "task.toml").read_text(encoding="utf-8")) + assert doc["environment"]["env"]["ANTHROPIC_API_KEY"] == "${ANTHROPIC_API_KEY:-}" + assert doc["environment"]["env"]["AWS_BEARER_TOKEN_BEDROCK"] == "${AWS_BEARER_TOKEN_BEDROCK:-}" + assert doc["verifier"]["env"]["ANTHROPIC_API_KEY"] == "${ANTHROPIC_API_KEY:-}" + # No literal value ever appears -- only the `${NAME:-}` template (empty default, never required). + assert "$" in doc["environment"]["env"]["ANTHROPIC_API_KEY"] + + def test_home_is_excluded_even_though_it_is_in_the_default_allowlist(self, tmp_path: Path) -> None: + # HOME is intentional in coder-eval's OWN docker driver only because it also + # bind-mounts ~/.claude; Harbor's container has no such mount, so forwarding + # the host's literal HOME would point the container at a directory that + # doesn't exist in it. + task_file = _write_task(tmp_path) + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + doc = tomllib.loads((out_dir / "task.toml").read_text(encoding="utf-8")) + assert "HOME" not in doc["environment"]["env"] + assert "HOME" not in doc["verifier"]["env"] + + def test_env_passthrough_extra_is_included_too(self, tmp_path: Path) -> None: + task_file = _write_task( + tmp_path, + {"sandbox": {"driver": "docker", "docker": {"env_passthrough_extra": ["MY_CUSTOM_TOKEN"]}}}, + ) + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + doc = tomllib.loads((out_dir / "task.toml").read_text(encoding="utf-8")) + assert doc["environment"]["env"]["MY_CUSTOM_TOKEN"] == "${MY_CUSTOM_TOKEN:-}" + + def test_no_env_section_when_the_resolved_allowlist_is_empty(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path, {"sandbox": {"driver": "docker", "docker": {"env_passthrough": []}}}) + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + doc = tomllib.loads((out_dir / "task.toml").read_text(encoding="utf-8")) + assert "env" not in doc["environment"] + assert "verifier" not in doc # no timeout_sec either in this fixture -- section fully omitted + + +class TestTemplateSourcesCopy: + """``TemplateDirSource`` directories must be copied into the export -- otherwise + ``environment/task.yaml`` would name an absolute HOST path (see + ``task_loader.resolve_template_source_paths``) that does not exist inside the + container the ``CoderEvalAgent`` embed actually runs in. + """ + + def _write_template_dir(self, tmp_path: Path, name: str = "starter") -> Path: + template_dir = tmp_path / name + template_dir.mkdir() + (template_dir / "main.py").write_text("def stub(): ...\n", encoding="utf-8") + return template_dir + + def test_template_dir_is_copied_and_path_rewritten(self, tmp_path: Path) -> None: + template_dir = self._write_template_dir(tmp_path) + task_file = _write_task( + tmp_path, + { + "sandbox": { + "driver": "docker", + "docker": {"image": "byod-custom-image:0.1.0", "network": "none"}, + "template_sources": [{"type": "template_dir", "path": str(template_dir)}], + } + }, + ) + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + copied = out_dir / "environment" / "templates" / "00-starter" / "main.py" + assert copied.read_text(encoding="utf-8") == "def stub(): ...\n" + + emitted = yaml.safe_load((out_dir / "environment" / "task.yaml").read_text(encoding="utf-8")) + assert emitted["sandbox"]["template_sources"][0]["path"] == "/opt/coder-eval-task/templates/00-starter" + + dockerfile_text = (out_dir / "environment" / "Dockerfile").read_text(encoding="utf-8") + assert "COPY templates/ /opt/coder-eval-task/templates/" in dockerfile_text + + def test_agent_phase_sandbox_preserves_python_and_limits(self, tmp_path: Path) -> None: + task_file = _write_task( + tmp_path, + { + "sandbox": { + "driver": "docker", + "docker": {"image": "byod-custom-image:0.1.0", "network": "none"}, + "python": {"env_packages": ["pytest"]}, + } + }, + ) + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + emitted = yaml.safe_load((out_dir / "environment" / "task.yaml").read_text(encoding="utf-8")) + assert emitted["sandbox"]["driver"] == "tempdir" + assert emitted["sandbox"]["python"]["env_packages"] == ["pytest"] + assert "docker" not in emitted["sandbox"] + + def test_no_templates_dir_or_copy_line_when_the_task_has_no_template_sources(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) + out_dir = tmp_path / "out" + + export_task(task_file, out_dir) + + assert not (out_dir / "environment" / "templates").exists() + dockerfile_text = (out_dir / "environment" / "Dockerfile").read_text(encoding="utf-8") + assert "templates" not in dockerfile_text + + def test_nonexistent_template_dir_warns_instead_of_crashing(self, tmp_path: Path) -> None: + missing = tmp_path / "does-not-exist" + task_file = _write_task( + tmp_path, + { + "sandbox": { + "driver": "docker", + "docker": {"image": "byod-custom-image:0.1.0", "network": "none"}, + "template_sources": [{"type": "template_dir", "path": str(missing)}], + } + }, + ) + out_dir = tmp_path / "out" + + result = export_task(task_file, out_dir) + + assert any("is not a directory" in w for w in result.warnings) + assert not (out_dir / "environment" / "templates").exists() From 0d8a7a669a962dd8c930bc0b06c7f0c4afe24891 Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Fri, 11 Sep 2026 11:05:41 -0700 Subject: [PATCH 06/12] ci(harbor): add Harbor E2E workflow (export + CoderEvalAgent round trip) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Installs harbor==0.22.0 alongside coder-eval in the same venv, builds the coder-eval-agent and byod-custom-image Docker images, then exports and runs four scenarios via `hb run -a coder_eval.harbor.agent:CoderEvalAgent`: baseline (plain docker driver, default image), llm_judge (real judge call inside the verifier phase), docker_custom_image (reuses tasks/byod_smoke_test.yaml's dockerfile_path + custom image), and template_sources (TemplateDirSource copy-in). Each scenario asserts reward.json == 1.0, a real ATIF trajectory.json, and both agent- and verifier-side task.json. Verified locally end-to-end against real Docker + real Bedrock credentials before committing (all four scenarios: reward 1.0, no exceptions). Triggered on workflow_dispatch/schedule/push-to-main only (not pull_request) so it's informational rather than a required PR check for now — it depends on live model calls and Docker builds, and Harbor's own release cadence is independent of this repo's. Co-Authored-By: Claude Sonnet 5 --- .github/scripts/harbor_e2e.py | 163 ++++++++++++++++++ .github/workflows/harbor-e2e.yml | 97 +++++++++++ .../harbor_e2e/fixtures/docker_baseline.yaml | 35 ++++ tests/harbor_e2e/fixtures/llm_judge.yaml | 44 +++++ .../harbor_e2e/fixtures/template_sources.yaml | 38 ++++ 5 files changed, 377 insertions(+) create mode 100644 .github/scripts/harbor_e2e.py create mode 100644 .github/workflows/harbor-e2e.yml create mode 100644 tests/harbor_e2e/fixtures/docker_baseline.yaml create mode 100644 tests/harbor_e2e/fixtures/llm_judge.yaml create mode 100644 tests/harbor_e2e/fixtures/template_sources.yaml diff --git a/.github/scripts/harbor_e2e.py b/.github/scripts/harbor_e2e.py new file mode 100644 index 00000000..2dd436a2 --- /dev/null +++ b/.github/scripts/harbor_e2e.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""Harbor end-to-end smoke test. + +Exports a handful of coder-eval tasks to Harbor (`coder-eval export --format +harbor`), runs each with `hb run -a coder_eval.harbor.agent:CoderEvalAgent` +against real Docker, and asserts that: + + - the verifier phase wrote `reward.json` with `reward == 1.0` + - the agent phase wrote a real ATIF `trajectory.json` + - both the agent and verifier phases left a `task.json` behind + +Invoked by `.github/workflows/harbor-e2e.yml` -- deliberately NOT part of +`make test`: it shells out to a real `hb` CLI, real Docker builds, and (for +the llm_judge scenario) a real model call, none of which belong in the fast +unit-test suite. +""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +WORK_DIR = REPO_ROOT / "tmp" / "harbor_e2e" +AGENT_IMPORT_PATH = "coder_eval.harbor.agent:CoderEvalAgent" + + +@dataclass(frozen=True) +class Scenario: + """One (task.yaml, export flags) pair to round-trip through Harbor.""" + + name: str + task_file: Path + allow_credentials: bool = False + + +SCENARIOS: list[Scenario] = [ + # "tmpdir" baseline: plain docker driver, default coder-eval-agent image, + # no dockerfile_path / template_sources / llm_judge -- exercises the bare + # export -> CoderEvalAgent -> --workspace-dir -> verifier round trip. + Scenario("baseline", REPO_ROOT / "tests/harbor_e2e/fixtures/docker_baseline.yaml"), + # llm_judge: real model call inside the VERIFIER phase, not just the agent. + Scenario("llm_judge", REPO_ROOT / "tests/harbor_e2e/fixtures/llm_judge.yaml", allow_credentials=True), + # Custom (BYOD) Docker image via dockerfile_path -- reuses the in-tree + # byod_smoke_test task/image rather than duplicating it. + Scenario("docker_custom_image", REPO_ROOT / "tasks/byod_smoke_test.yaml"), + # template_sources: TemplateDirSource copy-in + rewritten path, plus + # sandbox.python.env_packages surviving the agent-phase task.yaml merge. + Scenario("template_sources", REPO_ROOT / "tests/harbor_e2e/fixtures/template_sources.yaml"), +] + + +def _run(cmd: list[str]) -> subprocess.CompletedProcess[str]: + print(f"+ {' '.join(cmd)}", flush=True) + return subprocess.run(cmd, check=False, text=True, capture_output=True) + + +def export_task(scenario: Scenario, out_dir: Path) -> None: + if out_dir.exists(): + shutil.rmtree(out_dir) + cmd = ["coder-eval", "export", str(scenario.task_file), "-o", str(out_dir)] + if scenario.allow_credentials: + cmd.append("--allow-credentials") + result = _run(cmd) + print(result.stdout) + print(result.stderr, file=sys.stderr) + if result.returncode != 0: + raise RuntimeError(f"[{scenario.name}] `coder-eval export` failed (exit {result.returncode})") + + +def run_harbor(scenario: Scenario, export_dir: Path, jobs_dir: Path) -> Path: + if jobs_dir.exists(): + shutil.rmtree(jobs_dir) + cmd = [ + "hb", + "run", + "-p", + str(export_dir), + "-a", + AGENT_IMPORT_PATH, + "--jobs-dir", + str(jobs_dir), + "-n", + "1", + "-y", + ] + result = _run(cmd) + print(result.stdout) + print(result.stderr, file=sys.stderr) + if result.returncode != 0: + raise RuntimeError(f"[{scenario.name}] `hb run` failed (exit {result.returncode})") + + # ///{agent,verifier}/... -- glob for + # the one directory two levels down that actually holds a verifier/ output, + # rather than assuming a fixed trial-name shape Harbor doesn't guarantee. + trial_dirs = [d for d in jobs_dir.glob("*/*/") if (d / "verifier").is_dir()] + if len(trial_dirs) != 1: + raise RuntimeError( + f"[{scenario.name}] expected exactly one trial directory under {jobs_dir}, found {len(trial_dirs)}" + ) + return trial_dirs[0] + + +def assert_scenario_artifacts(scenario: Scenario, trial_dir: Path) -> None: + reward_path = trial_dir / "verifier" / "reward.json" + if not reward_path.is_file(): + raise RuntimeError(f"[{scenario.name}] missing {reward_path}") + reward = json.loads(reward_path.read_text(encoding="utf-8")) + if reward.get("reward") != 1.0: + raise RuntimeError(f"[{scenario.name}] expected reward 1.0, got {reward!r} ({reward_path})") + + trajectory_path = trial_dir / "agent" / "trajectory.json" + if not trajectory_path.is_file(): + raise RuntimeError(f"[{scenario.name}] missing {trajectory_path}") + trajectory = json.loads(trajectory_path.read_text(encoding="utf-8")) + if "schema_version" not in trajectory: + raise RuntimeError(f"[{scenario.name}] {trajectory_path} is missing 'schema_version'") + + agent_task_jsons = list((trial_dir / "agent").glob("**/task.json")) + if not agent_task_jsons: + raise RuntimeError(f"[{scenario.name}] no task.json found under {trial_dir / 'agent'}") + verifier_task_json = trial_dir / "verifier" / "task.json" + if not verifier_task_json.is_file(): + raise RuntimeError(f"[{scenario.name}] missing {verifier_task_json}") + + print( + f"[{scenario.name}] OK: reward=1.0, trajectory.json present, " + + f"{len(agent_task_jsons)} agent task.json + verifier/task.json present" + ) + + +def main() -> int: + WORK_DIR.mkdir(parents=True, exist_ok=True) + failures: list[str] = [] + for scenario in SCENARIOS: + export_dir = WORK_DIR / scenario.name / "export" + jobs_dir = WORK_DIR / scenario.name / "jobs" + print(f"\n=== {scenario.name} ===", flush=True) + try: + export_task(scenario, export_dir) + trial_dir = run_harbor(scenario, export_dir, jobs_dir) + assert_scenario_artifacts(scenario, trial_dir) + except Exception as exc: + print(f"[{scenario.name}] FAILED: {exc}", file=sys.stderr) + failures.append(scenario.name) + + print("\n=== Summary ===") + for scenario in SCENARIOS: + print(f" {scenario.name}: {'FAILED' if scenario.name in failures else 'OK'}") + + if failures: + print(f"\n{len(failures)}/{len(SCENARIOS)} scenario(s) failed: {', '.join(failures)}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/harbor-e2e.yml b/.github/workflows/harbor-e2e.yml new file mode 100644 index 00000000..c123be68 --- /dev/null +++ b/.github/workflows/harbor-e2e.yml @@ -0,0 +1,97 @@ +name: Harbor E2E + +# Deliberately NOT triggered on pull_request: this exercises real Docker +# builds, a real `harbor` install, and (for the llm_judge scenario) a real +# model call, so it is informational rather than a required PR check for now +# (see .github/scripts/harbor_e2e.py's module docstring). workflow_dispatch +# lets a maintainer run it on demand; the nightly schedule catches drift +# between coder-eval's own release and Harbor's own upstream releases without +# blocking anyone's PR. +on: + workflow_dispatch: + schedule: + - cron: "17 5 * * *" # nightly, off the hour to avoid GitHub's peak-load pile-up + push: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + TELEMETRY_ENABLED: "false" + # Route through Bedrock, mirroring pr-checks.yml's smoke-pass job -- keeps + # Anthropic-credit spend off this path; DirectRoute is exercised elsewhere. + API_BACKEND: "bedrock" + CLAUDE_CODE_USE_BEDROCK: "1" + AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }} + AWS_REGION: ${{ secrets.AWS_REGION }} + BEDROCK_MODEL: ${{ secrets.BEDROCK_MODEL }} + # Pinned to the version this integration was built and verified against + # (see src/coder_eval/harbor/*.py's module docstrings) -- bump deliberately, + # not implicitly via an unpinned `harbor` install. + HARBOR_VERSION: "0.22.0" + +jobs: + harbor-e2e: + name: Harbor export + CoderEvalAgent round trip + runs-on: uipath-ubuntu-latest + timeout-minutes: 20 + + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Python 3.13 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.13" + + - name: Set up Node.js 20 + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "20" + + - name: Install Claude CLI + run: npm install -g @anthropic-ai/claude-code + + - name: Cache dependencies + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: | + ~/.cache/uv + ~/.cache/pip + key: ${{ runner.os }}-py3.13-harbor-e2e-${{ env.HARBOR_VERSION }}-${{ hashFiles('pyproject.toml', 'uv.lock') }} + restore-keys: | + ${{ runner.os }}-py3.13-harbor-e2e-${{ env.HARBOR_VERSION }}- + ${{ runner.os }}-py3.13-harbor-e2e- + + - name: Install uv + run: | + python -m pip install --upgrade "pip>=26.2" + pip install uv + + - name: Install project dependencies (hash-verified from uv.lock) + run: uv sync --frozen --extra dev + + - name: Install harbor into the project venv + # Same venv as coder-eval: CoderEvalAgent is resolved by `hb run -a + # coder_eval.harbor.agent:CoderEvalAgent` on the HOST process, so + # `harbor` and `coder_eval` must be importable from the same + # interpreter (see harbor/agent.py's module docstring). + run: uv pip install --python .venv/bin/python "harbor==${{ env.HARBOR_VERSION }}" + + - name: Put the project venv on PATH + run: echo "${{ github.workspace }}/.venv/bin" >> "$GITHUB_PATH" + + - name: Build coder-eval-agent base Docker image + run: make docker-image + + - name: Build BYOD template Docker image + run: docker build -t byod-custom-image:0.1.0 templates/byod_smoke_test/ + + - name: Run Harbor E2E scenarios + run: python .github/scripts/harbor_e2e.py diff --git a/tests/harbor_e2e/fixtures/docker_baseline.yaml b/tests/harbor_e2e/fixtures/docker_baseline.yaml new file mode 100644 index 00000000..a078b5b2 --- /dev/null +++ b/tests/harbor_e2e/fixtures/docker_baseline.yaml @@ -0,0 +1,35 @@ +task_id: "harbor_e2e_baseline" +description: > + Harbor E2E baseline: a real claude-code agent turn against the default + coder-eval-agent image (no dockerfile_path, no template_sources, no + llm_judge). Exercises the plain export -> CoderEvalAgent -> --workspace-dir + -> verifier round trip with nothing else layered on top. + +agent: + type: "claude-code" + permission_mode: "acceptEdits" + allowed_tools: ["Read", "Write", "Bash"] + +sandbox: + driver: docker + docker: + image: coder-eval-agent:latest + +initial_prompt: > + Create a Python file named app.py in the current working directory that + prints 'Hello, Claude!' on one line, and today's date in YYYY-MM-DD format + on the next line. Use the datetime module. Then run the script with: + python app.py + +success_criteria: + - type: "file_exists" + path: "app.py" + description: "The file app.py must be created." + - type: "file_contains" + path: "app.py" + includes: ["Hello, Claude!", "datetime"] + description: "The script must contain the required string and import." + - type: "run_command" + command: "python app.py" + timeout: 10 + description: "The script must execute successfully." diff --git a/tests/harbor_e2e/fixtures/llm_judge.yaml b/tests/harbor_e2e/fixtures/llm_judge.yaml new file mode 100644 index 00000000..d8f1cbb7 --- /dev/null +++ b/tests/harbor_e2e/fixtures/llm_judge.yaml @@ -0,0 +1,44 @@ +task_id: "harbor_e2e_llm_judge" +description: > + Harbor E2E llm_judge case: a real claude-code agent turn, graded by a real + llm_judge call inside the Harbor verifier phase (--allow-credentials). + Catches regressions in judge-prompt construction / route resolution / JSON + verdict parsing when run through a Harbor-exported task, not just through + coder-eval's own `run`/`evaluate`. + +run_limits: + max_turns: 2 + +agent: + type: "claude-code" + permission_mode: "acceptEdits" + allowed_tools: ["Write"] + +sandbox: + driver: docker + docker: + image: coder-eval-agent:latest + +initial_prompt: > + Use the Write tool to create poem.txt containing a 2-line poem about the + Python programming language. The poem must mention 'Python' explicitly and + must be exactly 2 lines (one newline between them, no trailing blank line). + +success_criteria: + - type: "file_exists" + path: "poem.txt" + description: "Sanity check: agent must have written the file." + - type: "llm_judge" + files: ["poem.txt"] + max_tokens: 200 + pass_threshold: 0.8 + prompt: > + You are grading a 2-line poem about Python. The agent was instructed to + produce exactly 2 lines and explicitly mention 'Python'. + + Score 1.0 if the file contains exactly 2 non-empty lines AND the word + 'Python' (case-insensitive) appears at least once. Score 0.0 otherwise. + Do not penalize style, rhyme, or meter; this is a structural check. + + Return JSON: {"score": <0.0|1.0>, "rationale": ""} + description: "LLM-graded structural check on the poem output." diff --git a/tests/harbor_e2e/fixtures/template_sources.yaml b/tests/harbor_e2e/fixtures/template_sources.yaml new file mode 100644 index 00000000..5ada088e --- /dev/null +++ b/tests/harbor_e2e/fixtures/template_sources.yaml @@ -0,0 +1,38 @@ +task_id: "harbor_e2e_template_sources" +description: > + Harbor E2E template_sources case: a real claude-code agent turn against a + task using sandbox.template_sources + sandbox.python.env_packages, on + driver: docker. Exercises the packager copying the TemplateDirSource + directory into environment/templates/ and rewriting its path, plus + environment/task.yaml's sandbox block being field-merged (not replaced). + +run_limits: + expected_turns: 3 + +agent: + type: "claude-code" + permission_mode: "acceptEdits" + allowed_tools: ["Bash"] + +sandbox: + driver: docker + docker: + image: coder-eval-agent:latest + python: + env_packages: + - pytest + template_sources: + - type: template_dir + path: "../../../templates/fibonacci-starter" + +initial_prompt: > + Implement the fibonacci function in src/main.py. + The file has a stub function already defined with a docstring. + The tests in tests/test_main.py define the expected behavior. + Make all tests pass. + +success_criteria: + - type: run_command + description: "All tests pass" + command: "python -m pytest tests/" + timeout: 60 From 6c47e1f6b92fb9db9f063a7f0a49dd341551b13d Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Fri, 11 Sep 2026 11:32:18 -0700 Subject: [PATCH 07/12] fix(harbor): suppress pyright reportMissingImports for the harbor package `harbor` is intentionally not a project dependency -- coder_eval.harbor.agent only ever runs inside a Harbor trial container, where it's already present (see the module's own ImportError message). CI's Quality Gate installs the codex/litellm extras, which cleared those pre-existing pyright errors, so these 3 harbor.* import errors were the only real failures blocking PR #166 -- scoped `# pyright: ignore[reportMissingImports]` on the same 3 lines, matching the existing pattern for openai_codex's optional import in codex_agent.py. Co-Authored-By: Claude Sonnet 5 --- src/coder_eval/harbor/agent.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coder_eval/harbor/agent.py b/src/coder_eval/harbor/agent.py index 345cefe5..98a8a31c 100644 --- a/src/coder_eval/harbor/agent.py +++ b/src/coder_eval/harbor/agent.py @@ -49,11 +49,11 @@ if TYPE_CHECKING: - from harbor.environments.base import BaseEnvironment - from harbor.models.agent.context import AgentContext + from harbor.environments.base import BaseEnvironment # pyright: ignore[reportMissingImports] + from harbor.models.agent.context import AgentContext # pyright: ignore[reportMissingImports] try: - from harbor.agents.installed.base import BaseInstalledAgent + from harbor.agents.installed.base import BaseInstalledAgent # pyright: ignore[reportMissingImports] except ImportError as e: # pragma: no cover - exercised only where `harbor` is installed raise ImportError( "coder_eval.harbor.agent requires the `harbor` package (not installed here). This module is " From 502916b891c8a025e2442b261b62665cc1942f81 Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Fri, 11 Sep 2026 11:46:29 -0700 Subject: [PATCH 08/12] fix(tests): harden harbor CLI-error assertions and Windows chmod checks Two more CI-only failures found running PR #166's full Quality Gate/Windows Smoke Test jobs (masked previously by the typecheck failure that aborted before the test step ran): - test_execute_format_harbor.py / test_evaluate_format_harbor.py assert a plain substring ("Unsupported --format", "--trajectory") against a raised Click/Typer error's rendered output. CI renders that error box with color, which splits "--format"/"--trajectory" across ANSI escape sequences and silently breaks a naive substring check -- reproduced locally with FORCE_COLOR=1. Fixed with click.unstyle(result.output) before asserting. - Two harbor packager tests assert the exported test.sh's chmod executable bit, which is meaningless on NTFS (Windows Smoke Test runs on Windows). Skipped/guarded the same way test_sandbox.py already does for other POSIX-only behavior. Co-Authored-By: Claude Sonnet 5 --- tests/test_evaluate_format_harbor.py | 5 ++++- tests/test_execute_format_harbor.py | 6 +++++- tests/test_harbor_export_golden.py | 3 +++ tests/test_harbor_packager.py | 4 +++- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/test_evaluate_format_harbor.py b/tests/test_evaluate_format_harbor.py index 1d21455d..298129cf 100644 --- a/tests/test_evaluate_format_harbor.py +++ b/tests/test_evaluate_format_harbor.py @@ -10,6 +10,7 @@ import json +import click from typer.testing import CliRunner from coder_eval.cli import app @@ -120,4 +121,6 @@ def test_evaluate_format_harbor_requires_trajectory(tmp_path): result = runner.invoke(app, ["evaluate", str(task_file), str(work_dir), "--format", "harbor"]) assert result.exit_code != 0 - assert "--trajectory" in result.output + # click.unstyle strips ANSI color codes -- see test_execute_format_harbor.py's + # equivalent assertion for why a plain substring check is not portable here. + assert "--trajectory" in click.unstyle(result.output) diff --git a/tests/test_execute_format_harbor.py b/tests/test_execute_format_harbor.py index 76c7f12c..55e326ab 100644 --- a/tests/test_execute_format_harbor.py +++ b/tests/test_execute_format_harbor.py @@ -2,6 +2,7 @@ import json +import click import pytest from typer.testing import CliRunner @@ -80,4 +81,7 @@ def test_unknown_format_value_errors_cleanly(tmp_path, success_task): ["execute", str(success_task), "--run-dir", str(run_dir), "--format", "nonsense"], ) assert result.exit_code != 0 - assert "Unsupported --format" in result.output + # click.unstyle strips ANSI color codes -- CI renders Click's own error box + # with color (option names highlighted char-by-char), which would otherwise + # split "--format" across escape sequences and silently break this check. + assert "Unsupported --format" in click.unstyle(result.output) diff --git a/tests/test_harbor_export_golden.py b/tests/test_harbor_export_golden.py index 57f3f4ef..f543ba9b 100644 --- a/tests/test_harbor_export_golden.py +++ b/tests/test_harbor_export_golden.py @@ -26,6 +26,8 @@ class of bug directly, cheaper than enumerating every field in prose. import os from pathlib import Path +import pytest + from coder_eval.harbor.packager import export_task @@ -70,6 +72,7 @@ def test_export_matches_the_committed_golden_tree(tmp_path: Path) -> None: assert actual[rel_path] == expected[rel_path], f"content drifted for {rel_path}" +@pytest.mark.skipif(os.name == "nt", reason="POSIX-only: NTFS has no chmod executable bit") def test_test_sh_is_executable_in_the_golden_tree() -> None: """The executable bit is part of C1.1's contract and isn't captured by file content.""" test_sh = _EXPECTED_DIR / "tests" / "test.sh" diff --git a/tests/test_harbor_packager.py b/tests/test_harbor_packager.py index 3cfba738..0bf522d0 100644 --- a/tests/test_harbor_packager.py +++ b/tests/test_harbor_packager.py @@ -8,6 +8,7 @@ from __future__ import annotations +import os import tomllib from pathlib import Path @@ -129,7 +130,8 @@ def test_test_sh_is_executable_and_references_the_resolved_workdir(self, tmp_pat result = export_task(task_file, out_dir) test_sh = out_dir / "tests" / "test.sh" - assert test_sh.stat().st_mode & 0o111, "test.sh must be executable" + if os.name != "nt": # NTFS has no chmod executable bit + assert test_sh.stat().st_mode & 0o111, "test.sh must be executable" content = test_sh.read_text(encoding="utf-8") assert f'coder-eval evaluate /tests/task.yaml "{result.workdir}"' in content assert "coder-eval harbor reward /logs/verifier --out /logs/verifier/reward.json" in content From c2b79d7dc1065e0e0c2e4b1c6ba1ce0689f50743 Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Fri, 11 Sep 2026 12:32:24 -0700 Subject: [PATCH 09/12] fix(harbor): address code review findings from full 8-axis review Fixes surfaced by /coder-eval-code-review-full over two iterations: - Refuse a task with empty success_criteria under `run`/`evaluate` (grade=True) -- it previously loaded and would silently score FinalStatus.SUCCESS at weighted_score 0.0 since validate_success_criteria was removed to let the criteria-free Harbor agent-phase task.yaml round-trip through `execute`. `execute` (grade=False) still accepts it. Scoped to the post-`--resume` task set so an already-finalized row is never blocked. - Refuse a path-traversal-shaped variant_id/task_id/row_id in experiment export (`_out_subdir`) instead of silently writing outside `-o`'s output directory. - Convert `--workspace-dir` misuse (docker driver, >1 task) from an unhandled ValueError/traceback into a clean typer.BadParameter. - Escalate a missing `template_sources` directory during export from a warning to a hard failure -- the agent-phase task.yaml still references it, so a silently-skipped copy shipped an export with no starter code. - Add a `harbor` optional extra (pinned harbor==0.22.0) so harbor/agent.py type-checks against real types instead of scoped pyright ignores; wire it into the two CI jobs that run pyright and simplify harbor-e2e.yml's install step to use it. - New test coverage: harbor/agent.py's run()/populate_context_post_run (previously zero coverage), the empty-criteria and workspace_dir guards, the path-traversal refusal, and the template-source warn vs. hard-fail branches. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/harbor-e2e.yml | 18 +- .github/workflows/pr-checks.yml | 8 +- pyproject.toml | 17 ++ src/coder_eval/cli/run_command.py | 72 ++++- src/coder_eval/harbor/agent.py | 10 +- src/coder_eval/harbor/experiment_packager.py | 29 +- src/coder_eval/harbor/packager.py | 14 +- src/coder_eval/orchestration/batch.py | 2 +- tests/test_detached_grading_guards.py | 112 ++++++++ tests/test_harbor_agent.py | 171 ++++++++++++ tests/test_harbor_experiment_packager.py | 30 +++ tests/test_harbor_packager.py | 37 ++- uv.lock | 265 ++++++++++++++++++- 13 files changed, 738 insertions(+), 47 deletions(-) create mode 100644 tests/test_harbor_agent.py diff --git a/.github/workflows/harbor-e2e.yml b/.github/workflows/harbor-e2e.yml index c123be68..d11933d3 100644 --- a/.github/workflows/harbor-e2e.yml +++ b/.github/workflows/harbor-e2e.yml @@ -30,10 +30,6 @@ env: AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }} AWS_REGION: ${{ secrets.AWS_REGION }} BEDROCK_MODEL: ${{ secrets.BEDROCK_MODEL }} - # Pinned to the version this integration was built and verified against - # (see src/coder_eval/harbor/*.py's module docstrings) -- bump deliberately, - # not implicitly via an unpinned `harbor` install. - HARBOR_VERSION: "0.22.0" jobs: harbor-e2e: @@ -64,9 +60,8 @@ jobs: path: | ~/.cache/uv ~/.cache/pip - key: ${{ runner.os }}-py3.13-harbor-e2e-${{ env.HARBOR_VERSION }}-${{ hashFiles('pyproject.toml', 'uv.lock') }} + key: ${{ runner.os }}-py3.13-harbor-e2e-${{ hashFiles('pyproject.toml', 'uv.lock') }} restore-keys: | - ${{ runner.os }}-py3.13-harbor-e2e-${{ env.HARBOR_VERSION }}- ${{ runner.os }}-py3.13-harbor-e2e- - name: Install uv @@ -75,14 +70,13 @@ jobs: pip install uv - name: Install project dependencies (hash-verified from uv.lock) - run: uv sync --frozen --extra dev - - - name: Install harbor into the project venv - # Same venv as coder-eval: CoderEvalAgent is resolved by `hb run -a + # --extra harbor installs `harbor` into the SAME venv as coder-eval: + # CoderEvalAgent is resolved by `hb run -a # coder_eval.harbor.agent:CoderEvalAgent` on the HOST process, so # `harbor` and `coder_eval` must be importable from the same - # interpreter (see harbor/agent.py's module docstring). - run: uv pip install --python .venv/bin/python "harbor==${{ env.HARBOR_VERSION }}" + # interpreter (see harbor/agent.py's module docstring). The version is + # pinned once, in pyproject.toml's `harbor` extra -- bump it there. + run: uv sync --frozen --extra dev --extra harbor - name: Put the project venv on PATH run: echo "${{ github.workspace }}/.venv/bin" >> "$GITHUB_PATH" diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 58c450cd..cd0150be 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -80,7 +80,9 @@ jobs: pip install uv - name: Install project dependencies (hash-verified from uv.lock) - run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm + # --extra harbor: pyright below type-checks src/coder_eval/harbor/agent.py + # against harbor's real types, not a scoped ignore. + run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm --extra harbor # PHASE 1: Fast checks (fail early) - name: Check code formatting (ruff format) @@ -385,7 +387,9 @@ jobs: pip install uv - name: Install project dependencies (hash-verified from uv.lock) - run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm + # --extra harbor: pyright below type-checks src/coder_eval/harbor/agent.py + # against harbor's real types, not a scoped ignore. + run: uv sync --frozen --extra dev --extra uipath --extra codex --extra litellm --extra harbor - name: Check code formatting (ruff format) run: .venv/Scripts/ruff format --check src/ tests/ diff --git a/pyproject.toml b/pyproject.toml index 234c7921..fab9a43e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -161,6 +161,23 @@ opencode = [] # the CLI on PATH the framework still installs and runs; Pi tasks fail at start() # with a clear hint pointing here. See docs/agents/PI.md. pi = [] +# Optional extra for the Harbor-interop direction: coder-eval AS a Harbor agent +# (`coder_eval.harbor.agent:CoderEvalAgent`, a `harbor.agents.installed.base.BaseInstalledAgent` +# subclass) run via `harbor run -a coder_eval.harbor.agent:CoderEvalAgent`. `harbor` itself +# is never imported by anything OUTSIDE `coder_eval.harbor` (the export/packager side of +# the integration reads/writes Harbor's task.toml directly and needs no `harbor` import at +# all), so this extra exists only to satisfy that one module -- a task-image author (or a CI +# job driving `hb run`) installs it with `pip install 'coder-eval[harbor]'` into the SAME +# interpreter that also has `coder_eval`, since Harbor resolves the agent class on the host +# process (see harbor/agent.py's module docstring). Pinned exactly, mirroring the other +# harness pins: Harbor's own trial/environment/agent-context models are a load-bearing +# runtime contract for this class, not a loose API. Without this extra the framework still +# installs and runs; `coder_eval.harbor.agent` fails to import with a clear hint pointing +# back here (it is not needed to EXPORT a task to Harbor format, only to run coder-eval +# itself as Harbor's agent). +harbor = [ + "harbor==0.22.0", +] [project.scripts] coder-eval = "coder_eval.cli:app" diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py index 9c94e740..f3b5cacb 100644 --- a/src/coder_eval/cli/run_command.py +++ b/src/coder_eval/cli/run_command.py @@ -980,6 +980,40 @@ def _reject_simulation_under_execute(resolved: list[ResolvedTask], *, grade: boo ) +def _reject_empty_criteria_under_grade(resolved: list[ResolvedTask], *, grade: bool) -> None: + """Refuse a task with zero ``success_criteria`` under ``run``/``evaluate`` rather than scoring it. + + ``TaskDefinition.success_criteria`` accepts an empty list at the model level + (needed so the Harbor agent-phase ``task.yaml`` -- criteria-free by design, + see ``harbor/packager.py::_write_agent_phase_task_yaml`` -- can round-trip + through ``coder-eval execute``, which never grades). But `EvaluationResult`'s + scoring is vacuous over an empty list: `all_criteria_passed` returns `True` + and `calculate_weighted_score` returns `0.0`, so a criteria-free task graded + under `run` would silently finalize as `FinalStatus.SUCCESS` with + `weighted_score: 0.0` -- an internally contradictory "successful" result for + what is actually a misconfigured task (a typo, a bad merge, a `-D` override + that cleared the list). `execute` (`grade=False`) is exactly the case this + is legal for, so the check is scoped to `grade` the same way + ``_reject_simulation_under_execute`` scopes its own check. + + Callers must pass the POST-`--resume` set (``to_run``, not the full + ``resolved``): a resumed, already-finalized row is folded back from + ``prior_results`` and never re-executed or re-graded, so its own + (possibly empty) criteria are moot to this run and must not block one + that is not actually going to grade it. + """ + if not grade: + return + empty = sorted(rt.task.task_id for rt in resolved if not rt.task.success_criteria) + if empty: + raise typer.BadParameter( + "task(s) with no `success_criteria` cannot be graded (they would silently score " + + "SUCCESS at weighted_score 0.0): " + + ", ".join(empty) + + ". Add at least one criterion, or use `coder-eval execute` to run without grading." + ) + + async def _run_with_experiment( all_task_files: list[Path], config: BatchRunConfig, @@ -1098,21 +1132,35 @@ async def _run_with_experiment( resolved, grade=grade, allow_host_grading=allow_host_grading ) + # Checked against `to_run`, not `resolved`: a `--resume` peels off tasks + # already finalized (folded back from `prior_results`, never re-executed + # or re-graded), so an already-finalized row with empty success_criteria + # (e.g. it was originally run via `execute`) must not block a `run + # --resume` that isn't actually going to grade it. + _reject_empty_criteria_under_grade(to_run, grade=grade) + # Print execution mode print_execution_mode(len(to_run), max_parallel) - summary, task_results = await _run_with_callbacks( - execute_fn=lambda **kwargs: run_batch( - resolved_tasks=to_run, - config=config, - skipped_tasks=skipped, - prior_results=prior_results, - prior_resolved=prior_resolved, - **kwargs, - ), - task_count=len(to_run), - stream_mode=stream_mode, - ) + try: + summary, task_results = await _run_with_callbacks( + execute_fn=lambda **kwargs: run_batch( + resolved_tasks=to_run, + config=config, + skipped_tasks=skipped, + prior_results=prior_results, + prior_resolved=prior_resolved, + **kwargs, + ), + task_count=len(to_run), + stream_mode=stream_mode, + ) + except ValueError as e: + # run_batch's own resolution-time guards (e.g. --workspace-dir requiring + # exactly one non-docker task) raise a plain ValueError -- convert it to + # the same clean CLI error every other resolution-time refusal in this + # function gets, instead of an unhandled traceback. + raise typer.BadParameter(str(e)) from e # Generate experiment reports experiment_result = aggregate_results( diff --git a/src/coder_eval/harbor/agent.py b/src/coder_eval/harbor/agent.py index 98a8a31c..e4fb4fe3 100644 --- a/src/coder_eval/harbor/agent.py +++ b/src/coder_eval/harbor/agent.py @@ -49,17 +49,17 @@ if TYPE_CHECKING: - from harbor.environments.base import BaseEnvironment # pyright: ignore[reportMissingImports] - from harbor.models.agent.context import AgentContext # pyright: ignore[reportMissingImports] + from harbor.environments.base import BaseEnvironment + from harbor.models.agent.context import AgentContext try: - from harbor.agents.installed.base import BaseInstalledAgent # pyright: ignore[reportMissingImports] + from harbor.agents.installed.base import BaseInstalledAgent except ImportError as e: # pragma: no cover - exercised only where `harbor` is installed raise ImportError( "coder_eval.harbor.agent requires the `harbor` package (not installed here). This module is " + "meant to run INSIDE a Harbor trial container, where `harbor` is already present -- it is not " - + "a coder-eval runtime dependency. If you are trying to use this as a Harbor agent, install/pin " - + "`harbor` in the image the packager builds." + + "a coder-eval runtime dependency. Install it with `pip install 'coder-eval[harbor]'` into the " + + "same interpreter as coder_eval (the image the packager builds, or a `hb run` host)." ) from e try: diff --git a/src/coder_eval/harbor/experiment_packager.py b/src/coder_eval/harbor/experiment_packager.py index 86f90d11..55a110b5 100644 --- a/src/coder_eval/harbor/experiment_packager.py +++ b/src/coder_eval/harbor/experiment_packager.py @@ -98,13 +98,39 @@ def _unhonorable_override_reason(resolved: ResolvedTask) -> str | None: return None +class UnsafeExportPathError(ValueError): + """A resolved ``variant_id``/``task_id``/``row_id`` would export outside ``out_dir``.""" + + def _out_subdir(out_dir: Path, resolved: ResolvedTask, *, needs_replicate_segment: bool) -> Path: - """``///[/]rep/`` — row/replicate segments only when they fan out.""" + """``///[/]rep/`` — row/replicate segments only when they fan out. + + ``variant_id``/``task_id``/``row_id`` are free-form, author-controlled + strings (``row_id`` in particular comes straight from a dataset row's + ``id_field``, which may be sourced from an external CSV/JSONL — less + trusted than the task YAML itself). None of them are validated as + filesystem-safe elsewhere, so a value like ``"../../../etc"`` would + otherwise let a crafted experiment/dataset write Harbor's exported + directory (including an executable ``tests/test.sh``) anywhere the + invoking user can write. Resolve the computed destination and refuse it + outright if it would land outside ``out_dir``, rather than trusting the + segments to be well-formed. + """ + out_dir_resolved = out_dir.resolve() dest = out_dir / resolved.variant_id / resolved.task.task_id if resolved.task.row_id: dest = dest / resolved.task.row_id if needs_replicate_segment: dest = dest / f"rep{resolved.replicate_index:02d}" + # `resolve()` on a path that doesn't exist yet still normalizes `..` + # segments against its (existing) parents, so this catches traversal + # without requiring `dest` to already exist. + if not dest.resolve().is_relative_to(out_dir_resolved): + raise UnsafeExportPathError( + f"resolved export path for variant {resolved.variant_id!r}, task {resolved.task.task_id!r} " + + f"would land outside the output directory ({dest.resolve()} is not under {out_dir_resolved}) -- " + + "check variant_id/task_id/row_id for path-traversal sequences." + ) return dest @@ -196,5 +222,6 @@ def export_experiment( __all__ = [ "ExperimentExportResult", "SkippedVariantExport", + "UnsafeExportPathError", "export_experiment", ] diff --git a/src/coder_eval/harbor/packager.py b/src/coder_eval/harbor/packager.py index 2d891bec..801eaa6e 100644 --- a/src/coder_eval/harbor/packager.py +++ b/src/coder_eval/harbor/packager.py @@ -433,11 +433,17 @@ def _copy_template_sources(task: TaskDefinition, env_dir: Path, warnings: list[s dest_name = f"{i:02d}-{source_path.name}" dest = templates_dir / dest_name if not source_path.is_dir(): - warnings.append( - f"template_sources[{i}].path {source_path} is not a directory -- not copied into the export." + # A hard failure, not a warning: the agent-phase task.yaml + # still references this template (starter files, or a pytest + # suite the prompt expects), so a silently-skipped copy ships + # an export whose agent has no starter code -- every criterion + # then reads "file does not exist" indistinguishable from a + # real agent failure (the exact CE039 anti-pattern, one layer + # up at the export boundary instead of the grading boundary). + raise TaskNotExportableError( + f"template_sources[{i}].path {source_path} is not a directory -- cannot copy it into the " + + "export. Fix the task's template_sources entry before exporting." ) - rewritten.append(dumped) - continue if dest.exists(): shutil.rmtree(dest) shutil.copytree(source_path, dest) diff --git a/src/coder_eval/orchestration/batch.py b/src/coder_eval/orchestration/batch.py index a78b29b7..2dc2379d 100644 --- a/src/coder_eval/orchestration/batch.py +++ b/src/coder_eval/orchestration/batch.py @@ -124,7 +124,7 @@ async def run_batch( + "It names a single in-place directory every task would otherwise collide on." ) (_workspace_dir_task,) = resolved_tasks - if _workspace_dir_task.task.sandbox is not None and _workspace_dir_task.task.sandbox.driver == "docker": + if _workspace_dir_task.task.sandbox.driver == "docker": raise ValueError( "--workspace-dir is not for sandbox.driver: docker tasks -- the docker driver already " + "aligns automatically via sandbox.docker.working_dir (see DockerRunner)." diff --git a/tests/test_detached_grading_guards.py b/tests/test_detached_grading_guards.py index 2a1c5a28..6cc4cdcc 100644 --- a/tests/test_detached_grading_guards.py +++ b/tests/test_detached_grading_guards.py @@ -91,6 +91,45 @@ def test_run_still_accepts_the_same_simulation_task(tmp_path: Path) -> None: assert "does not support simulation" not in result.output +# -------------------------------------------------------------------------- +# `run` refuses a task with zero success_criteria; `execute` still accepts it +# -------------------------------------------------------------------------- + +_NO_CRITERIA = """task_id: t +description: d +agent: + type: none +success_criteria: [] +""" + + +def test_run_refuses_a_task_with_no_success_criteria(tmp_path: Path) -> None: + """An empty `success_criteria:` scores vacuously (all_criteria_passed([]) + is True, calculate_weighted_score([]) is 0.0), so a graded run of such a + task would silently finalize as SUCCESS at weighted_score 0.0 -- a + misconfigured task, not a real result. `run` must refuse it by name.""" + path = tmp_path / "t.yaml" + path.write_text(_NO_CRITERIA, encoding="utf-8") + + result = runner.invoke(app, ["run", str(path), "--run-dir", str(tmp_path / "r")]) + + assert result.exit_code != 0 + assert "success_criteria" in result.output + assert "t" in result.output + + +def test_execute_still_accepts_a_task_with_no_success_criteria(tmp_path: Path) -> None: + """The control: `execute` never grades, so a criteria-free task.yaml (the + shape the Harbor agent-phase export deliberately produces) is legal there.""" + path = tmp_path / "t.yaml" + path.write_text(_NO_CRITERIA, encoding="utf-8") + + with patch("coder_eval.cli.run_command._run_with_experiment", new=AsyncMock(return_value=(MagicMock(), 0))): + result = runner.invoke(app, ["execute", str(path), "--run-dir", str(tmp_path / "r")]) + + assert "success_criteria" not in result.output + + # -------------------------------------------------------------------------- # The evaluate-only path refuses grade=False # -------------------------------------------------------------------------- @@ -360,3 +399,76 @@ def test_the_recorded_task_defaults_to_the_task_being_run(tmp_path: Path) -> Non ) orch = Orchestrator(task=task, run_dir=tmp_path, variant_id="v") assert orch.recorded_task is task + + +# -------------------------------------------------------------------------- +# `--workspace-dir` misuse is a clean CLI error, not a traceback +# -------------------------------------------------------------------------- + +_DOCKER_TASK = """task_id: t +description: d +initial_prompt: p +agent: + type: claude-code +sandbox: + driver: docker + docker: + image: some-image:latest +success_criteria: + - type: file_exists + path: proof.txt + description: x +""" + + +def test_workspace_dir_with_docker_driver_is_a_clean_cli_error(tmp_path: Path) -> None: + """run_batch's own guard raises a plain ValueError; the CLI must convert it + to typer.BadParameter (exit 2, clean message) instead of an unhandled + traceback -- --workspace-dir is not for sandbox.driver: docker tasks.""" + path = tmp_path / "t.yaml" + path.write_text(_DOCKER_TASK, encoding="utf-8") + + result = runner.invoke( + app, ["run", str(path), "--run-dir", str(tmp_path / "r"), "--workspace-dir", str(tmp_path / "ws")] + ) + + assert result.exit_code != 0 + assert "--workspace-dir" in result.output + assert "Traceback" not in result.output + + +# -------------------------------------------------------------------------- +# The empty-criteria guard checks post-`--resume` `to_run`, not the full +# `resolved` set -- an already-finalized row folded back from prior_results +# is never re-graded, so its own (possibly empty) criteria are moot. +# -------------------------------------------------------------------------- + + +def test_empty_criteria_guard_ignores_an_already_finalized_resumed_row(tmp_path: Path) -> None: + import typer + + from coder_eval.cli.run_command import _reject_empty_criteria_under_grade + from coder_eval.models import ResolvedTask, TaskDefinition + + finalized_but_empty = ResolvedTask( + task=TaskDefinition( + task_id="already-done", + description="d", + agent=parse_agent_config(type=AgentKind.NONE), + success_criteria=[], + ), + task_file=tmp_path / "t.yaml", + run_dir=tmp_path, + variant_id="v", + ) + + # The full `resolved` set (pre-resume) still refuses when actually graded + # -- this is the control, proving the guard is not simply disabled. + with pytest.raises(typer.BadParameter): + _reject_empty_criteria_under_grade([finalized_but_empty], grade=True) + + # But `to_run` (post-resume) is what a real call site must pass: an + # already-finalized row is peeled off by `_apply_resume` and folded back + # from `prior_results` without being re-graded, so it is NOT in `to_run` + # -- the empty `to_run` a resumed run would actually see must not raise. + _reject_empty_criteria_under_grade([], grade=True) diff --git a/tests/test_harbor_agent.py b/tests/test_harbor_agent.py new file mode 100644 index 00000000..d012533b --- /dev/null +++ b/tests/test_harbor_agent.py @@ -0,0 +1,171 @@ +"""``coder_eval.harbor.agent.CoderEvalAgent`` — the Harbor-agent extension point. + +``harbor`` is not a project dependency (it only needs to be present INSIDE a +Harbor trial container, see ``harbor/agent.py``'s module docstring), so this +module cannot simply be imported in a normal test run. Rather than skip it +entirely (leaving `run()`'s command construction and +`populate_context_post_run()`'s trajectory parsing with zero coverage), stub +just enough of the `harbor` package tree to satisfy the one import +(`harbor.agents.installed.base.BaseInstalledAgent`) and import the real module +against that stub. +""" + +from __future__ import annotations + +import sys +import types +from pathlib import Path + +import pytest + +from coder_eval.harbor.atif_models import AtifAgent, FinalMetrics, Step, Trajectory + + +class _FakeBaseInstalledAgent: + """Just enough of Harbor's `BaseInstalledAgent` for `CoderEvalAgent` to subclass.""" + + +@pytest.fixture +def coder_eval_agent_module(monkeypatch: pytest.MonkeyPatch): + """Import `coder_eval.harbor.agent` against a stubbed `harbor` package tree.""" + harbor_mod = types.ModuleType("harbor") + agents_mod = types.ModuleType("harbor.agents") + installed_mod = types.ModuleType("harbor.agents.installed") + base_mod = types.ModuleType("harbor.agents.installed.base") + base_mod.BaseInstalledAgent = _FakeBaseInstalledAgent # type: ignore[attr-defined] + + for name, mod in ( + ("harbor", harbor_mod), + ("harbor.agents", agents_mod), + ("harbor.agents.installed", installed_mod), + ("harbor.agents.installed.base", base_mod), + ): + monkeypatch.setitem(sys.modules, name, mod) + + # `coder_eval.harbor.agent` may already be cached (unlikely -- nothing else + # imports it), but force a fresh import against the stub either way. + monkeypatch.delitem(sys.modules, "coder_eval.harbor.agent", raising=False) + import importlib + + return importlib.import_module("coder_eval.harbor.agent") + + +def _make_agent(module, *, logs_dir: Path): + """Build a `CoderEvalAgent` instance without running Harbor's real `__init__`.""" + agent = object.__new__(module.CoderEvalAgent) + agent.logs_dir = logs_dir + agent.environment_logs_dir = logs_dir + import logging + + agent.logger = logging.getLogger("test-coder-eval-agent") + return agent + + +class TestRunCommandConstruction: + async def test_run_shells_out_to_execute_with_workspace_dir_and_run_dir(self, coder_eval_agent_module, tmp_path): + """`--workspace-dir "$(pwd)"` is Gap 2's real fix: without it the agent's + tempdir sandbox writes outside the container's WORKDIR, where Harbor's + verifier phase looks. `--run-dir` must point at `environment_logs_dir` + (bind-mounted from Harbor's `self.logs_dir` on the host).""" + agent = _make_agent(coder_eval_agent_module, logs_dir=tmp_path) + + captured: dict[str, object] = {} + + async def _fake_exec(environment, command): + captured["environment"] = environment + captured["command"] = command + + agent._exec = _fake_exec # type: ignore[method-assign] + + fake_environment = object() + await agent.run("unused instruction", fake_environment, context=None) + + assert captured["environment"] is fake_environment + command = captured["command"] + assert isinstance(command, str) + assert command.startswith("coder-eval execute ") + assert "--format harbor" in command + assert f"--run-dir {tmp_path.as_posix()}" in command + assert '--workspace-dir "$(pwd)"' in command + + +class TestPopulateContextPostRun: + def _write_trajectory(self, path: Path, *, final_metrics: FinalMetrics | None) -> None: + trajectory = Trajectory( + agent=AtifAgent(name="coder-eval", version="0.0.0"), + steps=[Step(step_id=1, source="agent", message="did the thing")], + final_metrics=final_metrics, + ) + path.write_text(trajectory.model_dump_json(exclude_none=True), encoding="utf-8") + + def test_fills_cost_and_token_fields_from_trajectory_json(self, coder_eval_agent_module, tmp_path): + agent = _make_agent(coder_eval_agent_module, logs_dir=tmp_path) + self._write_trajectory( + tmp_path / "trajectory.json", + final_metrics=FinalMetrics( + total_prompt_tokens=100, + total_completion_tokens=50, + total_cached_tokens=10, + total_cost_usd=0.25, + ), + ) + + class _Context: + cost_usd = None + n_input_tokens = None + n_cache_tokens = None + n_output_tokens = None + + context = _Context() + agent.populate_context_post_run(context) + + assert context.cost_usd == 0.25 + assert context.n_input_tokens == 100 + assert context.n_cache_tokens == 10 + assert context.n_output_tokens == 50 + + def test_no_trajectory_json_is_a_silent_noop(self, coder_eval_agent_module, tmp_path): + """coder-eval execute may have crashed before writing anything -- this + must not raise, since it runs on every trial regardless.""" + agent = _make_agent(coder_eval_agent_module, logs_dir=tmp_path) + + class _Context: + cost_usd = "untouched" + + context = _Context() + agent.populate_context_post_run(context) + + assert context.cost_usd == "untouched" + + def test_trajectory_with_no_final_metrics_is_a_silent_noop(self, coder_eval_agent_module, tmp_path): + agent = _make_agent(coder_eval_agent_module, logs_dir=tmp_path) + self._write_trajectory(tmp_path / "trajectory.json", final_metrics=None) + + class _Context: + cost_usd = "untouched" + + context = _Context() + agent.populate_context_post_run(context) + + assert context.cost_usd == "untouched" + + def test_malformed_trajectory_json_is_a_silent_noop(self, coder_eval_agent_module, tmp_path): + """Best-effort context enrichment: a parse failure must never fail the trial.""" + agent = _make_agent(coder_eval_agent_module, logs_dir=tmp_path) + (tmp_path / "trajectory.json").write_text("not json", encoding="utf-8") + + class _Context: + cost_usd = "untouched" + + context = _Context() + agent.populate_context_post_run(context) + + assert context.cost_usd == "untouched" + + +def test_name_and_version(coder_eval_agent_module): + from coder_eval import __version__ + + assert coder_eval_agent_module.CoderEvalAgent.name() == "coder-eval" + agent = object.__new__(coder_eval_agent_module.CoderEvalAgent) + assert agent.version() == __version__ diff --git a/tests/test_harbor_experiment_packager.py b/tests/test_harbor_experiment_packager.py index 4715cde9..d834ac7d 100644 --- a/tests/test_harbor_experiment_packager.py +++ b/tests/test_harbor_experiment_packager.py @@ -232,3 +232,33 @@ def test_non_docker_driver_variant_is_skipped_not_fatal(self, tmp_path: Path) -> assert result.exported == [] assert len(result.skipped) == 1 assert "sandbox.driver" in result.skipped[0].reason + + +class TestExportPathContainment: + """A crafted `variant_id`/`task_id`/`row_id` must not escape `out_dir`. + + None of these identifiers are validated as filesystem-safe anywhere in the + resolution chain (`variant_id` is a free string on `ExperimentVariant`), + and `row_id` in particular can come from an externally-sourced dataset + row. `_out_subdir` must refuse rather than silently write outside + `out_dir`. + """ + + def test_traversal_variant_id_is_refused(self, tmp_path: Path) -> None: + from coder_eval.harbor.experiment_packager import UnsafeExportPathError + + task_file = _write_task(tmp_path) + exp_file = _write_experiment( + tmp_path, + { + "experiment_id": "evil", + "variants": [{"variant_id": "../../../../tmp/pwned"}], + }, + ) + out_dir = tmp_path / "out" + + with pytest.raises(UnsafeExportPathError): + export_experiment([task_file], exp_file, out_dir) + + # Nothing should have been written outside out_dir. + assert not (tmp_path.parent / "tmp" / "pwned").exists() diff --git a/tests/test_harbor_packager.py b/tests/test_harbor_packager.py index 0bf522d0..7299f2a9 100644 --- a/tests/test_harbor_packager.py +++ b/tests/test_harbor_packager.py @@ -608,7 +608,11 @@ def test_no_templates_dir_or_copy_line_when_the_task_has_no_template_sources(sel dockerfile_text = (out_dir / "environment" / "Dockerfile").read_text(encoding="utf-8") assert "templates" not in dockerfile_text - def test_nonexistent_template_dir_warns_instead_of_crashing(self, tmp_path: Path) -> None: + def test_nonexistent_template_dir_is_a_hard_export_failure(self, tmp_path: Path) -> None: + """A missing template dir is NOT downgraded to a warning: the agent-phase + task.yaml still references it, so a silently-skipped copy would ship an + export whose agent has no starter code -- every criterion then reads + "file does not exist" indistinguishable from a real agent failure.""" missing = tmp_path / "does-not-exist" task_file = _write_task( tmp_path, @@ -622,7 +626,34 @@ def test_nonexistent_template_dir_warns_instead_of_crashing(self, tmp_path: Path ) out_dir = tmp_path / "out" - result = export_task(task_file, out_dir) + with pytest.raises(packager.TaskNotExportableError, match="is not a directory"): + export_task(task_file, out_dir) - assert any("is not a directory" in w for w in result.warnings) assert not (out_dir / "environment" / "templates").exists() + + def test_non_template_dir_source_is_carried_over_unchanged_with_a_warning(self, tmp_path: Path) -> None: + """A `RepoSource`/`StarterFilesSource` template_sources entry resolves + entirely inside the container already (clone at runtime / inline file + content) -- it is legitimately not locally copyable, so it warns and is + carried over unchanged rather than failing the export.""" + task_file = _write_task( + tmp_path, + { + "sandbox": { + "driver": "docker", + "docker": {"image": "byod-custom-image:0.1.0", "network": "none"}, + "template_sources": [ + {"type": "starter_files", "files": [{"path": "README.md", "content": "hello"}]}, + ], + } + }, + ) + out_dir = tmp_path / "out" + + result = export_task(task_file, out_dir) + + assert any("StarterFilesSource" in w and "not a TemplateDirSource" in w for w in result.warnings) + emitted = yaml.safe_load((out_dir / "environment" / "task.yaml").read_text(encoding="utf-8")) + template_sources = emitted["sandbox"]["template_sources"] + assert len(template_sources) == 1 + assert template_sources[0]["files"] == [{"path": "README.md", "content": "hello"}] diff --git a/uv.lock b/uv.lock index b738e789..990baa92 100644 --- a/uv.lock +++ b/uv.lock @@ -521,6 +521,9 @@ dev = [ { name = "pytest-xdist", extra = ["psutil"] }, { name = "ruff" }, ] +harbor = [ + { name = "harbor" }, +] litellm = [ { name = "litellm" }, ] @@ -538,6 +541,7 @@ requires-dist = [ { name = "click", specifier = ">=8.3.3" }, { name = "defusedxml", marker = "extra == 'dev'", specifier = ">=0.7.1" }, { name = "google-antigravity", marker = "extra == 'antigravity'", specifier = "==0.1.8" }, + { name = "harbor", marker = "extra == 'harbor'", specifier = "==0.22.0" }, { name = "httpx2", specifier = ">=2.12.0,<3.0.0" }, { name = "jmespath", specifier = ">=1.1.0" }, { name = "jsonschema", specifier = ">=4.26.0" }, @@ -566,7 +570,7 @@ requires-dist = [ { name = "typer", specifier = ">=0.24.1" }, { name = "uipath", marker = "extra == 'uipath'", specifier = ">=2.10.31" }, ] -provides-extras = ["dev", "uipath", "litellm", "codex", "antigravity", "opencode", "pi"] +provides-extras = ["dev", "uipath", "litellm", "codex", "antigravity", "opencode", "pi", "harbor"] [[package]] name = "colorama" @@ -720,6 +724,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, ] +[[package]] +name = "deprecation" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/d3/8ae2869247df154b64c1884d7346d412fed0c49df84db635aab2d1c40e62/deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff", size = 173788, upload-time = "2020-04-20T14:23:38.738Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/c3/253a89ee03fc9b9682f1541728eb66db7db22148cd94f89ab22528cd1e1b/deprecation-2.1.0-py2.py3-none-any.whl", hash = "sha256:a10811591210e1fb0e768a8c25517cabeabcba6f0bf96564f8ff45189f90b14a", size = 11178, upload-time = "2020-04-20T14:23:36.581Z" }, +] + +[[package]] +name = "dirhash" +version = "0.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "scantree" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1d/70/49f93897f3a4f7ab5f20a854ebc91aad47854e9fb2cd169e3a4452fa3f5e/dirhash-0.5.0.tar.gz", hash = "sha256:e60760f0ab2e935d8cb088923ea2c6492398dca42cec785df778985fd4cd5386", size = 21377, upload-time = "2024-08-03T22:14:13.322Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/1f/c8bf92552b7f0a13b9f12b85e3de8df6d9814240e0f8ce8f37433df028b3/dirhash-0.5.0-py3-none-any.whl", hash = "sha256:523dfd6b058c64f45b31604376926c6e2bd2ea301d0df23095d4055674e38b09", size = 13119, upload-time = "2024-08-03T22:14:11.688Z" }, +] + [[package]] name = "distlib" version = "0.4.0" @@ -756,6 +784,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, ] +[[package]] +name = "fastapi" +version = "0.141.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8a/02/91e3416a8fdd715abb903a952a6bec7cdd8d14eed55d415fc8595524c319/fastapi-0.141.1.tar.gz", hash = "sha256:e8822fc40db1e1858054d7a949a888695bc9bdce70139178e33bd2871a453ca1", size = 425799, upload-time = "2026-07-29T17:18:05.568Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/03/10388a42375ee7e4ac9b94eb2c5c569c8b5795e377e701c9ac3ad63de890/fastapi-0.141.1-py3-none-any.whl", hash = "sha256:bfb91aa2d334c61cb35ba9a116fc123b3d3df31640b801cf57a7a78ec3f603b3", size = 131954, upload-time = "2026-07-29T17:18:04.364Z" }, +] + [[package]] name = "fastuuid" version = "0.14.0" @@ -788,11 +832,11 @@ wheels = [ [[package]] name = "filelock" -version = "3.25.2" +version = "3.32.6" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/94/b8/00651a0f559862f3bb7d6f7477b192afe3f583cc5e26403b44e59a55ab34/filelock-3.25.2.tar.gz", hash = "sha256:b64ece2b38f4ca29dd3e810287aa8c48182bbecd1ae6e9ae126c9b35f1382694", size = 40480, upload-time = "2026-03-11T20:45:38.487Z" } +sdist = { url = "https://files.pythonhosted.org/packages/38/46/126b1831dca12060d4a8296bf9c4fe5c93c4f22197fa239cb0cc82042bba/filelock-3.32.6.tar.gz", hash = "sha256:a3f55a18af3652a94d8f47d6055df434f254ca1d02ef2524850c6d249ca2512c", size = 225172, upload-time = "2026-09-08T22:57:11.528Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a4/a5/842ae8f0c08b61d6484b52f99a03510a3a72d23141942d216ebe81fefbce/filelock-3.25.2-py3-none-any.whl", hash = "sha256:ca8afb0da15f229774c9ad1b455ed96e85a81373065fb10446672f64444ddf70", size = 26759, upload-time = "2026-03-11T20:45:37.437Z" }, + { url = "https://files.pythonhosted.org/packages/cc/06/4f138f618dbea66803291274f228f01daf29f306fe8b96bc30dab765df75/filelock-3.32.6-py3-none-any.whl", hash = "sha256:3f16ecd0117feae0dfc147e8c62eb5daeccd8bd800378c3ddf416de9b4feb6b1", size = 100189, upload-time = "2026-09-08T22:57:10.182Z" }, ] [[package]] @@ -955,6 +999,51 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] +[[package]] +name = "h2" +version = "4.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "hpack" }, + { name = "hyperframe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e7/85/7c366e69d84c17bb778fe41419e1fbcce3033d5b7ce29bbffff0a98b859f/h2-4.4.1.tar.gz", hash = "sha256:4e866ffb1a869ae14dd9b5e6beb5c24a13da0495ad72b65925ded182521c1516", size = 2157281, upload-time = "2026-08-03T11:45:09.509Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/22/e85faf23bd72a92d1921e37d674ca56eb298a3c8be31fdecef0ff2b3aaac/h2-4.4.1-py3-none-any.whl", hash = "sha256:0e25f1462b23c9cb82d9eb02e28bc706dac2a68cb457c6a0d74d63c8a2a5d0e6", size = 62636, upload-time = "2026-08-03T11:44:59.164Z" }, +] + +[[package]] +name = "harbor" +version = "0.22.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dirhash" }, + { name = "fastapi" }, + { name = "filelock" }, + { name = "httpx" }, + { name = "jinja2" }, + { name = "litellm" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "platformdirs" }, + { name = "pydantic" }, + { name = "pyjwt" }, + { name = "python-dotenv" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "rich" }, + { name = "shortuuid" }, + { name = "supabase" }, + { name = "tenacity" }, + { name = "toml" }, + { name = "typer" }, + { name = "uvicorn" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/12/a0/a37532fb647b93ef709b63a7fbdf613a84837a569fdd88f69dd7105bf093/harbor-0.22.0.tar.gz", hash = "sha256:becf0ce354026cc37899855e0a0d2687cd5188034a43635849245069aad0938b", size = 1832554, upload-time = "2026-08-22T03:22:39.798Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/12/d517f1ca18738f7be78a210b13264c64bace531fdf644632128966aca530/harbor-0.22.0-py3-none-any.whl", hash = "sha256:4c4c6571b3d160ed0cb45b82918136751fb08e7b8596412723ac00dde12eeabb", size = 2058718, upload-time = "2026-08-22T03:22:38.071Z" }, +] + [[package]] name = "hf-xet" version = "1.6.0" @@ -979,6 +1068,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/63/ca511b6f802f28cf3489b280fe77475bcca8de85e81a6299d7916b5b5555/hf_xet-1.6.0-cp38-abi3-win_arm64.whl", hash = "sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3", size = 3859359, upload-time = "2026-08-03T22:33:11.725Z" }, ] +[[package]] +name = "hpack" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/26/5b/fcabf6028144a8723726318b07a32c2f3314acdff6265743cf08a344b18e/hpack-4.2.0.tar.gz", hash = "sha256:0895cfa3b5531fc65fe439c05eb65144f123bf7a394fcaa56aa423548d8e45c0", size = 51300, upload-time = "2026-06-23T18:34:46.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/b4/4a9fcfb2aef6ba44d9073ecd301443aa00b3dac95de5619f2a7de7ec8a91/hpack-4.2.0-py3-none-any.whl", hash = "sha256:858ac0b02280fa582b5080d68db0899c62a80375e0e5413a74970c5e518b6986", size = 34246, upload-time = "2026-06-23T18:34:45.472Z" }, +] + [[package]] name = "httpcore" version = "1.0.9" @@ -1020,6 +1118,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[package.optional-dependencies] +http2 = [ + { name = "h2" }, +] + [[package]] name = "httpx-sse" version = "0.4.3" @@ -1074,6 +1177,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/0e/eafef18f1a75e125e68395db21131db0cf868a128ecd2fce69b4df6c584b/huggingface_hub-1.28.0-py3-none-any.whl", hash = "sha256:58a8bacb03072edfc38067065e9dc24bbb34805410fcd36a1632de0b329660bb", size = 793202, upload-time = "2026-08-18T12:27:12.719Z" }, ] +[[package]] +name = "hyperframe" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/02/e7/94f8232d4a74cc99514c13a9f995811485a6903d48e5d952771ef6322e30/hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08", size = 26566, upload-time = "2025-01-22T21:41:49.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, +] + [[package]] name = "identify" version = "2.6.18" @@ -1741,6 +1853,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/f9/690a8600b93c332de3ab4a344a4ac34f00c8f104917061f779db6a918ed6/pathlib-1.0.1-py3-none-any.whl", hash = "sha256:f35f95ab8b0f59e6d354090350b44a80a80635d22efdedfa84c7ad1cf0a74147", size = 14363, upload-time = "2022-05-04T13:37:20.585Z" }, ] +[[package]] +name = "pathspec" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, +] + [[package]] name = "pip" version = "26.2.1" @@ -1798,11 +1919,11 @@ wheels = [ [[package]] name = "platformdirs" -version = "4.9.4" +version = "4.11.8" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/19/56/8d4c30c8a1d07013911a8fdbd8f89440ef9f08d07a1b50ab8ca8be5a20f9/platformdirs-4.9.4.tar.gz", hash = "sha256:1ec356301b7dc906d83f371c8f487070e99d3ccf9e501686456394622a01a934", size = 28737, upload-time = "2026-03-05T18:34:13.271Z" } +sdist = { url = "https://files.pythonhosted.org/packages/53/18/f3bb8ef0d3b930692343da8aa4d3cbcd6749477c053959395ac81965a6e9/platformdirs-4.11.8.tar.gz", hash = "sha256:f23abafea7dd4276d1f29104b83598d7dcc567cafd07c9c951e66665645437fc", size = 37182, upload-time = "2026-09-08T22:20:42.866Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/63/d7/97f7e3a6abb67d8080dd406fd4df842c2be0efaf712d1c899c32a075027c/platformdirs-4.9.4-py3-none-any.whl", hash = "sha256:68a9a4619a666ea6439f2ff250c12a853cd1cbd5158d258bd824a7df6be2f868", size = 21216, upload-time = "2026-03-05T18:34:12.172Z" }, + { url = "https://files.pythonhosted.org/packages/f4/e1/5b7b8bbb55084d1425bcb9bc823ff519e1b2be05f6ebb0089e2eacc38413/platformdirs-4.11.8-py3-none-any.whl", hash = "sha256:52f2f181bbfde907966932cc8312d967d02976422d66d537ea16092b8e291081", size = 24027, upload-time = "2026-09-08T22:20:41.537Z" }, ] [[package]] @@ -1814,6 +1935,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "postgrest" +version = "2.31.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "deprecation" }, + { name = "httpx", extra = ["http2"] }, + { name = "pydantic" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e2/22/88c470d8838d2678a44e0172d061630b8837cba3fb7fb492e28f6578c309/postgrest-2.31.0.tar.gz", hash = "sha256:2f395d84b2ee34fc57622ff2f711df603e2ede625f98e5015240741888f7bd0c", size = 14419, upload-time = "2026-06-04T13:37:20.474Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/3e/41909586cb148db0259e0208310067afda4cc097f4c7a779e829c1e68c46/postgrest-2.31.0-py3-none-any.whl", hash = "sha256:c2fd47c94e13ee8335111c4f03c9a24ea9766ce9d35fc3cd7330057c9e7ea0c3", size = 23098, upload-time = "2026-06-04T13:37:19.452Z" }, +] + [[package]] name = "pre-commit" version = "4.5.1" @@ -2340,6 +2476,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/93/f7/d00d9b4a0313a6be3a3e0818e6375e15da6d7076f4ae47d1324e7ca986a1/radon-6.0.1-py2.py3-none-any.whl", hash = "sha256:632cc032364a6f8bb1010a2f6a12d0f14bc7e5ede76585ef29dc0cecf4cd8859", size = 52784, upload-time = "2023-03-26T06:24:33.949Z" }, ] +[[package]] +name = "realtime" +version = "2.31.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "typing-extensions" }, + { name = "websockets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/34/54a1eaaefa24db5cb12596fd74792e08efa53ed30dc5bce2c0a68ded6146/realtime-2.31.0.tar.gz", hash = "sha256:9e641cb4d77ca0fe768515f8cf9f83550c79f49ce1550a95afc2dc0e252be8c9", size = 18716, upload-time = "2026-06-04T13:37:22.089Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/60/164246615e8b059f6d53d34648a0784260421ec98a07eb1e45160f063221/realtime-2.31.0-py3-none-any.whl", hash = "sha256:f6e494b53d6a6e80b6efcee6711c8dd40413a52e766271de1bce8ced6c36cc1d", size = 22374, upload-time = "2026-06-04T13:37:21.162Z" }, +] + [[package]] name = "referencing" version = "0.37.0" @@ -2569,6 +2719,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/e7/5c595c75e9f41a44f30e526eda465ea0b4eec93470e074e4a111b253f13a/s3transfer-0.19.2-py3-none-any.whl", hash = "sha256:d8168eccca828cbb2cd573675333f3bddd254313a9c42494b84c76b539e8ba25", size = 90216, upload-time = "2026-07-22T19:30:43.251Z" }, ] +[[package]] +name = "scantree" +version = "0.0.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "pathspec" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/e4/40998faefc72ba1ddeb640a44fba92935353525dba110488806da8339c0b/scantree-0.0.4.tar.gz", hash = "sha256:15bd5cb24483b04db2c70653604e8ea3522e98087db7e38ab8482f053984c0ac", size = 24643, upload-time = "2024-08-03T20:08:59.413Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/ce/828467ddfa0d2fe473673026442d2032d552a168e42cfbf25fd0e5264e0c/scantree-0.0.4-py3-none-any.whl", hash = "sha256:7616ab65aa6b7f16fcf8e6fa1d9afaa99a27ab72bba05c61b691853b96763174", size = 20690, upload-time = "2024-08-03T20:08:58.137Z" }, +] + [[package]] name = "shellingham" version = "1.5.4" @@ -2578,6 +2741,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, ] +[[package]] +name = "shortuuid" +version = "1.0.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/e2/bcf761f3bff95856203f9559baf3741c416071dd200c0fc19fad7f078f86/shortuuid-1.0.13.tar.gz", hash = "sha256:3bb9cf07f606260584b1df46399c0b87dd84773e7b25912b7e391e30797c5e72", size = 9662, upload-time = "2024-03-11T20:11:06.879Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/44/21d6bf170bf40b41396480d8d49ad640bca3f2b02139cd52aa1e272830a5/shortuuid-1.0.13-py3-none-any.whl", hash = "sha256:a482a497300b49b4953e15108a7913244e1bb0d41f9d332f5e9925dba33a3c5a", size = 10529, upload-time = "2024-03-11T20:11:04.807Z" }, +] + [[package]] name = "simple-websocket" version = "1.1.0" @@ -2660,6 +2832,76 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/69/06/36d260a695f383345ab5bbc3fd447249594ae2fa8dfd19c533d5ae23f46b/stevedore-5.7.0-py3-none-any.whl", hash = "sha256:fd25efbb32f1abb4c9e502f385f0018632baac11f9ee5d1b70f88cc5e22ad4ed", size = 54483, upload-time = "2026-02-20T13:27:05.561Z" }, ] +[[package]] +name = "storage3" +version = "2.31.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "deprecation" }, + { name = "httpx", extra = ["http2"] }, + { name = "pydantic" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/13/30/fee43d523d3f680a833a4aae5bf8094de0b9031b0c2bddb3e0bc6e829e1b/storage3-2.31.0.tar.gz", hash = "sha256:d2161e2ea650dc115a1787c30e09b118365589ac772f4dd8643e3a503ecfc667", size = 20348, upload-time = "2026-06-04T13:37:23.703Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/b2/60d86a3a99ae743e8a00a6df912f85269b3bc882ba217b8ce1690881c669/storage3-2.31.0-py3-none-any.whl", hash = "sha256:4bf46e8bea320743179a6beafdc7531c5242495e00e0cc22af7c7a9d69d4ed84", size = 28492, upload-time = "2026-06-04T13:37:22.792Z" }, +] + +[[package]] +name = "strenum" +version = "0.4.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/ad/430fb60d90e1d112a62ff57bdd1f286ec73a2a0331272febfddd21f330e1/StrEnum-0.4.15.tar.gz", hash = "sha256:878fb5ab705442070e4dd1929bb5e2249511c0bcf2b0eeacf3bcd80875c82eff", size = 23384, upload-time = "2023-06-29T22:02:58.399Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/69/297302c5f5f59c862faa31e6cb9a4cd74721cd1e052b38e464c5b402df8b/StrEnum-0.4.15-py3-none-any.whl", hash = "sha256:a30cda4af7cc6b5bf52c8055bc4bf4b2b6b14a93b574626da33df53cf7740659", size = 8851, upload-time = "2023-06-29T22:02:56.947Z" }, +] + +[[package]] +name = "supabase" +version = "2.31.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx" }, + { name = "postgrest" }, + { name = "realtime" }, + { name = "storage3" }, + { name = "supabase-auth" }, + { name = "supabase-functions" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fe/8e/54a2f950629689b1613434a61fc3bff5f92f84ba6b20213f5b2add05c1bb/supabase-2.31.0.tar.gz", hash = "sha256:3467b09d00482b9a0138235bdbde7a350426f93cf2a1342372eaddfc669f1206", size = 9805, upload-time = "2026-06-04T13:37:25.22Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/06/5e6f4bf89dedadf81f893832115c1866da1aa093142c9989c2818780dae3/supabase-2.31.0-py3-none-any.whl", hash = "sha256:25f2a99207a75f2d9377e2332783b4389cf56b02cbebdaf0c1743112dcbb704e", size = 16728, upload-time = "2026-06-04T13:37:24.278Z" }, +] + +[[package]] +name = "supabase-auth" +version = "2.31.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx", extra = ["http2"] }, + { name = "pydantic" }, + { name = "pyjwt", extra = ["crypto"] }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/8a/408689cf39820f0d46d2731d6747ff94dbefc87ae977b4b5c4066da5b070/supabase_auth-2.31.0.tar.gz", hash = "sha256:0945b33fa96239c76dc8eaf96d7d2c94991950d24b4cfe4a5c2da9aa5e909663", size = 39151, upload-time = "2026-06-04T13:37:27.375Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/5e/22f3b0546bb1f0985f06fb1ebf5f6405a3b1bb7a5db01142c26cf148e988/supabase_auth-2.31.0-py3-none-any.whl", hash = "sha256:5e9c8b4ecdee6af04dbcb06455ce78cb15674806fcb6b425170455307d70b0ee", size = 48363, upload-time = "2026-06-04T13:37:26.26Z" }, +] + +[[package]] +name = "supabase-functions" +version = "2.31.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx", extra = ["http2"] }, + { name = "strenum" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/84/5d/61c2446ed26a57fa5543f9c270a731320911569202340d15341f72cdba7c/supabase_functions-2.31.0.tar.gz", hash = "sha256:4ad027b3ae3bd28b31233339f4db1da6965affd3546f655b421baf40cee2690f", size = 4683, upload-time = "2026-06-04T13:37:28.878Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/79/1a8162ce7d705381a4f2668c0da68e097b103c1aa701418a31da52905c7b/supabase_functions-2.31.0-py3-none-any.whl", hash = "sha256:3fdc4c4766152bfda63bdd0e286fc8a06f50e1280711fae4a1dfc9b7e9ebabc6", size = 8794, upload-time = "2026-06-04T13:37:28.022Z" }, +] + [[package]] name = "tenacity" version = "9.1.4" @@ -2743,6 +2985,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cd/2b/2be299bab55fc595e3d38567edb1a87f86e594842968fa9515a07bdcf422/tokenizers-0.23.1-cp310-abi3-win_arm64.whl", hash = "sha256:a26197957d8e4425dfba746315f3c425ea00cfa8367c5fbc4ec73447893dcea9", size = 2664127, upload-time = "2026-04-27T14:43:26.949Z" }, ] +[[package]] +name = "toml" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f", size = 22253, upload-time = "2020-11-01T01:40:22.204Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588, upload-time = "2020-11-01T01:40:20.672Z" }, +] + [[package]] name = "tomli" version = "2.4.0" From 04204912e64d4e392d18ed7c19cd2848fe626686 Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Fri, 11 Sep 2026 12:40:49 -0700 Subject: [PATCH 10/12] fix(test): strip ANSI color before asserting --workspace-dir in CLI output CI renders Click's error box with color, splitting "--workspace-dir" across escape sequences and breaking the new test_workspace_dir_with_docker_driver_is_a_clean_cli_error test that raw-substring-matched it. Same fix as the existing --format/--trajectory tests: assert against click.unstyle(result.output) instead. Co-Authored-By: Claude Sonnet 5 --- tests/test_detached_grading_guards.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_detached_grading_guards.py b/tests/test_detached_grading_guards.py index 6cc4cdcc..f3939774 100644 --- a/tests/test_detached_grading_guards.py +++ b/tests/test_detached_grading_guards.py @@ -12,6 +12,7 @@ from pathlib import Path from unittest.mock import AsyncMock, MagicMock, patch +import click import pytest from typer.testing import CliRunner @@ -433,7 +434,11 @@ def test_workspace_dir_with_docker_driver_is_a_clean_cli_error(tmp_path: Path) - ) assert result.exit_code != 0 - assert "--workspace-dir" in result.output + # click.unstyle strips ANSI color codes -- CI renders Click's error box + # with color (option names highlighted char-by-char), which would + # otherwise split "--workspace-dir" across escape sequences and silently + # break this check (see test_execute_format_harbor.py's equivalent). + assert "--workspace-dir" in click.unstyle(result.output) assert "Traceback" not in result.output From eb0978f6723ed8cf82dd0bea02124d4db33d2af0 Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Fri, 11 Sep 2026 16:39:51 -0700 Subject: [PATCH 11/12] fix(harbor): address PR review blockers (score-integrity, security, drops) Closes out the reviewer's Top-5 priority actions from the full 8-axis review of this branch, plus the two CodeQL findings: - Restore the empty-`success_criteria` refusal at the layers `run --to_run`'s guard didn't cover: `regrade_in_place` (shared by `evaluate ` and `run --resume`'s `to_grade` set), the orchestrator-direct `evaluate` branch, and the resume `to_grade` partition itself. Without this, a criteria-less task graded SUCCESS at weighted_score 0.0 and exited 0. - Widen `write_reward`'s unmeasured-row guard to also refuse `final_status.category == "error"`, since a grading-time infra crash (JudgeInfrastructureError/CheckerMisuseError/ReferenceTamperedError) finalizes ERROR with weighted_score 0.0, not None -- publishing a crash to Harbor as a measured zero. - Read `result_status`/`duration_ms` back out of ATIF's ToolCall/ ObservationResult `extra` in atif_hydrate, so `command_executed (require_success: true)` no longer silently scores 0.0 on the `evaluate --format harbor` path for a command that succeeded. - Replace the two hand-picked export payloads: carry `run_limits` (both agent-phase and verifier task.yaml) and `checker_context` (verifier) through instead of dropping them, and refuse exporting a `dataset:`- backed task (unexpanded `${row.*}` placeholders) or an enabled `simulation:` block. - `_unhonorable_override_reason` no longer skips agent-override variants: CoderEvalAgent (C1.2) now carries `task.agent` verbatim and executes it, so `export -e experiment.yaml` exports one directory per variant instead of exiting with zero. - shlex.quote the task-YAML-controlled `working_dir` interpolated into the generated Harbor verifier `tests/test.sh` (command injection via a crafted `sandbox.docker.working_dir`). - Drop symlinks (never dereference) when copying template/reference trees into an export, matching every other task-authored-tree copy in `src/`; wrap the reference copy's OSError as TaskNotExportableError. - `--` before the task-controlled image name in `docker image inspect`. - Fix the SubagentTrajectoryRef vendored ATIF model to match harbor==0.22.0 exactly (verified against the installed package): all four fields optional, `session_id` added -- the old required-`trajectory_id` shape rejected two of harbor's own valid document forms. - `evaluate --format harbor`'s write-back now checks `target.mode is RUN_DIR`, not just `prior is not None`: a synthetic prior seeded from `--trajectory` on the WORK_DIR shape was planting a spurious task.json into the Harbor-synced work dir. - Narrow the DIRECT_WRITE stale-artifact warning suppression for `workspace_dir` to `IN_CONTAINER_ENV` (CE056) instead of the field itself: `--workspace-dir` is now also a host-reachable CLI flag, where the named directory persists across runs exactly like DIRECT_WRITE's own target. - Remove an unnecessary `del environment` (CodeQL) and fix a test's dual import style (`import X.Y` + `from X.Y import Z`, CodeQL) for the same module. Regenerated the golden export fixture for the `run_limits` addition. Full suite green (5706 passed), ruff/pyright/lint clean. Co-Authored-By: Claude Sonnet 5 --- src/coder_eval/cli/evaluate_command.py | 23 ++++- src/coder_eval/cli/run_command.py | 7 ++ src/coder_eval/harbor/agent.py | 1 - src/coder_eval/harbor/atif_hydrate.py | 10 +++ src/coder_eval/harbor/atif_models.py | 21 ++++- src/coder_eval/harbor/experiment_packager.py | 40 +++------ src/coder_eval/harbor/packager.py | 81 ++++++++++++++++- src/coder_eval/harbor/reward.py | 24 +++-- src/coder_eval/orchestration/regrade.py | 15 ++++ src/coder_eval/orchestrator.py | 17 +++- .../expected/environment/task.yaml | 5 ++ .../expected/tests/task.yaml | 5 ++ tests/test_atif_emit.py | 7 +- tests/test_atif_models.py | 37 ++++++-- tests/test_harbor_experiment_packager.py | 88 ++++++++++++++----- tests/test_orchestrator.py | 73 +++++++++++++++ 16 files changed, 372 insertions(+), 82 deletions(-) diff --git a/src/coder_eval/cli/evaluate_command.py b/src/coder_eval/cli/evaluate_command.py index 050a6a62..d8f47ad5 100644 --- a/src/coder_eval/cli/evaluate_command.py +++ b/src/coder_eval/cli/evaluate_command.py @@ -431,6 +431,19 @@ def run_evaluation( task_description=task.description, ) + if not task.success_criteria: + # `evaluate` always grades -- unlike `execute`, there is no legal reason + # for a zero-criteria task to reach here. `regrade_in_place` guards its + # own delegating branch; this guard covers the sibling orchestrator-direct + # branch below (fresh work-dir grading, or `--copy`), which never calls + # `regrade_in_place` and would otherwise finalize a criteria-free task as + # SUCCESS at weighted_score 0.0. + console.print( + f"[red]✗ Task {task.task_id!r} has no `success_criteria` and cannot be graded " + + "(it would silently score SUCCESS at weighted_score 0.0). Add at least one criterion.[/red]" + ) + raise typer.Exit(1) + grade_in_place = resolve_grade_in_place(target, in_place) try: @@ -612,7 +625,15 @@ def _report_and_exit( if result.sandbox_path: console.print(f"[dim]Artifacts: {result.sandbox_path}[/dim]") - if prior is not None: + # `prior is not None` alone is not enough: `--format harbor` seeds a + # SYNTHETIC prior on the WORK_DIR shape (from the supplied + # `--trajectory`), which is not a run directory and carries no + # `task.execute.json` sibling to preserve. `_write_back` is documented as + # "replace the graded RUN's task.json" and writes into `target.target`, + # which in WORK_DIR mode is the directory being graded, not a run dir -- + # writing there planted a spurious task.json into the Harbor-synced + # workdir and wedged a later `evaluate` on it into RUN_DIR mode. + if prior is not None and target.mode is EvaluateMode.RUN_DIR: console.print( f"[dim]Re-graded {prior.final_status.value} → {result.final_status.value} " + f"over {len(result.iterations)} recorded turn(s).[/dim]" diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py index f3b5cacb..8e234de7 100644 --- a/src/coder_eval/cli/run_command.py +++ b/src/coder_eval/cli/run_command.py @@ -942,6 +942,13 @@ async def _apply_resume( # to_grade is deliberately NOT cleared: its artifacts are the run's output # and the very thing being graded. cleared = clear_rerun_artifacts(part.to_run) + # `to_grade` rows are about to be graded by `_grade_resumed_tasks` below + # (which delegates to `regrade_in_place`), so the same refusal `to_run` + # gets via `_reject_empty_criteria_under_grade` applies here too -- checked + # explicitly rather than relying solely on `regrade_in_place`'s own guard + # so the whole batch is refused up front (exit 2) instead of one row at a + # time turning into a per-task "could not grade" warning mid-resume. + _reject_empty_criteria_under_grade(part.to_grade, grade=grade) console.print( f"[cyan]↻ Resume:[/] {len(prior_results)} task(s) already complete, " + f"running {len(part.to_run)} remaining" diff --git a/src/coder_eval/harbor/agent.py b/src/coder_eval/harbor/agent.py index e4fb4fe3..bccfcab0 100644 --- a/src/coder_eval/harbor/agent.py +++ b/src/coder_eval/harbor/agent.py @@ -95,7 +95,6 @@ async def install(self, environment: BaseEnvironment) -> None: time -- re-installing it here would fight whatever version the image pins. """ - del environment async def run(self, instruction: str, environment: BaseEnvironment, context: AgentContext) -> None: """Run ``coder-eval execute --format harbor`` inside the environment. diff --git a/src/coder_eval/harbor/atif_hydrate.py b/src/coder_eval/harbor/atif_hydrate.py index 509d973d..729fed17 100644 --- a/src/coder_eval/harbor/atif_hydrate.py +++ b/src/coder_eval/harbor/atif_hydrate.py @@ -45,12 +45,22 @@ def _tool_call_to_command( result = results_by_id.get(call.tool_call_id) content = result.content if result is not None else None result_summary = content if isinstance(content, str) else (None if content is None else str(content)) + # The mirror of atif_emit's `_tool_calls_for`: `result_status` rides on the + # ToolCall's own `extra` (call_extra) and `duration_ms` on its matching + # ObservationResult's `extra` (result_extra) -- read both back, or + # `command_executed(require_success: true)` silently scores a successful + # command 0.0 on every hydrated trajectory (result_status defaults to None, + # not "success"). + call_extra = call.extra or {} + result_extra = (result.extra or {}) if result is not None else {} return CommandTelemetry( tool_name=call.function_name, tool_id=call.tool_call_id, timestamp=datetime.now(UTC), parameters=call.arguments, result_summary=result_summary, + result_status=call_extra.get("result_status"), + duration_ms=result_extra.get("duration_ms"), assistant_turn_index=assistant_turn_index, sequence_number=assistant_turn_index, ) diff --git a/src/coder_eval/harbor/atif_models.py b/src/coder_eval/harbor/atif_models.py index 3923ccd0..22ab153b 100644 --- a/src/coder_eval/harbor/atif_models.py +++ b/src/coder_eval/harbor/atif_models.py @@ -1,6 +1,6 @@ """Vendored ATIF (Agent Trajectory Interchange Format) models, v1.7. -Mirrors the schema in ``harbor.models.trajectories`` (harbor 0.20.0) so +Mirrors the schema in ``harbor.models.trajectories`` (harbor 0.22.0) so coder_eval can emit and parse ATIF trajectories with ZERO runtime dependency on the ``harbor`` pip package. Fidelity is guarded by a frozen fixture in ``tests/fixtures/atif/`` that was validated once against the real harbor @@ -10,7 +10,7 @@ - ``schema_version`` is a pattern-validated ``str`` (``^ATIF-v1\\.\\d+$``) instead of a closed Literal, so trajectories written by a FUTURE harbor - minor version (e.g. ``ATIF-v1.9``) still parse — harbor 0.20.0 itself + minor version (e.g. ``ATIF-v1.9``) still parse — harbor 0.22.0 itself would reject them. Major-version bumps (``ATIF-v2.0``) are rejected. - harbor's ``Agent`` model is named :class:`AtifAgent` here to avoid clashing with ``coder_eval.agent.Agent``. @@ -72,11 +72,23 @@ class ToolCall(BaseModel): class SubagentTrajectoryRef(BaseModel): - """Reference to a delegated subagent trajectory.""" + """Reference to a delegated subagent trajectory. + + All four fields optional, mirroring ``harbor==0.22.0``'s real shape + (verified by installing it and diffing ``model_fields``): a document may + carry ``session_id`` alone (no ``trajectory_id``) or ``trajectory_path`` + alone (the spec's file-ref form), and ``extra`` is a documented carry-all. + The earlier version made ``trajectory_id`` required and omitted + ``session_id``/``extra`` entirely, which rejected both of those valid + shapes under ``extra="forbid"``. + """ model_config = ConfigDict(extra="forbid") - trajectory_id: str = Field(description="trajectory_id of the referenced subagent trajectory.") + trajectory_id: str | None = Field(default=None, description="trajectory_id of the referenced subagent trajectory.") + session_id: str | None = Field( + default=None, description="session_id of the referenced subagent trajectory, when addressed that way." + ) trajectory_path: str | None = Field( default=None, description=( @@ -84,6 +96,7 @@ class SubagentTrajectoryRef(BaseModel): "entry in the root trajectory's subagent_trajectories array." ), ) + extra: dict[str, Any] | None = Field(default=None, description="Custom reference-level metadata.") class ObservationResult(BaseModel): diff --git a/src/coder_eval/harbor/experiment_packager.py b/src/coder_eval/harbor/experiment_packager.py index 55a110b5..a8e3e614 100644 --- a/src/coder_eval/harbor/experiment_packager.py +++ b/src/coder_eval/harbor/experiment_packager.py @@ -60,35 +60,19 @@ class ExperimentExportResult: def _unhonorable_override_reason(resolved: ResolvedTask) -> str | None: """Why this resolved task's Harbor export can't honor an experiment-introduced override. - Harbor's agent phase is Harbor's own (or the unbuilt C1.2 - coder-eval-as-agent embed adapter, see ``tmp/harborframework.md`` Part A - step 8) — entirely independent of coder-eval's ``agent``/``simulation`` - config. A variant that changes neither is fine to export (the exported - directory is then identical to the base task's own export, which is the - existing, already-documented single-task limitation — coder-eval's - ``agent`` field is never read for the exported verifier ``tests/task.yaml`` - either). But a variant that *introduces* an agent or simulation override - is the entire point of an A/B experiment: silently dropping it would - export two indistinguishable Harbor directories claiming to compare two - different arms. Detected via ``ResolvedTask.config_lineage`` — a field - counts only when its most-specific source is the variant or the - experiment defaults, never the task's own YAML or the baseline defaults. + An agent override (model/plugins/system_prompt/tools/type) IS honorable + now: ``packager.py::_write_agent_phase_task_yaml`` carries ``task.agent`` + verbatim into ``environment/task.yaml``, which ``CoderEvalAgent.run()`` + (``harbor/agent.py``, C1.2) executes via ``coder-eval execute`` — so a + variant that only touches ``agent.*`` exports its own distinct directory + and is NOT skipped. ``simulation`` is the one override that remains + unhonorable: Harbor's ``Trial`` model has no multi-turn user-simulator + concept, and the dialog's turn-continuation logic reads coder-eval's own + criteria results mid-run, which a single-shot verifier export cannot + represent. Detected via ``ResolvedTask.config_lineage`` — a field counts + only when its most-specific source is the variant or the experiment + defaults, never the task's own YAML or the baseline defaults. """ - agent_sources = sorted( - { - entry.source - for key, entry in resolved.config_lineage.items() - if key.startswith("agent.") and entry.source in _EXPERIMENT_INTRODUCED_SOURCES - } - ) - if agent_sources: - return ( - f"variant {resolved.variant_id!r} sets an agent override (source: {', '.join(agent_sources)}) that " - "Harbor's verifier-only export cannot honor -- Harbor's agent phase is independent of coder-eval's " - "agent config. Building the C1.2 coder-eval-as-agent embed adapter (tmp/harborframework.md, Part A " - "step 8) is the fix; until then this variant is skipped rather than exported as an indistinguishable " - "duplicate of another arm." - ) sim_entry = resolved.config_lineage.get("simulation") if sim_entry is not None and sim_entry.source in _EXPERIMENT_INTRODUCED_SOURCES: return ( diff --git a/src/coder_eval/harbor/packager.py b/src/coder_eval/harbor/packager.py index 801eaa6e..2624534b 100644 --- a/src/coder_eval/harbor/packager.py +++ b/src/coder_eval/harbor/packager.py @@ -44,6 +44,7 @@ from __future__ import annotations +import shlex import shutil import subprocess from dataclasses import dataclass, field @@ -56,6 +57,7 @@ from coder_eval.harbor.portability import PortabilityIssue, audit_criteria from coder_eval.models import TaskDefinition, TemplateDirSource from coder_eval.orchestration.task_loader import load_task +from coder_eval.path_utils import REFERENCE_COPY_IGNORE, ignore_patterns_and_symlinks DEFAULT_WORKDIR = "/app" @@ -170,6 +172,27 @@ def export_resolved_task( + "Set sandbox.driver: docker (with dockerfile_path or a custom image) to export this task." ) + if task.dataset is not None: + # `export_task`/`export_resolved_task` calls `load_task`, which does + # NOT run `expand_dataset` -- fan-out happens later, in the experiment + # pipeline (`export_experiment` resolves it per row before reaching + # here). Exporting a raw dataset-backed task would silently emit ONE + # Harbor task whose prompt and criteria still contain literal + # `${row.}` placeholders: never expressible, but scored anyway. + raise TaskNotExportableError( + f"Task {task.task_id!r} has a `dataset:` block, which this function does not expand -- its " + + "`${row.*}` placeholders would export unsubstituted. Export via `coder-eval export ... " + + "-e ` (harbor.experiment_packager.export_experiment), which resolves one " + + "Harbor task directory per dataset row." + ) + + if task.simulation is not None and task.simulation.enabled: + raise TaskNotExportableError( + f"Task {task.task_id!r} has an enabled `simulation:` block -- its turn-continuation logic reads " + + "coder-eval's own criteria results mid-dialog, which has no Harbor equivalent. Simulation tasks " + + "cannot be exported." + ) + warnings: list[str] = [] if task.pre_run: warnings.append( @@ -343,7 +366,9 @@ def _inspect_image_workdir(image: str) -> str | None: """ try: result = subprocess.run( - ["docker", "image", "inspect", image, "--format", "{{.Config.WorkingDir}}"], + # `--` before `image` (task-YAML-controlled) stops it from being + # parsed as an option if it happens to start with "-". + ["docker", "image", "inspect", "--format", "{{.Config.WorkingDir}}", "--", image], capture_output=True, text=True, encoding="utf-8", @@ -404,6 +429,21 @@ def _write_verifier_task_yaml(task: TaskDefinition, out_dir: Path) -> None: } if task.reference is not None: payload["reference"] = {"directory": "reference"} + if task.run_limits is not None: + # Carried through so `coder-eval evaluate` (run by tests/test.sh inside + # Harbor's verifier container) sees the same turn/token/USD caps the + # task author declared, rather than silently falling back to the + # packaged default experiment's -- grading itself has no agent loop to + # cap, but `run_limits.task_timeout` still bounds the verifier + # invocation, and a future criterion or judge call reading `run_limits` + # off the resolved task should see the real value, not the default. + payload["run_limits"] = task.run_limits.model_dump(mode="json", exclude_none=True) + if task.checker_context is not None: + # The judge-route override (`checker_context.api_route`) determines + # which backend an `llm_judge`/`agent_judge` criterion dispatches + # through at verify time -- dropping it silently replaces a pinned + # route with the verifier environment's own default. + payload["checker_context"] = task.checker_context.model_dump(mode="json", exclude_none=True) (out_dir / "tests" / "task.yaml").write_text( yaml.safe_dump(payload, sort_keys=False, allow_unicode=True), encoding="utf-8" ) @@ -446,7 +486,14 @@ def _copy_template_sources(task: TaskDefinition, env_dir: Path, warnings: list[s ) if dest.exists(): shutil.rmtree(dest) - shutil.copytree(source_path, dest) + # Symlinks dereferenced by default (`shutil.copytree`'s default + # `symlinks=False`) would write a symlink TARGET's content into the + # distributable export -- e.g. a `creds -> /root/.aws/credentials` + # plant. Drop symlinks outright rather than following them, same + # rule as the sibling reference copy below and every other + # task-authored-tree copy in `src/` (`orchestration/evaluation.py`, + # `isolation/docker_runner.py`, `evaluation/sub_agent.py`). + shutil.copytree(source_path, dest, ignore=ignore_patterns_and_symlinks(REFERENCE_COPY_IGNORE)) dumped["path"] = f"{AGENT_TASK_TEMPLATES_DIR}/{dest_name}" else: warnings.append( @@ -511,13 +558,27 @@ def _write_agent_phase_task_yaml( } if not is_agentless: payload["initial_prompt"] = initial_prompt + if task.run_limits is not None: + # `CoderEvalAgent.run()` invokes `coder-eval execute` against this + # file, which enforces `max_turns`/`turn_timeout`/`task_timeout`/the + # token+USD budget caps during the agent phase itself -- dropping this + # silently replaced a declared cap with the packaged default + # experiment's (`max_turns: 100`, `turn_timeout: 300`, no `max_usd` / + # token ceiling at all). + payload["run_limits"] = task.run_limits.model_dump(mode="json", exclude_none=True) (env_dir / "task.yaml").write_text(yaml.safe_dump(payload, sort_keys=False, allow_unicode=True), encoding="utf-8") return (env_dir / "templates").is_dir() def _write_test_sh(out_dir: Path, *, workdir: str) -> None: path = out_dir / "tests" / "test.sh" - path.write_text(_TEST_SH_TEMPLATE.format(workdir=workdir), encoding="utf-8") + # `workdir` comes from task-YAML-controlled `sandbox.docker.working_dir` + # (or a derived default), and the only validator on that field checks for a + # leading "/" -- it does not reject quotes, `$(...)`, backticks or + # newlines. shlex.quote it before interpolating into the generated /bin/sh + # script so a crafted working_dir can't break out of the argument it's + # meant to be. + path.write_text(_TEST_SH_TEMPLATE.format(workdir=shlex.quote(workdir)), encoding="utf-8") path.chmod(0o755) @@ -528,7 +589,19 @@ def _write_reference(task: TaskDefinition, task_file: Path, out_dir: Path) -> No dest = out_dir / "tests" / "reference" if dest.exists(): shutil.rmtree(dest) - shutil.copytree(source, dest) + # Same rule as the template copy above: drop symlinks rather than + # dereferencing them into the distributable export. Wrapped in + # TaskNotExportableError rather than left to raise a bare OSError: the + # `export` CLI only catches `(TaskNotExportableError, + # CriteriaNotExportableError)`, so an unreadable/missing reference tree + # would otherwise surface as an uncaught traceback (single-task path) or + # abort a whole experiment export the docstring promises it won't abort. + try: + shutil.copytree(source, dest, ignore=ignore_patterns_and_symlinks(REFERENCE_COPY_IGNORE)) + except OSError as e: + raise TaskNotExportableError( + f"Task {task.task_id!r}'s reference directory {source} could not be copied into the export: {e}" + ) from e def _write_task_toml(task: TaskDefinition, out_dir: Path, *, workdir: str) -> None: diff --git a/src/coder_eval/harbor/reward.py b/src/coder_eval/harbor/reward.py index 426a485d..265028ca 100644 --- a/src/coder_eval/harbor/reward.py +++ b/src/coder_eval/harbor/reward.py @@ -23,6 +23,17 @@ instead — the same principle as CE049 (never coalesce a possibly-unmeasured score to a numeric literal), one level up, at the artifact-writing boundary rather than the in-process one. + +A grading-time INFRASTRUCTURE failure is the same case in disguise: +``EvaluationResult.calculate_weighted_score`` (``models/results.py``) +short-circuits an empty ``success_criteria_results`` list to a hard ``0.0``, +not ``None`` -- so a checker that raises ``JudgeInfrastructureError`` / +``CheckerMisuseError`` / ``ReferenceTamperedError`` (escalating exceptions +that deliberately propagate out of grading rather than being captured into a +scored-0.0 result) finalizes the row ``FinalStatus.ERROR`` with +``weighted_score == 0.0``, not ``None``. That is not a measurement either, so +it gets the same "write nothing" treatment via ``final_status.category == +"error"``. """ from __future__ import annotations @@ -59,14 +70,17 @@ def compute_reward(run_dir: Path) -> dict[str, float]: ``EvaluationResult`` (propagated from ``load_prior_result`` — an infrastructure failure, same "write nothing" contract as below). RewardWriteSkippedError: the row's ``weighted_score`` is ``None`` — an - unmeasured row, never to be coalesced to ``0.0``. + unmeasured row, never to be coalesced to ``0.0`` — or its + ``final_status.category`` is ``"error"``, a grading-time + infrastructure crash that also carries no real measurement despite + ``calculate_weighted_score`` writing a literal ``0.0`` for it. """ result = load_prior_result(run_dir) - if result.weighted_score is None: + if result.weighted_score is None or result.final_status.category == "error": raise RewardWriteSkippedError( - f"{run_dir} carries no weighted_score (final_status={result.final_status.value!r}); " - + "writing no reward file so Harbor's own missing-reward path masks the trial " - + "instead of scoring it 0.0" + f"{run_dir} carries no measured verdict (final_status={result.final_status.value!r}, " + + f"weighted_score={result.weighted_score!r}); writing no reward file so Harbor's own " + + "missing-reward path masks the trial instead of scoring it 0.0" ) return {"reward": result.weighted_score} diff --git a/src/coder_eval/orchestration/regrade.py b/src/coder_eval/orchestration/regrade.py index 70526d3f..ae4b19aa 100644 --- a/src/coder_eval/orchestration/regrade.py +++ b/src/coder_eval/orchestration/regrade.py @@ -1128,6 +1128,21 @@ async def regrade_in_place( """ from coder_eval.orchestrator import Orchestrator + # Every path through this function grades, so an empty `success_criteria` + # is never legal here the way it is for `execute` (which never calls this + # function at all). Checked once, at the single choke point every re-grade + # entry point (`evaluate `, `run --resume`'s `to_grade` set, and + # the container-dispatch branch below) shares -- a criteria-free task would + # otherwise finalize as `FinalStatus.SUCCESS` at `weighted_score: 0.0` + # (`all_criteria_passed([])` is vacuously `True`, + # `calculate_weighted_score([])` writes `0.0`), an internally contradictory + # "successful" result for what is actually a misconfigured task. + if not task.success_criteria: + raise RegradeError( + f"task {task.task_id!r} has no `success_criteria` and cannot be graded (it would silently " + + "score SUCCESS at weighted_score 0.0). Add at least one criterion before re-grading it." + ) + # A `driver: docker` row is graded INSIDE a container of the same image, # which is the only place its criteria mean what they meant during the run. # Dispatched before anything else here, including the reference check, so the diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index 281906e9..659d3863 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -35,6 +35,7 @@ from .models import ( CONTAINER_REFERENCE_DIR, DEFAULT_STOP_EARLY_GATE_THRESHOLD, + IN_CONTAINER_ENV, ROUTE_NAMES, AgentJudgeCriterion, AgentKind, @@ -1650,11 +1651,19 @@ async def _setup(self) -> None: # DIRECT_WRITE deliberately does NOT clear the target dir, so a reused # --run-dir (or --resume) can leave a prior run's files alongside this # run's outputs and silently perturb file-based criteria. Surface it. - # Skipped in workspace_dir mode: a WORKDIR (/root, /app) legitimately holds - # the image's baked inputs — not stale prior-run files — so the warning - # would fire on essentially every run and cry wolf. + # Suppressed in workspace_dir mode ONLY inside a container + # (IN_CONTAINER_ENV, per CE056 -- never on the field itself): the + # original writer was exclusively `run_task_internal_command`, where + # the WORKDIR is a fresh container filesystem every run, so a WORKDIR + # (/root, /app) legitimately holds the image's baked inputs there, not + # stale prior-run files. `--workspace-dir` is now also a host-reachable + # CLI flag on `run`/`execute`, where the named directory persists + # across invocations exactly like DIRECT_WRITE's own target -- keying + # the suppression on `workspace_dir is None` silently disabled the + # warning on precisely the new path where it is needed. + in_container = os.environ.get(IN_CONTAINER_ENV) == "1" if ( - self.workspace_dir is None + not (self.workspace_dir is not None and in_container) and direct_target is not None and direct_target.exists() and any(direct_target.iterdir()) diff --git a/tests/_fixtures/harbor_export_golden/expected/environment/task.yaml b/tests/_fixtures/harbor_export_golden/expected/environment/task.yaml index e9a3e841..d45b9d7d 100644 --- a/tests/_fixtures/harbor_export_golden/expected/environment/task.yaml +++ b/tests/_fixtures/harbor_export_golden/expected/environment/task.yaml @@ -18,3 +18,8 @@ sandbox: ignore_patterns: [] success_criteria: [] initial_prompt: Write 'hello world' to greeting.txt, matching reference/greeting.txt. +run_limits: + task_timeout: 300 + count_cached_input: false + count_cache_creation: false + stop_early_gate_threshold: 1.0 diff --git a/tests/_fixtures/harbor_export_golden/expected/tests/task.yaml b/tests/_fixtures/harbor_export_golden/expected/tests/task.yaml index c889ab53..2cb1dca6 100644 --- a/tests/_fixtures/harbor_export_golden/expected/tests/task.yaml +++ b/tests/_fixtures/harbor_export_golden/expected/tests/task.yaml @@ -21,3 +21,8 @@ success_criteria: similarity_threshold: 0.8 reference: directory: reference +run_limits: + task_timeout: 300 + count_cached_input: false + count_cache_creation: false + stop_early_gate_threshold: 1.0 diff --git a/tests/test_atif_emit.py b/tests/test_atif_emit.py index d932b0c3..a0b60769 100644 --- a/tests/test_atif_emit.py +++ b/tests/test_atif_emit.py @@ -3,6 +3,7 @@ import json from datetime import UTC, datetime +from coder_eval.harbor import atif_emit from coder_eval.harbor.atif_emit import evaluation_result_to_trajectory, write_trajectory_json from coder_eval.harbor.atif_models import Trajectory from coder_eval.models import ( @@ -350,11 +351,9 @@ def test_zero_step_result_writes_nothing(self, tmp_path): assert not path.exists() def test_converter_exception_swallowed(self, tmp_path, monkeypatch): - import coder_eval.harbor.atif_emit as emit - - monkeypatch.setattr(emit, "evaluation_result_to_trajectory", lambda _: 1 / 0) + monkeypatch.setattr(atif_emit, "evaluation_result_to_trajectory", lambda _: 1 / 0) path = tmp_path / "trajectory.json" - assert emit.write_trajectory_json(_result([_turn()]), path) is None + assert atif_emit.write_trajectory_json(_result([_turn()]), path) is None assert not path.exists() diff --git a/tests/test_atif_models.py b/tests/test_atif_models.py index a5640f51..550517e8 100644 --- a/tests/test_atif_models.py +++ b/tests/test_atif_models.py @@ -1,16 +1,17 @@ """Tests for the vendored ATIF models (coder_eval.harbor.atif_models). Fixture compatibility guarantee: ``tests/fixtures/atif/known_good_trajectory.json`` -was validated ONCE against the real harbor package (harbor==0.20.0) and then -frozen — CI never installs harbor. Reproducible re-validation procedure: +was originally validated against harbor==0.20.0. ``harbor`` is now a real, +installed optional extra (``pyproject.toml``'s ``harbor`` extra, pinned to +0.22.0 — see ``pyproject.toml`` and both pyright CI jobs), so this fixture and +the vendored models above can be re-verified directly against the installed +package: - python3 -m venv /tmp/harbor-check && /tmp/harbor-check/bin/pip install -q 'harbor==0.20.0' - /tmp/harbor-check/bin/python -c "from harbor.models.trajectories.trajectory import Trajectory; \\ + uv run python -c "from harbor.models.trajectories.trajectory import Trajectory; \\ import json; Trajectory.model_validate(json.load(open('tests/fixtures/atif/known_good_trajectory.json'))); \\ print('OK')" - rm -rf /tmp/harbor-check -Last validated: harbor 0.20.0 (2026-07-20). If the vendored models and this +Last validated: harbor 0.22.0 (2026-09-11). If the vendored models and this fixture ever disagree with harbor, re-run the procedure and reconcile. """ @@ -103,7 +104,7 @@ def test_duplicate_subagent_trajectory_ids_rejected(self): Trajectory(agent=_agent(), steps=[_step(1)], subagent_trajectories=[sub1, sub2]) def test_agent_version_required(self): - # Verified against harbor 0.20.0: Agent.version is REQUIRED, not optional. + # Verified against harbor 0.22.0: Agent.version is REQUIRED, not optional. with pytest.raises(ValidationError, match="version"): AtifAgent(name="claude-code") # type: ignore[call-arg] @@ -157,6 +158,24 @@ def test_unknown_step_field_rejected(self): with pytest.raises(ValidationError): Step(step_id=1, source="agent", message="x", bogus=True) # type: ignore[call-arg] - def test_subagent_ref_requires_trajectory_id(self): + def test_subagent_ref_rejects_unknown_field(self): + # All four real fields (trajectory_id, session_id, trajectory_path, + # extra) are optional in harbor==0.22.0 -- a bare SubagentTrajectoryRef() + # is a valid document (e.g. a session_id-only or trajectory_path-only + # reference need not repeat the others), so only an actually-unknown + # field should be rejected under extra="forbid". + SubagentTrajectoryRef() with pytest.raises(ValidationError): - SubagentTrajectoryRef() # type: ignore[call-arg] + SubagentTrajectoryRef(bogus=1) # type: ignore[call-arg] + + def test_subagent_ref_accepts_session_id_only_form(self): + """harbor==0.22.0's session_id-addressed shape, verified against a real install.""" + ref = SubagentTrajectoryRef(session_id="sess-123") + assert ref.trajectory_id is None + assert ref.session_id == "sess-123" + + def test_subagent_ref_accepts_trajectory_path_only_form(self): + """The spec's file-ref form: trajectory_path alone, no trajectory_id.""" + ref = SubagentTrajectoryRef(trajectory_path="subagents/child.json") + assert ref.trajectory_id is None + assert ref.trajectory_path == "subagents/child.json" diff --git a/tests/test_harbor_experiment_packager.py b/tests/test_harbor_experiment_packager.py index d834ac7d..2be63d64 100644 --- a/tests/test_harbor_experiment_packager.py +++ b/tests/test_harbor_experiment_packager.py @@ -106,10 +106,12 @@ def test_variant_prompt_override_lands_in_environment_task_yaml(self, tmp_path: assert emitted["initial_prompt"] == "Write 'howdy' to greeting.txt." -class TestUnhonorableOverridesAreSkipped: - """Agent/simulation overrides an exported Harbor directory cannot express.""" +class TestAgentOverridesAreHonored: + """Agent overrides ARE honorable now: CoderEvalAgent (C1.2) carries `task.agent` + verbatim into `environment/task.yaml` and executes it, so each variant exports + its own distinct directory rather than being skipped as indistinguishable.""" - def test_variant_agent_model_override_is_skipped_not_exported(self, tmp_path: Path) -> None: + def test_variant_agent_model_override_is_exported_not_skipped(self, tmp_path: Path) -> None: task_file = _write_task(tmp_path) exp_file = _write_experiment( tmp_path, @@ -125,11 +127,14 @@ def test_variant_agent_model_override_is_skipped_not_exported(self, tmp_path: Pa result = export_experiment([task_file], exp_file, out_dir) - assert result.exported == [] - assert {s.variant_id for s in result.skipped} == {"sonnet", "opus"} - assert all("agent override" in s.reason for s in result.skipped) + assert {r.out_dir for r in result.exported} == {out_dir / "sonnet" / "greet", out_dir / "opus" / "greet"} + assert result.skipped == [] + sonnet_yaml = (out_dir / "sonnet" / "greet" / "environment" / "task.yaml").read_text(encoding="utf-8") + opus_yaml = (out_dir / "opus" / "greet" / "environment" / "task.yaml").read_text(encoding="utf-8") + assert "claude-sonnet-5" in sonnet_yaml + assert "claude-opus-5" in opus_yaml - def test_experiment_defaults_agent_override_is_skipped(self, tmp_path: Path) -> None: + def test_experiment_defaults_agent_override_is_exported(self, tmp_path: Path) -> None: task_file = _write_task(tmp_path) exp_file = _write_experiment( tmp_path, @@ -143,18 +148,46 @@ def test_experiment_defaults_agent_override_is_skipped(self, tmp_path: Path) -> result = export_experiment([task_file], exp_file, out_dir) - assert result.exported == [] - assert "experiment-defaults" in result.skipped[0].reason + assert len(result.exported) == 1 + assert result.skipped == [] + + def test_task_level_agent_config_alone_does_not_trip_the_skip(self, tmp_path: Path) -> None: + """The task's OWN agent.type (source='task') must not be mistaken for an experiment override.""" + task_file = _write_task(tmp_path) # agent: {type: claude-code} is the task's own, not the experiment's + exp_file = _write_experiment( + tmp_path, + { + "experiment_id": "no-override", + "variants": [{"variant_id": "only"}], + }, + ) + out_dir = tmp_path / "out" + + result = export_experiment([task_file], exp_file, out_dir) + + assert len(result.exported) == 1 + assert result.skipped == [] + + +class TestUnhonorableOverridesAreSkipped: + """A simulation override is the one thing an exported Harbor directory still cannot express.""" - def test_variant_with_no_agent_override_still_exports_alongside_a_skipped_one(self, tmp_path: Path) -> None: + def test_variant_simulation_override_is_skipped_not_exported(self, tmp_path: Path) -> None: task_file = _write_task(tmp_path) exp_file = _write_experiment( tmp_path, { - "experiment_id": "mixed", + "experiment_id": "sim-ab", "variants": [ - {"variant_id": "unmodified"}, - {"variant_id": "sonnet", "agent": {"model": "claude-sonnet-5"}}, + { + "variant_id": "dialog", + "simulation": { + "enabled": True, + "persona": "a confused new user", + "goal": "get the greeting written", + "max_turns": 3, + }, + }, ], }, ) @@ -162,18 +195,28 @@ def test_variant_with_no_agent_override_still_exports_alongside_a_skipped_one(se result = export_experiment([task_file], exp_file, out_dir) - assert len(result.exported) == 1 - assert result.exported[0].out_dir == out_dir / "unmodified" / "greet" - assert [s.variant_id for s in result.skipped] == ["sonnet"] + assert result.exported == [] + assert [s.variant_id for s in result.skipped] == ["dialog"] + assert "simulation" in result.skipped[0].reason - def test_task_level_agent_config_alone_does_not_trip_the_skip(self, tmp_path: Path) -> None: - """The task's OWN agent.type (source='task') must not be mistaken for an experiment override.""" - task_file = _write_task(tmp_path) # agent: {type: claude-code} is the task's own, not the experiment's + def test_variant_with_no_simulation_override_still_exports_alongside_a_skipped_one(self, tmp_path: Path) -> None: + task_file = _write_task(tmp_path) exp_file = _write_experiment( tmp_path, { - "experiment_id": "no-override", - "variants": [{"variant_id": "only"}], + "experiment_id": "mixed", + "variants": [ + {"variant_id": "unmodified"}, + { + "variant_id": "dialog", + "simulation": { + "enabled": True, + "persona": "a confused new user", + "goal": "get the greeting written", + "max_turns": 3, + }, + }, + ], }, ) out_dir = tmp_path / "out" @@ -181,7 +224,8 @@ def test_task_level_agent_config_alone_does_not_trip_the_skip(self, tmp_path: Pa result = export_experiment([task_file], exp_file, out_dir) assert len(result.exported) == 1 - assert result.skipped == [] + assert result.exported[0].out_dir == out_dir / "unmodified" / "greet" + assert [s.variant_id for s in result.skipped] == ["dialog"] class TestReplicateFanOut: diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index a1fbeed1..b894ac6f 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -878,6 +878,79 @@ async def create_dummy_agent(_self): await orchestrator._cleanup() +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("in_container", "expect_warning"), + [(False, True), (True, False)], + ids=["host_reachable_warns", "in_container_silent"], +) +async def test_workspace_dir_staleness_warning_keys_on_in_container_not_field( + tmp_path, monkeypatch, caplog, in_container, expect_warning +): + """`--workspace-dir` on the host must warn on a pre-populated dir exactly like + DIRECT_WRITE does; only the in-container writer (a fresh container filesystem + every run) legitimately suppresses it. Regression for the bug where the + suppression keyed on `workspace_dir is None` instead of `IN_CONTAINER_ENV`.""" + import logging + from datetime import datetime + + from coder_eval import orchestrator as orchestrator_module + from coder_eval.models import IN_CONTAINER_ENV, ApiBackend, DirectRoute, EvaluationResult + + class DummyAgent: + async def start(self, working_directory, *, env_path_prepend=None, plugin_tools_dir=None): + self.working_directory = working_directory + + def get_sdk_options(self): + return {"env": {"PATH": os.environ.get("PATH", "")}} + + def get_environment_info(self): + return {} + + async def create_dummy_agent(_self): + return DummyAgent() + + task_file = Path("tasks/hello_date.yaml") + task, _ = load_task(task_file) + task.sandbox.python = None + + run_dir = tmp_path / "test_run" / "hello_date" + ws = tmp_path / "ws" + ws.mkdir(parents=True) + (ws / "stale.txt").write_text("from a prior run") + + orchestrator = Orchestrator(task=task, run_dir=run_dir, variant_id="test-variant", workspace_dir=ws) + orchestrator.task_file = task_file + orchestrator.result = EvaluationResult( + task_id=task.task_id, + task_description=task.description, + variant_id="test-variant", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime.now(), + final_status="FAILURE", + iteration_count=0, + environment_info={}, + ) + + if in_container: + monkeypatch.setenv(IN_CONTAINER_ENV, "1") + else: + monkeypatch.delenv(IN_CONTAINER_ENV, raising=False) + + monkeypatch.setattr(orchestrator_module.settings, "api_backend", ApiBackend.DIRECT) + monkeypatch.setattr(type(orchestrator_module.settings), "validate_api_keys", lambda _self, _agent_type: None) + monkeypatch.setattr(orchestrator_module, "resolve_route", lambda _settings: DirectRoute(judge_transport=None)) + monkeypatch.setattr(Orchestrator, "_create_agent", create_dummy_agent) + + with caplog.at_level(logging.WARNING, logger="coder_eval.orchestrator"): + await orchestrator._setup() + + warned = any("already exists and is non-empty" in r.message for r in caplog.records) + assert warned is expect_warning + + await orchestrator._cleanup() + + @pytest.mark.asyncio async def test_orchestrator_cleanup_persistent_sandbox(tmp_path): """DIRECT_WRITE: sandbox already lives in artifacts; _cleanup keeps it in place (no move).""" From 79c2f8c85c1d661de76484a49c4181d3f6699a44 Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Fri, 11 Sep 2026 17:15:57 -0700 Subject: [PATCH 12/12] build(docker): bake the harbor extra into the coder-eval-agent image Lets the same per-task agent image double as a Harbor agent image (coder_eval.harbor.agent:CoderEvalAgent, a harbor.agents.installed.base.BaseInstalledAgent subclass) without a separate build, matching the harbor extra already wired into both pyright CI jobs. Co-Authored-By: Claude Sonnet 5 --- docker/Dockerfile | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 99467fac..e3ff04ab 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -54,15 +54,19 @@ COPY src/ ./src/ # experiments/default.yaml is force-included by hatchling per pyproject.toml. COPY experiments/default.yaml ./experiments/default.yaml -# Codex, Antigravity, and litellm are always baked into the image -- codex/ -# antigravity are peers to the claude-code agent installed above; litellm backs -# the `checker_context.api_route.route: litellm` judge dispatch (see -# pyproject.toml's `litellm` extra), which every DockerRunner-isolated task -# needs available IN-container since the checker runs there too, not just on the -# host. `openai-codex` (+ its pinned cli-bin), `google-antigravity` (which -# bundles its `localharness` binary as a manylinux wheel), and `litellm` all +# Codex, Antigravity, litellm, and harbor are always baked into the image -- +# codex/antigravity are peers to the claude-code agent installed above; +# litellm backs the `checker_context.api_route.route: litellm` judge dispatch +# (see pyproject.toml's `litellm` extra), which every DockerRunner-isolated +# task needs available IN-container since the checker runs there too, not +# just on the host; harbor lets this same image double as a Harbor agent +# image (`coder_eval.harbor.agent:CoderEvalAgent`, a +# `harbor.agents.installed.base.BaseInstalledAgent` subclass -- see +# src/coder_eval/harbor/agent.py) without a separate build. `openai-codex` +# (+ its pinned cli-bin), `google-antigravity` (which bundles its +# `localharness` binary as a manylinux wheel), `litellm`, and `harbor` all # come from public PyPI, so this needs no private-index credentials. The RUN -# below always passes `--extra codex --extra antigravity --extra litellm`. +# below always passes `--extra codex --extra antigravity --extra litellm --extra harbor`. # # `pi` IS baked above (pinned PI_VERSION) and its OPENROUTER_API_KEY provider # credential is in DockerDriverConfig.env_passthrough, so `--driver docker --type pi` @@ -90,7 +94,7 @@ ARG SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS="openai-codex-cli-bin,openai-codex # All extras (codex, antigravity, litellm, and the opt-in uipath) resolve from # public PyPI per uv.lock, so the build needs no private-index credentials. RUN export SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS="${SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS}" && \ - uv export --frozen --extra codex --extra antigravity --extra litellm ${CODER_EVAL_UV_EXTRAS} | uv pip install --system -r /dev/stdin + uv export --frozen --extra codex --extra antigravity --extra litellm --extra harbor ${CODER_EVAL_UV_EXTRAS} | uv pip install --system -r /dev/stdin # Sanity check: the in-container entrypoint subcommand must be wired up. RUN coder-eval _run-task-internal --help > /dev/null