diff --git a/CHANGELOG.md b/CHANGELOG.md index 63b1a0b5..4b1e6cdd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ All notable changes to SkillOpt are documented here. This project adheres to ## [Unreleased] ### Added +- **GitHub Copilot CLI backend**, in two forms: `copilot_chat` (usable as both + optimizer and target) and `copilot_exec` (target-only execution harness). + Because the Copilot CLI carries its own sign-in, `--backend copilot` selects + `copilot_chat` for both roles and runs a complete train/eval loop with no + separate provider API key; inference still uses the GitHub Copilot cloud + service. Chat calls disable all built-in tools, built-in MCP servers, and + custom instructions, strip inherited `COPILOT_ALLOW_ALL`, and never pass + `--allow-all-tools`; `copilot_exec` requires an explicit + `copilot_exec_allow_all_tools` opt-in before granting unattended tool use. + The CLI reports no token counts, so usage totals are zero for these backends. - A non-destructive Devin installer and SessionEnd activity marker, preserving existing project hooks across repeated installation. - Per-night SkillOpt-Sleep `evidence.jsonl` chains for reconstructing harvest, diff --git a/README.md b/README.md index 8836325f..76444185 100644 --- a/README.md +++ b/README.md @@ -65,8 +65,8 @@ https://github.com/user-attachments/assets/eb12d3bc-371c-467f-904d-91b61f339ed7 ### Adding a new backend A backend = a chat / exec target (e.g. `openai_chat`, `claude_chat`, -`qwen_chat`, `minimax_chat`, `openai_compatible`, `codex_exec`, -`claude_code_exec`, `cursor_exec`). If a provider implements the OpenAI Chat Completions +`qwen_chat`, `minimax_chat`, `copilot_chat`, `openai_compatible`, `codex_exec`, +`claude_code_exec`, `cursor_exec`, `copilot_exec`). If a provider implements the OpenAI Chat Completions protocol, try the built-in `openai_compatible` backend before adding code. See [`docs/guide/new-backend.md`](docs/guide/new-backend.md) for the full contract. Chat backends add a `skillopt/model/_backend.py` module; diff --git a/configs/_base_/default.yaml b/configs/_base_/default.yaml index c8c068d1..77dc5d26 100644 --- a/configs/_base_/default.yaml +++ b/configs/_base_/default.yaml @@ -26,6 +26,15 @@ model: claude_code_exec_max_thinking_tokens: 16384 cursor_exec_path: "" # blank uses CURSOR_EXEC_PATH or cursor-agent cursor_exec_sandbox: "" # blank uses CURSOR_EXEC_SANDBOX or enabled + # GitHub Copilot CLI. `--backend copilot` selects copilot_chat for BOTH roles, + # using CLI authentication rather than a separate provider API key. + copilot_exec_path: "" # blank uses COPILOT_EXEC_PATH or copilot + copilot_exec_home: "" # blank leaves COPILOT_HOME untouched + # null preserves COPILOT_EXEC_ALLOW_ALL_TOOLS (default off); copilot_exec only + copilot_exec_allow_all_tools: null # required for file-edit rollouts + copilot_chat_optimizer_model: "" # blank lets the CLI pick its default model + copilot_chat_target_model: "" + copilot_chat_timeout: null # preserves COPILOT_CHAT_TIMEOUT or the built-in default codex_trace_to_optimizer: true azure_openai_endpoint: "" # e.g. "https://your-resource.openai.azure.com/" azure_openai_api_version: "2024-12-01-preview" diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 2a72a64a..2e23f3b5 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -43,9 +43,21 @@ model: | `claude_chat` | ✓ | ✓ | Claude Code CLI (`claude -p`) | | `qwen_chat` | ✓ | ✓ | Qwen served through an OpenAI-compatible local endpoint | | `minimax_chat` | ✓ | ✓ | MiniMax API | +| `copilot_chat` | ✓ | ✓ | GitHub Copilot CLI (`copilot -p`); alias `copilot` | | `codex_exec` | — | ✓ | Codex CLI execution harness | | `claude_code_exec` | — | ✓ | Claude Code CLI execution harness | | `cursor_exec` | — | ✓ | Cursor Agent CLI execution harness | +| `copilot_exec` | — | ✓ | GitHub Copilot CLI execution harness | + +`copilot_chat` fills **both** roles through a locally installed, +CLI-authenticated client, so `--backend copilot` runs a complete training loop +without a separate provider API key. Inference still uses the GitHub Copilot +cloud service. Sign in once with `copilot` (GitHub Copilot CLI) beforehand. +Expect roughly 20-40 s per call: the CLI is an agent, not a completions +endpoint, so a full-size run is far slower than a hosted backend. It also +reports no token counts, so usage totals are zero. +It does not support caller-supplied tools or structured tool calls; environments +that require those features must use another chat backend. The current MiniMax adapter has one shared deployment. Set `model.minimax_model` when MiniMax is the target; a mixed-backend run cannot @@ -191,6 +203,12 @@ Model credentials are loaded from environment variables: | `QWEN_CHAT_MODEL` | `qwen_chat` | Served model name for direct library use; train/eval YAML role models take precedence | | `MINIMAX_BASE_URL` | `minimax_chat` | MiniMax-compatible base URL | | `MINIMAX_API_KEY` | `minimax_chat` | MiniMax API key | +| `COPILOT_EXEC_PATH` | `copilot_chat`, `copilot_exec` | Optional path to `copilot`; defaults to `copilot` | +| `COPILOT_EXEC_HOME` | `copilot_chat`, `copilot_exec` | Optional `COPILOT_HOME` override isolating CLI config; sign-in state lives outside it | +| `COPILOT_EXEC_ALLOW_ALL_TOOLS` | `copilot_exec` | Opt in to `--allow-all-tools` for file-edit rollouts; `false` by default | +| `COPILOT_CHAT_OPTIMIZER_MODEL` | `copilot_chat` | Optional model ID passed as `--model` for optimizer calls | +| `COPILOT_CHAT_TARGET_MODEL` | `copilot_chat` | Optional model ID passed as `--model` for target calls | +| `COPILOT_CHAT_TIMEOUT` | `copilot_chat` | Per-call timeout in seconds | `OPTIMIZER_` and `TARGET_` prefixes provide per-role overrides for the Azure, OpenAI-compatible, and Qwen variable families. See the @@ -209,6 +227,23 @@ Cursor sandbox by default and rejects file-edit rollouts if it is disabled; read-only Ask-mode rollouts may explicitly disable it. SkillOpt does not approve MCP servers automatically. +`copilot_chat` drives the GitHub Copilot CLI as a chat model for either role. +Because the CLI carries its own sign-in, selecting it for both roles +(`--backend copilot`) avoids separate provider API-key configuration: + +```bash +copilot login +skillopt-train --config configs/searchqa/default.yaml --backend copilot +``` + +Chat calls use an empty tool allowlist and disable built-in MCP servers and +custom instructions, so the model sees only the prompt SkillOpt sends. +`COPILOT_ALLOW_ALL` is removed from child environments, and +`--allow-all-tools` is never passed for chat calls. `copilot_exec` is the +separate target-only harness that runs the CLI as an agent inside a benchmark +workspace; unlike the other exec harnesses it does not grant unattended tool +use unless `COPILOT_EXEC_ALLOW_ALL_TOOLS` is set. + ### Three OpenAI-compatible paths - Research, generic provider: select `openai_compatible` and use diff --git a/docs/reference/api.md b/docs/reference/api.md index 34176d7c..1b9ad2d8 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -189,10 +189,12 @@ not via a base class subclass. Supported values (as of this writing): | `claude_chat` | ✓ | ✓ | | `qwen_chat` | ✓ | ✓ | | `minimax_chat` | ✓ | ✓ | +| `copilot_chat` | ✓ | ✓ | | `openai_compatible` | ✓ | ✓ | | `codex_exec` | ✓ | ✓ | | `claude_code_exec` | — | ✓ | | `cursor_exec` | — | ✓ | +| `copilot_exec` | — | ✓ | See `skillopt/model/backend_config.py` for the live whitelist and [`docs/reference/config.md`](./config.md) for the per-backend diff --git a/docs/reference/config.md b/docs/reference/config.md index b8243c77..cdf44e61 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -16,9 +16,11 @@ selecting the generic OpenAI-compatible backend. | `claude_chat` | ✓ | ✓ | | `qwen_chat` | ✓ | ✓ | | `minimax_chat` | ✓ | ✓ | +| `copilot_chat` | ✓ | ✓ | | `codex_exec` | ✓ | ✓ | | `claude_code_exec` | — | ✓ | | `cursor_exec` | — | ✓ | +| `copilot_exec` | — | ✓ | MiniMax currently has one shared deployment. `model.minimax_model` is applied when MiniMax is the target; mixed-backend runs cannot independently choose a @@ -67,6 +69,11 @@ defaults to `claude` and can be overridden with `CLAUDE_CLI_BIN`. | `model.claude_code_exec_*` | Claude path, profile, SDK mode, effort, and thinking-token cap | | `model.cursor_exec_path` | Cursor Agent executable path; default `cursor-agent` | | `model.cursor_exec_sandbox` | Cursor sandbox mode: `enabled` (default) or `disabled`; file-edit rollouts require `enabled` | +| `model.copilot_exec_path` | GitHub Copilot CLI executable path; default `copilot` | +| `model.copilot_exec_home` | Optional `COPILOT_HOME` override isolating CLI config | +| `model.copilot_exec_allow_all_tools` | Optional opt-in to `--allow-all-tools`; unset by default so `COPILOT_EXEC_ALLOW_ALL_TOOLS` remains authoritative | +| `model.copilot_chat_optimizer_model` / `model.copilot_chat_target_model` | Optional per-role `--model` IDs for `copilot_chat` | +| `model.copilot_chat_timeout` | Per-call timeout in seconds for `copilot_chat` | ## Training (`train`) diff --git a/scripts/eval_only.py b/scripts/eval_only.py index a40a4bd5..5658f766 100644 --- a/scripts/eval_only.py +++ b/scripts/eval_only.py @@ -28,19 +28,34 @@ configure_azure_openai, configure_claude_code_exec, configure_codex_exec, + configure_copilot_chat, + configure_copilot_exec, configure_cursor_exec, - configure_qwen_chat, configure_minimax_chat, + configure_qwen_chat, + set_optimizer_backend, + set_optimizer_deployment, set_reasoning_effort, set_target_backend, set_target_deployment, - set_optimizer_backend, - set_optimizer_deployment, ) from skillopt.model.common import default_model_for_backend, normalize_backend_name +from skillopt.utils import compute_score _OPENAI_DEFAULT_MODEL_SENTINELS = {"gpt-5.4", "gpt-5.5"} -from skillopt.utils import compute_score +_ROLE_BACKEND_DEFAULTS = (None, "", "openai_chat") + + +def _set_role_if_default( + cfg: dict, + key: str, + value: str, + *, + explicitly_overridden: bool, +) -> None: + """Apply a high-level backend mapping without clobbering a named role.""" + if not explicitly_overridden and cfg.get(key) in _ROLE_BACKEND_DEFAULTS: + cfg[key] = value # ── Reuse registry from train.py ─────────────────────────────────────────── @@ -140,7 +155,7 @@ def parse_args() -> argparse.Namespace: # Legacy flat overrides p.add_argument("--env", type=str) p.add_argument("--backend", type=str, - choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec", "cursor", "cursor_exec", "minimax", "minimax_chat"]) + choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec", "cursor", "cursor_exec", "copilot", "copilot_chat", "copilot_exec", "minimax", "minimax_chat"]) p.add_argument("--optimizer_model", type=str) p.add_argument("--target_model", type=str) p.add_argument("--optimizer_backend", type=str) @@ -184,6 +199,12 @@ def parse_args() -> argparse.Namespace: p.add_argument("--claude_code_exec_max_thinking_tokens", type=int) p.add_argument("--cursor_exec_path", type=str) p.add_argument("--cursor_exec_sandbox", type=str) + p.add_argument("--copilot_exec_path", type=str) + p.add_argument("--copilot_exec_home", type=str) + p.add_argument("--copilot_exec_allow_all_tools", type=_BOOL) + p.add_argument("--copilot_chat_optimizer_model", type=str) + p.add_argument("--copilot_chat_target_model", type=str) + p.add_argument("--copilot_chat_timeout", type=int) p.add_argument("--minimax_base_url", type=str) p.add_argument("--minimax_api_key", type=str) p.add_argument("--minimax_model", type=str) @@ -267,6 +288,12 @@ def main() -> None: "claude_code_exec_max_thinking_tokens": "model.claude_code_exec_max_thinking_tokens", "cursor_exec_path": "model.cursor_exec_path", "cursor_exec_sandbox": "model.cursor_exec_sandbox", + "copilot_exec_path": "model.copilot_exec_path", + "copilot_exec_home": "model.copilot_exec_home", + "copilot_exec_allow_all_tools": "model.copilot_exec_allow_all_tools", + "copilot_chat_optimizer_model": "model.copilot_chat_optimizer_model", + "copilot_chat_target_model": "model.copilot_chat_target_model", + "copilot_chat_timeout": "model.copilot_chat_timeout", "minimax_base_url": "model.minimax_base_url", "minimax_api_key": "model.minimax_api_key", "minimax_model": "model.minimax_model", @@ -321,28 +348,43 @@ def _has_model_override(dotted_key: str, legacy_key: str) -> bool: if explicit_backend is not None: backend = normalize_backend_name(explicit_backend) cfg["model_backend"] = backend - if backend in {"claude", "claude_chat"}: - cfg.setdefault("optimizer_backend", "claude_chat") - cfg.setdefault("target_backend", "claude_chat") + + def _set_role(key: str, value: str) -> None: + _set_role_if_default( + cfg, + key, + value, + explicitly_overridden=_has_model_override(f"model.{key}", key), + ) + + if backend == "claude_chat": + _set_role("optimizer_backend", "claude_chat") + _set_role("target_backend", "claude_chat") elif backend in {"codex", "codex_exec"}: - if not _has_model_override("model.optimizer_backend", "optimizer_backend"): - cfg["optimizer_backend"] = "codex_exec" - if not _has_model_override("model.target_backend", "target_backend"): - cfg["target_backend"] = "codex_exec" + _set_role("optimizer_backend", "codex_exec") + _set_role("target_backend", "codex_exec") elif backend == "claude_code_exec": - cfg.setdefault("optimizer_backend", "openai_chat") - cfg.setdefault("target_backend", "claude_code_exec") + _set_role("optimizer_backend", "openai_chat") + _set_role("target_backend", "claude_code_exec") elif backend == "cursor_exec": - if not _has_model_override("model.optimizer_backend", "optimizer_backend"): - cfg["optimizer_backend"] = "openai_chat" - if not _has_model_override("model.target_backend", "target_backend"): - cfg["target_backend"] = "cursor_exec" - elif backend in {"minimax", "minimax_chat"}: - cfg.setdefault("optimizer_backend", "openai_chat") - cfg.setdefault("target_backend", "minimax_chat") + _set_role("optimizer_backend", "openai_chat") + _set_role("target_backend", "cursor_exec") + elif backend == "copilot_chat": + # Both roles use the locally installed, CLI-authenticated backend. + _set_role("optimizer_backend", "copilot_chat") + _set_role("target_backend", "copilot_chat") + elif backend == "copilot_exec": + _set_role("optimizer_backend", "openai_chat") + _set_role("target_backend", "copilot_exec") + elif backend == "minimax_chat": + _set_role("optimizer_backend", "openai_chat") + _set_role("target_backend", "minimax_chat") + elif backend == "openai_compatible": + _set_role("optimizer_backend", "openai_compatible") + _set_role("target_backend", "openai_compatible") else: - cfg.setdefault("optimizer_backend", "openai_chat") - cfg.setdefault("target_backend", "openai_chat") + _set_role("optimizer_backend", "openai_chat") + _set_role("target_backend", "openai_chat") else: cfg.setdefault("optimizer_backend", "openai_chat") cfg.setdefault("target_backend", "openai_chat") @@ -371,6 +413,13 @@ def _has_model_override(dotted_key: str, legacy_key: str) -> bool: and not _has_model_override("model.target", "target_model") ): cfg["target_model"] = default_model_for_backend("cursor_exec") + if cfg.get("target_backend") == "copilot_exec": + if ( + str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.target", "target_model") + ): + # Copilot CLI model IDs are independent of Azure deployment names. + cfg["target_model"] = "" if cfg.get("target_backend") == "minimax_chat": if ( str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS @@ -449,6 +498,16 @@ def _has_model_override(dotted_key: str, legacy_key: str) -> bool: path=cfg.get("cursor_exec_path") or None, sandbox=cfg.get("cursor_exec_sandbox") or None, ) + configure_copilot_exec( + path=cfg.get("copilot_exec_path") or None, + home=cfg.get("copilot_exec_home") or None, + allow_all_tools=cfg.get("copilot_exec_allow_all_tools"), + ) + configure_copilot_chat( + optimizer_model=cfg.get("copilot_chat_optimizer_model") or None, + target_model=cfg.get("copilot_chat_target_model") or None, + timeout=cfg.get("copilot_chat_timeout") or None, + ) configure_qwen_chat( base_url=cfg.get("qwen_chat_base_url") or None, api_key=cfg.get("qwen_chat_api_key") or None, diff --git a/scripts/train.py b/scripts/train.py index 8b253777..f8f4be76 100644 --- a/scripts/train.py +++ b/scripts/train.py @@ -29,6 +29,7 @@ from skillopt.model.common import default_model_for_backend, normalize_backend_name _OPENAI_DEFAULT_MODEL_SENTINELS = {"gpt-5.4", "gpt-5.5"} +_ROLE_BACKEND_DEFAULTS = (None, "", "openai_chat") # ── Environment registry ──────────────────────────────────────────────────── @@ -137,7 +138,7 @@ def parse_args() -> argparse.Namespace: # Legacy flat CLI overrides (still work, prefer --cfg-options for new usage) p.add_argument("--env", type=str) p.add_argument("--backend", type=str, - choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec", "cursor", "cursor_exec", "qwen", "qwen_chat", "minimax", "minimax_chat"]) + choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec", "cursor", "cursor_exec", "copilot", "copilot_chat", "copilot_exec", "qwen", "qwen_chat", "minimax", "minimax_chat"]) p.add_argument("--optimizer_model", type=str) p.add_argument("--target_model", type=str) p.add_argument("--optimizer_backend", type=str) @@ -207,6 +208,12 @@ def parse_args() -> argparse.Namespace: p.add_argument("--claude_code_exec_max_thinking_tokens", type=int) p.add_argument("--cursor_exec_path", type=str) p.add_argument("--cursor_exec_sandbox", type=str) + p.add_argument("--copilot_exec_path", type=str) + p.add_argument("--copilot_exec_home", type=str) + p.add_argument("--copilot_exec_allow_all_tools", type=_BOOL) + p.add_argument("--copilot_chat_optimizer_model", type=str) + p.add_argument("--copilot_chat_target_model", type=str) + p.add_argument("--copilot_chat_timeout", type=int) p.add_argument("--codex_trace_to_optimizer", type=_BOOL) p.add_argument("--skill_init", type=str) p.add_argument("--num_epochs", type=int) @@ -347,6 +354,12 @@ def parse_args() -> argparse.Namespace: "claude_code_exec_max_thinking_tokens": "model.claude_code_exec_max_thinking_tokens", "cursor_exec_path": "model.cursor_exec_path", "cursor_exec_sandbox": "model.cursor_exec_sandbox", + "copilot_exec_path": "model.copilot_exec_path", + "copilot_exec_home": "model.copilot_exec_home", + "copilot_exec_allow_all_tools": "model.copilot_exec_allow_all_tools", + "copilot_chat_optimizer_model": "model.copilot_chat_optimizer_model", + "copilot_chat_target_model": "model.copilot_chat_target_model", + "copilot_chat_timeout": "model.copilot_chat_timeout", "codex_trace_to_optimizer": "model.codex_trace_to_optimizer", "num_epochs": "train.num_epochs", "train_size": "train.train_size", @@ -495,31 +508,44 @@ def _has_model_override(dotted_key: str, legacy_key: str) -> bool: if explicit_backend is not None: backend = normalize_backend_name(explicit_backend) flat["model_backend"] = backend - if backend in {"claude", "claude_chat"}: - flat.setdefault("optimizer_backend", "claude_chat") - flat.setdefault("target_backend", "claude_chat") + + def _set_role(key: str, value: str) -> None: + if ( + not _has_model_override(f"model.{key}", key) + and flat.get(key) in _ROLE_BACKEND_DEFAULTS + ): + flat[key] = value + + if backend == "claude_chat": + _set_role("optimizer_backend", "claude_chat") + _set_role("target_backend", "claude_chat") elif backend in {"codex", "codex_exec"}: - if not _has_model_override("model.optimizer_backend", "optimizer_backend"): - flat["optimizer_backend"] = "codex_exec" - if not _has_model_override("model.target_backend", "target_backend"): - flat["target_backend"] = "codex_exec" + _set_role("optimizer_backend", "codex_exec") + _set_role("target_backend", "codex_exec") elif backend == "claude_code_exec": - flat.setdefault("optimizer_backend", "openai_chat") - flat.setdefault("target_backend", "claude_code_exec") + _set_role("optimizer_backend", "openai_chat") + _set_role("target_backend", "claude_code_exec") elif backend == "cursor_exec": - if not _has_model_override("model.optimizer_backend", "optimizer_backend"): - flat["optimizer_backend"] = "openai_chat" - if not _has_model_override("model.target_backend", "target_backend"): - flat["target_backend"] = "cursor_exec" - elif backend in {"qwen", "qwen_chat"}: - flat.setdefault("optimizer_backend", "openai_chat") - flat.setdefault("target_backend", "qwen_chat") - elif backend in {"minimax", "minimax_chat"}: - flat.setdefault("optimizer_backend", "openai_chat") - flat.setdefault("target_backend", "minimax_chat") + _set_role("optimizer_backend", "openai_chat") + _set_role("target_backend", "cursor_exec") + elif backend == "copilot_chat": + _set_role("optimizer_backend", "copilot_chat") + _set_role("target_backend", "copilot_chat") + elif backend == "copilot_exec": + _set_role("optimizer_backend", "openai_chat") + _set_role("target_backend", "copilot_exec") + elif backend == "qwen_chat": + _set_role("optimizer_backend", "openai_chat") + _set_role("target_backend", "qwen_chat") + elif backend == "minimax_chat": + _set_role("optimizer_backend", "openai_chat") + _set_role("target_backend", "minimax_chat") + elif backend == "openai_compatible": + _set_role("optimizer_backend", "openai_compatible") + _set_role("target_backend", "openai_compatible") else: - flat.setdefault("optimizer_backend", "openai_chat") - flat.setdefault("target_backend", "openai_chat") + _set_role("optimizer_backend", "openai_chat") + _set_role("target_backend", "openai_chat") else: flat.setdefault("optimizer_backend", "openai_chat") flat.setdefault("target_backend", "openai_chat") @@ -554,6 +580,13 @@ def _has_model_override(dotted_key: str, legacy_key: str) -> bool: and not _has_model_override("model.target", "target_model") ): flat["target_model"] = default_model_for_backend("cursor_exec") + if flat.get("target_backend") == "copilot_exec": + if ( + str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.target", "target_model") + ): + # Copilot CLI model IDs are independent of Azure deployment names. + flat["target_model"] = "" if flat.get("target_backend") == "qwen_chat": if ( str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS diff --git a/skillopt/config.py b/skillopt/config.py index c47e6d9b..acc7d5b3 100644 --- a/skillopt/config.py +++ b/skillopt/config.py @@ -53,6 +53,12 @@ "model.claude_code_exec_max_thinking_tokens": "claude_code_exec_max_thinking_tokens", "model.cursor_exec_path": "cursor_exec_path", "model.cursor_exec_sandbox": "cursor_exec_sandbox", + "model.copilot_exec_path": "copilot_exec_path", + "model.copilot_exec_home": "copilot_exec_home", + "model.copilot_exec_allow_all_tools": "copilot_exec_allow_all_tools", + "model.copilot_chat_optimizer_model": "copilot_chat_optimizer_model", + "model.copilot_chat_target_model": "copilot_chat_target_model", + "model.copilot_chat_timeout": "copilot_chat_timeout", "model.codex_trace_to_optimizer": "codex_trace_to_optimizer", "model.azure_endpoint": "azure_endpoint", "model.azure_api_version": "azure_api_version", diff --git a/skillopt/engine/trainer.py b/skillopt/engine/trainer.py index 1eae4209..bd148f94 100644 --- a/skillopt/engine/trainer.py +++ b/skillopt/engine/trainer.py @@ -63,7 +63,9 @@ configure_azure_openai, configure_claude_code_exec, configure_codex_exec, - configure_cursor_exec, + configure_copilot_chat, + configure_copilot_exec, + configure_cursor_exec, configure_minimax_chat, configure_qwen_chat, get_token_summary, @@ -74,6 +76,7 @@ set_optimizer_backend, set_optimizer_deployment, ) +from skillopt.model.common import normalize_backend_name from skillopt.utils import compute_score, skill_hash @@ -445,6 +448,72 @@ def _resolve_train_size(cfg: dict, dataloader) -> int: return int(train_size) +# Role backends the shipped base config sets, which must not defeat --backend. +_ROLE_BACKEND_DEFAULTS = (None, "", "openai_chat") + + +def _resolve_role_backends( + backend: str, optimizer_backend: str | None, target_backend: str | None +) -> tuple[str, str]: + """Map a high-level ``--backend`` label onto (optimizer, target) backends. + + ``configs/_base_/default.yaml`` pins both roles to ``openai_chat``, so a + resolution guarded only by "is either role unset?" never fired for runs + using the shipped defaults and ``--backend`` was silently ignored. A role + left at its default value counts as unset; a role the operator pointed at + something else always wins. + """ + backend = normalize_backend_name(backend) + if backend == "claude_chat": + # A chat backend fills BOTH roles, so -- like copilot -- a role pinned + # to a default (including the base config's truthy openai_chat) must be + # overridden. `x = x or ...` would leave openai_chat in place. + if optimizer_backend in _ROLE_BACKEND_DEFAULTS: + optimizer_backend = "claude_chat" + if target_backend in _ROLE_BACKEND_DEFAULTS: + target_backend = "claude_chat" + elif backend in {"codex", "codex_exec"}: + if optimizer_backend in _ROLE_BACKEND_DEFAULTS: + optimizer_backend = "codex_exec" + if target_backend in _ROLE_BACKEND_DEFAULTS: + target_backend = "codex_exec" + elif backend == "claude_code_exec": + optimizer_backend = optimizer_backend or "openai_chat" + if target_backend in _ROLE_BACKEND_DEFAULTS: + target_backend = "claude_code_exec" + elif backend == "cursor_exec": + optimizer_backend = optimizer_backend or "openai_chat" + if target_backend in _ROLE_BACKEND_DEFAULTS: + target_backend = "cursor_exec" + elif backend == "copilot_chat": + # Both roles use the locally installed, CLI-authenticated backend. + if optimizer_backend in _ROLE_BACKEND_DEFAULTS: + optimizer_backend = "copilot_chat" + if target_backend in _ROLE_BACKEND_DEFAULTS: + target_backend = "copilot_chat" + elif backend == "copilot_exec": + optimizer_backend = optimizer_backend or "openai_chat" + if target_backend in _ROLE_BACKEND_DEFAULTS: + target_backend = "copilot_exec" + elif backend == "qwen_chat": + optimizer_backend = optimizer_backend or "openai_chat" + if target_backend in _ROLE_BACKEND_DEFAULTS: + target_backend = "qwen_chat" + elif backend == "minimax_chat": + optimizer_backend = optimizer_backend or "openai_chat" + if target_backend in _ROLE_BACKEND_DEFAULTS: + target_backend = "minimax_chat" + elif backend == "openai_compatible": + if optimizer_backend in _ROLE_BACKEND_DEFAULTS: + optimizer_backend = "openai_compatible" + if target_backend in _ROLE_BACKEND_DEFAULTS: + target_backend = "openai_compatible" + else: + optimizer_backend = optimizer_backend or "openai_chat" + target_backend = target_backend or "openai_chat" + return optimizer_backend, target_backend + + def _compute_task_type_buckets(results: list[dict], task_types: list[str]) -> dict[str, dict]: """Compute per-task-type success rates.""" buckets: dict[str, dict] = {} @@ -668,31 +737,11 @@ def _build_eval_env(split: str, env_num: int, seed: int): cfg.get("target_azure_openai_managed_identity_client_id") or None ), ) - optimizer_backend = cfg.get("optimizer_backend") - target_backend = cfg.get("target_backend") - if not optimizer_backend or not target_backend: - if backend in {"claude", "claude_chat"}: - optimizer_backend = optimizer_backend or "claude_chat" - target_backend = target_backend or "claude_chat" - elif backend in {"codex", "codex_exec"}: - if optimizer_backend in (None, "", "openai_chat"): - optimizer_backend = "codex_exec" - if target_backend in (None, "", "openai_chat"): - target_backend = "codex_exec" - elif backend == "claude_code_exec": - optimizer_backend = optimizer_backend or "openai_chat" - target_backend = target_backend or "claude_code_exec" - elif backend in {"cursor", "cursor_exec"}: - optimizer_backend = optimizer_backend or "openai_chat" - target_backend = target_backend or "cursor_exec" - elif backend in {"qwen", "qwen_chat"}: - optimizer_backend = optimizer_backend or "openai_chat" - target_backend = target_backend or "qwen_chat" - else: - optimizer_backend = optimizer_backend or "openai_chat" - target_backend = target_backend or "openai_chat" - cfg["optimizer_backend"] = optimizer_backend - cfg["target_backend"] = target_backend + optimizer_backend, target_backend = _resolve_role_backends( + backend, cfg.get("optimizer_backend"), cfg.get("target_backend") + ) + cfg["optimizer_backend"] = optimizer_backend + cfg["target_backend"] = target_backend set_optimizer_backend(optimizer_backend) set_target_backend(target_backend) set_optimizer_deployment(cfg["optimizer_model"]) @@ -715,10 +764,20 @@ def _build_eval_env(split: str, env_num: int, seed: int): effort=cfg.get("claude_code_exec_effort", cfg.get("reasoning_effort", "medium")), max_thinking_tokens=cfg.get("claude_code_exec_max_thinking_tokens", 16384), ) - configure_cursor_exec( - path=cfg.get("cursor_exec_path") or None, - sandbox=cfg.get("cursor_exec_sandbox") or None, - ) + configure_cursor_exec( + path=cfg.get("cursor_exec_path") or None, + sandbox=cfg.get("cursor_exec_sandbox") or None, + ) + configure_copilot_exec( + path=cfg.get("copilot_exec_path") or None, + home=cfg.get("copilot_exec_home") or None, + allow_all_tools=cfg.get("copilot_exec_allow_all_tools"), + ) + configure_copilot_chat( + optimizer_model=cfg.get("copilot_chat_optimizer_model") or None, + target_model=cfg.get("copilot_chat_target_model") or None, + timeout=cfg.get("copilot_chat_timeout") or None, + ) configure_qwen_chat( base_url=cfg.get("qwen_chat_base_url") or None, api_key=cfg.get("qwen_chat_api_key") or None, diff --git a/skillopt/model/__init__.py b/skillopt/model/__init__.py index 6c84e6b9..d90719ca 100644 --- a/skillopt/model/__init__.py +++ b/skillopt/model/__init__.py @@ -7,15 +7,20 @@ from skillopt.model import azure_openai as _openai from skillopt.model import claude_backend as _claude from skillopt.model import codex_backend as _codex +from skillopt.model import copilot_backend as _copilot from skillopt.model import minimax_backend as _minimax from skillopt.model import openai_compatible_backend as _openai_compat from skillopt.model import qwen_backend as _qwen from skillopt.model.backend_config import ( # noqa: F401 configure_claude_code_exec, configure_codex_exec, + configure_copilot_chat, + configure_copilot_exec, configure_cursor_exec, get_claude_code_exec_config, get_codex_exec_config, + get_copilot_chat_config, + get_copilot_exec_config, get_cursor_exec_config, get_optimizer_backend, get_target_backend, @@ -25,6 +30,7 @@ set_optimizer_backend, set_target_backend, ) +from skillopt.model.common import normalize_backend_name def set_backend(name: str | None) -> str: @@ -34,20 +40,16 @@ def set_backend(name: str | None) -> str: target. Keep that entry point so older scripts continue to work, while mapping it onto the split optimizer/target backend model. """ - normalized = str(name or "azure_openai").strip().lower() - if normalized in {"azure_openai", "openai_chat", "azure", "azure-openai"}: + normalized = normalize_backend_name(name) + if normalized in {"azure_openai", "openai_chat"}: set_optimizer_backend("openai_chat") set_target_backend("openai_chat") return "azure_openai" - if normalized in {"claude", "claude_chat", "anthropic"}: + if normalized == "claude_chat": set_optimizer_backend("claude_chat") set_target_backend("claude_chat") return "claude_chat" - if normalized == "codex": - set_optimizer_backend("codex_exec") - set_target_backend("codex_exec") - return "codex" - if normalized == "codex_exec": + if normalized in {"codex", "codex_exec"}: set_optimizer_backend("codex_exec") set_target_backend("codex_exec") return normalized @@ -55,19 +57,29 @@ def set_backend(name: str | None) -> str: set_optimizer_backend("openai_chat") set_target_backend(normalized) return normalized - if normalized in {"cursor", "cursor_agent", "cursor_exec"}: + if normalized == "cursor_exec": set_optimizer_backend("openai_chat") set_target_backend("cursor_exec") return "cursor_exec" - if normalized in {"qwen", "qwen_chat"}: + if normalized == "copilot_chat": + # The CLI-authenticated backend drives both roles without a separate + # provider API key; inference still uses the Copilot cloud service. + set_optimizer_backend("copilot_chat") + set_target_backend("copilot_chat") + return "copilot_chat" + if normalized == "copilot_exec": + set_optimizer_backend("openai_chat") + set_target_backend("copilot_exec") + return "copilot_exec" + if normalized == "qwen_chat": set_optimizer_backend("openai_chat") set_target_backend("qwen_chat") return "qwen_chat" - if normalized in {"minimax", "minimax_chat"}: + if normalized == "minimax_chat": set_optimizer_backend("openai_chat") set_target_backend("minimax_chat") return "minimax_chat" - if normalized in {"openai_compatible", "openai_compatible_chat", "openai-compatible", "compat"}: + if normalized == "openai_compatible": set_optimizer_backend("openai_compatible") set_target_backend("openai_compatible") return "openai_compatible" @@ -82,6 +94,8 @@ def get_backend_name() -> str: return "claude_chat" if optimizer == "qwen_chat" and target == "qwen_chat": return "qwen_chat" + if optimizer == "copilot_chat" and target == "copilot_chat": + return "copilot_chat" if optimizer == "openai_chat" and target == "openai_chat": return "azure_openai" if optimizer == "codex_exec" and target == "codex_exec": @@ -92,6 +106,8 @@ def get_backend_name() -> str: return "minimax_chat" if optimizer == "openai_chat" and target == "cursor_exec": return "cursor_exec" + if optimizer == "openai_chat" and target == "copilot_exec": + return "copilot_exec" if optimizer == "openai_compatible" and target == "openai_compatible": return "openai_compatible" return f"{optimizer}+{target}" @@ -115,6 +131,16 @@ def chat_optimizer( stage=stage, timeout=timeout, ) + if get_optimizer_backend() == "copilot_chat": + return _copilot.chat_optimizer( + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + timeout=timeout, + ) if get_optimizer_backend() == "qwen_chat": return _qwen.chat_optimizer( system=system, @@ -212,10 +238,20 @@ def chat_target( reasoning_effort=reasoning_effort, timeout=timeout, ) + if get_target_backend() == "copilot_chat": + return _copilot.chat_target( + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + timeout=timeout, + ) if not is_target_chat_backend(): raise NotImplementedError( "chat_target is only supported with target_backend=openai_chat, claude_chat, qwen_chat, minimax_chat, " - "or openai_compatible. Exec backends are handled in environment-specific rollout code." + "copilot_chat, or openai_compatible. Exec backends are handled in environment-specific rollout code." ) return _openai.chat_target( system=system, @@ -240,6 +276,17 @@ def chat_optimizer_messages( return_message: bool = False, timeout: int | None = None, ) -> tuple[Any, dict]: + if get_optimizer_backend() == "copilot_chat": + return _copilot.chat_optimizer_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) if get_optimizer_backend() == "claude_chat": return _claude.chat_optimizer_messages( messages=messages, @@ -323,6 +370,17 @@ def chat_target_messages( return_message: bool = False, timeout: int | None = None, ) -> tuple[Any, dict]: + if get_target_backend() == "copilot_chat": + return _copilot.chat_target_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) if get_target_backend() == "claude_chat": return _claude.chat_target_messages( messages=messages, @@ -372,7 +430,8 @@ def chat_target_messages( if not is_target_chat_backend(): raise NotImplementedError( "chat_target_messages is only supported with target_backend=openai_chat, claude_chat, qwen_chat, " - "minimax_chat, or openai_compatible. Exec backends are handled in environment-specific rollout code." + "minimax_chat, copilot_chat, or openai_compatible. Exec backends are handled in environment-specific " + "rollout code." ) return _openai.chat_target_messages( messages=messages, @@ -493,6 +552,17 @@ def get_token_summary() -> dict: summary[stage]["prompt_tokens"] += values["prompt_tokens"] summary[stage]["completion_tokens"] += values["completion_tokens"] summary[stage]["total_tokens"] += values["total_tokens"] + copilot_summary = _copilot.get_token_summary() + for stage, values in copilot_summary.items(): + if stage == "_total": + continue + if stage not in summary: + summary[stage] = values + continue + summary[stage]["calls"] += values["calls"] + summary[stage]["prompt_tokens"] += values["prompt_tokens"] + summary[stage]["completion_tokens"] += values["completion_tokens"] + summary[stage]["total_tokens"] += values["total_tokens"] total = { "calls": 0, "prompt_tokens": 0, @@ -517,6 +587,7 @@ def reset_token_tracker() -> None: _minimax.reset_token_tracker() _openai_compat.reset_token_tracker() _codex.reset_token_tracker() + _copilot.reset_token_tracker() def configure_azure_openai( diff --git a/skillopt/model/backend_config.py b/skillopt/model/backend_config.py index d48b050b..dbca7694 100644 --- a/skillopt/model/backend_config.py +++ b/skillopt/model/backend_config.py @@ -3,7 +3,7 @@ import os -from skillopt.model.common import default_model_for_backend, normalize_backend_name +from skillopt.model.common import normalize_backend_name def _parse_bool(value: str | None, default: bool) -> bool: @@ -30,6 +30,14 @@ def _parse_bool(value: str | None, default: bool) -> bool: CLAUDE_CODE_EXEC_EFFORT = os.environ.get("CLAUDE_CODE_EXEC_EFFORT", "medium") CURSOR_EXEC_PATH = os.environ.get("CURSOR_EXEC_PATH", "cursor-agent") CURSOR_EXEC_SANDBOX = os.environ.get("CURSOR_EXEC_SANDBOX", "enabled") +COPILOT_EXEC_PATH = os.environ.get("COPILOT_EXEC_PATH", "copilot") +COPILOT_EXEC_HOME = os.environ.get("COPILOT_EXEC_HOME", "") +COPILOT_EXEC_ALLOW_ALL_TOOLS = ( + "1" if _parse_bool(os.environ.get("COPILOT_EXEC_ALLOW_ALL_TOOLS"), False) else "0" +) +COPILOT_CHAT_OPTIMIZER_MODEL = os.environ.get("COPILOT_CHAT_OPTIMIZER_MODEL", "") +COPILOT_CHAT_TARGET_MODEL = os.environ.get("COPILOT_CHAT_TARGET_MODEL", "") +COPILOT_CHAT_TIMEOUT = os.environ.get("COPILOT_CHAT_TIMEOUT", "600") def _parse_int(value: str | None, default: int) -> int: @@ -57,12 +65,13 @@ def set_optimizer_backend(backend: str) -> None: "qwen_chat", "minimax_chat", "openai_compatible", + "copilot_chat", "codex_exec", }: raise ValueError( f"Unsupported optimizer backend: {OPTIMIZER_BACKEND!r}. " "Supported values are 'openai_chat', 'claude_chat', 'qwen_chat', 'minimax_chat', " - "'openai_compatible', and 'codex_exec'." + "'openai_compatible', 'copilot_chat', and 'codex_exec'." ) os.environ["OPTIMIZER_BACKEND"] = OPTIMIZER_BACKEND @@ -74,11 +83,12 @@ def get_optimizer_backend() -> str: def set_target_backend(backend: str) -> None: global TARGET_BACKEND TARGET_BACKEND = normalize_backend_name(backend or "openai_chat") - if TARGET_BACKEND not in {"openai_chat", "claude_chat", "qwen_chat", "minimax_chat", "openai_compatible", "codex_exec", "claude_code_exec", "cursor_exec"}: + if TARGET_BACKEND not in {"openai_chat", "claude_chat", "qwen_chat", "minimax_chat", "openai_compatible", "copilot_chat", "codex_exec", "claude_code_exec", "cursor_exec", "copilot_exec"}: raise ValueError( f"Unsupported target backend: {TARGET_BACKEND!r}. " "Supported values are 'openai_chat', 'claude_chat', 'qwen_chat', 'minimax_chat', " - "'openai_compatible', 'codex_exec', 'claude_code_exec', and 'cursor_exec'." + "'openai_compatible', 'copilot_chat', 'codex_exec', 'claude_code_exec', " + "'cursor_exec', and 'copilot_exec'." ) os.environ["TARGET_BACKEND"] = TARGET_BACKEND @@ -88,7 +98,7 @@ def get_target_backend() -> str: def is_target_exec_backend() -> bool: - return TARGET_BACKEND in {"codex_exec", "claude_code_exec", "cursor_exec"} + return TARGET_BACKEND in {"codex_exec", "claude_code_exec", "cursor_exec", "copilot_exec"} def is_optimizer_chat_backend() -> bool: @@ -98,12 +108,13 @@ def is_optimizer_chat_backend() -> bool: "qwen_chat", "minimax_chat", "openai_compatible", + "copilot_chat", "codex_exec", } def is_target_chat_backend() -> bool: - return TARGET_BACKEND in {"openai_chat", "claude_chat", "qwen_chat", "minimax_chat", "openai_compatible"} + return TARGET_BACKEND in {"openai_chat", "claude_chat", "qwen_chat", "minimax_chat", "openai_compatible", "copilot_chat"} def configure_codex_exec( @@ -228,3 +239,95 @@ def get_cursor_exec_config() -> dict[str, str | int]: "sandbox": CURSOR_EXEC_SANDBOX, "empty_response_retries": EXEC_EMPTY_RESPONSE_RETRIES, } + + +def configure_copilot_exec( + *, + path: str | None = None, + home: str | None = None, + allow_all_tools: bool | str | None = None, +) -> None: + """Configure the GitHub Copilot CLI exec backend. + + ``home`` points ``COPILOT_HOME`` at an isolated config dir so the user's MCP + servers and custom instructions are not loaded during a rollout. Auth lives + outside ``COPILOT_HOME`` (OS credential store / token env vars), so isolating + the home does not break authentication. + """ + global COPILOT_EXEC_PATH, COPILOT_EXEC_HOME, COPILOT_EXEC_ALLOW_ALL_TOOLS + if path is not None: + COPILOT_EXEC_PATH = str(path).strip() or "copilot" + os.environ["COPILOT_EXEC_PATH"] = COPILOT_EXEC_PATH + if home is not None: + COPILOT_EXEC_HOME = str(home).strip() + os.environ["COPILOT_EXEC_HOME"] = COPILOT_EXEC_HOME + if allow_all_tools is not None: + normalized = str(allow_all_tools).strip().lower() + if normalized in {"1", "true", "yes", "on"}: + COPILOT_EXEC_ALLOW_ALL_TOOLS = "1" + elif normalized in {"0", "false", "no", "off", ""}: + COPILOT_EXEC_ALLOW_ALL_TOOLS = "0" + else: + raise ValueError( + "copilot_exec allow_all_tools must be a boolean-like value" + ) + os.environ["COPILOT_EXEC_ALLOW_ALL_TOOLS"] = COPILOT_EXEC_ALLOW_ALL_TOOLS + + +def get_copilot_exec_config() -> dict[str, str | int]: + if COPILOT_EXEC_ALLOW_ALL_TOOLS not in {"0", "1"}: + raise ValueError("copilot_exec allow_all_tools must be '0' or '1'") + return { + "path": COPILOT_EXEC_PATH, + "home": COPILOT_EXEC_HOME, + "allow_all_tools": COPILOT_EXEC_ALLOW_ALL_TOOLS, + "empty_response_retries": EXEC_EMPTY_RESPONSE_RETRIES, + } + + +def configure_copilot_chat( + *, + path: str | None = None, + home: str | None = None, + optimizer_model: str | None = None, + target_model: str | None = None, + timeout: int | str | None = None, +) -> None: + """Configure the Copilot CLI *chat* backend (optimizer and/or target). + + Unlike `copilot_exec`, this runs the CLI as a plain chat model so a whole + run can use CLI authentication without a separate provider API key. + """ + global COPILOT_EXEC_PATH, COPILOT_EXEC_HOME + global COPILOT_CHAT_OPTIMIZER_MODEL, COPILOT_CHAT_TARGET_MODEL, COPILOT_CHAT_TIMEOUT + if path is not None: + COPILOT_EXEC_PATH = str(path).strip() or "copilot" + os.environ["COPILOT_EXEC_PATH"] = COPILOT_EXEC_PATH + if home is not None: + COPILOT_EXEC_HOME = str(home).strip() + os.environ["COPILOT_EXEC_HOME"] = COPILOT_EXEC_HOME + if optimizer_model is not None: + COPILOT_CHAT_OPTIMIZER_MODEL = str(optimizer_model).strip() + os.environ["COPILOT_CHAT_OPTIMIZER_MODEL"] = COPILOT_CHAT_OPTIMIZER_MODEL + if target_model is not None: + COPILOT_CHAT_TARGET_MODEL = str(target_model).strip() + os.environ["COPILOT_CHAT_TARGET_MODEL"] = COPILOT_CHAT_TARGET_MODEL + if timeout is not None: + value = str(timeout).strip() + if not value.isdigit() or int(value) <= 0: + raise ValueError("copilot_chat timeout must be a positive integer") + COPILOT_CHAT_TIMEOUT = value + os.environ["COPILOT_CHAT_TIMEOUT"] = COPILOT_CHAT_TIMEOUT + + +def get_copilot_chat_config() -> dict[str, str | int]: + timeout = str(COPILOT_CHAT_TIMEOUT).strip() + if not timeout.isdigit() or int(timeout) <= 0: + raise ValueError("copilot_chat timeout must be a positive integer") + return { + "path": COPILOT_EXEC_PATH, + "home": COPILOT_EXEC_HOME, + "optimizer_model": COPILOT_CHAT_OPTIMIZER_MODEL, + "target_model": COPILOT_CHAT_TARGET_MODEL, + "timeout": int(timeout), + } diff --git a/skillopt/model/codex_harness.py b/skillopt/model/codex_harness.py index 6bbb0939..fad3983c 100644 --- a/skillopt/model/codex_harness.py +++ b/skillopt/model/codex_harness.py @@ -14,9 +14,14 @@ from skillopt.model.backend_config import ( get_claude_code_exec_config, get_codex_exec_config, + get_copilot_exec_config, get_cursor_exec_config, get_target_backend, ) +from skillopt.model.copilot_backend import ( + build_copilot_subprocess_env, + parse_copilot_jsonl, +) ANSWER_SCHEMA: dict[str, Any] = { "type": "object", @@ -1304,6 +1309,108 @@ def run_cursor_exec( raise RuntimeError(last_error) +def run_copilot_exec( + *, + work_dir: str, + prompt: str, + model: str, + timeout: int, + images: list[str] | None = None, + data_dirs: list[str] | None = None, + allow_file_edits: bool = False, +) -> tuple[str, str]: + """Run the GitHub Copilot CLI headlessly as a benchmark target. + + Uses ``copilot -p --output-format json`` and parses the emitted + JSONL event stream, concatenating ``assistant.message`` content. The plain + text / ``--silent`` modes do not reliably stream the response to stdout on + every platform, so JSONL is used for robust capture. + """ + config = get_copilot_exec_config() + retries = int(config.get("empty_response_retries", 0) or 0) + add_dirs = _validated_add_dirs(work_dir, data_dirs, images)[1:] + all_raw: list[str] = [] + last_error = "Copilot CLI returned no response" + allow_all_tools = str(config.get("allow_all_tools", "0")) == "1" + + for attempt in range(retries + 1): + attempt_prompt = _exec_prompt( + _retry_prompt(prompt, attempt), + allow_file_edits=allow_file_edits, + ) + cmd = [ + str(config["path"]), + "-p", + attempt_prompt, + "--output-format", + "json", + "--stream", + "off", + "--no-color", + "--log-level", + "none", + "-C", + work_dir, + "--disable-builtin-mcps", + "--no-custom-instructions", + ] + # Read-only rollouts keep the CLI's approval gate; only opt in to + # unattended tool use when the caller explicitly allows file edits and + # the operator has enabled it. + if allow_file_edits and allow_all_tools: + cmd.append("--allow-all-tools") + if model: + cmd.extend(["--model", model]) + for path in add_dirs: + cmd.extend(["--add-dir", path]) + + home = str(config.get("home") or "") + env = build_copilot_subprocess_env(home) + + try: + proc = subprocess.run( + cmd, + cwd=work_dir, + capture_output=True, + text=True, + timeout=timeout, + encoding="utf-8", + errors="replace", + env=env, + ) + except subprocess.TimeoutExpired: + # Nothing to capture here: all_raw is local and this path re-raises, + # and TimeoutExpired already carries .stdout/.stderr for the caller. + raise + except OSError as exc: + raise RuntimeError(f"Copilot CLI could not be executed: {exc}") from exc + + stdout = proc.stdout or "" + stderr = proc.stderr or "" + safe_raw = stdout + if stderr: + safe_raw = f"{safe_raw}\n[stderr]\n{stderr}" if safe_raw else f"[stderr]\n{stderr}" + all_raw.append(f"===== COPILOT CLI ATTEMPT {attempt + 1} =====\n{safe_raw}") + combined = "\n\n".join(all_raw) + + if proc.returncode != 0: + detail = (stderr or stdout).strip()[:4000] + raise RuntimeError( + f"Copilot CLI failed with exit code {proc.returncode}: {detail}" + ) + + response = parse_copilot_jsonl(stdout) + if response: + return response, combined + + combined = "\n\n".join(all_raw) + # Without this the caller gets a bare "returned no response" and no CLI + # output at all, since copilot_exec persists no artifacts; include a + # bounded tail so an empty/invalid JSONL stream is debuggable. + detail = combined.strip()[-4000:] + raise RuntimeError(f"{last_error}\n{detail}" if detail else last_error) + + def run_target_exec( *, work_dir: str, @@ -1353,4 +1460,14 @@ def run_target_exec( sandbox=sandbox, allow_file_edits=allow_file_edits, ) + if backend == "copilot_exec": + return run_copilot_exec( + work_dir=work_dir, + prompt=prompt, + model=model, + timeout=timeout, + images=images, + data_dirs=data_dirs, + allow_file_edits=allow_file_edits, + ) raise ValueError(f"Unsupported exec backend: {backend}") diff --git a/skillopt/model/common.py b/skillopt/model/common.py index 97e4be81..d3f4df52 100644 --- a/skillopt/model/common.py +++ b/skillopt/model/common.py @@ -6,7 +6,6 @@ from dataclasses import dataclass, field from typing import Any - _RESPONSES_API_MODELS = { "gpt-5.3-codex", "gpt-5.1-codex", @@ -25,6 +24,10 @@ "claude_chat": "claude-sonnet-4-6", "claude_code_exec": "claude-sonnet-4-6", "cursor_exec": "composer-2.5", + # copilot_chat / copilot_exec deliberately use the Azure default fallback + # from default_model_for_backend(). That fallback configures shared role + # deployments (not the CLI model, which uses the copilot_chat model keys) + # and keeps the openai_chat optimizer for copilot_exec non-empty. "qwen_chat": "Qwen/Qwen3.5-4B", "minimax_chat": "MiniMax-M2.7", "openai_compatible": "gpt-4o-mini", @@ -44,6 +47,11 @@ "cursor": "cursor_exec", "cursor_agent": "cursor_exec", "cursor_exec": "cursor_exec", + "copilot": "copilot_chat", + "copilot_cli": "copilot_chat", + "github_copilot": "copilot_chat", + "copilot_chat": "copilot_chat", + "copilot_exec": "copilot_exec", "anthropic": "claude_chat", "qwen": "qwen_chat", "qwen_chat": "qwen_chat", diff --git a/skillopt/model/copilot_backend.py b/skillopt/model/copilot_backend.py new file mode 100644 index 00000000..a56bca82 --- /dev/null +++ b/skillopt/model/copilot_backend.py @@ -0,0 +1,277 @@ +"""GitHub Copilot CLI chat backend. + +Drives the locally installed `copilot` CLI as a chat model so a full SkillOpt +run (optimizer *and* target) can execute with no separate provider API key. +Inference still uses the GitHub Copilot cloud service through the operator's +existing subscription and OS credential store. + +The CLI is a single-shot agent, not a chat-completions endpoint, so `system` +and `user` are composed into one prompt. Output is captured as JSONL +(`--output-format json`) and the `assistant.message` content is concatenated; +the plain-text/`--silent` modes do not reliably stream to stdout on all +platforms. + +The CLI does not report token usage, so usage counters are reported as zeros. +Cost/quota is governed by the Copilot subscription rather than per-token +billing, so this does not lose billable accounting. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import time +from typing import Any + +from .backend_config import get_copilot_chat_config +from .common import CompatAssistantMessage, TokenTracker + +tracker = TokenTracker() + +_ZERO_USAGE = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + + +def build_copilot_subprocess_env(home: str = "") -> dict[str, str]: + """Build a child environment without inherited unattended-tool approval.""" + env = os.environ.copy() + env.pop("COPILOT_ALLOW_ALL", None) + if home: + env["COPILOT_HOME"] = home + return env + + +def _compose_prompt(system: str, user: str) -> str: + system = (system or "").strip() + user = (user or "").strip() + if system and user: + return f"{system}\n\n---\n\n{user}" + return system or user + + +def _messages_to_prompt(messages: list[dict[str, Any]]) -> str: + parts: list[str] = [] + for message in messages or []: + content = message.get("content") + if isinstance(content, list): # OpenAI multi-part content + text_parts: list[str] = [] + for part in content: + if not isinstance(part, dict) or part.get("type") != "text": + part_type = part.get("type") if isinstance(part, dict) else type(part).__name__ + raise NotImplementedError( + "copilot_chat supports only text message parts; " + f"received unsupported multipart type {part_type!r}" + ) + text_parts.append(str(part.get("text", ""))) + content = "\n".join(text_parts) + elif content is not None and not isinstance(content, str): + raise TypeError( + "copilot_chat message content must be a string, a list of text parts, or None" + ) + content = str(content or "").strip() + if not content: + continue + role = str(message.get("role", "user")).strip().lower() + parts.append(content if role == "user" else f"[{role}]\n{content}") + return "\n\n---\n\n".join(parts) + + +def parse_copilot_jsonl(raw: str) -> str: + """Concatenate assistant text from a Copilot JSONL event stream.""" + parts: list[str] = [] + for line in (raw or "").splitlines(): + line = line.strip() + if not line or not line.startswith("{"): + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(obj, dict): + continue + if obj.get("type") == "assistant.message": + data = obj.get("data") + if not isinstance(data, dict): + continue + content = data.get("content") + if isinstance(content, str) and content: + parts.append(content) + return "\n".join(parts).strip() + + +def _invoke(prompt: str, *, model: str, timeout: float | None) -> str: + config = get_copilot_chat_config() + cmd = [ + str(config["path"]), + "-p", + prompt, + "--output-format", + "json", + "--stream", + "off", + "--no-color", + "--log-level", + "none", + "--disable-builtin-mcps", + "--no-custom-instructions", + # A chat backend must behave like a completions endpoint, not an agent. + # An empty allowlist removes built-in read/shell/write/web tools. + "--available-tools=", + ] + chosen = model or str(config.get("model") or "") + if chosen: + cmd.extend(["--model", chosen]) + + home = str(config.get("home") or "") + env = build_copilot_subprocess_env(home) + + proc = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=timeout if timeout else float(config["timeout"]), + encoding="utf-8", + errors="replace", + env=env, + ) + if proc.returncode != 0: + detail = ((proc.stderr or proc.stdout) or "").strip()[:4000] + raise RuntimeError(f"Copilot CLI failed with exit code {proc.returncode}: {detail}") + return parse_copilot_jsonl(proc.stdout or "") + + +def _chat_impl( + prompt: str, + retries: int, + stage: str, + *, + model: str = "", + timeout: float | None = None, +) -> tuple[str, dict[str, int]]: + last_err: Exception | None = None + for attempt in range(max(1, retries)): + try: + text = _invoke(prompt, model=model, timeout=timeout) + if not text: + raise RuntimeError("Copilot CLI returned an empty response") + tracker.record(stage, 0, 0) + return text, dict(_ZERO_USAGE) + except Exception as e: # noqa: BLE001 + last_err = e + if attempt < max(1, retries) - 1: + time.sleep(min(2**attempt, 30)) + raise RuntimeError(f"Copilot CLI chat failed after {max(1, retries)} retries: {last_err}") + + +def chat_optimizer( + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "optimizer", + reasoning_effort: str | None = None, + timeout: float | None = None, +) -> tuple[str, dict[str, int]]: + del max_completion_tokens, reasoning_effort + config = get_copilot_chat_config() + return _chat_impl( + _compose_prompt(system, user), + retries, + stage, + model=str(config.get("optimizer_model") or ""), + timeout=timeout, + ) + + +def chat_target( + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "target", + reasoning_effort: str | None = None, + timeout: float | None = None, +) -> tuple[str, dict[str, int]]: + del max_completion_tokens, reasoning_effort + config = get_copilot_chat_config() + return _chat_impl( + _compose_prompt(system, user), + retries, + stage, + model=str(config.get("target_model") or ""), + timeout=timeout, + ) + + +def chat_optimizer_messages( + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "optimizer", + reasoning_effort: str | None = None, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: float | None = None, +) -> tuple[str | CompatAssistantMessage, dict[str, int]]: + del max_completion_tokens, reasoning_effort + _reject_unsupported_tools(tools, tool_choice) + config = get_copilot_chat_config() + text, usage = _chat_impl( + _messages_to_prompt(messages), + retries, + stage, + model=str(config.get("optimizer_model") or ""), + timeout=timeout, + ) + return (CompatAssistantMessage(content=text) if return_message else text), usage + + +def chat_target_messages( + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "target", + reasoning_effort: str | None = None, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: float | None = None, +) -> tuple[str | CompatAssistantMessage, dict[str, int]]: + del max_completion_tokens, reasoning_effort + _reject_unsupported_tools(tools, tool_choice) + config = get_copilot_chat_config() + text, usage = _chat_impl( + _messages_to_prompt(messages), + retries, + stage, + model=str(config.get("target_model") or ""), + timeout=timeout, + ) + return (CompatAssistantMessage(content=text) if return_message else text), usage + + +def _reject_unsupported_tools( + tools: list[dict[str, Any]] | None, + tool_choice: str | dict[str, Any] | None, +) -> None: + if tools or tool_choice is not None: + raise NotImplementedError( + "copilot_chat does not support caller-supplied tools or tool_choice; " + "use a tool-capable chat backend for this environment" + ) + + +def get_token_summary() -> dict[str, dict[str, int]]: + """Per-stage usage, like every other backend. + + The Copilot CLI reports no token counts, so the token fields stay zero, but + the *call* counts are real and belong in the run's token snapshots. + """ + return tracker.summary() + + +def reset_token_tracker() -> None: + tracker.reset() diff --git a/tests/test_copilot_exec_backend.py b/tests/test_copilot_exec_backend.py new file mode 100644 index 00000000..6298b087 --- /dev/null +++ b/tests/test_copilot_exec_backend.py @@ -0,0 +1,577 @@ +from __future__ import annotations + +import ast +import json +import os +import subprocess +import sys +from collections.abc import Iterator +from pathlib import Path + +import pytest + +import scripts.train as train_script +import skillopt.model as model +from skillopt.config import flatten_config +from skillopt.model import backend_config, copilot_backend +from skillopt.model import codex_harness as harness +from skillopt.model.common import normalize_backend_name + + +@pytest.fixture(autouse=True) +def restore_backend_state() -> Iterator[None]: + optimizer_backend = backend_config.get_optimizer_backend() + target_backend = backend_config.get_target_backend() + copilot_path = backend_config.COPILOT_EXEC_PATH + copilot_home = backend_config.COPILOT_EXEC_HOME + copilot_tools = backend_config.COPILOT_EXEC_ALLOW_ALL_TOOLS + chat_optimizer_model = backend_config.COPILOT_CHAT_OPTIMIZER_MODEL + chat_target_model = backend_config.COPILOT_CHAT_TARGET_MODEL + chat_timeout = backend_config.COPILOT_CHAT_TIMEOUT + retries = backend_config.EXEC_EMPTY_RESPONSE_RETRIES + env = { + key: os.environ.get(key) + for key in ( + "OPTIMIZER_BACKEND", + "TARGET_BACKEND", + "COPILOT_EXEC_PATH", + "COPILOT_EXEC_HOME", + "COPILOT_EXEC_ALLOW_ALL_TOOLS", + "COPILOT_CHAT_OPTIMIZER_MODEL", + "COPILOT_CHAT_TARGET_MODEL", + "COPILOT_CHAT_TIMEOUT", + ) + } + yield + # copilot_backend records call counts, and those now feed the aggregate + # model.get_token_summary(); leaving them behind pollutes other suites. + copilot_backend.reset_token_tracker() + backend_config.OPTIMIZER_BACKEND = optimizer_backend + backend_config.TARGET_BACKEND = target_backend + backend_config.COPILOT_EXEC_PATH = copilot_path + backend_config.COPILOT_EXEC_HOME = copilot_home + backend_config.COPILOT_EXEC_ALLOW_ALL_TOOLS = copilot_tools + backend_config.COPILOT_CHAT_OPTIMIZER_MODEL = chat_optimizer_model + backend_config.COPILOT_CHAT_TARGET_MODEL = chat_target_model + backend_config.COPILOT_CHAT_TIMEOUT = chat_timeout + backend_config.EXEC_EMPTY_RESPONSE_RETRIES = retries + for key, value in env.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + +@pytest.mark.parametrize("alias", ["copilot_exec"]) +def test_aliases_normalize_to_copilot_exec(alias: str) -> None: + assert normalize_backend_name(alias) == "copilot_exec" + + +@pytest.mark.parametrize("alias", ["copilot", "copilot_cli", "github_copilot"]) +def test_bare_copilot_aliases_normalize_to_the_cli_authenticated_chat_backend( + alias: str, +) -> None: + assert normalize_backend_name(alias) == "copilot_chat" + assert model.set_backend(alias) == "copilot_chat" + assert backend_config.get_optimizer_backend() == "copilot_chat" + assert backend_config.get_target_backend() == "copilot_chat" + + +def test_set_backend_routes_target_to_copilot_and_keeps_chat_optimizer() -> None: + assert model.set_backend("copilot_exec") == "copilot_exec" + # The CLI agent is the *target*; the optimizer must stay a chat model + # because it has to emit structured skill edits. + assert backend_config.get_target_backend() == "copilot_exec" + assert backend_config.get_optimizer_backend() == "openai_chat" + assert backend_config.is_target_exec_backend() is True + assert model.get_backend_name() == "copilot_exec" + + +def test_chat_target_refuses_exec_backend() -> None: + model.set_backend("copilot_exec") + with pytest.raises(NotImplementedError): + model.chat_target(system="s", user="u") + + +def test_configure_rejects_non_boolean_allow_all_tools() -> None: + with pytest.raises(ValueError): + model.configure_copilot_exec(allow_all_tools="sometimes") + + +@pytest.mark.parametrize( + ("raw", "expected"), + [("true", "1"), ("false", "0"), ("yes", "1"), ("off", "0"), ("1", "1"), ("weird", "0")], +) +def test_env_allow_all_tools_is_normalized_on_load(raw, expected, monkeypatch) -> None: + # A user may set the env var to 'true'/'false'; loading it raw used to make + # get_copilot_exec_config() raise. It is now normalized to '0'/'1' (unknown + # values fall back to the safe '0'), matching the other boolean-ish flags. + import importlib + + monkeypatch.setenv("COPILOT_EXEC_ALLOW_ALL_TOOLS", raw) + importlib.reload(backend_config) + try: + assert backend_config.COPILOT_EXEC_ALLOW_ALL_TOOLS == expected + assert backend_config.get_copilot_exec_config()["allow_all_tools"] == expected + finally: + monkeypatch.delenv("COPILOT_EXEC_ALLOW_ALL_TOOLS", raising=False) + importlib.reload(backend_config) + + +def test_parse_jsonl_concatenates_assistant_messages_and_ignores_noise() -> None: + raw = "\n".join( + [ + "not json at all", + json.dumps({"type": "tool.call", "data": {"content": "ignored"}}), + json.dumps({"type": "assistant.message", "data": {"content": "first"}}), + "{ broken json", + "[]", + "null", + json.dumps({"type": "assistant.message", "data": "invalid"}), + json.dumps({"type": "assistant.message", "data": {"content": "second"}}), + json.dumps({"type": "assistant.message", "data": {}}), + ] + ) + assert copilot_backend.parse_copilot_jsonl(raw) == "first\nsecond" + + +def test_parse_jsonl_returns_empty_for_no_assistant_messages() -> None: + assert copilot_backend.parse_copilot_jsonl('{"type":"tool.call","data":{}}') == "" + assert copilot_backend.parse_copilot_jsonl("") == "" + + +def _fake_run(captured: dict, stdout: str, returncode: int = 0): + def _run(cmd, **kwargs): + captured["cmd"] = cmd + captured["env"] = kwargs.get("env") or {} + captured["cwd"] = kwargs.get("cwd") + return subprocess.CompletedProcess(cmd, returncode, stdout=stdout, stderr="") + + return _run + + +def test_run_copilot_exec_builds_isolated_readonly_command(monkeypatch, tmp_path) -> None: + model.set_backend("copilot") + model.configure_copilot_exec(path="copilot", home=str(tmp_path / "home"), allow_all_tools=False) + monkeypatch.setenv("COPILOT_ALLOW_ALL", "true") + captured: dict = {} + stdout = json.dumps({"type": "assistant.message", "data": {"content": "answer"}}) + monkeypatch.setattr(harness.subprocess, "run", _fake_run(captured, stdout)) + + response, raw = harness.run_copilot_exec(work_dir=str(tmp_path), prompt="solve it", model="", timeout=30) + + assert response == "answer" + assert "COPILOT CLI ATTEMPT 1" in raw + cmd = captured["cmd"] + # Prompt is a single argv element -- never interpolated into a shell string. + assert "solve it" in " ".join(cmd) + assert cmd[cmd.index("-p") + 1].endswith("solve it") or "solve it" in cmd[cmd.index("-p") + 1] + assert "--output-format" in cmd and cmd[cmd.index("--output-format") + 1] == "json" + # Startup isolation: no user MCP servers or custom instructions. + assert "--disable-builtin-mcps" in cmd + assert "--no-custom-instructions" in cmd + # Read-only rollout must NOT bypass the CLI approval gate. + assert "--allow-all-tools" not in cmd + assert "COPILOT_ALLOW_ALL" not in captured["env"] + assert captured["env"]["COPILOT_HOME"] == str(tmp_path / "home") + + +def test_copilot_chat_calls_reach_the_aggregate_token_summary(monkeypatch) -> None: + # The CLI reports no token counts (documented caveat), but the call counts + # are real and must not be dropped from the run's token snapshots. + model.set_backend("copilot") + model.reset_token_tracker() + monkeypatch.setattr(copilot_backend, "_invoke", lambda *a, **k: "hi") + + model.chat_target(system="s", user="u", stage="target") + + summary = model.get_token_summary() + assert summary["target"]["calls"] >= 1 + assert summary["_total"]["calls"] >= 1 + + model.reset_token_tracker() + assert model.get_token_summary()["_total"]["calls"] == 0 + + +def test_config_defaults_do_not_clobber_copilot_env_settings() -> None: + # The shipped base config must not pin this to false: trainer.py and + # eval_only.py pass cfg.get("copilot_exec_allow_all_tools") straight into + # configure_copilot_exec(), and a non-None value overwrites the env var -- + # which would make the documented COPILOT_EXEC_ALLOW_ALL_TOOLS=1 opt-in + # impossible. null leaves the env (default off) in charge. + import yaml + + root = Path(__file__).resolve().parent.parent + with open(root / "configs" / "_base_" / "default.yaml", encoding="utf-8") as fh: + base = yaml.safe_load(fh) + assert base["model"]["copilot_exec_allow_all_tools"] is None + assert base["model"]["copilot_chat_timeout"] is None + + # None must be a no-op, so prior environment-derived settings survive. + model.configure_copilot_exec(allow_all_tools=True) + model.configure_copilot_exec(allow_all_tools=None) + assert backend_config.COPILOT_EXEC_ALLOW_ALL_TOOLS == "1" + model.configure_copilot_chat(timeout=123) + model.configure_copilot_chat(timeout=None) + assert backend_config.COPILOT_CHAT_TIMEOUT == "123" + + +def test_copilot_backends_keep_a_real_deployment_fallback() -> None: + # This table also feeds the shared Azure deployment fallback in the entry + # points (cfg.get("optimizer_model", default_model_for_backend(backend))), + # and the shipped base config sets no optimizer_model/target_model -- so a + # "" here left `--backend copilot_exec` with an EMPTY optimizer deployment + # even though that role is still a real openai_chat model. The CLI's own + # model comes from copilot_chat_optimizer_model / _target_model instead. + from skillopt.model.common import default_model_for_backend + + assert default_model_for_backend("copilot_exec") == "gpt-4o" + assert default_model_for_backend("copilot_chat") == "gpt-4o" + + +def test_train_copilot_exec_omits_inherited_openai_target_model(monkeypatch) -> None: + root = Path(__file__).resolve().parents[1] + monkeypatch.setattr( + sys, + "argv", + [ + "skillopt-train", + "--config", + str(root / "configs" / "searchqa" / "default.yaml"), + "--backend", + "copilot_exec", + ], + ) + + cfg = train_script.load_config(train_script.parse_args()) + + assert cfg["target_backend"] == "copilot_exec" + assert cfg["target_model"] == "" + + +def test_train_copilot_exec_preserves_explicit_target_model(monkeypatch) -> None: + root = Path(__file__).resolve().parents[1] + monkeypatch.setattr( + sys, + "argv", + [ + "skillopt-train", + "--config", + str(root / "configs" / "searchqa" / "default.yaml"), + "--backend", + "copilot_exec", + "--target_model", + "copilot-model-id", + ], + ) + + cfg = train_script.load_config(train_script.parse_args()) + + assert cfg["target_backend"] == "copilot_exec" + assert cfg["target_model"] == "copilot-model-id" + + +def test_no_response_error_includes_cli_output(monkeypatch, tmp_path) -> None: + # copilot_exec persists no artifacts, so a bare "returned no response" + # would leave an empty/invalid JSONL stream undebuggable. + model.set_backend("copilot") + model.configure_copilot_exec(path="copilot", home="", allow_all_tools=False) + captured: dict = {} + noise = '{"type":"tool.call","data":{"content":"NOISE-MARKER"}}' + monkeypatch.setattr(harness.subprocess, "run", _fake_run(captured, noise)) + + with pytest.raises(RuntimeError) as exc: + harness.run_copilot_exec(work_dir=str(tmp_path), prompt="p", model="", timeout=30) + + assert "returned no response" in str(exc.value) + assert "NOISE-MARKER" in str(exc.value) + + +def test_allow_all_tools_requires_both_opt_in_and_file_edits(monkeypatch, tmp_path) -> None: + model.set_backend("copilot") + stdout = json.dumps({"type": "assistant.message", "data": {"content": "ok"}}) + + # Opted in, but read-only rollout -> still gated. + model.configure_copilot_exec(allow_all_tools=True) + captured: dict = {} + monkeypatch.setattr(harness.subprocess, "run", _fake_run(captured, stdout)) + harness.run_copilot_exec(work_dir=str(tmp_path), prompt="p", model="", timeout=30, allow_file_edits=False) + assert "--allow-all-tools" not in captured["cmd"] + + # File edits requested but operator did not opt in -> still gated. + model.configure_copilot_exec(allow_all_tools=False) + captured = {} + monkeypatch.setattr(harness.subprocess, "run", _fake_run(captured, stdout)) + harness.run_copilot_exec(work_dir=str(tmp_path), prompt="p", model="", timeout=30, allow_file_edits=True) + assert "--allow-all-tools" not in captured["cmd"] + + # Both -> allowed. + model.configure_copilot_exec(allow_all_tools=True) + captured = {} + monkeypatch.setattr(harness.subprocess, "run", _fake_run(captured, stdout)) + harness.run_copilot_exec(work_dir=str(tmp_path), prompt="p", model="", timeout=30, allow_file_edits=True) + assert "--allow-all-tools" in captured["cmd"] + + +def test_nonzero_exit_raises_with_detail(monkeypatch, tmp_path) -> None: + model.set_backend("copilot") + captured: dict = {} + monkeypatch.setattr(harness.subprocess, "run", _fake_run(captured, "", returncode=3)) + with pytest.raises(RuntimeError, match="exit code 3"): + harness.run_copilot_exec(work_dir=str(tmp_path), prompt="p", model="", timeout=30) + + +def test_run_target_exec_dispatches_to_copilot(monkeypatch, tmp_path) -> None: + model.set_backend("copilot_exec") + called: dict = {} + + def _fake(**kwargs): + called.update(kwargs) + return "resp", "raw" + + monkeypatch.setattr(harness, "run_copilot_exec", _fake) + response, raw = harness.run_target_exec(work_dir=str(tmp_path), prompt="p", model="m", timeout=15) + assert (response, raw) == ("resp", "raw") + assert called["model"] == "m" + + +# --- copilot_chat: CLI-authenticated path (no separate provider API key) ----- + + +def test_copilot_alias_selects_cli_authenticated_chat_backend() -> None: + assert normalize_backend_name("copilot") == "copilot_chat" + assert model.set_backend("copilot") == "copilot_chat" + # Both halves run through the CLI, so no separate provider API key is used. + assert backend_config.get_optimizer_backend() == "copilot_chat" + assert backend_config.get_target_backend() == "copilot_chat" + assert backend_config.is_optimizer_chat_backend() is True + assert backend_config.is_target_chat_backend() is True + assert backend_config.is_target_exec_backend() is False + # Both roles unify to the canonical label, like claude_chat/qwen_chat, not + # the generic "copilot_chat+copilot_chat". + assert model.get_backend_name() == "copilot_chat" + + +def test_copilot_exec_still_pairs_with_a_chat_optimizer() -> None: + # Guards the split: `copilot` fills both roles, `copilot_exec` does not. + assert model.set_backend("copilot_exec") == "copilot_exec" + assert backend_config.get_optimizer_backend() == "openai_chat" + assert backend_config.get_target_backend() == "copilot_exec" + + +def test_chat_backend_composes_system_and_user_into_one_prompt(monkeypatch) -> None: + model.set_backend("copilot") + captured: dict = {} + monkeypatch.setenv("COPILOT_ALLOW_ALL", "true") + stdout = json.dumps({"type": "assistant.message", "data": {"content": "done"}}) + monkeypatch.setattr(copilot_backend.subprocess, "run", _fake_run(captured, stdout)) + + text, usage = model.chat_target(system="SYS", user="USR", retries=1) + + assert text == "done" + assert usage == {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + cmd = captured["cmd"] + prompt = cmd[cmd.index("-p") + 1] + assert "SYS" in prompt and "USR" in prompt + assert "--disable-builtin-mcps" in cmd and "--no-custom-instructions" in cmd + assert "--available-tools=" in cmd + # A chat call must never be granted unattended tool use. + assert "--allow-all-tools" not in cmd + assert "COPILOT_ALLOW_ALL" not in captured["env"] + + +def test_chat_backend_routes_optimizer_and_target_models(monkeypatch) -> None: + model.set_backend("copilot") + model.configure_copilot_chat(optimizer_model="opt-m", target_model="tgt-m") + stdout = json.dumps({"type": "assistant.message", "data": {"content": "x"}}) + + captured: dict = {} + monkeypatch.setattr(copilot_backend.subprocess, "run", _fake_run(captured, stdout)) + model.chat_optimizer(system="s", user="u", retries=1) + assert captured["cmd"][captured["cmd"].index("--model") + 1] == "opt-m" + + captured = {} + monkeypatch.setattr(copilot_backend.subprocess, "run", _fake_run(captured, stdout)) + model.chat_target(system="s", user="u", retries=1) + assert captured["cmd"][captured["cmd"].index("--model") + 1] == "tgt-m" + + +def test_chat_backend_raises_on_empty_response(monkeypatch) -> None: + model.set_backend("copilot") + captured: dict = {} + monkeypatch.setattr(copilot_backend.subprocess, "run", _fake_run(captured, "")) + with pytest.raises(RuntimeError, match="after 1 retries"): + model.chat_target(system="s", user="u", retries=1) + + +def test_chat_backend_raises_on_nonzero_exit(monkeypatch) -> None: + model.set_backend("copilot") + captured: dict = {} + monkeypatch.setattr(copilot_backend.subprocess, "run", _fake_run(captured, "", returncode=2)) + with pytest.raises(RuntimeError, match="exit code 2"): + model.chat_target(system="s", user="u", retries=1) + + +def test_configure_copilot_chat_rejects_bad_timeout() -> None: + with pytest.raises(ValueError): + model.configure_copilot_chat(timeout="0") + with pytest.raises(ValueError): + model.configure_copilot_chat(timeout="soon") + + +def test_messages_variant_flattens_roles(monkeypatch) -> None: + model.set_backend("copilot") + captured: dict = {} + stdout = json.dumps({"type": "assistant.message", "data": {"content": "ok"}}) + monkeypatch.setattr(copilot_backend.subprocess, "run", _fake_run(captured, stdout)) + + text, _ = model.chat_target_messages( + messages=[ + {"role": "system", "content": "SYSTEM_TEXT"}, + {"role": "user", "content": "USER_TEXT"}, + ], + retries=1, + ) + assert text == "ok" + prompt = captured["cmd"][captured["cmd"].index("-p") + 1] + assert "SYSTEM_TEXT" in prompt and "USER_TEXT" in prompt + + +def test_messages_variant_rejects_non_text_multipart_content(monkeypatch) -> None: + model.set_backend("copilot") + invoked = False + + def _unexpected_invoke(*args, **kwargs): + nonlocal invoked + invoked = True + return "unexpected" + + monkeypatch.setattr(copilot_backend, "_invoke", _unexpected_invoke) + + with pytest.raises(NotImplementedError, match="unsupported multipart type 'image_url'"): + model.chat_target_messages( + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,..."}}, + ], + } + ], + retries=1, + ) + assert invoked is False + + +@pytest.mark.parametrize("role", ["optimizer", "target"]) +def test_messages_variant_returns_compat_message_when_requested(monkeypatch, role) -> None: + from skillopt.model.common import CompatAssistantMessage + + model.set_backend("copilot") + monkeypatch.setattr(copilot_backend, "_invoke", lambda *a, **k: "ok") + + chat_messages = getattr(model, f"chat_{role}_messages") + message, usage = chat_messages( + messages=[{"role": "user", "content": "hello"}], + return_message=True, + retries=1, + ) + + assert isinstance(message, CompatAssistantMessage) + assert message.content == "ok" + assert message.tool_calls == [] + assert usage == {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + + +@pytest.mark.parametrize( + ("role", "kwargs"), + [ + ( + "optimizer", + {"tools": [{"type": "function", "function": {"name": "search"}}]}, + ), + ("target", {"tool_choice": "auto"}), + ], +) +def test_messages_variant_fails_fast_for_unsupported_tools( + monkeypatch, role, kwargs +) -> None: + model.set_backend("copilot") + invoked = False + + def _unexpected_invoke(*args, **call_kwargs): + nonlocal invoked + invoked = True + return "unexpected" + + monkeypatch.setattr(copilot_backend, "_invoke", _unexpected_invoke) + + chat_messages = getattr(model, f"chat_{role}_messages") + with pytest.raises(NotImplementedError, match="does not support caller-supplied tools"): + chat_messages( + messages=[{"role": "user", "content": "hello"}], + retries=1, + **kwargs, + ) + assert invoked is False + + +def test_copilot_config_keys_survive_flattening() -> None: + # Guards the 4-place wiring: config.py map, both scripts, and trainer. + flat = flatten_config( + { + "model": { + "copilot_exec_path": "/opt/copilot", + "copilot_exec_home": "/tmp/cph", + "copilot_exec_allow_all_tools": True, + "copilot_chat_optimizer_model": "opt-m", + "copilot_chat_target_model": "tgt-m", + "copilot_chat_timeout": 900, + } + } + ) + assert flat["copilot_exec_path"] == "/opt/copilot" + assert flat["copilot_exec_home"] == "/tmp/cph" + assert flat["copilot_exec_allow_all_tools"] is True + assert flat["copilot_chat_optimizer_model"] == "opt-m" + assert flat["copilot_chat_target_model"] == "tgt-m" + assert flat["copilot_chat_timeout"] == 900 + + +@pytest.mark.parametrize("script", ["train", "eval_only"]) +def test_cli_exposes_copilot_backend_and_flags(script: str) -> None: + root = Path(__file__).resolve().parents[1] + script_path = root / "scripts" / f"{script}.py" + result = subprocess.run( + [sys.executable, str(script_path), "--help"], + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + assert "copilot_chat" in result.stdout + assert "copilot_exec" in result.stdout + for flag in ( + "--copilot_exec_path", + "--copilot_exec_home", + "--copilot_exec_allow_all_tools", + "--copilot_chat_optimizer_model", + "--copilot_chat_target_model", + "--copilot_chat_timeout", + ): + assert flag in result.stdout + # train.py delegates backend configuration to the trainer; eval_only wires + # it directly. Inspect call nodes rather than source spelling/formatting. + applier_path = ( + script_path + if script == "eval_only" + else root / "skillopt" / "engine" / "trainer.py" + ) + tree = ast.parse(applier_path.read_text(encoding="utf-8")) + calls = { + node.func.id + for node in ast.walk(tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) + } + assert {"configure_copilot_chat", "configure_copilot_exec"} <= calls diff --git a/tests/test_role_backend_resolution.py b/tests/test_role_backend_resolution.py new file mode 100644 index 00000000..afb92e5a --- /dev/null +++ b/tests/test_role_backend_resolution.py @@ -0,0 +1,172 @@ +"""``--backend`` must survive the role backends the base config sets.""" + +from __future__ import annotations + +import pytest + +from scripts.eval_only import _set_role_if_default +from skillopt.engine.trainer import _resolve_role_backends + +# What configs/_base_/default.yaml ships. +_BASE_CONFIG = ("openai_chat", "openai_chat") + + +@pytest.mark.parametrize("current", [None, "", "openai_chat"]) +def test_eval_only_backend_label_replaces_inherited_role_defaults(current) -> None: + cfg = {"target_backend": current} + _set_role_if_default( + cfg, + "target_backend", + "copilot_exec", + explicitly_overridden=False, + ) + assert cfg["target_backend"] == "copilot_exec" + + +def test_eval_only_backend_label_preserves_custom_yaml_role() -> None: + cfg = {"optimizer_backend": "minimax_chat"} + _set_role_if_default( + cfg, + "optimizer_backend", + "copilot_chat", + explicitly_overridden=False, + ) + assert cfg["optimizer_backend"] == "minimax_chat" + + +def test_eval_only_backend_label_preserves_explicit_cli_role_override() -> None: + cfg = {"optimizer_backend": "openai_chat"} + _set_role_if_default( + cfg, + "optimizer_backend", + "copilot_chat", + explicitly_overridden=True, + ) + assert cfg["optimizer_backend"] == "openai_chat" + + +@pytest.mark.parametrize( + ("backend", "expected"), + [ + ("cursor", ("openai_chat", "cursor_exec")), + ("cursor_exec", ("openai_chat", "cursor_exec")), + ("claude", ("claude_chat", "claude_chat")), + ("claude_chat", ("claude_chat", "claude_chat")), + ("claude_code_exec", ("openai_chat", "claude_code_exec")), + ("codex", ("codex_exec", "codex_exec")), + ("codex_exec", ("codex_exec", "codex_exec")), + ("qwen", ("openai_chat", "qwen_chat")), + ("qwen_chat", ("openai_chat", "qwen_chat")), + ], +) +def test_backend_flag_wins_over_base_config_defaults(backend, expected) -> None: + # Regression: the base config sets both roles to openai_chat, which used to + # skip resolution entirely and silently run --backend on openai_chat. + assert _resolve_role_backends(backend, *_BASE_CONFIG) == expected + + +@pytest.mark.parametrize( + ("alias", "expected"), + [ + ("anthropic", ("claude_chat", "claude_chat")), + ("openai", ("codex_exec", "codex_exec")), + ("cursor_agent", ("openai_chat", "cursor_exec")), + ("copilot_cli", ("copilot_chat", "copilot_chat")), + ("github_copilot", ("copilot_chat", "copilot_chat")), + ("minimax", ("openai_chat", "minimax_chat")), + ("compat", ("openai_compatible", "openai_compatible")), + ("openai-compatible", ("openai_compatible", "openai_compatible")), + ], +) +def test_backend_aliases_are_normalized_before_role_resolution(alias, expected) -> None: + assert _resolve_role_backends(alias, *_BASE_CONFIG) == expected + + +def test_azure_openai_stays_on_openai_chat() -> None: + assert _resolve_role_backends("azure_openai", *_BASE_CONFIG) == _BASE_CONFIG + + +def test_unset_roles_are_resolved() -> None: + assert _resolve_role_backends("cursor", "", "") == ("openai_chat", "cursor_exec") + assert _resolve_role_backends("cursor", None, None) == ("openai_chat", "cursor_exec") + + +def test_explicit_non_default_roles_are_preserved() -> None: + # An operator who names a role backend outranks the high-level label. + assert _resolve_role_backends("cursor", "qwen_chat", "minimax_chat") == ( + "qwen_chat", + "minimax_chat", + ) + + +@pytest.mark.parametrize( + ("backend", "expected_target"), + [ + ("claude", "claude_chat"), + ("codex", "codex_exec"), + ("claude_code_exec", "claude_code_exec"), + ("cursor", "cursor_exec"), + ("copilot", "copilot_chat"), + ("copilot_exec", "copilot_exec"), + ("qwen", "qwen_chat"), + ], +) +def test_explicit_optimizer_is_preserved_while_default_target_is_resolved( + backend, expected_target +) -> None: + assert _resolve_role_backends(backend, "minimax_chat", "openai_chat") == ( + "minimax_chat", + expected_target, + ) + + +@pytest.mark.parametrize( + ("backend", "expected_optimizer"), + [ + ("claude", "claude_chat"), + ("codex", "codex_exec"), + ("claude_code_exec", "openai_chat"), + ("cursor", "openai_chat"), + ("copilot", "copilot_chat"), + ("copilot_exec", "openai_chat"), + ("qwen", "openai_chat"), + ], +) +def test_default_optimizer_is_resolved_while_explicit_target_is_preserved( + backend, expected_optimizer +) -> None: + assert _resolve_role_backends(backend, "openai_chat", "minimax_chat") == ( + expected_optimizer, + "minimax_chat", + ) + + +def test_explicit_target_is_preserved_when_optimizer_is_default() -> None: + assert _resolve_role_backends("cursor", "openai_chat", "minimax_chat") == ( + "openai_chat", + "minimax_chat", + ) + + +def test_copilot_maps_both_roles_to_the_cli_authenticated_backend() -> None: + # No separate provider API key is needed because the CLI carries sign-in. + assert _resolve_role_backends("copilot", *_BASE_CONFIG) == ("copilot_chat", "copilot_chat") + assert _resolve_role_backends("copilot_chat", *_BASE_CONFIG) == ( + "copilot_chat", + "copilot_chat", + ) + + +def test_copilot_exec_keeps_a_chat_optimizer() -> None: + assert _resolve_role_backends("copilot_exec", *_BASE_CONFIG) == ( + "openai_chat", + "copilot_exec", + ) + + +def test_claude_maps_both_roles() -> None: + assert _resolve_role_backends("claude", None, None) == ("claude_chat", "claude_chat") + # And -- the regression -- when the base config pins both roles to the + # truthy openai_chat default, --backend claude must still win. + assert _resolve_role_backends("claude", *_BASE_CONFIG) == ("claude_chat", "claude_chat") + assert _resolve_role_backends("claude_chat", *_BASE_CONFIG) == ("claude_chat", "claude_chat")