Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
74 commits
Select commit Hold shift + click to select a range
bc30fe1
feat: add harness integration and lifecycle audit skill
nikhilachale Oct 10, 2026
bf6a30d
docs: require attached AO UI screenshots for harness claims
Oct 10, 2026
1816b30
fix: safely clear audit-owned cancelled drafts
Oct 10, 2026
1c7b3b4
fix: wait for audit draft persistence removal
Oct 10, 2026
35300d4
docs: attach fresh AO harness proof to skill report
Oct 10, 2026
14b8ad0
docs: qualify retained Tau failure capture provenance
Oct 10, 2026
f9c9b6f
docs: consolidate Strands and memcode AO recovery
Oct 10, 2026
cfcfffc
docs: pin corrected memcode evidence manifest scope
Oct 10, 2026
019e960
docs: preserve Hermes and OpenJarvis native AO evidence
Oct 11, 2026
f0a837f
docs: report scoped Hermes and OpenJarvis TUI results
Oct 11, 2026
419f70f
docs: spell native Hermes workspace arguments exactly
Oct 11, 2026
f18c771
docs: preserve Docker Agent ACP runtime and Electron evidence
Oct 11, 2026
5f9477f
fix: require cancellation process evidence
nikhilachale Oct 11, 2026
117c9a8
test: avoid cancellation witness race
nikhilachale Oct 11, 2026
dd88623
test: make cancellation witness deterministic
nikhilachale Oct 11, 2026
8245d9a
test: arm late witness after start
nikhilachale Oct 11, 2026
d49e43d
test: separate witness from poll interval
nikhilachale Oct 11, 2026
95269e8
fix: bind cancellation to process identity
nikhilachale Oct 11, 2026
17223fd
test: couple late witness to child process
nikhilachale Oct 11, 2026
f3e4895
test: widen stale cue fixture window
nikhilachale Oct 11, 2026
506eb72
test: terminate cancellation witness deterministically
nikhilachale Oct 11, 2026
270c3b3
test: fence cancellation side effect
nikhilachale Oct 11, 2026
434a6fa
fix: distinguish late witness probe errors
nikhilachale Oct 11, 2026
118e891
docs: preserve Mastra TUI qualification and native permission evidence
Oct 11, 2026
8d98673
docs: correct Mastra screenshot capture timestamp
Oct 11, 2026
92c6b0e
docs: record exact Mastra audit cleanup
Oct 11, 2026
36f0605
docs: record CodeBuddy native and AO TUI qualification
Oct 11, 2026
fdb2776
docs: retain Stakpak native cancellation and restore blockers
Oct 11, 2026
0c7b765
docs: retain Pool ACP provider routing qualification hold
Oct 11, 2026
42803b6
docs: record Freebuff instruction hold and remaining batch gates
Oct 11, 2026
f9da74d
docs: verify Pool alternate credential prerequisite
Oct 11, 2026
d4ccc7f
docs: qualify Pool native ACP recovery with real AO evidence
Oct 11, 2026
6f10905
docs: record Stakpak recovery ownership prerequisites
Oct 11, 2026
12601e9
docs: record Freebuff released instruction transport prerequisite
Oct 11, 2026
aa0a700
test: await mux fixture child before short cancellation deadline
Oct 11, 2026
cbef9c4
docs: record Stakpak native cancellation fix and prerelease AO evidence
Oct 12, 2026
384c80e
docs: record Freebuff native snapshot fix and exact prerelease AO evi…
Oct 12, 2026
60f694c
docs: retain ZCode protocol receipts and independent cleanup failures
Oct 12, 2026
0784a81
chore: reconcile current main documentation index for audit evidence CI
Oct 12, 2026
eaeed69
docs: preserve both documentation index additions after merge
Oct 12, 2026
0422af4
docs: keep harness evidence in its integration PR
Oct 12, 2026
def16c2
docs: clarify independent standing instruction proof
Oct 12, 2026
7411e7d
docs: focus harness skill on reusable integration guidance
nikhilachale Oct 13, 2026
139167d
docs: add harness audit operating recipe and journey map
nikhilachale Oct 13, 2026
397cabc
docs: clarify root terminal websocket route
nikhilachale Oct 13, 2026
4790046
docs: guide ACP integration through shared Chat UI
nikhilachale Oct 13, 2026
9d1195e
docs: require complete harness integration handoffs
nikhilachale Oct 14, 2026
b053c11
docs: require harness screenshots for every UI journey step
nikhilachale Oct 14, 2026
5af6c23
docs: qualify saved-credential screenshot route
nikhilachale Oct 14, 2026
396eb5d
docs: keep harness evidence in PR comments and attachments
nikhilachale Oct 15, 2026
78d01a4
docs: define complete harness capability and evidence workflow
nikhilachale Oct 16, 2026
bfb28b7
feat(harness-skill): interrupt-safe cleanup and touchpoint scanner
nikhilachale Oct 16, 2026
a67b5d2
feat(harness-skill): audit every Codex/Claude capability with per-fai…
nikhilachale Oct 21, 2026
2bca6e7
feat(harness-skill): self-driving fix loop and fixes for failures see…
nikhilachale Oct 21, 2026
efb55be
feat(harness-skill): prove system-prompt delivery per channel, with a…
nikhilachale Oct 21, 2026
5fea088
feat(harness-skill): ready-to-run local audit for Codex and Claude Code
nikhilachale Oct 22, 2026
7502771
feat(harness-skill): one clear integrate-and-audit workflow from a link
nikhilachale Oct 22, 2026
4d2a9a5
fix(harness-skill): UI audit follows the Harness page's Account/Model…
nikhilachale Oct 22, 2026
8f27654
fix(harness-skill): match current main's harness wiring conventions
nikhilachale Oct 22, 2026
af57840
Merge remote-tracking branch 'upstream/main' into feat/harness-integr…
nikhilachale Oct 22, 2026
f383c18
fix(harness-skill): give the AO build its own timeout and a readable …
nikhilachale Oct 22, 2026
f7cce9f
fix(harness-skill): read large native model catalogs whole
nikhilachale Oct 22, 2026
7a062f5
fix(harness-skill): prove instructions by behaviour, not by asking mo…
nikhilachale Oct 22, 2026
b25e750
fix(harness-skill): scope audit conventions to PROOF.json; defer firs…
nikhilachale Oct 22, 2026
0bc4c66
fix(harness-skill): live-run fixes for restore token, approvals and t…
nikhilachale Oct 22, 2026
d8336e6
feat(harness-skill): stop a running TUI agent the AO way, by archivin…
nikhilachale Oct 22, 2026
eda110d
fix(harness-skill): read the V1 role answer wherever the agent saved it
nikhilachale Oct 22, 2026
8d8df66
fix(harness-skill): find the harness row by ID, not by typing the ID …
nikhilachale Oct 22, 2026
21c42c5
fix(harness-skill): Claude Chat runtime, ready-cue screen text, slowe…
nikhilachale Oct 22, 2026
833297b
fix(harness-skill): match ready cues against the screen a user sees
nikhilachale Oct 22, 2026
5c14bc8
fix(harness-skill): approval test asks the agent to request escalatio…
nikhilachale Oct 22, 2026
7b1e3a6
fix(harness-skill): V1 role answer goes to the session artifacts folder
nikhilachale Oct 22, 2026
914f000
fix(harness-skill): read Claude Chat models from config options
nikhilachale Oct 22, 2026
89308d0
refactor(skill): make the harness audit generic for any harness
nikhilachale Oct 22, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
226 changes: 226 additions & 0 deletions .agents/skills/ao-agent-harness/SKILL.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,226 @@
---
name: ao-agent-harness
description: Use when given a link, repo or name of a coding-agent CLI to integrate into AO as a harness (TUI, Chat, installer, models, auth/login), when completing or reviewing a harness integration PR, or when auditing any integrated harness with step-by-step screenshots and a system-prompt check.
---

# AO harness: integrate and audit

Give this skill a link or a name (a GitHub repo, npm/PyPI package, docs page)
and it walks you from "is it already in AO?" to a working, audited harness:
the TUI and Chat adapters, installer, model list, auth detection, login button
and Connected state. It then verifies everything in the real app, with a
screenshot for every step, pass or fail.

It works the same for every harness. Nothing in it is specific to one agent:
everything harness-specific lives in that harness's **contract**, one JSON
entry you write in step 2. Codex and Claude Code are already integrated, so
use them as worked examples to copy from, not as the target.

Work through the steps in order. Each one says what to do, which tool runs it,
and when it is done.

| Step | What | Tool | Done when |
| --- | --- | --- | --- |
| 0 | **Intake:** already integrated? PR already open? | `scripts/harness_intake.mjs --link URL` | Decision printed: new, continue PR, complete existing, or update checkout |
| 1 | **Compatibility:** what the CLI supports and how much AO can use | [integration-path §1](references/integration-path.md#1-map-the-upstream-contract) | Compatibility table filled from the CLI's real `--help`/behaviour |
| 2 | **Build** in AO (order below) and **write its contract** | `scripts/harness_touchpoints.mjs --harness ID --like SIMILAR` | Every required touchpoint `ok`; contract entry added |
| 3 | **Local test:** standalone agent, New Task, TUI and Chat | `scripts/run_local.sh audit ID` | Live audit gates pass |
| 4 | **UI audit** with screenshots | `scripts/ui_audit.mjs` | Every checkpoint has a PASS/FAIL screenshot |
| 5 | **V1: system prompt** (worker vs orchestrator) | `--suites roles` (included in `audit`) | Both sessions name their own role |
| 6 | **Fix, then hand off** | `run_local.sh next`, `harness_pr_check.mjs` | Fix loop `done` (or only `needs_user`/upstream left); evidence posted on the PR |

## Step 0: Intake

```bash
node scripts/harness_intake.mjs --link https://github.com/org/agent-cli --binary agent --repo /path/to/ao
```

It scans the AO checkout for the harness, searches all PRs (open, merged,
closed) on `OrchestratorInc/agent-orchestrator`, and reads upstream facts
(release OS coverage, installers, and what the README and the local `--help`
mention). Follow its decision:

- **Open PR:** continue that PR, do not duplicate it.
- **Already integrated:** jump to step 3 and audit it.
- **Merged but missing locally:** update your checkout first.
- **New:** go to step 1.

## Step 1: Compatibility

Pin a released version. Then fill the table in
[integration-path §1](references/integration-path.md#1-map-the-upstream-contract)
from the CLI's real behaviour, not the README. The intake's hints are only a
starting point. Answer at least:

- Does it have an interactive TUI that AO can launch?
- Does it have a structured protocol for Chat: **ACP** (Agent Client Protocol)
or an SDK/app-server?
- Can it list models? How does it log in? Which OSes and installers does it
support?
- Can it resume a session by ID, and how does it take a system prompt?

Record what it does not support, with proof. Those become truthful "not
supported" states, not gaps.

## Step 2: Build, in this order

Copy the closest existing harness. `--like` prints where that harness is wired
in every file:

```bash
node scripts/harness_touchpoints.mjs --harness <id> --like <closest-harness> --repo /path/to/ao
```

1. **TUI adapter:** launch, prompt delivery, hooks/activity, native session ID,
resume, and the system prompt channel ([integration-path §3](references/integration-path.md#3-implement-the-tui-adapter)).
2. **Chat:** only if ACP or an SDK exists and passes its checks
([chat-integration](references/chat-integration.md)).
3. **Installer:** every OS and installer the CLI officially supports, so AO's
**Install** button works (touchpoints `install-target`, `install-plan`).
4. **Models:** list them (`selectionMode: catalog`), or declare free-form entry
(`text`) when the CLI has no list.
5. **Auth detection** on macOS, Linux and Windows. Report `authorized` only after
real verification. A key that is merely present is `configured`.
6. **Login/Setup button:** whatever the harness offers (CLI login, API key,
config), as an auth plan (touchpoint `auth-plan`).
7. **Connected state:** after auth and configuration the Agent Harness page
shows Connected. Otherwise it shows the same Not connected / Configure state
other harnesses show.

Then add branding (logo and licence), docs, and the reviewer if it is
supported. Read [lessons from earlier harness PRs](references/lessons-from-harness-prs.md)
before writing the adapter. The same mistakes failed 16 PRs.

### Write the harness contract

Copy the entry in [`assets/contract-template.json`](assets/contract-template.json)
into [`assets/local-contracts.json`](assets/local-contracts.json) under the new
agent ID, then fill it from the CLI's real behaviour (the `codex` and
`claude-code` entries there are filled-in examples):

| Field | What to put |
| --- | --- |
| `binary`, `versionArgs` | The CLI's command and its version flag |
| `integration` | The login-status command and the text it prints when logged in |
| `session` | A one-shot, non-interactive run with `{prompt}`, used to prove the CLI works outside AO |
| `models` | The model-list command, if the CLI has one |
| `surfaces` | `["tui"]`, `["chat"]` or both: what AO integrates |
| `adapters` | AO's directory names for its TUI adapter and Chat driver |
| `instructions` | How AO passes the system prompt, per surface: `flag`, `protocol`, `config` or `rule-file` |
| `restoredReady` | A regex matching only the empty input box after a restore |
| `supports` | Only to override the defaults below, e.g. `"chat_steer": "no"` |
| `hooks` | Only if AO writes the hooks into a file: its path and the text it must contain |

Every capability is **required** of a harness that has its surface (spawn,
instructions, messaging, kill and restore…) or **optional** (MCP status,
skills, steer, compaction, the TUI ↔ Chat switch). For an optional one, a
clean refusal is reported as `UNSUPPORTED`, not as a failure. Field details:
[local-cli-contract](references/local-cli-contract.md).

## Step 3: Local test (API audit)

```bash
scripts/run_local.sh doctor <id> # Node, AO checkout freshness, the CLI and its login
scripts/run_local.sh audit <id> # live audit with your system's existing logins
```

The runner builds AO from the checkout and runs it on an isolated daemon. It
then drives the harness through the API:

- install and auth readiness
- the model list
- a standalone (scratch) TUI session: spawn, prompt, follow-up, cancel, kill
and exact restore
- Chat: spawn, messages, approvals, interrupt and restore
- the TUI ↔ Chat switch
- the system-prompt checks

Without an ID it audits every harness in the contract file, which is how you
check that a shared change did not break the others.

## Step 4: UI audit with screenshots

Start the real desktop app from the AO checkout with a debugging port. The
first time only, run `npm ci` in `frontend/`. See the `ao-desktop-dev` skill for
details.

```bash
cd /path/to/ao/frontend
env -u AO_DATA_DIR -u AO_RUN_FILE -u AO_PORT npm run dev -- -- --remote-debugging-port=9222
```

Then, in another terminal, run:

```bash
scripts/run_local.sh ui <id> # add --install / --login to click those buttons too
```

It clicks through the same journey a user takes, and saves a screenshot at
every checkpoint, success or failure:

1. **Settings › Harness lists it**, and its detail view opens.
2. **Account tab: install.** Install is offered, or it is already installed.
3. **Health tab** shows Authentication, Executable and Last checked.
4. **Models tab** lists the harness's models (with their on/off switches).
5. **Account tab: login.** Login or Set up is offered, or it is already
Connected.
6. **Auth check:** Health › Check again says Connected.
7. **Update** on the Account tab. **Skipped in v0.**
8. **New Task** lists the agent and its models.
9. **Spawn a task**, and see the running TUI and/or Chat.

Each checkpoint is `PASS`, `FAIL`, `BLOCKED`, `NEEDS_USER` or `NOT_RUN`, with
its screenshot and reason, in `ui-report.md`. By default the audit installs and
logs in nothing: it reports what is offered. Selectors live in one table at
the top of `scripts/ui_audit.mjs`; update them there when the UI changes.

## Step 5: V1, the system prompt

`--suites roles` spawns one **worker** and one **orchestrator** and asks each:
"Are you a worker or an orchestrator?". Each must answer with its own role,
from AO's system prompt alone. A wrong answer means the role prompt never
reached the agent. The `instruction_channel*` gates then show where it was
lost: argv flag, Chat protocol, config file, or the last-resort rule file, as
declared in the contract (see [local-cli-contract](references/local-cli-contract.md)).

## Step 6: Fix, then hand off

Every run writes `<report>.next-actions.json`.

1. Run `scripts/run_local.sh next` to see the next fix.
2. Fix it, commit, then run `scripts/run_local.sh only <gate> <id>` to rerun
just that gate.
3. Stop when only `needs_user` items remain (a login or API key you must
provide) or upstream limits.

Before pushing, run `scripts/harness_pr_check.mjs --base upstream/main --pr N`.
Post the reports and screenshots as **comments on the integration PR**, never
commit them. Details: [fix loop](references/fix-loop.md),
[evidence and handoff](references/evidence-and-handoff.md).

## Rules

- **Auth:** use the machine's existing logins. Never log in or out, read
credential files, or paste keys yourself. When a login or API key is
missing, stop and ask the user for it.
- **Never pass a check by weakening it.** A pass must come from product behaviour.
- **Use a fresh report folder per attempt,** outside the Git checkout.
- **Audit an up-to-date AO checkout.** `run_local.sh doctor` shows how far
behind it is.
- **Missing screenshots stay `BLOCKED`** with their reason. They never count as
passes.

## Reference map

| Need | Read |
| --- | --- |
| Full build checklist and completion matrix | [integration-path](references/integration-path.md) |
| Chat (ACP/SDK) wiring | [chat-integration](references/chat-integration.md) |
| Local CLI contract fields | [local-cli-contract](references/local-cli-contract.md) |
| What every gate proves | [capability-suites](references/capability-suites.md) |
| Manual UI journeys and screenshot checklist | [audit-journeys](references/audit-journeys.md) |
| Failure classes and the fix loop | [fix-loop](references/fix-loop.md) |
| Gate contract and evidence rules | [evidence-and-handoff](references/evidence-and-handoff.md) |
| Mistakes from 16 earlier harness PRs | [lessons-from-harness-prs](references/lessons-from-harness-prs.md) |

Fixture tests (offline): `scripts/run_local.sh fixtures`.
37 changes: 37 additions & 0 deletions .agents/skills/ao-agent-harness/assets/contract-template.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
{
"schemaVersion": 1,
"agents": {
"my-agent": {
"binary": "my-agent",
"versionArgs": ["--version"],
"integration": {
"args": ["auth", "status"],
"expect": "Logged in"
},
"session": {
"args": ["run", "--yes", "{prompt}"],
"proofFile": "LOCAL_AUDIT_PROOF.json"
},
"models": {
"args": ["models", "--json"],
"format": "json"
},
"surfaces": ["tui", "chat"],
"adapters": { "tui": "myagent", "chat": "myagentacp" },
"instructions": {
"tui": { "channel": "flag" },
"chat": { "channel": "protocol" }
},
"restoredReady": {
"patterns": ["<regex that matches only the empty input box after a restore>"]
},
"supports": {
"chat_steer": "no"
},
"hooks": {
"file": ".my-agent/hooks.json",
"contains": ["hooks my-agent"]
}
}
}
}
73 changes: 73 additions & 0 deletions .agents/skills/ao-agent-harness/assets/local-contracts.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
{
"schemaVersion": 1,
"agents": {
"codex": {
"binary": "codex",
"versionArgs": ["--version"],
"integration": {
"args": ["login", "status"],
"expect": "Logged in"
},
"session": {
"args": ["exec", "--skip-git-repo-check", "--dangerously-bypass-approvals-and-sandbox", "{prompt}"],
"proofFile": "LOCAL_AUDIT_PROOF.json"
},
"models": {
"args": ["debug", "models"],
"format": "json"
},
"restoredReady": {
"patterns": ["›[ \\t\\u00a0]*(?:Ask Codex to do anything|Explain this codebase|Summarize recent commits|Implement \\{feature\\}|Find and fix a bug in @filename|Write tests for @filename|Improve documentation in @filename|Run /review on my current changes|Use /skills to list available skills)[ \\t]*\\n(?:[ \\t]*\\n)?[^\\n]*·[^\\n]*(?:\\n[^\\n]*){0,2}\\s*$"]
},
"instructions": {
"tui": {
"channel": "flag"
},
"chat": {
"channel": "protocol"
}
},
"surfaces": ["tui", "chat"],
"adapters": { "tui": "codex", "chat": "codexappserver" },
"supports": {
"chat_mcp": "yes",
"chat_skills": "yes",
"chat_steer": "yes",
"chat_compaction": "yes",
"interface_switch": "yes"
}
},
"claude-code": {
"binary": "claude",
"versionArgs": ["--version"],
"integration": {
"args": ["auth", "status"],
"expect": "\"loggedIn\": true"
},
"session": {
"args": ["-p", "--dangerously-skip-permissions", "{prompt}"],
"proofFile": "LOCAL_AUDIT_PROOF.json"
},
"restoredReady": {
"patterns": ["❯[ \\t\\u00a0]*\\n[ \\t]*─{10,}[ \\t]*(?:\\n(?![^\\n]*esc to interrupt)[^\\n]*){0,3}\\s*$"]
},
"instructions": {
"tui": {
"channel": "flag"
},
"chat": {
"channel": "protocol"
}
},
"surfaces": ["tui", "chat"],
"adapters": { "tui": "claudecode", "chat": "claudeacp" },
"supports": {
"interface_switch": "yes"
},
"hooks": {
"file": ".claude/settings.local.json",
"contains": ["hooks claude-code", "SessionStart", "UserPromptSubmit", "Stop"]
}
}
}
}
Loading
Loading