Skip to content
This repository was archived by the owner on Sep 13, 2026. It is now read-only.

Commit ecb9243

Browse files
committed
e2e: Add asciinema-based e2e test under e2e/cast
- Adds a new asciinema-based test for Claude - Adds README for new asciinema-based e2e test
1 parent 25cbbcb commit ecb9243

6 files changed

Lines changed: 623 additions & 0 deletions

File tree

‎e2e/cast/README.md‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
# Cast-Based E2E Testing
2+
3+
This directory contains a cast-based end-to-end (E2E) test for the AgentAPI project. The framework simulates realistic agent interactions by replaying asciicast v2 recordings.
4+
5+
## TL;DR
6+
7+
```shell
8+
go test ./e2e/cast
9+
```
10+
11+
## How it Works
12+
13+
The testing framework (`cast_test.go`) does the following:
14+
- Starts the AgentAPI server with a fake agent (`cmd/cast_agent.go`).
15+
- The fake agent replays a `.cast` file (asciicast v2 format), writing terminal output to stdout and validating stdin against the recorded input events.
16+
- The testing framework sends messages to the fake agent via the AgentAPI and validates the responses.
17+
18+
## Adding or Updating Fixtures
19+
20+
### Step 1: Record with asciinema
21+
22+
The recording captures both agentapi's output (what it sends to the agent) and the agent's responses.
23+
Use `asciinema` to record agentapi wrapping the agent. Below is an example for Claude:
24+
25+
```shell
26+
# Set terminal to minimum size Claude supports (80 columns)
27+
stty cols 80 rows 1000
28+
29+
# Build agentapi (or use an existing known-good version)
30+
agentapi server -t claude -- asciinema rec --stdin testdata/claude.cast --command 'echo hello | claude'
31+
32+
# Then interact with Claude via AgentAPI (either web UI or API):
33+
# 1. Wait for Claude to respond to the initial prompt
34+
# 2. Type a test message (e.g., "This is just a test.")
35+
# 3. Wait for Claude's reply
36+
# 4. Press Ctrl+C to exit
37+
```
38+
39+
**Important notes:**
40+
- The terminal dimensions (80x1000) must match the `--term-width` and `--term-height` flags in the test (see `defaultCmdFn` in `cast_test.go`).
41+
- Recording agentapi (not Claude directly) captures the exact byte sequences that agentapi sends, including bracketed paste mode escape sequences.
42+
- The `--stdin` flag captures your input, which the test uses to validate that agentapi sends the correct bytes.
43+
- To overwrite an existing fixture, add `--overwrite` to the `asciinema` invocation.
44+
45+
### Step 2: Create the sidecar script file
46+
47+
Create a matching sidecar file `testdata/my-fixture.txt` that lists the expected conversation in order, one entry per line:
48+
49+
```
50+
user hello
51+
agent Hello! How can I help you today?
52+
user This is just a test.
53+
agent Got it! Let me know if you need anything.
54+
```
55+
56+
Each line is `<role><TAB><message>`. Valid roles are `agent` or `user`.
57+
58+
**Flow explanation:**
59+
- Line 1: The initial prompt passed to the agent at startup
60+
- Line 2: The agent's reply to the initial prompt
61+
- Line 3: The test message sent via the AgentAPI
62+
- Line 4: The agent's reply to the test message
63+
64+
Note: The AgentAPI merges the startup sequence (welcome screen + initial prompt + initial reply) into the first agent message. The test expects 3 messages from the API, even though the script file has 4 entries.
65+
66+
### Step 3: Review and update the test
67+
68+
> **Caution:** Review the recording before committing. Remove or redact any API keys, tokens, or other sensitive data that may appear in the terminal output (stdout events), recorded keystrokes (`"i"` events from `--stdin`), or environment variables captured in the cast header.
69+
70+
To use a new fixture, update the `castFile` and `scriptFile` constants in `cast_test.go` to reference the new fixture files, then update the assertions in `TestE2E` to match the new conversation.

‎e2e/cast/cast_test.go‎

Lines changed: 259 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,259 @@
1+
package cast_test
2+
3+
import (
4+
"bufio"
5+
"bytes"
6+
"context"
7+
"fmt"
8+
"io"
9+
"net"
10+
"net/http"
11+
"os"
12+
"os/exec"
13+
"path/filepath"
14+
"strings"
15+
"sync"
16+
"testing"
17+
"time"
18+
19+
agentapisdk "github.com/coder/agentapi-sdk-go"
20+
"github.com/stretchr/testify/require"
21+
)
22+
23+
const (
24+
castTestTimeout = 30 * time.Second
25+
castOperationTimeout = 10 * time.Second
26+
castHealthCheckTimeout = 10 * time.Second
27+
castFile = "testdata/claude.cast"
28+
scriptFile = "testdata/claude.txt"
29+
)
30+
31+
type scriptEntry struct {
32+
Role string // "user" or "agent"
33+
Message string
34+
}
35+
36+
func loadScript(t testing.TB, path string) []scriptEntry {
37+
t.Helper()
38+
data, err := os.ReadFile(path)
39+
require.NoError(t, err)
40+
var entries []scriptEntry
41+
sc := bufio.NewScanner(bytes.NewReader(data))
42+
for sc.Scan() {
43+
line := sc.Text()
44+
if line == "" {
45+
continue
46+
}
47+
role, msg, ok := strings.Cut(line, "\t")
48+
require.True(t, ok, "malformed script line: %q", line)
49+
require.Contains(t, []string{"user", "agent"}, role,
50+
"unexpected role %q in script line: %q", role, line)
51+
entries = append(entries, scriptEntry{Role: role, Message: msg})
52+
}
53+
require.NoError(t, sc.Err())
54+
return entries
55+
}
56+
57+
func TestE2E(t *testing.T) {
58+
if testing.Short() {
59+
t.Skip("Skipping integration test in short mode")
60+
}
61+
62+
script := loadScript(t, scriptFile)
63+
require.GreaterOrEqual(t, len(script), 4, "claude.txt must have at least 4 entries")
64+
require.Equal(t, "user", script[0].Role) // initial prompt (startup)
65+
require.Equal(t, "agent", script[1].Role) // reply to initial prompt
66+
require.Equal(t, "user", script[2].Role) // test message sent via API
67+
require.Equal(t, "agent", script[3].Role) // reply to test message
68+
// Note: The API merges the startup sequence (initial prompt + initial reply)
69+
// into the first agent message, so we expect 3 messages from the API, not 4.
70+
initialPromptReply := script[1].Message
71+
userMessage := script[2].Message
72+
agentReply := script[3].Message
73+
74+
ctx, cancel := context.WithTimeout(context.Background(), castTestTimeout)
75+
defer cancel()
76+
77+
apiClient := setup(ctx, t)
78+
79+
// Agent should be running while processing the initial greeting.
80+
statusResp, err := apiClient.GetStatus(ctx)
81+
require.NoError(t, err)
82+
require.Equal(t, agentapisdk.StatusRunning, statusResp.Status)
83+
84+
require.NoError(t, waitAgentAPIStable(ctx, t, apiClient, castOperationTimeout, "initial stable"))
85+
86+
_, err = apiClient.PostMessage(ctx, agentapisdk.PostMessageParams{
87+
Content: userMessage,
88+
Type: agentapisdk.MessageTypeUser,
89+
})
90+
require.NoError(t, err, "failed to send message")
91+
92+
// Agent should be running while processing the reply.
93+
statusResp, err = apiClient.GetStatus(ctx)
94+
require.NoError(t, err)
95+
require.Equal(t, agentapisdk.StatusRunning, statusResp.Status)
96+
97+
require.NoError(t, waitAgentAPIStable(ctx, t, apiClient, castOperationTimeout, "post message"))
98+
99+
msgResp, err := apiClient.GetMessages(ctx)
100+
require.NoError(t, err, "failed to get messages")
101+
require.Len(t, msgResp.Messages, 3)
102+
// First message is agent greeting (contains startup screen + initial prompt reply)
103+
require.Contains(t, msgResp.Messages[0].Content, initialPromptReply)
104+
require.Contains(t, msgResp.Messages[1].Content, userMessage)
105+
require.Contains(t, msgResp.Messages[2].Content, agentReply)
106+
}
107+
108+
func defaultCmdFn(ctx context.Context, t testing.TB, serverPort int, binaryPath, cwd string) (string, []string) {
109+
// Terminal dimensions must match the cast file (80x1000) for correct ANSI escape sequence positioning.
110+
return binaryPath, []string{"server", fmt.Sprintf("--port=%d", serverPort), "--term-width=80", "--term-height=1000", "--", "go", "run", filepath.Join(cwd, "cmd", "cast_agent.go"), castFile}
111+
}
112+
113+
func setup(ctx context.Context, t testing.TB) *agentapisdk.Client {
114+
t.Helper()
115+
116+
binaryPath := os.Getenv("AGENTAPI_BINARY_PATH")
117+
if binaryPath == "" {
118+
cwd, err := os.Getwd()
119+
require.NoError(t, err, "Failed to get current working directory")
120+
// We're in e2e/cast, so go up two levels to reach the repo root
121+
binaryPath = filepath.Join(cwd, "..", "..", "out", "agentapi")
122+
t.Logf("Building binary at %s", binaryPath)
123+
buildCmd := exec.CommandContext(ctx, "go", "build", "-o", binaryPath, ".")
124+
buildCmd.Dir = filepath.Join(cwd, "..", "..")
125+
t.Logf("run: %s", buildCmd.String())
126+
require.NoError(t, buildCmd.Run(), "Failed to build binary")
127+
}
128+
129+
serverPort, err := getFreePort()
130+
require.NoError(t, err, "Failed to get free port for server")
131+
132+
cwd, err := os.Getwd()
133+
require.NoError(t, err, "Failed to get current working directory")
134+
135+
bin, args := defaultCmdFn(ctx, t, serverPort, binaryPath, cwd)
136+
t.Logf("Running command: %s %s", bin, strings.Join(args, " "))
137+
cmd := exec.CommandContext(ctx, bin, args...)
138+
139+
stdout, err := cmd.StdoutPipe()
140+
require.NoError(t, err, "Failed to create stdout pipe")
141+
142+
stderr, err := cmd.StderrPipe()
143+
require.NoError(t, err, "Failed to create stderr pipe")
144+
145+
err = cmd.Start()
146+
require.NoError(t, err, "Failed to start agentapi server")
147+
148+
var wg sync.WaitGroup
149+
wg.Add(2)
150+
151+
go func() {
152+
defer wg.Done()
153+
logOutput(t, "SERVER-STDOUT", stdout)
154+
}()
155+
156+
go func() {
157+
defer wg.Done()
158+
logOutput(t, "SERVER-STDERR", stderr)
159+
}()
160+
161+
t.Cleanup(func() {
162+
if cmd.Process != nil {
163+
_ = cmd.Process.Kill()
164+
_ = cmd.Wait()
165+
}
166+
wg.Wait()
167+
})
168+
169+
serverURL := fmt.Sprintf("http://localhost:%d", serverPort)
170+
require.NoError(t, waitForServer(ctx, t, serverURL, castHealthCheckTimeout), "Server not ready")
171+
apiClient, err := agentapisdk.NewClient(serverURL)
172+
require.NoError(t, err, "Failed to create agentapi SDK client")
173+
174+
return apiClient
175+
}
176+
177+
func logOutput(t testing.TB, prefix string, r io.Reader) {
178+
t.Helper()
179+
scanner := bufio.NewScanner(r)
180+
for scanner.Scan() {
181+
t.Logf("[%s] %s", prefix, scanner.Text())
182+
}
183+
}
184+
185+
func waitForServer(ctx context.Context, t testing.TB, url string, timeout time.Duration) error {
186+
t.Helper()
187+
client := &http.Client{Timeout: time.Second}
188+
healthCtx, cancel := context.WithTimeout(ctx, timeout)
189+
defer cancel()
190+
191+
ticker := time.NewTicker(100 * time.Millisecond)
192+
defer ticker.Stop()
193+
194+
for {
195+
select {
196+
case <-healthCtx.Done():
197+
require.Failf(t, "failed to start server", "server at %s not ready within timeout: %w", url, healthCtx.Err())
198+
case <-ticker.C:
199+
resp, err := client.Get(url)
200+
if err == nil {
201+
_ = resp.Body.Close()
202+
return nil
203+
}
204+
t.Logf("Server not ready yet: %s", err)
205+
}
206+
}
207+
}
208+
209+
func waitAgentAPIStable(ctx context.Context, t testing.TB, apiClient *agentapisdk.Client, waitFor time.Duration, msg string) error {
210+
t.Helper()
211+
waitCtx, waitCancel := context.WithTimeout(ctx, waitFor)
212+
defer waitCancel()
213+
214+
start := time.Now()
215+
var currStatus agentapisdk.AgentStatus
216+
var lastMessage string
217+
defer func() {
218+
elapsed := time.Since(start)
219+
t.Logf("%s: agent API status: %s (elapsed: %s)", msg, currStatus, elapsed.Round(100*time.Millisecond))
220+
if t.Failed() && lastMessage != "" {
221+
fmt.Fprintf(os.Stderr, "\n=== Last agent message ===\n%s\n=== End last agent message ===\n", lastMessage)
222+
}
223+
}()
224+
evts, errs, err := apiClient.SubscribeEvents(ctx)
225+
require.NoError(t, err, "failed to subscribe to events")
226+
for {
227+
select {
228+
case <-waitCtx.Done():
229+
return waitCtx.Err()
230+
case evt := <-evts:
231+
if esc, ok := evt.(agentapisdk.EventStatusChange); ok {
232+
currStatus = esc.Status
233+
if currStatus == agentapisdk.StatusStable {
234+
return nil
235+
}
236+
} else if emc, ok := evt.(agentapisdk.EventMessageUpdate); ok {
237+
lastMessage = emc.Message
238+
t.Logf("Got message event: id=%d role=%s len=%d", emc.Id, emc.Role, len(emc.Message))
239+
}
240+
case err := <-errs:
241+
return fmt.Errorf("read events: %w", err)
242+
}
243+
}
244+
}
245+
246+
func getFreePort() (int, error) {
247+
addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
248+
if err != nil {
249+
return 0, err
250+
}
251+
252+
l, err := net.ListenTCP("tcp", addr)
253+
if err != nil {
254+
return 0, err
255+
}
256+
defer func() { _ = l.Close() }()
257+
258+
return l.Addr().(*net.TCPAddr).Port, nil
259+
}

0 commit comments

Comments
 (0)