feat: execution recovery safeguards

.github/workflows/sync-cli-docs.yml

@@ -70,7 +70,7 @@ jobs:
           echo "Synced $COPIED command pages."
 
           # Sync hand-written guides (add frontmatter for Nextra)
-          for file in quickstart.md concepts.md; do
+          for file in quickstart.md concepts.md execution-recovery.md; do
             TITLE=$(head -1 "$CLI_DOCS/$file" | sed 's/^# //')
 
             # Build the Nextra-compatible version with frontmatter

cmd/execute/contract_call.go

@@ -1,14 +1,14 @@
 package execute
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"net/http"
 	"os"
 	"time"
 
 	"github.com/jedib0t/go-pretty/v6/table"
+	"github.com/keeperhub/cli/internal/execrecovery"
 	khhttp "github.com/keeperhub/cli/internal/http"
 	"github.com/keeperhub/cli/internal/output"
 	"github.com/keeperhub/cli/pkg/cmdutil"
@@ -61,6 +61,7 @@ func NewContractCallCmd(f *cmdutil.Factory) *cobra.Command {
 			abiFile, _ := cmd.Flags().GetString("abi-file")
 			wait, _ := cmd.Flags().GetBool("wait")
 			timeout, _ := cmd.Flags().GetDuration("timeout")
+			idemKeyFlag, _ := cmd.Flags().GetString("idempotency-key")
 
 			reqBody := contractCallRequest{
 				ContractAddress: contract,
@@ -88,13 +89,13 @@ func NewContractCallCmd(f *cmdutil.Factory) *cobra.Command {
 				return fmt.Errorf("marshalling request: %w", err)
 			}
 
-			req, err := client.NewRequest(http.MethodPost, khhttp.BuildBaseURL(host)+"/api/execute/contract-call", bytes.NewReader(bodyBytes))
+			idemKey, err := execrecovery.ResolveIdempotencyKey(idemKeyFlag)
 			if err != nil {
 				return err
 			}
-			req.Header.Set("Content-Type", "application/json")
 
-			resp, err := client.Do(req)
+			deadline := time.Now().Add(timeout)
+			resp, err := postIdempotentJSON(client, khhttp.BuildBaseURL(host)+"/api/execute/contract-call", bodyBytes, idemKey, deadline)
 			if err != nil {
 				return err
 			}
@@ -152,6 +153,7 @@ func NewContractCallCmd(f *cmdutil.Factory) *cobra.Command {
 	cmd.Flags().String("abi-file", "", "Path to local ABI JSON file")
 	cmd.Flags().Bool("wait", false, "Wait for completion")
 	cmd.Flags().Duration("timeout", 5*time.Minute, "Timeout when using --wait")
+	cmd.Flags().String("idempotency-key", "", "Stable Idempotency-Key for write intents (auto-generated if empty)")
 
 	_ = cmd.MarkFlagRequired("chain")
 	_ = cmd.MarkFlagRequired("contract")

cmd/execute/contract_call_test.go

@@ -6,11 +6,13 @@ import (
 	"net/http/httptest"
 	"os"
 	"strings"
+	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/keeperhub/cli/cmd/execute"
 	"github.com/keeperhub/cli/internal/config"
+	"github.com/keeperhub/cli/internal/execrecovery"
 	khhttp "github.com/keeperhub/cli/internal/http"
 	"github.com/keeperhub/cli/pkg/cmdutil"
 	"github.com/keeperhub/cli/pkg/iostreams"
@@ -20,7 +22,7 @@ func newContractCallFactory(ios *iostreams.IOStreams, srv *httptest.Server) *cmd
 	client := khhttp.NewClient(khhttp.ClientOptions{
 		Host:       srv.URL,
 		AppVersion: "test",
-		IOStreams:   ios,
+		IOStreams:  ios,
 	})
 	return &cmdutil.Factory{
 		IOStreams: ios,
@@ -304,6 +306,159 @@ func TestContractCallCmd_WaitWritePolls(t *testing.T) {
 	}
 }
 
+func TestContractCallCmd_SendsIdempotencyKey(t *testing.T) {
+	var gotKey string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotKey = r.Header.Get(execrecovery.IdempotencyHeader)
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusAccepted)
+		_, _ = w.Write([]byte(`{"executionId":"exec-cc-idem","status":"completed"}`))
+	}))
+	defer srv.Close()
+
+	ios, _, _, _ := iostreams.Test()
+	f := newContractCallFactory(ios, srv)
+	cmd := execute.NewContractCallCmd(f)
+	cmd.SetArgs([]string{"--chain", "1", "--contract", "0xcontract", "--method", "transfer", "--idempotency-key", "stable-cc-1"})
+
+	if err := cmd.Execute(); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if gotKey != "stable-cc-1" {
+		t.Fatalf("Idempotency-Key=%q, want stable-cc-1", gotKey)
+	}
+}
+
+func TestContractCallCmd_IdempotencyKeyStableAcrossHTTPRetries(t *testing.T) {
+	var keys []string
+	var calls atomic.Int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		n := calls.Add(1)
+		keys = append(keys, r.Header.Get(execrecovery.IdempotencyHeader))
+		if n == 1 {
+			w.WriteHeader(http.StatusBadGateway)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusAccepted)
+		_, _ = w.Write([]byte(`{"executionId":"exec-cc-retry","status":"completed"}`))
+	}))
+	defer srv.Close()
+
+	ios, _, _, _ := iostreams.Test()
+	f := newContractCallFactory(ios, srv)
+	cmd := execute.NewContractCallCmd(f)
+	cmd.SetArgs([]string{"--chain", "1", "--contract", "0xcontract", "--method", "transfer", "--idempotency-key", "retry-stable-cc"})
+
+	if err := cmd.Execute(); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if calls.Load() < 2 {
+		t.Fatalf("expected HTTP retry, got %d calls", calls.Load())
+	}
+	for i, k := range keys {
+		if k != "retry-stable-cc" {
+			t.Fatalf("call %d Idempotency-Key=%q, want retry-stable-cc", i, k)
+		}
+	}
+}
+
+func TestContractCallCmd_IdempotencyKeyStableAcross504(t *testing.T) {
+	var keys []string
+	var calls atomic.Int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		n := calls.Add(1)
+		keys = append(keys, r.Header.Get(execrecovery.IdempotencyHeader))
+		if n == 1 {
+			w.WriteHeader(http.StatusGatewayTimeout)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusAccepted)
+		_, _ = w.Write([]byte(`{"executionId":"exec-cc-504","status":"completed"}`))
+	}))
+	defer srv.Close()
+
+	ios, _, _, _ := iostreams.Test()
+	f := newContractCallFactory(ios, srv)
+	cmd := execute.NewContractCallCmd(f)
+	cmd.SetArgs([]string{"--chain", "1", "--contract", "0xcontract", "--method", "transfer", "--idempotency-key", "cc-504"})
+
+	if err := cmd.Execute(); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if calls.Load() < 2 {
+		t.Fatalf("expected HTTP retry, got %d calls", calls.Load())
+	}
+	for i, k := range keys {
+		if k != "cc-504" {
+			t.Fatalf("call %d key=%q", i, k)
+		}
+	}
+}
+
+func TestContractCallCmd_IdempotencyInProgressRetriesSameKey(t *testing.T) {
+	var keys []string
+	var calls atomic.Int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		n := calls.Add(1)
+		keys = append(keys, r.Header.Get(execrecovery.IdempotencyHeader))
+		w.Header().Set("Content-Type", "application/json")
+		if n == 1 {
+			w.WriteHeader(http.StatusConflict)
+			_, _ = w.Write([]byte(`{"error":"in flight","code":"idempotency_in_progress","retryable":true}`))
+			return
+		}
+		w.WriteHeader(http.StatusAccepted)
+		_, _ = w.Write([]byte(`{"executionId":"exec-cc-inprog","status":"completed"}`))
+	}))
+	defer srv.Close()
+
+	ios, _, _, _ := iostreams.Test()
+	f := newContractCallFactory(ios, srv)
+	cmd := execute.NewContractCallCmd(f)
+	cmd.SetArgs([]string{"--chain", "1", "--contract", "0xcontract", "--method", "transfer", "--idempotency-key", "cc-inprog", "--timeout", "10s"})
+
+	if err := cmd.Execute(); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if calls.Load() != 2 {
+		t.Fatalf("got %d POSTs, want 2", calls.Load())
+	}
+	for i, k := range keys {
+		if k != "cc-inprog" {
+			t.Fatalf("call %d key=%q", i, k)
+		}
+	}
+}
+
+func TestContractCallCmd_IdempotencyConflictFails(t *testing.T) {
+	var calls atomic.Int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		calls.Add(1)
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusConflict)
+		_, _ = w.Write([]byte(`{"error":"Idempotency-Key was reused with a different request payload.","code":"idempotency_conflict","retryable":false}`))
+	}))
+	defer srv.Close()
+
+	ios, _, _, _ := iostreams.Test()
+	f := newContractCallFactory(ios, srv)
+	cmd := execute.NewContractCallCmd(f)
+	cmd.SetArgs([]string{"--chain", "1", "--contract", "0xcontract", "--method", "transfer", "--idempotency-key", "cc-conflict"})
+
+	err := cmd.Execute()
+	if err == nil {
+		t.Fatal("expected conflict")
+	}
+	if !strings.Contains(err.Error(), "do not retry with a new key") {
+		t.Fatalf("got %v", err)
+	}
+	if calls.Load() != 1 {
+		t.Fatalf("got %d POSTs, want 1", calls.Load())
+	}
+}
+
 func TestContractCallCmd_WaitFailsWhenWriteResponseAlreadyFailed(t *testing.T) {
 	pollCount := 0
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

cmd/execute/idempotent_write.go

@@ -0,0 +1,76 @@
+package execute
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/keeperhub/cli/internal/execrecovery"
+	khhttp "github.com/keeperhub/cli/internal/http"
+)
+
+const inProgressBackoff = 200 * time.Millisecond
+const inProgressBackoffMax = 2 * time.Second
+
+// postIdempotentJSON POSTs body with a stable Idempotency-Key.
+//
+// HTTP 5xx retries are handled by the retryable client (same key).
+// HTTP 409 is classified by body code from lib/idempotency.ts:
+//
+//	idempotency_in_progress -> retry the same key until deadline
+//	idempotency_conflict    -> fail; never mint a new key
+func postIdempotentJSON(client *khhttp.Client, url string, body []byte, idemKey string, deadline time.Time) (*http.Response, error) {
+	backoff := inProgressBackoff
+	for {
+		req, err := client.NewRequest(http.MethodPost, url, bytes.NewReader(body))
+		if err != nil {
+			return nil, err
+		}
+		req.Header.Set("Content-Type", "application/json")
+		req.Header.Set(execrecovery.IdempotencyHeader, idemKey)
+
+		resp, err := client.Do(req)
+		if err != nil {
+			return nil, err
+		}
+
+		if resp.StatusCode != http.StatusConflict {
+			return resp, nil
+		}
+
+		raw, readErr := io.ReadAll(resp.Body)
+		_ = resp.Body.Close()
+		if readErr != nil {
+			return nil, fmt.Errorf("reading 409 body: %w", readErr)
+		}
+
+		info, ok := execrecovery.ParseIdempotencyBody(raw)
+		if ok && info.IsInProgress() {
+			if !deadline.IsZero() && time.Now().After(deadline) {
+				return nil, execrecovery.InProgressTimeoutError{Key: idemKey}
+			}
+			time.Sleep(backoff)
+			if backoff < inProgressBackoffMax {
+				backoff *= 2
+				if backoff > inProgressBackoffMax {
+					backoff = inProgressBackoffMax
+				}
+			}
+			continue
+		}
+		if ok && info.IsConflict() {
+			return nil, execrecovery.ConflictError{Body: info, Key: idemKey}
+		}
+
+		msg := string(raw)
+		if info.Error != "" {
+			msg = info.Error
+		}
+		if msg == "" {
+			msg = http.StatusText(http.StatusConflict)
+		}
+		return nil, &khhttp.APIError{StatusCode: http.StatusConflict, Body: raw, Message: msg}
+	}
+}

cmd/execute/status.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/jedib0t/go-pretty/v6/table"
+	"github.com/keeperhub/cli/internal/execrecovery"
 	khhttp "github.com/keeperhub/cli/internal/http"
 	"github.com/keeperhub/cli/internal/output"
 	"github.com/keeperhub/cli/pkg/cmdutil"
@@ -44,33 +45,12 @@ func nextPollDelay(resp *http.Response) (time.Duration, bool) {
 	return time.Duration(min(secs, maxPollIntervalSecs)) * time.Second, false
 }
 
-// ExecStatusResponse represents the execution status API response.
-// Shared by transfer, contract-call and status commands.
-type ExecStatusResponse struct {
-	ExecutionID     string        `json:"executionId"`
-	Status          string        `json:"status"`
-	Type            string        `json:"type"`
-	TransactionHash *string       `json:"transactionHash"`
-	TransactionLink *string       `json:"transactionLink"`
-	Result          any           `json:"result"`
-	Error           *string       `json:"error"`
-	CreatedAt       string        `json:"createdAt"`
-	CompletedAt     *string       `json:"completedAt"`
-	Receipts        []ExecReceipt `json:"receipts"`
-}
+// ExecStatusResponse is the GET /api/execute/{id}/status wire type.
+// Canonical definition: execrecovery.DirectStatus.
+type ExecStatusResponse = execrecovery.DirectStatus
 
-// ExecReceipt is a chain-re-fetched proof entry attached to an execution.
-// A transactionHash alone proves a transaction was submitted; a receipt with
-// verified=true and receiptStatus="success" proves it landed onchain.
-type ExecReceipt struct {
-	Hash          string  `json:"hash"`
-	ChainID       int64   `json:"chainId"`
-	Verified      bool    `json:"verified"`
-	ReceiptStatus string  `json:"receiptStatus"`
-	BlockNumber   *int64  `json:"blockNumber,omitempty"`
-	GasUsed       *string `json:"gasUsed,omitempty"`
-	VerifiedAt    *string `json:"verifiedAt,omitempty"`
-}
+// ExecReceipt is DirectExecutionReceiptEntry on the wire.
+type ExecReceipt = execrecovery.Receipt
 
 func NewStatusCmd(f *cmdutil.Factory) *cobra.Command {
 	cmd := &cobra.Command{
@@ -182,12 +162,8 @@ func renderExecStatus(p *output.Printer, f *cmdutil.Factory, sr *ExecStatusRespo
 		printUnconfirmedNotice(f, sr.ExecutionID, sr.TransactionHash)
 	}
 
-	if sr.Status == "failed" {
-		msg := fmt.Sprintf("execution %s failed", sr.ExecutionID)
-		if sr.Error != nil && *sr.Error != "" {
-			msg = *sr.Error
-		}
-		return fmt.Errorf("%s", msg)
+	if err := execOutcomeError(sr); err != nil {
+		return err
 	}
 
 	return nil
@@ -236,6 +212,8 @@ func watchExecStatus(f *cmdutil.Factory, client *khhttp.Client, host, executionI
 	for {
 		sr, delay, serverSaysTerminal, err := fetchExecStatus(client, host, executionID)
 		if err != nil {
+			// HTTP 404 is terminal for --watch (mistyped id / other org).
+			// Cold-start 404 tolerance lives only in pollExecStatus (--wait).
 			return err
 		}