diff --git a/cmd/context-guru-proxy/main.go b/cmd/context-guru-proxy/main.go index cf0ffb63c..bd97fdb82 100644 --- a/cmd/context-guru-proxy/main.go +++ b/cmd/context-guru-proxy/main.go @@ -19,7 +19,6 @@ import ( "net/http" "os" "os/signal" - "path/filepath" "strconv" "strings" "sync" @@ -206,11 +205,15 @@ func main() { "hosted mode: max in-flight requests per tenant (0 = unlimited)") cheapConcurrent = flag.Int("cheap-model-concurrent", envInt("CHEAP_MODEL_CONCURRENT", 4), "process-wide cap on concurrent compaction-model calls, so one tenant's extract_llm cannot stall everyone's agents (0 = unlimited)") - - dashBench = flag.String("dashboard-bench-dirs", envOr("DASHBOARD_BENCH_DIRS", ""), - "comma-separated directories of benchmark runs (each with summary.json + rows-*.json) to ingest") + // The Benchmarks tab is gone. The flag stays as a no-op so an existing unit file or + // environment that still passes it starts instead of exiting on an unknown flag. + dashBenchIgnored = flag.String("dashboard-bench-dirs", envOr("DASHBOARD_BENCH_DIRS", ""), + "deprecated and ignored: the dashboard no longer has a Benchmarks tab") ) flag.Parse() + if *dashBenchIgnored != "" { + slog.Warn("--dashboard-bench-dirs / DASHBOARD_BENCH_DIRS is ignored: the dashboard no longer has a Benchmarks tab") + } // --version before anything else: an installer asks a binary what it is, and it must be // able to ask without starting a server or needing a config. buildinfo.Version was already @@ -327,7 +330,6 @@ func main() { QueueSize: *dashQueue, LifecycleSignals: *dashLifecycle, TrustedCIDRs: splitComma(*dashCIDRs), - BenchDirs: splitComma(*dashBench), DiskHighWatermark: *dashDiskHigh, DiskLowWatermark: *dashDiskLow, @@ -379,31 +381,6 @@ func main() { } else { slog.Info("dashboard: recovered session recency across the restart", "sessions", n) } - if runs, tasks := rec.DB().IngestBenchRoots(opts.BenchDirs); runs > 0 { - slog.Info("dashboard: ingested benchmark runs", "runs", runs, "tasks", tasks) - } else if len(opts.BenchDirs) > 0 { - // Asked to ingest and found nothing. Say so: ingestion runs ONCE at startup, - // so a silent zero means the Benchmarks tab stays empty forever with no clue - // why, and the most likely cause is invisible from inside the process. - // - // That cause is PrivateTmp. The shipped unit sets PrivateTmp=true, so the - // service gets its OWN empty /tmp — a directory that plainly exists in the - // operator's shell is simply not there for this process. Pointing - // DASHBOARD_BENCH_DIRS at /tmp/ therefore looks correct, changes - // nothing, and reports nothing. Naming the path we actually looked at is what - // makes that diagnosable in one glance at the log. - hint := "" - for _, d := range opts.BenchDirs { - if strings.HasPrefix(filepath.Clean(d), "/tmp/") { - hint = "a /tmp path is NOT visible to this process when the unit sets " + - "PrivateTmp=true; copy the run directory somewhere readable " + - "(e.g. under the state dir) and point DASHBOARD_BENCH_DIRS there" - break - } - } - slog.Warn("dashboard: no benchmark runs ingested; the Benchmarks tab will be empty", - "dirs", strings.Join(opts.BenchDirs, ","), "hint", hint) - } slog.Info("dashboard enabled", "url", "http://"+addr+"/dashboard/", "db", rec.DB().Path(), "content_capture", *dashContent) } diff --git a/dash/api.go b/dash/api.go index c95278952..60535a9a9 100644 --- a/dash/api.go +++ b/dash/api.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "log/slog" "net" "net/http" @@ -333,7 +334,10 @@ func (a *API) serveJSON(w http.ResponseWriter, r *http.Request, c *jsonCache, ke // Detaching it means the result is always cached even if nobody is left to receive it, so the // next reader gets a hit rather than starting again. Each caller still gets its own deadline // from the handler timeout, which is what bounds THEIR wait; this bounds the WORK. - v, err, _ := a.jsonInflight.Do(key, func() (any, error) { + // The flight key names the CACHE as well as the entry: /api/stats and /api/tools, for one account and + // the same filters, have the same entry key, and two concurrent requests for them collapsed onto one + // computation, so /api/stats answered with the tools report. + v, err, _ := a.jsonInflight.Do(fmt.Sprintf("%p\x00%s", c, key), func() (any, error) { ctx, cancel := context.WithTimeout(context.Background(), dashComputeTimeout) defer cancel() body, err := compute(a.rec.DB().WithContext(ctx)) @@ -651,8 +655,6 @@ func (a *API) routes() []route { {"GET /api/breakdown", scopeTenant, a.breakdown}, {"GET /api/facets", scopeTenant, a.facets}, {"GET /api/config", scopeManager, a.config}, - {"GET /api/benchmarks", scopeManager, a.benchmarks}, - {"GET /api/benchmarks/{id}/tasks", scopeManager, a.benchmarkTasks}, {"GET /api/capture", scopeTenant, a.capture}, {"GET /api/whoami", scopePublic, a.whoamiHandler}, {"GET /api/archive", scopeTenant, a.archive}, @@ -1586,43 +1588,6 @@ func (a *API) config(w http.ResponseWriter, r *http.Request) { writeJSON(w, body) } -// benchmarks lists ingested harbor runs. A MANAGER view in hosted mode: the runs are -// the operator's own eval history, not any tenant's traffic — and ?refresh=1 walks the -// filesystem and inserts rows, which unauthenticated is a writer denial of service -// anyone can repeat. -func (a *API) benchmarks(w http.ResponseWriter, r *http.Request) { - if !a.requireManager(w, r, "benchmark runs") { - return - } - if r.URL.Query().Get("refresh") == "1" { - // The DIRS go back with the counts. Without them a re-scan of a mistyped path and a - // correct scan of an empty directory are the same response — and the UI reported both - // as "nothing happened" because it discarded the body entirely. - dirs := a.rec.Opts().BenchDirs - runs, tasks := a.db(r).IngestBenchRoots(dirs) - writeJSON(w, map[string]any{"ingested_runs": runs, "ingested_tasks": tasks, "dirs": dirs}) - return - } - runs, err := a.db(r).BenchRuns() - if err != nil { - httpErr(w, http.StatusInternalServerError, err.Error()) - return - } - writeJSON(w, map[string]any{"runs": runs}) -} - -func (a *API) benchmarkTasks(w http.ResponseWriter, r *http.Request) { - if !a.requireManager(w, r, "benchmark runs") { - return - } - rows, err := a.db(r).BenchTasks(atoi64(r.PathValue("id")), r.URL.Query().Get("arm")) - if err != nil { - httpErr(w, http.StatusInternalServerError, err.Error()) - return - } - writeJSON(w, map[string]any{"tasks": rows}) -} - // capture reports the capture pipeline's own health, drops included. Exposed as a // first-class endpoint (and rendered in the UI) because a dashboard that hides its // own coverage gaps cannot be trusted about anything else. diff --git a/dash/api_test.go b/dash/api_test.go index aa0de3038..0bd170c6d 100644 --- a/dash/api_test.go +++ b/dash/api_test.go @@ -71,7 +71,7 @@ func TestAPIRoutesServeJSON(t *testing.T) { for _, path := range []string{ "/api/stats", "/api/series?bucket=60000", "/api/requests", "/api/sessions", - "/api/components", "/api/facets", "/api/benchmarks", "/api/capture", + "/api/components", "/api/facets", "/api/capture", } { w, body := get(t, a, path, "127.0.0.1:1234") if w.Code != http.StatusOK { @@ -307,35 +307,20 @@ func TestUIHasTestIDsForEveryStatTile(t *testing.T) { } source := string(js) + string(html) - // Stat tiles get their testid as "tile-"+key from the tile() helper, so assert on - // the call site: tile('', … . Renaming a tile then fails here, before the - // Playwright checks and the docs screenshots silently go stale. - for _, key := range []string{ - "requests", "tokens-before", "tokens-after", "saved-gross", "saved-unique", - "saved-adjusted", "overcount", "cost-baseline", "cost-actual", "cost-cg", - "saved-usd", "total-saved-usd", "cachesplit-saved", - "split-requests", "split-tail-moved", "split-credited", "split-hit-rate", - "split-historical", - "cache-read", "cache-write", "fresh-input", "output", - "cg-latency", "upstream-latency", "expands", "reverts", "passthroughs", - } { - if !strings.Contains(source, "tile('"+key+"'") { - t.Errorf("stat tile %q is not rendered (expected a tile('%s', …) call); "+ - "data-testid=tile-%s is asserted by the UI checks and the docs screenshots", key, key, key) - } - } + // The Overview's three savings rows carry their testids literally (see the list below). The + // old grid of stat tiles was removed on purpose: Overview now shows only the money saved by + // keep-alive, compaction and the tools and skills filter. // Every other panel carries its testid literally. for _, id := range []string{ - "denominators", "waterfall", "safety-cost", "cache-miss", "uncompressed-reasons", - "accounting", "chart-cost", "chart-tokens", "chart-cache", "chart-latency", + "saved-ledger", "saved-cache", "saved-compaction", "saved-filter", "recommendations", + "tab-map", "overview-diagnostics", "waterfall", "chart-cost", "chart-tokens", "chart-cache", "chart-latency", "chart-volume", "chart-components", "components-table", "sessions-table", "requests-table", "requests-page", "requests-next", "requests-prev", - "bench-list", "bench-run", "bench-scatter", "bench-tasks", "config-body", "capture-body", "capture-warning", "observe-banner", "drawer-body", "diff-block", "detail-summary", "detail-components", "live-table", "live-indicator", "theme-toggle", "tab-overview", "tab-components", "tab-sessions", "tab-requests", - "tab-benchmarks", "tab-config", "filter-q", "filter-range", "filter-model", + "tab-config", "filter-q", "filter-range", "filter-model", // The nav's first level. tab-overview above is the Overview group's button: that // group is also a view, so it is the one that carries data-view as well, and its // testid stays the way to reach the Overview view. tab-note is the one line that @@ -360,7 +345,7 @@ func TestUIHasTestIDsForEveryStatTile(t *testing.T) { "filter-from", "filter-to", "filter-range-apply", // The manager's scope control, and the two things the money pass added to the page: // the not-netted prefix-change diagnostic and the unfinished-amortization pill. - "filter-tenant", "prefix-change-cost", "in-flight", + "filter-tenant", "in-flight", } { if !strings.Contains(source, `"`+id+`"`) && !strings.Contains(source, "'"+id+"'") { t.Errorf("data-testid %q is not produced by the UI; a check or screenshot depends on it", id) @@ -380,7 +365,6 @@ func TestManagerOnlyTabsCarryTheGatingAttributes(t *testing.T) { } want := map[string][]string{ "components": {"data-manager", "data-local-ok"}, - "benchmarks": {"data-manager", "data-local-ok"}, "config": {"data-manager", "data-local-ok"}, "tenants": {"data-manager"}, } @@ -407,167 +391,6 @@ func TestManagerOnlyTabsCarryTheGatingAttributes(t *testing.T) { } } -// TestBenchIngestCommitsNothingForARunWithNoTasks pins the counter/table agreement. -// IngestBenchDir used to INSERT the bench_runs row before it knew whether any rows-* -// file parsed, then return tasks=0 — so IngestBenchRoots did not count the run but the -// row was already committed. A real jobs root produced "runs=17" in the log and 42 rows -// from the API, 25 of them with no arms, padding the Benchmarks tab with empty shells -// and making the PR's own "42 runs ingested" claim wrong. -func TestBenchIngestCommitsNothingForARunWithNoTasks(t *testing.T) { - dir := t.TempDir() - write := func(sub, name, body string) { - if err := os.MkdirAll(filepath.Join(dir, sub), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, sub, name), []byte(body), 0o600); err != nil { - t.Fatal(err) - } - } - // A summary with no rows files at all: the shape of an abandoned or in-progress run. - write("smoke-hd", "summary.json", `{"model":"m","dataset":"d"}`) - // A summary whose only rows file is truncated, so nothing parses. - write("dbg1", "summary.json", `{"model":"m","dataset":"d"}`) - write("dbg1", "rows-off.json", `[{"task":"t1",`) - // And one good run, so this proves selectivity rather than a blanket refusal. - write("real", "summary.json", `{"model":"m","dataset":"d"}`) - write("real", "rows-off.json", `[{"task":"t1","reward":1.0,"steps":3,"agent_cost":0.1}]`) - - db := openTestDB(t) - runs, tasks := db.IngestBenchRoots([]string{dir}) - if runs != 1 || tasks != 1 { - t.Errorf("ingested %d runs / %d tasks; want 1/1 (the two contentless dirs must not count)", runs, tasks) - } - - got, err := db.BenchRuns() - if err != nil { - t.Fatal(err) - } - // The assertion that actually failed before: the API's row count must EQUAL the - // counter the log reports. - if len(got) != runs { - names := make([]string, len(got)) - for i, r := range got { - names[i] = r.Name - } - t.Errorf("BenchRuns returned %d rows but the ingest counted %d runs: %v", len(got), runs, names) - } - for _, r := range got { - if len(r.Arms) == 0 { - t.Errorf("run %q was committed with no arms; the UI would render an empty row", r.Name) - } - } -} - -func TestBenchIngestFromRealHarnessArtifacts(t *testing.T) { - dir := t.TempDir() - run := filepath.Join(dir, "study-run") - if err := os.MkdirAll(run, 0o755); err != nil { - t.Fatal(err) - } - // Exactly the shape deploy/harbor/*.py writes. - summary := `{"model":"aws/claude-sonnet-5","dataset":"swe-bench-verified", - "price":[2e-06,1e-05,2e-07,2.5e-06],"tasks":2, - "configs":[{"config":"codesmart","solved":1,"solve_rate":0.5}]}` - rowsOff := `[{"task":"t1","reward":1.0,"steps":20,"prompt_tokens":1000,"completion_tokens":50, - "cache_read":900,"cache_write":90,"fresh_input":10,"agent_cost":0.5,"norm_cost":0.4, - "wall_s":100.0,"exception":false}, - {"task":"t2","reward":0.0,"steps":30,"prompt_tokens":2000,"completion_tokens":60, - "cache_read":1800,"cache_write":180,"fresh_input":20,"agent_cost":1.0,"norm_cost":0.8, - "wall_s":200.0,"exception":true}]` - rowsCG := `[{"task":"t1","reward":1.0,"steps":18,"prompt_tokens":800,"completion_tokens":45, - "cache_read":740,"cache_write":50,"fresh_input":10,"agent_cost":0.4,"norm_cost":0.32, - "wall_s":95.0,"exception":false}, - {"task":"t2","reward":1.0,"steps":25,"prompt_tokens":1600,"completion_tokens":55, - "cache_read":1500,"cache_write":90,"fresh_input":10,"agent_cost":0.8,"norm_cost":0.64, - "wall_s":180.0,"exception":false}]` - write := func(name, body string) { - if err := os.WriteFile(filepath.Join(run, name), []byte(body), 0o600); err != nil { - t.Fatal(err) - } - } - write("summary.json", summary) - write("rows-off.json", rowsOff) - write("rows-codesmart.json", rowsCG) - // A directory that is not a run must be skipped silently, since a jobs root is - // full of in-progress and unrelated directories. - if err := os.MkdirAll(filepath.Join(dir, "not-a-run"), 0o755); err != nil { - t.Fatal(err) - } - // A truncated rows file must not abort the whole ingest. - write("rows-broken.json", `[{"task":"t1",`) - - db := openTestDB(t) - runs, tasks := db.IngestBenchRoots([]string{dir}) - if runs != 1 || tasks != 4 { - t.Fatalf("ingested %d runs / %d tasks; want 1/4", runs, tasks) - } - - got, err := db.BenchRuns() - if err != nil { - t.Fatal(err) - } - if len(got) != 1 { - t.Fatalf("BenchRuns = %d", len(got)) - } - r := got[0] - if r.Model != "aws/claude-sonnet-5" || r.Dataset != "swe-bench-verified" { - t.Errorf("run metadata: %+v", r) - } - if len(r.Arms) != 2 { - t.Fatalf("arms = %d; want 2 (off, codesmart)", len(r.Arms)) - } - byArm := map[string]BenchArm{} - for _, a := range r.Arms { - byArm[a.Arm] = a - } - off, cg := byArm["off"], byArm["codesmart"] - if off.Tasks != 2 || off.Solved != 1 || off.Exceptions != 1 { - t.Errorf("off arm = %+v", off) - } - // Solve rate is over SCORED trials — an exception is not a failed solve. - if off.Scored != 1 || off.SolveRate != 1 { - t.Errorf("off solve rate = %v over %d scored; an exception must not count as unsolved", - off.SolveRate, off.Scored) - } - if cg.Solved != 2 || cg.TotalCostUSD < 1.19 || cg.TotalCostUSD > 1.21 { - t.Errorf("codesmart arm = %+v", cg) - } - if cg.CacheHitRate <= 0 || cg.CacheHitRate >= 1 { - t.Errorf("cache hit rate = %v; want a fraction", cg.CacheHitRate) - } - - // Per-task drill-down. - tasksOut, err := db.BenchTasks(r.ID, "codesmart") - if err != nil { - t.Fatal(err) - } - if len(tasksOut) != 2 { - t.Fatalf("per-task rows = %d; want 2", len(tasksOut)) - } - all, err := db.BenchTasks(r.ID, "") - if err != nil { - t.Fatal(err) - } - if len(all) != 4 { - t.Errorf("all-arm rows = %d; want 4", len(all)) - } - - // Re-ingest must REPLACE, not duplicate — a proxy restart pointed at a jobs root - // would otherwise double every historical run. - runs2, tasks2 := db.IngestBenchRoots([]string{dir}) - if runs2 != 1 || tasks2 != 4 { - t.Fatalf("re-ingest = %d runs / %d tasks", runs2, tasks2) - } - after, _ := db.BenchRuns() - if len(after) != 1 { - t.Errorf("re-ingest duplicated the run: %d runs", len(after)) - } - allAfter, _ := db.BenchTasks(after[0].ID, "") - if len(allAfter) != 4 { - t.Errorf("re-ingest duplicated tasks: %d rows", len(allAfter)) - } -} - func TestAPIBadRequestIDs(t *testing.T) { a, _ := newTestAPI(t, Options{}) for _, path := range []string{"/api/requests/0", "/api/requests/abc"} { @@ -714,8 +537,8 @@ func TestUIClaimsOnlyTheCacheSavingWeMeasure(t *testing.T) { t.Fatal(err) } src := string(js) - if !strings.Contains(src, "o.cachesplit_saved_usd") { - t.Error("the overview does not render cachesplit_saved_usd, the one cache saving we claim") + if !strings.Contains(src, "o.keepalive_net_usd") { + t.Error("the overview does not render keepalive_net_usd, the cache saving we claim") } if strings.Contains(src, "cache_saved_protected_usd") { t.Error("cache_saved_protected_usd is back on the page; it measured co-occurrence, not cause") @@ -723,12 +546,17 @@ func TestUIClaimsOnlyTheCacheSavingWeMeasure(t *testing.T) { // The provider figure may appear ONCE in the per-request drawer and ONCE in the A/B // comparison (where it is the control variable, not the credit) — never in a tile. for _, forbidden := range []string{ - "tile('cache-saved'", "'Prompt-cache savings'", "o.cache_saved_usd", + "tile('cache-saved'", "'Prompt-cache savings'", } { if strings.Contains(src, forbidden) { t.Errorf("the provider's cache saving is presented as ours again: %q", forbidden) } } + // renderSavedLedger reads the provider figure exactly once, to recover the input price for the + // keep-alive token estimate. It is never displayed; a second use would need a second look. + if n := strings.Count(src, "o.cache_saved_usd"); n != 1 { + t.Errorf("o.cache_saved_usd appears %d times in app.js, want the single read in renderSavedLedger", n) + } } // TestSettingsFormSpeaksTheSameFieldNamesAsTheServer is the UI half of the descriptor @@ -998,7 +826,6 @@ func TestUINeverShowsABareCost(t *testing.T) { {ComponentRow{}, "net_usd", "verdict() would have to improvise a net client-side"}, {SessionRow{}, "in_flight", "a young session's incomplete amortization would read as a verdict"}, {SessionRow{}, "tenant_id", "a manager's service-wide list would be unattributable"}, - {Overview{}, "prefix_change_cost_usd", "the largest figure on the corpus would be invisible"}, // The value pass. Every one of these is the missing half of a pair that already // shipped: a component dollar column populated on 6 rows of 100,579, a saving // divided by a bill that is mostly output tokens, a freeze shown as a cost with no @@ -1007,10 +834,6 @@ func TestUINeverShowsABareCost(t *testing.T) { {ComponentRow{}, "saved_usd_estimated", "the components tab would show $0.00 for all history predating the column"}, {ComponentRow{}, "net_usd_with_estimate", "the verdict would judge a component's whole spend against six rows of its saving"}, {ComponentRow{}, "acted_structural", "a component whose job is cache placement would read 0% act rate forever"}, - {Overview{}, "replay_projected_tokens", "the headroom the cache freeze forgoes would have no number"}, - {Overview{}, "prefix_change_cost_all_usd", "the whole exposure of the failure mode would stay below the fold"}, - {Overview{}, "split_credited_moved", "credit paid on a forgotten session would be indistinguishable from a moved snapshot"}, - {SafetyCost{}, "frozen_write_risk_usd", "the freeze would keep being shown as a cost with no counterpart"}, {TierCosts{}, "addressable_usd", "savings would keep being divided by output tokens nothing can touch"}, } { rt := reflect.TypeOf(c.v) @@ -1040,7 +863,7 @@ func TestUINeverShowsABareCost(t *testing.T) { t.Error("verdict() does not read c.net_usd") } // The Saved $ / Net $ columns are static markup, so their absence is a markup fact. - for _, th := range []string{`Saved $`, `Net $`} { + for _, th := range []string{`>Saved`, `>Net`} { if !strings.Contains(string(html), th) { t.Errorf("the components table has no %s column header", th) } diff --git a/dash/assetversion_test.go b/dash/assetversion_test.go index 64fd89fee..883296526 100644 --- a/dash/assetversion_test.go +++ b/dash/assetversion_test.go @@ -43,21 +43,19 @@ func TestServedUIVersionsEveryAssetItReferences(t *testing.T) { // Both entry points must work and must be versioned: the plain directory URL and // the explicit filename. Both serve the same markup, so `checked` — the number of // local references found in it — is reused by the rewrite-count assertion below. - // Two documents: the five-page home (the default) and the classic dashboard. "/dashboard/" is - // the same markup as home.html, so `checked` counts each document's references once. checked := 0 - for _, entry := range []string{"/dashboard/", "/dashboard/home.html", "/dashboard/index.html"} { + for _, entry := range []string{"/dashboard/", "/dashboard/index.html"} { w := get(entry) if cc := w.Header().Get("Cache-Control"); cc != "no-cache" { t.Errorf("%s Cache-Control = %q; the HTML must not be cached, or it cannot hand out new asset URLs", entry, cc) } - docRefs := 0 + checked = 0 for _, mm := range localRef.FindAllStringSubmatch(w.Body.String(), -1) { ref := mm[1] if strings.HasPrefix(ref, "data:") || strings.HasPrefix(ref, "#") || strings.Contains(ref, "://") { continue } - docRefs++ + checked++ if !strings.Contains(ref, "?v="+assetVersion) { t.Errorf("%s references %q with no version token — a stale cached copy of it will be paired with this HTML", entry, ref) continue @@ -70,11 +68,8 @@ func TestServedUIVersionsEveryAssetItReferences(t *testing.T) { // Fail loudly if the reference extraction stops finding anything: a reshaped // index.html that no longer matches must break this test, not quietly ship // unversioned URLs. - if docRefs < 4 { - t.Errorf("%s: found only %d local asset references; the pages reference at least four assets each", entry, docRefs) - } - if entry != "/dashboard/" { - checked += docRefs + if checked < 4 { + t.Errorf("%s: found only %d local asset references; index.html references at least style.css, app.js, tools.js and kvcache.js", entry, checked) } } @@ -89,12 +84,14 @@ func TestServedUIVersionsEveryAssetItReferences(t *testing.T) { // campaigns.js is manager-only and not - - - diff --git a/dash/ui/index.html b/dash/ui/index.html index 3e55cd7f9..448b5e869 100644 --- a/dash/ui/index.html +++ b/dash/ui/index.html @@ -60,14 +60,13 @@