diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 84cae86631..b6a456ea1f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -159,6 +159,10 @@ jobs: - name: Test run: make test + - name: Summarize tests + if: always() && hashFiles('tmp/go-test.json') != '' + run: go run ./hack/cmd/test-summary tmp/go-test.json >> "$GITHUB_STEP_SUMMARY" + - name: Upload coverage if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 diff --git a/AGENTS.md b/AGENTS.md index 58d87f814f..58a1371da7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -59,6 +59,9 @@ unbounded-kube is organized into several directories: - `make build` compiles all Go packages (`go build ./...`). - `make vulncheck` runs `govulncheck` and fails only on vulnerabilities that are both reachable from our code and have a final-release fix, since those are the ones a production module bump resolves. Reachable ones with no final fix are reported and allowed through; prerelease and Go pseudo-version fixes are also reported but do not block. - `make fmt` formats Go source (gofumpt + wsl_v5 blank-line rules); `make lint` runs golangci-lint; `make test` runs all tests. +- `make test` runs through gotestsum: the output matches `go test -v`, and it ends with every skipped and failed test + and its reason. The JSON report is written to `tmp/go-test.json`, and CI turns it into the job summary. Other test + targets use `$(GOTEST)`, which also passes `-v`. - `make lint` runs the same checks locally and in CI and does NOT auto-fix. Always run `make fmt` before committing to satisfy the linter (gofumpt and wsl_v5 are enforced by `make lint`/CI); do not hand-format. - Locally `test` implies `lint`. In CI (`CI=1`), each runs independently. diff --git a/Makefile b/Makefile index 10baa65208..eedc4442c5 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,12 @@ GOCMD=go GOFMT=gofumpt GOBUILD=$(GOCMD) build -GOTEST=$(GOCMD) test +GOTEST=$(GOCMD) test -v +# gotestsum runs `go test -json`, prints the same output as `go test -v`, and +# ends with a summary of every skipped and failed test with its output. The +# JSON report feeds the CI job summary (hack/cmd/test-summary). +GO_TEST_REPORT ?= $(CURDIR)/tmp/go-test.json +GOTESTSUM=$(GOCMD) tool gotestsum --format standard-verbose --jsonfile $(GO_TEST_REPORT) GOMOD=$(GOCMD) mod GOLINT=golangci-lint run -c .golangci.yaml ENVTEST_K8S_VERSION ?= 1.37.0 @@ -569,13 +574,15 @@ ifdef CI # In CI each job is independent; skip chained prerequisites. test: machina-manifests token-refresher-manifests machine-ops-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Run all tests with race detector - $(GOTEST) -race ./... + @mkdir -p $(dir $(GO_TEST_REPORT)) + $(GOTESTSUM) -- -race ./... else # Locally, chain test -> lint for convenience. test: lint machina-manifests token-refresher-manifests machine-ops-manifests playpen-manifests net-manifests unbounded-operator-manifests gantry-manifests ## Run all tests (implies lint) - $(GOTEST) ./... + @mkdir -p $(dir $(GO_TEST_REPORT)) + $(GOTESTSUM) -- ./... endif diff --git a/cmd/kubectl-unbounded/app/overrides_test.go b/cmd/kubectl-unbounded/app/overrides_test.go index 75d8cfece6..25752d2a6c 100644 --- a/cmd/kubectl-unbounded/app/overrides_test.go +++ b/cmd/kubectl-unbounded/app/overrides_test.go @@ -485,12 +485,14 @@ data: // TestOverridesValidateShippedExample checks the exact file the repository // ships, since its own comments instruct the reader to run this command on it. +// The file is tracked, so a missing file means it moved and this path must +// follow. func TestOverridesValidateShippedExample(t *testing.T) { path := filepath.Join("..", "..", "..", "deploy", "unbounded-operator", "examples", "component-overrides.example.yaml") if _, err := os.Stat(path); err != nil { - t.Skipf("example not present: %v", err) + t.Fatalf("shipped example not found: %v", err) } var out bytes.Buffer diff --git a/go.mod b/go.mod index 8a8f8690cc..bf42924583 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.27.4 tool ( golang.org/x/vuln/cmd/govulncheck + gotest.tools/gotestsum sigs.k8s.io/controller-tools/cmd/controller-gen ) @@ -147,6 +148,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bitfield/gotestdox v0.2.2 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect @@ -165,6 +167,7 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 // indirect + github.com/dnephin/pflag v1.0.7 // indirect github.com/docker/go-connections v0.7.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dunglas/httpsfv v1.1.1 // indirect @@ -201,6 +204,7 @@ require ( github.com/google/go-cmp v0.7.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gopacket v1.1.19 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect github.com/guillaumemichel/reservedpool v0.3.0 // indirect @@ -356,6 +360,7 @@ require ( gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gotest.tools/gotestsum v1.13.0 // indirect k8s.io/code-generator v0.37.2 // indirect k8s.io/component-base v0.37.2 // indirect k8s.io/gengo/v2 v2.0.0-20260408192533-25e2208e0dc3 // indirect diff --git a/go.sum b/go.sum index 6f4e84db64..3ed6f2829c 100644 --- a/go.sum +++ b/go.sum @@ -138,6 +138,8 @@ github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bitfield/gotestdox v0.2.2 h1:x6RcPAbBbErKLnapz1QeAlf3ospg8efBsedU93CDsnE= +github.com/bitfield/gotestdox v0.2.2/go.mod h1:D+gwtS0urjBrzguAkTM2wodsTQYFHdpx8eqRJ3N+9pY= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bougou/go-ipmi v0.8.3 h1:jegrU+5lx5Moe/MiggLFIHCwDoVPd/BZuIOKYZS3BUY= @@ -228,6 +230,8 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3h github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk= +github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE= github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -377,6 +381,8 @@ github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/renameio/v2 v2.0.2 h1:qKZs+tfn+arruZZhQ7TKC/ergJunuJicWS6gLDt/dGw= github.com/google/renameio/v2 v2.0.2/go.mod h1:OX+G6WHHpHq3NVj7cAOleLOwJfcQ1s3uUJQCrr78SWo= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -950,6 +956,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/gotestsum v1.13.0 h1:+Lh454O9mu9AMG1APV4o0y7oDYKyik/3kBOiCqiEpRo= +gotest.tools/gotestsum v1.13.0/go.mod h1:7f0NS5hFb0dWr4NtcsAsF0y1kzjEFfAil0HiBQJE03Q= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= k8s.io/api v0.37.2 h1:PnxqnUFy93jr2GcpWDkKOm8wfy25kT9wa572MuQS8oc= diff --git a/hack/cmd/forge/forge/site/inventory_machina_test.go b/hack/cmd/forge/forge/site/inventory_machina_test.go index c883a881c6..e7127f413c 100644 --- a/hack/cmd/forge/forge/site/inventory_machina_test.go +++ b/hack/cmd/forge/forge/site/inventory_machina_test.go @@ -51,8 +51,6 @@ func TestMachineHost(t *testing.T) { } func TestWriteInventoryAsMachina(t *testing.T) { - t.Skip() - t.Parallel() inv := &azuredev.Inventory{ diff --git a/hack/cmd/test-summary/main.go b/hack/cmd/test-summary/main.go new file mode 100644 index 0000000000..0599bbf9bf --- /dev/null +++ b/hack/cmd/test-summary/main.go @@ -0,0 +1,289 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +// test-summary turns a `go test -json` event stream into a Markdown summary for +// the GitHub Actions job page: the counts, every skipped test with its reason, +// and every failure. +// +// Tests run verbosely, but a skipped test is still one line among tens of +// thousands in the log. A test that skips on every run, because a variable is +// never set or a tool is never installed, passes review and CI unnoticed +// indefinitely. The job summary puts the list where people look. +// +// It reports and never judges: a failed test is the test run's verdict to +// give, so this exits zero whatever the results, and non-zero only when the +// stream cannot be read or holds no test results at all. +// +// Usage: +// +// test-summary [go-test-json-file] >> "$GITHUB_STEP_SUMMARY" +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "sort" + "strings" +) + +// modulePrefix is trimmed from package paths to keep the tables readable. +const modulePrefix = "github.com/Azure/unbounded/" + +// maxReason bounds a skip reason, so one chatty test cannot swamp the table. +const maxReason = 300 + +// event is one line of `go test -json` output. See `go doc test2json`. +type event struct { + Action string + Package string + Test string + Output string + // OutputType is "frame" for the lines go test writes itself, such as + // "=== RUN" and "--- SKIP". Older toolchains leave it unset. + OutputType string +} + +// testID names one test, subtests included. +type testID struct { + pkg string + test string +} + +// outcome is a test that skipped or failed. +type outcome struct { + testID + + // reason is what a skipped test logged, framing lines removed. + reason string +} + +// results is everything the summary reports. +type results struct { + passed int + skips []outcome + fails []outcome + // brokenPackages failed with no failing test of their own: a build error, + // a panic outside a test, or a TestMain that refused to run. + brokenPackages []string +} + +var errNoResults = errors.New("no test results found: the tests did not run, or the report was truncated") + +func main() { + if err := run(os.Args[1:], os.Stdin, os.Stdout); err != nil { + fmt.Fprintf(os.Stderr, "test-summary: %v\n", err) + os.Exit(1) + } +} + +func run(args []string, stdin io.Reader, stdout io.Writer) error { + var input io.Reader + + switch len(args) { + case 0: + input = stdin + case 1: + file, err := os.Open(args[0]) + if err != nil { + return fmt.Errorf("open test report: %w", err) + } + + defer file.Close() //nolint:errcheck // read-only + + input = file + default: + return errors.New("usage: test-summary [go-test-json-file]") + } + + res, err := parse(input) + if err != nil { + return err + } + + _, err = io.WriteString(stdout, res.markdown()) + + return err +} + +// parse reads the event stream. Output is held only until its test finishes, +// and kept only for tests that skipped. +func parse(r io.Reader) (*results, error) { + decoder := json.NewDecoder(r) + output := map[testID][]string{} + failedTests := map[string]bool{} + failedPackages := map[string]bool{} + res := &results{} + + for { + var ev event + + err := decoder.Decode(&ev) + if errors.Is(err, io.EOF) { + break + } + + if err != nil { + return nil, fmt.Errorf("parse test report: %w", err) + } + + if ev.Test == "" { + if ev.Action == "fail" && ev.Package != "" { + failedPackages[ev.Package] = true + } + + continue + } + + id := testID{pkg: ev.Package, test: ev.Test} + + switch ev.Action { + case "output": + if !isFrame(ev) { + output[id] = append(output[id], ev.Output) + } + case "pass": + res.passed++ + case "skip": + res.skips = append(res.skips, outcome{testID: id, reason: reason(output[id])}) + case "fail": + res.fails = append(res.fails, outcome{testID: id}) + failedTests[ev.Package] = true + } + + switch ev.Action { + case "pass", "skip", "fail": + delete(output, id) + } + } + + for pkg := range failedPackages { + if !failedTests[pkg] { + res.brokenPackages = append(res.brokenPackages, pkg) + } + } + + if res.passed+len(res.skips)+len(res.fails)+len(res.brokenPackages) == 0 { + return nil, errNoResults + } + + sortOutcomes(res.skips) + sortOutcomes(res.fails) + sort.Strings(res.brokenPackages) + + return res, nil +} + +// isFrame reports whether ev is a line go test writes around a test, rather +// than something the test logged. +func isFrame(ev event) bool { + if ev.OutputType == "frame" { + return true + } + + line := strings.TrimSpace(ev.Output) + + for _, prefix := range []string{"=== ", "--- PASS", "--- SKIP", "--- FAIL"} { + if strings.HasPrefix(line, prefix) { + return true + } + } + + return false +} + +// reason joins what a test logged into one line. +func reason(lines []string) string { + var parts []string + + for _, line := range lines { + if trimmed := strings.TrimSpace(line); trimmed != "" { + parts = append(parts, trimmed) + } + } + + joined := strings.Join(parts, " ") + if runes := []rune(joined); len(runes) > maxReason { + joined = string(runes[:maxReason]) + "..." + } + + return joined +} + +func sortOutcomes(outcomes []outcome) { + sort.Slice(outcomes, func(i, j int) bool { + if outcomes[i].pkg != outcomes[j].pkg { + return outcomes[i].pkg < outcomes[j].pkg + } + + return outcomes[i].test < outcomes[j].test + }) +} + +func (r *results) markdown() string { + var b strings.Builder + + b.WriteString("## Go tests\n\n") + fmt.Fprintf(&b, "%d passed, %d skipped, %d failed", r.passed, len(r.skips), len(r.fails)) + + if n := len(r.brokenPackages); n > 0 { + fmt.Fprintf(&b, ", %d %s failed outside any test", n, pluralize(n, "package", "packages")) + } + + b.WriteString(".\n") + + if len(r.fails) > 0 || len(r.brokenPackages) > 0 { + fmt.Fprintf(&b, "\n### Failed (%d)\n\nDetails are in the Test step's log.\n\n", len(r.fails)+len(r.brokenPackages)) + + for _, pkg := range r.brokenPackages { + fmt.Fprintf(&b, "- `%s` (package)\n", shortPackage(pkg)) + } + + for _, fail := range r.fails { + fmt.Fprintf(&b, "- `%s` `%s`\n", shortPackage(fail.pkg), fail.test) + } + } + + if len(r.skips) == 0 { + b.WriteString("\nNo tests were skipped.\n") + + return b.String() + } + + fmt.Fprintf(&b, "\n### Skipped (%d)\n\n", len(r.skips)) + b.WriteString("| Package | Test | Reason |\n| --- | --- | --- |\n") + + for _, skip := range r.skips { + why := skip.reason + if why == "" { + why = "(no reason given)" + } + + fmt.Fprintf(&b, "| `%s` | `%s` | %s |\n", shortPackage(skip.pkg), escapeCell(skip.test), escapeCell(why)) + } + + return b.String() +} + +func shortPackage(pkg string) string { + if short := strings.TrimPrefix(pkg, modulePrefix); short != "" { + return short + } + + return pkg +} + +// escapeCell keeps a reason inside its Markdown table cell. +func escapeCell(s string) string { + return strings.ReplaceAll(s, "|", `\|`) +} + +func pluralize(n int, one, many string) string { + if n == 1 { + return one + } + + return many +} diff --git a/hack/cmd/test-summary/main_test.go b/hack/cmd/test-summary/main_test.go new file mode 100644 index 0000000000..42245d9998 --- /dev/null +++ b/hack/cmd/test-summary/main_test.go @@ -0,0 +1,311 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +const pkg = modulePrefix + "internal/widget" + +// stream encodes events the way `go test -json` writes them, one per line. +func stream(t *testing.T, events ...event) string { + t.Helper() + + var b strings.Builder + + for _, ev := range events { + line, err := json.Marshal(ev) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + b.Write(line) + b.WriteByte('\n') + } + + return b.String() +} + +func out(test, text string) event { + return event{Action: "output", Package: pkg, Test: test, Output: text} +} + +func frame(test, text string) event { + return event{Action: "output", Package: pkg, Test: test, Output: text, OutputType: "frame"} +} + +func done(action, test string) event { + return event{Action: action, Package: pkg, Test: test} +} + +func summarize(t *testing.T, input string) string { + t.Helper() + + var stdout bytes.Buffer + if err := run(nil, strings.NewReader(input), &stdout); err != nil { + t.Fatalf("run: %v", err) + } + + return stdout.String() +} + +// realSkip is a skipped test exactly as go1.27 `go test -json` reported it. +const realSkip = `{"Action":"run","Package":"github.com/Azure/unbounded/internal/net/html","Test":"TestClusterStatusFS"} +{"Action":"output","Package":"github.com/Azure/unbounded/internal/net/html","Test":"TestClusterStatusFS","Output":"=== RUN TestClusterStatusFS\n","OutputType":"frame"} +{"Action":"output","Package":"github.com/Azure/unbounded/internal/net/html","Test":"TestClusterStatusFS","Output":" pages_test.go:21: frontend not built; run ` + "`make net-frontend`" + ` to enable this test\n"} +{"Action":"output","Package":"github.com/Azure/unbounded/internal/net/html","Test":"TestClusterStatusFS","Output":"--- SKIP: TestClusterStatusFS (0.00s)\n","OutputType":"frame"} +{"Action":"skip","Package":"github.com/Azure/unbounded/internal/net/html","Test":"TestClusterStatusFS","Elapsed":0} +{"Action":"output","Package":"github.com/Azure/unbounded/internal/net/html","Output":"PASS\n"} +{"Action":"pass","Package":"github.com/Azure/unbounded/internal/net/html","Elapsed":0.01} +` + +func TestSummaryListsARealSkipWithItsReason(t *testing.T) { + t.Parallel() + + got := summarize(t, realSkip) + + for _, want := range []string{ + "0 passed, 1 skipped, 0 failed.", + "### Skipped (1)", + "| `internal/net/html` | `TestClusterStatusFS` | pages_test.go:21: frontend not built; run `make net-frontend` to enable this test |", + } { + if !strings.Contains(got, want) { + t.Errorf("summary does not contain %q:\n%s", want, got) + } + } + + if strings.Contains(got, "=== RUN") || strings.Contains(got, "--- SKIP") { + t.Errorf("summary includes framing lines:\n%s", got) + } +} + +func TestSummaryCountsAndSortsResults(t *testing.T) { + t.Parallel() + + const other = modulePrefix + "cmd/alpha" + + got := summarize(t, stream(t, + done("pass", "TestA"), + done("pass", "TestB"), + done("pass", "TestB/sub"), + out("TestZ", " z_test.go:9: needs root\n"), + done("skip", "TestZ"), + out("TestC/sub", " c_test.go:4: no fixture\n"), + done("skip", "TestC/sub"), + done("pass", "TestC"), + event{Action: "output", Package: other, Test: "TestY", Output: " y_test.go:1: set Y_DIR\n"}, + event{Action: "skip", Package: other, Test: "TestY"}, + )) + + if !strings.Contains(got, "4 passed, 3 skipped, 0 failed.") { + t.Errorf("wrong counts:\n%s", got) + } + + alpha := strings.Index(got, "`cmd/alpha` | `TestY`") + sub := strings.Index(got, "`internal/widget` | `TestC/sub`") + z := strings.Index(got, "`internal/widget` | `TestZ`") + + if alpha < 0 || sub < 0 || z < 0 || alpha >= sub || sub >= z { + t.Errorf("skips are missing or not sorted by package, then test:\n%s", got) + } + + if strings.Contains(got, "### Failed") { + t.Errorf("summary has a Failed section with no failures:\n%s", got) + } +} + +// TestSummaryAttributesInterleavedOutput covers parallel tests, whose output +// arrives interleaved. Each line belongs to the test the event names. +func TestSummaryAttributesInterleavedOutput(t *testing.T) { + t.Parallel() + + got := summarize(t, stream(t, + frame("TestOne", "=== CONT TestOne\n"), + out("TestOne", " one_test.go:1: reason one\n"), + frame("TestTwo", "=== CONT TestTwo\n"), + out("TestTwo", " two_test.go:1: reason two\n"), + done("skip", "TestTwo"), + done("skip", "TestOne"), + )) + + for _, want := range []string{ + "| `TestOne` | one_test.go:1: reason one |", + "| `TestTwo` | two_test.go:1: reason two |", + } { + if !strings.Contains(got, want) { + t.Errorf("summary does not contain %q:\n%s", want, got) + } + } +} + +// TestSummaryDropsFramesFromOlderToolchains covers streams with no OutputType, +// where framing lines are recognized by their prefix. +func TestSummaryDropsFramesFromOlderToolchains(t *testing.T) { + t.Parallel() + + got := summarize(t, stream(t, + out("TestOld", "=== RUN TestOld\n"), + out("TestOld", " old_test.go:3: first line\n"), + out("TestOld", " second line\n"), + out("TestOld", "--- SKIP: TestOld (0.00s)\n"), + done("skip", "TestOld"), + )) + + if !strings.Contains(got, "| `TestOld` | old_test.go:3: first line second line |") { + t.Errorf("framing lines were not dropped, or lines were not joined:\n%s", got) + } +} + +func TestSummaryOutputDoesNotLeakBetweenRuns(t *testing.T) { + t.Parallel() + + // The same test passes in one package run and skips in a later one, as + // with -count=2. Only the second run's output is the skip's reason. + got := summarize(t, stream(t, + out("TestFlip", " flip_test.go:1: passing chatter\n"), + done("pass", "TestFlip"), + out("TestFlip", " flip_test.go:2: skipping now\n"), + done("skip", "TestFlip"), + )) + + if !strings.Contains(got, "| `TestFlip` | flip_test.go:2: skipping now |") { + t.Errorf("reason includes output from an earlier run:\n%s", got) + } +} + +func TestSummaryKeepsTheTableIntact(t *testing.T) { + t.Parallel() + + long := strings.Repeat("x", maxReason+50) + + got := summarize(t, stream(t, + out("TestPipe", " p_test.go:1: a|b\n"), + done("skip", "TestPipe"), + out("TestLong", " "+long+"\n"), + done("skip", "TestLong"), + done("skip", "TestSilent"), + done("skip", "TestTable/a|b"), + )) + + for _, want := range []string{ + `| p_test.go:1: a\|b |`, + "| " + strings.Repeat("x", maxReason) + "... |", + "| `TestSilent` | (no reason given) |", + "| `TestTable/a\\|b` |", + } { + if !strings.Contains(got, want) { + t.Errorf("summary does not contain %q:\n%s", want, got) + } + } +} + +func TestSummaryListsFailures(t *testing.T) { + t.Parallel() + + const broken = modulePrefix + "internal/broken" + + got := summarize(t, stream(t, + done("pass", "TestOK"), + out("TestBad", " bad_test.go:7: got 1, want 2\n"), + done("fail", "TestBad"), + event{Action: "fail", Package: pkg}, + event{Action: "output", Package: broken, Output: "rendered manifests are missing\n"}, + event{Action: "fail", Package: broken}, + )) + + for _, want := range []string{ + "1 passed, 0 skipped, 1 failed, 1 package failed outside any test.", + "### Failed (2)", + "- `internal/broken` (package)", + "- `internal/widget` `TestBad`", + "No tests were skipped.", + } { + if !strings.Contains(got, want) { + t.Errorf("summary does not contain %q:\n%s", want, got) + } + } + + // internal/widget failed because TestBad did, so it is not also reported + // as failing outside a test. + if strings.Contains(got, "`internal/widget` (package)") { + t.Errorf("a package with a failing test is reported as broken:\n%s", got) + } +} + +func TestSummaryShortensOnlyThisModulesPackages(t *testing.T) { + t.Parallel() + + got := summarize(t, stream(t, + event{Action: "skip", Package: "example.com/elsewhere", Test: "TestX"}, + )) + + if !strings.Contains(got, "| `example.com/elsewhere` | `TestX` |") { + t.Errorf("a foreign package path was altered:\n%s", got) + } +} + +func TestSummaryRejectsAReportWithNoResults(t *testing.T) { + t.Parallel() + + for name, input := range map[string]string{ + "empty": "", + "build started": `{"Action":"start","Package":"` + pkg + `"}` + "\n", + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + err := run(nil, strings.NewReader(input), &bytes.Buffer{}) + if !errors.Is(err, errNoResults) { + t.Errorf("err = %v, want errNoResults", err) + } + }) + } +} + +func TestSummaryRejectsMalformedInput(t *testing.T) { + t.Parallel() + + err := run(nil, strings.NewReader("PASS\nok \tpkg\t0.1s\n"), &bytes.Buffer{}) + if err == nil || !strings.Contains(err.Error(), "parse test report") { + t.Errorf("err = %v, want a parse error", err) + } +} + +func TestRunReadsAFile(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "go-test.json") + if err := os.WriteFile(path, []byte(realSkip), 0o600); err != nil { + t.Fatal(err) + } + + var stdout bytes.Buffer + if err := run([]string{path}, strings.NewReader(""), &stdout); err != nil { + t.Fatalf("run: %v", err) + } + + if !strings.Contains(stdout.String(), "TestClusterStatusFS") { + t.Errorf("summary does not come from the file:\n%s", stdout.String()) + } +} + +func TestRunRejectsBadArguments(t *testing.T) { + t.Parallel() + + if err := run([]string{filepath.Join(t.TempDir(), "absent.json")}, nil, &bytes.Buffer{}); err == nil { + t.Error("run with a missing file: want an error") + } + + err := run([]string{"a.json", "b.json"}, nil, &bytes.Buffer{}) + if err == nil || !strings.Contains(err.Error(), "usage") { + t.Errorf("err = %v, want a usage error", err) + } +} diff --git a/internal/gantry/tracing/tracing_test.go b/internal/gantry/tracing/tracing_test.go index 855e52951f..b5f31bf414 100644 --- a/internal/gantry/tracing/tracing_test.go +++ b/internal/gantry/tracing/tracing_test.go @@ -318,14 +318,15 @@ func TestResourceMergeKeepsServiceName(t *testing.T) { } // TestResourceMergeRejectsMismatchedSchema documents WHY the override above is -// schemaless. If this ever stops failing, the SDK changed its default schema -// handling and the schemaless workaround can be revisited. +// schemaless. It fails once the merge stops failing: the SDK changed its +// default schema handling, and the schemaless workaround can be revisited. A +// skip would hide that change in a passing run. func TestResourceMergeRejectsMismatchedSchema(t *testing.T) { _, err := sdkresource.Merge( sdkresource.Default(), sdkresource.NewWithAttributes(semconv.SchemaURL, semconv.ServiceName(defaultServiceName)), ) if err == nil { - t.Skip("SDK default resource no longer conflicts with the v1.26.0 schema; revisit the schemaless override") + t.Fatal("SDK default resource no longer conflicts with the v1.26.0 schema; revisit the schemaless override") } } diff --git a/internal/net/netlink/netlink_test.go b/internal/net/netlink/netlink_test.go index 35d7c6ace9..5ff09a8b81 100644 --- a/internal/net/netlink/netlink_test.go +++ b/internal/net/netlink/netlink_test.go @@ -255,17 +255,6 @@ func TestProcessNetworkNamespacePaths(t *testing.T) { }) } -// TestLinkManager_Integration tests link manager integration. -func TestLinkManager_Integration(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test in short mode") - } - - // This would test actual interface creation/deletion - // Skip unless running as root - t.Skip("integration test requires root privileges") -} - // TestWireGuardManager_prefixToIPNet tests wire guard manager prefix to ipnet. func TestWireGuardManager_prefixToIPNet(t *testing.T) { tests := []struct {