diff --git a/README.md b/README.md index f81a7eb..72c427b 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,9 @@ go run ./ \ For `pull_request` events, checkout must use the PR head SHA, for example `github.event.pull_request.head.sha`. The default synthetic merge ref is rejected because the checked-out `HEAD` must match `pull_request.head.sha`. -The matrix JSON contains `include` rows with `package`, `run_regex`, and `test_count`. By default, each row represents one safe Go package pattern and a precise regex for the directly changed runnable tests in that package. The generator does not emit whole-package fallback rows. +The matrix JSON contains `selected_test_count` plus `include` rows with `package`, `run_regex`, and `test_count`. By default, each row represents one safe Go package pattern and a precise regex for the directly changed runnable tests in that package. The generator does not emit whole-package fallback rows. + +`--max-selected-tests` limits the size of the execution plan. When the exact package-aware selection exceeds the limit, `whichtests` preserves the selection in the Markdown summary but emits an empty `include` array so workflow consumers can skip execution. A limit of zero disables this behavior. ## Coalescing into a single matrix row diff --git a/cli.go b/cli.go index 62a6297..1f5f316 100644 --- a/cli.go +++ b/cli.go @@ -21,6 +21,7 @@ func main() { flags.StringVar(&cfg.OutSummary, "out-summary", cfg.OutSummary, "path to write Markdown summary, or - for stdout") flags.BoolVar(&cfg.GitHubActions, "github-actions", cfg.GitHubActions, "read diff range and output paths from GitHub Actions environment") flags.BoolVar(&cfg.Coalesce, "coalesce", cfg.Coalesce, "emit a single matrix row whose package list and run-regex union every selected package and test") + flags.IntVar(&cfg.MaxSelectedTests, "max-selected-tests", cfg.MaxSelectedTests, "emit an empty execution matrix when more than this many tests are selected; zero disables the limit") if err := flags.Parse(os.Args[1:]); err != nil { _, _ = fmt.Fprintln(os.Stderr, err) os.Exit(2) @@ -32,6 +33,9 @@ func main() { } func runCommand(ctx context.Context, cfg commandConfig, stdout, stderr io.Writer, git gitRunner, fetch gitFetcher) error { + if cfg.MaxSelectedTests < 0 { + return errors.New("--max-selected-tests must not be negative") + } var ( req runRequest err error @@ -71,7 +75,10 @@ func explicitRunRequest(cfg config) (runRequest, error) { OutMatrix: cfg.OutMatrix, OutSummary: cfg.OutSummary, }, - Plan: planOptions{Coalesce: cfg.Coalesce}, + Plan: planOptions{ + Coalesce: cfg.Coalesce, + MaxSelectedTests: cfg.MaxSelectedTests, + }, }, nil } @@ -80,10 +87,11 @@ func executeRunRequest(ctx context.Context, req runRequest, stdout, stderr io.Wr return err } selectorCfg := config{ - RepoRoot: req.RepoRoot, - BaseSHA: req.Range.BaseSHA, - HeadSHA: req.Range.HeadSHA, - Coalesce: req.Plan.Coalesce, + RepoRoot: req.RepoRoot, + BaseSHA: req.Range.BaseSHA, + HeadSHA: req.Range.HeadSHA, + Coalesce: req.Plan.Coalesce, + MaxSelectedTests: req.Plan.MaxSelectedTests, } changedFiles, result, err := selectTestPlan(ctx, selectorCfg, git) if err != nil { diff --git a/cli_test.go b/cli_test.go index b139f86..87d33e0 100644 --- a/cli_test.go +++ b/cli_test.go @@ -21,7 +21,10 @@ func TestRunValidationErrors(t *testing.T) { return gitResult{}, errors.New("git should not be called") } - err := runCommand(t.Context(), commandConfig{config: config{OutMatrix: "matrix.json"}}, &stdout, &stderr, neverGit, nil) + err := runCommand(t.Context(), commandConfig{config: config{MaxSelectedTests: -1, OutMatrix: "matrix.json"}}, &stdout, &stderr, neverGit, nil) + require.EqualError(t, err, "--max-selected-tests must not be negative") + + err = runCommand(t.Context(), commandConfig{config: config{OutMatrix: "matrix.json"}}, &stdout, &stderr, neverGit, nil) require.EqualError(t, err, "--base-sha is required") err = runCommand(t.Context(), commandConfig{config: config{BaseSHA: "base"}}, &stdout, &stderr, neverGit, nil) @@ -108,6 +111,7 @@ func TestShared(t *testing.T) { matrixData, err := os.ReadFile(matrixPath) require.NoError(t, err) require.NoError(t, json.Unmarshal(matrixData, &matrix)) + require.Equal(t, 2, matrix.SelectedTestCount) require.Len(t, matrix.Include, 2) require.Equal(t, "./pkgone", matrix.Include[0].Package) require.Equal(t, "^(TestShared)(/.*)?$", matrix.Include[0].RunRegex) @@ -194,6 +198,7 @@ func TestSharedTwo(t *testing.T) { matrixData, err := os.ReadFile(matrixPath) require.NoError(t, err) require.NoError(t, json.Unmarshal(matrixData, &matrix)) + require.Equal(t, 2, matrix.SelectedTestCount) require.Len(t, matrix.Include, 1) require.Equal(t, "./pkgone ./pkgtwo", matrix.Include[0].Package) require.Equal(t, "^(TestSharedOne|TestSharedTwo)(/.*)?$", matrix.Include[0].RunRegex) @@ -206,6 +211,80 @@ func TestSharedTwo(t *testing.T) { require.Contains(t, string(summary), "### `./pkgtwo`") } +func TestRunSkipsExecutionAboveSelectedTestLimit(t *testing.T) { + t.Parallel() + + repoRoot := t.TempDir() + baseFiles := map[string]string{ + "pkg/sample_test.go": `package sample + +import "testing" + +func TestAlpha(t *testing.T) { + t.Log("before alpha") +} + +func TestBeta(t *testing.T) { + t.Log("before beta") +} +`, + } + headFiles := map[string]string{ + "pkg/sample_test.go": `package sample + +import "testing" + +func TestAlpha(t *testing.T) { + t.Log("after alpha") +} + +func TestBeta(t *testing.T) { + t.Log("after beta") +} +`, + } + repo := fakeGitRepo{ + changes: []testFileChange{{Kind: changeModified, OldPath: "pkg/sample_test.go", NewPath: "pkg/sample_test.go"}}, + revisions: map[string]map[string]string{"base": baseFiles, "head": headFiles}, + diffOutputs: map[string]string{ + "pkg/sample_test.go": diffForChange( + rangeSpan( + singleLineRange(t, baseFiles["pkg/sample_test.go"], `t.Log("before alpha")`), + singleLineRange(t, baseFiles["pkg/sample_test.go"], `t.Log("before beta")`), + ), + rangeSpan( + singleLineRange(t, headFiles["pkg/sample_test.go"], `t.Log("after alpha")`), + singleLineRange(t, headFiles["pkg/sample_test.go"], `t.Log("after beta")`), + ), + ), + }, + } + + matrixPath := filepath.Join(repoRoot, "matrix.json") + summaryPath := filepath.Join(repoRoot, "summary.md") + var stdout bytes.Buffer + var stderr bytes.Buffer + err := runCommand(t.Context(), commandConfig{config: config{ + RepoRoot: repoRoot, BaseSHA: "base", HeadSHA: "head", + OutMatrix: matrixPath, OutSummary: summaryPath, Coalesce: true, MaxSelectedTests: 1, + }}, &stdout, &stderr, repo.runner(t), nil) + require.NoError(t, err) + + var matrix matrixOutput + matrixData, err := os.ReadFile(matrixPath) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(matrixData, &matrix)) + require.Equal(t, 2, matrix.SelectedTestCount) + require.Empty(t, matrix.Include) + + summary, err := os.ReadFile(summaryPath) + require.NoError(t, err) + require.Contains(t, string(summary), "Selected 2 tests across 1 package targets") + require.Contains(t, string(summary), "Skipping execution because 2 selected tests exceeds the limit of 1.") + require.Contains(t, string(summary), "TestAlpha") + require.Contains(t, string(summary), "TestBeta") +} + func TestRunWritesSummaryToStdout(t *testing.T) { t.Parallel() diff --git a/config.go b/config.go index 0d42e37..faf3455 100644 --- a/config.go +++ b/config.go @@ -12,12 +12,13 @@ const ( ) type config struct { - RepoRoot string - BaseSHA string - HeadSHA string - OutMatrix string - OutSummary string - Coalesce bool + RepoRoot string + BaseSHA string + HeadSHA string + OutMatrix string + OutSummary string + Coalesce bool + MaxSelectedTests int } func defaultConfig() config { diff --git a/githubactions.go b/githubactions.go index 15f9586..619d7ef 100644 --- a/githubactions.go +++ b/githubactions.go @@ -76,7 +76,10 @@ func githubActionsRunRequest(ctx context.Context, cfg commandConfig, git gitRunn GitHubOutput: githubOutput, GitHubStepSummary: stepSummary, }, - Plan: planOptions{Coalesce: baseCfg.Coalesce}, + Plan: planOptions{ + Coalesce: baseCfg.Coalesce, + MaxSelectedTests: baseCfg.MaxSelectedTests, + }, } switch eventName { diff --git a/githubactions_test.go b/githubactions_test.go index 916332c..d7afe06 100644 --- a/githubactions_test.go +++ b/githubactions_test.go @@ -11,6 +11,22 @@ import ( "github.com/stretchr/testify/require" ) +func TestRunCommandGitHubActionsRejectsNegativeSelectedTestLimit(t *testing.T) { + t.Parallel() + + var stdout bytes.Buffer + var stderr bytes.Buffer + neverGit := func(_ context.Context, _ string, _ ...string) (gitResult, error) { + return gitResult{}, errors.New("git should not be called") + } + + err := runCommand(t.Context(), commandConfig{ + config: config{MaxSelectedTests: -1}, + GitHubActions: true, + }, &stdout, &stderr, neverGit, nil) + require.EqualError(t, err, "--max-selected-tests must not be negative") +} + func TestGitHubActionsRunRequestPullRequest(t *testing.T) { eventPath := writeGitHubEvent(t, `{ "pull_request": { @@ -30,7 +46,7 @@ func TestGitHubActionsRunRequestPullRequest(t *testing.T) { t.Setenv("UNRELATED_EXTRA_ENV", "ignored") req, err := githubActionsRunRequest(t.Context(), commandConfig{ - config: config{RepoRoot: "/repo", OutMatrix: "matrix.json"}, + config: config{RepoRoot: "/repo", OutMatrix: "matrix.json", MaxSelectedTests: 100}, }, fakeGitRepo{headSHA: "head123"}.runner(t)) require.NoError(t, err) require.Equal(t, "/repo", req.RepoRoot) @@ -42,6 +58,7 @@ func TestGitHubActionsRunRequestPullRequest(t *testing.T) { require.Equal(t, "matrix.json", req.Sinks.OutMatrix) require.Equal(t, "output.txt", req.Sinks.GitHubOutput) require.Equal(t, "summary.md", req.Sinks.GitHubStepSummary) + require.Equal(t, 100, req.Plan.MaxSelectedTests) } func TestGitHubActionsRunRequestVerifiesPullRequestHead(t *testing.T) { @@ -176,7 +193,7 @@ func TestAlpha(t *testing.T) { matrixData, err := os.ReadFile(matrixPath) require.NoError(t, err) - require.JSONEq(t, `{"include":[{"package":"./pkg","run_regex":"^(TestAlpha)(/.*)?$","test_count":"10"}]}`, string(matrixData)) + require.JSONEq(t, `{"selected_test_count":1,"include":[{"package":"./pkg","run_regex":"^(TestAlpha)(/.*)?$","test_count":"10"}]}`, string(matrixData)) outputData, err := os.ReadFile(outputPath) require.NoError(t, err) require.Equal(t, "matrix="+string(bytes.TrimSpace(matrixData))+"\n", string(outputData)) diff --git a/plan.go b/plan.go index 2f8949b..8840fe1 100644 --- a/plan.go +++ b/plan.go @@ -16,7 +16,8 @@ var ( ) type matrixOutput struct { - Include []matrixEntry `json:"include"` + SelectedTestCount int `json:"selected_test_count"` + Include []matrixEntry `json:"include"` } type matrixEntry struct { @@ -26,7 +27,8 @@ type matrixEntry struct { } type summaryReport struct { - Entries []summaryEntry + Entries []summaryEntry + SkipReason string } type summaryEntry struct { @@ -67,7 +69,10 @@ func selectTestPlan(ctx context.Context, cfg config, git gitRunner) ([]string, b } } - result, err := buildExecutionPlan(selections, planOptions{Coalesce: cfg.Coalesce}) + result, err := buildExecutionPlan(selections, planOptions{ + Coalesce: cfg.Coalesce, + MaxSelectedTests: cfg.MaxSelectedTests, + }) if err != nil { return nil, buildResult{}, err } @@ -114,10 +119,27 @@ func buildExecutionPlan(selections map[packageKey]*packageSelection, opts planOp Notes: entry.Notes, }) } + result.Matrix.SelectedTestCount = selectedTestCount(result.Summary) + if opts.MaxSelectedTests > 0 && result.Matrix.SelectedTestCount > opts.MaxSelectedTests { + result.Summary.SkipReason = fmt.Sprintf( + "Skipping execution because %d selected tests exceeds the limit of %d.", + result.Matrix.SelectedTestCount, + opts.MaxSelectedTests, + ) + return result, nil + } result.Matrix.Include = buildMatrixInclude(accumulators, orderedPackages, opts) return result, nil } +func selectedTestCount(summary summaryReport) int { + count := 0 + for _, entry := range summary.Entries { + count += len(entry.Tests) + } + return count +} + // planOptions configures non-essential plan-shaping behavior. Validation and // selection are governed by config; planOptions controls only the matrix // output shape so that future knobs (e.g. test_count overrides) can be added @@ -129,6 +151,9 @@ type planOptions struct { // (for example, to amplify scheduling contention in a flake hunt) at the // cost of giving up per-package precision in -run. Coalesce bool + // MaxSelectedTests emits an empty execution matrix when the selection is + // larger than this value. Zero disables the limit. + MaxSelectedTests int } // buildMatrixInclude renders the matrix include rows for the given @@ -228,6 +253,9 @@ func renderSummary(changedFiles []string, summary summaryReport) string { totalTests += len(entry.Tests) } _, _ = fmt.Fprintf(&builder, "Selected %d tests across %d package targets.\n\n", totalTests, len(summary.Entries)) + if summary.SkipReason != "" { + _, _ = builder.WriteString(summary.SkipReason + "\n\n") + } for _, entry := range summary.Entries { _, _ = builder.WriteString("### `" + entry.Label + "`\n\n") _, _ = builder.WriteString("Files:\n") diff --git a/plan_test.go b/plan_test.go index 166f235..84fe5b1 100644 --- a/plan_test.go +++ b/plan_test.go @@ -208,3 +208,41 @@ func TestBuildExecutionPlanCoalesceWithNoSelectionsEmitsEmptyMatrix(t *testing.T require.Empty(t, result.Matrix.Include) require.Empty(t, result.Summary.Entries) } + +func TestBuildExecutionPlanSkipsMatrixAboveSelectedTestLimit(t *testing.T) { + t.Parallel() + + selections := map[packageKey]*packageSelection{ + {Dir: "pkgone", Name: "one"}: { + Key: packageKey{Dir: "pkgone", Name: "one"}, + Tests: map[string]struct{}{"TestAlpha": {}, "TestBeta": {}}, + Files: map[string]struct{}{"pkgone/one_test.go": {}}, + }, + {Dir: "pkgtwo", Name: "two"}: { + Key: packageKey{Dir: "pkgtwo", Name: "two"}, + Tests: map[string]struct{}{"TestAlpha": {}}, + Files: map[string]struct{}{"pkgtwo/two_test.go": {}}, + }, + } + result, err := buildExecutionPlan(selections, planOptions{Coalesce: true, MaxSelectedTests: 2}) + require.NoError(t, err) + require.Equal(t, 3, result.Matrix.SelectedTestCount) + require.Empty(t, result.Matrix.Include) + require.Equal(t, "Skipping execution because 3 selected tests exceeds the limit of 2.", result.Summary.SkipReason) + require.Contains(t, renderSummary([]string{"pkgone/one_test.go", "pkgtwo/two_test.go"}, result.Summary), result.Summary.SkipReason) +} + +func TestBuildExecutionPlanRunsAtSelectedTestLimit(t *testing.T) { + t.Parallel() + + selection := &packageSelection{ + Key: packageKey{Dir: "pkg", Name: "sample"}, + Tests: map[string]struct{}{"TestAlpha": {}, "TestBeta": {}}, + Files: map[string]struct{}{"pkg/sample_test.go": {}}, + } + result, err := buildExecutionPlan(map[packageKey]*packageSelection{selection.Key: selection}, planOptions{MaxSelectedTests: 2}) + require.NoError(t, err) + require.Equal(t, 2, result.Matrix.SelectedTestCount) + require.Len(t, result.Matrix.Include, 1) + require.Empty(t, result.Summary.SkipReason) +} diff --git a/publish_test.go b/publish_test.go index 26eb816..af4bf44 100644 --- a/publish_test.go +++ b/publish_test.go @@ -23,12 +23,12 @@ func TestPublishPlanWritesCompactGitHubOutputs(t *testing.T) { OutSummary: summaryPath, GitHubOutput: outputPath, GitHubStepSummary: stepSummaryPath, - }, matrixOutput{Include: []matrixEntry{{Package: "./pkg", RunRegex: "^(TestAlpha)(/.*)?$", TestCount: "10"}}}, summary, nil) + }, matrixOutput{SelectedTestCount: 1, Include: []matrixEntry{{Package: "./pkg", RunRegex: "^(TestAlpha)(/.*)?$", TestCount: "10"}}}, summary, nil) require.NoError(t, err) matrixData, err := os.ReadFile(matrixPath) require.NoError(t, err) - wantMatrix := `{"include":[{"package":"./pkg","run_regex":"^(TestAlpha)(/.*)?$","test_count":"10"}]}` + wantMatrix := `{"selected_test_count":1,"include":[{"package":"./pkg","run_regex":"^(TestAlpha)(/.*)?$","test_count":"10"}]}` require.Equal(t, wantMatrix+"\n", string(matrixData)) outputData, err := os.ReadFile(outputPath) @@ -50,7 +50,7 @@ func TestPublishPlanWritesEmptyMatrixAndRejectsUnsafeOutput(t *testing.T) { matrixData, err := marshalMatrix(matrixOutput{}) require.NoError(t, err) - require.Equal(t, `{"include":[]}`, string(matrixData)) + require.Equal(t, `{"selected_test_count":0,"include":[]}`, string(matrixData)) err = ensureGitHubOutputFits("matrix", "first\nsecond", defaultGitHubOutputValueLimit) require.ErrorContains(t, err, "single line")