Skip to content

Commit a255e26

Browse files
peyton-altclaude
andcommitted
test(agent): enforce the two-error composition contract instead of documenting it
The riskiest part of this PR is the #964 reconciliation (51c8b26), where two error models were fused by hand. The contract that came out of it — one error must satisfy errors.As(*TextGenError) for the label, errors.Is(sentinel) for the ctx branches, and errors.As(*TextGenerationError) for the timeout diagnostic — lived only in comments. Nothing failed if a return dropped its evidence wrapper; timeoutDiagnostic would just quietly fall to its no-information branch ("provider produced no output") with a blank stderr row. All three of the serious bugs in this PR came from that seam, so the durable fix is to make it fail loudly rather than rely on a reviewer noticing. - assertComposition() asserts all three lookups at once, applied to the CLIMissing and AuthFrom401 cases across all four injectable providers. - A new CanceledCarriesSentinelAndEvidence case covers the ctx path, which had zero coverage in the matrix: sentinel reachable, evidence populated, and the captured stderr and partial-stdout byte count both correct. This is the path where classification is meaningless, so evidence is the entire payload. Mutation-verified: dropping withEvidence from the classified-failure return in HandleTextGenResult now fails every provider with "errors.As(*TextGenerationError) failed — evidence lost, timeout diagnostic degrades silently". Before this commit the suite stayed green. Test-only; no production change. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1 parent c1241eb commit a255e26

1 file changed

Lines changed: 59 additions & 16 deletions

File tree

cmd/entire/cli/agent/generate_matrix_test.go

Lines changed: 59 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"runtime"
88
"strings"
99
"testing"
10+
"time"
1011

1112
"github.com/entireio/cli/cmd/entire/cli/agent"
1213
"github.com/entireio/cli/cmd/entire/cli/agent/codex"
@@ -60,6 +61,32 @@ func TestGenerateText_Matrix(t *testing.T) {
6061
}},
6162
}
6263

64+
// assertComposition pins the contract the #964/#1005 reconciliation created:
65+
// one error must satisfy all three lookups. Classification (*TextGenError)
66+
// drives the user-facing label; evidence (*TextGenerationError) drives the
67+
// timeout diagnostic and is the only signal on the ctx path.
68+
//
69+
// Enforced here rather than left to review because dropping withEvidence
70+
// from any single return degrades timeoutDiagnostic to its no-information
71+
// branch, and every other assertion in this file would still pass.
72+
assertComposition := func(t *testing.T, err error, wantKind agent.TextGenErrorKind, wantProvider types.AgentName) {
73+
t.Helper()
74+
var tge *agent.TextGenError
75+
if !errors.As(err, &tge) {
76+
t.Fatalf("errors.As(*TextGenError) failed: %T %v", err, err)
77+
}
78+
if tge.Kind != wantKind {
79+
t.Errorf("Kind = %q; want %q", tge.Kind, wantKind)
80+
}
81+
if tge.Provider != wantProvider {
82+
t.Errorf("Provider = %q; want %q", tge.Provider, wantProvider)
83+
}
84+
var failure *agent.TextGenerationError
85+
if !errors.As(err, &failure) {
86+
t.Errorf("errors.As(*TextGenerationError) failed — evidence lost, timeout diagnostic degrades silently: %T", err)
87+
}
88+
}
89+
6390
missing := func(ctx context.Context, _ string, _ ...string) *exec.Cmd {
6491
return exec.CommandContext(ctx, "/nonexistent/binary/that/does/not/exist")
6592
}
@@ -77,6 +104,11 @@ func TestGenerateText_Matrix(t *testing.T) {
77104
stdoutOnly := func(ctx context.Context, _ string, _ ...string) *exec.Cmd {
78105
return exec.CommandContext(ctx, "sh", "-c", `printf 'HTTP 429: quota exhausted, add credits'; exit 1`)
79106
}
107+
// Emits on both streams then stalls until the ctx kills it.
108+
stall := func(ctx context.Context, _ string, _ ...string) *exec.Cmd {
109+
return exec.CommandContext(ctx, "sh", "-c",
110+
"printf 'partial'; printf 'stalled talking to API' 1>&2; exec sleep 10")
111+
}
80112
// stdout carrying the model's PROSE about an auth error, not a diagnostic.
81113
// Must NOT classify — a summary that discusses a 401 is not a 401.
82114
stdoutProse := func(ctx context.Context, _ string, _ ...string) *exec.Cmd {
@@ -88,27 +120,14 @@ func TestGenerateText_Matrix(t *testing.T) {
88120
t.Run(a.name+"/CLIMissing", func(t *testing.T) {
89121
t.Parallel()
90122
_, err := a.make(missing).GenerateText(context.Background(), "prompt", "")
91-
var tge *agent.TextGenError
92-
if !errors.As(err, &tge) {
93-
t.Fatalf("CLIMissing: err = %v; want *agent.TextGenError", err)
94-
}
95-
if tge.Kind != agent.TextGenErrorCLIMissing {
96-
t.Errorf("CLIMissing: Kind = %q; want cli_missing", tge.Kind)
97-
}
98-
if tge.Provider != a.provider {
99-
t.Errorf("CLIMissing: Provider = %q; want %q", tge.Provider, a.provider)
100-
}
123+
assertComposition(t, err, agent.TextGenErrorCLIMissing, a.provider)
101124
})
102125
t.Run(a.name+"/AuthFrom401", func(t *testing.T) {
103126
t.Parallel()
104127
_, err := a.make(auth401).GenerateText(context.Background(), "prompt", "")
128+
assertComposition(t, err, agent.TextGenErrorAuth, a.provider)
105129
var tge *agent.TextGenError
106-
if !errors.As(err, &tge) {
107-
t.Fatalf("AuthFrom401: err = %v; want *agent.TextGenError", err)
108-
}
109-
if tge.Kind != agent.TextGenErrorAuth {
110-
t.Errorf("AuthFrom401: Kind = %q; want auth", tge.Kind)
111-
}
130+
_ = errors.As(err, &tge)
112131
// The CLI's own stderr must reach the user verbatim, and the real
113132
// exit code must be carried — this is the whole point of the typed
114133
// error for non-Claude providers.
@@ -167,6 +186,30 @@ func TestGenerateText_Matrix(t *testing.T) {
167186
t.Errorf("EmptyStdout: Message = %q; want %q", tge.Message, a.emptyMsg)
168187
}
169188
})
189+
t.Run(a.name+"/CanceledCarriesSentinelAndEvidence", func(t *testing.T) {
190+
t.Parallel()
191+
ctx, cancel := context.WithCancel(context.Background())
192+
go func() { time.Sleep(50 * time.Millisecond); cancel() }()
193+
_, err := a.make(stall).GenerateText(ctx, "prompt", "")
194+
195+
// On the ctx path classification is meaningless, so the sentinel and
196+
// the evidence are the entire payload. Returning a bare sentinel
197+
// (which is what this code did before the reconciliation) loses the
198+
// stderr the timeout diagnostic renders.
199+
if !errors.Is(err, context.Canceled) {
200+
t.Fatalf("errors.Is(context.Canceled) failed: %T %v", err, err)
201+
}
202+
var failure *agent.TextGenerationError
203+
if !errors.As(err, &failure) {
204+
t.Fatalf("errors.As(*TextGenerationError) failed — evidence lost: %T", err)
205+
}
206+
if failure.Stderr != "stalled talking to API" {
207+
t.Errorf("Stderr = %q; want the stderr captured before the kill", failure.Stderr)
208+
}
209+
if failure.StdoutBytes == 0 {
210+
t.Error("StdoutBytes = 0; want the partial stdout counted")
211+
}
212+
})
170213
t.Run(a.name+"/Success", func(t *testing.T) {
171214
t.Parallel()
172215
out, err := a.make(success).GenerateText(context.Background(), "prompt", "")

0 commit comments

Comments
 (0)