Skip to content

Commit b2b331e

Browse files
prekshivyasprekshivyas
committed
fix(e2e): keep terminal failures out of retries
Signed-off-by: Prekshi Vyas <prekshiv@nvidia.com>
1 parent fa32e4a commit b2b331e

15 files changed

Lines changed: 760 additions & 151 deletions

‎test/e2e/RETRY_INVENTORY.md‎

Lines changed: 18 additions & 13 deletions
Large diffs are not rendered by default.

‎test/e2e/fixtures/inference-switch-retry.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ import { runBoundedRetry, type RetryEvidence } from "./retry-policy.ts";
66

77
const TRANSIENT_INFERENCE_SET_FAILURE =
88
/timed? out|timeout|ETIMEDOUT|ECONNRESET|EAI_AGAIN|ENOTFOUND|failed to connect|error sending request|\b50[234]\b/iu;
9+
const TERMINAL_INFERENCE_SET_FAILURE =
10+
/authentication failed|authorization failed|unauthorized|forbidden|HTTP 40[13]\b|\b40[13]\b|denied by network policy|network policy denied|policy (?:update |validation )?failed|malformed|invalid (?:provider|model|configuration|request|[^\r\n]*(?:credential|api[_ -]?key))|(?:model|route|verification) mismatch|expected (?:model|provider|route)[^\r\n]*(?:got|found)/iu;
911

1012
export interface InferenceSwitchRetryArtifactSink {
1113
writeJson(path: string, value: unknown): Promise<string>;
@@ -28,7 +30,10 @@ export function inferenceSetAttemptCount(raw: string | undefined, fallback = 3):
2830
}
2931

3032
export function isTransientInferenceSetFailure(result: ShellProbeResult): boolean {
31-
return TRANSIENT_INFERENCE_SET_FAILURE.test(`${result.stdout}\n${result.stderr}`);
33+
const output = `${result.stdout}\n${result.stderr}`;
34+
return (
35+
!TERMINAL_INFERENCE_SET_FAILURE.test(output) && TRANSIENT_INFERENCE_SET_FAILURE.test(output)
36+
);
3237
}
3338

3439
export function inferenceResponseModel(raw: string): string {

‎test/e2e/lib/inference-switch-retry.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@
77
# returns. Exhausting the transient retry budget remains a verified failure.
88

99
is_transient_inference_set_failure() {
10+
if grep -qiE 'authentication failed|authorization failed|unauthorized|forbidden|HTTP 40[13]|(^|[^0-9])40[13]([^0-9]|$)|denied by network policy|network policy denied|policy (update |validation )?failed|malformed|invalid (provider|model|configuration|request|.*(credential|api[_ -]?key))|(model|route|verification) mismatch|expected (model|provider|route).*(got|found)' <<<"$1"; then
11+
return 1
12+
fi
1013
grep -qiE 'timed? out|timeout|ETIMEDOUT|ECONNRESET|EAI_AGAIN|ENOTFOUND|failed to connect|error sending request|(^|[^0-9])50[234]([^0-9]|$)' <<<"$1"
1114
}
1215

‎test/e2e/live/cloud-inference-provider-skip.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ const ENDPOINT_VALIDATION_RE =
1010
/endpoint validation failed|failed to verify inference endpoint|Chat Completions API validation/i;
1111
const RATE_LIMIT_OR_SANITIZED_EXTERNAL_RE =
1212
/HTTP 429|\b429\b|rate[- ]?limit|too many requests|quota|temporar|timed? out|timeout|ETIMEDOUT|ECONNRESET|EAI_AGAIN|ENOTFOUND|failed to connect|error sending request|\b(redacted|sanitized)\b/i;
13-
const CREDENTIAL_OR_AUTH_RE =
14-
/invalid.*(api[_-]?key|credential)|unauthorized|forbidden|HTTP 40[13]\b|\b40[13]\b/i;
13+
const TERMINAL_ENDPOINT_VALIDATION_RE =
14+
/invalid.*(api[_ -]?key|credential|configuration|request|json)|authentication failed|authorization failed|unauthorized|forbidden|HTTP 40[13]\b|\b40[13]\b|denied by network policy|network policy denied|policy .*failed|routing .*failed|route .*failed|proxy .*failed|hop-by-hop|header stripping|malformed/i;
1515
const TRANSIENT_CHAT_FAILURE =
1616
/timed? out|timeout|ETIMEDOUT|ECONNRESET|EAI_AGAIN|ENOTFOUND|failed to connect|rate[- ]?limit/iu;
1717
const CLOUD_CHAT_HTTP_STATUS_MARKER = "__NEMOCLAW_HTTP_STATUS__:";
@@ -89,7 +89,7 @@ export function classifyPreContractExternalProviderFailure(
8989
): PreContractExternalProviderFailure | null {
9090
const output = resultText(result);
9191
if (!ENDPOINT_VALIDATION_RE.test(output)) return null;
92-
if (CREDENTIAL_OR_AUTH_RE.test(output)) return null;
92+
if (TERMINAL_ENDPOINT_VALIDATION_RE.test(output)) return null;
9393
if (isTransientProviderValidationFailure(result)) {
9494
return {
9595
classifier: "transient-endpoint-validation",

‎test/e2e/live/common-egress-agent-helpers.ts‎

Lines changed: 65 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ export function runHermesAgentAssertionRetry(
8787

8888
export function runOpenClawAgentAssertionRetry(
8989
options: AgentAssertionRetryOptions & {
90-
reconcile: (attempt: AgentAssertionAttempt, attemptNumber: number) => Promise<boolean>;
90+
recover: (attempt: AgentAssertionAttempt, attemptNumber: number) => Promise<boolean>;
9191
},
9292
): Promise<BoundedRetryResult<AgentAssertionAttempt>> {
9393
return runBoundedRetry({
@@ -98,20 +98,20 @@ export function runOpenClawAgentAssertionRetry(
9898
delayMs: options.delayMs,
9999
onEvidence: options.onEvidence,
100100
run: options.run,
101+
reconcile: async (attempt, _error, attemptNumber) => {
102+
if (!attempt?.recoveryRequired) return false;
103+
try {
104+
return await options.recover(attempt, attemptNumber);
105+
} catch {
106+
return false;
107+
}
108+
},
101109
sleep: options.sleep,
102110
classify: (value, error) => {
103111
if (error !== undefined) return { outcome: "failed", failureClass: "deterministic" };
104112
if (value?.passed) return { outcome: "passed" };
105113
return { outcome: "failed", failureClass: value?.failureClass ?? "deterministic" };
106114
},
107-
reconcile: async (value, _error, attemptNumber) => {
108-
if (!value) return false;
109-
try {
110-
return await options.reconcile(value, attemptNumber);
111-
} catch {
112-
return false;
113-
}
114-
},
115115
});
116116
}
117117

@@ -170,8 +170,19 @@ function compactAgentReply(value: string): string {
170170
return value.replace(/\s+/gu, "");
171171
}
172172

173+
const AUTHENTICATION_AGENT_FAILURE_RE =
174+
/authentication failed|unauthorized|HTTP 401\b|\b401\b|invalid (?:credential|api[_ -]?key)/iu;
175+
const AUTHORIZATION_AGENT_FAILURE_RE = /authorization failed|forbidden|HTTP 403\b|\b403\b/iu;
176+
const POLICY_AGENT_FAILURE_RE =
177+
/SsrFBlockedError|Blocked hostname|denied by network policy|network policy denied|policy (?:update |validation )?failed/iu;
178+
const MALFORMED_AGENT_FAILURE_RE = /malformed|invalid request/iu;
179+
const TERMINAL_PROVIDER_VALIDATION_RE =
180+
/invalid.*(api[_ -]?key|credential|configuration|request|json)|authentication failed|authorization failed|unauthorized|forbidden|HTTP 40[13]\b|\b40[13]\b|denied by network policy|network policy denied|policy .*failed|routing .*failed|route .*failed|proxy .*failed|hop-by-hop|header stripping|malformed/iu;
181+
const TRANSIENT_AGENT_FAILURE_RE =
182+
/ECONNREFUSED|EAI_AGAIN|ECONNRESET|ETIMEDOUT|gateway unavailable|network connection error|DNS error|fetch failed|LLM request timed out|FailoverError|inference service unavailable|rawError=503/iu;
183+
173184
function isOpenClawPolicyBlock(output: string): boolean {
174-
return /SsrFBlockedError|Blocked hostname/i.test(output);
185+
return POLICY_AGENT_FAILURE_RE.test(output);
175186
}
176187

177188
function isOpenClawScopeUpgradePending(output: string): boolean {
@@ -181,8 +192,12 @@ function isOpenClawScopeUpgradePending(output: string): boolean {
181192
}
182193

183194
function isOpenClawTransientAgentError(output: string): boolean {
184-
return /ECONNREFUSED|EAI_AGAIN|ECONNRESET|ETIMEDOUT|gateway unavailable|network connection error|DNS error|fetch failed|LLM request timed out|FailoverError|inference service unavailable|rawError=503/i.test(
185-
output,
195+
return (
196+
!AUTHENTICATION_AGENT_FAILURE_RE.test(output) &&
197+
!AUTHORIZATION_AGENT_FAILURE_RE.test(output) &&
198+
!POLICY_AGENT_FAILURE_RE.test(output) &&
199+
!MALFORMED_AGENT_FAILURE_RE.test(output) &&
200+
TRANSIENT_AGENT_FAILURE_RE.test(output)
186201
);
187202
}
188203

@@ -200,17 +215,20 @@ export function classifyOpenClawAgentAssertion(
200215
if (isOpenClawPolicyBlock(result.response)) {
201216
return { passed: false, failureClass: "policy-denial" };
202217
}
203-
if (/\b401\b|unauthorized|authentication failed|invalid api key/iu.test(result.response)) {
218+
if (AUTHENTICATION_AGENT_FAILURE_RE.test(result.response)) {
204219
return { passed: false, failureClass: "authentication" };
205220
}
206-
if (/\b403\b|forbidden/iu.test(result.response)) {
221+
if (AUTHORIZATION_AGENT_FAILURE_RE.test(result.response)) {
207222
return { passed: false, failureClass: "authorization" };
208223
}
224+
if (MALFORMED_AGENT_FAILURE_RE.test(result.response)) {
225+
return { passed: false, failureClass: "malformed-input" };
226+
}
209227
const recoveryRequired = isOpenClawScopeUpgradePending(result.response);
210228
return {
211229
passed: false,
212230
failureClass:
213-
recoveryRequired || isOpenClawTransientAgentError(result.response)
231+
recoveryRequired || (result.exitCode !== 0 && isOpenClawTransientAgentError(result.response))
214232
? "transient-external"
215233
: "deterministic",
216234
recoveryRequired,
@@ -233,6 +251,18 @@ export function classifyHermesAgentAssertion(
233251
if (result.httpStatus === "403") {
234252
return { passed: false, failureClass: "authorization" };
235253
}
254+
if (AUTHENTICATION_AGENT_FAILURE_RE.test(result.response)) {
255+
return { passed: false, failureClass: "authentication" };
256+
}
257+
if (AUTHORIZATION_AGENT_FAILURE_RE.test(result.response)) {
258+
return { passed: false, failureClass: "authorization" };
259+
}
260+
if (POLICY_AGENT_FAILURE_RE.test(result.response)) {
261+
return { passed: false, failureClass: "policy-denial" };
262+
}
263+
if (MALFORMED_AGENT_FAILURE_RE.test(result.response)) {
264+
return { passed: false, failureClass: "malformed-input" };
265+
}
236266
return {
237267
passed: false,
238268
failureClass: isHermesTransientAgentFailure(result.httpStatus, result.response)
@@ -243,13 +273,19 @@ export function classifyHermesAgentAssertion(
243273

244274
/** Recognize transport/provider failures without retrying a successful product response. */
245275
export function isHermesTransientAgentFailure(httpStatus: string, output: string): boolean {
246-
if (httpStatus === "200") return false;
247-
return (
248-
/^(408|429|5[0-9]{2})$/u.test(httpStatus) ||
249-
/ECONNREFUSED|EAI_AGAIN|ECONNRESET|ETIMEDOUT|gateway unavailable|network connection error|DNS error|fetch failed|inference service unavailable/iu.test(
250-
output,
251-
)
252-
);
276+
if (
277+
httpStatus === "200" ||
278+
/^(401|403)$/u.test(httpStatus) ||
279+
AUTHENTICATION_AGENT_FAILURE_RE.test(output) ||
280+
AUTHORIZATION_AGENT_FAILURE_RE.test(output) ||
281+
POLICY_AGENT_FAILURE_RE.test(output) ||
282+
MALFORMED_AGENT_FAILURE_RE.test(output)
283+
) {
284+
return false;
285+
}
286+
if (/^(408|429|5[0-9]{2})$/u.test(httpStatus)) return true;
287+
const hasNoResponseStatus = httpStatus === "" || httpStatus === "000";
288+
return hasNoResponseStatus && TRANSIENT_AGENT_FAILURE_RE.test(output);
253289
}
254290

255291
export function classifyPreContractProviderValidationSkip(
@@ -260,11 +296,16 @@ export function classifyPreContractProviderValidationSkip(
260296
/endpoint validation failed|failed to verify inference endpoint|Chat Completions API validation/i.test(
261297
output,
262298
);
263-
const transientProviderValidationFailure = isTransientProviderValidationFailure(result);
299+
const terminalProviderValidationFailure = TERMINAL_PROVIDER_VALIDATION_RE.test(output);
300+
const transientProviderValidationFailure =
301+
!terminalProviderValidationFailure && isTransientProviderValidationFailure(result);
264302
const http429ProviderValidationFailure =
265-
providerValidation && /HTTP\s*429|\b429\b|rate[- ]?limit|too many requests/i.test(output);
303+
providerValidation &&
304+
!terminalProviderValidationFailure &&
305+
/HTTP\s*429|\b429\b|rate[- ]?limit|too many requests/i.test(output);
266306
const sanitizedEndpointValidationFailure =
267307
providerValidation &&
308+
!terminalProviderValidationFailure &&
268309
/Validation details were omitted to avoid exposing credentials/i.test(output) &&
269310
process.env.GITHUB_ACTIONS === "true";
270311

‎test/e2e/live/common-egress-agent.test.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -486,8 +486,7 @@ async function runOpenClawAgentAssertion(
486486
response: combined,
487487
});
488488
},
489-
reconcile: async (attempt, attemptNumber) => {
490-
if (!attempt.recoveryRequired) return true;
489+
recover: async (_attempt, attemptNumber) => {
491490
const recover = await host.command("node", [CLI_ENTRYPOINT, args.sandboxName, "recover"], {
492491
artifactName: `${args.label}-recover-after-attempt-${attemptNumber}`,
493492
env: commandEnv(),

‎test/e2e/live/hermes-inference-switch-helpers.ts‎

Lines changed: 109 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,11 @@ import {
4343
writeInferenceSwitchRetryEvidence,
4444
} from "../fixtures/inference-switch-retry.ts";
4545
import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts";
46+
import {
47+
runBoundedRetry,
48+
type RetryEvidence,
49+
type RetryFailureClass,
50+
} from "../fixtures/retry-policy.ts";
4651
import type { ShellProbeResult } from "../fixtures/shell-probe.ts";
4752
import { stripAnsi } from "./json-envelope.ts";
4853
import { isTransientProviderValidationFailure } from "./network-policy-transient-provider.ts";
@@ -327,52 +332,115 @@ export function chatContent(raw: string): string {
327332
);
328333
}
329334

330-
export async function runHermesPongWithRetry(options: {
331-
attempts?: number;
332-
delay?: (milliseconds: number) => Promise<void>;
333-
expectedModel: string;
334-
run: (attempt: number) => Promise<ShellProbeResult>;
335-
}): Promise<ShellProbeResult> {
336-
const attempts = options.attempts ?? 3;
337-
const delay =
338-
options.delay ??
339-
((milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)));
340-
let last: ShellProbeResult | undefined;
341-
for (let attempt = 1; attempt <= attempts; attempt += 1) {
342-
last = await options.run(attempt);
343-
let pong = false;
344-
if (last.exitCode === 0) {
345-
try {
346-
pong =
347-
inferenceResponseModel(last.stdout) === options.expectedModel &&
348-
/PONG/iu.test(chatContent(last.stdout));
349-
} catch {}
350-
}
351-
if (pong || attempt === attempts) return last;
352-
await delay(5_000);
335+
const TERMINAL_HERMES_PROBE_RE =
336+
/authentication failed|authorization failed|unauthorized|forbidden|HTTP 40[13]\b|\b40[13]\b|denied by network policy|network policy denied|policy (?:update |validation )?failed|malformed|invalid (?:credential|api[_ -]?key|request|json)/iu;
337+
const TRANSIENT_HERMES_PROBE_RE =
338+
/ECONNREFUSED|EAI_AGAIN|ECONNRESET|ETIMEDOUT|timed? out|gateway unavailable|network connection error|DNS error|fetch failed|inference service unavailable|rawError=503/iu;
339+
340+
function hermesProbeFailureClass(result: ShellProbeResult): RetryFailureClass {
341+
const output = resultText(result);
342+
if (/authentication failed|unauthorized|HTTP 401\b|\b401\b/iu.test(output)) {
343+
return "authentication";
344+
}
345+
if (/authorization failed|forbidden|HTTP 403\b|\b403\b/iu.test(output)) {
346+
return "authorization";
353347
}
354-
throw new Error("Hermes live probe retry loop completed without running an attempt.");
348+
if (
349+
/denied by network policy|network policy denied|policy (?:update |validation )?failed/iu.test(
350+
output,
351+
)
352+
) {
353+
return "policy-denial";
354+
}
355+
if (/malformed|invalid (?:request|json)/iu.test(output)) return "malformed-input";
356+
if (
357+
result.exitCode !== 0 &&
358+
!TERMINAL_HERMES_PROBE_RE.test(output) &&
359+
TRANSIENT_HERMES_PROBE_RE.test(output)
360+
) {
361+
return "transient-external";
362+
}
363+
return "deterministic";
355364
}
356365

357-
export async function runHermesCliPongWithRetry(options: {
358-
accept?: (result: ShellProbeResult, attempt: number) => boolean;
366+
interface HermesProbeRetryOptions {
359367
attempts?: number;
360368
delay?: (milliseconds: number) => Promise<void>;
361-
run: (attempt: number) => Promise<ShellProbeResult>;
362-
}): Promise<ShellProbeResult> {
363-
const attempts = options.attempts ?? 3;
364-
const delay =
365-
options.delay ??
366-
((milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)));
367-
let last: ShellProbeResult | undefined;
368-
for (let attempt = 1; attempt <= attempts; attempt += 1) {
369-
last = await options.run(attempt);
370-
const accepted = options.accept?.(last, attempt) ?? true;
371-
if ((last.exitCode === 0 && /\bPONG\b/iu.test(last.stdout) && accepted) || attempt === attempts)
372-
return last;
373-
await delay(5_000);
374-
}
375-
throw new Error("Hermes CLI retry loop completed without running an attempt.");
369+
onEvidence?: (evidence: RetryEvidence) => Promise<void> | void;
370+
}
371+
372+
export async function runHermesPongWithRetry(
373+
options: HermesProbeRetryOptions & {
374+
expectedModel: string;
375+
run: (attempt: number) => Promise<ShellProbeResult>;
376+
},
377+
): Promise<ShellProbeResult> {
378+
const execution = await runBoundedRetry({
379+
operation: "hermes-inference-switch.pong",
380+
owner: "inference-provider",
381+
idempotence: "read-only",
382+
maxAttempts: options.attempts ?? 3,
383+
delayMs: 5_000,
384+
onEvidence: options.onEvidence,
385+
run: async (attempt) => {
386+
const result = await options.run(attempt);
387+
let passed = false;
388+
if (result.exitCode === 0) {
389+
try {
390+
passed =
391+
inferenceResponseModel(result.stdout) === options.expectedModel &&
392+
/PONG/iu.test(chatContent(result.stdout));
393+
} catch {}
394+
}
395+
return { passed, result };
396+
},
397+
sleep: options.delay,
398+
classify: (value, error) => {
399+
if (error !== undefined || !value) {
400+
return { outcome: "failed", failureClass: "deterministic" };
401+
}
402+
if (value.passed) return { outcome: "passed" };
403+
return { outcome: "failed", failureClass: hermesProbeFailureClass(value.result) };
404+
},
405+
});
406+
if (execution.value) return execution.value.result;
407+
throw new Error("Hermes live probe completed without an attempt result.");
408+
}
409+
410+
export async function runHermesCliPongWithRetry(
411+
options: HermesProbeRetryOptions & {
412+
accept?: (result: ShellProbeResult, attempt: number) => boolean;
413+
run: (attempt: number) => Promise<ShellProbeResult>;
414+
},
415+
): Promise<ShellProbeResult> {
416+
const execution = await runBoundedRetry({
417+
operation: "hermes-inference-switch.cli-pong",
418+
owner: "inference-provider",
419+
idempotence: "read-only",
420+
maxAttempts: options.attempts ?? 3,
421+
delayMs: 5_000,
422+
onEvidence: options.onEvidence,
423+
run: async (attempt) => {
424+
const result = await options.run(attempt);
425+
return {
426+
passed:
427+
result.exitCode === 0 &&
428+
/\bPONG\b/iu.test(result.stdout) &&
429+
(options.accept?.(result, attempt) ?? true),
430+
result,
431+
};
432+
},
433+
sleep: options.delay,
434+
classify: (value, error) => {
435+
if (error !== undefined || !value) {
436+
return { outcome: "failed", failureClass: "deterministic" };
437+
}
438+
if (value.passed) return { outcome: "passed" };
439+
return { outcome: "failed", failureClass: hermesProbeFailureClass(value.result) };
440+
},
441+
});
442+
if (execution.value) return execution.value.result;
443+
throw new Error("Hermes CLI probe completed without an attempt result.");
376444
}
377445

378446
export async function cleanupHermesSwitch(

0 commit comments

Comments
 (0)