Skip to content

Commit 00764b6

Browse files
committed
fix(e2e): harden unit gap evidence reads
Signed-off-by: Carlos Villela <cvillela@nvidia.com>
1 parent 03c57f8 commit 00764b6

4 files changed

Lines changed: 72 additions & 24 deletions

File tree

test/e2e/README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -668,6 +668,8 @@ reviews them; redaction reduces exposure but does not prove that a report is
668668
credential-free. The command exits nonzero when a selected run is unfinished or
669669
failed-run evidence is unavailable. Do not accept a partial report as the
670670
weekly ledger.
671+
Every GitHub read names `NVIDIA/NemoClaw`, so a fork or different checkout remote
672+
cannot substitute another repository's run data.
671673
The command also stops when a workflow reaches the 1,000-run collection limit.
672674
Narrow the selected range and retry so the report cannot omit older runs silently.
673675

test/e2e/support/e2e-unit-test-gaps.test.ts

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,12 @@ import {
1111
normalizeFailureSignature,
1212
type RunLogEvidence,
1313
} from "../../../tools/e2e/unit-test-gaps-core.mts";
14-
import { requireCompleteRunSelection, rollingRange } from "../../../tools/e2e/unit-test-gaps.mts";
14+
import {
15+
failedRunLogArgs,
16+
listRunsArgs,
17+
requireCompleteRunSelection,
18+
rollingRange,
19+
} from "../../../tools/e2e/unit-test-gaps.mts";
1520

1621
function evidence(overrides: Partial<RunLogEvidence> = {}): RunLogEvidence {
1722
return {
@@ -69,6 +74,23 @@ describe("weekly E2E unit-test gap analysis", () => {
6974
);
7075
});
7176

77+
it("binds workflow and failed-log reads to the canonical repository", () => {
78+
expect(
79+
listRunsArgs("e2e.yaml", {
80+
from: "2026-08-09T20:00:00.000Z",
81+
to: "2026-08-16T20:00:00.000Z",
82+
}),
83+
).toEqual(expect.arrayContaining(["--repo", "NVIDIA/NemoClaw", "--workflow", "e2e.yaml"]));
84+
expect(failedRunLogArgs(12345678)).toEqual([
85+
"run",
86+
"view",
87+
"12345678",
88+
"--repo",
89+
"NVIDIA/NemoClaw",
90+
"--log-failed",
91+
]);
92+
});
93+
7294
it("groups volatile BuildKit references under the missing build-input contract", () => {
7395
expect(
7496
normalizeFailureSignature(
@@ -96,6 +118,20 @@ describe("weekly E2E unit-test gap analysis", () => {
96118
]);
97119
});
98120

121+
it("strips terminal controls after a timestamp and preserves an unprefixed message", () => {
122+
expect(
123+
extractJobSignatures(
124+
[
125+
"online\tstep\t2026-08-12T10:00:00.0000000Z \u001b[31mError: colored failure\u001b[0m",
126+
"offline\tstep\tError: offline evidence failed",
127+
].join("\n"),
128+
),
129+
).toEqual([
130+
{ job: "online", signature: "Error: colored failure" },
131+
{ job: "offline", signature: "Error: offline evidence failed" },
132+
]);
133+
});
134+
99135
it("keeps a failed job in the queue when its causal line needs manual review", () => {
100136
expect(
101137
extractJobSignatures(

tools/e2e/unit-test-gaps-core.mts

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
22
// SPDX-License-Identifier: Apache-2.0
33

4+
import { stripVTControlCharacters } from "node:util";
5+
46
type SanitizationModule = typeof import("../../src/lib/readiness/sanitize.ts");
57

68
const loadedSanitization = (await import("../../src/lib/readiness/sanitize.ts")) as unknown as {
@@ -56,7 +58,6 @@ export interface UnitGapReport {
5658
groups: UnitGapGroup[];
5759
}
5860

59-
const ANSI_PATTERN = /\u001b\[[0-9;]*[A-Za-z]/gu;
6061
const TIMESTAMP_PATTERN = /^\uFEFF?\d{4}-\d{2}-\d{2}T\S+Z\s+/u;
6162
const UUID_PATTERN =
6263
/\b[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\b/giu;
@@ -84,11 +85,9 @@ function stripLogPrefix(line: string): { job: string; message: string } | null {
8485
const fields = line.split("\t");
8586
if (fields.length < 3) return null;
8687
const job = fields[0]!.trim();
87-
const message = fields
88-
.slice(2)
89-
.join("\t")
90-
.replace(ANSI_PATTERN, "")
91-
.replace(TIMESTAMP_PATTERN, "");
88+
const message = stripVTControlCharacters(
89+
fields.slice(2).join("\t").replace(TIMESTAMP_PATTERN, ""),
90+
);
9291
return job.length === 0 ? null : { job, message: message.trim() };
9392
}
9493

tools/e2e/unit-test-gaps.mts

Lines changed: 28 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import {
1515
} from "./unit-test-gaps-core.mts";
1616

1717
const execFileAsync = promisify(execFile);
18+
const NEMOCLAW_REPOSITORY = "NVIDIA/NemoClaw";
1819
const DEFAULT_WORKFLOWS = ["e2e.yaml", "portable-profile-e2e.yaml"];
1920
const MAX_GH_BUFFER_BYTES = 128 * 1024 * 1024;
2021
const MAX_RUNS_PER_WORKFLOW = 1000;
@@ -140,28 +141,38 @@ export function requireCompleteRunSelection(workflow: string, runCount: number):
140141
);
141142
}
142143

144+
export function listRunsArgs(workflow: string, range: { from: string; to: string }): string[] {
145+
return [
146+
"run",
147+
"list",
148+
"--repo",
149+
NEMOCLAW_REPOSITORY,
150+
"--workflow",
151+
workflow,
152+
"--branch",
153+
"main",
154+
"--event",
155+
"push",
156+
"--created",
157+
`${range.from}..${range.to}`,
158+
"--limit",
159+
String(MAX_RUNS_PER_WORKFLOW),
160+
"--json",
161+
"attempt,conclusion,createdAt,databaseId,event,headBranch,headSha,name,status,url",
162+
];
163+
}
164+
165+
export function failedRunLogArgs(databaseId: number): string[] {
166+
return ["run", "view", String(databaseId), "--repo", NEMOCLAW_REPOSITORY, "--log-failed"];
167+
}
168+
143169
async function collectRuns(
144170
workflows: readonly string[],
145171
range: { from: string; to: string },
146172
): Promise<E2ERunRecord[]> {
147173
const records = await Promise.all(
148174
workflows.map(async (workflow) => {
149-
const output = await gh([
150-
"run",
151-
"list",
152-
"--workflow",
153-
workflow,
154-
"--branch",
155-
"main",
156-
"--event",
157-
"push",
158-
"--created",
159-
`${range.from}..${range.to}`,
160-
"--limit",
161-
String(MAX_RUNS_PER_WORKFLOW),
162-
"--json",
163-
"attempt,conclusion,createdAt,databaseId,event,headBranch,headSha,name,status,url",
164-
]);
175+
const output = await gh(listRunsArgs(workflow, range));
165176
const parsed = JSON.parse(output) as unknown;
166177
if (!Array.isArray(parsed)) throw new Error(`GitHub returned malformed runs for ${workflow}`);
167178
requireCompleteRunSelection(workflow, parsed.length);
@@ -197,7 +208,7 @@ async function collectEvidence(runs: readonly E2ERunRecord[]): Promise<RunLogEvi
197208
const logs = new Map<number, RunLogEvidence>();
198209
await parallelMap(failures, DEFAULT_CONCURRENCY, async (run) => {
199210
try {
200-
const log = await gh(["run", "view", String(run.databaseId), "--log-failed"]);
211+
const log = await gh(failedRunLogArgs(run.databaseId));
201212
logs.set(run.databaseId, { log, run });
202213
} catch (error) {
203214
logs.set(run.databaseId, {

0 commit comments

Comments
 (0)