|
| 1 | +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. |
| 2 | +// SPDX-License-Identifier: Apache-2.0 |
| 3 | + |
| 4 | +import { describe, expect, it } from "vitest"; |
| 5 | + |
| 6 | +import { |
| 7 | + buildUnitGapReport, |
| 8 | + classifyFailureSignature, |
| 9 | + extractJobSignatures, |
| 10 | + formatUnitGapReport, |
| 11 | + normalizeFailureSignature, |
| 12 | + type RunLogEvidence, |
| 13 | +} from "../../../tools/e2e/unit-test-gaps-core.mts"; |
| 14 | +import { rollingRange } from "../../../tools/e2e/unit-test-gaps.mts"; |
| 15 | + |
| 16 | +function evidence(overrides: Partial<RunLogEvidence> = {}): RunLogEvidence { |
| 17 | + return { |
| 18 | + log: "job\tstep\t2026-08-12T10:00:00.0000000Z AssertionError: expected UPGRADE, received 400\n", |
| 19 | + run: { |
| 20 | + attempt: 1, |
| 21 | + conclusion: "failure", |
| 22 | + createdAt: "2026-08-12T10:00:00Z", |
| 23 | + databaseId: 12345678, |
| 24 | + event: "push", |
| 25 | + headBranch: "main", |
| 26 | + headSha: "1234567890abcdef1234567890abcdef12345678", |
| 27 | + name: "E2E main", |
| 28 | + status: "completed", |
| 29 | + url: "https://github.com/NVIDIA/NemoClaw/actions/runs/12345678", |
| 30 | + }, |
| 31 | + ...overrides, |
| 32 | + }; |
| 33 | +} |
| 34 | + |
| 35 | +describe("weekly E2E unit-test gap analysis", () => { |
| 36 | + it("redacts volatile identifiers, paths, URLs, sandboxes, and durations", () => { |
| 37 | + const signature = normalizeFailureSignature( |
| 38 | + "Error: sandbox e2e-sbx-a at /home/runner/work/NemoClaw failed after 180000ms for 1234567890abcdef1234567890abcdef12345678 via https://example.test/path?token=secret", |
| 39 | + ); |
| 40 | + |
| 41 | + expect(signature).toBe( |
| 42 | + "Error: sandbox <sandbox> at <path> failed after <duration> for <sha> via <url>", |
| 43 | + ); |
| 44 | + expect(signature).not.toContain("secret"); |
| 45 | + }); |
| 46 | + |
| 47 | + it("redacts credential-shaped values before writing a cause candidate", () => { |
| 48 | + const signature = normalizeFailureSignature( |
| 49 | + "Error: Authorization: Bearer ghp_EXAMPLE012345678901234 AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE session=eyJhbGciOiJIUzI1NiJ9.cGF5bG9hZA.c2lnbmF0dXJl", |
| 50 | + ); |
| 51 | + |
| 52 | + expect(signature).toBe( |
| 53 | + "Error: Authorization: Bearer <REDACTED> AWS_ACCESS_KEY_ID=<REDACTED> session=<REDACTED>", |
| 54 | + ); |
| 55 | + expect(signature).not.toContain("EXAMPLE"); |
| 56 | + }); |
| 57 | + |
| 58 | + it("uses an exact rolling window for --days", () => { |
| 59 | + expect(rollingRange(7, new Date("2026-08-16T19:30:00.000Z"))).toEqual({ |
| 60 | + from: "2026-08-09T19:30:00.000Z", |
| 61 | + to: "2026-08-16T19:30:00.000Z", |
| 62 | + }); |
| 63 | + }); |
| 64 | + |
| 65 | + it("groups volatile BuildKit references under the missing build-input contract", () => { |
| 66 | + expect( |
| 67 | + normalizeFailureSignature( |
| 68 | + 'ERROR: failed to build: failed to solve: failed to compute cache key: failed to calculate checksum of ref 12345678-1234-4234-9234-123456789abc::sztmu18osbm95fj41qvxlgdie: "/tools/mcp-tool-discovery-runtime/reviewed-runtime-bundle/mcp-tool-discovery/mcp-tool-discovery.bundle": not found', |
| 69 | + ), |
| 70 | + ).toBe("ERROR: reviewed runtime bundle is missing from the image build context"); |
| 71 | + }); |
| 72 | + |
| 73 | + it("uses the earliest high-specificity causal line and ignores wrappers and echoed shell", () => { |
| 74 | + const signatures = extractJobSignatures( |
| 75 | + [ |
| 76 | + "portable-launch\tstep\t2026-08-12T10:00:00.0000000Z ##[error]Process completed with exit code 1.", |
| 77 | + 'portable-launch\tstep\t2026-08-12T10:00:00.5000000Z echo "::error::a shell guard failed"', |
| 78 | + "portable-launch\tstep\t2026-08-12T10:00:01.0000000Z Error: Portable Podman readiness failed at service activation", |
| 79 | + "rootless-linux\tstep\t2026-08-12T10:00:02.0000000Z npm error code EAI_AGAIN", |
| 80 | + ].join("\n"), |
| 81 | + ); |
| 82 | + |
| 83 | + expect(signatures).toEqual([ |
| 84 | + { |
| 85 | + job: "portable-launch", |
| 86 | + signature: "Error: Portable Podman readiness failed at service activation", |
| 87 | + }, |
| 88 | + { job: "rootless-linux", signature: "npm error code EAI_AGAIN" }, |
| 89 | + ]); |
| 90 | + }); |
| 91 | + |
| 92 | + it("keeps a failed job in the queue when its causal line needs manual review", () => { |
| 93 | + expect( |
| 94 | + extractJobSignatures( |
| 95 | + "job\tstep\t2026-08-12T10:00:00.0000000Z ##[error]Process completed with exit code 1.\n", |
| 96 | + ), |
| 97 | + ).toEqual([{ job: "job", signature: "Failed job log requires manual causal-line review" }]); |
| 98 | + }); |
| 99 | + |
| 100 | + it.each([ |
| 101 | + ["AssertionError: expected UPGRADE, received 400", "deterministic"], |
| 102 | + ["npm error code EAI_AGAIN", "external"], |
| 103 | + ["Error: E2E cleanup failed: gateway unavailable", "harness"], |
| 104 | + ["Error: Local BuildKit build failed", "needs-triage"], |
| 105 | + ] as const)("classifies %s as %s", (signature, classification) => { |
| 106 | + expect(classifyFailureSignature(signature)).toBe(classification); |
| 107 | + }); |
| 108 | + |
| 109 | + it("groups the same normalized cause across runs and keeps the required test action", () => { |
| 110 | + const first = evidence(); |
| 111 | + const second = evidence({ |
| 112 | + run: { |
| 113 | + ...evidence().run, |
| 114 | + databaseId: 23456789, |
| 115 | + url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23456789", |
| 116 | + }, |
| 117 | + }); |
| 118 | + const report = buildUnitGapReport( |
| 119 | + [first, second], |
| 120 | + { from: "2026-08-09", to: "2026-08-16" }, |
| 121 | + "2026-08-16T20:00:00.000Z", |
| 122 | + ); |
| 123 | + |
| 124 | + expect(report.groups).toHaveLength(1); |
| 125 | + expect(report.groups[0]).toMatchObject({ |
| 126 | + classification: "deterministic", |
| 127 | + regressionTest: null, |
| 128 | + reviewStatus: "open", |
| 129 | + runCount: 2, |
| 130 | + runIds: [12345678, 23456789], |
| 131 | + }); |
| 132 | + expect(report.groups[0]!.requiredAction).toContain("unit or package-contract regression test"); |
| 133 | + }); |
| 134 | + |
| 135 | + it("fails the evidence ledger visibly when a failed log is unavailable", () => { |
| 136 | + const report = buildUnitGapReport( |
| 137 | + [evidence({ error: "too many API requests needed to fetch logs", log: undefined })], |
| 138 | + { from: "2026-08-09", to: "2026-08-16" }, |
| 139 | + "2026-08-16T20:00:00.000Z", |
| 140 | + ); |
| 141 | + const markdown = formatUnitGapReport(report); |
| 142 | + |
| 143 | + expect(report.incompleteRuns).toEqual([ |
| 144 | + { |
| 145 | + error: "too many API requests needed to fetch logs", |
| 146 | + runId: 12345678, |
| 147 | + url: "https://github.com/NVIDIA/NemoClaw/actions/runs/12345678", |
| 148 | + }, |
| 149 | + ]); |
| 150 | + expect(markdown).toContain("The report is incomplete."); |
| 151 | + }); |
| 152 | + |
| 153 | + it("keeps a run active at the cutoff from producing a complete ledger", () => { |
| 154 | + const active = evidence({ |
| 155 | + log: undefined, |
| 156 | + run: { ...evidence().run, conclusion: "", status: "in_progress" }, |
| 157 | + }); |
| 158 | + const report = buildUnitGapReport( |
| 159 | + [active], |
| 160 | + { from: "2026-08-09T20:00:00.000Z", to: "2026-08-16T20:00:00.000Z" }, |
| 161 | + "2026-08-16T20:00:00.000Z", |
| 162 | + ); |
| 163 | + |
| 164 | + expect(report.incompleteRuns).toEqual([ |
| 165 | + { |
| 166 | + error: "run was in_progress at the collection cutoff", |
| 167 | + runId: 12345678, |
| 168 | + url: "https://github.com/NVIDIA/NemoClaw/actions/runs/12345678", |
| 169 | + }, |
| 170 | + ]); |
| 171 | + }); |
| 172 | +}); |
0 commit comments