Skip to content

Commit f095514

Browse files
HOYALIMHOYALIM
committed
feat(e2e): report same-commit reliability
Signed-off-by: Ho Lim <subhoya@gmail.com>
1 parent f5198b8 commit f095514

6 files changed

Lines changed: 1114 additions & 32 deletions

File tree

‎.github/workflows/e2e-main-retry.yaml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,3 +66,57 @@ jobs:
6666
path: ${{ runner.temp }}/e2e-main-retry-evidence.json
6767
if-no-files-found: warn
6868
retention-days: 14
69+
70+
report-same-commit-reliability:
71+
needs: evaluate
72+
if: >-
73+
${{
74+
always() &&
75+
github.run_attempt == 1 &&
76+
github.repository == 'NVIDIA/NemoClaw' &&
77+
github.event.workflow_run.status == 'completed' &&
78+
(github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'workflow_dispatch') &&
79+
github.event.workflow_run.path == '.github/workflows/e2e.yaml' &&
80+
startsWith(github.event.workflow_run.display_title, 'E2E ') &&
81+
github.event.workflow_run.head_branch == 'main' &&
82+
github.event.workflow_run.head_repository.full_name == 'NVIDIA/NemoClaw'
83+
}}
84+
runs-on: ubuntu-latest
85+
timeout-minutes: 10
86+
permissions:
87+
actions: read
88+
contents: read
89+
steps:
90+
- name: Checkout trusted reliability reporter
91+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
92+
with:
93+
ref: ${{ github.workflow_sha }}
94+
persist-credentials: false
95+
96+
- name: Setup Node.js
97+
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
98+
with:
99+
node-version: "22"
100+
101+
- name: Build advisory same-commit reliability report
102+
env:
103+
GITHUB_TOKEN: ${{ github.token }}
104+
RELIABILITY_REPORT_PATH: ${{ runner.temp }}/same-commit-reliability.json
105+
SOURCE_RUN_ID: ${{ github.event.workflow_run.id }}
106+
shell: bash
107+
run: |
108+
set -euo pipefail
109+
node --experimental-strip-types --no-warnings \
110+
tools/e2e/same-commit-reliability.mts \
111+
| tee "${RUNNER_TEMP}/same-commit-reliability.md"
112+
cat "${RUNNER_TEMP}/same-commit-reliability.md" >>"${GITHUB_STEP_SUMMARY}"
113+
114+
- name: Upload advisory reliability evidence
115+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
116+
with:
117+
name: same-commit-reliability-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}
118+
path: |
119+
${{ runner.temp }}/same-commit-reliability.json
120+
${{ runner.temp }}/same-commit-reliability.md
121+
if-no-files-found: error
122+
retention-days: 14

‎scripts/scorecard/read-artifact-zip.mts‎

Lines changed: 128 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -51,10 +51,109 @@ function isSafeExpectedFile(expectedFile: string): boolean {
5151
!expectedFile.endsWith("/") &&
5252
!expectedFile.includes("\\") &&
5353
!expectedFile.includes("\0") &&
54-
segments.every((segment) => segment !== "" && segment !== "." && segment !== "..")
54+
segments.every(
55+
(segment) => segment !== "" && segment !== "." && segment !== "..",
56+
)
5557
);
5658
}
5759

60+
/**
61+
* Lists safe regular-file entries from a GitHub artifact ZIP without
62+
* extracting or inflating their contents. Structural ambiguity, links,
63+
* encryption, split archives, ZIP64, duplicate names, and excess entries are
64+
* rejected before callers select an allowlisted evidence file.
65+
*/
66+
export function listValidatedArtifactZipEntries(
67+
archive: Buffer,
68+
options: { maxEntries?: number },
69+
): string[] | null {
70+
const maxEntries = options.maxEntries ?? 1000;
71+
if (maxEntries < 1) return null;
72+
const endOffset = findZipEndOfCentralDirectory(archive);
73+
if (endOffset < 0) return null;
74+
const entriesOnDisk = archive.readUInt16LE(endOffset + 8);
75+
const totalEntries = archive.readUInt16LE(endOffset + 10);
76+
const centralDirectorySize = archive.readUInt32LE(endOffset + 12);
77+
const centralDirectoryOffset = archive.readUInt32LE(endOffset + 16);
78+
if (
79+
archive.readUInt16LE(endOffset + 4) !== 0 ||
80+
archive.readUInt16LE(endOffset + 6) !== 0 ||
81+
entriesOnDisk !== totalEntries ||
82+
totalEntries < 1 ||
83+
totalEntries > maxEntries ||
84+
centralDirectoryOffset + centralDirectorySize !== endOffset
85+
) {
86+
return null;
87+
}
88+
89+
const names: string[] = [];
90+
const seen = new Set<string>();
91+
let offset = centralDirectoryOffset;
92+
for (let index = 0; index < totalEntries; index += 1) {
93+
if (
94+
offset + 46 > endOffset ||
95+
archive.readUInt32LE(offset) !== ZIP_CENTRAL_DIRECTORY_SIGNATURE
96+
) {
97+
return null;
98+
}
99+
const creatorSystem = archive.readUInt8(offset + 5);
100+
const flags = archive.readUInt16LE(offset + 8);
101+
const compressionMethod = archive.readUInt16LE(offset + 10);
102+
const compressedSize = archive.readUInt32LE(offset + 20);
103+
const fileNameLength = archive.readUInt16LE(offset + 28);
104+
const extraLength = archive.readUInt16LE(offset + 30);
105+
const commentLength = archive.readUInt16LE(offset + 32);
106+
const diskStart = archive.readUInt16LE(offset + 34);
107+
const externalAttributes = archive.readUInt32LE(offset + 38);
108+
const localHeaderOffset = archive.readUInt32LE(offset + 42);
109+
const entryEnd = offset + 46 + fileNameLength + extraLength + commentLength;
110+
if (entryEnd > endOffset) return null;
111+
const nameBytes = archive.subarray(
112+
offset + 46,
113+
offset + 46 + fileNameLength,
114+
);
115+
let name: string;
116+
try {
117+
name = new TextDecoder("utf-8", { fatal: true }).decode(nameBytes);
118+
} catch {
119+
return null;
120+
}
121+
const unixFileType = (externalAttributes >>> 16) & 0xf000;
122+
if (
123+
!isSafeExpectedFile(name) ||
124+
seen.has(name) ||
125+
diskStart !== 0 ||
126+
(flags & 0x1) !== 0 ||
127+
(compressionMethod !== 0 && compressionMethod !== 8) ||
128+
(creatorSystem !== 0 && creatorSystem !== 3) ||
129+
(creatorSystem === 3 && unixFileType !== 0 && unixFileType !== 0x8000) ||
130+
localHeaderOffset + 30 > centralDirectoryOffset ||
131+
archive.readUInt32LE(localHeaderOffset) !== ZIP_LOCAL_FILE_SIGNATURE
132+
) {
133+
return null;
134+
}
135+
const localFlags = archive.readUInt16LE(localHeaderOffset + 6);
136+
const localCompressionMethod = archive.readUInt16LE(localHeaderOffset + 8);
137+
const localNameLength = archive.readUInt16LE(localHeaderOffset + 26);
138+
const localExtraLength = archive.readUInt16LE(localHeaderOffset + 28);
139+
const localNameEnd = localHeaderOffset + 30 + localNameLength;
140+
const dataEnd = localNameEnd + localExtraLength + compressedSize;
141+
if (
142+
localNameEnd > centralDirectoryOffset ||
143+
dataEnd > centralDirectoryOffset ||
144+
localFlags !== flags ||
145+
localCompressionMethod !== compressionMethod ||
146+
!archive.subarray(localHeaderOffset + 30, localNameEnd).equals(nameBytes)
147+
) {
148+
return null;
149+
}
150+
seen.add(name);
151+
names.push(name);
152+
offset = entryEnd;
153+
}
154+
return offset === endOffset ? names.sort() : null;
155+
}
156+
58157
/**
59158
* Reads the exact bytes of one safe relative file from a GitHub artifact ZIP
60159
* without extracting paths to disk. Duplicate targets, links, encryption,
@@ -67,7 +166,11 @@ export function readValidatedArtifactZipEntryBytes(
67166
): Buffer | null {
68167
const maxEntries = options.maxEntries ?? 1000;
69168
const expectedFileName = Buffer.from(expectedFile, "utf8");
70-
if (!isSafeExpectedFile(expectedFile) || options.maxBytes < 1 || maxEntries < 1) {
169+
if (
170+
!isSafeExpectedFile(expectedFile) ||
171+
options.maxBytes < 1 ||
172+
maxEntries < 1
173+
) {
71174
return null;
72175
}
73176

@@ -96,14 +199,16 @@ export function readValidatedArtifactZipEntryBytes(
96199
for (let entryIndex = 0; entryIndex < totalEntries; entryIndex += 1) {
97200
if (
98201
centralEntryOffset + 46 > endOffset ||
99-
archive.readUInt32LE(centralEntryOffset) !== ZIP_CENTRAL_DIRECTORY_SIGNATURE
202+
archive.readUInt32LE(centralEntryOffset) !==
203+
ZIP_CENTRAL_DIRECTORY_SIGNATURE
100204
) {
101205
return null;
102206
}
103207
const fileNameLength = archive.readUInt16LE(centralEntryOffset + 28);
104208
const extraLength = archive.readUInt16LE(centralEntryOffset + 30);
105209
const commentLength = archive.readUInt16LE(centralEntryOffset + 32);
106-
const centralEntryEnd = centralEntryOffset + 46 + fileNameLength + extraLength + commentLength;
210+
const centralEntryEnd =
211+
centralEntryOffset + 46 + fileNameLength + extraLength + commentLength;
107212
if (centralEntryEnd > endOffset) return null;
108213
const fileName = archive.subarray(
109214
centralEntryOffset + 46,
@@ -164,23 +269,31 @@ export function readValidatedArtifactZipEntryBytes(
164269
localFileNameEnd > centralDirectoryOffset ||
165270
localFlags !== flags ||
166271
localCompressionMethod !== compressionMethod ||
167-
!archive.subarray(localHeaderOffset + 30, localFileNameEnd).equals(expectedFileName) ||
272+
!archive
273+
.subarray(localHeaderOffset + 30, localFileNameEnd)
274+
.equals(expectedFileName) ||
168275
compressedDataEnd > centralDirectoryOffset
169276
) {
170277
return null;
171278
}
172279

173-
const compressedData = archive.subarray(compressedDataOffset, compressedDataEnd);
280+
const compressedData = archive.subarray(
281+
compressedDataOffset,
282+
compressedDataEnd,
283+
);
174284
let contents: Buffer;
175285
try {
176286
contents =
177287
compressionMethod === 0
178288
? Buffer.from(compressedData)
179-
: zlib.inflateRawSync(compressedData, { maxOutputLength: options.maxBytes });
289+
: zlib.inflateRawSync(compressedData, {
290+
maxOutputLength: options.maxBytes,
291+
});
180292
} catch {
181293
return null;
182294
}
183-
if (contents.length !== uncompressedSize || crc32(contents) !== expectedCrc) return null;
295+
if (contents.length !== uncompressedSize || crc32(contents) !== expectedCrc)
296+
return null;
184297
return contents;
185298
}
186299

@@ -190,5 +303,11 @@ export function readValidatedArtifactZipEntry(
190303
expectedFile: string,
191304
options: { maxBytes: number; maxEntries?: number },
192305
): string | null {
193-
return readValidatedArtifactZipEntryBytes(archive, expectedFile, options)?.toString("utf8") ?? null;
306+
return (
307+
readValidatedArtifactZipEntryBytes(
308+
archive,
309+
expectedFile,
310+
options,
311+
)?.toString("utf8") ?? null
312+
);
194313
}

‎test/e2e/README.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -581,6 +581,12 @@ graph as the live targets:
581581

582582
- GitHub Actions run history is the authoritative record for push and
583583
manual E2E results.
584+
- `E2E / Main Retry` publishes an advisory same-commit reliability table for
585+
trusted-main and explicit manual qualification runs. It keeps first-pass
586+
success, pass-after-retry, exhausted retries, and pass/fail flips distinct,
587+
consumes only fixed retry and terminal runner classifications, and leaves
588+
missing or malformed evidence unclassified. The table never changes a
589+
required check, release conclusion, or rerun decision.
584590
- Automated issue routing and the workflow's `issues: write` capability are
585591
retired. Any future issue escalation should use a separately reviewed
586592
exceptional threshold, such as the same lane failing twice consecutively or

0 commit comments

Comments
 (0)