Skip to content

Commit 363a8a3

Browse files
SirTenzinSirTenzin
committed
fix(tw): stale warm checkout — origin-first resolution + sha assertions
Run mroteobc-tsc05k silently executed 72cd1d2 while its warm image was named tw-warm-ea5797a367aa: the modal fast-forward correctly checked out origin's tip (detached FETCH_HEAD), then warmup.sh's ref-ensure step resolved refs/heads/$REF FIRST — the stale local branch baked into the tw-warm:latest base image — and force-checked that out, reverting the correct code one step after it landed. The wrong tree was then snapshotted and published under the sha tag AND :latest, poisoning the cross-run cache. Three layers of fix: - warmup.sh: skip checkout when HEAD already matches the expected commit (don't undo the provider's fast-forward), otherwise resolve origin/$REF BEFORE the local branch; hard-fail (exit 1 -> run aborts, no workers forked) if the final HEAD doesn't match the expected sha when provided - run.ts / refreshWarm.ts: pass the resolved sha into warmup.sh so the assertion is always armed - modal.ts: verify a published warm image's baked repo HEAD against its sha tag before trusting a cache HIT; mismatch narrates "warm cache POISONED" and falls through to a rebuild, which republishes correct code over the bad tags (self-heals the existing poisoned tw-warm-ea5797a367aa without manual cleanup)
1 parent ea5797a commit 363a8a3

4 files changed

Lines changed: 94 additions & 17 deletions

File tree

‎scripts/tw/commands/refreshWarm.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -206,7 +206,7 @@ const refreshModalWarm = async ({
206206
const warmup = await timed(
207207
"warmup.sh (checkout → install → migrate → seed)",
208208
() =>
209-
runStreaming(warm, ["bash", WARMUP_SCRIPT, sha], (text) =>
209+
runStreaming(warm, ["bash", WARMUP_SCRIPT, sha, sha], (text) =>
210210
process.stdout.write(text),
211211
),
212212
);
@@ -291,7 +291,7 @@ export const refreshWarm = async (args: string[]): Promise<number> => {
291291
const warmup = await timed(
292292
"warmup.sh (checkout → install → migrate → seed)",
293293
() =>
294-
runStreaming(warm, ["bash", WARMUP_SCRIPT, sha], (text) =>
294+
runStreaming(warm, ["bash", WARMUP_SCRIPT, sha, sha], (text) =>
295295
process.stdout.write(text),
296296
),
297297
);

‎scripts/tw/commands/run.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -846,7 +846,9 @@ const getOrBuildWarmParent = async ({
846846
);
847847
const warmRun = await runStreaming(
848848
warm,
849-
["bash", WARMUP_SCRIPT, ref],
849+
// Pass the resolved sha so warmup.sh hard-fails instead of silently
850+
// building the warm on a stale checkout (the image is NAMED after it).
851+
["bash", WARMUP_SCRIPT, ref, sha],
850852
(text) => sink(text),
851853
{ signal, swallowStreamClose: true },
852854
);

‎scripts/tw/helpers/modal.ts‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,54 @@ const lookupPublishedWarmImage = async (
226226
}
227227
};
228228

229+
/**
230+
* A published warm image is only trustworthy if its baked repo HEAD matches the
231+
* sha it's named after — a warm built from a stale checkout otherwise gets
232+
* reused silently forever. Errors count as verified (don't block on infra).
233+
*/
234+
const verifyWarmImageHead = async (
235+
image: Image,
236+
sha12: string,
237+
v2: boolean,
238+
): Promise<boolean> => {
239+
const verifyName = `${WARM_SANDBOX_PREFIX}-${sha12}-verify`;
240+
let sandbox: Sandbox | undefined;
241+
try {
242+
sandbox = await createFromImage(
243+
image,
244+
{
245+
name: verifyName,
246+
env: {},
247+
tags: { kind: "bun-tw-warm-verify", sha: sha12 },
248+
cpu: WORKER_CPU,
249+
memoryMiB: WORKER_MEMORY_MIB,
250+
timeout: 120_000,
251+
},
252+
v2,
253+
);
254+
const proc = await sandbox.exec(
255+
["bash", "-lc", `git -C ${MODAL_REPO_ROOT} rev-parse HEAD`],
256+
{ stdout: "pipe", stderr: "pipe", workdir: "/" },
257+
);
258+
let out = "";
259+
await pumpStream(proc.stdout, (text) => {
260+
out += text;
261+
});
262+
await proc.wait();
263+
return out.trim().startsWith(sha12);
264+
} catch {
265+
return true;
266+
} finally {
267+
if (sandbox) {
268+
liveSandboxes.delete(verifyName);
269+
liveSandboxes.delete(sandbox.sandboxId);
270+
await sandbox.terminate().catch(() => {
271+
/* best-effort */
272+
});
273+
}
274+
}
275+
};
276+
229277
/** Publish the warm image under its sha tag AND as the rolling `:latest`. */
230278
const publishWarmImage = async (image: Image, sha12: string): Promise<void> => {
231279
try {
@@ -696,6 +744,14 @@ const makeModalProvider = (v2: boolean): ProviderImpl => {
696744
if (sha12) {
697745
const exact = await lookupPublishedWarmImage(sha12);
698746
if (exact) {
747+
if (!(await verifyWarmImageHead(exact, sha12, v2))) {
748+
narrate(
749+
chalk.yellow(
750+
`[modal] warm cache POISONED (${WARM_IMAGE_REPO}:${sha12} repo HEAD mismatch) — rebuilding warm`,
751+
),
752+
);
753+
return undefined;
754+
}
699755
warmImageByName.set(name, exact);
700756
narrate(
701757
chalk.magenta(

‎scripts/tw/image/warmup.sh‎

Lines changed: 33 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@ BIN_DIR="${TW_BIN_DIR:-$TW_PREFIX/bin}"
3232
# The ref under test. Defaults to current HEAD when not passed (plan §8.6:
3333
# --ref defaults to HEAD). First positional arg or TW_REF env.
3434
REF="${1:-${TW_REF:-HEAD}}"
35+
# The commit the caller resolved REF to (the warm image is named after it).
36+
# When set, checkout MUST land on it — a mismatch aborts the whole warm build.
37+
EXPECTED_HEAD="${2:-${TW_EXPECTED_HEAD:-}}"
3538

3639
export PATH="$HOME/.bun/bin:$BIN_DIR:$PATH"
3740
command -v bun >/dev/null 2>&1 || die "bun not on PATH (run build-base.sh)"
@@ -68,23 +71,39 @@ cd "$REPO_ROOT"
6871
# ---------------------------------------------------------------------------
6972
# 1. git checkout <ref>
7073
# ---------------------------------------------------------------------------
71-
log "Ensuring working tree is at ref: $REF"
72-
git fetch --quiet --all --tags 2>/dev/null \
73-
|| log "WARN: git fetch failed (offline / shallow clone) — using what's checked out"
74-
# Vercel clones the `revision` in a DETACHED state with no local branch, so a
75-
# plain `git checkout <branch>` fails ("pathspec did not match"). Resolve the ref
76-
# however it exists; if it's not a local branch/tag/remote ref, the clone is
77-
# already AT it (Vercel checked out the revision at create) — just proceed.
78-
if git rev-parse --verify --quiet "refs/heads/$REF" >/dev/null 2>&1; then
79-
git checkout --quiet --force "$REF"
80-
elif git rev-parse --verify --quiet "refs/remotes/origin/$REF" >/dev/null 2>&1; then
81-
git checkout --quiet --force -B "$REF" "origin/$REF"
82-
elif git rev-parse --verify --quiet "$REF" >/dev/null 2>&1; then
83-
git checkout --quiet --force "$REF"
74+
log "Ensuring working tree is at ref: $REF${EXPECTED_HEAD:+ (expect $EXPECTED_HEAD)}"
75+
if [ -n "$EXPECTED_HEAD" ] \
76+
&& [ "$(git rev-parse HEAD | cut -c1-${#EXPECTED_HEAD})" = "$EXPECTED_HEAD" ]; then
77+
# The provider's fast-forward checkout already put us on the exact commit;
78+
# re-resolving the branch here could revert onto a stale local ref.
79+
log "working tree already at expected commit — skipping checkout"
8480
else
85-
log "ref '$REF' is not a local branch/tag — assuming the clone is already at it"
81+
git fetch --quiet --all --tags \
82+
|| log "WARN: git fetch failed (offline / shallow clone) — using what's checked out"
83+
# Prefer origin's tip: fast-forward base images carry a STALE local branch,
84+
# so refs/heads/$REF can be behind origin/$REF and must not win.
85+
# Vercel clones the `revision` in a DETACHED state with no local branch, so a
86+
# plain `git checkout <branch>` fails ("pathspec did not match"). Resolve the ref
87+
# however it exists; if it's not a branch/tag/remote ref, the clone is
88+
# already AT it (Vercel checked out the revision at create) — just proceed.
89+
if git rev-parse --verify --quiet "refs/remotes/origin/$REF" >/dev/null 2>&1; then
90+
git checkout --quiet --force -B "$REF" "origin/$REF"
91+
elif git rev-parse --verify --quiet "refs/heads/$REF" >/dev/null 2>&1; then
92+
git checkout --quiet --force "$REF"
93+
elif git rev-parse --verify --quiet "$REF" >/dev/null 2>&1; then
94+
git checkout --quiet --force "$REF"
95+
else
96+
log "ref '$REF' is not a local branch/tag — assuming the clone is already at it"
97+
fi
8698
fi
8799
log "HEAD at $(git rev-parse --short HEAD)"
100+
if [ -n "$EXPECTED_HEAD" ]; then
101+
ACTUAL_HEAD="$(git rev-parse HEAD)"
102+
case "$ACTUAL_HEAD" in
103+
"$EXPECTED_HEAD"*) ;;
104+
*) die "HEAD $ACTUAL_HEAD != expected $EXPECTED_HEAD — refusing to build a stale warm" ;;
105+
esac
106+
fi
88107

89108
# ---------------------------------------------------------------------------
90109
# 2. bun install --frozen-lockfile (delta only — deps baked into base)

0 commit comments

Comments
 (0)