ensure-project-worktree: refuse a silently stale tree when the branch fetch fails

The branch fetch swallowed every failure (2>/dev/null || true), so a
transient network/auth blip silently left refs/heads/$branch at a stale
local SHA and the gardener worked an old tree with no warning. Observed
2026-07-06: endo-but-for-bots@llm delivered 8 weeks stale at 68246ad9
(missing the docs the job named) while origin/llm was at 11322892.

After the fetch, verify the local head against the authoritative remote
tip (git ls-remote), retry once on divergence, then die rather than hand
back a stale tree. The remote lookup gets one retry past a blip too. The
legitimate no-op case (a detached ref/sha on neither side) is preserved.

Adds a hermetic regression to project-worktree-isolation-test: advance
the upstream tip but delete its objects so ls-remote still advertises it
while the fetch fails, and assert the helper refuses.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: endolinbot

scripts/jobs/ensure-project-worktree.sh

@@ -114,8 +114,49 @@ mkdir -p "$GARDEN_SCRATCH"
 # <branch>`; fetching it explicitly into refs/heads/ makes the name resolve. The
 # `+` forces an update if the local head already exists. If the branch is on
 # neither side the fetch is a no-op and the add below surfaces a clear error.
-git --git-dir="$bare" fetch --quiet origin \
-  "+refs/heads/${branch}:refs/heads/${branch}" 2>/dev/null || true
+#
+# ── Silent stale-fetch guard (the 2026-07-06 regression) ─────────────────────
+# A bare `2>/dev/null || true` on the fetch swallows EVERY failure, so a transient
+# network/auth blip silently leaves refs/heads/$branch at a STALE local SHA and
+# the gardener works an old tree with no warning. Observed 2026-07-06 (job
+# design-daemon-agent-tools-reconcile-mount-git-capabilities): endo-but-for-bots@llm
+# was delivered 8 weeks stale at 68246ad9 — missing the very docs the job named —
+# while origin/llm was at 11322892, and a manual `git fetch origin llm` succeeded
+# moments later. So we verify the local head against the AUTHORITATIVE remote tip
+# (`git ls-remote`) after the fetch, retry once on any divergence, and die rather
+# than hand back a stale tree. The remote lookup itself gets one retry so a blip
+# there does not defeat the guard.
+remote_branch_sha() {
+  git --git-dir="$bare" ls-remote origin "refs/heads/${branch}" 2>/dev/null | cut -f1
+}
+local_branch_sha() {
+  git --git-dir="$bare" rev-parse --verify --quiet "refs/heads/${branch}" 2>/dev/null || true
+}
+fetch_branch() {
+  git --git-dir="$bare" fetch --quiet origin \
+    "+refs/heads/${branch}:refs/heads/${branch}" 2>/dev/null
+}
+
+remote_sha="$(remote_branch_sha)"
+[ -z "$remote_sha" ] && remote_sha="$(remote_branch_sha)"   # one retry past a blip
+
+fetch_branch || true
+
+if [ -n "$remote_sha" ]; then
+  # The branch exists upstream: the local head MUST equal the remote tip, else the
+  # fetch silently failed or delivered a stale ref. Retry once, then refuse.
+  have_sha="$(local_branch_sha)"
+  if [ "$have_sha" != "$remote_sha" ]; then
+    log "WARN: ensure-project-worktree: refs/heads/${branch} is ${have_sha:-<absent>} but origin/${branch} is ${remote_sha}; retrying fetch"
+    fetch_branch || true
+  fi
+  now_sha="$(local_branch_sha)"
+  if [ "$now_sha" != "$remote_sha" ]; then
+    die "ensure-project-worktree: could not fetch ${repo}@${branch} to ${remote_sha} (local head is ${now_sha:-absent}, likely a transient network/auth failure); refusing to hand back a stale tree"
+  fi
+fi
+# else: the branch is on neither side (a legitimate detached ref/sha checkout) —
+# the fetch is a no-op and the add below resolves $ref locally or errors clearly.
 
 git --git-dir="$bare" worktree add --detach "$wt" "$ref" >/dev/null \
   || die "ensure-project-worktree: could not check out $repo@$ref into $wt"

scripts/jobs/test/project-worktree-isolation-test.sh

@@ -128,6 +128,38 @@ P3="$(run_helper garden-fix-error-trace endojs/endo main)"
 [ "$P3" != "$P1" ] && ok "same base, different repo → DISTINCT path (no self-collision)" \
   || bad "same base collided across repos ('$P3' == '$P1')"
 
+# === 6: silent stale-fetch guard — remote advances but the fetch can't deliver =
+# Regression for the 2026-07-06 stale-tree delivery: when origin advertises a NEW
+# tip (ls-remote succeeds) but the fetch of it fails (a transient blip), the
+# helper must REFUSE rather than silently hand back the stale local ref.
+make_fork endojs stale-guard main
+UP="$TR/upstream-endojs-stale-guard.git"
+SEED="$TR/seed-endojs-stale-guard"
+SG_BARE="$GROOT/worktrees/endojs-stale-guard.git"
+git -C "$UP" config gc.auto 0 >/dev/null 2>&1; git -C "$UP" config receive.autogc false >/dev/null 2>&1
+# establish a fresh baseline (bare refs/heads/main = T1) via one good run
+run_helper garden-stale-baseline endojs/stale-guard main >/dev/null
+# advance upstream to T2, then delete T2's objects so ls-remote still advertises
+# the new tip but a fetch of it fails — the exact transient shape we must catch.
+find "$UP/objects" -type f | sort > "$TR/obj-before"
+( cd "$SEED"
+  printf 'upstream T2 — must-not-be-skipped\n' > error-trace.js
+  git "${git_id[@]}" commit -qam T2
+  git push -q origin main ) >/dev/null 2>&1
+find "$UP/objects" -type f | sort > "$TR/obj-after"
+comm -13 "$TR/obj-before" "$TR/obj-after" | while read -r f; do rm -f "$f"; done
+# sanity: ls-remote still advertises the (now unfetchable) T2 tip
+adv="$(git --git-dir="$SG_BARE" ls-remote origin refs/heads/main 2>/dev/null | cut -f1)"
+seed_t2="$(git -C "$SEED" rev-parse HEAD)"
+[ -n "$adv" ] && [ "$adv" = "$seed_t2" ] && ok "ls-remote advertises the advanced (unfetchable) tip" \
+  || bad "test setup: ls-remote did not advertise T2 (adv='$adv' t2='$seed_t2')"
+# the guard must make the helper DIE, not emit a stale-tree path
+if out="$(run_helper garden-stale-victim endojs/stale-guard main 2>/dev/null)"; then
+  bad "helper handed back a tree despite an undeliverable remote tip (stale!): '$out'"
+else
+  ok "helper REFUSES (dies) rather than delivering a stale tree"
+fi
+
 # --- summary -----------------------------------------------------------------
 echo "----------------------------------------------------------------"
 echo "project-worktree-isolation: $PASS passed, $FAIL failed"