V48 Gate 3 (impl-only): Fix sandbox tsx path

Resolve tsx from pipeline-hosts workspace paths so
Pipeliner deposit runs no longer hang on bare
node --import tsx / runtime npm install. Write absolute
stdout/stderr/exit/telemetry markers immediately so the
host always gets exit-code or early telemetry.

Author: geraldarthurdavis

containers/images/pipeliner/Dockerfile

@@ -55,6 +55,13 @@ RUN mkdir -p /opt/bitcode/.proofs/pipeline-host /opt/bitcode/pipeline \
   && cp containers/images/pipeliner/dist/run-live-asset-pack-pipeline.mjs /opt/bitcode/.proofs/pipeline-host/ \
   && cp containers/images/pipeliner/dist/run-pipeline.mjs /opt/bitcode/pipeline/run-pipeline.mjs
 
+# tsx must be loadable for the live AssetPack runner (imports monorepo .ts).
+# pnpm keeps it under packages/pipeline-hosts; host plan resolves that path.
+# Assert at image build so a missing dep fails the image, not a stuck sandbox.
+RUN test -f packages/pipeline-hosts/node_modules/tsx/dist/loader.mjs \
+  || test -f node_modules/tsx/dist/loader.mjs \
+  || (echo "pipeliner image missing tsx loader (pipeline-hosts dep)" >&2 && exit 1)
+
 ENV BITCODE_MONOREPO_ROOT=/opt/bitcode
 ENV BITCODE_PIPELINE_IMAGE_ENTRY=/opt/bitcode/pipeline/run-pipeline.mjs
 WORKDIR /vercel/sandbox

packages/pipeline-hosts/src/__tests__/asset-pack-host-plan.test.ts

@@ -107,9 +107,18 @@ describe('asset-pack sandbox host plan', () => {
     ]);
     const run = plan.commands.find((c) => c.label === 'asset-pack-pipeline-run');
     const runScript = run?.args?.join(' ') ?? '';
-    // Image path runs sandbox-uploaded runner with tsx (loads monorepo .ts packages).
+    // Image path: workspace tsx loader (pnpm does not hoist to monorepo root),
+    // absolute log/exit markers, cwd sandbox root — no runtime npm install hang.
+    expect(run?.cwd).toBe('/vercel/sandbox');
+    expect(run?.detached).toBe(true);
+    expect(run?.exitCodePath).toBe('.proofs/pipeline-host/pipeline.exit-code');
     expect(runScript).toContain('run-live-asset-pack-pipeline.mjs');
-    expect(runScript).toContain('tsx');
+    expect(runScript).toContain('/opt/bitcode/packages/pipeline-hosts/node_modules/tsx/dist/loader.mjs');
+    expect(runScript).toContain('/vercel/sandbox/.proofs/pipeline-host/pipeline.exit-code');
+    expect(runScript).toContain('/vercel/sandbox/.proofs/pipeline-host/telemetry.jsonl');
+    expect(runScript).toContain('pipeline-shell-start');
+    expect(runScript).toContain('Refusing runtime npm install');
+    expect(runScript).not.toContain('npm install -g tsx');
     expect(runScript).toContain('/opt/bitcode');
   });
 

packages/pipeline-hosts/src/asset-pack-host-constants.ts

@@ -15,6 +15,17 @@ export const PIPELINE_STDOUT_PATH = `${HOST_RUN_DIRECTORY}/pipeline.stdout.log`;
 export const PIPELINE_STDERR_PATH = `${HOST_RUN_DIRECTORY}/pipeline.stderr.log`;
 export const PIPELINE_EXIT_CODE_PATH = `${HOST_RUN_DIRECTORY}/pipeline.exit-code`;
 export const SANDBOX_WORKING_DIRECTORY = '/vercel/sandbox' as const;
+/** Absolute host-run dir inside the sandbox microVM (Vercel default mount). */
+export const SANDBOX_HOST_RUN_DIRECTORY =
+  `${SANDBOX_WORKING_DIRECTORY}/${HOST_RUN_DIRECTORY}` as const;
+export const SANDBOX_PIPELINE_STDOUT_PATH =
+  `${SANDBOX_HOST_RUN_DIRECTORY}/pipeline.stdout.log` as const;
+export const SANDBOX_PIPELINE_STDERR_PATH =
+  `${SANDBOX_HOST_RUN_DIRECTORY}/pipeline.stderr.log` as const;
+export const SANDBOX_PIPELINE_EXIT_CODE_PATH =
+  `${SANDBOX_HOST_RUN_DIRECTORY}/pipeline.exit-code` as const;
+export const SANDBOX_TELEMETRY_PATH =
+  `${SANDBOX_HOST_RUN_DIRECTORY}/telemetry.jsonl` as const;
 export const DEFAULT_LONG_TIMEOUT_MS = 45 * 60 * 1000;
 export const SANDBOX_PNPM_VERSION = '10.33.0';
 
@@ -33,6 +44,22 @@ export const PIPELINER_IMAGE_DEFAULT = `${PIPELINER_VCR_REPOSITORY}:latest`;
 export const PIPELINE_IMAGE_ENTRY_DEFAULT = '/opt/bitcode/pipeline/run-pipeline.mjs';
 export const PIPELINE_IMAGE_MONOREPO_ROOT_DEFAULT = '/opt/bitcode';
 
+/**
+ * Absolute tsx ESM loader paths inside the Pipeliner monorepo install.
+ * pnpm does not hoist `tsx` to `/opt/bitcode/node_modules`, so bare
+ * `node --import tsx` fails with ERR_MODULE_NOT_FOUND from monorepo root.
+ * Prefer the pipeline-hosts workspace install (declared dependency).
+ */
+export const PIPELINE_IMAGE_TSX_LOADER_CANDIDATES = [
+  `${PIPELINE_IMAGE_MONOREPO_ROOT_DEFAULT}/packages/pipeline-hosts/node_modules/tsx/dist/loader.mjs`,
+  `${PIPELINE_IMAGE_MONOREPO_ROOT_DEFAULT}/node_modules/tsx/dist/loader.mjs`,
+  `${PIPELINE_IMAGE_MONOREPO_ROOT_DEFAULT}/node_modules/.pnpm/node_modules/tsx/dist/loader.mjs`,
+] as const;
+export const PIPELINE_IMAGE_TSX_CLI_CANDIDATES = [
+  `${PIPELINE_IMAGE_MONOREPO_ROOT_DEFAULT}/packages/pipeline-hosts/node_modules/tsx/dist/cli.mjs`,
+  `${PIPELINE_IMAGE_MONOREPO_ROOT_DEFAULT}/node_modules/tsx/dist/cli.mjs`,
+] as const;
+
 /** Env: when set, host plans use Sandbox.create({ image }) instead of runtime. */
 export const PIPELINE_SANDBOX_IMAGE_ENV = 'BITCODE_PIPELINE_SANDBOX_IMAGE';
 export const PIPELINE_IMAGE_ENTRY_ENV = 'BITCODE_PIPELINE_IMAGE_ENTRY';

packages/pipeline-hosts/src/asset-pack-host-plan.ts

@@ -32,10 +32,16 @@ import {
   PIPELINE_IMAGE_ENTRY_DEFAULT,
   PIPELINE_IMAGE_ENTRY_ENV,
   PIPELINE_IMAGE_MONOREPO_ROOT_DEFAULT,
+  PIPELINE_IMAGE_TSX_CLI_CANDIDATES,
+  PIPELINE_IMAGE_TSX_LOADER_CANDIDATES,
   PIPELINE_SANDBOX_IMAGE_ENV,
   PIPELINE_STDERR_PATH,
   PIPELINE_STDOUT_PATH,
+  SANDBOX_PIPELINE_EXIT_CODE_PATH,
+  SANDBOX_PIPELINE_STDERR_PATH,
+  SANDBOX_PIPELINE_STDOUT_PATH,
   SANDBOX_PNPM_VERSION,
+  SANDBOX_TELEMETRY_PATH,
   SANDBOX_WORKING_DIRECTORY,
   SOURCE_OVERLAY_PATCH_PATH,
   TELEMETRY_PATH,
@@ -293,33 +299,18 @@ function buildPipelineImageCommands(
 
   if (mode === 'asset_pack_pipeline') {
     // Prefer sandbox-uploaded runner (hot-fixed) over image-baked copy.
-    // tsx loads monorepo .ts package sources under BITCODE_MONOREPO_ROOT.
-    const sandboxLiveRunner = `${SANDBOX_WORKING_DIRECTORY}/${HOST_RUN_DIRECTORY}/run-live-asset-pack-pipeline.mjs`;
-    const monorepoRoot = PIPELINE_IMAGE_MONOREPO_ROOT_DEFAULT;
-    const pipelineRunScript = [
-      `cd ${shellQuote(monorepoRoot)}`,
-      // tsx loads monorepo .ts package sources (plain node cannot).
-      `if ! node --import tsx -e "process.exit(0)" >/dev/null 2>&1; then`,
-      `  npm install -g tsx@4.19.3 || npm install --no-save --prefix ${shellQuote(monorepoRoot)} tsx@4.19.3`,
-      `fi`,
-      `RUNNER=${shellQuote(sandboxLiveRunner)}`,
-      `if [ ! -f "$RUNNER" ]; then RUNNER=${shellQuote(`${monorepoRoot}/.proofs/pipeline-host/run-live-asset-pack-pipeline.mjs`)}; fi`,
-      `if [ ! -f "$RUNNER" ]; then echo "live runner missing; falling back to image dispatcher" >&2; node ${shellQuote(pipelineImageEntry)}; else node --import tsx "$RUNNER"; fi`,
-    ].join(' && ');
+    // Live runner imports monorepo .ts packages — needs tsx, resolved from the
+    // image workspace (pnpm does not hoist to monorepo root). Never hang on
+    // runtime npm install; fail closed with logs + exit-code for host poll.
+    const pipelineRunScript = buildPipelineImageRunShellScript(pipelineImageEntry);
     commands.push({
       label: 'asset-pack-pipeline-run',
       cmd: 'sh',
-      args: [
-        '-lc',
-        [
-          `( ${pipelineRunScript} ) > ${shellQuote(PIPELINE_STDOUT_PATH)} 2> ${shellQuote(PIPELINE_STDERR_PATH)}`,
-          'code=$?',
-          `printf "%s" "$code" > ${shellQuote(PIPELINE_EXIT_CODE_PATH)}`,
-          'exit "$code"',
-        ].join('; '),
-      ],
+      args: ['-lc', pipelineRunScript],
+      cwd: SANDBOX_WORKING_DIRECTORY,
       env: commandEnvironment,
       detached: true,
+      // Relative paths: Sandbox readFile is rooted at /vercel/sandbox.
       exitCodePath: PIPELINE_EXIT_CODE_PATH,
       stdoutPath: PIPELINE_STDOUT_PATH,
       stderrPath: PIPELINE_STDERR_PATH,
@@ -456,6 +447,7 @@ function buildCommands(
           'exit "$code"',
         ].join('; '),
       ],
+      cwd: SANDBOX_WORKING_DIRECTORY,
       env: commandEnvironment,
       detached: true,
       exitCodePath: PIPELINE_EXIT_CODE_PATH,
@@ -480,6 +472,96 @@ function buildCommands(
   return commands;
 }
 
+/**
+ * Detached in-box shell for Pipeliner image mode.
+ *
+ * Guarantees (for host poll + operator debug):
+ * - Absolute stdout/stderr/exit-code under /vercel/sandbox/.proofs/pipeline-host
+ * - Early telemetry.jsonl line so UI is not stuck on bare command-started
+ * - tsx resolved from workspace install paths (no runtime npm install hang)
+ * - Always writes pipeline.exit-code so waitForDetachedCommand terminates
+ */
+export function buildPipelineImageRunShellScript(pipelineImageEntry: string): string {
+  const monorepoRoot = PIPELINE_IMAGE_MONOREPO_ROOT_DEFAULT;
+  const sandboxLiveRunner = `${SANDBOX_WORKING_DIRECTORY}/${HOST_RUN_DIRECTORY}/run-live-asset-pack-pipeline.mjs`;
+  const imageLiveRunner = `${monorepoRoot}/.proofs/pipeline-host/run-live-asset-pack-pipeline.mjs`;
+  const stdoutPath = SANDBOX_PIPELINE_STDOUT_PATH;
+  const stderrPath = SANDBOX_PIPELINE_STDERR_PATH;
+  const exitCodePath = SANDBOX_PIPELINE_EXIT_CODE_PATH;
+  const telemetryPath = SANDBOX_TELEMETRY_PATH;
+
+  const loaderCandidates = PIPELINE_IMAGE_TSX_LOADER_CANDIDATES.map(
+    (p) => shellQuote(p),
+  ).join(' ');
+  const cliCandidates = PIPELINE_IMAGE_TSX_CLI_CANDIDATES.map((p) => shellQuote(p)).join(' ');
+
+  // Single shell program: markers first, then resolve tsx, then run.
+  // Redirects use absolute paths so cwd surprises cannot hide artifacts.
+  return [
+    'set +e',
+    `HOST_DIR=${shellQuote(SANDBOX_WORKING_DIRECTORY + '/' + HOST_RUN_DIRECTORY)}`,
+    'mkdir -p "$HOST_DIR"',
+    `STDOUT=${shellQuote(stdoutPath)}`,
+    `STDERR=${shellQuote(stderrPath)}`,
+    `EXITF=${shellQuote(exitCodePath)}`,
+    `TELEM=${shellQuote(telemetryPath)}`,
+    // Truncate / create markers immediately (host + interactive debug).
+    ': > "$STDOUT"',
+    ': > "$STDERR"',
+    'TS=$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)',
+    'echo "pipeline-shell: start ts=$TS monorepo=' +
+      monorepoRoot +
+      ' node=$(command -v node 2>/dev/null)" >> "$STDERR"',
+    'printf "%s\\n" "{\\"type\\":\\"pipeline-shell-start\\",\\"timestamp\\":\\"$TS\\",\\"cwd\\":\\"$(pwd)\\"}" >> "$TELEM"',
+    '(',
+    `  cd ${shellQuote(monorepoRoot)} || { echo "pipeline-shell: cd monorepo failed" >&2; exit 127; }`,
+    '  TSX_LOADER=""',
+    `  for c in ${loaderCandidates}; do`,
+    '    if [ -f "$c" ]; then TSX_LOADER=$c; break; fi',
+    '  done',
+    '  TSX_CLI=""',
+    `  for c in ${cliCandidates}; do`,
+    '    if [ -f "$c" ]; then TSX_CLI=$c; break; fi',
+    '  done',
+    '  if [ -z "$TSX_LOADER" ] && [ -z "$TSX_CLI" ]; then',
+    '    if node --import tsx -e "process.exit(0)" >/dev/null 2>&1; then',
+    '      TSX_MODE=package',
+    '    else',
+    '      echo "pipeline-shell: tsx not found under monorepo (loader/cli/package). Refusing runtime npm install." >&2',
+    '      echo "pipeline-shell: candidates checked; rebuild Pipeliner with pipeline-hosts tsx dep." >&2',
+    '      exit 127',
+    '    fi',
+    '  else',
+    '    TSX_MODE=path',
+    '  fi',
+    `  RUNNER=${shellQuote(sandboxLiveRunner)}`,
+    `  if [ ! -f "$RUNNER" ]; then RUNNER=${shellQuote(imageLiveRunner)}; fi`,
+    '  echo "pipeline-shell: tsx_mode=$TSX_MODE loader=${TSX_LOADER:-none} cli=${TSX_CLI:-none} runner=$RUNNER" >&2',
+    `  if [ ! -f "$RUNNER" ]; then`,
+    `    echo "pipeline-shell: live runner missing; falling back to image dispatcher" >&2`,
+    `    node ${shellQuote(pipelineImageEntry)}`,
+    '    exit $?',
+    '  fi',
+    '  if [ -n "$TSX_LOADER" ]; then',
+    '    node --import "$TSX_LOADER" "$RUNNER"',
+    '    exit $?',
+    '  fi',
+    '  if [ -n "$TSX_CLI" ]; then',
+    '    node "$TSX_CLI" "$RUNNER"',
+    '    exit $?',
+    '  fi',
+    '  node --import tsx "$RUNNER"',
+    '  exit $?',
+    ') > "$STDOUT" 2>> "$STDERR"',
+    'code=$?',
+    'TS2=$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)',
+    'echo "pipeline-shell: finish ts=$TS2 exit=$code" >> "$STDERR"',
+    'printf "%s\\n" "{\\"type\\":\\"pipeline-shell-finish\\",\\"timestamp\\":\\"$TS2\\",\\"exitCode\\":$code}" >> "$TELEM"',
+    'printf "%s" "$code" > "$EXITF"',
+    'exit "$code"',
+  ].join('\n');
+}
+
 function shellQuote(value: string): string {
   return `'${value.replace(/'/g, `'\\''`)}'`;
 }