fix(server): fail closed on Postgres handoff errors (FAI-8942)

Co-Authored-By: Paperclip <noreply@paperclip.ing>

Author: Andrei Duduman

doc/DEVELOPING.md

@@ -119,6 +119,8 @@ Use `--drain-required` only when the deploy intentionally requires the old termi
 
 When the server owns embedded PostgreSQL, a validated hot restart leaves that database process running and opts out of the dependency's package-global signal hook. The predecessor issues a short-lived, one-time handoff bound to the validated restart intent and the PostgreSQL PID, start time, canonical data directory, and port. Exactly one replacement process can claim that handoff after the predecessor exits; an unrelated or mismatched server may reuse the live database but does not acquire authority to stop it. Normal shutdowns stop either the originally started or validly adopted embedded database.
 
+Handoff persistence fails closed. If the handoff cannot be written or the PostgreSQL identity changes before it is written, the predecessor stops its owned database instead of exiting and leaving it ownerless. If that stop also fails, the predecessor aborts its process exit so it retains ownership for operator recovery.
+
 A healthy guarded deploy must compare the report against `/api/health` (`version` or `serverVersion`) and treat any `lostRunIds` entry as a continuity failure that needs recovery before marking deployment complete.
 
 Tailscale/private-auth dev mode:

server/src/__tests__/heartbeat-process-recovery.test.ts

@@ -1456,6 +1456,9 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
         mode: "hot_restart",
         skipDrain: true,
         activeRunIds: [runId],
+        previousServerPid: process.pid,
+        requestedAt: "2026-03-19T00:05:00.000Z",
+        shutdownSnapshotCapturedAt: "2026-03-19T00:06:00.000Z",
       });
       expect(isPidAlive(child.pid)).toBe(true);
       const run = await db

server/src/index.ts

@@ -1414,49 +1414,63 @@ export async function startServer(): Promise<StartedServer> {
 
       if (stopOwnedEmbeddedPostgres) {
         try {
-          if (
+          const hotRestart = heartbeatShutdown.hotRestart;
+          const hotRestartHandoffContext = (
             shutdownLifecycle.preserveEmbeddedPostgres
-            && heartbeatShutdown.hotRestart?.previousServerPid === process.pid
-            && heartbeatShutdown.hotRestart.requestedAt
-            && heartbeatShutdown.hotRestart.shutdownSnapshotCapturedAt
-          ) {
-            const currentIdentity = await readEmbeddedPostgresProcessIdentity(
-              ownedEmbeddedPostgresIdentity?.dataDir ?? config.embeddedPostgresDataDir,
-            );
-            if (
-              currentIdentity
-              && ownedEmbeddedPostgresIdentity
-              && currentIdentity.pid === ownedEmbeddedPostgresIdentity.pid
-              && currentIdentity.startedAtEpochSeconds === ownedEmbeddedPostgresIdentity.startedAtEpochSeconds
-              && currentIdentity.port === ownedEmbeddedPostgresIdentity.port
-              && resolve(currentIdentity.dataDir) === resolve(ownedEmbeddedPostgresIdentity.dataDir)
-            ) {
-              const handoff = await writeEmbeddedPostgresHandoff({
-                hotRestartRequestedAt: heartbeatShutdown.hotRestart.requestedAt,
-                shutdownSnapshotCapturedAt: heartbeatShutdown.hotRestart.shutdownSnapshotCapturedAt,
-                predecessorServerPid: process.pid,
-                predecessorServerStartedAtEpochMs: serverStartedAtEpochMs,
-                postgres: currentIdentity,
-              });
-              logger.info({
-                postgresPid: currentIdentity.pid,
-                port: currentIdentity.port,
-                transferToken: handoff.transferToken,
-                expiresAt: handoff.expiresAt,
-              }, "Embedded PostgreSQL hot-restart ownership handoff issued");
-            } else {
-              logger.warn(
-                { expectedPostgresPid: ownedEmbeddedPostgresIdentity?.pid ?? null },
-                "Embedded PostgreSQL identity changed; preserving without issuing stop authority",
-              );
-            }
-          }
+            && hotRestart?.previousServerPid === process.pid
+            && hotRestart.requestedAt
+            && hotRestart.shutdownSnapshotCapturedAt
+          )
+            ? {
+                requestedAt: hotRestart.requestedAt,
+                shutdownSnapshotCapturedAt: hotRestart.shutdownSnapshotCapturedAt,
+              }
+            : null;
+          const persistedHandoff: {
+            value: Awaited<ReturnType<typeof writeEmbeddedPostgresHandoff>> | null;
+          } = { value: null };
           const postgresShutdown = await coordinateEmbeddedPostgresShutdown({
             ownedByThisProcess: true,
             stop: stopOwnedEmbeddedPostgres,
             lifecycle: shutdownLifecycle,
+            persistHotRestartHandoff: hotRestartHandoffContext
+              ? async () => {
+                  const currentIdentity = await readEmbeddedPostgresProcessIdentity(
+                    ownedEmbeddedPostgresIdentity?.dataDir ?? config.embeddedPostgresDataDir,
+                  );
+                  if (
+                    !currentIdentity
+                    || !ownedEmbeddedPostgresIdentity
+                    || currentIdentity.pid !== ownedEmbeddedPostgresIdentity.pid
+                    || currentIdentity.startedAtEpochSeconds !== ownedEmbeddedPostgresIdentity.startedAtEpochSeconds
+                    || currentIdentity.port !== ownedEmbeddedPostgresIdentity.port
+                    || resolve(currentIdentity.dataDir) !== resolve(ownedEmbeddedPostgresIdentity.dataDir)
+                  ) {
+                    throw new Error("Embedded PostgreSQL identity changed before hot-restart handoff persistence");
+                  }
+                  persistedHandoff.value = await writeEmbeddedPostgresHandoff({
+                    hotRestartRequestedAt: hotRestartHandoffContext.requestedAt,
+                    shutdownSnapshotCapturedAt: hotRestartHandoffContext.shutdownSnapshotCapturedAt,
+                    predecessorServerPid: process.pid,
+                    predecessorServerStartedAtEpochMs: serverStartedAtEpochMs,
+                    postgres: currentIdentity,
+                  });
+                }
+              : undefined,
+            onHotRestartHandoffFailure: (err) => {
+              logger.error(
+                { err, expectedPostgresPid: ownedEmbeddedPostgresIdentity?.pid ?? null },
+                "Embedded PostgreSQL hot-restart handoff failed; stopping owned database",
+              );
+            },
           });
           if (postgresShutdown === "preserved_for_hot_restart") {
+            logger.info({
+              postgresPid: persistedHandoff.value?.postgres.pid,
+              port: persistedHandoff.value?.postgres.port,
+              transferToken: persistedHandoff.value?.transferToken,
+              expiresAt: persistedHandoff.value?.expiresAt,
+            }, "Embedded PostgreSQL hot-restart ownership handoff issued");
             logger.info(
               { shutdownLifecycle },
               "Preserving embedded PostgreSQL for hot-restart adoption",
@@ -1466,6 +1480,13 @@ export async function startServer(): Promise<StartedServer> {
           }
         } catch (err) {
           logger.error({ err, shutdownLifecycle }, "Failed to stop embedded PostgreSQL cleanly");
+          if (shutdownLifecycle.preserveEmbeddedPostgres) {
+            logger.error(
+              { shutdownLifecycle },
+              "Aborting hot-restart process exit to retain embedded PostgreSQL ownership",
+            );
+            return;
+          }
         }
       }
 

server/src/shutdown.test.ts

@@ -90,6 +90,7 @@ describe("coordinateEmbeddedPostgresShutdown", () => {
       ownedByThisProcess: true,
       stop,
       lifecycle,
+      persistHotRestartHandoff: vi.fn(async () => undefined),
     });
 
     expect(result).toBe("preserved_for_hot_restart");
@@ -171,6 +172,29 @@ describe("coordinateEmbeddedPostgresShutdown", () => {
     });
   });
 
+  it("stops the owned database when hot-restart handoff persistence fails", async () => {
+    const writeError = new Error("forced handoff write failure");
+    const stop = vi.fn(async () => undefined);
+    const onHotRestartHandoffFailure = vi.fn();
+
+    const result = await coordinateEmbeddedPostgresShutdown({
+      ownedByThisProcess: true,
+      stop,
+      lifecycle: createShutdownLifecycleContext({
+        signal: "SIGTERM",
+        hotRestart: { skipDrain: true },
+      }),
+      persistHotRestartHandoff: vi.fn(async () => {
+        throw writeError;
+      }),
+      onHotRestartHandoffFailure,
+    });
+
+    expect(result).toBe("stopped");
+    expect(stop).toHaveBeenCalledOnce();
+    expect(onHotRestartHandoffFailure).toHaveBeenCalledWith(writeError);
+  });
+
   it("reads the PostgreSQL PID, start time, canonical data directory, and port as one identity", async () => {
     const homeDir = await fs.mkdtemp(resolve(os.tmpdir(), "paperclip-postgres-identity-"));
     const dataDir = resolve(homeDir, "postgres");
@@ -251,6 +275,7 @@ describe("coordinateEmbeddedPostgresShutdown", () => {
         signal: "SIGTERM",
         hotRestart: { skipDrain: true },
       }),
+      persistHotRestartHandoff: vi.fn(async () => undefined),
     });
 
     const stopAdoptedDatabase = adoptEmbeddedPostgres(replacement, {

server/src/shutdown.ts

@@ -46,9 +46,22 @@ export async function coordinateEmbeddedPostgresShutdown(input: {
   ownedByThisProcess: boolean;
   stop: (() => Promise<void>) | null;
   lifecycle: ShutdownLifecycleContext;
+  persistHotRestartHandoff?: () => Promise<void>;
+  onHotRestartHandoffFailure?: (error: unknown) => void;
 }): Promise<"not_owned" | "preserved_for_hot_restart" | "stopped"> {
   if (!input.ownedByThisProcess || !input.stop) return "not_owned";
-  if (input.lifecycle.preserveEmbeddedPostgres) return "preserved_for_hot_restart";
+  if (input.lifecycle.preserveEmbeddedPostgres && input.persistHotRestartHandoff) {
+    try {
+      await input.persistHotRestartHandoff();
+      return "preserved_for_hot_restart";
+    } catch (error) {
+      try {
+        input.onHotRestartHandoffFailure?.(error);
+      } catch {
+        // Diagnostic callbacks must never bypass the failure-closed stop below.
+      }
+    }
+  }
   await input.stop();
   return "stopped";
 }