fix(ingest): clear-then-derive friction/diagnostic events on full re-derive (#549) (#602)

Derived event rows are written via stable-key UPSERT, so when the
current derivation logic stops emitting a given row (e.g. a classifier
change like deriveCorrectionEdges no longer flagging Codex image-only
messages), the old row is never re-touched - it orphans with a stale
ts (the friction_event 1970-epoch residue). A full re-derive could
neither overwrite nor remove it.

On a FULL re-derive (no --since window) every friction/diagnostic row is
re-emitted from a complete scan, so cmdSignals now clears those two
standalone derived tables before re-inserting - orphans vanish. A bare
DELETE (no WHERE) sidesteps the DELETE-WHERE-on-indexed-field ghost-row
footgun (PR #141). A --since run keeps the UPSERT-only path: clearing
there would drop rows whose source falls outside the window. The gate is
the existing full-derive predicate (shouldDeriveAllTimeSkillPairs), the
same one that guards all-time skill-pair writes.

Scoped to the two tables named in the issue (friction_event,
diagnostic_event); other UPSERT-keyed derived edges can get the same
treatment as a follow-up.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: Necmttn

apps/axctl/src/ingest/derive-signals.ts

@@ -5,6 +5,7 @@ import { AppLayer } from "@ax/lib/layers";
 import type { DbError } from "@ax/lib/errors";
 import { executeStatementsWith } from "@ax/lib/shared/statement-exec";
 import {
+    buildClearDiagnosticEventStatement, buildClearFrictionEventStatement,
     buildCorrectedByStatements, buildDiagnosticEventStatements,
     buildFrictionEventStatements, buildProposedStatements,
     buildRecoveredStatements, buildSkillPairStatements,
@@ -248,6 +249,17 @@ export const deriveSignals = Effect.fn("derive.signals")(
                 attributes: { "signals.count": recoveryBatch.length },
             }),
         );
+        // On a FULL re-derive, clear the standalone derived-event tables before
+        // re-inserting so a row the current logic no longer emits cannot orphan
+        // with a stale ts (#549). `shouldWriteSkillPairs` is exactly the
+        // full-derive gate (sinceDays undefined / <=0); a --since run keeps the
+        // UPSERT-only path so it never drops rows whose source is out of window.
+        if (shouldWriteSkillPairs) {
+            yield* exec([
+                buildClearFrictionEventStatement(),
+                buildClearDiagnosticEventStatement(),
+            ]).pipe(Effect.withSpan("signals.clear.derived-events"));
+        }
         yield* exec(buildFrictionEventStatements(frictionBatch)).pipe(
             Effect.withSpan("signals.write.friction", {
                 attributes: { "signals.count": frictionBatch.length },

apps/axctl/src/ingest/signals/statements.test.ts

@@ -1,6 +1,8 @@
 import { describe, expect, test } from "bun:test";
 import type { CorrectionEdge, DerivedDiagnosticEvent, DerivedFrictionEvent } from "./types.ts";
 import {
+    buildClearDiagnosticEventStatement,
+    buildClearFrictionEventStatement,
     buildCorrectedByStatements,
     buildDiagnosticEventStatements,
     buildFrictionEventStatements,
@@ -157,3 +159,14 @@ describe("buildDiagnosticEventStatements", () => {
         expect(stmt).toContain('ts: d"2026-05-09T10:00:00.000Z" };');
     });
 });
+
+describe("clear-table statements (#549 orphan sweep)", () => {
+    test("bare DELETE (no WHERE) so a full re-derive removes orphaned rows", () => {
+        // Bare DELETE avoids the DELETE-WHERE-on-indexed-field ghost-row footgun
+        // (PR #141); only run on a FULL re-derive that re-emits every row.
+        expect(buildClearFrictionEventStatement()).toBe("DELETE friction_event RETURN NONE;");
+        expect(buildClearDiagnosticEventStatement()).toBe("DELETE diagnostic_event RETURN NONE;");
+        expect(buildClearFrictionEventStatement()).not.toContain("WHERE");
+        expect(buildClearDiagnosticEventStatement()).not.toContain("WHERE");
+    });
+});

apps/axctl/src/ingest/signals/statements.ts

@@ -100,6 +100,20 @@ export const buildRecoveredStatements = (edges: readonly RecoveryEdge[]): string
         return `RELATE turn:\`${e.fromTurnKey}\` -> recovered_by:\`${edgeId}\` -> skill:\`${e.skillKey}\` SET ts = d"${e.ts}", error_excerpt = ${excerpt};`;
     });
 
+/**
+ * Truncate the standalone derived event tables before a FULL re-derive (#549).
+ * These rows are written via stable-key UPSERT, so when the current derivation
+ * logic stops emitting a given row (e.g. a classifier change), the old row is
+ * never re-touched and orphans with a stale `ts` (the 1970-epoch residue). On a
+ * full re-derive every row is re-emitted from a complete scan, so clearing
+ * first removes orphans safely. A bare `DELETE` (no WHERE) sidesteps the
+ * DELETE-WHERE-on-indexed-field ghost-row footgun (PR #141). Only safe on a
+ * full derive - a `--since` run must NOT clear (it would drop rows whose source
+ * falls outside the window).
+ */
+export const buildClearFrictionEventStatement = (): string => "DELETE friction_event RETURN NONE;";
+export const buildClearDiagnosticEventStatement = (): string => "DELETE diagnostic_event RETURN NONE;";
+
 export const buildFrictionEventStatements = (
     events: readonly DerivedFrictionEvent[],
 ): string[] =>