feat(server): enforce fail-closed instance silent mode (#1442)

Author: FelixTJDietrich

.changeset/safe-silent-egress.md

@@ -0,0 +1,14 @@
+---
+"hephaestus": minor
+---
+
+Instance Silent Mode now fails closed and enforces the brake at every GitHub, GitLab, and Slack
+delivery gateway. Suppressed feedback remains auditable but is never replayed when the brake is
+released, and stale admin pages can no longer release a newer incident response.
+
+**Operators:** New installations and upgrades whose Silent Mode setting was never explicitly changed
+start engaged. On production, verify workspace delivery settings before releasing the brake from
+**Instance admin → Settings**; leave it engaged on staging clones and during disaster-recovery drills.
+
+**API clients:** Replace `PUT /admin/settings/silent-mode` with `PATCH`; the `PUT` operation has been
+removed.

MIGRATION.md

@@ -65,6 +65,22 @@ Entries exist only for releases that need operator action. Everything else is in
 
 ### Next release
 
+#### 🔴 Untouched instances start with Silent Mode engaged
+
+**Affected**: deployments where the instance Silent Mode setting has never been explicitly changed.
+
+The upgrade engages the instance-wide outbound brake before any new GitHub, GitLab, or Slack delivery
+can leave the application. Detection, persistence, synchronization, webhooks, OAuth, and administration
+continue normally; suppressed feedback is recorded and is never replayed.
+
+On production, verify each workspace's practice delivery settings and provider targets, then open
+**Instance admin → Settings** and release Silent Mode. Leave it engaged on staging clones and during
+disaster-recovery drills. Instances whose operator had already changed the setting keep that explicit
+choice.
+
+**API clients:** The Silent Mode update operation is now
+`PATCH /admin/settings/silent-mode`. Replace calls to the removed `PUT` operation before upgrading.
+
 #### 🔴 Practice-feedback delivery field renamed
 
 **Affected**: API clients that read or write user settings, or consume account-export JSON.

docs/contributor/erd/schema.mmd

@@ -438,6 +438,7 @@ erDiagram
         VARCHAR(500) silent_mode_reason
         TIMESTAMPTZ silent_mode_changed_at
         VARCHAR(255) silent_mode_changed_by
+        BIGINT version "NOT NULL"
     }
 
     Issue {

docs/contributor/instance-admin.md

@@ -45,6 +45,23 @@ All under `/admin`, all gated by `hasAuthority('app_admin')`:
 | `GET`/`PUT /admin/llm/settings` (`adminGetLlmSettings`, `adminUpdateLlmSettings`) | The instance LLM settings singleton: egress host allowlist and `allowWorkspaceConnections`, the switch that lets workspaces register their own provider connections. |
 | `GET /admin/llm/usage` (`adminGetLlmUsageReport`) | Cross-workspace monthly LLM usage and budget report, split by purse (shared models vs each workspace's own provider). |
 | `PUT /admin/workspaces/{workspaceSlug}/llm/budget` (`adminUpdateWorkspaceLlmBudget`) | Set **or clear** a workspace's monthly cap on **shared-model** spend — clearing is `PUT` with `monthlyBudgetUsd: null`, not `DELETE` (there is no `DELETE` mapping; it returns 405). The workspace's cap on its own provider is a different endpoint under `/workspaces/**`, set by the workspace's own admin. |
+| `GET /admin/settings` / `PATCH /admin/settings/silent-mode` | Read or change the instance-wide outbound delivery brake. Releasing requires `If-Match` with the ETag returned by `GET`, so a stale browser cannot release a newer incident response. |
+
+## Instance Silent Mode
+
+Silent Mode is an emergency and disaster-recovery brake, not a workspace rollout stage. It is
+**engaged by default** on new installs, when the singleton settings row is missing, and on upgrades
+whose seeded row was never explicitly changed. Detection, observation persistence, inbound webhook
+processing, synchronization, and admin access continue, but delivery writes to GitHub, GitLab, and
+Slack are refused at the provider gateway.
+
+Suppression is prospective: a suppressed review is recorded as `SUPPRESSED(INSTANCE_SILENCED)` for
+audit and preview, but is never queued for replay. Releasing the brake therefore sends nothing by
+itself; only a new source event can deliver. A re-review that would have edited an existing comment
+records the attempted replacement without superseding the live delivered ledger unit.
+
+OAuth/token lifecycle operations, webhook registration, and operator alerts remain available while
+Silent Mode is engaged.
 
 ## Impersonation time-box
 

pnpm-lock.yaml

@@ -1401,8 +1401,15 @@ paths:
       tags:
       - Instance Settings
   /admin/settings/silent-mode:
-    put:
+    patch:
       operationId: adminUpdateSilentMode
+      parameters:
+      - description: Current settings ETag; required when releasing Silent Mode
+        in: header
+        name: If-Match
+        required: false
+        schema:
+          type: string
       requestBody:
         content:
           application/json:
@@ -1415,7 +1422,19 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/InstanceSettings"
-          description: OK
+          description: Silent Mode updated
+        "412":
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ProblemDetail"
+          description: The supplied ETag is stale
+        "428":
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ProblemDetail"
+          description: If-Match is required when releasing Silent Mode
       summary: Engage or release the instance-wide silent mode
       tags:
       - Instance Settings
@@ -8384,6 +8403,8 @@ components:
     InstanceSettings:
       type: object
       properties:
+        etag:
+          type: string
         silentModeChangedAt:
           type: string
           format: date-time
@@ -8394,6 +8415,7 @@ components:
         silentModeReason:
           type: string
       required:
+      - etag
       - silentModeEngaged
     IntegrationCatalogEntry:
       type: object

server/openapi.yaml

@@ -1010,9 +1010,10 @@
                     <!-- Tag filtering: run integration tests, exclude live tests -->
                     <groups>${failsafe.includedGroups}</groups>
                     <excludedGroups>${failsafe.excludedGroups}</excludedGroups>
-                    <!-- Include *IntegrationTest.java pattern (project convention) -->
+                    <!-- Failsafe-discovered integration test patterns -->
                     <includes>
                         <include>**/*IntegrationTest.java</include>
+                        <include>**/*LiquibaseTest.java</include>
                     </includes>
                 </configuration>
                 <executions>

server/pom.xml

@@ -2,7 +2,9 @@
 
 import de.tum.cit.aet.hephaestus.agent.handler.PracticeDetectionResultParser.DiffNote;
 import de.tum.cit.aet.hephaestus.agent.handler.spi.JobDeliveryException;
+import de.tum.cit.aet.hephaestus.agent.handler.spi.JobDeliverySuppressedException;
 import de.tum.cit.aet.hephaestus.agent.job.AgentJob;
+import de.tum.cit.aet.hephaestus.integration.core.egress.OutboundEgressSuppressedException;
 import de.tum.cit.aet.hephaestus.integration.core.spi.FeedbackChannel;
 import de.tum.cit.aet.hephaestus.integration.core.spi.FeedbackDeliveryException;
 import de.tum.cit.aet.hephaestus.integration.core.spi.FindingAnchor;
@@ -74,6 +76,8 @@ DiffNoteResult reconcileInlineNotes(AgentJob job, List<DiffNote> diffNotes) {
         if (findings.isEmpty()) {
             try {
                 channel.clearStaleFindings(target, HEPHAESTUS_MARKER);
+            } catch (OutboundEgressSuppressedException e) {
+                throw new JobDeliverySuppressedException(e.getMessage(), e);
             } catch (RuntimeException e) {
                 log.warn(
                     "Stale inline-note clear failed (best-effort), continuing: kind={}, jobId={}, error={}",
@@ -94,7 +98,15 @@ DiffNoteResult reconcileInlineNotes(AgentJob job, List<DiffNote> diffNotes) {
                 result.failed(),
                 job.getId()
             );
-            return new DiffNoteResult(result.posted(), result.failed(), result.signals());
+            return new DiffNoteResult(
+                result.posted(),
+                result.failed(),
+                result.signals(),
+                result.suppressed(),
+                result.suppressedRecurrenceKeys()
+            );
+        } catch (OutboundEgressSuppressedException e) {
+            throw new JobDeliverySuppressedException(e.getMessage(), e);
         } catch (FeedbackDeliveryException e) {
             throw new JobDeliveryException(e.getMessage(), e);
         }
@@ -124,5 +136,15 @@ private List<InlineFindingChannel.InlineFinding> mapFindings(List<DiffNote> diff
         return findings;
     }
 
-    record DiffNoteResult(int posted, int failed, List<InlineFindingChannel.DeliveredSignal> signals) {}
+    record DiffNoteResult(
+        int posted,
+        int failed,
+        List<InlineFindingChannel.DeliveredSignal> signals,
+        boolean suppressed,
+        List<String> suppressedRecurrenceKeys
+    ) {
+        DiffNoteResult(int posted, int failed, List<InlineFindingChannel.DeliveredSignal> signals) {
+            this(posted, failed, signals, false, List.of());
+        }
+    }
 }

server/src/main/java/de/tum/cit/aet/hephaestus/agent/handler/DiffNotePoster.java

@@ -3,6 +3,7 @@
 import de.tum.cit.aet.hephaestus.agent.handler.PracticeDetectionResultParser.DeliveryContent;
 import de.tum.cit.aet.hephaestus.agent.handler.spi.ExistingDeliveryLookup;
 import de.tum.cit.aet.hephaestus.agent.handler.spi.JobDeliveryException;
+import de.tum.cit.aet.hephaestus.agent.handler.spi.JobDeliverySuppressedException;
 import de.tum.cit.aet.hephaestus.agent.job.AgentJob;
 import de.tum.cit.aet.hephaestus.integration.core.spi.InlineFindingChannel;
 import de.tum.cit.aet.hephaestus.integration.scm.domain.pullrequest.PullRequest;
@@ -87,6 +88,9 @@ void deliverFeedback(
 
         try {
             doDeliverEligible(job, delivery, summaryComposer, decision.artifact());
+        } catch (JobDeliverySuppressedException e) {
+            log.info("Delivery suppressed at egress: jobId={}", job.getId());
+            recordGateSuppressed(job, delivery, FeedbackSuppressionReason.INSTANCE_SILENCED);
         } catch (JobDeliveryException e) {
             if (job.getDeliveryCommentId() == null) {
                 recordUndelivered(job, delivery);
@@ -97,6 +101,21 @@ void deliverFeedback(
         }
     }
 
+    private void recordPartialSummaryDelivery(AgentJob job, DeliveryContent delivery) {
+        try {
+            feedbackLedgerRecorder.recordWithoutConversation(
+                job,
+                delivery,
+                WorkArtifact.PULL_REQUEST,
+                List.of(),
+                true,
+                false
+            );
+        } catch (RuntimeException e) {
+            log.warn("Partial delivery ledger record failed: jobId={}, error={}", job.getId(), e.getMessage());
+        }
+    }
+
     private void doDeliverEligible(
         AgentJob job,
         DeliveryContent delivery,
@@ -110,13 +129,29 @@ private void doDeliverEligible(
             : null;
 
         SummaryOutcome summaryOutcome = postSummaryNote(job, delivery, trend);
-        DiffNotePoster.DiffNoteResult inlineResult = postDiffNotes(job, delivery);
+        DiffNotePoster.DiffNoteResult inlineResult;
+        try {
+            inlineResult = postDiffNotes(job, delivery);
+        } catch (JobDeliverySuppressedException e) {
+            log.info("Inline delivery suppressed at egress: jobId={}", job.getId());
+            if (summaryOutcome == SummaryOutcome.DELIVERED) {
+                recordPartialSummaryDelivery(job, delivery);
+                recordSuppressedRemainder(job, delivery, List.of());
+            } else {
+                recordGateSuppressed(job, delivery, FeedbackSuppressionReason.INSTANCE_SILENCED);
+            }
+            return;
+        }
         List<InlineFindingChannel.DeliveredSignal> inlineSignals = inlineResult.signals();
 
-        if (summaryOutcome == SummaryOutcome.DELIVERED) {
+        if (summaryOutcome == SummaryOutcome.DELIVERED && !inlineResult.suppressed()) {
             reEditSummaryWithSignals(job, summaryComposer, inlineSignals, trend);
         }
         boolean inlineDelivered = inlineResult.posted() > 0;
+        if (inlineResult.suppressed() && summaryOutcome != SummaryOutcome.DELIVERED && !inlineDelivered) {
+            recordGateSuppressed(job, delivery, FeedbackSuppressionReason.INSTANCE_SILENCED);
+            return;
+        }
         if (summaryOutcome == SummaryOutcome.SKIPPED_EMPTY && !inlineDelivered) {
             recordGateSuppressed(job, delivery, FeedbackSuppressionReason.EMPTY_AFTER_SANITIZE);
             return;
@@ -126,21 +161,52 @@ private void doDeliverEligible(
         }
 
         try {
-            feedbackLedgerRecorder.record(
-                job,
-                delivery,
-                WorkArtifact.PULL_REQUEST,
-                inlineSignals,
-                summaryOutcome == SummaryOutcome.DELIVERED,
-                inlineDelivered
-            );
+            if (inlineResult.suppressed()) {
+                feedbackLedgerRecorder.recordWithoutConversation(
+                    job,
+                    delivery,
+                    WorkArtifact.PULL_REQUEST,
+                    inlineSignals,
+                    summaryOutcome == SummaryOutcome.DELIVERED,
+                    inlineDelivered
+                );
+            } else {
+                feedbackLedgerRecorder.record(
+                    job,
+                    delivery,
+                    WorkArtifact.PULL_REQUEST,
+                    inlineSignals,
+                    summaryOutcome == SummaryOutcome.DELIVERED,
+                    inlineDelivered
+                );
+            }
         } catch (RuntimeException e) {
             log.warn(
                 "Feedback ledger record failed (delivery unaffected): jobId={}, error={}",
                 job.getId(),
                 e.getMessage()
             );
         }
+        if (inlineResult.suppressed()) {
+            recordSuppressedRemainder(job, delivery, inlineResult.suppressedRecurrenceKeys());
+        }
+    }
+
+    private void recordSuppressedRemainder(
+        AgentJob job,
+        DeliveryContent delivery,
+        List<String> suppressedRecurrenceKeys
+    ) {
+        try {
+            feedbackLedgerRecorder.recordSuppressedRemainder(
+                job,
+                delivery,
+                FeedbackSuppressionReason.INSTANCE_SILENCED,
+                suppressedRecurrenceKeys
+            );
+        } catch (RuntimeException e) {
+            log.warn("Suppressed delivery ledger record failed: jobId={}, error={}", job.getId(), e.getMessage());
+        }
     }
 
     private void recordGateSuppressed(