fix(health): report live modality readiness

Author: alichherawalla

docs/P0_P2_INTEGRATION_COVERAGE.md

@@ -8,10 +8,10 @@ manual claim does not count as complete integration coverage.
 ## Current status - 2026-07-17
 
 - Status snapshot:
-  - P0: 74 total, 23 covered, 51 left.
+  - P0: 74 total, 24 covered, 50 left.
   - P1: 71 total, 6 covered, 65 left.
   - P2: 10 total, 1 covered, 9 left.
-  - Overall: 155 total, 30 covered, 125 left.
+  - Overall: 155 total, 31 covered, 124 left.
 - Green gates today:
   - `npm test`: 202 files passed, 1 skipped; 2,190 tests passed, 1 skipped.
   - `npm run test:coverage`: 96.75% statements, 91.54% branches, 96.14% functions,
@@ -29,6 +29,9 @@ manual claim does not count as complete integration coverage.
   new temp `OFFGRID_USER_DATA` directory and verifies first-run onboarding and the preload bridge.
 - #9 - Fresh onboarding completes. `e2e/smoke.spec.ts` drives the real onboarding flow and lands on
   Models.
+- #13 - System Health is truthful. `e2e/smoke.spec.ts` launches a fresh profile, compares the real
+  gateway `/health` payload with the System Health IPC components, verifies absent runtimes are
+  reported as `not_installed`, and confirms the unavailable chat engine port is actually down.
 - #43 - Chat survives relaunch. `e2e/chat-memory.spec.ts` creates multiple scoped and unscoped
   conversations with messages and context through production IPC, fully closes Electron, reopens
   the same profile, and verifies every association and payload through the reloaded preload path.
@@ -117,7 +120,6 @@ manual claim does not count as complete integration coverage.
 - #10 - Configure for me completes.
 - #11 - Manual setup path works.
 - #12 - Onboarding resumes after relaunch.
-- #13 - System Health is truthful.
 - #14 - Chat engine stderr is surfaced.
 - #15 - Required permissions granted.
 - #16 - Denied permission is recoverable.

e2e/smoke.spec.ts

@@ -108,6 +108,45 @@ test('system:health IPC returns the component list', async () => {
   const ids = health.components.map((c: { id: string }) => c.id)
   expect(ids).toContain('chat')
   expect(ids).toContain('gateway')
+
+  let gateway: { modalities?: Record<string, string> } = {}
+  await expect
+    .poll(
+      async () => {
+        try {
+          const response = await fetch('http://127.0.0.1:7878/health')
+          if (!response.ok) return false
+          gateway = await response.json()
+          return true
+        } catch {
+          return false
+        }
+      },
+      { timeout: 10000 }
+    )
+    .toBe(true)
+
+  const byId = new Map(
+    health.components.map((component: { id: string; status: string }) => [component.id, component])
+  )
+  expect(health.activeModel).toBeNull()
+  expect(byId.get('gateway')?.status).toBe('ready')
+  expect(byId.get('chat')?.status).toBe('not_installed')
+  const llamaReachable = await fetch('http://127.0.0.1:8439/health')
+    .then((response) => response.ok)
+    .catch(() => false)
+  expect(llamaReachable).toBe(false)
+
+  const gatewayBackedComponents = {
+    vision: 'vision_understanding',
+    embeddings: 'embeddings',
+    transcription: 'transcription',
+    speech: 'speech'
+  }
+  for (const [componentId, modalityId] of Object.entries(gatewayBackedComponents)) {
+    expect(byId.get(componentId)?.status).toBe(gateway.modalities?.[modalityId])
+  }
+  expect(byId.get('image')?.status).toBe(gateway.modalities?.image_generation)
 })
 
 test('gateway /v1/models serves active local models with modality metadata', async () => {

src/main/model-server.ts

@@ -51,13 +51,35 @@ import { isAsync, matchPollRoute } from './model-server/async-request'
 import { sanitizeChatMessages } from './model-server/chat-messages'
 import { parseMultipart } from './model-server/multipart'
 import { tagLlmEntries, modelEntry, ollamaMirror } from './model-server/models-list'
+import { buildGatewayModalities, type GatewayModalities } from './model-server/health'
 
 const UPSTREAM_HOST = '127.0.0.1'
 const UPSTREAM_PORT = LLAMA_SERVER_PORT // bundled llama-server (see llm.ts)
 const MAX_UPLOAD = 200 * 1024 * 1024 // 200MB upload cap (audio / init image)
 
 let server: http.Server | null = null
 
+async function liveGatewayModalities(imageAvailable: boolean): Promise<GatewayModalities> {
+  let installed: string[] = []
+  try {
+    const models = await import('./models-manager')
+    installed = await models.listInstalled()
+  } catch {
+    /* unavailable model registry means optional modalities are not ready */
+  }
+  const transcription = getActiveModal('transcription')
+  const speech = getActiveModal('speech')
+  const chat = llm.modelsExist()
+  return buildGatewayModalities({
+    chat,
+    vision: chat && llm.hasVision(),
+    embeddings: true,
+    transcription: !!whisperModel() || (!!transcription && installed.includes(transcription)),
+    speech: !!speech && installed.includes(speech),
+    image: imageAvailable
+  })
+}
+
 function json(res: http.ServerResponse, status: number, body: unknown): void {
   res.writeHead(status, { 'Content-Type': 'application/json' })
   res.end(JSON.stringify(body))
@@ -893,7 +915,7 @@ async function handleImageEdit(
 export function startModelServer(port = GATEWAY_PORT): void {
   if (server) return
 
-  server = http.createServer((req, res) => {
+  server = http.createServer(async (req, res) => {
     res.setHeader('Access-Control-Allow-Origin', '*')
     res.setHeader('Access-Control-Allow-Headers', '*')
     res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS')
@@ -921,21 +943,14 @@ export function startModelServer(port = GATEWAY_PORT): void {
 
     if (url === '/' || url === '/health') {
       const img = imageGenStatus()
+      const modalities = await liveGatewayModalities(img.available)
       json(res, 200, {
         name: 'Off Grid AI — local model gateway',
         openai_compatible: true,
         base_url: `http://127.0.0.1:${port}/v1`,
         docs: `http://127.0.0.1:${port}/docs`,
         mcp: `http://127.0.0.1:${port}/mcp`,
-        modalities: {
-          text: 'ready',
-          vision_understanding: 'ready',
-          embeddings: 'ready',
-          transcription: 'ready',
-          speech: 'ready',
-          image_generation: img.available ? 'ready' : 'not_installed',
-          image_edit: img.available ? 'ready' : 'not_installed'
-        },
+        modalities,
         image_models: img.models,
         image_reason: img.available ? undefined : img.reason
       })
@@ -944,15 +959,8 @@ export function startModelServer(port = GATEWAY_PORT): void {
 
     if (url === '/openapi.json') {
       const img = imageGenStatus()
-      const mods = {
-        text: 'ready',
-        vision_understanding: 'ready',
-        embeddings: 'ready',
-        transcription: 'ready',
-        speech: 'ready',
-        image_generation: img.available ? 'ready' : 'not_installed'
-      }
-      json(res, 200, openApiSpec(port, mods, img.models))
+      const modalities = await liveGatewayModalities(img.available)
+      json(res, 200, openApiSpec(port, modalities, img.models))
       return
     }
 

src/main/model-server/__tests__/health.test.ts

@@ -0,0 +1,46 @@
+import { describe, expect, it } from 'vitest'
+import { buildGatewayModalities } from '../health'
+
+describe('buildGatewayModalities', () => {
+  it('reports unavailable runtimes as not installed', () => {
+    expect(
+      buildGatewayModalities({
+        chat: false,
+        vision: false,
+        embeddings: true,
+        transcription: false,
+        speech: false,
+        image: false
+      })
+    ).toEqual({
+      text: 'not_installed',
+      vision_understanding: 'not_installed',
+      embeddings: 'ready',
+      transcription: 'not_installed',
+      speech: 'not_installed',
+      image_generation: 'not_installed',
+      image_edit: 'not_installed'
+    })
+  })
+
+  it('reports every runtime from its own capability fact', () => {
+    expect(
+      buildGatewayModalities({
+        chat: true,
+        vision: false,
+        embeddings: false,
+        transcription: true,
+        speech: true,
+        image: true
+      })
+    ).toEqual({
+      text: 'ready',
+      vision_understanding: 'not_installed',
+      embeddings: 'not_installed',
+      transcription: 'ready',
+      speech: 'ready',
+      image_generation: 'ready',
+      image_edit: 'ready'
+    })
+  })
+})

src/main/model-server/health.ts

@@ -0,0 +1,36 @@
+export type GatewayModalityStatus = 'ready' | 'not_installed'
+
+export interface GatewayCapabilityFacts {
+  chat: boolean
+  vision: boolean
+  embeddings: boolean
+  transcription: boolean
+  speech: boolean
+  image: boolean
+}
+
+export type GatewayModalities = Record<
+  | 'text'
+  | 'vision_understanding'
+  | 'embeddings'
+  | 'transcription'
+  | 'speech'
+  | 'image_generation'
+  | 'image_edit',
+  GatewayModalityStatus
+>
+
+/** Convert live runtime facts into the one modality contract shared by health and OpenAPI. */
+export function buildGatewayModalities(facts: GatewayCapabilityFacts): GatewayModalities {
+  const status = (available: boolean): GatewayModalityStatus =>
+    available ? 'ready' : 'not_installed'
+  return {
+    text: status(facts.chat),
+    vision_understanding: status(facts.vision),
+    embeddings: status(facts.embeddings),
+    transcription: status(facts.transcription),
+    speech: status(facts.speech),
+    image_generation: status(facts.image),
+    image_edit: status(facts.image)
+  }
+}