fix(flow): fail over on missing model errors (rebased on main)

Author: PhucHai123

docs/CONFIGURATION.md

@@ -37,6 +37,8 @@ Use `flowModelConfigs` in your Pi settings to define tiered model strategies. Ea
 
 Settings are merged: project `.pi/settings.json` overrides global `~/.pi/agent/settings.json`.
 
+Models referenced in `flowModelConfigs` are checked against the local `models.json` registry (usually `~/.pi/agent/models.json`). If a model is known to be missing from the registry, a warning is logged, but the model is still tried so that stale or incomplete registries do not silently block valid provider-side models. If **every** configured model for a flow or trace is missing from the registry, the invocation fails fast with a clear `Bad settings` error.
+
 ## Persistent Flow Mode Switch
 
 Switch the global active strategy quickly with `--flow-mode`:

src/config/config.ts

@@ -11,7 +11,7 @@ import * as path from "node:path";
 import { parseComplexity, type Complexity } from "../flow/complexity.js";
 import { type FlowTier } from "../flow/agents.js";
 import { logWarn } from "./log.js";
-import { resolveModelContextWindow as resolveModelContextWindowFromModels } from "./models.js";
+import { getModelsJsonPath, hasConfiguredModel, resolveModelContextWindow as resolveModelContextWindowFromModels } from "./models.js";
 import { getAgentDir, hasAgentDirOverride } from "./paths.js";
 import { atomicWriteFileSync, atomicWriteJsonAsync } from "../io/atomic-write.js";
 
@@ -615,34 +615,43 @@ export function resolveFlowModelCandidates(opts: {
 	cliTierOverride?: string;
 	strategy: FlowModelStrategy;
 	fallbackModel?: string;
-}): { primary: string | undefined; candidates: string[] } {
+}): { primary: string | undefined; candidates: string[]; invalidCandidates: string[] } {
 	const unique = new Set<string>();
 	const candidates: string[] = [];
+	const invalidCandidates: string[] = [];
+	// Read the registry once; models without a provider prefix or unknown providers
+	// are treated as "cannot answer authoritatively" and are still tried.
+	const registry = readSettingsJson(getModelsJsonPath());
 
 	const add = (value: string | undefined) => {
 		if (!value) return;
 		const normalized = value.trim();
 		if (!normalized || unique.has(normalized)) return;
 		unique.add(normalized);
+		const configured = hasConfiguredModel(normalized, registry);
+		if (configured === false) {
+			invalidCandidates.push(normalized);
+			logWarn(`[pi-agent-flow] Model "${normalized}" is not present in models.json; trying it anyway.`);
+		}
 		candidates.push(normalized);
 	};
 
 	if (opts.flowModel) {
 		add(opts.flowModel);
-		return { primary: candidates[0], candidates };
+		return { primary: candidates[0], candidates, invalidCandidates };
 	}
 
 	if (opts.cliTierOverride) {
 		add(opts.cliTierOverride);
-		return { primary: candidates[0], candidates };
+		return { primary: candidates[0], candidates, invalidCandidates };
 	}
 
 	const tierConfig = opts.strategy[opts.tier];
 	add(tierConfig?.primary);
 	for (const model of tierConfig?.failover ?? []) add(model);
 	add(opts.fallbackModel);
 
-	return { primary: candidates[0], candidates };
+	return { primary: candidates[0], candidates, invalidCandidates };
 }
 
 export function formatFlowModelStrategy(modeName: string, strategy: FlowModelStrategy): string {

src/config/models.ts

@@ -3,7 +3,7 @@ import * as path from "node:path";
 import { logWarn } from "./log.js";
 import { getAgentDir, hasAgentDirOverride } from "./paths.js";
 
-function getModelsJsonPath(): string {
+export function getModelsJsonPath(): string {
 	const agentDir = getAgentDir();
 	const defaultPath = path.join(agentDir, "models.json");
 	if (!hasAgentDirOverride() && !fs.existsSync(defaultPath)) {
@@ -56,3 +56,34 @@ export function resolveModelContextWindow(model: string): number | undefined {
 
 	return undefined;
 }
+
+/**
+ * Return whether a provider/model entry is known in models.json.
+ *
+ * `undefined` means the local model registry cannot answer authoritatively
+ * (for example, no provider/model form or unreadable models.json).
+ */
+export function hasConfiguredModel(
+	model: string,
+	registry?: Record<string, unknown> | null,
+): boolean | undefined {
+	const parts = model.split("/");
+	if (parts.length < 2) return undefined;
+
+	const providerKey = parts[0];
+	const modelId = parts.slice(1).join("/");
+
+	const raw = registry === undefined ? readSettingsJson(getModelsJsonPath()) : registry;
+	if (!isPlainObject(raw)) return undefined;
+
+	const providers = raw.providers;
+	if (!isPlainObject(providers)) return undefined;
+
+	const provider = providers[providerKey];
+	if (!isPlainObject(provider)) return undefined;
+
+	const models = provider.models;
+	if (!Array.isArray(models)) return undefined;
+
+	return models.some((m) => isPlainObject(m) && m.id === modelId);
+}

src/flow/cycle-guard.ts

@@ -33,7 +33,12 @@ export function shouldFailover(result: SingleResult): boolean {
 	if (result.stopReason === "aborted") return false;
 	const text = `${result.errorMessage ?? ""}\n${result.stderr ?? ""}`.toLowerCase();
 	if (!text.trim()) return false;
-	if (text.includes("permission") || text.includes("invalid tool") || text.includes("bad settings")) {
+	if (text.includes("permission") || text.includes("bad settings")) {
+		return false;
+	}
+	// Generic "invalid tool" failures are not retryable, but "invalid tool_call_id"
+	// errors are a specific provider-side rejection that failover can recover from.
+	if (text.includes("invalid tool") && !text.includes("tool_call_id")) {
 		return false;
 	}
 	if (result.exitCode > 0) return true;
@@ -47,6 +52,14 @@ export function shouldFailover(result: SingleResult): boolean {
 	if (!isFlowComplete(result) && text.includes("400") && text.includes("tool_call_id")) {
 		return true;
 	}
+	// Provider-side 404 / resource-not-found errors should fail over to the next
+	// configured model even when the process exits 0.
+	if (
+		!isFlowComplete(result) &&
+		(text.includes("resource_not_found_error") || text.includes("requested resource was not found"))
+	) {
+		return true;
+	}
 	return false;
 }
 

src/flow/execute-single.ts

@@ -63,18 +63,56 @@ export async function executeSingleFlow(
 
 	const shouldInheritContext = targetFlow?.inheritContext !== false;
 	const tier = targetFlow?.tier ?? "flash";
-	const { candidates } = resolveFlowModelCandidates({
+	const { candidates, invalidCandidates } = resolveFlowModelCandidates({
 		tier,
 		flowModel: targetFlow?.model,
 		cliTierOverride: tierOverrideResolver(tier),
 		strategy: selectedFlowModelConfig.strategy,
 		fallbackModel,
 	});
-	const attemptModels = candidates.length > 0 ? candidates : [undefined];
 	const attemptedModels: string[] = [];
 	let result = allResults[resultIndex];
 	const flowStart = Date.now();
 
+	// Fail fast when every configured model is known to be missing from models.json.
+	if (invalidCandidates.length > 0 && candidates.length === invalidCandidates.length) {
+		const badSettingsMessage = `Bad settings: all configured flow models are missing from models.json: ${invalidCandidates.join(", ")}`;
+		result = {
+			type: normalizedType,
+			agentSource: targetFlow?.source ?? "unknown",
+			intent: item.intent,
+			aim: item.aim,
+			exitCode: 1,
+			messages: [],
+			stderr: badSettingsMessage,
+			usage: emptyFlowUsage(),
+			model: invalidCandidates[0],
+			stopReason: "error",
+			errorMessage: badSettingsMessage,
+		};
+		const previous = allResults[resultIndex];
+		allResults[resultIndex] = result;
+		preserveMetadata(allResults[resultIndex], previous);
+		emitProgress();
+		if (onFlowMetrics) {
+			onFlowMetrics({
+				type: normalizedType,
+				durationMs: Date.now() - flowStart,
+				exitCode: result.exitCode,
+				success: false,
+				model: result.model,
+				failoverCount: 0,
+				connectionRetryCount: 0,
+				usage: result.usage,
+				source: result.agentSource,
+				depth: currentDepth + 1,
+			});
+		}
+		return result;
+	}
+
+	const attemptModels = candidates.length > 0 ? candidates : [undefined];
+
 	const maxRetries = deps.subAgentMaxRetries ?? DEFAULT_SUB_AGENT_MAX_RETRIES;
 	const baseDelayMs = deps.subAgentBaseDelayMs ?? DEFAULT_SUB_AGENT_BASE_DELAY_MS;
 	let retryCount = 0;

src/tools/trace.ts

@@ -211,7 +211,8 @@ export interface TraceToolOptions {
 	fallbackModel?: string;
 }
 
-function resolveTraceRuntime(
+/** Exported for unit testing; not part of the public API. */
+export function resolveTraceRuntime(
 	opts: TraceToolOptions,
 	traceFlow: FlowConfig,
 	ctx: ExtensionContext,
@@ -228,13 +229,18 @@ function resolveTraceRuntime(
 		);
 		selectedStrategy = selectedFlowModelConfig.strategy;
 	}
-	const { candidates } = resolveFlowModelCandidates({
+	const { candidates, invalidCandidates } = resolveFlowModelCandidates({
 		tier,
 		flowModel: traceFlow.model,
 		cliTierOverride: opts.tierOverrideResolver?.(tier),
 		strategy: selectedStrategy ?? {},
 		fallbackModel: opts.fallbackModel,
 	});
+	if (invalidCandidates.length > 0 && candidates.length === invalidCandidates.length) {
+		throw new Error(
+			`Bad settings: all configured trace models are missing from models.json: ${invalidCandidates.join(", ")}`,
+		);
+	}
 	const resolvedModel = candidates[0];
 	const maxContextTokens = resolveModelContextWindow(resolvedModel);
 	// INTENTIONALLY OMIT: `tier` and `compressionProfile`.

src/types/flow.ts

@@ -126,22 +126,31 @@ export function aggregateFlowUsage(results: SingleResult[]): UsageStats {
 	let weightedTps = 0;
 	let totalOutput = 0;
 	for (const r of results) {
-		total.input += r.usage.input;
-		total.output += r.usage.output;
-		total.cacheRead += r.usage.cacheRead;
-		total.cacheWrite += r.usage.cacheWrite;
-		total.cost += r.usage.cost;
-		total.turns += r.usage.turns;
-		total.toolCalls += r.usage.toolCalls;
-		if ((r.usage.smoothedTps ?? 0) > 0) {
-			weightedTps += (r.usage.smoothedTps ?? 0) * r.usage.output;
-			totalOutput += r.usage.output;
+		const usage = r.usage ?? emptyFlowUsage();
+		const input = usage.input ?? 0;
+		const output = usage.output ?? 0;
+		const cacheRead = usage.cacheRead ?? 0;
+		const cacheWrite = usage.cacheWrite ?? 0;
+		const cost = usage.cost ?? 0;
+		const turns = usage.turns ?? 0;
+		const toolCalls = usage.toolCalls ?? 0;
+
+		total.input += input;
+		total.output += output;
+		total.cacheRead += cacheRead;
+		total.cacheWrite += cacheWrite;
+		total.cost += cost;
+		total.turns += turns;
+		total.toolCalls += toolCalls;
+		if ((usage.smoothedTps ?? 0) > 0) {
+			weightedTps += (usage.smoothedTps ?? 0) * output;
+			totalOutput += output;
 		}
 	}
 	if (totalOutput > 0) {
 		total.smoothedTps = weightedTps / totalOutput;
 	} else if (results.length > 0) {
-		total.smoothedTps = Math.max(...results.map((r) => r.usage.smoothedTps ?? 0));
+		total.smoothedTps = Math.max(...results.map((r) => r.usage?.smoothedTps ?? 0));
 	}
 	return total;
 }