feat: add CRW as unified alternative to Tavily, Serper, and Firecrawl

apps/api/.env.example

@@ -116,6 +116,8 @@ OPENROUTER_API_KEY=                     # primary LLM route — warns if empty
 # Features degrade gracefully if keys are missing.
 # Custom API URLs can be set per provider (defaults to official endpoints).
 
+# CRW_API_KEY=
+# CRW_API_URL=https://fastcrw.com/api
 # TAVILY_API_KEY=
 # TAVILY_API_URL=https://api.tavily.com
 # SERPER_API_KEY=

apps/api/src/__tests__/unit-proxy-services.test.ts

@@ -191,7 +191,7 @@ describe('matchAllowedRoute', () => {
   describe('registry integrity', () => {
     test('proxy services registry contains expected services', () => {
       const serviceNames = Object.keys(getProxyServices()).sort();
-      expect(serviceNames).toEqual(['context7', 'firecrawl', 'replicate', 'serper', 'tavily']);
+      expect(serviceNames).toEqual(['anthropic', 'context7', 'crw', 'firecrawl', 'gemini', 'groq', 'openai', 'replicate', 'serper', 'tavily', 'xai']);
     });
 
     test('each service has required fields', () => {

apps/api/src/config.ts

@@ -79,7 +79,11 @@ const envSchema = z.object({
   KORTIX_BILLING_INTERNAL_ENABLED:  optBoolFalse,  // NOTE: overridden by ENV_MODE=cloud below
   KORTIX_DEPLOYMENTS_ENABLED:       optBoolFalse,
 
-  // ── Search Providers (optional — features degrade gracefully) ────────────
+  // ── CRW (search, scrape, crawl, map — replaces Tavily, Serper, Firecrawl) ─
+  CRW_API_URL:                 optUrl('https://fastcrw.com/api'),
+  CRW_API_KEY:                 optStr,
+
+  // ── Legacy Search Providers (kept for backward compat, optional) ────────
   TAVILY_API_URL:              optUrl('https://api.tavily.com'),
   TAVILY_API_KEY:              optStr,
   SERPER_API_URL:              optUrl('https://google.serper.dev'),
@@ -371,7 +375,11 @@ export const config = {
   // ─── API Key Hashing ──────────────────────────────────────────────────────
   API_KEY_SECRET: env.API_KEY_SECRET,
 
-  // ─── Search Providers ──────────────────────────────────────────────────────
+  // ─── CRW (unified search, scrape, crawl) ───────────────────────────────────
+  CRW_API_URL: env.CRW_API_URL,
+  CRW_API_KEY: env.CRW_API_KEY,
+
+  // ─── Legacy Search Providers ──────────────────────────────────────────────
   TAVILY_API_URL: env.TAVILY_API_URL,
   TAVILY_API_KEY: env.TAVILY_API_KEY,
   SERPER_API_URL: env.SERPER_API_URL,
@@ -597,6 +605,11 @@ export const TOOL_PRICING: Record<string, ToolPricing> = {
     perResultCost: 0,
     markupMultiplier: 2.0,
   },
+  proxy_crw: {
+    baseCost: 0.003,
+    perResultCost: 0,
+    markupMultiplier: 1.5,
+  },
   proxy_tavily: {
     baseCost: 0.005,
     perResultCost: 0,

apps/api/src/platform/providers/daytona.ts

@@ -69,6 +69,8 @@ export class DaytonaProvider implements SandboxProvider {
           REPLICATE_API_URL: `${routerBase}/replicate`,
           SERPER_API_URL: `${routerBase}/serper`,
           FIRECRAWL_API_URL: `${routerBase}/firecrawl`,
+          // Only inject CRW proxy URL when backend has CRW key configured
+          ...(config.CRW_API_KEY ? { CRW_API_URL: `${routerBase}/crw` } : {}),
           ...opts.envVars,
         },
         autoStopInterval: 15,

apps/api/src/platform/providers/justavps.ts

@@ -486,6 +486,8 @@ export class JustAVPSProvider implements SandboxProvider {
       REPLICATE_API_URL: `${routerBase}/replicate`,
       SERPER_API_URL: `${routerBase}/serper`,
       FIRECRAWL_API_URL: `${routerBase}/firecrawl`,
+      // Only inject CRW proxy URL when backend has CRW key configured
+      ...(config.CRW_API_KEY ? { CRW_API_URL: `${routerBase}/crw` } : {}),
       PUID: '911',
       PGID: '911',
       ...opts.envVars,

apps/api/src/platform/providers/local-docker.ts

@@ -584,6 +584,12 @@ export class LocalDockerProvider implements SandboxProvider {
       FIRECRAWL_API_URL: `${routerBase}/firecrawl`,
     };
 
+    // Only inject CRW proxy URL when the backend has a CRW key configured.
+    // Without this guard, sandbox tools would select CRW and get 503 "crw not configured".
+    if (config.CRW_API_KEY) {
+      desired.CRW_API_URL = `${routerBase}/crw`;
+    }
+
     // Read current state from the live master env (s6 env dir) — NOT from
     // Docker inspect which only has stale creation-time values.
     const authCandidates = getAuthCandidates(await this.getCanonicalServiceKey());
@@ -606,6 +612,13 @@ export class LocalDockerProvider implements SandboxProvider {
       }
     }
 
+    // Clean up stale CRW_API_URL from sandbox when CRW is no longer configured.
+    // Without this, disabling CRW leaves an orphan URL that makes sandbox tools
+    // select CRW and hit 503 "crw not configured".
+    if (!desired.CRW_API_URL && currentEnv.CRW_API_URL) {
+      stale.CRW_API_URL = '';  // empty value signals deletion to the env API
+    }
+
     if (Object.keys(stale).length === 0) {
       this._serviceKeySynced = true;
       console.log('[LOCAL-DOCKER] syncCoreEnvVars: all core vars in sync');
@@ -866,6 +879,7 @@ export class LocalDockerProvider implements SandboxProvider {
       'PROJECT_ID',
       'ENV_MODE',
       'CORS_ALLOWED_ORIGINS',
+      'CRW_API_URL',
       'TAVILY_API_URL',
       'REPLICATE_API_URL',
       'SERPER_API_URL',
@@ -915,6 +929,8 @@ export class LocalDockerProvider implements SandboxProvider {
       `REPLICATE_API_URL=${routerBase}/replicate`,
       `SERPER_API_URL=${routerBase}/serper`,
       `FIRECRAWL_API_URL=${routerBase}/firecrawl`,
+      // Only inject CRW proxy URL when the backend has a CRW key configured.
+      ...(config.CRW_API_KEY ? [`CRW_API_URL=${routerBase}/crw`] : []),
       ...(config.KORTIX_LOCAL_IMAGES ? ['KORTIX_LOCAL_SOURCE=1'] : []),
       `ENV_MODE=${config.KORTIX_BILLING_INTERNAL_ENABLED ? 'cloud' : 'local'}`,
       `CORS_ALLOWED_ORIGINS=${[config.FRONTEND_URL, config.KORTIX_URL].filter(Boolean).join(',')}`,

apps/api/src/pool/env-injector.ts

@@ -40,6 +40,13 @@ function buildEnvPayload(serviceKey: string, metadata?: Record<string, unknown>)
     TUNNEL_TOKEN: serviceKey,
   };
 
+  // Only inject CRW proxy URL when the backend has a CRW key configured.
+  // When CRW is disabled, explicitly write empty string so the /env POST clears
+  // any stale CRW_API_URL from previously-claimed pool sandboxes. getEnv() treats
+  // an existing-but-empty s6 file as "explicitly cleared" and won't fall through
+  // to stale process.env values.
+  payload.CRW_API_URL = config.CRW_API_KEY ? `${routerBase}/crw` : '';
+
   // Compute PUBLIC_BASE_URL from JustAVPS metadata so getMasterPublicBaseUrl()
   // returns a real public URL instead of localhost inside the sandbox.
   if (metadata) {

apps/api/src/providers/registry.ts

@@ -92,6 +92,16 @@ export const PROVIDER_REGISTRY: ProviderDef[] = [
   },
 
   // ─── Tool Providers ────────────────────────────────────────
+  {
+    id: 'crw',
+    name: 'CRW',
+    category: 'tool',
+    envKeys: ['CRW_API_KEY'],
+    envUrlKey: 'CRW_API_URL',
+    defaultUrl: 'https://fastcrw.com/api',
+    helpUrl: 'https://fastcrw.com',
+    description: 'Web Search, Image Search, Scraping & Crawling (replaces Tavily, Serper, Firecrawl)',
+  },
   {
     id: 'tavily',
     name: 'Tavily',

apps/api/src/router/config/proxy-services.ts

@@ -50,6 +50,27 @@ export interface ProxyServiceConfig {
 
 export function getProxyServices(): Record<string, ProxyServiceConfig> {
   return {
+    // ─── CRW (unified: search, scrape, crawl, map — replaces Tavily, Serper, Firecrawl) ─
+    crw: {
+      name: 'crw',
+      targetBaseUrl: config.CRW_API_URL,
+      getKortixApiKey: () => config.CRW_API_KEY,
+      keyInjection: { type: 'header', headerName: 'Authorization', prefix: 'Bearer ' },
+      allowedRoutes: [
+        { path: '/v1/scrape', methods: ['POST'] },
+        { path: '/v1/crawl', methods: ['POST', 'GET', 'DELETE'], prefixMatch: true },
+        { path: '/v1/map', methods: ['POST'] },
+        { path: '/v1/search', methods: ['POST'] },
+        { path: '/v2/scrape', methods: ['POST'] },
+        { path: '/v2/crawl', methods: ['POST', 'GET', 'DELETE'], prefixMatch: true },
+        { path: '/v2/map', methods: ['POST'] },
+        { path: '/v2/search', methods: ['POST'] },
+      ],
+      billingToolName: 'proxy_crw',
+    },
+
+    // ─── Legacy providers (kept for backward compat / gradual migration) ────
+
     tavily: {
       name: 'tavily',
       targetBaseUrl: config.TAVILY_API_URL,
@@ -86,7 +107,6 @@ export function getProxyServices(): Record<string, ProxyServiceConfig> {
         { path: '/v1/crawl', methods: ['POST', 'GET'], prefixMatch: true },
         { path: '/v1/map', methods: ['POST'] },
         { path: '/v1/search', methods: ['POST'] },
-        // Firecrawl JS SDK v2+ uses /v2 endpoints
         { path: '/v2/scrape', methods: ['POST'] },
         { path: '/v2/crawl', methods: ['POST', 'GET'], prefixMatch: true },
         { path: '/v2/map', methods: ['POST'] },
@@ -101,7 +121,6 @@ export function getProxyServices(): Record<string, ProxyServiceConfig> {
       getKortixApiKey: () => config.REPLICATE_API_TOKEN,
       keyInjection: { type: 'header', headerName: 'Authorization', prefix: 'Token ' },
       allowedRoutes: [
-        // Allowed models — locked to specific models, each with own billing
         {
           path: '/v1/models/google/nano-banana/predictions',
           methods: ['POST'],

apps/api/src/router/routes/proxy.ts

@@ -14,6 +14,20 @@ import { calculateCost, extractUsage } from '../services/llm';
 
 const proxy = new Hono();
 
+/**
+ * Strip hop-by-hop and transport-encoding headers that become invalid after
+ * Bun's fetch() auto-decompresses the upstream response body.
+ * Without this, clients see Content-Encoding: gzip on an already-decompressed
+ * body and get ZlibError when they try to decompress again.
+ */
+function stripTransportHeaders(headers: Headers): Headers {
+  const cleaned = new Headers(headers);
+  cleaned.delete('content-encoding');
+  cleaned.delete('content-length'); // length is wrong after decompression
+  cleaned.delete('transfer-encoding');
+  return cleaned;
+}
+
 const services = getProxyServices();
 
 for (const [prefix, serviceConfig] of Object.entries(services)) {
@@ -131,7 +145,7 @@ async function handleKortixProxy(
     return new Response(upstream.body, {
       status: upstream.status,
       statusText: upstream.statusText,
-      headers: upstream.headers,
+      headers: stripTransportHeaders(upstream.headers),
     });
   }
 
@@ -148,7 +162,7 @@ async function handleKortixProxy(
   return new Response(upstream.body, {
     status: upstream.status,
     statusText: upstream.statusText,
-    headers: upstream.headers,
+    headers: stripTransportHeaders(upstream.headers),
   });
 }
 
@@ -378,7 +392,7 @@ async function handleKortixPassthrough(
     return new Response(upstream.body, {
       status: upstream.status,
       statusText: upstream.statusText,
-      headers: upstream.headers,
+      headers: stripTransportHeaders(upstream.headers),
     });
   }
 
@@ -395,7 +409,7 @@ async function handleKortixPassthrough(
   return new Response(upstream.body, {
     status: upstream.status,
     statusText: upstream.statusText,
-    headers: upstream.headers,
+    headers: stripTransportHeaders(upstream.headers),
   });
 }
 
@@ -426,7 +440,7 @@ async function handlePassthrough(
   return new Response(upstream.body, {
     status: upstream.status,
     statusText: upstream.statusText,
-    headers: upstream.headers,
+    headers: stripTransportHeaders(upstream.headers),
   });
 }