Merge pull request #172 from Macnelson9/idempotency

Idempotency

Author: jhayniffy

backend/src/__tests__/idempotency.middleware.test.ts

@@ -0,0 +1,194 @@
+/**
+ * IdempotencyMiddleware unit tests.
+ *
+ * Uses in-memory stubs for Redis — no real Redis required.
+ */
+
+import { BadRequestException } from '@nestjs/common';
+import { IdempotencyMiddleware, IDEMPOTENCY_VERSION } from '../common/middleware/idempotency.middleware';
+import * as cache from '../redis/cache';
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+
+function makeReq(overrides: Partial<{
+  method: string;
+  path: string;
+  headers: Record<string, string>;
+  user: { sub: string };
+}> = {}): any {
+  return {
+    method: 'POST',
+    path: '/ipfs/upload',
+    headers: {},
+    ...overrides,
+  };
+}
+
+function makeRes(): any {
+  const res: any = {
+    statusCode: 200,
+    _headers: {} as Record<string, string>,
+    _body: undefined as unknown,
+    setHeader(k: string, v: string) { this._headers[k] = v; },
+    status(code: number) { this.statusCode = code; return this; },
+    json(body: unknown) { this._body = body; return this; },
+  };
+  res.json = res.json.bind(res);
+  return res;
+}
+
+const VALID_KEY = '550e8400-e29b-41d4-a716-446655440000';
+
+// ── Tests ─────────────────────────────────────────────────────────────────────
+
+describe('IdempotencyMiddleware', () => {
+  let middleware: IdempotencyMiddleware;
+  let getEntry: jest.SpyInstance;
+  let setEntry: jest.SpyInstance;
+
+  beforeEach(() => {
+    middleware = new IdempotencyMiddleware();
+    getEntry = jest.spyOn(cache, 'getIdempotencyEntry').mockResolvedValue(null);
+    setEntry = jest.spyOn(cache, 'setIdempotencyEntry').mockResolvedValue(undefined);
+  });
+
+  afterEach(() => jest.restoreAllMocks());
+
+  test('passes through when no Idempotency-Key header', async () => {
+    const req = makeReq();
+    const res = makeRes();
+    const next = jest.fn();
+    await middleware.use(req, res, next);
+    expect(next).toHaveBeenCalled();
+    expect(getEntry).not.toHaveBeenCalled();
+  });
+
+  test('rejects malformed key (not UUID v4) with 400', async () => {
+    const req = makeReq({ headers: { 'idempotency-key': 'not-a-uuid' } });
+    await expect(middleware.use(req, makeRes(), jest.fn())).rejects.toBeInstanceOf(BadRequestException);
+  });
+
+  test('rejects key with wrong UUID version (v1)', async () => {
+    const v1Key = '550e8400-e29b-11d4-a716-446655440000'; // version digit = 1
+    const req = makeReq({ headers: { 'idempotency-key': v1Key } });
+    await expect(middleware.use(req, makeRes(), jest.fn())).rejects.toBeInstanceOf(BadRequestException);
+  });
+
+  test('cache miss: calls next and stores response on json()', async () => {
+    const req = makeReq({ headers: { 'idempotency-key': VALID_KEY } });
+    const res = makeRes();
+    const next = jest.fn();
+
+    await middleware.use(req, res, next);
+    expect(next).toHaveBeenCalled();
+
+    // Simulate handler writing a response
+    res.json({ cid: 'Qm123' });
+
+    // Wait for the async setEntry call
+    await new Promise(resolve => setImmediate(resolve));
+    expect(setEntry).toHaveBeenCalledWith(
+      expect.any(String),
+      { status: 200, body: { cid: 'Qm123' }, version: IDEMPOTENCY_VERSION },
+      expect.any(Number),
+    );
+  });
+
+  test('cache hit: replays stored response without calling next', async () => {
+    getEntry.mockResolvedValue({ status: 200, body: { cid: 'Qm123' }, version: IDEMPOTENCY_VERSION });
+    const req = makeReq({ headers: { 'idempotency-key': VALID_KEY } });
+    const res = makeRes();
+    const next = jest.fn();
+
+    await middleware.use(req, res, next);
+
+    expect(next).not.toHaveBeenCalled();
+    expect(res._body).toEqual({ cid: 'Qm123' });
+    expect(res._headers['Idempotency-Replayed']).toBe('true');
+  });
+
+  test('double-submit returns identical body on second call', async () => {
+    // First call — cache miss
+    const req1 = makeReq({ headers: { 'idempotency-key': VALID_KEY } });
+    const res1 = makeRes();
+    await middleware.use(req1, res1, jest.fn());
+    res1.json({ cid: 'QmABC' });
+    await new Promise(resolve => setImmediate(resolve));
+
+    // Capture what was stored
+    const stored = setEntry.mock.calls[0][1] as cache.IdempotencyEntry;
+
+    // Second call — cache hit
+    getEntry.mockResolvedValue(stored);
+    const req2 = makeReq({ headers: { 'idempotency-key': VALID_KEY } });
+    const res2 = makeRes();
+    const next2 = jest.fn();
+    await middleware.use(req2, res2, next2);
+
+    expect(next2).not.toHaveBeenCalled();
+    expect(res2._body).toEqual({ cid: 'QmABC' });
+  });
+
+  test('5xx responses are NOT cached', async () => {
+    const req = makeReq({ headers: { 'idempotency-key': VALID_KEY } });
+    const res = makeRes();
+    res.statusCode = 503;
+    await middleware.use(req, res, jest.fn());
+    res.json({ error: 'service_unavailable' });
+    await new Promise(resolve => setImmediate(resolve));
+    expect(setEntry).not.toHaveBeenCalled();
+  });
+
+  test('4xx error responses ARE cached (replay error to client)', async () => {
+    const req = makeReq({ headers: { 'idempotency-key': VALID_KEY } });
+    const res = makeRes();
+    res.statusCode = 400;
+    await middleware.use(req, res, jest.fn());
+    res.json({ error: 'invalid_file' });
+    await new Promise(resolve => setImmediate(resolve));
+    expect(setEntry).toHaveBeenCalledWith(
+      expect.any(String),
+      { status: 400, body: { error: 'invalid_file' }, version: IDEMPOTENCY_VERSION },
+      expect.any(Number),
+    );
+  });
+
+  test('different subjects produce different cache keys (scope isolation)', async () => {
+    const capturedKeys: string[] = [];
+    setEntry.mockImplementation(async (key: string) => { capturedKeys.push(key); });
+
+    const req1 = makeReq({ headers: { 'idempotency-key': VALID_KEY }, user: { sub: 'userA' } });
+    const res1 = makeRes();
+    await middleware.use(req1, res1, jest.fn());
+    res1.json({ ok: true });
+
+    const req2 = makeReq({ headers: { 'idempotency-key': VALID_KEY }, user: { sub: 'userB' } });
+    const res2 = makeRes();
+    await middleware.use(req2, res2, jest.fn());
+    res2.json({ ok: true });
+
+    await new Promise(resolve => setImmediate(resolve));
+    expect(capturedKeys[0]).not.toBe(capturedKeys[1]);
+  });
+
+  test('Redis unavailable: fails open and calls next', async () => {
+    getEntry.mockRejectedValue(new Error('Redis down'));
+    const req = makeReq({ headers: { 'idempotency-key': VALID_KEY } });
+    const next = jest.fn();
+    // Should not throw — fail open
+    await expect(middleware.use(req, makeRes(), next)).resolves.toBeUndefined();
+    expect(next).toHaveBeenCalled();
+  });
+
+  test('version mismatch treated as cache miss', async () => {
+    // Stored entry has old version
+    getEntry.mockResolvedValue({ status: 200, body: { old: true }, version: IDEMPOTENCY_VERSION - 1 });
+    // getIdempotencyEntry already filters by version — returns null for mismatch
+    getEntry.mockResolvedValue(null);
+
+    const req = makeReq({ headers: { 'idempotency-key': VALID_KEY } });
+    const next = jest.fn();
+    await middleware.use(req, makeRes(), next);
+    expect(next).toHaveBeenCalled();
+  });
+});

backend/src/app.module.ts

@@ -22,6 +22,7 @@ import { RequestContextMiddleware } from './common/middleware/request-context.mi
 import { AppLoggerService } from './common/logger/app-logger.service';
 import { OracleHooksController } from './experimental/oracle-hooks.controller';
 import { BetaCalculatorsController } from './experimental/beta-calculators.controller';
+import { IdempotencyMiddleware } from './common/middleware/idempotency.middleware';
 
 @Module({
   imports: [

backend/src/common/middleware/idempotency.middleware.ts

@@ -0,0 +1,112 @@
+/**
+ * IdempotencyMiddleware — safe retries for POST /ipfs/upload and POST /tx/submit.
+ *
+ * ## How it works
+ *
+ * 1. Client sends a POST with `Idempotency-Key: <uuid-v4>` header.
+ * 2. Middleware hashes `SHA-256(method + path + key + subject)` to form the
+ *    Redis cache key.  Scoping by subject prevents one user replaying another's
+ *    key on the same endpoint.
+ * 3. On cache hit (same status + body stored): response is replayed immediately,
+ *    no handler is invoked.  Header `Idempotency-Replayed: true` is set.
+ * 4. On cache miss: request proceeds normally.  The response interceptor stores
+ *    the result before flushing to the client.
+ *
+ * ## Client responsibilities
+ *
+ * - Generate a fresh UUID v4 per *logical* operation (not per retry).
+ * - Reuse the same key on retries of the same operation.
+ * - Keys must be 36 characters (UUID v4 format: xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx).
+ *   Shorter/longer keys are rejected with 400.
+ * - Key collision probability with UUID v4 is negligible (~1 in 5.3×10³⁶).
+ *
+ * ## TTL and eviction
+ *
+ * Cached responses expire after `TTL.IDEMPOTENCY_SECONDS` (24 h).  After expiry
+ * the key is evicted and a fresh request is processed normally.  TTL is always
+ * set unconditionally — Redis growth is bounded.
+ *
+ * ## Schema versioning
+ *
+ * `IDEMPOTENCY_VERSION` is embedded in every cached entry.  Bump it when the
+ * response shape of a covered endpoint changes.  Old entries with a stale
+ * version are treated as cache misses and overwritten.
+ *
+ * ## Redis unavailability (FAIL OPEN)
+ *
+ * If Redis is down, the middleware logs a warning and lets the request through
+ * normally.  A duplicate submission may be processed, but the service stays
+ * available.  Operators should alert on Redis unavailability and restore quickly.
+ * This behaviour is intentional and documented — idempotency is best-effort
+ * when the cache layer is unavailable.
+ *
+ * Contrast with nonce storage (auth), which FAILS CLOSED.
+ */
+
+import { Injectable, NestMiddleware, BadRequestException, Logger } from '@nestjs/common';
+import { Request, Response, NextFunction } from 'express';
+import { createHash } from 'crypto';
+import { getIdempotencyEntry, setIdempotencyEntry } from '../redis/cache';
+import { TTL } from '../redis/config';
+
+/** Bump this when any covered endpoint's response schema changes. */
+export const IDEMPOTENCY_VERSION = 1;
+
+/** UUID v4 pattern — the only accepted key format. */
+const UUID_V4_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
+
+@Injectable()
+export class IdempotencyMiddleware implements NestMiddleware {
+  private readonly logger = new Logger(IdempotencyMiddleware.name);
+
+  async use(req: Request, res: Response, next: NextFunction): Promise<void> {
+    const rawKey = req.headers['idempotency-key'] as string | undefined;
+
+    // No key — pass through (idempotency is opt-in)
+    if (!rawKey) {
+      next();
+      return;
+    }
+
+    // Validate format
+    if (!UUID_V4_RE.test(rawKey)) {
+      throw new BadRequestException(
+        'Idempotency-Key must be a valid UUID v4 (xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx)',
+      );
+    }
+
+    // Build scoped cache key: hash(method + path + rawKey + subject)
+    // Subject is the authenticated wallet address when present, otherwise 'anon'.
+    const subject: string = (req as Request & { user?: { sub?: string } }).user?.sub ?? 'anon';
+    const cacheKey = createHash('sha256')
+      .update(`${req.method}:${req.path}:${rawKey}:${subject}`)
+      .digest('hex');
+
+    // Cache hit — replay stored response
+    const cached = await getIdempotencyEntry(cacheKey, IDEMPOTENCY_VERSION);
+    if (cached) {
+      this.logger.debug(`Idempotency replay: key=${rawKey} subject=${subject} path=${req.path}`);
+      res.setHeader('Idempotency-Replayed', 'true');
+      res.status(cached.status).json(cached.body);
+      return;
+    }
+
+    // Cache miss — intercept the response to store it
+    const originalJson = res.json.bind(res);
+    res.json = (body: unknown): Response => {
+      // Only cache 2xx and 4xx responses; never cache 5xx (transient errors)
+      if (res.statusCode < 500) {
+        setIdempotencyEntry(
+          cacheKey,
+          { status: res.statusCode, body, version: IDEMPOTENCY_VERSION },
+          TTL.IDEMPOTENCY_SECONDS,
+        ).catch((err: unknown) => {
+          this.logger.warn(`Failed to store idempotency entry: ${String(err)}`);
+        });
+      }
+      return originalJson(body);
+    };
+
+    next();
+  }
+}

backend/src/main.ts

@@ -67,7 +67,7 @@ async function bootstrap() {
     },
     credentials: true,
     methods: ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"],
-    allowedHeaders: ["Authorization", "Content-Type", "X-Requested-With"],
+    allowedHeaders: ["Authorization", "Content-Type", "X-Requested-With", "Idempotency-Key"],
     maxAge: 86400,
     preflightContinue: false,
     optionsSuccessStatus: 204,

backend/src/redis/cache.ts

@@ -145,3 +145,43 @@ export async function incrementRateLimit(identifier: string): Promise<number> {
     return 0;
   }
 }
+
+// ── Idempotency (FAIL OPEN) ───────────────────────────────────────────────────
+//
+// Idempotency keys map a hashed (method + path + key + subject) to a cached
+// response envelope { status, body, version }.  If Redis is unavailable the
+// request is processed normally (fail open) — a duplicate may go through, but
+// the service remains available.  This is documented behaviour; clients must
+// treat Redis unavailability as a best-effort guarantee.
+
+export interface IdempotencyEntry {
+  status: number;
+  body: unknown;
+  /** Schema version — bump when response shape changes to invalidate old entries. */
+  version: number;
+}
+
+/**
+ * Store an idempotency response.  TTL is always set (bounded Redis growth).
+ * Silently swallows Redis errors (fail open).
+ */
+export async function setIdempotencyEntry(
+  key: string,
+  entry: IdempotencyEntry,
+  ttlSeconds: number,
+): Promise<void> {
+  await cacheSet(`idempotency:${key}`, entry, ttlSeconds);
+}
+
+/**
+ * Retrieve a cached idempotency response.
+ * Returns null on cache miss, Redis error, or version mismatch.
+ */
+export async function getIdempotencyEntry(
+  key: string,
+  currentVersion: number,
+): Promise<IdempotencyEntry | null> {
+  const entry = await cacheGet<IdempotencyEntry>(`idempotency:${key}`);
+  if (!entry || entry.version !== currentVersion) return null;
+  return entry;
+}

backend/src/redis/config.ts

@@ -74,4 +74,16 @@ export const TTL = {
   POLICY_CACHE_SECONDS: 30,
   /** Claim read cache. Lower TTL because claim status changes frequently. */
   CLAIM_CACHE_SECONDS: 10,
+  /**
+   * Idempotency key TTL: 24 hours.
+   *
+   * A client may safely retry any idempotent POST within this window and
+   * receive the exact same status code + body as the original response.
+   * After expiry the key is evicted and a fresh request is processed normally.
+   *
+   * Eviction note: TTL is always set unconditionally — Redis growth is bounded.
+   * Schema versioning: if a response schema changes, bump IDEMPOTENCY_VERSION
+   * in idempotency.middleware.ts; old cached entries will be ignored.
+   */
+  IDEMPOTENCY_SECONDS: 24 * 60 * 60,
 } as const;

backend/src/redis/index.ts

@@ -19,5 +19,8 @@ export {
   setNonce,
   consumeNonce,
   incrementRateLimit,
+  setIdempotencyEntry,
+  getIdempotencyEntry,
+  type IdempotencyEntry,
 } from "./cache";
 export { collectRedisMetrics } from "./metrics";