feat(#217): OpenTelemetry distributed tracing for HTTP, DB, Redis, and Soroban RPC

- Add @opentelemetry/sdk-node with auto-instrumentation (HTTP, pg, ioredis)
- Configure OTLP exporter via OTEL_EXPORTER_OTLP_ENDPOINT env var; no-op when unset
- Configurable sampling ratio via OTEL_SAMPLING_RATIO (default 1.0 dev / 0.1 prod)
- Propagate x-request-id as span attribute 'request.id' for log/trace correlation
- Add withSorobanSpan() helper for Soroban RPC simulation spans (contractId + method only)
- Sensitive data policy: XDR and private keys never appear as span attributes
- Import tracing.ts before all other modules in main.ts for correct patching order
- Add local Jaeger dev setup docs and environment variable reference

Closes #217

Author: Coredevjay

backend/docs/opentelemetry-tracing.md

@@ -0,0 +1,62 @@
+# OpenTelemetry Tracing
+
+Distributed tracing is implemented via `@opentelemetry/sdk-node` with auto-instrumentation for HTTP, Prisma/PostgreSQL, and Redis (ioredis). Soroban RPC simulation calls are wrapped with a custom span helper.
+
+## Environment Variables
+
+| Variable | Default | Description |
+|---|---|---|
+| `OTEL_EXPORTER_OTLP_ENDPOINT` | _(unset — no-op)_ | OTLP HTTP endpoint, e.g. `http://localhost:4318` |
+| `OTEL_SERVICE_NAME` | `niffyinsure-backend` | Service name in traces |
+| `OTEL_SAMPLING_RATIO` | `1.0` (dev) / `0.1` (prod) | Head-sampling ratio 0.0–1.0 |
+
+When `OTEL_EXPORTER_OTLP_ENDPOINT` is not set, the SDK runs with a no-op exporter — no traces are exported and there is no performance overhead.
+
+## Local Development with Jaeger
+
+Run Jaeger all-in-one (includes OTLP HTTP receiver on port 4318):
+
+```bash
+docker run --rm -d \
+  -p 16686:16686 \
+  -p 4318:4318 \
+  --name jaeger \
+  jaegertracing/all-in-one:latest
+```
+
+Then set in your `.env`:
+
+```
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
+OTEL_SERVICE_NAME=niffyinsure-backend
+OTEL_SAMPLING_RATIO=1.0
+```
+
+Open the Jaeger UI at http://localhost:16686 and select the `niffyinsure-backend` service.
+
+## Trace Correlation with Logs
+
+Every incoming request attaches `x-request-id` as a span attribute (`request.id`). Structured log entries include the same `requestId` field, enabling correlation between traces and logs.
+
+## Sensitive Data Policy
+
+- XDR payloads and private keys **must never** appear as span attributes.
+- Request/response bodies are not captured by auto-instrumentation (body capture hooks are disabled).
+- Only `soroban.contract_id` and `soroban.method` are recorded for Soroban RPC spans.
+
+## Soroban RPC Spans
+
+Use `withSorobanSpan` from `src/common/tracing/soroban-span.ts` to wrap simulation calls:
+
+```typescript
+import { withSorobanSpan } from '@/common/tracing/soroban-span'
+
+const result = await withSorobanSpan(
+  { contractId: 'C...', method: 'simulateTransaction', requestId },
+  () => server.simulateTransaction(tx),
+)
+```
+
+## Production Sampling
+
+The default production sampling ratio is `0.1` (10%). Adjust via `OTEL_SAMPLING_RATIO` without redeployment.

backend/package.json

@@ -47,6 +47,13 @@
     "@nestjs/swagger": "^7.4.2",
     "@nestjs/terminus": "^10.2.1",
     "@nestjs/throttler": "^6.5.0",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/auto-instrumentations-node": "^0.54.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.57.0",
+    "@opentelemetry/resources": "^1.30.0",
+    "@opentelemetry/sdk-node": "^0.57.0",
+    "@opentelemetry/sdk-trace-base": "^1.30.0",
+    "@opentelemetry/semantic-conventions": "^1.28.0",
     "@prisma/client": "^6.6.0",
     "@stellar/stellar-sdk": "^14.6.1",
     "@types/form-data": "^2.2.1",

backend/src/common/middleware/request-context.middleware.ts

@@ -1,6 +1,7 @@
 import { Injectable, NestMiddleware } from '@nestjs/common';
 import { Request, Response, NextFunction } from 'express';
 import { randomUUID } from 'crypto';
+import { trace, context as otelContext, SpanStatusCode } from '@opentelemetry/api';
 import { MetricsService } from '../../metrics/metrics.service';
 import { AppLoggerService, redactHeaders } from '../logger/app-logger.service';
 
@@ -34,6 +35,12 @@ export class RequestContextMiddleware implements NestMiddleware {
     // Echo back so clients can correlate
     res.setHeader('x-request-id', requestId);
 
+    // Propagate requestId as a span attribute on the active OTel span (if any)
+    const activeSpan = trace.getActiveSpan()
+    if (activeSpan) {
+      activeSpan.setAttribute('request.id', requestId)
+    }
+
     const start = Date.now();
 
     this.logger.structured('info', 'request_received', {

backend/src/common/tracing/soroban-span.ts

@@ -0,0 +1,61 @@
+/**
+ * Utility for wrapping Soroban RPC simulation calls with an OpenTelemetry span.
+ *
+ * Usage:
+ *   const result = await withSorobanSpan(
+ *     { contractId: 'C...', method: 'vote' },
+ *     () => server.simulateTransaction(tx),
+ *   )
+ *
+ * Sensitive data policy:
+ *   - XDR payloads and private keys MUST NOT be passed as span attributes.
+ *   - Only contractId and method are recorded.
+ */
+
+import { trace, SpanStatusCode, context } from '@opentelemetry/api'
+
+const tracer = trace.getTracer('soroban-rpc')
+
+export interface SorobanSpanOptions {
+  /** Contract ID (C... address) — safe to record as a span attribute. */
+  contractId: string
+  /** RPC method name, e.g. "simulateTransaction", "sendTransaction". */
+  method: string
+  /** Optional x-request-id for correlation with structured logs. */
+  requestId?: string
+}
+
+/**
+ * Wraps a Soroban RPC call in an OTel span.
+ * Records contractId and method as span attributes.
+ * Never records XDR, private keys, or other sensitive parameters.
+ */
+export async function withSorobanSpan<T>(
+  opts: SorobanSpanOptions,
+  fn: () => Promise<T>,
+): Promise<T> {
+  const span = tracer.startSpan(`soroban.${opts.method}`, {
+    attributes: {
+      'soroban.contract_id': opts.contractId,
+      'soroban.method': opts.method,
+      ...(opts.requestId ? { 'request.id': opts.requestId } : {}),
+    },
+  })
+
+  return context.with(trace.setSpan(context.active(), span), async () => {
+    try {
+      const result = await fn()
+      span.setStatus({ code: SpanStatusCode.OK })
+      return result
+    } catch (err) {
+      span.setStatus({
+        code: SpanStatusCode.ERROR,
+        message: err instanceof Error ? err.message : String(err),
+      })
+      span.recordException(err instanceof Error ? err : new Error(String(err)))
+      throw err
+    } finally {
+      span.end()
+    }
+  })
+}

backend/src/main.ts

@@ -1,3 +1,7 @@
+// OpenTelemetry instrumentation MUST be imported before any other module
+// so that auto-instrumentation patches are applied at load time.
+import './tracing'
+
 import { NestFactory } from "@nestjs/core";
 import { ValidationPipe, Logger } from "@nestjs/common";
 import { SwaggerModule, DocumentBuilder } from "@nestjs/swagger";

backend/src/tracing.ts

@@ -0,0 +1,74 @@
+/**
+ * OpenTelemetry instrumentation bootstrap.
+ *
+ * This file MUST be imported before any other module (i.e. at the very top of
+ * main.ts) so that auto-instrumentation patches are applied before the
+ * libraries they instrument are loaded.
+ *
+ * Configuration via environment variables:
+ *   OTEL_EXPORTER_OTLP_ENDPOINT  — OTLP gRPC/HTTP endpoint (e.g. http://localhost:4318)
+ *                                   Defaults to no-op (no export) when unset.
+ *   OTEL_SERVICE_NAME             — Service name reported in traces (default: niffyinsure-backend)
+ *   OTEL_SAMPLING_RATIO           — Head-sampling ratio 0.0–1.0 (default: 1.0 in dev, 0.1 in prod)
+ *
+ * Sensitive data policy:
+ *   - XDR payloads and private keys MUST NOT appear as span attributes.
+ *   - Request bodies are never captured by auto-instrumentation (HTTP body capture is disabled).
+ */
+
+import { NodeSDK } from '@opentelemetry/sdk-node'
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'
+import { Resource } from '@opentelemetry/resources'
+import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions'
+import { TraceIdRatioBasedSampler } from '@opentelemetry/sdk-trace-base'
+import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node'
+
+const SERVICE_NAME = process.env.OTEL_SERVICE_NAME ?? 'niffyinsure-backend'
+const OTLP_ENDPOINT = process.env.OTEL_EXPORTER_OTLP_ENDPOINT
+
+// Sampling: configurable without redeployment via env var.
+// Default: 1.0 (sample everything) unless NODE_ENV=production, then 0.1.
+const defaultRatio = process.env.NODE_ENV === 'production' ? 0.1 : 1.0
+const samplingRatio = parseFloat(process.env.OTEL_SAMPLING_RATIO ?? String(defaultRatio))
+
+// Only configure an exporter when an endpoint is explicitly set.
+// In development (no endpoint), the SDK runs with a no-op exporter.
+const traceExporter = OTLP_ENDPOINT
+  ? new OTLPTraceExporter({ url: `${OTLP_ENDPOINT}/v1/traces` })
+  : undefined
+
+const sdk = new NodeSDK({
+  resource: new Resource({
+    [ATTR_SERVICE_NAME]: SERVICE_NAME,
+    [ATTR_SERVICE_VERSION]: process.env.npm_package_version ?? '0.0.0',
+  }),
+  sampler: new TraceIdRatioBasedSampler(samplingRatio),
+  ...(traceExporter ? { traceExporter } : {}),
+  instrumentations: [
+    getNodeAutoInstrumentations({
+      // HTTP instrumentation — captures incoming/outgoing HTTP spans.
+      // Body capture is disabled to prevent XDR/key leakage.
+      '@opentelemetry/instrumentation-http': {
+        enabled: true,
+        // Do not capture request/response bodies
+        requestHook: () => undefined,
+        responseHook: () => undefined,
+      },
+      // Prisma / pg instrumentation for DB spans
+      '@opentelemetry/instrumentation-pg': { enabled: true },
+      // Redis instrumentation for cache spans
+      '@opentelemetry/instrumentation-ioredis': { enabled: true },
+      // Disable noisy fs instrumentation
+      '@opentelemetry/instrumentation-fs': { enabled: false },
+    }),
+  ],
+})
+
+sdk.start()
+
+// Graceful shutdown
+process.on('SIGTERM', () => {
+  sdk.shutdown().catch((err) => console.error('OTel shutdown error', err))
+})
+
+export { sdk }