Perf/stress test findings revisions (#623)

Author: nofurtherinformation

backend/app/core/db.py

@@ -6,6 +6,10 @@
 engine = create_engine(
     str(settings.SQLALCHEMY_DATABASE_URI),
     echo=settings.ECHO_DB,
+    # 60/task; RDS max_connections ~900 caps us at ~14 tasks. Keep in step
+    # with the anyio threadpool limiter (app/main.py lifespan).
+    pool_size=40,
+    max_overflow=20,
     pool_pre_ping=True,
     pool_recycle=3600,
 )

backend/app/evaluation/graph.py

@@ -53,7 +53,9 @@ def get_gerrydb_graph(file_path: str) -> Graph:
         return pickle.load(f)
 
 
-_GRAPH_CACHE_MAX_SIZE = 10
+# Must exceed the distinct-map working set or evictions force multi-second
+# cold S3 reloads; each cached graph costs real memory, so raise with care.
+_GRAPH_CACHE_MAX_SIZE = 15
 
 
 @lru_cache(maxsize=_GRAPH_CACHE_MAX_SIZE)

backend/app/main.py

@@ -8,6 +8,7 @@
 )
 from fastapi.responses import JSONResponse, Response
 from typing import Annotated, Any
+import anyio
 import msgpack
 import psutil
 import time
@@ -116,6 +117,9 @@
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
+    # Sync-route concurrency; default 40 would cap below the DB pool
+    # (60/task, app/core/db.py). Needs a running event loop, hence lifespan.
+    anyio.to_thread.current_default_thread_limiter().total_tokens = 80
     yield
 
 

backend/stress_test/runner/provision.sh

@@ -20,6 +20,14 @@ REPO_SHA="${REPO_SHA:?Set REPO_SHA to the pinned commit containing backend/stres
 # Backend S3 bucket (pulumi config s3BucketName / task env R2_BUCKET_NAME);
 # artifacts live under stress-test/.
 RESULTS_BUCKET="${RESULTS_BUCKET:?Set RESULTS_BUCKET to the backend S3 bucket name}"
+# Easy slip: the ALB access-logs bucket, which the IAM policy can't write to.
+case "$RESULTS_BUCKET" in
+  *alb-logs*)
+    echo "RESULTS_BUCKET ($RESULTS_BUCKET) looks like the ALB access-logs bucket;" \
+      "use the backend data bucket (pulumi config s3BucketName, e.g. districtr-cdn-data-f1eb3d1)" >&2
+    exit 1
+    ;;
+esac
 
 export AWS_DEFAULT_REGION="$REGION"
 

backend/stress_test/runner/run.sh

@@ -13,6 +13,14 @@ RUN_ID="${RUN_ID:?Set RUN_ID}"
 SCALE="${SCALE:-0.01}"
 WINDOW_SECONDS="${WINDOW_SECONDS:-900}"
 RESULTS_BUCKET="${RESULTS_BUCKET:?Set RESULTS_BUCKET to the backend S3 bucket name}"
+# Same guard as provision.sh: the ALB access-logs bucket is not writable here.
+case "$RESULTS_BUCKET" in
+  *alb-logs*)
+    echo "RESULTS_BUCKET ($RESULTS_BUCKET) looks like the ALB access-logs bucket;" \
+      "use the backend data bucket (e.g. districtr-cdn-data-f1eb3d1)" >&2
+    exit 1
+    ;;
+esac
 BASE_URL="${BASE_URL:-https://api.beta.districtr.org}"
 CLUSTER="${CLUSTER:-districtr-prod}"
 export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-2}"

infra/backend.ts

@@ -169,6 +169,24 @@ export function createBackend(
     },
   });
 
+  // The workload is I/O-bound, so CPU stays low while requests queue and the
+  // policy above never fires. ECS scales out when either policy demands it.
+  new aws.appautoscaling.Policy(`${name}-backend-req-scaling-policy`, {
+    policyType: "TargetTrackingScaling",
+    serviceNamespace: scalingTarget.serviceNamespace,
+    scalableDimension: scalingTarget.scalableDimension,
+    resourceId: scalingTarget.resourceId,
+    targetTrackingScalingPolicyConfiguration: {
+      predefinedMetricSpecification: {
+        predefinedMetricType: "ALBRequestCountPerTarget",
+        resourceLabel: pulumi.interpolate`${alb.alb.arnSuffix}/${alb.backendTargetGroup.arnSuffix}`,
+      },
+      targetValue: config.backendRequestsPerTarget,
+      scaleOutCooldown: 60,
+      scaleInCooldown: 300,
+    },
+  });
+
   return {service};
 }
 

infra/config.ts

@@ -60,6 +60,8 @@ export const config = {
   backendMemory: cfg.getNumber("backendMemory") ?? 8192,
   backendMinCount: cfg.getNumber("backendMinCount") ?? (isProd ? 2 : 1),
   backendMaxCount: cfg.getNumber("backendMaxCount") ?? (isProd ? 6 : 2),
+  // ALB requests/min per backend task before scale-out.
+  backendRequestsPerTarget: cfg.getNumber("backendRequestsPerTarget") ?? 600,
   frontendCpu: cfg.getNumber("frontendCpu") ?? (isProd ? 1024 : 512),
   frontendMemory: cfg.getNumber("frontendMemory") ?? 2048,
   frontendMinCount: cfg.getNumber("frontendMinCount") ?? (isProd ? 2 : 1),