Add ephemeral EC2 Harbor backend

.github/workflows/modal-deploy.yml

@@ -42,6 +42,22 @@ jobs:
       ODDISH_GKE_AUTO_PROVISION_CLUSTER: "true"
       ODDISH_GKE_AUTO_BUILD_MISSING_IMAGE: "true"
       ODDISH_GKE_IDLE_CLUSTER_TTL_HOURS: "1"
+      # Ephemeral Harbor CPU sandboxes. Credentials and the SSH private key are
+      # attached from dedicated Modal secrets; only platform-owned launch
+      # coordinates are declared in the deployment environment.
+      ODDISH_EC2_ENABLED: "true"
+      ODDISH_EC2_CONTROL_SECRET_NAME: oddish-ec2-control
+      ODDISH_EC2_SSH_SECRET_NAME: oddish-ec2-ssh
+      ODDISH_EC2_REGION: us-west-2
+      ODDISH_EC2_AMI_ID: ami-0ac74609c6396bed3
+      ODDISH_EC2_INSTANCE_TYPE: m7i-flex.2xlarge
+      ODDISH_EC2_SUBNET_ID: subnet-0da0349bc81aa34d8
+      ODDISH_EC2_SECURITY_GROUP_IDS: '["sg-04d0011ae0f6b6c00"]'
+      ODDISH_EC2_KEY_NAME: oddish-harbor
+      ODDISH_EC2_SSH_USER: ubuntu
+      ODDISH_EC2_ROOT_VOLUME_SIZE_GB: "80"
+      ODDISH_EC2_USE_PUBLIC_IP: "true"
+      ODDISH_EC2_BOOTSTRAP_DOCKER: "true"
       # Point uv at the .venv pre-built into the base image.  `uv sync
       # --frozen` then validates that venv against the current lockfile
       # and only patches the editable oddish path — typically a no-op

.github/workflows/pr-preview.yml

@@ -174,6 +174,20 @@ jobs:
       ODDISH_MODAL_WORKER_BUFFER_CONTAINERS: "0"
       ODDISH_MODAL_WORKER_MAX_CONTAINERS: "2"
       ODDISH_GATE_LLM_ON_BASELINES: "1"
+      ODDISH_EC2_ENABLED: "true"
+      ODDISH_EC2_CONTROL_SECRET_NAME: oddish-ec2-control
+      ODDISH_EC2_SSH_SECRET_NAME: oddish-ec2-ssh
+      ODDISH_EC2_REGION: us-west-2
+      ODDISH_EC2_AMI_ID: ami-0ac74609c6396bed3
+      ODDISH_EC2_INSTANCE_TYPE: m7i-flex.2xlarge
+      ODDISH_EC2_SUBNET_ID: subnet-0da0349bc81aa34d8
+      ODDISH_EC2_SECURITY_GROUP_IDS: '["sg-04d0011ae0f6b6c00"]'
+      ODDISH_EC2_KEY_NAME: oddish-harbor
+      ODDISH_EC2_SSH_USER: ubuntu
+      ODDISH_EC2_ROOT_VOLUME_SIZE_GB: "80"
+      ODDISH_EC2_USE_PUBLIC_IP: "true"
+      ODDISH_EC2_BOOTSTRAP_DOCKER: "true"
+      ODDISH_EC2_MAX_CONCURRENT_INSTANCES: "2"
     defaults:
       run:
         working-directory: backend

.github/workflows/staging-deploy.yml

@@ -61,6 +61,22 @@ jobs:
       ODDISH_GKE_AUTO_PROVISION_CLUSTER: "true"
       ODDISH_GKE_AUTO_BUILD_MISSING_IMAGE: "true"
       ODDISH_GKE_IDLE_CLUSTER_TTL_HOURS: "1"
+      # Ephemeral Harbor CPU sandboxes. Credentials and the SSH private key are
+      # attached from dedicated Modal secrets; only platform-owned launch
+      # coordinates are declared in the deployment environment.
+      ODDISH_EC2_ENABLED: "true"
+      ODDISH_EC2_CONTROL_SECRET_NAME: oddish-ec2-control
+      ODDISH_EC2_SSH_SECRET_NAME: oddish-ec2-ssh
+      ODDISH_EC2_REGION: us-west-2
+      ODDISH_EC2_AMI_ID: ami-0ac74609c6396bed3
+      ODDISH_EC2_INSTANCE_TYPE: m7i-flex.2xlarge
+      ODDISH_EC2_SUBNET_ID: subnet-0da0349bc81aa34d8
+      ODDISH_EC2_SECURITY_GROUP_IDS: '["sg-04d0011ae0f6b6c00"]'
+      ODDISH_EC2_KEY_NAME: oddish-harbor
+      ODDISH_EC2_SSH_USER: ubuntu
+      ODDISH_EC2_ROOT_VOLUME_SIZE_GB: "80"
+      ODDISH_EC2_USE_PUBLIC_IP: "true"
+      ODDISH_EC2_BOOTSTRAP_DOCKER: "true"
       UV_PROJECT_ENVIRONMENT: /opt/venvs/backend
       # Low caps: staging carries no production traffic.
       ODDISH_MODAL_API_MIN_CONTAINERS: "0"

AGENTS.md

@@ -514,6 +514,49 @@ Settings are loaded from `oddish/.env`; see `oddish/env.example`,
 Keep these routing rules in sync with `oddish/src/oddish/config.py` and
 `oddish/src/oddish/workers/harbor/runner.py`:
 
+- EC2 is an explicit, opt-in Harbor backend: `ODDISH_EC2_ENABLED=true` registers
+  it and permits hosted `environment=ec2`, but capability ordering keeps Daytona
+  as the CPU default. V1 launches one ephemeral CPU instance per trial and uses
+  public-IP, key-only SSH. It does not support accelerators, attach/retain mode,
+  private networking, Spot, or AWS infrastructure provisioning.
+- An EC2 deployment must provide an existing Ubuntu-compatible AMI, subnet,
+  security group, EC2 key pair/private key, region, and instance type. The
+  security group must allow TCP/22 from the Modal worker network path. Keep
+  `ODDISH_EC2_SSH_PRIVATE_KEY` in the dedicated worker secret, materialize it
+  mode `0600`, and never bake it into an image or attach it to API, dispatcher,
+  or reconciler functions.
+- EC2 control credentials must be least privilege: workers need
+  `sts:GetCallerIdentity` plus launch, describe, image lookup, tagging, and
+  termination actions; reconciliation needs `sts:GetCallerIdentity`, describe,
+  and tag-scoped termination. Store them under the namespaced
+  `ODDISH_EC2_AWS_*` settings; workers materialize a mode-`0600` AWS profile and
+  scrub the raw values before starting Harbor. API cancellation delegates to a
+  dedicated Modal teardown function, so API and dispatcher containers receive
+  neither EC2 control nor SSH secrets. An optional platform-owned
+  `ODDISH_EC2_INSTANCE_PROFILE` may be attached; it is visible to tenant code,
+  so keep it task-scoped and grant the control identity `iam:PassRole` only for
+  that role. Oddish always requires IMDSv2 so cloud-init can retrieve the EC2
+  launch key: the response hop limit is one without an instance profile and two
+  when a profile is explicitly exposed to Docker containers.
+- Oddish does not create the VPC, subnet, security group, AMI, key pair, or IAM
+  policy. Every instance and root volume must carry protected Oddish ownership,
+  deployment, task/trial, worker-job, worker-attempt, sandbox-run, unguessable
+  launch-token, and Harbor-session tags. A durable `sandbox_runs` row is created
+  before launch; Harbor's `environment-provisioned` event binds the structured
+  handle before SSH/bootstrap. Normal teardown, cancellation, stale-heartbeat
+  cleanup, and reconciliation terminate only after the full ledger/tag tuple
+  agrees.
+- EC2 orphan reconciliation snapshots deployment-tagged instances before the
+  shared cleanup transaction, evaluates worker liveness using the database clock,
+  and terminates only after the transaction commits. It preserves live linked
+  jobs and conservatively preserves unlinked trial startup for 30 minutes, then
+  reaps terminal and stale owners with an exact ledger match; missing or
+  mismatched ledgers are ownership refusals, never destructive guesses. The
+  protected 14-hour hard maximum age overrides worker liveness only for exactly
+  owned instances. `ODDISH_EC2_MAX_CONCURRENT_INSTANCES` is enforced globally
+  with heartbeat-renewed `sandbox_capacity_leases`, independent of model/variant
+  queue slots. Inventory and termination failures stay visible in logs/metrics
+  while the rest of queue cleanup continues.
 - Claude trials run through AWS Bedrock by default. `CLAUDE_CODE_USE_BEDROCK=1` is
   baked into the Modal image, and Claude model aliases must normalize to an
   invokable inference profile (`global.` / `us.` / ARN) via
@@ -568,6 +611,27 @@ Storage defaults:
 - Harbor job outputs: `/tmp/harbor-jobs`
 - Modal workers also check `/mnt/oddish-tasks` before falling back to the S3 download path
 
+EC2 canary procedure:
+
+1. In a non-production AWS account, create the Ubuntu-compatible AMI, subnet,
+   public-IP route, SSH security group, key pair, and least-privilege worker IAM
+   credentials. Enable the backend with the `ODDISH_EC2_*` settings documented
+   in `backend/.env.example`.
+2. Submit a small CPU-only task with `oddish run <task> --env ec2 --background`.
+   Confirm the trial records provider `ec2` and an external instance handle, and
+   confirm the instance and root volume have the protected Oddish tags.
+3. Verify SSH/bootstrap, Docker Compose execution, result/artifact collection,
+   and terminal instance state. Confirm the instance has the configured IAM
+   profile (or none), and that metadata is IMDSv2-only with response hop limit
+   one without a profile or two with a profile.
+4. Start a longer canary, cancel it with `oddish cancel <trial-or-task-id>`, and
+   confirm the tagged instance terminates exactly once.
+5. In the non-production deployment only, deliberately interrupt a worker after
+   launch. Confirm stale-heartbeat/orphan reconciliation preserves it during the
+   grace window and terminates it afterward. Also verify the hard maximum-age
+   path. Review logs/metrics for the candidate, ownership decision, and terminate
+   result before enabling production traffic.
+
 ### Using as a Library
 
 ```python

DOCS.md

@@ -85,7 +85,7 @@ Options
 - `--task-name`, `-t TEXT` - Include task glob filter; can be passed multiple times
 - `--exclude-task-name`, `-x TEXT` - Exclude task glob filter; can be passed multiple times
 - `--n-tasks`, `-l INTEGER` - Limit the number of selected tasks after filtering
-- `--env`, `-e` - Execution environment: `docker`, `daytona`, `e2b`, `modal`, `runloop`, or `gke`
+- `--env`, `-e` - Execution environment: `docker`, `daytona`, `ec2`, `e2b`, `modal`, `runloop`, or `gke`. Hosted EC2 is opt-in and must be enabled by the deployment operator; Daytona remains the CPU default.
 - `--priority`, `-P TEXT` - Queue priority, typically `low` or `high`
 - `--experiment`, `-E TEXT` - Reuse or create an experiment ID/name
 - `--user`, `-u TEXT` - Override the author attached to the run. Defaults to the authenticated identity (Clerk-linked email for API keys / dashboard sessions); set this only to attribute a run to someone other than yourself.
@@ -123,6 +123,22 @@ Options
 - `--api TEXT` - Override the API URL
 - `--json` - Emit JSON for scripts and CI; implies `--background`
 
+### Run on ephemeral EC2
+
+An EC2-enabled deployment can run a trial on one disposable CPU VM by selecting
+the backend explicitly:
+
+```bash
+oddish run ./my-task --env ec2 -a claude-code -m anthropic/claude-sonnet-4-5
+```
+
+The hosted API rejects `--env ec2` when its operator has not enabled and fully
+configured the backend. EC2 is not an automatic fallback: CPU-only hosted runs
+without `--env` continue to use Daytona. V1 does not accept GPU/TPU requests,
+attach mode, retained instances, or caller overrides of platform EC2 settings.
+It uses a public address and key-only SSH; the instance is terminated after the
+trial or cancellation.
+
 ### Re-run with `--retry`
 
 `oddish run --retry` re-runs existing work instead of submitting new trials. It

backend/.env.example

@@ -41,6 +41,42 @@ ODDISH_S3_ENDPOINT_URL=...
 # ODDISH_SAURON_S3_BUCKET=abundant-github-workflows-bucket
 # ODDISH_SAURON_S3_ORG=oddish       # top-level segment for CLI runs: {org}/runs/run-{id}/
 
+# =============================================================================
+# EC2 Harbor backend (optional, CPU-only)
+# =============================================================================
+# Opt-in one-instance-per-trial backend. Daytona remains the default for hosted
+# CPU tasks; EC2 runs only when a request explicitly selects --env ec2. V1 uses
+# public-IP, key-only SSH and requires pre-existing AWS infrastructure.
+# ODDISH_EC2_ENABLED=true
+# ODDISH_EC2_REGION=us-east-1
+# ODDISH_EC2_AMI_ID=ami-...
+# ODDISH_EC2_INSTANCE_TYPE=m7i-flex.large
+# ODDISH_EC2_SUBNET_ID=subnet-...
+# ODDISH_EC2_SECURITY_GROUP_IDS=["sg-..."]
+# ODDISH_EC2_KEY_NAME=oddish-harbor
+# ODDISH_EC2_SSH_USER=ubuntu
+# Optional platform-owned instance-profile name or ARN. Enabling this exposes
+# the profile to sandbox code via IMDSv2; keep its permissions task-scoped.
+# ODDISH_EC2_INSTANCE_PROFILE=oddish-harbor-sandbox
+# ODDISH_EC2_ROOT_VOLUME_SIZE_GB=80
+# ODDISH_EC2_USE_PUBLIC_IP=true
+# ODDISH_EC2_BOOTSTRAP_DOCKER=true
+# ODDISH_EC2_MAX_CONCURRENT_INSTANCES=16
+# Deploy-time Modal secret names (the secrets themselves are not stored here):
+# ODDISH_EC2_CONTROL_SECRET_NAME=oddish-ec2-control
+# ODDISH_EC2_SSH_SECRET_NAME=oddish-ec2-ssh
+# The control secret contains ODDISH_EC2_AWS_ACCESS_KEY_ID,
+# ODDISH_EC2_AWS_SECRET_ACCESS_KEY, and optional ODDISH_EC2_AWS_SESSION_TOKEN.
+# The SSH secret contains only ODDISH_EC2_SSH_PRIVATE_KEY.
+# The matching security group must admit TCP/22 from the Modal worker path.
+# EC2 control credentials need least-privilege sts:GetCallerIdentity plus
+# RunInstances, DescribeInstances, DescribeInstanceStatus, DescribeImages,
+# CreateTags, and TerminateInstances.
+# Add iam:PassRole for the exact role only when ODDISH_EC2_INSTANCE_PROFILE is
+# configured. IMDSv2 is always required: hop limit 1 lets host cloud-init fetch
+# the launch SSH key, while configured profiles use hop limit 2 so Harbor's
+# Docker containers can receive replies.
+
 # =============================================================================
 # GKE TPU backend (optional)
 # =============================================================================

backend/Dockerfile

@@ -1,6 +1,6 @@
 FROM python:3.13-slim-bookworm
 
-RUN apt-get update && apt-get install -y --no-install-recommends git \
+RUN apt-get update && apt-get install -y --no-install-recommends git openssh-client \
     && rm -rf /var/lib/apt/lists/*
 
 RUN pip install --no-cache-dir uv

backend/README.md

@@ -42,7 +42,8 @@ Scheduled functions
        - Dispatches to the registered handler
        - Writes heartbeats, records outcome, exits
   ▼
-Modal sandboxes (Harbor execution, logs/artifacts to S3)
+Harbor execution on Modal, Daytona, or opt-in ephemeral EC2
+  - logs/artifacts persisted to S3
 ```
 
 ### Worker architecture
@@ -223,6 +224,35 @@ throughput (`ODDISH_MODAL_MAX_WORKERS_PER_POLL`, default `256`;
 
 Local `backend/.env` values are layered on top of the shared Modal secret for local deploys.
 
+### Ephemeral EC2 Harbor backend
+
+EC2 is an explicit CPU-only provider; Daytona remains the default. Set the
+non-secret `ODDISH_EC2_*` launch coordinates from `backend/.env.example` and
+name two dedicated Modal secrets:
+
+- The control secret contains `ODDISH_EC2_AWS_ACCESS_KEY_ID`,
+  `ODDISH_EC2_AWS_SECRET_ACCESS_KEY`, and optional
+  `ODDISH_EC2_AWS_SESSION_TOKEN`.
+- The worker-only SSH secret contains `ODDISH_EC2_SSH_PRIVATE_KEY`.
+
+Only dedicated `ec2_trial` worker functions receive both secrets. Generic trial
+workers receive neither; the reconciler and dedicated `teardown_ec2_sandbox`
+function receive only control credentials. API and dispatcher functions receive
+neither. API cancellation delegates one teardown call to that dedicated
+function. EC2 workers materialize the AWS credentials and SSH key as mode-`0600`
+temporary files, pass only the named AWS profile and key path to Harbor, and
+remove the raw secret variables from the Harbor child. Set
+`ODDISH_EC2_MAX_CONCURRENT_INSTANCES` to the provider-wide instance cap; this is
+enforced across every model and Harbor variant.
+The control policy must include `sts:GetCallerIdentity` in addition to the EC2
+launch, describe, image lookup, tag, and terminate actions listed in
+`.env.example`.
+
+Standalone hosts installed with `oddish[worker]` must also provide the OpenSSH
+client (`openssh-client` on Debian/Ubuntu), because Harbor invokes `ssh` to reach
+the VM. The shared `backend/Dockerfile` already installs this package for the
+Railway/Docker deployment path, and the Modal worker image installs it as well.
+
 ### oddish runtime patching
 
 `endpoints.py`, `serve.py`, and `worker/runtime.py` patch oddish settings at startup:
@@ -248,7 +278,7 @@ All routes require auth unless marked public.
 | GET | `/tasks` | List tasks (org-scoped, paginated/filtered) |
 | GET | `/tasks/browse` | Browse latest task versions with pagination and search |
 | GET | `/tasks/{task_id}` | Task details |
-| POST | `/tasks/cancel` | Cancel in-flight trials and queue jobs for one or more tasks (org-scoped); Modal workers terminated when applicable |
+| POST | `/tasks/cancel` | Cancel in-flight trials and queue jobs for one or more tasks (org-scoped); Modal workers and supported remote sandboxes are terminated when applicable |
 | POST | `/tasks/{task_id}/qa/retry` | Re-run task QA: classify trials and synthesize the verdict |
 | POST | `/tasks/{task_id}/qa/cancel` | Cancel a task's in-flight QA job |
 | GET | `/tasks/{task_id}/trials` | Trials for task |

backend/endpoints.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import os
+
 from oddish.config import Settings
 
 # API containers are warm and long-lived (min_containers >= 1).  Reuse pooled
@@ -50,6 +52,19 @@
     runtime_secrets,
 )
 from api.app import create_app
+from oddish.core.helpers import register_provider_teardown_delegate
+
+
+async def _teardown_ec2_sandbox(external_id: str) -> bool:
+    function = modal.Function.from_name(
+        os.environ.get("MODAL_APP_NAME", "oddish"),
+        "teardown_ec2_sandbox",
+        environment_name=os.environ.get("MODAL_ENVIRONMENT") or None,
+    )
+    return bool(await function.remote.aio(external_id))
+
+
+register_provider_teardown_delegate("ec2", _teardown_ec2_sandbox)
 
 api = create_app()