test(e2e): migrate builder suite to Hetzner-provisioned cluster

Replace pre-existing cluster requirement with ephemeral 2-VM Hetzner
setup (A = central + builder, B = coold-only). All scenarios collapsed
into single `builder_lifecycle` test so one cluster is reused across
every dispatch/cancel/restart/artifact-perm scenario.

Drop manual BUILDER_HOST/COOLD_ONLY_HOST/BUILDER_MGMT/COOLD_ONLY_MGMT/
CENTRAL_HOST env vars — now driven by same HETZNER_TOKEN/PROJECT/SSH_KEY
vars as the install suite. Update .env.example and E2E.md accordingly.

Author: andrasbacsai

e2e-tests/.env.example

@@ -1,7 +1,7 @@
 # Copy to .env and fill in. Never committed — root .gitignore covers it.
 # Values only apply when not already exported in the shell.
 
-# ─── Install suite (Hetzner-provisioned) ─────────────────────────────────────
+# ─── Hetzner-provisioned suites (install + builder) ──────────────────────────
 HETZNER_TOKEN=
 HETZNER_PROJECT=my-coold-e2e
 SSH_KEY=/Users/you/.ssh/id_ed25519
@@ -10,13 +10,5 @@ COOLIFY_BIN=coolify
 # HETZNER_IMAGE=ubuntu-24.04
 # HETZNER_SERVER_TYPE=cx23
 
-# ─── Builder suite (pre-existing cluster) ────────────────────────────────────
-# BUILDER_HOST=
-# COOLD_ONLY_HOST=
-# BUILDER_MGMT=
-# COOLD_ONLY_MGMT=
-# CENTRAL_HOST=
-# SSH_USER=root
-
 # ─── Sweeper opt-in ──────────────────────────────────────────────────────────
 # CONFIRM_SWEEP=1

e2e-tests/E2E.md

@@ -19,37 +19,20 @@ $EDITOR e2e-tests/.env
 cargo test -p e2e-tests --no-run
 ```
 
-## Suite 1 — `builder.rs` (pre-installed cluster)
+## Suite 1 — `builder.rs` (Hetzner-provisioned)
 
-Requires an already-bootstrapped cluster. Env vars:
+Provisions 2 VMs (A = central + builder, B = coold-only), runs
+`coolify init apply`, then executes every dispatch / cancel / restart
+/ artifact-perm scenario on the shared cluster. VMs destroyed on drop.
+Uses the same env vars as the install suite (see below):
 
 ```bash
-export BUILDER_HOST=<ssh-addr-of-builder-host>
-export COOLD_ONLY_HOST=<ssh-addr-of-coold-only-host>
-export BUILDER_MGMT=<wg0-ip-of-builder-host>
-export COOLD_ONLY_MGMT=<wg0-ip-of-coold-only-host>
-export CENTRAL_HOST=<ssh-addr-of-central>
-export SSH_KEY=~/.ssh/<key>
-# optional:
-export SSH_USER=root
-```
-
-### Run all in suite
-
-```bash
-cargo test -p e2e-tests --test builder -- --ignored --nocapture --test-threads=1
+cargo test -p e2e-tests --test builder builder_lifecycle -- --ignored --nocapture --test-threads=1
 ```
 
-### Individual tests
-
-```bash
-cargo test -p e2e-tests --test builder pin_to_builder_host             -- --ignored --nocapture
-cargo test -p e2e-tests --test builder pin_to_coold_only_host_returns_503 -- --ignored --nocapture
-cargo test -p e2e-tests --test builder unknown_host_id_returns_503     -- --ignored --nocapture
-cargo test -p e2e-tests --test builder load_balance_picks_builder_host -- --ignored --nocapture
-cargo test -p e2e-tests --test builder build_cancel_emits_stage_cancel -- --ignored --nocapture
-cargo test -p e2e-tests --test builder coold_restart_adopts_in_flight_build -- --ignored --nocapture
-```
+The whole suite is a single `#[test] fn builder_lifecycle` — there are
+no longer individual scenario tests (running each separately would
+provision its own cluster, which is wasteful).
 
 ## Suite 2 — `install.rs` (Hetzner-provisioned)
 

e2e-tests/src/lib.rs

@@ -1,27 +1,26 @@
 //! Live-server test harness for the coold/broker/builder stack.
 //!
-//! Tests are written as Rust integration tests under `tests/`, marked
-//! `#[ignore]` so default `cargo test` skips them. Run with:
+//! Tests are Rust integration tests under `tests/`, marked `#[ignore]` so
+//! default `cargo test` skips them. Every suite provisions its own
+//! ephemeral Hetzner cluster via [`hetzner::EphemeralCluster`], runs
+//! `coolify init apply` from the local `coolify` binary, then exercises
+//! the black-box HTTP/UDS/systemd contract over SSH. No broker/coold
+//! code is linked.
+//!
+//! Run with:
 //!
 //! ```text
-//! BUILDER_HOST=<host-a> \
-//! COOLD_ONLY_HOST=<host-b> \
-//! BUILDER_MGMT=<wg0-ip-of-host-a> \
-//! COOLD_ONLY_MGMT=<wg0-ip-of-host-b> \
-//! CENTRAL_HOST=<central-host> \
+//! HETZNER_TOKEN=... HETZNER_PROJECT=... \
 //! SSH_KEY=~/.ssh/<key> \
-//! cargo test -p e2e-tests -- --ignored --test-threads=1
+//! COOLIFY_BIN=$(which coolify) \
+//! cargo test -p e2e-tests -- --ignored --nocapture --test-threads=1
 //! ```
 //!
-//! `--test-threads=1` is mandatory: the tests dispatch real builds against
-//! a shared cluster, and running them in parallel overwhelms the
-//! `COOLD_BUILDER_CAPACITY` semaphore and races on `buildah images` state
-//! shared across hosts.
-//!
-//! The harness drives the broker UDS via `ssh + curl --unix-socket` on the
-//! central host and asserts remote state via `buildah images` and
-//! `systemctl is-active`. No broker/coold code is linked — tests exercise
-//! the black-box contract.
+//! `--test-threads=1` is mandatory: the tests dispatch real builds and
+//! running them in parallel overwhelms the `COOLD_BUILDER_CAPACITY`
+//! semaphore and races on `buildah images` state shared across hosts.
+//! VMs are deleted via [`hetzner::EphemeralCluster`]'s RAII `Drop`, so
+//! panics during assertions still clean up paid resources.
 
 use std::process::Command;
 use std::thread;
@@ -90,37 +89,34 @@ pub struct Env {
     pub cool_only_mgmt: String,
     pub central_host: String,
     pub ssh_key: String,
-    pub ssh_user: String,
 }
 
 impl Env {
-    pub fn from_env() -> Self {
-        load_dotenv();
+    /// Build from a live [`hetzner::EphemeralCluster`] provisioned with
+    /// 2 hosts (A = central + builder, B = coold-only) and the wg0
+    /// addresses resolved on each host after `coolify init apply`.
+    pub fn from_cluster(
+        cluster: &hetzner::EphemeralCluster,
+        builder_mgmt: String,
+        cool_only_mgmt: String,
+    ) -> Self {
+        let hosts = cluster.hosts();
+        assert_eq!(hosts.len(), 2, "Env::from_cluster expects 2 hosts");
         Self {
-            builder_host: must("BUILDER_HOST"),
-            cool_only_host: must("COOLD_ONLY_HOST"),
-            builder_mgmt: must("BUILDER_MGMT"),
-            cool_only_mgmt: must("COOLD_ONLY_MGMT"),
-            central_host: must("CENTRAL_HOST"),
-            ssh_key: must("SSH_KEY"),
-            ssh_user: std::env::var("SSH_USER").unwrap_or_else(|_| "root".into()),
+            builder_host: hosts[0].ipv4.clone(),
+            cool_only_host: hosts[1].ipv4.clone(),
+            builder_mgmt,
+            cool_only_mgmt,
+            central_host: hosts[0].ipv4.clone(),
+            ssh_key: cluster.ssh_key.clone(),
         }
     }
 
     pub fn ssh(&self, host: &str, cmd: &str) -> Result<String, String> {
         let out = Command::new("ssh")
-            .args([
-                "-i",
-                &self.ssh_key,
-                "-o",
-                "StrictHostKeyChecking=accept-new",
-                "-o",
-                "BatchMode=yes",
-                "-o",
-                "ConnectTimeout=10",
-                &format!("{}@{}", self.ssh_user, host),
-                cmd,
-            ])
+            .args(hetzner::ephemeral_ssh_args(&self.ssh_key))
+            .arg(format!("root@{host}"))
+            .arg(cmd)
             .output()
             .map_err(|e| format!("spawn ssh: {e}"))?;
         if !out.status.success() {
@@ -267,10 +263,6 @@ impl Env {
     }
 }
 
-fn must(key: &str) -> String {
-    std::env::var(key).unwrap_or_else(|_| panic!("env {key} required"))
-}
-
 /// Lowercase request_id suitable for use as an OCI image tag (OCI rejects
 /// uppercase in repository names).
 pub fn uniq_req_id(prefix: &str) -> String {