integration-tests: add buildkit autoscaling scale test

Burst 8 parallel buildctl builds per arch (each holds a maxconn=1 slot
~10m via sleep). With amd64_min=2 they serialize ~43m > timeout 30m and
fail unless KEDA scales the pool up; one wave ~18m when it does.


ghstack-source-id: e1a1a45
Pull-Request: #725

Author: huydhn

osdc/integration-tests/docker/test-buildkit-scale/Dockerfile

@@ -0,0 +1,6 @@
+# Holds one BuildKit slot (~10 min) so a burst of parallel builds exceeds the
+# warm baseline and forces KEDA scale-up. CACHEBUST differs per build so each
+# actually runs (no layer reuse).
+FROM alpine:3.21
+ARG CACHEBUST
+RUN echo "osdc buildkit scale-test ${CACHEBUST}" && sleep 600

osdc/integration-tests/scripts/python/phases.py

@@ -276,15 +276,17 @@ def prepare_pr(
     # Write integration test workflow
     (workflows_dir / "integration-test.yaml").write_text(workflow_content)
 
-    # Copy build-image reusable workflow
-    build_wf_src = upstream_dir / "integration-tests" / "workflows" / "build-image.yaml"
-    (workflows_dir / "build-image.yaml").write_text(build_wf_src.read_text())
-
-    # Copy test Dockerfile
-    docker_dir = canary_path / "docker" / "test-buildkit"
-    docker_dir.mkdir(parents=True, exist_ok=True)
-    dockerfile_src = upstream_dir / "integration-tests" / "docker" / "test-buildkit" / "Dockerfile"
-    (docker_dir / "Dockerfile").write_text(dockerfile_src.read_text())
+    # Copy reusable BuildKit workflows
+    wf_root = upstream_dir / "integration-tests" / "workflows"
+    for wf in ("build-image.yaml", "build-image-scale.yaml"):
+        (workflows_dir / wf).write_text((wf_root / wf).read_text())
+
+    # Copy test Dockerfiles (connectivity + autoscaling scale test)
+    docker_root = upstream_dir / "integration-tests" / "docker"
+    for name in ("test-buildkit", "test-buildkit-scale"):
+        dst = canary_path / "docker" / name
+        dst.mkdir(parents=True, exist_ok=True)
+        (dst / "Dockerfile").write_text((docker_root / name / "Dockerfile").read_text())
 
     # Commit
     run_cmd(["git", "add", "-A"], cwd=canary_path)

osdc/integration-tests/scripts/python/test_run.py

@@ -112,11 +112,13 @@ def workflow_template(tmp_path):
     )
     (wf_dir / "integration-test.yaml.tpl").write_text(template)
 
-    # Also create build-image.yaml and Dockerfile for prepare_pr
+    # Also create reusable workflows and Dockerfiles for prepare_pr
     (wf_dir / "build-image.yaml").write_text("name: build-image\n")
-    docker_dir = upstream / "integration-tests" / "docker" / "test-buildkit"
-    docker_dir.mkdir(parents=True)
-    (docker_dir / "Dockerfile").write_text("FROM alpine\n")
+    (wf_dir / "build-image-scale.yaml").write_text("name: build-image-scale\n")
+    for name in ("test-buildkit", "test-buildkit-scale"):
+        docker_dir = upstream / "integration-tests" / "docker" / name
+        docker_dir.mkdir(parents=True)
+        (docker_dir / "Dockerfile").write_text("FROM alpine\n")
 
     return upstream
 

osdc/integration-tests/workflows/build-image-scale.yaml

@@ -0,0 +1,59 @@
+# Reusable workflow: BuildKit autoscaling scale test.
+# Launches a burst of 8 parallel builds against one arch's remote BuildKit, each
+# holding a slot (server maxconn=1) for ~10 min via a sleep. The warm baseline
+# (amd64_min / arm64_min) is below the burst, so the builds finish within
+# timeout-minutes only if KEDA scales the pool up. Without scale-up they
+# serialize through the baseline pods and the back of the queue times out — i.e.
+# this job FAILS when autoscaling does not happen.
+name: BuildKit Scale Test
+
+on:
+  workflow_call:
+    inputs:
+      arch:
+        description: "Target architecture (amd64 or arm64)"
+        required: true
+        type: string
+      runner_label:
+        description: "Runner label to use (includes cluster prefix)"
+        required: false
+        type: string
+        default: "l-x86iavx512-2-4"
+
+jobs:
+  scale:
+    # Runs on x86 — BuildKit is a *remote* builder; arch selects the endpoint.
+    # timeout-minutes is the gate: scaled-up ~18 min, serialized ~43 min.
+    runs-on: ${{ inputs.runner_label }}
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        replica: [1, 2, 3, 4, 5, 6, 7, 8]
+    container:
+      image: ghcr.io/actions/actions-runner:latest
+    steps:
+      - name: Install buildctl
+        run: |
+          BUILDKIT_VERSION="v0.29.0"
+          mkdir -p "$HOME/.local/bin"
+          curl -sSL "https://github.com/moby/buildkit/releases/download/${BUILDKIT_VERSION}/buildkit-${BUILDKIT_VERSION}.linux-amd64.tar.gz" \
+            | tar xz --strip-components=1 -C "$HOME/.local/bin" bin/buildctl
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+          "$HOME/.local/bin/buildctl" --version
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Occupy a BuildKit slot (~10 min) to drive autoscaling
+        run: |
+          ENDPOINT="tcp://buildkitd-${{ inputs.arch }}.buildkit:1234"
+          echo "Build ${{ matrix.replica }} -> $ENDPOINT"
+          buildctl --addr "$ENDPOINT" build \
+            --frontend dockerfile.v0 \
+            --local context=docker/test-buildkit-scale \
+            --local dockerfile=docker/test-buildkit-scale \
+            --opt build-arg:CACHEBUST=${{ inputs.arch }}-${{ matrix.replica }}-${{ github.run_id }} \
+            --no-cache \
+            --output type=cacheonly
+          echo "PASS: build ${{ matrix.replica }} finished within timeout"

osdc/integration-tests/workflows/integration-test.yaml.tpl

@@ -1514,6 +1514,20 @@ jobs:
       arch: arm64
       runner_label: {{PREFIX}}l-x86iamx-8-32
 
+  # ── BuildKit Autoscaling Scale Test ───────────────────────────────────
+  # Bursts 8 parallel builds per arch; fails if KEDA does not scale the pool up.
+  buildkit-scale-amd64:
+    uses: ./.github/workflows/build-image-scale.yaml
+    with:
+      arch: amd64
+      runner_label: {{PREFIX}}l-x86iamx-8-32
+
+  buildkit-scale-arm64:
+    uses: ./.github/workflows/build-image-scale.yaml
+    with:
+      arch: arm64
+      runner_label: {{PREFIX}}l-x86iamx-8-32
+
   # ── Harbor Cache Test ─────────────────────────────────────────────────
   test-harbor:
     runs-on: {{PREFIX}}l-x86iamx-8-32