Add App Studio assistant engine contracts #174
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tilt E2E | |
| # End-to-end test against the operator-deployed, multi-shard Tilt stack | |
| # (Tiltfile.cluster / `make tilt-cluster`): kcp-operator + root/theseus shards + | |
| # front-proxy + the in-cluster kedge hub + the host-run providers + the kro | |
| # runtime cluster. This is the only suite that exercises the real multi-shard | |
| # operator topology (cross-shard CachedResource projection, MCP federation, the | |
| # caller-identity gate) — the embedded-kcp suites can't. | |
| # | |
| # Unlike the other e2e jobs (which spawn their own embedded kcp), this job has | |
| # to STAND UP the whole stack, drive a couple of manual Tilt resources (provider | |
| # register + init), wait for kcp to reconcile the provider's CatalogEntry to | |
| # Ready, and only then run the read-only test suite — which fails fast (no | |
| # internal retries) if the stack isn't fully settled. | |
| on: | |
| pull_request: | |
| branches: [main] | |
| push: | |
| branches: [main] | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| # kcp source the Tilt setup deploys. Tiltfile.cluster needs a kcp checkout for | |
| # its contrib/tilt/Tiltfile.static (operator manifests, kubeconfig extract, | |
| # static-token wiring) — it deploys upstream PREBUILT kcp images, so no kcp `go | |
| # build` happens here. Pin KCP_REF to a SHA if main's contrib/tilt drifts from | |
| # the image the operator deploys. | |
| env: | |
| KCP_REF: main | |
| # Skip the observability stack (grafana/loki/promtail/prometheus) in the | |
| # kcp Tilt setup — read by contrib/tilt/Tiltfile.static via os.getenv. Those | |
| # pods are pure overhead on CI and add CPU/memory pressure to the single | |
| # kind node (the multi-shard job has hit "Insufficient cpu" scheduling the | |
| # theseus shard); we don't assert on dashboards/metrics here. | |
| KCP_OBSERVABILITY_ENABLED: "false" | |
| jobs: | |
| e2e-tilt-cluster: | |
| name: Tilt-cluster (operator multi-shard kcp + in-cluster hub) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| # Non-blocking for now: the multi-shard stack over-subscribes the single | |
| # 2-vCPU runner (root-kcp won't schedule, shards/proxies crash-loop on | |
| # probe timeouts), so this job flakes. continue-on-error keeps it visible | |
| # (it still runs and reports) without failing the PR. Remove once the stack | |
| # is trimmed to fit (shared etcd ✓, observability off ✓, replicas=1 TODO). | |
| continue-on-error: true | |
| steps: | |
| - name: Checkout kedge | |
| uses: actions/checkout@v4 | |
| with: | |
| path: kedge | |
| - name: Checkout kcp (for the Tilt operator setup) | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: kcp-dev/kcp | |
| ref: ${{ env.KCP_REF }} | |
| path: kcp | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version: v1.26.1 | |
| cache-dependency-path: kedge/go.sum | |
| # Node + npm are needed by `make build-portal` (vite), which the Tilt | |
| # `portal-build` resource runs before compiling the hub binary. | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Install kind | |
| uses: helm/kind-action@v1 | |
| with: | |
| install_only: true | |
| - name: Install Helm | |
| uses: azure/setup-helm@v4 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install kubectl | |
| uses: azure/setup-kubectl@v4 | |
| - name: Install Tilt | |
| run: | | |
| curl -fsSL https://raw.githubusercontent.com/tilt-dev/tilt/master/scripts/install.sh | bash | |
| tilt version | |
| # The kcp front-proxy/shards and the hub are reached on the host via Tilt | |
| # port-forwards to *.kcp.localhost:8443; the `.localhost` TLD is not | |
| # auto-resolved on Linux, so map the hostnames (and the hub vhost) to | |
| # loopback. Matches the local-dev /etc/hosts the README documents. | |
| - name: Map kcp + hub hostnames to loopback | |
| run: | | |
| echo "127.0.0.1 kcp.localhost root.kcp.localhost theseus.kcp.localhost kedge.localhost" \ | |
| | sudo tee -a /etc/hosts | |
| # Bring the stack up. Tiltfile.cluster creates the `kcp-tilt` kind cluster | |
| # at parse time, but the Makefile pre-creates it + exports the context | |
| # first to avoid a Tilt deploy-client race (it caches an empty config if | |
| # the context doesn't yet exist) — we replicate that here. Tilt then runs | |
| # detached so we can drive it with `tilt wait` / `tilt trigger` from the | |
| # following steps; the process keeps running across steps on the runner. | |
| - name: Bring up the Tilt stack | |
| working-directory: kedge | |
| run: | | |
| set -euo pipefail | |
| kind get clusters 2>/dev/null | grep -qx kcp-tilt || kind create cluster --name kcp-tilt | |
| kind export kubeconfig --name kcp-tilt | |
| # The Tilt `portal-build` resource copies the built SPA with | |
| # `cp -r portal/dist pkg/hub/portal/`. On a fresh checkout | |
| # pkg/hub/portal/ doesn't exist (it's untracked), so cp would create | |
| # it AS a copy of dist — landing files at pkg/hub/portal/* instead of | |
| # pkg/hub/portal/dist/*, which breaks the hub's `//go:embed | |
| # portal/dist/*` (tags portal_embed). Pre-create the dir so the cp | |
| # nests correctly. (Locally this dir already exists from prior runs.) | |
| mkdir -p pkg/hub/portal/dist | |
| nohup tilt up -f Tiltfile.cluster --stream=true --port 10350 -- \ | |
| --kcp-dir="${GITHUB_WORKSPACE}/kcp" \ | |
| > "${GITHUB_WORKSPACE}/tilt-up.log" 2>&1 & | |
| echo $! > "${GITHUB_WORKSPACE}/tilt.pid" | |
| tilt_pid=$(cat "${GITHUB_WORKSPACE}/tilt.pid") | |
| # `tilt up` needs a few seconds to bring up its apiserver on :10350; | |
| # `tilt wait` errors with "No tilt apiserver found" until then. Poll | |
| # until the apiserver answers (or the process dies), then proceed. | |
| echo "Waiting for the Tilt apiserver to come up..." | |
| for i in $(seq 1 90); do | |
| if ! kill -0 "$tilt_pid" 2>/dev/null; then | |
| echo "tilt up exited before its apiserver came up; log:" | |
| cat "${GITHUB_WORKSPACE}/tilt-up.log" || true | |
| exit 1 | |
| fi | |
| if tilt get uiresources >/dev/null 2>&1; then | |
| echo "Tilt apiserver is up." | |
| break | |
| fi | |
| sleep 2 | |
| done | |
| tilt get uiresources >/dev/null 2>&1 || { | |
| echo "Tilt apiserver never came up; log:" | |
| cat "${GITHUB_WORKSPACE}/tilt-up.log" || true | |
| exit 1 | |
| } | |
| # `tilt wait` errors "not found" if a resource isn't registered yet | |
| # (the Tiltfile is still loading), so use our own poller that waits for | |
| # the resource to APPEAR and reach a ready status (and bails on error | |
| # or if tilt dies). Shared with the register/init step via a sourced | |
| # helper file. | |
| cat > "${GITHUB_WORKSPACE}/tilt-helpers.sh" <<'SH' | |
| # dump_diag prints why a resource is stuck. A resource at | |
| # update=pending never STARTED — it's blocked behind an upstream | |
| # resource_dep (for kedge-hub: hub-kcp-secrets → 'kcp-admin extract' | |
| # → the operator's rootshard/front-proxy). So `tilt logs <name>` is | |
| # empty in that case; the signal lives in the OTHER resources. We | |
| # therefore dump: (1) the status of EVERY tilt resource (pinpoints | |
| # which upstream dep is the real blocker), (2) the full tilt-up.log | |
| # tail (all resources' build output, incl. the operator/extract), and | |
| # (3) the named resource's own logs + a cluster-wide pod/event | |
| # snapshot. Wrapped in a collapsible ::group:: so it doesn't drown | |
| # the main log. | |
| dump_diag() { | |
| local name="$1" | |
| echo "::group::diagnostics for stuck resource '$name'" | |
| echo "----- all tilt resources (which upstream dep is blocking?) -----" | |
| tilt get uiresources -o json 2>/dev/null \ | |
| | jq -r '.items[] | " \(.metadata.name): update=\(.status.updateStatus // "<none>") runtime=\(.status.runtimeStatus // "<none>")"' 2>/dev/null \ | |
| || tilt get uiresources 2>/dev/null || true | |
| echo "----- tilt-up.log (last 300 lines: operator / kcp-admin extract / build output) -----" | |
| tail -n 300 "${GITHUB_WORKSPACE}/tilt-up.log" 2>/dev/null || true | |
| echo "----- tilt logs '$name' (last 400 lines; empty if it never started) -----" | |
| tilt logs "$name" 2>/dev/null | tail -n 400 || true | |
| echo "----- kubectl get pods -A -----" | |
| kubectl get pods -A -o wide 2>/dev/null || true | |
| echo "----- describe + logs for non-Running pods -----" | |
| kubectl get pods -A --no-headers 2>/dev/null | awk '$4 != "Running" && $4 != "Completed" {print $1" "$2}' | while read -r ns pod; do | |
| echo "===== $ns/$pod describe ====="; kubectl -n "$ns" describe pod "$pod" 2>/dev/null | tail -n 60 || true | |
| echo "===== $ns/$pod logs (all containers, tail 200) ====="; kubectl -n "$ns" logs "$pod" --all-containers --tail=200 2>/dev/null || true | |
| done | |
| echo "----- recent events (all namespaces) -----" | |
| kubectl get events -A --sort-by=.lastTimestamp 2>/dev/null | tail -n 60 || true | |
| echo "::endgroup::" | |
| } | |
| wait_resource() { | |
| local name="$1" timeout="$2" upd run deadline last="" | |
| deadline=$(( $(date +%s) + timeout )) | |
| echo "waiting for tilt resource '$name' (<= ${timeout}s)" | |
| while :; do | |
| if [ -f "${GITHUB_WORKSPACE}/tilt.pid" ] && ! kill -0 "$(cat "${GITHUB_WORKSPACE}/tilt.pid")" 2>/dev/null; then | |
| echo "tilt process exited unexpectedly; tail of log:"; tail -n 80 "${GITHUB_WORKSPACE}/tilt-up.log" || true; dump_diag "$name"; return 1 | |
| fi | |
| json=$(tilt get uiresource "$name" -o json 2>/dev/null || true) | |
| upd=$(printf '%s' "$json" | jq -r '.status.updateStatus // empty' 2>/dev/null || true) | |
| run=$(printf '%s' "$json" | jq -r '.status.runtimeStatus // empty' 2>/dev/null || true) | |
| if [ "$upd $run" != "$last" ]; then echo " $name: update=${upd:-<none>} runtime=${run:-<none>}"; last="$upd $run"; fi | |
| if [ "$upd" = "error" ] || [ "$run" = "error" ]; then | |
| echo "resource '$name' entered error state"; dump_diag "$name"; return 1 | |
| fi | |
| if [ "$upd" = "ok" ] && { [ "$run" = "ok" ] || [ "$run" = "not_applicable" ] || [ "$run" = "none" ]; }; then | |
| echo "resource '$name' is ready (update=$upd runtime=$run)"; return 0 | |
| fi | |
| if [ "$(date +%s)" -ge "$deadline" ]; then | |
| echo "timeout waiting for '$name' (update=${upd:-<none>} runtime=${run:-<none>})"; dump_diag "$name"; return 1 | |
| fi | |
| sleep 5 | |
| done | |
| } | |
| SH | |
| source "${GITHUB_WORKSPACE}/tilt-helpers.sh" | |
| echo "Waiting for the kcp + hub + provider stack to become Ready..." | |
| # Generous: covers Tiltfile load (cert-manager/operator/kcp install) + | |
| # hub image build + in-cluster deploy before kedge-hub even appears. | |
| wait_resource kedge-hub 1800 | |
| wait_resource kro-mgmt-up 600 | |
| wait_resource infrastructure 600 | |
| # Host-side access to kcp goes through the envoy gateway on | |
| # kcp.localhost:8443. Upstream kcp's Tilt `ingress` resource | |
| # (contrib/tilt/port-forward.bash) is supposed to provide that forward, | |
| # but it's racy on CI — its `kubectl wait gateway/eg --for=create` can hit | |
| # "the server doesn't have a resource type gateway" before the Gateway CRD | |
| # is established, leaving nothing on :8443. The host-side kubectl | |
| # (register/init) AND the e2e suite both need kcp.localhost:8443, so run | |
| # our own self-healing port-forward to the gateway service instead. | |
| - name: Port-forward the kcp gateway to localhost:8443 | |
| working-directory: kedge | |
| run: | | |
| set -euo pipefail | |
| echo "waiting for the envoy gateway service..." | |
| svc="" | |
| for i in $(seq 1 60); do | |
| svc=$(kubectl -n envoy-gateway-system get svc \ | |
| -l gateway.envoyproxy.io/owning-gateway-name=eg \ | |
| -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) | |
| [ -n "$svc" ] && break | |
| sleep 5 | |
| done | |
| [ -n "$svc" ] || { echo "envoy gateway service never appeared"; kubectl -n envoy-gateway-system get svc || true; exit 1; } | |
| echo "envoy gateway service: $svc" | |
| # Self-healing: kubectl port-forward can drop; the wrapper reconnects | |
| # so :8443 stays up across register/init + the CatalogEntry gate + e2e. | |
| nohup bash -c "while true; do kubectl -n envoy-gateway-system port-forward svc/$svc 8443:8443 --address 127.0.0.1; echo 'port-forward dropped; reconnecting'; sleep 2; done" \ | |
| > "${GITHUB_WORKSPACE}/port-forward-8443.log" 2>&1 & | |
| echo $! > "${GITHUB_WORKSPACE}/pf8443.pid" | |
| echo "waiting for :8443 to accept connections..." | |
| for i in $(seq 1 60); do | |
| if timeout 3 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8443' 2>/dev/null; then | |
| echo ":8443 is reachable"; exit 0 | |
| fi | |
| sleep 2 | |
| done | |
| echo ":8443 never became reachable; port-forward log:"; cat "${GITHUB_WORKSPACE}/port-forward-8443.log" || true; exit 1 | |
| # The provider's CatalogEntry registration + workspace bootstrap are MANUAL | |
| # Tilt resources in local dev (devs click ▶). In CI we trigger them and | |
| # wait for each one-shot job to finish. init writes the runtime kubeconfig | |
| # the provider watches, so the `infrastructure` resource restarts and | |
| # re-enables its controllers afterwards. | |
| - name: Register + initialize the infrastructure provider | |
| working-directory: kedge | |
| run: | | |
| set -euo pipefail | |
| source "${GITHUB_WORKSPACE}/tilt-helpers.sh" | |
| tilt trigger infrastructure-register | |
| wait_resource infrastructure-register 300 | |
| tilt trigger infrastructure-init | |
| wait_resource infrastructure-init 300 | |
| # Provider restarts on the runtime-kubeconfig write; wait for it back. | |
| # (The CatalogEntry reconcile gate below is the real readiness check.) | |
| wait_resource infrastructure 300 | |
| # The suite reads kcp state directly and fails fast (no Eventually/retry), | |
| # so block until kcp has actually reconciled the provider's CatalogEntry to | |
| # Ready before running it. This absorbs the controller/CachedResource | |
| # settle time between init finishing and the catalog being projected. | |
| - name: Wait for the CatalogEntry to reconcile Ready | |
| working-directory: kedge | |
| run: | | |
| set -euo pipefail | |
| deadline=$(( $(date +%s) + 600 )) | |
| while :; do | |
| status=$(kubectl --kubeconfig=tilt-frontproxy.kubeconfig \ | |
| --server=https://kcp.localhost:8443/clusters/root:kedge:providers \ | |
| --insecure-skip-tls-verify \ | |
| get catalogentries.providers.kedge.faros.sh infrastructure \ | |
| -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true) | |
| if [ "$status" = "True" ]; then | |
| echo "CatalogEntry infrastructure is Ready." | |
| break | |
| fi | |
| if [ "$(date +%s)" -ge "$deadline" ]; then | |
| echo "timed out waiting for CatalogEntry Ready (last status: '${status:-<none>}')" | |
| kubectl --kubeconfig=tilt-frontproxy.kubeconfig \ | |
| --server=https://kcp.localhost:8443/clusters/root:kedge:providers \ | |
| --insecure-skip-tls-verify \ | |
| get catalogentries.providers.kedge.faros.sh infrastructure -o yaml || true | |
| exit 1 | |
| fi | |
| sleep 10 | |
| done | |
| # `make e2e-tilt-cluster` prechecks hub + provider /healthz, then runs the | |
| # suite against the live stack. Retry a couple of times to absorb any | |
| # last-moment reconcile lag (the tests are read-only and idempotent). | |
| - name: Run tilt-cluster e2e | |
| working-directory: kedge | |
| run: | | |
| set -euo pipefail | |
| for attempt in 1 2 3; do | |
| echo "=== e2e-tilt-cluster attempt ${attempt} ===" | |
| if make e2e-tilt-cluster E2E_TIMEOUT=15m; then | |
| exit 0 | |
| fi | |
| echo "attempt ${attempt} failed; retrying after 60s..." | |
| sleep 60 | |
| done | |
| echo "e2e-tilt-cluster failed after 3 attempts" | |
| exit 1 | |
| - name: Tilt resource status on failure | |
| if: failure() | |
| working-directory: kedge | |
| run: tilt get uiresources -o json || true | |
| - name: Collect logs | |
| if: failure() | |
| working-directory: kedge | |
| run: | | |
| mkdir -p "${GITHUB_WORKSPACE}/tilt-logs" | |
| cp "${GITHUB_WORKSPACE}/tilt-up.log" "${GITHUB_WORKSPACE}/tilt-logs/" 2>/dev/null || true | |
| cp *.kubeconfig "${GITHUB_WORKSPACE}/tilt-logs/" 2>/dev/null || true | |
| for cluster in $(kind get clusters 2>/dev/null); do | |
| kind export logs "${GITHUB_WORKSPACE}/tilt-logs/kind-$cluster" --name "$cluster" || true | |
| done | |
| - name: Upload logs on failure | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: tilt-e2e-logs | |
| path: tilt-logs/ | |
| retention-days: 3 | |
| # Always tear down: `tilt down` removes the in-cluster objects, then kill | |
| # the detached Tilt process and delete the kind cluster. `|| true` so a | |
| # half-up stack still cleans up. | |
| - name: Tear down | |
| if: always() | |
| working-directory: kedge | |
| run: | | |
| tilt down -f Tiltfile.cluster -- --kcp-dir="${GITHUB_WORKSPACE}/kcp" || true | |
| [ -f "${GITHUB_WORKSPACE}/pf8443.pid" ] && kill "$(cat "${GITHUB_WORKSPACE}/pf8443.pid")" 2>/dev/null || true | |
| pkill -f "port-forward.*8443" 2>/dev/null || true | |
| [ -f "${GITHUB_WORKSPACE}/tilt.pid" ] && kill "$(cat "${GITHUB_WORKSPACE}/tilt.pid")" 2>/dev/null || true | |
| kind delete cluster --name kcp-tilt || true |