One script for end-to-end deployment on substrate

.gitignore

@@ -11,6 +11,7 @@ test-sandbox-config.yaml
 sandbox-router
 examples/sandbox_agent/cloudbuild.yaml
 tasklog/*
+.ax-dev-env.sh
 python/__pycache__
 python/proto/__pycache__
 /axepp

go.mod

@@ -5,7 +5,7 @@ go 1.26.3
 require (
 	charm.land/huh/v2 v2.0.3
 	charm.land/lipgloss/v2 v2.0.3
-	github.com/agent-substrate/substrate v0.0.0-20260622205654-fe93d160a1df
+	github.com/agent-substrate/substrate v0.0.0-20260629060549-505b3a44d2be
 	github.com/google/uuid v1.6.0
 	github.com/jackc/pgx/v5 v5.10.0
 	github.com/spf13/cobra v1.10.2

go.sum

@@ -10,8 +10,8 @@ cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdB
 cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
 github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
 github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
-github.com/agent-substrate/substrate v0.0.0-20260622205654-fe93d160a1df h1:Qq7cujws6kbtE/ykSlCBcZQWlpK/HQQZcmbUjJbWiUs=
-github.com/agent-substrate/substrate v0.0.0-20260622205654-fe93d160a1df/go.mod h1:TgdtEUV6iaflJTwmS8ONiGsyyJD+5okPZj2H6mM8WlA=
+github.com/agent-substrate/substrate v0.0.0-20260629060549-505b3a44d2be h1:7OYxdwo+uXGF2ISOUGMVXBnPNe64wpkUrfzwhI944ts=
+github.com/agent-substrate/substrate v0.0.0-20260629060549-505b3a44d2be/go.mod h1:2ri5mTz714LWinayPQG1d4G/kMuzbqvRXlrUrpc7N9Y=
 github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
 github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
 github.com/aymanbagabas/go-udiff v0.4.1 h1:OEIrQ8maEeDBXQDoGCbbTTXYJMYRCRO1fnodZ12Gv5o=

hack/ax-dev-env.sh.example

@@ -0,0 +1,56 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Example environment for deploying Agent Substrate + AX to GKE.
+#
+# Copy this file to .ax-dev-env.sh and customize it for your environment:
+#
+#   cp hack/ax-dev-env.sh.example .ax-dev-env.sh
+#
+# hack/install-ax.sh sources .ax-dev-env.sh automatically (set NO_DEV_ENV=1 to
+# skip). The substrate version is NOT configured here: it is taken automatically
+# from AX's go.mod pin.
+
+# --- Required: set these for your environment (no defaults) ---
+export PROJECT_ID=
+export CLUSTER_NAME=
+
+# --- GCP project ---
+# PROJECT_NUMBER is derived from PROJECT_ID (used by --create-cluster).
+if [[ -n "${PROJECT_ID}" ]]; then
+  export PROJECT_NUMBER=$(gcloud projects describe "${PROJECT_ID}" --format="value(projectNumber)")
+fi
+
+# --- Region / networking ---
+export GCE_REGION=us-central1
+export CLUSTER_LOCATION=us-central1-c
+export NETWORK=default
+export SUBNETWORK=default
+
+# --- Cluster / gVisor node pool ---
+export CLUSTER_VERSION=1.35.5-gke.1163012
+export NODE_POOL_NAME=substrate-node-pool
+export NODE_POOL_VERSION=1.35.5-gke.1163012
+export GVISOR_NODE_MACHINE_TYPE=n2-standard-4
+
+# Set this to target an existing cluster by its kube context name. When empty,
+# the deploy resolves the current kube context (or uses gcloud credentials).
+export KUBECTL_CONTEXT=
+
+# --- Storage / image registry ---
+export AX_SNAPSHOTS_BUCKET=snapshot-substrate-test-${PROJECT_ID}
+export KO_DOCKER_REPO="gcr.io/${PROJECT_ID}"
+
+# Override the default build platforms if needed.
+export KO_DEFAULTPLATFORMS=linux/amd64

hack/install-ax.sh

@@ -20,8 +20,18 @@ set -o pipefail
 ROOT=$(git rev-parse --show-toplevel)
 cd "${ROOT}"
 
+# Source developer environment overrides if present. Copy
+# hack/ax-dev-env.sh.example to .ax-dev-env.sh to configure your deployment.
+# Set NO_DEV_ENV=1 to skip.
+if [[ -f .ax-dev-env.sh ]] && [[ -z "${NO_DEV_ENV:-}" ]]; then
+  # shellcheck source=/dev/null
+  source .ax-dev-env.sh
+fi
+
 if [[ -n "${PROJECT_ID:-}" ]]; then
-  export KO_DOCKER_REPO="gcr.io/${PROJECT_ID}"
+  # Default the image repo to the project's GCR, but respect an explicit
+  # override (e.g. from .ax-dev-env.sh).
+  export KO_DOCKER_REPO="${KO_DOCKER_REPO:-gcr.io/${PROJECT_ID}}"
   echo "Using KO_DOCKER_REPO: ${KO_DOCKER_REPO}" >&2
 fi
 
@@ -70,9 +80,22 @@ wait_with_spinner() {
 function usage() {
   echo "Usage: $0 [options]"
   echo ""
-  echo "Options:"
-  echo "  --deploy-ax-server                    Build images and deploy AX server and components"
-  echo "  --delete-ax-server                    Delete AX server and components, preserving the event-log database"
+  echo "Deploys Agent Substrate and AX together. The substrate version is taken"
+  echo "automatically from AX's go.mod pin; you never choose a substrate commit."
+  echo ""
+  echo "One-time cluster setup (run once per cluster):"
+  echo ""
+  echo "  --create-cluster                      Provision GCP resources (GKE cluster, GCS bucket, IAM)"
+  echo "  --delete-cluster                      Tear down the provisioned GCP resources"
+  echo ""
+  echo "Deploy / delete (re-run as you update your code):"
+  echo ""
+  echo "  --deploy-ate-system                   Deploy the substrate control plane (at AX's pinned version)"
+  echo "  --delete-ate-system                   Delete the substrate control plane"
+  echo "  --deploy-ax                           Build images and deploy AX server and components"
+  echo "  --delete-ax                           Delete AX server and components, preserving the event-log database"
+  echo "  --delete-all                          Delete both the AX server and the substrate control plane"
+  echo ""
   echo "  -h, --help                            Show this help message"
 }
 
@@ -184,20 +207,164 @@ build_ateom_image() {
   echo "${ateom_ref}"
 }
 
-deploy_ax_server() {
-  log_step "deploy_ax_server"
+# --- Agent Substrate (compute layer) ---------------------------------------
+#
+# AX runs on Agent Substrate. The substrate version is taken automatically from
+# AX's go.mod pin, so developers never choose a substrate commit.
+
+# require_env exits with a clear message if any named environment variable is
+# unset or empty.
+require_env() {
+  local missing=() v
+  for v in "$@"; do
+    [[ -n "${!v:-}" ]] || missing+=("${v}")
+  done
+  if [[ "${#missing[@]}" -gt 0 ]]; then
+    echo "Error: missing required environment variables: ${missing[*]}" >&2
+    echo "Copy hack/ax-dev-env.sh.example to .ax-dev-env.sh, fill it in, and re-run." >&2
+    exit 1
+  fi
+}
+
+# ensure_substrate_src materializes a git checkout of Agent Substrate at the
+# exact version AX pins in go.mod and echoes its directory on stdout. By default
+# it maintains a managed clone under the user cache; set AX_SUBSTRATE_DIR to use
+# your own substrate checkout instead.
+ensure_substrate_src() {
+  local commit
+  commit="$(go list -m -f '{{.Version}}' github.com/agent-substrate/substrate | sed 's/.*-//')"
+  if [[ -z "${commit}" ]]; then
+    echo "Error: could not determine the pinned substrate version from go.mod." >&2
+    exit 1
+  fi
+
+  local dir="${AX_SUBSTRATE_DIR:-${XDG_CACHE_HOME:-${HOME}/.cache}/ax/substrate}"
+
+  if [[ ! -d "${dir}/.git" ]]; then
+    if [[ -n "${AX_SUBSTRATE_DIR:-}" ]]; then
+      echo "Error: AX_SUBSTRATE_DIR=${AX_SUBSTRATE_DIR} is not a git checkout." >&2
+      exit 1
+    fi
+    log_step "clone substrate -> ${dir}" >&2
+    mkdir -p "$(dirname "${dir}")"
+    git clone --quiet https://github.com/agent-substrate/substrate "${dir}" >&2
+  fi
+
+  # Never clobber uncommitted work in a developer-provided checkout.
+  if [[ -n "${AX_SUBSTRATE_DIR:-}" ]] && [[ -n "$(git -C "${dir}" status --porcelain)" ]]; then
+    echo "Error: ${dir} has uncommitted changes; refusing to check out ${commit}." >&2
+    echo "Commit or stash them, or unset AX_SUBSTRATE_DIR." >&2
+    exit 1
+  fi
+
+  # Ensure the pinned commit is present locally, then check it out (detached).
+  if ! git -C "${dir}" cat-file -e "${commit}^{commit}" 2>/dev/null; then
+    log_step "fetch substrate ${commit}" >&2
+    git -C "${dir}" fetch --quiet origin >&2 || true
+  fi
+  if ! git -C "${dir}" cat-file -e "${commit}^{commit}" 2>/dev/null; then
+    git -C "${dir}" fetch --quiet origin "${commit}" >&2 \
+      || { echo "Error: could not fetch substrate commit ${commit}." >&2; exit 1; }
+  fi
+  git -C "${dir}" checkout --quiet --detach "${commit}" >&2
+
+  echo "${dir}"
+}
+
+# run_substrate runs a substrate hack/ script from the substrate checkout, in a
+# subshell with the checkout as the working directory so substrate's own
+# `git rev-parse --show-toplevel` resolves to the substrate tree (not AX's). It
+# forwards the current kube context so substrate targets the same cluster.
+run_substrate() {
+  local src
+  src="$(ensure_substrate_src)"
+  # The managed clone takes all config from the inherited environment. Remove any
+  # leftover .ate-dev-env.sh.
+  if [[ -z "${AX_SUBSTRATE_DIR:-}" ]]; then
+    rm -f "${src}/.ate-dev-env.sh"
+  fi
+  ( cd "${src}" \
+    && KUBECTL_CONTEXT="${KUBECTL_CONTEXT:-$(kubectl config current-context 2>/dev/null || true)}" \
+       "$@" )
+}
+
+# create_cluster provisions the GCP resources substrate needs (GKE cluster with
+# Workload Identity and the required beta APIs, the GCS snapshot bucket, and IAM
+# bindings) via substrate's setup-gcp tool. This is a one-time step per cluster.
+create_cluster() {
+  log_step "create_cluster"
+  require_env PROJECT_ID PROJECT_NUMBER CLUSTER_NAME CLUSTER_LOCATION CLUSTER_VERSION \
+    NETWORK SUBNETWORK NODE_POOL_NAME NODE_POOL_VERSION GCE_REGION AX_SNAPSHOTS_BUCKET \
+    GVISOR_NODE_MACHINE_TYPE
+  local src
+  src="$(ensure_substrate_src)"
+  # substrate's setup-gcp reads BUCKET_NAME for the snapshot bucket.
+  ( cd "${src}" && BUCKET_NAME="${AX_SNAPSHOTS_BUCKET}" go run ./tools/setup-gcp --all )
+}
+
+# delete_cluster tears down the GCP resources created by create_cluster.
+#
+# Substrate's teardown.sh requires a .ate-dev-env.sh file. We generate one transiently
+# in the checkout from the current environment, then remove it so it never
+# lingers to shadow a later deploy.
+delete_cluster() {
+  log_step "delete_cluster"
+  require_env PROJECT_ID PROJECT_NUMBER AX_SNAPSHOTS_BUCKET NODE_POOL_NAME CLUSTER_NAME \
+    CLUSTER_LOCATION
+  # substrate's teardown.sh reads BUCKET_NAME for the snapshot bucket; the
+  # generated .ate-dev-env.sh below writes it from AX_SNAPSHOTS_BUCKET.
+  local BUCKET_NAME="${AX_SNAPSHOTS_BUCKET}"
+  local src
+  src="$(ensure_substrate_src)"
+
+  local env_file="${src}/.ate-dev-env.sh"
+  local created_env_file=false
+  if [[ -z "${AX_SUBSTRATE_DIR:-}" ]] || [[ ! -f "${env_file}" ]]; then
+    created_env_file=true
+    local v
+    : >"${env_file}"
+    for v in PROJECT_ID PROJECT_NUMBER BUCKET_NAME NODE_POOL_NAME CLUSTER_NAME \
+      CLUSTER_LOCATION; do
+      printf 'export %s=%q\n' "${v}" "${!v}" >>"${env_file}"
+    done
+  fi
+
+  local rc=0
+  ( cd "${src}" && ./hack/teardown.sh --all ) || rc=$?
+  if [[ "${created_env_file}" == true ]]; then
+    rm -f "${env_file}"
+  fi
+  return "${rc}"
+}
+
+# deploy_ate_system deploys the substrate control plane (CRDs, ateapi,
+# atecontroller, atelet, atenet, valkey, pod-cert controller) at AX's pinned
+# version. Idempotent: re-applying an unchanged version is a no-op.
+deploy_ate_system() {
+  log_step "deploy_ate_system"
+  run_substrate ./hack/install-ate.sh --deploy-ate-system
+}
+
+# delete_ate_system removes the substrate control plane.
+delete_ate_system() {
+  log_step "delete_ate_system"
+  run_substrate ./hack/install-ate.sh --delete-ate-system
+}
+
+deploy_ax() {
+  log_step "deploy_ax"
 
   # Check dependencies
   if [[ -z "${GEMINI_API_KEY:-}" ]]; then
     echo "Error: GEMINI_API_KEY environment variable must be set" >&2
     exit 1
   fi
-  if [[ -z "${BUCKET_NAME:-}" ]]; then
-    echo "Error: BUCKET_NAME environment variable must be set" >&2
+  if [[ -z "${AX_SNAPSHOTS_BUCKET:-}" ]]; then
+    echo "Error: AX_SNAPSHOTS_BUCKET environment variable must be set" >&2
     exit 1
   fi
 
-  echo "Using GCS Bucket: ${BUCKET_NAME}"
+  echo "Using GCS Bucket: ${AX_SNAPSHOTS_BUCKET}"
 
   # Build and push the images, capturing their digest-pinned references.
   local ax_image ateom_image
@@ -216,16 +383,16 @@ deploy_ax_server() {
 
   # Render the manifest and apply it.
   if ! sed -e "s|\${GEMINI_API_KEY}|${GEMINI_API_KEY}|g" \
-      -e "s|\${BUCKET_NAME}|${BUCKET_NAME}|g" \
+      -e "s|\${AX_SNAPSHOTS_BUCKET}|${AX_SNAPSHOTS_BUCKET}|g" \
       -e "s|\${AX_IMAGE}|${ax_image}|g" \
       -e "s|\${ATEOM_IMAGE}|${ateom_image}|g" \
       -e "s|\${POSTGRES_PASSWORD}|${pg_password}|g" \
       manifests/ax-deployment.yaml \
       | run_kubectl apply -f -; then
     echo >&2
     echo "Error: cluster rejected the manifest. An \"unknown field\" error usually means the" >&2
-    echo "cluster's substrate is incompatible with AX's go.mod pin — see" >&2
-    echo "manifests/README.md (\"Substrate compatibility\")." >&2
+    echo "cluster's substrate is incompatible with AX's go.mod pin. Sync the control" >&2
+    echo "plane to the pinned version with: $0 --deploy-ate-system" >&2
     exit 1
   fi
 
@@ -246,12 +413,12 @@ deploy_ax_server() {
   echo "kubectl port-forward -n ax rs/ax-server 8494:8494"
 }
 
-# delete_ax_server removes the AX server and harness resources but preserves the
+# delete_ax removes the AX server and harness resources but preserves the
 # event-log database: it leaves the namespace and the Postgres subsystem
 # (Service/Secret/StatefulSet and its PVC) intact so a later redeploy reuses the
 # existing data.
-delete_ax_server() {
-  log_step "delete_ax_server"
+delete_ax() {
+  log_step "delete_ax"
 
   run_kubectl -n ax delete --ignore-not-found \
     replicaset/ax-server \
@@ -260,6 +427,13 @@ delete_ax_server() {
     workerpool/ax-harness-workerpool
 }
 
+# delete_all removes the AX server and then the substrate control plane. It does
+# not delete the cluster or GCP resources (use --delete-cluster for that).
+delete_all() {
+  delete_ax
+  delete_ate_system
+}
+
 if [ "$#" -eq 0 ]; then
   usage
   exit 1
@@ -275,10 +449,18 @@ for arg in "$@"; do
   esac
 done
 
+# PROJECT_ID and CLUSTER_NAME are required for all operations.
+require_env PROJECT_ID CLUSTER_NAME
+
 while [[ "$#" -gt 0 ]]; do
   case $1 in
-    --deploy-ax-server) deploy_ax_server ;;
-    --delete-ax-server) delete_ax_server ;;
+    --create-cluster) create_cluster ;;
+    --delete-cluster) delete_cluster ;;
+    --delete-all) delete_all ;;
+    --deploy-ate-system) deploy_ate_system ;;
+    --delete-ate-system) delete_ate_system ;;
+    --deploy-ax) deploy_ax ;;
+    --delete-ax) delete_ax ;;
     *)
       echo "Error: unknown option: $1" >&2
       echo ""