E2E Install Test #12
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # End-to-end install test for example app configs. | |
| # | |
| # For each selected app it creates a real install in the stage org, applies the | |
| # Nuon CloudFormation install stack into a test AWS account (assumed via GitHub | |
| # OIDC), waits for the runner + sandbox + components to go active, runs the | |
| # app's verify action, then tears the install down. | |
| # | |
| # The target control-plane environment is chosen at dispatch (`environment` | |
| # input: stage or prod); all secrets resolve from the matching GitHub Environment. | |
| # | |
| # Teardown only runs when the app's install succeeded. If any step fails the | |
| # install is left running so it can be inspected. A second matrix job, `cleanup`, | |
| # runs only on failure and is gated behind the `<environment>-cleanup` GitHub | |
| # Environment: it pauses for manual approval (inspect first), then tears down the | |
| # exact install this run created for the failed app. Installs are named | |
| # deterministically as <app>-ci-<run_id>-<run_attempt> so cleanup can find them | |
| # without any ids. | |
| # | |
| # Prerequisites (per environment, e.g. `stage` and `prod`): | |
| # - `<env>` GitHub Environment (no required reviewers, so tests run un-gated): | |
| # - secret NUON_CONFIG : ~/.nuon contents (api_url, org_id, api_token) | |
| # - secret AWS_ROLE_ARN : IAM role to assume via OIDC; must be able to create | |
| # and delete the install stack (IAM, VPC/EC2, ECS/EKS, | |
| # Lambda, CFN nested stacks, Logs, custom resources) | |
| # - `<env>-cleanup` GitHub Environment WITH required reviewer(s): gates the | |
| # cleanup job so it waits for an approval before tearing a failed install | |
| # down. Needs the same NUON_CONFIG / AWS_ROLE_ARN secrets as `<env>`. | |
| # | |
| # NOTE: the AWS_ROLE_ARN role is currently provisioned MANUALLY (not IaC) in the | |
| # demo account (949309607565) as gha-example-app-configs-e2e: | |
| # - trust: GitHub OIDC, sub=repo:nuonco/example-app-configs:environment:{stage,prod,stage-cleanup,prod-cleanup} | |
| # - policy: AdministratorAccess | |
| # TODO: move this to mono IaC (infra/aws/lib/accounts.ts). | |
| # | |
| # Components are pinned to branch=main, so this always tests released config. | |
| name: E2E Install Test | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| environment: | |
| description: "Nuon control-plane environment to test against" | |
| type: choice | |
| options: | |
| - stage | |
| - prod | |
| default: stage | |
| apps: | |
| description: "Apps to test (space or comma separated)" | |
| type: string | |
| default: "aws-lambda ecs-simple eks-simple" | |
| keep_alive: | |
| description: Skip teardown even on success (leave installs running) | |
| type: boolean | |
| default: false | |
| NUON_DEBUG: | |
| description: Enable nuon CLI debug logging | |
| type: boolean | |
| default: false | |
| permissions: | |
| contents: read | |
| id-token: write | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| setup: | |
| name: Build test matrix | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.build.outputs.matrix }} | |
| steps: | |
| - id: build | |
| env: | |
| APPS_INPUT: ${{ inputs.apps }} | |
| run: | | |
| set -euo pipefail | |
| # Known apps -> per-app test config. Add new apps here. | |
| known='{ | |
| "aws-lambda": {"app":"aws-lambda","region":"us-east-1","install_config":"aws-lambda/installs/aws-lambda-ci-stage.toml","verify_action":"http_healthcheck"}, | |
| "ecs-simple": {"app":"ecs-simple","region":"us-east-1","install_config":"ecs-simple/installs/ecs-simple-ci-stage.toml","verify_action":"curl_endpoint"}, | |
| "eks-simple": {"app":"eks-simple","region":"us-east-1","install_config":"eks-simple/installs/eks-simple-ci-stage.toml","verify_action":"alb_healthcheck"} | |
| }' | |
| include='[]' | |
| for a in $(echo "$APPS_INPUT" | tr ',' ' '); do | |
| entry=$(echo "$known" | jq -c --arg a "$a" '.[$a] // empty') | |
| if [ -z "$entry" ]; then | |
| echo "::error::unknown app '$a' (known: $(echo "$known" | jq -r 'keys | join(", ")'))" | |
| exit 1 | |
| fi | |
| include=$(echo "$include" | jq -c --argjson e "$entry" '. + [$e]') | |
| done | |
| if [ "$(echo "$include" | jq 'length')" = "0" ]; then | |
| echo "::error::no apps selected" | |
| exit 1 | |
| fi | |
| echo "matrix={\"include\":$include}" >> "$GITHUB_OUTPUT" | |
| echo "Testing apps: $(echo "$include" | jq -r '[.[].app] | join(", ")')" | |
| install-test: | |
| needs: setup | |
| name: ${{ matrix.app }} | |
| runs-on: ubuntu-latest | |
| environment: ${{ inputs.environment }} | |
| timeout-minutes: 150 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.setup.outputs.matrix) }} | |
| env: | |
| NUON_DEBUG: ${{ inputs.NUON_DEBUG }} | |
| NUON_PREVIEW: "true" | |
| AWS_REGION: ${{ matrix.region }} | |
| APP: ${{ matrix.app }} | |
| INSTALL_CONFIG: ${{ matrix.install_config }} | |
| VERIFY_ACTION: ${{ matrix.verify_action }} | |
| STACK_PREFIX: nuon-e2e | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install nuon CLI | |
| run: ./scripts/install-cli.sh | |
| - name: Configure nuon auth | |
| env: | |
| NUON_CONFIG: ${{ secrets.NUON_CONFIG }} | |
| run: printf '%s' "$NUON_CONFIG" > "$HOME/.nuon" | |
| - name: Configure AWS credentials (OIDC) | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_ROLE_ARN }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Sync app config | |
| run: nuon apps sync "$APP" --create | |
| - name: Create install | |
| run: | | |
| set -euo pipefail | |
| # Deterministic per-run name so the cleanup job can recompute it and | |
| # target this exact install without carrying any ids across jobs. | |
| INSTALL_NAME="${APP}-ci-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" | |
| echo "INSTALL_NAME=$INSTALL_NAME" >> "$GITHUB_ENV" | |
| echo "Using install name: $INSTALL_NAME" | |
| # Render a per-run config from the template with a unique name so | |
| # concurrent runs don't collide (installs sync is non-pruning). | |
| cfg="$(mktemp -d)/install.toml" | |
| sed -E "s|^name[[:space:]]*=.*|name = \"${INSTALL_NAME}\"|" "$INSTALL_CONFIG" > "$cfg" | |
| nuon installs sync -a "$APP" -d "$cfg" -y | |
| INSTALL_ID=$(nuon installs list -a "$APP" --output json \ | |
| | jq -r --arg n "$INSTALL_NAME" '.[] | select(.name==$n) | .id' | head -1) | |
| if [ -z "$INSTALL_ID" ] || [ "$INSTALL_ID" = "null" ]; then | |
| echo "::error::could not resolve install id for $INSTALL_NAME" | |
| exit 1 | |
| fi | |
| echo "INSTALL_ID=$INSTALL_ID" >> "$GITHUB_ENV" | |
| echo "STACK_NAME=${STACK_PREFIX}-${INSTALL_ID}" >> "$GITHUB_ENV" | |
| echo "Created install $INSTALL_ID" | |
| - name: Apply CloudFormation install stack | |
| run: | | |
| set -euo pipefail | |
| # The install stack template is generated asynchronously after the | |
| # install is created; until then, `stacks latest` prints a non-JSON | |
| # "no stack versions found" message, so tolerate parse failures. | |
| deadline=$(( $(date +%s) + 300 )) | |
| while :; do | |
| stack=$(nuon installs stacks latest --install-id "$INSTALL_ID" --output json 2>/dev/null || true) | |
| TEMPLATE_URL=$(echo "$stack" | jq -r '.template_url // ""' 2>/dev/null || echo "") | |
| if [ -n "$TEMPLATE_URL" ] && [ "$TEMPLATE_URL" != "null" ]; then | |
| break | |
| fi | |
| echo "waiting for install stack template to generate..." | |
| if [ "$(date +%s)" -ge "$deadline" ]; then | |
| echo "::error::timed out waiting for install stack template to generate" | |
| exit 1 | |
| fi | |
| sleep 10 | |
| done | |
| echo "Applying stack $STACK_NAME from $TEMPLATE_URL" | |
| aws cloudformation create-stack \ | |
| --stack-name "$STACK_NAME" \ | |
| --template-url "$TEMPLATE_URL" \ | |
| --capabilities CAPABILITY_NAMED_IAM CAPABILITY_AUTO_EXPAND \ | |
| --region "$AWS_REGION" | |
| aws cloudformation wait stack-create-complete \ | |
| --stack-name "$STACK_NAME" --region "$AWS_REGION" | |
| echo "Stack $STACK_NAME create complete" | |
| - name: Wait for install to go active | |
| run: | | |
| set -euo pipefail | |
| # The runner boots from an autoscaling group while the CloudFormation | |
| # stack is still creating. Until the stack's PhoneHome lambda reports | |
| # the install's AWS account id (at stack completion), the runner's IID | |
| # auth returns 403 and the VM cycles, so runner_status is transiently | |
| # "error" ("waiting for health check to mark healthy"). Treat those as | |
| # transient and only fail on the overall deadline. | |
| deadline=$(( $(date +%s) + 7200 )) | |
| while :; do | |
| json=$(nuon installs get -i "$INSTALL_ID" --output json) | |
| runner=$(echo "$json" | jq -r '.runner_status // "unknown"') | |
| sandbox=$(echo "$json" | jq -r '.sandbox_status // "unknown"') | |
| comp=$(echo "$json" | jq -r '.composite_component_status // "unknown"') | |
| echo "runner=$runner sandbox=$sandbox components=$comp" | |
| if [ "$runner" = "active" ] && [ "$sandbox" = "active" ] && [ "$comp" = "active" ]; then | |
| echo "install is active" | |
| break | |
| fi | |
| if [ "$(date +%s)" -ge "$deadline" ]; then | |
| echo "::error::timed out waiting for install to go active" | |
| echo "$json" | jq '{runner_status,runner_status_description,sandbox_status,composite_component_status,composite_component_status_description}' | |
| exit 1 | |
| fi | |
| sleep 20 | |
| done | |
| - name: Verify install | |
| run: | | |
| set -euo pipefail | |
| ACW=$(nuon installs actions list -i "$INSTALL_ID" --output json \ | |
| | jq -r --arg n "$VERIFY_ACTION" '.[] | select(.action_workflow.name==$n) | .action_workflow.id' | head -1) | |
| if [ -z "$ACW" ] || [ "$ACW" = "null" ]; then | |
| echo "::error::verify action '$VERIFY_ACTION' not found" | |
| exit 1 | |
| fi | |
| # Some verify actions also auto-trigger when the install goes active, | |
| # and those early runs can fail transiently (e.g. TLS before the ACM | |
| # cert/DNS is ready). Trigger our own run, only track runs created at | |
| # or after our trigger (never recent-runs[0], which races with an auto | |
| # run), and retry on a transient error until it succeeds. When the run | |
| # exposes an http_status_code output, additionally require 200. | |
| attempts=6 | |
| for attempt in $(seq 1 "$attempts"); do | |
| start_ts=$(date +%s) | |
| nuon actions create-run -w "$ACW" -i "$INSTALL_ID" | |
| rid=""; status="" | |
| deadline=$(( $(date +%s) + 180 )) | |
| while :; do | |
| run=$(nuon actions recent-runs -w "$ACW" -i "$INSTALL_ID" --output json \ | |
| | jq -r --argjson ts "$start_ts" \ | |
| '[.[] | select((.created_at | sub("\\.[0-9]+Z$";"Z") | fromdateiso8601) >= $ts)] | .[0] // empty') | |
| rid=$(echo "$run" | jq -r '.id // empty') | |
| status=$(echo "$run" | jq -r '.status // empty') | |
| echo "attempt=$attempt run=${rid:-none} status=${status:-none}" | |
| case "$status" in | |
| finished|error|failed) break ;; | |
| esac | |
| if [ "$(date +%s)" -ge "$deadline" ]; then | |
| echo "run did not settle in time; retrying" | |
| status="timeout"; break | |
| fi | |
| sleep 10 | |
| done | |
| if [ "$status" = "finished" ]; then | |
| code=$(nuon actions get-run --run-id "$rid" -i "$INSTALL_ID" --output json \ | |
| | jq -r '.outputs.http_status_code // .outputs.steps.check.http_status_code // empty') | |
| if [ -n "$code" ]; then | |
| echo "http_status_code=$code" | |
| if [ "$code" = "200" ]; then | |
| echo "$VERIFY_ACTION healthy (200)" | |
| exit 0 | |
| fi | |
| echo "endpoint returned $code; retrying" | |
| else | |
| echo "$VERIFY_ACTION finished successfully" | |
| exit 0 | |
| fi | |
| fi | |
| sleep 15 | |
| done | |
| echo "::error::$VERIFY_ACTION did not succeed after $attempts attempts" | |
| exit 1 | |
| - name: Teardown | |
| # Only tear down when this app's install succeeded. On any failure the | |
| # install is left running for inspection; the approval-gated `cleanup` | |
| # job below tears it down once you approve. | |
| if: ${{ success() && inputs.keep_alive != true && env.INSTALL_ID != '' }} | |
| run: | | |
| set -uo pipefail | |
| echo "Deprovisioning install $INSTALL_ID" | |
| nuon installs deprovision -i "$INSTALL_ID" || true | |
| # sandbox_status reports "unknown" once torn down, so key off a | |
| # completed deprovision sandbox run plus all components inactive. | |
| deadline=$(( $(date +%s) + 900 )) | |
| while :; do | |
| runs=$(nuon installs sandbox-runs --install-id "$INSTALL_ID" --output json 2>/dev/null || echo '[]') | |
| comps=$(nuon installs get -i "$INSTALL_ID" --output json 2>/dev/null \ | |
| | jq -r '(.component_statuses // {}) | [.[]] as $s | ($s|length>0) and ($s|all(.=="inactive"))' 2>/dev/null || echo false) | |
| deprov=$(echo "$runs" | jq -r 'any(.[]; .run_type=="deprovision" and .status=="deprovisioned")' 2>/dev/null || echo false) | |
| # If the install never provisioned a sandbox there is nothing to wait on. | |
| never_provisioned=$(echo "$runs" | jq -r 'length==0' 2>/dev/null || echo false) | |
| echo "components_inactive=$comps deprovisioned=$deprov never_provisioned=$never_provisioned" | |
| if { [ "$comps" = "true" ] && [ "$deprov" = "true" ]; } || [ "$never_provisioned" = "true" ]; then | |
| echo "nothing left to deprovision" | |
| break | |
| fi | |
| [ "$(date +%s)" -ge "$deadline" ] && { echo "::warning::deprovision wait timed out, continuing to stack delete"; break; } | |
| sleep 20 | |
| done | |
| if [ -n "${STACK_NAME:-}" ]; then | |
| echo "Deleting stack $STACK_NAME" | |
| aws cloudformation delete-stack --stack-name "$STACK_NAME" --region "$AWS_REGION" || true | |
| aws cloudformation wait stack-delete-complete --stack-name "$STACK_NAME" --region "$AWS_REGION" || true | |
| fi | |
| echo "Deleting install $INSTALL_ID" | |
| nuon installs delete -i "$INSTALL_ID" --confirm || true | |
| cleanup: | |
| needs: [setup, install-test] | |
| # Runs only when a test leg failed (setup itself must have succeeded so we | |
| # have a matrix). Gated behind the `e2e-cleanup` environment: the job waits | |
| # for a reviewer to approve before touching anything, so failed installs can | |
| # be inspected first. It fans out over the same apps; legs whose install | |
| # already succeeded (and auto-torn-down) find nothing and no-op, so only the | |
| # failed app's install is actually torn down. | |
| if: ${{ failure() && needs.setup.result == 'success' }} | |
| name: cleanup ${{ matrix.app }} | |
| runs-on: ubuntu-latest | |
| environment: ${{ inputs.environment }}-cleanup | |
| timeout-minutes: 90 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.setup.outputs.matrix) }} | |
| env: | |
| NUON_DEBUG: ${{ inputs.NUON_DEBUG }} | |
| NUON_PREVIEW: "true" | |
| AWS_REGION: ${{ matrix.region }} | |
| APP: ${{ matrix.app }} | |
| STACK_PREFIX: nuon-e2e | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install nuon CLI | |
| run: ./scripts/install-cli.sh | |
| - name: Configure nuon auth | |
| env: | |
| NUON_CONFIG: ${{ secrets.NUON_CONFIG }} | |
| run: printf '%s' "$NUON_CONFIG" > "$HOME/.nuon" | |
| - name: Configure AWS credentials (OIDC) | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_ROLE_ARN }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Teardown failed install | |
| run: | | |
| set -uo pipefail | |
| # Recompute the exact name this run used for this app. | |
| INSTALL_NAME="${APP}-ci-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" | |
| INSTALL_ID=$(nuon installs list -a "$APP" --output json \ | |
| | jq -r --arg n "$INSTALL_NAME" '.[] | select(.name==$n) | .id' | head -1) | |
| if [ -z "$INSTALL_ID" ] || [ "$INSTALL_ID" = "null" ]; then | |
| echo "no leftover install '$INSTALL_NAME' (already cleaned up); nothing to do" | |
| exit 0 | |
| fi | |
| STACK_NAME="${STACK_PREFIX}-${INSTALL_ID}" | |
| echo "Cleaning up install $INSTALL_ID ($INSTALL_NAME)" | |
| nuon installs deprovision -i "$INSTALL_ID" || true | |
| deadline=$(( $(date +%s) + 900 )) | |
| while :; do | |
| runs=$(nuon installs sandbox-runs --install-id "$INSTALL_ID" --output json 2>/dev/null || echo '[]') | |
| comps=$(nuon installs get -i "$INSTALL_ID" --output json 2>/dev/null \ | |
| | jq -r '(.component_statuses // {}) | [.[]] as $s | ($s|length>0) and ($s|all(.=="inactive"))' 2>/dev/null || echo false) | |
| deprov=$(echo "$runs" | jq -r 'any(.[]; .run_type=="deprovision" and .status=="deprovisioned")' 2>/dev/null || echo false) | |
| never_provisioned=$(echo "$runs" | jq -r 'length==0' 2>/dev/null || echo false) | |
| echo "components_inactive=$comps deprovisioned=$deprov never_provisioned=$never_provisioned" | |
| if { [ "$comps" = "true" ] && [ "$deprov" = "true" ]; } || [ "$never_provisioned" = "true" ]; then | |
| echo "nothing left to deprovision" | |
| break | |
| fi | |
| [ "$(date +%s)" -ge "$deadline" ] && { echo "::warning::deprovision wait timed out, continuing to stack delete"; break; } | |
| sleep 20 | |
| done | |
| echo "Deleting stack $STACK_NAME" | |
| aws cloudformation delete-stack --stack-name "$STACK_NAME" --region "$AWS_REGION" || true | |
| aws cloudformation wait stack-delete-complete --stack-name "$STACK_NAME" --region "$AWS_REGION" || true | |
| echo "Deleting install $INSTALL_ID" | |
| nuon installs delete -i "$INSTALL_ID" --confirm || true |