weekly-integration.yml

# Weekly integration for AccountFoundation. Cron Sunday 04:00 UTC +
# workflow_dispatch for ad-hoc verification. OIDC only — no long-lived
# AWS credentials. 30-min job timeout (15-min Pulumi up + 10 min destroy
# + 5 min slop). Teardown runs on failure (cost safety per global red
# lines).
#
# State backend: prefer a self-managed, private S3 backend via the
# PULUMI_BACKEND_URL repository secret. Pulumi Cloud remains supported via
# PULUMI_ACCESS_TOKEN for users who opt in. If neither is present, the
# workflow runs CONTRACT-ONLY paths and logs that no pulumi up will run.
# Documented at docs/integration-testing.md.

name: weekly-integration
on:
  schedule:
    - cron: "0 4 * * 0"
  workflow_dispatch:
    inputs:
      tier:
        description: "Which tier to test (sandbox | startup-hardened | both)"
        required: false
        default: "both"
        type: choice
        options:
          - sandbox
          - startup-hardened
          - both

permissions:
  id-token: write
  contents: read

concurrency:
  group: weekly-integration-${{ github.ref }}
  cancel-in-progress: false

jobs:
  integration:
    name: account-foundation integration (real AWS)
    runs-on: ubuntu-latest
    timeout-minutes: 30
    # Scheduled runs do not need an environment review (no actor), but
    # the workflow_dispatch path is fired with the dispatching actor's
    # permissions and consumes OIDC against the sandbox account. Gate
    # behind a protected environment so any manual dispatch must clear
    # required reviewers. (Maintainer must configure
    # `aws-weekly-integration` in repo settings — see PR body.)
    environment: aws-weekly-integration
    strategy:
      fail-fast: false
      # AccountFoundation touches account-wide services such as Config,
      # GuardDuty, Security Hub, and IAM password policy. Keep tiers
      # serialized so the sandbox account is never mutated by two jobs at
      # once.
      max-parallel: 1
      matrix:
        tier: ${{ fromJSON(github.event_name == 'workflow_dispatch' && inputs.tier == 'sandbox' && '["sandbox"]' || github.event_name == 'workflow_dispatch' && inputs.tier == 'startup-hardened' && '["startup-hardened"]' || '["sandbox","startup-hardened"]') }}
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v6

      - name: Confirm required repo secrets/variables are set
        id: env
        run: |
          : "${ACCOUNT_ID:?AWS_SANDBOX_ACCOUNT_ID secret not set}"
          : "${ROLE_ARN:?AWS_SANDBOX_OIDC_ROLE_ARN secret not set}"
          : "${REGION:?AWS_SANDBOX_REGION variable not set}"
          node <<'NODE'
          const fs = require("node:fs");

          const out = process.env.GITHUB_OUTPUT;
          const backendUrl = (process.env.PULUMI_BACKEND_URL || "").trim();
          const token = (process.env.PULUMI_ACCESS_TOKEN || "").trim();
          const region = (process.env.REGION || "").trim();
          const accountId = (process.env.ACCOUNT_ID || "").trim();
          const roleArn = (process.env.ROLE_ARN || "").trim();

          function write(name, value) {
            fs.appendFileSync(out, `${name}=${value}\n`);
          }

          for (const value of [accountId, roleArn, backendUrl, token]) {
            if (value) {
              console.log(`::add-mask::${value}`);
            }
          }

          if (backendUrl && token) {
            console.error(
              "Set exactly one Pulumi backend: PULUMI_BACKEND_URL (preferred S3 backend) or PULUMI_ACCESS_TOKEN (Pulumi Cloud).",
            );
            process.exit(1);
          }

          if (!backendUrl && !token) {
            console.log(
              "No Pulumi state backend configured — running in CONTRACT-ONLY mode (no pulumi up).",
            );
            write("contract_only", "true");
            write("backend_mode", "none");
            process.exit(0);
          }

          if (backendUrl) {
            let parsed;
            try {
              parsed = new URL(backendUrl);
            } catch (err) {
              console.error(`PULUMI_BACKEND_URL is not a valid URL: ${err.message}`);
              process.exit(1);
            }
            if (parsed.protocol !== "s3:") {
              console.error(
                "PULUMI_BACKEND_URL must use s3:// for this public-repo workflow. Refusing file:// and other local/ambient backends.",
              );
              process.exit(1);
            }
            const bucket = parsed.hostname;
            const backendRegion = parsed.searchParams.get("region") || region;
            if (!bucket || !backendRegion) {
              console.error("PULUMI_BACKEND_URL must include an S3 bucket and region.");
              process.exit(1);
            }
            if (!bucket.startsWith("hulumi-") || !bucket.endsWith(accountId)) {
              console.error(
                "State bucket name must start with 'hulumi-' and end with the sandbox account ID to avoid shared/prod buckets.",
              );
              process.exit(1);
            }
            console.log(`::add-mask::${bucket}`);
            write("contract_only", "false");
            write("backend_mode", "s3");
            write("state_bucket", bucket);
            write("state_region", backendRegion);
            process.exit(0);
          }

          write("contract_only", "false");
          write("backend_mode", "pulumi-cloud");
          NODE
        env:
          ACCOUNT_ID: ${{ secrets.AWS_SANDBOX_ACCOUNT_ID }}
          ROLE_ARN: ${{ secrets.AWS_SANDBOX_OIDC_ROLE_ARN }}
          REGION: ${{ vars.AWS_SANDBOX_REGION }}
          PULUMI_BACKEND_URL: ${{ secrets.PULUMI_BACKEND_URL }}
          PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}

      - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
        with:
          version: 9.12.0
      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
        with:
          node-version: 20
          cache: pnpm
      - run: pnpm install --frozen-lockfile

      - name: Configure AWS credentials via OIDC
        uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6
        with:
          role-to-assume: ${{ secrets.AWS_SANDBOX_OIDC_ROLE_ARN }}
          aws-region: ${{ vars.AWS_SANDBOX_REGION }}
          role-session-name: hulumi-weekly-integration-${{ matrix.tier }}

      - name: Verify state bucket is owned by the sandbox account
        if: steps.env.outputs.backend_mode == 's3'
        env:
          STATE_BUCKET: ${{ steps.env.outputs.state_bucket }}
          AWS_SANDBOX_ACCOUNT_ID: ${{ secrets.AWS_SANDBOX_ACCOUNT_ID }}
        run: |
          # Name-shape checks (hulumi- prefix + account-id suffix) do not
          # prove ownership: an attacker can pre-create a bucket whose
          # name embeds our account ID in another AWS account. Resolve
          # the canonical owner and fail closed on mismatch BEFORE any
          # state I/O or bucket hardening writes. The `s3api` calls below
          # additionally pass --expected-bucket-owner as defense in depth.
          : "${AWS_SANDBOX_ACCOUNT_ID:?AWS_SANDBOX_ACCOUNT_ID secret not set}"
          if aws s3api head-bucket \
            --bucket "$STATE_BUCKET" \
            --expected-bucket-owner "$AWS_SANDBOX_ACCOUNT_ID" >/dev/null 2>&1; then
            OWNER_ID="$(aws s3api get-bucket-acl \
              --bucket "$STATE_BUCKET" \
              --expected-bucket-owner "$AWS_SANDBOX_ACCOUNT_ID" \
              --query 'Owner.ID' --output text)"
            CANONICAL_ID="$(aws s3api list-buckets \
              --query 'Owner.ID' --output text)"
            if [ -z "$OWNER_ID" ] || [ "$OWNER_ID" != "$CANONICAL_ID" ]; then
              echo "::error::State bucket owner does not match the sandbox account canonical user. Refusing to touch it."
              exit 1
            fi
            echo "State bucket ownership verified against the sandbox account."
          else
            # head-bucket failed under --expected-bucket-owner: either the
            # bucket does not exist yet (we will create it below, owned by
            # this account) or it exists but is NOT owned by the sandbox
            # account (403). Distinguish the two — fail closed on 403.
            if aws s3api head-bucket --bucket "$STATE_BUCKET" >/dev/null 2>&1; then
              echo "::error::State bucket exists but is not owned by the sandbox account. Refusing to touch it."
              exit 1
            fi
            echo "State bucket does not exist yet — it will be created in this account by the hardening step."
          fi

      - name: Ensure S3 Pulumi state backend is hardened
        if: steps.env.outputs.backend_mode == 's3'
        env:
          STATE_BUCKET: ${{ steps.env.outputs.state_bucket }}
          STATE_REGION: ${{ steps.env.outputs.state_region }}
          AWS_SANDBOX_ACCOUNT_ID: ${{ secrets.AWS_SANDBOX_ACCOUNT_ID }}
        run: |
          : "${AWS_SANDBOX_ACCOUNT_ID:?AWS_SANDBOX_ACCOUNT_ID secret not set}"
          if ! aws s3api head-bucket --bucket "$STATE_BUCKET" >/dev/null 2>&1; then
            if [ "$STATE_REGION" = "us-east-1" ]; then
              aws s3api create-bucket --bucket "$STATE_BUCKET" --region "$STATE_REGION"
            else
              aws s3api create-bucket \
                --bucket "$STATE_BUCKET" \
                --region "$STATE_REGION" \
                --create-bucket-configuration "LocationConstraint=$STATE_REGION"
            fi
          fi

          aws s3api put-public-access-block \
            --bucket "$STATE_BUCKET" \
            --expected-bucket-owner "$AWS_SANDBOX_ACCOUNT_ID" \
            --public-access-block-configuration \
              BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true

          aws s3api put-bucket-versioning \
            --bucket "$STATE_BUCKET" \
            --expected-bucket-owner "$AWS_SANDBOX_ACCOUNT_ID" \
            --versioning-configuration Status=Enabled

          aws s3api put-bucket-encryption \
            --bucket "$STATE_BUCKET" \
            --expected-bucket-owner "$AWS_SANDBOX_ACCOUNT_ID" \
            --server-side-encryption-configuration \
              '{"Rules":[{"ApplyServerSideEncryptionByDefault":{"SSEAlgorithm":"AES256"},"BucketKeyEnabled":true}]}'

      - name: Build packages
        run: pnpm -r build

      - name: Run mock unit + integration scenarios (always)
        env:
          HULUMI_INTEGRATION: "0"
        run: pnpm --filter @hulumi/baseline test

      - name: Run real-AWS integration (only when a Pulumi backend is configured)
        if: steps.env.outputs.contract_only == 'false'
        env:
          HULUMI_INTEGRATION: "1"
          HULUMI_TIER: ${{ matrix.tier }}
          HULUMI_IAC_ROLE_ARN: ${{ secrets.AWS_SANDBOX_OIDC_ROLE_ARN }}
          PULUMI_BACKEND_URL: ${{ secrets.PULUMI_BACKEND_URL }}
          PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
          AWS_REGION: ${{ vars.AWS_SANDBOX_REGION }}
        run: |
          # Real-AWS path — uses Pulumi Automation API to drive the
          # examples/account-foundation-smoke program against the sandbox
          # account. Implementation lives in the integration test file;
          # this step is the trigger.
          pnpm --filter @hulumi/baseline test -- tests/integration/

      - name: Run drift-classify integration (after AccountFoundation up)
        if: steps.env.outputs.contract_only == 'false' && vars.HULUMI_DRIFT_INTEGRATION == '1'
        env:
          HULUMI_INTEGRATION: "1"
          HULUMI_TIER: ${{ matrix.tier }}
          PULUMI_BACKEND_URL: ${{ secrets.PULUMI_BACKEND_URL }}
          PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
          AWS_REGION: ${{ vars.AWS_SANDBOX_REGION }}
        run: pnpm --filter @hulumi/drift exec vitest run --config vitest.integration.config.ts tests/integration/drift-classify.integration.test.ts

      - name: Run drift reconciler integration (explicit execute gate)
        if: steps.env.outputs.contract_only == 'false' && vars.HULUMI_RECONCILER_AWS_INTEGRATION == '1'
        env:
          HULUMI_INTEGRATION: "1"
          HULUMI_RECONCILER_AWS_INTEGRATION: "1"
          AWS_REGION: ${{ vars.AWS_SANDBOX_REGION }}
        run: pnpm --filter @hulumi/drift exec vitest run --config vitest.integration.config.ts tests/integration/reconciler-cloudwatch-log-group.integration.test.ts tests/integration/reconciler-s3.integration.test.ts

      - name: Run k8s-baseline integration lanes (contract-only by default)
        # The kind / EKS lanes skip cleanly when their flags are unset, so
        # this step always runs. When HULUMI_INTEGRATION_KIND or
        # HULUMI_INTEGRATION_EKS is set on the workflow run (via dispatch
        # input or secrets in a future milestone), the lanes exercise real
        # clusters.
        run: |
          pnpm --filter @hulumi/k8s-baseline run test:integration:kind
          pnpm --filter @hulumi/k8s-baseline run test:integration:eks

      - name: Upload stack export on failure
        if: failure() && steps.env.outputs.contract_only == 'false'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: hulumi-stack-export-${{ matrix.tier }}
          path: |
            packages/baseline/.pulumi/
            packages/baseline/*.log
          if-no-files-found: ignore

      - name: Teardown (always runs on failure too)
        if: always() && steps.env.outputs.contract_only == 'false'
        env:
          HULUMI_INTEGRATION: "1"
          HULUMI_TEARDOWN_ONLY: "1"
          HULUMI_TIER: ${{ matrix.tier }}
          PULUMI_BACKEND_URL: ${{ secrets.PULUMI_BACKEND_URL }}
          PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
          AWS_REGION: ${{ vars.AWS_SANDBOX_REGION }}
        run: |
          # Best-effort teardown. Failures here are logged; the integration
          # test itself owns afterAll() teardown for the primary path.
          echo "Teardown step — stack-level afterAll() in the integration test handled the primary destroy path."