Reduce GTFS-RT archiver log ingestion costs #1351
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Terraform Plan | |
| on: | |
| pull_request: | |
| paths: | |
| - 'iac/**' | |
| - 'services/gtfs-rt-archiver/**' | |
| env: | |
| SERVICE_ACCOUNT: 'github-actions-terraform@cal-itp-data-infra.iam.gserviceaccount.com' | |
| WORKLOAD_IDENTITY_PROVIDER: 'projects/1005246706141/locations/global/workloadIdentityPools/github-actions/providers/data-infra' | |
| PROJECT_ID: 'cal-itp-data-infra' | |
| STAGING_SERVICE_ACCOUNT: 'github-actions-terraform@cal-itp-data-infra-staging.iam.gserviceaccount.com' | |
| STAGING_WORKLOAD_IDENTITY_PROVIDER: 'projects/473674835135/locations/global/workloadIdentityPools/github-actions/providers/data-infra' | |
| STAGING_PROJECT_ID: 'cal-itp-data-infra-staging' | |
| jobs: | |
| targets: | |
| name: Find targets | |
| runs-on: ubuntu-latest | |
| outputs: | |
| staging: ${{ steps.compute.outputs.staging }} | |
| production: ${{ steps.compute.outputs.production }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Staging Changes | |
| uses: tj-actions/changed-files@v46 | |
| id: staging-changes | |
| with: | |
| matrix: true | |
| dir_names: true | |
| files: 'iac/cal-itp-data-infra-staging/**/*.tf' | |
| - name: Production Changes | |
| uses: tj-actions/changed-files@v46 | |
| id: production-changes | |
| with: | |
| matrix: true | |
| dir_names: true | |
| files: 'iac/cal-itp-data-infra/**/*.tf' | |
| # A pure source change under services/gtfs-rt-archiver does not touch any | |
| # *.tf, but it bumps the archive zip hash, so the archiver iac dir must be | |
| # planned to surface the Cloud Functions redeploy. | |
| - name: Archiver source changes | |
| uses: tj-actions/changed-files@v46 | |
| id: archiver-changes | |
| with: | |
| files: 'services/gtfs-rt-archiver/**' | |
| - name: Compute targets | |
| id: compute | |
| env: | |
| STAGING_TF: ${{ steps.staging-changes.outputs.all_changed_files }} | |
| PRODUCTION_TF: ${{ steps.production-changes.outputs.all_changed_files }} | |
| ARCHIVER_CHANGED: ${{ steps.archiver-changes.outputs.any_changed }} | |
| run: | | |
| staging="${STAGING_TF:-[]}" | |
| production="${PRODUCTION_TF:-[]}" | |
| if [ "$ARCHIVER_CHANGED" = "true" ]; then | |
| staging=$(echo "$staging" | jq -c '. + ["iac/cal-itp-data-infra-staging/gtfs-rt-archiver/us"] | unique') | |
| production=$(echo "$production" | jq -c '. + ["iac/cal-itp-data-infra/gtfs-rt-archiver/us"] | unique') | |
| fi | |
| echo "staging=$staging" >> "$GITHUB_OUTPUT" | |
| echo "production=$production" >> "$GITHUB_OUTPUT" | |
| echo "Staging targets: $staging" | |
| echo "Production targets: $production" | |
| staging: | |
| name: Staging | |
| needs: [targets] | |
| if: ${{ needs.targets.outputs.staging != '[]' && needs.targets.outputs.staging != '' }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| id-token: write | |
| pull-requests: write | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| path: ${{ fromJson(needs.targets.outputs.staging) }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Authenticate Google Service Account | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| create_credentials_file: true | |
| project_id: ${{ env.STAGING_PROJECT_ID }} | |
| workload_identity_provider: ${{ env.STAGING_WORKLOAD_IDENTITY_PROVIDER }} | |
| service_account: ${{ env.STAGING_SERVICE_ACCOUNT }} | |
| - name: Setup GCloud utilities | |
| uses: google-github-actions/setup-gcloud@v3 | |
| - name: Terraform Formatting | |
| uses: dflook/terraform-fmt-check@v1 | |
| with: | |
| path: ${{ matrix.path }} | |
| - name: Terraform Validation | |
| uses: dflook/terraform-validate@v1 | |
| with: | |
| path: ${{ matrix.path }} | |
| - name: Note archiver redeploy | |
| if: contains(matrix.path, 'gtfs-rt-archiver') | |
| run: echo "::notice title=Archiver redeploy::Planning ${{ matrix.path }} — GTFS-RT archiver source (services/gtfs-rt-archiver) changed; zip hash bumped, so the Cloud Functions redeploy." | |
| - name: Terraform Plan | |
| uses: dflook/terraform-plan@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| TERRAFORM_PRE_RUN: | | |
| git config --global --add safe.directory $GITHUB_WORKSPACE | |
| with: | |
| add_github.qkg1.topment: changes-only | |
| path: ${{ matrix.path }} | |
| production: | |
| name: Production | |
| needs: [targets] | |
| if: ${{ needs.targets.outputs.production != '[]' && needs.targets.outputs.production != '' }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| id-token: write | |
| pull-requests: write | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| path: ${{ fromJson(needs.targets.outputs.production) }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Authenticate Google Service Account | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| create_credentials_file: true | |
| project_id: ${{ env.PROJECT_ID }} | |
| workload_identity_provider: ${{ env.WORKLOAD_IDENTITY_PROVIDER }} | |
| service_account: ${{ env.SERVICE_ACCOUNT }} | |
| - name: Setup GCloud utilities | |
| uses: google-github-actions/setup-gcloud@v3 | |
| - name: Terraform Format Check | |
| uses: dflook/terraform-fmt-check@v1 | |
| with: | |
| path: ${{ matrix.path }} | |
| - name: Terraform Validate | |
| uses: dflook/terraform-validate@v1 | |
| with: | |
| path: ${{ matrix.path }} | |
| - name: Note archiver redeploy | |
| if: contains(matrix.path, 'gtfs-rt-archiver') | |
| run: echo "::notice title=Archiver redeploy::Planning ${{ matrix.path }} — GTFS-RT archiver source (services/gtfs-rt-archiver) changed; zip hash bumped, so the Cloud Functions redeploy." | |
| - name: Terraform Plan | |
| uses: dflook/terraform-plan@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| TERRAFORM_PRE_RUN: | | |
| git config --global --add safe.directory $GITHUB_WORKSPACE | |
| with: | |
| add_github.qkg1.topment: changes-only | |
| path: ${{ matrix.path }} |