scrape and alert #5
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: scrape and alert | |
| on: | |
| schedule: | |
| # Daily at 7am ET (12:00 UTC). The city ArcGIS feeds usually refresh overnight. | |
| - cron: "0 12 * * *" | |
| # Sunday at 7am ET. Triggers the weekly digest job below. | |
| - cron: "0 12 * * 0" | |
| workflow_dispatch: | |
| jobs: | |
| scrape: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: check secrets | |
| id: secrets | |
| env: | |
| DATABASE_URL: ${{ secrets.DATABASE_URL }} | |
| run: | | |
| if [ -n "$DATABASE_URL" ]; then | |
| echo "has_db=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "has_db=false" >> "$GITHUB_OUTPUT" | |
| echo "::notice title=civic-philly scrape skipped::DATABASE_URL secret not configured. Run 'gh secret set DATABASE_URL --repo c-tonneslan/civic-philly' to enable the scheduled scrape." | |
| fi | |
| - uses: actions/setup-node@v4 | |
| if: steps.secrets.outputs.has_db == 'true' | |
| with: | |
| node-version: "20" | |
| cache: "npm" | |
| - if: steps.secrets.outputs.has_db == 'true' | |
| run: npm ci | |
| - name: run all scrapers | |
| if: steps.secrets.outputs.has_db == 'true' | |
| env: | |
| DATABASE_URL: ${{ secrets.DATABASE_URL }} | |
| SCRAPE_REQUIRE_DB: "true" | |
| run: npm run scrape:all | |
| - name: refresh demolition permits + violations + owners + transfers | |
| if: steps.secrets.outputs.has_db == 'true' | |
| env: | |
| DATABASE_URL: ${{ secrets.DATABASE_URL }} | |
| run: | | |
| npm run load:displacement | |
| npm run load:violations | |
| npm run load:owners | |
| npm run load:transfers | |
| - name: backfill developer + spatial joins for new rows | |
| if: steps.secrets.outputs.has_db == 'true' | |
| env: | |
| DATABASE_URL: ${{ secrets.DATABASE_URL }} | |
| run: | | |
| npm run backfill:developers | |
| npm run backfill:spatial | |
| - name: send alerts | |
| if: steps.secrets.outputs.has_db == 'true' | |
| env: | |
| DATABASE_URL: ${{ secrets.DATABASE_URL }} | |
| RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }} | |
| ALERT_FROM_EMAIL: ${{ secrets.ALERT_FROM_EMAIL }} | |
| NEXT_PUBLIC_BASE_URL: ${{ secrets.PUBLIC_BASE_URL }} | |
| run: npm run alerts:send | |
| digest: | |
| # Sunday at noon UTC (7am ET). Weekly per-district summary. | |
| if: github.event_name == 'workflow_dispatch' || (github.event_name == 'schedule' && contains(github.event.schedule, '0 12 * * 0')) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: check secrets | |
| id: secrets | |
| env: | |
| DATABASE_URL: ${{ secrets.DATABASE_URL }} | |
| run: | | |
| if [ -n "$DATABASE_URL" ]; then | |
| echo "has_db=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "has_db=false" >> "$GITHUB_OUTPUT" | |
| echo "::notice title=civic-philly digest skipped::DATABASE_URL secret not configured." | |
| fi | |
| - uses: actions/setup-node@v4 | |
| if: steps.secrets.outputs.has_db == 'true' | |
| with: { node-version: "20", cache: "npm" } | |
| - if: steps.secrets.outputs.has_db == 'true' | |
| run: npm ci | |
| - name: send weekly digests | |
| if: steps.secrets.outputs.has_db == 'true' | |
| env: | |
| DATABASE_URL: ${{ secrets.DATABASE_URL }} | |
| RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }} | |
| ALERT_FROM_EMAIL: ${{ secrets.ALERT_FROM_EMAIL }} | |
| NEXT_PUBLIC_BASE_URL: ${{ secrets.PUBLIC_BASE_URL }} | |
| run: npm run digest:send |