scrape: skip cleanly in CI when DATABASE_URL secret is absent

The repo doesn't have DATABASE_URL configured yet, so the daily
scheduled scrape has been failing red for the last three days even
though the scraper code is fine — the workflow just has no way to
authenticate to a database that doesn't exist.

Two changes:

- scripts/scrape-all.mjs: in CI without DATABASE_URL, exit 0 with a
  workflow ::notice instead of failing. Local runs and any run with
  SCRAPE_REQUIRE_DB=true still fail loudly, so dev work isn't
  silenced.
- workflow: a 'check secrets' step gates every downstream step on
  whether DATABASE_URL is set. When unset the run renders one notice
  in the summary and exits green; when set it runs exactly the way
  it used to.

Signed-off-by: Charlie Tonneslan <cst0520@gmail.com>

Author: c-tonneslan

.github/workflows/scrape.yml

@@ -13,16 +13,32 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - name: check secrets
+        id: secrets
+        env:
+          DATABASE_URL: ${{ secrets.DATABASE_URL }}
+        run: |
+          if [ -n "$DATABASE_URL" ]; then
+            echo "has_db=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "has_db=false" >> "$GITHUB_OUTPUT"
+            echo "::notice title=civic-philly scrape skipped::DATABASE_URL secret not configured. Run 'gh secret set DATABASE_URL --repo c-tonneslan/civic-philly' to enable the scheduled scrape."
+          fi
       - uses: actions/setup-node@v4
+        if: steps.secrets.outputs.has_db == 'true'
         with:
           node-version: "20"
           cache: "npm"
-      - run: npm ci
+      - if: steps.secrets.outputs.has_db == 'true'
+        run: npm ci
       - name: run all scrapers
+        if: steps.secrets.outputs.has_db == 'true'
         env:
           DATABASE_URL: ${{ secrets.DATABASE_URL }}
+          SCRAPE_REQUIRE_DB: "true"
         run: npm run scrape:all
       - name: refresh demolition permits + violations + owners + transfers
+        if: steps.secrets.outputs.has_db == 'true'
         env:
           DATABASE_URL: ${{ secrets.DATABASE_URL }}
         run: |
@@ -31,12 +47,14 @@ jobs:
           npm run load:owners
           npm run load:transfers
       - name: backfill developer + spatial joins for new rows
+        if: steps.secrets.outputs.has_db == 'true'
         env:
           DATABASE_URL: ${{ secrets.DATABASE_URL }}
         run: |
           npm run backfill:developers
           npm run backfill:spatial
       - name: send alerts
+        if: steps.secrets.outputs.has_db == 'true'
         env:
           DATABASE_URL: ${{ secrets.DATABASE_URL }}
           RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}
@@ -50,10 +68,24 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - name: check secrets
+        id: secrets
+        env:
+          DATABASE_URL: ${{ secrets.DATABASE_URL }}
+        run: |
+          if [ -n "$DATABASE_URL" ]; then
+            echo "has_db=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "has_db=false" >> "$GITHUB_OUTPUT"
+            echo "::notice title=civic-philly digest skipped::DATABASE_URL secret not configured."
+          fi
       - uses: actions/setup-node@v4
+        if: steps.secrets.outputs.has_db == 'true'
         with: { node-version: "20", cache: "npm" }
-      - run: npm ci
+      - if: steps.secrets.outputs.has_db == 'true'
+        run: npm ci
       - name: send weekly digests
+        if: steps.secrets.outputs.has_db == 'true'
         env:
           DATABASE_URL: ${{ secrets.DATABASE_URL }}
           RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}

scripts/scrape-all.mjs

@@ -1,6 +1,11 @@
 // Run every scraper in sequence. Used by the scheduled job (cron / GH Action).
 //
 //   node scripts/scrape-all.mjs
+//
+// In CI without a DATABASE_URL secret configured the script exits 0 with a
+// workflow notice instead of failing every scheduled run. Set
+// SCRAPE_REQUIRE_DB=true to force a hard fail (useful for manual
+// workflow_dispatch runs once the secret is in place, and for local dev).
 
 import { spawn } from "node:child_process";
 import { fileURLToPath } from "node:url";
@@ -9,6 +14,19 @@ import { dirname, join } from "node:path";
 const here = dirname(fileURLToPath(import.meta.url));
 const scripts = ["scrape-housing.mjs", "scrape-zoning.mjs", "scrape-infrastructure.mjs", "scrape-septa.mjs"];
 
+if (!process.env.DATABASE_URL) {
+  const inCI = process.env.CI === "true" || process.env.GITHUB_ACTIONS === "true";
+  const required = process.env.SCRAPE_REQUIRE_DB === "true";
+  if (inCI && !required) {
+    const msg =
+      "DATABASE_URL is not set. Configure the repository secret to enable the scheduled scrape (gh secret set DATABASE_URL --repo c-tonneslan/civic-philly).";
+    console.log(`::notice title=civic-philly scrape skipped::${msg}`);
+    process.exit(0);
+  }
+  console.error("DATABASE_URL is not set.");
+  process.exit(1);
+}
+
 function run(script) {
   return new Promise((resolve) => {
     const path = join(here, script);