data-refresh.yml

name: Refresh intelligence data

on:
  schedule:
    # Daily at 20:17 Asia/Shanghai; the weekly Issue remains Sunday-gated below.
    - cron: "17 12 * * *"
  workflow_dispatch:
    inputs:
      mode:
        description: Refresh mode
        required: true
        default: incremental
        type: choice
        options:
          - incremental
          - backfill
      publish_weekly:
        description: Publish or re-render the current weekly review
        required: false
        default: false
        type: boolean

permissions:
  actions: write
  contents: write
  issues: write

concurrency:
  group: agent-pulse-repository-data-main
  cancel-in-progress: false

jobs:
  refresh:
    runs-on: ubuntu-latest
    timeout-minutes: 35
    env:
      DATABASE_URL: sqlite:./var/automation.db
      PUBLIC_SITE_URL: https://barretlee.github.io/agent-pulse/
      REFRESH_MODE: ${{ inputs.mode || 'incremental' }}
      PUBLISH_WEEKLY: ${{ inputs.publish_weekly || false }}
    steps:
      - name: Check out main
        uses: actions/checkout@v7
        with:
          ref: main
          fetch-depth: 0

      - name: Set up Node.js
        uses: actions/setup-node@v6
        with:
          node-version: 24
          cache: npm

      - name: Install locked dependencies
        run: npm ci

      - name: Restore auditable repository snapshot
        run: npm run db:snapshot -- restore

      - name: Capture the current public content fingerprint
        shell: bash
        run: |
          set -euo pipefail
          npm run export -- --skip-seed
          npm run --silent public:fingerprint > "$RUNNER_TEMP/public-before.sha256"

      - name: Collect and cluster signals
        shell: bash
        run: |
          set -euo pipefail
          if [[ "$REFRESH_MODE" == "backfill" ]]; then
            npm run collect -- --backfill
          else
            npm run collect
          fi

      - name: Observe direct research feeds in shadow
        shell: bash
        run: |
          set -u
          for source in microsoft-research google-research; do
            if ! npm run collect -- --source="$source"; then
              echo "::warning title=Direct research source unavailable::$source was not collected; its lifecycle state is unchanged and the batch will continue"
            fi
          done

      - name: Reconcile sources and source radar
        run: npm run ops:reconcile

      - name: Reconcile observation and activate qualified sources
        shell: bash
        run: |
          set -euo pipefail
          npm run observe:sources -- --confirm
          npm run activate:auto

      - name: Remove unattached aggregator provenance debt
        run: npm run repair:provenance -- --confirm

      - name: Audit research impact before publication
        env:
          OPENALEX_API_KEY: ${{ secrets.OPENALEX_API_KEY }}
        run: npm run --silent research:impact -- --skip-seed

      - name: Enrich evidence-ready review Events with DeepSeek
        id: enrichment
        env:
          AI_ENRICHMENT_ENABLED: "true"
          AI_ENRICHMENT_MAX_EVENTS: "8"
          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
          DEEPSEEK_MODEL: deepseek-v4-flash
        shell: bash
        run: |
          set -euo pipefail
          set +e
          npm run --silent ai:enrich -- --require-success | tee "$RUNNER_TEMP/ai-enrichment.json"
          status="${PIPESTATUS[0]}"
          if [[ "$status" -eq 0 ]] && ! node -e 'JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8"))' "$RUNNER_TEMP/ai-enrichment.json"; then
            status=1
          fi
          set -e
          if [[ "$status" -eq 0 ]]; then
            echo "status=ok" >> "$GITHUB_OUTPUT"
          else
            if [[ ! -s "$RUNNER_TEMP/ai-enrichment.json" ]]; then
              printf '%s\n' '{"enabled":true,"workflowStatus":"failed","error":"ai_enrichment_failed"}' > "$RUNNER_TEMP/ai-enrichment.json"
            fi
            echo "status=failed" >> "$GITHUB_OUTPUT"
            echo "::warning title=AI Event enrichment unavailable::Eligible Events remain in review; deterministic collection and snapshot persistence will continue."
          fi

      - name: Generate and publish qualified Scout opportunities
        run: npm run scout:generate -- 12

      - name: Publish every Event and Scout item that clears hard gates
        run: npm run auto:publish

      - name: Merge latest remote repository snapshot
        shell: bash
        run: |
          set -euo pipefail
          git fetch origin main
          git show origin/main:data/snapshot/v1.json > "$RUNNER_TEMP/remote-snapshot.json"
          git show origin/main:data/narratives/stage-promotions.json > data/narratives/stage-promotions.json
          git show origin/main:data/reports/system-evaluation.json > data/reports/system-evaluation.json
          npm run db:snapshot -- merge --file="$RUNNER_TEMP/remote-snapshot.json"

      - name: Re-apply source reconciliation after remote merge
        run: npm run ops:reconcile

      - name: Persist measured system quality and ranked improvement actions
        shell: bash
        run: |
          set -euo pipefail
          npm run --silent evaluate:system -- \
            --skip-bootstrap \
            --baseline=data/reports/system-evaluation.json \
            --output=data/reports/system-evaluation.json \
            --summary="$GITHUB_STEP_SUMMARY" | tee "$RUNNER_TEMP/evaluation.json"
          node -e 'JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8"))' "$RUNNER_TEMP/evaluation.json"

      - name: Assess a major narrative stage promotion with DeepSeek V4 Pro
        id: stage_proposal
        env:
          AI_STAGE_PROMOTION_ENABLED: "true"
          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
          DEEPSEEK_STAGE_MODEL: deepseek-v4-pro
        shell: bash
        run: |
          set -euo pipefail
          set +e
          npm run --silent narrative:stage -- propose \
            --candidate-path "$RUNNER_TEMP/stage-promotion-candidate.json" \
            | tee "$RUNNER_TEMP/stage-promotion.json"
          status="${PIPESTATUS[0]}"
          set -e
          if [[ "$status" -ne 0 ]] || ! node -e 'JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8"))' "$RUNNER_TEMP/stage-promotion.json"; then
            printf '%s\n' '{"status":"failed","model":"deepseek-v4-pro","errorCode":"stage_proposal_failed"}' > "$RUNNER_TEMP/stage-promotion.json"
            echo "status=failed" >> "$GITHUB_OUTPUT"
            echo "proposed=false" >> "$GITHUB_OUTPUT"
            echo "::warning title=Stage promotion assessment unavailable::No stage will be added; deterministic snapshot persistence will continue."
            exit 0
          fi
          proposal_status="$(node -e 'const x=JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8")); process.stdout.write(String(x.status || "unknown"))' "$RUNNER_TEMP/stage-promotion.json")"
          if [[ "$proposal_status" == "candidate" && -s "$RUNNER_TEMP/stage-promotion-candidate.json" ]]; then
            echo "proposed=true" >> "$GITHUB_OUTPUT"
          else
            echo "proposed=false" >> "$GITHUB_OUTPUT"
          fi
          echo "status=$proposal_status" >> "$GITHUB_OUTPUT"

      - name: Create the milestone Issue and apply the validated stage
        id: stage_apply
        if: steps.stage_proposal.outputs.proposed == 'true'
        env:
          GH_TOKEN: ${{ github.token }}
          ACTIONS_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
        shell: bash
        run: |
          set -euo pipefail
          candidate="$RUNNER_TEMP/stage-promotion-candidate.json"
          body="$RUNNER_TEMP/stage-promotion-issue.md"
          title_file="$RUNNER_TEMP/stage-promotion-title.txt"
          apply_report="$RUNNER_TEMP/stage-promotion-apply.json"
          set +e
          npm run --silent narrative:stage -- render-issue --candidate-path "$candidate" --actions-url "$ACTIONS_URL" > "$body"
          render_status=$?
          npm run --silent narrative:stage -- render-issue --candidate-path "$candidate" --title > "$title_file"
          title_status=$?
          set -e
          if [[ "$render_status" -ne 0 || "$title_status" -ne 0 ]]; then
            printf '%s\n' '{"status":"failed","errorCode":"stage_issue_render_failed"}' > "$apply_report"
            echo "status=failed" >> "$GITHUB_OUTPUT"
            echo "::warning title=Stage milestone Issue skipped::Validated candidate could not be rendered safely."
            exit 0
          fi
          marker="$(node -e 'const x=JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8")); process.stdout.write(x.marker)' "$candidate")"
          title="$(<"$title_file")"
          set +e
          gh label create "stage:milestone" --color 5319e7 --description "Evidence-gated major narrative stage" --force
          label_status=$?
          issue_number="$(gh issue list --state all --label "stage:milestone" --limit 100 --json number,body \
            --jq "first(.[] | select(.body | contains(\"$marker\"))) | .number // empty")"
          list_status=$?
          set -e
          if [[ "$label_status" -ne 0 || "$list_status" -ne 0 ]]; then
            printf '%s\n' '{"status":"failed","errorCode":"stage_issue_lookup_failed"}' > "$apply_report"
            echo "status=failed" >> "$GITHUB_OUTPUT"
            echo "::warning title=Stage milestone Issue skipped::The repository label or existing milestone Issue could not be checked."
            exit 0
          fi
          set +e
          if [[ -n "$issue_number" ]]; then
            gh issue edit "$issue_number" --title "$title" --body-file "$body" --add-label "stage:milestone"
            issue_status=$?
            if [[ "$issue_status" -eq 0 ]]; then gh issue reopen "$issue_number" >/dev/null 2>&1 || true; fi
            issue_url="$(gh issue view "$issue_number" --json url --jq .url 2>/dev/null)"
          else
            issue_url="$(gh issue create --title "$title" --body-file "$body" --label "stage:milestone")"
            issue_status=$?
            issue_number="${issue_url##*/}"
          fi
          set -e
          if [[ "$issue_status" -ne 0 || -z "$issue_number" || -z "$issue_url" ]]; then
            printf '%s\n' '{"status":"failed","errorCode":"stage_issue_upsert_failed"}' > "$apply_report"
            echo "status=failed" >> "$GITHUB_OUTPUT"
            echo "::warning title=Stage milestone Issue skipped::The candidate was not persisted because its dedicated Issue could not be created."
            exit 0
          fi
          set +e
          npm run --silent narrative:stage -- apply \
            --candidate-path "$candidate" \
            --issue-number "$issue_number" \
            --issue-url "$issue_url" | tee "$apply_report"
          apply_status="${PIPESTATUS[0]}"
          set -e
          if [[ "$apply_status" -ne 0 ]]; then
            printf '%s\n' '{"status":"failed","errorCode":"stage_apply_failed"}' > "$apply_report"
            echo "status=failed" >> "$GITHUB_OUTPUT"
            gh issue comment "$issue_number" --body "⚠️ 本次 Actions 未能把候选阶段写入版本化数据，因此该阶段尚未生效。请以关联 Actions 与后续数据提交为准。" >/dev/null 2>&1 || true
            echo "::warning title=Stage candidate not persisted::The milestone Issue remains as visible audit evidence; deterministic snapshot persistence will continue."
            exit 0
          fi
          echo "status=applied" >> "$GITHUB_OUTPUT"

      - name: Export converged public content and detect a material change
        id: public
        shell: bash
        run: |
          set -euo pipefail
          npm run export -- --skip-seed
          npm run --silent public:validate -- --output="$RUNNER_TEMP/public-integrity.json"
          npm run --silent public:fingerprint > "$RUNNER_TEMP/public-after.sha256"
          if cmp -s "$RUNNER_TEMP/public-before.sha256" "$RUNNER_TEMP/public-after.sha256"; then
            echo "No material public content changes."
            echo "changed=false" >> "$GITHUB_OUTPUT"
          else
            echo "Material public content changed."
            echo "changed=true" >> "$GITHUB_OUTPUT"
          fi

      - name: Write privacy-safe text snapshot
        run: npm run db:snapshot -- write

      - name: Validate snapshot diff
        shell: bash
        run: |
          set -euo pipefail
          test -s data/snapshot/v1.json
          test -s data/narratives/stage-promotions.json
          test -s data/reports/research-impact.json
          test -s data/reports/system-evaluation.json
          node -e 'JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8"))' data/narratives/stage-promotions.json
          node -e 'JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8"))' data/reports/research-impact.json
          node -e 'JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8"))' data/reports/system-evaluation.json
          git diff --check -- data/snapshot/v1.json data/narratives/stage-promotions.json data/reports/research-impact.json data/reports/system-evaluation.json
          if grep -E '"(token|secret|password|cookie|authorization|api[_-]?key|reasoning|prompt|completion)"[[:space:]]*:' data/snapshot/v1.json data/narratives/stage-promotions.json data/reports/research-impact.json data/reports/system-evaluation.json; then
            echo "Sensitive or raw model material detected in repository data" >&2
            exit 1
          fi
          if grep -E -- '-----BEGIN (RSA |EC |OPENSSH )?PRIVATE KEY-----|/Users/[^/]+/|/home/runner/' data/snapshot/v1.json data/narratives/stage-promotions.json data/reports/research-impact.json data/reports/system-evaluation.json; then
            echo "Private material or local path detected in repository data" >&2
            exit 1
          fi

      - name: Commit snapshot when data changed
        id: commit
        shell: bash
        run: |
          set -euo pipefail
          git add -- data/snapshot/v1.json data/narratives/stage-promotions.json data/reports/research-impact.json data/reports/system-evaluation.json
          if git diff --cached --quiet; then
            echo "No material data changes; skipping commit."
            echo "changed=false" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git commit -m "chore(data): refresh intelligence snapshot"
          git rebase origin/main
          git push origin HEAD:main
          echo "changed=true" >> "$GITHUB_OUTPUT"

      # Pushes made with GITHUB_TOKEN intentionally do not trigger other push workflows.
      - name: Dispatch Pages deployment after the daily refresh
        env:
          GH_TOKEN: ${{ github.token }}
        run: gh workflow run pages.yml --ref main

      - name: Render an eligible weekly intelligence review
        id: weekly
        env:
          AI_ENRICHMENT_ENABLED: "true"
          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
          DEEPSEEK_MODEL: deepseek-v4-flash
        shell: bash
        run: |
          set -euo pipefail
          report_date="$(TZ=Asia/Shanghai date +%F)"
          report_week="$(TZ=Asia/Shanghai date +%G-W%V)"
          weekday="$(TZ=Asia/Shanghai date +%u)"
          hour="$(TZ=Asia/Shanghai date +%H)"
          publish_weekly=false
          weekly_status=not_requested
          weekly_attempted=false
          weekly_error=""
          if [[ "$PUBLISH_WEEKLY" == "true" || ( "$weekday" == "7" && "$hour" -ge 20 ) ]]; then
            weekly_attempted=true
            set +e
            npm run --silent weekly:issue -- \
              --ai \
              --require-success \
              --timeline dist/data/timeline.json \
              --scout dist/data/scout.json \
              --product dist/data/product.json \
              --end-date "$report_date" \
              --time-zone Asia/Shanghai > "$RUNNER_TEMP/weekly-brief.md" 2> "$RUNNER_TEMP/weekly-error.log"
            status=$?
            set -e
            if [[ "$status" -ne 0 ]]; then
              weekly_status=failed
              weekly_error="$(sed -n -E 's/^(DeepSeekError|Error): //p' "$RUNNER_TEMP/weekly-error.log" | tail -n 1 | tr -cd '[:alnum:]_:-')"
              weekly_error="${weekly_error:-weekly_render_failed}"
              echo "::warning title=AI weekly brief skipped::$weekly_error"
            elif [[ -s "$RUNNER_TEMP/weekly-brief.md" ]]; then
              weekly_status=ready
              publish_weekly=true
            else
              weekly_status=empty
              echo "No public Event cleared this week's gate; skipping the weekly Issue."
            fi
          fi
          WEEKLY_STATUS="$weekly_status" WEEKLY_ATTEMPTED="$weekly_attempted" WEEKLY_ERROR="$weekly_error" \
            node --input-type=module <<'NODE' > "$RUNNER_TEMP/weekly-status.json"
          const status = process.env.WEEKLY_STATUS || "unknown";
          process.stdout.write(`${JSON.stringify({
            status,
            attempted: process.env.WEEKLY_ATTEMPTED === "true",
            published: false,
            errorCode: process.env.WEEKLY_ERROR || null,
          }, null, 2)}\n`);
          NODE
          echo "publish_weekly=$publish_weekly" >> "$GITHUB_ENV"
          echo "report_week=$report_week" >> "$GITHUB_ENV"
          echo "status=$weekly_status" >> "$GITHUB_OUTPUT"

      - name: Create or update this week's GitHub Issue
        id: weekly_issue
        if: env.publish_weekly == 'true'
        env:
          GH_TOKEN: ${{ github.token }}
        shell: bash
        run: |
          set -euo pipefail
          marker="agent-pulse-weekly-brief:${report_week}"
          title="Agent Pulse AI 周报 · ${report_week}"
          gh label create weekly-brief --color 087e70 --description "Automated weekly intelligence brief" 2>/dev/null || true
          issue_number="$(gh issue list --state all --label weekly-brief --limit 100 --json number,body \
            --jq ".[] | select(.body | contains(\"$marker\")) | .number" | head -1)"
          if [[ -n "$issue_number" ]]; then
            gh issue edit "$issue_number" --title "$title" --body-file "$RUNNER_TEMP/weekly-brief.md" --add-label weekly-brief
            gh issue reopen "$issue_number" >/dev/null 2>&1 || true
          else
            gh issue create --title "$title" --body-file "$RUNNER_TEMP/weekly-brief.md" --label weekly-brief
          fi
          WEEKLY_STATUS_PATH="$RUNNER_TEMP/weekly-status.json" node --input-type=module <<'NODE'
          import { readFile, writeFile } from "node:fs/promises";
          const path = process.env.WEEKLY_STATUS_PATH;
          const status = JSON.parse(await readFile(path, "utf8"));
          await writeFile(path, `${JSON.stringify({ ...status, status: "published", published: true }, null, 2)}\n`);
          NODE

      - name: Record refresh summary
        if: always()
        shell: bash
        run: |
          {
            echo "## Data Refresh"
            echo "- AI enrichment: ${{ steps.enrichment.outputs.status || 'not-run' }}"
            echo "- Stage proposal: ${{ steps.stage_proposal.outputs.status || 'not-run' }}"
            echo "- Stage apply: ${{ steps.stage_apply.outputs.status || 'skipped' }}"
            echo "- Snapshot changed: ${{ steps.commit.outputs.changed || 'unknown' }}"
            echo "- Weekly render: ${{ steps.weekly.outputs.status || 'not-run' }}"
            echo "- Weekly Issue: ${{ steps.weekly_issue.outcome || 'skipped' }}"
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Upload quality evidence
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: quality-evidence-${{ github.run_id }}
          path: |
            ${{ runner.temp }}/evaluation.json
            ${{ runner.temp }}/ai-enrichment.json
            ${{ runner.temp }}/stage-promotion.json
            ${{ runner.temp }}/stage-promotion-apply.json
            ${{ runner.temp }}/weekly-status.json
            ${{ runner.temp }}/public-integrity.json
            data/reports/research-impact.json
            data/reports/system-evaluation.json
          if-no-files-found: ignore
          retention-days: 14