Refresh intelligence data #36
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Refresh intelligence data | |
| on: | |
| schedule: | |
| # Daily at 20:17 Asia/Shanghai; the weekly Issue remains Sunday-gated below. | |
| - cron: "17 12 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| mode: | |
| description: Refresh mode | |
| required: true | |
| default: incremental | |
| type: choice | |
| options: | |
| - incremental | |
| - backfill | |
| publish_weekly: | |
| description: Publish or re-render the current weekly review | |
| required: false | |
| default: false | |
| type: boolean | |
| permissions: | |
| actions: write | |
| contents: write | |
| issues: write | |
| concurrency: | |
| group: agent-pulse-repository-data-main | |
| cancel-in-progress: false | |
| jobs: | |
| refresh: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 35 | |
| env: | |
| DATABASE_URL: sqlite:./var/automation.db | |
| PUBLIC_SITE_URL: https://barretlee.github.io/agent-pulse/ | |
| REFRESH_MODE: ${{ inputs.mode || 'incremental' }} | |
| PUBLISH_WEEKLY: ${{ inputs.publish_weekly || false }} | |
| steps: | |
| - name: Check out main | |
| uses: actions/checkout@v7 | |
| with: | |
| ref: main | |
| fetch-depth: 0 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: 24 | |
| cache: npm | |
| - name: Install locked dependencies | |
| run: npm ci | |
| - name: Restore auditable repository snapshot | |
| run: npm run db:snapshot -- restore | |
| - name: Capture the current public content fingerprint | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| npm run export -- --skip-seed | |
| npm run --silent public:fingerprint > "$RUNNER_TEMP/public-before.sha256" | |
| - name: Collect and cluster signals | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| if [[ "$REFRESH_MODE" == "backfill" ]]; then | |
| npm run collect -- --backfill | |
| else | |
| npm run collect | |
| fi | |
| - name: Observe direct research feeds in shadow | |
| shell: bash | |
| run: | | |
| set -u | |
| for source in microsoft-research google-research; do | |
| if ! npm run collect -- --source="$source"; then | |
| echo "::warning title=Direct research source unavailable::$source was not collected; its lifecycle state is unchanged and the batch will continue" | |
| fi | |
| done | |
| - name: Reconcile sources and source radar | |
| run: npm run ops:reconcile | |
| - name: Reconcile observation and activate qualified sources | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| npm run observe:sources -- --confirm | |
| npm run activate:auto | |
| - name: Remove unattached aggregator provenance debt | |
| run: npm run repair:provenance -- --confirm | |
| - name: Audit research impact before publication | |
| env: | |
| OPENALEX_API_KEY: ${{ secrets.OPENALEX_API_KEY }} | |
| run: npm run --silent research:impact -- --skip-seed | |
| - name: Enrich evidence-ready review Events with DeepSeek | |
| id: enrichment | |
| env: | |
| AI_ENRICHMENT_ENABLED: "true" | |
| AI_ENRICHMENT_MAX_EVENTS: "8" | |
| DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} | |
| DEEPSEEK_MODEL: deepseek-v4-flash | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| set +e | |
| npm run --silent ai:enrich -- --require-success | tee "$RUNNER_TEMP/ai-enrichment.json" | |
| status="${PIPESTATUS[0]}" | |
| if [[ "$status" -eq 0 ]] && ! node -e 'JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8"))' "$RUNNER_TEMP/ai-enrichment.json"; then | |
| status=1 | |
| fi | |
| set -e | |
| if [[ "$status" -eq 0 ]]; then | |
| echo "status=ok" >> "$GITHUB_OUTPUT" | |
| else | |
| if [[ ! -s "$RUNNER_TEMP/ai-enrichment.json" ]]; then | |
| printf '%s\n' '{"enabled":true,"workflowStatus":"failed","error":"ai_enrichment_failed"}' > "$RUNNER_TEMP/ai-enrichment.json" | |
| fi | |
| echo "status=failed" >> "$GITHUB_OUTPUT" | |
| echo "::warning title=AI Event enrichment unavailable::Eligible Events remain in review; deterministic collection and snapshot persistence will continue." | |
| fi | |
| - name: Generate and publish qualified Scout opportunities | |
| run: npm run scout:generate -- 12 | |
| - name: Publish every Event and Scout item that clears hard gates | |
| run: npm run auto:publish | |
| - name: Merge latest remote repository snapshot | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| git fetch origin main | |
| git show origin/main:data/snapshot/v1.json > "$RUNNER_TEMP/remote-snapshot.json" | |
| git show origin/main:data/narratives/stage-promotions.json > data/narratives/stage-promotions.json | |
| git show origin/main:data/reports/system-evaluation.json > data/reports/system-evaluation.json | |
| npm run db:snapshot -- merge --file="$RUNNER_TEMP/remote-snapshot.json" | |
| - name: Re-apply source reconciliation after remote merge | |
| run: npm run ops:reconcile | |
| - name: Persist measured system quality and ranked improvement actions | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| npm run --silent evaluate:system -- \ | |
| --skip-bootstrap \ | |
| --baseline=data/reports/system-evaluation.json \ | |
| --output=data/reports/system-evaluation.json \ | |
| --summary="$GITHUB_STEP_SUMMARY" | tee "$RUNNER_TEMP/evaluation.json" | |
| node -e 'JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8"))' "$RUNNER_TEMP/evaluation.json" | |
| - name: Assess a major narrative stage promotion with DeepSeek V4 Pro | |
| id: stage_proposal | |
| env: | |
| AI_STAGE_PROMOTION_ENABLED: "true" | |
| DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} | |
| DEEPSEEK_STAGE_MODEL: deepseek-v4-pro | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| set +e | |
| npm run --silent narrative:stage -- propose \ | |
| --candidate-path "$RUNNER_TEMP/stage-promotion-candidate.json" \ | |
| | tee "$RUNNER_TEMP/stage-promotion.json" | |
| status="${PIPESTATUS[0]}" | |
| set -e | |
| if [[ "$status" -ne 0 ]] || ! node -e 'JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8"))' "$RUNNER_TEMP/stage-promotion.json"; then | |
| printf '%s\n' '{"status":"failed","model":"deepseek-v4-pro","errorCode":"stage_proposal_failed"}' > "$RUNNER_TEMP/stage-promotion.json" | |
| echo "status=failed" >> "$GITHUB_OUTPUT" | |
| echo "proposed=false" >> "$GITHUB_OUTPUT" | |
| echo "::warning title=Stage promotion assessment unavailable::No stage will be added; deterministic snapshot persistence will continue." | |
| exit 0 | |
| fi | |
| proposal_status="$(node -e 'const x=JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8")); process.stdout.write(String(x.status || "unknown"))' "$RUNNER_TEMP/stage-promotion.json")" | |
| if [[ "$proposal_status" == "candidate" && -s "$RUNNER_TEMP/stage-promotion-candidate.json" ]]; then | |
| echo "proposed=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "proposed=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| echo "status=$proposal_status" >> "$GITHUB_OUTPUT" | |
| - name: Create the milestone Issue and apply the validated stage | |
| id: stage_apply | |
| if: steps.stage_proposal.outputs.proposed == 'true' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| ACTIONS_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| candidate="$RUNNER_TEMP/stage-promotion-candidate.json" | |
| body="$RUNNER_TEMP/stage-promotion-issue.md" | |
| title_file="$RUNNER_TEMP/stage-promotion-title.txt" | |
| apply_report="$RUNNER_TEMP/stage-promotion-apply.json" | |
| set +e | |
| npm run --silent narrative:stage -- render-issue --candidate-path "$candidate" --actions-url "$ACTIONS_URL" > "$body" | |
| render_status=$? | |
| npm run --silent narrative:stage -- render-issue --candidate-path "$candidate" --title > "$title_file" | |
| title_status=$? | |
| set -e | |
| if [[ "$render_status" -ne 0 || "$title_status" -ne 0 ]]; then | |
| printf '%s\n' '{"status":"failed","errorCode":"stage_issue_render_failed"}' > "$apply_report" | |
| echo "status=failed" >> "$GITHUB_OUTPUT" | |
| echo "::warning title=Stage milestone Issue skipped::Validated candidate could not be rendered safely." | |
| exit 0 | |
| fi | |
| marker="$(node -e 'const x=JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8")); process.stdout.write(x.marker)' "$candidate")" | |
| title="$(<"$title_file")" | |
| set +e | |
| gh label create "stage:milestone" --color 5319e7 --description "Evidence-gated major narrative stage" --force | |
| label_status=$? | |
| issue_number="$(gh issue list --state all --label "stage:milestone" --limit 100 --json number,body \ | |
| --jq "first(.[] | select(.body | contains(\"$marker\"))) | .number // empty")" | |
| list_status=$? | |
| set -e | |
| if [[ "$label_status" -ne 0 || "$list_status" -ne 0 ]]; then | |
| printf '%s\n' '{"status":"failed","errorCode":"stage_issue_lookup_failed"}' > "$apply_report" | |
| echo "status=failed" >> "$GITHUB_OUTPUT" | |
| echo "::warning title=Stage milestone Issue skipped::The repository label or existing milestone Issue could not be checked." | |
| exit 0 | |
| fi | |
| set +e | |
| if [[ -n "$issue_number" ]]; then | |
| gh issue edit "$issue_number" --title "$title" --body-file "$body" --add-label "stage:milestone" | |
| issue_status=$? | |
| if [[ "$issue_status" -eq 0 ]]; then gh issue reopen "$issue_number" >/dev/null 2>&1 || true; fi | |
| issue_url="$(gh issue view "$issue_number" --json url --jq .url 2>/dev/null)" | |
| else | |
| issue_url="$(gh issue create --title "$title" --body-file "$body" --label "stage:milestone")" | |
| issue_status=$? | |
| issue_number="${issue_url##*/}" | |
| fi | |
| set -e | |
| if [[ "$issue_status" -ne 0 || -z "$issue_number" || -z "$issue_url" ]]; then | |
| printf '%s\n' '{"status":"failed","errorCode":"stage_issue_upsert_failed"}' > "$apply_report" | |
| echo "status=failed" >> "$GITHUB_OUTPUT" | |
| echo "::warning title=Stage milestone Issue skipped::The candidate was not persisted because its dedicated Issue could not be created." | |
| exit 0 | |
| fi | |
| set +e | |
| npm run --silent narrative:stage -- apply \ | |
| --candidate-path "$candidate" \ | |
| --issue-number "$issue_number" \ | |
| --issue-url "$issue_url" | tee "$apply_report" | |
| apply_status="${PIPESTATUS[0]}" | |
| set -e | |
| if [[ "$apply_status" -ne 0 ]]; then | |
| printf '%s\n' '{"status":"failed","errorCode":"stage_apply_failed"}' > "$apply_report" | |
| echo "status=failed" >> "$GITHUB_OUTPUT" | |
| gh issue comment "$issue_number" --body "⚠️ 本次 Actions 未能把候选阶段写入版本化数据，因此该阶段尚未生效。请以关联 Actions 与后续数据提交为准。" >/dev/null 2>&1 || true | |
| echo "::warning title=Stage candidate not persisted::The milestone Issue remains as visible audit evidence; deterministic snapshot persistence will continue." | |
| exit 0 | |
| fi | |
| echo "status=applied" >> "$GITHUB_OUTPUT" | |
| - name: Export converged public content and detect a material change | |
| id: public | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| npm run export -- --skip-seed | |
| npm run --silent public:validate -- --output="$RUNNER_TEMP/public-integrity.json" | |
| npm run --silent public:fingerprint > "$RUNNER_TEMP/public-after.sha256" | |
| if cmp -s "$RUNNER_TEMP/public-before.sha256" "$RUNNER_TEMP/public-after.sha256"; then | |
| echo "No material public content changes." | |
| echo "changed=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "Material public content changed." | |
| echo "changed=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Write privacy-safe text snapshot | |
| run: npm run db:snapshot -- write | |
| - name: Validate snapshot diff | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| test -s data/snapshot/v1.json | |
| test -s data/narratives/stage-promotions.json | |
| test -s data/reports/research-impact.json | |
| test -s data/reports/system-evaluation.json | |
| node -e 'JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8"))' data/narratives/stage-promotions.json | |
| node -e 'JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8"))' data/reports/research-impact.json | |
| node -e 'JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8"))' data/reports/system-evaluation.json | |
| git diff --check -- data/snapshot/v1.json data/narratives/stage-promotions.json data/reports/research-impact.json data/reports/system-evaluation.json | |
| if grep -E '"(token|secret|password|cookie|authorization|api[_-]?key|reasoning|prompt|completion)"[[:space:]]*:' data/snapshot/v1.json data/narratives/stage-promotions.json data/reports/research-impact.json data/reports/system-evaluation.json; then | |
| echo "Sensitive or raw model material detected in repository data" >&2 | |
| exit 1 | |
| fi | |
| if grep -E -- '-----BEGIN (RSA |EC |OPENSSH )?PRIVATE KEY-----|/Users/[^/]+/|/home/runner/' data/snapshot/v1.json data/narratives/stage-promotions.json data/reports/research-impact.json data/reports/system-evaluation.json; then | |
| echo "Private material or local path detected in repository data" >&2 | |
| exit 1 | |
| fi | |
| - name: Commit snapshot when data changed | |
| id: commit | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| git add -- data/snapshot/v1.json data/narratives/stage-promotions.json data/reports/research-impact.json data/reports/system-evaluation.json | |
| if git diff --cached --quiet; then | |
| echo "No material data changes; skipping commit." | |
| echo "changed=false" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git commit -m "chore(data): refresh intelligence snapshot" | |
| git rebase origin/main | |
| git push origin HEAD:main | |
| echo "changed=true" >> "$GITHUB_OUTPUT" | |
| # Pushes made with GITHUB_TOKEN intentionally do not trigger other push workflows. | |
| - name: Dispatch Pages deployment after the daily refresh | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: gh workflow run pages.yml --ref main | |
| - name: Render an eligible weekly intelligence review | |
| id: weekly | |
| env: | |
| AI_ENRICHMENT_ENABLED: "true" | |
| DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} | |
| DEEPSEEK_MODEL: deepseek-v4-flash | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| report_date="$(TZ=Asia/Shanghai date +%F)" | |
| report_week="$(TZ=Asia/Shanghai date +%G-W%V)" | |
| weekday="$(TZ=Asia/Shanghai date +%u)" | |
| hour="$(TZ=Asia/Shanghai date +%H)" | |
| publish_weekly=false | |
| weekly_status=not_requested | |
| weekly_attempted=false | |
| weekly_error="" | |
| if [[ "$PUBLISH_WEEKLY" == "true" || ( "$weekday" == "7" && "$hour" -ge 20 ) ]]; then | |
| weekly_attempted=true | |
| set +e | |
| npm run --silent weekly:issue -- \ | |
| --ai \ | |
| --require-success \ | |
| --timeline dist/data/timeline.json \ | |
| --scout dist/data/scout.json \ | |
| --product dist/data/product.json \ | |
| --end-date "$report_date" \ | |
| --time-zone Asia/Shanghai > "$RUNNER_TEMP/weekly-brief.md" 2> "$RUNNER_TEMP/weekly-error.log" | |
| status=$? | |
| set -e | |
| if [[ "$status" -ne 0 ]]; then | |
| weekly_status=failed | |
| weekly_error="$(sed -n -E 's/^(DeepSeekError|Error): //p' "$RUNNER_TEMP/weekly-error.log" | tail -n 1 | tr -cd '[:alnum:]_:-')" | |
| weekly_error="${weekly_error:-weekly_render_failed}" | |
| echo "::warning title=AI weekly brief skipped::$weekly_error" | |
| elif [[ -s "$RUNNER_TEMP/weekly-brief.md" ]]; then | |
| weekly_status=ready | |
| publish_weekly=true | |
| else | |
| weekly_status=empty | |
| echo "No public Event cleared this week's gate; skipping the weekly Issue." | |
| fi | |
| fi | |
| WEEKLY_STATUS="$weekly_status" WEEKLY_ATTEMPTED="$weekly_attempted" WEEKLY_ERROR="$weekly_error" \ | |
| node --input-type=module <<'NODE' > "$RUNNER_TEMP/weekly-status.json" | |
| const status = process.env.WEEKLY_STATUS || "unknown"; | |
| process.stdout.write(`${JSON.stringify({ | |
| status, | |
| attempted: process.env.WEEKLY_ATTEMPTED === "true", | |
| published: false, | |
| errorCode: process.env.WEEKLY_ERROR || null, | |
| }, null, 2)}\n`); | |
| NODE | |
| echo "publish_weekly=$publish_weekly" >> "$GITHUB_ENV" | |
| echo "report_week=$report_week" >> "$GITHUB_ENV" | |
| echo "status=$weekly_status" >> "$GITHUB_OUTPUT" | |
| - name: Create or update this week's GitHub Issue | |
| id: weekly_issue | |
| if: env.publish_weekly == 'true' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| marker="agent-pulse-weekly-brief:${report_week}" | |
| title="Agent Pulse AI 周报 · ${report_week}" | |
| gh label create weekly-brief --color 087e70 --description "Automated weekly intelligence brief" 2>/dev/null || true | |
| issue_number="$(gh issue list --state all --label weekly-brief --limit 100 --json number,body \ | |
| --jq ".[] | select(.body | contains(\"$marker\")) | .number" | head -1)" | |
| if [[ -n "$issue_number" ]]; then | |
| gh issue edit "$issue_number" --title "$title" --body-file "$RUNNER_TEMP/weekly-brief.md" --add-label weekly-brief | |
| gh issue reopen "$issue_number" >/dev/null 2>&1 || true | |
| else | |
| gh issue create --title "$title" --body-file "$RUNNER_TEMP/weekly-brief.md" --label weekly-brief | |
| fi | |
| WEEKLY_STATUS_PATH="$RUNNER_TEMP/weekly-status.json" node --input-type=module <<'NODE' | |
| import { readFile, writeFile } from "node:fs/promises"; | |
| const path = process.env.WEEKLY_STATUS_PATH; | |
| const status = JSON.parse(await readFile(path, "utf8")); | |
| await writeFile(path, `${JSON.stringify({ ...status, status: "published", published: true }, null, 2)}\n`); | |
| NODE | |
| - name: Record refresh summary | |
| if: always() | |
| shell: bash | |
| run: | | |
| { | |
| echo "## Data Refresh" | |
| echo "- AI enrichment: ${{ steps.enrichment.outputs.status || 'not-run' }}" | |
| echo "- Stage proposal: ${{ steps.stage_proposal.outputs.status || 'not-run' }}" | |
| echo "- Stage apply: ${{ steps.stage_apply.outputs.status || 'skipped' }}" | |
| echo "- Snapshot changed: ${{ steps.commit.outputs.changed || 'unknown' }}" | |
| echo "- Weekly render: ${{ steps.weekly.outputs.status || 'not-run' }}" | |
| echo "- Weekly Issue: ${{ steps.weekly_issue.outcome || 'skipped' }}" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Upload quality evidence | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: quality-evidence-${{ github.run_id }} | |
| path: | | |
| ${{ runner.temp }}/evaluation.json | |
| ${{ runner.temp }}/ai-enrichment.json | |
| ${{ runner.temp }}/stage-promotion.json | |
| ${{ runner.temp }}/stage-promotion-apply.json | |
| ${{ runner.temp }}/weekly-status.json | |
| ${{ runner.temp }}/public-integrity.json | |
| data/reports/research-impact.json | |
| data/reports/system-evaluation.json | |
| if-no-files-found: ignore | |
| retention-days: 14 |