Generate Daily Digest Draft #299
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Generate Daily Digest Draft | |
| on: | |
| schedule: | |
| # Run at ~5 AM PT (13:17 UTC). Offset off the top-of-hour to reduce | |
| # GitHub's best-effort scheduler dispatch delay (scheduled workflows are | |
| # deprioritized under load, worst at :00). Send-digest runs ~1h later | |
| # (14:17) preserving the intended human review window before publish. | |
| - cron: '17 13 * * *' | |
| workflow_dispatch: # Allow manual triggering for testing | |
| jobs: | |
| generate-draft: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write # Allow pushing commits | |
| pull-requests: read # Allow reading PRs if needed | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| version: "latest" | |
| - name: Install dependencies | |
| run: | | |
| # Install dependencies using uv (project standard) | |
| uv sync | |
| uv pip install -e . | |
| # Verify key dependencies are installed | |
| uv run python -c "import dotenv; print('dotenv installed')" | |
| uv run python -c "import httpx; print('httpx installed')" | |
| uv run python -c "import langchain_core; print('langchain_core installed')" | |
| - name: Validate required secrets | |
| run: | | |
| # Check for required secrets | |
| if [ -z "${{ secrets.COINGECKO_API_KEY }}" ]; then | |
| echo "❌ ERROR: COINGECKO_API_KEY secret is not set!" | |
| echo "This is REQUIRED for market snapshot generation." | |
| echo "Please add COINGECKO_API_KEY to your GitHub repository secrets." | |
| exit 1 | |
| fi | |
| if [ -z "${{ secrets.PERPLEXITY_API_KEY }}" ]; then | |
| echo "❌ ERROR: PERPLEXITY_API_KEY secret is not set!" | |
| exit 1 | |
| fi | |
| echo "✅ All required secrets are configured" | |
| - name: Clear stale price cache for today | |
| run: | | |
| # Clear any cached price data for today to ensure fresh data with API key | |
| # This prevents using stale/incomplete cache from previous failed runs | |
| TODAY=$(date -u '+%Y%m%d') | |
| if [ -d ".cache/prices" ]; then | |
| echo "Clearing price cache files that include today's date..." | |
| find .cache/prices -name "*_*${TODAY}.json" -delete 2>/dev/null || true | |
| echo "✓ Cleared stale cache files" | |
| else | |
| echo "No cache directory found, skipping" | |
| fi | |
| - name: Generate digest | |
| env: | |
| PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }} | |
| COINGECKO_API_KEY: ${{ secrets.COINGECKO_API_KEY }} | |
| SENDGRID_API_KEY: ${{ secrets.SENDGRID_API_KEY }} | |
| ADMIN_EMAILS: ${{ secrets.ADMIN_EMAILS || 'squid@leviathannews.xyz' }} | |
| PUBLIC_EMAILS: ${{ secrets.PUBLIC_EMAILS || 'squid@leviathannews.xyz,kraken@leviathannews.xyz' }} | |
| run: | | |
| # Generate signals version only | |
| echo "Generating signals version..." | |
| ACTIVE_PROMPT=signals uv run python scripts/digest.py --verbose | |
| if [ $? -ne 0 ]; then | |
| echo "Signals generation failed with exit code $?" | |
| exit 1 | |
| fi | |
| # Debug: List created files | |
| TODAY=$(date -u '+%Y-%m-%d') | |
| YEAR=$(date -u '+%Y') | |
| MONTH=$(date -u '+%m') | |
| DAY=$(date -u '+%d') | |
| echo "Files created in writeup/${YEAR}/${MONTH}/${DAY}/:" | |
| ls -la writeup/${YEAR}/${MONTH}/${DAY}/ || echo "No files in date directory" | |
| echo "Files created in writeup/${YEAR}/${MONTH}/${DAY}/thinking_logs/:" | |
| ls -la writeup/${YEAR}/${MONTH}/${DAY}/thinking_logs/ || echo "No thinking_logs directory" | |
| - name: Upload diagnostics | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: squid-digest-diagnostics-${{ github.run_id }} | |
| path: | | |
| writeup/**/diagnostic_*.txt | |
| writeup/**/diagnostic_*_*.txt | |
| .data/leviathan_news.json | |
| .data/leviathan_tokens.json | |
| if-no-files-found: ignore | |
| - name: Surface signals_status | |
| if: always() | |
| run: | | |
| TODAY=$(date -u '+%Y-%m-%d') | |
| YEAR=$(date -u '+%Y') | |
| MONTH=$(date -u '+%m') | |
| DAY=$(date -u '+%d') | |
| META_FILE="writeup/${YEAR}/${MONTH}/${DAY}/meta_${TODAY}.json" | |
| if [ ! -f "$META_FILE" ]; then | |
| echo "No meta file found at $META_FILE — likely the digest step did not complete." | |
| exit 0 | |
| fi | |
| STATUS=$(uv run python -c "import json,sys; d=json.load(open('$META_FILE')); print(d.get('signals_status','(unset)'))" 2>/dev/null || echo "(unread)") | |
| echo "Today's signals_status: $STATUS" | |
| if [ "$STATUS" != "ok" ]; then | |
| { | |
| echo "## ⚠️ Signals were \`$STATUS\` today" | |
| echo "" | |
| echo "The strict prompt did not produce parseable signals. The recovery path taken was \`$STATUS\`." | |
| echo "" | |
| echo "- \`reformatted\`: a third LLM pass converted prose to signals — published as normal." | |
| echo "- \`skipped\`: all three passes failed — banner published in place of signals." | |
| echo "" | |
| echo "See diagnostic artifacts and \`writeup/${YEAR}/${MONTH}/${DAY}/\` for details." | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| - name: Validate signals output | |
| run: | | |
| TODAY=$(date -u '+%Y-%m-%d') | |
| YEAR=$(date -u '+%Y') | |
| MONTH=$(date -u '+%m') | |
| DAY=$(date -u '+%d') | |
| SIGNALS_FILE="writeup/${YEAR}/${MONTH}/${DAY}/signals_${TODAY}.md" | |
| META_FILE="writeup/${YEAR}/${MONTH}/${DAY}/meta_${TODAY}.json" | |
| echo "Validating signals file..." | |
| # Pass --meta so the validator can run relaxed checks when signals_status="skipped". | |
| uv run python scripts/validate_signals_output.py "$SIGNALS_FILE" --meta "$META_FILE" | |
| if [ $? -ne 0 ]; then | |
| echo "❌ Signal validation FAILED!" | |
| echo "This indicates the LLM failed to generate trading signals." | |
| echo "Check logs above for diagnostic details." | |
| exit 1 | |
| fi | |
| - name: Commit and push digest | |
| run: | | |
| git config --local user.email "action@github.com" | |
| git config --local user.name "GitHub Action" | |
| echo "=== Git Status Before Digest Commit ===" | |
| git status --porcelain | |
| echo "========================================" | |
| # Stage and commit ALL changes in one atomic operation | |
| # This ensures we capture any files modified by the digest script (feed.xml, portfolio state, etc) | |
| # even if they were modified after the digest script completed | |
| git add -A | |
| if git diff --cached --quiet; then | |
| echo "No changes to commit" | |
| else | |
| echo "=== Files to be Committed ===" | |
| git diff --cached --name-only | |
| echo "================================" | |
| git commit -m "🤖 Auto-generated daily digest draft - $(date -u '+%Y-%m-%d %H:%M UTC')" | |
| git pull --rebase --autostash origin main | |
| git push | |
| echo "✓ Digest committed and pushed successfully" | |
| fi | |
| - name: Update README with today's signals | |
| continue-on-error: true | |
| run: | | |
| echo "Updating README with today's signals..." | |
| if uv run python scripts/update_readme.py --allow-empty-headlines; then | |
| echo "✓ README updated successfully" | |
| else | |
| echo "⚠ Warning: README update failed or incomplete" | |
| echo "This may be due to:" | |
| echo " - Signals file not found or empty" | |
| echo " - Unexpected file format" | |
| echo " - Missing Top Stories section" | |
| echo "" | |
| echo "Workflow will continue, but README may not be updated." | |
| echo "Check the logs above for diagnostic information." | |
| exit 0 # Don't fail the workflow | |
| fi | |
| - name: Update writeup README with recent headlines | |
| run: | | |
| echo "Updating writeup README with recent headlines and navigation..." | |
| uv run python scripts/update_writeup_readme.py | |
| if [ $? -ne 0 ]; then | |
| echo "Warning: Writeup README update failed, but continuing workflow" | |
| else | |
| echo "✓ Writeup README updated successfully" | |
| fi | |
| - name: Commit and push README updates | |
| run: | | |
| git config --local user.email "action@github.com" | |
| git config --local user.name "GitHub Action" | |
| echo "=== Git Status Before README Commit ===" | |
| git status --porcelain | |
| echo "========================================" | |
| # Stage portfolio state files if modified (ROOT CAUSE FIX) | |
| if [ -f "writeup/portfolio_state_buy.json" ] || [ -f "writeup/portfolio_state_sell.json" ]; then | |
| echo "Checking for portfolio state changes..." | |
| if ! git diff --quiet writeup/portfolio_state_buy.json 2>/dev/null || ! git diff --quiet writeup/portfolio_state_sell.json 2>/dev/null; then | |
| echo "Portfolio state files were modified, staging them..." | |
| git add writeup/portfolio_state_buy.json writeup/portfolio_state_sell.json 2>/dev/null || true | |
| fi | |
| fi | |
| # Check if any changes to commit (READMEs or portfolio state) | |
| if git diff --quiet README.md writeup/README.md && git diff --cached --quiet; then | |
| echo "No changes to commit" | |
| else | |
| git add README.md writeup/README.md | |
| echo "=== Files Staged for Commit ===" | |
| git diff --cached --name-only | |
| echo "================================" | |
| git commit -m "📊 Update READMEs with daily signals and navigation - $(date -u '+%Y-%m-%d')" | |
| git pull --rebase --autostash origin main | |
| git push | |
| echo "✓ README changes pushed successfully" | |
| fi | |
| - name: Send admin notification | |
| env: | |
| GHOST_URL: ${{ secrets.GHOST_URL }} | |
| GHOST_ADMIN_API_KEY: ${{ secrets.GHOST_ADMIN_API_KEY }} | |
| ADMIN_EMAILS: ${{ secrets.ADMIN_EMAILS || 'squid@leviathannews.xyz' }} | |
| run: | | |
| # Create Ghost drafts with actual digest content for both versions | |
| echo "Creating Ghost drafts for admin review..." | |
| # Use today's date to find the signals file we just generated | |
| TODAY=$(date -u '+%Y-%m-%d') | |
| YEAR=$(date -u '+%Y') | |
| MONTH=$(date -u '+%m') | |
| DAY=$(date -u '+%d') | |
| SIGNALS_FILE="writeup/${YEAR}/${MONTH}/${DAY}/signals_${TODAY}.md" | |
| echo "Looking for today's signals file:" | |
| echo "Signals file: $SIGNALS_FILE" | |
| # Check if today's signals file exists | |
| if [ ! -f "$SIGNALS_FILE" ]; then | |
| echo "Error: Could not find today's signals file" | |
| echo "Expected: $SIGNALS_FILE" | |
| echo "Available files in date directory:" | |
| ls -la writeup/${YEAR}/${MONTH}/${DAY}/ || echo "Date directory does not exist" | |
| exit 1 | |
| fi | |
| # Create Ghost draft with actual content and capture ghost_url for distribution | |
| uv run python -c " | |
| import sys, json | |
| sys.path.insert(0, 'src') | |
| from squid_digest.email import GhostEmailClient | |
| from squid_digest.config import save_meta, get_digest_title | |
| from pathlib import Path | |
| from datetime import datetime | |
| client = GhostEmailClient() | |
| # Create signals draft | |
| signals_path = Path('$SIGNALS_FILE') | |
| if signals_path.exists(): | |
| content = signals_path.read_text() | |
| html_content = client.format_digest_html(content) | |
| date_str = signals_path.stem.split('_')[-1] | |
| title = get_digest_title(datetime.strptime(date_str, '%Y-%m-%d')) | |
| post = client.create_post(title, html_content, status='draft') | |
| post_id = post.get('id', 'unknown') | |
| print(f'✓ Created signals draft: {post_id}') | |
| # Save draft post ID and URL to metadata for send-digest.yml to reuse | |
| ghost_url = post.get('url') or post.get('canonical_url') | |
| date_str = signals_path.stem.split('_')[-1] | |
| date = datetime.strptime(date_str, '%Y-%m-%d') | |
| meta_update = {'draft_post_id': post_id} | |
| if ghost_url: | |
| meta_update['draft_ghost_url'] = ghost_url | |
| save_meta(date, meta_update) | |
| print(f'✓ Draft metadata saved (post_id: {post_id})') | |
| print('✓ Draft created successfully for admin review') | |
| " | |
| - name: Post to Telegram | |
| env: | |
| TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }} | |
| TELEGRAM_CHANNEL_ID: ${{ secrets.TELEGRAM_CHANNEL_ID }} | |
| TELEGRAM_CHANNEL_USERNAME: ${{ secrets.TELEGRAM_CHANNEL_USERNAME }} | |
| continue-on-error: true | |
| run: | | |
| TODAY=$(date -u '+%Y-%m-%d') | |
| if [ -z "$TELEGRAM_BOT_TOKEN" ] || [ -z "$TELEGRAM_CHANNEL_ID" ]; then | |
| echo "Warning: Telegram credentials not configured, skipping" | |
| exit 0 | |
| fi | |
| uv run python scripts/post_telegram.py --date "$TODAY" | |
| - name: Post to SQUID Cave | |
| env: | |
| TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }} | |
| TELEGRAM_CAVE_CHANNEL_ID: ${{ secrets.TELEGRAM_CAVE_CHANNEL_ID }} | |
| continue-on-error: true | |
| run: | | |
| # Post page 1 of digest to SQUID Cave channel with masthead and links | |
| echo "Posting to SQUID Cave..." | |
| TODAY=$(date -u '+%Y-%m-%d') | |
| YEAR=$(date -u '+%Y') | |
| MONTH=$(date -u '+%m') | |
| DAY=$(date -u '+%d') | |
| SIGNALS_FILE="writeup/${YEAR}/${MONTH}/${DAY}/signals_${TODAY}.md" | |
| if [ ! -f "$SIGNALS_FILE" ]; then | |
| echo "Warning: Signals file not found: $SIGNALS_FILE" | |
| echo "Skipping SQUID Cave post" | |
| exit 0 | |
| fi | |
| if [ -z "$TELEGRAM_CAVE_CHANNEL_ID" ]; then | |
| echo "Warning: TELEGRAM_CAVE_CHANNEL_ID not configured" | |
| echo "Skipping SQUID Cave post (add secret to enable)" | |
| exit 0 | |
| fi | |
| if [ -z "$TELEGRAM_BOT_TOKEN" ]; then | |
| echo "Warning: TELEGRAM_BOT_TOKEN not configured" | |
| echo "Skipping SQUID Cave post" | |
| exit 0 | |
| fi | |
| uv run python scripts/post_telegram_cave.py --date "$TODAY" | |
| - name: Commit distribution metadata | |
| run: | | |
| git config --local user.email "action@github.com" | |
| git config --local user.name "GitHub Action" | |
| git add writeup/**/meta_*.json 2>/dev/null || true | |
| if git diff --cached --quiet; then | |
| echo "No distribution metadata changes to commit" | |
| else | |
| git commit -m "📡 Distribution metadata for $(date -u '+%Y-%m-%d')" | |
| git pull --rebase --autostash origin main | |
| git push | |
| echo "✓ Distribution metadata committed" | |
| fi |