full-pipeline.yml

name: Full Pipeline

on:
  schedule:
    # Saturday 11pm UTC (adjust for your timezone: 5pm MDT, 4pm MST)
    - cron: '0 23 * * 6'
  workflow_dispatch:
    inputs:
      sync_batch_size:
        description: 'Docs to sync to RAGFlow (0 = all)'
        required: false
        default: '0'
      convert_batch_size:
        description: 'Docs to convert to MkDocs (0 = all)'
        required: false
        default: '0'
      linkfix_batch_size:
        description: 'Docs to run link-fix on (0 = all eligible)'
        required: false
        default: '0'
      completions_batch_size:
        description: 'Docs to generate completions for (0 = all pending)'
        required: false
        default: '0'
      sync_verify:
        description: 'Verify synced docs exist in RAGFlow before syncing'
        required: false
        default: false
        type: boolean
      sync_force:
        description: 'Force full re-sync by clearing ragflow_source_hash'
        required: false
        default: false
        type: boolean

permissions:
  contents: write

env:
  PYTHONUNBUFFERED: "1"

jobs:
  download:
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Set up venv
        run: |
          python -m venv venv
          source venv/bin/activate
          pip install -r requirements.txt

      - name: Download wiki pages
        run: |
          source venv/bin/activate
          python scripts/download-wiki.py

      - name: Download events
        run: |
          source venv/bin/activate
          python scripts/download-events.py

      - name: Download status
        run: |
          source venv/bin/activate
          python scripts/download-status.py

      - name: Sync to RAGFlow
        env:
          RAGFLOW_API_KEY: ${{ secrets.RAGFLOW_API_KEY }}
          RAGFLOW_BASE_URL: ${{ secrets.RAGFLOW_BASE_URL }}
          BATCH_SIZE: ${{ github.event.inputs.sync_batch_size || '0' }}
        run: |
          source venv/bin/activate
          ARGS="--allow-failures"
          if [ "${{ github.event.inputs.sync_verify }}" = "true" ]; then ARGS="$ARGS --verify"; fi
          if [ "${{ github.event.inputs.sync_force }}" = "true" ]; then ARGS="$ARGS --force"; fi
          python scripts/sync-ragflow.py $ARGS

      - name: Convert to MkDocs
        env:
          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
          BATCH_SIZE: ${{ github.event.inputs.convert_batch_size || '0' }}
        run: |
          source venv/bin/activate
          python scripts/convert-to-mkdocs.py

      - name: Fix internal MkDocs links
        env:
          BATCH_SIZE: ${{ github.event.inputs.linkfix_batch_size || '0' }}
        run: |
          source venv/bin/activate
          python scripts/fix-mkdocs-links.py

      - name: Regenerate homepage
        run: |
          source venv/bin/activate
          python scripts/generate-homepage.py

      - name: Smoke build MkDocs site
        run: |
          source venv/bin/activate
          mkdocs build -f mkdocs-site/mkdocs.yml

      - name: Generate prompt completions
        env:
          RAGFLOW_API_KEY: ${{ secrets.RAGFLOW_API_KEY }}
          RAGFLOW_BASE_URL: ${{ secrets.RAGFLOW_BASE_URL }}
          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
          BATCH_SIZE: ${{ github.event.inputs.completions_batch_size || '0' }}
        run: |
          source venv/bin/activate
          python scripts/generate-completions.py

      - name: Export training data
        run: |
          source venv/bin/activate
          python scripts/export-training-data.py

      - name: Configure Git
        run: |
          git config --global user.name "github-actions[bot]"
          git config --global user.email "github-actions[bot]@users.noreply.github.com"

      - name: Commit and push changes
        run: |
          git add -A docs/ mkdocs-site/ completions/ config/processing-state.json manifest.json training-data.jsonl
          
          if git diff --staged --quiet; then
            echo "No changes to commit."
            exit 0
          fi
          
          ts="$(date -u +"%Y-%m-%d %H:%M:%S UTC")"
          git commit -m "Full pipeline run ($ts)"
          git push origin HEAD:${{ github.ref_name }}

      - name: Deploy to gh-pages branch
        run: |
          cd mkdocs-site/site
          git init
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"
          git add -A
          git commit -m "Deploy docs $(date -u +'%Y-%m-%d %H:%M:%S UTC')"
          git push --force https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git HEAD:gh-pages