Full Pipeline #28
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Full Pipeline | |
| on: | |
| schedule: | |
| # Saturday 11pm UTC (adjust for your timezone: 5pm MDT, 4pm MST) | |
| - cron: '0 23 * * 6' | |
| workflow_dispatch: | |
| inputs: | |
| sync_batch_size: | |
| description: 'Docs to sync to RAGFlow (0 = all)' | |
| required: false | |
| default: '0' | |
| convert_batch_size: | |
| description: 'Docs to convert to MkDocs (0 = all)' | |
| required: false | |
| default: '0' | |
| linkfix_batch_size: | |
| description: 'Docs to run link-fix on (0 = all eligible)' | |
| required: false | |
| default: '0' | |
| completions_batch_size: | |
| description: 'Docs to generate completions for (0 = all pending)' | |
| required: false | |
| default: '0' | |
| sync_verify: | |
| description: 'Verify synced docs exist in RAGFlow before syncing' | |
| required: false | |
| default: false | |
| type: boolean | |
| sync_force: | |
| description: 'Force full re-sync by clearing ragflow_source_hash' | |
| required: false | |
| default: false | |
| type: boolean | |
| permissions: | |
| contents: write | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| jobs: | |
| download: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Set up venv | |
| run: | | |
| python -m venv venv | |
| source venv/bin/activate | |
| pip install -r requirements.txt | |
| - name: Download wiki pages | |
| run: | | |
| source venv/bin/activate | |
| python scripts/download-wiki.py | |
| - name: Download events | |
| run: | | |
| source venv/bin/activate | |
| python scripts/download-events.py | |
| - name: Download status | |
| run: | | |
| source venv/bin/activate | |
| python scripts/download-status.py | |
| - name: Sync to RAGFlow | |
| env: | |
| RAGFLOW_API_KEY: ${{ secrets.RAGFLOW_API_KEY }} | |
| RAGFLOW_BASE_URL: ${{ secrets.RAGFLOW_BASE_URL }} | |
| BATCH_SIZE: ${{ github.event.inputs.sync_batch_size || '0' }} | |
| run: | | |
| source venv/bin/activate | |
| ARGS="--allow-failures" | |
| if [ "${{ github.event.inputs.sync_verify }}" = "true" ]; then ARGS="$ARGS --verify"; fi | |
| if [ "${{ github.event.inputs.sync_force }}" = "true" ]; then ARGS="$ARGS --force"; fi | |
| python scripts/sync-ragflow.py $ARGS | |
| - name: Convert to MkDocs | |
| env: | |
| GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} | |
| BATCH_SIZE: ${{ github.event.inputs.convert_batch_size || '0' }} | |
| run: | | |
| source venv/bin/activate | |
| python scripts/convert-to-mkdocs.py | |
| - name: Fix internal MkDocs links | |
| env: | |
| BATCH_SIZE: ${{ github.event.inputs.linkfix_batch_size || '0' }} | |
| run: | | |
| source venv/bin/activate | |
| python scripts/fix-mkdocs-links.py | |
| - name: Regenerate homepage | |
| run: | | |
| source venv/bin/activate | |
| python scripts/generate-homepage.py | |
| - name: Smoke build MkDocs site | |
| run: | | |
| source venv/bin/activate | |
| mkdocs build -f mkdocs-site/mkdocs.yml | |
| - name: Generate prompt completions | |
| env: | |
| RAGFLOW_API_KEY: ${{ secrets.RAGFLOW_API_KEY }} | |
| RAGFLOW_BASE_URL: ${{ secrets.RAGFLOW_BASE_URL }} | |
| GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} | |
| BATCH_SIZE: ${{ github.event.inputs.completions_batch_size || '0' }} | |
| run: | | |
| source venv/bin/activate | |
| python scripts/generate-completions.py | |
| - name: Export training data | |
| run: | | |
| source venv/bin/activate | |
| python scripts/export-training-data.py | |
| - name: Configure Git | |
| run: | | |
| git config --global user.name "github-actions[bot]" | |
| git config --global user.email "github-actions[bot]@users.noreply.github.com" | |
| - name: Commit and push changes | |
| run: | | |
| git add -A docs/ mkdocs-site/ completions/ config/processing-state.json manifest.json training-data.jsonl | |
| if git diff --staged --quiet; then | |
| echo "No changes to commit." | |
| exit 0 | |
| fi | |
| ts="$(date -u +"%Y-%m-%d %H:%M:%S UTC")" | |
| git commit -m "Full pipeline run ($ts)" | |
| git push origin HEAD:${{ github.ref_name }} | |
| - name: Deploy to gh-pages branch | |
| run: | | |
| cd mkdocs-site/site | |
| git init | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add -A | |
| git commit -m "Deploy docs $(date -u +'%Y-%m-%d %H:%M:%S UTC')" | |
| git push --force https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git HEAD:gh-pages |