Generate RC Test Plan #3814
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Generate RC Test Plan | |
| # Trigger when Bitrise posts "RC Builds Ready for Testing" comment | |
| on: | |
| issue_comment: | |
| types: [created] | |
| jobs: | |
| generate-test-plan: | |
| name: Generate AI Test Plan | |
| # Only run when: | |
| # 1. Comment is on a PR (not an issue) | |
| # 2. Comment contains "RC Builds Ready for Testing" | |
| # 3. Comment is from github-actions bot (Bitrise posts via this) | |
| if: | | |
| github.event.issue.pull_request && | |
| contains(github.event.comment.body, 'RC Builds Ready for Testing') && | |
| github.event.comment.user.login == 'github-actions[bot]' | |
| runs-on: ubuntu-latest | |
| environment: release-ci | |
| timeout-minutes: 15 | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| issues: write | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| E2E_CLAUDE_API_KEY: ${{ secrets.E2E_CLAUDE_API_KEY }} | |
| E2E_OPENAI_API_KEY: ${{ secrets.E2E_OPENAI_API_KEY }} | |
| E2E_GEMINI_API_KEY: ${{ secrets.E2E_GEMINI_API_KEY }} | |
| PR_NUMBER: ${{ github.event.issue.number }} | |
| steps: | |
| - name: Check if release PR | |
| id: check-release | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const pr = await github.rest.pulls.get({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: process.env.PR_NUMBER | |
| }); | |
| const isRelease = pr.data.head.ref.startsWith('release/'); | |
| console.log(`PR branch: ${pr.data.head.ref}, isRelease: ${isRelease}`); | |
| if (!isRelease) { | |
| console.log('Not a release PR, skipping test plan generation'); | |
| return; | |
| } | |
| // Extract version from branch name (e.g., release/7.70.0 -> 7.70.0) | |
| // Sanitize to prevent shell injection - only allow semver chars | |
| const rawVersion = pr.data.head.ref.replace('release/', ''); | |
| const version = rawVersion.replace(/[^0-9.]/g, ''); | |
| if (!version || !/^\d+\.\d+\.\d+$/.test(version)) { | |
| console.log(`Invalid version format: ${rawVersion}`); | |
| return; | |
| } | |
| core.setOutput('version', version); | |
| core.setOutput('is_release', 'true'); | |
| core.setOutput('pr_title', pr.data.title); | |
| - name: Checkout repository | |
| if: steps.check-release.outputs.is_release == 'true' | |
| uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| if: steps.check-release.outputs.is_release == 'true' | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| cache: 'yarn' | |
| - name: Install dependencies | |
| if: steps.check-release.outputs.is_release == 'true' | |
| run: yarn install --frozen-lockfile | |
| - name: Extract build number from comment | |
| if: steps.check-release.outputs.is_release == 'true' | |
| id: extract-build | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const comment = context.payload.comment.body; | |
| // Match build number from "RC X.Y.Z (BUILD)" pattern specifically | |
| // e.g., "RC 7.65.0 (4025)" captures "4025" | |
| const buildMatch = comment.match(/RC\s+\d+\.\d+\.\d+\s*\((\d+)\)/i); | |
| if (buildMatch) { | |
| const buildNumber = buildMatch[1]; | |
| console.log(`Extracted build number: ${buildNumber}`); | |
| core.setOutput('build_number', buildNumber); | |
| } else { | |
| console.log('Could not extract build number from comment'); | |
| core.setOutput('build_number', ''); | |
| } | |
| - name: Generate test plan | |
| if: steps.check-release.outputs.is_release == 'true' | |
| id: generate | |
| run: | | |
| VERSION="${{ steps.check-release.outputs.version }}" | |
| RAW_BUILD="${{ steps.extract-build.outputs.build_number }}" | |
| # Sanitize BUILD to only allow digits | |
| BUILD=$(echo "$RAW_BUILD" | tr -cd '0-9') | |
| echo "Generating test plan for version: $VERSION, build: $BUILD" | |
| # Sanitize PR_NUMBER to only allow digits | |
| PR_NUM=$(echo "${{ env.PR_NUMBER }}" | tr -cd '0-9') | |
| # Run the analyzer | |
| if node -r esbuild-register tests/tools/e2e-ai-analyzer \ | |
| --mode generate-test-plan \ | |
| --pr "$PR_NUM" \ | |
| --auto-ff \ | |
| -v "$VERSION"; then | |
| echo "test_plan_generated=true" >> "${GITHUB_OUTPUT}" | |
| else | |
| echo "Warning: Test plan generation failed" | |
| echo "test_plan_generated=false" >> "${GITHUB_OUTPUT}" | |
| fi | |
| - name: Generate HTML viewer | |
| if: steps.generate.outputs.test_plan_generated == 'true' | |
| run: | | |
| VERSION="${{ steps.check-release.outputs.version }}" | |
| BUILD="${{ steps.extract-build.outputs.build_number }}" | |
| # Create test-plans directory | |
| mkdir -p test-plans | |
| # Move JSON | |
| mv release-test-plan.json "test-plans/test-plan-${VERSION}.json" | |
| # Generate HTML viewer | |
| node -e " | |
| const fs = require('fs'); | |
| const plan = JSON.parse(fs.readFileSync('test-plans/test-plan-${VERSION}.json', 'utf8')); | |
| // Escape HTML to prevent XSS from LLM-generated content | |
| const esc = (s) => (s == null ? '' : String(s)).replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/\"/g, '"'); | |
| const html = \`<!DOCTYPE html> | |
| <html lang=\"en\"> | |
| <head> | |
| <meta charset=\"UTF-8\"> | |
| <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\"> | |
| <title>RC \${esc(plan.version) || '${VERSION}'} Test Plan</title> | |
| <style> | |
| * { box-sizing: border-box; } | |
| body { | |
| font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; | |
| max-width: 900px; margin: 0 auto; padding: 20px; | |
| background: #f5f5f5; color: #333; | |
| } | |
| h1 { color: #1a1a1a; border-bottom: 3px solid #037dd6; padding-bottom: 10px; } | |
| h2 { color: #037dd6; margin-top: 30px; } | |
| h3 { color: #444; } | |
| .summary { background: #fff; padding: 20px; border-radius: 8px; margin: 20px 0; box-shadow: 0 2px 4px rgba(0,0,0,0.1); } | |
| .summary-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); gap: 15px; } | |
| .stat { text-align: center; padding: 15px; background: #f8f9fa; border-radius: 6px; } | |
| .stat-value { font-size: 24px; font-weight: bold; color: #037dd6; } | |
| .stat-label { font-size: 12px; color: #666; margin-top: 5px; } | |
| .risk-high { border-left: 4px solid #d73a49; } | |
| .risk-medium { border-left: 4px solid #f9a825; } | |
| .scenario { background: #fff; padding: 20px; margin: 15px 0; border-radius: 8px; box-shadow: 0 2px 4px rgba(0,0,0,0.1); } | |
| .scenario h3 { margin-top: 0; } | |
| .badge { display: inline-block; padding: 3px 8px; border-radius: 4px; font-size: 12px; font-weight: 600; } | |
| .badge-high { background: #ffeef0; color: #d73a49; } | |
| .badge-medium { background: #fff8e1; color: #f57c00; } | |
| .steps { background: #f8f9fa; padding: 15px; border-radius: 6px; margin-top: 10px; } | |
| .steps li { margin: 8px 0; } | |
| .preconditions { background: #e3f2fd; padding: 10px 15px; border-radius: 6px; margin: 10px 0; } | |
| .outcomes { background: #e8f5e9; padding: 10px 15px; border-radius: 6px; margin: 10px 0; } | |
| .executive { background: linear-gradient(135deg, #037dd6 0%, #0260a8 100%); color: white; padding: 25px; border-radius: 8px; margin: 20px 0; } | |
| .executive h2 { color: white; margin-top: 0; } | |
| .teams { display: flex; flex-wrap: wrap; gap: 8px; margin: 15px 0; } | |
| .team { background: #e3f2fd; padding: 5px 12px; border-radius: 20px; font-size: 13px; } | |
| .footer { text-align: center; margin-top: 40px; padding: 20px; color: #666; font-size: 13px; } | |
| a { color: #037dd6; } | |
| </style> | |
| </head> | |
| <body> | |
| <h1>🧪 RC \${esc(plan.version) || '${VERSION}'} Test Plan</h1> | |
| <p>Build: \${plan.buildNumber || '${BUILD}'} | Generated: \${new Date(plan.generatedAt).toLocaleString()}</p> | |
| \${plan.executiveSummary ? \` | |
| <div class=\"executive\"> | |
| <h2>📊 Executive Summary</h2> | |
| <p><strong>\${esc(plan.executiveSummary.releaseFocus)}</strong></p> | |
| <p><strong>Key Changes:</strong></p> | |
| <ul>\${plan.executiveSummary.keyChanges.map(c => '<li>' + esc(c) + '</li>').join('')}</ul> | |
| <p><strong>Risk Level:</strong> \${esc(plan.executiveSummary.overallRisk).toUpperCase()}</p> | |
| <p><strong>Recommendation:</strong> \${esc(plan.executiveSummary.recommendation)}</p> | |
| </div> | |
| \` : ''} | |
| <div class=\"summary\"> | |
| <h2>📈 Summary</h2> | |
| <div class=\"summary-grid\"> | |
| <div class=\"stat\"><div class=\"stat-value\">\${plan.summary?.releaseRiskScore || '0/100'}</div><div class=\"stat-label\">Risk Score</div></div> | |
| <div class=\"stat\"><div class=\"stat-value\">\${plan.summary?.totalFiles || plan.summary?.totalFilesChanged || 0}</div><div class=\"stat-label\">Files Changed</div></div> | |
| <div class=\"stat\"><div class=\"stat-value\">\${plan.summary?.highImpactFiles || 0}</div><div class=\"stat-label\">High Impact</div></div> | |
| <div class=\"stat\"><div class=\"stat-value\">\${plan.summary?.highRiskCount || plan.summary?.highRiskScenarios || 0}</div><div class=\"stat-label\">High Risk</div></div> | |
| <div class=\"stat\"><div class=\"stat-value\">\${plan.summary?.mediumRiskCount || plan.summary?.mediumRiskScenarios || 0}</div><div class=\"stat-label\">Medium Risk</div></div> | |
| </div> | |
| </div> | |
| \${(plan.signOffs?.needsAttention?.length || plan.teamsNeedingSignOff?.length) ? \` | |
| <h2>👥 Teams Needing Sign-off</h2> | |
| <div class=\"teams\">\${(plan.signOffs?.needsAttention || plan.teamsNeedingSignOff || []).map(t => '<span class=\"team\">⏳ ' + esc(t) + '</span>').join('')}</div> | |
| \` : ''} | |
| \${plan.testScenarios?.cherryPickScenarios?.length ? \` | |
| <h2>🍒 Cherry-Pick Scenarios</h2> | |
| \${plan.testScenarios.cherryPickScenarios.map((s, i) => \` | |
| <div class=\"scenario risk-high\"> | |
| <h3>\${i + 1}. \${esc(s.area)} <span class=\"badge badge-high\">CHERRY-PICK</span></h3> | |
| <p><strong>Why:</strong> \${esc(s.whyThisMatters)}</p> | |
| <div class=\"steps\"><strong>Test Steps:</strong><ol>\${(s.testSteps || []).map(step => '<li>' + esc(step) + '</li>').join('')}</ol></div> | |
| </div> | |
| \`).join('')} | |
| \` : ''} | |
| <h2>🔴 High Risk Areas</h2> | |
| \${(plan.scenarios || plan.testScenarios?.initialScenarios || []).filter(s => s.riskLevel === 'high').map((s, i) => \` | |
| <div class=\"scenario risk-high\"> | |
| <h3>\${i + 1}. \${esc(s.area)} <span class=\"badge badge-high\">HIGH</span></h3> | |
| <p><strong>Why:</strong> \${esc(s.whyThisMatters)}</p> | |
| \${s.preconditions?.length ? '<div class=\"preconditions\"><strong>Preconditions:</strong><ul>' + s.preconditions.map(p => '<li>' + esc(p) + '</li>').join('') + '</ul></div>' : ''} | |
| <div class=\"steps\"><strong>Test Steps:</strong><ol>\${(s.testSteps || []).map(step => '<li>' + esc(step) + '</li>').join('')}</ol></div> | |
| \${s.expectedOutcomes?.length ? '<div class=\"outcomes\"><strong>Expected Outcomes:</strong><ul>' + s.expectedOutcomes.map(o => '<li>✓ ' + esc(o) + '</li>').join('') + '</ul></div>' : ''} | |
| </div> | |
| \`).join('')} | |
| <h2>🟡 Medium Risk Areas</h2> | |
| \${(plan.scenarios || plan.testScenarios?.initialScenarios || []).filter(s => s.riskLevel === 'medium').map((s, i) => \` | |
| <div class=\"scenario risk-medium\"> | |
| <h3>\${i + 1}. \${esc(s.area)} <span class=\"badge badge-medium\">MEDIUM</span></h3> | |
| <p><strong>Why:</strong> \${esc(s.whyThisMatters)}</p> | |
| \${s.preconditions?.length ? '<div class=\"preconditions\"><strong>Preconditions:</strong><ul>' + s.preconditions.map(p => '<li>' + esc(p) + '</li>').join('') + '</ul></div>' : ''} | |
| <div class=\"steps\"><strong>Test Steps:</strong><ol>\${(s.testSteps || []).map(step => '<li>' + esc(step) + '</li>').join('')}</ol></div> | |
| \${s.expectedOutcomes?.length ? '<div class=\"outcomes\"><strong>Expected Outcomes:</strong><ul>' + s.expectedOutcomes.map(o => '<li>✓ ' + esc(o) + '</li>').join('') + '</ul></div>' : ''} | |
| </div> | |
| \`).join('')} | |
| <div class=\"footer\"> | |
| <p>Generated by AI Test Plan Analyzer | <a href=\"https://github.com/MetaMask/metamask-mobile\">MetaMask Mobile</a></p> | |
| <p><a href=\"test-plan-${VERSION}.json\">Download JSON</a></p> | |
| </div> | |
| </body> | |
| </html>\`; | |
| fs.writeFileSync('test-plans/test-plan-${VERSION}.html', html); | |
| console.log('Generated HTML viewer'); | |
| " | |
| - name: Deploy to GitHub Pages | |
| if: steps.generate.outputs.test_plan_generated == 'true' | |
| run: | | |
| VERSION="${{ steps.check-release.outputs.version }}" | |
| # Save generated files to temp before switching branches | |
| cp test-plans/test-plan-${VERSION}.json /tmp/ | |
| cp test-plans/test-plan-${VERSION}.html /tmp/ | |
| # Clean up to avoid conflicts when switching branches | |
| rm -rf test-plans | |
| # Configure git | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| # Fetch gh-pages branch or create it | |
| git fetch origin gh-pages:gh-pages 2>/dev/null || echo "gh-pages doesn't exist yet" | |
| # Switch to gh-pages (create orphan if it doesn't exist) | |
| if git checkout gh-pages 2>/dev/null; then | |
| echo "Switched to existing gh-pages branch" | |
| else | |
| # Create orphan branch and clear the index to avoid committing entire repo | |
| git checkout --orphan gh-pages | |
| git rm -rf . 2>/dev/null || true | |
| git clean -fd 2>/dev/null || true | |
| fi | |
| # Create test-plans directory | |
| mkdir -p test-plans | |
| # Copy files from temp (overwrites if exists - handles re-runs) | |
| cp /tmp/test-plan-${VERSION}.json test-plans/ | |
| cp /tmp/test-plan-${VERSION}.html test-plans/ | |
| # Add and commit | |
| git add test-plans/ | |
| git commit -m "Add test plan for RC ${VERSION}" || echo "No changes to commit" | |
| # Push to gh-pages | |
| git push origin gh-pages | |
| - name: Update build comment with test plan links | |
| if: steps.generate.outputs.test_plan_generated == 'true' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const version = '${{ steps.check-release.outputs.version }}'; | |
| const buildNumber = '${{ steps.extract-build.outputs.build_number }}'; | |
| const commentId = context.payload.comment.id; | |
| // Fetch latest comment body to avoid race conditions | |
| const { data: comment } = await github.rest.issues.getComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: commentId | |
| }); | |
| const currentBody = comment.body; | |
| const baseUrl = `https://metamask.github.io/metamask-mobile/test-plans`; | |
| const htmlUrl = `${baseUrl}/test-plan-${version}.html`; | |
| const jsonUrl = `${baseUrl}/test-plan-${version}.json`; | |
| // Add test plan row to the existing comment | |
| const testPlanSection = ` | |
| --- | |
| 🤖 **AI Test Plan:** [View](${htmlUrl}) | [JSON](${jsonUrl})`; | |
| // Update the comment | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: commentId, | |
| body: currentBody + testPlanSection | |
| }); | |
| - name: Post failure notice | |
| if: steps.check-release.outputs.is_release == 'true' && steps.generate.outputs.test_plan_generated != 'true' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const version = '${{ steps.check-release.outputs.version }}'; | |
| const commentId = context.payload.comment.id; | |
| const runId = context.runId; | |
| const logsUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${runId}`; | |
| // Fetch latest comment body to avoid race conditions | |
| const { data: comment } = await github.rest.issues.getComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: commentId | |
| }); | |
| const currentBody = comment.body; | |
| const failureSection = ` | |
| --- | |
| ⚠️ **AI Test Plan generation failed** - [View logs](${logsUrl})`; | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: commentId, | |
| body: currentBody + failureSection | |
| }); |