UI & Performance Testing #885
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: UI & Performance Testing | |
| on: | |
| push: | |
| branches: [ main, develop ] | |
| tags-ignore: | |
| - '**' # Don't run on tag pushes - release.yml handles those | |
| pull_request: | |
| branches: [ main ] | |
| schedule: | |
| # Run daily at 6 AM UTC to catch regressions | |
| - cron: '0 6 * * *' | |
| workflow_dispatch: | |
| # Concurrency: cancel in-progress runs for same branch | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| build: | |
| name: Build APKs | |
| runs-on: ubuntu-latest | |
| outputs: | |
| apk-path: ${{ steps.build.outputs.apk-path }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Java 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: '17' | |
| distribution: 'temurin' | |
| - name: Setup Gradle | |
| uses: gradle/actions/setup-gradle@v4 | |
| - name: Cache Gradle packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.gradle/caches | |
| ~/.gradle/wrapper | |
| key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} | |
| restore-keys: | | |
| ${{ runner.os }}-gradle- | |
| - name: Build debug APK | |
| id: build | |
| run: | | |
| chmod +x gradlew | |
| ./gradlew assembleDebug --stacktrace | |
| # Find the x86_64 APK for emulator testing | |
| X86_APK=$(find build/outputs/apk/debug -name "*x86_64*.apk" | head -1) | |
| if [ -z "$X86_APK" ]; then | |
| # Fallback to universal if no x86_64 specific | |
| X86_APK=$(find build/outputs/apk/debug -name "*universal*.apk" | head -1) | |
| fi | |
| echo "apk-path=$X86_APK" >> $GITHUB_OUTPUT | |
| echo "Found APK for testing: $X86_APK" | |
| - name: Upload APK for testing | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: apks | |
| path: build/outputs/apk/debug/*x86_64*.apk | |
| retention-days: 1 | |
| if-no-files-found: warn | |
| - name: Upload universal APK as fallback | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: apks-universal | |
| path: build/outputs/apk/debug/*universal*.apk | |
| retention-days: 1 | |
| if-no-files-found: warn | |
| ui-tests: | |
| name: UI Tests (API ${{ matrix.api-level }}) | |
| runs-on: ubuntu-latest | |
| needs: build | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Focus on key API levels: min supported, common, latest | |
| api-level: [21, 29, 34] | |
| include: | |
| - api-level: 21 | |
| target: default | |
| arch: x86_64 | |
| - api-level: 29 | |
| target: default | |
| arch: x86_64 | |
| - api-level: 34 | |
| target: google_apis | |
| arch: x86_64 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Java 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: '17' | |
| distribution: 'temurin' | |
| - name: Setup Gradle | |
| uses: gradle/actions/setup-gradle@v4 | |
| - name: Download APK | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: apks | |
| path: ./apk/ | |
| - name: Verify APK download | |
| run: | | |
| echo "Downloaded APK files:" | |
| ls -la ./apk/ | |
| APK_FILE=$(find ./apk -name "*.apk" | head -1) | |
| echo "APK_FILE=$APK_FILE" >> $GITHUB_ENV | |
| - name: Enable KVM | |
| run: | | |
| echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules | |
| sudo udevadm control --reload-rules | |
| sudo udevadm trigger --name-match=kvm | |
| - name: Cache AVD | |
| uses: actions/cache@v4 | |
| id: avd-cache | |
| with: | |
| path: | | |
| ~/.android/avd/* | |
| ~/.android/adb* | |
| key: avd-${{ matrix.api-level }}-${{ matrix.target }}-${{ matrix.arch }}-v2 | |
| - name: Create AVD and generate snapshot | |
| if: steps.avd-cache.outputs.cache-hit != 'true' | |
| uses: reactivecircus/android-emulator-runner@v2 | |
| with: | |
| api-level: ${{ matrix.api-level }} | |
| target: ${{ matrix.target }} | |
| arch: ${{ matrix.arch }} | |
| force-avd-creation: false | |
| emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none | |
| disable-animations: false | |
| script: echo "Generated AVD snapshot for caching." | |
| - name: Run UI Tests on Emulator | |
| uses: reactivecircus/android-emulator-runner@v2 | |
| with: | |
| api-level: ${{ matrix.api-level }} | |
| target: ${{ matrix.target }} | |
| arch: ${{ matrix.arch }} | |
| force-avd-creation: false | |
| emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none | |
| disable-animations: true | |
| script: | | |
| # Install CleverKeys APK | |
| echo "Installing APK: ${{ env.APK_FILE }}" | |
| adb install "${{ env.APK_FILE }}" | |
| # List available input methods | |
| echo "Available input methods:" | |
| adb shell ime list -a | |
| # Try to enable CleverKeys (may fail on some API levels, that's OK) | |
| adb shell ime enable tribixbite.cleverkeys/tribixbite.cleverkeys.CleverKeysService || true | |
| adb shell ime set tribixbite.cleverkeys/tribixbite.cleverkeys.CleverKeysService || true | |
| # Basic smoke test - launch settings activity | |
| adb shell am start -n tribixbite.cleverkeys/tribixbite.cleverkeys.SettingsActivity || true | |
| sleep 2 | |
| # Capture screenshot | |
| adb exec-out screencap -p > settings-screenshot.png || true | |
| echo "Smoke test completed" | |
| - name: Upload test reports | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ui-test-reports-api${{ matrix.api-level }} | |
| path: | | |
| build/reports/androidTests/ | |
| build/outputs/androidTest-results/ | |
| retention-days: 7 | |
| if-no-files-found: ignore | |
| visual-regression: | |
| name: Visual Regression Tests | |
| runs-on: ubuntu-latest | |
| needs: build | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Java 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: '17' | |
| distribution: 'temurin' | |
| - name: Download APK | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: apks | |
| path: ./apk/ | |
| - name: Verify APK download | |
| run: | | |
| APK_FILE=$(find ./apk -name "*.apk" | head -1) | |
| echo "APK_FILE=$APK_FILE" >> $GITHUB_ENV | |
| - name: Enable KVM | |
| run: | | |
| echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules | |
| sudo udevadm control --reload-rules | |
| sudo udevadm trigger --name-match=kvm | |
| - name: Run Visual Regression Tests | |
| uses: reactivecircus/android-emulator-runner@v2 | |
| with: | |
| api-level: 33 | |
| target: google_apis | |
| arch: x86_64 | |
| force-avd-creation: false | |
| emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none | |
| disable-animations: true | |
| script: | | |
| # Install CleverKeys APK | |
| adb install "${{ env.APK_FILE }}" | |
| # Try to enable keyboard (may fail, that's OK) | |
| adb shell ime enable tribixbite.cleverkeys/tribixbite.cleverkeys.CleverKeysService || true | |
| adb shell ime set tribixbite.cleverkeys/tribixbite.cleverkeys.CleverKeysService || true | |
| # Capture screenshots | |
| mkdir -p screenshots | |
| # Launch settings to verify app works | |
| adb shell am start -n tribixbite.cleverkeys/tribixbite.cleverkeys.SettingsActivity || true | |
| sleep 2 | |
| adb exec-out screencap -p > screenshots/settings-screen.png || true | |
| echo "Visual regression test completed" | |
| - name: Upload visual regression artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: visual-regression-screenshots | |
| path: screenshots/ | |
| retention-days: 30 | |
| if-no-files-found: ignore | |
| - name: Generate summary | |
| run: | | |
| echo "## Visual Regression Test Results" >> $GITHUB_STEP_SUMMARY | |
| if [ -d screenshots ] && [ "$(ls -A screenshots)" ]; then | |
| echo "- Keyboard Layout: Captured" >> $GITHUB_STEP_SUMMARY | |
| echo "- Shift State: Captured" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "- No screenshots captured (keyboard may not have opened)" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| accessibility-testing: | |
| name: Accessibility Tests | |
| runs-on: ubuntu-latest | |
| needs: build | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Java 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: '17' | |
| distribution: 'temurin' | |
| - name: Download APK | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: apks | |
| path: ./apk/ | |
| - name: Verify APK download | |
| run: | | |
| APK_FILE=$(find ./apk -name "*.apk" | head -1) | |
| echo "APK_FILE=$APK_FILE" >> $GITHUB_ENV | |
| - name: Enable KVM | |
| run: | | |
| echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules | |
| sudo udevadm control --reload-rules | |
| sudo udevadm trigger --name-match=kvm | |
| - name: Run Accessibility Tests | |
| uses: reactivecircus/android-emulator-runner@v2 | |
| with: | |
| api-level: 33 | |
| target: google_apis | |
| arch: x86_64 | |
| force-avd-creation: false | |
| emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none | |
| disable-animations: true | |
| script: | | |
| # Install CleverKeys APK | |
| adb install "${{ env.APK_FILE }}" | |
| # Try to enable keyboard (may fail, that's OK) | |
| adb shell ime enable tribixbite.cleverkeys/tribixbite.cleverkeys.CleverKeysService || true | |
| adb shell ime set tribixbite.cleverkeys/tribixbite.cleverkeys.CleverKeysService || true | |
| # Enable accessibility services for testing | |
| adb shell settings put secure accessibility_enabled 1 || true | |
| # Verify keyboard is accessible | |
| adb shell dumpsys input_method | head -20 | |
| echo "Accessibility test completed" | |
| - name: Generate accessibility summary | |
| run: | | |
| echo "## Accessibility Test Results" >> $GITHUB_STEP_SUMMARY | |
| echo "- Keyboard installed and enabled" >> $GITHUB_STEP_SUMMARY | |
| echo "- Accessibility services verified" >> $GITHUB_STEP_SUMMARY | |
| device-farm-testing: | |
| name: Firebase Test Lab | |
| runs-on: ubuntu-latest | |
| needs: build | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' && vars.ENABLE_FIREBASE == 'true' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Download APKs | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: apks | |
| path: build/outputs/apk/ | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: ${{ secrets.FIREBASE_SERVICE_ACCOUNT }} | |
| - name: Run tests on Firebase Test Lab | |
| run: | | |
| # Install gcloud CLI | |
| curl -sSL https://sdk.cloud.google.com | bash -s -- --disable-prompts | |
| source ~/google-cloud-sdk/path.bash.inc | |
| # Run tests (requires Firebase project setup) | |
| gcloud firebase test android run \ | |
| --type instrumentation \ | |
| --app build/outputs/apk/debug/tribixbite.cleverkeys.apk \ | |
| --test build/outputs/apk/androidTest/debug/*.apk \ | |
| --device model=Pixel6,version=33 \ | |
| --timeout 10m \ | |
| --results-bucket gs://${{ secrets.FIREBASE_BUCKET }} || true | |
| echo "## Firebase Test Lab Results" >> $GITHUB_STEP_SUMMARY | |
| echo "- Tests submitted to Firebase Test Lab" >> $GITHUB_STEP_SUMMARY | |
| test-summary: | |
| name: Test Summary | |
| runs-on: ubuntu-latest | |
| needs: [ui-tests, visual-regression, accessibility-testing] | |
| if: always() | |
| steps: | |
| - name: Generate test summary | |
| run: | | |
| echo "# CleverKeys Testing Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "## Test Results Overview" >> $GITHUB_STEP_SUMMARY | |
| echo "- **UI Tests**: ${{ needs.ui-tests.result }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Visual Regression**: ${{ needs.visual-regression.result }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Accessibility Tests**: ${{ needs.accessibility-testing.result }}" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "## Key Metrics Tested" >> $GITHUB_STEP_SUMMARY | |
| echo "- UI responsiveness across Android versions (API 21, 29, 34)" >> $GITHUB_STEP_SUMMARY | |
| echo "- Visual consistency of keyboard layouts" >> $GITHUB_STEP_SUMMARY | |
| echo "- Accessibility compliance" >> $GITHUB_STEP_SUMMARY |