UI & Performance Testing #932
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: UI & Performance Testing | |
| on: | |
| pull_request: | |
| branches: [ main ] | |
| schedule: | |
| # Run daily at 6 AM UTC to catch regressions | |
| - cron: '0 6 * * *' | |
| workflow_dispatch: | |
| # Concurrency: cancel in-progress runs for same branch | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| build: | |
| name: Build APKs | |
| runs-on: ubuntu-latest | |
| outputs: | |
| apk-path: ${{ steps.build.outputs.apk-path }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Java 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: '17' | |
| distribution: 'temurin' | |
| - name: Setup Gradle | |
| uses: gradle/actions/setup-gradle@v4 | |
| - name: Cache Gradle packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.gradle/caches | |
| ~/.gradle/wrapper | |
| key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} | |
| restore-keys: | | |
| ${{ runner.os }}-gradle- | |
| - name: Build debug APK | |
| id: build | |
| run: | | |
| chmod +x gradlew | |
| ./gradlew assembleDebug --stacktrace | |
| # Find the x86_64 APK for emulator testing | |
| X86_APK=$(find build/outputs/apk/debug -name "*x86_64*.apk" | head -1) | |
| if [ -z "$X86_APK" ]; then | |
| # Fallback to universal if no x86_64 specific | |
| X86_APK=$(find build/outputs/apk/debug -name "*universal*.apk" | head -1) | |
| fi | |
| echo "apk-path=$X86_APK" >> $GITHUB_OUTPUT | |
| echo "Found APK for testing: $X86_APK" | |
| - name: Upload APK for testing | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: apks | |
| path: build/outputs/apk/debug/*x86_64*.apk | |
| retention-days: 1 | |
| if-no-files-found: warn | |
| - name: Upload universal APK as fallback | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: apks-universal | |
| path: build/outputs/apk/debug/*universal*.apk | |
| retention-days: 1 | |
| if-no-files-found: warn | |
| ui-tests: | |
| name: UI Tests (API ${{ matrix.api-level }}) | |
| runs-on: ubuntu-latest | |
| needs: build | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Focus on key API levels: min supported, common, latest | |
| api-level: [21, 29, 34] | |
| include: | |
| - api-level: 21 | |
| target: default | |
| arch: x86_64 | |
| - api-level: 29 | |
| target: default | |
| arch: x86_64 | |
| - api-level: 34 | |
| target: google_apis | |
| arch: x86_64 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Java 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: '17' | |
| distribution: 'temurin' | |
| - name: Setup Gradle | |
| uses: gradle/actions/setup-gradle@v4 | |
| - name: Download APK | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: apks | |
| path: ./apk/ | |
| - name: Verify APK download | |
| run: | | |
| echo "Downloaded APK files:" | |
| ls -la ./apk/ | |
| APK_FILE=$(find ./apk -name "*.apk" | head -1) | |
| echo "APK_FILE=$APK_FILE" >> $GITHUB_ENV | |
| - name: Enable KVM | |
| run: | | |
| echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules | |
| sudo udevadm control --reload-rules | |
| sudo udevadm trigger --name-match=kvm | |
| - name: Cache AVD | |
| uses: actions/cache@v4 | |
| id: avd-cache | |
| with: | |
| path: | | |
| ~/.android/avd/* | |
| ~/.android/adb* | |
| key: avd-${{ matrix.api-level }}-${{ matrix.target }}-${{ matrix.arch }}-v2 | |
| - name: Create AVD and generate snapshot | |
| if: steps.avd-cache.outputs.cache-hit != 'true' | |
| uses: reactivecircus/android-emulator-runner@v2 | |
| with: | |
| api-level: ${{ matrix.api-level }} | |
| target: ${{ matrix.target }} | |
| arch: ${{ matrix.arch }} | |
| force-avd-creation: false | |
| emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none | |
| disable-animations: false | |
| script: echo "Generated AVD snapshot for caching." | |
| - name: Run UI Tests on Emulator | |
| uses: reactivecircus/android-emulator-runner@v2 | |
| with: | |
| api-level: ${{ matrix.api-level }} | |
| target: ${{ matrix.target }} | |
| arch: ${{ matrix.arch }} | |
| force-avd-creation: false | |
| emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none | |
| disable-animations: true | |
| script: | | |
| set -euo pipefail | |
| # Debug build applies applicationIdSuffix '.debug'; component classes | |
| # keep their original (non-suffixed) fully-qualified names. | |
| PKG=tribixbite.cleverkeys.debug | |
| IME_COMPONENT="$PKG/tribixbite.cleverkeys.CleverKeysService" | |
| echo "Installing APK: ${{ env.APK_FILE }}" | |
| adb install "${{ env.APK_FILE }}" | |
| echo "Available input methods:" | |
| adb shell ime list -a | |
| adb shell ime enable "$IME_COMPONENT" | |
| adb shell ime set "$IME_COMPONENT" | |
| # Hard-assert the IME registered — a silent no-op used to pass here. | |
| adb shell dumpsys input_method | grep -q "$PKG" \ | |
| || { echo "::error::CleverKeys IME not registered"; exit 1; } | |
| adb shell am start -W -n "$PKG/tribixbite.cleverkeys.SettingsActivity" | |
| sleep 2 | |
| adb shell pidof "$PKG" || { echo "::error::App process died"; exit 1; } | |
| # Capture screenshot (best-effort — not a gate). | |
| adb exec-out screencap -p > settings-screenshot.png || true | |
| echo "Smoke test completed" | |
| - name: Upload settings screenshot | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ui-settings-screenshot-api${{ matrix.api-level }} | |
| path: settings-screenshot.png | |
| retention-days: 7 | |
| if-no-files-found: ignore | |
| visual-regression: | |
| name: Visual Regression Tests | |
| runs-on: ubuntu-latest | |
| needs: build | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Java 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: '17' | |
| distribution: 'temurin' | |
| - name: Download APK | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: apks | |
| path: ./apk/ | |
| - name: Verify APK download | |
| run: | | |
| APK_FILE=$(find ./apk -name "*.apk" | head -1) | |
| echo "APK_FILE=$APK_FILE" >> $GITHUB_ENV | |
| - name: Enable KVM | |
| run: | | |
| echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules | |
| sudo udevadm control --reload-rules | |
| sudo udevadm trigger --name-match=kvm | |
| - name: Run Visual Regression Tests | |
| uses: reactivecircus/android-emulator-runner@v2 | |
| with: | |
| api-level: 33 | |
| target: google_apis | |
| arch: x86_64 | |
| force-avd-creation: false | |
| emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none | |
| disable-animations: true | |
| script: | | |
| set -euo pipefail | |
| PKG=tribixbite.cleverkeys.debug | |
| IME_COMPONENT="$PKG/tribixbite.cleverkeys.CleverKeysService" | |
| adb install "${{ env.APK_FILE }}" | |
| adb shell ime enable "$IME_COMPONENT" | |
| adb shell ime set "$IME_COMPONENT" | |
| adb shell dumpsys input_method | grep -q "$PKG" \ | |
| || { echo "::error::CleverKeys IME not registered"; exit 1; } | |
| mkdir -p screenshots | |
| adb shell am start -W -n "$PKG/tribixbite.cleverkeys.SettingsActivity" | |
| sleep 2 | |
| adb shell pidof "$PKG" || { echo "::error::App process died"; exit 1; } | |
| adb exec-out screencap -p > screenshots/settings-screen.png || true | |
| echo "Visual regression test completed" | |
| - name: Upload visual regression artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: visual-regression-screenshots | |
| path: screenshots/ | |
| retention-days: 30 | |
| if-no-files-found: ignore | |
| - name: Generate summary | |
| run: | | |
| echo "## Visual Regression Test Results" >> $GITHUB_STEP_SUMMARY | |
| if [ -d screenshots ] && [ "$(ls -A screenshots)" ]; then | |
| echo "- Keyboard Layout: Captured" >> $GITHUB_STEP_SUMMARY | |
| echo "- Shift State: Captured" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "- No screenshots captured (keyboard may not have opened)" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| accessibility-testing: | |
| name: Accessibility Tests | |
| runs-on: ubuntu-latest | |
| needs: build | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Java 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: '17' | |
| distribution: 'temurin' | |
| - name: Download APK | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: apks | |
| path: ./apk/ | |
| - name: Verify APK download | |
| run: | | |
| APK_FILE=$(find ./apk -name "*.apk" | head -1) | |
| echo "APK_FILE=$APK_FILE" >> $GITHUB_ENV | |
| - name: Enable KVM | |
| run: | | |
| echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules | |
| sudo udevadm control --reload-rules | |
| sudo udevadm trigger --name-match=kvm | |
| - name: Run Accessibility Tests | |
| uses: reactivecircus/android-emulator-runner@v2 | |
| with: | |
| api-level: 33 | |
| target: google_apis | |
| arch: x86_64 | |
| force-avd-creation: false | |
| emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none | |
| disable-animations: true | |
| script: | | |
| set -euo pipefail | |
| PKG=tribixbite.cleverkeys.debug | |
| IME_COMPONENT="$PKG/tribixbite.cleverkeys.CleverKeysService" | |
| adb install "${{ env.APK_FILE }}" | |
| adb shell ime enable "$IME_COMPONENT" | |
| adb shell ime set "$IME_COMPONENT" | |
| # Enable accessibility services for testing | |
| adb shell settings put secure accessibility_enabled 1 | |
| # Hard-assert the IME registered before claiming accessibility passed. | |
| adb shell dumpsys input_method | grep -q "$PKG" \ | |
| || { echo "::error::CleverKeys IME not registered"; exit 1; } | |
| echo "Accessibility test completed" | |
| - name: Generate accessibility summary | |
| if: success() | |
| run: | | |
| echo "## Accessibility Test Results" >> $GITHUB_STEP_SUMMARY | |
| echo "- Keyboard installed and enabled" >> $GITHUB_STEP_SUMMARY | |
| echo "- IME registration verified via dumpsys" >> $GITHUB_STEP_SUMMARY | |
| test-summary: | |
| name: Test Summary | |
| runs-on: ubuntu-latest | |
| needs: [ui-tests, visual-regression, accessibility-testing] | |
| if: always() | |
| steps: | |
| - name: Generate test summary | |
| run: | | |
| echo "# CleverKeys Testing Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "## Test Results Overview" >> $GITHUB_STEP_SUMMARY | |
| echo "- **UI Tests**: ${{ needs.ui-tests.result }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Visual Regression**: ${{ needs.visual-regression.result }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Accessibility Tests**: ${{ needs.accessibility-testing.result }}" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "## Key Metrics Tested" >> $GITHUB_STEP_SUMMARY | |
| echo "- UI responsiveness across Android versions (API 21, 29, 34)" >> $GITHUB_STEP_SUMMARY | |
| echo "- Visual consistency of keyboard layouts" >> $GITHUB_STEP_SUMMARY | |
| echo "- Accessibility compliance" >> $GITHUB_STEP_SUMMARY |