[QA-1126] Adding sanity check against real devices for Prod builds

.github/workflows/build.yml

@@ -113,8 +113,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        variant: ["prod", "dev"]
-        artifact: ["apk", "aab"]
+        variant: [ "prod", "dev" ]
+        artifact: [ "apk", "aab" ]
     steps:
       - name: Check out repo
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -602,3 +602,149 @@ jobs:
           bundle exec fastlane distributeReleaseFDroidToFirebase \
           actionUrl:${{ env.GITHUB_ACTION_RUN_URL }} \
           service_credentials_file:${{ env.APP_FDROID_FIREBASE_CREDS_PATH }}
+
+  test_on_device:
+    name: Running sanity check on a real device
+    needs: publish_playstore
+    runs-on: ubuntu-24.04
+    if: github.ref == 'refs/heads/main'
+    env:
+      _JAVA_VERSION: 17
+      _GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Validate Gradle wrapper
+        uses: gradle/actions/wrapper-validation@06832c7b30a0129d7fb559bcc6e43d26f6374244 # v4.3.1
+
+      - name: Cache Gradle files
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+          key: ${{ runner.os }}-gradle-v2-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties', '**/libs.versions.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-v2-
+
+      - name: Cache build output
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ${{ github.workspace }}/build-cache
+          key: ${{ runner.os }}-build-cache-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-build-
+
+      - name: Configure JDK
+        uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
+        with:
+          distribution: "temurin"
+          java-version: ${{ env._JAVA_VERSION }}
+
+      - name: Configure Ruby
+        uses: ruby/setup-ruby@ca041f971d66735f3e5ff1e21cc13e2d51e7e535 # v1.233.0
+        with:
+          bundler-cache: true
+
+      - name: Install Fastlane
+        run: |
+          gem install bundler:2.2.27
+          bundle config path vendor/bundle
+          bundle install --jobs 4 --retry 3
+
+      - name: Install saucectl
+        run: |
+          npm i -g saucectl
+
+      - name: Log in to Azure
+        uses: Azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2.3.0
+        with:
+          creds: ${{ secrets.AZURE_KV_CI_SERVICE_PRINCIPAL }}
+
+      - name: Retrieve secrets
+        env:
+          ACCOUNT_NAME: bitwardenci
+          CONTAINER_NAME: mobile
+        run: |
+          mkdir -p ${{ github.workspace }}/secrets
+          mkdir -p ${{ github.workspace }}/app/src/standardRelease
+
+          az storage blob download --account-name $ACCOUNT_NAME --container-name $CONTAINER_NAME \
+          --name app_play-keystore.jks --file ${{ github.workspace }}/keystores/app_play-keystore.jks --output none
+          az storage blob download --account-name $ACCOUNT_NAME --container-name $CONTAINER_NAME \
+          --name google-services.json --file ${{ github.workspace }}/app/src/standardRelease/google-services.json --output none
+
+      - name: Log in to Azure (Bitwarden)
+        uses: bitwarden/gh-actions/azure-login@main
+        with:
+          subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          tenant_id: ${{ secrets.AZURE_TENANT_ID }}
+          client_id: ${{ secrets.AZURE_CLIENT_ID }}
+
+      - name: Get E2E test secrets from Azure
+        uses: bitwarden/gh-actions/get-keyvault-secrets@main
+        with:
+          keyvault: gh-android
+          secrets: "BWS-ACCESS-TOKEN, SAUCE-LABS-USERNAME, SAUCE-LABS-ACCESS-KEY"
+        id: get-e2e-secrets
+
+      - name: Log out from Azure
+        uses: bitwarden/gh-actions/azure-logout@main
+
+      - name: Retrieve test data
+        uses: bitwarden/sm-action@14f92f1d294ae3c2b6a3845d389cd2c318b0dfd8 # v2.2.0
+        with:
+          access_token: ${{ steps.get-e2e-secrets.outputs.BWS-ACCESS-TOKEN }}
+          secrets: |
+            63e93f73-5118-4a62-9db8-b3160176aa8a > TEST_ACCOUNT_CREDS
+
+      - name: Configure .json test data file
+        run: printf %s '${{ env.TEST_ACCOUNT_CREDS }}' > app/src/androidTest/assets/TestData.json
+
+      - name: Build test APK (espresso)
+        run: |
+          ./gradlew :app:assembleStandardReleaseAndroidTest
+
+      - name: Download release APK artifact
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: com.x8bit.bitwarden.apk
+          path: app/build/outputs/apk/standard/release/
+
+      - name: Signing test APK
+        run: |
+          $ANDROID_SDK_ROOT/build-tools/34.0.0/apksigner sign \
+            --ks keystores/app_play-keystore.jks \
+            --ks-key-alias bitwarden \
+            --ks-pass pass:${{ secrets.PLAY_KEYSTORE_PASSWORD }} \
+            --key-pass pass:${{ secrets.PLAY_KEYSTORE_PASSWORD }} \
+            app/build/outputs/apk/androidTest/standard/release/com.x8bit.bitwarden-standard-release-androidTest.apk
+
+      - name: Upload APK to SauceLabs storage
+        run: |
+          saucectl storage upload app/build/outputs/apk/standard/release/com.x8bit.bitwarden.apk
+        env:
+          SAUCE_USERNAME: ${{ steps.get-e2e-secrets.outputs.SAUCE-LABS-USERNAME }}
+          SAUCE_ACCESS_KEY: ${{ steps.get-e2e-secrets.outputs.SAUCE-LABS-ACCESS-KEY }}
+
+      - name: Upload test APK to SauceLabs storage
+        run: |
+          saucectl storage upload app/build/outputs/apk/androidTest/standard/release/com.x8bit.bitwarden-standard-release-androidTest.apk
+        env:
+          SAUCE_USERNAME: ${{ steps.get-e2e-secrets.outputs.SAUCE-LABS-USERNAME }}
+          SAUCE_ACCESS_KEY: ${{ steps.get-e2e-secrets.outputs.SAUCE-LABS-ACCESS-KEY }}
+
+      - name: Run tests on SauceLabs
+        run: saucectl run --config .sauce/config.yml
+        env:
+          SAUCE_USERNAME: ${{ steps.get-e2e-secrets.outputs.SAUCE-LABS-USERNAME }}
+          SAUCE_ACCESS_KEY: ${{ steps.get-e2e-secrets.outputs.SAUCE-LABS-ACCESS-KEY }}
+
+      - name: Upload SauceLabs test report
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: saucectl-report
+          path: saucectl-report.xml

.github/workflows/test-device.yml

@@ -5,12 +5,155 @@ on:
 
 permissions:
   contents: read
+  actions: read
+  checks: write
+  id-token: write
 
 jobs:
-  test:
-    name: Test Device
+  test-device:
+    name: Check main build against real devices
     runs-on: ubuntu-24.04
+    env:
+      _JAVA_VERSION: 17
+      _GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
     steps:
-    - name: Placeholder step
-      run: echo "Placeholder workflow step"
+      - name: Check out repo
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Validate Gradle wrapper
+        uses: gradle/actions/wrapper-validation@06832c7b30a0129d7fb559bcc6e43d26f6374244 # v4.3.1
+
+      - name: Cache Gradle files
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+          key: ${{ runner.os }}-gradle-v2-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties', '**/libs.versions.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-v2-
+
+      - name: Cache build output
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ${{ github.workspace }}/build-cache
+          key: ${{ runner.os }}-build-cache-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-build-
+
+      - name: Configure JDK
+        uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
+        with:
+          distribution: "temurin"
+          java-version: ${{ env.JAVA_VERSION }}
+
+      - name: Configure Ruby
+        uses: ruby/setup-ruby@ca041f971d66735f3e5ff1e21cc13e2d51e7e535 # v1.233.0
+        with:
+          bundler-cache: true
+
+      - name: Install Fastlane
+        run: |
+          gem install bundler:2.2.27
+          bundle config path vendor/bundle
+          bundle install --jobs 4 --retry 3
+
+      - name: Install saucectl
+        run: |
+          npm i -g saucectl
+
+      - name: Log in to Azure
+        uses: Azure/login@e15b166166a8746d1a47596803bd8c1b595455cf # v1.6.0
+        with:
+          creds: ${{ secrets.AZURE_KV_CI_SERVICE_PRINCIPAL }}
+
+      - name: Retrieve secrets
+        env:
+          ACCOUNT_NAME: bitwardenci
+          CONTAINER_NAME: mobile
+        run: |
+          mkdir -p ${{ github.workspace }}/secrets
+          mkdir -p ${{ github.workspace }}/app/src/standardRelease
+
+          az storage blob download --account-name $ACCOUNT_NAME --container-name $CONTAINER_NAME \
+          --name app_play-keystore.jks --file ${{ github.workspace }}/keystores/app_play-keystore.jks --output none
+          az storage blob download --account-name $ACCOUNT_NAME --container-name $CONTAINER_NAME \
+          --name google-services.json --file ${{ github.workspace }}/app/src/standardRelease/google-services.json --output none
+
+      - name: Log in to Azure
+        uses: bitwarden/gh-actions/azure-login@main
+        with:
+          subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          tenant_id: ${{ secrets.AZURE_TENANT_ID }}
+          client_id: ${{ secrets.AZURE_CLIENT_ID }}
+
+      - name: Get E2E secrets from Azure
+        uses: bitwarden/gh-actions/get-keyvault-secrets@main
+        with:
+          keyvault: gh-android
+          secrets: "BWS-ACCESS-TOKEN, SAUCE-LABS-USERNAME, SAUCE-LABS-ACCESS-KEY"
+        id: get-e2e-secrets
+
+      - name: Log out from Azure
+        uses: bitwarden/gh-actions/azure-logout@main
+
+      - name: Retrieve test data
+        uses: bitwarden/sm-action@14f92f1d294ae3c2b6a3845d389cd2c318b0dfd8 # v2.2.0
+        with:
+          access_token: ${{ steps.get-e2e-secrets.outputs.BWS-ACCESS-TOKEN }}
+          secrets: |
+            63e93f73-5118-4a62-9db8-b3160176aa8a > TEST_ACCOUNT_CREDS
+
+      - name: Configure .json test data file
+        run: printf %s '${{ env.TEST_ACCOUNT_CREDS }}' > app/src/androidTest/assets/TestData.json
+
+      - name: Build release APK
+        env:
+          PLAY_KEYSTORE_PASSWORD: ${{ secrets.PLAY_KEYSTORE_PASSWORD }}
+        run: |
+          bundle exec fastlane assemblePlayStoreReleaseApk \
+            storeFile:app_play-keystore.jks \
+            storePassword:'${{ env.PLAY_KEYSTORE_PASSWORD }}' \
+            keyAlias:bitwarden \
+            keyPassword:'${{ env.PLAY_KEYSTORE_PASSWORD }}'
+
+      - name: Build test APK (espresso)
+        run: |
+          ./gradlew :app:assembleStandardReleaseAndroidTest
+
+      - name: Signing test APK
+        run: |
+          $ANDROID_SDK_ROOT/build-tools/34.0.0/apksigner sign \
+            --ks keystores/app_play-keystore.jks \
+            --ks-key-alias bitwarden \
+            --ks-pass pass:${{ secrets.PLAY_KEYSTORE_PASSWORD }} \
+            --key-pass pass:${{ secrets.PLAY_KEYSTORE_PASSWORD }} \
+            app/build/outputs/apk/androidTest/standard/release/com.x8bit.bitwarden-standard-release-androidTest.apk
+
+      - name: Upload app APK to SauceLabs storage
+        run: |
+          saucectl storage upload app/build/outputs/apk/standard/release/com.x8bit.bitwarden.apk
+        env:
+          SAUCE_USERNAME: ${{ steps.get-e2e-secrets.outputs.SAUCE-LABS-USERNAME }}
+          SAUCE_ACCESS_KEY: ${{ steps.get-e2e-secrets.outputs.SAUCE-LABS-ACCESS-KEY }}
+
+      - name: Upload test APK to SauceLabs storage
+        run: |
+          saucectl storage upload app/build/outputs/apk/androidTest/standard/release/com.x8bit.bitwarden-standard-release-androidTest.apk
+        env:
+          SAUCE_USERNAME: ${{ steps.get-e2e-secrets.outputs.SAUCE-LABS-USERNAME }}
+          SAUCE_ACCESS_KEY: ${{ steps.get-e2e-secrets.outputs.SAUCE-LABS-ACCESS-KEY }}
+
+      - name: Run tests on SauceLabs
+        run: saucectl run --config .sauce/config.yml
+        env:
+          SAUCE_USERNAME: ${{ steps.get-e2e-secrets.outputs.SAUCE-LABS-USERNAME }}
+          SAUCE_ACCESS_KEY: ${{ steps.get-e2e-secrets.outputs.SAUCE-LABS-ACCESS-KEY }}
+
+      - name: Upload SauceLabs test report
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: saucectl-report
+          path: saucectl-report.xml

.sauce/config.yml

@@ -0,0 +1,32 @@
+apiVersion: v1alpha
+kind: espresso
+defaults:
+  timeout: 10m
+sauce:
+  region: us-west-1
+  # Controls how many suites are executed at the same time (sauce test env only).
+  concurrency: 1
+  retries: 1
+  visibility: team
+  metadata:
+    tags:
+      - Android
+      - sanity-e2e
+    build: Sanity check on Real devices
+reporters:
+  junit:
+    enabled: true
+    filename: saucectl-report.xml
+espresso:
+  app: storage:filename=com.x8bit.bitwarden.apk
+  testApp: storage:filename=com.x8bit.bitwarden-standard-release-androidTest.apk
+suites:
+  - name: "Android - Sanity"
+    devices:
+      - name: "Google.*"
+        platformVersion: "^1[3456].*"
+        options:
+          deviceType: PHONE
+    testOptions:
+      package: e2e.tests
+      resigningEnabled: false

app/build.gradle.kts

@@ -47,6 +47,8 @@ android {
     namespace = "com.x8bit.bitwarden"
     compileSdk = libs.versions.compileSdk.get().toInt()
 
+    testBuildType = "release"
+
     room {
         schemaDirectory("$projectDir/schemas")
     }
@@ -248,6 +250,10 @@ dependencies {
     implementation(libs.androidx.lifecycle.runtime.compose)
     implementation(libs.androidx.lifecycle.runtime.ktx)
     implementation(libs.androidx.navigation.compose)
+    implementation(libs.androidx.uiautomator)
+    implementation(libs.androidx.espresso.core)
+    implementation(libs.androidx.junit.ktx)
+    implementation(libs.androidx.ui.test.junit4.android)
     ksp(libs.androidx.room.compiler)
     implementation(libs.androidx.room.ktx)
     implementation(libs.androidx.room.runtime)
@@ -284,7 +290,6 @@ dependencies {
     testImplementation(testFixtures(project(":network")))
     testImplementation(testFixtures(project(":ui")))
 
-    testImplementation(libs.androidx.compose.ui.test)
     testImplementation(libs.google.hilt.android.testing)
     testImplementation(platform(libs.junit.bom))
     testRuntimeOnly(libs.junit.platform.launcher)
@@ -294,6 +299,11 @@ dependencies {
     testImplementation(libs.mockk.mockk)
     testImplementation(libs.robolectric.robolectric)
     testImplementation(libs.square.turbine)
+    androidTestImplementation(libs.androidx.uiautomator)
+    androidTestImplementation(libs.androidx.espresso.core)
+    androidTestImplementation(libs.androidx.junit.ktx)
+    androidTestImplementation(libs.androidx.ui.test.junit4.android)
+    androidTestImplementation(libs.androidx.compose.ui.test)
 }
 
 tasks {