Run end to end tests on more devices for nightly builds.

.github/workflows/instrumentation_tests.yml

@@ -14,7 +14,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/stripe_setup
       - name: 'Build'
-        run: ./gradlew :paymentsheet-example:assembleDebugAndroidTest :paymentsheet-example:assembleDebug
+        run: ./gradlew :paymentsheet-example:assembleDebugAndroidTest :paymentsheet-example:assembleDebug -PIS_NIGHTLY_BUILD=true
 
       - name: 'Install pip modules'
         run: pip install requests_toolbelt requests
@@ -23,7 +23,7 @@ jobs:
         env:
           BROWSERSTACK_USERNAME: ${{ secrets.BROWSERSTACK_USERNAME }}
           BROWSERSTACK_ACCESS_KEY: ${{ secrets.BROWSERSTACK_ACCESS_KEY }}
-        run: python scripts/browserstack.py --test --apk paymentsheet-example/build/outputs/apk/debug/paymentsheet-example-debug.apk --espresso paymentsheet-example/build/outputs/apk/androidTest/debug/paymentsheet-example-debug-androidTest.apk --is-nightly --num-retries 2
+        run: python scripts/browserstack.py --test --apk paymentsheet-example/build/outputs/apk/debug/paymentsheet-example-debug.apk --espresso paymentsheet-example/build/outputs/apk/androidTest/debug/paymentsheet-example-debug-androidTest.apk --is-nightly --num-retries 3
 
       - name: Notify failure endpoint
         id: notifyFailureEndpoint

build-configuration/build-environment.gradle

@@ -1,2 +1,4 @@
 gradle.ext.isCi = System.getenv().containsKey("CI") && System.getenv("CI").contains("true")
 println "Running on CI: ${gradle.ext.isCi}"
+gradle.ext.isNightlyBuild = project.hasProperty("IS_NIGHTLY_BUILD") && project.IS_NIGHTLY_BUILD.contains("true")
+println "IS_NIGHTLY_BUILD: ${gradle.ext.isNightlyBuild}"

paymentsheet-example/build.gradle

@@ -86,6 +86,7 @@ android {
         testInstrumentationRunner "com.karumi.shot.ShotTestRunner"
 
         buildConfigField "boolean", "IS_RUNNING_IN_CI", "$gradle.ext.isCi"
+        buildConfigField "boolean", "IS_NIGHTLY_BUILD", "$gradle.ext.isNightlyBuild"
 
         manifestPlaceholders = [
                 BACKEND_URL: getBackendUrl(),

paymentsheet-example/src/androidTest/java/com/stripe/android/lpm/TestBacs.kt

@@ -60,13 +60,13 @@ internal class TestBacs : BasePlaygroundTest() {
     ): TestParameters {
         return TestParameters.create(
             paymentMethodCode = "bacs_debit",
+            authorizationAction = bacsAuthAction,
+            executeInNightlyRun = true,
         ) { settings ->
             settings[AutomaticPaymentMethodsSettingsDefinition] = true
             settings[DelayedPaymentMethodsSettingsDefinition] = true
             settings[CountrySettingsDefinition] = Country.GB
             settings[CurrencySettingsDefinition] = Currency.GBP
-        }.copy(
-            authorizationAction = bacsAuthAction
-        )
+        }
     }
 }

paymentsheet-example/src/androidTest/java/com/stripe/android/lpm/TestBoleto.kt

@@ -23,6 +23,7 @@ import org.junit.runner.RunWith
 internal class TestBoleto : BasePlaygroundTest() {
     private val testParameters = TestParameters.create(
         paymentMethodCode = "boleto",
+        executeInNightlyRun = true,
     ) { settings ->
         settings[CustomerSettingsDefinition] = CustomerType.GUEST
         settings[CountrySettingsDefinition] = Country.BR

paymentsheet-example/src/androidTest/java/com/stripe/android/lpm/TestCard.kt

@@ -23,15 +23,18 @@ import org.junit.runner.RunWith
 
 @RunWith(AndroidJUnit4::class)
 internal class TestCard : BasePlaygroundTest() {
+    private val testParameters = TestParameters.create(
+        paymentMethodCode = "card",
+        authorizationAction = null,
+        executeInNightlyRun = true,
+    ).copy(
+        saveForFutureUseCheckboxVisible = true,
+    )
+
     @Test
     fun testCard() {
         testDriver.confirmNewOrGuestComplete(
-            TestParameters.create(
-                paymentMethodCode = "card",
-            ).copy(
-                authorizationAction = null,
-                saveForFutureUseCheckboxVisible = true,
-            ),
+            testParameters,
             populateCustomLpmFields = {
                 populateCardDetails()
             },
@@ -41,9 +44,7 @@ internal class TestCard : BasePlaygroundTest() {
     @Test
     fun testCardWithCustomBillingDetailsCollection() {
         testDriver.confirmNewOrGuestComplete(
-            TestParameters.create(
-                paymentMethodCode = "card",
-            ) { settings ->
+            testParameters.copyPlaygroundSettings { settings ->
                 settings[DefaultBillingAddressSettingsDefinition] = DefaultBillingAddress.Off
                 settings[CollectNameSettingsDefinition] =
                     PaymentSheet.BillingDetailsCollectionConfiguration.CollectionMode.Always
@@ -53,10 +54,7 @@ internal class TestCard : BasePlaygroundTest() {
                     PaymentSheet.BillingDetailsCollectionConfiguration.CollectionMode.Always
                 settings[CollectAddressSettingsDefinition] =
                     PaymentSheet.BillingDetailsCollectionConfiguration.AddressCollectionMode.Full
-            }.copy(
-                authorizationAction = null,
-                saveForFutureUseCheckboxVisible = true,
-            ),
+            },
             populateCustomLpmFields = {
                 populateCardDetails()
                 populateEmail()
@@ -70,9 +68,7 @@ internal class TestCard : BasePlaygroundTest() {
     @Test
     fun testCardWithCustomBillingDetailsCollectionWithDefaults() {
         testDriver.confirmNewOrGuestComplete(
-            TestParameters.create(
-                paymentMethodCode = "card",
-            ) { settings ->
+            testParameters.copyPlaygroundSettings { settings ->
                 settings[DefaultBillingAddressSettingsDefinition] = DefaultBillingAddress.On
                 settings[CollectNameSettingsDefinition] =
                     PaymentSheet.BillingDetailsCollectionConfiguration.CollectionMode.Always
@@ -82,10 +78,7 @@ internal class TestCard : BasePlaygroundTest() {
                     PaymentSheet.BillingDetailsCollectionConfiguration.CollectionMode.Always
                 settings[CollectAddressSettingsDefinition] =
                     PaymentSheet.BillingDetailsCollectionConfiguration.AddressCollectionMode.Full
-            }.copy(
-                authorizationAction = null,
-                saveForFutureUseCheckboxVisible = true,
-            ),
+            },
             populateCustomLpmFields = {
                 populateCardDetails()
             },
@@ -95,12 +88,7 @@ internal class TestCard : BasePlaygroundTest() {
     @Test
     fun testCardInCustomFlow() {
         testDriver.confirmCustom(
-            TestParameters.create(
-                paymentMethodCode = "card",
-            ).copy(
-                authorizationAction = null,
-                saveForFutureUseCheckboxVisible = true,
-            ),
+            testParameters,
             populateCustomLpmFields = {
                 populateCardDetails()
             },
@@ -118,11 +106,7 @@ internal class TestCard : BasePlaygroundTest() {
     @Test
     fun testDefaultSavedPaymentMethodUsedAfterSingleSave() {
         val cardNumber = "6011111111111117"
-        val testParameters = TestParameters.create(
-            paymentMethodCode = "card",
-        ).copy(
-            authorizationAction = null,
-            saveForFutureUseCheckboxVisible = true,
+        val testParameters = testParameters.copy(
             saveCheckboxValue = true,
         )
 
@@ -160,11 +144,7 @@ internal class TestCard : BasePlaygroundTest() {
         val firstCardNumber = "6011111111111117"
         val secondCardNumber = "6011000990139424"
 
-        val testParameters = TestParameters.create(
-            paymentMethodCode = "card",
-        ).copy(
-            authorizationAction = null,
-            saveForFutureUseCheckboxVisible = true,
+        val testParameters = testParameters.copy(
             saveCheckboxValue = true,
         )
 
@@ -212,11 +192,7 @@ internal class TestCard : BasePlaygroundTest() {
     fun testDefaultSavedPaymentMethodUsedAfterSingleSaveInCustomFlow() {
         val cardNumber = "6011111111111117"
 
-        val testParameters = TestParameters.create(
-            paymentMethodCode = "card",
-        ).copy(
-            authorizationAction = null,
-            saveForFutureUseCheckboxVisible = true,
+        val testParameters = testParameters.copy(
             saveCheckboxValue = true,
         )
 
@@ -253,11 +229,7 @@ internal class TestCard : BasePlaygroundTest() {
         val cardNumber = "6011111111111117"
         val secondCardNumber = "6011000990139424"
 
-        val testParameters = TestParameters.create(
-            paymentMethodCode = "card",
-        ).copy(
-            authorizationAction = null,
-            saveForFutureUseCheckboxVisible = true,
+        val testParameters = testParameters.copy(
             saveCheckboxValue = true,
         )
 
@@ -300,11 +272,8 @@ internal class TestCard : BasePlaygroundTest() {
     @Test
     fun testCardWith3ds2() {
         testDriver.confirmNewOrGuestComplete(
-            TestParameters.create(
-                paymentMethodCode = "card",
-            ).copy(
+            testParameters.copy(
                 authorizationAction = AuthorizeAction.Authorize3ds2,
-                saveForFutureUseCheckboxVisible = true,
             ),
             populateCustomLpmFields = {
                 populateCardDetails()

paymentsheet-example/src/androidTest/java/com/stripe/android/lpm/TestCardInCustomerSheet.kt

@@ -14,13 +14,19 @@ import org.junit.runner.RunWith
 
 @RunWith(AndroidJUnit4::class)
 internal class TestCardInCustomerSheet : BasePlaygroundTest() {
+    private val testParameters = TestParameters.create(
+        paymentMethodCode = "card",
+        executeInNightlyRun = true,
+    ).copyPlaygroundSettings { settings ->
+        settings[CustomerSettingsDefinition] = CustomerType.NEW
+        settings[CustomerSheetPaymentMethodModeDefinition] = PaymentMethodMode.CreateAndAttach
+    }
+
     @Test
     fun testCard() {
         testDriver.savePaymentMethodInCustomerSheet(
-            TestParameters.create(paymentMethodCode = "card").copyPlaygroundSettings { settings ->
-                settings[CustomerSettingsDefinition] = CustomerType.NEW
+            testParameters.copyPlaygroundSettings { settings ->
                 settings[CountrySettingsDefinition] = Country.US
-                settings[CustomerSheetPaymentMethodModeDefinition] = PaymentMethodMode.CreateAndAttach
             },
             populateCustomLpmFields = {
                 populateCardDetails()
@@ -31,10 +37,8 @@ internal class TestCardInCustomerSheet : BasePlaygroundTest() {
     @Test
     fun testCardWithNonUsMerchant() {
         testDriver.savePaymentMethodInCustomerSheet(
-            TestParameters.create(paymentMethodCode = "card").copyPlaygroundSettings { settings ->
-                settings[CustomerSettingsDefinition] = CustomerType.NEW
+            testParameters.copyPlaygroundSettings { settings ->
                 settings[CountrySettingsDefinition] = Country.FR
-                settings[CustomerSheetPaymentMethodModeDefinition] = PaymentMethodMode.CreateAndAttach
             },
             populateCustomLpmFields = {
                 populateCardDetails()

paymentsheet-example/src/androidTest/java/com/stripe/android/lpm/TestCashApp.kt

@@ -20,6 +20,7 @@ internal class TestCashApp : BasePlaygroundTest() {
 
     private val testParameters = TestParameters.create(
         paymentMethodCode = "cashapp",
+        executeInNightlyRun = true,
     ) { settings ->
         settings[CountrySettingsDefinition] = Country.US
         settings[CurrencySettingsDefinition] = Currency.USD

paymentsheet-example/src/androidTest/java/com/stripe/android/lpm/TestExternalPaymentMethod.kt

@@ -18,6 +18,7 @@ internal class TestExternalPaymentMethod : BasePlaygroundTest() {
 
     private val testParameters = TestParameters.create(
         paymentMethodCode = externalFawryCode,
+        executeInNightlyRun = true,
     ) { settings ->
         settings[ExternalPaymentMethodSettingsDefinition] = ExternalPaymentMethodType.Fawry
         settings[PaymentMethodOrderSettingsDefinition] = externalFawryCode

paymentsheet-example/src/androidTest/java/com/stripe/android/lpm/TestKlarna.kt

@@ -17,6 +17,7 @@ import org.junit.runner.RunWith
 internal class TestKlarna : BasePlaygroundTest() {
     private val testParameters = TestParameters.create(
         paymentMethodCode = "klarna",
+        executeInNightlyRun = true,
     ) { settings ->
         settings[CountrySettingsDefinition] = Country.US
         settings[CurrencySettingsDefinition] = Currency.USD

paymentsheet-example/src/androidTest/java/com/stripe/android/lpm/TestUSBankAccount.kt

@@ -20,6 +20,7 @@ import org.junit.runner.RunWith
 internal class TestUSBankAccount : BasePlaygroundTest() {
     private val testParameters = TestParameters.create(
         paymentMethodCode = "us_bank_account",
+        executeInNightlyRun = true,
     ) { settings ->
         settings[CountrySettingsDefinition] = Country.US
         settings[CurrencySettingsDefinition] = Currency.USD

paymentsheet-example/src/androidTest/java/com/stripe/android/test/core/PlaygroundTestDriver.kt

@@ -29,6 +29,7 @@ import androidx.test.espresso.matcher.ViewMatchers.withText
 import androidx.test.uiautomator.UiDevice
 import com.google.common.truth.Truth.assertThat
 import com.karumi.shot.ScreenshotTest
+import com.stripe.android.paymentsheet.example.BuildConfig
 import com.stripe.android.paymentsheet.example.playground.PaymentSheetPlaygroundActivity
 import com.stripe.android.paymentsheet.example.playground.PlaygroundState
 import com.stripe.android.paymentsheet.example.playground.SUCCESS_RESULT
@@ -48,6 +49,8 @@ import com.stripe.android.test.core.ui.clickTextInWebView
 import kotlinx.coroutines.launch
 import org.junit.Assert.fail
 import org.junit.Assume
+import org.junit.Assume.assumeFalse
+import org.junit.Assume.assumeTrue
 import java.util.concurrent.CountDownLatch
 import java.util.concurrent.TimeUnit
 import kotlin.time.Duration
@@ -865,7 +868,7 @@ internal class PlaygroundTestDriver(
         }
         if (authorizeAction == AuthorizeAction.DisplayQrCode) {
             // Browserstack tests fail on pixel 2 API 26.
-            Assume.assumeFalse("walleye + 26" == "${Build.DEVICE} + ${Build.VERSION.SDK_INT}")
+            assumeFalse("walleye + 26" == "${Build.DEVICE} + ${Build.VERSION.SDK_INT}")
         }
     }
 
@@ -1080,6 +1083,15 @@ internal class PlaygroundTestDriver(
     }
 
     internal fun setup(testParameters: TestParameters) {
+        if (BuildConfig.IS_NIGHTLY_BUILD) {
+            assumeTrue(testParameters.executeInNightlyRun)
+        }
+
+        if (Build.VERSION.SDK_INT <= 28) {
+            val unsupportedAuthorizeActions = setOf(AuthorizeAction.Authorize3ds2, AuthorizeAction.DisplayQrCode)
+            assumeFalse(unsupportedAuthorizeActions.contains(testParameters.authorizationAction))
+        }
+
         this.testParameters = testParameters
         this.selectors = Selectors(device, composeTestRule, testParameters)
 

paymentsheet-example/src/androidTest/java/com/stripe/android/test/core/TestParameters.kt

@@ -22,6 +22,7 @@ internal data class TestParameters(
     val saveForFutureUseCheckboxVisible: Boolean,
     val useBrowser: Browser? = null,
     val authorizationAction: AuthorizeAction? = null,
+    val executeInNightlyRun: Boolean = false,
     val playgroundSettingsSnapshot: PlaygroundSettings.Snapshot = playgroundSettings().snapshot(),
 ) {
     fun copyPlaygroundSettings(block: (PlaygroundSettings) -> Unit): TestParameters {
@@ -34,14 +35,16 @@ internal data class TestParameters(
         fun create(
             paymentMethodCode: String,
             requiresBrowser: Boolean = true,
-            authorizationAction: AuthorizeAction = AuthorizeAction.AuthorizePayment(requiresBrowser),
-            playgroundSettingsBlock: (PlaygroundSettings) -> Unit = {}
+            authorizationAction: AuthorizeAction? = AuthorizeAction.AuthorizePayment(requiresBrowser),
+            executeInNightlyRun: Boolean = false,
+            playgroundSettingsBlock: (PlaygroundSettings) -> Unit = {},
         ): TestParameters {
             return TestParameters(
                 paymentMethodCode = paymentMethodCode,
                 saveCheckboxValue = false,
                 saveForFutureUseCheckboxVisible = false,
                 authorizationAction = authorizationAction,
+                executeInNightlyRun = executeInNightlyRun,
                 playgroundSettingsSnapshot = playgroundSettings(playgroundSettingsBlock).snapshot()
             )
         }

scripts/browserstack.py

@@ -184,7 +184,7 @@ def getAllTestClassNames():
 
 def testShards(isNightly, testClassNames):
     # We only have 25 parallel runs, and we want multiple PRs to run at the same time.
-    numberOfShards = 8.0 if isNightly else 10.0
+    numberOfShards = 2.0 if isNightly else 10.0
     testClassesPerShard = math.ceil(len(testClassNames) / numberOfShards)
 
     shards = []
@@ -229,8 +229,15 @@ def executeTests(appUrl, testUrl, isNightly, testClasses):
     devices = []
     if isNightly:
         devices = [
+            "Google Pixel 8-14.0",
             "Google Pixel 7-13.0",
             "Samsung Galaxy S22-12.0",
+            "Google Pixel 5-11.0",
+            "Google Pixel 4 XL-10.0",
+            "Google Pixel 3-9.0",
+            "Samsung Galaxy S9-8.0",
+            "Samsung Galaxy S8-7.0",
+            "Samsung Galaxy S7-6.0",
         ]
     else:
         devices = [