tests now search the envelope among all the ones sent to the mock relay server and make assertion on them, instead of relying on the order the envelopes were created (#2422)

stefanosiano · web-flow · commit 703d523e46bf · 2022-12-09T12:33:09.000+01:00
* ui tests now search the envelope among all the ones sent to the mock relay server and make assertion on them, instead of relying on the order the envelopes were created
* added androidx test orchestrator to saucelabs configuration with "clearPackageData" option enabled
* updated androidx orchestrator dependency
diff --git a/.sauce/sentry-uitest-android-benchmark-lite.yml b/.sauce/sentry-uitest-android-benchmark-lite.yml
@@ -19,6 +19,9 @@ espresso:
 suites:
 
   - name: "Android 11 (api 30)"
+    testOptions:
+      clearPackageData: true
+      useTestOrchestrator: true
     devices:
       - id: Google_Pixel_3a_real # Google Pixel 3a - api 30 (11)
 
diff --git a/.sauce/sentry-uitest-android-benchmark.yml b/.sauce/sentry-uitest-android-benchmark.yml
@@ -20,17 +20,26 @@ suites:
 
   # Devices are chosen so that there is a high-end and a low-end device for each api level
   - name: "Android 12 (api 31)"
+    testOptions:
+      clearPackageData: true
+      useTestOrchestrator: true
     devices:
       - id: Google_Pixel_6_Pro_real_us # Google Pixel 6 Pro - api 31 (12) - high end
       - id: Google_Pixel_6a_real_us # Google Pixel 6a - api 31 (12) - low end
 
   - name: "Android 11 (api 30)"
+    testOptions:
+      clearPackageData: true
+      useTestOrchestrator: true
     devices:
       - id: OnePlus_9_Pro_real_us # OnePlus 9 Pro - api 30 (11) - high end
       - id: Google_Pixel_4_real_us # Google Pixel 4 - api 30 (11) - mid end
       - id: Google_Pixel_3a_real # Google Pixel 3a - api 30 (11) - low end
 
   - name: "Android 10 (api 29)"
+    testOptions:
+      clearPackageData: true
+      useTestOrchestrator: true
     devices:
       - id: Google_Pixel_4_XL_real_us1 # Google Pixel 4 XL - api 29 (10)
       - id: Nokia_7_1_real_us # Nokia 7.1 - api 29 (10)
diff --git a/.sauce/sentry-uitest-android-ui.yml b/.sauce/sentry-uitest-android-ui.yml
@@ -19,18 +19,30 @@ espresso:
 suites:
 
   - name: "Android 13 Ui test (api 33)"
+    testOptions:
+      clearPackageData: true
+      useTestOrchestrator: true
     devices:
       - id: Google_Pixel_5_13_real_us # Google Pixel 5 - api 33 (13)
 
   - name: "Android 12 Ui test (api 31)"
+    testOptions:
+      clearPackageData: true
+      useTestOrchestrator: true
     devices:
       - id: Samsung_Galaxy_S22_Ultra_5G_real_us # Samsung Galaxy S22 Ultra 5G - api 31 (12)
 
   - name: "Android 11 Ui test (api 30)"
+    testOptions:
+      clearPackageData: true
+      useTestOrchestrator: true
     devices:
       - id: OnePlus_9_Pro_real_us # OnePlus 9 Pro - api 30 (11)
 
   - name: "Android 10 Ui test (api 29)"
+    testOptions:
+      clearPackageData: true
+      useTestOrchestrator: true
     devices:
       - id: OnePlus_7T_real_us # OnePlus 7T - api 29 (10)
 
diff --git a/buildSrc/src/main/java/Config.kt b/buildSrc/src/main/java/Config.kt
@@ -167,7 +167,7 @@ object Config {
         val androidxTestRules = "androidx.test:rules:$androidxTestVersion"
         val espressoCore = "androidx.test.espresso:espresso-core:$espressoVersion"
         val espressoIdlingResource = "androidx.test.espresso:espresso-idling-resource:$espressoVersion"
-        val androidxTestOrchestrator = "androidx.test:orchestrator:1.4.1"
+        val androidxTestOrchestrator = "androidx.test:orchestrator:1.4.2"
         val androidxJunit = "androidx.test.ext:junit:1.1.3"
         val androidxCoreKtx = "androidx.core:core-ktx:1.7.0"
         val robolectric = "org.robolectric:robolectric:4.7.3"
diff --git a/sentry-android-integration-tests/sentry-uitest-android-benchmark/build.gradle.kts b/sentry-android-integration-tests/sentry-uitest-android-benchmark/build.gradle.kts
@@ -26,8 +26,11 @@ android {
         // https://developer.android.com/training/testing/instrumented-tests/androidx-test-libraries/runner#enable-gradle
         // This doesn't work on some devices with Android 11+. Clearing package data resets permissions.
         // Check the readme for more info.
-        // Test orchestrator was removed due to issues with SauceLabs
-//        testInstrumentationRunnerArguments["clearPackageData"] = "true"
+        testInstrumentationRunnerArguments["clearPackageData"] = "true"
+    }
+
+    testOptions {
+        execution = "ANDROIDX_TEST_ORCHESTRATOR"
     }
 
     buildFeatures {
diff --git a/sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts b/sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts
@@ -25,8 +25,11 @@ android {
         // https://developer.android.com/training/testing/instrumented-tests/androidx-test-libraries/runner#enable-gradle
         // This doesn't work on some devices with Android 11+. Clearing package data resets permissions.
         // Check the readme for more info.
-        // Test orchestrator was removed due to issues with SauceLabs
-//        testInstrumentationRunnerArguments["clearPackageData"] = "true"
+        testInstrumentationRunnerArguments["clearPackageData"] = "true"
+    }
+
+    testOptions {
+        execution = "ANDROIDX_TEST_ORCHESTRATOR"
     }
 
     buildFeatures {
diff --git a/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/EnvelopeTests.kt
@@ -12,6 +12,7 @@ import io.sentry.ProfilingTraceData
 import io.sentry.Sentry
 import io.sentry.SentryEvent
 import io.sentry.android.core.SentryAndroidOptions
+import io.sentry.assertEnvelopeItem
 import io.sentry.profilemeasurements.ProfileMeasurement
 import io.sentry.protocol.SentryTransaction
 import org.junit.runner.RunWith
@@ -34,7 +35,7 @@ class EnvelopeTests : BaseUiTest() {
         Sentry.captureMessage("Message captured during test")
 
         relay.assert {
-            assertEnvelope {
+            assertFirstEnvelope {
                 val event: SentryEvent = it.assertItem()
                 it.assertNoOtherItems()
                 assertTrue(event.message?.formatted == "Message captured during test")
@@ -53,7 +54,7 @@ class EnvelopeTests : BaseUiTest() {
 
         relayIdlingResource.increment()
         IdlingRegistry.getInstance().register(ProfilingSampleActivity.scrollingIdlingResource)
-        val transaction = Sentry.startTransaction("e2etests", "test1")
+        val transaction = Sentry.startTransaction("profiledTransaction", "test1")
         val sampleScenario = launchActivity<ProfilingSampleActivity>()
         swipeList(1)
         sampleScenario.moveToState(Lifecycle.State.DESTROYED)
@@ -63,21 +64,29 @@ class EnvelopeTests : BaseUiTest() {
 
         transaction.finish()
         relay.assert {
-            assertEnvelope {
+            findEnvelope {
+                assertEnvelopeItem<SentryTransaction>(it.items.toList()).transaction == "ProfilingSampleActivity"
+            }.assert {
                 val transactionItem: SentryTransaction = it.assertItem()
                 it.assertNoOtherItems()
                 assertEquals("ProfilingSampleActivity", transactionItem.transaction)
             }
-            assertEnvelope {
+
+            findEnvelope {
+                assertEnvelopeItem<SentryTransaction>(it.items.toList()).transaction == "profiledTransaction"
+            }.assert {
                 val transactionItem: SentryTransaction = it.assertItem()
                 it.assertNoOtherItems()
-                assertEquals("e2etests", transactionItem.transaction)
+                assertEquals("profiledTransaction", transactionItem.transaction)
             }
-            assertEnvelope {
+
+            findEnvelope {
+                assertEnvelopeItem<ProfilingTraceData>(it.items.toList()).transactionName == "profiledTransaction"
+            }.assert {
                 val profilingTraceData: ProfilingTraceData = it.assertItem()
                 it.assertNoOtherItems()
                 assertEquals(profilingTraceData.transactionId, transaction.eventId.toString())
-                assertEquals("e2etests", profilingTraceData.transactionName)
+                assertEquals("profiledTransaction", profilingTraceData.transactionName)
                 assertTrue(profilingTraceData.environment.isNotEmpty())
                 assertTrue(profilingTraceData.cpuArchitecture.isNotEmpty())
                 assertTrue(profilingTraceData.transactions.isNotEmpty())
@@ -127,24 +136,31 @@ class EnvelopeTests : BaseUiTest() {
         transaction3.finish()
 
         relay.assert {
-            assertEnvelope {
+            findEnvelope {
+                assertEnvelopeItem<SentryTransaction>(it.items.toList()).transaction == "e2etests"
+            }.assert {
                 val transactionItem: SentryTransaction = it.assertItem()
                 it.assertNoOtherItems()
                 assertEquals(transaction.eventId.toString(), transactionItem.eventId.toString())
             }
-            assertEnvelope {
+            findEnvelope {
+                assertEnvelopeItem<SentryTransaction>(it.items.toList()).transaction == "e2etests1"
+            }.assert {
                 val transactionItem: SentryTransaction = it.assertItem()
                 it.assertNoOtherItems()
                 assertEquals(transaction2.eventId.toString(), transactionItem.eventId.toString())
             }
-            // The profile is sent only in the last transaction envelope
-            assertEnvelope {
+            findEnvelope {
+                assertEnvelopeItem<SentryTransaction>(it.items.toList()).transaction == "e2etests2"
+            }.assert {
                 val transactionItem: SentryTransaction = it.assertItem()
                 it.assertNoOtherItems()
                 assertEquals(transaction3.eventId.toString(), transactionItem.eventId.toString())
             }
-            // The profile is sent only in the last transaction envelope
-            assertEnvelope {
+            // The profile is sent in its own envelope
+            findEnvelope {
+                assertEnvelopeItem<ProfilingTraceData>(it.items.toList()).transactionName == "e2etests2"
+            }.assert {
                 val profilingTraceData: ProfilingTraceData = it.assertItem()
                 it.assertNoOtherItems()
                 assertEquals("e2etests2", profilingTraceData.transactionName)
@@ -191,7 +207,7 @@ class EnvelopeTests : BaseUiTest() {
         relayIdlingResource.increment()
         relayIdlingResource.increment()
         val profilesDirPath = Sentry.getCurrentHub().options.profilingTracesDirPath
-        val transaction = Sentry.startTransaction("e2etests", "test empty")
+        val transaction = Sentry.startTransaction("emptyProfileTransaction", "test empty")
 
         var finished = false
         Thread {
@@ -209,13 +225,15 @@ class EnvelopeTests : BaseUiTest() {
         }
 
         relay.assert {
-            assertEnvelope {
+            findEnvelope {
+                assertEnvelopeItem<SentryTransaction>(it.items.toList()).transaction == "emptyProfileTransaction"
+            }.assert {
                 val transactionItem: SentryTransaction = it.assertItem()
                 it.assertNoOtherItems()
-                assertEquals("e2etests", transactionItem.transaction)
+                assertEquals("emptyProfileTransaction", transactionItem.transaction)
             }
             // The profile failed to be sent. Trying to read the envelope from the data transmitted throws an exception
-            assertFailsWith<IllegalArgumentException> { assertEnvelope {} }
+            assertFailsWith<IllegalArgumentException> { assertFirstEnvelope {} }
             assertNoOtherEnvelopes()
             assertNoOtherRequests()
         }
@@ -230,14 +248,16 @@ class EnvelopeTests : BaseUiTest() {
             options.profilesSampleRate = 1.0
         }
         relayIdlingResource.increment()
-        Sentry.startTransaction("e2etests", "testTimeout")
+        Sentry.startTransaction("timedOutProfile", "testTimeout")
         // We don't call transaction.finish() and let the timeout do its job
 
         relay.assert {
-            assertEnvelope {
+            findEnvelope {
+                assertEnvelopeItem<ProfilingTraceData>(it.items.toList()).transactionName == "timedOutProfile"
+            }.assert {
                 val profilingTraceData: ProfilingTraceData = it.assertItem()
                 it.assertNoOtherItems()
-                assertEquals("e2etests", profilingTraceData.transactionName)
+                assertEquals("timedOutProfile", profilingTraceData.transactionName)
                 assertEquals(ProfilingTraceData.TRUNCATION_REASON_TIMEOUT, profilingTraceData.truncationReason)
             }
             assertNoOtherEnvelopes()
diff --git a/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/RelayAsserter.kt b/sentry-android-integration-tests/sentry-uitest-android/src/androidTest/java/io/sentry/uitest/android/mockservers/RelayAsserter.kt
@@ -2,8 +2,10 @@ package io.sentry.uitest.android.mockservers
 
 import io.sentry.EnvelopeReader
 import io.sentry.Sentry
+import io.sentry.SentryEnvelope
 import okhttp3.mockwebserver.MockResponse
 import okhttp3.mockwebserver.RecordedRequest
+import java.io.IOException
 import java.util.zip.GZIPInputStream
 
 /** Class used to assert requests sent to [MockRelay]. */
@@ -13,19 +15,17 @@ class RelayAsserter(
 ) {
 
     /**
-     * Asserts an envelope request exists and allows to make other assertions on it and its response.
-     * The asserted envelope request is then removed from internal list of unasserted envelope.
+     * Asserts an envelope request exists and allows to make other assertions on the first one and on its response.
+     * The asserted envelope request is then removed from internal list of unasserted envelopes.
      */
-    fun assertRawEnvelope(assertion: ((request: RecordedRequest, response: MockResponse) -> Unit)? = null) {
+    fun assertFirstRawEnvelope(assertion: ((relayResponse: RelayResponse) -> Unit)? = null) {
         val relayResponse = unassertedEnvelopes.removeFirstOrNull()
             ?: throw AssertionError("No envelope request found")
-        assertion?.let {
-            it(relayResponse.request, relayResponse.response)
-        }
+        assertion?.let { it(relayResponse) }
     }
 
     /**
-     * Asserts a request exists and makes other assertions on it and its response.
+     * Asserts a request exists and makes other assertions on the first one and on its response.
      * The asserted request is then removed from internal list of unasserted requests.
      */
     fun assertRawRequest(assertion: ((request: RecordedRequest, response: MockResponse) -> Unit)? = null) {
@@ -37,19 +37,27 @@ class RelayAsserter(
     }
 
     /**
-     * Asserts a request exists, parses it as an envelope and makes other assertions through a [EnvelopeAsserter].
+     * Parses the first request as an envelope and makes other assertions through a [EnvelopeAsserter].
      * The asserted envelope is then removed from internal list of unasserted envelopes.
      */
-    fun assertEnvelope(assertion: (asserter: EnvelopeAsserter) -> Unit) {
-        assertRawEnvelope { request, response ->
-            // Parse the request to rebuild the original envelope. If it fails we throw an assertion error.
-            val envelope = EnvelopeReader(Sentry.getCurrentHub().options.serializer)
-                .read(GZIPInputStream(request.body.inputStream()))
-                ?: throw AssertionError("Was unable to parse the request as an envelope: $request")
-            assertion(EnvelopeAsserter(envelope, response))
+    fun assertFirstEnvelope(assertion: (asserter: EnvelopeAsserter) -> Unit) {
+        assertFirstRawEnvelope { relayResponse ->
+            relayResponse.assert(assertion)
         }
     }
 
+    /**
+     * Returns the first request that can be parsed as an envelope and that satisfies [filter].
+     * Throws an [AssertionError] if the envelope was not found.
+     */
+    fun findEnvelope(
+        filter: (envelope: SentryEnvelope) -> Boolean = { true }
+    ): RelayResponse {
+        val relayResponseIndex = unassertedEnvelopes.indexOfFirst { it.envelope?.let(filter) ?: false }
+        if (relayResponseIndex == -1) throw AssertionError("No envelope request found with specified filter")
+        return unassertedEnvelopes.removeAt(relayResponseIndex)
+    }
+
     /** Asserts no other envelopes were sent. */
     fun assertNoOtherEnvelopes() {
         if (unassertedEnvelopes.isNotEmpty()) {
@@ -71,5 +79,24 @@ class RelayAsserter(
         assertNoOtherRawRequests()
     }
 
-    data class RelayResponse(val request: RecordedRequest, val response: MockResponse)
+    data class RelayResponse(val request: RecordedRequest, val response: MockResponse) {
+
+        /** Request parsed as envelope. */
+        val envelope: SentryEnvelope? by lazy {
+            try {
+                EnvelopeReader(Sentry.getCurrentHub().options.serializer)
+                    .read(GZIPInputStream(request.body.inputStream()))
+            } catch (e: IOException) {
+                null
+            }
+        }
+
+        /** Run [assertion] on this request parsed as an envelope. */
+        fun assert(assertion: (asserter: EnvelopeAsserter) -> Unit) {
+            // Parse the request to rebuild the original envelope. If it fails we throw an assertion error.
+            envelope?.let {
+                assertion(EnvelopeAsserter(it, response))
+            } ?: throw AssertionError("Was unable to parse the request as an envelope: $request")
+        }
+    }
 }
diff --git a/sentry-test-support/src/main/kotlin/io/sentry/Assertions.kt b/sentry-test-support/src/main/kotlin/io/sentry/Assertions.kt
@@ -32,7 +32,10 @@ fun checkTransaction(predicate: (SentryTransaction) -> Unit): SentryEnvelope {
 /**
  * Asserts an envelope item of [T] exists in [items] and returns the first one. Otherwise it throws an [AssertionError].
  */
-inline fun <reified T> assertEnvelopeItem(items: List<SentryEnvelopeItem>, predicate: (index: Int, item: T) -> Unit): T {
+inline fun <reified T> assertEnvelopeItem(
+    items: List<SentryEnvelopeItem>,
+    predicate: (index: Int, item: T) -> Unit = { _, _ -> }
+): T {
     val item = items.mapIndexedNotNull { index, it ->
         val deserialized = JsonSerializer(SentryOptions()).deserialize(it.data.inputStream().reader(), T::class.java)
         deserialized?.let { Pair(index, it) }
