From f8749f08246ea57f95cd7440ecc3cbe827f821d6 Mon Sep 17 00:00:00 2001 From: tiagocandido Date: Wed, 19 Aug 2026 15:00:34 +0200 Subject: [PATCH] Instrument Android checkout failures --- platforms/android/README.md | 9 + platforms/android/lib/api/lib.api | 16 + .../shopify/checkoutkit/CheckoutProtocol.kt | 9 + .../shopify/checkoutkit/CheckoutWebView.kt | 97 +++++++ .../com/shopify/checkoutkit/Configuration.kt | 7 + .../EmbeddedCheckoutProtocolBridge.kt | 17 +- .../shopify/checkoutkit/ShopifyCheckoutKit.kt | 4 + .../java/com/shopify/checkoutkit/Telemetry.kt | 118 ++++++++ .../checkoutkit/CheckoutWebViewClientTest.kt | 2 + .../shopify/checkoutkit/ConfigurationTest.kt | 15 + .../EmbeddedCheckoutProtocolBridgeTest.kt | 2 + .../checkoutkit/PreloadObservabilityTest.kt | 2 + .../checkoutkit/TelemetryIntegrationTest.kt | 274 ++++++++++++++++++ 13 files changed, 569 insertions(+), 3 deletions(-) create mode 100644 platforms/android/lib/src/main/java/com/shopify/checkoutkit/Telemetry.kt create mode 100644 platforms/android/lib/src/test/java/com/shopify/checkoutkit/TelemetryIntegrationTest.kt diff --git a/platforms/android/README.md b/platforms/android/README.md index a8d40ed15..174b95311 100644 --- a/platforms/android/README.md +++ b/platforms/android/README.md @@ -282,6 +282,7 @@ ShopifyCheckoutKit.configure { ), ) it.logLevel = LogLevel.ERROR + it.telemetry = Telemetry(enabled = false) } ``` @@ -293,6 +294,14 @@ ShopifyCheckoutKit.configure { | `preloading` | `Preloading(enabled = true)` | Enables best-effort checkout preloading before presentation. | | `title` | `null` | Runtime override for the checkout sheet header title. When `null`, the SDK uses the localized `checkout_web_view_title` string resource. | | `allowedMessageOrigins` | `emptySet()` | Extra origins allowed to send checkout protocol messages. | +| `telemetry` | `Telemetry(enabled = true)` | Sends anonymous diagnostic metrics to Shopify. Set `enabled` to `false` to opt out. | + +Checkout Kit reports bounded counts for checkout errors, protocol decoding +failures, and navigation retries, plus navigation duration histograms. These +diagnostics never include checkout URLs, message payloads, buyer data, or +checkout, order, customer, or shop identifiers. Disabling telemetry stops new +collection and discards measurements that have not already been handed to the +operating system for delivery. ### Color schemes diff --git a/platforms/android/lib/api/lib.api b/platforms/android/lib/api/lib.api index f537565e7..afa597124 100644 --- a/platforms/android/lib/api/lib.api +++ b/platforms/android/lib/api/lib.api @@ -494,6 +494,7 @@ public final class com/shopify/checkoutkit/Configuration { public final fun component5 ()Lcom/shopify/checkoutkit/Preloading; public final fun component6 ()Ljava/lang/String; public final fun component7 ()Ljava/util/Set; + public final fun component8 ()Lcom/shopify/checkoutkit/Telemetry; public fun equals (Ljava/lang/Object;)Z public final fun getAllowedMessageOrigins ()Ljava/util/Set; public final fun getAppearance ()Lcom/shopify/checkoutkit/CheckoutAppearance; @@ -501,6 +502,7 @@ public final class com/shopify/checkoutkit/Configuration { public final fun getPlatform ()Lcom/shopify/checkoutkit/Platform; public final fun getPreloading ()Lcom/shopify/checkoutkit/Preloading; public final fun getSheet ()Lcom/shopify/checkoutkit/CheckoutSheetOptions; + public final fun getTelemetry ()Lcom/shopify/checkoutkit/Telemetry; public final fun getTitle ()Ljava/lang/String; public fun hashCode ()I public final fun setAllowedMessageOrigins (Ljava/util/Set;)V @@ -509,6 +511,7 @@ public final class com/shopify/checkoutkit/Configuration { public final fun setPlatform (Lcom/shopify/checkoutkit/Platform;)V public final fun setPreloading (Lcom/shopify/checkoutkit/Preloading;)V public final fun setSheet (Lcom/shopify/checkoutkit/CheckoutSheetOptions;)V + public final fun setTelemetry (Lcom/shopify/checkoutkit/Telemetry;)V public final fun setTitle (Ljava/lang/String;)V public fun toString ()Ljava/lang/String; } @@ -710,3 +713,16 @@ public abstract interface class com/shopify/checkoutkit/StorefrontCustomizer { public abstract fun customize (Lcom/shopify/checkoutkit/ColorsBuilder;)V } +public final class com/shopify/checkoutkit/Telemetry { + public fun ()V + public fun (Z)V + public synthetic fun (ZILkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun component1 ()Z + public final fun copy (Z)Lcom/shopify/checkoutkit/Telemetry; + public static synthetic fun copy$default (Lcom/shopify/checkoutkit/Telemetry;ZILjava/lang/Object;)Lcom/shopify/checkoutkit/Telemetry; + public fun equals (Ljava/lang/Object;)Z + public final fun getEnabled ()Z + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + diff --git a/platforms/android/lib/src/main/java/com/shopify/checkoutkit/CheckoutProtocol.kt b/platforms/android/lib/src/main/java/com/shopify/checkoutkit/CheckoutProtocol.kt index fe09f92d5..2fb223d92 100644 --- a/platforms/android/lib/src/main/java/com/shopify/checkoutkit/CheckoutProtocol.kt +++ b/platforms/android/lib/src/main/java/com/shopify/checkoutkit/CheckoutProtocol.kt @@ -2,6 +2,9 @@ package com.shopify.checkoutkit import android.os.Looper import com.shopify.checkoutkit.ShopifyCheckoutKit.log +import com.shopify.checkoutkit.telemetry.TelemetryDecodeFailureType +import com.shopify.checkoutkit.telemetry.TelemetryProtocolDecodeErrorMetric +import com.shopify.checkoutkit.telemetry.TelemetryProtocolMethod import com.shopify.ucp.embedded.checkout.Checkout import com.shopify.ucp.embedded.checkout.EcpRequest import com.shopify.ucp.embedded.checkout.EmbeddedCheckoutProtocol @@ -122,6 +125,12 @@ public object CheckoutProtocol { ProtocolClient().onDecodeError { method, error, params -> log.e(LOG_TAG, "Failed to decode $method params", error) log.d(LOG_TAG, "Raw $method params: $params") + CheckoutTelemetry.recorder.recordProtocolDecodeError( + TelemetryProtocolDecodeErrorMetric( + method = TelemetryProtocolMethod.fromMethod(method), + failureType = TelemetryDecodeFailureType.Params, + ), + ) }, ) diff --git a/platforms/android/lib/src/main/java/com/shopify/checkoutkit/CheckoutWebView.kt b/platforms/android/lib/src/main/java/com/shopify/checkoutkit/CheckoutWebView.kt index 50a894fcf..2b9210c02 100644 --- a/platforms/android/lib/src/main/java/com/shopify/checkoutkit/CheckoutWebView.kt +++ b/platforms/android/lib/src/main/java/com/shopify/checkoutkit/CheckoutWebView.kt @@ -8,6 +8,7 @@ import android.net.Uri import android.os.Build import android.os.Handler import android.os.Looper +import android.os.SystemClock import android.util.AttributeSet import android.view.MotionEvent import android.view.View @@ -33,8 +34,18 @@ import androidx.core.net.toUri import androidx.webkit.WebSettingsCompat import androidx.webkit.WebViewFeature import com.shopify.checkoutkit.ShopifyCheckoutKit.log +import com.shopify.checkoutkit.telemetry.TelemetryErrorCategory +import com.shopify.checkoutkit.telemetry.TelemetryErrorCode +import com.shopify.checkoutkit.telemetry.TelemetryErrorMetric +import com.shopify.checkoutkit.telemetry.TelemetryErrorStage +import com.shopify.checkoutkit.telemetry.TelemetryNavigationDurationMetric +import com.shopify.checkoutkit.telemetry.TelemetryNavigationDurationResult +import com.shopify.checkoutkit.telemetry.TelemetryNavigationRetryMetric +import com.shopify.checkoutkit.telemetry.TelemetryNavigationRetryReason +import com.shopify.checkoutkit.telemetry.TelemetryNavigationRetryResult import java.util.concurrent.CountDownLatch +@Suppress("TooManyFunctions") internal class CheckoutWebView private constructor( context: Context, attributeSet: AttributeSet?, @@ -67,6 +78,8 @@ internal class CheckoutWebView private constructor( */ internal var hasHandledTerminalFailure = false + private var checkoutRequestRetryReason: TelemetryNavigationRetryReason? = null + private var navigationTiming: NavigationTiming? = null private val touchHandler = CheckoutWebViewTouchHandler() /** Origin of the loaded checkout URL, trusted as a safe default for incoming-message validation. */ @@ -150,6 +163,8 @@ internal class CheckoutWebView private constructor( ) checkoutRequest = request didRetryCheckoutRequest = false + checkoutRequestRetryReason = null + navigationTiming = NavigationTiming(SystemClock.elapsedRealtime(), preloaded = isPreload) loadCheckoutRequest(request) } } @@ -171,6 +186,7 @@ internal class CheckoutWebView private constructor( private fun resetCheckoutRequestRetryState() { checkoutRequest = null didRetryCheckoutRequest = false + checkoutRequestRetryReason = null } private data class CheckoutRequest( @@ -178,6 +194,36 @@ internal class CheckoutWebView private constructor( val headers: Map, ) + private data class NavigationTiming( + val startedAtMillis: Long, + val preloaded: Boolean, + ) + + internal fun recordTerminalProtocolFailureTelemetry() { + CheckoutTelemetry.recorder.recordError( + TelemetryErrorMetric( + category = TelemetryErrorCategory.Protocol, + stage = TelemetryErrorStage.Message, + code = TelemetryErrorCode.Unknown, + retryable = false, + isRetry = didRetryCheckoutRequest, + ), + ) + recordNavigationDuration(TelemetryNavigationDurationResult.Failure) + } + + private fun recordNavigationDuration(result: TelemetryNavigationDurationResult) { + val timing = navigationTiming ?: return + navigationTiming = null + CheckoutTelemetry.recorder.recordNavigationDuration( + TelemetryNavigationDurationMetric( + milliseconds = (SystemClock.elapsedRealtime() - timing.startedAtMillis).toDouble(), + result = result, + preloaded = timing.preloaded, + ), + ) + } + internal fun markPreloadConsumed() { isPreloadRequest = false } @@ -201,6 +247,17 @@ internal class CheckoutWebView private constructor( ) if (wasBackgroundedUnconsumedPreload || !shouldDeliverLifecycleFailure) return true + CheckoutTelemetry.recorder.recordError( + TelemetryErrorMetric( + category = TelemetryErrorCategory.RenderProcess, + stage = TelemetryErrorStage.Presentation, + code = TelemetryErrorCode.Unknown, + retryable = false, + isRetry = didRetryCheckoutRequest, + ), + ) + recordNavigationDuration(TelemetryNavigationDurationResult.Failure) + // didCrash is API 26; framework delivery of this callback also begins on API 26. val didCrash = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) detail.didCrash() else null log.e( @@ -225,6 +282,7 @@ internal class CheckoutWebView private constructor( loadComplete = true preloadCache.transition(this@CheckoutWebView, PreloadState.Ready) listener.onCheckoutViewLoadComplete() + recordNavigationDuration(TelemetryNavigationDurationResult.Success) resetCheckoutRequestRetryState() } @@ -236,6 +294,13 @@ internal class CheckoutWebView private constructor( if (shouldRetryCheckoutRequest(request, error)) { val checkoutRequest = requireNotNull(checkoutRequest) didRetryCheckoutRequest = true + checkoutRequestRetryReason = CheckoutTelemetry.retryReason(error?.errorCode ?: 0) + CheckoutTelemetry.recorder.recordNavigationRetry( + TelemetryNavigationRetryMetric( + reason = requireNotNull(checkoutRequestRetryReason), + result = TelemetryNavigationRetryResult.Started, + ), + ) log.w( LOG_TAG, "Retrying checkout navigation. Error code: ${error?.errorCode}, " + @@ -247,6 +312,14 @@ internal class CheckoutWebView private constructor( val isMainFrame = request?.isForMainFrame == true if (isMainFrame) { + if (didRetryCheckoutRequest) { + CheckoutTelemetry.recorder.recordNavigationRetry( + TelemetryNavigationRetryMetric( + reason = checkoutRequestRetryReason ?: TelemetryNavigationRetryReason.Unknown, + result = TelemetryNavigationRetryResult.Failed, + ), + ) + } preloadCache.evict( PreloadState.Failed( PreloadState.FailureReason.NavigationFailed, @@ -260,6 +333,16 @@ internal class CheckoutWebView private constructor( handleClientError(request, it) } if (isMainFrame) { + CheckoutTelemetry.recorder.recordError( + TelemetryErrorMetric( + category = TelemetryErrorCategory.Navigation, + stage = TelemetryErrorStage.Load, + code = CheckoutTelemetry.errorCode(error?.errorCode ?: 0), + retryable = error?.errorCode in RETRYABLE_CHECKOUT_ERROR_CODES, + isRetry = didRetryCheckoutRequest, + ), + ) + recordNavigationDuration(TelemetryNavigationDurationResult.Failure) resetCheckoutRequestRetryState() } } @@ -272,6 +355,20 @@ internal class CheckoutWebView private constructor( val isMainFrame = request?.isForMainFrame == true if (isMainFrame) { val statusCode = errorResponse?.statusCode ?: 0 + CheckoutTelemetry.recorder.recordError( + TelemetryErrorMetric( + category = TelemetryErrorCategory.Http, + stage = TelemetryErrorStage.Load, + code = when (statusCode) { + in 400..499 -> TelemetryErrorCode.Client + in 500..599 -> TelemetryErrorCode.Server + else -> TelemetryErrorCode.Unknown + }, + retryable = statusCode >= 500, + isRetry = didRetryCheckoutRequest, + ), + ) + recordNavigationDuration(TelemetryNavigationDurationResult.Failure) preloadCache.evict( PreloadState.Failed( PreloadState.FailureReason.HttpError(statusCode), diff --git a/platforms/android/lib/src/main/java/com/shopify/checkoutkit/Configuration.kt b/platforms/android/lib/src/main/java/com/shopify/checkoutkit/Configuration.kt index 04dc93cd9..cfe1039a5 100644 --- a/platforms/android/lib/src/main/java/com/shopify/checkoutkit/Configuration.kt +++ b/platforms/android/lib/src/main/java/com/shopify/checkoutkit/Configuration.kt @@ -23,6 +23,7 @@ public data class Configuration internal constructor( var preloading: Preloading = Preloading(), var title: String? = null, var allowedMessageOrigins: Set = emptySet(), + var telemetry: Telemetry = Telemetry(), ) /** @@ -35,6 +36,12 @@ public data class Preloading( public val enabled: Boolean = true, ) +/** Controls anonymous diagnostic metrics sent by Checkout Kit. */ +public data class Telemetry( + /** Set to false to prevent Checkout Kit from recording or sending diagnostic metrics. */ + public val enabled: Boolean = true, +) + public enum class LogLevel { DEBUG, WARN, ERROR, NONE } diff --git a/platforms/android/lib/src/main/java/com/shopify/checkoutkit/EmbeddedCheckoutProtocolBridge.kt b/platforms/android/lib/src/main/java/com/shopify/checkoutkit/EmbeddedCheckoutProtocolBridge.kt index 40295ffbe..8bbdcbcf2 100644 --- a/platforms/android/lib/src/main/java/com/shopify/checkoutkit/EmbeddedCheckoutProtocolBridge.kt +++ b/platforms/android/lib/src/main/java/com/shopify/checkoutkit/EmbeddedCheckoutProtocolBridge.kt @@ -2,6 +2,9 @@ package com.shopify.checkoutkit import androidx.core.net.toUri import com.shopify.checkoutkit.ShopifyCheckoutKit.log +import com.shopify.checkoutkit.telemetry.TelemetryDecodeFailureType +import com.shopify.checkoutkit.telemetry.TelemetryProtocolDecodeErrorMetric +import com.shopify.checkoutkit.telemetry.TelemetryProtocolMethod import com.shopify.ucp.embedded.checkout.InstrumentsChangeResultUcp import com.shopify.ucp.embedded.checkout.ReadyResult import com.shopify.ucp.embedded.checkout.UCPCheckoutResponseSchemaStatus @@ -155,9 +158,16 @@ internal class EmbeddedCheckoutProtocolBridge( } } catch (e: SerializationException) { log.d(LOG_TAG, "Failed to decode ECP message: $e raw=$message") - val isTerminalError = runCatching { - Json.parseToJsonElement(message).jsonObject["method"]?.jsonPrimitive?.content == CheckoutProtocol.error.method - }.getOrDefault(false) + val decodedMethod = runCatching { + Json.parseToJsonElement(message).jsonObject["method"]?.jsonPrimitive?.content + }.getOrNull() + CheckoutTelemetry.recorder.recordProtocolDecodeError( + TelemetryProtocolDecodeErrorMetric( + method = TelemetryProtocolMethod.fromMethod(decodedMethod.orEmpty()), + failureType = TelemetryDecodeFailureType.Envelope, + ), + ) + val isTerminalError = decodedMethod == CheckoutProtocol.error.method if (isTerminalError) { handleTerminalError(message, null) } else { @@ -245,6 +255,7 @@ internal class EmbeddedCheckoutProtocolBridge( ) { return@onMainThread } + view.recordTerminalProtocolFailureTelemetry() view.listener.onCheckoutViewFailedWithError(failure) } } diff --git a/platforms/android/lib/src/main/java/com/shopify/checkoutkit/ShopifyCheckoutKit.kt b/platforms/android/lib/src/main/java/com/shopify/checkoutkit/ShopifyCheckoutKit.kt index 03bbc0b1c..0e9423513 100644 --- a/platforms/android/lib/src/main/java/com/shopify/checkoutkit/ShopifyCheckoutKit.kt +++ b/platforms/android/lib/src/main/java/com/shopify/checkoutkit/ShopifyCheckoutKit.kt @@ -43,7 +43,11 @@ public object ShopifyCheckoutKit { */ @JvmStatic public fun configure(setter: ConfigurationUpdater) { + val wasTelemetryEnabled = configuration.telemetry.enabled setter.configure(configuration) + if (wasTelemetryEnabled && !configuration.telemetry.enabled) { + CheckoutTelemetry.disable() + } CheckoutWebView.clearCache() } diff --git a/platforms/android/lib/src/main/java/com/shopify/checkoutkit/Telemetry.kt b/platforms/android/lib/src/main/java/com/shopify/checkoutkit/Telemetry.kt new file mode 100644 index 000000000..c24080ca5 --- /dev/null +++ b/platforms/android/lib/src/main/java/com/shopify/checkoutkit/Telemetry.kt @@ -0,0 +1,118 @@ +package com.shopify.checkoutkit + +import android.webkit.WebViewClient.ERROR_CONNECT +import android.webkit.WebViewClient.ERROR_HOST_LOOKUP +import android.webkit.WebViewClient.ERROR_IO +import android.webkit.WebViewClient.ERROR_TIMEOUT +import com.shopify.checkoutkit.telemetry.CheckoutKitTelemetry +import com.shopify.checkoutkit.telemetry.TelemetryErrorCode +import com.shopify.checkoutkit.telemetry.TelemetryErrorMetric +import com.shopify.checkoutkit.telemetry.TelemetryNavigationDurationMetric +import com.shopify.checkoutkit.telemetry.TelemetryNavigationRetryMetric +import com.shopify.checkoutkit.telemetry.TelemetryNavigationRetryReason +import com.shopify.checkoutkit.telemetry.TelemetryPlatform +import com.shopify.checkoutkit.telemetry.TelemetryProtocolDecodeErrorMetric + +internal interface CheckoutTelemetryRecording { + fun recordError(metric: TelemetryErrorMetric) + fun recordProtocolDecodeError(metric: TelemetryProtocolDecodeErrorMetric) + fun recordNavigationRetry(metric: TelemetryNavigationRetryMetric) + fun recordNavigationDuration(metric: TelemetryNavigationDurationMetric) +} + +private interface CheckoutTelemetryClient : CheckoutTelemetryRecording { + fun shutdown() +} + +private class DefaultCheckoutTelemetryRecorder : CheckoutTelemetryClient { + private val telemetry = CheckoutKitTelemetry( + sdkVersion = BuildConfig.SDK_VERSION, + platform = if (ShopifyCheckoutKit.configuration.platform is Platform.ReactNative) { + TelemetryPlatform.ReactNativeAndroid + } else { + TelemetryPlatform.Android + }, + ).also { it.start() } + + override fun recordError(metric: TelemetryErrorMetric) = telemetry.recordError(metric) + + override fun recordProtocolDecodeError(metric: TelemetryProtocolDecodeErrorMetric) = + telemetry.recordProtocolDecodeError(metric) + + override fun recordNavigationRetry(metric: TelemetryNavigationRetryMetric) = + telemetry.recordNavigationRetry(metric) + + override fun recordNavigationDuration(metric: TelemetryNavigationDurationMetric) = + telemetry.recordNavigationDuration(metric) + + override fun shutdown() = telemetry.shutdown(discardPending = true) +} + +private object NoOpCheckoutTelemetryRecorder : CheckoutTelemetryRecording { + override fun recordError(metric: TelemetryErrorMetric) = Unit + + override fun recordProtocolDecodeError(metric: TelemetryProtocolDecodeErrorMetric) = Unit + + override fun recordNavigationRetry(metric: TelemetryNavigationRetryMetric) = Unit + + override fun recordNavigationDuration(metric: TelemetryNavigationDurationMetric) = Unit +} + +internal object CheckoutTelemetry { + private val lock = Any() + + @Volatile + private var configuredRecorder: CheckoutTelemetryClient? = null + + @Volatile + private var recorderOverride: CheckoutTelemetryRecording? = null + + val recorder: CheckoutTelemetryRecording + get() { + if (!ShopifyCheckoutKit.configuration.telemetry.enabled) { + return NoOpCheckoutTelemetryRecorder + } + recorderOverride?.let { return it } + configuredRecorder?.let { return it } + return synchronized(lock) { + // Re-check under the lock so a concurrent disable() cannot race a + // recorder creation that would keep exporting after opt-out. + if (!ShopifyCheckoutKit.configuration.telemetry.enabled) { + return NoOpCheckoutTelemetryRecorder + } + recorderOverride + ?: configuredRecorder + ?: runCatching { DefaultCheckoutTelemetryRecorder() } + .getOrNull() + ?.also { configuredRecorder = it } + ?: NoOpCheckoutTelemetryRecorder + } + } + + fun disable() { + val recorder = synchronized(lock) { + configuredRecorder.also { configuredRecorder = null } + } + recorder?.shutdown() + } + + fun overrideRecorderForTesting(recorder: CheckoutTelemetryRecording?) { + recorderOverride = recorder + } + + fun errorCode(errorCode: Int): TelemetryErrorCode = when (errorCode) { + ERROR_TIMEOUT -> TelemetryErrorCode.Timeout + ERROR_CONNECT -> TelemetryErrorCode.CannotConnect + ERROR_HOST_LOOKUP -> TelemetryErrorCode.Dns + ERROR_IO -> TelemetryErrorCode.ConnectionLost + else -> TelemetryErrorCode.Unknown + } + + fun retryReason(errorCode: Int): TelemetryNavigationRetryReason = when (errorCode(errorCode)) { + TelemetryErrorCode.Timeout -> TelemetryNavigationRetryReason.Timeout + TelemetryErrorCode.ConnectionLost -> TelemetryNavigationRetryReason.ConnectionLost + TelemetryErrorCode.CannotConnect -> TelemetryNavigationRetryReason.CannotConnect + TelemetryErrorCode.Dns -> TelemetryNavigationRetryReason.Dns + else -> TelemetryNavigationRetryReason.Unknown + } +} diff --git a/platforms/android/lib/src/test/java/com/shopify/checkoutkit/CheckoutWebViewClientTest.kt b/platforms/android/lib/src/test/java/com/shopify/checkoutkit/CheckoutWebViewClientTest.kt index c6e3166f2..aa384a46d 100644 --- a/platforms/android/lib/src/test/java/com/shopify/checkoutkit/CheckoutWebViewClientTest.kt +++ b/platforms/android/lib/src/test/java/com/shopify/checkoutkit/CheckoutWebViewClientTest.kt @@ -44,6 +44,7 @@ class CheckoutWebViewClientTest { @Before fun setUp() { + CheckoutTelemetry.overrideRecorderForTesting(NoOpTestCheckoutTelemetryRecorder) activity = Robolectric.buildActivity(ComponentActivity::class.java).get() // Mirror real-Android behavior: startActivity throws ActivityNotFoundException when // no activity resolves the intent. Robolectric defaults to silently recording the @@ -53,6 +54,7 @@ class CheckoutWebViewClientTest { @After fun tearDown() { + CheckoutTelemetry.overrideRecorderForTesting(null) CheckoutWebView.clearCache() ShadowLooper.shadowMainLooper().idle() } diff --git a/platforms/android/lib/src/test/java/com/shopify/checkoutkit/ConfigurationTest.kt b/platforms/android/lib/src/test/java/com/shopify/checkoutkit/ConfigurationTest.kt index 703af83d0..506482f92 100644 --- a/platforms/android/lib/src/test/java/com/shopify/checkoutkit/ConfigurationTest.kt +++ b/platforms/android/lib/src/test/java/com/shopify/checkoutkit/ConfigurationTest.kt @@ -21,6 +21,7 @@ class ConfigurationTest { it.appearance = initialConfiguration.appearance it.sheet = initialConfiguration.sheet it.preloading = initialConfiguration.preloading + it.telemetry = initialConfiguration.telemetry it.platform = initialConfiguration.platform it.logLevel = initialConfiguration.logLevel } @@ -81,6 +82,11 @@ class ConfigurationTest { assertThat(ShopifyCheckoutKit.getConfiguration().preloading.enabled).isTrue() } + @Test + fun `telemetry defaults to enabled`() { + assertThat(ShopifyCheckoutKit.getConfiguration().telemetry.enabled).isTrue() + } + @Test fun `appearance defaults to storefront`() { assertThat(ShopifyCheckoutKit.getConfiguration().appearance).isEqualTo(CheckoutAppearance.Storefront()) @@ -101,6 +107,15 @@ class ConfigurationTest { assertThat(ShopifyCheckoutKit.getConfiguration().preloading.enabled).isFalse() } + @Test + fun `can disable telemetry via configure function`() { + ShopifyCheckoutKit.configure { + it.telemetry = Telemetry(enabled = false) + } + + assertThat(ShopifyCheckoutKit.getConfiguration().telemetry.enabled).isFalse() + } + @Test fun `can set sheet via configure function`() { val sheet = CheckoutSheetOptions( diff --git a/platforms/android/lib/src/test/java/com/shopify/checkoutkit/EmbeddedCheckoutProtocolBridgeTest.kt b/platforms/android/lib/src/test/java/com/shopify/checkoutkit/EmbeddedCheckoutProtocolBridgeTest.kt index 06fc02198..81f9185c3 100644 --- a/platforms/android/lib/src/test/java/com/shopify/checkoutkit/EmbeddedCheckoutProtocolBridgeTest.kt +++ b/platforms/android/lib/src/test/java/com/shopify/checkoutkit/EmbeddedCheckoutProtocolBridgeTest.kt @@ -40,6 +40,7 @@ class EmbeddedCheckoutProtocolBridgeTest { @Before fun setUp() { + CheckoutTelemetry.overrideRecorderForTesting(NoOpTestCheckoutTelemetryRecorder) CheckoutWebView.clearCache() shadowOf(Looper.getMainLooper()).idle() activity = Robolectric.buildActivity(ComponentActivity::class.java).setup().get() @@ -59,6 +60,7 @@ class EmbeddedCheckoutProtocolBridgeTest { @After fun tearDown() { + CheckoutTelemetry.overrideRecorderForTesting(null) CheckoutWebView.clearCache() shadowOf(Looper.getMainLooper()).idle() } diff --git a/platforms/android/lib/src/test/java/com/shopify/checkoutkit/PreloadObservabilityTest.kt b/platforms/android/lib/src/test/java/com/shopify/checkoutkit/PreloadObservabilityTest.kt index c22e5ecbd..1b3220478 100644 --- a/platforms/android/lib/src/test/java/com/shopify/checkoutkit/PreloadObservabilityTest.kt +++ b/platforms/android/lib/src/test/java/com/shopify/checkoutkit/PreloadObservabilityTest.kt @@ -31,6 +31,7 @@ class PreloadObservabilityTest { @Before fun setUp() { + CheckoutTelemetry.overrideRecorderForTesting(NoOpTestCheckoutTelemetryRecorder) initialConfiguration = ShopifyCheckoutKit.getConfiguration() webMessageTransport = FakeWebMessageTransport() CheckoutWebView.clearCache() @@ -41,6 +42,7 @@ class PreloadObservabilityTest { @After fun tearDown() { + CheckoutTelemetry.overrideRecorderForTesting(null) CheckoutWebView.clearCache() ShadowLooper.shadowMainLooper().idle() CheckoutWebView.cacheClock = PreloadCache.Clock() diff --git a/platforms/android/lib/src/test/java/com/shopify/checkoutkit/TelemetryIntegrationTest.kt b/platforms/android/lib/src/test/java/com/shopify/checkoutkit/TelemetryIntegrationTest.kt new file mode 100644 index 000000000..2e5fe19fd --- /dev/null +++ b/platforms/android/lib/src/test/java/com/shopify/checkoutkit/TelemetryIntegrationTest.kt @@ -0,0 +1,274 @@ +package com.shopify.checkoutkit + +import android.net.Uri +import android.os.Looper +import android.webkit.WebResourceError +import android.webkit.WebResourceRequest +import android.webkit.WebViewClient.ERROR_FAILED_SSL_HANDSHAKE +import android.webkit.WebViewClient.ERROR_IO +import android.webkit.WebViewClient.ERROR_TIMEOUT +import androidx.activity.ComponentActivity +import com.shopify.checkoutkit.telemetry.TelemetryErrorCategory +import com.shopify.checkoutkit.telemetry.TelemetryErrorCode +import com.shopify.checkoutkit.telemetry.TelemetryErrorMetric +import com.shopify.checkoutkit.telemetry.TelemetryNavigationDurationMetric +import com.shopify.checkoutkit.telemetry.TelemetryNavigationDurationResult +import com.shopify.checkoutkit.telemetry.TelemetryNavigationRetryMetric +import com.shopify.checkoutkit.telemetry.TelemetryNavigationRetryReason +import com.shopify.checkoutkit.telemetry.TelemetryNavigationRetryResult +import com.shopify.checkoutkit.telemetry.TelemetryProtocolDecodeErrorMetric +import org.assertj.core.api.Assertions.assertThat +import org.junit.After +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.mockito.kotlin.mock +import org.mockito.kotlin.whenever +import org.robolectric.Robolectric +import org.robolectric.RobolectricTestRunner +import org.robolectric.Shadows.shadowOf +import org.robolectric.annotation.Config + +@RunWith(RobolectricTestRunner::class) +@Config(shadows = [RecordingShadowWebView::class]) +class TelemetryIntegrationTest { + private lateinit var activity: ComponentActivity + private lateinit var originalConfiguration: Configuration + private lateinit var recorder: RecordingTelemetry + + @Before + fun setUp() { + activity = Robolectric.buildActivity(ComponentActivity::class.java).get() + originalConfiguration = ShopifyCheckoutKit.getConfiguration() + ShopifyCheckoutKit.configure { it.telemetry = Telemetry(enabled = true) } + recorder = RecordingTelemetry() + CheckoutTelemetry.overrideRecorderForTesting(recorder) + } + + @After + fun tearDown() { + CheckoutTelemetry.overrideRecorderForTesting(null) + ShopifyCheckoutKit.configure { it.telemetry = originalConfiguration.telemetry } + CheckoutWebView.clearCache() + shadowOf(Looper.getMainLooper()).runToEndOfTasks() + } + + @Test + fun `records retry failure and navigation failure without raw error data`() { + val view = CheckoutWebView(activity, FakeWebMessageTransport()) + view.loadCheckout("https://checkout-sdk.myshopify.com/cart/secret") + shadowOf(Looper.getMainLooper()).runToEndOfTasks() + val request = mainFrameRequest(requireNotNull(shadowOf(view).lastLoadedUrl)) + val error = mock().also { + whenever(it.errorCode).thenReturn(ERROR_TIMEOUT) + whenever(it.description).thenReturn("sensitive failure details") + } + val client = view.CheckoutWebViewClient() + + client.onReceivedError(view, request, error) + val retryFailure = mock().also { + whenever(it.errorCode).thenReturn(ERROR_FAILED_SSL_HANDSHAKE) + whenever(it.description).thenReturn("different sensitive details") + } + client.onReceivedError(view, request, retryFailure) + + assertThat(recorder.retries.map { it.result }).containsExactly( + TelemetryNavigationRetryResult.Started, + TelemetryNavigationRetryResult.Failed, + ) + assertThat(recorder.retries.map { it.reason }).containsOnly( + CheckoutTelemetry.retryReason(ERROR_TIMEOUT), + ) + assertThat(recorder.errors).hasSize(1) + assertThat(recorder.errors.single().isRetry).isTrue() + assertThat(recorder.durations.single().result) + .isEqualTo(TelemetryNavigationDurationResult.Failure) + } + + @Test + fun `records terminal protocol error and navigation failure after lifecycle guards`() { + val transport = FakeWebMessageTransport() + val view = CheckoutWebView(activity, transport) + view.loadCheckout("https://checkout-sdk.myshopify.com/cart/secret") + shadowOf(Looper.getMainLooper()).runToEndOfTasks() + val bridge = EmbeddedCheckoutProtocolBridge( + view = view, + webMessageTransport = transport, + protocolMessageExecutor = { command -> command.run() }, + ) + + bridge.receiveMessage(ecErrorMessage()) + bridge.receiveMessage(ecErrorMessage()) + shadowOf(Looper.getMainLooper()).runToEndOfTasks() + + assertThat(recorder.errors).hasSize(1) + assertThat(recorder.errors.single().category).isEqualTo(TelemetryErrorCategory.Protocol) + assertThat(recorder.errors.single().isRetry).isFalse() + assertThat(recorder.durations).hasSize(1) + assertThat(recorder.durations.single().result) + .isEqualTo(TelemetryNavigationDurationResult.Failure) + } + + @Test + fun `does not record surfaced terminal metrics for backgrounded preload errors`() { + val transport = FakeWebMessageTransport() + CheckoutWebView.preload("https://checkout-sdk.myshopify.com/cart/secret", activity, transport) + // idle() runs only due tasks; running delayed tasks would fire the + // preload TTL expiry and evict the cached view under test. + shadowOf(Looper.getMainLooper()).idle() + val preloadedView = requireNotNull(CheckoutWebView.cachedPreloadViewForTesting()) + val bridge = EmbeddedCheckoutProtocolBridge( + view = preloadedView, + webMessageTransport = transport, + protocolMessageExecutor = { command -> command.run() }, + ) + + bridge.receiveMessage(ecErrorMessage()) + shadowOf(Looper.getMainLooper()).idle() + + assertThat(recorder.errors).isEmpty() + assertThat(recorder.durations).isEmpty() + } + + @Test + fun `preserves retry context on terminal protocol errors`() { + val transport = FakeWebMessageTransport() + val view = CheckoutWebView(activity, transport) + view.loadCheckout("https://checkout-sdk.myshopify.com/cart/secret") + shadowOf(Looper.getMainLooper()).runToEndOfTasks() + val request = mainFrameRequest(requireNotNull(shadowOf(view).lastLoadedUrl)) + val timeout = mock().also { + whenever(it.errorCode).thenReturn(ERROR_TIMEOUT) + whenever(it.description).thenReturn("sensitive timeout") + } + + view.CheckoutWebViewClient().onReceivedError(view, request, timeout) + EmbeddedCheckoutProtocolBridge( + view = view, + webMessageTransport = transport, + protocolMessageExecutor = { command -> command.run() }, + ).receiveMessage(ecErrorMessage()) + shadowOf(Looper.getMainLooper()).runToEndOfTasks() + + assertThat(recorder.retries.single().reason).isEqualTo(TelemetryNavigationRetryReason.Timeout) + assertThat(recorder.retries.single().result).isEqualTo(TelemetryNavigationRetryResult.Started) + assertThat(recorder.errors.single().isRetry).isTrue() + } + + @Test + fun `records malformed protocol messages with a bounded method`() { + val view = CheckoutWebView(activity, FakeWebMessageTransport()) + val bridge = EmbeddedCheckoutProtocolBridge( + view = view, + webMessageTransport = FakeWebMessageTransport(), + protocolMessageExecutor = { command -> command.run() }, + ) + + bridge.receiveMessage("not-json containing sensitive data") + + assertThat(recorder.decodeErrors.single().method.wireValue).isEqualTo("unknown") + } + + @Test + fun `does not record metrics when telemetry is disabled`() { + ShopifyCheckoutKit.configure { it.telemetry = Telemetry(enabled = false) } + val view = CheckoutWebView(activity, FakeWebMessageTransport()) + val bridge = EmbeddedCheckoutProtocolBridge( + view = view, + webMessageTransport = FakeWebMessageTransport(), + protocolMessageExecutor = { command -> command.run() }, + ) + + bridge.receiveMessage("not-json") + + assertThat(recorder.decodeErrors).isEmpty() + } + + @Test + fun `captures preload attribution when navigation starts`() { + val view = CheckoutWebView(activity, FakeWebMessageTransport()) + view.loadCheckout("https://checkout-sdk.myshopify.com/cart/secret", isPreload = true) + shadowOf(Looper.getMainLooper()).runToEndOfTasks() + view.markPreloadConsumed() + + view.CheckoutWebViewClient().onPageFinished(view, requireNotNull(shadowOf(view).lastLoadedUrl)) + + assertThat(recorder.durations.single().preloaded).isTrue() + } + + @Test + fun `maps IO errors to connection lost`() { + assertThat(CheckoutTelemetry.errorCode(ERROR_IO)) + .isEqualTo(TelemetryErrorCode.ConnectionLost) + } + + @Test + fun `does not retry IO errors even though they map to connection lost`() { + val view = CheckoutWebView(activity, FakeWebMessageTransport()) + view.loadCheckout("https://checkout-sdk.myshopify.com/cart/secret") + shadowOf(Looper.getMainLooper()).runToEndOfTasks() + val request = mainFrameRequest(requireNotNull(shadowOf(view).lastLoadedUrl)) + val error = mock().also { + whenever(it.errorCode).thenReturn(ERROR_IO) + whenever(it.description).thenReturn("sensitive io details") + } + + view.CheckoutWebViewClient().onReceivedError(view, request, error) + + assertThat(recorder.retries).isEmpty() + assertThat(recorder.errors.single().code).isEqualTo(TelemetryErrorCode.ConnectionLost) + assertThat(recorder.errors.single().retryable).isFalse() + } + + private fun mainFrameRequest(url: String): WebResourceRequest = mock().also { + whenever(it.url).thenReturn(Uri.parse(url)) + whenever(it.isForMainFrame).thenReturn(true) + } + + private fun ecErrorMessage(): String { + val error = """ + |{ + | "ucp":{"version":"2026-04-08","status":"error"}, + | "messages":[ + | { + | "type":"error", + | "code":"session_failed", + | "content":"Session failed", + | "severity":"unrecoverable" + | } + | ] + |} + """.trimMargin() + return """{"jsonrpc":"2.0","method":"ec.error","params":{"error":$error}}""" + } +} + +private class RecordingTelemetry : CheckoutTelemetryRecording { + val errors = mutableListOf() + val decodeErrors = mutableListOf() + val retries = mutableListOf() + val durations = mutableListOf() + + override fun recordError(metric: TelemetryErrorMetric) { + errors += metric + } + + override fun recordProtocolDecodeError(metric: TelemetryProtocolDecodeErrorMetric) { + decodeErrors += metric + } + + override fun recordNavigationRetry(metric: TelemetryNavigationRetryMetric) { + retries += metric + } + + override fun recordNavigationDuration(metric: TelemetryNavigationDurationMetric) { + durations += metric + } +} + +internal object NoOpTestCheckoutTelemetryRecorder : CheckoutTelemetryRecording { + override fun recordError(metric: TelemetryErrorMetric) = Unit + override fun recordProtocolDecodeError(metric: TelemetryProtocolDecodeErrorMetric) = Unit + override fun recordNavigationRetry(metric: TelemetryNavigationRetryMetric) = Unit + override fun recordNavigationDuration(metric: TelemetryNavigationDurationMetric) = Unit +}