feat: Send missing analytics data from Flank to Mixpanel

docs/flank_mixpanel_metrics.md

@@ -0,0 +1,57 @@
+# Mixpanel events in Flank
+
+Flank is currently tracking the following events:
+
+- configuration
+- bundle id and package name with information about the device type
+- total cost
+- cost per device type
+- test duration
+- flank version with information if tests run on corellium or firebase
+
+Every event contains a session id and project id. By ```project id``` you can find all events from every execution with specific ```project id```. By ```session id``` you can find events from specific Flank execution.
+
+## Configuration
+
+This event contains information about the configuration executed in Flank. This event allows us to
+track information about devices and features used in Flank.
+It could be useful to check what features are most important for the community.
+
+Fields reflect configuration names.
+
+## Bundle and package id [app_id]
+
+This event contains information about the bundle id for ios and the package id for android project. Additionally,
+contains platform type (android, ios). This event allows us to implement a ```Who uses Flank``` report with additional breakdown by ```device type```.
+
+Fields:
+
+- ```app_id```
+- ```device_type```
+
+## Total cost and cost per device type [devices_cost]
+
+With these event's we can realize the report ```How many millions per month in spend is Flank responsible for on Firebase Test Lab? ```
+
+Fields:
+
+- ```virtual_cost```
+- ```physical_cost```
+- ```total_cost```
+
+## Test duration [total_test_time]
+
+With these event's we can check how long tests took.
+
+Fields:
+
+- ```test_duration```
+
+## Flank Version [flank_version]
+
+By these event's we can check how frequently users upgrade the Flank version.
+
+Fields:
+
+- ```version```
+- ```test_platform```

test_runner/src/main/kotlin/ftl/analytics/SendUsageStatistics.kt

@@ -7,14 +7,31 @@ import ftl.args.IosArgs
 import ftl.args.blockSendingUsageStatistics
 import ftl.util.isGoogleAnalyticsDisabled
 
+private const val CONFIGURATION_KEY = "configuration"
+private const val APP_ID = "app_id"
+private const val DEVICE_TYPE = "device_type"
+const val FLANK_VERSION = "flank_version"
+const val FLANK_VERSION_PROPERTY = "version"
+const val TEST_PLATFORM = "test_platform"
+const val FIREBASE = "firebase"
+
 fun AndroidArgs.sendConfiguration() = sendConfiguration(events = createEventMap())
 
 fun IosArgs.sendConfiguration() = sendConfiguration(events = createEventMap())
 
-fun IArgs.sendConfiguration(events: Map<String, Any?>, rootPath: String = userHome) =
+fun IArgs.sendConfiguration(
+    events: Map<String, Any?>,
+    rootPath: String = userHome,
+    eventName: String = CONFIGURATION_KEY
+) =
     takeUnless { blockSendingUsageStatistics || isGoogleAnalyticsDisabled(rootPath) }?.run {
         registerUser()
         events
-            .toEvent(project)
+            .toEvent(project, eventName)
             .send()
     }
+
+fun IArgs.sendAppId(appId: String) = when (this) {
+    is AndroidArgs -> sendConfiguration(mapOf(APP_ID to appId, DEVICE_TYPE to "android"), eventName = APP_ID)
+    else -> sendConfiguration(mapOf(APP_ID to appId, DEVICE_TYPE to "ios"), eventName = APP_ID)
+}

test_runner/src/main/kotlin/ftl/analytics/UsageStatisticsClient.kt

@@ -10,7 +10,6 @@ import ftl.util.sessionId
 import org.json.JSONObject
 
 private const val MIXPANEL_API_TOKEN = "d9728b2c8e6ca9fd6de1fcd32dd8cdc2"
-private const val CONFIGURATION_KEY = "configuration"
 
 internal val messageBuilder by lazy {
     MessageBuilder(MIXPANEL_API_TOKEN)
@@ -29,7 +28,7 @@ internal val objectMapper by lazy {
 @VisibleForTesting
 internal fun JSONObject.send() = apiClient.sendMessage(this)
 
-internal fun Map<String, Any?>.toEvent(projectId: String) =
+internal fun Map<String, Any?>.toEvent(projectId: String, eventName: String) =
     (this + Pair(SESSION_ID, sessionId)).run {
-        messageBuilder.event(projectId, CONFIGURATION_KEY, toJSONObject())
+        messageBuilder.event(projectId, eventName, toJSONObject())
     }

test_runner/src/main/kotlin/ftl/client/google/AppDetails.kt

@@ -0,0 +1,7 @@
+package ftl.client.google
+
+import com.google.testing.model.FileReference
+
+fun getAndroidAppDetails(gcsAppPath: String): String =
+    GcTesting.get.ApplicationDetailService().getApkDetails(FileReference().apply { gcsPath = gcsAppPath })
+        .execute()?.apkDetail?.apkManifest?.packageName?.toString().orEmpty()

test_runner/src/main/kotlin/ftl/domain/RunTestAndroid.kt

@@ -1,6 +1,10 @@
 package ftl.domain
 
 import flank.common.logLn
+import ftl.analytics.FIREBASE
+import ftl.analytics.FLANK_VERSION
+import ftl.analytics.FLANK_VERSION_PROPERTY
+import ftl.analytics.TEST_PLATFORM
 import ftl.analytics.sendConfiguration
 import ftl.args.createAndroidArgs
 import ftl.args.setupLogLevel
@@ -22,6 +26,7 @@ import ftl.util.StopWatch
 import ftl.util.TEST_TYPE
 import ftl.util.loadFile
 import ftl.util.printVersionInfo
+import ftl.util.readVersion
 import ftl.util.setCrashReportTag
 import kotlinx.coroutines.runBlocking
 import java.nio.file.Paths
@@ -56,6 +61,7 @@ operator fun RunTestAndroid.invoke() {
             TEST_TYPE to type?.name.orEmpty()
         )
         sendConfiguration()
+        sendConfiguration(mapOf(FLANK_VERSION_PROPERTY to readVersion(), TEST_PLATFORM to FIREBASE), eventName = FLANK_VERSION)
     }.validate().also { args ->
         runBlocking {
             if (dumpShards)

test_runner/src/main/kotlin/ftl/domain/RunTestIos.kt

@@ -1,6 +1,8 @@
 package ftl.domain
 
 import flank.common.logLn
+import ftl.analytics.FLANK_VERSION
+import ftl.analytics.FLANK_VERSION_PROPERTY
 import ftl.analytics.sendConfiguration
 import ftl.args.createIosArgs
 import ftl.args.setupLogLevel
@@ -22,6 +24,7 @@ import ftl.util.StopWatch
 import ftl.util.TEST_TYPE
 import ftl.util.loadFile
 import ftl.util.printVersionInfo
+import ftl.util.readVersion
 import ftl.util.setCrashReportTag
 import kotlinx.coroutines.runBlocking
 import java.nio.file.Paths
@@ -56,6 +59,7 @@ operator fun RunIosTest.invoke() {
             TEST_TYPE to type?.name.orEmpty()
         )
         sendConfiguration()
+        sendConfiguration(mapOf(FLANK_VERSION_PROPERTY to readVersion()), eventName = FLANK_VERSION)
         if (dumpShards.not()) logLn(this)
     }.validate().run {
         if (dumpShards) dumpShards()

test_runner/src/main/kotlin/ftl/ios/xctest/XcTestData.kt

@@ -1,13 +1,15 @@
 package ftl.ios.xctest
 
 import com.dd.plist.NSDictionary
+import ftl.analytics.sendAppId
 import ftl.args.ArgsHelper.calculateShards
 import ftl.args.IosArgs
 import ftl.args.isXcTest
 import ftl.ios.xctest.common.XcTestRunVersion
 import ftl.ios.xctest.common.XcTestRunVersion.V1
 import ftl.ios.xctest.common.XcTestRunVersion.V2
 import ftl.ios.xctest.common.XctestrunMethods
+import ftl.ios.xctest.common.getBundleId
 import ftl.ios.xctest.common.getXcTestRunVersion
 import ftl.ios.xctest.common.mapToRegex
 import ftl.ios.xctest.common.parseToNSDictionary
@@ -47,6 +49,8 @@ private fun IosArgs.calculateXcTest(): XcTestRunData {
         )
     }
 
+    reportBundleId()
+
     return XcTestRunData(
         rootDir = xcTestRoot,
         nsDict = xcTestNsDictionary,
@@ -55,6 +59,8 @@ private fun IosArgs.calculateXcTest(): XcTestRunData {
     )
 }
 
+private fun IosArgs.reportBundleId() = sendAppId(getBundleId())
+
 private inline fun <reified T> createCustomSharding(shardingJsonPath: String) =
     fromJson<T>(Paths.get(shardingJsonPath).toFile().readText())
 

test_runner/src/main/kotlin/ftl/ios/xctest/common/Util.kt

@@ -3,6 +3,7 @@ package ftl.ios.xctest.common
 import com.dd.plist.NSArray
 import com.dd.plist.NSDictionary
 import com.dd.plist.PropertyListParser
+import ftl.args.IosArgs
 import ftl.run.exception.FlankConfigurationError
 import ftl.run.exception.FlankGeneralError
 import java.io.ByteArrayOutputStream
@@ -14,6 +15,7 @@ enum class XcTestRunVersion { V1, V2 }
 
 internal const val XCTEST_METADATA = "__xctestrun_metadata__"
 internal const val FORMAT_VERSION = "FormatVersion"
+internal const val BUNDLE_ID = "TestHostBundleIdentifier"
 internal const val TEST_CONFIGURATIONS = "TestConfigurations"
 internal const val TEST_TARGETS = "TestTargets"
 internal const val TEST_PLAN = "TestPlan"
@@ -56,6 +58,12 @@ private fun NSDictionary.getName(): String = get(NAME)
 
 internal fun NSDictionary.getBlueprintName() = get(BLUEPRINT_NAME).toString()
 
+internal fun IosArgs.getBundleId(): String =
+    File(xctestrunFile).parentFile.walk().maxDepth(3).first { it.extension == "plist" && it.parent.endsWith(".app") }
+        .let { plist ->
+            PropertyListParser.parse(plist) as NSDictionary
+        }["CFBundleIdentifier"]?.toString().orEmpty()
+
 internal fun NSDictionary.toByteArray(): ByteArray {
     val out = ByteArrayOutputStream()
     PropertyListParser.saveAsXML(this, out)

test_runner/src/main/kotlin/ftl/mock/MockServer.kt

@@ -26,6 +26,7 @@ import com.google.api.services.toolresults.model.TestExecutionStep
 import com.google.api.services.toolresults.model.TestSuiteOverview
 import com.google.api.services.toolresults.model.TestTiming
 import com.google.gson.GsonBuilder
+import com.google.gson.JsonObject
 import com.google.gson.LongSerializationPolicy
 import com.google.testing.model.AndroidDevice
 import com.google.testing.model.AndroidDeviceCatalog
@@ -379,6 +380,10 @@ object MockServer {
                         )
                     call.respond(performanceMetricsSummary)
                 }
+
+                post("/v1/applicationDetailService/getApkDetails") {
+                    call.respond(JsonObject())
+                }
             }
         }
     }

test_runner/src/main/kotlin/ftl/reports/CostReport.kt

@@ -1,12 +1,16 @@
 package ftl.reports
 
 import flank.common.println
+import ftl.analytics.sendConfiguration
 import ftl.api.JUnitTest
 import ftl.args.IArgs
 import ftl.config.FtlConstants.indent
 import ftl.json.MatrixMap
 import ftl.reports.util.IReport
 import ftl.reports.util.ReportManager
+import ftl.util.calculatePhysicalCost
+import ftl.util.calculateTotalCost
+import ftl.util.calculateVirtualCost
 import ftl.util.estimateCosts
 import java.io.StringWriter
 
@@ -15,7 +19,7 @@ object CostReport : IReport {
 
     override val extension = ".txt"
 
-    private fun estimate(matrices: MatrixMap): String {
+    private fun estimate(args: IArgs, matrices: MatrixMap): String {
         var totalBillableVirtualMinutes = 0L
         var totalBillablePhysicalMinutes = 0L
 
@@ -24,11 +28,23 @@ object CostReport : IReport {
             totalBillablePhysicalMinutes += it.billableMinutes.physical
         }
 
+        args.sendConfiguration(
+            events = mapOf(
+                "virtual_cost" to calculateVirtualCost(totalBillableVirtualMinutes.toBigDecimal()),
+                "physical_cost" to calculatePhysicalCost(totalBillablePhysicalMinutes.toBigDecimal()),
+                "total_cost" to calculateTotalCost(
+                    totalBillablePhysicalMinutes.toBigDecimal(),
+                    totalBillableVirtualMinutes.toBigDecimal()
+                )
+            ),
+            eventName = "devices_cost"
+        )
+
         return estimateCosts(totalBillableVirtualMinutes, totalBillablePhysicalMinutes)
     }
 
-    private fun generate(matrices: MatrixMap): String {
-        val cost = estimate(matrices)
+    private fun generate(args: IArgs, matrices: MatrixMap): String {
+        val cost = estimate(args, matrices)
         StringWriter().use { writer ->
             writer.println(reportName())
             cost.split("\n").forEach { writer.println(indent + it) }
@@ -37,7 +53,7 @@ object CostReport : IReport {
     }
 
     override fun run(matrices: MatrixMap, result: JUnitTest.Result?, printToStdout: Boolean, args: IArgs) {
-        val output = generate(matrices)
+        val output = generate(args, matrices)
         if (printToStdout) print(output)
         write(matrices, output, args)
         ReportManager.uploadReportResult(output, args, fileName())

test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt

@@ -2,18 +2,24 @@ package ftl.run.platform
 
 import flank.common.join
 import flank.common.logLn
+import ftl.analytics.sendAppId
 import ftl.api.RemoteStorage
 import ftl.api.TestMatrixAndroid
 import ftl.api.executeTestMatrixAndroid
 import ftl.api.uploadToRemoteStorage
 import ftl.args.AndroidArgs
+import ftl.args.IArgs
 import ftl.args.isInstrumentationTest
 import ftl.args.shardsFilePath
+import ftl.client.google.getAndroidAppDetails
 import ftl.config.FtlConstants
 import ftl.run.exception.FlankGeneralError
 import ftl.run.model.AndroidMatrixTestShards
 import ftl.run.model.AndroidTestContext
+import ftl.run.model.GameLoopContext
 import ftl.run.model.InstrumentationTestContext
+import ftl.run.model.RoboTestContext
+import ftl.run.model.SanityRoboTestContext
 import ftl.run.model.TestResult
 import ftl.run.platform.android.asMatrixTestShards
 import ftl.run.platform.android.createAndroidTestConfig
@@ -45,6 +51,7 @@ internal suspend fun AndroidArgs.runAndroidTests(): TestResult = coroutineScope
                 ignoredTestsShardChunks += context.ignoredTestCases
                 allTestShardChunks += context.shards
             }
+            context.reportPackageName(args)
         }
         .map { createTestSetup(it) }
         .run { executeTestMatrixAndroid(this) }
@@ -89,3 +96,12 @@ private suspend fun createTestSetup(context: AndroidTestContext) = TestMatrixAnd
     config = createAndroidTestConfig(context.args),
     type = context.args.createAndroidTestMatrixType(context)
 )
+
+private fun AndroidTestContext.reportPackageName(args: IArgs) = when (this) {
+    is InstrumentationTestContext -> getAndroidAppDetails(app.remote)
+    is RoboTestContext -> getAndroidAppDetails(app.remote)
+    is SanityRoboTestContext -> getAndroidAppDetails(app.remote)
+    is GameLoopContext -> getAndroidAppDetails(app.remote)
+}.sendPackageName(args)
+
+private fun String.sendPackageName(args: IArgs) = args.sendAppId(this)