Enable caching of analytics tracking data (#11667)

airbyte-analytics/README.md

@@ -28,6 +28,17 @@ environment variable. In addition to the type of analytics tracker, the followin
 In order to use this module in a service at runtime,  add the following configuration to the service's `application.yml` file:
 
 ```yaml
+micronaut:
+  caches:
+    # used by the analytics tracking client to cache calls to resolve the deployment and identity (workspace) for
+    # track events
+    analytics-tracking-deployments:
+      charset: "UTF-8"
+      expire-after-access: 10m
+    analytics-tracking-identity:
+      charset: "UTF-8"
+      expire-after-access: 10m
+
 airbyte:
   deployment-mode: ${DEPLOYMENT_MODE:OSS}
   role: ${AIRBYTE_ROLE:dev}

airbyte-analytics/build.gradle.kts

@@ -11,6 +11,7 @@ dependencies {
 
     api(libs.segment.java.analytics)
     api(libs.micronaut.http)
+    api(libs.micronaut.cache.caffeine)
     api(libs.bundles.micronaut.annotation)
     api(libs.bundles.micronaut.kotlin)
     api(libs.kotlin.logging)

airbyte-analytics/src/main/kotlin/io/airbyte/analytics/TrackingClient.kt

@@ -14,6 +14,8 @@ import com.segment.analytics.messages.TrackMessage
 import io.airbyte.api.client.model.generated.DeploymentMetadataRead
 import io.airbyte.api.client.model.generated.WorkspaceRead
 import io.github.oshai.kotlinlogging.KotlinLogging
+import io.micronaut.cache.annotation.CacheConfig
+import io.micronaut.cache.annotation.Cacheable
 import io.micronaut.context.annotation.Requires
 import io.micronaut.context.annotation.Value
 import io.micronaut.http.context.ServerRequestContext
@@ -337,19 +339,23 @@ class LoggingTrackingClient(
 }
 
 @Singleton
-class DeploymentFetcher(
+@CacheConfig("analytics-tracking-deployments")
+open class DeploymentFetcher(
   @Named("deploymentSupplier") val deploymentFetcher: Supplier<DeploymentMetadataRead>,
 ) : Supplier<Deployment> {
+  @Cacheable
   override fun get(): Deployment {
     val deploymentMetadata = deploymentFetcher.get()
     return Deployment(deploymentMetadata)
   }
 }
 
 @Singleton
-class TrackingIdentityFetcher(
+@CacheConfig("analytics-tracking-identity")
+open class TrackingIdentityFetcher(
   @Named("workspaceFetcher") val workspaceFetcher: Function<UUID, WorkspaceRead>,
 ) : Function<UUID, TrackingIdentity> {
+  @Cacheable
   override fun apply(workspaceId: UUID): TrackingIdentity {
     val workspaceRead = workspaceFetcher.apply(workspaceId)
     val email: String? =

airbyte-api-server/build.gradle.kts

@@ -31,6 +31,7 @@ dependencies {
     implementation(libs.log4j.slf4j2.impl)
     implementation(libs.bundles.jackson)
     implementation(libs.bundles.micronaut)
+    implementation(libs.bundles.micronaut.cache)
     implementation(libs.bundles.micronaut.data.jdbc)
     implementation(libs.bundles.micronaut.metrics)
     implementation(libs.micronaut.jaxrs.server)

airbyte-api-server/src/main/resources/application.yml

@@ -1,6 +1,15 @@
 micronaut:
   application:
     name: airbyte-api-server
+  caches:
+    # used by the analytics tracking client to cache calls to resolve the deployment and identity (workspace) for
+    # track events
+    analytics-tracking-deployments:
+      charset: "UTF-8"
+      expire-after-access: 10m
+    analytics-tracking-identity:
+      charset: "UTF-8"
+      expire-after-access: 10m
   env:
     cloud-deduction: true
   executors:
@@ -92,6 +101,9 @@ endpoints:
   beans:
     enabled: true
     sensitive: false
+  caches:
+    enabled: true
+    sensitive: false
   env:
     enabled: true
     sensitive: false

airbyte-connector-builder-server/build.gradle.kts

@@ -24,6 +24,7 @@ dependencies {
     implementation(libs.guava)
     implementation(platform(libs.micronaut.platform))
     implementation(libs.bundles.micronaut)
+    implementation(libs.bundles.micronaut.cache)
     implementation(libs.micronaut.http)
     implementation(libs.micronaut.security)
     implementation(libs.jakarta.annotation.api)

airbyte-connector-builder-server/src/main/resources/application.yml

@@ -1,6 +1,15 @@
 micronaut:
   application:
     name: airbyte-connector-builder-server
+  caches:
+    # used by the analytics tracking client to cache calls to resolve the deployment and identity (workspace) for
+    # track events
+    analytics-tracking-deployments:
+      charset: "UTF-8"
+      expire-after-access: 10m
+    analytics-tracking-identity:
+      charset: "UTF-8"
+      expire-after-access: 10m
   env:
     cloud-deduction: true
   metrics:
@@ -77,6 +86,9 @@ endpoints:
   beans:
     enabled: true
     sensitive: false
+  caches:
+    enabled: true
+    sensitive: false
   env:
     enabled: true
     sensitive: false

airbyte-container-orchestrator/build.gradle.kts

@@ -44,6 +44,7 @@ dependencies {
 
     implementation(platform(libs.micronaut.platform))
     implementation(libs.bundles.micronaut)
+    implementation(libs.bundles.micronaut.cache)
     implementation(libs.bundles.micronaut.metrics)
     implementation(libs.guava)
     implementation(libs.s3)

airbyte-container-orchestrator/src/main/resources/application.yml

@@ -1,6 +1,15 @@
 micronaut:
   application:
     name: airbyte-container-orchestrator
+  caches:
+    # used by the analytics tracking client to cache calls to resolve the deployment and identity (workspace) for
+    # track events
+    analytics-tracking-deployments:
+      charset: "UTF-8"
+      expire-after-access: 10m
+    analytics-tracking-identity:
+      charset: "UTF-8"
+      expire-after-access: 10m
   env:
     cloud-deduction: true
   server:
@@ -129,6 +138,9 @@ endpoints:
   beans:
     enabled: true
     sensitive: false
+  caches:
+    enabled: true
+    sensitive: false
   env:
     enabled: true
     sensitive: false

airbyte-cron/build.gradle.kts

@@ -17,6 +17,7 @@ dependencies {
 
     implementation(platform(libs.micronaut.platform))
     implementation(libs.bundles.micronaut)
+    implementation(libs.bundles.micronaut.cache)
     implementation(libs.bundles.micronaut.metrics)
     implementation(libs.bundles.kubernetes.client)
     implementation(libs.bundles.temporal)

airbyte-cron/src/main/resources/application.yml

@@ -1,6 +1,17 @@
 micronaut:
   application:
     name: airbyte-cron
+  caches:
+    # used by the analytics tracking client to cache calls to resolve the deployment and identity (workspace) for
+    # track events
+    analytics-tracking-deployments:
+      charset: "UTF-8"
+      expire-after-access: 10m
+    analytics-tracking-identity:
+      charset: "UTF-8"
+      expire-after-access: 10m
+    remote-definitions-provider:
+      expire-after-write: 15s
   env:
     cloud-deduction: true
   metrics:
@@ -12,9 +23,6 @@ micronaut:
         step: ${MICROMETER_METRICS_STEP:PT30S}
         host: ${STATSD_HOST:localhost}
         port: ${STATSD_PORT:8125}
-  caches:
-    remote-definitions-provider:
-      expire-after-write: 15s
   server:
     port: 9001
 
@@ -106,6 +114,9 @@ endpoints:
   beans:
     enabled: true
     sensitive: false
+  caches:
+    enabled: true
+    sensitive: false
   env:
     enabled: true
     sensitive: false

airbyte-server/build.gradle.kts

@@ -31,6 +31,7 @@ dependencies {
 
     implementation(platform(libs.micronaut.platform))
     implementation(libs.bundles.micronaut)
+    implementation(libs.bundles.micronaut.cache)
     implementation(libs.bundles.micronaut.data.jdbc)
     implementation(libs.bundles.micronaut.metrics)
     implementation(libs.micronaut.jaxrs.server)

airbyte-server/src/main/resources/application.yml

@@ -1,6 +1,15 @@
 micronaut:
   application:
     name: airbyte-server
+  caches:
+    # used by the analytics tracking client to cache calls to resolve the deployment and identity (workspace) for
+    # track events
+    analytics-tracking-deployments:
+      charset: "UTF-8"
+      expire-after-access: 10m
+    analytics-tracking-identity:
+      charset: "UTF-8"
+      expire-after-access: 10m
   env:
     cloud-deduction: true
   executors:
@@ -344,6 +353,9 @@ endpoints:
   beans:
     enabled: true
     sensitive: false
+  caches:
+    enabled: true
+    sensitive: false
   env:
     enabled: true
     sensitive: false

airbyte-workers/build.gradle.kts

@@ -48,8 +48,8 @@ dependencies {
     implementation(platform(libs.micronaut.platform))
     implementation(libs.google.cloud.storage)
     implementation(libs.bundles.micronaut)
+    implementation(libs.bundles.micronaut.cache)
     implementation(libs.bundles.micronaut.metrics)
-    implementation(libs.micronaut.cache.caffeine)
     implementation(libs.jooq)
     implementation(libs.s3)
     implementation(libs.sts)

airbyte-workers/src/main/resources/application.yml

@@ -9,6 +9,14 @@ micronaut:
         max-content-length: 52428800 # 50MB
     port: 9000
   caches:
+    # used by the analytics tracking client to cache calls to resolve the deployment and identity (workspace) for
+    # track events
+    analytics-tracking-deployments:
+      charset: "UTF-8"
+      expire-after-access: 10m
+    analytics-tracking-identity:
+      charset: "UTF-8"
+      expire-after-access: 10m
     # used by RecordMetricActivity to cache repeated calls to workspaceApi.getWorkspaceByConnectionId.
     # entries are UUID connectionIds, so 100,000 entries should be roughly 1.6MB
     connection-workspace-id:
@@ -275,6 +283,9 @@ endpoints:
   beans:
     enabled: true
     sensitive: false
+  caches:
+    enabled: true
+    sensitive: false
   env:
     enabled: true
     sensitive: false

deps.toml

@@ -21,6 +21,7 @@ kubernetes-client = "6.5.1"
 log4j = "2.22.1"
 lombok = "1.18.30"
 micronaut = "4.3.5"
+micronaut-cache = "4.2.2"
 micronaut-data = "4.6.1"
 micronaut-email = "2.4.0"
 micronaut-jaxrs = "4.3.0"
@@ -184,7 +185,8 @@ vault-java-driver = { module = "com.bettercloud:vault-java-driver", version = "5
 
 # Micronaut-related dependencies
 h2-database = { module = "com.h2database:h2", version = "2.2.224" }
-micronaut-cache-caffeine = { module = "io.micronaut.cache:micronaut-cache-caffeine", version = "4.2.2" }
+micronaut-cache-caffeine = { module = "io.micronaut.cache:micronaut-cache-caffeine", version.ref = "micronaut-cache" }
+micronaut-cache-management = { module = "io.micronaut.cache:micronaut-cache-management", version.ref = "micronaut-cache" }
 micronaut-data-connection = { module = "io.micronaut.data:micronaut-data-connection", version.ref = "micronaut-data" }
 micronaut-data-processor = { module = "io.micronaut.data:micronaut-data-processor", version.ref = "micronaut-data" }
 micronaut-data-jdbc = { module = "io.micronaut.data:micronaut-data-jdbc", version.ref = "micronaut-data" }
@@ -238,6 +240,7 @@ micronaut = ["jakarta-annotation-api", "jakarta-transaction-api", "micronaut-htt
 micronaut-light = ["jakarta-annotation-api", "micronaut-inject-java", "micronaut-runtime"]
 micronaut-annotation = ["jakarta-annotation-api", "micronaut-inject-java", "micronaut-inject-kotlin"]
 micronaut-annotation-processor = ["micronaut-inject-java", "micronaut-management", "micronaut-validation", "micronaut-data-processor", "micronaut-jaxrs-processor", "micronaut-http-validation"]
+micronaut-cache = ["micronaut-cache-caffeine", "micronaut-cache-management"]
 micronaut-data-jdbc = [ "micronaut-data-connection", "micronaut-data-jdbc", "micronaut-data-model", "micronaut-data-tx", "jakarta-persistence-api", "micronaut-jdbc-hikari"]
 micronaut-email = ["micronaut-email", "micronaut-email-sendgrid"]
 micronaut-kotlin = ["micronaut-kotlin-extension-functions", "micronaut-kotlin-runtime"]