From d9eec27f5a9d5fdf0bffaec398b2d8ef2ba9ca01 Mon Sep 17 00:00:00 2001 From: Jonathan Pearlin Date: Thu, 14 Mar 2024 12:49:17 -0400 Subject: [PATCH] Enable caching of analytics tracking data (#11667) --- airbyte-analytics/README.md | 11 +++++++++++ airbyte-analytics/build.gradle.kts | 1 + .../io/airbyte/analytics/TrackingClient.kt | 10 ++++++++-- airbyte-api-server/build.gradle.kts | 1 + .../src/main/resources/application.yml | 12 ++++++++++++ .../build.gradle.kts | 1 + .../src/main/resources/application.yml | 12 ++++++++++++ airbyte-container-orchestrator/build.gradle.kts | 1 + .../src/main/resources/application.yml | 12 ++++++++++++ airbyte-cron/build.gradle.kts | 1 + airbyte-cron/src/main/resources/application.yml | 17 ++++++++++++++--- airbyte-server/build.gradle.kts | 1 + .../src/main/resources/application.yml | 12 ++++++++++++ airbyte-workers/build.gradle.kts | 2 +- .../src/main/resources/application.yml | 11 +++++++++++ deps.toml | 5 ++++- 16 files changed, 103 insertions(+), 7 deletions(-) diff --git a/airbyte-analytics/README.md b/airbyte-analytics/README.md index 26bc90f1be0..9fcef5cd3ca 100644 --- a/airbyte-analytics/README.md +++ b/airbyte-analytics/README.md @@ -28,6 +28,17 @@ environment variable. In addition to the type of analytics tracker, the followin In order to use this module in a service at runtime, add the following configuration to the service's `application.yml` file: ```yaml +micronaut: + caches: + # used by the analytics tracking client to cache calls to resolve the deployment and identity (workspace) for + # track events + analytics-tracking-deployments: + charset: "UTF-8" + expire-after-access: 10m + analytics-tracking-identity: + charset: "UTF-8" + expire-after-access: 10m + airbyte: deployment-mode: ${DEPLOYMENT_MODE:OSS} role: ${AIRBYTE_ROLE:dev} diff --git a/airbyte-analytics/build.gradle.kts b/airbyte-analytics/build.gradle.kts index 30e9f5349fd..e35e89dd2e1 100644 --- a/airbyte-analytics/build.gradle.kts +++ b/airbyte-analytics/build.gradle.kts @@ -11,6 +11,7 @@ dependencies { api(libs.segment.java.analytics) api(libs.micronaut.http) + api(libs.micronaut.cache.caffeine) api(libs.bundles.micronaut.annotation) api(libs.bundles.micronaut.kotlin) api(libs.kotlin.logging) diff --git a/airbyte-analytics/src/main/kotlin/io/airbyte/analytics/TrackingClient.kt b/airbyte-analytics/src/main/kotlin/io/airbyte/analytics/TrackingClient.kt index e632bbf5ddb..6208cafabfe 100644 --- a/airbyte-analytics/src/main/kotlin/io/airbyte/analytics/TrackingClient.kt +++ b/airbyte-analytics/src/main/kotlin/io/airbyte/analytics/TrackingClient.kt @@ -14,6 +14,8 @@ import com.segment.analytics.messages.TrackMessage import io.airbyte.api.client.model.generated.DeploymentMetadataRead import io.airbyte.api.client.model.generated.WorkspaceRead import io.github.oshai.kotlinlogging.KotlinLogging +import io.micronaut.cache.annotation.CacheConfig +import io.micronaut.cache.annotation.Cacheable import io.micronaut.context.annotation.Requires import io.micronaut.context.annotation.Value import io.micronaut.http.context.ServerRequestContext @@ -337,9 +339,11 @@ class LoggingTrackingClient( } @Singleton -class DeploymentFetcher( +@CacheConfig("analytics-tracking-deployments") +open class DeploymentFetcher( @Named("deploymentSupplier") val deploymentFetcher: Supplier, ) : Supplier { + @Cacheable override fun get(): Deployment { val deploymentMetadata = deploymentFetcher.get() return Deployment(deploymentMetadata) @@ -347,9 +351,11 @@ class DeploymentFetcher( } @Singleton -class TrackingIdentityFetcher( +@CacheConfig("analytics-tracking-identity") +open class TrackingIdentityFetcher( @Named("workspaceFetcher") val workspaceFetcher: Function, ) : Function { + @Cacheable override fun apply(workspaceId: UUID): TrackingIdentity { val workspaceRead = workspaceFetcher.apply(workspaceId) val email: String? = diff --git a/airbyte-api-server/build.gradle.kts b/airbyte-api-server/build.gradle.kts index 4d2b3e4c2f9..6e5d5fae649 100644 --- a/airbyte-api-server/build.gradle.kts +++ b/airbyte-api-server/build.gradle.kts @@ -31,6 +31,7 @@ dependencies { implementation(libs.log4j.slf4j2.impl) implementation(libs.bundles.jackson) implementation(libs.bundles.micronaut) + implementation(libs.bundles.micronaut.cache) implementation(libs.bundles.micronaut.data.jdbc) implementation(libs.bundles.micronaut.metrics) implementation(libs.micronaut.jaxrs.server) diff --git a/airbyte-api-server/src/main/resources/application.yml b/airbyte-api-server/src/main/resources/application.yml index a88d8752d18..7ffe8095bfb 100644 --- a/airbyte-api-server/src/main/resources/application.yml +++ b/airbyte-api-server/src/main/resources/application.yml @@ -1,6 +1,15 @@ micronaut: application: name: airbyte-api-server + caches: + # used by the analytics tracking client to cache calls to resolve the deployment and identity (workspace) for + # track events + analytics-tracking-deployments: + charset: "UTF-8" + expire-after-access: 10m + analytics-tracking-identity: + charset: "UTF-8" + expire-after-access: 10m env: cloud-deduction: true executors: @@ -92,6 +101,9 @@ endpoints: beans: enabled: true sensitive: false + caches: + enabled: true + sensitive: false env: enabled: true sensitive: false diff --git a/airbyte-connector-builder-server/build.gradle.kts b/airbyte-connector-builder-server/build.gradle.kts index 36528111045..23613e4d2a6 100644 --- a/airbyte-connector-builder-server/build.gradle.kts +++ b/airbyte-connector-builder-server/build.gradle.kts @@ -24,6 +24,7 @@ dependencies { implementation(libs.guava) implementation(platform(libs.micronaut.platform)) implementation(libs.bundles.micronaut) + implementation(libs.bundles.micronaut.cache) implementation(libs.micronaut.http) implementation(libs.micronaut.security) implementation(libs.jakarta.annotation.api) diff --git a/airbyte-connector-builder-server/src/main/resources/application.yml b/airbyte-connector-builder-server/src/main/resources/application.yml index e5057520694..daa4d6fb7b0 100644 --- a/airbyte-connector-builder-server/src/main/resources/application.yml +++ b/airbyte-connector-builder-server/src/main/resources/application.yml @@ -1,6 +1,15 @@ micronaut: application: name: airbyte-connector-builder-server + caches: + # used by the analytics tracking client to cache calls to resolve the deployment and identity (workspace) for + # track events + analytics-tracking-deployments: + charset: "UTF-8" + expire-after-access: 10m + analytics-tracking-identity: + charset: "UTF-8" + expire-after-access: 10m env: cloud-deduction: true metrics: @@ -77,6 +86,9 @@ endpoints: beans: enabled: true sensitive: false + caches: + enabled: true + sensitive: false env: enabled: true sensitive: false diff --git a/airbyte-container-orchestrator/build.gradle.kts b/airbyte-container-orchestrator/build.gradle.kts index 3ce31bbfef1..b09d9fb1d76 100644 --- a/airbyte-container-orchestrator/build.gradle.kts +++ b/airbyte-container-orchestrator/build.gradle.kts @@ -44,6 +44,7 @@ dependencies { implementation(platform(libs.micronaut.platform)) implementation(libs.bundles.micronaut) + implementation(libs.bundles.micronaut.cache) implementation(libs.bundles.micronaut.metrics) implementation(libs.guava) implementation(libs.s3) diff --git a/airbyte-container-orchestrator/src/main/resources/application.yml b/airbyte-container-orchestrator/src/main/resources/application.yml index 9fdf0bb73ad..6584b8e36a9 100644 --- a/airbyte-container-orchestrator/src/main/resources/application.yml +++ b/airbyte-container-orchestrator/src/main/resources/application.yml @@ -1,6 +1,15 @@ micronaut: application: name: airbyte-container-orchestrator + caches: + # used by the analytics tracking client to cache calls to resolve the deployment and identity (workspace) for + # track events + analytics-tracking-deployments: + charset: "UTF-8" + expire-after-access: 10m + analytics-tracking-identity: + charset: "UTF-8" + expire-after-access: 10m env: cloud-deduction: true server: @@ -129,6 +138,9 @@ endpoints: beans: enabled: true sensitive: false + caches: + enabled: true + sensitive: false env: enabled: true sensitive: false diff --git a/airbyte-cron/build.gradle.kts b/airbyte-cron/build.gradle.kts index 03aafeb83f3..706f2d5c2e4 100644 --- a/airbyte-cron/build.gradle.kts +++ b/airbyte-cron/build.gradle.kts @@ -17,6 +17,7 @@ dependencies { implementation(platform(libs.micronaut.platform)) implementation(libs.bundles.micronaut) + implementation(libs.bundles.micronaut.cache) implementation(libs.bundles.micronaut.metrics) implementation(libs.bundles.kubernetes.client) implementation(libs.bundles.temporal) diff --git a/airbyte-cron/src/main/resources/application.yml b/airbyte-cron/src/main/resources/application.yml index 9397777cf98..ce823361448 100644 --- a/airbyte-cron/src/main/resources/application.yml +++ b/airbyte-cron/src/main/resources/application.yml @@ -1,6 +1,17 @@ micronaut: application: name: airbyte-cron + caches: + # used by the analytics tracking client to cache calls to resolve the deployment and identity (workspace) for + # track events + analytics-tracking-deployments: + charset: "UTF-8" + expire-after-access: 10m + analytics-tracking-identity: + charset: "UTF-8" + expire-after-access: 10m + remote-definitions-provider: + expire-after-write: 15s env: cloud-deduction: true metrics: @@ -12,9 +23,6 @@ micronaut: step: ${MICROMETER_METRICS_STEP:PT30S} host: ${STATSD_HOST:localhost} port: ${STATSD_PORT:8125} - caches: - remote-definitions-provider: - expire-after-write: 15s server: port: 9001 @@ -106,6 +114,9 @@ endpoints: beans: enabled: true sensitive: false + caches: + enabled: true + sensitive: false env: enabled: true sensitive: false diff --git a/airbyte-server/build.gradle.kts b/airbyte-server/build.gradle.kts index 36b22f08082..36b244db7cd 100644 --- a/airbyte-server/build.gradle.kts +++ b/airbyte-server/build.gradle.kts @@ -31,6 +31,7 @@ dependencies { implementation(platform(libs.micronaut.platform)) implementation(libs.bundles.micronaut) + implementation(libs.bundles.micronaut.cache) implementation(libs.bundles.micronaut.data.jdbc) implementation(libs.bundles.micronaut.metrics) implementation(libs.micronaut.jaxrs.server) diff --git a/airbyte-server/src/main/resources/application.yml b/airbyte-server/src/main/resources/application.yml index c3b6a231f7f..07caf1e1504 100644 --- a/airbyte-server/src/main/resources/application.yml +++ b/airbyte-server/src/main/resources/application.yml @@ -1,6 +1,15 @@ micronaut: application: name: airbyte-server + caches: + # used by the analytics tracking client to cache calls to resolve the deployment and identity (workspace) for + # track events + analytics-tracking-deployments: + charset: "UTF-8" + expire-after-access: 10m + analytics-tracking-identity: + charset: "UTF-8" + expire-after-access: 10m env: cloud-deduction: true executors: @@ -344,6 +353,9 @@ endpoints: beans: enabled: true sensitive: false + caches: + enabled: true + sensitive: false env: enabled: true sensitive: false diff --git a/airbyte-workers/build.gradle.kts b/airbyte-workers/build.gradle.kts index 1f77a3c3be8..85d06cd3d30 100644 --- a/airbyte-workers/build.gradle.kts +++ b/airbyte-workers/build.gradle.kts @@ -48,8 +48,8 @@ dependencies { implementation(platform(libs.micronaut.platform)) implementation(libs.google.cloud.storage) implementation(libs.bundles.micronaut) + implementation(libs.bundles.micronaut.cache) implementation(libs.bundles.micronaut.metrics) - implementation(libs.micronaut.cache.caffeine) implementation(libs.jooq) implementation(libs.s3) implementation(libs.sts) diff --git a/airbyte-workers/src/main/resources/application.yml b/airbyte-workers/src/main/resources/application.yml index 89078945fc8..60418103694 100644 --- a/airbyte-workers/src/main/resources/application.yml +++ b/airbyte-workers/src/main/resources/application.yml @@ -9,6 +9,14 @@ micronaut: max-content-length: 52428800 # 50MB port: 9000 caches: + # used by the analytics tracking client to cache calls to resolve the deployment and identity (workspace) for + # track events + analytics-tracking-deployments: + charset: "UTF-8" + expire-after-access: 10m + analytics-tracking-identity: + charset: "UTF-8" + expire-after-access: 10m # used by RecordMetricActivity to cache repeated calls to workspaceApi.getWorkspaceByConnectionId. # entries are UUID connectionIds, so 100,000 entries should be roughly 1.6MB connection-workspace-id: @@ -275,6 +283,9 @@ endpoints: beans: enabled: true sensitive: false + caches: + enabled: true + sensitive: false env: enabled: true sensitive: false diff --git a/deps.toml b/deps.toml index cca2140cf84..80ecb23574c 100644 --- a/deps.toml +++ b/deps.toml @@ -21,6 +21,7 @@ kubernetes-client = "6.5.1" log4j = "2.22.1" lombok = "1.18.30" micronaut = "4.3.5" +micronaut-cache = "4.2.2" micronaut-data = "4.6.1" micronaut-email = "2.4.0" micronaut-jaxrs = "4.3.0" @@ -184,7 +185,8 @@ vault-java-driver = { module = "com.bettercloud:vault-java-driver", version = "5 # Micronaut-related dependencies h2-database = { module = "com.h2database:h2", version = "2.2.224" } -micronaut-cache-caffeine = { module = "io.micronaut.cache:micronaut-cache-caffeine", version = "4.2.2" } +micronaut-cache-caffeine = { module = "io.micronaut.cache:micronaut-cache-caffeine", version.ref = "micronaut-cache" } +micronaut-cache-management = { module = "io.micronaut.cache:micronaut-cache-management", version.ref = "micronaut-cache" } micronaut-data-connection = { module = "io.micronaut.data:micronaut-data-connection", version.ref = "micronaut-data" } micronaut-data-processor = { module = "io.micronaut.data:micronaut-data-processor", version.ref = "micronaut-data" } micronaut-data-jdbc = { module = "io.micronaut.data:micronaut-data-jdbc", version.ref = "micronaut-data" } @@ -238,6 +240,7 @@ micronaut = ["jakarta-annotation-api", "jakarta-transaction-api", "micronaut-htt micronaut-light = ["jakarta-annotation-api", "micronaut-inject-java", "micronaut-runtime"] micronaut-annotation = ["jakarta-annotation-api", "micronaut-inject-java", "micronaut-inject-kotlin"] micronaut-annotation-processor = ["micronaut-inject-java", "micronaut-management", "micronaut-validation", "micronaut-data-processor", "micronaut-jaxrs-processor", "micronaut-http-validation"] +micronaut-cache = ["micronaut-cache-caffeine", "micronaut-cache-management"] micronaut-data-jdbc = [ "micronaut-data-connection", "micronaut-data-jdbc", "micronaut-data-model", "micronaut-data-tx", "jakarta-persistence-api", "micronaut-jdbc-hikari"] micronaut-email = ["micronaut-email", "micronaut-email-sendgrid"] micronaut-kotlin = ["micronaut-kotlin-extension-functions", "micronaut-kotlin-runtime"]