Merge pull request #392 from Aiven-Open/ivanyu/separate-distro

Exclude s3 from main distribution
Aiven-Open · Sep 19, 2023 · 71b1ebb · 71b1ebb
2 parents 98d60a2 + 2639ed0
commit 71b1ebb
Show file tree

Hide file tree

Showing 9 changed files with 39 additions and 14 deletions.
diff --git a/Makefile b/Makefile
@@ -28,24 +28,30 @@ clean:
 checkstyle:
 	./gradlew checkstyleMain checkstyleTest checkstyleIntegrationTest
 
-build: build/distributions/tiered-storage-for-apache-kafka-$(VERSION).tgz
+build: build/distributions/tiered-storage-for-apache-kafka-$(VERSION).tgz storage/s3/build/distributions/s3-$(VERSION).tgz storage/gcs/build/distributions/gcs-$(VERSION).tgz
 
 build/distributions/tiered-storage-for-apache-kafka-$(VERSION).tgz:
 	./gradlew build distTar -x test -x integrationTest -x e2e:test
 
-test: build/distributions/tiered-storage-for-apache-kafka-$(VERSION).tgz
+storage/s3/build/distributions/s3-$(VERSION).tgz:
+	./gradlew build :storage:s3:distTar -x test -x integrationTest -x e2e:test
+
+storage/gcs/build/distributions/gcs-$(VERSION).tgz:
+	./gradlew build :storage:gcs:distTar -x test -x integrationTest -x e2e:test
+
+test: build
 	./gradlew test -x e2e:test
 
-integration_test: build/distributions/tiered-storage-for-apache-kafka-$(VERSION).tgz
+integration_test: build
 	./gradlew integrationTest
 
 E2E_TEST=LocalSystem
 
-e2e_test:  build/distributions/tiered-storage-for-apache-kafka-$(VERSION).tgz
+e2e_test: build
 	./gradlew e2e:test --tests $(E2E_TEST)*
 
 .PHONY: docker_image
-docker_image: build/distributions/tiered-storage-for-apache-kafka-$(VERSION).tgz
+docker_image: build
 	docker build . \
 		-f docker/Dockerfile \
 		--build-arg _VERSION=$(VERSION) \

diff --git a/build.gradle b/build.gradle
@@ -208,6 +208,11 @@ subprojects {
             }
         }
     }
+
+    tasks.distTar {
+        compression = Compression.GZIP
+        duplicatesStrategy DuplicatesStrategy.EXCLUDE
+    }
 }
 
 distributions {
@@ -217,8 +222,6 @@ distributions {
             from(project(":core").configurations.runtimeClasspath)
             from(project(":storage:core").jar)
             from(project(":storage:core").configurations.runtimeClasspath)
-            from(project(":storage:s3").jar)
-            from(project(":storage:s3").configurations.runtimeClasspath)
             from(project(":storage:filesystem").jar)
             from(project(":storage:filesystem").configurations.runtimeClasspath)
         }

diff --git a/demo/compose-local-fs.yml b/demo/compose-local-fs.yml
@@ -55,7 +55,7 @@ services:
       KAFKA_REMOTE_LOG_METADATA_MANAGER_LISTENER_NAME: "BROKER"
       KAFKA_RLMM_CONFIG_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR: 1
       # Remote storage manager
-      KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_PATH: "/tiered-storage-for-apache-kafka/*"
+      KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_PATH: "/tiered-storage-for-apache-kafka/core/*"
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_NAME: "io.aiven.kafka.tieredstorage.RemoteStorageManager"
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_IMPL_PREFIX: "rsm.config."
       KAFKA_RSM_CONFIG_CHUNK_SIZE: 5242880 # 5MiB

diff --git a/demo/compose-s3-aws.yml b/demo/compose-s3-aws.yml
@@ -55,7 +55,7 @@ services:
       KAFKA_REMOTE_LOG_METADATA_MANAGER_LISTENER_NAME: "BROKER"
       KAFKA_RLMM_CONFIG_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR: 1
       # Remote storage manager
-      KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_PATH: "/tiered-storage-for-apache-kafka/*"
+      KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_PATH: "/tiered-storage-for-apache-kafka/core/*:/tiered-storage-for-apache-kafka/s3/*"
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_NAME: "io.aiven.kafka.tieredstorage.RemoteStorageManager"
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_IMPL_PREFIX: "rsm.config."
       KAFKA_RSM_CONFIG_CHUNK_SIZE: 5242880 # 5MiB

diff --git a/demo/compose-s3-minio.yml b/demo/compose-s3-minio.yml
@@ -56,7 +56,7 @@ services:
       KAFKA_REMOTE_LOG_METADATA_MANAGER_LISTENER_NAME: "BROKER"
       KAFKA_RLMM_CONFIG_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR: 1
       # Remote storage manager
-      KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_PATH: "/tiered-storage-for-apache-kafka/*"
+      KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_PATH: "/tiered-storage-for-apache-kafka/core/*:/tiered-storage-for-apache-kafka/s3/*"
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_NAME: "io.aiven.kafka.tieredstorage.RemoteStorageManager"
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_IMPL_PREFIX: "rsm.config."
       KAFKA_RSM_CONFIG_CHUNK_SIZE: 5242880 # 5MiB

diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -18,12 +18,26 @@ FROM aivenoy/kafka:kips_KIP-405_3.6-2023-09-12
 ARG _VERSION
 
 USER root
-RUN mkdir /tiered-storage-for-apache-kafka
-COPY build/distributions/tiered-storage-for-apache-kafka-${_VERSION}.tgz /tiered-storage-for-apache-kafka
-RUN cd /tiered-storage-for-apache-kafka \
+
+RUN mkdir -p /tiered-storage-for-apache-kafka/core \
+    &&  mkdir -p /tiered-storage-for-apache-kafka/s3 \
+    &&  mkdir -p /tiered-storage-for-apache-kafka/gcs
+
+COPY build/distributions/tiered-storage-for-apache-kafka-${_VERSION}.tgz /tiered-storage-for-apache-kafka/core
+RUN cd /tiered-storage-for-apache-kafka/core \
     && tar -xf tiered-storage-for-apache-kafka-${_VERSION}.tgz --strip-components=1 \
     && rm tiered-storage-for-apache-kafka-${_VERSION}.tgz
 
+COPY storage/s3/build/distributions/s3-${_VERSION}.tgz /tiered-storage-for-apache-kafka/s3
+RUN cd /tiered-storage-for-apache-kafka/s3 \
+    && tar -xf s3-${_VERSION}.tgz --strip-components=1 \
+    && rm s3-${_VERSION}.tgz
+
+COPY storage/gcs/build/distributions/gcs-${_VERSION}.tgz /tiered-storage-for-apache-kafka/gcs
+RUN cd /tiered-storage-for-apache-kafka/gcs \
+    && tar -xf gcs-${_VERSION}.tgz --strip-components=1 \
+    && rm gcs-${_VERSION}.tgz
+
 # Installing JMX exporter agent
 ARG JMX_EXPORTER_VERSION=0.18.0
 RUN mkdir -p /opt/prometheus/jmx-exporter

diff --git a/e2e/src/test/java/io/aiven/kafka/tieredstorage/e2e/LocalSystemSingleBrokerTest.java b/e2e/src/test/java/io/aiven/kafka/tieredstorage/e2e/LocalSystemSingleBrokerTest.java
@@ -42,6 +42,7 @@ static void init() throws Exception {
             tieredDataDir.toFile().setWritable(true, false);
 
             kafka
+                .withEnv("KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_PATH", "/tiered-storage-for-apache-kafka/core/*")
                 .withEnv("KAFKA_RSM_CONFIG_STORAGE_BACKEND_CLASS",
                     "io.aiven.kafka.tieredstorage.storage.filesystem.FileSystemStorage")
                 .withEnv("KAFKA_RSM_CONFIG_STORAGE_ROOT", TS_DATA_DIR_CONTAINER)

diff --git a/e2e/src/test/java/io/aiven/kafka/tieredstorage/e2e/S3MinioSingleBrokerTest.java b/e2e/src/test/java/io/aiven/kafka/tieredstorage/e2e/S3MinioSingleBrokerTest.java
@@ -65,6 +65,8 @@ static void init() throws Exception {
 
         setupKafka(kafka -> kafka.withEnv("KAFKA_RSM_CONFIG_STORAGE_BACKEND_CLASS",
                 "io.aiven.kafka.tieredstorage.storage.s3.S3Storage")
+            .withEnv("KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_PATH",
+                "/tiered-storage-for-apache-kafka/core/*:/tiered-storage-for-apache-kafka/s3/*")
             .withEnv("KAFKA_RSM_CONFIG_STORAGE_S3_BUCKET_NAME", BUCKET)
             .withEnv("KAFKA_RSM_CONFIG_STORAGE_S3_REGION", REGION)
             .withEnv("KAFKA_RSM_CONFIG_STORAGE_S3_PATH_STYLE_ACCESS_ENABLED", "true")

diff --git a/e2e/src/test/java/io/aiven/kafka/tieredstorage/e2e/SingleBrokerTest.java b/e2e/src/test/java/io/aiven/kafka/tieredstorage/e2e/SingleBrokerTest.java
@@ -148,7 +148,6 @@ static void setupKafka(final Consumer<KafkaContainer> tsPluginSetup) throws Exce
             .withEnv("KAFKA_REMOTE_LOG_METADATA_MANAGER_LISTENER_NAME", "BROKER")
             .withEnv("KAFKA_RLMM_CONFIG_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR", "1")
             // remote storage manager
-            .withEnv("KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_PATH", "/tiered-storage-for-apache-kafka/*")
             .withEnv("KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_NAME",
                 "io.aiven.kafka.tieredstorage.RemoteStorageManager")
             .withEnv("KAFKA_REMOTE_LOG_STORAGE_MANAGER_IMPL_PREFIX", "rsm.config.")