feat(e2e): move e2e tests

.github/workflows/codeql-analysis.yml

@@ -74,8 +74,15 @@ jobs:
           java-version: 11
           distribution: temurin
 
+      - name: Read version from properties file
+        id: read-version
+        run: |
+          version=$(grep -oP 'version=\K[^[:space:]]+' gradle.properties)
+          echo "Version found: $version"
+          echo "VERSION=$version" >> $GITHUB_ENV
+
       - name: Build with Gradle
-        run: ./gradlew build
+        run: make build/distributions/tiered-storage-for-apache-kafka-${{ env.VERSION }}.tgz
 
       - name: Perform CodeQL Analysis
         uses: github/codeql-action/analyze@v2

.github/workflows/main_push_and_pull_request_workflow.yml

@@ -25,12 +25,46 @@ jobs:
           distribution: temurin
 
       - name: Build with Gradle
-        run: ./gradlew build distTar -x integrationTest
+        run: make build
 
       - name: Run integration tests
-        run: ./gradlew integrationTest
+        run: make integration_test
+
+      - name: Upload Build Artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: build-${{ matrix.java-version }}
+          path: build/distributions/*.tgz
+  e2e_test:
+    strategy:
+      matrix:
+        java-version: [ 11, 17 ]
+        runs-on: [ ubuntu-latest ]
+    name: E2E tests on ${{ matrix.runs-on }} with jdk ${{ matrix.java-version }}
+    runs-on: ${{ matrix.runs-on }}
+    needs:
+      - build
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Download Build Artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: build-${{ matrix.java-version }}
+          path: build/distributions
+
+      - name: Display structure of downloaded files
+        run: ls -R
+        working-directory: build/distributions
 
       - name: Build Docker image
         run: make docker_image
 
+      - name: Run E2E tests
+        timeout-minutes: 30
+        run: make e2e_test
+
       # TODO: publish docker image

Makefile

@@ -13,19 +13,30 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ##
-VERSION=0.0.1-SNAPSHOT
-IMAGE_TAG=aivenoy/kafka:3.3-2022-10-06-tiered-storage-1-ts-2
+VERSION := $(shell grep -oP 'version=\K[^[:space:]]+' gradle.properties)
+IMAGE_NAME := aivenoy/kafka-with-ts-plugin
+IMAGE_VERSION := latest
+IMAGE_TAG := $(IMAGE_NAME):$(IMAGE_VERSION)
+
+.PHONY: clean checkstyle build integration_test e2e_test docker_image docker_push
 
-.PHONY: clean
 clean:
 	./gradlew clean
 
+checkstyle:
+	./gradlew checkstyleMain checkstyleTest checkstyleIntegrationTest
+
+build: build/distributions/tiered-storage-for-apache-kafka-$(VERSION).tgz
+
 build/distributions/tiered-storage-for-apache-kafka-$(VERSION).tgz:
-	./gradlew build distTar -x integrationTest
+	./gradlew build distTar -x integrationTest -x e2e:test
 
 integration_test: build/distributions/tiered-storage-for-apache-kafka-$(VERSION).tgz
 	./gradlew integrationTest
 
+e2e_test:  build/distributions/tiered-storage-for-apache-kafka-$(VERSION).tgz
+	./gradlew e2e:test
+
 .PHONY: docker_image
 docker_image: build/distributions/tiered-storage-for-apache-kafka-$(VERSION).tgz
 	docker build . \

build.gradle

@@ -103,14 +103,18 @@ subprojects {
         slf4jVersion = "1.7.36"
 
         // Don't bump this version without need, as this is the min supported version for the plugin.
-        kafkaVersion = "3.0.0"
+        kafkaVersion = "3.3.2"
 
         assertJVersion = "3.24.2"
 
         apacheCommonsIOVersion = "2.13.0"
 
         jacksonVersion = "2.15.2"
 
+        awaitilityVersion = "4.2.0"
+
+        awsSdkVersion = "1.12.520"
+
         testcontainersVersion = "1.18.3"
     }
 
@@ -133,6 +137,8 @@ subprojects {
         testImplementation "org.mockito:mockito-core:$mockitoVersion"
         testImplementation "org.mockito:mockito-junit-jupiter:$mockitoVersion"
 
+        testImplementation "org.awaitility:awaitility:$awaitilityVersion"
+
         testRuntimeOnly "org.slf4j:slf4j-log4j12:$slf4jVersion"
     }
 
@@ -253,3 +259,8 @@ tasks.register('validateDependencies') {
         assert !conflictsFound : "Dependency conflicts found!"
     }
 }
+
+// TODO fix GCP dependency issues
+//tasks.named("check") {
+//    dependsOn(tasks.named("validateDependencies"))
+//}

checkstyle/suppressions.xml

@@ -29,7 +29,10 @@
     <suppress checks="ClassDataAbstractionCoupling" files="Metrics.java"/>
     <suppress checks="ClassDataAbstractionCoupling" files="MetricCollector.java"/>
     <suppress checks="CyclomaticComplexity" files="MetricCollector.java"/>
+    <suppress checks="CyclomaticComplexity" files="SingleBrokerTest.java"/>
+    <suppress checks="NPathComplexity" files="SingleBrokerTest.java"/>
     <suppress checks="JavaNCSSCheck" files="MetricsRegistry.java"/>
     <suppress checks="JavaNCSSCheck" files="RemoteStorageManagerMetricsTest.java"/>
+    <suppress checks="JavaNCSSCheck" files="SingleBrokerTest.java"/>
     <suppress checks="NPathComplexity" files="CredentialsBuilder.java"/>
 </suppressions>

core/build.gradle

@@ -27,8 +27,6 @@ ext {
     caffeineVersion = "3.1.7"
 
     zstdVersion = "1.5.5-5"
-
-    awaitilityVersion = "4.2.0"
 }
 
 dependencies {
@@ -51,6 +49,5 @@ dependencies {
     testImplementation(project(":storage:filesystem"))
 
     testImplementation "com.github.luben:zstd-jni:$zstdVersion"
-    testImplementation "org.awaitility:awaitility:$awaitilityVersion"
     integrationTestImplementation sourceSets.test.output
 }

demo/compose-local-fs.yml

@@ -23,7 +23,7 @@ services:
       ZOOKEEPER_CLIENT_PORT: 2181
 
   kafka:
-    image: "aivenoy/kafka:3.3-2022-10-06-tiered-storage-1-ts-2"
+    image: "aivenoy/kafka-with-ts-plugin"
     container_name: "kafka-ts"
     depends_on:
       - zookeeper

demo/compose-s3-aws.yml

@@ -23,7 +23,7 @@ services:
       ZOOKEEPER_CLIENT_PORT: 2181
 
   kafka:
-    image: "aivenoy/kafka:3.3-2022-10-06-tiered-storage-1-ts-2"
+    image: "aivenoy/kafka-with-ts-plugin"
     container_name: "kafka-ts"
     depends_on:
       - zookeeper

demo/compose-s3-minio.yml

@@ -23,7 +23,7 @@ services:
       ZOOKEEPER_CLIENT_PORT: 2181
 
   kafka:
-    image: "aivenoy/kafka:3.3-2022-10-06-tiered-storage-1-ts-2"
+    image: "aivenoy/kafka-with-ts-plugin"
     container_name: "kafka-ts"
     depends_on:
       - zookeeper

e2e/README.md

@@ -0,0 +1,8 @@
+# End-to-end tests for Kafka tiered storage
+
+## Usage
+
+Docker is needed for running the tests.
+
+1. Build the image with < TBD >.
+2. `./gradlew test`

e2e/build.gradle

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2023 Aiven Oy
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+dependencies {
+    testImplementation "org.apache.kafka:kafka-clients:$kafkaVersion"
+    testImplementation "org.apache.kafka:kafka-storage:$kafkaVersion"
+    testImplementation "org.apache.kafka:kafka-storage-api:$kafkaVersion"
+
+    testImplementation "commons-io:commons-io:$apacheCommonsIOVersion"
+    testImplementation "com.amazonaws:aws-java-sdk-s3:$awsSdkVersion"
+
+    testImplementation "org.testcontainers:junit-jupiter:$testcontainersVersion"
+    testImplementation "org.testcontainers:kafka:$testcontainersVersion"
+
+    testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:$junitVersion"
+    testRuntimeOnly "org.slf4j:slf4j-log4j12:$slf4jVersion"
+}
+
+tasks.named('test') {
+    // Use junit platform for unit tests.
+    useJUnitPlatform()
+    testLogging {
+        events 'passed', 'skipped', 'failed'
+        showStandardStreams = true
+        showExceptions = true
+        showStackTraces = true
+        showCauses = true
+        exceptionFormat "full"
+    }
+}

e2e/src/test/java/io/aiven/kafka/tieredstorage/e2e/LocalSystemSingleBrokerTest.java

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2023 Aiven Oy
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.aiven.kafka.tieredstorage.e2e;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.Uuid;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+
+public class LocalSystemSingleBrokerTest extends SingleBrokerTest {
+    static final String TS_DATA_SUBDIR_HOST = "ts-data";
+    static final String TS_DATA_DIR_CONTAINER = "/home/appuser/kafka-tiered-storage";
+
+    static Path tieredDataDir;
+
+    @BeforeAll
+    static void init() throws Exception {
+        setupKafka(kafka -> {
+            tieredDataDir = baseDir.resolve(TS_DATA_SUBDIR_HOST);
+            tieredDataDir.toFile().mkdirs();
+            tieredDataDir.toFile().setWritable(true, false);
+
+            kafka
+                .withEnv("KAFKA_RSM_CONFIG_STORAGE_BACKEND_CLASS",
+                    "io.aiven.kafka.tieredstorage.storage.filesystem.FileSystemStorage")
+                .withEnv("KAFKA_RSM_CONFIG_STORAGE_ROOT", TS_DATA_DIR_CONTAINER)
+                .withFileSystemBind(tieredDataDir.toString(), TS_DATA_DIR_CONTAINER);
+        });
+    }
+
+    @AfterAll
+    static void cleanup() {
+        stopKafka();
+        cleanupStorage();
+    }
+
+    @Override
+    boolean assertNoTopicDataOnTierStorage(final String topicName, final Uuid topicId) {
+        final String prefix = String.format("%s-%s", topicName, topicId.toString());
+        try (final var files = Files.list(tieredDataDir)) {
+            return files.noneMatch(path -> path.getFileName().startsWith(prefix));
+        } catch (final IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    List<String> remotePartitionFiles(final TopicIdPartition topicIdPartition) {
+        final Path dir = tieredDataDir.resolve(
+            String.format("%s-%s/%s",
+                topicIdPartition.topic(),
+                topicIdPartition.topicId().toString(),
+                topicIdPartition.partition()));
+        try (final var paths = Files.list(dir)) {
+            return paths.map(Path::getFileName)
+                .map(Path::toString)
+                .sorted()
+                .collect(Collectors.toList());
+        } catch (final IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}