fix: Include functional tests jar in docker images (#4274)

* build new server image with functional tests jar * added missing dependencies
confluentinc · Jan 29, 2020 · 2559b2f · 2559b2f
1 parent c1faef9
commit 2559b2f
Show file tree

Hide file tree

Showing 10 changed files with 273 additions and 18 deletions.
diff --git a/ksql-cli/Dockerfile b/ksql-cli/Dockerfile
diff --git a/ksql-cli/pom.xml b/ksql-cli/pom.xml
@@ -32,8 +32,8 @@
         <main-class>io.confluent.ksql.Ksql</main-class>
         <cli.skip-execute>false</cli.skip-execute>
         <cli.main-class>${main-class}</cli.main-class>
-        <docker.skip-build>${skip.docker.build}</docker.skip-build>
-        <docker.skip-test>${skip.docker.test}</docker.skip-test>
+        <docker.skip-build>true</docker.skip-build>
+        <docker.skip-test>true</docker.skip-test>
     </properties>
 
     <dependencies>

diff --git a/ksql-rest-app/Dockerfile → ksql-docker/Dockerfile b/ksql-rest-app/Dockerfile → ksql-docker/Dockerfile
@@ -6,7 +6,7 @@ FROM ${DOCKER_UPSTREAM_REGISTRY}confluentinc/cp-base-new:${DOCKER_UPSTREAM_TAG}
 ARG PROJECT_VERSION
 ARG ARTIFACT_ID
 
-ADD target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/
+ADD target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/ksql-rest-app/
 ADD target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/bin/* /usr/bin/
 ADD target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/bin/docker/* /usr/bin/docker/
 ADD target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/etc/* /etc/ksql/

diff --git a/ksql-docker/pom.xml b/ksql-docker/pom.xml
@@ -0,0 +1,106 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright 2018 Confluent Inc.
+  ~
+  ~ Licensed under the Confluent Community License (the "License"); you may not use
+  ~ this file except in compliance with the License.  You may obtain a copy of the
+  ~ License at
+  ~
+  ~ http://www.confluent.io/confluent-community-license
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  ~ WARRANTIES OF ANY KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <artifactId>ksql-parent</artifactId>
+    <groupId>io.confluent.ksql</groupId>
+    <version>5.5.0-SNAPSHOT</version>
+  </parent>
+
+  <artifactId>ksql-docker</artifactId>
+
+  <properties>
+    <air.main.basedir>${project.parent.basedir}</air.main.basedir>
+    <docker.skip-build>${skip.docker.build}</docker.skip-build>
+    <docker.skip-test>${skip.docker.test}</docker.skip-test>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>io.confluent.ksql</groupId>
+      <artifactId>ksql-rest-app</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>io.confluent.ksql</groupId>
+      <artifactId>ksql-cli</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>io.confluent.ksql</groupId>
+      <artifactId>ksql-functional-tests</artifactId>
+    </dependency>
+
+    <!-- Need to explicitly add dependencies with classifier as they are not automatically pulled
+         in. -->
+    <dependency>
+      <groupId>org.apache.kafka</groupId>
+      <artifactId>kafka-clients</artifactId>
+      <version>${kafka.version}</version>
+      <classifier>test</classifier>
+      <scope>compile</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.kafka</groupId>
+      <artifactId>kafka_${kafka.scala.version}</artifactId>
+      <classifier>test</classifier>
+      <scope>compile</scope>
+    </dependency>
+
+  </dependencies>
+
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <configuration>
+          <descriptors>
+            <descriptor>src/assembly/package.xml</descriptor>
+          </descriptors>
+          <attach>false</attach>
+        </configuration>
+        <executions>
+          <execution>
+            <id>make-assembly</id>
+            <phase>package</phase>
+            <goals>
+              <goal>single</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+        <executions>
+          <execution>
+            <goals>
+              <goal>test-jar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>
diff --git a/ksql-docker/src/assembly/package.xml b/ksql-docker/src/assembly/package.xml
@@ -0,0 +1,76 @@
+<!--
+  ~ Copyright 2018 Confluent Inc.
+  ~
+  ~ Licensed under the Confluent Community License (the "License"); you may not use
+  ~ this file except in compliance with the License.  You may obtain a copy of the
+  ~ License at
+  ~
+  ~ http://www.confluent.io/confluent-community-license
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  ~ WARRANTIES OF ANY KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations under the License.
+  -->
+
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2
+          http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+    <!-- Assembles a package that can run the ksql server given dependency packages. This is used to
+         construct packages OS packages where underlying libraries, like rest-utils, can be shared
+         -->
+    <id>package</id>
+    <formats>
+        <format>dir</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+    <fileSets>
+        <fileSet>
+            <directory>${project.parent.basedir}</directory>
+            <outputDirectory>share/doc/ksql-rest-app/</outputDirectory>
+            <includes>
+                <include>version.txt</include>
+                <include>COPYRIGHT*</include>
+            </includes>
+        </fileSet>
+        <fileSet>
+            <directory>${project.parent.basedir}</directory>
+            <outputDirectory></outputDirectory>
+            <includes>
+                <include>bin/*</include>
+            </includes>
+        </fileSet>
+        <fileSet>
+            <directory>${project.parent.basedir}/config</directory>
+            <outputDirectory>etc/ksql-rest-app</outputDirectory>
+            <includes>
+                <include>*</include>
+            </includes>
+        </fileSet>
+        <fileSet>
+            <directory>${project.basedir}/src/include</directory>
+            <outputDirectory>bin</outputDirectory>
+            <includes>
+                <include>docker/*</include>
+            </includes>
+        </fileSet>
+    </fileSets>
+    <dependencySets>
+        <dependencySet>
+            <outputDirectory>share/java/ksql-docker</outputDirectory>
+            <useProjectArtifact>true</useProjectArtifact>
+            <!-- Warning: Enabling useTransitiveFiltering is potentially unsafe,
+                 cf. https://github.com/miguno/maven-assembly-transitive-deps-woes
+                 and https://issues.apache.org/jira/browse/MASSEMBLY-504.  If
+                 dependencies happen to be missing in the assembly you may want to
+                 disable transitive filtering. -->
+            <useTransitiveFiltering>true</useTransitiveFiltering>
+            <excludes>
+                <exclude>io.confluent:rest-utils</exclude>
+                <exclude>io.confluent:common-*</exclude>
+                <exclude>com.google.guava:guava</exclude>
+            </excludes>
+        </dependencySet>
+    </dependencySets>
+</assembly>
diff --git a/ksql-docker/src/include/docker/configure b/ksql-docker/src/include/docker/configure
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+#
+# Copyright 2020 Confluent Inc.
+#
+# Licensed under the Confluent Community License (the "License"); you may not use
+# this file except in compliance with the License.  You may obtain a copy of the
+# License at
+#
+# http://www.confluent.io/confluent-community-license
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OF ANY KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations under the License.
+
+
+dub ensure KSQL_CONNECT_BOOTSTRAP_SERVERS
+dub ensure KSQL_CONNECT_GROUP_ID
+dub ensure KSQL_CONNECT_CONFIG_STORAGE_TOPIC
+dub ensure KSQL_CONNECT_OFFSET_STORAGE_TOPIC
+dub ensure KSQL_CONNECT_STATUS_STORAGE_TOPIC
+dub ensure KSQL_CONNECT_KEY_CONVERTER
+dub ensure KSQL_CONNECT_VALUE_CONVERTER
+# This is required to avoid config bugs. You should set this to a value that is
+# resolvable by all containers.
+dub ensure KSQL_CONNECT_REST_ADVERTISED_HOST_NAME
+
+# Default to 8083, which matches the mesos-overrides. This is here in case we extend the containers to remove the mesos overrides.
+if [ -z "$KSQL_CONNECT_REST_PORT" ]; then
+  export KSQL_CONNECT_REST_PORT=8083
+fi
+
+if [[ $KSQL_CONNECT_KEY_CONVERTER == "io.confluent.connect.avro.AvroConverter" ]]
+then
+  dub ensure KSQL_CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL
+fi
+
+if [[ $KSQL_CONNECT_VALUE_CONVERTER == "io.confluent.connect.avro.AvroConverter" ]]
+then
+  dub ensure KSQL_CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL
+fi
+
diff --git a/ksql-docker/src/include/docker/run b/ksql-docker/src/include/docker/run
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+#
+# Copyright 2019 Confluent Inc.
+#
+# Licensed under the Confluent Community License (the "License"); you may not use
+# this file except in compliance with the License.  You may obtain a copy of the
+# License at
+#
+# http://www.confluent.io/confluent-community-license
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OF ANY KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations under the License.
+#
+
+
+echo "===> Configuring ksqlDB..."
+dub template "/etc/ksql/connect.properties.template" "/etc/ksql/connect.properties"
+
+# KSQL_CONNECT_GROUP_ID is a required configuration for running embedded connect,
+# so we can proxy it to check whether or not to start an embedded connect worker
+if ! [[ -z "${KSQL_CONNECT_GROUP_ID}" ]]; then
+  echo "===> Configuring Embedded Connect"
+  /usr/bin/docker/configure
+
+  echo "===> Enabling Embedded Connect"
+  export KSQL_KSQL_CONNECT_WORKER_CONFIG="/etc/ksql/connect.properties"
+fi
+
+dub template "/etc/ksql/ksqldb-server.properties.template" "/etc/ksql/ksqldb-server.properties"
+
+echo "===> Launching ksqlDB Server..."
+/usr/bin/ksql-server-start /etc/ksql/ksqldb-server.properties
diff --git a/ksql-functional-tests/pom.xml b/ksql-functional-tests/pom.xml
@@ -106,7 +106,7 @@
       <groupId>io.confluent.ksql</groupId>
       <artifactId>ksql-rest-app</artifactId>
       <version>${project.version}</version>
-      <scope>test</scope>
+      <scope>compile</scope>
     </dependency>
 
     <dependency>

diff --git a/ksql-rest-app/pom.xml b/ksql-rest-app/pom.xml
@@ -33,8 +33,8 @@
         <main-class>io.confluent.ksql.rest.server.KsqlRestApplication</main-class>
         <cli.skip-execute>false</cli.skip-execute>
         <cli.main-class>${main-class}</cli.main-class>
-        <docker.skip-build>${skip.docker.build}</docker.skip-build>
-        <docker.skip-test>${skip.docker.test}</docker.skip-test>
+        <docker.skip-build>true</docker.skip-build>
+        <docker.skip-test>true</docker.skip-test>
     </properties>
 
     <dependencies>

diff --git a/pom.xml b/pom.xml
@@ -63,6 +63,7 @@
         <module>ksql-functional-tests</module>
         <module>ksql-rocksdb-config-setter</module>
         <module>ksql-api</module>
+        <module>ksql-docker</module>
     </modules>
 
     <repositories>
@@ -216,6 +217,12 @@
                 <version>${project.version}</version>
             </dependency>
 
+            <dependency>
+                <groupId>io.confluent.ksql</groupId>
+                <artifactId>ksql-functional-tests</artifactId>
+                <version>${project.version}</version>
+            </dependency>
+
             <dependency>
                 <groupId>io.confluent.ksql</groupId>
                 <artifactId>ksql-version-metrics-client</artifactId>