feat: add confluent-hub to ksqlDB docker image (#4729)

docker-compose.yml

@@ -1,6 +1,6 @@
-# Docker compose from bringing up a local KSQL cluster and dependencies.
+# Docker compose from bringing up a local ksqlDB cluster and dependencies.
 #
-# By default, the cluster has two KSQL servers. You can scale the number of KSQL nodes in the
+# By default, the cluster has two ksqlDB servers. You can scale the number of ksqlDB nodes in the
 # cluster by using the docker `--scale` command line arg.
 #
 # e.g. for a 4 node cluster run:
@@ -13,25 +13,12 @@
 # difference is that the primary node has a well-known port exposed so clients can connect, where
 # as the additional nodes use auto-port assignment so that ports don't clash.
 #
-# If you wish to run with locally built KSQL docker images then build then:
+# If you wish to run with locally built ksqlDB docker images then:
 #
-# 1. Ensure logged in to docker:
-# > docker login
+# 1. Follow the steps in https://github.com/confluentinc/ksql/blob/master/ksql-docker/README.md
+# to build a ksqlDB docker image with local changes.
 #
-# 2. Ensure you can log in to AWS ECR:
-# > aws ecr get-login --no-include-email | sh
-#
-# 3. Build docker images from local changes 
-#    (NOTE: This currently depends on a Docker image that is not publicly available.
-#           Access to Confluent docker registry is required, and should be specified in docker.upstream-registry)
-#    Change `docker.upstream-tag` if you want to depend on anything other than the latest master upstream, e.g. 5.4.x-latest
-# > mvn -Pdocker package -DskipTests -Dspotbugs.skip -Dcheckstyle.skip  -Ddockerfile.skip=false -Dskip.docker.build=false -Ddocker.upstream-tag=master-latest -Ddocker.tag=local.build  -Ddocker.upstream-registry='XXXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/'
-#
-# 4. check images build:
-# > docker image ls | grep ksql.local.build
-# You should see your new images listed
-#
-# 5. update this file below replacing all references to "confluentinc/ksqldb-server:latest" with your image, e.g. "placeholder/confluentinc/ksql-rest-app:local.build"
+# 2. Update this file below replacing all references to "confluentinc/ksqldb-server:latest" with your image, e.g. "placeholder/confluentinc/ksql-docker:local.build"
 
 ---
 version: '2'

docs-md/tutorials/embedded-connect.md

@@ -112,7 +112,7 @@ services:
       KSQL_CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
       KSQL_CONNECT_PLUGIN_PATH: "/usr/share/kafka/plugins"
     volumes:
-      # - ./confluentinc-kafka-connect-jdbc-VERSION_YOU_DOWNLOADED:/usr/share/kafka/plugins/jdbc
+      - ./confluent-hub-components/confluentinc-kafka-connect-jdbc:/usr/share/kafka/plugins/jdbc
 
   ksqldb-cli:
     image: confluentinc/ksqldb-cli:0.7.0
@@ -136,16 +136,15 @@ services:
 2. Get the JDBC connector
 -------------------------
 
-[Download the JDBC connector](https://www.confluent.io/hub/confluentinc/kafka-connect-jdbc)
-to your local working directory. Next, unzip the downloaded archive:
+The easiest way to download connectors for use in ksqlDB with embedded {{ site.kconnect }}
+is via the [Confluent Hub Client](https://docs.confluent.io/current/connect/managing/confluent-hub/client.html)
+included in the `ksqlDB-server` Docker image.
 
+To download the JDBC connector, use the following command:
 ```bash
-unzip confluentinc-kafka-connect-jdbc-*.zip
+docker run -v $PWD/confluent-hub-components:/share/confluent-hub-components confluentinc/ksqldb-server:0.7.0 confluent-hub install --no-prompt confluentinc/kafka-connect-jdbc:{{ site.cprelease }}
 ```
-
-Make sure to update the `docker-compose.yml` to uncomment the line mounting the volume in 
-`ksqldb-server`.
-
+This command downloads the JDBC connector into the directory `./confluent-hub-components`.
 
 3. Start ksqlDB and PostgreSQL
 ------------------------------

ksql-docker/Dockerfile

@@ -6,12 +6,17 @@ FROM ${DOCKER_UPSTREAM_REGISTRY}confluentinc/cp-base-new:${DOCKER_UPSTREAM_TAG}
 ARG PROJECT_VERSION
 ARG ARTIFACT_ID
 
-ADD target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/ksql-rest-app/
+ADD target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/
 ADD target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/bin/* /usr/bin/
 ADD target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/bin/docker/* /usr/bin/docker/
 ADD target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/etc/* /etc/ksql/
 ADD target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/
 
+RUN echo "===> Installing confluent-hub..." \
+    && wget http://client.hub.confluent.io/confluent-hub-client-latest.tar.gz \
+    && tar xf confluent-hub-client-latest.tar.gz \
+    && rm confluent-hub-client-latest.tar.gz
+
 RUN chmod +x /usr/bin/docker/configure
 RUN chmod +x /usr/bin/docker/run
 

ksql-docker/README.md

@@ -0,0 +1,24 @@
+# ksql-docker
+
+Module for building ksqlDB docker images.
+
+## To build locally
+
+To build a new image with local changes:
+
+1. Ensure you're logged in to docker:
+    ```
+    > docker login
+    ```
+
+1. Build docker images from local changes.
+    ```
+    > mvn -Pdocker package -DskipTests -Dspotbugs.skip -Dcheckstyle.skip  -Ddockerfile.skip=false -Dskip.docker.build=false -Ddocker.upstream-tag=latest -Ddocker.tag=local.build  -Ddocker.upstream-registry=''
+    ```
+   Change `docker.upstream-tag` if you want to depend on anything other than the latest master upstream, e.g. 5.4.x-latest.
+
+1. Check the image was built:
+    ```
+    > docker image ls | grep local.build
+    ```
+    You should see the new image listed.

ksql-docker/src/assembly/package.xml

@@ -28,7 +28,7 @@
     <fileSets>
         <fileSet>
             <directory>${project.parent.basedir}</directory>
-            <outputDirectory>share/doc/ksql-rest-app/</outputDirectory>
+            <outputDirectory>share/doc/ksql-docker/</outputDirectory>
             <includes>
                 <include>version.txt</include>
                 <include>COPYRIGHT*</include>
@@ -43,7 +43,7 @@
         </fileSet>
         <fileSet>
             <directory>${project.parent.basedir}/config</directory>
-            <outputDirectory>etc/ksql-rest-app</outputDirectory>
+            <outputDirectory>etc/ksql-docker</outputDirectory>
             <includes>
                 <include>*</include>
             </includes>