From b5308ca8e4b7e83ca492aff3d4224f7d6ce778d2 Mon Sep 17 00:00:00 2001 From: FANNG Date: Thu, 1 Aug 2024 19:50:17 +0800 Subject: [PATCH] [#4059] feat(iceberg-rest-server): support Iceberg REST server docker image (#4138) ### What changes were proposed in this pull request? add gravitino iceberg rest service docker image ### Why are the changes needed? 1. setup Gravition Iceberg REST service easily for test 2. user could add customer changes and setup a new docker image for product. Fix: #4059 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? in local machine, setup docker image and check it works --- dev/docker/build-docker.sh | 4 +- dev/docker/gravitino/Dockerfile | 4 +- dev/docker/iceberg-rest-server/Dockerfile | 29 +++++++++++++ .../iceberg-rest-server-dependency.sh | 42 +++++++++++++++++++ docs/docker-image-details.md | 20 ++++++++- 5 files changed, 94 insertions(+), 5 deletions(-) create mode 100644 dev/docker/iceberg-rest-server/Dockerfile create mode 100755 dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh diff --git a/dev/docker/build-docker.sh b/dev/docker/build-docker.sh index c604f6b37f5..71c47c0ea9b 100755 --- a/dev/docker/build-docker.sh +++ b/dev/docker/build-docker.sh @@ -27,7 +27,7 @@ usage() { cat << EOF Usage: -./build-docker.sh --platform [all|linux/amd64|linux/arm64] --type [gravitino|hive|kerberos-hive|trino|doris|ranger] --image {image_name} --tag {tag_name} --latest +./build-docker.sh --platform [all|linux/amd64|linux/arm64] --type [gravitino|hive|kerberos-hive|trino|doris|ranger|iceberg-rest-server] --image {image_name} --tag {tag_name} --latest Notice: You shouldn't use 'all' for the platform if you don't use the Github action to publish the Docker image. EOF @@ -113,6 +113,8 @@ elif [ "${component_type}" == "doris" ]; then elif [ "${component_type}" == "ranger" ]; then . ${script_dir}/ranger/ranger-dependency.sh build_args="--build-arg RANGER_PACKAGE_NAME=${RANGER_PACKAGE_NAME} --build-arg MYSQL_CONNECTOR_PACKAGE_NAME=${MYSQL_CONNECTOR_PACKAGE_NAME} --build-arg LOG4JDBC_PACKAGE_NAME=${LOG4JDBC_PACKAGE_NAME} --build-arg RANGER_VERSION=${RANGER_VERSION}" +elif [ "${component_type}" == "iceberg-rest-server" ]; then + . ${script_dir}/iceberg-rest-server/iceberg-rest-server-dependency.sh else echo "ERROR : ${component_type} is not a valid component type" usage diff --git a/dev/docker/gravitino/Dockerfile b/dev/docker/gravitino/Dockerfile index 157822d9c43..763884fc8ba 100644 --- a/dev/docker/gravitino/Dockerfile +++ b/dev/docker/gravitino/Dockerfile @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. # -FROM openjdk:8-jdk-buster +FROM openjdk:17 LABEL maintainer="dev@gravitino.apache.org" WORKDIR /root/gravitino @@ -26,4 +26,4 @@ COPY packages/gravitino /root/gravitino EXPOSE 8090 EXPOSE 9001 -ENTRYPOINT ["/bin/bash", "/root/gravitino/bin/gravitino.sh", "start"] \ No newline at end of file +ENTRYPOINT ["/bin/bash", "/root/gravitino/bin/gravitino.sh", "start"] diff --git a/dev/docker/iceberg-rest-server/Dockerfile b/dev/docker/iceberg-rest-server/Dockerfile new file mode 100644 index 00000000000..a85f6707275 --- /dev/null +++ b/dev/docker/iceberg-rest-server/Dockerfile @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +FROM openjdk:17 + +LABEL maintainer="dev@gravitino.apache.org" + +WORKDIR /root/gravitino-iceberg-rest-server + +COPY packages/gravitino-iceberg-rest-server /root/gravitino-iceberg-rest-server + +EXPOSE 9001 + +ENTRYPOINT ["/bin/bash", "/root/gravitino-iceberg-rest-server/bin/gravitino-iceberg-rest-server.sh", "start"] diff --git a/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh b/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh new file mode 100755 index 00000000000..8581cc5be2e --- /dev/null +++ b/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +set -ex +iceberg_rest_server_dir="$(dirname "${BASH_SOURCE-$0}")" +iceberg_rest_server_dir="$(cd "${iceberg_rest_server_dir}">/dev/null; pwd)" +gravitino_home="$(cd "${iceberg_rest_server_dir}/../../..">/dev/null; pwd)" + +# Prepare the Iceberg REST server packages +cd ${gravitino_home} +./gradlew clean assembleIcebergRESTServer -x test + +# Removed old packages +rm -rf "${iceberg_rest_server_dir}/packages" +mkdir -p "${iceberg_rest_server_dir}/packages" + +cd distribution +tar xfz gravitino-iceberg-rest-server-*.tar.gz +cp -r gravitino-iceberg-rest-server*-bin ${iceberg_rest_server_dir}/packages/gravitino-iceberg-rest-server + +# Keeping the container running at all times +cat <> "${iceberg_rest_server_dir}/packages/gravitino-iceberg-rest-server/bin/gravitino-iceberg-rest-server.sh" + +# Keeping a process running in the background +tail -f /dev/null +EOF diff --git a/docs/docker-image-details.md b/docs/docker-image-details.md index d9f7638e8e7..16c7a965932 100644 --- a/docs/docker-image-details.md +++ b/docs/docker-image-details.md @@ -7,8 +7,6 @@ license: "This software is licensed under the Apache License version 2." # User Docker images -There are two kinds of Docker images you can use: the Apache Gravitino Docker image and playground Docker images. - ## Apache Gravitino Docker image You can deploy the service with the Gravitino Docker image. @@ -44,6 +42,24 @@ Changelog - `8090` Gravitino Web UI - `9001` Iceberg REST service +## Apache Gravitino Iceberg REST server Docker image + +You can deploy the standalone Gravitino Iceberg REST server with the Docker image. + +Container startup commands + +```shell +docker run --rm -d -p 9001:9001 datastrato/gravitino-iceberg-rest-server +``` + +Changelog + +- gravitino-iceberg-rest-server:0.6.0 + - Docker image `datastrato/gravitino-iceberg-rest-server:0.6.0` + - Gravitino Iceberg REST Server with memory catalog backend. + - Expose ports: + - `9001` Iceberg REST service + ## Playground Docker image You can use the [playground](https://github.com/apache/gravitino-playground) to experience the whole Gravitino system with other components.