-
Notifications
You must be signed in to change notification settings - Fork 386
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Browse the repository at this point in the history
### What changes were proposed in this pull request? - Build a minimal Trino Docker image. - Provides a choice of the Docker image type when the build Docker image in the GitHub Action.  ### Why are the changes needed? Gravitino uses Trino for integration tests. Trino has more than 40 plugins, Which makes the Trino server start up very slowly, usually taking 2~3 minutes. We need to rebuild the Gravitino Trino Docker image and remove unused plugins. Fix: #614 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? I build Trino Docker image success. https://hub.docker.com/repository/docker/datastrato/gravitino-ci-trino/general
- Loading branch information
Showing
8 changed files
with
230 additions
and
130 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,99 @@ | ||
| #!/bin/bash | ||
| # | ||
| # Copyright 2023 Datastrato. | ||
| # This software is licensed under the Apache License version 2. | ||
| # | ||
| set -ex | ||
| script_dir="$(dirname "${BASH_SOURCE-$0}")" | ||
| script_dir="$(cd "${script_dir}">/dev/null; pwd)" | ||
|
|
||
| # Build docker image for multi-arch | ||
| USAGE="-e Usage: ./build-docker.sh --platform [all|linux/amd64|linux/arm64] --type [hive|trino] --image {image_name} --tag {tag_name} --latest" | ||
|
|
||
| # Get platform type | ||
| if [[ "$1" == "--platform" ]]; then | ||
| shift | ||
| platform_type="$1" | ||
| if [[ "${platform_type}" == "linux/amd64" || "${platform_type}" == "linux/arm64" || "${platform_type}" == "all" ]]; then | ||
| echo "INFO : platform type is ${platform_type}" | ||
| else | ||
| echo "ERROR : ${platform_type} is not a valid platform type" | ||
| echo ${USAGE} | ||
| exit 1 | ||
| fi | ||
| shift | ||
| else | ||
| platform_type="all" | ||
| fi | ||
|
|
||
| # Get component type | ||
| if [[ "$1" == "--type" ]]; then | ||
| shift | ||
| component_type="$1" | ||
| shift | ||
| else | ||
| echo "ERROR : must specify component type" | ||
| echo ${USAGE} | ||
| exit 1 | ||
| fi | ||
|
|
||
| # Get docker image name | ||
| if [[ "$1" == "--image" ]]; then | ||
| shift | ||
| image_name="$1" | ||
| shift | ||
| else | ||
| echo "ERROR : must specify image name" | ||
| echo ${USAGE} | ||
| exit 1 | ||
| fi | ||
|
|
||
| # Get docker image tag | ||
| if [[ "$1" == "--tag" ]]; then | ||
| shift | ||
| tag_name="$1" | ||
| shift | ||
| fi | ||
|
|
||
| # Get latest flag | ||
| build_latest=0 | ||
| if [[ "$1" == "--latest" ]]; then | ||
| shift | ||
| build_latest=1 | ||
| fi | ||
|
|
||
| if [[ "${component_type}" == "hive" ]]; then | ||
| . ${script_dir}/hive/hive-dependency.sh | ||
| build_args="--build-arg HADOOP_PACKAGE_NAME=${HADOOP_PACKAGE_NAME} --build-arg HIVE_PACKAGE_NAME=${HIVE_PACKAGE_NAME}" | ||
| elif [ "${component_type}" == "trino" ]; then | ||
| true # Placeholder, do nothing | ||
| else | ||
| echo "ERROR : ${component_type} is not a valid component type" | ||
| echo ${USAGE} | ||
| exit 1 | ||
| fi | ||
|
|
||
| # Create multi-arch builder | ||
| BUILDER_NAME="gravitino-builder" | ||
| builders=$(docker buildx ls) | ||
| if echo "${builders}" | grep -q "${BUILDER_NAME}"; then | ||
| echo "BuildKit builder '${BUILDER_NAME}' already exists." | ||
| else | ||
| echo "BuildKit builder '${BUILDER_NAME}' does not exist." | ||
| docker buildx create --platform linux/amd64,linux/arm64 --use --name ${BUILDER_NAME} | ||
| fi | ||
|
|
||
| cd ${script_dir}/${component_type} | ||
| if [[ "${platform_type}" == "all" ]]; then | ||
| if [ ${build_latest} -eq 1 ]; then | ||
| docker buildx build --platform=linux/amd64,linux/arm64 ${build_args} --push --progress plain -f Dockerfile -t ${image_name}:latest -t ${image_name}:${tag_name} . | ||
| else | ||
| docker buildx build --platform=linux/amd64,linux/arm64 ${build_args} --push --progress plain -f Dockerfile -t ${image_name}:${tag_name} . | ||
| fi | ||
| else | ||
| if [ ${build_latest} -eq 1 ]; then | ||
| docker buildx build --platform=${platform_type} ${build_args} --output type=docker --progress plain -f Dockerfile -t ${image_name}:latest -t ${image_name}:${tag_name} . | ||
| else | ||
| docker buildx build --platform=${platform_type} ${build_args} --output type=docker --progress plain -f Dockerfile -t ${image_name}:${tag_name} . | ||
| fi | ||
| fi |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| #!/bin/bash | ||
| # | ||
| # Copyright 2023 Datastrato. | ||
| # This software is licensed under the Apache License version 2. | ||
| # | ||
| set -ex | ||
| hive_dir="$(dirname "${BASH_SOURCE-$0}")" | ||
| hive_dir="$(cd "${hive_dir}">/dev/null; pwd)" | ||
|
|
||
| # Environment variables definition | ||
| HADOOP_VERSION="2.7.3" | ||
| HIVE_VERSION="2.3.9" | ||
|
|
||
| HADOOP_PACKAGE_NAME="hadoop-${HADOOP_VERSION}.tar.gz" # Must export this variable for Dockerfile | ||
| HADOOP_DOWNLOAD_URL="http://archive.apache.org/dist/hadoop/core/hadoop-${HADOOP_VERSION}/${HADOOP_PACKAGE_NAME}" | ||
|
|
||
| HIVE_PACKAGE_NAME="apache-hive-${HIVE_VERSION}-bin.tar.gz" # Must export this variable for Dockerfile | ||
| HIVE_DOWNLOAD_URL="https://archive.apache.org/dist/hive/hive-${HIVE_VERSION}/${HIVE_PACKAGE_NAME}" | ||
|
|
||
| # Prepare download packages | ||
| if [[ ! -d "${hive_dir}/packages" ]]; then | ||
| mkdir -p "${hive_dir}/packages" | ||
| fi | ||
|
|
||
| if [ ! -f "${hive_dir}/packages/${HADOOP_PACKAGE_NAME}" ]; then | ||
| curl -s -o "${hive_dir}/packages/${HADOOP_PACKAGE_NAME}" ${HADOOP_DOWNLOAD_URL} | ||
| fi | ||
|
|
||
| if [ ! -f "${hive_dir}/packages/${HIVE_PACKAGE_NAME}" ]; then | ||
| curl -s -o "${hive_dir}/packages/${HIVE_PACKAGE_NAME}" ${HIVE_DOWNLOAD_URL} | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| # | ||
| # Copyright 2023 Datastrato. | ||
| # This software is licensed under the Apache License version 2. | ||
| # | ||
| FROM trinodb/trino:426 | ||
| LABEL maintainer="[email protected]" | ||
|
|
||
| # Only mysql, hudi, iceberg, mariadb, jmx, memory, tpch, tpcds, hive plugin are kept | ||
| RUN rm -rf /usr/lib/trino/plugin/accumulo \ | ||
| && rm -rf /usr/lib/trino/plugin/blackhole \ | ||
| && rm -rf /usr/lib/trino/plugin/delta-lake \ | ||
| && rm -rf /usr/lib/trino/plugin/example-http \ | ||
| && rm -rf /usr/lib/trino/plugin/geospatial \ | ||
| && rm -rf /usr/lib/trino/plugin/kafka \ | ||
| && rm -rf /usr/lib/trino/plugin/local-file \ | ||
| && rm -rf /usr/lib/trino/plugin/ml \ | ||
| && rm -rf /usr/lib/trino/plugin/mysql-event-listener \ | ||
| && rm -rf /usr/lib/trino/plugin/phoenix5 \ | ||
| && rm -rf /usr/lib/trino/plugin/prometheus \ | ||
| && rm -rf /usr/lib/trino/plugin/redshift \ | ||
| && rm -rf /usr/lib/trino/plugin/singlestore \ | ||
| && rm -rf /usr/lib/trino/plugin/thrift \ | ||
| && rm -rf /usr/lib/trino/plugin/atop \ | ||
| && rm -rf /usr/lib/trino/plugin/cassandra \ | ||
| && rm -rf /usr/lib/trino/plugin/druid \ | ||
| && rm -rf /usr/lib/trino/plugin/exchange-filesystem \ | ||
| && rm -rf /usr/lib/trino/plugin/google-sheets \ | ||
| && rm -rf /usr/lib/trino/plugin/http-event-listener \ | ||
| && rm -rf /usr/lib/trino/plugin/ignite \ | ||
| && rm -rf /usr/lib/trino/plugin/kinesis \ | ||
| && rm -rf /usr/lib/trino/plugin/mongodb \ | ||
| && rm -rf /usr/lib/trino/plugin/oracle \ | ||
| && rm -rf /usr/lib/trino/plugin/pinot \ | ||
| && rm -rf /usr/lib/trino/plugin/raptor-legacy \ | ||
| && rm -rf /usr/lib/trino/plugin/resource-group-managers \ | ||
| && rm -rf /usr/lib/trino/plugin/sqlserver \ | ||
| && rm -rf /usr/lib/trino/plugin/bigquery \ | ||
| && rm -rf /usr/lib/trino/plugin/clickhouse \ | ||
| && rm -rf /usr/lib/trino/plugin/elasticsearch \ | ||
| && rm -rf /usr/lib/trino/plugin/exchange-hdfs \ | ||
| && rm -rf /usr/lib/trino/plugin/hudi \ | ||
| && rm -rf /usr/lib/trino/plugin/kudu \ | ||
| && rm -rf /usr/lib/trino/plugin/password-authenticators \ | ||
| && rm -rf /usr/lib/trino/plugin/postgresql \ | ||
| && rm -rf /usr/lib/trino/plugin/redis \ | ||
| && rm -rf /usr/lib/trino/plugin/session-property-managers \ | ||
| && rm -rf /usr/lib/trino/plugin/teradata-functions |
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Oops, something went wrong.