[#3776] docker(ranger): Use Ranger binary release package to build Ra…

…nger Docker (#3775)

### What changes were proposed in this pull request?

Fix: #3776 

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Unit and Integration tests.

TODO:
- [x] Change ranger image name in ContainerSuite.java back to default
- [x] Change `RANGER_DOWNLOAD_URL` to the location of
`datastrato/apache-ranger`.
- [x] @xunliu needs to push new image release to docker hub.
- [x] @unknowntpo needs to update `docker-image-details.md`

dev/docker/build-docker.sh

@@ -85,8 +85,8 @@ elif [ "${component_type}" == "doris" ]; then
   . ${script_dir}/doris/doris-dependency.sh --platform ${platform_type}
   build_args="--build-arg DORIS_VERSION=${DORIS_VERSION}"
 elif [ "${component_type}" == "ranger" ]; then
-  # Multiple plugins can be passed using commas, e.g. `plugin-trino,plugin-hive`
-  build_args="--build-arg RANGER_VERSION=2.4.0 --build-arg RANGER_PLUGINS=plugin-trino,plugin-hive"
+  . ${script_dir}/ranger/ranger-dependency.sh
+  build_args="--build-arg RANGER_PACKAGE_NAME=${RANGER_PACKAGE_NAME} --build-arg MYSQL_CONNECTOR_PACKAGE_NAME=${MYSQL_CONNECTOR_PACKAGE_NAME} --build-arg LOG4JDBC_PACKAGE_NAME=${LOG4JDBC_PACKAGE_NAME} --build-arg RANGER_VERSION=${RANGER_VERSION}"
 else
   echo "ERROR : ${component_type} is not a valid component type"
   usage

dev/docker/ranger/Dockerfile

@@ -2,55 +2,25 @@
 # Copyright 2023 Datastrato Pvt Ltd.
 # This software is licensed under the Apache License version 2.
 #
-# Apache Ranger compile Docker image
-FROM debian:buster as compile-ranger
+# Apache Ranger Admin runtime Docker image
+FROM debian:buster
 LABEL maintainer="[email protected]"
 
-ARG RANGER_VERSION=2.4.0
-# Multiple plugins can be passed using commas, e.g. `plugin-trino,plugin-hive`
-ARG RANGER_PLUGINS=plugin-trino
-
 WORKDIR /root
 
 RUN apt-get -q update && \
-    apt-get install -y -q python python3 gcc mariadb-server vim curl wget openjdk-11-jdk git procps
-
-RUN wget https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.tar.gz && \
-    tar zxvf apache-maven-3.6.3-bin.tar.gz && \
-    ln -s /root/apache-maven-3.6.3/bin/mvn /usr/local/bin/mvn
-
-ENV JAVA_HOME=/usr/local/jdk
-RUN ARCH=$(uname -m) && \
-    if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then \
-    ln -s /usr/lib/jvm/java-11-openjdk-arm64 ${JAVA_HOME}; \
-    else \
-    ln -s /usr/lib/jvm/java-11-openjdk-amd64 ${JAVA_HOME}; \
-    fi
-
-RUN wget https://downloads.apache.org/ranger/${RANGER_VERSION}/apache-ranger-${RANGER_VERSION}.tar.gz && \
-    tar zxvf apache-ranger-${RANGER_VERSION}.tar.gz && \
-    ln -s apache-ranger-${RANGER_VERSION} apache-ranger && \
-    cd apache-ranger && \
-    mvn -pl ${RANGER_PLUGINS},jisql,agents-audit,agents-common,agents-cred,agents-installer,credentialbuilder,embeddedwebserver,security-admin,ranger-util,ranger-plugin-classloader,ranger-tools,distro -am -DskipTests=true compile package
-
-# Apache Ranger Admin runtime Docker image
-FROM debian:buster
-LABEL maintainer="[email protected]"
+    apt-get install -y -q python python3 gcc mariadb-server vim curl wget openjdk-11-jdk git procps && \
+    rm -rf /var/lib/apt/lists/*
 
 ARG RANGER_VERSION=2.4.0
-# Multiple plugins can be passed using commas, e.g. `plugin-trino,plugin-hive`
-ARG RANGER_PLUGINS=plugin-trino
 ENV RANGER_PASSWORD=rangerR0cks!
-
-WORKDIR /root
+ARG MYSQL_CONNECTOR_PACKAGE_NAME
+ARG LOG4JDBC_PACKAGE_NAME
 
 COPY init-mysql.sql.template /tmp/
 COPY start-ranger-services.sh /tmp/
 RUN chmod +x /tmp/start-ranger-services.sh
 
-RUN apt-get -q update && \
-    apt-get install -y -q python python3 gcc mariadb-server vim curl wget openjdk-11-jdk git procps
-
 ENV JAVA_HOME=/usr/local/jdk
 RUN ARCH=$(uname -m) && \
     if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then \
@@ -59,20 +29,17 @@ RUN ARCH=$(uname -m) && \
     ln -s /usr/lib/jvm/java-11-openjdk-amd64 ${JAVA_HOME}; \
     fi
 
-COPY --from=compile-ranger /root/apache-ranger/target/ranger-${RANGER_VERSION}-admin.tar.gz /opt
-RUN cd /opt && \
-    tar zxvf ranger-${RANGER_VERSION}-admin.tar.gz && \
-    ln -s ranger-${RANGER_VERSION}-admin ranger-admin
+# install ranger
+ENV RANGER_HOME=/opt/ranger-admin
+ADD packages/ranger-${RANGER_VERSION}-admin.tar.gz /opt/
+RUN ln -s /opt/ranger-${RANGER_VERSION}-admin ${RANGER_HOME}
 
 # Initialize Ranger envirioment
-RUN curl -L https://search.maven.org/remotecontent?filepath=mysql/mysql-connector-java/8.0.28/mysql-connector-java-8.0.28.jar --output /opt/ranger-admin/ews/webapp/WEB-INF/lib/mysql-connector-java-8.0.28.jar && \
-    cp /opt/ranger-admin/ews/webapp/WEB-INF/lib/mysql-connector-java-8.0.28.jar /opt/ranger-admin/jisql/lib/ && \
-    curl -L https://repo1.maven.org/maven2/com/googlecode/log4jdbc/log4jdbc/1.2/log4jdbc-1.2.jar --output /opt/ranger-admin/ews/webapp/WEB-INF/lib/log4jdbc-1.2.jar && \
-    cp -r /opt/ranger-admin/ews/webapp/WEB-INF/classes/conf.dist/ /opt/ranger-admin/ews/webapp/WEB-INF/classes/conf && \
-    mkdir /opt/ranger-admin/ews/logs
-
-# Clean up
-RUN rm -rf /var/lib/apt/lists/*
+# We download these packages because ranger-admin required thiese packages, but it seems not included in tar file.
+# And in ${RANGER_HOME}/setup.sh, it will use copy_db_connector() function to copy it to correct location.
+COPY packages/${MYSQL_CONNECTOR_PACKAGE_NAME} /usr/share/java/mysql-connector-java.jar
+RUN cp -r ${RANGER_HOME}/ews/webapp/WEB-INF/classes/conf.dist/ ${RANGER_HOME}/ews/webapp/WEB-INF/classes/conf && \
+    mkdir ${RANGER_HOME}/ews/logs
 
 EXPOSE 6080
 

dev/docker/ranger/ranger-dependency.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+#
+# Copyright 2024 Datastrato Pvt Ltd.
+# This software is licensed under the Apache License version 2.
+#
+set -ex
+ranger_dir="$(dirname "${BASH_SOURCE-$0}")"
+ranger_dir="$(cd "${ranger_dir}">/dev/null; pwd)"
+
+# Environment variables definition
+RANGER_VERSION=2.4.0
+RANGER_PACKAGE_NAME="ranger-${RANGER_VERSION}-admin.tar.gz" # Must export this variable for Dockerfile
+RANGER_DOWNLOAD_URL=https://github.com/datastrato/apache-ranger/releases/download/release-ranger-${RANGER_VERSION}/ranger-${RANGER_VERSION}-admin.tar.gz
+
+MYSQL_CONNECTOR_VERSION=8.0.28
+MYSQL_CONNECTOR_PACKAGE_NAME="mysql-connector-java-${MYSQL_CONNECTOR_VERSION}.jar"
+MYSQL_CONNECTOR_DOWNLOAD_URL=https://search.maven.org/remotecontent?filepath=mysql/mysql-connector-java/${MYSQL_CONNECTOR_VERSION}/mysql-connector-java-${MYSQL_CONNECTOR_VERSION}.jar
+
+# Prepare download packages
+if [[ ! -d "${ranger_dir}/packages" ]]; then
+  mkdir -p "${ranger_dir}/packages"
+fi
+
+if [ ! -f "${ranger_dir}/packages/${RANGER_PACKAGE_NAME}" ]; then
+  curl -L -s -o "${ranger_dir}/packages/${RANGER_PACKAGE_NAME}" ${RANGER_DOWNLOAD_URL}
+fi
+
+if [ ! -f "${ranger_dir}/packages/${MYSQL_CONNECTOR_PACKAGE_NAME}" ]; then
+  curl -L -s -o "${ranger_dir}/packages/${MYSQL_CONNECTOR_PACKAGE_NAME}" ${MYSQL_CONNECTOR_DOWNLOAD_URL}
+fi

dev/docker/ranger/start-ranger-services.sh

@@ -15,12 +15,6 @@ sed -i "s/rangerAdmin_password=/rangerAdmin_password=${RANGER_PASSWORD}/g" /opt/
 sed -i "s/rangerTagsync_password=/rangerTagsync_password=${RANGER_PASSWORD}/g" /opt/ranger-admin/install.properties
 sed -i "s/rangerUsersync_password=/rangerUsersync_password=${RANGER_PASSWORD}/g" /opt/ranger-admin/install.properties
 sed -i "s/keyadmin_password=/keyadmin_password=${RANGER_PASSWORD}/g" /opt/ranger-admin/install.properties
-sed -i 's/check_java_version/#check_java_version/g' /opt/ranger-admin/setup.sh
-sed -i 's/#check_java_version()/check_java_version()/g' /opt/ranger-admin/setup.sh
-sed -i 's/check_db_connector/#check_db_connector/g' /opt/ranger-admin/setup.sh
-sed -i 's/#check_db_connector()/check_db_connector()/g' /opt/ranger-admin/setup.sh
-sed -i 's/copy_db_connector/#copy_db_connector/g' /opt/ranger-admin/setup.sh
-sed -i 's/#copy_db_connector()/copy_db_connector()/g' /opt/ranger-admin/setup.sh
 cd /opt/ranger-admin && /opt/ranger-admin/setup.sh
 
 # Start Ranger Admin

docs/docker-image-details.md

@@ -238,6 +238,12 @@ You can use this image to control Trino's permissions.
 
 Changelog
 
+- gravitino-ci-ranger:0.1.1
+  - Docker image datastrato/gravitino-ci-ranger:0.1.1
+  - Use `ranger-admin` release from `datastrato/apache-ranger:2.4.0` to build docker image.
+  - Remove unnecessary hack in `start-ranger-service.sh`.
+  - Reduce docker image build time from `~1h` to `~5min`.
+
 - gravitino-ci-ranger:0.1.0
   - Docker image `datastrato/gravitino-ci-ranger:0.1.0`
   - Support Apache Ranger 2.4.0