[#4072] improvement(docker-image): Transfer docker hub from datastrat…

…o to apache (#4523) ### What changes were proposed in this pull request? Transfer docker image used Gravitino from user `datastrato` to Apache. ### Why are the changes needed? Apache project should use the apache docker hub as the user to keep docker image. Fix: #4072 ### Does this PR introduce _any_ user-facing change? N/A. ### How was this patch tested? Existing CI. --------- Co-authored-by: xunliu <[email protected]>
apache · Aug 20, 2024 · 4f484a1 · 4f484a1
1 parent 837b53b
commit 4f484a1
Show file tree

Hide file tree

Showing 32 changed files with 164 additions and 216 deletions.
diff --git a/.github/workflows/cron-integration-test.yml b/.github/workflows/cron-integration-test.yml
@@ -11,7 +11,7 @@ concurrency:
 
 jobs:
   changes:
-    if: github.repository == 'datastrato/gravitino'
+    if: github.repository == 'apache/gravitino'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml
@@ -10,17 +10,22 @@ on:
         default: 'gravitino'
         options:
           - 'gravitino'
-          - 'gravitino-ci-hive'
-          - 'gravitino-ci-kerberos-hive'
-          - 'gravitino-ci-trino'
-          - 'gravitino-ci-doris'
-          - 'gravitino-ci-ranger'
+          - 'gravitino-ci:hive'
+          - 'gravitino-ci:kerberos-hive'
+          - 'gravitino-ci:trino'
+          - 'gravitino-ci:doris'
+          - 'gravitino-ci:ranger'
+          - 'gravitino-playground:trino'
+          - 'gravitino-playground:hive'
+          - 'gravitino-playground:ranger'
           - 'gravitino-iceberg-rest-server'
-          - 'trino'
-          - 'hive'
-          - 'ranger'
-      tag:
-        description: 'Docker tag to apply to this image'
+      version:
+        description: 'Docker version to apply to this image'
+        required: true
+        type: string
+
+      username:
+        description: 'Docker username'
         required: true
         type: string
       token:
@@ -38,36 +43,45 @@ jobs:
     steps:
       - name: Set environment variables
         run: |
-          if [ "${{ github.event.inputs.image }}" == "gravitino-ci-hive" ]; then
+          if [ "${{ github.event.inputs.image }}" == "gravitino-ci:hive" ]; then
             echo "image_type=hive" >> $GITHUB_ENV
-            echo "image_name=datastrato/gravitino-ci-hive" >> $GITHUB_ENV
-          elif [ "${{ github.event.inputs.image }}" == "gravitino-ci-kerberos-hive" ]; then
+            echo "image_name=apache/gravitino-ci" >> $GITHUB_ENV
+            echo "tag_name=hive" >> $GITHUB_ENV
+          elif [ "${{ github.event.inputs.image }}" == "gravitino-ci:kerberos-hive" ]; then
             echo "image_type=kerberos-hive" >> $GITHUB_ENV
-            echo "image_name=datastrato/gravitino-ci-kerberos-hive" >> $GITHUB_ENV
-          elif [ "${{ github.event.inputs.image }}" == "gravitino-ci-trino" ]; then
+            echo "image_name=apache/gravitino-ci" >> $GITHUB_ENV
+            echo "tag_name=kerberos-hive" >> $GITHUB_ENV
+          elif [ "${{ github.event.inputs.image }}" == "gravitino-ci:trino" ]; then
             echo "image_type=trino" >> $GITHUB_ENV
-            echo "image_name=datastrato/gravitino-ci-trino" >> $GITHUB_ENV
-          elif [ "${{ github.event.inputs.image }}" == "gravitino-ci-doris" ]; then
+            echo "image_name=apache/gravitino-ci" >> $GITHUB_ENV
+            echo "tag_name=trino" >> $GITHUB_ENV
+          elif [ "${{ github.event.inputs.image }}" == "gravitino-ci:doris" ]; then
             echo "image_type=doris" >> $GITHUB_ENV
-            echo "image_name=datastrato/gravitino-ci-doris" >> $GITHUB_ENV
-          elif [ "${{ github.event.inputs.image }}" == "gravitino-ci-ranger" ]; then
+            echo "image_name=apache/gravitino-ci" >> $GITHUB_ENV
+            echo "tag_name=doris" >> $GITHUB_ENV
+          elif [ "${{ github.event.inputs.image }}" == "gravitino-ci:ranger" ]; then
             echo "image_type=ranger" >> $GITHUB_ENV
-            echo "image_name=datastrato/gravitino-ci-ranger" >> $GITHUB_ENV
+            echo "image_name=apache/gravitino-ci" >> $GITHUB_ENV
+            echo "tag_name=ranger" >> $GITHUB_ENV
           elif [ "${{ github.event.inputs.image }}" == "gravitino" ]; then
             echo "image_type=gravitino" >> $GITHUB_ENV
-            echo "image_name=datastrato/gravitino" >> $GITHUB_ENV
-          elif [ "${{ github.event.inputs.image }}" == "trino" ]; then
+            echo "image_name=apache/gravitino" >> $GITHUB_ENV
+            # `apache/gravitino` is the default image name, didn't need to tag alias name
+          elif [ "${{ github.event.inputs.image }}" == "gravitino-playground:trino" ]; then
             echo "image_type=trino" >> $GITHUB_ENV
-            echo "image_name=datastrato/trino" >> $GITHUB_ENV
-          elif [ "${{ github.event.inputs.image }}" == "hive" ]; then
+            echo "image_name=apache/gravitino-playground" >> $GITHUB_ENV
+            echo "tag_name=trino" >> $GITHUB_ENV
+          elif [ "${{ github.event.inputs.image }}" == "gravitino-playground:hive" ]; then
             echo "image_type=hive" >> $GITHUB_ENV
-            echo "image_name=datastrato/hive" >> $GITHUB_ENV
-          elif [ "${{ github.event.inputs.image }}" == "ranger" ]; then
+            echo "image_name=apache/gravitino-playground" >> $GITHUB_ENV
+            echo "tag_name=hive" >> $GITHUB_ENV
+          elif [ "${{ github.event.inputs.image }}" == "gravitino-playground:ranger" ]; then
             echo "image_type=ranger" >> $GITHUB_ENV
-            echo "image_name=datastrato/ranger" >> $GITHUB_ENV
+            echo "image_name=apache/gravitino-playground" >> $GITHUB_ENV
+            echo "tag_name=ranger" >> $GITHUB_ENV
           elif [ "${{ github.event.inputs.image }}" == "gravitino-iceberg-rest-server" ]; then
             echo "image_type=iceberg-rest-server" >> $GITHUB_ENV
-            echo "image_name=datastrato/gravitino-iceberg-rest-server" >> $GITHUB_ENV
+            echo "image_name=apache/gravitino-iceberg-rest" >> $GITHUB_ENV
           fi
 
       - name: Check publish Docker token
@@ -83,7 +97,7 @@ jobs:
       - name: Login to Docker Hub
         uses: docker/login-action@v2
         with:
-          username: datastrato
+          username: ${{ github.event.inputs.username }}
           password: ${{ secrets.DOCKER_REPOSITORY_PASSWORD }}
 
       - name: Set up Docker Buildx
@@ -100,4 +114,9 @@ jobs:
         run: | 
           sudo rm -rf /usr/local/lib/android
           sudo rm -rf /opt/hostedtoolcache/CodeQL
-          ./dev/docker/build-docker.sh --platform all --type ${image_type} --image ${image_name} --tag ${{ github.event.inputs.tag }} --latest
+          
+          if [[ "${image_type}" == "gravitino" || "{image_type}" == "iceberg-rest-server" ]]; then
+            ./dev/docker/build-docker.sh --platform all --type ${image_type} --image ${image_name} --tag ${{ github.event.inputs.version }} --latest
+          else
+            ./dev/docker/build-docker.sh --platform all --type ${image_type} --image ${image_name} --tag "${tag_name}-${{ github.event.inputs.version }}"
+          fi
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -61,7 +61,7 @@ cd gravitino
 
 ### Development Setup
 
-Once you have cloned the [GitHub repository](https://github.com/apache/gravitino), see [how to build](/docs/how-to-build.md) for instructions on how to build, or you can use the provided docker images at [Datastrato's DockerHub repository](https://hub.docker.com/u/datastrato).
+Once you have cloned the [GitHub repository](https://github.com/apache/gravitino), see [how to build](/docs/how-to-build.md) for instructions on how to build, or you can use the provided docker images at [Apache DockerHub repository](https://hub.docker.com/u/apache).
 
 To stop and start a local Gravitino server via `bin/gravitino.sh start` and `bin/gravitino.sh stop` in a Gravitino distribution, see [how to build](/docs/how-to-build.md) for more instructions.
 

diff --git a/build.gradle.kts b/build.gradle.kts
@@ -156,7 +156,7 @@ allprojects {
     }
   }
 
-  val setIntegrationTestEnvironment: (Test) -> Unit = { param ->
+  val setTestEnvironment: (Test) -> Unit = { param ->
     param.doFirst {
       param.jvmArgs(project.property("extraJvmArgs") as List<*>)
 
@@ -167,6 +167,14 @@ allprojects {
       param.environment("HADOOP_HOME", "/tmp")
       param.environment("PROJECT_VERSION", project.version)
 
+      // Gravitino CI Docker image
+      param.environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "apache/gravitino-ci:hive-0.1.13")
+      param.environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE", "apache/gravitino-ci:kerberos-hive-0.1.5")
+      param.environment("GRAVITINO_CI_DORIS_DOCKER_IMAGE", "apache/gravitino-ci:doris-0.1.5")
+      param.environment("GRAVITINO_CI_TRINO_DOCKER_IMAGE", "apache/gravitino-ci:trino-0.1.6")
+      param.environment("GRAVITINO_CI_RANGER_DOCKER_IMAGE", "apache/gravitino-ci:ranger-0.1.1")
+      param.environment("GRAVITINO_CI_KAFKA_DOCKER_IMAGE", "apache/kafka:3.7.0")
+
       val dockerRunning = project.rootProject.extra["dockerRunning"] as? Boolean ?: false
       val macDockerConnector = project.rootProject.extra["macDockerConnector"] as? Boolean ?: false
       if (OperatingSystem.current().isMacOsX() &&
@@ -205,7 +213,7 @@ allprojects {
     }
   }
 
-  extra["initIntegrationTest"] = setIntegrationTestEnvironment
+  extra["initTestParam"] = setTestEnvironment
 }
 
 nexusPublishing {
@@ -409,6 +417,11 @@ subprojects {
   }
 
   tasks.configureEach<Test> {
+    if (project.name != "server-common") {
+      val initTest = project.extra.get("initTestParam") as (Test) -> Unit
+      initTest(this)
+    }
+
     testLogging {
       exceptionFormat = TestExceptionFormat.FULL
       showExceptions = true

diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts
@@ -125,14 +125,6 @@ tasks.test {
     exclude("**/integration/**")
   } else {
     dependsOn(tasks.jar)
-
-    doFirst {
-      environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.13")
-      environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-kerberos-hive:0.1.5")
-    }
-
-    val init = project.extra.get("initIntegrationTest") as (Test) -> Unit
-    init(this)
   }
 }
 

diff --git a/catalogs/catalog-hive/build.gradle.kts b/catalogs/catalog-hive/build.gradle.kts
@@ -179,14 +179,6 @@ tasks.test {
     exclude("**/integration/**")
   } else {
     dependsOn(tasks.jar)
-
-    doFirst {
-      environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.13")
-      environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-kerberos-hive:0.1.5")
-    }
-
-    val init = project.extra.get("initIntegrationTest") as (Test) -> Unit
-    init(this)
   }
 }
 

diff --git a/catalogs/catalog-jdbc-doris/build.gradle.kts b/catalogs/catalog-jdbc-doris/build.gradle.kts
@@ -85,24 +85,16 @@ tasks {
 
 tasks.test {
   val skipUTs = project.hasProperty("skipTests")
-  doFirst {
-    environment("GRAVITINO_CI_DORIS_DOCKER_IMAGE", "datastrato/gravitino-ci-doris:0.1.5")
-  }
-
   if (skipUTs) {
     // Only run integration tests
     include("**/integration/**")
   }
-
   val skipITs = project.hasProperty("skipITs")
   if (skipITs) {
     // Exclude integration tests
     exclude("**/integration/**")
   } else {
     dependsOn(tasks.jar)
-
-    val init = project.extra.get("initIntegrationTest") as (Test) -> Unit
-    init(this)
   }
 }
 

diff --git a/catalogs/catalog-jdbc-mysql/build.gradle.kts b/catalogs/catalog-jdbc-mysql/build.gradle.kts
@@ -98,9 +98,6 @@ tasks.test {
     exclude("**/integration/**")
   } else {
     dependsOn(tasks.jar)
-
-    val init = project.extra.get("initIntegrationTest") as (Test) -> Unit
-    init(this)
   }
 }
 

diff --git a/catalogs/catalog-jdbc-postgresql/build.gradle.kts b/catalogs/catalog-jdbc-postgresql/build.gradle.kts
@@ -103,9 +103,6 @@ tasks.test {
     // PG will use project jdbc-mysql/build/libs directory, so we add the task dependency here.
     dependsOn(":catalogs:catalog-jdbc-mysql:jar")
     dependsOn(tasks.jar)
-
-    val init = project.extra.get("initIntegrationTest") as (Test) -> Unit
-    init(this)
   }
 }
 

diff --git a/catalogs/catalog-kafka/build.gradle.kts b/catalogs/catalog-kafka/build.gradle.kts
@@ -107,12 +107,5 @@ tasks.test {
     exclude("**/integration/**")
   } else {
     dependsOn(tasks.jar)
-
-    doFirst {
-      environment("GRAVITINO_CI_KAFKA_DOCKER_IMAGE", "apache/kafka:3.7.0")
-    }
-
-    val init = project.extra.get("initIntegrationTest") as (Test) -> Unit
-    init(this)
   }
 }
diff --git a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts
@@ -150,14 +150,6 @@ tasks.test {
     exclude("**/integration/**")
   } else {
     dependsOn(tasks.jar)
-
-    doFirst {
-      environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.13")
-      environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-kerberos-hive:0.1.5")
-    }
-
-    val init = project.extra.get("initIntegrationTest") as (Test) -> Unit
-    init(this)
   }
 }
 

diff --git a/catalogs/catalog-lakehouse-paimon/build.gradle.kts b/catalogs/catalog-lakehouse-paimon/build.gradle.kts
@@ -138,14 +138,6 @@ tasks.test {
     exclude("**/integration/**")
   } else {
     dependsOn(tasks.jar)
-
-    doFirst {
-      environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.13")
-      environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-kerberos-hive:0.1.3")
-    }
-
-    val init = project.extra.get("initIntegrationTest") as (Test) -> Unit
-    init(this)
   }
 }
 

diff --git a/clients/client-python/build.gradle.kts b/clients/client-python/build.gradle.kts
@@ -222,7 +222,7 @@ tasks {
       "GRAVITINO_HOME" to project.rootDir.path + "/distribution/package",
       "START_EXTERNAL_GRAVITINO" to "true",
       "DOCKER_TEST" to dockerTest.toString(),
-      "GRAVITINO_CI_HIVE_DOCKER_IMAGE" to "datastrato/gravitino-ci-hive:0.1.13",
+      "GRAVITINO_CI_HIVE_DOCKER_IMAGE" to "apache/gravitino-ci:hive-0.1.13",
       // Set the PYTHONPATH to the client-python directory, make sure the tests can import the
       // modules from the client-python directory.
       "PYTHONPATH" to "${project.rootDir.path}/clients/client-python"

diff --git a/dev/docker/build-docker.sh b/dev/docker/build-docker.sh
@@ -121,7 +121,7 @@ else
   exit 1
 fi
 
-build_args="${build_args} --build-arg IMAGE_NAME=${image_name}"
+build_args="${build_args} --build-arg IMAGE_NAME=${image_name} --build-arg TAG_NAME=${tag_name}"
 
 # Create multi-arch builder
 BUILDER_NAME="gravitino-builder"

diff --git a/dev/docker/trino/Dockerfile b/dev/docker/trino/Dockerfile
@@ -67,7 +67,8 @@ RUN mkdir /tmp/gravitino
 COPY --chown=trino:trino packages/gravitino-trino-connector /tmp/gravitino
 
 ARG IMAGE_NAME
-RUN if [ "$IMAGE_NAME" = "datastrato/trino" ] ; then \
+ARG TAG_NAME
+RUN if [ "$TAG_NAME" = "trino-*" ] ; then \
     mv /tmp/gravitino /usr/lib/trino/plugin/; \
     else echo "Copying files for other images"; \
     fi

diff --git a/docs/assets/publish-docker-image.jpg b/docs/assets/publish-docker-image.jpg
diff --git a/docs/assets/publish-docker-image.png b/docs/assets/publish-docker-image.png