ARROW-12600: [CI] Push docker images from crossbow tasks

Manually submitted build: https://github.com/ursacomputing/crossbow/runs/2467679252 Closes apache#10197 from kszucs/push-docker-images-from-crossbow-tasks Authored-by: Krisztián Szűcs <[email protected]> Signed-off-by: Krisztián Szűcs <[email protected]>
kszucs · Apr 30, 2021 · 9da02ce · 9da02ce
1 parent 7430bbd
commit 9da02ce
Show file tree

Hide file tree

Showing 2 changed files with 40 additions and 27 deletions.
diff --git a/dev/tasks/docker-tests/github.linux.yml b/dev/tasks/docker-tests/github.linux.yml
@@ -23,6 +23,12 @@ jobs:
   test:
     name: Docker Test
     runs-on: ubuntu-latest
+  {% if env is defined %}
+    env:
+    {% for key, value in env.items() %}
+      {{ key }}: {{ value }}
+    {% endfor %}
+  {% endif %}
     steps:
       {{ macros.github_checkout_arrow()|indent }}
       {{ macros.github_install_archery()|indent }}
@@ -33,10 +39,11 @@ jobs:
 
       - name: Execute Docker Build
         shell: bash
-        {% if env is defined %}
-        env:
-        {% for key, value in env.items() %}
-          {{ key }}: {{ value }}
-        {% endfor %}
-        {% endif %}
-        run: archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION="{{ arrow.no_rc_version }}" {{ run }}
+        run: archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION="{{ arrow.no_rc_version }}" {{ flags|default("") }} {{ image }}
+
+    {% if arrow.branch == 'master' %}
+      {{ macros.github_login_dockerhub()|indent }}
+      - name: Push Docker Image
+        shell: bash
+        run: archery docker push {{ image }}
+    {% endif %}
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
@@ -1234,7 +1234,7 @@ tasks:
     ci: github
     template: docker-tests/github.linux.yml
     params:
-      run: {{ image }}
+      image: {{ image }}
 {% endfor %}
 
 {% for ubuntu_version in ["18.04", "20.04"] %}
@@ -1244,7 +1244,7 @@ tasks:
     params:
       env:
         UBUNTU: {{ ubuntu_version }}
-      run: ubuntu-cpp
+      image: ubuntu-cpp
 {% endfor %}
 
   test-debian-10-cpp:
@@ -1253,31 +1253,33 @@ tasks:
     params:
       env:
         DEBIAN: 10
-      run: debian-cpp
+      image: debian-cpp
 
   test-fedora-33-cpp:
     ci: github
     template: docker-tests/github.linux.yml
     params:
       env:
         FEDORA: 33
-      run: fedora-cpp
+      image: fedora-cpp
 
   test-ubuntu-18.04-cpp-release:
     ci: github
     template: docker-tests/github.linux.yml
     params:
       env:
         UBUNTU: 18.04
-      run: "-e ARROW_BUILD_TYPE=release ubuntu-cpp"
+      flags: "-e ARROW_BUILD_TYPE=release"
+      image: ubuntu-cpp
 
   test-ubuntu-18.04-cpp-static:
     ci: github
     template: docker-tests/github.linux.yml
     params:
       env:
         UBUNTU: 18.04
-      run: "-e ARROW_BUILD_SHARED=OFF -e ARROW_BUILD_STATIC=ON -e ARROW_TEST_LINKAGE=static ubuntu-cpp"
+      flags: "-e ARROW_BUILD_SHARED=OFF -e ARROW_BUILD_STATIC=ON -e ARROW_TEST_LINKAGE=static"
+      image: ubuntu-cpp
 
 {% for cpp_standard in [14, 17] %}
   test-ubuntu-20.04-cpp-{{ cpp_standard }}:
@@ -1286,7 +1288,8 @@ tasks:
     params:
       env:
         UBUNTU: 20.04
-      run: "-e CMAKE_ARGS=-DCMAKE_CXX_STANDARD={{ cpp_standard }} ubuntu-cpp"
+      flags: "-e CMAKE_ARGS=-DCMAKE_CXX_STANDARD={{ cpp_standard }}"
+      image: ubuntu-cpp
 {% endfor %}
 
   test-ubuntu-20.04-cpp-thread-sanitizer:
@@ -1296,7 +1299,7 @@ tasks:
       env:
         CLANG_TOOLS: 11
         UBUNTU: 20.04
-      run: ubuntu-cpp-thread-sanitizer
+      image: ubuntu-cpp-thread-sanitizer
 
 {% for python_version in ["3.6", "3.7", "3.8", "3.9"] %}
   test-conda-python-{{ python_version }}:
@@ -1305,7 +1308,7 @@ tasks:
     params:
       env:
         PYTHON: {{ python_version }}
-      run: conda-python
+      image: conda-python
 {% endfor %}
 
   test-conda-python-3.8-hypothesis:
@@ -1318,7 +1321,7 @@ tasks:
         PYTHON: 3.8
         # limit to execute hypothesis tests only
         PYTEST_ARGS: "-m hypothesis"
-      run: conda-python-pandas
+      image: conda-python-pandas
 
   test-debian-10-python-3:
     ci: azure
@@ -1464,11 +1467,10 @@ tasks:
         PYTHON: {{ python_version }}
         PANDAS: {{ pandas_version }}
     {% if cache_leaf %}
-      run: conda-python-pandas
-    {% else %}
       # use the latest pandas release, so prevent reusing any cached layers
-      run: --no-leaf-cache conda-python-pandas
+      flags: --no-leaf-cache
     {% endif %}
+      image: conda-python-pandas
 {% endfor %}
 
 {% for dask_version in ["latest", "master"] %}
@@ -1480,7 +1482,8 @@ tasks:
         PYTHON: 3.7
         DASK: {{ dask_version }}
       # use the latest dask release, so prevent reusing any cached layers
-      run: --no-leaf-cache conda-python-dask
+      flags: --no-leaf-cache
+      image: conda-python-dask
 {% endfor %}
 
 {% for turbodbc_version in ["latest", "master"] %}
@@ -1492,7 +1495,8 @@ tasks:
         PYTHON: 3.7
         TURBODBC: {{ turbodbc_version }}
       # use the latest turbodbc release, so prevent reusing any cached layers
-      run: --no-leaf-cache conda-python-turbodbc
+      flags: --no-leaf-cache
+      image: conda-python-turbodbc
 {% endfor %}
 
 {% for kartothek_version in ["latest", "master"] %}
@@ -1503,7 +1507,8 @@ tasks:
       env:
         PYTHON: 3.7
         KARTOTHEK: {{ kartothek_version }}
-      run: --no-leaf-cache conda-python-kartothek
+      flags: --no-leaf-cache
+      image: conda-python-kartothek
 {% endfor %}
 
 {% for hdfs_version in ["2.9.2", "3.2.1"] %}
@@ -1514,7 +1519,7 @@ tasks:
       env:
         PYTHON: 3.7
         HDFS: {{ hdfs_version }}
-      run: conda-python-hdfs
+      image: conda-python-hdfs
 {% endfor %}
 
 {% for python_version, spark_version, test_pyarrow_only in [("3.7", "branch-3.0", "true"),
@@ -1528,7 +1533,8 @@ tasks:
         SPARK: {{ spark_version }}
         TEST_PYARROW_ONLY: {{ test_pyarrow_only }}
       # use the branch-3.0 of spark, so prevent reusing any layers
-      run: --no-leaf-cache conda-python-spark
+      flags: --no-leaf-cache
+      image: conda-python-spark
 {% endfor %}
 
   test-conda-python-3.8-jpype:
@@ -1537,14 +1543,14 @@ tasks:
     params:
       env:
         PYTHON: 3.8
-      run: conda-python-jpype
+      image: conda-python-jpype
 
   # Remove the "skipped-" prefix in ARROW-8475
   skipped-test-conda-cpp-hiveserver2:
     ci: github
     template: docker-tests/github.linux.yml
     params:
-      run: conda-cpp-hiveserver2
+      image: conda-cpp-hiveserver2
 
 {% for kind in ["static", "static-system-dependency"] %}
   example-cpp-minimal-build-{{ kind }}: