From 78480f4692e2d26e3288804b1c35506edefca330 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Thu, 18 Aug 2022 13:26:58 +0200 Subject: [PATCH 01/14] add env macro --- dev/tasks/docker-tests/github.linux.yml | 7 +------ dev/tasks/fuzz-tests/github.oss-fuzz.yml | 7 +------ dev/tasks/macros.jinja | 9 +++++++++ dev/tasks/python-minimal-build/github.linux.yml | 7 +------ 4 files changed, 12 insertions(+), 18 deletions(-) diff --git a/dev/tasks/docker-tests/github.linux.yml b/dev/tasks/docker-tests/github.linux.yml index 41d54981047d9..c74731cec01d0 100644 --- a/dev/tasks/docker-tests/github.linux.yml +++ b/dev/tasks/docker-tests/github.linux.yml @@ -24,12 +24,7 @@ jobs: name: | Docker Test {{ flags|default("") }} {{ image }} {{ command|default("") }} runs-on: ubuntu-latest - {% if env is defined %} - env: - {% for key, value in env.items() %} - {{ key }}: "{{ value }}" - {% endfor %} - {% endif %} +{{ macros.github_set_env(env) }} steps: {{ macros.github_checkout_arrow(fetch_depth=fetch_depth if fetch_depth is defined else 1)|indent }} {{ macros.github_install_archery()|indent }} diff --git a/dev/tasks/fuzz-tests/github.oss-fuzz.yml b/dev/tasks/fuzz-tests/github.oss-fuzz.yml index e49d73d8d11e7..d7cf516266831 100644 --- a/dev/tasks/fuzz-tests/github.oss-fuzz.yml +++ b/dev/tasks/fuzz-tests/github.oss-fuzz.yml @@ -23,12 +23,7 @@ jobs: test: name: OSS-Fuzz build test runs-on: ubuntu-latest - {% if env is defined %} - env: - {% for key, value in env.items() %} - {{ key }}: "{{ value }}" - {% endfor %} - {% endif %} +{{ macros.github_set_env(env) }} steps: {{ macros.github_checkout_arrow()|indent }} diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 09a6696a377cc..3e87d507e77e1 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -352,3 +352,12 @@ on: {% set r_release = '4.2' %} {% set r_oldrel = '4.1' %} + +{%- macro github_set_env(env) -%} + {% if env is defined %} + env: + {% for key, value in env.items() %} + {{ key }}: "{{ value }}" + {% endfor %} + {% endif %} +{%- endmacro -%} diff --git a/dev/tasks/python-minimal-build/github.linux.yml b/dev/tasks/python-minimal-build/github.linux.yml index 887197d17bf04..e776312b93f95 100644 --- a/dev/tasks/python-minimal-build/github.linux.yml +++ b/dev/tasks/python-minimal-build/github.linux.yml @@ -24,12 +24,7 @@ jobs: name: | Docker Python Minimal Build {{ flags|default("") }} {{ image }} {{ command|default("") }} runs-on: ubuntu-latest - {% if env is defined %} - env: - {% for key, value in env.items() %} - {{ key }}: "{{ value }}" - {% endfor %} - {% endif %} +{{ macros.github_set_env(env) }} steps: {{ macros.github_checkout_arrow(submodules=false)|indent }} From 3c992b8732c4f9fcaab6b9ffcdee4c1be6afa9c6 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Thu, 18 Aug 2022 14:09:24 +0200 Subject: [PATCH 02/14] add docs preview job --- dev/archery/archery/bot.py | 3 +- dev/tasks/docker-tests/github.linux.yml | 2 +- dev/tasks/docs/github.linux.yml | 56 +++++++++++++++++++++++++ dev/tasks/tasks.yml | 9 ++++ 4 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 dev/tasks/docs/github.linux.yml diff --git a/dev/archery/archery/bot.py b/dev/archery/archery/bot.py index fb300f554c03d..020c299e0c7ad 100644 --- a/dev/archery/archery/bot.py +++ b/dev/archery/archery/bot.py @@ -259,7 +259,8 @@ def submit(obj, tasks, groups, params, arrow_version): # parse additional job parameters params = dict([p.split("=") for p in params]) - + params['pr_number'] = pull_request.number + # instantiate the job object job = Job.from_config(config=config, target=target, tasks=tasks, groups=groups, params=params) diff --git a/dev/tasks/docker-tests/github.linux.yml b/dev/tasks/docker-tests/github.linux.yml index c74731cec01d0..f7fd6a0be6eff 100644 --- a/dev/tasks/docker-tests/github.linux.yml +++ b/dev/tasks/docker-tests/github.linux.yml @@ -26,7 +26,7 @@ jobs: runs-on: ubuntu-latest {{ macros.github_set_env(env) }} steps: - {{ macros.github_checkout_arrow(fetch_depth=fetch_depth if fetch_depth is defined else 1)|indent }} + {{ macros.github_checkout_arrow(fetch_depth=fetch_depth|default(1))|indent }} {{ macros.github_install_archery()|indent }} - name: Execute Docker Build diff --git a/dev/tasks/docs/github.linux.yml b/dev/tasks/docs/github.linux.yml new file mode 100644 index 0000000000000..b48ac0b0b3eec --- /dev/null +++ b/dev/tasks/docs/github.linux.yml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +jobs: + test: + name: Docs Preview + runs-on: ubuntu-latest +{{ macros.github_set_env(env) }} + steps: + {{ macros.github_checkout_arrow(fetch_depth=fetch_depth|default(1))|indent }} + {{ macros.github_install_archery()|indent }} + + - name: Execute Docker Build + shell: bash + run: | + archery docker run \ + -e SETUPTOOLS_SCM_PRETEND_VERSION="{{ arrow.no_rc_version }}" \ + {{ flags|default("") }} \ + {{ image }} \ + {{ command|default("") }} + - name: Checkout Crossbow + uses: actions/checkout@v3 + with: + ref: {{ default_branch|default("master") }} + path: crossbow + fetch-depth: 1 + - name: Move docs + run: | + sudo chown -R ${USER}: build + mv --force build crossbow/docs/pr/{{ pr_number }} + - name: Push changes + run: | + cd crossbow + git config --local user.name "Github Actions" + git config --local user.email "github-actions@users.noreply.github.com" + git add docs/pr/{{ pr_number }} + git commit -m "Add docs preview for PR {{ pr_number }}" + git push \ No newline at end of file diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 386c95945ed93..3b1852ddad97c 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1607,3 +1607,12 @@ tasks: type: minimal_build run: {{ kind }} {% endfor %} + +############################## Utility tasks ############################ + preview-docs: + ci: github + template: docs/github.linux.yml + params: + artifacts: "build/docs.tar.gz" + flags: "-v $PWD/build/:/build/" + image: ubuntu-docs \ No newline at end of file From 287cd8f54197e68bbc4fc8dbf74c4433bca9100d Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Thu, 18 Aug 2022 14:51:42 +0200 Subject: [PATCH 03/14] add default value for params --- dev/tasks/docs/github.linux.yml | 4 +++- dev/tasks/r/github.packages.yml | 2 +- dev/tasks/tasks.yml | 3 +++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/dev/tasks/docs/github.linux.yml b/dev/tasks/docs/github.linux.yml index b48ac0b0b3eec..d419d78171f83 100644 --- a/dev/tasks/docs/github.linux.yml +++ b/dev/tasks/docs/github.linux.yml @@ -16,7 +16,7 @@ # under the License. {% import 'macros.jinja' as macros with context %} - +{% set pr_number = pr_number|replace("Unset", arrow.branch) %} {{ macros.github_header() }} jobs: @@ -25,6 +25,8 @@ jobs: runs-on: ubuntu-latest {{ macros.github_set_env(env) }} steps: + - name: Check that PR number set + {{ macros.github_checkout_arrow(fetch_depth=fetch_depth|default(1))|indent }} {{ macros.github_install_archery()|indent }} diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index 7ca8bd7a5f7f2..d1a95f64caa51 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -20,7 +20,7 @@ # This allows us to set a custom version via param: # crossbow submit --param custom_version=8.5.3 r-binary-packages # if the param is unset defaults to the usual Ymd naming scheme -{% set package_version = custom_version|default("\\2.\'\"$(date +%Y%m%d)\"\'") %} +{% set package_version = custom_version|replace("Unset", "\\2.\'\"$(date +%Y%m%d)\"\'") %} {% set is_fork = macros.is_fork %} {{ macros.github_header() }} diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 3b1852ddad97c..fa5cbf969cb19 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -969,6 +969,8 @@ tasks: r-binary-packages: ci: github template: r/github.packages.yml + params: + custom_version: Unset artifacts: - r-lib__libarrow__bin__windows__arrow-[0-9\.]+\.zip - r-lib__libarrow__bin__centos-7__arrow-[0-9\.]+\.zip @@ -1613,6 +1615,7 @@ tasks: ci: github template: docs/github.linux.yml params: + pr_number: Unset artifacts: "build/docs.tar.gz" flags: "-v $PWD/build/:/build/" image: ubuntu-docs \ No newline at end of file From 3628cdf0a43916b41188867726af79a6f4440c1f Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Thu, 18 Aug 2022 16:54:29 +0200 Subject: [PATCH 04/14] remove line --- dev/tasks/docs/github.linux.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dev/tasks/docs/github.linux.yml b/dev/tasks/docs/github.linux.yml index d419d78171f83..7c0be808f0c72 100644 --- a/dev/tasks/docs/github.linux.yml +++ b/dev/tasks/docs/github.linux.yml @@ -25,8 +25,6 @@ jobs: runs-on: ubuntu-latest {{ macros.github_set_env(env) }} steps: - - name: Check that PR number set - {{ macros.github_checkout_arrow(fetch_depth=fetch_depth|default(1))|indent }} {{ macros.github_install_archery()|indent }} @@ -55,4 +53,4 @@ jobs: git config --local user.email "github-actions@users.noreply.github.com" git add docs/pr/{{ pr_number }} git commit -m "Add docs preview for PR {{ pr_number }}" - git push \ No newline at end of file + git push From de4a4bf67e255b881bf6734ab3b04c2c7a97d7cb Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Thu, 18 Aug 2022 18:08:06 +0200 Subject: [PATCH 05/14] fix dir --- dev/tasks/docs/github.linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/docs/github.linux.yml b/dev/tasks/docs/github.linux.yml index 7c0be808f0c72..aef5f20e71e56 100644 --- a/dev/tasks/docs/github.linux.yml +++ b/dev/tasks/docs/github.linux.yml @@ -45,7 +45,7 @@ jobs: - name: Move docs run: | sudo chown -R ${USER}: build - mv --force build crossbow/docs/pr/{{ pr_number }} + mv --force build/docs crossbow/docs/pr/{{ pr_number }} - name: Push changes run: | cd crossbow From 2f9e769f2c9063869b0b6e2a29963ba3c1d399f0 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Thu, 18 Aug 2022 18:24:54 +0200 Subject: [PATCH 06/14] skip git plugin --- ci/scripts/java_build.sh | 5 +++++ docker-compose.yml | 1 + 2 files changed, 6 insertions(+) diff --git a/ci/scripts/java_build.sh b/ci/scripts/java_build.sh index b0362868b0a4e..120d04ffc2df9 100755 --- a/ci/scripts/java_build.sh +++ b/ci/scripts/java_build.sh @@ -64,6 +64,11 @@ if [[ "$(uname -s)" == "Linux" ]] && [[ "$(uname -m)" == "s390x" ]]; then fi mvn="mvn -B -DskipTests -Drat.skip=true -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" + +if [ $ARROW_JAVA_SKIP_GIT_PLUGIN ]; then + mvn="${mvn} -Dmaven.gitcommitid.skip=true" +fi + # Use `2 * ncores` threads mvn="${mvn} -T 2C" diff --git a/docker-compose.yml b/docker-compose.yml index e4363f9180bf7..b97a6a641a8f5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1642,6 +1642,7 @@ services: base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-3 environment: <<: *ccache + ARROW_JAVA_SKIP_GIT_PLUGIN: "TRUE" ARROW_CUDA: "ON" BUILD_DOCS_C_GLIB: "ON" BUILD_DOCS_CPP: "ON" From f6a5425fbd354107297c05882495ef7db7778487 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Thu, 18 Aug 2022 18:31:02 +0200 Subject: [PATCH 07/14] update git plugin --- java/pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 08bc54c140e4e..aeaf43efe4d5c 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -249,9 +249,9 @@ - pl.project13.maven - git-commit-id-plugin - 2.2.2 + io.github.git-commit-id + git-commit-id-maven-plugin + 5.0 for-jars From 7f064b683a84453785662ef045131b1c854d3a5e Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Thu, 18 Aug 2022 19:24:34 +0200 Subject: [PATCH 08/14] fix version --- java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/pom.xml b/java/pom.xml index aeaf43efe4d5c..30f79d54ea43d 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -251,7 +251,7 @@ io.github.git-commit-id git-commit-id-maven-plugin - 5.0 + 5.0.0 for-jars From 4c318f104fb6cf6e0283efa2c633bfe7a601da82 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Thu, 18 Aug 2022 19:36:51 +0200 Subject: [PATCH 09/14] lower git plugin version for jdk 8 supoport --- java/pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 30f79d54ea43d..a1e3ffc7b8f82 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -249,9 +249,9 @@ - io.github.git-commit-id - git-commit-id-maven-plugin - 5.0.0 + pl.project13.maven + git-commit-id-plugin + 4.0.5 for-jars From 03a82332e9ad7852c35db0284a9c1c903a7bdb71 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Fri, 19 Aug 2022 12:04:01 +0200 Subject: [PATCH 10/14] fix lint [skip ci] --- dev/archery/archery/bot.py | 2 +- dev/tasks/docs/github.linux.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/archery/archery/bot.py b/dev/archery/archery/bot.py index 020c299e0c7ad..c548e9a2a473c 100644 --- a/dev/archery/archery/bot.py +++ b/dev/archery/archery/bot.py @@ -260,7 +260,7 @@ def submit(obj, tasks, groups, params, arrow_version): # parse additional job parameters params = dict([p.split("=") for p in params]) params['pr_number'] = pull_request.number - + # instantiate the job object job = Job.from_config(config=config, target=target, tasks=tasks, groups=groups, params=params) diff --git a/dev/tasks/docs/github.linux.yml b/dev/tasks/docs/github.linux.yml index aef5f20e71e56..ed73a46dc8cb0 100644 --- a/dev/tasks/docs/github.linux.yml +++ b/dev/tasks/docs/github.linux.yml @@ -45,6 +45,7 @@ jobs: - name: Move docs run: | sudo chown -R ${USER}: build + mkdir -p crossbow/docs/pr/{{ pr_number }} mv --force build/docs crossbow/docs/pr/{{ pr_number }} - name: Push changes run: | From fe0475673d95bf4f9112f54d0ee2419a8bd50782 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Fri, 19 Aug 2022 14:12:09 +0200 Subject: [PATCH 11/14] return default behavor to git-commit plugin --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index b97a6a641a8f5..9ab6c0dd14873 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1642,7 +1642,7 @@ services: base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-3 environment: <<: *ccache - ARROW_JAVA_SKIP_GIT_PLUGIN: "TRUE" + ARROW_JAVA_SKIP_GIT_PLUGIN: ARROW_CUDA: "ON" BUILD_DOCS_C_GLIB: "ON" BUILD_DOCS_CPP: "ON" From 4d0d4eec8e5273d77ccf28e0a42a547e3a733f52 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Fri, 19 Aug 2022 14:12:29 +0200 Subject: [PATCH 12/14] use rsync --- dev/tasks/docs/github.linux.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev/tasks/docs/github.linux.yml b/dev/tasks/docs/github.linux.yml index ed73a46dc8cb0..e75547c92aa99 100644 --- a/dev/tasks/docs/github.linux.yml +++ b/dev/tasks/docs/github.linux.yml @@ -30,6 +30,8 @@ jobs: - name: Execute Docker Build shell: bash + env: + ARROW_JAVA_SKIP_GIT_PLUGIN: true run: | archery docker run \ -e SETUPTOOLS_SCM_PRETEND_VERSION="{{ arrow.no_rc_version }}" \ @@ -46,7 +48,7 @@ jobs: run: | sudo chown -R ${USER}: build mkdir -p crossbow/docs/pr/{{ pr_number }} - mv --force build/docs crossbow/docs/pr/{{ pr_number }} + rsync -a --delete build/docs/ crossbow/docs/pr/{{ pr_number }} - name: Push changes run: | cd crossbow From 33ac5d5e0074f42a3a73972aa241e1b7ce671e87 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Fri, 19 Aug 2022 19:32:08 +0200 Subject: [PATCH 13/14] add md summary --- dev/tasks/docs/github.linux.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/dev/tasks/docs/github.linux.yml b/dev/tasks/docs/github.linux.yml index e75547c92aa99..6dd8efaa8d117 100644 --- a/dev/tasks/docs/github.linux.yml +++ b/dev/tasks/docs/github.linux.yml @@ -46,14 +46,21 @@ jobs: fetch-depth: 1 - name: Move docs run: | + # build files are created by the docker user sudo chown -R ${USER}: build - mkdir -p crossbow/docs/pr/{{ pr_number }} - rsync -a --delete build/docs/ crossbow/docs/pr/{{ pr_number }} + PR_DIR=crossbow/docs/pr_docs/{{ pr_number }} + mkdir -p $PR_DIR + rsync -a --delete build/docs/ $PR_DIR - name: Push changes run: | cd crossbow git config --local user.name "Github Actions" git config --local user.email "github-actions@users.noreply.github.com" - git add docs/pr/{{ pr_number }} + + URL_PATH=pr_docs/{{ pr_number }} + URL=https://crossbow.voltrondata.com/$URL_PATH + + git add docs/$URL_PATH git commit -m "Add docs preview for PR {{ pr_number }}" git push + echo ":open_book: You can find the preview here $URL" >> $GITHUB_STEP_SUMMARY From 5aaa225683b4fa427d98306d347f7ac4bd3a7397 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Fri, 19 Aug 2022 19:33:00 +0200 Subject: [PATCH 14/14] typo --- dev/tasks/docs/github.linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/docs/github.linux.yml b/dev/tasks/docs/github.linux.yml index 6dd8efaa8d117..853729dd9a5ae 100644 --- a/dev/tasks/docs/github.linux.yml +++ b/dev/tasks/docs/github.linux.yml @@ -63,4 +63,4 @@ jobs: git add docs/$URL_PATH git commit -m "Add docs preview for PR {{ pr_number }}" git push - echo ":open_book: You can find the preview here $URL" >> $GITHUB_STEP_SUMMARY + echo ":open_book: You can find the preview here: $URL" >> $GITHUB_STEP_SUMMARY