ARROW-12958: [CI][Developer] Build + host the docs for PR branches (#…

…13913) Authored-by: Jacob Wujciak-Jens <[email protected]> Signed-off-by: Rok <[email protected]>
apache · Aug 19, 2022 · 28cf3f9 · 28cf3f9
1 parent 4af3df0
commit 28cf3f9
Show file tree

Hide file tree

Showing 11 changed files with 100 additions and 21 deletions.
diff --git a/ci/scripts/java_build.sh b/ci/scripts/java_build.sh
@@ -64,6 +64,11 @@ if [[ "$(uname -s)" == "Linux" ]] && [[ "$(uname -m)" == "s390x" ]]; then
 fi
 
 mvn="mvn -B -DskipTests -Drat.skip=true -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
+
+if [ $ARROW_JAVA_SKIP_GIT_PLUGIN ]; then
+  mvn="${mvn} -Dmaven.gitcommitid.skip=true"
+fi
+
 # Use `2 * ncores` threads
 mvn="${mvn} -T 2C"
 

diff --git a/dev/archery/archery/bot.py b/dev/archery/archery/bot.py
@@ -259,6 +259,7 @@ def submit(obj, tasks, groups, params, arrow_version):
 
         # parse additional job parameters
         params = dict([p.split("=") for p in params])
+        params['pr_number'] = pull_request.number
 
         # instantiate the job object
         job = Job.from_config(config=config, target=target, tasks=tasks,

diff --git a/dev/tasks/docker-tests/github.linux.yml b/dev/tasks/docker-tests/github.linux.yml
@@ -24,14 +24,9 @@ jobs:
     name: |
       Docker Test {{ flags|default("") }} {{ image }} {{ command|default("") }}
     runs-on: ubuntu-latest
-  {% if env is defined %}
-    env:
-    {% for key, value in env.items() %}
-      {{ key }}: "{{ value }}"
-    {% endfor %}
-  {% endif %}
+{{ macros.github_set_env(env) }}
     steps:
-      {{ macros.github_checkout_arrow(fetch_depth=fetch_depth if fetch_depth is defined else 1)|indent }}
+      {{ macros.github_checkout_arrow(fetch_depth=fetch_depth|default(1))|indent }}
       {{ macros.github_install_archery()|indent }}
 
       - name: Execute Docker Build

diff --git a/dev/tasks/docs/github.linux.yml b/dev/tasks/docs/github.linux.yml
@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+{% import 'macros.jinja' as macros with context %}
+{% set pr_number = pr_number|replace("Unset", arrow.branch) %}
+{{ macros.github_header() }}
+
+jobs:
+  test:
+    name: Docs Preview 
+    runs-on: ubuntu-latest
+{{ macros.github_set_env(env) }}
+    steps:
+      {{ macros.github_checkout_arrow(fetch_depth=fetch_depth|default(1))|indent }}
+      {{ macros.github_install_archery()|indent }}
+
+      - name: Execute Docker Build
+        shell: bash
+        env:
+          ARROW_JAVA_SKIP_GIT_PLUGIN: true
+        run: |
+          archery docker run \
+            -e SETUPTOOLS_SCM_PRETEND_VERSION="{{ arrow.no_rc_version }}" \
+            {{ flags|default("") }} \
+            {{ image }} \
+            {{ command|default("") }}
+      - name: Checkout Crossbow
+        uses: actions/checkout@v3
+        with:
+          ref: {{ default_branch|default("master") }}
+          path: crossbow
+          fetch-depth: 1
+      - name: Move docs
+        run: |
+          # build files are created by the docker user
+          sudo chown -R ${USER}: build
+          PR_DIR=crossbow/docs/pr_docs/{{ pr_number }}
+          mkdir -p $PR_DIR
+          rsync -a --delete build/docs/ $PR_DIR
+      - name: Push changes
+        run: |
+          cd crossbow
+          git config --local user.name "Github Actions"
+          git config --local user.email "[email protected]"
+          
+          URL_PATH=pr_docs/{{ pr_number }}
+          URL=https://crossbow.voltrondata.com/$URL_PATH
+
+          git add docs/$URL_PATH
+          git commit -m "Add docs preview for PR {{ pr_number }}"
+          git push
+          echo ":open_book: You can find the preview here: $URL" >> $GITHUB_STEP_SUMMARY
diff --git a/dev/tasks/fuzz-tests/github.oss-fuzz.yml b/dev/tasks/fuzz-tests/github.oss-fuzz.yml
@@ -23,12 +23,7 @@ jobs:
   test:
     name: OSS-Fuzz build test
     runs-on: ubuntu-latest
-  {% if env is defined %}
-    env:
-    {% for key, value in env.items() %}
-      {{ key }}: "{{ value }}"
-    {% endfor %}
-  {% endif %}
+{{ macros.github_set_env(env) }}
     steps:
       {{ macros.github_checkout_arrow()|indent }}
 

diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja
@@ -352,3 +352,12 @@ on:
 
 {% set r_release = '4.2' %}
 {% set r_oldrel = '4.1' %}
+
+{%- macro github_set_env(env) -%}
+  {% if env is defined %}
+    env:
+    {% for key, value in env.items() %}
+      {{ key }}: "{{ value }}"
+    {% endfor %}
+  {% endif %}
+{%- endmacro -%}
diff --git a/dev/tasks/python-minimal-build/github.linux.yml b/dev/tasks/python-minimal-build/github.linux.yml
@@ -24,12 +24,7 @@ jobs:
     name: |
       Docker Python Minimal Build {{ flags|default("") }} {{ image }} {{ command|default("") }}
     runs-on: ubuntu-latest
-  {% if env is defined %}
-    env:
-    {% for key, value in env.items() %}
-      {{ key }}: "{{ value }}"
-    {% endfor %}
-  {% endif %}
+{{ macros.github_set_env(env) }}
     steps:
       {{ macros.github_checkout_arrow(submodules=false)|indent }}
 

diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml
@@ -20,7 +20,7 @@
 # This allows us to set a custom version via param:
 # crossbow submit --param custom_version=8.5.3 r-binary-packages
 # if the param is unset defaults to the usual Ymd naming scheme
-{% set package_version = custom_version|default("\\2.\'\"$(date +%Y%m%d)\"\'") %}
+{% set package_version = custom_version|replace("Unset", "\\2.\'\"$(date +%Y%m%d)\"\'") %}
 {% set is_fork = macros.is_fork %}
 
 {{ macros.github_header() }}

diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
@@ -969,6 +969,8 @@ tasks:
   r-binary-packages:
     ci: github
     template: r/github.packages.yml
+    params:
+      custom_version: Unset
     artifacts:
       - r-lib__libarrow__bin__windows__arrow-[0-9\.]+\.zip
       - r-lib__libarrow__bin__centos-7__arrow-[0-9\.]+\.zip
@@ -1607,3 +1609,13 @@ tasks:
       type: minimal_build
       run: {{ kind }}
 {% endfor %}
+
+############################## Utility tasks ############################
+  preview-docs:
+    ci: github
+    template: docs/github.linux.yml
+    params:
+      pr_number: Unset
+      artifacts: "build/docs.tar.gz"
+      flags: "-v $PWD/build/:/build/"
+      image: ubuntu-docs
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1642,6 +1642,7 @@ services:
         base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-3
     environment:
       <<: *ccache
+      ARROW_JAVA_SKIP_GIT_PLUGIN:
       ARROW_CUDA: "ON"
       BUILD_DOCS_C_GLIB: "ON"
       BUILD_DOCS_CPP: "ON"

diff --git a/java/pom.xml b/java/pom.xml
@@ -251,7 +251,7 @@
       <plugin>
         <groupId>pl.project13.maven</groupId>
         <artifactId>git-commit-id-plugin</artifactId>
-        <version>2.2.2</version>
+        <version>4.0.5</version>
         <executions>
           <execution>
             <id>for-jars</id>