From 6049dfd951c64f0631a44e77efe248656773412e Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 17 Sep 2024 11:50:44 +0900 Subject: [PATCH 1/4] GH-43874: [CI][Integration][Go] Use apache/arrow-go --- .github/workflows/integration.yml | 6 ++++++ ci/scripts/integration_arrow.sh | 4 +--- ci/scripts/integration_arrow_build.sh | 13 ++++++------- dev/archery/archery/cli.py | 3 ++- docker-compose.yml | 1 + 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index bffc1c597b4c9..fd71889151824 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -81,6 +81,11 @@ jobs: with: repository: apache/arrow-nanoarrow path: nanoarrow + - name: Checkout Arrow Go + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + with: + repository: apache/arrow-go + path: go - name: Free up disk space run: | ci/scripts/util_free_space.sh @@ -104,6 +109,7 @@ jobs: source ci/scripts/util_enable_core_dumps.sh archery docker run \ -e ARCHERY_DEFAULT_BRANCH=${{ github.event.repository.default_branch }} \ + -e ARCHERY_INTEGRATION_WITH_GO=1 \ -e ARCHERY_INTEGRATION_WITH_NANOARROW=1 \ -e ARCHERY_INTEGRATION_WITH_RUST=1 \ conda-integration diff --git a/ci/scripts/integration_arrow.sh b/ci/scripts/integration_arrow.sh index ecb6b261ef1ee..079521d9a368a 100755 --- a/ci/scripts/integration_arrow.sh +++ b/ci/scripts/integration_arrow.sh @@ -26,11 +26,10 @@ gold_dir=$arrow_dir/testing/data/arrow-ipc-stream/integration : ${ARROW_INTEGRATION_CPP:=ON} : ${ARROW_INTEGRATION_CSHARP:=ON} -: ${ARROW_INTEGRATION_GO:=ON} : ${ARROW_INTEGRATION_JAVA:=ON} : ${ARROW_INTEGRATION_JS:=ON} -: ${ARCHERY_INTEGRATION_TARGET_LANGUAGES:=cpp,csharp,go,java,js} +: ${ARCHERY_INTEGRATION_TARGET_LANGUAGES:=cpp,csharp,java,js} export ARCHERY_INTEGRATION_TARGET_LANGUAGES . ${arrow_dir}/ci/scripts/util_log.sh @@ -67,7 +66,6 @@ time archery integration \ --run-flight \ --with-cpp=$([ "$ARROW_INTEGRATION_CPP" == "ON" ] && echo "1" || echo "0") \ --with-csharp=$([ "$ARROW_INTEGRATION_CSHARP" == "ON" ] && echo "1" || echo "0") \ - --with-go=$([ "$ARROW_INTEGRATION_GO" == "ON" ] && echo "1" || echo "0") \ --with-java=$([ "$ARROW_INTEGRATION_JAVA" == "ON" ] && echo "1" || echo "0") \ --with-js=$([ "$ARROW_INTEGRATION_JS" == "ON" ] && echo "1" || echo "0") \ --gold-dirs=$gold_dir/0.14.1 \ diff --git a/ci/scripts/integration_arrow_build.sh b/ci/scripts/integration_arrow_build.sh index 8fca0d434b75e..242c8bc163938 100755 --- a/ci/scripts/integration_arrow_build.sh +++ b/ci/scripts/integration_arrow_build.sh @@ -24,7 +24,6 @@ build_dir=${2} : ${ARROW_INTEGRATION_CPP:=ON} : ${ARROW_INTEGRATION_CSHARP:=ON} -: ${ARROW_INTEGRATION_GO:=ON} : ${ARROW_INTEGRATION_JAVA:=ON} : ${ARROW_INTEGRATION_JS:=ON} @@ -38,6 +37,12 @@ github_actions_group_begin "Integration: Build: nanoarrow" ${arrow_dir}/ci/scripts/nanoarrow_build.sh ${arrow_dir} ${build_dir} github_actions_group_end +github_actions_group_begin "Integration: Build: Go" +if [ "${ARCHERY_INTEGRATION_WITH_GO}" -gt "0" ]; then + ${arrow_dir}/go/ci/scripts/build.sh ${arrow_dir} ${build_dir} +fi +github_actions_group_end + github_actions_group_begin "Integration: Build: C++" if [ "${ARROW_INTEGRATION_CPP}" == "ON" ]; then ${arrow_dir}/ci/scripts/cpp_build.sh ${arrow_dir} ${build_dir} @@ -50,12 +55,6 @@ if [ "${ARROW_INTEGRATION_CSHARP}" == "ON" ]; then fi github_actions_group_end -github_actions_group_begin "Integration: Build: Go" -if [ "${ARROW_INTEGRATION_GO}" == "ON" ]; then - ${arrow_dir}/ci/scripts/go_build.sh ${arrow_dir} ${build_dir} -fi -github_actions_group_end - github_actions_group_begin "Integration: Build: Java" if [ "${ARROW_INTEGRATION_JAVA}" == "ON" ]; then export ARROW_JAVA_CDATA="ON" diff --git a/dev/archery/archery/cli.py b/dev/archery/archery/cli.py index 64481b8ff62a4..49699e81d57f5 100644 --- a/dev/archery/archery/cli.py +++ b/dev/archery/archery/cli.py @@ -737,7 +737,8 @@ def _set_default(opt, default): @click.option('--with-js', type=bool, default=False, help='Include JavaScript in integration tests') @click.option('--with-go', type=bool, default=False, - help='Include Go in integration tests') + help='Include Go in integration tests', + envvar="ARCHERY_INTEGRATION_WITH_GO") @click.option('--with-nanoarrow', type=bool, default=False, help='Include nanoarrow in integration tests', envvar="ARCHERY_INTEGRATION_WITH_NANOARROW") diff --git a/docker-compose.yml b/docker-compose.yml index 66607157318a2..7750575a88bae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1870,6 +1870,7 @@ services: volumes: *conda-volumes environment: <<: [*common, *ccache] + ARCHERY_INTEGRATION_WITH_GO: 0 ARCHERY_INTEGRATION_WITH_NANOARROW: 0 ARCHERY_INTEGRATION_WITH_RUST: 0 # Tell Archery where Arrow binaries are located From beb1daa9319102ef10fa8b180db90fe53753c234 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 17 Sep 2024 12:47:14 +0900 Subject: [PATCH 2/4] Fix args --- ci/scripts/integration_arrow_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/integration_arrow_build.sh b/ci/scripts/integration_arrow_build.sh index 242c8bc163938..4dfcf8768c71f 100755 --- a/ci/scripts/integration_arrow_build.sh +++ b/ci/scripts/integration_arrow_build.sh @@ -39,7 +39,7 @@ github_actions_group_end github_actions_group_begin "Integration: Build: Go" if [ "${ARCHERY_INTEGRATION_WITH_GO}" -gt "0" ]; then - ${arrow_dir}/go/ci/scripts/build.sh ${arrow_dir} ${build_dir} + ${arrow_dir}/go/ci/scripts/build.sh ${arrow_dir}/go fi github_actions_group_end From b5decda5e885eac964b06d97c7d2fc4307726183 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 17 Sep 2024 14:44:04 +0900 Subject: [PATCH 3/4] Set safe.directory --- docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7750575a88bae..a76ee49490a6e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1878,7 +1878,8 @@ services: ARROW_NANOARROW_PATH: /build/nanoarrow ARROW_RUST_EXE_PATH: /build/rust/debug command: - ["/arrow/ci/scripts/integration_arrow_build.sh /arrow /build && + ["git config --global --add safe.directory /arrow/go && + /arrow/ci/scripts/integration_arrow_build.sh /arrow /build && /arrow/ci/scripts/integration_arrow.sh /arrow /build"] ################################ Docs ####################################### From d3d8fe651b641d774a008a4d1e10d57106398e64 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 18 Sep 2024 09:27:40 +0900 Subject: [PATCH 4/4] Use the latest actions/checkout --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index fd71889151824..af9a98ed437f8 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -82,7 +82,7 @@ jobs: repository: apache/arrow-nanoarrow path: nanoarrow - name: Checkout Arrow Go - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: repository: apache/arrow-go path: go