From 48cd142525d29f15feae7fb8190be5a99f6d21df Mon Sep 17 00:00:00 2001 From: Isaac Brodsky Date: Thu, 6 Jun 2024 18:30:59 -0700 Subject: [PATCH 01/10] use Files.createTempFile --- .../java/com/uber/h3core/H3CoreLoader.java | 20 ++++++++++++------- .../com/uber/h3core/TestH3CoreLoader.java | 3 ++- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/uber/h3core/H3CoreLoader.java b/src/main/java/com/uber/h3core/H3CoreLoader.java index e40df91..1502256 100644 --- a/src/main/java/com/uber/h3core/H3CoreLoader.java +++ b/src/main/java/com/uber/h3core/H3CoreLoader.java @@ -1,5 +1,5 @@ /* - * Copyright 2017-2018 Uber Technologies, Inc. + * Copyright 2017-2018, 2024 Uber Technologies, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,7 +20,12 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; import java.util.Locale; +import java.util.Set; /** Extracts the native H3 core library to the local filesystem and loads it. */ public final class H3CoreLoader { @@ -60,11 +65,6 @@ private static void copyStream(InputStream in, OutputStream out) throws IOExcept * @throws UnsatisfiedLinkError The resource path does not exist */ static void copyResource(String resourcePath, File newH3LibFile) throws IOException { - // Set the permissions - newH3LibFile.setReadable(true); - newH3LibFile.setWritable(true, true); - newH3LibFile.setExecutable(true, true); - // Shove the resource into the file and close it try (InputStream resource = H3CoreLoader.class.getResourceAsStream(resourcePath)) { if (resource == null) { @@ -115,8 +115,14 @@ public static synchronized NativeMethods loadNatives(OperatingSystem os, String final String dirName = String.format("%s-%s", os.getDirName(), arch); final String libName = String.format("libh3-java%s", os.getSuffix()); - final File newLibraryFile = File.createTempFile("libh3-java", os.getSuffix()); + final FileAttribute> attr = + PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------")); + final File newLibraryFile = Files.createTempFile("libh3-java", os.getSuffix(), attr).toFile(); + // Set permissions in case the Posix permissions were not applied (non-Posix OS) + newLibraryFile.setReadable(true, true); + newLibraryFile.setWritable(true, true); + newLibraryFile.setExecutable(true, true); newLibraryFile.deleteOnExit(); copyResource(String.format("/%s/%s", dirName, libName), newLibraryFile); diff --git a/src/test/java/com/uber/h3core/TestH3CoreLoader.java b/src/test/java/com/uber/h3core/TestH3CoreLoader.java index cd0b8ea..f822b19 100644 --- a/src/test/java/com/uber/h3core/TestH3CoreLoader.java +++ b/src/test/java/com/uber/h3core/TestH3CoreLoader.java @@ -19,6 +19,7 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Files; import org.junit.Test; /** H3CoreLoader is mostly tested by {@link TestH3CoreFactory}. This also tests OS detection. */ @@ -60,7 +61,7 @@ public void testDetectArch() { @Test(expected = UnsatisfiedLinkError.class) public void testExtractNonexistant() throws IOException { - File tempFile = File.createTempFile("test-extract-resource", null); + File tempFile = Files.createTempFile("test-extract-resource", null).toFile(); tempFile.deleteOnExit(); From 54dbd60b2793e0b6276879fdeac7febdb1cbb59a Mon Sep 17 00:00:00 2001 From: Isaac Brodsky Date: Thu, 6 Jun 2024 18:55:09 -0700 Subject: [PATCH 02/10] Don't always apply POSIX permissions --- .../java/com/uber/h3core/H3CoreLoader.java | 41 +++++++++++++++---- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/uber/h3core/H3CoreLoader.java b/src/main/java/com/uber/h3core/H3CoreLoader.java index 1502256..9533e64 100644 --- a/src/main/java/com/uber/h3core/H3CoreLoader.java +++ b/src/main/java/com/uber/h3core/H3CoreLoader.java @@ -93,6 +93,24 @@ public static NativeMethods loadNatives() throws IOException { return loadNatives(os, arch); } + private static File createTempLibraryFile(OperatingSystem os) throws IOException { + if (os.isPosix()) { + // Note this is already done by the implementation of Files.createTempFile that I looked at, + // but the javadoc does not seem to gaurantee the permissions will be restricted to owner + // write. + final FileAttribute> attr = + PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------")); + return Files.createTempFile("libh3-java", os.getSuffix(), attr).toFile(); + } else { + // When not a POSIX OS, try to ensure the permissions are secure + final File f = Files.createTempFile("libh3-java", os.getSuffix()).toFile(); + f.setReadable(true, true); + f.setWritable(true, true); + f.setExecutable(true, true); + return f; + } + } + /** * For use when the H3 library should be unpacked from the JAR and loaded. The native library for * the specified operating system and architecture will be extract. @@ -115,14 +133,7 @@ public static synchronized NativeMethods loadNatives(OperatingSystem os, String final String dirName = String.format("%s-%s", os.getDirName(), arch); final String libName = String.format("libh3-java%s", os.getSuffix()); - final FileAttribute> attr = - PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------")); - final File newLibraryFile = Files.createTempFile("libh3-java", os.getSuffix(), attr).toFile(); - - // Set permissions in case the Posix permissions were not applied (non-Posix OS) - newLibraryFile.setReadable(true, true); - newLibraryFile.setWritable(true, true); - newLibraryFile.setExecutable(true, true); + final File newLibraryFile = createTempLibraryFile(os); newLibraryFile.deleteOnExit(); copyResource(String.format("/%s/%s", dirName, libName), newLibraryFile); @@ -152,13 +163,20 @@ public enum OperatingSystem { ANDROID(".so"), DARWIN(".dylib"), FREEBSD(".so"), - WINDOWS(".dll"), + WINDOWS(".dll", false), LINUX(".so"); private final String suffix; + private final boolean posix; OperatingSystem(String suffix) { this.suffix = suffix; + this.posix = true; + } + + OperatingSystem(String suffix, boolean posix) { + this.suffix = suffix; + this.posix = posix; } /** Suffix for native libraries. */ @@ -170,6 +188,11 @@ public String getSuffix() { public String getDirName() { return name().toLowerCase(H3_CORE_LOCALE); } + + /** Whether to try to use POSIX file permissions when creating the native library temp file. */ + public boolean isPosix() { + return this.posix; + } } /** From 8ec486b76d2136b73f414778663e05b14a975881 Mon Sep 17 00:00:00 2001 From: Isaac Brodsky Date: Thu, 6 Jun 2024 18:59:25 -0700 Subject: [PATCH 03/10] back out permissions change --- src/main/java/com/uber/h3core/H3CoreLoader.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/uber/h3core/H3CoreLoader.java b/src/main/java/com/uber/h3core/H3CoreLoader.java index 9533e64..5be0196 100644 --- a/src/main/java/com/uber/h3core/H3CoreLoader.java +++ b/src/main/java/com/uber/h3core/H3CoreLoader.java @@ -65,6 +65,13 @@ private static void copyStream(InputStream in, OutputStream out) throws IOExcept * @throws UnsatisfiedLinkError The resource path does not exist */ static void copyResource(String resourcePath, File newH3LibFile) throws IOException { + // When not a POSIX OS, try to ensure the permissions are secure. + // If possible, we try to create the file with the right permissions the first time in + // createTempLibraryFile. + newH3LibFile.setReadable(true, true); + newH3LibFile.setWritable(true, true); + newH3LibFile.setExecutable(true, true); + // Shove the resource into the file and close it try (InputStream resource = H3CoreLoader.class.getResourceAsStream(resourcePath)) { if (resource == null) { @@ -102,12 +109,7 @@ private static File createTempLibraryFile(OperatingSystem os) throws IOException PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------")); return Files.createTempFile("libh3-java", os.getSuffix(), attr).toFile(); } else { - // When not a POSIX OS, try to ensure the permissions are secure - final File f = Files.createTempFile("libh3-java", os.getSuffix()).toFile(); - f.setReadable(true, true); - f.setWritable(true, true); - f.setExecutable(true, true); - return f; + return Files.createTempFile("libh3-java", os.getSuffix()).toFile(); } } From 51b5e8299aa3dd7f3d15eada86901a5c1291700c Mon Sep 17 00:00:00 2001 From: Isaac Brodsky Date: Thu, 6 Jun 2024 19:04:33 -0700 Subject: [PATCH 04/10] Revert "back out permissions change" This reverts commit 8ec486b76d2136b73f414778663e05b14a975881. --- src/main/java/com/uber/h3core/H3CoreLoader.java | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/uber/h3core/H3CoreLoader.java b/src/main/java/com/uber/h3core/H3CoreLoader.java index 5be0196..9533e64 100644 --- a/src/main/java/com/uber/h3core/H3CoreLoader.java +++ b/src/main/java/com/uber/h3core/H3CoreLoader.java @@ -65,13 +65,6 @@ private static void copyStream(InputStream in, OutputStream out) throws IOExcept * @throws UnsatisfiedLinkError The resource path does not exist */ static void copyResource(String resourcePath, File newH3LibFile) throws IOException { - // When not a POSIX OS, try to ensure the permissions are secure. - // If possible, we try to create the file with the right permissions the first time in - // createTempLibraryFile. - newH3LibFile.setReadable(true, true); - newH3LibFile.setWritable(true, true); - newH3LibFile.setExecutable(true, true); - // Shove the resource into the file and close it try (InputStream resource = H3CoreLoader.class.getResourceAsStream(resourcePath)) { if (resource == null) { @@ -109,7 +102,12 @@ private static File createTempLibraryFile(OperatingSystem os) throws IOException PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------")); return Files.createTempFile("libh3-java", os.getSuffix(), attr).toFile(); } else { - return Files.createTempFile("libh3-java", os.getSuffix()).toFile(); + // When not a POSIX OS, try to ensure the permissions are secure + final File f = Files.createTempFile("libh3-java", os.getSuffix()).toFile(); + f.setReadable(true, true); + f.setWritable(true, true); + f.setExecutable(true, true); + return f; } } From 0d911cb03f5e10a363cceade74674b6d60b6e49b Mon Sep 17 00:00:00 2001 From: Isaac Brodsky Date: Thu, 6 Jun 2024 19:13:23 -0700 Subject: [PATCH 05/10] pin back windows --- .github/workflows/tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d36f68d..7cdfbe6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -94,7 +94,8 @@ jobs: strategy: matrix: # TODO: Docker on macos-latest running is not working - os: [macos-latest, windows-latest] + # TODO: Windows pinned back + os: [macos-latest, windows-2019] java-distribution: [adopt] java-version: [8, 11, 15, 17] From 9cd779882c8ad96970d9877ce26b30fe9453e5b8 Mon Sep 17 00:00:00 2001 From: Isaac Brodsky Date: Thu, 6 Jun 2024 19:31:35 -0700 Subject: [PATCH 06/10] add slack alert & cron --- .github/workflows/slack-alert.yml | 22 ++++++++++++++++++++++ .github/workflows/tests.yml | 5 ++++- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/slack-alert.yml diff --git a/.github/workflows/slack-alert.yml b/.github/workflows/slack-alert.yml new file mode 100644 index 0000000..8f65cf5 --- /dev/null +++ b/.github/workflows/slack-alert.yml @@ -0,0 +1,22 @@ +name: slack-alert + +on: + workflow_run: + workflows: [tests] + types: [completed] + +jobs: + on-failure: + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'failure' }} + steps: + - name: Send Slack Alert + id: slack + uses: slackapi/slack-github-action@v1.26.0 + with: + payload: | + Github Actions ${{ github.event.workflow_run.conclusion }} + Repo: ${{github.event.workflow_run.repository.name }} + Workflow URL: ${{ github.event.workflow_run.html_url }} + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7cdfbe6..fdb748c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,6 +5,9 @@ on: branches: [master, stable-*] pull_request: branches: [master, stable-*] + schedule: + # Every Sunday, rerun + - cron: "0 12 * * 0" jobs: tests: @@ -64,7 +67,7 @@ jobs: os: [ubuntu-latest] java-distribution: [adopt] java-version: [17] - dockcross-tag: ["20220906-e88a3ce", "20230116-670f7f7"] + dockcross-tag: ["20230116-670f7f7", "20240529-0dade71", "latest"] steps: - uses: actions/checkout@v2.1.1 From 70131c897dc835ebefe88d93b37cc548bbe4bc2f Mon Sep 17 00:00:00 2001 From: Isaac Brodsky Date: Thu, 6 Jun 2024 19:34:18 -0700 Subject: [PATCH 07/10] broken dockcross tag --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fdb748c..2a0583a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -67,7 +67,7 @@ jobs: os: [ubuntu-latest] java-distribution: [adopt] java-version: [17] - dockcross-tag: ["20230116-670f7f7", "20240529-0dade71", "latest"] + dockcross-tag: ["20230116-670f7f7", "20240418-88c04a4", "latest"] steps: - uses: actions/checkout@v2.1.1 From cdb2863b85c4f22b1801eab04db29f4e399d5385 Mon Sep 17 00:00:00 2001 From: Isaac Brodsky Date: Thu, 20 Jun 2024 07:22:12 -0700 Subject: [PATCH 08/10] docker rmi --- src/main/c/h3-java/build-h3.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/c/h3-java/build-h3.sh b/src/main/c/h3-java/build-h3.sh index 741a023..4cb1835 100755 --- a/src/main/c/h3-java/build-h3.sh +++ b/src/main/c/h3-java/build-h3.sh @@ -210,6 +210,9 @@ for image in android-arm android-arm64 linux-arm64 linux-armv5 linux-armv7 linux if [ -e $BUILD_ROOT/lib/libh3-java.dll ]; then cp $BUILD_ROOT/lib/libh3-java.dll $OUTPUT_ROOT ; fi if $SYSTEM_PRUNE; then + # Remove the image we just ran + docker rmi dockcross/$image:$DOCKCROSS_TAG + # Aggressively try to free more disk space docker system prune --force --all docker system df rm $BUILD_ROOT/dockcross From 5de22340d9b2454b736937dd997dc07c18e14397 Mon Sep 17 00:00:00 2001 From: Isaac Brodsky Date: Thu, 20 Jun 2024 08:01:29 -0700 Subject: [PATCH 09/10] build only specific artifact per test --- .github/workflows/tests.yml | 5 +++-- pom.xml | 2 ++ src/main/c/h3-java/build-h3.sh | 16 ++++++++++++---- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2a0583a..f77a083 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -59,7 +59,7 @@ jobs: if-no-files-found: error tests-new-dockcross: - name: Dockcross ${{ matrix.dockcross-tag }} Java ${{ matrix.java-version }} ${{ matrix.os }} + name: Dockcross ${{ matrix.dockcross-tag }} Java ${{ matrix.java-version }} ${{ matrix.os }} ${{ matrix.dockcross-only }} runs-on: ${{ matrix.os }} strategy: @@ -68,6 +68,7 @@ jobs: java-distribution: [adopt] java-version: [17] dockcross-tag: ["20230116-670f7f7", "20240418-88c04a4", "latest"] + dockcross-only: ["android-arm", "android-arm64", "linux-arm64", "linux-armv5", "linux-armv7", "linux-s390x", "linux-ppc64le", "linux-x64", "linux-x86", "windows-static-x64", "windows-static-x86"] steps: - uses: actions/checkout@v2.1.1 @@ -88,7 +89,7 @@ jobs: ${{ runner.os }}-maven- - name: Tests - run: mvn "-Dh3.system.prune=true" "-Dh3.dockcross.tag=${{ matrix.dockcross-tag }}" -B -V clean test + run: mvn "-Dh3.system.prune=true" "-Dh3.dockcross.tag=${{ matrix.dockcross-tag }}" "-Dh3.dockcross.only=${{ matrix.dockcross-only }}" -B -V clean test tests-no-docker: name: Java (No Docker) ${{ matrix.java-version }} ${{ matrix.os }} diff --git a/pom.xml b/pom.xml index 1d6e3e8..968e5b3 100644 --- a/pom.xml +++ b/pom.xml @@ -72,6 +72,7 @@ true false 20210624-de7b1b0 + > false @@ -270,6 +271,7 @@ ${h3.use.docker} ${h3.system.prune} ${h3.dockcross.tag} + ${h3.dockcross.only} ${h3.github.artifacts.use} ${h3.github.artifacts.by_run} diff --git a/src/main/c/h3-java/build-h3.sh b/src/main/c/h3-java/build-h3.sh index 4cb1835..a3019d6 100755 --- a/src/main/c/h3-java/build-h3.sh +++ b/src/main/c/h3-java/build-h3.sh @@ -25,6 +25,7 @@ # system prune will be run after each step # (i.e. for disk space constrained environments like CI) # dockcross-tag - Tag name for dockcross +# dockcross-only - When set, build only a specific architecture with dockcross. # github-artifacts - When set, all build artifacts are retrieved from Github # Actions artifacts rather than built locally (overrides # all other settings.) @@ -44,8 +45,9 @@ GIT_REVISION=$2 USE_DOCKER=$3 SYSTEM_PRUNE=$4 DOCKCROSS_TAG=$5 -GITHUB_ARTIFACTS=$6 -GITHUB_ARTIFACTS_RUN=$7 +DOCKCROSS_ONLY=$6 +GITHUB_ARTIFACTS=$7 +GITHUB_ARTIFACTS_RUN=$8 if $GITHUB_ARTIFACTS; then src/main/c/h3-java/pull-from-github.sh "$GITHUB_ARTIFACTS_RUN" @@ -180,10 +182,16 @@ UPGRADE_CMAKE=true CMAKE_ROOT=target/cmake-3.23.2-linux-x86_64 mkdir -p $CMAKE_ROOT +DOCKCROSS_IMAGES=android-arm android-arm64 linux-arm64 linux-armv5 linux-armv7 linux-s390x \ + linux-ppc64le linux-x64 linux-x86 windows-static-x64 windows-static-x86 +if ! $DOCKCROSS_ONLY; then + DOCKCROSS_IMAGES=$DOCKCROSS_ONLY + echo Building only: $DOCKCROSS_IMAGES +fi + # linux-armv6 excluded because of build failure # linux-mips excluded due to manifest error -for image in android-arm android-arm64 linux-arm64 linux-armv5 linux-armv7 linux-s390x \ - linux-ppc64le linux-x64 linux-x86 windows-static-x64 windows-static-x86; do +for image in $DOCKCROSS_IMAGES; do # Setup for using dockcross BUILD_ROOT=target/h3-java-build-$image From f60d5baf27da01c9474e0b30a8435badb1398337 Mon Sep 17 00:00:00 2001 From: Isaac Brodsky Date: Thu, 20 Jun 2024 08:23:24 -0700 Subject: [PATCH 10/10] fix --- pom.xml | 2 +- src/main/c/h3-java/build-h3.sh | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 968e5b3..9ebd6a2 100644 --- a/pom.xml +++ b/pom.xml @@ -72,7 +72,7 @@ true false 20210624-de7b1b0 - > + false diff --git a/src/main/c/h3-java/build-h3.sh b/src/main/c/h3-java/build-h3.sh index a3019d6..883b60d 100755 --- a/src/main/c/h3-java/build-h3.sh +++ b/src/main/c/h3-java/build-h3.sh @@ -182,8 +182,7 @@ UPGRADE_CMAKE=true CMAKE_ROOT=target/cmake-3.23.2-linux-x86_64 mkdir -p $CMAKE_ROOT -DOCKCROSS_IMAGES=android-arm android-arm64 linux-arm64 linux-armv5 linux-armv7 linux-s390x \ - linux-ppc64le linux-x64 linux-x86 windows-static-x64 windows-static-x86 +DOCKCROSS_IMAGES="android-arm android-arm64 linux-arm64 linux-armv5 linux-armv7 linux-s390x linux-ppc64le linux-x64 linux-x86 windows-static-x64 windows-static-x86" if ! $DOCKCROSS_ONLY; then DOCKCROSS_IMAGES=$DOCKCROSS_ONLY echo Building only: $DOCKCROSS_IMAGES