Skip to content

Commit

Permalink
Add support for Apple Silicon to RocksJava (#9254)
Browse files Browse the repository at this point in the history
    Summary:
    Fixes facebook/rocksdb#7720

    Updated Makefile with flags to define target architecture when compiling/linking,
    and added goal `rocksdbjavastaticosxub` to build a OS X Universal Binary native library.

    Pull Request resolved: facebook/rocksdb#9254

    Reviewed By: mrambacher

    Differential Revision: D33551160

    Pulled By: pdillinger

    fbshipit-source-id: 9ce9962e03aacf55014545a6cdf638b5b14b8fa9
  • Loading branch information
Myasuka committed Jul 18, 2022
1 parent f75e983 commit dac2c60
Show file tree
Hide file tree
Showing 4 changed files with 139 additions and 30 deletions.
42 changes: 38 additions & 4 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,13 @@ commands:
command: |
HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake
install-jdk8-on-macos:
steps:
- run:
name: Install JDK 8 on macos
command: |
brew install --cask adoptopenjdk/openjdk/adoptopenjdk8
increase-max-open-files-on-macos:
steps:
- run:
Expand Down Expand Up @@ -53,7 +60,7 @@ commands:
pre-steps-macos:
steps:
- pre-steps:
python-version: "3.6.0"
python-version: "3.7.8"

post-steps:
steps:
Expand Down Expand Up @@ -486,7 +493,7 @@ jobs:

build-macos-java-static:
macos:
xcode: 11.3.0
xcode: 12.5.1
resource_class: medium
environment:
JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
Expand All @@ -495,6 +502,7 @@ jobs:
- install-pyenv-on-macos
- install-gflags-on-macos
- install-cmake-on-macos
- install-jdk8-on-macos
- pre-steps-macos
- run:
name: "Set Java Environment"
Expand All @@ -504,8 +512,33 @@ jobs:
which java && java -version
which javac && javac -version
- run:
name: "Build RocksDBJava Static Library"
command: make V=1 J=8 -j8 rocksdbjavastatic | .circleci/cat_ignore_eagain
name: "Build RocksDBJava x86 and ARM Static Libraries"
command: make V=1 J=8 -j8 rocksdbjavastaticosx 2>&1 | .circleci/cat_ignore_eagain
- post-steps

build-macos-java-static-universal:
macos:
xcode: 12.5.1
resource_class: medium
environment:
JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
steps:
- increase-max-open-files-on-macos
- install-pyenv-on-macos
- install-gflags-on-macos
- install-cmake-on-macos
- install-jdk8-on-macos
- pre-steps-macos
- run:
name: "Set Java Environment"
command: |
echo "JAVA_HOME=${JAVA_HOME}"
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
which java && java -version
which javac && javac -version
- run:
name: "Build RocksDBJava Universal Binary Static Library"
command: make V=1 J=8 -j8 rocksdbjavastaticosx_ub 2>&1 | .circleci/cat_ignore_eagain
- post-steps

build-examples:
Expand Down Expand Up @@ -659,6 +692,7 @@ workflows:
- build-linux-java-static
- build-macos-java
- build-macos-java-static
- build-macos-java-static-universal
build-examples:
jobs:
- build-examples
Expand Down
107 changes: 84 additions & 23 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -136,12 +136,6 @@ CFLAGS += -DHAVE_POWER8
HAVE_POWER8=1
endif

ifeq (,$(shell $(CXX) -fsyntax-only -march=armv8-a+crc+crypto -xc /dev/null 2>&1))
CXXFLAGS += -march=armv8-a+crc+crypto
CFLAGS += -march=armv8-a+crc+crypto
ARMCRC_SOURCE=1
endif

# if we're compiling for shared libraries, add the shared flags
ifeq ($(LIB_MODE),shared)
CXXFLAGS += $(PLATFORM_SHARED_CFLAGS) -DROCKSDB_DLL
Expand Down Expand Up @@ -252,6 +246,16 @@ ROCKSDB_PLUGIN_SOURCES = $(foreach plugin, $(ROCKSDB_PLUGINS), $(foreach source,
ROCKSDB_PLUGIN_HEADERS = $(foreach plugin, $(ROCKSDB_PLUGINS), $(foreach header, $($(plugin)_HEADERS), plugin/$(plugin)/$(header)))
PLATFORM_LDFLAGS += $(foreach plugin, $(ROCKSDB_PLUGINS), $($(plugin)_LDFLAGS))

CXXFLAGS += $(ARCHFLAG)

ifeq (,$(shell $(CXX) -fsyntax-only -march=armv8-a+crc+crypto -xc /dev/null 2>&1))
ifneq ($(PLATFORM),OS_MACOSX)
CXXFLAGS += -march=armv8-a+crc+crypto
CFLAGS += -march=armv8-a+crc+crypto
ARMCRC_SOURCE=1
endif
endif

export JAVAC_ARGS
CLEAN_FILES += make_config.mk rocksdb.pc

Expand Down Expand Up @@ -301,6 +305,19 @@ ifeq ($(LIB_MODE),shared)
EXEC_LDFLAGS += -Wl,-rpath -Wl,'$$ORIGIN'
endif

ifeq ($(PLATFORM), OS_MACOSX)
ifeq ($(ARCHFLAG), -arch arm64)
ifneq ($(MACHINE), arm64)
# If we're building on a non-arm64 machine but targeting arm64 Mac, we need to disable
# linking with jemalloc (as it won't be arm64-compatible) and remove some other options
# set during platform detection
DISABLE_JEMALLOC=1
PLATFORM_CFLAGS := $(filter-out -march=native -DHAVE_SSE42, $(PLATFORM_CFLAGS))
PLATFORM_CXXFLAGS := $(filter-out -march=native -DHAVE_SSE42, $(PLATFORM_CXXFLAGS))
endif
endif
endif

# ASAN doesn't work well with jemalloc. If we're compiling with ASAN, we should use regular malloc.
ifdef COMPILE_WITH_ASAN
DISABLE_JEMALLOC=1
Expand Down Expand Up @@ -1954,11 +1971,13 @@ ifneq ($(origin JNI_LIBC), undefined)
JNI_LIBC_POSTFIX = -$(JNI_LIBC)
endif

ifneq (,$(filter ppc% arm64 aarch64 sparc64, $(MACHINE)))
ifeq (,$(ROCKSDBJNILIB))
ifneq (,$(filter ppc% s390x arm64 aarch64 sparc64, $(MACHINE)))
ROCKSDBJNILIB = librocksdbjni-linux-$(MACHINE)$(JNI_LIBC_POSTFIX).so
else
ROCKSDBJNILIB = librocksdbjni-linux$(ARCH)$(JNI_LIBC_POSTFIX).so
endif
endif
ROCKSDB_JAVA_VERSION ?= $(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)
ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-linux$(ARCH)$(JNI_LIBC_POSTFIX).jar
ROCKSDB_JAR_ALL = rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar
Expand All @@ -1984,7 +2003,13 @@ ZSTD_DOWNLOAD_BASE ?= https://github.com/facebook/zstd/archive
CURL_SSL_OPTS ?= --tlsv1

ifeq ($(PLATFORM), OS_MACOSX)
ifeq (,$(findstring librocksdbjni-osx,$(ROCKSDBJNILIB)))
ifeq ($(MACHINE),arm64)
ROCKSDBJNILIB = librocksdbjni-osx-aarch64.jnilib
else
ROCKSDBJNILIB = librocksdbjni-osx.jnilib
endif
endif
ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-osx.jar
SHA256_CMD = openssl sha256 -r
ifneq ("$(wildcard $(JAVA_HOME)/include/darwin)","")
Expand All @@ -1993,6 +2018,7 @@ else
JAVA_INCLUDE = -I/System/Library/Frameworks/JavaVM.framework/Headers/
endif
endif

ifeq ($(PLATFORM), OS_FREEBSD)
JAVA_INCLUDE = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/freebsd
ROCKSDBJNILIB = librocksdbjni-freebsd$(ARCH).so
Expand Down Expand Up @@ -2027,7 +2053,11 @@ zlib-$(ZLIB_VER).tar.gz:
libz.a: zlib-$(ZLIB_VER).tar.gz
-rm -rf zlib-$(ZLIB_VER)
tar xvzf zlib-$(ZLIB_VER).tar.gz
cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' ./configure --static && $(MAKE)
if [ -n"$(ARCHFLAG)" ]; then \
cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' ./configure --static --archs="$(ARCHFLAG)" && $(MAKE); \
else \
cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' ./configure --static && $(MAKE); \
fi
cp zlib-$(ZLIB_VER)/libz.a .

bzip2-$(BZIP2_VER).tar.gz:
Expand All @@ -2041,7 +2071,7 @@ bzip2-$(BZIP2_VER).tar.gz:
libbz2.a: bzip2-$(BZIP2_VER).tar.gz
-rm -rf bzip2-$(BZIP2_VER)
tar xvzf bzip2-$(BZIP2_VER).tar.gz
cd bzip2-$(BZIP2_VER) && $(MAKE) CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64 ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' AR='ar ${EXTRA_ARFLAGS}'
cd bzip2-$(BZIP2_VER) && $(MAKE) CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64 $(ARCHFLAG) ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' AR='ar ${EXTRA_ARFLAGS}' libbz2.a
cp bzip2-$(BZIP2_VER)/libbz2.a .

snappy-$(SNAPPY_VER).tar.gz:
Expand All @@ -2056,7 +2086,7 @@ libsnappy.a: snappy-$(SNAPPY_VER).tar.gz
-rm -rf snappy-$(SNAPPY_VER)
tar xvzf snappy-$(SNAPPY_VER).tar.gz
mkdir snappy-$(SNAPPY_VER)/build
cd snappy-$(SNAPPY_VER)/build && CFLAGS='${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' CXXFLAGS='${JAVA_STATIC_DEPS_CXXFLAGS} ${EXTRA_CXXFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON ${PLATFORM_CMAKE_FLAGS} .. && $(MAKE) ${SNAPPY_MAKE_TARGET}
cd snappy-$(SNAPPY_VER)/build && CFLAGS='$(ARCHFLAG) ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' CXXFLAGS='$(ARCHFLAG) ${JAVA_STATIC_DEPS_CXXFLAGS} ${EXTRA_CXXFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON ${PLATFORM_CMAKE_FLAGS} .. && $(MAKE) ${SNAPPY_MAKE_TARGET}
cp snappy-$(SNAPPY_VER)/build/libsnappy.a .

lz4-$(LZ4_VER).tar.gz:
Expand All @@ -2070,7 +2100,7 @@ lz4-$(LZ4_VER).tar.gz:
liblz4.a: lz4-$(LZ4_VER).tar.gz
-rm -rf lz4-$(LZ4_VER)
tar xvzf lz4-$(LZ4_VER).tar.gz
cd lz4-$(LZ4_VER)/lib && $(MAKE) CFLAGS='-fPIC -O2 ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' all
cd lz4-$(LZ4_VER)/lib && $(MAKE) CFLAGS='-fPIC -O2 $(ARCHFLAG) ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' all
cp lz4-$(LZ4_VER)/lib/liblz4.a .

zstd-$(ZSTD_VER).tar.gz:
Expand All @@ -2084,7 +2114,7 @@ zstd-$(ZSTD_VER).tar.gz:
libzstd.a: zstd-$(ZSTD_VER).tar.gz
-rm -rf zstd-$(ZSTD_VER)
tar xvzf zstd-$(ZSTD_VER).tar.gz
cd zstd-$(ZSTD_VER)/lib && DESTDIR=. PREFIX= $(MAKE) CFLAGS='-fPIC -O2 ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' libzstd.a
cd zstd-$(ZSTD_VER)/lib && DESTDIR=. PREFIX= $(MAKE) CFLAGS='-fPIC -O2 $(ARCHFLAG) ${JAVA_STATIC_DEPS_CCFLAGS} ${EXTRA_CFLAGS}' LDFLAGS='${JAVA_STATIC_DEPS_LDFLAGS} ${EXTRA_LDFLAGS}' libzstd.a
cp zstd-$(ZSTD_VER)/lib/libzstd.a .

# A version of each $(LIB_OBJECTS) compiled with -fPIC and a fixed set of static compression libraries
Expand All @@ -2106,7 +2136,37 @@ endif
$(MAKE) rocksdbjavastatic_deps
$(MAKE) rocksdbjavastatic_libobjects
$(MAKE) rocksdbjavastatic_javalib
$(MAKE) rocksdbjavastatic_jar

rocksdbjavastaticosx: rocksdbjavastaticosx_archs
mv java/target/librocksdbjni-osx-x86_64.jnilib java/target/librocksdbjni-osx.jnilib
mv java/target/librocksdbjni-osx-arm64.jnilib java/target/librocksdbjni-osx-aarch64.jnilib

rocksdbjavastaticosx_ub: rocksdbjavastaticosx_archs
lipo -create -output ./java/target/$(ROCKSDBJNILIB) java/target/librocksdbjni-osx-x86_64.jnilib java/target/librocksdbjni-osx-arm64.jnilib
$(MAKE) rocksdbjavastatic_jar

rocksdbjavastaticosx_archs:
$(MAKE) rocksdbjavastaticosx_arch_x86_64
$(MAKE) rocksdbjavastaticosx_arch_arm64

rocksdbjavastaticosx_arch_%:
ifeq ($(JAVA_HOME),)
$(error JAVA_HOME is not set)
endif
$(MAKE) clean-ext-libraries-bin
$(MAKE) clean-rocks
ARCHFLAG="-arch $*" $(MAKE) rocksdbjavastatic_deps
ARCHFLAG="-arch $*" $(MAKE) rocksdbjavastatic_libobjects
ARCHFLAG="-arch $*" ROCKSDBJNILIB="librocksdbjni-osx-$*.jnilib" $(MAKE) rocksdbjavastatic_javalib

ifeq ($(JAR_CMD),)
ifneq ($(JAVA_HOME),)
JAR_CMD := $(JAVA_HOME)/bin/jar
else
JAR_CMD := jar
endif
endif
rocksdbjavastatic_javalib:
cd java; SHA256_CMD='$(SHA256_CMD)' $(MAKE) javalib
rm -f java/target/$(ROCKSDBJNILIB)
Expand All @@ -2117,11 +2177,13 @@ rocksdbjavastatic_javalib:
cd java/target;if [ "$(DEBUG_LEVEL)" == "0" ]; then \
strip $(STRIPFLAGS) $(ROCKSDBJNILIB); \
fi
cd java;jar -cf target/$(ROCKSDB_JAR) HISTORY*.md
cd java/target;jar -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB)
cd java/target/classes;jar -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class
cd java/target/apidocs;jar -cf ../$(ROCKSDB_JAVADOCS_JAR) *
cd java/src/main/java;jar -cf ../../../target/$(ROCKSDB_SOURCES_JAR) org

rocksdbjavastatic_jar:
cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR) HISTORY*.md
cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB)
cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class
cd java/target/apidocs; $(JAR_CMD) -cf ../$(ROCKSDB_JAVADOCS_JAR) *
cd java/src/main/java; $(JAR_CMD) -cf ../../../target/$(ROCKSDB_SOURCES_JAR) org
openssl sha1 java/target/$(ROCKSDB_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR).sha1
openssl sha1 java/target/$(ROCKSDB_JAVADOCS_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAVADOCS_JAR).sha1
openssl sha1 java/target/$(ROCKSDB_SOURCES_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_SOURCES_JAR).sha1
Expand All @@ -2130,18 +2192,17 @@ rocksdbjavastatic_deps: $(JAVA_COMPRESSIONS)

rocksdbjavastatic_libobjects: $(LIB_OBJECTS)

rocksdbjavastaticrelease: rocksdbjavastatic
rocksdbjavastaticrelease: rocksdbjavastaticosx
cd java/crossbuild && (vagrant destroy -f || true) && vagrant up linux32 && vagrant halt linux32 && vagrant up linux64 && vagrant halt linux64 && vagrant up linux64-musl && vagrant halt linux64-musl
cd java;jar -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md
cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
cd java/target/classes;jar -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class
openssl sha1 java/target/$(ROCKSDB_JAR_ALL) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR_ALL).sha1

rocksdbjavastaticreleasedocker: rocksdbjavastatic rocksdbjavastaticdockerx86 rocksdbjavastaticdockerx86_64 rocksdbjavastaticdockerx86musl rocksdbjavastaticdockerx86_64musl
cd java;jar -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md
jar -uf java/target/$(ROCKSDB_JAR_ALL) HISTORY*.md
cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib librocksdbjni-win64.dll
cd java/target/classes;jar -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class
rocksdbjavastaticreleasedocker: rocksdbjavastaticosx rocksdbjavastaticdockerx86 rocksdbjavastaticdockerx86_64 rocksdbjavastaticdockerx86musl rocksdbjavastaticdockerx86_64musl
cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md
cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class
openssl sha1 java/target/$(ROCKSDB_JAR_ALL) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR_ALL).sha1

frocksdbjavastaticreleasedocker: rocksdbjavastaticreleasedocker
Expand Down
6 changes: 5 additions & 1 deletion java/src/main/java/org/rocksdb/util/Environment.java
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,11 @@ public static String getJniLibraryName(final String name) {
return String.format("%sjni-linux%s%s", name, arch, getLibcPostfix());
}
} else if (isMac()) {
return String.format("%sjni-osx", name);
if (isAarch64()) {
return String.format("%sjni-osx-%s", name, ARCH);
} else {
return String.format("%sjni-osx", name);
}
} else if (isFreeBSD()) {
return String.format("%sjni-freebsd%s", name, is64Bit() ? "64" : "32");
} else if (isAix() && is64Bit()) {
Expand Down
14 changes: 12 additions & 2 deletions java/src/test/java/org/rocksdb/util/EnvironmentTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ public void mac32() {

@Test
public void mac64() {
setEnvironmentClassFields("mac", "64");
setEnvironmentClassFields("mac", "x86-64");
assertThat(Environment.isWindows()).isFalse();
assertThat(Environment.getJniLibraryExtension()).
isEqualTo(".jnilib");
Expand All @@ -53,6 +53,16 @@ public void mac64() {
isEqualTo("librocksdbjni.dylib");
}

@Test
public void macAarch64() {
setEnvironmentClassFields("mac", "aarch64");
assertThat(Environment.isWindows()).isFalse();
assertThat(Environment.getJniLibraryExtension()).isEqualTo(".jnilib");
assertThat(Environment.getJniLibraryFileName("rocksdb"))
.isEqualTo("librocksdbjni-osx-aarch64.jnilib");
assertThat(Environment.getSharedLibraryFileName("rocksdb")).isEqualTo("librocksdbjni.dylib");
}

@Test
public void nix32() {
// Linux
Expand Down Expand Up @@ -184,7 +194,7 @@ public void ppc64le() {
}

@Test
public void aarch64() {
public void linuxArch64() {
setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, false);
setEnvironmentClassFields("Linux", "aarch64");
assertThat(Environment.isUnix()).isTrue();
Expand Down

0 comments on commit dac2c60

Please sign in to comment.