From 10dc4772a99af6af4903e3900ca1d382a43552f8 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 18 Jul 2024 01:22:26 +0800 Subject: [PATCH 01/10] setup --- hugegraph-pd/Dockerfile | 2 +- hugegraph-pd/hg-pd-dist/pom.xml | 1 - hugegraph-pd/pom.xml | 2 +- .../src/assembly/descriptor/assembly.xml | 11 ---- .../src/assembly/travis/start-pd.sh | 2 +- .../src/assembly/travis/start-store.sh | 2 +- hugegraph-server/pom.xml | 3 +- hugegraph-store/Dockerfile | 2 +- hugegraph-store/hg-store-dist/pom.xml | 1 - hugegraph-store/pom.xml | 2 +- install-dist/pom.xml | 62 +++++++++++++++++++ install-dist/scripts/apache-release.sh | 4 +- pom.xml | 1 + 13 files changed, 71 insertions(+), 24 deletions(-) diff --git a/hugegraph-pd/Dockerfile b/hugegraph-pd/Dockerfile index a86b5b9bd4..e507900ade 100644 --- a/hugegraph-pd/Dockerfile +++ b/hugegraph-pd/Dockerfile @@ -30,7 +30,7 @@ RUN mvn package $MAVEN_ARGS -e -B -ntp -DskipTests -Dmaven.javadoc.skip=true && # Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13 FROM openjdk:11-slim -COPY --from=build /pkg/hugegraph-pd/apache-hugegraph-incubating-pd-*/ /hugegraph-pd/ +COPY --from=build /pkg/hugegraph-pd/apache-hugegraph-pd-incubating-*/ /hugegraph-pd/ LABEL maintainer="HugeGraph Docker Maintainers " # TODO: use g1gc or zgc as default diff --git a/hugegraph-pd/hg-pd-dist/pom.xml b/hugegraph-pd/hg-pd-dist/pom.xml index 096c27790b..f669cf5ae5 100644 --- a/hugegraph-pd/hg-pd-dist/pom.xml +++ b/hugegraph-pd/hg-pd-dist/pom.xml @@ -31,7 +31,6 @@ ${project.parent.basedir} - bash ${project.basedir}/src/assembly ${assembly.dir}/descriptor ${assembly.dir}/static diff --git a/hugegraph-pd/pom.xml b/hugegraph-pd/pom.xml index 34f25b5d8b..b2547a7dc4 100644 --- a/hugegraph-pd/pom.xml +++ b/hugegraph-pd/pom.xml @@ -43,7 +43,7 @@ 2.17.0 - apache-${release.name}-incubating-pd-${project.version} + apache-${release.name}-pd-incubating-${project.version} diff --git a/hugegraph-server/hugegraph-dist/src/assembly/descriptor/assembly.xml b/hugegraph-server/hugegraph-dist/src/assembly/descriptor/assembly.xml index b1c293dfb1..b7f9ee0487 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/descriptor/assembly.xml +++ b/hugegraph-server/hugegraph-dist/src/assembly/descriptor/assembly.xml @@ -46,17 +46,6 @@ *.jar - - ${top.level.dir}/hugegraph-dist/release-docs/ - / - - - ${top.level.dir}/../ - / - - DISCLAIMER* - - diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/start-pd.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/start-pd.sh index c9200fefb3..9f694d5c3c 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/start-pd.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/start-pd.sh @@ -18,7 +18,7 @@ set -ev HOME_DIR=$(pwd) -PD_DIR=$HOME_DIR/hugegraph-pd/apache-hugegraph-incubating-pd-1.5.0 +PD_DIR=$HOME_DIR/hugegraph-pd/apache-hugegraph-pd-incubating-1.5.0 pushd $PD_DIR . bin/start-hugegraph-pd.sh diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/start-store.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/start-store.sh index c4c11c2c35..23e8f2297c 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/start-store.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/start-store.sh @@ -18,7 +18,7 @@ set -ev HOME_DIR=$(pwd) -STORE_DIR=$HOME_DIR/hugegraph-store/apache-hugegraph-incubating-store-1.5.0 +STORE_DIR=$HOME_DIR/hugegraph-store/apache-hugegraph-store-incubating-1.5.0 pushd $STORE_DIR . bin/start-hugegraph-store.sh diff --git a/hugegraph-server/pom.xml b/hugegraph-server/pom.xml index 2eb0158570..6234d17b26 100644 --- a/hugegraph-server/pom.xml +++ b/hugegraph-server/pom.xml @@ -37,7 +37,7 @@ ${project.basedir}/.. - apache-${release.name}-incubating-server-${project.version} + apache-${release.name}-server-incubating-${project.version} 1.7.5 1.2.17 2.17.1 @@ -49,7 +49,6 @@ 3.0.3 4.2.4 3.21.0-GA - bash 3.1.2 8.45 1.47.0 diff --git a/hugegraph-store/Dockerfile b/hugegraph-store/Dockerfile index c7329a8f81..042ceef42f 100644 --- a/hugegraph-store/Dockerfile +++ b/hugegraph-store/Dockerfile @@ -30,7 +30,7 @@ RUN mvn package $MAVEN_ARGS -e -B -ntp -DskipTests -Dmaven.javadoc.skip=true && # Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13 FROM openjdk:11-slim -COPY --from=build /pkg/hugegraph-store/apache-hugegraph-incubating-store-*/ /hugegraph-store/ +COPY --from=build /pkg/hugegraph-store/apache-hugegraph-store-incubating-*/ /hugegraph-store/ LABEL maintainer="HugeGraph Docker Maintainers " # TODO: use g1gc or zgc as default diff --git a/hugegraph-store/hg-store-dist/pom.xml b/hugegraph-store/hg-store-dist/pom.xml index dc6b1bf34f..ce2bd5824e 100644 --- a/hugegraph-store/hg-store-dist/pom.xml +++ b/hugegraph-store/hg-store-dist/pom.xml @@ -32,7 +32,6 @@ ${project.parent.basedir} - bash ${project.basedir}/src/assembly ${assembly.dir}/descriptor ${assembly.dir}/static diff --git a/hugegraph-store/pom.xml b/hugegraph-store/pom.xml index be1bbfc7ff..22c96a77bd 100644 --- a/hugegraph-store/pom.xml +++ b/hugegraph-store/pom.xml @@ -47,7 +47,7 @@ 2.15.0 - apache-${release.name}-incubating-store-${project.version} + apache-${release.name}-store-incubating-${project.version} diff --git a/install-dist/pom.xml b/install-dist/pom.xml index d351fcf908..ff4fb92c2f 100644 --- a/install-dist/pom.xml +++ b/install-dist/pom.xml @@ -27,4 +27,66 @@ install-dist + + + apache-${release.name}-incubating-${project.version} + + + + + + maven-antrun-plugin + + + package + + run + + + + + + root_path=$(cd $(dirname $0)/..; pwd) + cd $root_path || exit + + mkdir -p ${final.name} + cp -r -v $root_path/hugegraph-pd/apache-hugegraph-pd-incubating-${project.version} ${final.name}/ || exit + cp -r -v $root_path/hugegraph-store/apache-hugegraph-store-incubating-${project.version} ${final.name}/ || exit + cp -r -v $root_path/hugegraph-server/apache-hugegraph-server-incubating-${project.version} ${final.name}/ || exit + cp -r -v $root_path/install-dist/release-docs/* $root_path/DISCLAIMER ${final.name}/ || exit + + tar zcvf $root_path/target/${final.name}.tar.gz ./${final.name} || exit 1 + + md5sum $root_path/target/${final.name}.tar.gz + echo -n "hugegraph tar.gz available at: " + echo "$root_path/target/${final.name}.tar.gz" + rm -f ${project.basedir}/dist.sh + + + + + + + + + + + maven-clean-plugin + + + + ${project.basedir}/.. + + ${final.name}.tar.gz + + false + + + ${project.basedir}/../${final.name} + + + + + + diff --git a/install-dist/scripts/apache-release.sh b/install-dist/scripts/apache-release.sh index 859a635620..f8a44f5476 100755 --- a/install-dist/scripts/apache-release.sh +++ b/install-dist/scripts/apache-release.sh @@ -47,9 +47,7 @@ cd - || exit # step2: copy the binary file (Optional) # Note: it's optional for project to generate binary package (skip this step if not need) -cp -v ../../hugegraph-server/apache-${REPO}-incubating-server-"${RELEASE_VERSION}".tar.gz \ - dist/apache-${REPO} || exit -# TODO: pd & store +cp -v ../../apache-${REPO}-incubating-"${RELEASE_VERSION}".tar.gz dist/apache-${REPO} || exit # step3: sign + hash ##### 3.1 sign in source & binary package diff --git a/pom.xml b/pom.xml index 0f8bc3e228..44be456375 100644 --- a/pom.xml +++ b/pom.xml @@ -92,6 +92,7 @@ UTF-8 11 11 + bash From 67f51b31fe828d978ab552022b4741c6d393fdf8 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 18 Jul 2024 01:37:05 +0800 Subject: [PATCH 02/10] update .gitattributes --- .gitattributes | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitattributes b/.gitattributes index 636f55aa01..c4a1edec5a 100755 --- a/.gitattributes +++ b/.gitattributes @@ -8,5 +8,7 @@ apache-release.sh export-ignore # ignored directory .github/ export-ignore -hugegraph-dist/scripts/ export-ignore -#assembly/ export-ignore +.idea/ export-ignore +install-dist/scripts/ export-ignore +hugegraph-store/hg-store-dist/src/assembly/static/bin/libjemalloc.so export-ignore +hugegraph-store/hg-store-dist/src/assembly/static/bin/libjemalloc_aarch64.so export-ignore From 43bb853b05800b1a9263322efc2f8bc633dd543e Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 18 Jul 2024 01:40:29 +0800 Subject: [PATCH 03/10] update --- .gitattributes | 5 +++-- install-dist/scripts/apache-release.sh | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitattributes b/.gitattributes index c4a1edec5a..ca5e57db70 100755 --- a/.gitattributes +++ b/.gitattributes @@ -5,10 +5,11 @@ apache-release.sh export-ignore .licenserc.yaml export-ignore .editorconfig export-ignore +hugegraph-store/hg-store-dist/src/assembly/static/bin/libjemalloc.so export-ignore +hugegraph-store/hg-store-dist/src/assembly/static/bin/libjemalloc_aarch64.so export-ignore # ignored directory .github/ export-ignore .idea/ export-ignore install-dist/scripts/ export-ignore -hugegraph-store/hg-store-dist/src/assembly/static/bin/libjemalloc.so export-ignore -hugegraph-store/hg-store-dist/src/assembly/static/bin/libjemalloc_aarch64.so export-ignore +docker/ export-ignore diff --git a/install-dist/scripts/apache-release.sh b/install-dist/scripts/apache-release.sh index f8a44f5476..168217c425 100755 --- a/install-dist/scripts/apache-release.sh +++ b/install-dist/scripts/apache-release.sh @@ -47,7 +47,7 @@ cd - || exit # step2: copy the binary file (Optional) # Note: it's optional for project to generate binary package (skip this step if not need) -cp -v ../../apache-${REPO}-incubating-"${RELEASE_VERSION}".tar.gz dist/apache-${REPO} || exit +cp -v ../../target/apache-${REPO}-incubating-"${RELEASE_VERSION}".tar.gz dist/apache-${REPO} || exit # step3: sign + hash ##### 3.1 sign in source & binary package From 00cdac5bab6795402e004ec1a47a4048429a7b69 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 18 Jul 2024 10:32:27 +0800 Subject: [PATCH 04/10] fixup --- .gitignore | 3 +++ .../src/assembly/travis/run-api-test-for-raft.sh | 2 +- .../hugegraph-dist/src/assembly/travis/run-api-test.sh | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index bf4993e030..c51d74da61 100644 --- a/.gitignore +++ b/.gitignore @@ -58,6 +58,9 @@ build/ # maven ignore apache-hugegraph-incubating-*/ +apache-hugegraph-pd-incubating-*/ +apache-hugegraph-store-incubating-*/ +apache-hugegraph-server-incubating-*/ output/ *.war *.zip diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test-for-raft.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test-for-raft.sh index 85c45b8c47..529874f58b 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test-for-raft.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test-for-raft.sh @@ -23,7 +23,7 @@ REPORT_FILE=$REPORT_DIR/jacoco-api-test.xml TRAVIS_DIR=$(dirname $0) VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) -SERVER_DIR=hugegraph-server/apache-hugegraph-incubating-server-$VERSION +SERVER_DIR=hugegraph-server/apache-hugegraph-server-incubating-$VERSION RAFT1_DIR=hugegraph-raft1 RAFT2_DIR=hugegraph-raft2 RAFT3_DIR=hugegraph-raft3 diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh index 17d4722dc2..8008f39cdb 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh @@ -23,7 +23,7 @@ REPORT_FILE=$REPORT_DIR/jacoco-api-test-for-raft.xml TRAVIS_DIR=$(dirname $0) VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) -SERVER_DIR=hugegraph-server/apache-hugegraph-incubating-server-$VERSION/ +SERVER_DIR=hugegraph-server/apache-hugegraph-server-incubating-$VERSION/ CONF=$SERVER_DIR/conf/graphs/hugegraph.properties REST_SERVER_CONF=$SERVER_DIR/conf/rest-server.properties GREMLIN_SERVER_CONF=$SERVER_DIR/conf/gremlin-server.yaml From a07ebd828b52f016d27c65b5526b254bd6b4b402 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 18 Jul 2024 11:23:20 +0800 Subject: [PATCH 05/10] exclude *.so for binary package --- .../assembly/descriptor/server-assembly.xml | 3 ++ .../static/bin/start-hugegraph-store.sh | 54 ++++++++++++++++--- 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/hugegraph-store/hg-store-dist/src/assembly/descriptor/server-assembly.xml b/hugegraph-store/hg-store-dist/src/assembly/descriptor/server-assembly.xml index 3e7a6aeb55..522a93c917 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/descriptor/server-assembly.xml +++ b/hugegraph-store/hg-store-dist/src/assembly/descriptor/server-assembly.xml @@ -32,6 +32,9 @@ * + + *.so + 755 diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh index 991d42767a..ad93b2ac82 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh @@ -36,14 +36,54 @@ LOGS="$TOP/logs" OUTPUT=${LOGS}/hugegraph-store-server.log GITHUB="https://github.com" PID_FILE="$BIN/pid" -arch=$(arch) -echo "Current arch: ", "${arch}" -#if [[ $arch =~ "aarch64" ]];then -# export LD_PRELOAD="$TOP/bin/libjemalloc_aarch64.so" -#else -export LD_PRELOAD="$TOP/bin/libjemalloc.so" -#fi +arch=$(uname -m) +echo "Current arch: $arch" + +download_and_verify() { + local url=$1 + local filepath=$2 + local expected_md5=$3 + + if [[ -f $filepath ]]; then + echo "File $filepath exists. Verifying MD5 checksum..." + actual_md5=$(md5sum $filepath | awk '{ print $1 }') + if [[ $actual_md5 != $expected_md5 ]]; then + echo "MD5 checksum verification failed for $filepath. Expected: $expected_md5, but got: $actual_md5" + echo "Deleting $filepath..." + rm -f $filepath + else + echo "MD5 checksum verification succeeded for $filepath." + return + fi + fi + + echo "Downloading $filepath..." + curl -L -o $filepath $url + + actual_md5=$(md5sum $filepath | awk '{ print $1 }') + if [[ $actual_md5 != $expected_md5 ]]; then + echo "MD5 checksum verification failed for $filepath after download. Expected: $expected_md5, but got: $actual_md5" + exit 1 + fi +} + +if [[ $arch == "aarch64" || $arch == "arm64" ]]; then + libfile="$TOP/bin/libjemalloc_aarch64.so" + download_url="https://github.com/apache/incubator-hugegraph/raw/master/hugegraph-store/hg-store-dist/src/assembly/static/bin/libjemalloc_aarch64.so" + expected_md5="2a631d2f81837f9d5864586761c5e380" + download_and_verify $download_url $libfile $expected_md5 + export LD_PRELOAD=$libfile +elif [[ $arch == "x86_64" ]]; then + libfile="$TOP/bin/libjemalloc.so" + download_url="https://github.com/apache/incubator-hugegraph/raw/master/hugegraph-store/hg-store-dist/src/assembly/static/bin/libjemalloc.so" + expected_md5="fd61765eec3bfea961b646c269f298df" + download_and_verify $download_url $libfile $expected_md5 + export LD_PRELOAD=$libfile +else + echo "Unsupported architecture: $arch" + exit 1 +fi ##pd/store max user processes, ulimit -u # Reduce the maximum number of processes that can be opened by a normal dev/user From 126e834a44343a9c6255d9b0c1f83aee5fc867ab Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 18 Jul 2024 16:24:16 +0800 Subject: [PATCH 06/10] handle hugegraph-server.keystore --- hugegraph-server/hugegraph-dist/pom.xml | 80 ++++++++++--------- .../assembly/static/bin/hugegraph-server.sh | 9 +-- .../assembly/static/bin/start-hugegraph.sh | 5 ++ .../static/bin/start-hugegraph-store.sh | 25 +++--- 4 files changed, 63 insertions(+), 56 deletions(-) diff --git a/hugegraph-server/hugegraph-dist/pom.xml b/hugegraph-server/hugegraph-dist/pom.xml index 50e232ba33..419f60f1f5 100644 --- a/hugegraph-server/hugegraph-dist/pom.xml +++ b/hugegraph-server/hugegraph-dist/pom.xml @@ -222,45 +222,47 @@ - - download-keystore - prepare-package - - run - - - - - - - - - - - cp-keystore - package - - run - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh index 8174082d49..8b6a16d5ad 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh @@ -61,14 +61,7 @@ ensure_path_writable "$PLUGINS" # The maximum and minimum heap memory that service can use MAX_MEM=$((32 * 1024)) MIN_MEM=$((1 * 512)) -# TODO: upgrade to Java 11 in 1.5.0 -MIN_JAVA_VERSION=8 - -# Note: Download for HTTPS, could comment out if you don't need it -# TODO: only download it when we config https (check the conf file) -if [[ ! -e "${CONF}/hugegraph-server.keystore" ]]; then - download "${CONF}" "${GITHUB}/apache/hugegraph-doc/raw/binary-1.0/dist/server/hugegraph-server.keystore" -fi +MIN_JAVA_VERSION=11 # Add the slf4j-log4j12 binding CP=$(find -L $LIB -name 'log4j-slf4j-impl*.jar' | sort | tr '\n' ':') diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh index 5e14397f7e..836c11836f 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh @@ -80,6 +80,11 @@ REST_SERVER_URL=$(read_property "$CONF/rest-server.properties" "restserver.url") check_port "$GREMLIN_SERVER_URL" check_port "$REST_SERVER_URL" +# Note: Only download hugegraph-server.keystore when we config https (check the conf file) +if [[ $REST_SERVER_URL == https* && ! -e "${CONF}/hugegraph-server.keystore" ]]; then + download "${CONF}" "https://github.com/apache/hugegraph-doc/raw/binary-1.0/dist/server/hugegraph-server.keystore" +fi + if [ ! -d "$LOGS" ]; then mkdir -p "$LOGS" fi diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh index ad93b2ac82..2450992488 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh @@ -54,7 +54,7 @@ download_and_verify() { rm -f $filepath else echo "MD5 checksum verification succeeded for $filepath." - return + return 0 fi fi @@ -64,25 +64,32 @@ download_and_verify() { actual_md5=$(md5sum $filepath | awk '{ print $1 }') if [[ $actual_md5 != $expected_md5 ]]; then echo "MD5 checksum verification failed for $filepath after download. Expected: $expected_md5, but got: $actual_md5" - exit 1 + return 1 fi + + return 0 } if [[ $arch == "aarch64" || $arch == "arm64" ]]; then - libfile="$TOP/bin/libjemalloc_aarch64.so" + lib_file="$TOP/bin/libjemalloc_aarch64.so" download_url="https://github.com/apache/incubator-hugegraph/raw/master/hugegraph-store/hg-store-dist/src/assembly/static/bin/libjemalloc_aarch64.so" expected_md5="2a631d2f81837f9d5864586761c5e380" - download_and_verify $download_url $libfile $expected_md5 - export LD_PRELOAD=$libfile + if download_and_verify $download_url $lib_file $expected_md5; then + export LD_PRELOAD=$lib_file + else + echo "Failed to verify or download $lib_file, skip it" + fi elif [[ $arch == "x86_64" ]]; then - libfile="$TOP/bin/libjemalloc.so" + lib_file="$TOP/bin/libjemalloc.so" download_url="https://github.com/apache/incubator-hugegraph/raw/master/hugegraph-store/hg-store-dist/src/assembly/static/bin/libjemalloc.so" expected_md5="fd61765eec3bfea961b646c269f298df" - download_and_verify $download_url $libfile $expected_md5 - export LD_PRELOAD=$libfile + if download_and_verify $download_url $lib_file $expected_md5; then + export LD_PRELOAD=$lib_file + else + echo "Failed to verify or download $lib_file, skip it" + fi else echo "Unsupported architecture: $arch" - exit 1 fi ##pd/store max user processes, ulimit -u From bb174d85c81bc0f7656a1f27c7ba3d96731bcf0f Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Sat, 3 Aug 2024 00:39:50 +0800 Subject: [PATCH 07/10] fetch binary from doc repo --- .../assembly/static/bin/start-hugegraph.sh | 2 +- .../static/bin/start-hugegraph-store.sh | 41 ++----------------- .../src/assembly/static/bin/util.sh | 30 ++++++++++++++ 3 files changed, 35 insertions(+), 38 deletions(-) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh index 836c11836f..e36c33abc7 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh @@ -82,7 +82,7 @@ check_port "$REST_SERVER_URL" # Note: Only download hugegraph-server.keystore when we config https (check the conf file) if [[ $REST_SERVER_URL == https* && ! -e "${CONF}/hugegraph-server.keystore" ]]; then - download "${CONF}" "https://github.com/apache/hugegraph-doc/raw/binary-1.0/dist/server/hugegraph-server.keystore" + download "${CONF}" "https://github.com/apache/hugegraph-doc/raw/binary/dist/server/hugegraph-server.keystore" fi if [ ! -d "$LOGS" ]; then diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh index 45549664d8..6e5eee6656 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh @@ -37,42 +37,14 @@ OUTPUT=${LOGS}/hugegraph-store-server.log GITHUB="https://github.com" PID_FILE="$BIN/pid" +. "$BIN"/util.sh + arch=$(uname -m) echo "Current arch: $arch" -download_and_verify() { - local url=$1 - local filepath=$2 - local expected_md5=$3 - - if [[ -f $filepath ]]; then - echo "File $filepath exists. Verifying MD5 checksum..." - actual_md5=$(md5sum $filepath | awk '{ print $1 }') - if [[ $actual_md5 != $expected_md5 ]]; then - echo "MD5 checksum verification failed for $filepath. Expected: $expected_md5, but got: $actual_md5" - echo "Deleting $filepath..." - rm -f $filepath - else - echo "MD5 checksum verification succeeded for $filepath." - return 0 - fi - fi - - echo "Downloading $filepath..." - curl -L -o $filepath $url - - actual_md5=$(md5sum $filepath | awk '{ print $1 }') - if [[ $actual_md5 != $expected_md5 ]]; then - echo "MD5 checksum verification failed for $filepath after download. Expected: $expected_md5, but got: $actual_md5" - return 1 - fi - - return 0 -} - if [[ $arch == "aarch64" || $arch == "arm64" ]]; then lib_file="$TOP/bin/libjemalloc_aarch64.so" - download_url="https://github.com/apache/incubator-hugegraph/raw/master/hugegraph-store/hg-store-dist/src/assembly/static/bin/libjemalloc_aarch64.so" + download_url="https://github.com/apache/hugegraph-doc/raw/binary/dist/server/libjemalloc_aarch64.so" expected_md5="2a631d2f81837f9d5864586761c5e380" if download_and_verify $download_url $lib_file $expected_md5; then export LD_PRELOAD=$lib_file @@ -81,7 +53,7 @@ if [[ $arch == "aarch64" || $arch == "arm64" ]]; then fi elif [[ $arch == "x86_64" ]]; then lib_file="$TOP/bin/libjemalloc.so" - download_url="https://github.com/apache/incubator-hugegraph/raw/master/hugegraph-store/hg-store-dist/src/assembly/static/bin/libjemalloc.so" + download_url="https://github.com/apache/hugegraph-doc/raw/binary/dist/server/libjemalloc.so" expected_md5="fd61765eec3bfea961b646c269f298df" if download_and_verify $download_url $lib_file $expected_md5; then export LD_PRELOAD=$lib_file @@ -139,11 +111,6 @@ while getopts "g:j:y:" arg; do esac done - - - -. "$BIN"/util.sh - ensure_path_writable "$LOGS" ensure_path_writable "$PLUGINS" diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh index 93eae7b890..93b55311cb 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh @@ -282,6 +282,36 @@ function download() { fi } +download_and_verify() { + local url=$1 + local filepath=$2 + local expected_md5=$3 + + if [[ -f $filepath ]]; then + echo "File $filepath exists. Verifying MD5 checksum..." + actual_md5=$(md5sum $filepath | awk '{ print $1 }') + if [[ $actual_md5 != $expected_md5 ]]; then + echo "MD5 checksum verification failed for $filepath. Expected: $expected_md5, but got: $actual_md5" + echo "Deleting $filepath..." + rm -f $filepath + else + echo "MD5 checksum verification succeeded for $filepath." + return 0 + fi + fi + + echo "Downloading $filepath..." + curl -L -o $filepath $url + + actual_md5=$(md5sum $filepath | awk '{ print $1 }') + if [[ $actual_md5 != $expected_md5 ]]; then + echo "MD5 checksum verification failed for $filepath after download. Expected: $expected_md5, but got: $actual_md5" + return 1 + fi + + return 0 +} + function ensure_package_exist() { local path=$1 local dir=$2 From 10660f430bedf1606022d4c77e61caf9a695d54c Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Sat, 3 Aug 2024 11:05:38 +0800 Subject: [PATCH 08/10] update .gitignore --- .gitignore | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index c51d74da61..c8c2e1eb55 100644 --- a/.gitignore +++ b/.gitignore @@ -57,10 +57,7 @@ build/ *.pyc # maven ignore -apache-hugegraph-incubating-*/ -apache-hugegraph-pd-incubating-*/ -apache-hugegraph-store-incubating-*/ -apache-hugegraph-server-incubating-*/ +apache-hugegraph-*/ output/ *.war *.zip From 1744d3b54b783f26694aec5aa27cdb5dfc137475 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Mon, 5 Aug 2024 18:56:13 +0800 Subject: [PATCH 09/10] binary -> binary-1.5 --- .../hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh | 2 +- .../src/assembly/static/bin/start-hugegraph-store.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh index e36c33abc7..30c2764989 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh @@ -82,7 +82,7 @@ check_port "$REST_SERVER_URL" # Note: Only download hugegraph-server.keystore when we config https (check the conf file) if [[ $REST_SERVER_URL == https* && ! -e "${CONF}/hugegraph-server.keystore" ]]; then - download "${CONF}" "https://github.com/apache/hugegraph-doc/raw/binary/dist/server/hugegraph-server.keystore" + download "${CONF}" "https://github.com/apache/hugegraph-doc/raw/binary-1.5/dist/server/hugegraph-server.keystore" fi if [ ! -d "$LOGS" ]; then diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh index 6e5eee6656..27ae9b64b6 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh @@ -44,7 +44,7 @@ echo "Current arch: $arch" if [[ $arch == "aarch64" || $arch == "arm64" ]]; then lib_file="$TOP/bin/libjemalloc_aarch64.so" - download_url="https://github.com/apache/hugegraph-doc/raw/binary/dist/server/libjemalloc_aarch64.so" + download_url="https://github.com/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc_aarch64.so" expected_md5="2a631d2f81837f9d5864586761c5e380" if download_and_verify $download_url $lib_file $expected_md5; then export LD_PRELOAD=$lib_file @@ -53,7 +53,7 @@ if [[ $arch == "aarch64" || $arch == "arm64" ]]; then fi elif [[ $arch == "x86_64" ]]; then lib_file="$TOP/bin/libjemalloc.so" - download_url="https://github.com/apache/hugegraph-doc/raw/binary/dist/server/libjemalloc.so" + download_url="https://github.com/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc.so" expected_md5="fd61765eec3bfea961b646c269f298df" if download_and_verify $download_url $lib_file $expected_md5; then export LD_PRELOAD=$lib_file From fb4a92ca6d350223d14f32475934761aa4c547ce Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 6 Aug 2024 14:43:50 +0800 Subject: [PATCH 10/10] Update start-hugegraph-store.sh --- .../src/assembly/static/bin/start-hugegraph-store.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh index 27ae9b64b6..b363b7ab45 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh @@ -44,7 +44,7 @@ echo "Current arch: $arch" if [[ $arch == "aarch64" || $arch == "arm64" ]]; then lib_file="$TOP/bin/libjemalloc_aarch64.so" - download_url="https://github.com/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc_aarch64.so" + download_url="${GITHUB}/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc_aarch64.so" expected_md5="2a631d2f81837f9d5864586761c5e380" if download_and_verify $download_url $lib_file $expected_md5; then export LD_PRELOAD=$lib_file @@ -53,7 +53,7 @@ if [[ $arch == "aarch64" || $arch == "arm64" ]]; then fi elif [[ $arch == "x86_64" ]]; then lib_file="$TOP/bin/libjemalloc.so" - download_url="https://github.com/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc.so" + download_url="${GITHUB}/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc.so" expected_md5="fd61765eec3bfea961b646c269f298df" if download_and_verify $download_url $lib_file $expected_md5; then export LD_PRELOAD=$lib_file