From c9b3a51edc3c5a6a3f53acf917a6642b17874de6 Mon Sep 17 00:00:00 2001 From: "Daniel Doubrovkine (dB.)" Date: Tue, 12 Apr 2022 21:58:17 -0400 Subject: [PATCH] Added notifications-core and notifications to 2.0 and add integTest manifest (#1957) * Added notifications-core and notifications to 2.0. Signed-off-by: dblock * Remove scripts from opensearch-build. Signed-off-by: dblock * Remove dashboards notifications scripts to use the default one Signed-off-by: Peter Zhu * Remove CI checks for lack of gradlew. Signed-off-by: dblock * Fix the second subfolder location after core Signed-off-by: Peter Zhu * Add notifications related build scripts to build repo Signed-off-by: Peter Zhu * Tweak build scripts to correctly move the zips Signed-off-by: Peter Zhu Co-authored-by: Peter Zhu --- manifests/2.0.0/opensearch-2.0.0-test.yml | 116 ++++++++++++++++++ manifests/2.0.0/opensearch-2.0.0.yml | 8 ++ .../build.sh | 30 ++--- scripts/components/notifications/build.sh | 26 ++-- 4 files changed, 151 insertions(+), 29 deletions(-) create mode 100644 manifests/2.0.0/opensearch-2.0.0-test.yml rename scripts/components/{notificationsDashboards => notifications-core}/build.sh (54%) mode change 100755 => 100644 mode change 100755 => 100644 scripts/components/notifications/build.sh diff --git a/manifests/2.0.0/opensearch-2.0.0-test.yml b/manifests/2.0.0/opensearch-2.0.0-test.yml new file mode 100644 index 0000000000..e0955e50f4 --- /dev/null +++ b/manifests/2.0.0/opensearch-2.0.0-test.yml @@ -0,0 +1,116 @@ +--- +schema-version: '1.0' +name: OpenSearch +ci: + image: + name: opensearchstaging/ci-runner-centos7-opensearch-build-v1 + args: -e JAVA_HOME=/opt/java/openjdk-17 +components: + - name: index-management + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + additional-cluster-configs: + path.repo: [/tmp] + bwc-test: + test-configs: + - with-security + - without-security + + - name: anomaly-detection + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + - without-security + + - name: asynchronous-search + integ-test: + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + - without-security + + - name: alerting + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + plugins.destination.host.deny_list: [10.0.0.0/8, 127.0.0.1] + bwc-test: + test-configs: + - with-security + - without-security + + - name: notifications + working-directory: notifications + integ-test: + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + - without-security + + - name: sql + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + script.context.field.max_compilations_rate: 1000/1m + bwc-test: + test-configs: + - with-security + - without-security + + - name: k-NN + integ-test: + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + - without-security + + - name: dashboards-reports + working-directory: reports-scheduler + integ-test: + test-configs: + - without-security + bwc-test: + test-configs: + - without-security + + - name: observability + working-directory: opensearch-observability + integ-test: + test-configs: + - without-security + bwc-test: + test-configs: + - without-security + + - name: ml-commons + integ-test: + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + - without-security diff --git a/manifests/2.0.0/opensearch-2.0.0.yml b/manifests/2.0.0/opensearch-2.0.0.yml index 32e0069a3e..bcc267fd53 100644 --- a/manifests/2.0.0/opensearch-2.0.0.yml +++ b/manifests/2.0.0/opensearch-2.0.0.yml @@ -42,6 +42,14 @@ components: checks: - gradle:properties:version - gradle:dependencies:opensearch.version + - name: notifications-core + repository: https://github.com/opensearch-project/notifications.git + ref: main + working_directory: notifications/core + - name: notifications + repository: https://github.com/opensearch-project/notifications.git + ref: main + working_directory: notifications/notifications - name: alerting repository: https://github.com/opensearch-project/alerting.git ref: main diff --git a/scripts/components/notificationsDashboards/build.sh b/scripts/components/notifications-core/build.sh old mode 100755 new mode 100644 similarity index 54% rename from scripts/components/notificationsDashboards/build.sh rename to scripts/components/notifications-core/build.sh index fe2046e603..841f9257a3 --- a/scripts/components/notificationsDashboards/build.sh +++ b/scripts/components/notifications-core/build.sh @@ -13,7 +13,7 @@ function usage() { echo "" echo "Arguments:" echo -e "-v VERSION\t[Required] OpenSearch version." - echo -e "-q QUALIFIER\t[Optional] Version qualifier." + echo -e "-q QUALIFIER\t[Optional] Build qualifier." echo -e "-s SNAPSHOT\t[Optional] Build a snapshot, default is 'false'." echo -e "-p PLATFORM\t[Optional] Platform, ignored." echo -e "-a ARCHITECTURE\t[Optional] Build architecture, ignored." @@ -58,25 +58,21 @@ while getopts ":h:v:q:s:o:p:a:" arg; do done if [ -z "$VERSION" ]; then - echo "Error: You must specify the OpenSearch Dashboards version" + echo "Error: You must specify the OpenSearch version" usage exit 1 fi +[[ ! -z "$QUALIFIER" ]] && VERSION=$VERSION-$QUALIFIER +[[ "$SNAPSHOT" == "true" ]] && VERSION=$VERSION-SNAPSHOT [ -z "$OUTPUT" ] && OUTPUT=artifacts -[ ! -z "$QUALIFIER" ] && QUALIFIER_IDENTIFIER="-$QUALIFIER" -mkdir -p $OUTPUT/plugins -# For hybrid plugin it actually resides in 'notificationsDashboards/dashboards-notifications' -PLUGIN_FOLDER=$(basename "$PWD") -PLUGIN_NAME=$(basename $(dirname "$PWD")) -# TODO: [CLEANUP] Needed OpenSearch Dashboards git repo to build the required modules for plugins -# This makes it so there is a dependency on having Dashboards pulled already. -cp -r ../$PLUGIN_FOLDER/ ../../OpenSearch-Dashboards/plugins -echo "BUILD MODULES FOR $PLUGIN_NAME" -(cd ../../OpenSearch-Dashboards && source $NVM_DIR/nvm.sh && nvm use && yarn osd bootstrap) -echo "BUILD RELEASE ZIP FOR $PLUGIN_NAME" -(cd ../../OpenSearch-Dashboards && source $NVM_DIR/nvm.sh && nvm use && cd plugins/$PLUGIN_FOLDER && yarn plugin_helpers build --opensearch-dashboards-version=$VERSION$QUALIFIER_IDENTIFIER) -echo "COPY $PLUGIN_NAME.zip" -cp -r ../../OpenSearch-Dashboards/plugins/$PLUGIN_FOLDER/build/$PLUGIN_NAME-$VERSION$QUALIFIER_IDENTIFIER.zip $OUTPUT/plugins/ -rm -rf ../../OpenSearch-Dashboards/plugins/$PLUGIN_FOLDER \ No newline at end of file + +# Go to the first notifications folder, which holds the core folder and the second notifications folder +cd ../ +./gradlew publishToMavenLocal -PexcludeTests="**/SesChannelIT*" -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER +./gradlew assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER + +mkdir -p core/$OUTPUT/plugins +notifCoreZipPath=$(ls core/build/distributions/ | grep .zip) +cp -v core/build/distributions/$notifCoreZipPath core/$OUTPUT/plugins diff --git a/scripts/components/notifications/build.sh b/scripts/components/notifications/build.sh old mode 100755 new mode 100644 index 1749a768aa..298bdf15e1 --- a/scripts/components/notifications/build.sh +++ b/scripts/components/notifications/build.sh @@ -13,6 +13,7 @@ function usage() { echo "" echo "Arguments:" echo -e "-v VERSION\t[Required] OpenSearch version." + echo -e "-q QUALIFIER\t[Optional] Build qualifier." echo -e "-s SNAPSHOT\t[Optional] Build a snapshot, default is 'false'." echo -e "-p PLATFORM\t[Optional] Platform, ignored." echo -e "-a ARCHITECTURE\t[Optional] Build architecture, ignored." @@ -20,7 +21,7 @@ function usage() { echo -e "-h help" } -while getopts ":h:v:s:o:p:a:" arg; do +while getopts ":h:v:q:s:o:p:a:" arg; do case $arg in h) usage @@ -29,6 +30,9 @@ while getopts ":h:v:s:o:p:a:" arg; do v) VERSION=$OPTARG ;; + q) + QUALIFIER=$OPTARG + ;; s) SNAPSHOT=$OPTARG ;; @@ -59,19 +63,17 @@ if [ -z "$VERSION" ]; then exit 1 fi +[[ ! -z "$QUALIFIER" ]] && VERSION=$VERSION-$QUALIFIER [[ "$SNAPSHOT" == "true" ]] && VERSION=$VERSION-SNAPSHOT [ -z "$OUTPUT" ] && OUTPUT=artifacts -mkdir -p $OUTPUT/maven -mkdir -p $OUTPUT/plugins +# Go to the first notifications folder, which holds the core folder and the second notifications folder +cd ../ +./gradlew publishToMavenLocal -PexcludeTests="**/SesChannelIT*" -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER +./gradlew assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER + +mkdir -p notifications/$OUTPUT/plugins -cd notifications -./gradlew publishToMavenLocal -PexcludeTests="**/SesChannelIT*" -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -./gradlew assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -cd .. +notifCoreZipPath=$(ls notifications/build/distributions/ | grep .zip) +cp -v notifications/build/distributions/$notifCoreZipPath notifications/$OUTPUT/plugins -zipPath=$(find . -path \*notifications/build/distributions/*.zip) -distributions="$(dirname "${zipPath}")" -echo "COPY ${distributions}/*.zip" -mkdir -p $OUTPUT/plugins -cp ${distributions}/*.zip ./$OUTPUT/plugins \ No newline at end of file