Added notifications-core and notifications to 2.0 and add integTest m…

…anifest (#1957) * Added notifications-core and notifications to 2.0. Signed-off-by: dblock <[email protected]> * Remove scripts from opensearch-build. Signed-off-by: dblock <[email protected]> * Remove dashboards notifications scripts to use the default one Signed-off-by: Peter Zhu <[email protected]> * Remove CI checks for lack of gradlew. Signed-off-by: dblock <[email protected]> * Fix the second subfolder location after core Signed-off-by: Peter Zhu <[email protected]> * Add notifications related build scripts to build repo Signed-off-by: Peter Zhu <[email protected]> * Tweak build scripts to correctly move the zips Signed-off-by: Peter Zhu <[email protected]> Co-authored-by: Peter Zhu <[email protected]>
opensearch-project · Apr 13, 2022 · c9b3a51 · c9b3a51
1 parent 4d76087
commit c9b3a51
Show file tree

Hide file tree

Showing 4 changed files with 151 additions and 29 deletions.
diff --git a/manifests/2.0.0/opensearch-2.0.0-test.yml b/manifests/2.0.0/opensearch-2.0.0-test.yml
@@ -0,0 +1,116 @@
+---
+schema-version: '1.0'
+name: OpenSearch
+ci:
+  image:
+    name: opensearchstaging/ci-runner-centos7-opensearch-build-v1
+    args: -e JAVA_HOME=/opt/java/openjdk-17
+components:
+  - name: index-management
+    integ-test:
+      build-dependencies:
+        - job-scheduler
+      test-configs:
+        - with-security
+        - without-security
+      additional-cluster-configs:
+        path.repo: [/tmp]
+    bwc-test:
+      test-configs:
+        - with-security
+        - without-security
+
+  - name: anomaly-detection
+    integ-test:
+      build-dependencies:
+        - job-scheduler
+      test-configs:
+        - with-security
+        - without-security
+    bwc-test:
+      test-configs:
+        - with-security
+        - without-security
+
+  - name: asynchronous-search
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+    bwc-test:
+      test-configs:
+        - with-security
+        - without-security
+
+  - name: alerting
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+      additional-cluster-configs:
+        plugins.destination.host.deny_list: [10.0.0.0/8, 127.0.0.1]
+    bwc-test:
+      test-configs:
+        - with-security
+        - without-security
+
+  - name: notifications
+    working-directory: notifications
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+    bwc-test:
+      test-configs:
+        - with-security
+        - without-security
+
+  - name: sql
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+      additional-cluster-configs:
+        script.context.field.max_compilations_rate: 1000/1m
+    bwc-test:
+      test-configs:
+        - with-security
+        - without-security
+
+  - name: k-NN
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+    bwc-test:
+      test-configs:
+        - with-security
+        - without-security
+
+  - name: dashboards-reports
+    working-directory: reports-scheduler
+    integ-test:
+      test-configs:
+        - without-security
+    bwc-test:
+      test-configs:
+        - without-security
+
+  - name: observability
+    working-directory: opensearch-observability
+    integ-test:
+      test-configs:
+        - without-security
+    bwc-test:
+      test-configs:
+        - without-security
+
+  - name: ml-commons
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+    bwc-test:
+      test-configs:
+        - with-security
+        - without-security
diff --git a/manifests/2.0.0/opensearch-2.0.0.yml b/manifests/2.0.0/opensearch-2.0.0.yml
@@ -42,6 +42,14 @@ components:
     checks:
       - gradle:properties:version
       - gradle:dependencies:opensearch.version
+  - name: notifications-core
+    repository: https://github.com/opensearch-project/notifications.git
+    ref: main
+    working_directory: notifications/core
+  - name: notifications
+    repository: https://github.com/opensearch-project/notifications.git
+    ref: main
+    working_directory: notifications/notifications
   - name: alerting
     repository: https://github.com/opensearch-project/alerting.git
     ref: main

diff --git a/...mponents/notificationsDashboards/build.sh → ...ts/components/notifications-core/build.sh b/...mponents/notificationsDashboards/build.sh → ...ts/components/notifications-core/build.sh
@@ -13,7 +13,7 @@ function usage() {
     echo ""
     echo "Arguments:"
     echo -e "-v VERSION\t[Required] OpenSearch version."
-    echo -e "-q QUALIFIER\t[Optional] Version qualifier."
+    echo -e "-q QUALIFIER\t[Optional] Build qualifier."
     echo -e "-s SNAPSHOT\t[Optional] Build a snapshot, default is 'false'."
     echo -e "-p PLATFORM\t[Optional] Platform, ignored."
     echo -e "-a ARCHITECTURE\t[Optional] Build architecture, ignored."
@@ -58,25 +58,21 @@ while getopts ":h:v:q:s:o:p:a:" arg; do
 done
 
 if [ -z "$VERSION" ]; then
-    echo "Error: You must specify the OpenSearch Dashboards version"
+    echo "Error: You must specify the OpenSearch version"
     usage
     exit 1
 fi
 
+[[ ! -z "$QUALIFIER" ]] && VERSION=$VERSION-$QUALIFIER
+[[ "$SNAPSHOT" == "true" ]] && VERSION=$VERSION-SNAPSHOT
 [ -z "$OUTPUT" ] && OUTPUT=artifacts
-[ ! -z "$QUALIFIER" ] && QUALIFIER_IDENTIFIER="-$QUALIFIER"
 
-mkdir -p $OUTPUT/plugins
-# For hybrid plugin it actually resides in 'notificationsDashboards/dashboards-notifications'
-PLUGIN_FOLDER=$(basename "$PWD")
-PLUGIN_NAME=$(basename $(dirname "$PWD"))
-# TODO: [CLEANUP] Needed OpenSearch Dashboards git repo to build the required modules for plugins
-# This makes it so there is a dependency on having Dashboards pulled already.
-cp -r ../$PLUGIN_FOLDER/ ../../OpenSearch-Dashboards/plugins
-echo "BUILD MODULES FOR $PLUGIN_NAME"
-(cd ../../OpenSearch-Dashboards && source $NVM_DIR/nvm.sh && nvm use && yarn osd bootstrap)
-echo "BUILD RELEASE ZIP FOR $PLUGIN_NAME"
-(cd ../../OpenSearch-Dashboards && source $NVM_DIR/nvm.sh && nvm use && cd plugins/$PLUGIN_FOLDER && yarn plugin_helpers build --opensearch-dashboards-version=$VERSION$QUALIFIER_IDENTIFIER)
-echo "COPY $PLUGIN_NAME.zip"
-cp -r ../../OpenSearch-Dashboards/plugins/$PLUGIN_FOLDER/build/$PLUGIN_NAME-$VERSION$QUALIFIER_IDENTIFIER.zip $OUTPUT/plugins/
-rm -rf ../../OpenSearch-Dashboards/plugins/$PLUGIN_FOLDER
+
+# Go to the first notifications folder, which holds the core folder and the second notifications folder
+cd ../
+./gradlew publishToMavenLocal -PexcludeTests="**/SesChannelIT*" -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER
+./gradlew assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER
+
+mkdir -p core/$OUTPUT/plugins
+notifCoreZipPath=$(ls core/build/distributions/ | grep .zip)
+cp -v core/build/distributions/$notifCoreZipPath core/$OUTPUT/plugins
diff --git a/scripts/components/notifications/build.sh b/scripts/components/notifications/build.sh
@@ -13,14 +13,15 @@ function usage() {
     echo ""
     echo "Arguments:"
     echo -e "-v VERSION\t[Required] OpenSearch version."
+    echo -e "-q QUALIFIER\t[Optional] Build qualifier."
     echo -e "-s SNAPSHOT\t[Optional] Build a snapshot, default is 'false'."
     echo -e "-p PLATFORM\t[Optional] Platform, ignored."
     echo -e "-a ARCHITECTURE\t[Optional] Build architecture, ignored."
     echo -e "-o OUTPUT\t[Optional] Output path, default is 'artifacts'."
     echo -e "-h help"
 }
 
-while getopts ":h:v:s:o:p:a:" arg; do
+while getopts ":h:v:q:s:o:p:a:" arg; do
     case $arg in
         h)
             usage
@@ -29,6 +30,9 @@ while getopts ":h:v:s:o:p:a:" arg; do
         v)
             VERSION=$OPTARG
             ;;
+        q)
+            QUALIFIER=$OPTARG
+            ;;
         s)
             SNAPSHOT=$OPTARG
             ;;
@@ -59,19 +63,17 @@ if [ -z "$VERSION" ]; then
     exit 1
 fi
 
+[[ ! -z "$QUALIFIER" ]] && VERSION=$VERSION-$QUALIFIER
 [[ "$SNAPSHOT" == "true" ]] && VERSION=$VERSION-SNAPSHOT
 [ -z "$OUTPUT" ] && OUTPUT=artifacts
 
-mkdir -p $OUTPUT/maven
-mkdir -p $OUTPUT/plugins
+# Go to the first notifications folder, which holds the core folder and the second notifications folder
+cd ../
+./gradlew publishToMavenLocal -PexcludeTests="**/SesChannelIT*" -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER
+./gradlew assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER
+
+mkdir -p notifications/$OUTPUT/plugins
 
-cd notifications
-./gradlew publishToMavenLocal -PexcludeTests="**/SesChannelIT*" -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT
-./gradlew assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT
-cd ..
+notifCoreZipPath=$(ls notifications/build/distributions/ | grep .zip)
+cp -v notifications/build/distributions/$notifCoreZipPath notifications/$OUTPUT/plugins
 
-zipPath=$(find . -path \*notifications/build/distributions/*.zip)
-distributions="$(dirname "${zipPath}")"
-echo "COPY ${distributions}/*.zip"
-mkdir -p $OUTPUT/plugins
-cp ${distributions}/*.zip ./$OUTPUT/plugins