Test k-NN on Secure Cluster #136
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test k-NN on Secure Cluster | |
| on: | |
| schedule: | |
| - cron: '0 0 * * *' # every night | |
| push: | |
| branches: | |
| - "*" | |
| - "feature/**" | |
| pull_request: | |
| branches: | |
| - "*" | |
| - "feature/**" | |
| jobs: | |
| Build-ad: | |
| strategy: | |
| matrix: | |
| java: [ 11,17 ] | |
| os: [ubuntu-latest] | |
| fail-fast: true | |
| name: Test k-NN on Secure Cluster | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout k-NN | |
| uses: actions/checkout@v1 | |
| - name: Setup Java ${{ matrix.java }} | |
| uses: actions/setup-java@v1 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| - name: Install dependencies on ubuntu | |
| if: startsWith(matrix.os,'ubuntu') | |
| run: | | |
| sudo apt-get install libopenblas-dev gfortran -y | |
| - name: Assemble k-NN | |
| run: | | |
| ./gradlew assemble | |
| # example of variables: | |
| # plugin = opensearch-knn-2.7.0.0-SNAPSHOT.zip | |
| # version = 2.7.0 | |
| # plugin_version = 2.7.0.0 | |
| # qualifier = `SNAPSHOT` | |
| - name: Pull and Run Docker | |
| run: | | |
| plugin=`basename $(ls build/distributions/*.zip)` | |
| version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-3` | |
| plugin_version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-4` | |
| qualifier=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-1` | |
| if [ $qualifier != `SNAPSHOT` ]; | |
| then | |
| docker_version=$version-$qualifier | |
| else | |
| docker_version=$version | |
| fi | |
| echo plugin version plugin_version qualifier docker_version | |
| echo "($plugin) ($version) ($plugin_version) ($qualifier) ($docker_version)" | |
| cd .. | |
| if docker pull opensearchstaging/opensearch:$docker_version | |
| then | |
| echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile | |
| # knn plugin cannot be deleted until there are plugin that has dependency on it | |
| echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-neural-search ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-neural-search; fi" >> Dockerfile | |
| echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-performance-analyzer ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-performance-analyzer; fi" >> Dockerfile | |
| # saving pre-built artifacts of native libraries as we can't build it with gradle assemle | |
| echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-knn ]; then cp -r /usr/share/opensearch/plugins/opensearch-knn/lib /usr/share/opensearch/knn-libs; fi" >> Dockerfile | |
| echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-knn ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-knn; fi" >> Dockerfile | |
| echo "ADD k-NN/build/distributions/$plugin /tmp/" >> Dockerfile | |
| echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile | |
| # moving pre-built artifacts of native libraries back to plugin folder | |
| echo "RUN if [ -d /usr/share/opensearch/knn-libs ]; then mv /usr/share/opensearch/knn-libs /usr/share/opensearch/plugins/opensearch-knn/lib; fi" >> Dockerfile | |
| docker build -t opensearch-knn:test . | |
| echo "imagePresent=true" >> $GITHUB_ENV | |
| else | |
| echo "imagePresent=false" >> $GITHUB_ENV | |
| fi | |
| - name: Run Docker Image | |
| if: env.imagePresent == 'true' | |
| run: | | |
| cd .. | |
| docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" opensearch-knn:test | |
| sleep 90 | |
| - name: Run k-NN Integ Test | |
| if: env.imagePresent == 'true' | |
| run: | | |
| security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opensearch-security|wc -l` | |
| if [ $security -gt 0 ] | |
| then | |
| echo "Security plugin is available" | |
| ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin | |
| else | |
| echo "Security plugin is NOT available, skipping integration tests" | |
| fi |