Build and Test k-NN #1057
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Test k-NN | |
| on: | |
| schedule: | |
| - cron: '0 0 * * *' # every night | |
| push: | |
| branches: | |
| - "*" | |
| - "feature/**" | |
| pull_request: | |
| branches: | |
| - "*" | |
| - "feature/**" | |
| jobs: | |
| Build-k-NN: | |
| strategy: | |
| matrix: | |
| java: [11, 17] | |
| os: [ubuntu-latest, macos-latest] | |
| name: Build and Test k-NN Plugin | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout k-NN | |
| uses: actions/checkout@v1 | |
| - name: Setup Java ${{ matrix.java }} | |
| uses: actions/setup-java@v1 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| - name: Install dependencies on ubuntu | |
| if: startsWith(matrix.os,'ubuntu') | |
| run: | | |
| sudo apt-get install libopenblas-dev gfortran -y | |
| - name: Install dependencies on macos | |
| if: startsWith(matrix.os, 'macos') | |
| run: | | |
| brew reinstall gcc | |
| export FC=/usr/local/Cellar/gcc/12.2.0/bin/gfortran | |
| - name: Run build | |
| run: | | |
| ./gradlew build | |
| - name: Upload Coverage Report | |
| if: startsWith(matrix.os,'ubuntu') | |
| uses: codecov/codecov-action@v1 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| Build-k-NN-Windows: | |
| strategy: | |
| matrix: | |
| java: [ 11, 17 ] | |
| name: Build and Test k-NN Plugin on Windows | |
| runs-on: windows-latest | |
| steps: | |
| - name: Checkout k-NN | |
| uses: actions/checkout@v1 | |
| - name: Setup Java ${{ matrix.java }} | |
| uses: actions/setup-java@v1 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| - name: Install MinGW Using Scoop | |
| run: | | |
| iex "& {$(irm get.scoop.sh)} -RunAsAdmin" | |
| scoop bucket add main | |
| scoop install mingw | |
| - name: Add MinGW to PATH | |
| run: | | |
| echo "C:/Users/runneradmin/scoop/apps/mingw/current/bin" >> $env:GITHUB_PATH | |
| refreshenv | |
| - name: Download OpenBLAS | |
| run: | | |
| curl -L -O https://github.com/xianyi/OpenBLAS/releases/download/v0.3.21/OpenBLAS-0.3.21-x64.zip | |
| mkdir OpenBLAS | |
| Expand-Archive -Path .\OpenBLAS-0.3.21-x64.zip -DestinationPath .\OpenBLAS\ | |
| mkdir ./src/main/resources/windowsDependencies | |
| cp ./OpenBLAS/bin/libopenblas.dll ./src/main/resources/windowsDependencies/ | |
| rm .\OpenBLAS-0.3.21-x64.zip | |
| rm -r .\OpenBLAS\ | |
| - name: Run build | |
| run: | | |
| ./gradlew.bat build | |
| # - name: Pull and Run Docker for security tests | |
| # run: | | |
| # plugin=`ls build/distributions/*.zip` | |
| # version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-3` | |
| # plugin_version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-4` | |
| # echo $version | |
| # cd .. | |
| # if docker pull opendistroforelasticsearch/opendistroforelasticsearch:$version | |
| # then | |
| # echo "FROM opendistroforelasticsearch/opendistroforelasticsearch:$version" >> Dockerfile | |
| # echo "RUN if [ -d /usr/share/elasticsearch/plugins/opendistro-knn ]; then /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro-knn; fi" >> Dockerfile | |
| # echo "RUN yum -y update \ && yum -y groupinstall "Development Tools" \ && yum install -y unzip glibc.x86_64 cmake \ && yum clean all" >> Dockerfile | |
| # echo "RUN git clone --recursive --branch ${GITHUB_REF##*/} https://github.com/opendistro-for-elasticsearch/k-NN.git /usr/share/elasticsearch/k-NN \ " >> Dockerfile | |
| # echo "&& cd /usr/share/elasticsearch/k-NN/jni \ && sed -i 's/-march=native/-march=x86-64/g' external/nmslib/similarity_search/CMakeLists.txt \ && cmake . \ && make \ " >> Dockerfile | |
| # echo "&& mkdir /tmp/jni/ && cp release/*.so /tmp/jni/ && ls -ltr /tmp/jni/ \ && cp /tmp/jni/libKNNIndex*.so /usr/lib \ && rm -rf /usr/share/elasticsearch/k-NN" >> Dockerfile | |
| # echo "RUN cd /usr/share/elasticsearch/" >> Dockerfile | |
| # echo "ADD k-NN/build/distributions/opendistro-knn-$plugin_version.zip /tmp/" >> Dockerfile | |
| # echo "RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch file:/tmp/opendistro-knn-$plugin_version.zip" >> Dockerfile | |
| # docker build -t odfe-knn:test . | |
| # echo "imagePresent=true" >> $GITHUB_ENV | |
| # else | |
| # echo "imagePresent=false" >> $GITHUB_ENV | |
| # fi | |
| # - name: Run Docker Image | |
| # if: env.imagePresent == 'true' | |
| # run: | | |
| # cd .. | |
| # docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" odfe-knn:test | |
| # sleep 90 | |
| # - name: Run k-NN Test | |
| # if: env.imagePresent == 'true' | |
| # run: | | |
| # security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opendistro_security|wc -l` | |
| # if [ $security -gt 0 ] | |
| # then | |
| # echo "Security plugin is available. Running tests in security mode" | |
| # ./gradlew :integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin | |
| # else | |
| # echo "Security plugin is NOT available. Skipping tests as they are already ran part of ./gradlew build" | |
| # fi |