From 10d73e167436bf64b1e37892f66d4b3005b54c47 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Fri, 17 Jul 2020 09:29:23 -0700 Subject: [PATCH 1/7] build libraries in containers --- .github/workflows/CD.yml | 114 ++++++++++++++++++++++++++++----------- 1 file changed, 84 insertions(+), 30 deletions(-) diff --git a/.github/workflows/CD.yml b/.github/workflows/CD.yml index b0659af1..001c983f 100644 --- a/.github/workflows/CD.yml +++ b/.github/workflows/CD.yml @@ -5,15 +5,14 @@ on: - v* jobs: - Build-k-NN: + release-plugin-artifacts: strategy: matrix: java: [14] compiler: [g++-5] name: Build and Release k-NN Plugin - runs-on: ubuntu-16.04 - + runs-on: ubuntu-latest steps: - name: Checkout k-NN uses: actions/checkout@v2 @@ -30,33 +29,6 @@ jobs: with: java-version: ${{ matrix.java }} - - name: Build and ship library artifacts - env: - CXX: ${{ matrix.compiler }} - run: | - cd jni - git submodule update --init -- jni/external/nmslib - sed -i 's/-march=native/-march=x86-64/g' external/nmslib/similarity_search/CMakeLists.txt - cmake . - make package - - cd packages - folder_name=`ls ./*.rpm | sed 's|\(.*\)\..*|\1|'` - zip_name=$folder_name".zip" - mkdir $folder_name - cp ../release/*.so $folder_name - zip -r $zip_name $folder_name/* - cd .. - - zip_artifact=`ls packages/*.zip` - rpm_artifact=`ls packages/*.rpm` - deb_artifact=`ls packages/*.deb` - - aws s3 cp $zip_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/opendistro-libs/opendistro-knnlib/ - aws s3 cp $rpm_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/rpms/opendistro-knnlib/ - aws s3 cp $deb_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/debs/opendistro-knnlib/ - aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/*" - - name: Build and ship plugin artifacts run: | ./gradlew buildPackages --console=plain -Dbuild.snapshot=false @@ -68,3 +40,85 @@ jobs: aws s3 cp $rpm_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/rpms/opendistro-knn/ aws s3 cp $deb_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/debs/opendistro-knn/ aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/*" + + library-build-and-ship-zip-and-rpm: + name: Build k-NN JNI Library for Zip and RPM + runs-on: ubuntu-latest + container: + image: centos:7 + steps: + - name: Checkout k-NN + - uses: actions/checkout@v2 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Setup Java ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + - name: Build and ship library RPM and ZIP + env: + CXX: ${{ matrix.compiler }} + run: | + cd jni + git submodule update --init -- jni/external/nmslib + sed -i 's/-march=native/-march=x86-64/g' external/nmslib/similarity_search/CMakeLists.txt + cmake . + make package + + cd packages + folder_name=`ls ./*.rpm | sed 's|\(.*\)\..*|\1|'` + zip_name=$folder_name".zip" + mkdir $folder_name + cp ../release/*.so $folder_name + zip -r $zip_name $folder_name/* + cd .. + + zip_artifact=`ls packages/*.zip` + rpm_artifact=`ls packages/*.rpm` + + aws s3 cp $zip_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/opendistro-libs/opendistro-knnlib/ + aws s3 cp $rpm_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/rpms/opendistro-knnlib/ + aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/*" + + library-build-and-ship-deb: + name: Build k-NN JNI Library for Zip and RPM + runs-on: ubuntu-latest + container: + image: ubuntu:16.04 + steps: + - name: Checkout k-NN + - uses: actions/checkout@v2 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Setup Java ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + - name: Build and ship library DEB + env: + CXX: ${{ matrix.compiler }} + run: | + cd jni + git submodule update --init -- jni/external/nmslib + sed -i 's/-march=native/-march=x86-64/g' external/nmslib/similarity_search/CMakeLists.txt + cmake . + make package + + deb_artifact=`ls packages/*.deb` + + aws s3 cp $deb_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/debs/opendistro-knnlib/ + aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/*" From ed5a3704fbc5f980de01194d8658b679dce78e3f Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Fri, 17 Jul 2020 09:32:17 -0700 Subject: [PATCH 2/7] fix library artifact builds --- .github/workflows/CD.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CD.yml b/.github/workflows/CD.yml index 001c983f..30a15b9e 100644 --- a/.github/workflows/CD.yml +++ b/.github/workflows/CD.yml @@ -9,7 +9,6 @@ jobs: strategy: matrix: java: [14] - compiler: [g++-5] name: Build and Release k-NN Plugin runs-on: ubuntu-latest @@ -42,10 +41,15 @@ jobs: aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/*" library-build-and-ship-zip-and-rpm: + name: Build k-NN JNI Library for Zip and RPM runs-on: ubuntu-latest container: image: centos:7 + strategy: + matrix: + java: [14] + compiler: [g++] steps: - name: Checkout k-NN - uses: actions/checkout@v2 @@ -92,6 +96,10 @@ jobs: runs-on: ubuntu-latest container: image: ubuntu:16.04 + strategy: + matrix: + java: [14] + compiler: [g++-5] steps: - name: Checkout k-NN - uses: actions/checkout@v2 From dd8fde231ca9dead21ea3c446cdc5c56d3cea458 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Fri, 17 Jul 2020 12:33:47 -0700 Subject: [PATCH 3/7] fixed RPM/DEB/ZIP build and release --- .github/workflows/CD.yml | 121 +++++++++++++++++++++------------------ 1 file changed, 66 insertions(+), 55 deletions(-) diff --git a/.github/workflows/CD.yml b/.github/workflows/CD.yml index 30a15b9e..94787a8c 100644 --- a/.github/workflows/CD.yml +++ b/.github/workflows/CD.yml @@ -40,38 +40,43 @@ jobs: aws s3 cp $deb_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/debs/opendistro-knn/ aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/*" - library-build-and-ship-zip-and-rpm: - - name: Build k-NN JNI Library for Zip and RPM - runs-on: ubuntu-latest - container: - image: centos:7 - strategy: - matrix: - java: [14] - compiler: [g++] - steps: - - name: Checkout k-NN - - uses: actions/checkout@v2 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - - name: Setup Java ${{ matrix.java }} - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - - - name: Build and ship library RPM and ZIP + library-build-and-ship-zip-and-rpm: + name: Build k-NN JNI Library for Zip and RPM + runs-on: ubuntu-latest + container: + image: centos:7 + strategy: + matrix: + java: [14] + compiler: [g++] + steps: + - name: Checkout k-NN + uses: actions/checkout@v1 + with: + submodules: true + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Setup Java ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + - name: Install dependencies + run: | + yum update -y + yum install -y cmake rpm-build gcc-c++ make + + - name: Build and ship library RPM and ZIP env: CXX: ${{ matrix.compiler }} run: | cd jni - git submodule update --init -- jni/external/nmslib sed -i 's/-march=native/-march=x86-64/g' external/nmslib/similarity_search/CMakeLists.txt cmake . make package @@ -91,37 +96,43 @@ jobs: aws s3 cp $rpm_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/rpms/opendistro-knnlib/ aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/*" - library-build-and-ship-deb: - name: Build k-NN JNI Library for Zip and RPM - runs-on: ubuntu-latest - container: - image: ubuntu:16.04 - strategy: - matrix: - java: [14] - compiler: [g++-5] - steps: - - name: Checkout k-NN - - uses: actions/checkout@v2 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - - name: Setup Java ${{ matrix.java }} - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - - - name: Build and ship library DEB + library-build-and-ship-deb: + name: Build k-NN JNI Library for DEB + runs-on: ubuntu-latest + container: + image: ubuntu:16.04 + strategy: + matrix: + java: [14] + compiler: [g++-5] + steps: + - name: Checkout k-NN + uses: actions/checkout@v1 + with: + submodules: true + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Setup Java ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + - name: Install dependencies + run: | + apt-get update -y + apt-get install -y cmake rpm ${{ matrix.compiler }} + + - name: Build and ship library DEB env: CXX: ${{ matrix.compiler }} run: | cd jni - git submodule update --init -- jni/external/nmslib sed -i 's/-march=native/-march=x86-64/g' external/nmslib/similarity_search/CMakeLists.txt cmake . make package From 527e0097ddc0a3be327a1914f33a20818cf5c4aa Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Fri, 17 Jul 2020 12:38:54 -0700 Subject: [PATCH 4/7] Update README with information about library artifacts --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e78f59c1..8d07a258 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ make The library will be placed in the `jni/release` directory. ## JNI Library Artifacts -We build and distribute binary library artifacts with Opendistro for Elasticsearch. We build the library binary, RPM and DEB in [this GitHub action](https://github.com/opendistro-for-elasticsearch/k-NN/blob/master/.github/workflows/CD.yml). We use Ubuntu 16.04 with g++ 5.4.0 to build the library. Additionally, in order to provide as much general compatibility as possible, we compile the library without optimized instruction sets enabled. For users that want to get the most out of the library, they should follow [this section](##Build JNI Library RPM/DEB) and build the library from source in their production environment, so that if their environment has optimized instruction sets, they take advantage of them. +We build and distribute binary library artifacts with Opendistro for Elasticsearch. We build the library binary, RPM and DEB in [this GitHub action](https://github.com/opendistro-for-elasticsearch/k-NN/blob/master/.github/workflows/CD.yml). We use Ubuntu 16.04 with g++ 5.4.0 to build the DEB and Centos 7 with g++ 4.8.5 to build the RPM and ZIP. Additionally, in order to provide as much general compatibility as possible, we compile the library without optimized instruction sets enabled. For users that want to get the most out of the library, they should follow [this section](##Build JNI Library RPM/DEB) and build the library from source in their production environment, so that if their environment has optimized instruction sets, they take advantage of them. ## Build JNI Library RPM/DEB From d0688c5df0c661b5795eb02bf78e36ac50d394ca Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Fri, 17 Jul 2020 14:13:07 -0700 Subject: [PATCH 5/7] switch to centos for DEB --- .github/workflows/CD.yml | 56 +++++----------------------------------- README.md | 2 +- 2 files changed, 7 insertions(+), 51 deletions(-) diff --git a/.github/workflows/CD.yml b/.github/workflows/CD.yml index 94787a8c..4978a9fd 100644 --- a/.github/workflows/CD.yml +++ b/.github/workflows/CD.yml @@ -5,12 +5,12 @@ on: - v* jobs: - release-plugin-artifacts: + plugin-build-and-ship-artifacts: strategy: matrix: java: [14] - name: Build and Release k-NN Plugin + name: Build and release plugin artifacts runs-on: ubuntu-latest steps: - name: Checkout k-NN @@ -40,8 +40,8 @@ jobs: aws s3 cp $deb_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/debs/opendistro-knn/ aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/*" - library-build-and-ship-zip-and-rpm: - name: Build k-NN JNI Library for Zip and RPM + library-build-and-ship-artifacts: + name: Build and release JNI library artifacts runs-on: ubuntu-latest container: image: centos:7 @@ -72,7 +72,7 @@ jobs: yum update -y yum install -y cmake rpm-build gcc-c++ make - - name: Build and ship library RPM and ZIP + - name: Build and ship library artifacts env: CXX: ${{ matrix.compiler }} run: | @@ -91,53 +91,9 @@ jobs: zip_artifact=`ls packages/*.zip` rpm_artifact=`ls packages/*.rpm` + deb_artifact=`ls packages/*.deb` aws s3 cp $zip_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/opendistro-libs/opendistro-knnlib/ aws s3 cp $rpm_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/rpms/opendistro-knnlib/ - aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/*" - - library-build-and-ship-deb: - name: Build k-NN JNI Library for DEB - runs-on: ubuntu-latest - container: - image: ubuntu:16.04 - strategy: - matrix: - java: [14] - compiler: [g++-5] - steps: - - name: Checkout k-NN - uses: actions/checkout@v1 - with: - submodules: true - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - - name: Setup Java ${{ matrix.java }} - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - - - name: Install dependencies - run: | - apt-get update -y - apt-get install -y cmake rpm ${{ matrix.compiler }} - - - name: Build and ship library DEB - env: - CXX: ${{ matrix.compiler }} - run: | - cd jni - sed -i 's/-march=native/-march=x86-64/g' external/nmslib/similarity_search/CMakeLists.txt - cmake . - make package - - deb_artifact=`ls packages/*.deb` - aws s3 cp $deb_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/debs/opendistro-knnlib/ aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/*" diff --git a/README.md b/README.md index 8d07a258..a006329e 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ make The library will be placed in the `jni/release` directory. ## JNI Library Artifacts -We build and distribute binary library artifacts with Opendistro for Elasticsearch. We build the library binary, RPM and DEB in [this GitHub action](https://github.com/opendistro-for-elasticsearch/k-NN/blob/master/.github/workflows/CD.yml). We use Ubuntu 16.04 with g++ 5.4.0 to build the DEB and Centos 7 with g++ 4.8.5 to build the RPM and ZIP. Additionally, in order to provide as much general compatibility as possible, we compile the library without optimized instruction sets enabled. For users that want to get the most out of the library, they should follow [this section](##Build JNI Library RPM/DEB) and build the library from source in their production environment, so that if their environment has optimized instruction sets, they take advantage of them. +We build and distribute binary library artifacts with Opendistro for Elasticsearch. We build the library binary, RPM and DEB in [this GitHub action](https://github.com/opendistro-for-elasticsearch/k-NN/blob/master/.github/workflows/CD.yml). We use Centos 7 with g++ 4.8.5 to build the DEB, RPM and ZIP. Additionally, in order to provide as much general compatibility as possible, we compile the library without optimized instruction sets enabled. For users that want to get the most out of the library, they should follow [this section](##Build JNI Library RPM/DEB) and build the library from source in their production environment, so that if their environment has optimized instruction sets, they take advantage of them. ## Build JNI Library RPM/DEB From ab411494846dd359a511befc9fe5c39a337627dc Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Fri, 17 Jul 2020 15:46:56 -0700 Subject: [PATCH 6/7] updated centos to install dpkg --- .github/workflows/CD.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CD.yml b/.github/workflows/CD.yml index 4978a9fd..106ced81 100644 --- a/.github/workflows/CD.yml +++ b/.github/workflows/CD.yml @@ -70,7 +70,8 @@ jobs: - name: Install dependencies run: | yum update -y - yum install -y cmake rpm-build gcc-c++ make + yum install -y cmake rpm-build gcc-c++ make epel-release + yum install -y dpkg - name: Build and ship library artifacts env: From 2284e4f15a09452dc883bf894af2cc66c4b86276 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Fri, 17 Jul 2020 15:48:33 -0700 Subject: [PATCH 7/7] add refresh repolist --- .github/workflows/CD.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/CD.yml b/.github/workflows/CD.yml index 106ced81..ba9036c1 100644 --- a/.github/workflows/CD.yml +++ b/.github/workflows/CD.yml @@ -71,6 +71,7 @@ jobs: run: | yum update -y yum install -y cmake rpm-build gcc-c++ make epel-release + yum repolist yum install -y dpkg - name: Build and ship library artifacts