diff --git a/.github/workflows/.require-approval.yml.swp b/.github/workflows/.require-approval.yml.swp new file mode 100644 index 0000000000..56454a0e0d Binary files /dev/null and b/.github/workflows/.require-approval.yml.swp differ diff --git a/.github/workflows/CI-workflow.yml b/.github/workflows/CI-workflow.yml index debe6958b6..0082977d09 100644 --- a/.github/workflows/CI-workflow.yml +++ b/.github/workflows/CI-workflow.yml @@ -1,11 +1,11 @@ name: Build and Test ml-commons # This workflow is triggered on pull requests and push to any branches on: - push: - branches-ignore: - - 'backport/**' - - 'create-pull-request/**' - - 'dependabot/**' +# push: +# branches-ignore: +# - 'backport/**' +# - 'create-pull-request/**' +# - 'dependabot/**' pull_request_target: types: [opened, synchronize, reopened] @@ -14,13 +14,16 @@ permissions: contents: read jobs: + Get-Require-Approval: + uses: ./.github/workflows/require-approval.yml + Get-CI-Image-Tag: uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main with: product: opensearch Build-ml-linux: - needs: Get-CI-Image-Tag + needs: [Get-Require-Approval, Get-CI-Image-Tag] strategy: matrix: java: [21] @@ -41,185 +44,186 @@ jobs: options: --user root steps: + - run: echo test ${{ needs.Get-Require-Approval.outputs.is-require-approval }} - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v1 with: java-version: ${{ matrix.java }} - - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ secrets.ML_ROLE }} - aws-region: us-west-2 - - - name: Checkout MLCommons - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: Build and Run Tests - id: step-build-test-linux - run: | - chown -R 1000:1000 `pwd` - su `id -un 1000` -c 'whoami && java -version && - export OPENAI_KEY=`aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text` && - export COHERE_KEY=`aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text` && - echo "::add-mask::$OPENAI_KEY" && - echo "::add-mask::$COHERE_KEY" && - echo "build and run tests" && ./gradlew build && - echo "Publish to Maven Local" && ./gradlew publishToMavenLocal && - echo "Multi Nodes Integration Testing" && ./gradlew integTest -PnumNodes=3' - plugin=`basename $(ls plugin/build/distributions/*.zip)` - echo $plugin - mv -v plugin/build/distributions/$plugin ./ - echo "build-test-linux=$plugin" >> $GITHUB_OUTPUT - - - name: Upload Coverage Report - uses: codecov/codecov-action@v1 - with: - flags: ml-commons - token: ${{ secrets.CODECOV_TOKEN }} - - - uses: actions/upload-artifact@v3 - with: - name: ml-plugin-linux-${{ matrix.java }} - path: ${{ steps.step-build-test-linux.outputs.build-test-linux }} - if-no-files-found: error - - - Test-ml-linux-docker: - needs: Build-ml-linux - strategy: - matrix: - java: [21] - - name: Test MLCommons Plugin on linux docker - if: github.repository == 'opensearch-project/ml-commons' - environment: ml-commons-cicd-env - runs-on: ubuntu-latest - - steps: - - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ secrets.ML_ROLE }} - aws-region: us-west-2 - - - name: Setup Java ${{ matrix.java }} - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - - - name: Checkout MLCommons - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - uses: actions/download-artifact@v3 - with: - name: ml-plugin-linux-${{ matrix.java }} - - - name: Pull and Run Docker - run: | - plugin=${{ needs.Build-ml-linux.outputs.build-test-linux }} - version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-3` - plugin_version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-4` - qualifier=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-1` - - if [ -n "$qualifier" ] && [ "$qualifier" != "SNAPSHOT" ]; then - qualifier=-${qualifier} - else - qualifier="" - fi - - docker_version=$version$qualifier - - echo plugin version plugin_version qualifier docker_version - echo "($plugin) ($version) ($plugin_version) ($qualifier) ($docker_version)" - - pwd && ls -l ./$plugin - - if docker pull opensearchstaging/opensearch:$docker_version - then - echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile - echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-skills ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-skills; fi" >> Dockerfile - echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-ml ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-ml; fi" >> Dockerfile - echo "COPY $plugin /tmp/" >> Dockerfile - echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile - docker build -t opensearch-ml:test . - echo "imagePresent=true" >> $GITHUB_ENV - else - echo "imagePresent=false" >> $GITHUB_ENV - fi - - name: Generate Password For Admin - id: genpass - run: | - PASSWORD=$(openssl rand -base64 20 | tr -dc 'A-Za-z0-9!@#$%^&*()_+=-') - echo "password={$PASSWORD}" >> $GITHUB_OUTPUT - - name: Run Docker Image - if: env.imagePresent == 'true' - run: | - cd .. - docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" -e OPENSEARCH_INITIAL_ADMIN_PASSWORD=${{ steps.genpass.outputs.password }} opensearch-ml:test - sleep 90 - - name: Run MLCommons Test - if: env.imagePresent == 'true' - run: | - security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:${{ steps.genpass.outputs.password }} --insecure |grep opensearch-security|wc -l` - export OPENAI_KEY=$(aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text) - export COHERE_KEY=$(aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text) - echo "::add-mask::$OPENAI_KEY" - echo "::add-mask::$COHERE_KEY" - if [ $security -gt 0 ] - then - echo "Security plugin is available" - ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=${{ steps.genpass.outputs.password }} - else - echo "Security plugin is NOT available" - ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" - fi - - - name: Upload Coverage Report - uses: codecov/codecov-action@v3 - with: - flags: ml-commons - token: ${{ secrets.CODECOV_TOKEN }} - - Build-ml-windows: - strategy: - matrix: - java: [21] - name: Build and Test MLCommons Plugin on Windows - if: github.repository == 'opensearch-project/ml-commons' - environment: ml-commons-cicd-env - runs-on: windows-latest - - steps: - - name: Setup Java ${{ matrix.java }} - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - - - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ secrets.ML_ROLE }} - aws-region: us-west-2 - - # ml-commons - - name: Checkout MLCommons - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: Build and Run Tests - shell: bash - run: | - export OPENAI_KEY=$(aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text) - export COHERE_KEY=$(aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text) - echo "::add-mask::$OPENAI_KEY" - echo "::add-mask::$COHERE_KEY" - ./gradlew.bat build - - name: Publish to Maven Local - run: | - ./gradlew publishToMavenLocal +## - uses: aws-actions/configure-aws-credentials@v2 +## with: +## role-to-assume: ${{ secrets.ML_ROLE }} +## aws-region: us-west-2 +## +## - name: Checkout MLCommons +## uses: actions/checkout@v3 +## with: +## ref: ${{ github.event.pull_request.head.sha }} +## +## - name: Build and Run Tests +## id: step-build-test-linux +## run: | +## chown -R 1000:1000 `pwd` +## su `id -un 1000` -c 'whoami && java -version && +## export OPENAI_KEY=`aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text` && +## export COHERE_KEY=`aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text` && +## echo "::add-mask::$OPENAI_KEY" && +## echo "::add-mask::$COHERE_KEY" && +## echo "build and run tests" && ./gradlew build && +## echo "Publish to Maven Local" && ./gradlew publishToMavenLocal && +## echo "Multi Nodes Integration Testing" && ./gradlew integTest -PnumNodes=3' +## plugin=`basename $(ls plugin/build/distributions/*.zip)` +## echo $plugin +## mv -v plugin/build/distributions/$plugin ./ +## echo "build-test-linux=$plugin" >> $GITHUB_OUTPUT +## +## - name: Upload Coverage Report +## uses: codecov/codecov-action@v1 +## with: +## flags: ml-commons +## token: ${{ secrets.CODECOV_TOKEN }} +## +## - uses: actions/upload-artifact@v3 +## with: +## name: ml-plugin-linux-${{ matrix.java }} +## path: ${{ steps.step-build-test-linux.outputs.build-test-linux }} +## if-no-files-found: error +## +## +## Test-ml-linux-docker: +## needs: Build-ml-linux +## strategy: +## matrix: +## java: [21] +## +## name: Test MLCommons Plugin on linux docker +## if: github.repository == 'opensearch-project/ml-commons' +## environment: ml-commons-cicd-env +## runs-on: ubuntu-latest +## +## steps: +## - uses: aws-actions/configure-aws-credentials@v2 +## with: +## role-to-assume: ${{ secrets.ML_ROLE }} +## aws-region: us-west-2 +## +## - name: Setup Java ${{ matrix.java }} +## uses: actions/setup-java@v1 +## with: +## java-version: ${{ matrix.java }} +## +## - name: Checkout MLCommons +## uses: actions/checkout@v3 +## with: +## ref: ${{ github.event.pull_request.head.sha }} +## +## - uses: actions/download-artifact@v3 +## with: +## name: ml-plugin-linux-${{ matrix.java }} +## +## - name: Pull and Run Docker +## run: | +## plugin=${{ needs.Build-ml-linux.outputs.build-test-linux }} +## version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-3` +## plugin_version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-4` +## qualifier=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-1` +## +## if [ -n "$qualifier" ] && [ "$qualifier" != "SNAPSHOT" ]; then +## qualifier=-${qualifier} +## else +## qualifier="" +## fi +## +## docker_version=$version$qualifier +## +## echo plugin version plugin_version qualifier docker_version +## echo "($plugin) ($version) ($plugin_version) ($qualifier) ($docker_version)" +## +## pwd && ls -l ./$plugin +## +## if docker pull opensearchstaging/opensearch:$docker_version +## then +## echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile +## echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-skills ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-skills; fi" >> Dockerfile +## echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-ml ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-ml; fi" >> Dockerfile +## echo "COPY $plugin /tmp/" >> Dockerfile +## echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile +## docker build -t opensearch-ml:test . +## echo "imagePresent=true" >> $GITHUB_ENV +## else +## echo "imagePresent=false" >> $GITHUB_ENV +## fi +## - name: Generate Password For Admin +## id: genpass +## run: | +## PASSWORD=$(openssl rand -base64 20 | tr -dc 'A-Za-z0-9!@#$%^&*()_+=-') +## echo "password={$PASSWORD}" >> $GITHUB_OUTPUT +## - name: Run Docker Image +## if: env.imagePresent == 'true' +## run: | +## cd .. +## docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" -e OPENSEARCH_INITIAL_ADMIN_PASSWORD=${{ steps.genpass.outputs.password }} opensearch-ml:test +## sleep 90 +## - name: Run MLCommons Test +## if: env.imagePresent == 'true' +## run: | +## security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:${{ steps.genpass.outputs.password }} --insecure |grep opensearch-security|wc -l` +## export OPENAI_KEY=$(aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text) +## export COHERE_KEY=$(aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text) +## echo "::add-mask::$OPENAI_KEY" +## echo "::add-mask::$COHERE_KEY" +## if [ $security -gt 0 ] +## then +## echo "Security plugin is available" +## ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=${{ steps.genpass.outputs.password }} +## else +## echo "Security plugin is NOT available" +## ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" +## fi +## +## - name: Upload Coverage Report +## uses: codecov/codecov-action@v3 +## with: +## flags: ml-commons +## token: ${{ secrets.CODECOV_TOKEN }} +## +## Build-ml-windows: +## strategy: +## matrix: +## java: [21] +## name: Build and Test MLCommons Plugin on Windows +## if: github.repository == 'opensearch-project/ml-commons' +## environment: ml-commons-cicd-env +## runs-on: windows-latest +## +## steps: +## - name: Setup Java ${{ matrix.java }} +## uses: actions/setup-java@v1 +## with: +## java-version: ${{ matrix.java }} +## +## - uses: aws-actions/configure-aws-credentials@v2 +## with: +## role-to-assume: ${{ secrets.ML_ROLE }} +## aws-region: us-west-2 +## +## # ml-commons +## - name: Checkout MLCommons +## uses: actions/checkout@v3 +## with: +## ref: ${{ github.event.pull_request.head.sha }} +## +## - name: Build and Run Tests +## shell: bash +## run: | +## export OPENAI_KEY=$(aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text) +## export COHERE_KEY=$(aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text) +## echo "::add-mask::$OPENAI_KEY" +## echo "::add-mask::$COHERE_KEY" +## ./gradlew.bat build +## - name: Publish to Maven Local +## run: | +## ./gradlew publishToMavenLocal # - name: Multi Nodes Integration Testing # shell: bash # run: | diff --git a/.github/workflows/add-untriaged.yml b/.github/workflows/add-untriaged.yml deleted file mode 100644 index 9dcc7020d2..0000000000 --- a/.github/workflows/add-untriaged.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Apply 'untriaged' label during issue lifecycle - -on: - issues: - types: [opened, reopened, transferred] - -jobs: - apply-label: - runs-on: ubuntu-latest - steps: - - uses: actions/github-script@v6 - with: - script: | - github.rest.issues.addLabels({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - labels: ['untriaged'] - }) diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml deleted file mode 100644 index 214283feeb..0000000000 --- a/.github/workflows/auto-release.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Releases - -on: - push: - tags: - - '*' - -jobs: - build: - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - name: GitHub App token - id: github_app_token - uses: tibdex/github-app-token@v1.5.0 - with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.APP_PRIVATE_KEY }} - installation_id: 22958780 - - name: Get tag - id: tag - uses: dawidd6/action-get-tag@v1 - - uses: actions/checkout@v2 - - uses: ncipollo/release-action@v1 - with: - github_token: ${{ steps.github_app_token.outputs.token }} - bodyFile: release-notes/opensearch-ml-common.release-notes-${{steps.tag.outputs.tag}}.md \ No newline at end of file diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml deleted file mode 100644 index cc046cc5a3..0000000000 --- a/.github/workflows/backport.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Backport -on: - pull_request_target: - types: - - closed - - labeled - -jobs: - backport: - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - name: Backport - steps: - - name: GitHub App token - id: github_app_token - uses: tibdex/github-app-token@v1.5.0 - with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.APP_PRIVATE_KEY }} - installation_id: 22958780 - - - name: Backport - uses: VachaShah/backport@v1.1.4 - with: - github_token: ${{ steps.github_app_token.outputs.token }} - branch_name: backport/backport-${{ github.event.number }} \ No newline at end of file diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml deleted file mode 100644 index 5881df9279..0000000000 --- a/.github/workflows/delete_backport_branch.yml +++ /dev/null @@ -1,15 +0,0 @@ -name: Delete merged branch of the backport PRs -on: - pull_request: - types: - - closed - -jobs: - delete-branch: - runs-on: ubuntu-latest - if: startsWith(github.event.pull_request.head.ref,'backport/') - steps: - - name: Delete merged branch - uses: SvanBoxel/delete-merged-branch@main - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/draft-release-notes-workflow.yml b/.github/workflows/draft-release-notes-workflow.yml deleted file mode 100644 index 5e67f9a5ab..0000000000 --- a/.github/workflows/draft-release-notes-workflow.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Release Drafter - -on: - push: - branches: - - main - -jobs: - update_release_draft: - name: Update draft release notes - runs-on: ubuntu-latest - steps: - - name: Update draft release notes - uses: release-drafter/release-drafter@v5 - with: - config-name: draft-release-notes-config.yml - name: Version (set here) - tag: (None) - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml deleted file mode 100644 index 2f51301a1b..0000000000 --- a/.github/workflows/labeler.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: "Pull Request Labeler" -on: - pull_request_target: - branches: - - main - types: - - opened - -jobs: - label: - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - steps: - - name: GitHub App token - id: github_app_token - uses: tibdex/github-app-token@v1.5.0 - with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.APP_PRIVATE_KEY }} - installation_id: 22958780 - - name: Label - uses: actions/labeler@v4 - with: - repo-token: ${{ steps.github_app_token.outputs.token }} diff --git a/.github/workflows/maven-publish.yml b/.github/workflows/maven-publish.yml deleted file mode 100644 index 283b3b77df..0000000000 --- a/.github/workflows/maven-publish.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Publish snapshots to maven - -on: - workflow_dispatch: - push: - branches: - - main - - 1.* - - 2.* - -jobs: - build-and-publish-snapshots: - strategy: - fail-fast: false - if: github.repository == 'opensearch-project/ml-commons' - runs-on: ubuntu-latest - - permissions: - id-token: write - contents: write - - steps: - - uses: actions/setup-java@v3 - with: - distribution: temurin # Temurin is a distribution of adoptium - java-version: 21 - - uses: actions/checkout@v3 - - uses: aws-actions/configure-aws-credentials@v1.7.0 - with: - role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }} - aws-region: us-east-1 - - name: publish snapshots to maven - run: | - export SONATYPE_USERNAME=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-username --query SecretString --output text) - export SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text) - echo "::add-mask::$SONATYPE_USERNAME" - echo "::add-mask::$SONATYPE_PASSWORD" - ./gradlew publishPluginZipPublicationToSnapshotsRepository publishShadowPublicationToSnapshotsRepository diff --git a/.github/workflows/require-approval.yml b/.github/workflows/require-approval.yml new file mode 100644 index 0000000000..2a890ef953 --- /dev/null +++ b/.github/workflows/require-approval.yml @@ -0,0 +1,30 @@ +--- +name: Check if the workflow require approval +on: + workflow_call: + outputs: + is-require-approval: + description: The ci image version for linux build + value: ${{ jobs.Require-Approval.outputs.output-is-require-approval }} + +jobs: + Require-Approval: + runs-on: ubuntu-latest + outputs: + output-is-require-approval: ${{ steps.step-is-require-approval.outputs.is-require-approval }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.sha }} + - name: Get CodeOwner List + id: step-is-require-approval + run: | + approvers=$(cat .github/CODEOWNERS | grep @ | tr -d '* ' | sed 's/@/,/g' | sed 's/,//1') + author=${{ github.event.pull_request.user.login }} + if [[ "$approvers" =~ "$author" ]]; then + echo "$authoer is in the approval list" + echo "is-require-approval=ml-commons-cicd-env" >> $GITHUB_OUTPUT + else + echo "$author is not in the approval list" + echo "is-require-approval=ml-commons-cicd-env-require-approval" >> $GITHUB_OUTPUT + fi diff --git a/README.md b/README.md index 832febe399..0af7e8855b 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ +test123 [![Test Workflow](https://github.com/opensearch-project/ml-commons/workflows/Build%20and%20Test%20ml-commons/badge.svg)](https://github.com/opensearch-project/ml-commons/actions) [![codecov](https://codecov.io/gh/opensearch-project/ml-commons/branch/main/graph/badge.svg)](https://codecov.io/gh/opensearch-project/ml-commons) [![Documentation](https://img.shields.io/badge/api-reference-blue.svg)](https://opensearch.org/docs/latest/ml-commons-plugin/api/)