From 5dbee258b81960fcd1deaf75f11d0a0018299006 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Thu, 8 Feb 2024 17:21:24 +0100 Subject: [PATCH 01/39] sdl tests --- .github/workflows/sdl.yml | 50 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/sdl.yml diff --git a/.github/workflows/sdl.yml b/.github/workflows/sdl.yml new file mode 100644 index 000000000..bf32258d7 --- /dev/null +++ b/.github/workflows/sdl.yml @@ -0,0 +1,50 @@ +name: SDL checks +on: + workflow_dispatch: + pull_request: + push: + branches: + - master + - 'releases/**' + +concurrency: + # github.ref is not unique in post-commit + group: ${{ github.event_name == 'push' && github.run_id || github.ref }}-linux-sdl + cancel-in-progress: true + +env: + PYTHON_VERSION: '3.11' + +jobs: + sdl_tests: + name: OpenVINO tokenizers sdl tests + timeout-minutes: 5 + defaults: + run: + shell: bash + runs-on: ubuntu-latest + + env: + OPENVINO_TOKENIZERS_REPO: ${{ github.workspace }}/openvino_tokenizers + + steps: + - name: Clone Openvino tokenizers sources and tests + uses: actions/checkout@v4 + with: + path: ${{ env.OPENVINO_TOKENIZERS_REPO }} + + - name: Setup Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + + - name: Install Python tests dependencies + run: | + python3 -m pip install bandit + + - name: Tokenizers Bandit tests + run: | + bandit -c pyproject.toml -r python + working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }} + From e810ae0914f17fcd42dc1fbd80b0d39e0e62dc36 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Thu, 8 Feb 2024 17:23:58 +0100 Subject: [PATCH 02/39] removed bandit tests from linux workflow --- .github/workflows/linux.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index f63f06777..4ed651f83 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -289,11 +289,6 @@ jobs: python3 -m pip install $wheel_name[dev] popd - - name: Tokenizers Bandit tests - run: | - bandit -c pyproject.toml -r python - working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }} - - name: Tokenizers regression tests run: | python3 -m pytest tokenizers_test.py From 99c703186a72a0ab17162e893f8a3e5a5bea2aa7 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Thu, 8 Feb 2024 17:38:08 +0100 Subject: [PATCH 03/39] deps --- .github/workflows/mac.yml | 5 +++++ .github/workflows/sdl.yml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 23cd3dff5..a1e3e4080 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -1,5 +1,9 @@ name: macOS (13, Python 3.11) on: + workflow_run: + workflows: ["SDL Tests"] + types: + - completed workflow_dispatch: pull_request: push: @@ -17,6 +21,7 @@ env: jobs: openvino_build: + if: ${{ github.event.workflow_run.conclusion == 'success' }} timeout-minutes: 150 defaults: run: diff --git a/.github/workflows/sdl.yml b/.github/workflows/sdl.yml index bf32258d7..9472a0a4a 100644 --- a/.github/workflows/sdl.yml +++ b/.github/workflows/sdl.yml @@ -1,4 +1,4 @@ -name: SDL checks +name: SDL Tests on: workflow_dispatch: pull_request: From a1a1a5dfdb5d890a7272f0006493a511e7f918bc Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Thu, 8 Feb 2024 17:40:59 +0100 Subject: [PATCH 04/39] fixed mac triggers --- .github/workflows/mac.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index a1e3e4080..c695e18b7 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -5,11 +5,6 @@ on: types: - completed workflow_dispatch: - pull_request: - push: - branches: - - master - - 'releases/**' concurrency: # github.ref is not unique in post-commit From ebe944a39fe3d6177a1681e11796c3d807a11f21 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Thu, 8 Feb 2024 17:47:45 +0100 Subject: [PATCH 05/39] fixed trigger --- .github/workflows/mac.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index c695e18b7..f518034d6 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -1,7 +1,7 @@ name: macOS (13, Python 3.11) on: workflow_run: - workflows: ["SDL Tests"] + workflows: [SDL Tests] types: - completed workflow_dispatch: From cf5fae7dcaed8221340e2200a6fa41643a2b2c40 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Thu, 8 Feb 2024 17:52:02 +0100 Subject: [PATCH 06/39] reverted triggers --- .github/workflows/mac.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index f518034d6..c0c40afe7 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -5,6 +5,11 @@ on: types: - completed workflow_dispatch: + pull_request: + push: + branches: + - master + - 'releases/**' concurrency: # github.ref is not unique in post-commit From 5024fb66b6a2866a8159dc7ca72ce4876ca26089 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Thu, 8 Feb 2024 17:55:46 +0100 Subject: [PATCH 07/39] workflow_run only --- .github/workflows/mac.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index c0c40afe7..5ac056f41 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -4,12 +4,6 @@ on: workflows: [SDL Tests] types: - completed - workflow_dispatch: - pull_request: - push: - branches: - - master - - 'releases/**' concurrency: # github.ref is not unique in post-commit From 15a690b4b271f7bb3c7561588ba956ba2ba2bf0c Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Thu, 8 Feb 2024 18:00:24 +0100 Subject: [PATCH 08/39] workflow_run branch --- .github/workflows/mac.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 5ac056f41..cba950103 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -2,6 +2,7 @@ name: macOS (13, Python 3.11) on: workflow_run: workflows: [SDL Tests] + branches: [master] types: - completed From ea674ca6a312069d25f53fcc879839fa035def18 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Thu, 8 Feb 2024 18:09:40 +0100 Subject: [PATCH 09/39] all types --- .github/workflows/mac.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index cba950103..998f18575 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -2,9 +2,6 @@ name: macOS (13, Python 3.11) on: workflow_run: workflows: [SDL Tests] - branches: [master] - types: - - completed concurrency: # github.ref is not unique in post-commit From 18f2087d97f32e5231343151e02f32b25474f06e Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Thu, 8 Feb 2024 18:59:19 +0100 Subject: [PATCH 10/39] trivy scan --- .github/workflows/sdl.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/sdl.yml b/.github/workflows/sdl.yml index 9472a0a4a..94603b872 100644 --- a/.github/workflows/sdl.yml +++ b/.github/workflows/sdl.yml @@ -48,3 +48,23 @@ jobs: bandit -c pyproject.toml -r python working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }} + trivy_scan: + name: code_scanning + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Run Trivy vulnerability scanner in repo mode + uses: aquasecurity/trivy-action@master + with: + scan-type: 'fs' + ignore-unfixed: true + format: 'sarif' + output: 'trivy-results.sarif' + severity: 'CRITICAL' + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: 'trivy-results.sarif' \ No newline at end of file From b64971ffec1875b7016b9399643de5c3047e98a4 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Thu, 8 Feb 2024 18:59:38 +0100 Subject: [PATCH 11/39] reverted mac --- .github/workflows/mac.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 998f18575..23cd3dff5 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -1,7 +1,11 @@ name: macOS (13, Python 3.11) on: - workflow_run: - workflows: [SDL Tests] + workflow_dispatch: + pull_request: + push: + branches: + - master + - 'releases/**' concurrency: # github.ref is not unique in post-commit @@ -13,7 +17,6 @@ env: jobs: openvino_build: - if: ${{ github.event.workflow_run.conclusion == 'success' }} timeout-minutes: 150 defaults: run: From f221e5e9dd54e4cfec8241debe540072fbff1277 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Thu, 8 Feb 2024 19:01:55 +0100 Subject: [PATCH 12/39] updated action --- .github/workflows/sdl.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sdl.yml b/.github/workflows/sdl.yml index 94603b872..daaf5a38a 100644 --- a/.github/workflows/sdl.yml +++ b/.github/workflows/sdl.yml @@ -65,6 +65,6 @@ jobs: severity: 'CRITICAL' - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/upload-sarif@v3 with: sarif_file: 'trivy-results.sarif' \ No newline at end of file From cc2f1f1bf7480a4222bb8bea44aed15f4d8cf61e Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Thu, 8 Feb 2024 19:05:56 +0100 Subject: [PATCH 13/39] set permissions --- .github/workflows/sdl.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/sdl.yml b/.github/workflows/sdl.yml index daaf5a38a..2a7474d2c 100644 --- a/.github/workflows/sdl.yml +++ b/.github/workflows/sdl.yml @@ -7,6 +7,11 @@ on: - master - 'releases/**' +permissions: + actions: read + contents: read + security-events: write + concurrency: # github.ref is not unique in post-commit group: ${{ github.event_name == 'push' && github.run_id || github.ref }}-linux-sdl From 3f0538dfac7354db8a44e6f84c1c951b6ea19828 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Thu, 8 Feb 2024 19:17:31 +0100 Subject: [PATCH 14/39] scan artifacts --- .github/workflows/sdl.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sdl.yml b/.github/workflows/sdl.yml index 2a7474d2c..ffbd743ca 100644 --- a/.github/workflows/sdl.yml +++ b/.github/workflows/sdl.yml @@ -69,7 +69,15 @@ jobs: output: 'trivy-results.sarif' severity: 'CRITICAL' - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 + # - name: Upload Trivy scan results to GitHub Security tab + # uses: github/codeql-action/upload-sarif@v3 + # with: + # sarif_file: 'trivy-results.sarif' + + - name: Upload Trivy scan results + if: ${{ always() }} + uses: actions/upload-artifact@v4 with: - sarif_file: 'trivy-results.sarif' \ No newline at end of file + name: trivy_results + path: 'trivy-results.sarif' + if-no-files-found: 'error' From 81281fc8c524fdd8f0bcebd2586abea8a8d3abb3 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Fri, 9 Feb 2024 12:04:00 +0100 Subject: [PATCH 15/39] revert trivy_scan --- .github/workflows/sdl.yml | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/.github/workflows/sdl.yml b/.github/workflows/sdl.yml index ffbd743ca..74ac15e66 100644 --- a/.github/workflows/sdl.yml +++ b/.github/workflows/sdl.yml @@ -53,31 +53,3 @@ jobs: bandit -c pyproject.toml -r python working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }} - trivy_scan: - name: code_scanning - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Run Trivy vulnerability scanner in repo mode - uses: aquasecurity/trivy-action@master - with: - scan-type: 'fs' - ignore-unfixed: true - format: 'sarif' - output: 'trivy-results.sarif' - severity: 'CRITICAL' - - # - name: Upload Trivy scan results to GitHub Security tab - # uses: github/codeql-action/upload-sarif@v3 - # with: - # sarif_file: 'trivy-results.sarif' - - - name: Upload Trivy scan results - if: ${{ always() }} - uses: actions/upload-artifact@v4 - with: - name: trivy_results - path: 'trivy-results.sarif' - if-no-files-found: 'error' From 80fe4561d3dc1754813d39342e6aec8e54cd36ba Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Fri, 9 Feb 2024 12:05:31 +0100 Subject: [PATCH 16/39] try deps --- .github/workflows/mac.yml | 10 ++++------ .github/workflows/sdl.yml | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 23cd3dff5..139987bb7 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -1,11 +1,8 @@ name: macOS (13, Python 3.11) on: - workflow_dispatch: - pull_request: - push: - branches: - - master - - 'releases/**' + workflow_run: + workflows: [SDL] + types: [completed] concurrency: # github.ref is not unique in post-commit @@ -17,6 +14,7 @@ env: jobs: openvino_build: + if: ${{ github.event.workflow_run.conclusion == 'success' }} timeout-minutes: 150 defaults: run: diff --git a/.github/workflows/sdl.yml b/.github/workflows/sdl.yml index 74ac15e66..57df3e27f 100644 --- a/.github/workflows/sdl.yml +++ b/.github/workflows/sdl.yml @@ -1,4 +1,4 @@ -name: SDL Tests +name: SDL on: workflow_dispatch: pull_request: From aecae5d206896ab53505c95e2870a129dec53b4e Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Fri, 9 Feb 2024 12:25:15 +0100 Subject: [PATCH 17/39] dep on linux --- .github/workflows/mac.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 139987bb7..ef5e0ad06 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -1,7 +1,7 @@ name: macOS (13, Python 3.11) on: workflow_run: - workflows: [SDL] + workflows: [Linux (Ubuntu 20.04, Python 3.11)] types: [completed] concurrency: From 26ecd3346bed47917467111ea0be36ff6e91b46b Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Fri, 9 Feb 2024 16:07:01 +0100 Subject: [PATCH 18/39] remove job condition --- .github/workflows/mac.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index ef5e0ad06..19acb116a 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -14,7 +14,6 @@ env: jobs: openvino_build: - if: ${{ github.event.workflow_run.conclusion == 'success' }} timeout-minutes: 150 defaults: run: From b64d52aa41489cde5ddb5584699aed7651d09469 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 13 Feb 2024 11:54:28 +0100 Subject: [PATCH 19/39] Revert "revert trivy_scan" This reverts commit 81281fc8c524fdd8f0bcebd2586abea8a8d3abb3. --- .github/workflows/sdl.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/sdl.yml b/.github/workflows/sdl.yml index 57df3e27f..71a7c5193 100644 --- a/.github/workflows/sdl.yml +++ b/.github/workflows/sdl.yml @@ -53,3 +53,31 @@ jobs: bandit -c pyproject.toml -r python working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }} + trivy_scan: + name: code_scanning + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Run Trivy vulnerability scanner in repo mode + uses: aquasecurity/trivy-action@master + with: + scan-type: 'fs' + ignore-unfixed: true + format: 'sarif' + output: 'trivy-results.sarif' + severity: 'CRITICAL' + + # - name: Upload Trivy scan results to GitHub Security tab + # uses: github/codeql-action/upload-sarif@v3 + # with: + # sarif_file: 'trivy-results.sarif' + + - name: Upload Trivy scan results + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: trivy_results + path: 'trivy-results.sarif' + if-no-files-found: 'error' From b56f5f874728d6179f51daca376a42f7cb96ac1b Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 13 Feb 2024 12:06:04 +0100 Subject: [PATCH 20/39] added trivy scan --- .github/workflows/mac.yml | 9 ++++++--- .github/workflows/sdl.yml | 30 ++++++++++++++++++------------ 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 19acb116a..23cd3dff5 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -1,8 +1,11 @@ name: macOS (13, Python 3.11) on: - workflow_run: - workflows: [Linux (Ubuntu 20.04, Python 3.11)] - types: [completed] + workflow_dispatch: + pull_request: + push: + branches: + - master + - 'releases/**' concurrency: # github.ref is not unique in post-commit diff --git a/.github/workflows/sdl.yml b/.github/workflows/sdl.yml index 71a7c5193..dccd2ccf8 100644 --- a/.github/workflows/sdl.yml +++ b/.github/workflows/sdl.yml @@ -60,24 +60,30 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Run Trivy vulnerability scanner in repo mode + - name: Run Trivy vulnerability scanner in fs mode uses: aquasecurity/trivy-action@master with: scan-type: 'fs' - ignore-unfixed: true - format: 'sarif' - output: 'trivy-results.sarif' - severity: 'CRITICAL' + scan-ref: '.' + + # - name: Run Trivy vulnerability scanner in repo mode + # uses: aquasecurity/trivy-action@master + # with: + # scan-type: 'fs' + # ignore-unfixed: true + # format: 'sarif' + # output: 'trivy-results.sarif' + # severity: 'CRITICAL' # - name: Upload Trivy scan results to GitHub Security tab # uses: github/codeql-action/upload-sarif@v3 # with: # sarif_file: 'trivy-results.sarif' - - name: Upload Trivy scan results - if: ${{ always() }} - uses: actions/upload-artifact@v4 - with: - name: trivy_results - path: 'trivy-results.sarif' - if-no-files-found: 'error' + # - name: Upload Trivy scan results + # if: ${{ always() }} + # uses: actions/upload-artifact@v4 + # with: + # name: trivy_results + # path: 'trivy-results.sarif' + # if-no-files-found: 'error' From 04f77f0a59403544d62260c9b80c6e3bee4a461e Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 13 Feb 2024 12:10:31 +0100 Subject: [PATCH 21/39] update names --- .github/workflows/sdl.yml | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/.github/workflows/sdl.yml b/.github/workflows/sdl.yml index dccd2ccf8..3c8a87287 100644 --- a/.github/workflows/sdl.yml +++ b/.github/workflows/sdl.yml @@ -1,4 +1,4 @@ -name: SDL +name: OpenVINO tokenizers sdl tests on: workflow_dispatch: pull_request: @@ -22,7 +22,7 @@ env: jobs: sdl_tests: - name: OpenVINO tokenizers sdl tests + name: bandit_tests timeout-minutes: 5 defaults: run: @@ -54,7 +54,7 @@ jobs: working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }} trivy_scan: - name: code_scanning + name: tryvi_tests runs-on: ubuntu-latest steps: - name: Checkout code @@ -66,24 +66,3 @@ jobs: scan-type: 'fs' scan-ref: '.' - # - name: Run Trivy vulnerability scanner in repo mode - # uses: aquasecurity/trivy-action@master - # with: - # scan-type: 'fs' - # ignore-unfixed: true - # format: 'sarif' - # output: 'trivy-results.sarif' - # severity: 'CRITICAL' - - # - name: Upload Trivy scan results to GitHub Security tab - # uses: github/codeql-action/upload-sarif@v3 - # with: - # sarif_file: 'trivy-results.sarif' - - # - name: Upload Trivy scan results - # if: ${{ always() }} - # uses: actions/upload-artifact@v4 - # with: - # name: trivy_results - # path: 'trivy-results.sarif' - # if-no-files-found: 'error' From 541ff9ab62f7137abf9487ebff63332a1b1b34c4 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 13 Feb 2024 12:22:45 +0100 Subject: [PATCH 22/39] clone repo --- .github/workflows/sdl.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sdl.yml b/.github/workflows/sdl.yml index 3c8a87287..628bd862a 100644 --- a/.github/workflows/sdl.yml +++ b/.github/workflows/sdl.yml @@ -53,12 +53,15 @@ jobs: bandit -c pyproject.toml -r python working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }} + trivy_scan: name: tryvi_tests runs-on: ubuntu-latest steps: - - name: Checkout code + - name: Clone Openvino tokenizers sources and tests uses: actions/checkout@v4 + with: + path: ${{ env.OPENVINO_TOKENIZERS_REPO }} - name: Run Trivy vulnerability scanner in fs mode uses: aquasecurity/trivy-action@master From 92aad6356e97d183e62b34883a695e8490ae7a54 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 13 Feb 2024 12:26:06 +0100 Subject: [PATCH 23/39] changed scan-ref --- .github/workflows/sdl.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/sdl.yml b/.github/workflows/sdl.yml index 628bd862a..e16c8cfce 100644 --- a/.github/workflows/sdl.yml +++ b/.github/workflows/sdl.yml @@ -67,5 +67,4 @@ jobs: uses: aquasecurity/trivy-action@master with: scan-type: 'fs' - scan-ref: '.' - + scan-ref: ${{ env.OPENVINO_TOKENIZERS_REPO }} From 340ceca9f7ba78462fa71aba3b2670156bc6d0ff Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 13 Feb 2024 12:28:18 +0100 Subject: [PATCH 24/39] syntax --- .github/workflows/sdl.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sdl.yml b/.github/workflows/sdl.yml index e16c8cfce..9dc31e443 100644 --- a/.github/workflows/sdl.yml +++ b/.github/workflows/sdl.yml @@ -67,4 +67,4 @@ jobs: uses: aquasecurity/trivy-action@master with: scan-type: 'fs' - scan-ref: ${{ env.OPENVINO_TOKENIZERS_REPO }} + scan-ref: '${{ env.OPENVINO_TOKENIZERS_REPO }}' From 9ab6058f9476cf16ba3df6af7d8ca2208e7100ca Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 13 Feb 2024 12:31:39 +0100 Subject: [PATCH 25/39] scan all sources --- .github/workflows/sdl.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sdl.yml b/.github/workflows/sdl.yml index 9dc31e443..44195c1a0 100644 --- a/.github/workflows/sdl.yml +++ b/.github/workflows/sdl.yml @@ -67,4 +67,4 @@ jobs: uses: aquasecurity/trivy-action@master with: scan-type: 'fs' - scan-ref: '${{ env.OPENVINO_TOKENIZERS_REPO }}' + scan-ref: '.' From 1259564aec9172f7f3d77e9c5cbc97956efca427 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 13 Feb 2024 12:37:01 +0100 Subject: [PATCH 26/39] env --- .github/workflows/sdl.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/sdl.yml b/.github/workflows/sdl.yml index 44195c1a0..d8e337efb 100644 --- a/.github/workflows/sdl.yml +++ b/.github/workflows/sdl.yml @@ -19,6 +19,7 @@ concurrency: env: PYTHON_VERSION: '3.11' + OPENVINO_TOKENIZERS_REPO: ${{ github.workspace }}/openvino_tokenizers jobs: sdl_tests: @@ -29,9 +30,6 @@ jobs: shell: bash runs-on: ubuntu-latest - env: - OPENVINO_TOKENIZERS_REPO: ${{ github.workspace }}/openvino_tokenizers - steps: - name: Clone Openvino tokenizers sources and tests uses: actions/checkout@v4 @@ -53,12 +51,11 @@ jobs: bandit -c pyproject.toml -r python working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }} - trivy_scan: name: tryvi_tests runs-on: ubuntu-latest steps: - - name: Clone Openvino tokenizers sources and tests + - name: Clone Openvino tokenizers sources uses: actions/checkout@v4 with: path: ${{ env.OPENVINO_TOKENIZERS_REPO }} @@ -67,4 +64,5 @@ jobs: uses: aquasecurity/trivy-action@master with: scan-type: 'fs' - scan-ref: '.' + scan-ref: ${{ env.OPENVINO_TOKENIZERS_REPO }} + From fac41d8466cc20f81545aee8cc552e9147fbb848 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 13 Feb 2024 12:40:20 +0100 Subject: [PATCH 27/39] changed env --- .github/workflows/sdl.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sdl.yml b/.github/workflows/sdl.yml index d8e337efb..47441b687 100644 --- a/.github/workflows/sdl.yml +++ b/.github/workflows/sdl.yml @@ -19,7 +19,6 @@ concurrency: env: PYTHON_VERSION: '3.11' - OPENVINO_TOKENIZERS_REPO: ${{ github.workspace }}/openvino_tokenizers jobs: sdl_tests: @@ -30,6 +29,9 @@ jobs: shell: bash runs-on: ubuntu-latest + env: + OPENVINO_TOKENIZERS_REPO: ${{ github.workspace }}/openvino_tokenizers + steps: - name: Clone Openvino tokenizers sources and tests uses: actions/checkout@v4 @@ -54,6 +56,10 @@ jobs: trivy_scan: name: tryvi_tests runs-on: ubuntu-latest + + env: + OPENVINO_TOKENIZERS_REPO: ${{ github.workspace }}/openvino_tokenizers + steps: - name: Clone Openvino tokenizers sources uses: actions/checkout@v4 From 9b3373b860d61e993c482a86c794e3a22f07c84e Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 13 Feb 2024 12:44:25 +0100 Subject: [PATCH 28/39] simplify pipeline --- .github/workflows/sdl.yml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/.github/workflows/sdl.yml b/.github/workflows/sdl.yml index 47441b687..c164e0009 100644 --- a/.github/workflows/sdl.yml +++ b/.github/workflows/sdl.yml @@ -29,14 +29,9 @@ jobs: shell: bash runs-on: ubuntu-latest - env: - OPENVINO_TOKENIZERS_REPO: ${{ github.workspace }}/openvino_tokenizers - steps: - name: Clone Openvino tokenizers sources and tests uses: actions/checkout@v4 - with: - path: ${{ env.OPENVINO_TOKENIZERS_REPO }} - name: Setup Python ${{ env.PYTHON_VERSION }} uses: actions/setup-python@v5 @@ -51,24 +46,18 @@ jobs: - name: Tokenizers Bandit tests run: | bandit -c pyproject.toml -r python - working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }} trivy_scan: name: tryvi_tests runs-on: ubuntu-latest - env: - OPENVINO_TOKENIZERS_REPO: ${{ github.workspace }}/openvino_tokenizers - steps: - name: Clone Openvino tokenizers sources uses: actions/checkout@v4 - with: - path: ${{ env.OPENVINO_TOKENIZERS_REPO }} - name: Run Trivy vulnerability scanner in fs mode uses: aquasecurity/trivy-action@master with: scan-type: 'fs' - scan-ref: ${{ env.OPENVINO_TOKENIZERS_REPO }} + scan-ref: '.' From 36841d19e7ada517025ae40e29729c9823949d1e Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 13 Feb 2024 12:53:12 +0100 Subject: [PATCH 29/39] checkmarx_scan --- .github/workflows/sdl.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/sdl.yml b/.github/workflows/sdl.yml index c164e0009..7d85e3b97 100644 --- a/.github/workflows/sdl.yml +++ b/.github/workflows/sdl.yml @@ -61,3 +61,18 @@ jobs: scan-type: 'fs' scan-ref: '.' + checkmarx_scan: + name: tryvi_tests + runs-on: ubuntu-latest + + steps: + - name: Clone Openvino tokenizers sources + uses: actions/checkout@v4 + + - name: Checkmarx AST CLI Action + uses: checkmarx/ast-github-action@main + with: + base_uri: https://ast.checkmarx.net/ + cx_tenant: your_tenant + cx_client_id: ${{ secrets.CX_CLIENT_ID }} + cx_client_secret: ${{ secrets.CX_CLIENT_SECRET }} \ No newline at end of file From 67cefaab4708b658ebf81642b843a7ac567ef545 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 13 Feb 2024 12:57:41 +0100 Subject: [PATCH 30/39] renamed --- .github/workflows/sdl.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sdl.yml b/.github/workflows/sdl.yml index 7d85e3b97..2f03772ff 100644 --- a/.github/workflows/sdl.yml +++ b/.github/workflows/sdl.yml @@ -62,7 +62,7 @@ jobs: scan-ref: '.' checkmarx_scan: - name: tryvi_tests + name: checkmarx_tests runs-on: ubuntu-latest steps: From c9202d80e7052eb370647b678abb035f2b59a0ff Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 13 Feb 2024 12:58:08 +0100 Subject: [PATCH 31/39] removed checkmarx tests --- .github/workflows/sdl.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.github/workflows/sdl.yml b/.github/workflows/sdl.yml index 2f03772ff..c9fc8a552 100644 --- a/.github/workflows/sdl.yml +++ b/.github/workflows/sdl.yml @@ -60,19 +60,3 @@ jobs: with: scan-type: 'fs' scan-ref: '.' - - checkmarx_scan: - name: checkmarx_tests - runs-on: ubuntu-latest - - steps: - - name: Clone Openvino tokenizers sources - uses: actions/checkout@v4 - - - name: Checkmarx AST CLI Action - uses: checkmarx/ast-github-action@main - with: - base_uri: https://ast.checkmarx.net/ - cx_tenant: your_tenant - cx_client_id: ${{ secrets.CX_CLIENT_ID }} - cx_client_secret: ${{ secrets.CX_CLIENT_SECRET }} \ No newline at end of file From c12a4a2f58b1c4d6fcf4f39a0f3d41e5da30be66 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 13 Feb 2024 14:50:26 +0100 Subject: [PATCH 32/39] set mac deployment target --- .github/workflows/mac.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 23cd3dff5..c1118b8eb 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -14,6 +14,7 @@ concurrency: env: PYTHON_VERSION: '3.11' + MACOSX_DEPLOYMENT_TARGET: '10.12' jobs: openvino_build: From 8142d17dc3b53b9e82cb079c67942ecfa0d85bf0 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 13 Feb 2024 14:52:48 +0100 Subject: [PATCH 33/39] simplified pipeline --- .github/workflows/sdl.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/sdl.yml b/.github/workflows/sdl.yml index c9fc8a552..4989bfcfb 100644 --- a/.github/workflows/sdl.yml +++ b/.github/workflows/sdl.yml @@ -22,7 +22,7 @@ env: jobs: sdl_tests: - name: bandit_tests + name: SDL tests timeout-minutes: 5 defaults: run: @@ -47,16 +47,9 @@ jobs: run: | bandit -c pyproject.toml -r python - trivy_scan: - name: tryvi_tests - runs-on: ubuntu-latest - - steps: - - name: Clone Openvino tokenizers sources - uses: actions/checkout@v4 - - name: Run Trivy vulnerability scanner in fs mode uses: aquasecurity/trivy-action@master with: scan-type: 'fs' scan-ref: '.' + From b56d314561c85ec3dccd9d7e27b44cf1d9b791d1 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 13 Feb 2024 23:06:49 +0100 Subject: [PATCH 34/39] move install rules at the end --- src/CMakeLists.txt | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 44b84f7bd..17955b134 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -9,20 +9,6 @@ add_library(${TARGET_NAME} SHARED ${SRC}) target_compile_definitions(${TARGET_NAME} PRIVATE IMPLEMENT_OPENVINO_EXTENSION_API) target_link_libraries(${TARGET_NAME} PRIVATE openvino::runtime) -if(rpaths) - set_target_properties(${TARGET_NAME} PROPERTIES INSTALL_RPATH "${rpaths}") -endif() - -# Installing the extension module to the root of the package -install(TARGETS ${TARGET_NAME} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) - -# cpack configuration -set(CPACK_PACKAGE_NAME ${TARGET_NAME}) -set(CPACK_PACKAGE_VERSION "${CMAKE_PROJECT_VERSION}") -set(CPACK_SOURCE_GENERATOR "") # not used - if(POLICY CMP0135) cmake_policy(SET CMP0135 NEW) endif() @@ -75,6 +61,7 @@ FetchContent_Declare( URL_HASH SHA256=63617eaf56c7a3857597dcd8780461f57dd21381b56a27716ef7d7e02e14ced4 ) FetchContent_MakeAvailable(sentencepiece) +set_property(DIRECTORY ${sentencepiece_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL ON) if(BUILD_FAST_TOKENIZERS) FetchContent_Declare( @@ -192,8 +179,6 @@ set_target_properties(${TARGET_NAME} PROPERTIES COMPILE_OPTIONS "${extra_flags}" # Post build steps to copy core_tokenizers dependencies # -set_property(DIRECTORY ${sentencepiece_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL ON) - if(BUILD_FAST_TOKENIZERS) install(TARGETS core_tokenizers LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} @@ -224,4 +209,18 @@ if(extra_libs) install(FILES ${extra_libs} DESTINATION ${extra_libs_location}) endif() +if(rpaths) + set_target_properties(${TARGET_NAME} PROPERTIES INSTALL_RPATH "${rpaths}") +endif() + +# Installing the extension module to the root of the package +install(TARGETS ${TARGET_NAME} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT tokenizers + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT tokenizers) + +# cpack configuration +set(CPACK_PACKAGE_NAME ${TARGET_NAME}) +set(CPACK_PACKAGE_VERSION "${CMAKE_PROJECT_VERSION}") +set(CPACK_SOURCE_GENERATOR "") # not used + include (CPack) From 62768231e21da0da3d165dcd94e83e3cd6078b0e Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Tue, 13 Feb 2024 23:23:26 +0100 Subject: [PATCH 35/39] git w/a --- .github/workflows/windows.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index e3babcb84..b8e0b1f7d 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -32,6 +32,9 @@ jobs: BUILD_DIR: ${{ github.workspace }}\\openvino\\build steps: + - name: git configuration + run: git config --system core.longpaths true + - name: Clone OpenVINO uses: actions/checkout@v4 with: From 31f25997955cf82af644d5eb60074e458a2b7f0f Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 14 Feb 2024 12:01:12 +0100 Subject: [PATCH 36/39] removed component splitting --- src/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 17955b134..96e058f6a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -215,8 +215,8 @@ endif() # Installing the extension module to the root of the package install(TARGETS ${TARGET_NAME} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT tokenizers - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT tokenizers) + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) # cpack configuration set(CPACK_PACKAGE_NAME ${TARGET_NAME}) From 5a2eb3d5c0a85db7f0a9535fa7fb3c557c67b3b2 Mon Sep 17 00:00:00 2001 From: Zlobin Vladimir Date: Wed, 14 Feb 2024 17:30:24 +0400 Subject: [PATCH 37/39] Enable Python3.8 (#20) --- python/openvino_tokenizers/tokenizer_pipeline.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/openvino_tokenizers/tokenizer_pipeline.py b/python/openvino_tokenizers/tokenizer_pipeline.py index 51fe4cc43..5e15f183c 100644 --- a/python/openvino_tokenizers/tokenizer_pipeline.py +++ b/python/openvino_tokenizers/tokenizer_pipeline.py @@ -2,6 +2,7 @@ # Copyright (C) 2018-2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations import logging import weakref from dataclasses import dataclass, field From 1a9c0ab7f821c06fad0367ad57f7d99b708c4fa0 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 14 Feb 2024 23:22:14 +0100 Subject: [PATCH 38/39] [CI] Exclude FetchContent targets (#22) * exlude_from_all for FetchContent * fixed typo * reverted fast_tokenizers fetch --- src/CMakeLists.txt | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 96e058f6a..f73ad7d0e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -60,8 +60,12 @@ FetchContent_Declare( URL https://github.com/google/sentencepiece/archive/refs/tags/v0.1.99.tar.gz URL_HASH SHA256=63617eaf56c7a3857597dcd8780461f57dd21381b56a27716ef7d7e02e14ced4 ) -FetchContent_MakeAvailable(sentencepiece) -set_property(DIRECTORY ${sentencepiece_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL ON) +FetchContent_GetProperties(sentencepiece) +if(NOT sentencepiece_POPULATED) + FetchContent_Populate(sentencepiece) + add_subdirectory(${sentencepiece_SOURCE_DIR} ${sentencepiece_BINARY_DIR} EXCLUDE_FROM_ALL) +endif() + if(BUILD_FAST_TOKENIZERS) FetchContent_Declare( @@ -102,8 +106,11 @@ else() URL https://github.com/google/re2/archive/refs/tags/2022-04-01.tar.gz URL_HASH SHA256=1ae8ccfdb1066a731bba6ee0881baad5efd2cd661acd9569b689f2586e1a50e9 ) - FetchContent_MakeAvailable(re2) - set_property(DIRECTORY ${re2_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL ON) + FetchContent_GetProperties(re2) + if(NOT re2_POPULATED) + FetchContent_Populate(re2) + add_subdirectory(${re2_SOURCE_DIR} ${re2_BINARY_DIR} EXCLUDE_FROM_ALL) + endif() elseif(LINUX AND X86_64) FetchContent_Declare( fast_tokenizer @@ -135,6 +142,7 @@ else() endif() FetchContent_MakeAvailable(fast_tokenizer) + # to allow find_library to work with conda-forge env set(_old_CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ${CMAKE_FIND_ROOT_PATH_MODE_LIBRARY}) set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY NEVER) From 6ab5521799be48cc35081cbcac1dbb273bcf35a0 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Thu, 15 Feb 2024 10:11:45 +0100 Subject: [PATCH 39/39] removed OpenVINODeveloperPackage (#24) --- CMakeLists.txt | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1491aa0f2..a56f22b8f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,12 +8,7 @@ project(openvino_tokenizers) include(cmake/platforms.cmake) -# Find OpenVINODeveloperPackage first to compile with SDL flags -find_package(OpenVINODeveloperPackage QUIET - PATHS "${OpenVINO_DIR}") -if(NOT OpenVINODeveloperPackage_FOUND) - find_package(OpenVINO REQUIRED COMPONENTS Runtime) -endif() +find_package(OpenVINO REQUIRED COMPONENTS Runtime) # set the project version using OpenVINO major, minor and patch versions set(CMAKE_PROJECT_VERSION_TWEAK 0)