refactor: use new grpc version (#6110)

Signed-off-by: dependabot[bot] <[email protected]> Signed-off-by: Joan Fontanals Martinez <[email protected]> Signed-off-by: Joan Martinez <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Jina Dev Bot <[email protected]> Co-authored-by: Joan Fontanals Martinez <[email protected]>
jina-ai · Dec 1, 2023 · ae399ad · ae399ad
1 parent 8a1e6b0
commit ae399ad
Show file tree

Hide file tree

Showing 31 changed files with 874 additions and 245 deletions.
diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
@@ -107,6 +107,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: [3.8]
+        protobuf-version: ['==3.19.6', '']
     steps:
       - uses: actions/[email protected]
       - name: Set up Python ${{ matrix.python-version }}
@@ -134,7 +135,11 @@ jobs:
           python -m pip install wheel
           WHEEL_FILE=$(ls dist/*whl)
           pip install "$WHEEL_FILE[common,devel,test]" --no-cache-dir
-          pip install -U protobuf${{ matrix.protobuf-version }}
+          if [[ "${{ matrix.protobuf-version }}" == "==3.19.6" ]]; then
+            pip install -U protobuf${{ matrix.protobuf-version }} grpcio==1.47.5 grpcio-reflection==1.47.5 grpcio-health-checking==1.47.5
+          else
+            pip install -U protobuf${{ matrix.protobuf-version }}
+          fi
           jina
           export JINA_LOG_LEVEL="ERROR"
       - name: Test
@@ -153,10 +158,69 @@ jobs:
         timeout-minutes: 45
         env:
           JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}"
+      - name: Check codecov file
+        id: check_files
+        uses: andstor/file-existence-action@v1
+        with:
+          files: "coverage.xml"
+      - name: Upload coverage from test to Codecov
+        uses: codecov/[email protected]
+        if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
+        with:
+          file: coverage.xml
+          name: ${{ matrix.test-path }}-codecov
+          flags: ${{ steps.test.outputs.codecov_flag }}
+          fail_ci_if_error: false
+
+  stateful-docarray-v-two-test:
+    needs: prep-testbed
+    runs-on: ubuntu-latest
+    env:
+      JINA_RANDOM_PORT_MIN: 16384
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.8]
+        protobuf-version: ['==3.19.6', '']
+    steps:
+      - uses: actions/[email protected]
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Set up Golang
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.19.5
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel
+          pip install git+https://github.com/jina-ai/setuptools-golang.git@feat-align-with-alaeddine-code
+          pip install cibuildwheel
+
+      - name: Build wheels with setuptools-golang-build-manylinux-wheel
+        run: |
+          setuptools-golang-build-manylinux-wheels --pythons cp38-cp38
+      - name: Prepare environment
+        run: |
+          docker build -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
+          python -m pip install --upgrade pip
+          python -m pip install wheel
+          WHEEL_FILE=$(ls dist/*whl)
+          pip install "$WHEEL_FILE[common,devel,test]" --no-cache-dir
+          if [[ "${{ matrix.protobuf-version }}" == "==3.19.6" ]]; then
+            pip install -U protobuf${{ matrix.protobuf-version }} grpcio==1.47.5 grpcio-reflection==1.47.5 grpcio-health-checking==1.47.5
+          else
+            pip install -U protobuf${{ matrix.protobuf-version }}
+          fi
+          jina
+          export JINA_LOG_LEVEL="ERROR"
       - name: Test stateful
         id: test_stateful
         run: |
-          JINA_LOG_LEVEL=DEBUG JINA_RANDOM_PORT_MAX="60535" pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/stateful
+          JINA_LOG_LEVEL=DEBUG JINA_RANDOM_PORT_MAX="60535" pytest --suppress-no-test-exit-code --maxfail 1 --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/stateful
           echo "flag it as jina for codeoverage"
           echo "codecov_flag=jina" >> $GITHUB_OUTPUT
         timeout-minutes: 30
@@ -811,7 +875,7 @@ jobs:
   # just for blocking the merge until all parallel core-test are successful
   success-all-steps:
     runs-on: ubuntu-latest
-    needs: [core-test, docarray-v-two-test, import-test, hub-test, k8s-flow-test, k8s-deployment-test, k8s-graceful-test, k8s-failures-test, k8s-otel-test, docker-compose-test, docker-image-test, benchmark-pre-release, update-schema, update-docker] #, pre-release]
+    needs: [core-test, docarray-v-two-test, stateful-docarray-v-two-test, import-test, hub-test, k8s-flow-test, k8s-deployment-test, k8s-graceful-test, k8s-failures-test, k8s-otel-test, docker-compose-test, docker-image-test, benchmark-pre-release, update-schema, update-docker] #, pre-release]
     if: always()
     steps:
       - uses: technote-space/workflow-conclusion-action@v2

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -31,9 +31,9 @@ jobs:
         run: |
           pip install flake8
           # stop the build if there are Python syntax errors or undefined names
-          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,jina/resources/,jina/proto/docarray_v1,jina/proto/docarray_v2
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,jina/resources/,jina/proto/docarray_v1,jina/proto/docarray_v2,jina/serve/consensus/add_voter/pb,jina/serve/consensus/add_voter/pb2
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,jina/proto/docarray_v1,jina/proto/docarray_v2
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,jina/proto/docarray_v1,jina/proto/docarray_v2,jina/serve/consensus/add_voter/pb,jina/serve/consensus/add_voter/pb2
 
   check-docstring:
     runs-on: ubuntu-latest
@@ -469,7 +469,11 @@ jobs:
           python -m pip install wheel
           WHEEL_FILE=$(ls dist/*whl)
           pip install "$WHEEL_FILE[common,devel,test]" --no-cache-dir
-          pip install -U protobuf${{ matrix.protobuf-version }}
+          if [[ "${{ matrix.protobuf-version }}" == "==3.19.6" ]]; then
+            pip install -U protobuf${{ matrix.protobuf-version }} grpcio==1.47.5 grpcio-reflection==1.47.5 grpcio-health-checking==1.47.5
+          else
+            pip install -U protobuf${{ matrix.protobuf-version }}
+          fi
           jina
           export JINA_LOG_LEVEL="ERROR"
       - name: Test
@@ -488,10 +492,70 @@ jobs:
         timeout-minutes: 45
         env:
           JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}"
+      - name: Check codecov file
+        id: check_files
+        uses: andstor/file-existence-action@v1
+        with:
+          files: "coverage.xml"
+      - name: Upload coverage from test to Codecov
+        uses: codecov/[email protected]
+        if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'
+        with:
+          file: coverage.xml
+          name: ${{ matrix.test-path }}-codecov
+          flags: ${{ steps.test.outputs.codecov_flag }}
+          fail_ci_if_error: false
+
+
+  stateful-docarray-v-two-test:
+    needs: prep-testbed
+    runs-on: ubuntu-latest
+    env:
+      JINA_RANDOM_PORT_MIN: 16384
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.8]
+        protobuf-version: ['==3.19.6', '']
+    steps:
+      - uses: actions/[email protected]
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Set up Golang
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.19.5
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel
+          pip install git+https://github.com/jina-ai/setuptools-golang.git@feat-align-with-alaeddine-code
+          pip install cibuildwheel
+
+      - name: Build wheels with setuptools-golang-build-manylinux-wheel
+        run: |
+          setuptools-golang-build-manylinux-wheels --pythons cp38-cp38
+      - name: Prepare environment
+        run: |
+          docker build -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
+          python -m pip install --upgrade pip
+          python -m pip install wheel
+          WHEEL_FILE=$(ls dist/*whl)
+          pip install "$WHEEL_FILE[common,devel,test]" --no-cache-dir
+          if [[ "${{ matrix.protobuf-version }}" == "==3.19.6" ]]; then
+            pip install -U protobuf${{ matrix.protobuf-version }} grpcio==1.47.5 grpcio-reflection==1.47.5 grpcio-health-checking==1.47.5
+          else
+            pip install -U protobuf${{ matrix.protobuf-version }}
+          fi
+          jina
+          export JINA_LOG_LEVEL="ERROR"
       - name: Test stateful
         id: test_stateful
         run: |
-          JINA_LOG_LEVEL=DEBUG JINA_RANDOM_PORT_MAX="60535" pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/stateful
+          JINA_LOG_LEVEL=DEBUG JINA_RANDOM_PORT_MAX="60535" pytest --suppress-no-test-exit-code --maxfail 1 --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/stateful
           echo "flag it as jina for codeoverage"
           echo "codecov_flag=jina" >> $GITHUB_OUTPUT
         timeout-minutes: 30
@@ -772,7 +836,7 @@ jobs:
   # just for blocking the merge until all parallel core-test are successful
   success-all-test:
     runs-on: ubuntu-latest
-    needs: [commit-lint, core-test, docarray-v-two-test, import-test, hub-test, k8s-flow-test, k8s-deployment-test, k8s-graceful-test, k8s-failures-test, k8s-otel-test, docker-compose-test, docker-image-test, check-docstring, check-black, code-injection, build-wheels]
+    needs: [commit-lint, core-test, docarray-v-two-test, stateful-docarray-v-two-test, import-test, hub-test, k8s-flow-test, k8s-deployment-test, k8s-graceful-test, k8s-failures-test, k8s-otel-test, docker-compose-test, docker-image-test, check-docstring, check-black, code-injection, build-wheels]
 
     if: always()
     steps:

diff --git a/extra-requirements.txt b/extra-requirements.txt
@@ -27,9 +27,9 @@
 
 numpy:                      core
 protobuf>=3.19.0:           core
-grpcio>=1.46.0,<1.48.1:     core
-grpcio-reflection>=1.46.0,<1.48.1:  core
-grpcio-health-checking>=1.46.0,<1.48.1:  core
+grpcio>=1.46.0,<=1.57.0:    core
+grpcio-reflection>=1.46.0,<=1.57.0:  core
+grpcio-health-checking>=1.46.0,<=1.57.0:  core
 pyyaml>=5.3.1:              core
 packaging>=20.0:            core
 docarray>=0.16.4:           core

diff --git a/jina/logging/logger.py b/jina/logging/logger.py
@@ -129,7 +129,6 @@ def __init__(
 
         if not name:
             name = os.getenv('JINA_DEPLOYMENT_NAME', context)
-
         self.logger = logging.getLogger(context)
         self.logger.propagate = False
 
@@ -138,7 +137,6 @@ def __init__(
             'uptime': __uptime__,
             'context': context,
         }
-
         self.add_handlers(log_config, **context_vars)
         self.debug = self.logger.debug
         self.warning = self.logger.warning