From ae399adf3f712074ffdc2e7a27261ee109a81b31 Mon Sep 17 00:00:00 2001 From: Joan Fontanals Date: Fri, 1 Dec 2023 14:03:18 +0100 Subject: [PATCH] refactor: use new grpc version (#6110) Signed-off-by: dependabot[bot] Signed-off-by: Joan Fontanals Martinez Signed-off-by: Joan Martinez Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Jina Dev Bot Co-authored-by: Joan Fontanals Martinez --- .github/workflows/cd.yml | 70 +++++++++- .github/workflows/ci.yml | 74 +++++++++- extra-requirements.txt | 6 +- jina/logging/logger.py | 2 - jina/orchestrate/deployments/__init__.py | 120 ++++++++-------- jina/orchestrate/flow/base.py | 20 +-- jina/serve/consensus/__init__.py | 0 jina/serve/consensus/add_voter/README.md | 42 ++++++ jina/serve/consensus/add_voter/__init__.py | 0 .../serve/consensus/add_voter/add_voter.proto | 26 ++++ .../consensus/add_voter/add_voter_pb2.py | 6 + .../consensus/add_voter/add_voter_pb2_grpc.py | 6 + .../add_voter/build-add-voter-proto.sh | 41 ++++++ .../consensus/add_voter/call_add_voter.py | 47 +++++++ jina/serve/consensus/add_voter/pb/__init__.py | 0 .../consensus/add_voter/pb/add_voter_pb2.py | 33 +++++ .../add_voter/pb/add_voter_pb2_grpc.py | 132 ++++++++++++++++++ .../serve/consensus/add_voter/pb2/__init__.py | 0 .../consensus/add_voter/pb2/add_voter_pb2.py | 67 +++++++++ .../add_voter/pb2/add_voter_pb2_grpc.py | 132 ++++++++++++++++++ jina/serve/consensus/go.mod | 8 +- jina/serve/consensus/go.sum | 85 ++--------- jina/serve/consensus/jina_raft/fsm.go | 4 +- jina/serve/consensus/jraft.go | 6 +- jina/serve/consensus/jraft.h | 12 +- jina/serve/consensus/run.go | 56 ++++---- jina/serve/executors/run.py | 3 +- .../serve/runtimes/worker/request_handling.py | 1 + scripts/docstrings_lint.sh | 1 + setup.py | 9 -- tests/integration/stateful/test_stateful.py | 110 ++++++++++----- 31 files changed, 874 insertions(+), 245 deletions(-) create mode 100644 jina/serve/consensus/__init__.py create mode 100644 jina/serve/consensus/add_voter/README.md create mode 100644 jina/serve/consensus/add_voter/__init__.py create mode 100644 jina/serve/consensus/add_voter/add_voter.proto create mode 100644 jina/serve/consensus/add_voter/add_voter_pb2.py create mode 100644 jina/serve/consensus/add_voter/add_voter_pb2_grpc.py create mode 100644 jina/serve/consensus/add_voter/build-add-voter-proto.sh create mode 100644 jina/serve/consensus/add_voter/call_add_voter.py create mode 100644 jina/serve/consensus/add_voter/pb/__init__.py create mode 100644 jina/serve/consensus/add_voter/pb/add_voter_pb2.py create mode 100644 jina/serve/consensus/add_voter/pb/add_voter_pb2_grpc.py create mode 100644 jina/serve/consensus/add_voter/pb2/__init__.py create mode 100644 jina/serve/consensus/add_voter/pb2/add_voter_pb2.py create mode 100644 jina/serve/consensus/add_voter/pb2/add_voter_pb2_grpc.py diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index dc0c3a507ba58..6362dab74d535 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -107,6 +107,7 @@ jobs: fail-fast: false matrix: python-version: [3.8] + protobuf-version: ['==3.19.6', ''] steps: - uses: actions/checkout@v2.5.0 - name: Set up Python ${{ matrix.python-version }} @@ -134,7 +135,11 @@ jobs: python -m pip install wheel WHEEL_FILE=$(ls dist/*whl) pip install "$WHEEL_FILE[common,devel,test]" --no-cache-dir - pip install -U protobuf${{ matrix.protobuf-version }} + if [[ "${{ matrix.protobuf-version }}" == "==3.19.6" ]]; then + pip install -U protobuf${{ matrix.protobuf-version }} grpcio==1.47.5 grpcio-reflection==1.47.5 grpcio-health-checking==1.47.5 + else + pip install -U protobuf${{ matrix.protobuf-version }} + fi jina export JINA_LOG_LEVEL="ERROR" - name: Test @@ -153,10 +158,69 @@ jobs: timeout-minutes: 45 env: JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}" + - name: Check codecov file + id: check_files + uses: andstor/file-existence-action@v1 + with: + files: "coverage.xml" + - name: Upload coverage from test to Codecov + uses: codecov/codecov-action@v3.1.1 + if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8' + with: + file: coverage.xml + name: ${{ matrix.test-path }}-codecov + flags: ${{ steps.test.outputs.codecov_flag }} + fail_ci_if_error: false + + stateful-docarray-v-two-test: + needs: prep-testbed + runs-on: ubuntu-latest + env: + JINA_RANDOM_PORT_MIN: 16384 + strategy: + fail-fast: false + matrix: + python-version: [3.8] + protobuf-version: ['==3.19.6', ''] + steps: + - uses: actions/checkout@v2.5.0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Set up Golang + uses: actions/setup-go@v2 + with: + go-version: 1.19.5 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel + pip install git+https://github.com/jina-ai/setuptools-golang.git@feat-align-with-alaeddine-code + pip install cibuildwheel + + - name: Build wheels with setuptools-golang-build-manylinux-wheel + run: | + setuptools-golang-build-manylinux-wheels --pythons cp38-cp38 + - name: Prepare environment + run: | + docker build -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip . + python -m pip install --upgrade pip + python -m pip install wheel + WHEEL_FILE=$(ls dist/*whl) + pip install "$WHEEL_FILE[common,devel,test]" --no-cache-dir + if [[ "${{ matrix.protobuf-version }}" == "==3.19.6" ]]; then + pip install -U protobuf${{ matrix.protobuf-version }} grpcio==1.47.5 grpcio-reflection==1.47.5 grpcio-health-checking==1.47.5 + else + pip install -U protobuf${{ matrix.protobuf-version }} + fi + jina + export JINA_LOG_LEVEL="ERROR" - name: Test stateful id: test_stateful run: | - JINA_LOG_LEVEL=DEBUG JINA_RANDOM_PORT_MAX="60535" pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/stateful + JINA_LOG_LEVEL=DEBUG JINA_RANDOM_PORT_MAX="60535" pytest --suppress-no-test-exit-code --maxfail 1 --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/stateful echo "flag it as jina for codeoverage" echo "codecov_flag=jina" >> $GITHUB_OUTPUT timeout-minutes: 30 @@ -811,7 +875,7 @@ jobs: # just for blocking the merge until all parallel core-test are successful success-all-steps: runs-on: ubuntu-latest - needs: [core-test, docarray-v-two-test, import-test, hub-test, k8s-flow-test, k8s-deployment-test, k8s-graceful-test, k8s-failures-test, k8s-otel-test, docker-compose-test, docker-image-test, benchmark-pre-release, update-schema, update-docker] #, pre-release] + needs: [core-test, docarray-v-two-test, stateful-docarray-v-two-test, import-test, hub-test, k8s-flow-test, k8s-deployment-test, k8s-graceful-test, k8s-failures-test, k8s-otel-test, docker-compose-test, docker-image-test, benchmark-pre-release, update-schema, update-docker] #, pre-release] if: always() steps: - uses: technote-space/workflow-conclusion-action@v2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6eaf4b2d0aab7..d0738c7359eae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,9 +31,9 @@ jobs: run: | pip install flake8 # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,jina/resources/,jina/proto/docarray_v1,jina/proto/docarray_v2 + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,jina/resources/,jina/proto/docarray_v1,jina/proto/docarray_v2,jina/serve/consensus/add_voter/pb,jina/serve/consensus/add_voter/pb2 # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,jina/proto/docarray_v1,jina/proto/docarray_v2 + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,jina/proto/docarray_v1,jina/proto/docarray_v2,jina/serve/consensus/add_voter/pb,jina/serve/consensus/add_voter/pb2 check-docstring: runs-on: ubuntu-latest @@ -469,7 +469,11 @@ jobs: python -m pip install wheel WHEEL_FILE=$(ls dist/*whl) pip install "$WHEEL_FILE[common,devel,test]" --no-cache-dir - pip install -U protobuf${{ matrix.protobuf-version }} + if [[ "${{ matrix.protobuf-version }}" == "==3.19.6" ]]; then + pip install -U protobuf${{ matrix.protobuf-version }} grpcio==1.47.5 grpcio-reflection==1.47.5 grpcio-health-checking==1.47.5 + else + pip install -U protobuf${{ matrix.protobuf-version }} + fi jina export JINA_LOG_LEVEL="ERROR" - name: Test @@ -488,10 +492,70 @@ jobs: timeout-minutes: 45 env: JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}" + - name: Check codecov file + id: check_files + uses: andstor/file-existence-action@v1 + with: + files: "coverage.xml" + - name: Upload coverage from test to Codecov + uses: codecov/codecov-action@v3.1.1 + if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8' + with: + file: coverage.xml + name: ${{ matrix.test-path }}-codecov + flags: ${{ steps.test.outputs.codecov_flag }} + fail_ci_if_error: false + + + stateful-docarray-v-two-test: + needs: prep-testbed + runs-on: ubuntu-latest + env: + JINA_RANDOM_PORT_MIN: 16384 + strategy: + fail-fast: false + matrix: + python-version: [3.8] + protobuf-version: ['==3.19.6', ''] + steps: + - uses: actions/checkout@v2.5.0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Set up Golang + uses: actions/setup-go@v2 + with: + go-version: 1.19.5 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel + pip install git+https://github.com/jina-ai/setuptools-golang.git@feat-align-with-alaeddine-code + pip install cibuildwheel + + - name: Build wheels with setuptools-golang-build-manylinux-wheel + run: | + setuptools-golang-build-manylinux-wheels --pythons cp38-cp38 + - name: Prepare environment + run: | + docker build -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip . + python -m pip install --upgrade pip + python -m pip install wheel + WHEEL_FILE=$(ls dist/*whl) + pip install "$WHEEL_FILE[common,devel,test]" --no-cache-dir + if [[ "${{ matrix.protobuf-version }}" == "==3.19.6" ]]; then + pip install -U protobuf${{ matrix.protobuf-version }} grpcio==1.47.5 grpcio-reflection==1.47.5 grpcio-health-checking==1.47.5 + else + pip install -U protobuf${{ matrix.protobuf-version }} + fi + jina + export JINA_LOG_LEVEL="ERROR" - name: Test stateful id: test_stateful run: | - JINA_LOG_LEVEL=DEBUG JINA_RANDOM_PORT_MAX="60535" pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/stateful + JINA_LOG_LEVEL=DEBUG JINA_RANDOM_PORT_MAX="60535" pytest --suppress-no-test-exit-code --maxfail 1 --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/stateful echo "flag it as jina for codeoverage" echo "codecov_flag=jina" >> $GITHUB_OUTPUT timeout-minutes: 30 @@ -772,7 +836,7 @@ jobs: # just for blocking the merge until all parallel core-test are successful success-all-test: runs-on: ubuntu-latest - needs: [commit-lint, core-test, docarray-v-two-test, import-test, hub-test, k8s-flow-test, k8s-deployment-test, k8s-graceful-test, k8s-failures-test, k8s-otel-test, docker-compose-test, docker-image-test, check-docstring, check-black, code-injection, build-wheels] + needs: [commit-lint, core-test, docarray-v-two-test, stateful-docarray-v-two-test, import-test, hub-test, k8s-flow-test, k8s-deployment-test, k8s-graceful-test, k8s-failures-test, k8s-otel-test, docker-compose-test, docker-image-test, check-docstring, check-black, code-injection, build-wheels] if: always() steps: diff --git a/extra-requirements.txt b/extra-requirements.txt index a69425580cd0f..8ea22555847e9 100644 --- a/extra-requirements.txt +++ b/extra-requirements.txt @@ -27,9 +27,9 @@ numpy: core protobuf>=3.19.0: core -grpcio>=1.46.0,<1.48.1: core -grpcio-reflection>=1.46.0,<1.48.1: core -grpcio-health-checking>=1.46.0,<1.48.1: core +grpcio>=1.46.0,<=1.57.0: core +grpcio-reflection>=1.46.0,<=1.57.0: core +grpcio-health-checking>=1.46.0,<=1.57.0: core pyyaml>=5.3.1: core packaging>=20.0: core docarray>=0.16.4: core diff --git a/jina/logging/logger.py b/jina/logging/logger.py index 75107721d1ae2..1d48f65cfc4c6 100644 --- a/jina/logging/logger.py +++ b/jina/logging/logger.py @@ -129,7 +129,6 @@ def __init__( if not name: name = os.getenv('JINA_DEPLOYMENT_NAME', context) - self.logger = logging.getLogger(context) self.logger.propagate = False @@ -138,7 +137,6 @@ def __init__( 'uptime': __uptime__, 'context': context, } - self.add_handlers(log_config, **context_vars) self.debug = self.logger.debug self.warning = self.logger.warning diff --git a/jina/orchestrate/deployments/__init__.py b/jina/orchestrate/deployments/__init__.py index ae19d52d88ff1..580370fd6373f 100644 --- a/jina/orchestrate/deployments/__init__.py +++ b/jina/orchestrate/deployments/__init__.py @@ -68,30 +68,64 @@ class DeploymentType(type(ExitStack), type(JAMLCompatible)): pass -def _call_add_voters(leader, voters, replica_ids, name, event_signal=None): +def _call_add_voters(leader, voters, replica_ids, logger): # this method needs to be run in multiprocess, importing jraft in main process # makes it impossible to do tests sequentially + from jina.serve.consensus.add_voter.call_add_voter import call_add_voter - import jraft - - logger = JinaLogger(context=f'add_voter-{name}', name=f'add_voter-{name}') logger.debug(f'Trying to add {len(replica_ids)} voters to leader {leader}') + success_lists = [] for voter_address, replica_id in zip(voters, replica_ids): logger.debug( f'Trying to add replica-{str(replica_id)} as voter with address {voter_address} to leader at {leader}' ) success = False for i in range(5): - try: - logger.debug(f'Trying {i}th time') - jraft.add_voter(leader, str(replica_id), voter_address) + logger.debug(f'Trying {i}th time') + success = call_add_voter(leader, str(replica_id), voter_address) + if success: logger.debug(f'Trying {i}th time succeeded') - success = True break - except ValueError: + else: logger.debug(f'Trying {i}th failed. Wait 2 seconds for next try') time.sleep(2.0) + success_lists.append(success) + if not success: + logger.warning( + f'Failed to add {str(replica_id)} as voter with address {voter_address} to leader at {leader}. This could be because {leader} is not the leader, ' + f'and maybe the cluster is restoring from a previous cluster state' + ) + else: + logger.success( + f'Replica-{str(replica_id)} successfully added as voter with address {voter_address} to leader at {leader}' + ) + logger.debug('Adding voters to leader finished') + return all(success_lists) + + +async def _async_call_add_voters(leader, voters, replica_ids, logger): + # this method needs to be run in multiprocess, importing jraft in main process + # makes it impossible to do tests sequentially + from jina.serve.consensus.add_voter.call_add_voter import async_call_add_voter + + logger.debug(f'Trying to add {len(replica_ids)} voters to leader {leader}') + success_lists = [] + for voter_address, replica_id in zip(voters, replica_ids): + logger.debug( + f'Trying to add replica-{str(replica_id)} as voter with address {voter_address} to leader at {leader}' + ) + success = False + for i in range(5): + logger.debug(f'Trying {i}th time') + success = await async_call_add_voter(leader, str(replica_id), voter_address) + if success: + logger.debug(f'Trying {i}th time succeeded') + break + else: + logger.debug(f'Trying {i}th failed. Wait 2 seconds for next try') + time.sleep(2.0) + success_lists.append(success) if not success: logger.warning( f'Failed to add {str(replica_id)} as voter with address {voter_address} to leader at {leader}. This could be because {leader} is not the leader, ' @@ -102,8 +136,7 @@ def _call_add_voters(leader, voters, replica_ids, name, event_signal=None): f'Replica-{str(replica_id)} successfully added as voter with address {voter_address} to leader at {leader}' ) logger.debug('Adding voters to leader finished') - if event_signal: - event_signal.set() + return all(success_lists) class Deployment(JAMLCompatible, PostMixin, BaseOrchestrator, metaclass=DeploymentType): @@ -135,66 +168,33 @@ def _add_voter_to_leader(self): leader_address = f'{self._pods[0].runtime_ctrl_address}' voter_addresses = [pod.runtime_ctrl_address for pod in self._pods[1:]] replica_ids = [pod.args.replica_id for pod in self._pods[1:]] - event_signal = multiprocessing.Event() self.logger.debug('Starting process to call Add Voters') - process = multiprocessing.Process( - target=_call_add_voters, - kwargs={ - 'leader': leader_address, - 'voters': voter_addresses, - 'replica_ids': replica_ids, - 'name': self.name, - 'event_signal': event_signal, - }, + res = _call_add_voters( + leader=leader_address, + voters=voter_addresses, + replica_ids=replica_ids, + logger=self.logger, ) - process.start() - start = time.time() - properly_closed = False - while time.time() - start < 20 * len(replica_ids): - if event_signal.is_set(): - properly_closed = True - break - else: - time.sleep(1.0) - if properly_closed: - self.logger.debug('Add Voters process finished') - process.terminate() + if res: + self.logger.debug('Finished adding voters') else: - self.logger.error('Add Voters process did not finish successfully') - process.kill() - self.logger.debug('Add Voters process finished') + self.logger.error('Adding Voters did not finish successfully') async def _async_add_voter_to_leader(self): leader_address = f'{self._pods[0].runtime_ctrl_address}' voter_addresses = [pod.runtime_ctrl_address for pod in self._pods[1:]] replica_ids = [pod.args.replica_id for pod in self._pods[1:]] - event_signal = multiprocessing.Event() self.logger.debug('Starting process to call Add Voters') - process = multiprocessing.Process( - target=_call_add_voters, - kwargs={ - 'leader': leader_address, - 'voters': voter_addresses, - 'replica_ids': replica_ids, - 'name': self.name, - 'event_signal': event_signal, - }, + res = await _async_call_add_voters( + leader=leader_address, + voters=voter_addresses, + replica_ids=replica_ids, + logger=self.logger, ) - process.start() - start = time.time() - properly_closed = False - while time.time() - start < 20 * len(replica_ids): - if event_signal.is_set(): - properly_closed = True - break - else: - await asyncio.sleep(1.0) - if properly_closed: - self.logger.debug('Add Voters process finished') - process.terminate() + if res: + self.logger.debug('Finished adding voters') else: - self.logger.error('Add Voters process did not finish successfully') - process.kill() + self.logger.error('Adding Voters did not finish successfully') @property def is_ready(self): diff --git a/jina/orchestrate/flow/base.py b/jina/orchestrate/flow/base.py index 93d82a3644cf8..3f93448c0015a 100644 --- a/jina/orchestrate/flow/base.py +++ b/jina/orchestrate/flow/base.py @@ -1961,17 +1961,17 @@ async def _async_wait_all(): for task in pending: task.cancel() - # kick off spinner thread - polling_status_thread = threading.Thread( - target=_polling_status, - args=(len(wait_for_ready_coros),), - daemon=True, - ) + if 'GITHUB_WORKFLOW' not in os.environ: + # kick off spinner thread + polling_status_thread = threading.Thread( + target=_polling_status, + args=(len(wait_for_ready_coros),), + daemon=True, + ) - polling_status_thread.start() + polling_status_thread.start() # kick off all deployments wait-ready tasks - try: _ = asyncio.get_event_loop() except: @@ -1991,7 +1991,6 @@ async def _f(): if not running_in_event_loop: asyncio.get_event_loop().run_until_complete(_async_wait_all()) else: - # TODO: the same logic that one fails all other fail should be done also here for k, v in self: wait_ready_threads.append( threading.Thread(target=_wait_ready, args=(k, v), daemon=True) @@ -1999,7 +1998,8 @@ async def _f(): for t in wait_ready_threads: t.start() - polling_status_thread.join() + if 'GITHUB_WORKFLOW' not in os.environ: + polling_status_thread.join() for t in wait_ready_threads: t.join() diff --git a/jina/serve/consensus/__init__.py b/jina/serve/consensus/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/jina/serve/consensus/add_voter/README.md b/jina/serve/consensus/add_voter/README.md new file mode 100644 index 0000000000000..4522160324e5b --- /dev/null +++ b/jina/serve/consensus/add_voter/README.md @@ -0,0 +1,42 @@ +#Build add_voter proto + +ina support two versions of protobuf, before 3.19 and after (which is a breaking change for python), therefore we have +duplicate python file generation from proto based on the installed protobuf version.) + +This complex setup lead to a not straightforward way to generate the python code from the proto + +this guide explain how to do it properly. + +# how to build the proto + +## 1. build docker image for protobuf generation + + +``` cmd +docker build -f Dockerfiles/protogen.Dockerfile -t jinaai/protogen:local . +``` + +This build the docker image that will be used to generate the python code from the proto for proto **before** 3.19 + +``` cmd +docker build -f Dockerfiles/protogen-3.21.Dockerfile -t jinaai/protogen-3.21:local . +``` + +This build the docker image that will be used to generate the python code from the proto for proto **after* 3.19 + +## 2. generate the python code from the proto + +note: you need to be in the root of the repo to do the following steps + + +``` cmd +docker run -it -v $(pwd)/jina/serve/consensus/add_voter:/jina/serve/consensus/add_voter --entrypoint=/bin/bash jinaai/protogen:local +cd /jina/serve/consensus/add_voter +bash build-add-voter-proto.sh /builder/grpc/cmake/build/grpc_python_plugin pb2 +``` + +``` +docker run -it -v $(pwd)/jina/serve/consensus/add_voter:/jina/serve/consensus/add_voter --entrypoint=/bin/bash jinaai/protogen-3.21:local +cd /jina/serve/consensus/add_voter +bash build-add-voter-proto.sh /builder/grpc/cmake/build/grpc_python_plugin pb +``` \ No newline at end of file diff --git a/jina/serve/consensus/add_voter/__init__.py b/jina/serve/consensus/add_voter/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/jina/serve/consensus/add_voter/add_voter.proto b/jina/serve/consensus/add_voter/add_voter.proto new file mode 100644 index 0000000000000..f62ce75ddff80 --- /dev/null +++ b/jina/serve/consensus/add_voter/add_voter.proto @@ -0,0 +1,26 @@ +syntax = "proto3"; + +service RaftAdmin { + rpc AddVoter(AddVoterRequest) returns (Future) {} + + rpc Await(Future) returns (AwaitResponse) {} + rpc Forget(Future) returns (ForgetResponse) {} +} + +message AwaitResponse { + string error = 1; + uint64 index = 2; +} + +message ForgetResponse { +} + +message Future { + string operation_token = 1; +} + +message AddVoterRequest { + string id = 1; + string address = 2; + uint64 previous_index = 3; +} \ No newline at end of file diff --git a/jina/serve/consensus/add_voter/add_voter_pb2.py b/jina/serve/consensus/add_voter/add_voter_pb2.py new file mode 100644 index 0000000000000..9dc096eeaffaa --- /dev/null +++ b/jina/serve/consensus/add_voter/add_voter_pb2.py @@ -0,0 +1,6 @@ +from google.protobuf import __version__ as __pb__version__ + +if __pb__version__.startswith('4'): + from jina.serve.consensus.add_voter.pb.add_voter_pb2 import * +else: + from jina.serve.consensus.add_voter.pb2.add_voter_pb2 import * diff --git a/jina/serve/consensus/add_voter/add_voter_pb2_grpc.py b/jina/serve/consensus/add_voter/add_voter_pb2_grpc.py new file mode 100644 index 0000000000000..96b2a9bdb11ca --- /dev/null +++ b/jina/serve/consensus/add_voter/add_voter_pb2_grpc.py @@ -0,0 +1,6 @@ +from google.protobuf import __version__ as __pb__version__ + +if __pb__version__.startswith('4'): + from jina.serve.consensus.add_voter.pb.add_voter_pb2_grpc import * +else: + from jina.serve.consensus.add_voter.pb2.add_voter_pb2_grpc import * diff --git a/jina/serve/consensus/add_voter/build-add-voter-proto.sh b/jina/serve/consensus/add_voter/build-add-voter-proto.sh new file mode 100644 index 0000000000000..388e4344cd5a7 --- /dev/null +++ b/jina/serve/consensus/add_voter/build-add-voter-proto.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -e + +# Do NOT use this directly, use jinaai/protogen image +# use jinaai/protogen:v21 in order to use compiler version == 21 (creates pb/docarray_pb2.py) +# and use jinaai/protogen:latest to use compiler version <= 20 (creates pb2/docarray_pb2.py) +# make sure to use jinaai/protogen:v21 to avoid overwriting the module +# +# current dir: jina root (the one with README.md) +# run the following in bash: +# docker run -v $(pwd)/jina/proto/docarray_v2:/jina/proto jinaai/protogen +# finally, set back owner of the generated files using: sudo chown -R $(id -u ${USER}):$(id -g ${USER}) ./jina/proto + +# The protogen docker image can also be build locally using: +# docker build -f Dockerfiles/protogen.Dockerfile -t jinaai/protogen:local . +# or +# docker build -f Dockerfiles/protogen-3.21.Dockerfile -t jinaai/protogen-3.21:local . + +SRC_DIR=./ +SRC_NAME="add_voter.proto" + +PB_NAME="${2:-pb}" +OUT_FOLDER="${PB_NAME}/" + +if [ "$#" -ne 1 ] && [ "$#" -ne 2 ]; then + echo "Error: Please specify the [PATH_TO_GRPC_PYTHON_PLUGIN], refer more details at " \ + "https://docs.jina.ai/" + printf "\n" + echo "USAGE:" + printf "\t" + echo "bash ./build-proto.sh [PATH_TO_GRPC_PYTHON_PLUGIN]" + exit 1 +fi + +PLUGIN_PATH=${1} # /Volumes/TOSHIBA-4T/Documents/grpc/bins/opt/grpc_python_plugin + +printf "\e[1;33mgenerating protobuf and grpc python interface\e[0m\n" + +protoc -I ${SRC_DIR} --python_out="${SRC_DIR}${OUT_FOLDER}" --grpc_python_out="${SRC_DIR}${OUT_FOLDER}" --plugin=protoc-gen-grpc_python=${PLUGIN_PATH} ${SRC_DIR}${SRC_NAME} +sed -i 's/import add_voter_pb2 as add__voter__pb2/from . import add_voter_pb2 as add__voter__pb2/' "${PB_NAME}/add_voter_pb2_grpc.py" +printf "\e[1;32mAll done!\e[0m\n" \ No newline at end of file diff --git a/jina/serve/consensus/add_voter/call_add_voter.py b/jina/serve/consensus/add_voter/call_add_voter.py new file mode 100644 index 0000000000000..06ab97d388509 --- /dev/null +++ b/jina/serve/consensus/add_voter/call_add_voter.py @@ -0,0 +1,47 @@ +import grpc +from jina.serve.consensus.add_voter.add_voter_pb2_grpc import RaftAdminStub +from jina.serve.consensus.add_voter.add_voter_pb2 import AddVoterRequest + + +def call_add_voter(target, replica_id, voter_address): + with grpc.insecure_channel(target) as channel: + stub = RaftAdminStub(channel) + + req = AddVoterRequest( + id=replica_id, + address=voter_address, + previous_index=0, + ) + + try: + future = stub.AddVoter(req) + add_voter_result = stub.Await(future) + _ = stub.Forget(future) + if not add_voter_result.error: + return True + else: + return False + except: + return False + + +async def async_call_add_voter(target, replica_id, voter_address): + async with grpc.aio.insecure_channel(target) as channel: + stub = RaftAdminStub(channel) + + req = AddVoterRequest( + id=replica_id, + address=voter_address, + previous_index=0, + ) + + try: + future = await stub.AddVoter(req) + add_voter_result = await stub.Await(future) + _ = await stub.Forget(future) + if not add_voter_result.error: + return True + else: + return False + except: + return False diff --git a/jina/serve/consensus/add_voter/pb/__init__.py b/jina/serve/consensus/add_voter/pb/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/jina/serve/consensus/add_voter/pb/add_voter_pb2.py b/jina/serve/consensus/add_voter/pb/add_voter_pb2.py new file mode 100644 index 0000000000000..ee5528f3fe13e --- /dev/null +++ b/jina/serve/consensus/add_voter/pb/add_voter_pb2.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: add_voter.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0f\x61\x64\x64_voter.proto\"-\n\rAwaitResponse\x12\r\n\x05\x65rror\x18\x01 \x01(\t\x12\r\n\x05index\x18\x02 \x01(\x04\"\x10\n\x0e\x46orgetResponse\"!\n\x06\x46uture\x12\x17\n\x0foperation_token\x18\x01 \x01(\t\"F\n\x0f\x41\x64\x64VoterRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0f\n\x07\x61\x64\x64ress\x18\x02 \x01(\t\x12\x16\n\x0eprevious_index\x18\x03 \x01(\x04\x32~\n\tRaftAdmin\x12\'\n\x08\x41\x64\x64Voter\x12\x10.AddVoterRequest\x1a\x07.Future\"\x00\x12\"\n\x05\x41wait\x12\x07.Future\x1a\x0e.AwaitResponse\"\x00\x12$\n\x06\x46orget\x12\x07.Future\x1a\x0f.ForgetResponse\"\x00\x62\x06proto3') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'add_voter_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _AWAITRESPONSE._serialized_start=19 + _AWAITRESPONSE._serialized_end=64 + _FORGETRESPONSE._serialized_start=66 + _FORGETRESPONSE._serialized_end=82 + _FUTURE._serialized_start=84 + _FUTURE._serialized_end=117 + _ADDVOTERREQUEST._serialized_start=119 + _ADDVOTERREQUEST._serialized_end=189 + _RAFTADMIN._serialized_start=191 + _RAFTADMIN._serialized_end=317 +# @@protoc_insertion_point(module_scope) diff --git a/jina/serve/consensus/add_voter/pb/add_voter_pb2_grpc.py b/jina/serve/consensus/add_voter/pb/add_voter_pb2_grpc.py new file mode 100644 index 0000000000000..e59cf3319fefd --- /dev/null +++ b/jina/serve/consensus/add_voter/pb/add_voter_pb2_grpc.py @@ -0,0 +1,132 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +from . import add_voter_pb2 as add__voter__pb2 + + +class RaftAdminStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.AddVoter = channel.unary_unary( + '/RaftAdmin/AddVoter', + request_serializer=add__voter__pb2.AddVoterRequest.SerializeToString, + response_deserializer=add__voter__pb2.Future.FromString, + ) + self.Await = channel.unary_unary( + '/RaftAdmin/Await', + request_serializer=add__voter__pb2.Future.SerializeToString, + response_deserializer=add__voter__pb2.AwaitResponse.FromString, + ) + self.Forget = channel.unary_unary( + '/RaftAdmin/Forget', + request_serializer=add__voter__pb2.Future.SerializeToString, + response_deserializer=add__voter__pb2.ForgetResponse.FromString, + ) + + +class RaftAdminServicer(object): + """Missing associated documentation comment in .proto file.""" + + def AddVoter(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Await(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Forget(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_RaftAdminServicer_to_server(servicer, server): + rpc_method_handlers = { + 'AddVoter': grpc.unary_unary_rpc_method_handler( + servicer.AddVoter, + request_deserializer=add__voter__pb2.AddVoterRequest.FromString, + response_serializer=add__voter__pb2.Future.SerializeToString, + ), + 'Await': grpc.unary_unary_rpc_method_handler( + servicer.Await, + request_deserializer=add__voter__pb2.Future.FromString, + response_serializer=add__voter__pb2.AwaitResponse.SerializeToString, + ), + 'Forget': grpc.unary_unary_rpc_method_handler( + servicer.Forget, + request_deserializer=add__voter__pb2.Future.FromString, + response_serializer=add__voter__pb2.ForgetResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'RaftAdmin', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class RaftAdmin(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def AddVoter(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/RaftAdmin/AddVoter', + add__voter__pb2.AddVoterRequest.SerializeToString, + add__voter__pb2.Future.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Await(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/RaftAdmin/Await', + add__voter__pb2.Future.SerializeToString, + add__voter__pb2.AwaitResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Forget(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/RaftAdmin/Forget', + add__voter__pb2.Future.SerializeToString, + add__voter__pb2.ForgetResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/jina/serve/consensus/add_voter/pb2/__init__.py b/jina/serve/consensus/add_voter/pb2/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/jina/serve/consensus/add_voter/pb2/add_voter_pb2.py b/jina/serve/consensus/add_voter/pb2/add_voter_pb2.py new file mode 100644 index 0000000000000..7ec7594f2afa7 --- /dev/null +++ b/jina/serve/consensus/add_voter/pb2/add_voter_pb2.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: add_voter.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0f\x61\x64\x64_voter.proto\"-\n\rAwaitResponse\x12\r\n\x05\x65rror\x18\x01 \x01(\t\x12\r\n\x05index\x18\x02 \x01(\x04\"\x10\n\x0e\x46orgetResponse\"!\n\x06\x46uture\x12\x17\n\x0foperation_token\x18\x01 \x01(\t\"F\n\x0f\x41\x64\x64VoterRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0f\n\x07\x61\x64\x64ress\x18\x02 \x01(\t\x12\x16\n\x0eprevious_index\x18\x03 \x01(\x04\x32~\n\tRaftAdmin\x12\'\n\x08\x41\x64\x64Voter\x12\x10.AddVoterRequest\x1a\x07.Future\"\x00\x12\"\n\x05\x41wait\x12\x07.Future\x1a\x0e.AwaitResponse\"\x00\x12$\n\x06\x46orget\x12\x07.Future\x1a\x0f.ForgetResponse\"\x00\x62\x06proto3') + + + +_AWAITRESPONSE = DESCRIPTOR.message_types_by_name['AwaitResponse'] +_FORGETRESPONSE = DESCRIPTOR.message_types_by_name['ForgetResponse'] +_FUTURE = DESCRIPTOR.message_types_by_name['Future'] +_ADDVOTERREQUEST = DESCRIPTOR.message_types_by_name['AddVoterRequest'] +AwaitResponse = _reflection.GeneratedProtocolMessageType('AwaitResponse', (_message.Message,), { + 'DESCRIPTOR' : _AWAITRESPONSE, + '__module__' : 'add_voter_pb2' + # @@protoc_insertion_point(class_scope:AwaitResponse) + }) +_sym_db.RegisterMessage(AwaitResponse) + +ForgetResponse = _reflection.GeneratedProtocolMessageType('ForgetResponse', (_message.Message,), { + 'DESCRIPTOR' : _FORGETRESPONSE, + '__module__' : 'add_voter_pb2' + # @@protoc_insertion_point(class_scope:ForgetResponse) + }) +_sym_db.RegisterMessage(ForgetResponse) + +Future = _reflection.GeneratedProtocolMessageType('Future', (_message.Message,), { + 'DESCRIPTOR' : _FUTURE, + '__module__' : 'add_voter_pb2' + # @@protoc_insertion_point(class_scope:Future) + }) +_sym_db.RegisterMessage(Future) + +AddVoterRequest = _reflection.GeneratedProtocolMessageType('AddVoterRequest', (_message.Message,), { + 'DESCRIPTOR' : _ADDVOTERREQUEST, + '__module__' : 'add_voter_pb2' + # @@protoc_insertion_point(class_scope:AddVoterRequest) + }) +_sym_db.RegisterMessage(AddVoterRequest) + +_RAFTADMIN = DESCRIPTOR.services_by_name['RaftAdmin'] +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _AWAITRESPONSE._serialized_start=19 + _AWAITRESPONSE._serialized_end=64 + _FORGETRESPONSE._serialized_start=66 + _FORGETRESPONSE._serialized_end=82 + _FUTURE._serialized_start=84 + _FUTURE._serialized_end=117 + _ADDVOTERREQUEST._serialized_start=119 + _ADDVOTERREQUEST._serialized_end=189 + _RAFTADMIN._serialized_start=191 + _RAFTADMIN._serialized_end=317 +# @@protoc_insertion_point(module_scope) diff --git a/jina/serve/consensus/add_voter/pb2/add_voter_pb2_grpc.py b/jina/serve/consensus/add_voter/pb2/add_voter_pb2_grpc.py new file mode 100644 index 0000000000000..e59cf3319fefd --- /dev/null +++ b/jina/serve/consensus/add_voter/pb2/add_voter_pb2_grpc.py @@ -0,0 +1,132 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +from . import add_voter_pb2 as add__voter__pb2 + + +class RaftAdminStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.AddVoter = channel.unary_unary( + '/RaftAdmin/AddVoter', + request_serializer=add__voter__pb2.AddVoterRequest.SerializeToString, + response_deserializer=add__voter__pb2.Future.FromString, + ) + self.Await = channel.unary_unary( + '/RaftAdmin/Await', + request_serializer=add__voter__pb2.Future.SerializeToString, + response_deserializer=add__voter__pb2.AwaitResponse.FromString, + ) + self.Forget = channel.unary_unary( + '/RaftAdmin/Forget', + request_serializer=add__voter__pb2.Future.SerializeToString, + response_deserializer=add__voter__pb2.ForgetResponse.FromString, + ) + + +class RaftAdminServicer(object): + """Missing associated documentation comment in .proto file.""" + + def AddVoter(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Await(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Forget(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_RaftAdminServicer_to_server(servicer, server): + rpc_method_handlers = { + 'AddVoter': grpc.unary_unary_rpc_method_handler( + servicer.AddVoter, + request_deserializer=add__voter__pb2.AddVoterRequest.FromString, + response_serializer=add__voter__pb2.Future.SerializeToString, + ), + 'Await': grpc.unary_unary_rpc_method_handler( + servicer.Await, + request_deserializer=add__voter__pb2.Future.FromString, + response_serializer=add__voter__pb2.AwaitResponse.SerializeToString, + ), + 'Forget': grpc.unary_unary_rpc_method_handler( + servicer.Forget, + request_deserializer=add__voter__pb2.Future.FromString, + response_serializer=add__voter__pb2.ForgetResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'RaftAdmin', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class RaftAdmin(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def AddVoter(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/RaftAdmin/AddVoter', + add__voter__pb2.AddVoterRequest.SerializeToString, + add__voter__pb2.Future.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Await(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/RaftAdmin/Await', + add__voter__pb2.Future.SerializeToString, + add__voter__pb2.AwaitResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Forget(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/RaftAdmin/Forget', + add__voter__pb2.Future.SerializeToString, + add__voter__pb2.ForgetResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/jina/serve/consensus/go.mod b/jina/serve/consensus/go.mod index 9fc388afeb117..fc854188ac592 100644 --- a/jina/serve/consensus/go.mod +++ b/jina/serve/consensus/go.mod @@ -6,12 +6,12 @@ require ( github.com/Jille/raft-grpc-leader-rpc v1.1.0 github.com/Jille/raft-grpc-transport v1.1.1 github.com/Jille/raftadmin v1.2.0 - github.com/golang/protobuf v1.5.2 + github.com/golang/protobuf v1.5.3 github.com/hashicorp/go-hclog v0.16.2 github.com/hashicorp/raft v1.3.11 github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0 - google.golang.org/grpc v1.47.0 - google.golang.org/protobuf v1.27.1 + google.golang.org/grpc v1.56.3 + google.golang.org/protobuf v1.30.0 ) require ( @@ -26,5 +26,5 @@ require ( golang.org/x/net v0.17.0 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect - google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83 // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect ) diff --git a/jina/serve/consensus/go.sum b/jina/serve/consensus/go.sum index d777ff6673a89..79726ae0112fd 100644 --- a/jina/serve/consensus/go.sum +++ b/jina/serve/consensus/go.sum @@ -1,5 +1,4 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= @@ -9,12 +8,10 @@ github.com/Jille/raft-grpc-transport v1.1.1 h1:VvFzYU3GuuEtehtBbgfubJnoPNBDHix0D github.com/Jille/raft-grpc-transport v1.1.1/go.mod h1:NG6sOvCFk8j7t93TTfJ40w5it4cbmB1kw0MjoEWzs6o= github.com/Jille/raftadmin v1.2.0 h1:hMLFUK7iKpeXP+CoIhNMWj+F53XOLSjMDSia0C60cps= github.com/Jille/raftadmin v1.2.0/go.mod h1:vtVEpToPGTUPVwwunypWDpi69JpdnHMhWRUlc/65U+Y= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= github.com/armon/go-metrics v0.3.8/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-metrics v0.3.9 h1:O2sNqxBdvq8Eq5xmzljcYzAORli6RWCvEym4cJf9m18= @@ -25,32 +22,21 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc= github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -69,24 +55,19 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.16.2 h1:K4ev2ib4LdQETX5cSZBG0DVLk1jwGqSPXBjdah3veNs= @@ -152,135 +133,91 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83 h1:3V2dxSZpz4zozWWUq36vUxXEKnSYitEH2LdsAx+RUmg= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8= -google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= +google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/jina/serve/consensus/jina_raft/fsm.go b/jina/serve/consensus/jina_raft/fsm.go index 9907cb18401d1..ea753f9aade1b 100644 --- a/jina/serve/consensus/jina_raft/fsm.go +++ b/jina/serve/consensus/jina_raft/fsm.go @@ -32,9 +32,9 @@ type executorFSM struct { } -func NewExecutorFSM(target string, LogLevel string, raftID string) *executorFSM { +func NewExecutorFSM(target string, LogLevel string, name string, raftID string) *executorFSM { fsm_logger := hclog.New(&hclog.LoggerOptions{ - Name: "executorFSM-" + raftID, + Name: "FSM-" + name, Level: hclog.LevelFromString(LogLevel), }) executor := &executor{ diff --git a/jina/serve/consensus/jraft.go b/jina/serve/consensus/jraft.go index a3640f108262b..b15e43ec7b5aa 100644 --- a/jina/serve/consensus/jraft.go +++ b/jina/serve/consensus/jraft.go @@ -8,9 +8,9 @@ package main // Workaround missing variadic function support // https://github.com/golang/go/issues/975 -int PyArg_ParseTuple_run(PyObject * args, PyObject * kwargs, char **myAddr, char **raftId, char **raftDir, char **executorTarget, int *HeartbeatTimeout, int *ElectionTimeout, int *CommitTimeout, int *MaxAppendEntries, bool *BatchApplyCh, bool *ShutdownOnRemove, uint64_t *TrailingLogs, int *snapshotInterval, uint64_t *SnapshotThreshold, int *LeaderLeaseTimeout, char **LogLevel, bool *NoSnapshotRestoreOnStart) { - static char *kwlist[] = {"myAddr", "raftId", "raftDir", "executorTarget", "HeartbeatTimeout", "ElectionTimeout", "CommitTimeout", "MaxAppendEntries", "BatchApplyCh", "ShutdownOnRemove", "TrailingLogs", "SnapshotInterval", "SnapshotThreshold", "LeaderLeaseTimeout", "LogLevel", "NoSnapshotRestoreOnStart", NULL}; - return PyArg_ParseTupleAndKeywords(args, kwargs, "ssss|llllppklklsp", kwlist, myAddr, raftId, raftDir, executorTarget, HeartbeatTimeout, ElectionTimeout, CommitTimeout, MaxAppendEntries, BatchApplyCh, ShutdownOnRemove, TrailingLogs, snapshotInterval, SnapshotThreshold, LeaderLeaseTimeout, LogLevel, NoSnapshotRestoreOnStart); +int PyArg_ParseTuple_run(PyObject * args, PyObject * kwargs, char **myAddr, char **raftId, char **raftDir, char **name, char **executorTarget, int *HeartbeatTimeout, int *ElectionTimeout, int *CommitTimeout, int *MaxAppendEntries, bool *BatchApplyCh, bool *ShutdownOnRemove, uint64_t *TrailingLogs, int *snapshotInterval, uint64_t *SnapshotThreshold, int *LeaderLeaseTimeout, char **LogLevel, bool *NoSnapshotRestoreOnStart) { + static char *kwlist[] = {"myAddr", "raftId", "raftDir", "name", "executorTarget", "HeartbeatTimeout", "ElectionTimeout", "CommitTimeout", "MaxAppendEntries", "BatchApplyCh", "ShutdownOnRemove", "TrailingLogs", "SnapshotInterval", "SnapshotThreshold", "LeaderLeaseTimeout", "LogLevel", "NoSnapshotRestoreOnStart", NULL}; + return PyArg_ParseTupleAndKeywords(args, kwargs, "sssss|llllppklklsp", kwlist, myAddr, raftId, raftDir, name, executorTarget, HeartbeatTimeout, ElectionTimeout, CommitTimeout, MaxAppendEntries, BatchApplyCh, ShutdownOnRemove, TrailingLogs, snapshotInterval, SnapshotThreshold, LeaderLeaseTimeout, LogLevel, NoSnapshotRestoreOnStart); } int PyArg_ParseTuple_add_voter(PyObject * args, char **a, char **b, char **c) { diff --git a/jina/serve/consensus/jraft.h b/jina/serve/consensus/jraft.h index 06753e107f9af..d9225c6ac770f 100644 --- a/jina/serve/consensus/jraft.h +++ b/jina/serve/consensus/jraft.h @@ -5,7 +5,7 @@ #line 1 "cgo-builtin-export-prolog" -#include /* for ptrdiff_t below */ +#include #ifndef GO_CGO_EXPORT_PROLOGUE_H #define GO_CGO_EXPORT_PROLOGUE_H @@ -22,7 +22,7 @@ typedef struct { const char *p; ptrdiff_t n; } _GoString_; #line 3 "run.go" #include #include - int PyArg_ParseTuple_run(PyObject * args, PyObject * kwargs, char **myAddr, char **raftId, char **raftDir, char **executorTarget, int *HeartbeatTimeout, int *ElectionTimeout, int *CommitTimeout, int *MaxAppendEntries, bool *BatchApplyCh, bool *ShutdownOnRemove, uint64_t *TrailingLogs, int *snapshotInterval, uint64_t *SnapshotThreshold, int *LeaderLeaseTimeout, char **LogLevel, bool *NoSnapshotRestoreOnStart); + int PyArg_ParseTuple_run(PyObject * args, PyObject * kwargs, char **myAddr, char **raftId, char **raftDir, char **name, char **executorTarget, int *HeartbeatTimeout, int *ElectionTimeout, int *CommitTimeout, int *MaxAppendEntries, bool *BatchApplyCh, bool *ShutdownOnRemove, uint64_t *TrailingLogs, int *snapshotInterval, uint64_t *SnapshotThreshold, int *LeaderLeaseTimeout, char **LogLevel, bool *NoSnapshotRestoreOnStart); int PyArg_ParseTuple_add_voter(PyObject * args, char **a, char **b, char **c); int PyArg_ParseTuple_get_configuration(PyObject * args, char **a, char **b); void raise_exception(char *msg); @@ -49,11 +49,17 @@ typedef long long GoInt64; typedef unsigned long long GoUint64; typedef GoInt64 GoInt; typedef GoUint64 GoUint; -typedef __SIZE_TYPE__ GoUintptr; +typedef size_t GoUintptr; typedef float GoFloat32; typedef double GoFloat64; +#ifdef _MSC_VER +#include +typedef _Fcomplex GoComplex64; +typedef _Dcomplex GoComplex128; +#else typedef float _Complex GoComplex64; typedef double _Complex GoComplex128; +#endif /* static assertion to make sure the file is being used on architecture diff --git a/jina/serve/consensus/run.go b/jina/serve/consensus/run.go index e14db997a0c15..b655b3b5373f7 100644 --- a/jina/serve/consensus/run.go +++ b/jina/serve/consensus/run.go @@ -2,7 +2,7 @@ package main // #include // #include -// int PyArg_ParseTuple_run(PyObject * args, PyObject * kwargs, char **myAddr, char **raftId, char **raftDir, char **executorTarget, int *HeartbeatTimeout, int *ElectionTimeout, int *CommitTimeout, int *MaxAppendEntries, bool *BatchApplyCh, bool *ShutdownOnRemove, uint64_t *TrailingLogs, int *snapshotInterval, uint64_t *SnapshotThreshold, int *LeaderLeaseTimeout, char **LogLevel, bool *NoSnapshotRestoreOnStart); +// int PyArg_ParseTuple_run(PyObject * args, PyObject * kwargs, char **myAddr, char **raftId, char **raftDir, char **name, char **executorTarget, int *HeartbeatTimeout, int *ElectionTimeout, int *CommitTimeout, int *MaxAppendEntries, bool *BatchApplyCh, bool *ShutdownOnRemove, uint64_t *TrailingLogs, int *snapshotInterval, uint64_t *SnapshotThreshold, int *LeaderLeaseTimeout, char **LogLevel, bool *NoSnapshotRestoreOnStart); // int PyArg_ParseTuple_add_voter(PyObject * args, char **a, char **b, char **c); // int PyArg_ParseTuple_get_configuration(PyObject * args, char **a, char **b); // void raise_exception(char *msg); @@ -35,6 +35,7 @@ import ( ) func NewRaft(ctx context.Context, + name string, myID string, myAddress string, raftDir string, @@ -66,7 +67,7 @@ func NewRaft(ctx context.Context, config.LogLevel = LogLevel config.NoSnapshotRestoreOnStart = NoSnapshotRestoreOnStart config.Logger = hclog.New(&hclog.LoggerOptions{ - Name: "raft-" + myID, + Name: "RAFT-" + name, Level: hclog.LevelFromString(LogLevel), }) @@ -120,6 +121,7 @@ func NewRaft(ctx context.Context, func Run(myAddr string, raftId string, raftDir string, + name string, executorTarget string, HeartbeatTimeout int, ElectionTimeout int, @@ -134,7 +136,7 @@ func Run(myAddr string, LogLevel string, NoSnapshotRestoreOnStart bool) { run_logger := hclog.New(&hclog.LoggerOptions{ - Name: "run_raft-" + raftId, + Name: "RAFT-" + name, Level: hclog.LevelFromString(LogLevel), }) if raftId == "" { @@ -155,9 +157,10 @@ func Run(myAddr string, } defer sock.Close() - executorFSM := jinaraft.NewExecutorFSM(executorTarget, LogLevel, raftId) + executorFSM := jinaraft.NewExecutorFSM(executorTarget, LogLevel, name, raftId) r, tm, err := NewRaft(ctx, + name, raftId, myAddr, raftDir, @@ -180,37 +183,23 @@ func Run(myAddr string, } grpcServer := grpc.NewServer() rpc_logger := hclog.New(&hclog.LoggerOptions{ - Name: "rpc-" + raftId, + Name: "RPC-" + name, Level: hclog.LevelFromString(LogLevel), }) - pb.RegisterJinaSingleDataRequestRPCServer(grpcServer, &jinaraft.RpcInterface{ - Executor: executorFSM, - Raft: r, - Logger: rpc_logger, - }) - pb.RegisterJinaDiscoverEndpointsRPCServer(grpcServer, &jinaraft.RpcInterface{ - Executor: executorFSM, - Raft: r, - Logger: rpc_logger, - }) - pb.RegisterJinaInfoRPCServer(grpcServer, &jinaraft.RpcInterface{ - Executor: executorFSM, - Raft: r, - Logger: rpc_logger, - }) - pb.RegisterJinaRPCServer(grpcServer, &jinaraft.RpcInterface{ - Executor: executorFSM, - Raft: r, - Logger: rpc_logger, - }) + rpc_interface := jinaraft.RpcInterface{ + Executor: executorFSM, + Raft: r, + Logger: rpc_logger, + } + + pb.RegisterJinaSingleDataRequestRPCServer(grpcServer, &rpc_interface) + pb.RegisterJinaDiscoverEndpointsRPCServer(grpcServer, &rpc_interface) + pb.RegisterJinaInfoRPCServer(grpcServer, &rpc_interface) + pb.RegisterJinaRPCServer(grpcServer, &rpc_interface) tm.Register(grpcServer) - healthpb.RegisterHealthServer(grpcServer, &jinaraft.RpcInterface{ - Executor: executorFSM, - Raft: r, - Logger: rpc_logger, - }) + healthpb.RegisterHealthServer(grpcServer, &rpc_interface) raftadmin.Register(grpcServer, r) reflection.Register(grpcServer) @@ -221,7 +210,9 @@ func Run(myAddr string, run_logger.Info("Received", "signal", sig) run_logger.Info("gRPCServer stopping") grpcServer.GracefulStop() + run_logger.Info("gRPCServer stopped, close socket") sock.Close() + run_logger.Info("Socket closed") run_logger.Info("call RAFT shutdown") shutdownResultFuture := r.Shutdown() err := shutdownResultFuture.Error() @@ -249,6 +240,7 @@ func findServerByID(servers []raft.Server, id raft.ServerID) *raft.Server { func main() { raftDefaultConfig := raft.DefaultConfig() + name := flag.String("name", "executor", "name to identify in the logger the Node") myAddr := flag.String("address", "localhost:50051", "TCP host+port for this node") raftId := flag.String("raft_id", "", "Node id used by Raft") raftDir := flag.String("raft_data_dir", "data/", "Raft data dir") @@ -269,6 +261,7 @@ func main() { Run(*myAddr, *raftId, *raftDir, + *name, *executorTarget, *HeartbeatTimeout, *ElectionTimeout, @@ -290,6 +283,7 @@ func run(self *C.PyObject, args *C.PyObject, kwargs *C.PyObject) *C.PyObject { var myAddr *C.char var raftId *C.char var raftDir *C.char + var name *C.char var executorTarget *C.char var HeartbeatTimeout C.int var ElectionTimeout C.int @@ -325,6 +319,7 @@ func run(self *C.PyObject, args *C.PyObject, kwargs *C.PyObject) *C.PyObject { &myAddr, &raftId, &raftDir, + &name, &executorTarget, &HeartbeatTimeout, &ElectionTimeout, @@ -341,6 +336,7 @@ func run(self *C.PyObject, args *C.PyObject, kwargs *C.PyObject) *C.PyObject { Run(C.GoString(myAddr), C.GoString(raftId), C.GoString(raftDir), + C.GoString(name), C.GoString(executorTarget), int(HeartbeatTimeout), int(ElectionTimeout), diff --git a/jina/serve/executors/run.py b/jina/serve/executors/run.py index 077e487352d4e..6901982e6548b 100644 --- a/jina/serve/executors/run.py +++ b/jina/serve/executors/run.py @@ -48,7 +48,7 @@ def pascal_case_dict(d): executor_target = f'{args.host}:{port + RAFT_TO_EXECUTOR_PORT}' # if the Executor was already persisted, retrieve its port and host configuration - logger = JinaLogger('run_raft', **vars(args)) + logger = JinaLogger(context=f'RAFT-{args.name}', **vars(args)) persisted_address = jraft.get_configuration(raft_id, raft_dir) if persisted_address: logger.debug(f'Configuration found on the node: Address {persisted_address}') @@ -65,6 +65,7 @@ def pascal_case_dict(d): address, raft_id, raft_dir, + args.name, executor_target, **raft_configuration, ) diff --git a/jina/serve/runtimes/worker/request_handling.py b/jina/serve/runtimes/worker/request_handling.py index 7e954efd53186..0849aaebb388d 100644 --- a/jina/serve/runtimes/worker/request_handling.py +++ b/jina/serve/runtimes/worker/request_handling.py @@ -1262,6 +1262,7 @@ async def restore(self, request: 'jina_pb2.RestoreSnapshotCommand', context): else: self._restore = self._create_restore_status() self._did_restore_raise_exception = threading.Event() + self._restore_thread = threading.Thread( target=self._executor._run_restore, args=(request.snapshot_file, self._did_restore_raise_exception), diff --git a/scripts/docstrings_lint.sh b/scripts/docstrings_lint.sh index 5e46148a317bd..f6b6f890b7ad7 100755 --- a/scripts/docstrings_lint.sh +++ b/scripts/docstrings_lint.sh @@ -16,6 +16,7 @@ for changed_file in $CHANGED_FILES; do jina/helloworld/* | \ jina/proto/* | \ jina/resources/* | \ + jina/serve/consensus/add_voter/* | \ docs/* | \ setup.py | \ fastentrypoints.py) diff --git a/setup.py b/setup.py index e8f8dd3715ce7..51083076d0e59 100644 --- a/setup.py +++ b/setup.py @@ -160,15 +160,6 @@ def get_extra_requires(path, add_all=True): elif os.environ.get('JINA_PIP_INSTALL_PERF'): final_deps = perf_deps -if sys.version_info.major == 3 and sys.version_info.minor >= 11: - for dep in list(final_deps): - if dep.startswith('grpcio'): - final_deps.remove(dep) - final_deps.add('grpcio>=1.49.0') - final_deps.add('grpcio-health-checking>=1.49.0') - final_deps.add('grpcio-reflection>=1.49.0') - - extra_golang_kw = {} ret_code = -1 diff --git a/tests/integration/stateful/test_stateful.py b/tests/integration/stateful/test_stateful.py index 432f459e7975c..73b575414e561 100644 --- a/tests/integration/stateful/test_stateful.py +++ b/tests/integration/stateful/test_stateful.py @@ -112,6 +112,7 @@ def test_stateful_index_search(executor_cls, shards, tmpdir, stateful_exec_docke for doc in docs: assert doc.text == 'similarity' assert len(doc.l) == len(index_da) # good merging of results + time.sleep(10) @pytest.mark.timeout(240) @@ -152,19 +153,37 @@ def test_stateful_index_search_restore(executor_cls, shards, tmpdir, stateful_ex # checking against the main read replica assert_is_indexed(dep, search_da) assert_all_replicas_indexed(dep, search_da) + time.sleep(10) # test restoring - with dep: + dep_restore = Deployment( + uses=executor_cls, + replicas=replicas, + workspace=tmpdir, + stateful=True, + raft_configuration={ + 'snapshot_interval': 10, + 'snapshot_threshold': 5, + 'trailing_logs': 10, + 'LogLevel': 'INFO', + }, + shards=shards, + volumes=[str(tmpdir) + ':' + '/workspace'], + peer_ports=peer_ports, + polling={'/index': 'ANY', '/search': 'ALL', '/similarity': 'ALL'} + ) + with dep_restore: index_da = DocumentArray[TextDocWithId]( [TextDocWithId(id=f'{i}', text=f'ID {i}') for i in range(100, 200)] ) - dep.index(inputs=index_da, request_size=1, return_type=DocumentArray[TextDocWithId]) + dep_restore.index(inputs=index_da, request_size=1, return_type=DocumentArray[TextDocWithId]) time.sleep(20) search_da = DocumentArray[TextDocWithId]([TextDocWithId(id=f'{i}') for i in range(200)]) - assert_all_replicas_indexed(dep, search_da) + assert_all_replicas_indexed(dep_restore, search_da) + time.sleep(10) -@pytest.mark.skip('Not sure how containerization will work with docarray v2') +@pytest.mark.skipif(not docarray_v2, reason='tests support for docarray>=0.30') @pytest.mark.parametrize('shards', [1, 2]) def test_stateful_index_search_container(shards, tmpdir, stateful_exec_docker_image_built): replicas = 3 @@ -196,34 +215,57 @@ def test_stateful_index_search_container(shards, tmpdir, stateful_exec_docker_im dep.index(inputs=index_da, request_size=1, return_type=DocumentArray[TextDocWithId]) # allowing some time for the state to be replicated - time.sleep(10) + time.sleep(20) # checking against the main read replica assert_is_indexed(dep, search_da) - assert_all_replicas_indexed(dep, search_da, key='random_num') + assert_all_replicas_indexed(dep, search_da, key='num') + time.sleep(10) + dep_restore = Deployment( + uses='docker://stateful-exec', + replicas=replicas, + stateful=True, + raft_configuration={ + 'snapshot_interval': 10, + 'snapshot_threshold': 5, + 'trailing_logs': 10, + 'LogLevel': 'INFO', + }, + shards=shards, + workspace='/workspace/tmp', + volumes=[str(tmpdir) + ':' + '/workspace/tmp'], + peer_ports=peer_ports, + polling={'/index': 'ANY', '/search': 'ALL', '/similarity': 'ALL'} + ) # test restoring - with dep: + with dep_restore: index_da = DocumentArray[TextDocWithId]( - [Document(id=f'{i}', text=f'ID {i}') for i in range(100, 200)] + [TextDocWithId(id=f'{i}', text=f'ID {i}') for i in range(100, 200)] ) - dep.index(inputs=index_da, request_size=1, return_type=DocumentArray[TextDocWithId]) - time.sleep(10) + dep_restore.index(inputs=index_da, request_size=1, return_type=DocumentArray[TextDocWithId]) + time.sleep(20) search_da = DocumentArray[TextDocWithId]([TextDocWithId(id=f'{i}') for i in range(200)]) - assert_all_replicas_indexed(dep, search_da, key='random_num') + assert_all_replicas_indexed(dep_restore, search_da, key='num') + time.sleep(10) -@pytest.mark.skip() -@pytest.mark.parametrize('executor_cls', [MyStateExecutor, MyStateExecutorNoSnapshot]) +@pytest.mark.skipif(not docarray_v2, reason='tests support for docarray>=0.30') +@pytest.mark.parametrize('executor_cls', [MyStateExecutor]) def test_add_new_replica(executor_cls, tmpdir): from jina.parsers import set_pod_parser from jina.orchestrate.pods.factory import PodFactory gateway_port = random_port() + replicas = 3 + peer_ports = {} + for shard in range(1): + peer_ports[shard] = [random_port() for _ in range(replicas)] ctx_mngr = Flow(port=gateway_port).add( uses=executor_cls, - replicas=3, + replicas=replicas, workspace=tmpdir, stateful=True, + peer_ports=peer_ports, raft_configuration={ 'snapshot_interval': 10, 'snapshot_threshold': 5, @@ -233,7 +275,7 @@ def test_add_new_replica(executor_cls, tmpdir): ) with ctx_mngr: index_da = DocumentArray[TextDocWithId]( - [Document(id=f'{i}', text=f'ID {i}') for i in range(100)] + [TextDocWithId(id=f'{i}', text=f'ID {i}') for i in range(100)] ) ctx_mngr.index(inputs=index_da, request_size=1) # allowing sometime for snapshots @@ -241,38 +283,34 @@ def test_add_new_replica(executor_cls, tmpdir): new_replica_port = random_port() args = set_pod_parser().parse_args([]) + args.name = 'new-replica' args.host = args.host[0] args.port = [new_replica_port] args.stateful = True args.workspace = str(tmpdir) args.uses = executor_cls.__name__ - args.replica_id = '4' - with PodFactory.build_pod(args) as p: - import psutil - current_pid = os.getpid() - ports = set() - for proc in psutil.process_iter(['pid', 'ppid', 'name']): - if proc.info['ppid'] == current_pid and proc.info['pid'] != current_pid: - for conn in proc.connections(): - if conn.status == 'LISTEN': - ports.add(conn.laddr.port) - for port in ports: - try: - leader_address = f'0.0.0.0:{port}' # detect the Pods addresses of the original Flow - voter_address = f'0.0.0.0:{new_replica_port}' - import jraft - jraft.add_voter( - leader_address, '4', voter_address - ) + args.replica_id = str(replicas + 1) + with PodFactory.build_pod(args): + for port in peer_ports[0]: + leader_address = f'127.0.0.1:{port}' # detect the Pods addresses of the original Flow + voter_address = f'127.0.0.1:{new_replica_port}' + + from jina.serve.consensus.add_voter.call_add_voter import call_add_voter + + ret = call_add_voter(leader_address, '4', voter_address) + if ret is True: break - except: - pass + time.sleep(10) + + index_da = DocumentArray[TextDocWithId]( [TextDocWithId(id=f'{i}', text=f'ID {i}') for i in range(100, 200)] ) ctx_mngr.index(inputs=index_da, request_size=1, return_type=DocumentArray[TextDocWithId]) - time.sleep(10) + + time.sleep(20) search_da = DocumentArray[TextDocWithId]([TextDocWithId(id=f'{i}') for i in range(200)]) client = Client(port=new_replica_port) assert_is_indexed(client, search_da=search_da) + time.sleep(10)