From 6ef78522f26fd55898b9acf605f0662eea7bbfe8 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Wed, 24 Apr 2024 03:03:10 +0400 Subject: [PATCH] chore: Install python dependencies with uv in workflows (#4086) * install dependencies in unit-tests with uv Signed-off-by: tokoko * install dependencies in unit-tests with uv Signed-off-by: tokoko * enable caching, change linter job Signed-off-by: tokoko * change local integration tests to uv Signed-off-by: tokoko * change all installs to uv Signed-off-by: tokoko * try adding uv cache Signed-off-by: tokoko * fix lambda cache step name Signed-off-by: tokoko * reenable caches for uv Signed-off-by: tokoko * remove dangling cache step Signed-off-by: tokoko --------- Signed-off-by: tokoko --- .github/workflows/linter.yml | 22 ++--------- .github/workflows/master_only.yml | 31 +++++++--------- .github/workflows/nightly-ci.yml | 37 ++++++++----------- .github/workflows/pr_integration_tests.yml | 31 +++++++--------- .../workflows/pr_local_integration_tests.yml | 31 +++++++--------- .github/workflows/unit_tests.yml | 32 +++++++--------- Makefile | 5 +++ 7 files changed, 78 insertions(+), 111 deletions(-) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index a1747db135..f235b6f3ba 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -11,31 +11,17 @@ jobs: - uses: actions/checkout@v3 - name: Setup Python id: setup-python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: "3.9" architecture: x64 - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache - uses: actions/cache@v2 - with: - path: | - ${{ steps.pip-cache.outputs.dir }} - /opt/hostedtoolcache/Python - /Users/runner/hostedtoolcache/Python - key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - restore-keys: | - ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Upgrade pip version run: | pip install --upgrade "pip>=21.3.1,<23.2" - - name: Install pip-tools - run: pip install pip-tools + - name: Install uv + run: pip install uv - name: Install dependencies run: | - make install-python-ci-dependencies + make install-python-ci-dependencies-uv - name: Lint python run: make lint-python diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 225f24a828..c355c55c23 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -81,7 +81,7 @@ jobs: --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v5 - name: Setup Python id: setup-python uses: actions/setup-python@v3 @@ -106,27 +106,22 @@ jobs: aws-region: us-west-2 - name: Use AWS CLI run: aws sts get-caller-identity - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache - uses: actions/cache@v2 - with: - path: | - ${{ steps.pip-cache.outputs.dir }} - /opt/hostedtoolcache/Python - /Users/runner/hostedtoolcache/Python - key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - restore-keys: | - ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Upgrade pip version run: | pip install --upgrade "pip>=21.3.1,<23.2" - - name: Install pip-tools - run: pip install pip-tools + - name: Install uv + run: pip install uv + - name: Get uv cache dir + id: uv-cache + run: | + echo "::set-output name=dir::$(uv cache dir)" + - name: uv cache + uses: actions/cache@v4 + with: + path: ${{ steps.uv-cache.outputs.dir }} + key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - name: Install dependencies - run: make install-python-ci-dependencies + run: make install-python-ci-dependencies-uv - name: Setup Redis Cluster run: | docker pull vishnunair/docker-redis-cluster:latest diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 3292d6bcb5..2db7087908 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -33,7 +33,7 @@ jobs: with: ref: master - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 id: setup-python with: python-version: "3.9" @@ -90,18 +90,18 @@ jobs: run: echo "::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`" - name: Cache Public ECR Image id: lambda_python_3_9 - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/cache key: lambda_python_3_9 - name: Handle Cache Miss (pull public ECR image & save it to tar file) - if: steps.cache-primes.outputs.cache-hit != 'true' + if: steps.lambda_python_3_9.outputs.cache-hit != 'true' run: | mkdir -p ~/cache docker pull public.ecr.aws/lambda/python:3.9 docker save public.ecr.aws/lambda/python:3.9 -o ~/cache/lambda_python_3_9.tar - name: Handle Cache Hit (load docker image from tar file) - if: steps.cache-primes.outputs.cache-hit == 'true' + if: steps.lambda_python_3_9.outputs.cache-hit == 'true' run: | docker load -i ~/cache/lambda_python_3_9.tar - name: Build and push @@ -145,7 +145,7 @@ jobs: ref: master submodules: recursive - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 id: setup-python with: python-version: ${{ matrix.python-version }} @@ -173,25 +173,20 @@ jobs: aws-region: us-west-2 - name: Use AWS CLI run: aws sts get-caller-identity - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache - uses: actions/cache@v2 - with: - path: | - ${{ steps.pip-cache.outputs.dir }} - /opt/hostedtoolcache/Python - /Users/runner/hostedtoolcache/Python - key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - restore-keys: | - ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Upgrade pip version run: | pip install --upgrade "pip>=21.3.1,<23.2" - - name: Install pip-tools - run: pip install pip-tools + - name: Install uv + run: pip install uv + - name: Get uv cache dir + id: uv-cache + run: | + echo "::set-output name=dir::$(uv cache dir)" + - name: uv cache + uses: actions/cache@v4 + with: + path: ${{ steps.uv-cache.outputs.dir }} + key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - name: Install apache-arrow on ubuntu if: matrix.os == 'ubuntu-latest' run: | diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 5e7287351b..4d28c6b456 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -110,7 +110,7 @@ jobs: ref: refs/pull/${{ github.event.pull_request.number }}/merge submodules: recursive - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 id: setup-python with: python-version: ${{ matrix.python-version }} @@ -133,27 +133,22 @@ jobs: aws-region: us-west-2 - name: Use AWS CLI run: aws sts get-caller-identity - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache - uses: actions/cache@v2 - with: - path: | - ${{ steps.pip-cache.outputs.dir }} - /opt/hostedtoolcache/Python - /Users/runner/hostedtoolcache/Python - key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - restore-keys: | - ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Upgrade pip version run: | pip install --upgrade "pip>=21.3.1,<23.2" - - name: Install pip-tools - run: pip install pip-tools + - name: Install uv + run: pip install uv + - name: Get uv cache dir + id: uv-cache + run: | + echo "::set-output name=dir::$(uv cache dir)" + - name: uv cache + uses: actions/cache@v4 + with: + path: ${{ steps.uv-cache.outputs.dir }} + key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - name: Install dependencies - run: make install-python-ci-dependencies + run: make install-python-ci-dependencies-uv - name: Setup Redis Cluster run: | docker pull vishnunair/docker-redis-cluster:latest diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml index 266cdcc9b9..be892ae910 100644 --- a/.github/workflows/pr_local_integration_tests.yml +++ b/.github/workflows/pr_local_integration_tests.yml @@ -33,32 +33,27 @@ jobs: ref: refs/pull/${{ github.event.pull_request.number }}/merge submodules: recursive - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 id: setup-python with: python-version: ${{ matrix.python-version }} architecture: x64 - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache - uses: actions/cache@v2 - with: - path: | - ${{ steps.pip-cache.outputs.dir }} - /opt/hostedtoolcache/Python - /Users/runner/hostedtoolcache/Python - key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - restore-keys: | - ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Upgrade pip version run: | pip install --upgrade "pip>=21.3.1,<23.2" - - name: Install pip-tools - run: pip install pip-tools + - name: Install uv + run: pip install uv + - name: Get uv cache dir + id: uv-cache + run: | + echo "::set-output name=dir::$(uv cache dir)" + - name: uv cache + uses: actions/cache@v4 + with: + path: ${{ steps.uv-cache.outputs.dir }} + key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - name: Install dependencies - run: make install-python-ci-dependencies + run: make install-python-ci-dependencies-uv - name: Test local integration tests if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak run: make test-python-integration-local diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index ff7c3d5e23..5d689d72de 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -19,31 +19,27 @@ jobs: - uses: actions/checkout@v3 - name: Setup Python id: setup-python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} architecture: x64 - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache - uses: actions/cache@v2 - with: - path: | - ${{ steps.pip-cache.outputs.dir }} - /opt/hostedtoolcache/Python - /Users/runner/hostedtoolcache/Python - key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - restore-keys: | - ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Upgrade pip version run: | pip install --upgrade "pip>=21.3.1,<23.2" - - name: Install pip-tools - run: pip install pip-tools + - name: Install uv + run: | + pip install uv + - name: Get uv cache dir + id: uv-cache + run: | + echo "::set-output name=dir::$(uv cache dir)" + - name: uv cache + uses: actions/cache@v4 + with: + path: ${{ steps.uv-cache.outputs.dir }} + key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} - name: Install dependencies - run: make install-python-ci-dependencies + run: make install-python-ci-dependencies-uv - name: Test Python run: make test-python-unit diff --git a/Makefile b/Makefile index bf2d876b7f..f42d5c1eda 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,11 @@ install-python-ci-dependencies: pip install --no-deps -e . python setup.py build_python_protos --inplace +install-python-ci-dependencies-uv: + uv pip sync --system sdk/python/requirements/py$(PYTHON)-ci-requirements.txt + uv pip install --system --no-deps -e . + python setup.py build_python_protos --inplace + lock-python-ci-dependencies: python -m piptools compile -U --extra ci --output-file sdk/python/requirements/py$(PYTHON)-ci-requirements.txt