Switch from --user to venv environment for PROD image

This PR introduces a joint way to treat the .local (--user) folder as both - venv and `--user` package installation. This allow two things: * we can use `uv` to built the production image, which gives 40%-50% saving for image build time. * user do not have to use `pip install --user` to install new packages * python -m venv --system-site-packages continues to use the .local packages from the .local installation (and not uses them if --system-site-packages is not used * you can use uv to install packages when you extend the image as a user * this PR switches to use `uv` by default for all prod images, but it adds a check if the image still builds with `pip` * we also switch to more PEP standard way of installing packages from local filesystem (package-name @ file:///FILE) Fixes: #37785
apache · Feb 29, 2024 · 64a37a1 · 64a37a1
1 parent 41ac676
commit 64a37a1
Show file tree

Hide file tree

Showing 22 changed files with 541 additions and 362 deletions.
diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
@@ -180,6 +180,7 @@ jobs:
       RUNS_ON: "${{ needs.build-info.outputs.runs-on }}"
       BACKEND: sqlite
       VERSION_SUFFIX_FOR_PYPI: "dev0"
+      USE_UV: "true"
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -258,6 +259,7 @@ jobs:
       BACKEND: sqlite
       VERSION_SUFFIX_FOR_PYPI: "dev0"
       INCLUDE_NOT_READY_PROVIDERS: "true"
+      USE_UV: "true"
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -282,6 +282,7 @@ jobs:
       # Force more parallelism for build even on public images
       PARALLELISM: 6
       VERSION_SUFFIX_FOR_PYPI: "dev0"
+      USE_UV: "true"
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -1863,6 +1864,7 @@ jobs:
       BACKEND: sqlite
       VERSION_SUFFIX_FOR_PYPI: "dev0"
       DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
+      USE_UV: "true"
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -1898,6 +1900,58 @@ jobs:
           PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
           DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}
 
+  build-prod-images-pip:
+    strategy:
+      matrix:
+        python-version: ${{fromJson(needs.build-info.outputs.python-versions)}}
+    timeout-minutes: 80
+    name: ${{needs.build-info.outputs.build-job-description}} PROD image pip (main) ${{matrix.python-version}}
+    runs-on: ["ubuntu-22.04"]
+    needs: [build-info, build-ci-images]
+    env:
+      DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
+      DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
+      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
+      BACKEND: sqlite
+      VERSION_SUFFIX_FOR_PYPI: "dev0"
+      DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
+      USE_UV: "false"
+    steps:
+      - name: Cleanup repo
+        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
+        if: >
+          needs.build-info.outputs.in-workflow-build == 'true' &&
+          needs.build-info.outputs.default-branch == 'main'
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.build-info.outputs.targetCommitSha }}
+          persist-credentials: false
+        if: >
+          needs.build-info.outputs.in-workflow-build == 'true' &&
+          needs.build-info.outputs.default-branch == 'main'
+      - name: "Install Breeze"
+        uses: ./.github/actions/breeze
+        with:
+          python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }}
+        if: >
+          needs.build-info.outputs.in-workflow-build == 'true' &&
+          needs.build-info.outputs.default-branch == 'main'
+      - name: Build PROD Image pip ${{ matrix.python-version }}:${{env.IMAGE_TAG}}
+        uses: ./.github/actions/build-prod-images
+        if: >
+          needs.build-info.outputs.in-workflow-build == 'true' &&
+          needs.build-info.outputs.default-branch == 'main'
+        with:
+          build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
+          chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
+          python-version: ${{ matrix.python-version }}
+        env:
+          UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
+          DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
+          PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
+          DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}
+          IMAGE_TAG: "pip-${{ github.event.pull_request.head.sha || github.sha }}"
+
   build-prod-images-bullseye:
     strategy:
       matrix:
@@ -1914,6 +1968,7 @@ jobs:
       BACKEND: sqlite
       VERSION_SUFFIX_FOR_PYPI: "dev0"
       DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
+      USE_UV: "true"
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -1970,6 +2025,7 @@ jobs:
       BACKEND: sqlite
       VERSION_SUFFIX_FOR_PYPI: "dev0"
       DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
+      USE_UV: "true"
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -2027,6 +2083,7 @@ jobs:
       BACKEND: sqlite
       VERSION_SUFFIX_FOR_PYPI: "dev0"
       DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
+      USE_UV: "true"
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -2078,6 +2135,7 @@ jobs:
       BACKEND: sqlite
       VERSION_SUFFIX_FOR_PYPI: "dev0"
       DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
+      USE_UV: "true"
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -2134,6 +2192,7 @@ jobs:
       BACKEND: sqlite
       VERSION_SUFFIX_FOR_PYPI: "dev0"
       DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
+      USE_UV: "true"
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -2530,6 +2589,7 @@ jobs:
       RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
       # Force more parallelism for build even on small instances
       PARALLELISM: 6
+      USE_UV: "true"
     if: >
       needs.build-info.outputs.in-workflow-build == 'true' &&
       needs.build-info.outputs.canary-run != 'true'