Migrate to charmcraft 3 poetry plugin

.github/workflows/ci.yaml

@@ -43,9 +43,10 @@ jobs:
 
   build:
     name: Build charms
-    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v24.0.6
+    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v26.0.0
     with:
-      cache: true
+      cache: false  # TODO: will change to `true` before merge
+      charmcraft-snap-channel: latest/beta/data-platform  # TODO: remove after charmcraft 3.3 stable release
       path-to-charm-directory: ${{ matrix.path }}
     strategy:
       matrix:

.github/workflows/release.yaml

@@ -13,17 +13,18 @@ jobs:
 
   build:
     name: Build charm
-    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v24.0.6
+    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v26.0.0
     with:
-      cache: false
+      charmcraft-snap-channel: latest/beta/data-platform  # TODO: remove after charmcraft 3.3 stable release
 
   publish:
     name: Release charm
     needs:
       - ci-tests
       - build
-    uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v24.0.6
+    uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v26.0.0
     with:
+      charmcraft-snap-channel: latest/beta/data-platform  # TODO: remove after charmcraft 3.3 stable release
       channel: latest/edge
       artifact-prefix: ${{ needs.build.outputs.artifact-prefix }}
     secrets:

charmcraft.yaml

@@ -1,31 +1,86 @@
-# Copyright 2024 Canonical Ltd.
+# Copyright 2022 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 type: charm
+platforms:
+  [email protected]:amd64:
+# Files implicitly created by charmcraft without a part:
+# - dispatch (https://github.com/canonical/charmcraft/pull/1898)
+# - manifest.yaml
+#   (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/services/package.py#L259)
+# Files implicitly copied/"primed" by charmcraft without a part:
+# - actions.yaml, config.yaml, metadata.yaml
+#   (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/services/package.py#L290-L293
+#   https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/services/package.py#L156-L157)
 parts:
-  charm:
-    build-snaps:
-      - rustup
+  # "poetry-deps" part name is a magic constant
+  # https://github.com/canonical/craft-parts/pull/901
+  poetry-deps:
+    plugin: nil
     build-packages:
-      - libffi-dev
-      - libssl-dev
-      - pkg-config
+      - curl
     override-build: |
-      # get latest rust stable toolchain
-      rustup default stable
+      # Use environment variable instead of `--break-system-packages` to avoid failing on older
+      # versions of pip that do not recognize `--break-system-packages`
+      # `--user` needed (in addition to `--break-system-packages`) for Ubuntu >=24.04
+      PIP_BREAK_SYSTEM_PACKAGES=true python3 -m pip install --user --upgrade pip==24.3.1  # renovate: charmcraft-pip-latest
 
-      # Convert subset of poetry.lock to requirements.txt
-      curl -sSL https://install.python-poetry.org | python3 -
-      /root/.local/bin/poetry self add poetry-plugin-export
-      /root/.local/bin/poetry export --only main,charm-libs --output requirements.txt
+      # Use uv to install poetry so that a newer version of Python can be installed if needed by poetry
+      curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.5.15/uv-installer.sh | sh  # renovate: charmcraft-uv-latest
+      # poetry 2.0.0 requires Python >=3.9
+      if ! "$HOME/.local/bin/uv" python find '>=3.9'
+      then
+        # Use first Python version that is >=3.9 and available in an Ubuntu LTS
+        # (to reduce the number of Python versions we use)
+        "$HOME/.local/bin/uv" python install 3.10.12  # renovate: charmcraft-python-ubuntu-22.04
+      fi
+      "$HOME/.local/bin/uv" tool install --no-python-downloads --python '>=3.9' poetry==2.0.0 --with poetry-plugin-export==1.8.0  # renovate: charmcraft-poetry-latest
+
+      ln -sf "$HOME/.local/bin/poetry" /usr/local/bin/poetry
+  # "charm-poetry" part name is arbitrary; use for consistency
+  # Avoid using "charm" part name since that has special meaning to charmcraft
+  charm-poetry:
+    # By default, the `poetry` plugin creates/primes these directories:
+    # - lib, src
+    #   (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/parts/plugins/_poetry.py#L76-L78)
+    # - venv
+    #   (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/parts/plugins/_poetry.py#L95
+    #   https://github.com/canonical/craft-parts/blob/afb0d652eb330b6aaad4f40fbd6e5357d358de47/craft_parts/plugins/base.py#L270)
+    plugin: poetry
+    source: .
+    after:
+      - poetry-deps
+    poetry-export-extra-args: ['--only', 'main,charm-libs']
+    build-packages:
+      - libffi-dev  # Needed to build Python dependencies with Rust from source
+      - libssl-dev  # Needed to build Python dependencies with Rust from source
+      - pkg-config  # Needed to build Python dependencies with Rust from source
+    override-build: |
+      # Workaround for https://github.com/canonical/charmcraft/issues/2068
+      # rustup used to install rustc and cargo, which are needed to build Python dependencies with Rust from source
+      if [[ "$CRAFT_PLATFORM" == [email protected]:* || "$CRAFT_PLATFORM" == [email protected]:* ]]
+      then
+        snap install rustup --classic
+      else
+        apt-get install rustup -y
+      fi
+
+      # If Ubuntu version < 24.04, rustup was installed from snap instead of from the Ubuntu
+      # archive—which means the rustup version could be updated at any time. Print rustup version
+      # to build log to make changes to the snap's rustup version easier to track
+      rustup --version
+
+      # rpds-py (Python package) >=0.19.0 requires rustc >=1.76, which is not available in the
+      # Ubuntu 22.04 archive. Install rustc and cargo using rustup instead of the Ubuntu archive
+      rustup set profile minimal
+      rustup default 1.83.0  # renovate: charmcraft-rust-latest
 
       craftctl default
-    charm-strict-dependencies: true
-    charm-requirements: [requirements.txt]
-bases:
-  - build-on:
-    - name: ubuntu
-      channel: "22.04"
-    run-on:
-    - name: ubuntu
-      channel: "22.04"
+      # Include requirements.txt in *.charm artifact for easier debugging
+      cp requirements.txt "$CRAFT_PART_INSTALL/requirements.txt"
+  # "files" part name is arbitrary; use for consistency
+  files:
+    plugin: dump
+    source: .
+    prime:
+      - LICENSE

pyproject.toml

@@ -80,7 +80,7 @@ pytest-operator = ">0.20"
 boto3 = ">=1.28.0"
 tenacity = "^8.2.2"
 spark8t = { git = "https://github.com/canonical/spark-k8s-toolkit-py.git", tag = "v0.0.7" }
-pytest-operator-cache = { git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.6", subdirectory = "python/pytest_plugins/pytest_operator_cache" }
+pytest-operator-cache = { git = "https://github.com/canonical/data-platform-workflows", tag = "v26.0.0", subdirectory = "python/pytest_plugins/pytest_operator_cache" }
 # pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v13.1.2", subdirectory = "python/pytest_plugins/pytest_operator_groups"}
 # pytest-operator-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v13.1.2", subdirectory = "python/pytest_plugins/pytest_operator_secrets"}
 

tests/integration/app-charm/charmcraft.yaml

@@ -1,11 +1,9 @@
-# Copyright 2024 Canonical Ltd.
+# Copyright 2022 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 type: charm
-bases:
-  - build-on:
-      - name: "ubuntu"
-        channel: "22.04"
-    run-on:
-      - name: "ubuntu"
-        channel: "22.04"
+platforms:
+  [email protected]:amd64:
+parts:
+  charm:
+    plugin: charm