Migrate to charmcraft 3 poetry plugin (#273)

data-platform-workflows and charmcraftcache are migrating from charmcraft 2 to 3 to enable 24.04 based charms & to enable use of the poetry plugin, which fixes several longstanding issues with charmcraft (e.g. canonical/charmcraft#1077)

.github/workflows/ci.yaml

@@ -20,13 +20,14 @@ on:
 jobs:
   lint:
     name: Lint
-    uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v24.0.6
+    uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v26.0.0
 
   build:
     name: Build charm
-    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v24.0.6
+    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v26.0.0
     with:
       cache: true
+      charmcraft-snap-channel: latest/beta/data-platform  # TODO: remove after charmcraft 3.3 stable release
 
   integration-test:
     strategy:
@@ -43,7 +44,7 @@ jobs:
     needs:
       - lint
       - build
-    uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v24.0.6
+    uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v26.0.0
     with:
       artifact-prefix: ${{ needs.build.outputs.artifact-prefix }}
       cloud: ${{ matrix.cloud }}

.github/workflows/release.yaml

@@ -22,15 +22,18 @@ jobs:
 
   build:
     name: Build charm
-    uses: canonical/data-platform-workflows/.github/workflows/[email protected]
+    uses: canonical/data-platform-workflows/.github/workflows/[email protected]
+    with:
+      charmcraft-snap-channel: latest/beta/data-platform  # TODO: remove after charmcraft 3.3 stable release
 
   release:
     name: Release charm
     needs:
       - ci-tests
       - build
-    uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v24.0.6
+    uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v26.0.0
     with:
+      charmcraft-snap-channel: latest/beta/data-platform  # TODO: remove after charmcraft 3.3 stable release
       channel: edge
       artifact-prefix: ${{ needs.build.outputs.artifact-prefix }}
     secrets:

charmcraft.yaml

@@ -1,25 +1,90 @@
-# Copyright 2023 Canonical Ltd.
+# Copyright 2022 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 type: charm
-bases:
-  - name: ubuntu
-    channel: "22.04"
-    architectures: [amd64]
-  - name: ubuntu
-    channel: "22.04"
-    architectures: [arm64]
+platforms:
+  [email protected]:amd64:
+  [email protected]:arm64:
+# Files implicitly created by charmcraft without a part:
+# - dispatch (https://github.com/canonical/charmcraft/pull/1898)
+# - manifest.yaml
+#   (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/services/package.py#L259)
+# Files implicitly copied/"primed" by charmcraft without a part:
+# - actions.yaml, config.yaml, metadata.yaml
+#   (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/services/package.py#L290-L293
+#   https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/services/package.py#L156-L157)
 parts:
-  charm:
+  # "poetry-deps" part name is a magic constant
+  # https://github.com/canonical/craft-parts/pull/901
+  poetry-deps:
+    plugin: nil
+    build-packages:
+      - curl
+    override-build: |
+      # Use environment variable instead of `--break-system-packages` to avoid failing on older
+      # versions of pip that do not recognize `--break-system-packages`
+      # `--user` needed (in addition to `--break-system-packages`) for Ubuntu >=24.04
+      PIP_BREAK_SYSTEM_PACKAGES=true python3 -m pip install --user --upgrade pip==24.3.1  # renovate: charmcraft-pip-latest
+
+      # Use uv to install poetry so that a newer version of Python can be installed if needed by poetry
+      curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.5.15/uv-installer.sh | sh  # renovate: charmcraft-uv-latest
+      # poetry 2.0.0 requires Python >=3.9
+      if ! "$HOME/.local/bin/uv" python find '>=3.9'
+      then
+        # Use first Python version that is >=3.9 and available in an Ubuntu LTS
+        # (to reduce the number of Python versions we use)
+        "$HOME/.local/bin/uv" python install 3.10.12  # renovate: charmcraft-python-ubuntu-22.04
+      fi
+      "$HOME/.local/bin/uv" tool install --no-python-downloads --python '>=3.9' poetry==2.0.0 --with poetry-plugin-export==1.8.0  # renovate: charmcraft-poetry-latest
+
+      ln -sf "$HOME/.local/bin/poetry" /usr/local/bin/poetry
+  # "charm-poetry" part name is arbitrary; use for consistency
+  # Avoid using "charm" part name since that has special meaning to charmcraft
+  charm-poetry:
+    # By default, the `poetry` plugin creates/primes these directories:
+    # - lib, src
+    #   (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/parts/plugins/_poetry.py#L76-L78)
+    # - venv
+    #   (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/parts/plugins/_poetry.py#L95
+    #   https://github.com/canonical/craft-parts/blob/afb0d652eb330b6aaad4f40fbd6e5357d358de47/craft_parts/plugins/base.py#L270)
+    plugin: poetry
+    source: .
+    after:
+      - poetry-deps
+    poetry-export-extra-args: ['--only', 'main,charm-libs']
+    build-packages:
+      - libffi-dev  # Needed to build Python dependencies with Rust from source
+      - libssl-dev  # Needed to build Python dependencies with Rust from source
+      - pkg-config  # Needed to build Python dependencies with Rust from source
     override-build: |
-      # Convert subset of poetry.lock to requirements.txt
-      curl -sSL https://install.python-poetry.org | python3 -
-      /root/.local/bin/poetry self add poetry-plugin-export
-      /root/.local/bin/poetry export --only main,charm-libs --output requirements.txt
+      # Workaround for https://github.com/canonical/charmcraft/issues/2068
+      # rustup used to install rustc and cargo, which are needed to build Python dependencies with Rust from source
+      if [[ "$CRAFT_PLATFORM" == [email protected]:* || "$CRAFT_PLATFORM" == [email protected]:* ]]
+      then
+        snap install rustup --classic
+      else
+        apt-get install rustup -y
+      fi
+
+      # If Ubuntu version < 24.04, rustup was installed from snap instead of from the Ubuntu
+      # archive—which means the rustup version could be updated at any time. Print rustup version
+      # to build log to make changes to the snap's rustup version easier to track
+      rustup --version
+
+      # rpds-py (Python package) >=0.19.0 requires rustc >=1.76, which is not available in the
+      # Ubuntu 22.04 archive. Install rustc and cargo using rustup instead of the Ubuntu archive
+      rustup set profile minimal
+      rustup default 1.83.0  # renovate: charmcraft-rust-latest
 
       craftctl default
-    charm-strict-dependencies: true
-    charm-requirements: [requirements.txt]
+      # Include requirements.txt in *.charm artifact for easier debugging
+      cp requirements.txt "$CRAFT_PART_INSTALL/requirements.txt"
+  # "files" part name is arbitrary; use for consistency
+  files:
+    plugin: dump
+    source: .
+    prime:
+      - LICENSE
   libpq:
     build-packages:
       - libpq-dev