From 7358c4a0e0bebec25fe0bf0fdf993f3ed27d7ae9 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 10 Jan 2025 11:41:38 +0100 Subject: [PATCH 1/5] Update data-platform-workflows to v26.0.0 --- .github/workflows/ci.yaml | 6 +++--- .github/workflows/release.yaml | 4 ++-- poetry.lock | 14 +++++++------- pyproject.toml | 6 +++--- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 44fa7a31..eeb616d6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,7 +16,7 @@ on: jobs: lint: name: Lint - uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v24.0.6 + uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v26.0.0 lib-check: name: Check libraries @@ -60,7 +60,7 @@ jobs: - . - tests/integration/application name: Build charms - uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v24.0.6 + uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v26.0.0 with: path-to-charm-directory: ${{ matrix.path }} @@ -80,7 +80,7 @@ jobs: - lint - unit-test - build - uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v24.0.6 + uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v26.0.0 with: artifact-prefix: packed-charm-cache-false # TODO revert to "packed-charm-cache-true" when cache re-enabled cloud: lxd diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 3c517d4c..d9e3397a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -12,7 +12,7 @@ jobs: build: name: Build charm - uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v24.0.6 + uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v26.0.0 with: artifact-prefix: temp-release-packed-charm # TODO: remove after caching re-enabled on PR @@ -21,7 +21,7 @@ jobs: needs: - ci-tests - build - uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v24.0.6 + uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v26.0.0 with: channel: 6/edge artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} diff --git a/poetry.lock b/poetry.lock index 224f9f82..ac5861e2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -33,8 +33,8 @@ pytest = "*" [package.source] type = "git" url = "https://github.com/canonical/data-platform-workflows" -reference = "v24.0.6" -resolved_reference = "11c673f692893a15d15ee63469420e91f91f8a95" +reference = "v26.0.0" +resolved_reference = "92d0a9f28a22c57b5965866c22f65b5021d3b115" subdirectory = "python/pytest_plugins/allure_pytest_collection_report" [[package]] @@ -1560,8 +1560,8 @@ pyyaml = "*" [package.source] type = "git" url = "https://github.com/canonical/data-platform-workflows" -reference = "v24.0.6" -resolved_reference = "11c673f692893a15d15ee63469420e91f91f8a95" +reference = "v26.0.0" +resolved_reference = "92d0a9f28a22c57b5965866c22f65b5021d3b115" subdirectory = "python/pytest_plugins/pytest_operator_cache" [[package]] @@ -1580,8 +1580,8 @@ pytest = "*" [package.source] type = "git" url = "https://github.com/canonical/data-platform-workflows" -reference = "v24.0.6" -resolved_reference = "11c673f692893a15d15ee63469420e91f91f8a95" +reference = "v26.0.0" +resolved_reference = "92d0a9f28a22c57b5965866c22f65b5021d3b115" subdirectory = "python/pytest_plugins/pytest_operator_groups" [[package]] @@ -2145,4 +2145,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = "^3.10.12" -content-hash = "d273d6c39cf28616020f8abaec77e080ff91620555ce64f0350f1ab236efa8df" +content-hash = "1abb97c413ed8813fe009149e09fcae51bc9fbbf3b988a5e64f177bb3afa51ac" diff --git a/pyproject.toml b/pyproject.toml index 7c33750f..2a467f36 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,9 +59,9 @@ juju = "^3.5.0" pytest = "^8.1.1" pytest-asyncio ="^0.21.1" pytest-operator = "^0.34.0" -pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.6", subdirectory = "python/pytest_plugins/pytest_operator_cache"} -pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.6", subdirectory = "python/pytest_plugins/pytest_operator_groups"} -allure-pytest-collection-report = {git = "https://github.com/canonical/data-platform-workflows", tag = "v24.0.6", subdirectory = "python/pytest_plugins/allure_pytest_collection_report"} +pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v26.0.0", subdirectory = "python/pytest_plugins/pytest_operator_cache"} +pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v26.0.0", subdirectory = "python/pytest_plugins/pytest_operator_groups"} +allure-pytest-collection-report = {git = "https://github.com/canonical/data-platform-workflows", tag = "v26.0.0", subdirectory = "python/pytest_plugins/allure_pytest_collection_report"} [build-system] build-backend = "poetry.core.masonry.api" From 5130d436966fa9b6ee9a6ea774f5ba8b2571a83f Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 10 Jan 2025 13:03:54 +0100 Subject: [PATCH 2/5] Migrate to charmcraft 3 poetry plugin --- .github/workflows/ci.yaml | 2 + .github/workflows/release.yaml | 3 +- charmcraft.yaml | 100 ++++++++++++++++++++++++++------- 3 files changed, 83 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index eeb616d6..372bc4ee 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -62,6 +62,8 @@ jobs: name: Build charms uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v26.0.0 with: + cache: false # TODO: will change to `true` before merge + charmcraft-snap-channel: latest/beta/data-platform # TODO: remove after charmcraft 3.3 stable release path-to-charm-directory: ${{ matrix.path }} integration-test: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d9e3397a..0f56291b 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -14,7 +14,7 @@ jobs: name: Build charm uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v26.0.0 with: - artifact-prefix: temp-release-packed-charm # TODO: remove after caching re-enabled on PR + charmcraft-snap-channel: latest/beta/data-platform # TODO: remove after charmcraft 3.3 stable release release-charm: name: Release charm @@ -23,6 +23,7 @@ jobs: - build uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v26.0.0 with: + charmcraft-snap-channel: latest/beta/data-platform # TODO: remove after charmcraft 3.3 stable release channel: 6/edge artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} secrets: diff --git a/charmcraft.yaml b/charmcraft.yaml index a59040e8..cb23cafb 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -1,30 +1,88 @@ -# This file configures Charmcraft. -# See https://juju.is/docs/sdk/charmcraft-config for guidance. +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. type: charm -bases: - - name: ubuntu - channel: "22.04" - architectures: [amd64] +platforms: + ubuntu@22.04:amd64: +# Files implicitly created by charmcraft without a part: +# - dispatch (https://github.com/canonical/charmcraft/pull/1898) +# - manifest.yaml +# (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/services/package.py#L259) +# Files implicitly copied/"primed" by charmcraft without a part: +# - actions.yaml, config.yaml, metadata.yaml +# (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/services/package.py#L290-L293 +# https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/services/package.py#L156-L157) parts: - charm: - build-snaps: - - rustup + # "poetry-deps" part name is a magic constant + # https://github.com/canonical/craft-parts/pull/901 + poetry-deps: + plugin: nil build-packages: - - libffi-dev - - libssl-dev - - pkg-config - - rustc - - cargo + - curl override-build: | - rustup default stable - # Convert subset of poetry.lock to requirements.txt - curl -sSL https://install.python-poetry.org | python3 - - /root/.local/bin/poetry self add poetry-plugin-export - /root/.local/bin/poetry export --only main,charm-libs --output requirements.txt + # Use environment variable instead of `--break-system-packages` to avoid failing on older + # versions of pip that do not recognize `--break-system-packages` + # `--user` needed (in addition to `--break-system-packages`) for Ubuntu >=24.04 + PIP_BREAK_SYSTEM_PACKAGES=true python3 -m pip install --user --upgrade pip==24.3.1 # renovate: charmcraft-pip-latest + + # Use uv to install poetry so that a newer version of Python can be installed if needed by poetry + curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.5.15/uv-installer.sh | sh # renovate: charmcraft-uv-latest + # poetry 2.0.0 requires Python >=3.9 + if ! "$HOME/.local/bin/uv" python find '>=3.9' + then + # Use first Python version that is >=3.9 and available in an Ubuntu LTS + # (to reduce the number of Python versions we use) + "$HOME/.local/bin/uv" python install 3.10.12 # renovate: charmcraft-python-ubuntu-22.04 + fi + "$HOME/.local/bin/uv" tool install --no-python-downloads --python '>=3.9' poetry==2.0.0 --with poetry-plugin-export==1.8.0 # renovate: charmcraft-poetry-latest + + ln -sf "$HOME/.local/bin/poetry" /usr/local/bin/poetry + # "charm-poetry" part name is arbitrary; use for consistency + # Avoid using "charm" part name since that has special meaning to charmcraft + charm-poetry: + # By default, the `poetry` plugin creates/primes these directories: + # - lib, src + # (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/parts/plugins/_poetry.py#L76-L78) + # - venv + # (https://github.com/canonical/charmcraft/blob/9ff19c328e23b50cc06f04e8a5ad4835740badf4/charmcraft/parts/plugins/_poetry.py#L95 + # https://github.com/canonical/craft-parts/blob/afb0d652eb330b6aaad4f40fbd6e5357d358de47/craft_parts/plugins/base.py#L270) + plugin: poetry + source: . + after: + - poetry-deps + poetry-export-extra-args: ['--only', 'main,charm-libs'] + build-packages: + - libffi-dev # Needed to build Python dependencies with Rust from source + - libssl-dev # Needed to build Python dependencies with Rust from source + - pkg-config # Needed to build Python dependencies with Rust from source + override-build: | + # Workaround for https://github.com/canonical/charmcraft/issues/2068 + # rustup used to install rustc and cargo, which are needed to build Python dependencies with Rust from source + if [[ "$CRAFT_PLATFORM" == ubuntu@20.04:* || "$CRAFT_PLATFORM" == ubuntu@22.04:* ]] + then + snap install rustup --classic + else + apt-get install rustup -y + fi + + # If Ubuntu version < 24.04, rustup was installed from snap instead of from the Ubuntu + # archive—which means the rustup version could be updated at any time. Print rustup version + # to build log to make changes to the snap's rustup version easier to track + rustup --version + + # rpds-py (Python package) >=0.19.0 requires rustc >=1.76, which is not available in the + # Ubuntu 22.04 archive. Install rustc and cargo using rustup instead of the Ubuntu archive + rustup set profile minimal + rustup default 1.83.0 # renovate: charmcraft-rust-latest + craftctl default + # Include requirements.txt in *.charm artifact for easier debugging + cp requirements.txt "$CRAFT_PART_INSTALL/requirements.txt" + # "files" part name is arbitrary; use for consistency + files: + plugin: dump + source: . prime: + - LICENSE - charm_version - workload_version - charm-strict-dependencies: true - charm-requirements: [requirements.txt] From de69f34bfec07da49544a8b4f588cfbafda9c8ec Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 10 Jan 2025 13:23:43 +0100 Subject: [PATCH 3/5] Migrate test charm(s) to `platforms` syntax --- tests/integration/application/charmcraft.yaml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/integration/application/charmcraft.yaml b/tests/integration/application/charmcraft.yaml index 1235c6e9..e7ff5f19 100644 --- a/tests/integration/application/charmcraft.yaml +++ b/tests/integration/application/charmcraft.yaml @@ -1,11 +1,9 @@ -# Copyright 2024 Canonical Ltd. +# Copyright 2022 Canonical Ltd. # See LICENSE file for licensing details. type: charm -bases: - - build-on: - - name: "ubuntu" - channel: "22.04" - run-on: - - name: "ubuntu" - channel: "22.04" +platforms: + ubuntu@22.04:amd64: +parts: + charm: + plugin: charm From af7969fcccfc0c7501ef53ff8ee1f9f4bee6d485 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 10 Jan 2025 14:15:49 +0100 Subject: [PATCH 4/5] Fix artifact prefix --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 372bc4ee..4e7d5be9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -84,7 +84,7 @@ jobs: - build uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v26.0.0 with: - artifact-prefix: packed-charm-cache-false # TODO revert to "packed-charm-cache-true" when cache re-enabled + artifact-prefix: packed-charm-cache-false # TODO: revert cache cloud: lxd juju-agent-version: ${{ matrix.juju.agent }} juju-snap-channel: ${{ matrix.juju.snap_channel }} From 8f3e3d08a3248d2aaccb90071a1721234536f315 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 10 Jan 2025 15:24:29 +0100 Subject: [PATCH 5/5] Remove extra juju 3 integration tests --- .github/workflows/ci.yaml | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4e7d5be9..976b510a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -67,17 +67,7 @@ jobs: path-to-charm-directory: ${{ matrix.path }} integration-test: - strategy: - fail-fast: false - matrix: - juju: - # This runs on all runs - - agent: 3.5.3 # renovate: juju-agent-pin-minor - allure_report: true - # This runs only on scheduled runs, DPW 21 specifics (scheduled + 3.6/X) - - snap_channel: 3.6/stable - allure_report: false - name: Integration test charm | ${{ matrix.juju.agent || matrix.juju.snap_channel }} + name: Integration test charm needs: - lint - unit-test @@ -86,8 +76,7 @@ jobs: with: artifact-prefix: packed-charm-cache-false # TODO: revert cache cloud: lxd - juju-agent-version: ${{ matrix.juju.agent }} - juju-snap-channel: ${{ matrix.juju.snap_channel }} - _beta_allure_report: ${{ matrix.juju.allure_report }} + juju-agent-version: 3.6.1 # renovate: juju-agent-pin-minor + _beta_allure_report: true permissions: contents: write