diff --git a/.dockerignore b/.dockerignore index dffae8f55..b23f4a4fc 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,7 @@ # Git -.git +# .git .gitignore -.gitattributes +# .gitattributes # Docker docker-compose.yml diff --git a/.github/ISSUE_TEMPLATE/new-release.md b/.github/ISSUE_TEMPLATE/new-release.md index c247625af..7ee589672 100644 --- a/.github/ISSUE_TEMPLATE/new-release.md +++ b/.github/ISSUE_TEMPLATE/new-release.md @@ -24,8 +24,6 @@ Release captain responsible - <@gh_username> - [ ] Create a new git branch for the release `git checkout -b release-2024.9.1` - [ ] Prepare the branch just in case `git clean -fxdq` -- [ ] Bump `conda-store` version in [`conda-store/conda-store/__init__.py`](https://github.com/conda-incubator/conda-store/blob/main/conda-store/conda_store/__init__.py) -- [ ] Bump `conda-store-server` version in [`conda-store-server/conda-store-server/__init__.py`](https://github.com/conda-incubator/conda-store/blob/main/conda-store/conda_store/__init__.py) - [ ] Update the `conda-store-ui` version used in `conda-store-server` [`conda-store-server/hatch_build.py`](https://github.com/conda-incubator/conda-store/blob/main/conda-store-server/hatch_build.py) - [ ] Update the [CHANGELOG.md](./CHANGELOG.md) file with the new version, release date, and relevant changes[^github-activity]. - [ ] Check the version locally with `hatch version` @@ -67,7 +65,6 @@ Release captain responsible - <@gh_username> - [The Docker images have been published](https://github.com/conda-incubator/conda-store/blob/main/.github/workflows/build_docker_image.yaml) - [ ] Update the [conda-forge feedstock version](https://github.com/conda-forge/conda-store-feedstock) through a PR or review and merge the regro-bot PR. - [ ] If needed - update `meta.yaml` or `recipe.yaml` and re-render the feedstock. -- [ ] Open a follow-up PR to bump `conda-store` and `conda-store-server` versions to the next dev-release number (for example `2024.10.1`). - [ ] Open a follow-up PR to bump the `conda-store-server` version in the [`conda-store-ui` compose file](https://github.com/conda-incubator/conda-store-ui/blob/main/docker-compose.yml). - [ ] Celebrate, you're done! 🎉 diff --git a/.github/actions/docker-setup/action.yaml b/.github/actions/docker-setup/action.yaml new file mode 100644 index 000000000..e68c0710b --- /dev/null +++ b/.github/actions/docker-setup/action.yaml @@ -0,0 +1,40 @@ +name: "Setup Docker" +description: "Setup tasks needed to build and publish Docker images" + +inputs: + docker-image: + description: "conda-store project to build the Docker image for: conda-store or conda-store-server" + required: true +outputs: + cs-version: + description: "conda-store current version - dirty tag" + value: ${{ steps.get-dev-version.outputs.cs-version }} + +runs: + using: "composite" + steps: + - name: "Get dev version 🏷️" + id: get-dev-version + shell: bash + run: | + echo "cs-version=$(git describe --tags)" >> $GITHUB_OUTPUT + + - name: "Get project's default Python version 🏷️" + shell: bash + run: | + echo "PYTHON_VERSION_DEFAULT=$(cat .python-version-default)" >> $GITHUB_ENV + + - name: "Lint Dockerfiles 🔍" + uses: jbergstroem/hadolint-gh-action@v1 + with: + dockerfile: ${{ inputs.docker-image }}/Dockerfile + output_format: tty + error_level: 0 + + - name: "Set up Docker Buildx 🏗" + uses: docker/setup-buildx-action@v3 + + - name: "Copy .dockerignore" + run: | + cp .dockerignore ${{ inputs.docker-image }}/.dockerignore + shell: bash diff --git a/.github/workflows/build_docker_image.yaml b/.github/workflows/build_docker_image.yaml index 36776a0ec..fb1c40831 100644 --- a/.github/workflows/build_docker_image.yaml +++ b/.github/workflows/build_docker_image.yaml @@ -27,10 +27,14 @@ jobs: steps: - name: "Checkout Repository 🛎" uses: actions/checkout@v4 + with: + fetch-depth: 0 - - name: "Get project's default Python version 🏷️" - run: | - echo "PYTHON_VERSION_DEFAULT=$(cat .python-version-default)" >> $GITHUB_ENV + - name: "Setup CI environment - Docker 🛠" + uses: ./.github/actions/docker-setup + id: setup-ci-docker + with: + docker-image: ${{ matrix.docker-image }} - name: "Retrieve secret from Vault 🗝" uses: hashicorp/vault-action@v3 @@ -45,9 +49,6 @@ jobs: kv/data/repository/conda-incubator/conda-store/shared_secrets QUAY_QUANSIGHT_USERNAME | QUAY_USERNAME; kv/data/repository/conda-incubator/conda-store/shared_secrets QUAY_QUANSIGHT_PASSWORD | QUAY_PASSWORD; - - name: "Set up Docker Buildx 🏗" - uses: docker/setup-buildx-action@v3 - - name: "Login to Docker Hub 🐳" uses: docker/login-action@v3 with: @@ -72,8 +73,11 @@ jobs: type=ref,event=tag type=sha - # we need this as we are setting the context to subproject directory - - name: "Copy .dockerignore" + - name: "Get project's default Python version 🏷️" + run: | + echo "PYTHON_VERSION_DEFAULT=$(cat .python-version-default)" >> $GITHUB_ENV + + - name: "Copy .dockerignore 📄" run: | cp .dockerignore ${{ matrix.docker-image }}/.dockerignore shell: bash @@ -92,4 +96,5 @@ jobs: cache-to: type=gha,mode=max build-args: | python_version=${{ env.PYTHON_VERSION_DEFAULT }} + PSEUDO_VERSION=${{ steps.setup-ci-docker.outputs.cs-version }} platforms: ${{ matrix.platform }} diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index f56765dc8..3d6e0e5f9 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -30,6 +30,7 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 + - name: "Check conda builds 📦" uses: jaimergp/conda-build-action@main with: diff --git a/.github/workflows/generate_api_docs.yaml b/.github/workflows/generate_api_docs.yaml index f41dc3c26..ffeb36284 100644 --- a/.github/workflows/generate_api_docs.yaml +++ b/.github/workflows/generate_api_docs.yaml @@ -33,7 +33,7 @@ jobs: jq . --sort-keys docusaurus-docs/static/openapi.json > docusaurus-docs/static/openapi.json.formatted mv docusaurus-docs/static/openapi.json.formatted docusaurus-docs/static/openapi.json - - name: Create Pull Request + - name: "Create Pull Request" uses: peter-evans/create-pull-request@v7 with: title: "[AUTO] Update openapi.json" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 4fcc18ec8..2723b7f2e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -5,6 +5,9 @@ on: types: [published] push: tags: ["*"] + # needed to publish to test PyPI when merging to main + branches: + - main workflow_dispatch: env: diff --git a/.github/workflows/test_build_docker_image.yaml b/.github/workflows/test_build_docker_image.yaml index 6d8424094..e43b4d48b 100644 --- a/.github/workflows/test_build_docker_image.yaml +++ b/.github/workflows/test_build_docker_image.yaml @@ -17,7 +17,6 @@ on: branches: - main - # ensuring only one instance is running at a given time concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} @@ -38,25 +37,14 @@ jobs: steps: - name: "Checkout Repository 🛎" uses: actions/checkout@v4 - - - name: "Get project's default Python version 🏷️" - run: | - echo "PYTHON_VERSION_DEFAULT=$(cat .python-version-default)" >> $GITHUB_ENV - - - name: "Set up Docker Buildx 🏗" - uses: docker/setup-buildx-action@v3 - - - name: "Lint Dockerfiles 🔍" - uses: jbergstroem/hadolint-gh-action@v1 with: - dockerfile: ${{ matrix.docker-image }}/Dockerfile - output_format: tty - error_level: 0 + fetch-depth: 0 - - name: "Copy .dockerignore" - run: | - cp .dockerignore ${{ matrix.docker-image }}/.dockerignore - shell: bash + - name: "Setup CI environment - Docker 🛠" + uses: ./.github/actions/docker-setup + id: setup-ci-docker + with: + docker-image: ${{ matrix.docker-image }} - name: "Add Docker metadata 📝" id: meta @@ -67,6 +55,15 @@ jobs: tags: | type=sha + - name: "Get project's default Python version 🏷️" + run: | + echo "PYTHON_VERSION_DEFAULT=$(cat .python-version-default)" >> $GITHUB_ENV + + - name: "Copy .dockerignore 📄" + run: | + cp .dockerignore ${{ matrix.docker-image }}/.dockerignore + shell: bash + - name: "Build Docker images 🐳" uses: docker/build-push-action@v5 with: @@ -81,4 +78,5 @@ jobs: cache-to: type=gha,mode=max build-args: | python_version=${{ env.PYTHON_VERSION_DEFAULT }} + PSEUDO_VERSION=${{ steps.setup-ci-docker.outputs.cs-version }} platforms: ${{ matrix.platform }} diff --git a/.github/workflows/test_conda_store_server_unit.yaml b/.github/workflows/test_conda_store_server_unit.yaml index 38000e041..1d098542c 100644 --- a/.github/workflows/test_conda_store_server_unit.yaml +++ b/.github/workflows/test_conda_store_server_unit.yaml @@ -30,6 +30,8 @@ jobs: steps: - name: "Checkout Repository 🛎" uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: "Set up Python 🐍" uses: actions/setup-python@v5 diff --git a/.gitignore b/.gitignore index 0ec82f551..89b2490ff 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ dist build *.egg-info .pytest_cache +**/_version.py +_version.py # data from deployments data diff --git a/.hadolint.yaml b/.hadolint.yaml index 83905e9d4..ff5928d38 100644 --- a/.hadolint.yaml +++ b/.hadolint.yaml @@ -1,5 +1,5 @@ --- ignored: - DL3008 # we do not pin apt deps - - DL3003 # already use workkdir + - DL3003 # already use workdir - DL3042 # already use --no-cache--dir diff --git a/conda-store-server/Dockerfile b/conda-store-server/Dockerfile index b9d400486..bce0ecb63 100644 --- a/conda-store-server/Dockerfile +++ b/conda-store-server/Dockerfile @@ -15,7 +15,7 @@ ENV PATH=/opt/conda/condabin:/opt/conda/envs/conda-store-server/bin:/opt/conda/b ENV TZ=Etc/UTC # must be passed at build environment (should use .python-version-default) -ARG python_version +ARG python_version="3.12" ARG conda_env_name="conda-store-server" ARG user_no=1000 @@ -62,8 +62,8 @@ COPY ./ /opt/conda-store-server/ USER 0:0 RUN chown -R ${user_no}:${user_no} /opt/conda-store-server/ && \ - mkdir -p /.local/share/conda-store && \ - chown -R ${user_no}:${user_no} /.local/share/conda-store + mkdir -p /.local/share/conda-store && \ + chown -R ${user_no}:${user_no} /.local/share/conda-store USER ${user_no}:${user_no} @@ -84,6 +84,9 @@ WORKDIR /var/lib/conda-store # --------------------------------------------------------------------------------- # for development images we install conda-store-server in editable mode FROM base AS dev +ARG PSEUDO_VERSION=1.0.0 +# we need to set the environment variable for setuptools_scm to work +ENV SETUPTOOLS_SCM_PRETEND_VERSION=${PSEUDO_VERSION} WORKDIR /opt/conda-store-server RUN which python && \ diff --git a/conda-store-server/conda_store_server/__init__.py b/conda-store-server/conda_store_server/__init__.py index 7ab38eb25..2b2eb500f 100644 --- a/conda-store-server/conda_store_server/__init__.py +++ b/conda-store-server/conda_store_server/__init__.py @@ -6,18 +6,43 @@ import datetime import hashlib import typing - -import platformdirs +from pathlib import Path if typing.TYPE_CHECKING: from ._internal.orm import Build from .app import CondaStore - -__version__ = "2024.11.2-dev" - - -CONDA_STORE_DIR = platformdirs.user_data_path(appname="conda-store") +# For runtime, we use platformdirs to get the user data directory and ensure +# this is cross-platform +try: + import platformdirs + + CONDA_STORE_DIR = platformdirs.user_data_path(appname="conda-store") +# at build time, we use Path.home() to default to the user's home directory +# this is a workaround as we cannot import platformdirs at build time +except ImportError: + CONDA_STORE_DIR = Path.home() / ".conda-store" + +# Since we are now using vcs we need to define the variable '__version__': +try: + # If setuptools_scm is installed (e.g. in a development environment with + # an editable install), then use it to determine the version dynamically. + from setuptools_scm import get_version + + # This will fail with LookupError if the package is not installed in + # editable mode or if Git is not installed. + __version__ = get_version(root="../..", relative_to=__file__) +except (ImportError, LookupError): + # As a fallback, use the version that is hard-coded in the file. + try: + from conda_store_server._version import __version__ # noqa: F401 + except ModuleNotFoundError: + # The user is probably trying to run this without having installed + # the package, so complain. + raise RuntimeError( + "conda-store-server is not correctly installed. " + "Please install it with pip." + ) class BuildKey: diff --git a/conda-store-server/environment-dev.yaml b/conda-store-server/environment-dev.yaml index 974f0465a..3ba470e82 100644 --- a/conda-store-server/environment-dev.yaml +++ b/conda-store-server/environment-dev.yaml @@ -36,3 +36,4 @@ dependencies: - pip: - pytest-playwright - build + - setuptools-scm>=6.4.0 diff --git a/conda-store-server/pyproject.toml b/conda-store-server/pyproject.toml index 1b1aa8d29..8db802e87 100644 --- a/conda-store-server/pyproject.toml +++ b/conda-store-server/pyproject.toml @@ -3,7 +3,7 @@ # license that can be found in the LICENSE file. [build-system] # https://github.com/ofek/hatch-vcs -requires = ["hatchling>=1.14.0", "hatch-vcs"] +requires = ["hatchling>=1.14.0", "hatch-vcs", "hatch-fancy-pypi-readme"] build-backend = "hatchling.build" [project] @@ -81,11 +81,16 @@ dynamic = ["version"] [project.urls] Homepage = "https://conda.store/" -Source = "https://github.com/conda-incubator/conda-store" +Source = "https://github.com/conda-incubator/conda-store/conda-store-server" Issues = "https://github.com/conda-incubator/conda-store/issues" [tool.hatch.version] -path = "conda_store_server/__init__.py" +source = "vcs" +# need to specify root as .git is one level up +raw-options = { root = "../", relative_to = "pyproject.toml", local_scheme = "node-and-date" } + +[tool.hatch.build.hooks.vcs] +version-file = "conda_store_server/_version.py" [tool.hatch.envs.dev] dependencies = [ @@ -100,6 +105,7 @@ dependencies = [ "twine>=5.0.0", "pkginfo>=1.10", # Needed to support metadata 2.3 "pytest-cov", + "setuptools-scm>=6.4.0" ] [tool.hatch.envs.lint] @@ -107,6 +113,8 @@ dependencies = ["pre-commit"] [tool.hatch.envs.lint.scripts] lint = ["pre-commit run --all"] + +[tool.hatch.envs.test.scripts] unit-test = "pytest -m 'not extended_prefix and not user_journey' tests" playwright-test = [ "playwright install", diff --git a/conda-store/.gitignore b/conda-store/.gitignore index 325e8c346..61e9f76e2 100644 --- a/conda-store/.gitignore +++ b/conda-store/.gitignore @@ -6,3 +6,4 @@ node_modules/ *.tsbuildinfo __pycache__ .DS_Store +**/_version.py diff --git a/conda-store/Dockerfile b/conda-store/Dockerfile index fd1870ea5..d7d79e8b1 100644 --- a/conda-store/Dockerfile +++ b/conda-store/Dockerfile @@ -5,13 +5,15 @@ # conda-store (core) Dockerfile # the generated Docker image is used with docker compose to run the conda-store # worker and public conda-store Docker images +# To build locally, run: +# docker build --rm -f "conda-store/Dockerfile" -t conda-store:latest "conda-store" --platform=linux/arm64 . FROM condaforge/miniforge3:24.7.1-0 AS base LABEL org.opencontainers.image.authors="conda-store development team" # must be passed at build environment (should use .python-version-default) -ARG python_version +ARG python_version="3.12" ARG conda_env_name="conda-store" ARG user_no=1000 @@ -37,6 +39,13 @@ RUN mamba create --name ${conda_env_name} \ conda clean --force-pkgs-dirs COPY ./ /opt/conda-store/ +COPY --from=repo_root .git/ /opt/conda-store/.git/ + +# needed to get the dynamic version for the project +WORKDIR /opt/conda-store +RUN PSEUDO_VERSION=$(git describe --tags) && \ + echo "PSEUDO_VERSION=${PSEUDO_VERSION}" > /etc/environment && \ + echo ${PSEUDO_VERSION} # --------------------------------------------------------------------------------- # for production-ready images we install a specific version of conda-store @@ -46,7 +55,6 @@ ARG RELEASE_VERSION WORKDIR /opt/conda-store - RUN which python && \ python -m pip install conda-store==${RELEASE_VERSION} --no-cache-dir @@ -58,12 +66,16 @@ WORKDIR /opt/jupyterhub # for development images we install conda-store-server in editable mode FROM base AS dev +# we need to set the environment variable for setuptools_scm to work +ENV SETUPTOOLS_SCM_PRETEND_VERSION_FOR_CONDA_STORE=${PSEUDO_VERSION} RUN --mount=type=cache,target=/root/.cache/pip WORKDIR /opt/conda-store RUN which python && \ - python -m pip install -e . --no-cache-dir + echo ${PSEUDO_VERSION} && \ + echo ${SETUPTOOLS_SCM_PRETEND_VERSION_FOR_CONDA_STORE} +# pip install -e . --no-cache-dir USER conda-store WORKDIR /opt/jupyterhub diff --git a/conda-store/conda_store/__init__.py b/conda-store/conda_store/__init__.py index 00a5d43e8..299290e8f 100644 --- a/conda-store/conda_store/__init__.py +++ b/conda-store/conda_store/__init__.py @@ -1,5 +1,19 @@ -# Copyright (c) conda-store development team. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. +# Define the variable '__version__': +try: + # If setuptools_scm is installed (e.g. in a development environment with + # an editable install), then use it to determine the version dynamically. + from setuptools_scm import get_version -__version__ = "2024.11.2-dev" + # This will fail with LookupError if the package is not installed in + # editable mode or if Git is not installed. + __version__ = get_version(root="../..", relative_to=__file__) +except (ImportError, LookupError): + # As a fallback, use the version that is hard-coded in the file. + try: + from conda_store._version import __version__ # noqa: F401 + except ModuleNotFoundError: + # The user is probably trying to run this without having installed + # the package, so complain. + raise RuntimeError( + "conda-store is not correctly installed. " "Please install it with pip." + ) diff --git a/conda-store/pyproject.toml b/conda-store/pyproject.toml index 4930325ab..a34982054 100644 --- a/conda-store/pyproject.toml +++ b/conda-store/pyproject.toml @@ -59,7 +59,12 @@ Source = "https://github.com/conda-incubator/conda-store" Issues = "https://github.com/conda-incubator/conda-store/issues" [tool.hatch.version] -path = "conda_store/__init__.py" +source = "vcs" +# need to specify root as .git is one level up +raw-options = { root = "../", relative_to = "pyproject.toml", local_scheme = "node-and-date" } + +[tool.hatch.build.hooks.vcs] +version-file = "conda_store/_version.py" [project.optional-dependencies] dev = ["build", "twine"] @@ -71,7 +76,7 @@ dependencies = [ "pytest", "twine>=5.0.0", "pkginfo >= 1.10", # Needed to support metadata 2.3 - + "setuptools-scm>=6.4.0" ] [tool.hatch.envs.lint]