diff --git a/.dockerignore b/.dockerignore index dd39463ba..dffae8f55 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,52 @@ +# Git +.git +.gitignore +.gitattributes + +# Docker +docker-compose.yml +Dockerfile +.docker +.dockerignore + +# python +**/__pycache__ +dist +build +*.egg-info +.pytest_cache + +# VSCode +.vscode/ + +# MacOS +**/.DS_Store + +# playwright +screenshots +test-results + +# ruff +.ruff_cache + +# conda-store-ui +conda-store-server/conda_store_server/server/static/conda-store-ui/ + +# Docusaurus dependencies +/node_modules + +# Docusaurus production +/build + +# markdown +*.md +!README.md + +# ------------------------ +# conda-store specific data examples/docker/data + +# test files +conda-store-server/tests +conda-store/tests diff --git a/.github/workflows/build_docker_image.yaml b/.github/workflows/build_docker_image.yaml index 18c7f980a..36776a0ec 100644 --- a/.github/workflows/build_docker_image.yaml +++ b/.github/workflows/build_docker_image.yaml @@ -28,6 +28,10 @@ jobs: - name: "Checkout Repository ๐Ÿ›Ž" uses: actions/checkout@v4 + - name: "Get project's default Python version ๐Ÿท๏ธ" + run: | + echo "PYTHON_VERSION_DEFAULT=$(cat .python-version-default)" >> $GITHUB_ENV + - name: "Retrieve secret from Vault ๐Ÿ—" uses: hashicorp/vault-action@v3 with: @@ -68,6 +72,12 @@ jobs: type=ref,event=tag type=sha + # we need this as we are setting the context to subproject directory + - name: "Copy .dockerignore" + run: | + cp .dockerignore ${{ matrix.docker-image }}/.dockerignore + shell: bash + - name: "Publish Docker image ๐Ÿš€" uses: docker/build-push-action@v5 with: @@ -80,4 +90,6 @@ jobs: labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max + build-args: | + python_version=${{ env.PYTHON_VERSION_DEFAULT }} platforms: ${{ matrix.platform }} diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 55513b4d8..f56765dc8 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -23,11 +23,8 @@ jobs: strategy: fail-fast: false matrix: - os: - - ubuntu-latest - - macos-latest - - macos-14 - - windows-latest + # ubuntu 22.04, macos 14, windows 2022 + os: ["ubuntu-latest", "macos-latest", "windows-latest"] steps: - name: "Checkout Repository ๐Ÿ›Ž" uses: actions/checkout@v4 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 7fa1384bd..fde0a92ad 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -20,7 +20,7 @@ permissions: jobs: # Always build & verify package. build-package: - name: Build & verify package + name: "Build & verify package" runs-on: ubuntu-latest strategy: matrix: @@ -39,7 +39,7 @@ jobs: - run: echo "Running on ${{ matrix.directory }}" - name: "Check package build - ${{ matrix.directory }} ๐Ÿ“ฆ" - uses: hynek/build-and-inspect-python-package@main + uses: hynek/build-and-inspect-python-package@v2 with: path: ${{ matrix.directory }} upload-name-suffix: "-${{ matrix.directory }}" @@ -55,7 +55,7 @@ jobs: # Upload to Test PyPI on every commit on main release-test-pypi: - name: Publish in-dev to test.pypi.org + name: "Publish in-dev to test.pypi.org" environment: release-test-pypi if: github.repository_owner == 'conda-incubator' && github.event_name == 'push' && github.ref == 'refs/heads/main' runs-on: ubuntu-latest @@ -82,9 +82,9 @@ jobs: print-hash: true verbose: true - + # Upload to PyPI on every tag/release release-pypi: - name: Publish released package to pypi.org + name: "Publish released package to pypi.org" environment: release-pypi if: github.repository_owner == 'conda-incubator' && github.event_name == 'release' && startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest @@ -128,6 +128,15 @@ jobs: - name: "Checkout Repository ๐Ÿ›Ž" uses: actions/checkout@v4 + - name: "Get project's default Python version ๐Ÿท๏ธ" + run: | + echo "PYTHON_VERSION_DEFAULT=$(cat .python-version-default)" >> $GITHUB_ENV + + - name: "Copy .dockerignore" + run: | + cp .dockerignore ${{ matrix.docker-image }}/.dockerignore + shell: bash + - name: "Retrieve secret from Vault ๐Ÿ—" uses: hashicorp/vault-action@v2 with: @@ -177,6 +186,7 @@ jobs: file: "${{ matrix.docker-image }}/Dockerfile" build-args: | RELEASE_VERSION=${{github.ref_name}} + python_version=${{ env.PYTHON_VERSION_DEFAULT }} tags: | ${{ steps.meta.outputs.tags }} push: true diff --git a/.github/workflows/test_build_docker_image.yaml b/.github/workflows/test_build_docker_image.yaml index 20cbada3e..6d8424094 100644 --- a/.github/workflows/test_build_docker_image.yaml +++ b/.github/workflows/test_build_docker_image.yaml @@ -1,8 +1,8 @@ name: "Test build Docker image" env: - DEFAULT_PYTHON_VERSION: "3.10" FORCE_COLOR: "1" # Make tools pretty. + PYTHONUNBUFFERED: "1" # Make stdout and stderr behave well on: pull_request: @@ -20,7 +20,7 @@ on: # ensuring only one instance is running at a given time concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: true jobs: @@ -39,7 +39,11 @@ jobs: - name: "Checkout Repository ๐Ÿ›Ž" uses: actions/checkout@v4 - - name: "Set up Docker Buildx" + - name: "Get project's default Python version ๐Ÿท๏ธ" + run: | + echo "PYTHON_VERSION_DEFAULT=$(cat .python-version-default)" >> $GITHUB_ENV + + - name: "Set up Docker Buildx ๐Ÿ—" uses: docker/setup-buildx-action@v3 - name: "Lint Dockerfiles ๐Ÿ”" @@ -49,7 +53,12 @@ jobs: output_format: tty error_level: 0 - - name: "Docker Meta" + - name: "Copy .dockerignore" + run: | + cp .dockerignore ${{ matrix.docker-image }}/.dockerignore + shell: bash + + - name: "Add Docker metadata ๐Ÿ“" id: meta uses: docker/metadata-action@v5 with: @@ -58,7 +67,7 @@ jobs: tags: | type=sha - - name: "Build Docker images" + - name: "Build Docker images ๐Ÿณ" uses: docker/build-push-action@v5 with: context: "${{ matrix.docker-image }}" @@ -70,4 +79,6 @@ jobs: labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max + build-args: | + python_version=${{ env.PYTHON_VERSION_DEFAULT }} platforms: ${{ matrix.platform }} diff --git a/.github/workflows/test_conda_store.yaml b/.github/workflows/test_conda_store.yaml index 1dff7cabe..67269811d 100644 --- a/.github/workflows/test_conda_store.yaml +++ b/.github/workflows/test_conda_store.yaml @@ -1,8 +1,8 @@ name: "Test conda-store" env: - DEFAULT_PYTHON_VERSION: "3.10" FORCE_COLOR: "1" # Make tools pretty. + PYTHONUNBUFFERED: "1" # Make stdout and stderr behave well on: pull_request: @@ -20,7 +20,7 @@ on: # ensuring only one instance is running at a given time concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: true jobs: @@ -40,34 +40,29 @@ jobs: - name: "Set up Python ๐Ÿ" uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version-file: .python-version-default + cache: "pip" - name: "Install Dependencies ๐Ÿ“ฆ" run: | - pip install hatch sudo apt install wait-for-it -y + python -m pip install hatch + # install conda-store + pip install . - name: "Linting Checks ๐Ÿงน" run: | hatch env run -e lint lint - - name: "Build package ๐Ÿ“ฆ" - run: | - hatch build - - - name: "Deploy docker-compose" + - name: "Deploy docker compose ๐Ÿ—๏ธ" run: | - docker-compose up -d + docker compose up -d docker ps wait-for-it localhost:5432 # postgresql wait-for-it localhost:9000 # minio wait-for-it localhost:8080 # conda-store-server - - name: "Install conda-store for tests ๐Ÿ“ฆ" - run: | - pip install . - - name: "Run basic tests - not authenticated" run: | sleep 20 @@ -88,4 +83,4 @@ jobs: - name: "Get Docker logs ๐Ÿ”" if: ${{ failure() }} run: | - docker-compose logs + docker compose logs diff --git a/.github/workflows/test_conda_store_server_integration.yaml b/.github/workflows/test_conda_store_server_integration.yaml index 19069d3d1..4edc6645d 100644 --- a/.github/workflows/test_conda_store_server_integration.yaml +++ b/.github/workflows/test_conda_store_server_integration.yaml @@ -1,8 +1,8 @@ name: "Test conda-store-server (integration)" env: - DEFAULT_PYTHON_VERSION: "3.10" FORCE_COLOR: "1" # Make tools pretty. + PYTHONUNBUFFERED: "1" # Make stdout and stderr behave well on: pull_request: @@ -20,12 +20,12 @@ on: # ensuring only one instance is running at a given time concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: true jobs: integration-test-conda-store-server: - name: "integration-test conda-store-server" + name: "integration-test - ${{ matrix.test-type }}" runs-on: ubuntu-latest strategy: matrix: @@ -38,20 +38,29 @@ jobs: - name: "Checkout Repository ๐Ÿ›Ž" uses: actions/checkout@v4 - - name: "Set up env ๐Ÿ" + - name: "Get project's default Python version ๐Ÿท๏ธ" + run: | + echo "PYTHON_VERSION_DEFAULT=$(cat .python-version-default)" >> $GITHUB_ENV + + - name: "Set up conda env ๐Ÿ" uses: conda-incubator/setup-miniconda@v3 with: - environment-file: conda-store-server/environment-dev.yaml - miniforge-version: latest + environment-file: conda-store-server/environment-dev.yaml + miniforge-version: latest + auto-activate-base: false + activate-environment: conda-store-server-dev + python-version: ${{ env.PYTHON_VERSION_DEFAULT }} - - name: "Install build dependencies ๐Ÿ“ฆ" + - name: "Install dependencies ๐Ÿ“ฆ" run: | - pip install hatch + which python sudo apt install wait-for-it -y + # install conda-store-server + python -m pip install . - - name: "Deploy docker-compose" + - name: "Deploy docker compose ๐Ÿ—๏ธ" run: | - docker-compose up -d + docker compose up -d docker ps wait-for-it localhost:5432 # postgresql @@ -61,29 +70,28 @@ jobs: - name: "Run Playwright tests ๐ŸŽญ" run: | - playwright install - pytest --video on ../tests/test_playwright.py + python -m playwright install --with-deps chromium + python -m pytest --video on ../tests/test_playwright.py if: matrix.test-type == 'playwright' - name: "Upload test results ๐Ÿ“ค" uses: actions/upload-artifact@v4 - if: ${{ always() }} && matrix.test-type == 'playwright' + if: matrix.test-type == 'playwright' with: name: playwright-tests path: conda-store-server/test-results - name: "Run integration tests โœ…" run: | - export PYTHONPATH=$PYTHONPATH:$PWD - pytest ../tests/test_api.py ../tests/test_metrics.py + python -m pytest ../tests/test_api.py ../tests/test_metrics.py if: matrix.test-type == 'integration' - name: "Run user journey tests โœ…" run: | - pytest -m "user_journey" + python -m pytest -m "user_journey" if: matrix.test-type == 'user-journey' - name: "Get Docker logs ๐Ÿ”" if: ${{ failure() }} run: | - docker-compose logs + docker compose logs diff --git a/.github/workflows/test_conda_store_server_unit.yaml b/.github/workflows/test_conda_store_server_unit.yaml index cb50a4873..36ffd0f91 100644 --- a/.github/workflows/test_conda_store_server_unit.yaml +++ b/.github/workflows/test_conda_store_server_unit.yaml @@ -1,7 +1,6 @@ name: "Test conda-store-server (unit)" env: - DEFAULT_PYTHON_VERSION: "3.10" FORCE_COLOR: "1" # Make tools pretty. on: @@ -17,26 +16,43 @@ on: branches: - main - # ensuring only one instance is running at a given time concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: true jobs: + linting: + runs-on: ubuntu-latest + defaults: + run: + working-directory: conda-store-server + steps: + - name: "Checkout Repository ๐Ÿ›Ž" + uses: actions/checkout@v4 + + - name: "Set up Python ๐Ÿ" + uses: actions/setup-python@v5 + with: + python-version-file: .python-version-default + cache: "pip" + + - name: "Install Dependencies ๐Ÿ“ฆ" + run: | + python -m pip install hatch + + - name: "Linting Checks ๐Ÿงน" + run: | + hatch env run -e lint lint + test-conda-store-server: - name: "unit-test conda-store-server" + name: "unit-test - ${{ matrix.os}} " strategy: matrix: - os: ["ubuntu", "macos", "windows"] - include: - - os: ubuntu - environment-file: conda-store-server/environment-dev.yaml - - os: macos - environment-file: conda-store-server/environment-macos-dev.yaml - - os: windows - environment-file: conda-store-server/environment-windows-dev.yaml - runs-on: ${{ matrix.os }}-latest + # ubuntu 22.04, macos 14, windows 2022 + # os: ["ubuntu-latest", "macos-latest", "windows-latest"] + os: ["ubuntu-latest"] + runs-on: ${{ matrix.os }} defaults: run: shell: bash -el {0} @@ -45,33 +61,39 @@ jobs: - name: "Checkout Repository ๐Ÿ›Ž" uses: actions/checkout@v4 + - name: "Get project's default Python version ๐Ÿท๏ธ" + run: | + echo "PYTHON_VERSION_DEFAULT=$(cat .python-version-default)" >> $GITHUB_ENV + - name: "Set up env ${{ matrix.os }} ๐Ÿ" - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: - environment-file: ${{ matrix.environment-file }} + environment-file: conda-store-server/environment-dev.yaml miniforge-version: latest + auto-activate-base: false + activate-environment: conda-store-server-dev + python-version: ${{ env.PYTHON_VERSION_DEFAULT }} # This fixes a "DLL not found" issue importing ctypes from the hatch env - - name: "Reinstall Python 3.10 on Windows runner" + - name: "Reinstall Python on Windows runner" uses: nick-fields/retry@v2.8.3 with: timeout_minutes: 9999 max_attempts: 6 command: - conda install --channel=conda-forge --quiet --yes python=${{ matrix.python }} - if: matrix.os == 'windows' - - - name: "Linting Checks ๐Ÿงน" - run: | - hatch env run -e lint lint + conda install --channel=conda-forge --quiet --yes python=${{ env.PYTHON_VERSION_DEFAULT }} + if: matrix.os == 'windows-latest' - - name: "Build package ๐Ÿ“ฆ" + - name: "Install conda-store server ๐Ÿ“ฆ" run: | - hatch build + python -m pip install . + # some checks + conda list + which python - name: "Unit tests โœ…" run: | - pytest -m "not extended_prefix and not user_journey" tests + python -m pytest -m "not extended_prefix and not user_journey" tests # https://github.com/actions/runner-images/issues/1052 - name: "Windows extended prefix unit tests โœ…" @@ -83,4 +105,4 @@ jobs: -Type DWord (Get-ItemProperty "HKLM:System\CurrentControlSet\Control\FileSystem").LongPathsEnabled pytest -m "extended_prefix" tests - if: matrix.os == 'windows' + if: matrix.os == 'windows-latest' diff --git a/.gitignore b/.gitignore index ffd282ce2..0ec82f551 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,9 @@ test-results # ruff .ruff_cache +# MacOS +**/.DS_Store + # conda-store-ui conda-store-server/conda_store_server/_internal/server/static/conda-store-ui/ @@ -45,12 +48,12 @@ conda-store-server/conda_store_server/_internal/server/static/conda-store-ui/ .cache-loader # Docusaurus misc -.DS_Store .env.local .env.development.local .env.test.local .env.production.local +# Node/bun npm-debug.log* yarn-debug.log* yarn-error.log* diff --git a/.hadolint.yaml b/.hadolint.yaml new file mode 100644 index 000000000..83905e9d4 --- /dev/null +++ b/.hadolint.yaml @@ -0,0 +1,5 @@ +--- +ignored: + - DL3008 # we do not pin apt deps + - DL3003 # already use workkdir + - DL3042 # already use --no-cache--dir diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d56b84139..6d13c9722 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,11 +31,10 @@ repos: rev: v4.6.0 hooks: - id: trailing-whitespace - files: ".*\\.py" - id: check-added-large-files - id: check-toml - id: end-of-file-fixer - - id: trailing-whitespace + exclude: ".python-version-default" - repo: https://github.com/pre-commit/mirrors-prettier rev: v4.0.0-alpha.8 diff --git a/.python-version-default b/.python-version-default new file mode 100644 index 000000000..fdcfcfdfc --- /dev/null +++ b/.python-version-default @@ -0,0 +1 @@ +3.12 \ No newline at end of file diff --git a/conda-store-server/Dockerfile b/conda-store-server/Dockerfile index a462a10d7..ae5c6da9a 100644 --- a/conda-store-server/Dockerfile +++ b/conda-store-server/Dockerfile @@ -1,10 +1,20 @@ -FROM condaforge/mambaforge:23.3.1-1 as base +# conda-store-server Dockerfile +# the generated Docker image is used with docker compose to run the conda-store +# server and public conda-store Docker images + +FROM condaforge/miniforge3:24.3.0-0 AS base LABEL org.opencontainers.image.authors="conda-store development team" +# ensure we are using the conda environment ENV PATH=/opt/conda/condabin:/opt/conda/envs/conda-store-server/bin:/opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH} ENV TZ=Etc/UTC +# must be passed at build environment (should use .python-version-default) +ARG python_version +ARG conda_env_name="conda-store-server" +ARG user_no=1000 + RUN apt-get update && \ # https://docs.anaconda.org/anaconda/install/linux/#installing-on-linux apt-get install -yq --no-install-recommends \ @@ -27,40 +37,50 @@ RUN apt-get update && \ # ensure that conda channels is empty by default # we do not want to tamper with the solveโ‰ˆ RUN printf 'channels: []\n' > /opt/conda/.condarc && \ - chown -R 1000:1000 /opt/conda && \ + chown -R ${user_no}:${user_no} /opt/conda && \ mkdir -p /opt/conda-store/conda-store && \ - chown 1000:1000 /opt/conda-store/conda-store && \ + chown ${user_no}:${user_no} /opt/conda-store/conda-store && \ mkdir -p /var/lib/conda-store && \ - chown 1000:1000 /var/lib/conda-store && \ + chown ${user_no}:${user_no} /var/lib/conda-store && \ mkdir -p /opt/conda-store/envs && \ - chown 1000:1000 /opt/conda-store/envs && \ + chown ${user_no}:${user_no} /opt/conda-store/envs && \ mkdir /.cache && \ - chown 1000:1000 /.cache - -USER 1000:1000 + chown ${user_no}:${user_no} /.cache -COPY environment.yaml /opt/conda-store-server/environment.yaml +USER ${user_no}:${user_no} -RUN mamba env create -f /opt/conda-store-server/environment.yaml && \ - conda clean --force-pkgs-dirs +RUN mamba create --name ${conda_env_name} \ + python=${python_version} conda \ + --channel conda-forge -y && \ + conda clean --force-pkgs-dirs --all -y COPY ./ /opt/conda-store-server/ USER 0:0 -RUN chown -R 1000:1000 /opt/conda-store-server/ -USER 1000:1000 +RUN chown -R ${user_no}:${user_no} /opt/conda-store-server/ +USER ${user_no}:${user_no} + +# --------------------------------------------------------------------------------- +# for production-ready images we install a specific version of conda-store-server +FROM base AS prod -FROM base as prod ARG RELEASE_VERSION -RUN cd /opt/conda-store-server && \ - /opt/conda/envs/conda-store-server/bin/pip install conda-store-server==${RELEASE_VERSION} && \ + +WORKDIR /opt/conda-store-server +RUN which python && \ + python -m pip install conda-store-server==${RELEASE_VERSION} --no-cache-dir && \ rm -rf /opt/conda-store-server/tests WORKDIR /var/lib/conda-store +# --------------------------------------------------------------------------------- -FROM base as dev +# --------------------------------------------------------------------------------- +# for development images we install conda-store-server in editable mode +FROM base AS dev -RUN cd /opt/conda-store-server && \ - /opt/conda/envs/conda-store-server/bin/pip install -e . +WORKDIR /opt/conda-store-server +RUN which python && \ + python -m pip install -e . --no-cache-dir WORKDIR /var/lib/conda-store +# --------------------------------------------------------------------------------- diff --git a/conda-store-server/conda_store_server/_internal/action/generate_constructor_installer.py b/conda-store-server/conda_store_server/_internal/action/generate_constructor_installer.py index 27100dc7e..95ae1a82a 100644 --- a/conda-store-server/conda_store_server/_internal/action/generate_constructor_installer.py +++ b/conda-store-server/conda_store_server/_internal/action/generate_constructor_installer.py @@ -40,6 +40,8 @@ def write_file(filename, s): # Checks if constructor is available try: command = [ + "python", + "-m", "constructor", "--help", ] @@ -114,9 +116,7 @@ def write_file(filename, s): } if sys.platform == "win32": - post_install = """\ -call "%PREFIX%\Scripts\\activate.bat" -""" + post_install = "\n" r'call "%PREFIX%\Scripts\activate.bat' "\n" else: post_install = """\ #!/usr/bin/env bash diff --git a/conda-store-server/conda_store_server/_internal/build.py b/conda-store-server/conda_store_server/_internal/build.py index 63258799f..2e56a3916 100644 --- a/conda-store-server/conda_store_server/_internal/build.py +++ b/conda-store-server/conda_store_server/_internal/build.py @@ -49,8 +49,13 @@ def append_to_logs(db: Session, conda_store, build, logs: typing.Union[str, byte except Exception: current_logs = b"" + if current_logs is None: + current_logs = b"" + if isinstance(logs, str): logs = logs.encode("utf-8") + elif logs is None: + logs = b"" conda_store.storage.set( db, diff --git a/conda-store-server/environment-dev.yaml b/conda-store-server/environment-dev.yaml index ba6ff6543..0ef1e428f 100644 --- a/conda-store-server/environment-dev.yaml +++ b/conda-store-server/environment-dev.yaml @@ -1,64 +1,34 @@ +# only development dependencies should be added to this file +# if you add a new dev dependency you must ensure to add it to the [tool.hatch.envs.dev] +# section of the pyproject.toml file too +# required conda-store-server or conda-store (core) dependencies MUST ONLY be added/updated in the [dependencies] +# section of the corresponding pyproject.toml file +# the only exceptions to this rule are conda and constructor which are only conda installable name: conda-store-server-dev channels: - conda-forge - - microsoft - nodefaults dependencies: - - python ==3.10 + # must be kept in sync with .python-version-default + - python=3.12 # conda builds - - conda ==23.5.2 - - conda-docker >= 0.1.2 - - python-docker - - conda-pack - - conda-lock >=1.0.5 - - conda-package-handling - - conda-package-streaming - # web server - - celery - - flower - - redis-py - - sqlalchemy<=1.4.47 - - psycopg2 - - pymysql - - requests - - uvicorn - - fastapi - - pydantic < 2.0 - - pyyaml - - traitlets - - yarl - - pyjwt - - filelock - - itsdangerous - - jinja2 - - python-multipart - - alembic - - platformdirs >=4.0,<5.0a0 - # artifact storage - - minio - # installer - - constructor - # CLI - - typer - + - conda # dev dependencies - aiohttp>=3.8.1 - - hatch - pytest - pytest-celery - pytest-mock - - black ==22.3.0 - - flake8 - - ruff - - sphinx - - myst-parser - - sphinx-panels - - sphinx-copybutton - - pydata-sphinx-theme - - playwright - - docker-py<7 # for docker-compose + - docker-py<=7 - docker-compose + # build dependencies + - hatch + - build + - twine>=5.0.0 + - pkginfo>=1.10 # needed to support metadata 2.3 + # installer + - constructor # runtime dep but must be conda installed + # linting + - pre_commit - pip - - pip: - pytest-playwright diff --git a/conda-store-server/environment-macos-dev.yaml b/conda-store-server/environment-macos-dev.yaml deleted file mode 100644 index fb9fa162b..000000000 --- a/conda-store-server/environment-macos-dev.yaml +++ /dev/null @@ -1,64 +0,0 @@ -name: conda-store-server-dev -channels: - - conda-forge - - microsoft - - nodefaults -dependencies: - - python ==3.10 - # conda builds - - conda ==23.5.2 - - python-docker - - conda-pack - - conda-lock >=1.0.5 - - mamba - - conda-package-handling - - conda-package-streaming - # web server - - celery - - flower - - redis-py - - sqlalchemy<=1.4.47 - - psycopg2 - - pymysql - - requests - - uvicorn - - fastapi - - pydantic < 2.0 - - pyyaml - - traitlets - - yarl - - pyjwt - - filelock - - itsdangerous - - jinja2 - - python-multipart - - alembic - - platformdirs >=4.0,<5.0a0 - # artifact storage - - minio - # installer - - constructor - # CLI - - typer - - # dev dependencies - - aiohttp>=3.8.1 - - hatch - - pytest - - pytest-celery - - pytest-mock - - black ==22.3.0 - - flake8 - - ruff - - sphinx - - myst-parser - - sphinx-panels - - sphinx-copybutton - - pydata-sphinx-theme - - playwright - - docker-py<7 # for docker-compose - - docker-compose - - pip - - - pip: - - pytest-playwright diff --git a/conda-store-server/environment-windows-dev.yaml b/conda-store-server/environment-windows-dev.yaml deleted file mode 100644 index fb9fa162b..000000000 --- a/conda-store-server/environment-windows-dev.yaml +++ /dev/null @@ -1,64 +0,0 @@ -name: conda-store-server-dev -channels: - - conda-forge - - microsoft - - nodefaults -dependencies: - - python ==3.10 - # conda builds - - conda ==23.5.2 - - python-docker - - conda-pack - - conda-lock >=1.0.5 - - mamba - - conda-package-handling - - conda-package-streaming - # web server - - celery - - flower - - redis-py - - sqlalchemy<=1.4.47 - - psycopg2 - - pymysql - - requests - - uvicorn - - fastapi - - pydantic < 2.0 - - pyyaml - - traitlets - - yarl - - pyjwt - - filelock - - itsdangerous - - jinja2 - - python-multipart - - alembic - - platformdirs >=4.0,<5.0a0 - # artifact storage - - minio - # installer - - constructor - # CLI - - typer - - # dev dependencies - - aiohttp>=3.8.1 - - hatch - - pytest - - pytest-celery - - pytest-mock - - black ==22.3.0 - - flake8 - - ruff - - sphinx - - myst-parser - - sphinx-panels - - sphinx-copybutton - - pydata-sphinx-theme - - playwright - - docker-py<7 # for docker-compose - - docker-compose - - pip - - - pip: - - pytest-playwright diff --git a/conda-store-server/environment.yaml b/conda-store-server/environment.yaml deleted file mode 100644 index 932504a14..000000000 --- a/conda-store-server/environment.yaml +++ /dev/null @@ -1,8 +0,0 @@ -name: conda-store-server -channels: - - conda-forge - - nodefaults -dependencies: - - python ==3.10 - # conda environment builds - - conda ==23.5.2 diff --git a/conda-store-server/pyproject.toml b/conda-store-server/pyproject.toml index c633e2fc6..d91869754 100644 --- a/conda-store-server/pyproject.toml +++ b/conda-store-server/pyproject.toml @@ -37,6 +37,7 @@ classifiers = [ dependencies = [ # env builds # conda (which cannot be installed via pip see https://github.com/conda/conda/issues/11715) + # constructor also not pip installable "python-docker", # we need platform dependent dependencies here "conda-docker; sys_platform == 'linux'", @@ -60,7 +61,7 @@ dependencies = [ "requests", "pydantic >=1.10.16,<2.0a0", "python-multipart", - "sqlalchemy<2", + "sqlalchemy<2.0a0", "traitlets", "uvicorn", "yarl", diff --git a/conda-store-server/tests/test_actions.py b/conda-store-server/tests/test_actions.py index 7a723d9e0..8fd086fea 100644 --- a/conda-store-server/tests/test_actions.py +++ b/conda-store-server/tests/test_actions.py @@ -11,6 +11,7 @@ import yaml import yarl +from celery.result import AsyncResult from conda.base.context import context as conda_base_context from constructor import construct from fastapi.responses import RedirectResponse @@ -387,7 +388,11 @@ def test_add_conda_prefix_packages(db, conda_store, simple_specification, conda_ def test_add_lockfile_packages( - db, conda_store, simple_specification, simple_conda_lock + db, + conda_store, + simple_specification, + simple_conda_lock, + celery_worker, ): task, solve_id = conda_store.register_solve(db, specification=simple_specification) @@ -400,6 +405,10 @@ def test_add_lockfile_packages( solve = api.get_solve(db, solve_id=solve_id) assert len(solve.package_builds) > 0 + result = AsyncResult(task) + result.get(timeout=30) + assert result.state == "SUCCESS" + @pytest.mark.parametrize( "is_legacy_build, build_key_version", diff --git a/conda-store-server/tests/test_app_api.py b/conda-store-server/tests/test_app_api.py index b8f816eec..08c787ed5 100644 --- a/conda-store-server/tests/test_app_api.py +++ b/conda-store-server/tests/test_app_api.py @@ -3,26 +3,31 @@ from celery.result import AsyncResult from conda_store_server import api -from conda_store_server._internal import schema +from conda_store_server._internal import action, conda_utils, schema -def test_conda_store_app_register_solve(db, conda_store, celery_worker): - conda_specification = schema.CondaSpecification( - name="pytest-name", - channels=["main"], - dependencies=["python"], +def test_conda_store_app_register_solve( + db, conda_store, simple_specification, simple_conda_lock, celery_worker +): + """Test that CondaStore can register Solve objects and dispatch solve tasks.""" + task_id, solve_id = conda_store.register_solve(db, simple_specification) + action.action_add_lockfile_packages( + db=db, + conda_lock_spec=simple_conda_lock, + solve_id=solve_id, ) - - task_id, solve_id = conda_store.register_solve(db, conda_specification) solve = api.get_solve(db, solve_id=solve_id) + platform = conda_utils.conda_platform() assert solve is not None assert solve.started_on is None assert solve.ended_on is None - assert solve.package_builds == [] - assert solve.specification.spec["name"] == conda_specification.name - assert solve.specification.spec["channels"] == conda_specification.channels - assert solve.specification.spec["dependencies"] == conda_specification.dependencies + assert len(solve.package_builds) == len( + [item for item in simple_conda_lock["package"] if item["platform"] == platform] + ) + assert solve.specification.spec["name"] == simple_specification.name + assert solve.specification.spec["channels"] == simple_specification.channels + assert solve.specification.spec["dependencies"] == simple_specification.dependencies task = AsyncResult(task_id) task.get(timeout=30) assert task.state == "SUCCESS" @@ -32,32 +37,27 @@ def test_conda_store_app_register_solve(db, conda_store, celery_worker): assert len(solve.package_builds) > 0 -def test_conda_store_register_environment_workflow(db, conda_store, celery_worker): - """Test entire environment build workflow""" - conda_specification = schema.CondaSpecification( - name="pytest-name", - channels=["main"], - dependencies=["python"], - ) +def test_conda_store_register_environment_workflow( + db, simple_specification, conda_store, celery_worker +): + """Test entire environment build workflow.""" namespace_name = "pytest-namespace" build_id = conda_store.register_environment( - db, specification=conda_specification.dict(), namespace=namespace_name + db, specification=simple_specification.dict(), namespace=namespace_name ) build = api.get_build(db, build_id=build_id) assert build is not None assert build.status == schema.BuildStatus.QUEUED - assert build.environment.name == conda_specification.name + assert build.environment.name == simple_specification.name assert build.environment.namespace.name == namespace_name - assert build.specification.spec["name"] == conda_specification.name - assert build.specification.spec["channels"] == conda_specification.channels - assert build.specification.spec["dependencies"] == conda_specification.dependencies + assert build.specification.spec["name"] == simple_specification.name + assert build.specification.spec["channels"] == simple_specification.channels + assert build.specification.spec["dependencies"] == simple_specification.dependencies # when new environment is created current_build should be the default build assert build.environment.current_build == build - from celery.result import AsyncResult - # wait for task to complete # build: environment, export, archive # docker is expected to fail will be fixed soon diff --git a/conda-store/Dockerfile b/conda-store/Dockerfile index e576bdf05..3a807874e 100644 --- a/conda-store/Dockerfile +++ b/conda-store/Dockerfile @@ -1,37 +1,66 @@ -FROM condaforge/mambaforge:23.3.1-1 as base +# conda-store (core) Dockerfile +# the generated Docker image is used with docker compose to run the conda-store +# worker and public conda-store Docker images + +FROM condaforge/miniforge3:24.3.0-0 AS base LABEL org.opencontainers.image.authors="conda-store development team" +# must be passed at build environment (should use .python-version-default) +ARG python_version +ARG conda_env_name="conda-store" +ARG user_no=1000 + +# ensure we are using the conda environment +ENV PATH=/opt/conda/condabin:/opt/conda/envs/conda-store/bin:/opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH} + RUN apt-get update && \ apt-get install -yq --no-install-recommends curl && \ apt-get clean && \ rm -rf /var/cache/apt/* &&\ rm -rf /var/lib/apt/lists/* &&\ rm -rf /tmp/* &&\ - groupadd -g 1000 conda-store &&\ - useradd -M -r -s /usr/sbin/nologin -u 1000 -g 1000 conda-store && \ + groupadd -g ${user_no} conda-store &&\ + useradd -M -r -s /usr/sbin/nologin -u ${user_no} -g ${user_no} conda-store && \ mkdir -p /opt/jupyterhub && \ chown -R conda-store:conda-store /opt/jupyterhub -COPY environment.yaml /opt/conda-store/environment.yaml - -RUN mamba env create -f /opt/conda-store/environment.yaml && \ -mamba clean --all -y && \ -conda clean --force-pkgs-dirs - -ENV PATH=/opt/conda/condabin:/opt/conda/envs/conda-store/bin:/opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH} +# MUST keep node in sync with conda-store-ui +RUN mamba create --name ${conda_env_name} \ + python=${python_version} nb_conda_kernels nodejs=18 yarn constructor \ + --channel conda-forge -y && \ + mamba clean --all -y && \ + conda clean --force-pkgs-dirs COPY ./ /opt/conda-store/ -FROM base as prod +# --------------------------------------------------------------------------------- +# for production-ready images we install a specific version of conda-store +FROM base AS prod + ARG RELEASE_VERSION -RUN cd /opt/conda-store && \ - pip install conda-store==${RELEASE_VERSION} + +WORKDIR /opt/conda-store + + +RUN which python && \ + python -m pip install conda-store==${RELEASE_VERSION} --no-cache-dir + USER conda-store WORKDIR /opt/jupyterhub +# --------------------------------------------------------------------------------- + +# --------------------------------------------------------------------------------- +# for development images we install conda-store-server in editable mode +FROM base AS dev + + +RUN --mount=type=cache,target=/root/.cache/pip +WORKDIR /opt/conda-store + +RUN which python && \ + python -m pip install -e . --no-cache-dir -FROM base as dev -RUN --mount=type=cache,target=/root/.cache/pip cd /opt/conda-store && \ - pip install -e . --no-cache USER conda-store WORKDIR /opt/jupyterhub +# --------------------------------------------------------------------------------- diff --git a/conda-store/environment.yaml b/conda-store/environment.yaml deleted file mode 100644 index a98574444..000000000 --- a/conda-store/environment.yaml +++ /dev/null @@ -1,8 +0,0 @@ -name: conda-store -channels: - - conda-forge -dependencies: - - nb_conda_kernels - - nodejs=18 - - yarn - - constructor # Runtime dependency, but must be conda-installed diff --git a/docusaurus-docs/community/contribute/contribute-code.md b/docusaurus-docs/community/contribute/contribute-code.md index f881a23ed..28820abc3 100644 --- a/docusaurus-docs/community/contribute/contribute-code.md +++ b/docusaurus-docs/community/contribute/contribute-code.md @@ -24,7 +24,7 @@ If you are a first-time contributor: Once you have a local copy of the `conda-store` repository, you can set up your development environment. There are two main ways to set up your local environment for development: -- Using [Docker and docker-compose(recommended)](#docker-based-development---conda-store-core) +- Using [Docker and Docker compose(recommended)](#docker-based-development---conda-store-core) - Local development [without Docker](#local-development-without-docker---conda-store-core) ### Docker-based development - conda-store-core @@ -32,12 +32,12 @@ There are two main ways to set up your local environment for development: Install the following dependencies before developing on `conda-store`. - [Docker](https://docs.docker.com/engine/install/) -- [docker-compose](https://docs.docker.com/compose/install/) +- [Docker compose](https://docs.docker.com/compose/install/) To deploy `conda-store` run the following command: ```bash -docker-compose up --build -d +docker compose up --build -d ``` :::important @@ -48,7 +48,7 @@ Otherwise, this workflow has been shown to run and build on OSX. **Notice** the `architecture: amd64` within the `docker-compose.yaml` files. ::: -After running the `docker-compose` command, the following resources will be available: +After running the `docker compose` command, the following resources will be available: | Resource | Localhost port | username | password | |----------|----------------|----------|----------| @@ -65,17 +65,17 @@ If you are making any changes to `conda-store-server` and would like to see those changes in the deployment, run: ```shell -docker-compose down -v # not always necessary -docker-compose up --build +docker compose down -v # not always necessary +docker compose up --build ``` To stop the deployment, run: ```bash -docker-compose stop +docker compose stop # optional to remove the containers -docker-compose rm -f +docker compose rm -f ``` ### Local development without Docker - conda-store-core @@ -125,7 +125,7 @@ If you are a first-time contributor: Once you have a local copy of the `conda-store` repository, you can set up your development environment. There are two main ways to set up your local environment for development: -- Using [Docker and docker-compose(recommended)](#docker-based-development---conda-store-ui) +- Using [Docker and Docker compose(recommended)](#docker-based-development---conda-store-ui) - Local development [without Docker](#local-development-without-docker---conda-store-ui) ### Pre-requisites @@ -137,9 +137,9 @@ There are two main ways to set up your local environment for development: Running conda-store-ui in Docker is the simplest way to set up your local development environment. -We use [docker-compose](https://docs.docker.com/compose/) to set up the infrastructure before starting, -you must ensure you have docker-compose installed. -If you need to install docker-compose, please see their [installation documentation](https://docs.docker.com/compose/install/). +We use [Docker compose](https://docs.docker.com/compose/) to set up the infrastructure before starting, +you must ensure you have Docker compose installed. +If you need to install Docker compose, please see their [installation documentation](https://docs.docker.com/compose/install/). 1. After cloning the repository change to the project directory: diff --git a/docusaurus-docs/community/contribute/testing.md b/docusaurus-docs/community/contribute/testing.md index 10596b735..c4cede077 100644 --- a/docusaurus-docs/community/contribute/testing.md +++ b/docusaurus-docs/community/contribute/testing.md @@ -23,8 +23,8 @@ docker. ```shell $ cd conda-store -$ docker-compose down -v # ensure you've cleared state -$ docker-compose up --build +$ docker compose down -v # ensure you've cleared state +$ docker compose up --build # wait until the conda-store-server is running check by visiting localhost:8080 $ pip install -e . @@ -66,8 +66,8 @@ docker. ```shell $ cd conda-store-server -$ docker-compose down -v # ensure you've cleared state -$ docker-compose up --build +$ docker compose down -v # ensure you've cleared state +$ docker compose up --build # wait until the conda-store-server is running check by visiting localhos:8080 $ hatch env run -e dev playwright-test $ hatch env run -e dev integration-test diff --git a/docusaurus-docs/conda-store-ui/how-tos/configure-ui.md b/docusaurus-docs/conda-store-ui/how-tos/configure-ui.md index 6de5011ec..5d1d71670 100644 --- a/docusaurus-docs/conda-store-ui/how-tos/configure-ui.md +++ b/docusaurus-docs/conda-store-ui/how-tos/configure-ui.md @@ -44,9 +44,9 @@ In your HTML file, add the following **before** loading the react app : ``` -## Docker-compose configuration +## Docker compose configuration -By default, docker-compose uses the latest release of conda-store-server, but there could be cases where a devloper wishes to test against a different versions, such as a release candidate. +By default, Docker compose uses the latest release of conda-store-server, but there could be cases where a devloper wishes to test against a different versions, such as a release candidate. Adding the `CONDA_STORE_SERVER_VERSION` variable to the `.env` file will allow overriding this default and setting whichever version of conda-store-server is desired. diff --git a/docusaurus-docs/conda-store/how-tos/install-docker.md b/docusaurus-docs/conda-store/how-tos/install-docker.md index 6febce5c1..63c660c5e 100644 --- a/docusaurus-docs/conda-store/how-tos/install-docker.md +++ b/docusaurus-docs/conda-store/how-tos/install-docker.md @@ -12,7 +12,7 @@ To install on a local docker daemon there is an existing `docker-compose.yaml` f The example files required are in `examples/docker`: ```shell -docker-compose up --build +docker compose up --build ``` On your web browser, visit: [https://conda-store.localhost/conda-store](https://conda-store.localhost/conda-store). diff --git a/docusaurus-docs/conda-store/references/database.md b/docusaurus-docs/conda-store/references/database.md index 0eb347fb0..a9f4d0a90 100644 --- a/docusaurus-docs/conda-store/references/database.md +++ b/docusaurus-docs/conda-store/references/database.md @@ -48,7 +48,7 @@ The procedure to modify the database is the following : - First, modify [the ORM Model](https://github.com/conda-incubator/conda-store/blob/main/conda-store-server/conda_store_server/orm.py) according to the changes you want to make - edit the file `conda-store-server/alembic.ini` and replace the value for entry `sqlalchemy.url` to match the connection URL of your database. -For example (when postgres was started via docker-compose): +For example (when postgres was started via Docker compose): ``` script_location = alembic sqlalchemy.url = postgresql+psycopg2://postgres:password@localhost:5432/conda-store diff --git a/docusaurus-docs/scripts/generate_openapi_json.py b/docusaurus-docs/scripts/generate_openapi_json.py index bc2dd145e..94346ba2d 100644 --- a/docusaurus-docs/scripts/generate_openapi_json.py +++ b/docusaurus-docs/scripts/generate_openapi_json.py @@ -4,7 +4,7 @@ from fastapi.openapi.utils import get_openapi -from conda_store_server.server import app as server_app +from conda_store_server._internal.server import app as server_app def gen_openapi_json(): diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 3c77704ca..6688b1a8e 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -1,4 +1,5 @@ {% set name = "conda-store" %} +# version MUST be updated when a new release is made {% set version = "2024.6.1" %} package: