From 2cc3c6bb7f58088667aa744dc016198ddfd9a3fb Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Mon, 1 Jul 2024 19:29:25 +0100 Subject: [PATCH 01/43] Pin baipp action --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 7fa1384bd..596352372 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -39,7 +39,7 @@ jobs: - run: echo "Running on ${{ matrix.directory }}" - name: "Check package build - ${{ matrix.directory }} ๐Ÿ“ฆ" - uses: hynek/build-and-inspect-python-package@main + uses: hynek/build-and-inspect-python-package@v2 with: path: ${{ matrix.directory }} upload-name-suffix: "-${{ matrix.directory }}" From 494160b95baebb4d16f470ab7c92eecb8f5d96c5 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Mon, 1 Jul 2024 19:33:29 +0100 Subject: [PATCH 02/43] Update concurrency group --- .github/workflows/test_build_docker_image.yaml | 3 ++- .github/workflows/test_conda_store.yaml | 3 ++- .github/workflows/test_conda_store_server_integration.yaml | 3 ++- .github/workflows/test_conda_store_server_unit.yaml | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_build_docker_image.yaml b/.github/workflows/test_build_docker_image.yaml index f1751c04d..42de64b96 100644 --- a/.github/workflows/test_build_docker_image.yaml +++ b/.github/workflows/test_build_docker_image.yaml @@ -3,6 +3,7 @@ name: "Test build Docker image" env: DEFAULT_PYTHON_VERSION: "3.10" FORCE_COLOR: "1" # Make tools pretty. + PYTHONUNBUFFERED: "1" # Make stdout and stderr behave well on: pull_request: @@ -20,7 +21,7 @@ on: # ensuring only one instance is running at a given time concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: true jobs: diff --git a/.github/workflows/test_conda_store.yaml b/.github/workflows/test_conda_store.yaml index 1dff7cabe..94d2ebfc3 100644 --- a/.github/workflows/test_conda_store.yaml +++ b/.github/workflows/test_conda_store.yaml @@ -3,6 +3,7 @@ name: "Test conda-store" env: DEFAULT_PYTHON_VERSION: "3.10" FORCE_COLOR: "1" # Make tools pretty. + PYTHONUNBUFFERED: "1" # Make stdout and stderr behave well on: pull_request: @@ -20,7 +21,7 @@ on: # ensuring only one instance is running at a given time concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: true jobs: diff --git a/.github/workflows/test_conda_store_server_integration.yaml b/.github/workflows/test_conda_store_server_integration.yaml index 19069d3d1..326b43602 100644 --- a/.github/workflows/test_conda_store_server_integration.yaml +++ b/.github/workflows/test_conda_store_server_integration.yaml @@ -3,6 +3,7 @@ name: "Test conda-store-server (integration)" env: DEFAULT_PYTHON_VERSION: "3.10" FORCE_COLOR: "1" # Make tools pretty. + PYTHONUNBUFFERED: "1" # Make stdout and stderr behave well on: pull_request: @@ -20,7 +21,7 @@ on: # ensuring only one instance is running at a given time concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: true jobs: diff --git a/.github/workflows/test_conda_store_server_unit.yaml b/.github/workflows/test_conda_store_server_unit.yaml index cb50a4873..853f7d74e 100644 --- a/.github/workflows/test_conda_store_server_unit.yaml +++ b/.github/workflows/test_conda_store_server_unit.yaml @@ -3,6 +3,7 @@ name: "Test conda-store-server (unit)" env: DEFAULT_PYTHON_VERSION: "3.10" FORCE_COLOR: "1" # Make tools pretty. + PYTHONUNBUFFERED: "1" # Make stdout and stderr behave well on: pull_request: @@ -20,7 +21,7 @@ on: # ensuring only one instance is running at a given time concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: true jobs: From 71a3a786feaa42ed0bbbe27731e6b59c7245b5ec Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Mon, 1 Jul 2024 19:39:01 +0100 Subject: [PATCH 03/43] chore: Update playwright installation command to include dependencies --- .github/workflows/test_conda_store_server_integration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_conda_store_server_integration.yaml b/.github/workflows/test_conda_store_server_integration.yaml index 326b43602..1c479c22a 100644 --- a/.github/workflows/test_conda_store_server_integration.yaml +++ b/.github/workflows/test_conda_store_server_integration.yaml @@ -62,7 +62,7 @@ jobs: - name: "Run Playwright tests ๐ŸŽญ" run: | - playwright install + playwright install --with-deps chromium pytest --video on ../tests/test_playwright.py if: matrix.test-type == 'playwright' From 74f9066a7f1146a9bb2f7507753c230b6e7c1fcd Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Mon, 1 Jul 2024 19:39:14 +0100 Subject: [PATCH 04/43] Update OS versions for unit testing in test_conda_store_server_unit.yaml --- .github/workflows/test_conda_store_server_unit.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_conda_store_server_unit.yaml b/.github/workflows/test_conda_store_server_unit.yaml index 853f7d74e..9b5813970 100644 --- a/.github/workflows/test_conda_store_server_unit.yaml +++ b/.github/workflows/test_conda_store_server_unit.yaml @@ -29,7 +29,8 @@ jobs: name: "unit-test conda-store-server" strategy: matrix: - os: ["ubuntu", "macos", "windows"] + # ubuntu 22.04, macos 14, windows 2022 + os: ["ubuntu-latest", "macos-latest", "windows-latest"] include: - os: ubuntu environment-file: conda-store-server/environment-dev.yaml From 3cc82f99be2d92fd3a63654e4148cb0328c32857 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Mon, 1 Jul 2024 19:44:05 +0100 Subject: [PATCH 05/43] Remove runtime dependencies from environment-dev.yaml --- conda-store-server/environment-dev.yaml | 66 +++++++------------------ 1 file changed, 17 insertions(+), 49 deletions(-) diff --git a/conda-store-server/environment-dev.yaml b/conda-store-server/environment-dev.yaml index ba6ff6543..66fd9da1d 100644 --- a/conda-store-server/environment-dev.yaml +++ b/conda-store-server/environment-dev.yaml @@ -1,64 +1,32 @@ +# only development dependencies should be added to this file, if you add a new dev dependency +# you must ensure to add it to the [tool.hatch.envs.dev] section of the pyproject.toml file too +# required conda-store-server or conda-store (core) dependencies MUST ONLY be added/updated in the [dependencies] +# section of the corresponding pyproject.toml file +# the only exceptions to this rule are conda and constructor which are only conda installable name: conda-store-server-dev channels: - conda-forge - - microsoft - nodefaults dependencies: - - python ==3.10 + - python=3.10 # conda builds - - conda ==23.5.2 - - conda-docker >= 0.1.2 - - python-docker - - conda-pack - - conda-lock >=1.0.5 - - conda-package-handling - - conda-package-streaming - # web server - - celery - - flower - - redis-py - - sqlalchemy<=1.4.47 - - psycopg2 - - pymysql - - requests - - uvicorn - - fastapi - - pydantic < 2.0 - - pyyaml - - traitlets - - yarl - - pyjwt - - filelock - - itsdangerous - - jinja2 - - python-multipart - - alembic - - platformdirs >=4.0,<5.0a0 - # artifact storage - - minio - # installer - - constructor - # CLI - - typer - + - conda # dev dependencies - aiohttp>=3.8.1 - - hatch - pytest - pytest-celery - pytest-mock - - black ==22.3.0 - - flake8 - - ruff - - sphinx - - myst-parser - - sphinx-panels - - sphinx-copybutton - - pydata-sphinx-theme - - playwright - - docker-py<7 # for docker-compose + - docker-py<=7 - docker-compose + # build dependencies + - hatch + - build + - twine>=5.0.0 + - pkginfo>=1.10 # needed to support metadata 2.3 + # installer + - constructor # runtime dep but must be conda installed + # linting + - pre_commit - pip - - pip: - pytest-playwright From 17604c33c7dfa7e7be78abc7f417c1962a9c6ba2 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Mon, 1 Jul 2024 19:57:05 +0100 Subject: [PATCH 06/43] Update installation command for conda-store server --- .github/workflows/test_conda_store_server_integration.yaml | 1 - .github/workflows/test_conda_store_server_unit.yaml | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_conda_store_server_integration.yaml b/.github/workflows/test_conda_store_server_integration.yaml index 1c479c22a..577797700 100644 --- a/.github/workflows/test_conda_store_server_integration.yaml +++ b/.github/workflows/test_conda_store_server_integration.yaml @@ -47,7 +47,6 @@ jobs: - name: "Install build dependencies ๐Ÿ“ฆ" run: | - pip install hatch sudo apt install wait-for-it -y - name: "Deploy docker-compose" diff --git a/.github/workflows/test_conda_store_server_unit.yaml b/.github/workflows/test_conda_store_server_unit.yaml index 9b5813970..8a2325124 100644 --- a/.github/workflows/test_conda_store_server_unit.yaml +++ b/.github/workflows/test_conda_store_server_unit.yaml @@ -67,9 +67,10 @@ jobs: run: | hatch env run -e lint lint - - name: "Build package ๐Ÿ“ฆ" + - name: "Install conda-store server ๐Ÿ“ฆ" run: | - hatch build + # and the dev dependencies + python -m pip install -e .[dev] - name: "Unit tests โœ…" run: | From 15bbcba86be32e6feeeffb609f6600928f9a0d50 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Tue, 2 Jul 2024 12:21:22 +0100 Subject: [PATCH 07/43] =?UTF-8?q?=F0=9F=93=8C=20Pin=20conda=20back=20again?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- conda-store-server/environment-dev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-store-server/environment-dev.yaml b/conda-store-server/environment-dev.yaml index 66fd9da1d..3d0560d58 100644 --- a/conda-store-server/environment-dev.yaml +++ b/conda-store-server/environment-dev.yaml @@ -10,7 +10,7 @@ channels: dependencies: - python=3.10 # conda builds - - conda + - conda ==23.5.2 # dev dependencies - aiohttp>=3.8.1 - pytest From e3d6a139b1d92e39539336733379836b8d0bec2b Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Wed, 3 Jul 2024 12:29:46 +0100 Subject: [PATCH 08/43] Simplify workflow for debugging --- .../test_conda_store_server_unit.yaml | 44 +++++++++---------- conda-store-server/environment-dev.yaml | 35 +++++++++++++++ conda-store-server/pyproject.toml | 1 + 3 files changed, 58 insertions(+), 22 deletions(-) diff --git a/.github/workflows/test_conda_store_server_unit.yaml b/.github/workflows/test_conda_store_server_unit.yaml index 8a2325124..2755013c0 100644 --- a/.github/workflows/test_conda_store_server_unit.yaml +++ b/.github/workflows/test_conda_store_server_unit.yaml @@ -1,6 +1,7 @@ name: "Test conda-store-server (unit)" env: + # needs to keep the same as the conda-store-server/environment-dev.yaml DEFAULT_PYTHON_VERSION: "3.10" FORCE_COLOR: "1" # Make tools pretty. PYTHONUNBUFFERED: "1" # Make stdout and stderr behave well @@ -18,7 +19,6 @@ on: branches: - main - # ensuring only one instance is running at a given time concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} @@ -26,19 +26,13 @@ concurrency: jobs: test-conda-store-server: - name: "unit-test conda-store-server" + name: "${{ matrix.os}} unit-test" strategy: matrix: # ubuntu 22.04, macos 14, windows 2022 - os: ["ubuntu-latest", "macos-latest", "windows-latest"] - include: - - os: ubuntu - environment-file: conda-store-server/environment-dev.yaml - - os: macos - environment-file: conda-store-server/environment-macos-dev.yaml - - os: windows - environment-file: conda-store-server/environment-windows-dev.yaml - runs-on: ${{ matrix.os }}-latest + os: ["ubuntu-latest",] + # os: ["ubuntu-latest", "macos-latest", "windows-latest"] + runs-on: ${{ matrix.os }} defaults: run: shell: bash -el {0} @@ -48,10 +42,12 @@ jobs: uses: actions/checkout@v4 - name: "Set up env ${{ matrix.os }} ๐Ÿ" - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: - environment-file: ${{ matrix.environment-file }} + environment-file: conda-store-server/environment-dev.yaml miniforge-version: latest + auto-activate-base: false + activate-environment: conda-store-server-dev # This fixes a "DLL not found" issue importing ctypes from the hatch env - name: "Reinstall Python 3.10 on Windows runner" @@ -60,21 +56,25 @@ jobs: timeout_minutes: 9999 max_attempts: 6 command: - conda install --channel=conda-forge --quiet --yes python=${{ matrix.python }} - if: matrix.os == 'windows' + conda install --channel=conda-forge --quiet --yes python=${{ env.DEFAULT_PYTHON_VERSION }} + if: matrix.os == 'windows-latest' - - name: "Linting Checks ๐Ÿงน" - run: | - hatch env run -e lint lint + # - name: "Lint code ๐Ÿงน" + # run: | + # pre-commit run --all-files - name: "Install conda-store server ๐Ÿ“ฆ" run: | - # and the dev dependencies - python -m pip install -e .[dev] + python -m pip install . + # some checks + conda list + which python - name: "Unit tests โœ…" run: | - pytest -m "not extended_prefix and not user_journey" tests + python -m pytest ./tests/test_actions.py + python -m pytest ./tests/test_app_api.py + # python -m pytest -m "not extended_prefix and not user_journey" tests # https://github.com/actions/runner-images/issues/1052 - name: "Windows extended prefix unit tests โœ…" @@ -86,4 +86,4 @@ jobs: -Type DWord (Get-ItemProperty "HKLM:System\CurrentControlSet\Control\FileSystem").LongPathsEnabled pytest -m "extended_prefix" tests - if: matrix.os == 'windows' + if: matrix.os == 'windows-latest' diff --git a/conda-store-server/environment-dev.yaml b/conda-store-server/environment-dev.yaml index 3d0560d58..3dc2a6f7e 100644 --- a/conda-store-server/environment-dev.yaml +++ b/conda-store-server/environment-dev.yaml @@ -11,6 +11,41 @@ dependencies: - python=3.10 # conda builds - conda ==23.5.2 + # -------------------------------------- + # - conda-docker >= 0.1.2 + # - python-docker + # - conda-pack + # - conda-lock >=1.0.5 + # - conda-package-handling + # - conda-package-streaming + # # web server + # - celery + # - flower + # - redis-py + # - sqlalchemy<=1.4.47 + # - psycopg2 + # - pymysql + # - requests + # - uvicorn + # - fastapi + # - pydantic < 2.0 + # - pyyaml + # - traitlets + # - yarl + # - pyjwt + # - filelock + # - itsdangerous + # - jinja2 + # - python-multipart + # - alembic + # - platformdirs >=4.0,<5.0a0 + # # artifact storage + # - minio + # # installer + # - constructor + # # CLI + # - typer + # -------------------------------------- # dev dependencies - aiohttp>=3.8.1 - pytest diff --git a/conda-store-server/pyproject.toml b/conda-store-server/pyproject.toml index c633e2fc6..aa91f4595 100644 --- a/conda-store-server/pyproject.toml +++ b/conda-store-server/pyproject.toml @@ -37,6 +37,7 @@ classifiers = [ dependencies = [ # env builds # conda (which cannot be installed via pip see https://github.com/conda/conda/issues/11715) + # constructor also not pip installable "python-docker", # we need platform dependent dependencies here "conda-docker; sys_platform == 'linux'", From 4c62c2fc6a4b191d7dae2dbf48d0186092acc214 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Wed, 3 Jul 2024 12:30:56 +0100 Subject: [PATCH 09/43] Avoid python call fails --- .../_internal/action/generate_constructor_installer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda-store-server/conda_store_server/_internal/action/generate_constructor_installer.py b/conda-store-server/conda_store_server/_internal/action/generate_constructor_installer.py index 27100dc7e..f44116d44 100644 --- a/conda-store-server/conda_store_server/_internal/action/generate_constructor_installer.py +++ b/conda-store-server/conda_store_server/_internal/action/generate_constructor_installer.py @@ -40,6 +40,8 @@ def write_file(filename, s): # Checks if constructor is available try: command = [ + "python", + "-m", "constructor", "--help", ] From 359b202d7f97a878bd3ae1399a841c2cef1bca3f Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Wed, 3 Jul 2024 12:31:11 +0100 Subject: [PATCH 10/43] Commenting env for debugging --- conda-store-server/environment-dev.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda-store-server/environment-dev.yaml b/conda-store-server/environment-dev.yaml index 3dc2a6f7e..8f9566c3b 100644 --- a/conda-store-server/environment-dev.yaml +++ b/conda-store-server/environment-dev.yaml @@ -10,13 +10,13 @@ channels: dependencies: - python=3.10 # conda builds - - conda ==23.5.2 + - conda # -------------------------------------- # - conda-docker >= 0.1.2 # - python-docker # - conda-pack # - conda-lock >=1.0.5 - # - conda-package-handling + # - conda-package- # - conda-package-streaming # # web server # - celery From 645116d951762081961790fb43ec2920b4ff2f8b Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Wed, 3 Jul 2024 12:31:34 +0100 Subject: [PATCH 11/43] Capture subprocess errors in tests --- conda-store-server/tests/test_actions.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/conda-store-server/tests/test_actions.py b/conda-store-server/tests/test_actions.py index de3ed5f6b..b8ef2ecbb 100644 --- a/conda-store-server/tests/test_actions.py +++ b/conda-store-server/tests/test_actions.py @@ -195,9 +195,16 @@ def test_generate_constructor_installer( # Runs the installer out_dir = pathlib.Path(tmp_dir) / "out" if sys.platform == "win32": - subprocess.check_output([installer, "/S", f"/D={out_dir}"]) + try: + subprocess.check_output([installer, "/S", f"/D={out_dir}"]) + except subprocess.CalledProcessError as e: + print(f"There was an error with the installer subprocess:\n{e.output}") else: - subprocess.check_output([installer, "-b", "-p", str(out_dir)]) + try: + subprocess.check_output([installer, "-b", "-p", str(out_dir)]) + # Adding this to debug since there have been failures during testing + except subprocess.CalledProcessError as e: + print(f"There was an error with the installer subprocess:\n{e.output}") # Checks the output directory assert out_dir.exists() From ead3fc3497d8fac6374a384b7e49c166d9e6ea5c Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Wed, 3 Jul 2024 12:54:00 +0100 Subject: [PATCH 12/43] Remove debug comments --- conda-store-server/environment-dev.yaml | 35 ------------------------- 1 file changed, 35 deletions(-) diff --git a/conda-store-server/environment-dev.yaml b/conda-store-server/environment-dev.yaml index 8f9566c3b..66fd9da1d 100644 --- a/conda-store-server/environment-dev.yaml +++ b/conda-store-server/environment-dev.yaml @@ -11,41 +11,6 @@ dependencies: - python=3.10 # conda builds - conda - # -------------------------------------- - # - conda-docker >= 0.1.2 - # - python-docker - # - conda-pack - # - conda-lock >=1.0.5 - # - conda-package- - # - conda-package-streaming - # # web server - # - celery - # - flower - # - redis-py - # - sqlalchemy<=1.4.47 - # - psycopg2 - # - pymysql - # - requests - # - uvicorn - # - fastapi - # - pydantic < 2.0 - # - pyyaml - # - traitlets - # - yarl - # - pyjwt - # - filelock - # - itsdangerous - # - jinja2 - # - python-multipart - # - alembic - # - platformdirs >=4.0,<5.0a0 - # # artifact storage - # - minio - # # installer - # - constructor - # # CLI - # - typer - # -------------------------------------- # dev dependencies - aiohttp>=3.8.1 - pytest From f15e622cbd13a6f65812cdaaaa1899c0c744d482 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Wed, 3 Jul 2024 13:05:31 +0100 Subject: [PATCH 13/43] :construction_worker: Add linting back to workflow --- .../test_conda_store_server_unit.yaml | 30 ++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test_conda_store_server_unit.yaml b/.github/workflows/test_conda_store_server_unit.yaml index 2755013c0..f7020075d 100644 --- a/.github/workflows/test_conda_store_server_unit.yaml +++ b/.github/workflows/test_conda_store_server_unit.yaml @@ -25,13 +25,33 @@ concurrency: cancel-in-progress: true jobs: + linting: + runs-on: ubuntu-latest + defaults: + run: + working-directory: conda-store-server + steps: + - name: "Checkout Repository ๐Ÿ›Ž" + uses: actions/checkout@v4 + + - name: "Set up Python ๐Ÿ" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.DEFAULT_PYTHON_VERSION }} + - name: "Install Dependencies ๐Ÿ“ฆ" + run: | + pip install hatch + + - name: "Linting Checks ๐Ÿงน" + run: | + hatch env run -e lint lint + test-conda-store-server: - name: "${{ matrix.os}} unit-test" + name: "unit-test - ${{ matrix.os}} " strategy: matrix: # ubuntu 22.04, macos 14, windows 2022 - os: ["ubuntu-latest",] - # os: ["ubuntu-latest", "macos-latest", "windows-latest"] + os: ["ubuntu-latest", "macos-latest", "windows-latest"] runs-on: ${{ matrix.os }} defaults: run: @@ -59,10 +79,6 @@ jobs: conda install --channel=conda-forge --quiet --yes python=${{ env.DEFAULT_PYTHON_VERSION }} if: matrix.os == 'windows-latest' - # - name: "Lint code ๐Ÿงน" - # run: | - # pre-commit run --all-files - - name: "Install conda-store server ๐Ÿ“ฆ" run: | python -m pip install . From a8b54456a029ef22efe159f4f662ee53a9b97d6a Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Wed, 3 Jul 2024 13:06:04 +0100 Subject: [PATCH 14/43] :pushpin: Fix dependency pin - sqlalchemy --- .github/workflows/conda.yml | 7 ++----- conda-store-server/pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 55513b4d8..f56765dc8 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -23,11 +23,8 @@ jobs: strategy: fail-fast: false matrix: - os: - - ubuntu-latest - - macos-latest - - macos-14 - - windows-latest + # ubuntu 22.04, macos 14, windows 2022 + os: ["ubuntu-latest", "macos-latest", "windows-latest"] steps: - name: "Checkout Repository ๐Ÿ›Ž" uses: actions/checkout@v4 diff --git a/conda-store-server/pyproject.toml b/conda-store-server/pyproject.toml index aa91f4595..d91869754 100644 --- a/conda-store-server/pyproject.toml +++ b/conda-store-server/pyproject.toml @@ -61,7 +61,7 @@ dependencies = [ "requests", "pydantic >=1.10.16,<2.0a0", "python-multipart", - "sqlalchemy<2", + "sqlalchemy<2.0a0", "traitlets", "uvicorn", "yarl", From a3c79d6ffadab8c4ffb61bf6968a7130051fed4e Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Wed, 3 Jul 2024 18:01:11 +0100 Subject: [PATCH 15/43] Remove Windows and Mac for now --- .github/workflows/test_conda_store_server_unit.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_conda_store_server_unit.yaml b/.github/workflows/test_conda_store_server_unit.yaml index f7020075d..1d47d8327 100644 --- a/.github/workflows/test_conda_store_server_unit.yaml +++ b/.github/workflows/test_conda_store_server_unit.yaml @@ -51,7 +51,8 @@ jobs: strategy: matrix: # ubuntu 22.04, macos 14, windows 2022 - os: ["ubuntu-latest", "macos-latest", "windows-latest"] + # os: ["ubuntu-latest", "macos-latest", "windows-latest"] + os: ["ubuntu-latest"] runs-on: ${{ matrix.os }} defaults: run: From e5ea78cf579dd315dff132d92e8916b452fdd4a3 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Wed, 3 Jul 2024 18:20:05 +0100 Subject: [PATCH 16/43] Remove conda pin --- .../test_conda_store_server_integration.yaml | 24 +++++++++++-------- conda-store-server/environment.yaml | 5 ++-- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test_conda_store_server_integration.yaml b/.github/workflows/test_conda_store_server_integration.yaml index 577797700..f9c50df02 100644 --- a/.github/workflows/test_conda_store_server_integration.yaml +++ b/.github/workflows/test_conda_store_server_integration.yaml @@ -26,7 +26,7 @@ concurrency: jobs: integration-test-conda-store-server: - name: "integration-test conda-store-server" + name: "integration-test - ${{ matrix.test-type }}" runs-on: ubuntu-latest strategy: matrix: @@ -42,14 +42,19 @@ jobs: - name: "Set up env ๐Ÿ" uses: conda-incubator/setup-miniconda@v3 with: - environment-file: conda-store-server/environment-dev.yaml - miniforge-version: latest + environment-file: conda-store-server/environment-dev.yaml + miniforge-version: latest + auto-activate-base: false + activate-environment: conda-store-server-dev - - name: "Install build dependencies ๐Ÿ“ฆ" + - name: "Install dependencies ๐Ÿ“ฆ" run: | + which python sudo apt install wait-for-it -y + # install conda-store-server + python -m pip install . - - name: "Deploy docker-compose" + - name: "Deploy docker-compose ๐Ÿ—๏ธ" run: | docker-compose up -d docker ps @@ -61,8 +66,8 @@ jobs: - name: "Run Playwright tests ๐ŸŽญ" run: | - playwright install --with-deps chromium - pytest --video on ../tests/test_playwright.py + python -m playwright install --with-deps chromium + python -m pytest --video on ../tests/test_playwright.py if: matrix.test-type == 'playwright' - name: "Upload test results ๐Ÿ“ค" @@ -74,13 +79,12 @@ jobs: - name: "Run integration tests โœ…" run: | - export PYTHONPATH=$PYTHONPATH:$PWD - pytest ../tests/test_api.py ../tests/test_metrics.py + python -m pytest ../tests/test_api.py ../tests/test_metrics.py if: matrix.test-type == 'integration' - name: "Run user journey tests โœ…" run: | - pytest -m "user_journey" + python -m pytest -m "user_journey" if: matrix.test-type == 'user-journey' - name: "Get Docker logs ๐Ÿ”" diff --git a/conda-store-server/environment.yaml b/conda-store-server/environment.yaml index 932504a14..91a96a044 100644 --- a/conda-store-server/environment.yaml +++ b/conda-store-server/environment.yaml @@ -1,8 +1,9 @@ +# docker-compose environment only name: conda-store-server channels: - conda-forge - nodefaults dependencies: - - python ==3.10 + - python>=3.10 # conda environment builds - - conda ==23.5.2 + - conda From d08491eb02549243ca1bb43b20b4eacce7effefe Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Wed, 3 Jul 2024 18:26:29 +0100 Subject: [PATCH 17/43] Update conda-store workflow - align with rest --- .github/workflows/test_conda_store.yaml | 14 ++++---------- .../workflows/test_conda_store_server_unit.yaml | 3 ++- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test_conda_store.yaml b/.github/workflows/test_conda_store.yaml index 94d2ebfc3..903087d89 100644 --- a/.github/workflows/test_conda_store.yaml +++ b/.github/workflows/test_conda_store.yaml @@ -45,18 +45,16 @@ jobs: - name: "Install Dependencies ๐Ÿ“ฆ" run: | - pip install hatch sudo apt install wait-for-it -y + python -m pip install hatch + # install conda-store + pip install . - name: "Linting Checks ๐Ÿงน" run: | hatch env run -e lint lint - - name: "Build package ๐Ÿ“ฆ" - run: | - hatch build - - - name: "Deploy docker-compose" + - name: "Deploy docker-compose ๐Ÿ—๏ธ" run: | docker-compose up -d docker ps @@ -65,10 +63,6 @@ jobs: wait-for-it localhost:9000 # minio wait-for-it localhost:8080 # conda-store-server - - name: "Install conda-store for tests ๐Ÿ“ฆ" - run: | - pip install . - - name: "Run basic tests - not authenticated" run: | sleep 20 diff --git a/.github/workflows/test_conda_store_server_unit.yaml b/.github/workflows/test_conda_store_server_unit.yaml index 1d47d8327..f7986743f 100644 --- a/.github/workflows/test_conda_store_server_unit.yaml +++ b/.github/workflows/test_conda_store_server_unit.yaml @@ -38,9 +38,10 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ env.DEFAULT_PYTHON_VERSION }} + - name: "Install Dependencies ๐Ÿ“ฆ" run: | - pip install hatch + python -m pip install hatch - name: "Linting Checks ๐Ÿงน" run: | From a361a8ced1a6a93e0c0e1bc74a6a7306db8a21e5 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Wed, 3 Jul 2024 18:32:03 +0100 Subject: [PATCH 18/43] :construction_worker: Only upload artefacts when needed --- .github/workflows/test_conda_store_server_integration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_conda_store_server_integration.yaml b/.github/workflows/test_conda_store_server_integration.yaml index f9c50df02..bd88351f2 100644 --- a/.github/workflows/test_conda_store_server_integration.yaml +++ b/.github/workflows/test_conda_store_server_integration.yaml @@ -72,7 +72,7 @@ jobs: - name: "Upload test results ๐Ÿ“ค" uses: actions/upload-artifact@v4 - if: ${{ always() }} && matrix.test-type == 'playwright' + if: matrix.test-type == 'playwright' with: name: playwright-tests path: conda-store-server/test-results From 8069f5cc739a6db6b8eed0858b277ef17dece25d Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Wed, 3 Jul 2024 19:03:09 +0100 Subject: [PATCH 19/43] Remove unnecessary env files --- conda-store-server/environment-dev.yaml | 5 +- conda-store-server/environment-macos-dev.yaml | 64 ------------------- .../environment-windows-dev.yaml | 64 ------------------- 3 files changed, 3 insertions(+), 130 deletions(-) delete mode 100644 conda-store-server/environment-macos-dev.yaml delete mode 100644 conda-store-server/environment-windows-dev.yaml diff --git a/conda-store-server/environment-dev.yaml b/conda-store-server/environment-dev.yaml index 66fd9da1d..1922281ac 100644 --- a/conda-store-server/environment-dev.yaml +++ b/conda-store-server/environment-dev.yaml @@ -1,5 +1,6 @@ -# only development dependencies should be added to this file, if you add a new dev dependency -# you must ensure to add it to the [tool.hatch.envs.dev] section of the pyproject.toml file too +# only development dependencies should be added to this file +# if you add a new dev dependency you must ensure to add it to the [tool.hatch.envs.dev] +# section of the pyproject.toml file too # required conda-store-server or conda-store (core) dependencies MUST ONLY be added/updated in the [dependencies] # section of the corresponding pyproject.toml file # the only exceptions to this rule are conda and constructor which are only conda installable diff --git a/conda-store-server/environment-macos-dev.yaml b/conda-store-server/environment-macos-dev.yaml deleted file mode 100644 index fb9fa162b..000000000 --- a/conda-store-server/environment-macos-dev.yaml +++ /dev/null @@ -1,64 +0,0 @@ -name: conda-store-server-dev -channels: - - conda-forge - - microsoft - - nodefaults -dependencies: - - python ==3.10 - # conda builds - - conda ==23.5.2 - - python-docker - - conda-pack - - conda-lock >=1.0.5 - - mamba - - conda-package-handling - - conda-package-streaming - # web server - - celery - - flower - - redis-py - - sqlalchemy<=1.4.47 - - psycopg2 - - pymysql - - requests - - uvicorn - - fastapi - - pydantic < 2.0 - - pyyaml - - traitlets - - yarl - - pyjwt - - filelock - - itsdangerous - - jinja2 - - python-multipart - - alembic - - platformdirs >=4.0,<5.0a0 - # artifact storage - - minio - # installer - - constructor - # CLI - - typer - - # dev dependencies - - aiohttp>=3.8.1 - - hatch - - pytest - - pytest-celery - - pytest-mock - - black ==22.3.0 - - flake8 - - ruff - - sphinx - - myst-parser - - sphinx-panels - - sphinx-copybutton - - pydata-sphinx-theme - - playwright - - docker-py<7 # for docker-compose - - docker-compose - - pip - - - pip: - - pytest-playwright diff --git a/conda-store-server/environment-windows-dev.yaml b/conda-store-server/environment-windows-dev.yaml deleted file mode 100644 index fb9fa162b..000000000 --- a/conda-store-server/environment-windows-dev.yaml +++ /dev/null @@ -1,64 +0,0 @@ -name: conda-store-server-dev -channels: - - conda-forge - - microsoft - - nodefaults -dependencies: - - python ==3.10 - # conda builds - - conda ==23.5.2 - - python-docker - - conda-pack - - conda-lock >=1.0.5 - - mamba - - conda-package-handling - - conda-package-streaming - # web server - - celery - - flower - - redis-py - - sqlalchemy<=1.4.47 - - psycopg2 - - pymysql - - requests - - uvicorn - - fastapi - - pydantic < 2.0 - - pyyaml - - traitlets - - yarl - - pyjwt - - filelock - - itsdangerous - - jinja2 - - python-multipart - - alembic - - platformdirs >=4.0,<5.0a0 - # artifact storage - - minio - # installer - - constructor - # CLI - - typer - - # dev dependencies - - aiohttp>=3.8.1 - - hatch - - pytest - - pytest-celery - - pytest-mock - - black ==22.3.0 - - flake8 - - ruff - - sphinx - - myst-parser - - sphinx-panels - - sphinx-copybutton - - pydata-sphinx-theme - - playwright - - docker-py<7 # for docker-compose - - docker-compose - - pip - - - pip: - - pytest-playwright From d2a6d2c20a235487dc0ecd7833cefe285057a6ff Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Thu, 4 Jul 2024 13:17:04 +0100 Subject: [PATCH 20/43] Remove workflow STDERR env var --- .github/workflows/test_conda_store_server_unit.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test_conda_store_server_unit.yaml b/.github/workflows/test_conda_store_server_unit.yaml index f7986743f..8737f4412 100644 --- a/.github/workflows/test_conda_store_server_unit.yaml +++ b/.github/workflows/test_conda_store_server_unit.yaml @@ -4,7 +4,6 @@ env: # needs to keep the same as the conda-store-server/environment-dev.yaml DEFAULT_PYTHON_VERSION: "3.10" FORCE_COLOR: "1" # Make tools pretty. - PYTHONUNBUFFERED: "1" # Make stdout and stderr behave well on: pull_request: From 00246725e680b415352236e62663ee3bccc781fe Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Fri, 5 Jul 2024 10:31:02 +0100 Subject: [PATCH 21/43] DEV - Simplify Docker images (#841) --- .dockerignore | 50 ++++++++++++++++ .github/workflows/build_docker_image.yaml | 6 ++ .../workflows/test_build_docker_image.yaml | 11 +++- .gitignore | 6 +- .hadolint.yaml | 5 ++ conda-store-server/Dockerfile | 57 ++++++++++++------ conda-store-server/environment.yaml | 9 --- conda-store/Dockerfile | 60 ++++++++++++++----- conda-store/environment.yaml | 8 --- 9 files changed, 155 insertions(+), 57 deletions(-) create mode 100644 .hadolint.yaml delete mode 100644 conda-store-server/environment.yaml delete mode 100644 conda-store/environment.yaml diff --git a/.dockerignore b/.dockerignore index dd39463ba..dffae8f55 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,52 @@ +# Git +.git +.gitignore +.gitattributes + +# Docker +docker-compose.yml +Dockerfile +.docker +.dockerignore + +# python +**/__pycache__ +dist +build +*.egg-info +.pytest_cache + +# VSCode +.vscode/ + +# MacOS +**/.DS_Store + +# playwright +screenshots +test-results + +# ruff +.ruff_cache + +# conda-store-ui +conda-store-server/conda_store_server/server/static/conda-store-ui/ + +# Docusaurus dependencies +/node_modules + +# Docusaurus production +/build + +# markdown +*.md +!README.md + +# ------------------------ +# conda-store specific data examples/docker/data + +# test files +conda-store-server/tests +conda-store/tests diff --git a/.github/workflows/build_docker_image.yaml b/.github/workflows/build_docker_image.yaml index e5d4d49b7..3f7b01f2f 100644 --- a/.github/workflows/build_docker_image.yaml +++ b/.github/workflows/build_docker_image.yaml @@ -65,6 +65,12 @@ jobs: type=ref,event=tag type=sha + # we need this as we are setting the context to subproject directory + - name: "Copy .dockerignore" + run: | + cp .dockerignore ${{ matrix.docker-image }}/.dockerignore + shell: bash + - name: "Publish Docker image ๐Ÿš€" uses: docker/build-push-action@v5 with: diff --git a/.github/workflows/test_build_docker_image.yaml b/.github/workflows/test_build_docker_image.yaml index 42de64b96..a8f793772 100644 --- a/.github/workflows/test_build_docker_image.yaml +++ b/.github/workflows/test_build_docker_image.yaml @@ -37,7 +37,7 @@ jobs: - name: "Checkout Repository ๐Ÿ›Ž" uses: actions/checkout@v4 - - name: "Set up Docker Buildx" + - name: "Set up Docker Buildx ๐Ÿ—" uses: docker/setup-buildx-action@v3 - name: "Lint Dockerfiles ๐Ÿ”" @@ -47,7 +47,12 @@ jobs: output_format: tty error_level: 0 - - name: "Docker Meta" + - name: "Copy .dockerignore" + run: | + cp .dockerignore ${{ matrix.docker-image }}/.dockerignore + shell: bash + + - name: "Add Docker metadata ๐Ÿ“" id: meta uses: docker/metadata-action@v5 with: @@ -56,7 +61,7 @@ jobs: tags: | type=sha - - name: "Build Docker images" + - name: "Build Docker images ๐Ÿณ" uses: docker/build-push-action@v5 with: context: "${{ matrix.docker-image }}" diff --git a/.gitignore b/.gitignore index 82cd2bc80..3c3bbabdc 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,9 @@ test-results # ruff .ruff_cache +# MacOS +**/.DS_Store + # conda-store-ui conda-store-server/conda_store_server/server/static/conda-store-ui/ @@ -45,14 +48,13 @@ conda-store-server/conda_store_server/server/static/conda-store-ui/ .cache-loader # Docusaurus misc -.DS_Store .env.local .env.development.local .env.test.local .env.production.local +# Node/bun npm-debug.log* yarn-debug.log* yarn-error.log* - *.lockb diff --git a/.hadolint.yaml b/.hadolint.yaml new file mode 100644 index 000000000..83905e9d4 --- /dev/null +++ b/.hadolint.yaml @@ -0,0 +1,5 @@ +--- +ignored: + - DL3008 # we do not pin apt deps + - DL3003 # already use workkdir + - DL3042 # already use --no-cache--dir diff --git a/conda-store-server/Dockerfile b/conda-store-server/Dockerfile index a462a10d7..7d79ba632 100644 --- a/conda-store-server/Dockerfile +++ b/conda-store-server/Dockerfile @@ -1,10 +1,19 @@ -FROM condaforge/mambaforge:23.3.1-1 as base +# conda-store-server Dockerfile +# the generated Docker image is used with docker-compose to run the conda-store +# server and public conda-store Docker images + +FROM condaforge/mambaforge:24.3.0-0 as base LABEL org.opencontainers.image.authors="conda-store development team" +# ensure we are using the conda environment ENV PATH=/opt/conda/condabin:/opt/conda/envs/conda-store-server/bin:/opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH} ENV TZ=Etc/UTC +ARG python_version=3.10 +ARG conda_env_name="conda-store-server" +ARG user_no=1000 + RUN apt-get update && \ # https://docs.anaconda.org/anaconda/install/linux/#installing-on-linux apt-get install -yq --no-install-recommends \ @@ -27,40 +36,50 @@ RUN apt-get update && \ # ensure that conda channels is empty by default # we do not want to tamper with the solveโ‰ˆ RUN printf 'channels: []\n' > /opt/conda/.condarc && \ - chown -R 1000:1000 /opt/conda && \ + chown -R ${user_no}:${user_no} /opt/conda && \ mkdir -p /opt/conda-store/conda-store && \ - chown 1000:1000 /opt/conda-store/conda-store && \ + chown ${user_no}:${user_no} /opt/conda-store/conda-store && \ mkdir -p /var/lib/conda-store && \ - chown 1000:1000 /var/lib/conda-store && \ + chown ${user_no}:${user_no} /var/lib/conda-store && \ mkdir -p /opt/conda-store/envs && \ - chown 1000:1000 /opt/conda-store/envs && \ + chown ${user_no}:${user_no} /opt/conda-store/envs && \ mkdir /.cache && \ - chown 1000:1000 /.cache - -USER 1000:1000 + chown ${user_no}:${user_no} /.cache -COPY environment.yaml /opt/conda-store-server/environment.yaml +USER ${user_no}:${user_no} -RUN mamba env create -f /opt/conda-store-server/environment.yaml && \ - conda clean --force-pkgs-dirs +RUN mamba create --name ${conda_env_name} \ + python=${python_version} conda \ + --channel conda-forge -y && \ + conda clean --force-pkgs-dirs --all -y COPY ./ /opt/conda-store-server/ USER 0:0 -RUN chown -R 1000:1000 /opt/conda-store-server/ -USER 1000:1000 +RUN chown -R ${user_no}:${user_no} /opt/conda-store-server/ +USER ${user_no}:${user_no} + +# --------------------------------------------------------------------------------- +# for production-ready images we install a specific version of conda-store-server +FROM base AS prod -FROM base as prod ARG RELEASE_VERSION -RUN cd /opt/conda-store-server && \ - /opt/conda/envs/conda-store-server/bin/pip install conda-store-server==${RELEASE_VERSION} && \ + +WORKDIR /opt/conda-store-server +RUN which python && \ + python -m pip install conda-store-server==${RELEASE_VERSION} --no-cache-dir && \ rm -rf /opt/conda-store-server/tests WORKDIR /var/lib/conda-store +# --------------------------------------------------------------------------------- -FROM base as dev +# --------------------------------------------------------------------------------- +# for development images we install conda-store-server in editable mode +FROM base AS dev -RUN cd /opt/conda-store-server && \ - /opt/conda/envs/conda-store-server/bin/pip install -e . +WORKDIR /opt/conda-store-server +RUN which python && \ + python -m pip install -e . --no-cache-dir WORKDIR /var/lib/conda-store +# --------------------------------------------------------------------------------- diff --git a/conda-store-server/environment.yaml b/conda-store-server/environment.yaml deleted file mode 100644 index 91a96a044..000000000 --- a/conda-store-server/environment.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# docker-compose environment only -name: conda-store-server -channels: - - conda-forge - - nodefaults -dependencies: - - python>=3.10 - # conda environment builds - - conda diff --git a/conda-store/Dockerfile b/conda-store/Dockerfile index e576bdf05..2c112fb25 100644 --- a/conda-store/Dockerfile +++ b/conda-store/Dockerfile @@ -1,37 +1,65 @@ -FROM condaforge/mambaforge:23.3.1-1 as base +# conda-store (core) Dockerfile +# the generated Docker image is used with docker-compose to run the conda-store +# worker and public conda-store Docker images + +FROM condaforge/mambaforge:24.3.0-0 AS base LABEL org.opencontainers.image.authors="conda-store development team" +ARG python_version=3.10 +ARG conda_env_name="conda-store" +ARG user_no=1000 + +# ensure we are using the conda environment +ENV PATH=/opt/conda/condabin:/opt/conda/envs/conda-store/bin:/opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH} + RUN apt-get update && \ apt-get install -yq --no-install-recommends curl && \ apt-get clean && \ rm -rf /var/cache/apt/* &&\ rm -rf /var/lib/apt/lists/* &&\ rm -rf /tmp/* &&\ - groupadd -g 1000 conda-store &&\ - useradd -M -r -s /usr/sbin/nologin -u 1000 -g 1000 conda-store && \ + groupadd -g ${user_no} conda-store &&\ + useradd -M -r -s /usr/sbin/nologin -u ${user_no} -g ${user_no} conda-store && \ mkdir -p /opt/jupyterhub && \ chown -R conda-store:conda-store /opt/jupyterhub -COPY environment.yaml /opt/conda-store/environment.yaml - -RUN mamba env create -f /opt/conda-store/environment.yaml && \ -mamba clean --all -y && \ -conda clean --force-pkgs-dirs - -ENV PATH=/opt/conda/condabin:/opt/conda/envs/conda-store/bin:/opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH} +# MUST keep node in sync with conda-store-ui +RUN mamba create --name ${conda_env_name} \ + python=${python_version} nb_conda_kernels nodejs=18 yarn constructor \ + --channel conda-forge -y && \ + mamba clean --all -y && \ + conda clean --force-pkgs-dirs COPY ./ /opt/conda-store/ -FROM base as prod +# --------------------------------------------------------------------------------- +# for production-ready images we install a specific version of conda-store +FROM base AS prod + ARG RELEASE_VERSION -RUN cd /opt/conda-store && \ - pip install conda-store==${RELEASE_VERSION} + +WORKDIR /opt/conda-store + + +RUN which python && \ + python -m pip install conda-store==${RELEASE_VERSION} --no-cache-dir + USER conda-store WORKDIR /opt/jupyterhub +# --------------------------------------------------------------------------------- + +# --------------------------------------------------------------------------------- +# for development images we install conda-store-server in editable mode +FROM base AS dev + + +RUN --mount=type=cache,target=/root/.cache/pip +WORKDIR /opt/conda-store + +RUN which python && \ + python -m pip install -e . --no-cache-dir -FROM base as dev -RUN --mount=type=cache,target=/root/.cache/pip cd /opt/conda-store && \ - pip install -e . --no-cache USER conda-store WORKDIR /opt/jupyterhub +# --------------------------------------------------------------------------------- diff --git a/conda-store/environment.yaml b/conda-store/environment.yaml deleted file mode 100644 index a98574444..000000000 --- a/conda-store/environment.yaml +++ /dev/null @@ -1,8 +0,0 @@ -name: conda-store -channels: - - conda-forge -dependencies: - - nb_conda_kernels - - nodejs=18 - - yarn - - constructor # Runtime dependency, but must be conda-installed From e8956f4c26bf8c785614cbdef0cf92c56956b56c Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Mon, 8 Jul 2024 19:32:48 +0100 Subject: [PATCH 22/43] DEV - Set a canonical default Python version (3.12) (#844) --- .github/workflows/build_docker_image.yaml | 6 ++++++ .github/workflows/release.yaml | 18 ++++++++++++++---- .github/workflows/test_build_docker_image.yaml | 7 ++++++- .github/workflows/test_conda_store.yaml | 4 ++-- .../test_conda_store_server_integration.yaml | 8 ++++++-- .../test_conda_store_server_unit.yaml | 14 +++++++++----- .pre-commit-config.yaml | 3 +-- .python-version-default | 1 + conda-store-server/Dockerfile | 5 +++-- conda-store-server/environment-dev.yaml | 3 ++- conda-store/Dockerfile | 3 ++- recipe/meta.yaml | 1 + 12 files changed, 53 insertions(+), 20 deletions(-) create mode 100644 .python-version-default diff --git a/.github/workflows/build_docker_image.yaml b/.github/workflows/build_docker_image.yaml index 3f7b01f2f..a03b8830e 100644 --- a/.github/workflows/build_docker_image.yaml +++ b/.github/workflows/build_docker_image.yaml @@ -25,6 +25,10 @@ jobs: - name: "Checkout Repository ๐Ÿ›Ž" uses: actions/checkout@v4 + - name: "Get project's default Python version ๐Ÿท๏ธ" + run: | + echo "PYTHON_VERSION_DEFAULT=$(cat .python-version-default)" >> $GITHUB_ENV + - name: "Retrieve secret from Vault ๐Ÿ—" uses: hashicorp/vault-action@v3 with: @@ -83,3 +87,5 @@ jobs: labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max + build-args: | + python_version=${{ env.PYTHON_VERSION_DEFAULT }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 596352372..fde0a92ad 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -20,7 +20,7 @@ permissions: jobs: # Always build & verify package. build-package: - name: Build & verify package + name: "Build & verify package" runs-on: ubuntu-latest strategy: matrix: @@ -55,7 +55,7 @@ jobs: # Upload to Test PyPI on every commit on main release-test-pypi: - name: Publish in-dev to test.pypi.org + name: "Publish in-dev to test.pypi.org" environment: release-test-pypi if: github.repository_owner == 'conda-incubator' && github.event_name == 'push' && github.ref == 'refs/heads/main' runs-on: ubuntu-latest @@ -82,9 +82,9 @@ jobs: print-hash: true verbose: true - + # Upload to PyPI on every tag/release release-pypi: - name: Publish released package to pypi.org + name: "Publish released package to pypi.org" environment: release-pypi if: github.repository_owner == 'conda-incubator' && github.event_name == 'release' && startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest @@ -128,6 +128,15 @@ jobs: - name: "Checkout Repository ๐Ÿ›Ž" uses: actions/checkout@v4 + - name: "Get project's default Python version ๐Ÿท๏ธ" + run: | + echo "PYTHON_VERSION_DEFAULT=$(cat .python-version-default)" >> $GITHUB_ENV + + - name: "Copy .dockerignore" + run: | + cp .dockerignore ${{ matrix.docker-image }}/.dockerignore + shell: bash + - name: "Retrieve secret from Vault ๐Ÿ—" uses: hashicorp/vault-action@v2 with: @@ -177,6 +186,7 @@ jobs: file: "${{ matrix.docker-image }}/Dockerfile" build-args: | RELEASE_VERSION=${{github.ref_name}} + python_version=${{ env.PYTHON_VERSION_DEFAULT }} tags: | ${{ steps.meta.outputs.tags }} push: true diff --git a/.github/workflows/test_build_docker_image.yaml b/.github/workflows/test_build_docker_image.yaml index a8f793772..6c6ef91a3 100644 --- a/.github/workflows/test_build_docker_image.yaml +++ b/.github/workflows/test_build_docker_image.yaml @@ -1,7 +1,6 @@ name: "Test build Docker image" env: - DEFAULT_PYTHON_VERSION: "3.10" FORCE_COLOR: "1" # Make tools pretty. PYTHONUNBUFFERED: "1" # Make stdout and stderr behave well @@ -37,6 +36,10 @@ jobs: - name: "Checkout Repository ๐Ÿ›Ž" uses: actions/checkout@v4 + - name: "Get project's default Python version ๐Ÿท๏ธ" + run: | + echo "PYTHON_VERSION_DEFAULT=$(cat .python-version-default)" >> $GITHUB_ENV + - name: "Set up Docker Buildx ๐Ÿ—" uses: docker/setup-buildx-action@v3 @@ -73,3 +76,5 @@ jobs: labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max + build-args: | + python_version=${{ env.PYTHON_VERSION_DEFAULT }} diff --git a/.github/workflows/test_conda_store.yaml b/.github/workflows/test_conda_store.yaml index 903087d89..30d6988bd 100644 --- a/.github/workflows/test_conda_store.yaml +++ b/.github/workflows/test_conda_store.yaml @@ -1,7 +1,6 @@ name: "Test conda-store" env: - DEFAULT_PYTHON_VERSION: "3.10" FORCE_COLOR: "1" # Make tools pretty. PYTHONUNBUFFERED: "1" # Make stdout and stderr behave well @@ -41,7 +40,8 @@ jobs: - name: "Set up Python ๐Ÿ" uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version-file: .python-version-default + cache: "pip" - name: "Install Dependencies ๐Ÿ“ฆ" run: | diff --git a/.github/workflows/test_conda_store_server_integration.yaml b/.github/workflows/test_conda_store_server_integration.yaml index bd88351f2..ea327ddbe 100644 --- a/.github/workflows/test_conda_store_server_integration.yaml +++ b/.github/workflows/test_conda_store_server_integration.yaml @@ -1,7 +1,6 @@ name: "Test conda-store-server (integration)" env: - DEFAULT_PYTHON_VERSION: "3.10" FORCE_COLOR: "1" # Make tools pretty. PYTHONUNBUFFERED: "1" # Make stdout and stderr behave well @@ -39,13 +38,18 @@ jobs: - name: "Checkout Repository ๐Ÿ›Ž" uses: actions/checkout@v4 - - name: "Set up env ๐Ÿ" + - name: "Get project's default Python version ๐Ÿท๏ธ" + run: | + echo "PYTHON_VERSION_DEFAULT=$(cat .python-version-default)" >> $GITHUB_ENV + + - name: "Set up conda env ๐Ÿ" uses: conda-incubator/setup-miniconda@v3 with: environment-file: conda-store-server/environment-dev.yaml miniforge-version: latest auto-activate-base: false activate-environment: conda-store-server-dev + python-version: ${{ env.PYTHON_VERSION_DEFAULT }} - name: "Install dependencies ๐Ÿ“ฆ" run: | diff --git a/.github/workflows/test_conda_store_server_unit.yaml b/.github/workflows/test_conda_store_server_unit.yaml index 8737f4412..b22f6a226 100644 --- a/.github/workflows/test_conda_store_server_unit.yaml +++ b/.github/workflows/test_conda_store_server_unit.yaml @@ -1,8 +1,6 @@ name: "Test conda-store-server (unit)" env: - # needs to keep the same as the conda-store-server/environment-dev.yaml - DEFAULT_PYTHON_VERSION: "3.10" FORCE_COLOR: "1" # Make tools pretty. on: @@ -36,7 +34,8 @@ jobs: - name: "Set up Python ๐Ÿ" uses: actions/setup-python@v5 with: - python-version: ${{ env.DEFAULT_PYTHON_VERSION }} + python-version-file: .python-version-default + cache: "pip" - name: "Install Dependencies ๐Ÿ“ฆ" run: | @@ -62,6 +61,10 @@ jobs: - name: "Checkout Repository ๐Ÿ›Ž" uses: actions/checkout@v4 + - name: "Get project's default Python version ๐Ÿท๏ธ" + run: | + echo "PYTHON_VERSION_DEFAULT=$(cat .python-version-default)" >> $GITHUB_ENV + - name: "Set up env ${{ matrix.os }} ๐Ÿ" uses: conda-incubator/setup-miniconda@v3 with: @@ -69,15 +72,16 @@ jobs: miniforge-version: latest auto-activate-base: false activate-environment: conda-store-server-dev + python-version: ${{ env.PYTHON_VERSION_DEFAULT }} # This fixes a "DLL not found" issue importing ctypes from the hatch env - - name: "Reinstall Python 3.10 on Windows runner" + - name: "Reinstall Python on Windows runner" uses: nick-fields/retry@v2.8.3 with: timeout_minutes: 9999 max_attempts: 6 command: - conda install --channel=conda-forge --quiet --yes python=${{ env.DEFAULT_PYTHON_VERSION }} + conda install --channel=conda-forge --quiet --yes python=${{ env.PYTHON_VERSION_DEFAULT }} if: matrix.os == 'windows-latest' - name: "Install conda-store server ๐Ÿ“ฆ" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d56b84139..6d13c9722 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,11 +31,10 @@ repos: rev: v4.6.0 hooks: - id: trailing-whitespace - files: ".*\\.py" - id: check-added-large-files - id: check-toml - id: end-of-file-fixer - - id: trailing-whitespace + exclude: ".python-version-default" - repo: https://github.com/pre-commit/mirrors-prettier rev: v4.0.0-alpha.8 diff --git a/.python-version-default b/.python-version-default new file mode 100644 index 000000000..fdcfcfdfc --- /dev/null +++ b/.python-version-default @@ -0,0 +1 @@ +3.12 \ No newline at end of file diff --git a/conda-store-server/Dockerfile b/conda-store-server/Dockerfile index 7d79ba632..664b4934b 100644 --- a/conda-store-server/Dockerfile +++ b/conda-store-server/Dockerfile @@ -2,7 +2,7 @@ # the generated Docker image is used with docker-compose to run the conda-store # server and public conda-store Docker images -FROM condaforge/mambaforge:24.3.0-0 as base +FROM condaforge/mambaforge:24.3.0-0 AS base LABEL org.opencontainers.image.authors="conda-store development team" @@ -10,7 +10,8 @@ LABEL org.opencontainers.image.authors="conda-store development team" ENV PATH=/opt/conda/condabin:/opt/conda/envs/conda-store-server/bin:/opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH} ENV TZ=Etc/UTC -ARG python_version=3.10 +# must be passed at build environment (should use .python-version-default) +ARG python_version ARG conda_env_name="conda-store-server" ARG user_no=1000 diff --git a/conda-store-server/environment-dev.yaml b/conda-store-server/environment-dev.yaml index 1922281ac..0ef1e428f 100644 --- a/conda-store-server/environment-dev.yaml +++ b/conda-store-server/environment-dev.yaml @@ -9,7 +9,8 @@ channels: - conda-forge - nodefaults dependencies: - - python=3.10 + # must be kept in sync with .python-version-default + - python=3.12 # conda builds - conda # dev dependencies diff --git a/conda-store/Dockerfile b/conda-store/Dockerfile index 2c112fb25..734c78bcb 100644 --- a/conda-store/Dockerfile +++ b/conda-store/Dockerfile @@ -6,7 +6,8 @@ FROM condaforge/mambaforge:24.3.0-0 AS base LABEL org.opencontainers.image.authors="conda-store development team" -ARG python_version=3.10 +# must be passed at build environment (should use .python-version-default) +ARG python_version ARG conda_env_name="conda-store" ARG user_no=1000 diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 3c77704ca..6688b1a8e 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -1,4 +1,5 @@ {% set name = "conda-store" %} +# version MUST be updated when a new release is made {% set version = "2024.6.1" %} package: From cd92d54db9f4546b841bfdea2c4e01b1ee512fac Mon Sep 17 00:00:00 2001 From: pdmurray Date: Tue, 23 Jul 2024 12:58:54 -0700 Subject: [PATCH 23/43] Added celery worker to test_add_lockfile_packages for testing... --- conda-store-server/tests/test_actions.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/conda-store-server/tests/test_actions.py b/conda-store-server/tests/test_actions.py index 7a723d9e0..8fd086fea 100644 --- a/conda-store-server/tests/test_actions.py +++ b/conda-store-server/tests/test_actions.py @@ -11,6 +11,7 @@ import yaml import yarl +from celery.result import AsyncResult from conda.base.context import context as conda_base_context from constructor import construct from fastapi.responses import RedirectResponse @@ -387,7 +388,11 @@ def test_add_conda_prefix_packages(db, conda_store, simple_specification, conda_ def test_add_lockfile_packages( - db, conda_store, simple_specification, simple_conda_lock + db, + conda_store, + simple_specification, + simple_conda_lock, + celery_worker, ): task, solve_id = conda_store.register_solve(db, specification=simple_specification) @@ -400,6 +405,10 @@ def test_add_lockfile_packages( solve = api.get_solve(db, solve_id=solve_id) assert len(solve.package_builds) > 0 + result = AsyncResult(task) + result.get(timeout=30) + assert result.state == "SUCCESS" + @pytest.mark.parametrize( "is_legacy_build, build_key_version", From 31c51a0979ab7b328d2b2cc7bd1479ae572f1e61 Mon Sep 17 00:00:00 2001 From: pdmurray Date: Tue, 23 Jul 2024 13:20:35 -0700 Subject: [PATCH 24/43] Test adding conda lock spec to db --- conda-store-server/tests/test_app_api.py | 30 +++++++++++++----------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/conda-store-server/tests/test_app_api.py b/conda-store-server/tests/test_app_api.py index b8f816eec..58663ee63 100644 --- a/conda-store-server/tests/test_app_api.py +++ b/conda-store-server/tests/test_app_api.py @@ -3,26 +3,30 @@ from celery.result import AsyncResult from conda_store_server import api -from conda_store_server._internal import schema +from conda_store_server._internal import action, conda_utils, schema -def test_conda_store_app_register_solve(db, conda_store, celery_worker): - conda_specification = schema.CondaSpecification( - name="pytest-name", - channels=["main"], - dependencies=["python"], +def test_conda_store_app_register_solve( + db, conda_store, simple_specification, simple_conda_lock, celery_worker +): + task_id, solve_id = conda_store.register_solve(db, simple_specification) + action.action_add_lockfile_packages( + db=db, + conda_lock_spec=simple_conda_lock, + solve_id=solve_id, ) - - task_id, solve_id = conda_store.register_solve(db, conda_specification) solve = api.get_solve(db, solve_id=solve_id) + platform = conda_utils.conda_platform() assert solve is not None assert solve.started_on is None assert solve.ended_on is None - assert solve.package_builds == [] - assert solve.specification.spec["name"] == conda_specification.name - assert solve.specification.spec["channels"] == conda_specification.channels - assert solve.specification.spec["dependencies"] == conda_specification.dependencies + assert len(solve.package_builds) == len( + filter(lambda item: item["platform"] == platform, simple_conda_lock["package"]) + ) + assert solve.specification.spec["name"] == simple_specification.name + assert solve.specification.spec["channels"] == simple_specification.channels + assert solve.specification.spec["dependencies"] == simple_specification.dependencies task = AsyncResult(task_id) task.get(timeout=30) assert task.state == "SUCCESS" @@ -56,8 +60,6 @@ def test_conda_store_register_environment_workflow(db, conda_store, celery_worke # when new environment is created current_build should be the default build assert build.environment.current_build == build - from celery.result import AsyncResult - # wait for task to complete # build: environment, export, archive # docker is expected to fail will be fixed soon From 9c03baad967e3efa4ad746cc76ac725fc865d397 Mon Sep 17 00:00:00 2001 From: pdmurray Date: Tue, 23 Jul 2024 16:14:48 -0700 Subject: [PATCH 25/43] Fix potential issue with popping `CONDA_FLAGS` env var --- .../_internal/action/generate_lockfile.py | 9 +-------- .../conda_store_server/_internal/utils.py | 18 ++++++++++++++++++ conda-store-server/tests/test_app_api.py | 5 +++-- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py b/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py index 6c79a2def..755d12c52 100644 --- a/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py +++ b/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py @@ -1,5 +1,4 @@ import json -import os import pathlib import typing @@ -53,11 +52,7 @@ def action_solve_lockfile( cuda_version = None # CONDA_FLAGS is used by conda-lock in conda_solver.solve_specs_for_arch - try: - conda_flags_name = "CONDA_FLAGS" - print(f"{conda_flags_name}={conda_flags}") - os.environ[conda_flags_name] = conda_flags - + with utils.set_environment(**{"CONDA_FLAGS": conda_flags}): run_lock( environment_files=[environment_filename], platforms=platforms, @@ -65,8 +60,6 @@ def action_solve_lockfile( conda_exe=conda_command, with_cuda=cuda_version, ) - finally: - os.environ.pop(conda_flags_name, None) with lockfile_filename.open() as f: return yaml.safe_load(f) diff --git a/conda-store-server/conda_store_server/_internal/utils.py b/conda-store-server/conda_store_server/_internal/utils.py index d47e3275d..019ccc9db 100644 --- a/conda-store-server/conda_store_server/_internal/utils.py +++ b/conda-store-server/conda_store_server/_internal/utils.py @@ -147,3 +147,21 @@ def callable_or_value(v, *args, **kwargs): if callable(v): return v(*args, **kwargs) return v + + +@contextlib.contextmanager +def set_environment(**update_env_vars: dict): + """Update the environment variables temporarily. + + Parameters + ---------- + **update_env_vars : dict + Environment variables to update + """ + old_env = os.environ.copy() + try: + os.environ.update(**update_env_vars) + yield + finally: + os.environ.clear() + os.environ = old_env diff --git a/conda-store-server/tests/test_app_api.py b/conda-store-server/tests/test_app_api.py index 58663ee63..2617a3689 100644 --- a/conda-store-server/tests/test_app_api.py +++ b/conda-store-server/tests/test_app_api.py @@ -9,6 +9,7 @@ def test_conda_store_app_register_solve( db, conda_store, simple_specification, simple_conda_lock, celery_worker ): + """Test that CondaStore can register Solve objects and dispatch solve tasks.""" task_id, solve_id = conda_store.register_solve(db, simple_specification) action.action_add_lockfile_packages( db=db, @@ -22,7 +23,7 @@ def test_conda_store_app_register_solve( assert solve.started_on is None assert solve.ended_on is None assert len(solve.package_builds) == len( - filter(lambda item: item["platform"] == platform, simple_conda_lock["package"]) + [item for item in simple_conda_lock["package"] if item["platform"] == platform] ) assert solve.specification.spec["name"] == simple_specification.name assert solve.specification.spec["channels"] == simple_specification.channels @@ -37,7 +38,7 @@ def test_conda_store_app_register_solve( def test_conda_store_register_environment_workflow(db, conda_store, celery_worker): - """Test entire environment build workflow""" + """Test entire environment build workflow.""" conda_specification = schema.CondaSpecification( name="pytest-name", channels=["main"], From 91bcb952f6b0f9ee908e02283ef2620c2682204b Mon Sep 17 00:00:00 2001 From: pdmurray Date: Tue, 23 Jul 2024 17:05:23 -0700 Subject: [PATCH 26/43] Add tmate action for debugging the runner --- .../test_conda_store_server_unit.yaml | 51 ++++++++++--------- conda-store-server/tests/conftest.py | 13 ++--- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/.github/workflows/test_conda_store_server_unit.yaml b/.github/workflows/test_conda_store_server_unit.yaml index b22f6a226..fe3c0190b 100644 --- a/.github/workflows/test_conda_store_server_unit.yaml +++ b/.github/workflows/test_conda_store_server_unit.yaml @@ -22,28 +22,28 @@ concurrency: cancel-in-progress: true jobs: - linting: - runs-on: ubuntu-latest - defaults: - run: - working-directory: conda-store-server - steps: - - name: "Checkout Repository ๐Ÿ›Ž" - uses: actions/checkout@v4 - - - name: "Set up Python ๐Ÿ" - uses: actions/setup-python@v5 - with: - python-version-file: .python-version-default - cache: "pip" - - - name: "Install Dependencies ๐Ÿ“ฆ" - run: | - python -m pip install hatch - - - name: "Linting Checks ๐Ÿงน" - run: | - hatch env run -e lint lint + # linting: + # runs-on: ubuntu-latest + # defaults: + # run: + # working-directory: conda-store-server + # steps: + # - name: "Checkout Repository ๐Ÿ›Ž" + # uses: actions/checkout@v4 + # + # - name: "Set up Python ๐Ÿ" + # uses: actions/setup-python@v5 + # with: + # python-version-file: .python-version-default + # cache: "pip" + # + # - name: "Install Dependencies ๐Ÿ“ฆ" + # run: | + # python -m pip install hatch + # + # - name: "Linting Checks ๐Ÿงน" + # run: | + # hatch env run -e lint lint test-conda-store-server: name: "unit-test - ${{ matrix.os}} " @@ -93,10 +93,13 @@ jobs: - name: "Unit tests โœ…" run: | - python -m pytest ./tests/test_actions.py - python -m pytest ./tests/test_app_api.py + python -m pytest -svv ./tests/test_actions.py::test_install_lockfile + # python -m pytest -svv ./tests/test_app_api.py # python -m pytest -m "not extended_prefix and not user_journey" tests + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + # https://github.com/actions/runner-images/issues/1052 - name: "Windows extended prefix unit tests โœ…" shell: pwsh diff --git a/conda-store-server/tests/conftest.py b/conda-store-server/tests/conftest.py index 9ea0b61e6..aa01d451c 100644 --- a/conda-store-server/tests/conftest.py +++ b/conda-store-server/tests/conftest.py @@ -37,7 +37,7 @@ def celery_config(tmp_path, conda_store): @pytest.fixture -def conda_store_config(tmp_path, request): +def conda_store_config(tmp_path): from traitlets.config import Config filename = tmp_path / ".conda-store" / "database.sqlite" @@ -47,6 +47,9 @@ def conda_store_config(tmp_path, request): storage.LocalStorage.storage_path = str(tmp_path / ".conda-store" / "storage") + original_sys_argv = list(sys.argv) + sys.argv = [sys.argv[0]] + with utils.chdir(tmp_path): yield Config( CondaStore=dict( @@ -56,13 +59,7 @@ def conda_store_config(tmp_path, request): ) ) - original_sys_argv = list(sys.argv) - sys.argv = [sys.argv[0]] - - def teardown(): - sys.argv = list(original_sys_argv) - - request.addfinalizer(teardown) + sys.argv = list(original_sys_argv) @pytest.fixture From 15191c795c8daa6d3b8121072dd969dfb07490d7 Mon Sep 17 00:00:00 2001 From: pdmurray Date: Tue, 23 Jul 2024 17:37:56 -0700 Subject: [PATCH 27/43] Safeguard against missing lockfile; ensure empty logs concat properly --- .../conda_store_server/_internal/action/generate_lockfile.py | 5 +++++ conda-store-server/conda_store_server/_internal/build.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py b/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py index 755d12c52..96c6dabb6 100644 --- a/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py +++ b/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py @@ -51,6 +51,11 @@ def action_solve_lockfile( else: cuda_version = None + # Check that the lockfile exists + if not lockfile_filename.exists(): + context.log.critical(f"Lockfile {lockfile_filename} does not exist.") + raise ValueError(f"Lockfile {lockfile_filename} does not exist.") + # CONDA_FLAGS is used by conda-lock in conda_solver.solve_specs_for_arch with utils.set_environment(**{"CONDA_FLAGS": conda_flags}): run_lock( diff --git a/conda-store-server/conda_store_server/_internal/build.py b/conda-store-server/conda_store_server/_internal/build.py index 63258799f..2e56a3916 100644 --- a/conda-store-server/conda_store_server/_internal/build.py +++ b/conda-store-server/conda_store_server/_internal/build.py @@ -49,8 +49,13 @@ def append_to_logs(db: Session, conda_store, build, logs: typing.Union[str, byte except Exception: current_logs = b"" + if current_logs is None: + current_logs = b"" + if isinstance(logs, str): logs = logs.encode("utf-8") + elif logs is None: + logs = b"" conda_store.storage.set( db, From fdc5a05b67e7eb24c4dd6783dca7b0d182bf6069 Mon Sep 17 00:00:00 2001 From: pdmurray Date: Tue, 23 Jul 2024 17:42:53 -0700 Subject: [PATCH 28/43] Reenable all unit tests; remove conda lockfile existence check --- .../test_conda_store_server_unit.yaml | 54 +++++++++---------- .../_internal/action/generate_lockfile.py | 5 -- 2 files changed, 27 insertions(+), 32 deletions(-) diff --git a/.github/workflows/test_conda_store_server_unit.yaml b/.github/workflows/test_conda_store_server_unit.yaml index fe3c0190b..1915a47b4 100644 --- a/.github/workflows/test_conda_store_server_unit.yaml +++ b/.github/workflows/test_conda_store_server_unit.yaml @@ -22,28 +22,28 @@ concurrency: cancel-in-progress: true jobs: - # linting: - # runs-on: ubuntu-latest - # defaults: - # run: - # working-directory: conda-store-server - # steps: - # - name: "Checkout Repository ๐Ÿ›Ž" - # uses: actions/checkout@v4 - # - # - name: "Set up Python ๐Ÿ" - # uses: actions/setup-python@v5 - # with: - # python-version-file: .python-version-default - # cache: "pip" - # - # - name: "Install Dependencies ๐Ÿ“ฆ" - # run: | - # python -m pip install hatch - # - # - name: "Linting Checks ๐Ÿงน" - # run: | - # hatch env run -e lint lint + linting: + runs-on: ubuntu-latest + defaults: + run: + working-directory: conda-store-server + steps: + - name: "Checkout Repository ๐Ÿ›Ž" + uses: actions/checkout@v4 + + - name: "Set up Python ๐Ÿ" + uses: actions/setup-python@v5 + with: + python-version-file: .python-version-default + cache: "pip" + + - name: "Install Dependencies ๐Ÿ“ฆ" + run: | + python -m pip install hatch + + - name: "Linting Checks ๐Ÿงน" + run: | + hatch env run -e lint lint test-conda-store-server: name: "unit-test - ${{ matrix.os}} " @@ -93,13 +93,10 @@ jobs: - name: "Unit tests โœ…" run: | - python -m pytest -svv ./tests/test_actions.py::test_install_lockfile - # python -m pytest -svv ./tests/test_app_api.py + python -m pytest -svv ./tests/test_actions.py + python -m pytest -svv ./tests/test_app_api.py # python -m pytest -m "not extended_prefix and not user_journey" tests - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - # https://github.com/actions/runner-images/issues/1052 - name: "Windows extended prefix unit tests โœ…" shell: pwsh @@ -111,3 +108,6 @@ jobs: (Get-ItemProperty "HKLM:System\CurrentControlSet\Control\FileSystem").LongPathsEnabled pytest -m "extended_prefix" tests if: matrix.os == 'windows-latest' + + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 diff --git a/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py b/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py index 96c6dabb6..755d12c52 100644 --- a/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py +++ b/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py @@ -51,11 +51,6 @@ def action_solve_lockfile( else: cuda_version = None - # Check that the lockfile exists - if not lockfile_filename.exists(): - context.log.critical(f"Lockfile {lockfile_filename} does not exist.") - raise ValueError(f"Lockfile {lockfile_filename} does not exist.") - # CONDA_FLAGS is used by conda-lock in conda_solver.solve_specs_for_arch with utils.set_environment(**{"CONDA_FLAGS": conda_flags}): run_lock( From 44030f2142e0426fc111390113770c965f5aa963 Mon Sep 17 00:00:00 2001 From: pdmurray Date: Tue, 23 Jul 2024 18:13:07 -0700 Subject: [PATCH 29/43] Revert to old environment setting method --- .../_internal/action/generate_lockfile.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py b/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py index 755d12c52..6c79a2def 100644 --- a/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py +++ b/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py @@ -1,4 +1,5 @@ import json +import os import pathlib import typing @@ -52,7 +53,11 @@ def action_solve_lockfile( cuda_version = None # CONDA_FLAGS is used by conda-lock in conda_solver.solve_specs_for_arch - with utils.set_environment(**{"CONDA_FLAGS": conda_flags}): + try: + conda_flags_name = "CONDA_FLAGS" + print(f"{conda_flags_name}={conda_flags}") + os.environ[conda_flags_name] = conda_flags + run_lock( environment_files=[environment_filename], platforms=platforms, @@ -60,6 +65,8 @@ def action_solve_lockfile( conda_exe=conda_command, with_cuda=cuda_version, ) + finally: + os.environ.pop(conda_flags_name, None) with lockfile_filename.open() as f: return yaml.safe_load(f) From 1f0530c0b05f22f7e0cbe9a45dbdf664649e4a50 Mon Sep 17 00:00:00 2001 From: pdmurray Date: Tue, 23 Jul 2024 18:29:10 -0700 Subject: [PATCH 30/43] Reenable tmate --- .github/workflows/test_conda_store_server_unit.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_conda_store_server_unit.yaml b/.github/workflows/test_conda_store_server_unit.yaml index 1915a47b4..c107c5d32 100644 --- a/.github/workflows/test_conda_store_server_unit.yaml +++ b/.github/workflows/test_conda_store_server_unit.yaml @@ -91,6 +91,9 @@ jobs: conda list which python + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + - name: "Unit tests โœ…" run: | python -m pytest -svv ./tests/test_actions.py @@ -108,6 +111,3 @@ jobs: (Get-ItemProperty "HKLM:System\CurrentControlSet\Control\FileSystem").LongPathsEnabled pytest -m "extended_prefix" tests if: matrix.os == 'windows-latest' - - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 From d9c032253aee2ee073e6d8dd7a6346ccb614f32a Mon Sep 17 00:00:00 2001 From: pdmurray Date: Tue, 23 Jul 2024 19:15:19 -0700 Subject: [PATCH 31/43] Remove tmate --- .github/workflows/test_conda_store_server_unit.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/test_conda_store_server_unit.yaml b/.github/workflows/test_conda_store_server_unit.yaml index c107c5d32..54fc64d1d 100644 --- a/.github/workflows/test_conda_store_server_unit.yaml +++ b/.github/workflows/test_conda_store_server_unit.yaml @@ -91,9 +91,6 @@ jobs: conda list which python - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - - name: "Unit tests โœ…" run: | python -m pytest -svv ./tests/test_actions.py From c3f4744e4ea997c011705fe5cdf13bc40817cd8f Mon Sep 17 00:00:00 2001 From: pdmurray Date: Wed, 24 Jul 2024 12:11:50 -0700 Subject: [PATCH 32/43] Fix invalid escape sequence in generate_constructor_installer --- .../_internal/action/generate_constructor_installer.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/conda-store-server/conda_store_server/_internal/action/generate_constructor_installer.py b/conda-store-server/conda_store_server/_internal/action/generate_constructor_installer.py index f44116d44..95ae1a82a 100644 --- a/conda-store-server/conda_store_server/_internal/action/generate_constructor_installer.py +++ b/conda-store-server/conda_store_server/_internal/action/generate_constructor_installer.py @@ -116,9 +116,7 @@ def write_file(filename, s): } if sys.platform == "win32": - post_install = """\ -call "%PREFIX%\Scripts\\activate.bat" -""" + post_install = "\n" r'call "%PREFIX%\Scripts\activate.bat' "\n" else: post_install = """\ #!/usr/bin/env bash From 1c4c16055651e6b529807407311e5c93fa3e3090 Mon Sep 17 00:00:00 2001 From: pdmurray Date: Wed, 24 Jul 2024 12:13:07 -0700 Subject: [PATCH 33/43] Add tmate back in --- .github/workflows/test_conda_store_server_unit.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test_conda_store_server_unit.yaml b/.github/workflows/test_conda_store_server_unit.yaml index 54fc64d1d..c107c5d32 100644 --- a/.github/workflows/test_conda_store_server_unit.yaml +++ b/.github/workflows/test_conda_store_server_unit.yaml @@ -91,6 +91,9 @@ jobs: conda list which python + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + - name: "Unit tests โœ…" run: | python -m pytest -svv ./tests/test_actions.py From f4378e60ce4ee45b1bdc5ff5f041367fbeffcc4d Mon Sep 17 00:00:00 2001 From: pdmurray Date: Wed, 24 Jul 2024 12:28:27 -0700 Subject: [PATCH 34/43] Explicitly tell conda to run tests with the expected environment --- .github/workflows/test_conda_store_server_unit.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_conda_store_server_unit.yaml b/.github/workflows/test_conda_store_server_unit.yaml index c107c5d32..5dad08c06 100644 --- a/.github/workflows/test_conda_store_server_unit.yaml +++ b/.github/workflows/test_conda_store_server_unit.yaml @@ -91,13 +91,13 @@ jobs: conda list which python - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 + # - name: Setup tmate session + # uses: mxschmitt/action-tmate@v3 - name: "Unit tests โœ…" run: | - python -m pytest -svv ./tests/test_actions.py - python -m pytest -svv ./tests/test_app_api.py + conda run -n conda-store-server-dev pytest ./tests/test_actions.py + conda run -n conda-store-server-dev pytest ./tests/test_app_api.py # python -m pytest -m "not extended_prefix and not user_journey" tests # https://github.com/actions/runner-images/issues/1052 From 63df5db6c29076a126ca632803bc3ef06a6beaeb Mon Sep 17 00:00:00 2001 From: pdmurray Date: Wed, 24 Jul 2024 12:59:07 -0700 Subject: [PATCH 35/43] Remove superfluous print; add DebuggingException with more context --- .../_internal/action/generate_lockfile.py | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py b/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py index 6c79a2def..077364e4d 100644 --- a/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py +++ b/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py @@ -11,6 +11,10 @@ from conda_store_server._internal.action.utils import logged_command +class DebuggingException(Exception): + pass + + @action.action def action_solve_lockfile( context, @@ -55,7 +59,7 @@ def action_solve_lockfile( # CONDA_FLAGS is used by conda-lock in conda_solver.solve_specs_for_arch try: conda_flags_name = "CONDA_FLAGS" - print(f"{conda_flags_name}={conda_flags}") + # print(f"{conda_flags_name}={conda_flags}") os.environ[conda_flags_name] = conda_flags run_lock( @@ -65,6 +69,25 @@ def action_solve_lockfile( conda_exe=conda_command, with_cuda=cuda_version, ) + except KeyError as exc: + with open(environment_filename, "r") as f: + env_file_content = f.read() + + raise DebuggingException( + "\n".join( + [ + f"environment_files={[environment_filename]}", + f"platforms={platforms}", + f"lockfile_path={lockfile_filename}", + f"conda_exe={conda_command}", + f"with_cuda={cuda_version}", + f"conda_flags_name={conda_flags_name}", + f"conda_flags={conda_flags}", + f"os.environ={os.environ}", + f"environment_files content={env_file_content}", + ] + ) + ) from exc finally: os.environ.pop(conda_flags_name, None) From 9cd2c3dd70199ac611897379869bc651feb4114d Mon Sep 17 00:00:00 2001 From: pdmurray Date: Wed, 24 Jul 2024 15:22:46 -0700 Subject: [PATCH 36/43] Use simple specification in test --- conda-store-server/tests/test_app_api.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/conda-store-server/tests/test_app_api.py b/conda-store-server/tests/test_app_api.py index 2617a3689..08c787ed5 100644 --- a/conda-store-server/tests/test_app_api.py +++ b/conda-store-server/tests/test_app_api.py @@ -37,27 +37,24 @@ def test_conda_store_app_register_solve( assert len(solve.package_builds) > 0 -def test_conda_store_register_environment_workflow(db, conda_store, celery_worker): +def test_conda_store_register_environment_workflow( + db, simple_specification, conda_store, celery_worker +): """Test entire environment build workflow.""" - conda_specification = schema.CondaSpecification( - name="pytest-name", - channels=["main"], - dependencies=["python"], - ) namespace_name = "pytest-namespace" build_id = conda_store.register_environment( - db, specification=conda_specification.dict(), namespace=namespace_name + db, specification=simple_specification.dict(), namespace=namespace_name ) build = api.get_build(db, build_id=build_id) assert build is not None assert build.status == schema.BuildStatus.QUEUED - assert build.environment.name == conda_specification.name + assert build.environment.name == simple_specification.name assert build.environment.namespace.name == namespace_name - assert build.specification.spec["name"] == conda_specification.name - assert build.specification.spec["channels"] == conda_specification.channels - assert build.specification.spec["dependencies"] == conda_specification.dependencies + assert build.specification.spec["name"] == simple_specification.name + assert build.specification.spec["channels"] == simple_specification.channels + assert build.specification.spec["dependencies"] == simple_specification.dependencies # when new environment is created current_build should be the default build assert build.environment.current_build == build From c5a10f7ed1391a9a4955d49f17dff8fd3d9a56cb Mon Sep 17 00:00:00 2001 From: pdmurray Date: Wed, 24 Jul 2024 15:41:07 -0700 Subject: [PATCH 37/43] Revert debugging changes --- .../test_conda_store_server_unit.yaml | 7 +----- .../_internal/action/generate_lockfile.py | 25 +------------------ .../conda_store_server/_internal/utils.py | 18 ------------- conda-store-server/tests/conftest.py | 13 ++++++---- 4 files changed, 10 insertions(+), 53 deletions(-) diff --git a/.github/workflows/test_conda_store_server_unit.yaml b/.github/workflows/test_conda_store_server_unit.yaml index 5dad08c06..36ffd0f91 100644 --- a/.github/workflows/test_conda_store_server_unit.yaml +++ b/.github/workflows/test_conda_store_server_unit.yaml @@ -91,14 +91,9 @@ jobs: conda list which python - # - name: Setup tmate session - # uses: mxschmitt/action-tmate@v3 - - name: "Unit tests โœ…" run: | - conda run -n conda-store-server-dev pytest ./tests/test_actions.py - conda run -n conda-store-server-dev pytest ./tests/test_app_api.py - # python -m pytest -m "not extended_prefix and not user_journey" tests + python -m pytest -m "not extended_prefix and not user_journey" tests # https://github.com/actions/runner-images/issues/1052 - name: "Windows extended prefix unit tests โœ…" diff --git a/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py b/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py index 077364e4d..6c79a2def 100644 --- a/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py +++ b/conda-store-server/conda_store_server/_internal/action/generate_lockfile.py @@ -11,10 +11,6 @@ from conda_store_server._internal.action.utils import logged_command -class DebuggingException(Exception): - pass - - @action.action def action_solve_lockfile( context, @@ -59,7 +55,7 @@ def action_solve_lockfile( # CONDA_FLAGS is used by conda-lock in conda_solver.solve_specs_for_arch try: conda_flags_name = "CONDA_FLAGS" - # print(f"{conda_flags_name}={conda_flags}") + print(f"{conda_flags_name}={conda_flags}") os.environ[conda_flags_name] = conda_flags run_lock( @@ -69,25 +65,6 @@ def action_solve_lockfile( conda_exe=conda_command, with_cuda=cuda_version, ) - except KeyError as exc: - with open(environment_filename, "r") as f: - env_file_content = f.read() - - raise DebuggingException( - "\n".join( - [ - f"environment_files={[environment_filename]}", - f"platforms={platforms}", - f"lockfile_path={lockfile_filename}", - f"conda_exe={conda_command}", - f"with_cuda={cuda_version}", - f"conda_flags_name={conda_flags_name}", - f"conda_flags={conda_flags}", - f"os.environ={os.environ}", - f"environment_files content={env_file_content}", - ] - ) - ) from exc finally: os.environ.pop(conda_flags_name, None) diff --git a/conda-store-server/conda_store_server/_internal/utils.py b/conda-store-server/conda_store_server/_internal/utils.py index 019ccc9db..d47e3275d 100644 --- a/conda-store-server/conda_store_server/_internal/utils.py +++ b/conda-store-server/conda_store_server/_internal/utils.py @@ -147,21 +147,3 @@ def callable_or_value(v, *args, **kwargs): if callable(v): return v(*args, **kwargs) return v - - -@contextlib.contextmanager -def set_environment(**update_env_vars: dict): - """Update the environment variables temporarily. - - Parameters - ---------- - **update_env_vars : dict - Environment variables to update - """ - old_env = os.environ.copy() - try: - os.environ.update(**update_env_vars) - yield - finally: - os.environ.clear() - os.environ = old_env diff --git a/conda-store-server/tests/conftest.py b/conda-store-server/tests/conftest.py index aa01d451c..9ea0b61e6 100644 --- a/conda-store-server/tests/conftest.py +++ b/conda-store-server/tests/conftest.py @@ -37,7 +37,7 @@ def celery_config(tmp_path, conda_store): @pytest.fixture -def conda_store_config(tmp_path): +def conda_store_config(tmp_path, request): from traitlets.config import Config filename = tmp_path / ".conda-store" / "database.sqlite" @@ -47,9 +47,6 @@ def conda_store_config(tmp_path): storage.LocalStorage.storage_path = str(tmp_path / ".conda-store" / "storage") - original_sys_argv = list(sys.argv) - sys.argv = [sys.argv[0]] - with utils.chdir(tmp_path): yield Config( CondaStore=dict( @@ -59,7 +56,13 @@ def conda_store_config(tmp_path): ) ) - sys.argv = list(original_sys_argv) + original_sys_argv = list(sys.argv) + sys.argv = [sys.argv[0]] + + def teardown(): + sys.argv = list(original_sys_argv) + + request.addfinalizer(teardown) @pytest.fixture From b80ba37afc66fb8aca74927c8eb8cdb23613d903 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Fri, 9 Aug 2024 16:01:32 +0100 Subject: [PATCH 38/43] Update conda-store-server/Dockerfile Co-authored-by: jaimergp --- conda-store-server/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-store-server/Dockerfile b/conda-store-server/Dockerfile index 664b4934b..7f6c76acd 100644 --- a/conda-store-server/Dockerfile +++ b/conda-store-server/Dockerfile @@ -2,7 +2,7 @@ # the generated Docker image is used with docker-compose to run the conda-store # server and public conda-store Docker images -FROM condaforge/mambaforge:24.3.0-0 AS base +FROM condaforge/miniforge:24.3.0-0 AS base LABEL org.opencontainers.image.authors="conda-store development team" From 0f82099f11eb26233a3cfec3cb8093e8360824b2 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Fri, 9 Aug 2024 16:08:10 +0100 Subject: [PATCH 39/43] Fix server_app import --- docusaurus-docs/scripts/generate_openapi_json.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docusaurus-docs/scripts/generate_openapi_json.py b/docusaurus-docs/scripts/generate_openapi_json.py index bc2dd145e..00cd87d29 100644 --- a/docusaurus-docs/scripts/generate_openapi_json.py +++ b/docusaurus-docs/scripts/generate_openapi_json.py @@ -1,11 +1,9 @@ import json - from pathlib import Path +from conda_store_server._internal.server import app as server_app from fastapi.openapi.utils import get_openapi -from conda_store_server.server import app as server_app - def gen_openapi_json(): cs_server = server_app.CondaStoreServer() From 6952c3abf61c9eaa74b5e649574b15e59239dcbd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 9 Aug 2024 15:08:31 +0000 Subject: [PATCH 40/43] [pre-commit.ci] Apply automatic pre-commit fixes --- docusaurus-docs/scripts/generate_openapi_json.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docusaurus-docs/scripts/generate_openapi_json.py b/docusaurus-docs/scripts/generate_openapi_json.py index 00cd87d29..94346ba2d 100644 --- a/docusaurus-docs/scripts/generate_openapi_json.py +++ b/docusaurus-docs/scripts/generate_openapi_json.py @@ -1,9 +1,11 @@ import json + from pathlib import Path -from conda_store_server._internal.server import app as server_app from fastapi.openapi.utils import get_openapi +from conda_store_server._internal.server import app as server_app + def gen_openapi_json(): cs_server = server_app.CondaStoreServer() From de01dc26ed00dad8fa707873fb2c609a358615c7 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Fri, 9 Aug 2024 16:48:18 +0100 Subject: [PATCH 41/43] Fix Docker base image --- conda-store-server/Dockerfile | 2 +- conda-store/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda-store-server/Dockerfile b/conda-store-server/Dockerfile index 7f6c76acd..74f5cf8dc 100644 --- a/conda-store-server/Dockerfile +++ b/conda-store-server/Dockerfile @@ -2,7 +2,7 @@ # the generated Docker image is used with docker-compose to run the conda-store # server and public conda-store Docker images -FROM condaforge/miniforge:24.3.0-0 AS base +FROM condaforge/miniforge3:24.3.0-0 AS base LABEL org.opencontainers.image.authors="conda-store development team" diff --git a/conda-store/Dockerfile b/conda-store/Dockerfile index 734c78bcb..abe89747e 100644 --- a/conda-store/Dockerfile +++ b/conda-store/Dockerfile @@ -2,7 +2,7 @@ # the generated Docker image is used with docker-compose to run the conda-store # worker and public conda-store Docker images -FROM condaforge/mambaforge:24.3.0-0 AS base +FROM condaforge/miniforge3:24.3.0-0 AS base LABEL org.opencontainers.image.authors="conda-store development team" From 85d78851f1b9518fd3e6c191d64d781dcb9eaec8 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Fri, 9 Aug 2024 17:08:27 +0100 Subject: [PATCH 42/43] Remove Docker compose v1 references --- .github/workflows/test_conda_store.yaml | 6 ++--- .../test_conda_store_server_integration.yaml | 6 ++--- conda-store-server/Dockerfile | 2 +- conda-store-server/environment-dev.yaml | 2 +- conda-store/Dockerfile | 2 +- .../community/contribute/contribute-code.md | 24 +++++++++---------- .../community/contribute/testing.md | 8 +++---- .../conda-store-ui/how-tos/configure-ui.md | 4 ++-- .../conda-store/how-tos/install-docker.md | 2 +- .../conda-store/references/database.md | 2 +- 10 files changed, 29 insertions(+), 29 deletions(-) diff --git a/.github/workflows/test_conda_store.yaml b/.github/workflows/test_conda_store.yaml index 30d6988bd..67269811d 100644 --- a/.github/workflows/test_conda_store.yaml +++ b/.github/workflows/test_conda_store.yaml @@ -54,9 +54,9 @@ jobs: run: | hatch env run -e lint lint - - name: "Deploy docker-compose ๐Ÿ—๏ธ" + - name: "Deploy docker compose ๐Ÿ—๏ธ" run: | - docker-compose up -d + docker compose up -d docker ps wait-for-it localhost:5432 # postgresql @@ -83,4 +83,4 @@ jobs: - name: "Get Docker logs ๐Ÿ”" if: ${{ failure() }} run: | - docker-compose logs + docker compose logs diff --git a/.github/workflows/test_conda_store_server_integration.yaml b/.github/workflows/test_conda_store_server_integration.yaml index ea327ddbe..4edc6645d 100644 --- a/.github/workflows/test_conda_store_server_integration.yaml +++ b/.github/workflows/test_conda_store_server_integration.yaml @@ -58,9 +58,9 @@ jobs: # install conda-store-server python -m pip install . - - name: "Deploy docker-compose ๐Ÿ—๏ธ" + - name: "Deploy docker compose ๐Ÿ—๏ธ" run: | - docker-compose up -d + docker compose up -d docker ps wait-for-it localhost:5432 # postgresql @@ -94,4 +94,4 @@ jobs: - name: "Get Docker logs ๐Ÿ”" if: ${{ failure() }} run: | - docker-compose logs + docker compose logs diff --git a/conda-store-server/Dockerfile b/conda-store-server/Dockerfile index 74f5cf8dc..ae5c6da9a 100644 --- a/conda-store-server/Dockerfile +++ b/conda-store-server/Dockerfile @@ -1,5 +1,5 @@ # conda-store-server Dockerfile -# the generated Docker image is used with docker-compose to run the conda-store +# the generated Docker image is used with docker compose to run the conda-store # server and public conda-store Docker images FROM condaforge/miniforge3:24.3.0-0 AS base diff --git a/conda-store-server/environment-dev.yaml b/conda-store-server/environment-dev.yaml index 0ef1e428f..c28d1548e 100644 --- a/conda-store-server/environment-dev.yaml +++ b/conda-store-server/environment-dev.yaml @@ -19,7 +19,7 @@ dependencies: - pytest-celery - pytest-mock - docker-py<=7 - - docker-compose + - docker compose # build dependencies - hatch - build diff --git a/conda-store/Dockerfile b/conda-store/Dockerfile index abe89747e..3a807874e 100644 --- a/conda-store/Dockerfile +++ b/conda-store/Dockerfile @@ -1,5 +1,5 @@ # conda-store (core) Dockerfile -# the generated Docker image is used with docker-compose to run the conda-store +# the generated Docker image is used with docker compose to run the conda-store # worker and public conda-store Docker images FROM condaforge/miniforge3:24.3.0-0 AS base diff --git a/docusaurus-docs/community/contribute/contribute-code.md b/docusaurus-docs/community/contribute/contribute-code.md index f881a23ed..28820abc3 100644 --- a/docusaurus-docs/community/contribute/contribute-code.md +++ b/docusaurus-docs/community/contribute/contribute-code.md @@ -24,7 +24,7 @@ If you are a first-time contributor: Once you have a local copy of the `conda-store` repository, you can set up your development environment. There are two main ways to set up your local environment for development: -- Using [Docker and docker-compose(recommended)](#docker-based-development---conda-store-core) +- Using [Docker and Docker compose(recommended)](#docker-based-development---conda-store-core) - Local development [without Docker](#local-development-without-docker---conda-store-core) ### Docker-based development - conda-store-core @@ -32,12 +32,12 @@ There are two main ways to set up your local environment for development: Install the following dependencies before developing on `conda-store`. - [Docker](https://docs.docker.com/engine/install/) -- [docker-compose](https://docs.docker.com/compose/install/) +- [Docker compose](https://docs.docker.com/compose/install/) To deploy `conda-store` run the following command: ```bash -docker-compose up --build -d +docker compose up --build -d ``` :::important @@ -48,7 +48,7 @@ Otherwise, this workflow has been shown to run and build on OSX. **Notice** the `architecture: amd64` within the `docker-compose.yaml` files. ::: -After running the `docker-compose` command, the following resources will be available: +After running the `docker compose` command, the following resources will be available: | Resource | Localhost port | username | password | |----------|----------------|----------|----------| @@ -65,17 +65,17 @@ If you are making any changes to `conda-store-server` and would like to see those changes in the deployment, run: ```shell -docker-compose down -v # not always necessary -docker-compose up --build +docker compose down -v # not always necessary +docker compose up --build ``` To stop the deployment, run: ```bash -docker-compose stop +docker compose stop # optional to remove the containers -docker-compose rm -f +docker compose rm -f ``` ### Local development without Docker - conda-store-core @@ -125,7 +125,7 @@ If you are a first-time contributor: Once you have a local copy of the `conda-store` repository, you can set up your development environment. There are two main ways to set up your local environment for development: -- Using [Docker and docker-compose(recommended)](#docker-based-development---conda-store-ui) +- Using [Docker and Docker compose(recommended)](#docker-based-development---conda-store-ui) - Local development [without Docker](#local-development-without-docker---conda-store-ui) ### Pre-requisites @@ -137,9 +137,9 @@ There are two main ways to set up your local environment for development: Running conda-store-ui in Docker is the simplest way to set up your local development environment. -We use [docker-compose](https://docs.docker.com/compose/) to set up the infrastructure before starting, -you must ensure you have docker-compose installed. -If you need to install docker-compose, please see their [installation documentation](https://docs.docker.com/compose/install/). +We use [Docker compose](https://docs.docker.com/compose/) to set up the infrastructure before starting, +you must ensure you have Docker compose installed. +If you need to install Docker compose, please see their [installation documentation](https://docs.docker.com/compose/install/). 1. After cloning the repository change to the project directory: diff --git a/docusaurus-docs/community/contribute/testing.md b/docusaurus-docs/community/contribute/testing.md index 10596b735..c4cede077 100644 --- a/docusaurus-docs/community/contribute/testing.md +++ b/docusaurus-docs/community/contribute/testing.md @@ -23,8 +23,8 @@ docker. ```shell $ cd conda-store -$ docker-compose down -v # ensure you've cleared state -$ docker-compose up --build +$ docker compose down -v # ensure you've cleared state +$ docker compose up --build # wait until the conda-store-server is running check by visiting localhost:8080 $ pip install -e . @@ -66,8 +66,8 @@ docker. ```shell $ cd conda-store-server -$ docker-compose down -v # ensure you've cleared state -$ docker-compose up --build +$ docker compose down -v # ensure you've cleared state +$ docker compose up --build # wait until the conda-store-server is running check by visiting localhos:8080 $ hatch env run -e dev playwright-test $ hatch env run -e dev integration-test diff --git a/docusaurus-docs/conda-store-ui/how-tos/configure-ui.md b/docusaurus-docs/conda-store-ui/how-tos/configure-ui.md index 6de5011ec..5d1d71670 100644 --- a/docusaurus-docs/conda-store-ui/how-tos/configure-ui.md +++ b/docusaurus-docs/conda-store-ui/how-tos/configure-ui.md @@ -44,9 +44,9 @@ In your HTML file, add the following **before** loading the react app : ``` -## Docker-compose configuration +## Docker compose configuration -By default, docker-compose uses the latest release of conda-store-server, but there could be cases where a devloper wishes to test against a different versions, such as a release candidate. +By default, Docker compose uses the latest release of conda-store-server, but there could be cases where a devloper wishes to test against a different versions, such as a release candidate. Adding the `CONDA_STORE_SERVER_VERSION` variable to the `.env` file will allow overriding this default and setting whichever version of conda-store-server is desired. diff --git a/docusaurus-docs/conda-store/how-tos/install-docker.md b/docusaurus-docs/conda-store/how-tos/install-docker.md index 6febce5c1..63c660c5e 100644 --- a/docusaurus-docs/conda-store/how-tos/install-docker.md +++ b/docusaurus-docs/conda-store/how-tos/install-docker.md @@ -12,7 +12,7 @@ To install on a local docker daemon there is an existing `docker-compose.yaml` f The example files required are in `examples/docker`: ```shell -docker-compose up --build +docker compose up --build ``` On your web browser, visit: [https://conda-store.localhost/conda-store](https://conda-store.localhost/conda-store). diff --git a/docusaurus-docs/conda-store/references/database.md b/docusaurus-docs/conda-store/references/database.md index 0eb347fb0..a9f4d0a90 100644 --- a/docusaurus-docs/conda-store/references/database.md +++ b/docusaurus-docs/conda-store/references/database.md @@ -48,7 +48,7 @@ The procedure to modify the database is the following : - First, modify [the ORM Model](https://github.com/conda-incubator/conda-store/blob/main/conda-store-server/conda_store_server/orm.py) according to the changes you want to make - edit the file `conda-store-server/alembic.ini` and replace the value for entry `sqlalchemy.url` to match the connection URL of your database. -For example (when postgres was started via docker-compose): +For example (when postgres was started via Docker compose): ``` script_location = alembic sqlalchemy.url = postgresql+psycopg2://postgres:password@localhost:5432/conda-store From 43e93dd49be54ee6381382ef410daa4b091231d6 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Fri, 9 Aug 2024 17:11:08 +0100 Subject: [PATCH 43/43] Revert to docker-compose in env file --- conda-store-server/environment-dev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-store-server/environment-dev.yaml b/conda-store-server/environment-dev.yaml index c28d1548e..0ef1e428f 100644 --- a/conda-store-server/environment-dev.yaml +++ b/conda-store-server/environment-dev.yaml @@ -19,7 +19,7 @@ dependencies: - pytest-celery - pytest-mock - docker-py<=7 - - docker compose + - docker-compose # build dependencies - hatch - build