From 8d1136e3d462671ff5f8630b874145ceb55596db Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Wed, 20 Mar 2024 16:32:01 +0000 Subject: [PATCH 01/26] :pushpin: Clean dev and runtime deps - server --- conda-store-server/environment-dev.yaml | 39 +------------- conda-store-server/environment-macos-dev.yaml | 38 +------------- .../environment-windows-dev.yaml | 39 +------------- conda-store-server/pyproject.toml | 52 +++++++++++++------ conda-store/environment.yaml | 11 +--- conda-store/pyproject.toml | 26 +++++----- 6 files changed, 54 insertions(+), 151 deletions(-) diff --git a/conda-store-server/environment-dev.yaml b/conda-store-server/environment-dev.yaml index b7dc836ad..2d600e48d 100644 --- a/conda-store-server/environment-dev.yaml +++ b/conda-store-server/environment-dev.yaml @@ -7,57 +7,20 @@ dependencies: - python ==3.10 # conda builds - conda ==23.5.2 - - conda-docker >= 0.1.2 - - python-docker - - conda-pack - - conda-lock >=1.0.5 - - conda-package-handling - - conda-package-streaming # web server - - celery - flower - - redis-py - - sqlalchemy<=1.4.47 - - psycopg2 - - pymysql - - requests - - uvicorn - - fastapi - - pydantic < 2.0 - - pyyaml - - traitlets - - yarl - - pyjwt - - filelock - - itsdangerous - - jinja2 - - python-multipart - - alembic - # artifact storage - - minio - # installer - - constructor - # CLI - - typer - # dev dependencies - aiohttp>=3.8.1 - hatch - pytest - pytest-celery - pytest-mock - - black ==22.3.0 + - black - flake8 - ruff - - sphinx - - myst-parser - - sphinx-panels - - sphinx-copybutton - - pydata-sphinx-theme - playwright - docker-py<7 # for docker-compose - docker-compose - pip - - pip: - pytest-playwright diff --git a/conda-store-server/environment-macos-dev.yaml b/conda-store-server/environment-macos-dev.yaml index 3cab8855d..32caa941d 100644 --- a/conda-store-server/environment-macos-dev.yaml +++ b/conda-store-server/environment-macos-dev.yaml @@ -8,56 +8,20 @@ dependencies: # conda builds - conda ==23.5.2 - python-docker - - conda-pack - - conda-lock >=1.0.5 - - mamba - - conda-package-handling - - conda-package-streaming # web server - - celery - flower - - redis-py - - sqlalchemy<=1.4.47 - - psycopg2 - - pymysql - - requests - - uvicorn - - fastapi - - pydantic < 2.0 - - pyyaml - - traitlets - - yarl - - pyjwt - - filelock - - itsdangerous - - jinja2 - - python-multipart - - alembic - # artifact storage - - minio - # installer - - constructor - # CLI - - typer - # dev dependencies - aiohttp>=3.8.1 - hatch - pytest - pytest-celery - pytest-mock - - black ==22.3.0 + - black - flake8 - ruff - - sphinx - - myst-parser - - sphinx-panels - - sphinx-copybutton - - pydata-sphinx-theme - playwright - docker-py<7 # for docker-compose - docker-compose - pip - - pip: - pytest-playwright diff --git a/conda-store-server/environment-windows-dev.yaml b/conda-store-server/environment-windows-dev.yaml index 3cab8855d..2d600e48d 100644 --- a/conda-store-server/environment-windows-dev.yaml +++ b/conda-store-server/environment-windows-dev.yaml @@ -7,57 +7,20 @@ dependencies: - python ==3.10 # conda builds - conda ==23.5.2 - - python-docker - - conda-pack - - conda-lock >=1.0.5 - - mamba - - conda-package-handling - - conda-package-streaming # web server - - celery - flower - - redis-py - - sqlalchemy<=1.4.47 - - psycopg2 - - pymysql - - requests - - uvicorn - - fastapi - - pydantic < 2.0 - - pyyaml - - traitlets - - yarl - - pyjwt - - filelock - - itsdangerous - - jinja2 - - python-multipart - - alembic - # artifact storage - - minio - # installer - - constructor - # CLI - - typer - # dev dependencies - aiohttp>=3.8.1 - hatch - pytest - pytest-celery - pytest-mock - - black ==22.3.0 + - black - flake8 - ruff - - sphinx - - myst-parser - - sphinx-panels - - sphinx-copybutton - - pydata-sphinx-theme - playwright - docker-py<7 # for docker-compose - docker-compose - pip - - pip: - pytest-playwright diff --git a/conda-store-server/pyproject.toml b/conda-store-server/pyproject.toml index c5c3e6093..4f1129157 100644 --- a/conda-store-server/pyproject.toml +++ b/conda-store-server/pyproject.toml @@ -13,6 +13,9 @@ keywords = ["conda"] authors = [ { name = "Christopher Ostrouchov", email = "crhsi.ostrouchov@gmail.com" }, ] +maintainers = [{ name = "Tania Allard", email = "trallard@bitsandchips.me" }, + {name= "Pavithra Eswaramoorthy", email = " pavithraes@outlook.com"}, + ] classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", @@ -25,32 +28,48 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3 :: Only", ] + +dynamic = ["version"] + dependencies = [ + # env builds + # conda (which cannot be installed via pip see https://github.com/conda/conda/issues/11715) + "python-docker", + # we need platform dependent dependencies here "conda-docker; sys_platform == 'linux'", + "conda-lock >=1.0.5", "conda-pack", + "conda-package-handling", + "conda-package-streaming", + # web server + "alembic", "celery", - "sqlalchemy", - "requests", "fastapi", + "filelock", + "itsdangerous", + "jinja2", + "pyjwt", + "psycopg2-binary", + "pymysql", "pyyaml", "redis", + "requests", "pydantic<2", - "minio", + "python-multipart", + "sqlalchemy<=1.4.47", "traitlets", - "pyjwt", + "uvicorn", "yarl", - "filelock", - "itsdangerous", - "jinja2", - "python-multipart", - "python-docker" - # conda (which should not be included here) + # installer + "constructor", + # artifact storage + "minio", ] -dynamic = ["version"] [project.urls] Homepage = "https://conda.store/" Source = "https://github.com/conda-incubator/conda-store" +Issues = "https://github.com/conda-incubator/conda-store/issues" [tool.hatch.version] path = "conda_store_server/__init__.py" @@ -60,13 +79,14 @@ dev = ["build", "twine"] [tool.hatch.envs.dev] dependencies = [ + "aiohttp>=3.8.1", + "docker-compose", + "docker-py<7", + "flower", + "playwright", "pre-commit", - "sphinx", - "myst-parser", - "sphinx-panels", - "sphinx-copybutton", - "pydata-sphinx-theme", "pytest", + "pytest-celery", "pytest-mock", "pytest-playwright", ] diff --git a/conda-store/environment.yaml b/conda-store/environment.yaml index 122cb1d1b..9e112c875 100644 --- a/conda-store/environment.yaml +++ b/conda-store/environment.yaml @@ -2,19 +2,12 @@ name: conda-store channels: - conda-forge dependencies: - - pip - jupyterhub<2.0.0 - jupyterlab>=3.0.0 - nb_conda_kernels - # - nb_conda_store_kernels >=0.1.4 - - nodejs=16 + - nodejs=18 - yarn - # conda-store client dependencies - - yarl - - aiohttp>=3.8.1 - - rich - - click - - ruamel.yaml + - pip - pip: - https://github.com/yuvipanda/jupyter-launcher-shortcuts/archive/refs/heads/master.zip - jupyterlab-conda-store diff --git a/conda-store/pyproject.toml b/conda-store/pyproject.toml index 012a8dded..1dad28105 100644 --- a/conda-store/pyproject.toml +++ b/conda-store/pyproject.toml @@ -13,8 +13,10 @@ keywords = ["conda"] authors = [ { name = "Christopher Ostrouchov", email = "chris.ostrouchov@gmail.com" }, ] -maintainers = [{ name = "Tania Allard", email = "trallard@bitsandchips.me" }] -# TODO: update maintainers +maintainers = [{ name = "Tania Allard", email = "trallard@bitsandchips.me" }, + {name= "Pavithra Eswaramoorthy", email = " pavithraes@outlook.com"}, + ] + classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", @@ -31,29 +33,28 @@ classifiers = [ "Intended Audience :: Science/Research", "Intended Audience :: System Administrators", ] -dependencies = ["rich", "click", "yarl", "aiohttp", "ruamel.yaml"] dynamic = ["version"] +dependencies = ["rich", + "click", + "yarl", + "aiohttp>=3.8.1", + "ruamel.yaml"] [project.urls] Homepage = "https://conda.store/" Source = "https://github.com/conda-incubator/conda-store" +Issues = "https://github.com/conda-incubator/conda-store/issues" [tool.hatch.version] path = "conda_store/__init__.py" -[project.optional-dependencies] -dev = ["build", "twine"] - [tool.hatch.envs.dev] dependencies = [ + "build", "pre-commit", - "sphinx", - "myst-parser", - "sphinx-panels", - "sphinx-copybutton", - "pydata-sphinx-theme", "pytest", "pytest-mock", + "twine", ] [tool.hatch.envs.dev.scripts] @@ -65,10 +66,9 @@ conda-store = "conda_store.__main__:main" [tool.black] line-length = 88 -target-version = ['py38', 'py39', 'py310'] +target-version = ['py38', 'py39', 'py310', 'py311'] [tool.ruff] ignore = [ "E501", # line-length - ] From b85208ee3daf881ed2dd5922acfa50be1c5b24c0 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Wed, 20 Mar 2024 16:59:13 +0000 Subject: [PATCH 02/26] :arrow_up: Bump pre-commit deps --- .pre-commit-config.yaml | 10 +++++----- conda-store-server/pyproject.toml | 5 ++--- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 164a8bf16..a7b165c51 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,21 +10,21 @@ repos: - repo: https://github.com/psf/black - rev: 23.9.1 + rev: 24.3.0 hooks: - id: black exclude: "examples|tests/assets" - repo: https://github.com/charliermarsh/ruff-pre-commit # Ruff version. - rev: "v0.0.289" + rev: "v0.3.3" hooks: - id: ruff exclude: "examples|tests/assets" args: ["--fix"] - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort name: isort @@ -34,7 +34,7 @@ repos: exclude: "conda-store-server/conda_store_server/action/__init__.py" - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: trailing-whitespace files: ".*\\.py" @@ -44,7 +44,7 @@ repos: - id: trailing-whitespace - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.3 + rev: v4.0.0-alpha.8 hooks: - id: prettier exclude: ^(examples/|templates/|) diff --git a/conda-store-server/pyproject.toml b/conda-store-server/pyproject.toml index 4f1129157..6d79afa42 100644 --- a/conda-store-server/pyproject.toml +++ b/conda-store-server/pyproject.toml @@ -74,12 +74,10 @@ Issues = "https://github.com/conda-incubator/conda-store/issues" [tool.hatch.version] path = "conda_store_server/__init__.py" -[project.optional-dependencies] -dev = ["build", "twine"] - [tool.hatch.envs.dev] dependencies = [ "aiohttp>=3.8.1", + "build", "docker-compose", "docker-py<7", "flower", @@ -89,6 +87,7 @@ dependencies = [ "pytest-celery", "pytest-mock", "pytest-playwright", + "twine" ] [tool.hatch.envs.dev.scripts] From 48ec413f8e98df06715ceff3f4f3f8a2d3d60c1f Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Wed, 20 Mar 2024 17:07:08 +0000 Subject: [PATCH 03/26] :heavy_minus_sign: Remove runtime deps --- conda-store-server/environment-dev.yaml | 1 + conda-store-server/environment.yaml | 28 ------------------------- conda-store/environment.yaml | 2 +- 3 files changed, 2 insertions(+), 29 deletions(-) diff --git a/conda-store-server/environment-dev.yaml b/conda-store-server/environment-dev.yaml index 2d600e48d..32caa941d 100644 --- a/conda-store-server/environment-dev.yaml +++ b/conda-store-server/environment-dev.yaml @@ -7,6 +7,7 @@ dependencies: - python ==3.10 # conda builds - conda ==23.5.2 + - python-docker # web server - flower # dev dependencies diff --git a/conda-store-server/environment.yaml b/conda-store-server/environment.yaml index f3b8e40f9..a865ffd32 100644 --- a/conda-store-server/environment.yaml +++ b/conda-store-server/environment.yaml @@ -6,33 +6,5 @@ dependencies: - python ==3.10 # conda environment builds - conda ==23.5.2 - - python-docker - - conda-docker >= 0.1.2 - - conda-pack - - conda-lock >=1.0.5 - - conda-package-handling - - conda-package-streaming # web server - - celery - flower - - redis-py - - sqlalchemy<=1.4.47 - - alembic - - psycopg2 - - pymysql - - requests - - pyyaml - - uvicorn - - fastapi - - pydantic < 2.0 - - traitlets - - yarl - - pyjwt - - filelock - - itsdangerous - - jinja2 - - python-multipart - # artifact storage - - minio - # installer - - constructor diff --git a/conda-store/environment.yaml b/conda-store/environment.yaml index 9e112c875..aadebdf6a 100644 --- a/conda-store/environment.yaml +++ b/conda-store/environment.yaml @@ -3,7 +3,7 @@ channels: - conda-forge dependencies: - jupyterhub<2.0.0 - - jupyterlab>=3.0.0 + - jupyterlab>4.0 - nb_conda_kernels - nodejs=18 - yarn From 757677fd0e0f81cd4e9bb68751b5072536972e99 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 20 Mar 2024 17:20:29 +0000 Subject: [PATCH 04/26] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../alembic/versions/03c839888c82_add_canceled_status.py | 1 + .../alembic/versions/0f7e23ff24ee_add_worker.py | 1 + ...6f65805dc8f_split_conda_package_into_conda_package_.py | 1 + .../versions/30b37e725c32_add_build_key_version.py | 1 + .../alembic/versions/48be4072fe58_initial_schema.py | 1 + .../alembic/versions/57cd11b949d5_add_installer.py | 1 + ...ad723de2abd_adding_container_registry_value_to_enum.py | 1 + .../alembic/versions/771180018e1b_add_v2_role_mappings.py | 1 + .../versions/8d63a091aff8_add_environment_description.py | 1 + .../versions/abd7248d5327_adding_a_settings_table.py | 1 + .../alembic/versions/b387747ca9b7_role_mapping.py | 1 + .../alembic/versions/d78e9889566a_add_status_info.py | 1 + conda-store-server/conda_store_server/server/views/api.py | 8 +++++--- .../conda_store_server/server/views/registry.py | 8 +++++--- .../tests/user_journeys/test_user_journeys.py | 1 + conda-store-server/tests/user_journeys/utils/api_utils.py | 1 + tests/test_api.py | 1 + 17 files changed, 25 insertions(+), 6 deletions(-) diff --git a/conda-store-server/conda_store_server/alembic/versions/03c839888c82_add_canceled_status.py b/conda-store-server/conda_store_server/alembic/versions/03c839888c82_add_canceled_status.py index 8c30d3b50..ac023d997 100644 --- a/conda-store-server/conda_store_server/alembic/versions/03c839888c82_add_canceled_status.py +++ b/conda-store-server/conda_store_server/alembic/versions/03c839888c82_add_canceled_status.py @@ -5,6 +5,7 @@ Create Date: 2024-01-29 03:56:36.889909 """ + import sqlalchemy as sa from alembic import op diff --git a/conda-store-server/conda_store_server/alembic/versions/0f7e23ff24ee_add_worker.py b/conda-store-server/conda_store_server/alembic/versions/0f7e23ff24ee_add_worker.py index ca086ebd0..19d0e4fd3 100644 --- a/conda-store-server/conda_store_server/alembic/versions/0f7e23ff24ee_add_worker.py +++ b/conda-store-server/conda_store_server/alembic/versions/0f7e23ff24ee_add_worker.py @@ -5,6 +5,7 @@ Create Date: 2023-12-13 21:01:45.546591 """ + import sqlalchemy as sa from alembic import op diff --git a/conda-store-server/conda_store_server/alembic/versions/16f65805dc8f_split_conda_package_into_conda_package_.py b/conda-store-server/conda_store_server/alembic/versions/16f65805dc8f_split_conda_package_into_conda_package_.py index dc6c07944..19d1e7067 100644 --- a/conda-store-server/conda_store_server/alembic/versions/16f65805dc8f_split_conda_package_into_conda_package_.py +++ b/conda-store-server/conda_store_server/alembic/versions/16f65805dc8f_split_conda_package_into_conda_package_.py @@ -5,6 +5,7 @@ Create Date: 2022-08-24 12:01:48.461989 """ + import sqlalchemy as sa from alembic import op diff --git a/conda-store-server/conda_store_server/alembic/versions/30b37e725c32_add_build_key_version.py b/conda-store-server/conda_store_server/alembic/versions/30b37e725c32_add_build_key_version.py index 0313a95f0..353a59555 100644 --- a/conda-store-server/conda_store_server/alembic/versions/30b37e725c32_add_build_key_version.py +++ b/conda-store-server/conda_store_server/alembic/versions/30b37e725c32_add_build_key_version.py @@ -5,6 +5,7 @@ Create Date: 2023-11-17 14:34:40.688865 """ + import sqlalchemy as sa from alembic import op diff --git a/conda-store-server/conda_store_server/alembic/versions/48be4072fe58_initial_schema.py b/conda-store-server/conda_store_server/alembic/versions/48be4072fe58_initial_schema.py index 7e3e15d87..bc83537c7 100644 --- a/conda-store-server/conda_store_server/alembic/versions/48be4072fe58_initial_schema.py +++ b/conda-store-server/conda_store_server/alembic/versions/48be4072fe58_initial_schema.py @@ -5,6 +5,7 @@ Create Date: 2022-06-01 18:37:12.396138 """ + import sqlalchemy as sa from alembic import op diff --git a/conda-store-server/conda_store_server/alembic/versions/57cd11b949d5_add_installer.py b/conda-store-server/conda_store_server/alembic/versions/57cd11b949d5_add_installer.py index db815fe1d..d4104a874 100644 --- a/conda-store-server/conda_store_server/alembic/versions/57cd11b949d5_add_installer.py +++ b/conda-store-server/conda_store_server/alembic/versions/57cd11b949d5_add_installer.py @@ -5,6 +5,7 @@ Create Date: 2024-01-28 14:31:35.723505 """ + import sqlalchemy as sa from alembic import op diff --git a/conda-store-server/conda_store_server/alembic/versions/5ad723de2abd_adding_container_registry_value_to_enum.py b/conda-store-server/conda_store_server/alembic/versions/5ad723de2abd_adding_container_registry_value_to_enum.py index 3398b7a36..282944eab 100644 --- a/conda-store-server/conda_store_server/alembic/versions/5ad723de2abd_adding_container_registry_value_to_enum.py +++ b/conda-store-server/conda_store_server/alembic/versions/5ad723de2abd_adding_container_registry_value_to_enum.py @@ -5,6 +5,7 @@ Create Date: 2022-08-05 22:14:34.110642 """ + import sqlalchemy as sa from alembic import op diff --git a/conda-store-server/conda_store_server/alembic/versions/771180018e1b_add_v2_role_mappings.py b/conda-store-server/conda_store_server/alembic/versions/771180018e1b_add_v2_role_mappings.py index a195b7a9f..02e859e0d 100644 --- a/conda-store-server/conda_store_server/alembic/versions/771180018e1b_add_v2_role_mappings.py +++ b/conda-store-server/conda_store_server/alembic/versions/771180018e1b_add_v2_role_mappings.py @@ -5,6 +5,7 @@ Create Date: 2023-11-29 09:02:35.835664 """ + import sqlalchemy as sa from alembic import op diff --git a/conda-store-server/conda_store_server/alembic/versions/8d63a091aff8_add_environment_description.py b/conda-store-server/conda_store_server/alembic/versions/8d63a091aff8_add_environment_description.py index 518362dbf..693bfa265 100644 --- a/conda-store-server/conda_store_server/alembic/versions/8d63a091aff8_add_environment_description.py +++ b/conda-store-server/conda_store_server/alembic/versions/8d63a091aff8_add_environment_description.py @@ -5,6 +5,7 @@ Create Date: 2022-07-15 14:22:00.351131 """ + import sqlalchemy as sa from alembic import op diff --git a/conda-store-server/conda_store_server/alembic/versions/abd7248d5327_adding_a_settings_table.py b/conda-store-server/conda_store_server/alembic/versions/abd7248d5327_adding_a_settings_table.py index 2fd34559d..0e1231dca 100644 --- a/conda-store-server/conda_store_server/alembic/versions/abd7248d5327_adding_a_settings_table.py +++ b/conda-store-server/conda_store_server/alembic/versions/abd7248d5327_adding_a_settings_table.py @@ -5,6 +5,7 @@ Create Date: 2023-05-11 16:38:12.210549 """ + import sqlalchemy as sa from alembic import op diff --git a/conda-store-server/conda_store_server/alembic/versions/b387747ca9b7_role_mapping.py b/conda-store-server/conda_store_server/alembic/versions/b387747ca9b7_role_mapping.py index 5f899a5bb..8a645298b 100644 --- a/conda-store-server/conda_store_server/alembic/versions/b387747ca9b7_role_mapping.py +++ b/conda-store-server/conda_store_server/alembic/versions/b387747ca9b7_role_mapping.py @@ -5,6 +5,7 @@ Create Date: 2023-07-04 14:35:48.177574 """ + import sqlalchemy as sa from alembic import op diff --git a/conda-store-server/conda_store_server/alembic/versions/d78e9889566a_add_status_info.py b/conda-store-server/conda_store_server/alembic/versions/d78e9889566a_add_status_info.py index d91938a11..e1cbdefc4 100644 --- a/conda-store-server/conda_store_server/alembic/versions/d78e9889566a_add_status_info.py +++ b/conda-store-server/conda_store_server/alembic/versions/d78e9889566a_add_status_info.py @@ -5,6 +5,7 @@ Create Date: 2023-11-07 12:25:04.416192 """ + import sqlalchemy as sa from alembic import op diff --git a/conda-store-server/conda_store_server/server/views/api.py b/conda-store-server/conda_store_server/server/views/api.py index 72c6826cf..ff4440b96 100644 --- a/conda-store-server/conda_store_server/server/views/api.py +++ b/conda-store-server/conda_store_server/server/views/api.py @@ -145,9 +145,11 @@ async def api_get_permissions( "status": "ok", "data": { "authenticated": authenticated, - "primary_namespace": entity.primary_namespace - if authenticated - else conda_store.default_namespace, + "primary_namespace": ( + entity.primary_namespace + if authenticated + else conda_store.default_namespace + ), "entity_permissions": entity_binding_permissions, "entity_roles": entity_binding_roles, "expiration": entity.exp if authenticated else None, diff --git a/conda-store-server/conda_store_server/server/views/registry.py b/conda-store-server/conda_store_server/server/views/registry.py index 7acd480c5..a43163d6e 100644 --- a/conda-store-server/conda_store_server/server/views/registry.py +++ b/conda-store-server/conda_store_server/server/views/registry.py @@ -159,9 +159,11 @@ def list_tags( try: auth.authorize_request( request, - image - if parts[0] != "conda-store-dynamic" - else "conda-store-dynamic/python", + ( + image + if parts[0] != "conda-store-dynamic" + else "conda-store-dynamic/python" + ), {Permissions.ENVIRONMENT_READ}, require=True, ) diff --git a/conda-store-server/tests/user_journeys/test_user_journeys.py b/conda-store-server/tests/user_journeys/test_user_journeys.py index 49584a5f0..a55e93b46 100644 --- a/conda-store-server/tests/user_journeys/test_user_journeys.py +++ b/conda-store-server/tests/user_journeys/test_user_journeys.py @@ -1,4 +1,5 @@ """User journey tests for the API.""" + import os import pytest diff --git a/conda-store-server/tests/user_journeys/utils/api_utils.py b/conda-store-server/tests/user_journeys/utils/api_utils.py index 0a50fd321..5030287de 100644 --- a/conda-store-server/tests/user_journeys/utils/api_utils.py +++ b/conda-store-server/tests/user_journeys/utils/api_utils.py @@ -1,4 +1,5 @@ """Helper functions for user journeys.""" + import time import uuid from enum import Enum diff --git a/tests/test_api.py b/tests/test_api.py index 9583249b1..d5f4aded2 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -8,6 +8,7 @@ ordering for tests. """ + import asyncio import collections import datetime From d365061cb3c97b4a3712b861f2278d5fe7ece156 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Wed, 20 Mar 2024 17:20:55 +0000 Subject: [PATCH 05/26] Bump GH actions workflows --- .github/workflows/build_docker_image.yaml | 2 +- .github/workflows/release.yaml | 8 ++++---- .github/workflows/tests.yaml | 25 ++++++++++++----------- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build_docker_image.yaml b/.github/workflows/build_docker_image.yaml index fa5489081..bb7dcb13c 100644 --- a/.github/workflows/build_docker_image.yaml +++ b/.github/workflows/build_docker_image.yaml @@ -23,7 +23,7 @@ jobs: uses: actions/checkout@v4 - name: "Retrieve secret from Vault ๐Ÿ—" - uses: hashicorp/vault-action@v2 + uses: hashicorp/vault-action@v3 with: method: jwt url: "https://quansight-vault-public-vault-b2379fa7.d415e30e.z1.hashicorp.cloud:8200" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e9b8f2545..f4fc8402b 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -39,13 +39,13 @@ jobs: - run: echo "Running on ${{ matrix.directory }}" - name: "Check package build - ${{ matrix.directory }} ๐Ÿ“ฆ" - uses: hynek/build-and-inspect-python-package@v1 + uses: hynek/build-and-inspect-python-package@v2 with: path: ${{ matrix.directory }} # Adding a separate upload for publishing - name: "Upload build artefacts ๐Ÿ“ค" - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ matrix.directory }}-package path: | @@ -69,7 +69,7 @@ jobs: steps: - name: "Download build artefacts ๐Ÿ“ฅ" - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ${{ matrix.directory }}-package path: dist @@ -101,7 +101,7 @@ jobs: steps: - name: "Download build artefacts ๐Ÿ“ฅ" - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ${{ matrix.directory }}-package path: dist diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index c078552b3..0a98a3fed 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -20,13 +20,14 @@ jobs: strategy: matrix: os: ["ubuntu", "macos", "windows"] + # TODO: come back and clean here include: - os: ubuntu environment-file: conda-store-server/environment-dev.yaml - os: macos - environment-file: conda-store-server/environment-macos-dev.yaml + environment-file: conda-store-server/environment-dev.yaml - os: windows - environment-file: conda-store-server/environment-windows-dev.yaml + environment-file: conda-store-server/environment-dev.yaml runs-on: ${{ matrix.os }}-latest defaults: run: @@ -37,7 +38,7 @@ jobs: uses: actions/checkout@v4 - name: "Set up env ${{ matrix.os }} ๐Ÿ" - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: environment-file: ${{ matrix.environment-file }} miniforge-version: latest @@ -89,7 +90,7 @@ jobs: uses: actions/checkout@v4 - name: "Set up env ๐Ÿ" - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: environment-file: conda-store-server/environment-dev.yaml miniforge-version: latest @@ -114,11 +115,11 @@ jobs: pytest --video on ../tests/test_playwright.py - name: "Upload test results ๐Ÿ“ค" - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: ${{ always() }} with: name: playwright-tests - path: conda-store-server/test-results + path: conda-store-server/test-results/ - name: "Run integration tests โœ…" run: | @@ -142,13 +143,13 @@ jobs: working-directory: conda-store strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - name: "Checkout Repository ๐Ÿ›Ž" uses: actions/checkout@v4 - name: "Set up Python ๐Ÿ" - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -212,17 +213,17 @@ jobs: - name: "Checkout Repository" uses: actions/checkout@v4 - - name: Set up Docker Buildx + - name: "Set up Docker Buildx" uses: docker/setup-buildx-action@v3 - - name: Lint Dockerfiles + - name: "Lint Dockerfiles" uses: jbergstroem/hadolint-gh-action@v1 with: dockerfile: ${{ matrix.docker-image }}/Dockerfile output_format: tty error_level: 0 - - name: Docker Meta + - name: "Docker Meta" id: meta uses: docker/metadata-action@v5 with: @@ -231,7 +232,7 @@ jobs: tags: | type=sha - - name: Build docker + - name: "Build docker" uses: docker/build-push-action@v5 with: context: "${{ matrix.docker-image }}" From e52de2f5049b915ad2f54772415e1b98dfc87274 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Wed, 20 Mar 2024 17:22:44 +0000 Subject: [PATCH 06/26] Add pre-commit ci --- .pre-commit-config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a7b165c51..cb104e8d1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,6 +8,11 @@ # - Register git hooks: pre-commit install --install-hooks # - Run on all files: pre-commit run --all-files +ci: + autoupdate_schedule: monthly + autofix_commit_msg: | + [pre-commit.ci] Apply automatic pre-commit fixes + repos: - repo: https://github.com/psf/black rev: 24.3.0 From 383236fb4e2c890210abec11105b5144294b0d35 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Wed, 20 Mar 2024 19:22:05 +0000 Subject: [PATCH 07/26] Update lint hatch script --- .github/workflows/tests.yaml | 4 ++-- conda-store-server/pyproject.toml | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 0a98a3fed..f55411a2c 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -55,7 +55,7 @@ jobs: - name: "Linting Checks ๐Ÿงน" run: | - hatch env run -e dev lint + hatch run lint:lint - name: "Build package ๐Ÿ“ฆ" run: | @@ -160,7 +160,7 @@ jobs: - name: "Linting Checks ๐Ÿงน" run: | - hatch env run -e dev lint + hatch run lint:lint - name: "Build package ๐Ÿ“ฆ" run: | diff --git a/conda-store-server/pyproject.toml b/conda-store-server/pyproject.toml index 6d79afa42..ef226b1e7 100644 --- a/conda-store-server/pyproject.toml +++ b/conda-store-server/pyproject.toml @@ -74,6 +74,9 @@ Issues = "https://github.com/conda-incubator/conda-store/issues" [tool.hatch.version] path = "conda_store_server/__init__.py" +[project.optional-dependencies] +dev=["build", "twine"] + [tool.hatch.envs.dev] dependencies = [ "aiohttp>=3.8.1", @@ -90,7 +93,9 @@ dependencies = [ "twine" ] -[tool.hatch.envs.dev.scripts] +[tool.hatch.envs.lint] +dependencies = ["pre-commit"] +[tool.hatch.envs.lint.scripts] lint = ["pre-commit run --all"] unit-test = "pytest tests -v" playwright-test = [ From 6a34329423579ab998f05bc1acf2777ba591a754 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Wed, 20 Mar 2024 19:25:17 +0000 Subject: [PATCH 08/26] Add updated intro docs --- conda-store-server/pyproject.toml | 1 + docusaurus-docs/conda-store/introduction.md | 35 ++++++++++++++++----- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/conda-store-server/pyproject.toml b/conda-store-server/pyproject.toml index ef226b1e7..401bfca56 100644 --- a/conda-store-server/pyproject.toml +++ b/conda-store-server/pyproject.toml @@ -95,6 +95,7 @@ dependencies = [ [tool.hatch.envs.lint] dependencies = ["pre-commit"] + [tool.hatch.envs.lint.scripts] lint = ["pre-commit run --all"] unit-test = "pytest tests -v" diff --git a/docusaurus-docs/conda-store/introduction.md b/docusaurus-docs/conda-store/introduction.md index dfbe5d7f0..dd15da355 100644 --- a/docusaurus-docs/conda-store/introduction.md +++ b/docusaurus-docs/conda-store/introduction.md @@ -9,11 +9,29 @@ description: Introduction to conda-store (core) documentation The [`conda-store` repository on GitHub][conda-store-repo] consists of two separate, yet related, packages: - **`conda-store-server`**: web server and workers that together provide the `conda-store` "service" through a REST API -- **`conda-store`** (client): a client which interacts with the service to offer user-facing command line interface +- **`conda-store`** (client): a client that interacts with the service to offer a user-facing command line interface -## Quickstart +## Pre-requisites -1. Install conda-store with `conda` or `pip`: +- Python `3.8` or later +- `conda`, if installing from scratch we recommend you install [`miniforge`](https://github.com/conda-forge/miniforge) or [`miniconda`](https://docs.anaconda.com/free/miniconda/miniconda-install/). + +:::warning + +`conda` is a hard requirement for `conda-store` and since it is not `pip` installable you need to have `conda` installed separately. + +::: + +## Quick start + +1. Create and activate new `conda` environment: + +```bash +conda create -n conda-store-env python= +conda activate conda-store-env +``` + +2. Install `conda-store` with `conda` or `pip`: import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; @@ -25,26 +43,29 @@ import TabItem from '@theme/TabItem'; ```bash conda install -c conda-forge conda-store conda-store-server ``` + ```bash -pip install conda-store conda-store-server +# note that we still recommend you use conda to create an environment +python -m pip install conda-store conda-store-server ``` + -2. Start a standalone local instance and use the conda-store UI +3. Start a standalone local instance and use the `conda-store` UI ```bash conda-store-server --standalone ``` -You can then access conda-store at `localhost:8080` of the machine running it. +You can then access `conda-store` at `localhost:8080` of the machine running it. -3. Use the CLI +4. Use the CLI ```bash conda-store --help From f1dfe9a28da8717f7834bb52b1ad122036cd931d Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Wed, 20 Mar 2024 19:25:56 +0000 Subject: [PATCH 09/26] Update hatch syntax --- .github/workflows/tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index f55411a2c..12fc32311 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -55,7 +55,7 @@ jobs: - name: "Linting Checks ๐Ÿงน" run: | - hatch run lint:lint + hatch env run -e lint lint - name: "Build package ๐Ÿ“ฆ" run: | @@ -160,7 +160,7 @@ jobs: - name: "Linting Checks ๐Ÿงน" run: | - hatch run lint:lint + hatch env run -e lint lint - name: "Build package ๐Ÿ“ฆ" run: | From d4c3b733446a9ce59f091d094c746fced4af3c07 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Wed, 20 Mar 2024 19:41:32 +0000 Subject: [PATCH 10/26] :construction_worker: Ensure we always install packages --- .github/workflows/tests.yaml | 10 +++++++--- conda-store-server/environment-dev.yaml | 8 ++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 12fc32311..6bfbf022a 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -61,6 +61,10 @@ jobs: run: | hatch build + - name: "Installed built conda-store-server ๐Ÿ“ฆ" + run: | + pip install dist/*.whl + - name: "Unit tests โœ…" run: | pytest -m "not extended_prefix and not user_journey" tests @@ -160,7 +164,7 @@ jobs: - name: "Linting Checks ๐Ÿงน" run: | - hatch env run -e lint lint + hatch env run -e dev lint - name: "Build package ๐Ÿ“ฆ" run: | @@ -175,9 +179,9 @@ jobs: wait-for-it localhost:9000 # minio wait-for-it localhost:8080 # conda-store-server - - name: "Install conda-store for tests ๐Ÿ“ฆ" + - name: "Install built conda-store for tests ๐Ÿ“ฆ" run: | - pip install . + pip install dist/*.whl - name: "Run basic tests - not authenticated" run: | diff --git a/conda-store-server/environment-dev.yaml b/conda-store-server/environment-dev.yaml index 32caa941d..4b6d6c1a3 100644 --- a/conda-store-server/environment-dev.yaml +++ b/conda-store-server/environment-dev.yaml @@ -12,16 +12,16 @@ dependencies: - flower # dev dependencies - aiohttp>=3.8.1 - - hatch - pytest - pytest-celery - pytest-mock - - black - - flake8 - - ruff - playwright - docker-py<7 # for docker-compose - docker-compose + # build dependencies + - hatch + - twine + - build - pip - pip: - pytest-playwright From 3f1b16e77ca0b55ec024e7cb5d20efe68db9ad5e Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Wed, 20 Mar 2024 19:48:33 +0000 Subject: [PATCH 11/26] :fire: Remove redundant env.yaml file --- .github/workflows/tests.yaml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 6bfbf022a..fb7ef280c 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -20,14 +20,6 @@ jobs: strategy: matrix: os: ["ubuntu", "macos", "windows"] - # TODO: come back and clean here - include: - - os: ubuntu - environment-file: conda-store-server/environment-dev.yaml - - os: macos - environment-file: conda-store-server/environment-dev.yaml - - os: windows - environment-file: conda-store-server/environment-dev.yaml runs-on: ${{ matrix.os }}-latest defaults: run: @@ -40,7 +32,7 @@ jobs: - name: "Set up env ${{ matrix.os }} ๐Ÿ" uses: conda-incubator/setup-miniconda@v3 with: - environment-file: ${{ matrix.environment-file }} + environment-file: conda-store-server/environment-dev.yaml miniforge-version: latest # This fixes a "DLL not found" issue importing ctypes from the hatch env From b7ea23ef054a24f8ab03a2bdae9f05bc58d3f449 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Wed, 20 Mar 2024 19:52:03 +0000 Subject: [PATCH 12/26] :fire: Remove yaml files --- conda-store-server/environment-macos-dev.yaml | 27 ------------------- .../environment-windows-dev.yaml | 26 ------------------ .../community/contribute/contribute-code.md | 1 + 3 files changed, 1 insertion(+), 53 deletions(-) delete mode 100644 conda-store-server/environment-macos-dev.yaml delete mode 100644 conda-store-server/environment-windows-dev.yaml diff --git a/conda-store-server/environment-macos-dev.yaml b/conda-store-server/environment-macos-dev.yaml deleted file mode 100644 index 32caa941d..000000000 --- a/conda-store-server/environment-macos-dev.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: conda-store-server-dev -channels: - - conda-forge - - microsoft - - nodefaults -dependencies: - - python ==3.10 - # conda builds - - conda ==23.5.2 - - python-docker - # web server - - flower - # dev dependencies - - aiohttp>=3.8.1 - - hatch - - pytest - - pytest-celery - - pytest-mock - - black - - flake8 - - ruff - - playwright - - docker-py<7 # for docker-compose - - docker-compose - - pip - - pip: - - pytest-playwright diff --git a/conda-store-server/environment-windows-dev.yaml b/conda-store-server/environment-windows-dev.yaml deleted file mode 100644 index 2d600e48d..000000000 --- a/conda-store-server/environment-windows-dev.yaml +++ /dev/null @@ -1,26 +0,0 @@ -name: conda-store-server-dev -channels: - - conda-forge - - microsoft - - nodefaults -dependencies: - - python ==3.10 - # conda builds - - conda ==23.5.2 - # web server - - flower - # dev dependencies - - aiohttp>=3.8.1 - - hatch - - pytest - - pytest-celery - - pytest-mock - - black - - flake8 - - ruff - - playwright - - docker-py<7 # for docker-compose - - docker-compose - - pip - - pip: - - pytest-playwright diff --git a/docusaurus-docs/community/contribute/contribute-code.md b/docusaurus-docs/community/contribute/contribute-code.md index bdfe2e687..6440a178c 100644 --- a/docusaurus-docs/community/contribute/contribute-code.md +++ b/docusaurus-docs/community/contribute/contribute-code.md @@ -57,6 +57,7 @@ docker-compose up --build - [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/install/linux.html) + 2. Install the development dependencies and activate the environment: ```shell From e9b03a1161d1a459fa64d5ef91ea68281c8be162 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Thu, 21 Mar 2024 10:34:10 +0000 Subject: [PATCH 13/26] Update conda-store/pyproject.toml Co-authored-by: Peyton Murray --- conda-store/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-store/pyproject.toml b/conda-store/pyproject.toml index 1dad28105..7c9f75e42 100644 --- a/conda-store/pyproject.toml +++ b/conda-store/pyproject.toml @@ -14,7 +14,7 @@ authors = [ { name = "Christopher Ostrouchov", email = "chris.ostrouchov@gmail.com" }, ] maintainers = [{ name = "Tania Allard", email = "trallard@bitsandchips.me" }, - {name= "Pavithra Eswaramoorthy", email = " pavithraes@outlook.com"}, + {name= "Pavithra Eswaramoorthy", email = "pavithraes@outlook.com"}, ] classifiers = [ From c8cdce9b8a626d5678d4376cffffb4dc14e25e9b Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Thu, 21 Mar 2024 12:41:22 +0000 Subject: [PATCH 14/26] Update pyproject.toml - isort and maintainers --- .pre-commit-config.yaml | 13 +---- README.md | 53 +++++++++------------ conda-store-server/pyproject.toml | 13 +++-- conda-store/pyproject.toml | 18 +++++-- docusaurus-docs/conda-store/introduction.md | 2 +- 5 files changed, 48 insertions(+), 51 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cb104e8d1..c44478ec2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,22 +21,11 @@ repos: exclude: "examples|tests/assets" - repo: https://github.com/charliermarsh/ruff-pre-commit - # Ruff version. rev: "v0.3.3" hooks: - id: ruff exclude: "examples|tests/assets" - args: ["--fix"] - - - repo: https://github.com/pycqa/isort - rev: 5.13.2 - hooks: - - id: isort - name: isort - additional_dependencies: [toml] - files: \.py$ - args: ["--profile", "black"] - exclude: "conda-store-server/conda_store_server/action/__init__.py" + args: [--fix, --exit-non-zero-on-fix] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 diff --git a/README.md b/README.md index 60ddef78a..8d2ec6f92 100644 --- a/README.md +++ b/README.md @@ -10,53 +10,46 @@ Flexible. Reproducible. Governable. --- -
- - - PyPi - - - - PyPi - - - - Conda - - - - Conda - - -
+| Information | Links| +| :---------- | :-----| +| Project | [![License](https://img.shields.io/badge/License-BSD%203--Clause-gray.svg?colorA=2b2d42&colorB=206532&style=flat.svg)](https://opensource.org/licenses/BSD-3-Clause) [![Read the docs](https://img.shields.io/badge/%F0%9F%93%96%20Read-the%20docs-gray.svg?colorA=2b2d42&colorB=206532&style=flat.svg)](https://conda.store)| +|Community | [![Open an issue](https://img.shields.io/badge/%F0%9F%93%9D%20Open-an%20issue-gray.svg?colorA=2b2d42&colorB=206532&style=flat.svg)](https://github.com/conda-incubator/conda-store/issues/new/choose) [![Community guidelines](https://img.shields.io/badge/๐Ÿค%20Community-guidelines-gray.svg?colorA=2b2d42&colorB=206532&style=flat.svg)](https://conda.store/community/introduction)| +|Releases | [![PyPI release conda-store](https://img.shields.io/pypi/v/conda-store)](https://badge.fury.io/py/conda-store?label=pypi%20conda-store&style=flat.svg) [![PyPI release conda-store-server](https://img.shields.io/pypi/v/conda-store-server?label=pypi%20conda-store-server)](https://badge.fury.io/py/conda-store-server) [![conda-forge release conda-store](https://img.shields.io/conda/vn/conda-forge/conda-store?label=conda-forge%20conda-store)]((https://anaconda.org/conda-forge/conda-store)) [![conda-forge release conda-store-server](https://img.shields.io/conda/vn/conda-forge/conda-store-server?label=conda-forge%20conda-store-server)]((https://anaconda.org/conda-forge/conda-store-server)) | --- -conda-store provides the familiarity and flexibility of conda environments, without compromising reliability for collaborative settings. +conda-store provides the familiarity and flexibility of conda environments, without compromising reliability for +collaborative settings. -conda-store is built to work for all team members from individual data scientists to administrators, while making sure your team follows best practices throughout the environment life cycle: from initial environment creation, to using environments in a production machine. +conda-store is built to work for all team members from individual data scientists to administrators, +while making sure your team follows best practices throughout the environment life cycle: +from initial environment creation to using environments in a production machine. ## Key features -- **Flexiblity**: +- **Flexibility**: - Users can create and update environments with the Graphical UI or a YAML editor. - The environments are automatically version-controlled and all versions are readily available. -- **Reproduciblity**: - - User can share environments quickly through the auto-generated artifacts including a lockfile, docker image, YAML file, and tarball. +- **Reproducibility**: + - Users can share environments quickly through the auto-generated artifacts including a lockfile, docker image, YAML file, and tarball. - conda-store pins exact versions of all packages and their dependencies in all the auto-generated artifacts. -- **Goverance**: - - Users have access to admin-approved packages and channels for their work, and can request new ones when needed. +- **Governance**: + - Users have access to admin-approved packages and channels for their work and can request new ones when needed. - Admins can insert or require certain packages and versions for organization-level compatibility. - - Admins can manage users' access-levels using "Namespaces", and allow users to share environments across (and only with) their team. + - Admins can manage users' access levels using "Namespaces", and allow users to share environments across (and only with) their team. ## Get started Learn more, including how to install, use, and contribute to conda-store in our documentation at [**conda.store**](https://conda.store/). -## Related repositories +## Contributing + +We welcome all types of contributions. Please read our [Contributing Guide](https://conda.store/community/contribute/) to get started. + +## Related repositories and projects -- We are working on a new UI for conda-store at: [`conda-incubator/conda-store-ui`](https://github.com/conda-incubator/conda-store-ui), and -- a JupyterLab extension at: [`conda-incubator/jupyterlab-conda-store`](https://github.com/conda-incubator/jupyterlab-conda-store). +- We are working on a new UI for conda-store at [`conda-incubator/conda-store-ui`](https://github.com/conda-incubator/conda-store-ui) and +- a JupyterLab extension at [`conda-incubator/jupyterlab-conda-store`](https://github.com/conda-incubator/jupyterlab-conda-store). ## Code of Conduct diff --git a/conda-store-server/pyproject.toml b/conda-store-server/pyproject.toml index 401bfca56..4c12624fc 100644 --- a/conda-store-server/pyproject.toml +++ b/conda-store-server/pyproject.toml @@ -9,12 +9,13 @@ description = "Conda Environment Management, Builds, and Serve" readme = "README.md" license = "BSD-3-Clause" requires-python = ">=3.8" -keywords = ["conda"] +keywords = ["conda", "dependency management", "environment management", "reproducibility"] authors = [ { name = "Christopher Ostrouchov", email = "crhsi.ostrouchov@gmail.com" }, ] maintainers = [{ name = "Tania Allard", email = "trallard@bitsandchips.me" }, {name= "Pavithra Eswaramoorthy", email = " pavithraes@outlook.com"}, + {name="Nikita Karetnikov", email="nkaretnikov@quansight.com"}, ] classifiers = [ "Development Status :: 3 - Alpha", @@ -98,7 +99,9 @@ dependencies = ["pre-commit"] [tool.hatch.envs.lint.scripts] lint = ["pre-commit run --all"] -unit-test = "pytest tests -v" + +[tool.hatch.envs.dev.scripts] +unit-test = "pytest -m 'not extended_prefix and not user_journey' tests" playwright-test = [ "playwright install", "pytest --video on ../tests/test_playwright.py" @@ -117,7 +120,11 @@ conda-store-worker = "conda_store_server.worker.__main__:main" [tool.black] line-length = 88 -target-version = ['py37', 'py38', 'py39'] +target-version = ['py38', 'py39', 'py310', 'py311', 'py312'] + +[tool.ruff.lint.isort] +lines-between-types = 1 +lines-after-imports = 2 [tool.ruff] ignore = [ diff --git a/conda-store/pyproject.toml b/conda-store/pyproject.toml index 7c9f75e42..c64117062 100644 --- a/conda-store/pyproject.toml +++ b/conda-store/pyproject.toml @@ -9,12 +9,13 @@ description = "conda-store client" readme = "README.md" license = "BSD-3-Clause" requires-python = ">=3.8" -keywords = ["conda"] +keywords = ["conda", "dependency management", "environment management", "reproducibility"] authors = [ { name = "Christopher Ostrouchov", email = "chris.ostrouchov@gmail.com" }, ] maintainers = [{ name = "Tania Allard", email = "trallard@bitsandchips.me" }, {name= "Pavithra Eswaramoorthy", email = "pavithraes@outlook.com"}, + {name="Nikita Karetnikov", email="nkaretnikov@quansight.com"}, ] classifiers = [ @@ -51,14 +52,18 @@ path = "conda_store/__init__.py" [tool.hatch.envs.dev] dependencies = [ "build", - "pre-commit", "pytest", - "pytest-mock", "twine", + "pytest-mock", ] -[tool.hatch.envs.dev.scripts] +[tool.hatch.envs.lint] +dependencies = ["pre-commit"] + +[tool.hatch.envs.lint.scripts] lint = ["pre-commit run --all"] + +[tool.hatch.envs.dev.scripts] test = "pytest tests" [project.scripts] @@ -66,7 +71,10 @@ conda-store = "conda_store.__main__:main" [tool.black] line-length = 88 -target-version = ['py38', 'py39', 'py310', 'py311'] + +[tool.ruff.lint.isort] +lines-between-types = 1 +lines-after-imports = 2 [tool.ruff] ignore = [ diff --git a/docusaurus-docs/conda-store/introduction.md b/docusaurus-docs/conda-store/introduction.md index dd15da355..59dbb4e87 100644 --- a/docusaurus-docs/conda-store/introduction.md +++ b/docusaurus-docs/conda-store/introduction.md @@ -24,7 +24,7 @@ The [`conda-store` repository on GitHub][conda-store-repo] consists of two separ ## Quick start -1. Create and activate new `conda` environment: +1. Create and activate a new `conda` environment: ```bash conda create -n conda-store-env python= From cf5df1ed098453b1df39bd373a3a84e9fd0a4eea Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Thu, 21 Mar 2024 12:41:44 +0000 Subject: [PATCH 15/26] :memo: Update contributing docs --- .../community/contribute/contribute-code.md | 215 +++++++++++++----- 1 file changed, 154 insertions(+), 61 deletions(-) diff --git a/docusaurus-docs/community/contribute/contribute-code.md b/docusaurus-docs/community/contribute/contribute-code.md index 6440a178c..6ac6d29b8 100644 --- a/docusaurus-docs/community/contribute/contribute-code.md +++ b/docusaurus-docs/community/contribute/contribute-code.md @@ -5,32 +5,50 @@ description: Contribute to conda-store's codebase # Contribute code :::warning -This page is in active development, content may be inaccurate and incomplete. +This page is in active development, content may be inaccurate and incomplete. If you encounter any content that needs improvement, please [create an issue](https://github.com/conda-incubator/conda-store/issues/new/choose) in our issue tracker. ::: -## Set up development environment - conda-store-core +## `conda-store-core` -### Docker (recommended) +Before setting up your `conda-store-core` development environment you will need to have a local copy of the `conda-store` repository. -Install the following dependencies before developing on conda-store. +If you are a first-time contributor: + +1. Go to [https://github.com/conda-incubator/conda-store/](https://github.com/conda-incubator/conda-store/) and click on the **Fork** button in the upper right corner to create your copy of the repository. +2. Clone the project to your local computer: + + ```bash + git clone https://github.com//conda-store/ + ``` + +Once you have a local copy of the `conda-store` repository, you can set up your development environment. +There are two main ways to set up your local environment for development: + +- Using [Docker and docker-compose(recommended)](#docker-based-development---conda-store-core) +- Local development [without Docker](#local-development-without-docker---conda-store-core) + +### Docker-based development - conda-store-core + +Install the following dependencies before developing on `conda-store`. - [Docker](https://docs.docker.com/engine/install/) - [docker-compose](https://docs.docker.com/compose/install/) -To deploy `conda-store` run the following command +To deploy `conda-store` run the following command: -```shell +```bash docker-compose up --build -d ``` :::important -Many of the conda-store docker images are built/tested for amd64(x86-64) +Many of the conda-store Docker images are built/tested for amd64(x86-64) there will be a performance impact when building and running on -arm architectures. Otherwise, this workflow has been shown to run and build on OSX. -Notice the `architecture: amd64` within the `docker-compose.yaml` files. +arm architectures. +Otherwise, this workflow has been shown to run and build on OSX. +**Notice** the `architecture: amd64` within the `docker-compose.yaml` files. ::: -The following resources will be available: +After running the docker-compose command, the following resources will be available: | Resource | Localhost port | username | password | |----------|----------------|----------|----------| @@ -40,7 +58,7 @@ The following resources will be available: | [PostgreSQL](https://www.postgresql.org/) (database: `conda-store`)| [localhost:5432](http://localhost:5432) | `admin` | `password` | | [Redis](https://www.redis.com/) | [localhost:6379](http://localhost:6379) | - | password | -On a fast machine this deployment should only take 10 or so seconds +On a fast machine, this deployment should only take 10 or so seconds assuming the docker images have been partially built before. If you are making any changes to `conda-store-server` and would like to see @@ -51,21 +69,26 @@ docker-compose down -v # not always necessary docker-compose up --build ``` -### Linux +### Local development without Docker - conda-store-core -1. Install the following dependencies before developing on conda-store: +You will need to install the following dependencies before developing on `conda-store`: - - [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/install/linux.html) +- [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html) - -2. Install the development dependencies and activate the environment: +1. Install the development dependencies and activate the environment: -```shell -# replace this with environment-macos-dev.yaml or environment-windows-dev.yaml -# if you are on Mac or Windows -conda env create -f conda-store-server/environment-dev.yaml -conda activate conda-store-server-dev -``` + ```bash + # from the root of the repository + conda env create -f conda-store-server/environment-dev.yaml + conda activate conda-store-server-dev + ``` + +2. Install the package in editable mode: + + ```bash + # from the root of the repository + pip install -e ./conda-store-server + ``` 3. Running `conda-store` in `--standalone` mode launches celery as a subprocess of the web server. @@ -74,50 +97,106 @@ subprocess of the web server. python -m conda_store_server.server --standalone ``` -1. Visit [localhost:8080](http://localhost:8080/) +4. You should now be able to access the `conda-store` server at [localhost:8080](http://localhost:8080/) from your web browser. + +## `conda-store-ui` -## Set up development environment -- conda-store-ui +Before setting up your `conda-store-ui` development environment you will need to have a local copy of the +`conda-store-ui` repository. -To get started with conda-store-ui development, there are a couple of options. This guide will help you to set up your local development environment. +If you are a first-time contributor: -### Prerequisites +1. Go to [https://github.com/conda-incubator/conda-store-ui/](https://github.com/conda-incubator/conda-store-ui/) and click on the **Fork** button in the upper right corner to create your copy of the repository. +2. Clone the project to your local computer: + + ```bash + git clone https://github.com//conda-store-ui/ + ``` -Before setting up conda-store-ui, you must prepare your environment. +Once you have a local copy of the `conda-store` repository, you can set up your development environment. +There are two main ways to set up your local environment for development: -We use [Docker Compose](https://docs.docker.com/compose/) to set up the infrastructure before starting ensure that you have docker-compose installed. If you need to install docker-compose, please see their [installation documentation](https://docs.docker.com/compose/install/) +- Using [Docker and docker-compose(recommended)](#docker-based-development---conda-store-ui) +- Local development [without Docker](#local-development-without-docker---conda-store-ui) -1. Clone the [conda-store-ui](https://github.com/conda-incubator/conda-store-ui.git) repository. -2. Copy `.env.example` to `.env`. All default settings should work, but if you want to test against a different version of conda-store-server, you can specify it in the `.env` file by setting the `CONDA_STORE_SERVER_VERSION` variable to the desired version +### Pre-requisites -### Local Development with conda-store-ui running in Docker +- [Node.js](https://nodejs.org/en/download/) >= 18 +- [Yarn](https://classic.yarnpkg.com/lang/en/docs/) + +### Docker-based development - conda-store-ui Running conda-store-ui in Docker is the simplest way to set up your local development environment. -1. Run `yarn run start:docker` to start the entire development stack. -Open your local browser and go to [http://localhost:8000](http://localhost:8000) so see conda-store-ui. -3. You can then log in using the default username of `username` and default password of `password`. +We use [docker-compose](https://docs.docker.com/compose/) to set up the infrastructure before starting, +you must ensure you have docker-compose installed. +If you need to install docker-compose, please see their [installation documentation](https://docs.docker.com/compose/install/). + +1. After cloning the repository change to the project directory: + + ```bash + # from your command line or terminal + cd conda-store-ui + ``` + +2. Copy `.env.example` to `.env`. All default settings should work, but if you want to test against a different version + of `conda-store-server`, you can specify it in the `.env` file by setting the `CONDA_STORE_SERVER_VERSION` variable + to the desired version. +3. Run `yarn run start:docker` to start the entire development stack. +4. Open your local browser and go to [http://localhost:8000](http://localhost:8000) so see conda-store-ui. +5. You can then log in using the default username of `username` and default password of `password`. + +:::note + +Hot reloading is enabled, so when you make changes to source files, your browser will automatically reload +and reflect the changes. + +::: -**Note:** Hot reloading is enabled, so when you make changes to source files, your browser will reload and reflect the changes. +### Local development without Docker - conda-store-ui -### Local Development without running conda-store-ui in Docker +:::note This setup still uses Docker for supporting services but runs conda-store-ui locally. +::: + #### Set up your environment -This project uses [Conda](https://conda.io) for package management. To set up Conda, please see their [installation documentation](https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html). -1. Change to the project root ` cd conda-store-ui` -2. From the project root create the conda environment `conda env create -f environment_dev.yml` -3. Activate the development environment `conda activate cs-ui-dev-env` -4. Install yarn dependencies `yarn install` +This project uses [conda](https://conda.io) for package management. +To set up conda, please see their [installation documentation](https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html). + +1. After cloning the repository change to the project directory: + + ```bash + # from your command line or terminal + cd conda-store-ui + ``` + +2. From the project root create and activate a new conda environment: + + ```bash + conda env create -f environment_dev.yml + conda activate cs-ui-dev-env + ``` + +3. Install node dependencies: + + ```bash + yarn install + ``` #### Run the application -1. Run `yarn run start` and wait for the application to finish starting up -Open your local browser and go to [http://localhost:8000](http://localhost:8000) so see conda-store-ui. -3. You can then log in using the default username of `username` and default password of `password`. +1. Run `yarn run start` and wait for the application to spin up. +Open your local browser and go to [http://localhost:8000](http://localhost:8000). +2. You can then log in using the default username of `username` and default password of `password`. + +:::note + +Hot reloading is enabled, so when you make changes to source files, your browser will reload and reflect the changes. +::: -**Note:** Hot reloading is enabled, so when you make changes to source files, your browser will reload and reflect the changes. +The REST API is considered somewhat stable. If any changes are made to the API make sure the update the OpenAPI/Swagger +specification in `docs/_static/openapi.json`. This may be downloaded from the `/docs` endpoint when running conda-store. +Ensure that the `c.CondaStoreServer.url_prefix` is set to `/` when generating the endpoints. -## Workflows +### Adding new dependencies to the libraries + +### `conda-store-core` + +Runtime-required dependencies should be added to the `pyproject.toml` file: + +- `conda-store-server/pyproject.toml` +- `conda-store/pyproject.toml` + +Development dependencies should be added to both the `environment-dev.yaml` and `pyproject.toml` files. -### Changes to API +This will ensure that conda-store distributions are properly built and tested with the correct dependencies. + +### `conda-store-ui` + +Dependencies should be added to the [`package.json`](https://github.com/conda-incubator/conda-store-ui/blob/main/package.json) file. + +### Linting and formatting + +We use pre-commit hooks to ensure that code is formatted and linted before being committed. +To install the pre-commit hooks, run: + +```bash +pre-commit install --install-hooks +``` -The REST API is considered somewhat stable. If any changes are made to -the API make sure the update the OpenAPI/Swagger specification in -`docs/_static/openapi.json`. This may be downloaded from the `/docs` -endpoint when running conda-store. Ensure that the -`c.CondaStoreServer.url_prefix` is set to `/` when generating the -endpoints. +Now every time you commit, the pre-commit hooks will run and check your code. +We also use [pre-commit.ci](https://pre-commit.com/) to automatically run the pre-commit hooks on every Pull Requests. From bcdb39a3ce1e9c813a5a35422bca807b48d90ced Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Thu, 21 Mar 2024 13:43:55 +0000 Subject: [PATCH 16/26] :construction_worker: Pin BAIPP to v1 --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f4fc8402b..2b8ecace2 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -39,7 +39,7 @@ jobs: - run: echo "Running on ${{ matrix.directory }}" - name: "Check package build - ${{ matrix.directory }} ๐Ÿ“ฆ" - uses: hynek/build-and-inspect-python-package@v2 + uses: hynek/build-and-inspect-python-package@v1 with: path: ${{ matrix.directory }} From 390fc5ffec9699d89db1400d1cb70f9dcc0aac0e Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Thu, 21 Mar 2024 13:57:55 +0000 Subject: [PATCH 17/26] More updates to GH actions --- .github/workflows/release.yaml | 11 ++++++++--- .github/workflows/tests.yaml | 23 +++++++++++------------ 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2b8ecace2..4a9f7d09a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -17,6 +17,11 @@ env: permissions: contents: read # This is required for actions/checkout +# ensuring only one instance is running at a given time +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + jobs: # Always build & verify package. build-package: @@ -45,7 +50,7 @@ jobs: # Adding a separate upload for publishing - name: "Upload build artefacts ๐Ÿ“ค" - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: ${{ matrix.directory }}-package path: | @@ -69,7 +74,7 @@ jobs: steps: - name: "Download build artefacts ๐Ÿ“ฅ" - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v3 with: name: ${{ matrix.directory }}-package path: dist @@ -101,7 +106,7 @@ jobs: steps: - name: "Download build artefacts ๐Ÿ“ฅ" - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v3 with: name: ${{ matrix.directory }}-package path: dist diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index fb7ef280c..209836f1c 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -2,6 +2,7 @@ name: Tests env: DEFAULT_PYTHON_VERSION: "3.10" + FORCE_COLOR: "1" # Make tools pretty. on: pull_request: @@ -55,11 +56,11 @@ jobs: - name: "Installed built conda-store-server ๐Ÿ“ฆ" run: | - pip install dist/*.whl + python -Im pip install dist/*.whl - name: "Unit tests โœ…" run: | - pytest -m "not extended_prefix and not user_journey" tests + python -Im pytest -m "not extended_prefix and not user_journey" tests # https://github.com/actions/runner-images/issues/1052 - name: "Windows extended prefix unit tests โœ…" @@ -93,7 +94,6 @@ jobs: - name: "Install build dependencies ๐Ÿ“ฆ" run: | - pip install hatch sudo apt install wait-for-it -y - name: "Deploy docker-compose" @@ -108,7 +108,7 @@ jobs: - name: "Run Playwright tests ๐ŸŽญ" run: | playwright install - pytest --video on ../tests/test_playwright.py + python -Im pytest --video on ../tests/test_playwright.py - name: "Upload test results ๐Ÿ“ค" uses: actions/upload-artifact@v4 @@ -120,11 +120,11 @@ jobs: - name: "Run integration tests โœ…" run: | export PYTHONPATH=$PYTHONPATH:$PWD - pytest ../tests/test_api.py ../tests/test_metrics.py + python -Im pytest ../tests/test_api.py ../tests/test_metrics.py - name: "Run user journey tests โœ…" run: | - pytest -m "user_journey" + python -Im pytest -m "user_journey" - name: "Get Docker logs ๐Ÿ”" if: ${{ failure() }} @@ -139,7 +139,7 @@ jobs: working-directory: conda-store strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - name: "Checkout Repository ๐Ÿ›Ž" uses: actions/checkout@v4 @@ -151,12 +151,11 @@ jobs: - name: "Install Dependencies ๐Ÿ“ฆ" run: | - pip install hatch sudo apt install wait-for-it -y - name: "Linting Checks ๐Ÿงน" run: | - hatch env run -e dev lint + hatch env run -e lint lint - name: "Build package ๐Ÿ“ฆ" run: | @@ -173,7 +172,7 @@ jobs: - name: "Install built conda-store for tests ๐Ÿ“ฆ" run: | - pip install dist/*.whl + python -Im pip install dist/*.whl - name: "Run basic tests - not authenticated" run: | @@ -206,13 +205,13 @@ jobs: - conda-store - conda-store-server steps: - - name: "Checkout Repository" + - name: "Checkout Repository ๐Ÿ›Ž" uses: actions/checkout@v4 - name: "Set up Docker Buildx" uses: docker/setup-buildx-action@v3 - - name: "Lint Dockerfiles" + - name: "Lint Dockerfiles ๐Ÿ”" uses: jbergstroem/hadolint-gh-action@v1 with: dockerfile: ${{ matrix.docker-image }}/Dockerfile From 69a49ca4c1bb587d18c1661a3cbf697f5fc1d6ca Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Thu, 21 Mar 2024 14:05:08 +0000 Subject: [PATCH 18/26] :heavy_plus_sign: Add hatch to action --- .github/workflows/tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 209836f1c..9f9777d71 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -151,6 +151,7 @@ jobs: - name: "Install Dependencies ๐Ÿ“ฆ" run: | + pyhton -Im pip install hatch sudo apt install wait-for-it -y - name: "Linting Checks ๐Ÿงน" From 4aa7d7b87f6fcead927b84e72cc1f10359342b6e Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Thu, 21 Mar 2024 14:07:49 +0000 Subject: [PATCH 19/26] Fix typo --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 9f9777d71..ae9a7f405 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -151,7 +151,7 @@ jobs: - name: "Install Dependencies ๐Ÿ“ฆ" run: | - pyhton -Im pip install hatch + python -Im pip install hatch sudo apt install wait-for-it -y - name: "Linting Checks ๐Ÿงน" From 14314192f1755b14d3094397b07dbd16a02c1d93 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Thu, 21 Mar 2024 14:22:28 +0000 Subject: [PATCH 20/26] Fix artefacts name --- .github/workflows/release.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 4a9f7d09a..e9b8f2545 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -17,11 +17,6 @@ env: permissions: contents: read # This is required for actions/checkout -# ensuring only one instance is running at a given time -concurrency: - group: ${{ github.ref }} - cancel-in-progress: true - jobs: # Always build & verify package. build-package: From 8fcf25920182d47e34488a56e5988d3e302fc74d Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Fri, 22 Mar 2024 12:35:18 +0000 Subject: [PATCH 21/26] :construction_worker: Improve deps in CI --- .github/workflows/build_docker_image.yaml | 3 ++ .github/workflows/release.yaml | 5 --- .github/workflows/tests.yaml | 37 ++++++++++------------- 3 files changed, 19 insertions(+), 26 deletions(-) diff --git a/.github/workflows/build_docker_image.yaml b/.github/workflows/build_docker_image.yaml index bb7dcb13c..e5d4d49b7 100644 --- a/.github/workflows/build_docker_image.yaml +++ b/.github/workflows/build_docker_image.yaml @@ -9,6 +9,9 @@ permissions: id-token: write # This is required for requesting the JWT contents: read # This is required for actions/checkout +env: + FORCE_COLOR: "1" # Make tools pretty. + jobs: build_and_push_docker_image: name: "Build Docker Images ๐Ÿ› " diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 4a9f7d09a..e9b8f2545 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -17,11 +17,6 @@ env: permissions: contents: read # This is required for actions/checkout -# ensuring only one instance is running at a given time -concurrency: - group: ${{ github.ref }} - cancel-in-progress: true - jobs: # Always build & verify package. build-package: diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index ae9a7f405..08996ba03 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -50,12 +50,9 @@ jobs: run: | hatch env run -e lint lint - - name: "Build package ๐Ÿ“ฆ" + - name: "Build and install conda-store-server ๐Ÿ“ฆ" run: | hatch build - - - name: "Installed built conda-store-server ๐Ÿ“ฆ" - run: | python -Im pip install dist/*.whl - name: "Unit tests โœ…" @@ -74,7 +71,6 @@ jobs: pytest -m "extended_prefix" tests if: matrix.os == 'windows' - integration-test-conda-store-server: name: "integration-test conda-store-server" runs-on: ubuntu-latest @@ -94,7 +90,8 @@ jobs: - name: "Install build dependencies ๐Ÿ“ฆ" run: | - sudo apt install wait-for-it -y + sudo apt update + sudo apt install -y wait-for-it - name: "Deploy docker-compose" run: | @@ -107,8 +104,8 @@ jobs: - name: "Run Playwright tests ๐ŸŽญ" run: | - playwright install - python -Im pytest --video on ../tests/test_playwright.py + playwright install --with-deps chromium + python -m pytest --video on ../tests/test_playwright.py - name: "Upload test results ๐Ÿ“ค" uses: actions/upload-artifact@v4 @@ -119,12 +116,11 @@ jobs: - name: "Run integration tests โœ…" run: | - export PYTHONPATH=$PYTHONPATH:$PWD - python -Im pytest ../tests/test_api.py ../tests/test_metrics.py + python -m pytest ../tests/test_api.py ../tests/test_metrics.py - name: "Run user journey tests โœ…" run: | - python -Im pytest -m "user_journey" + python -m pytest -m "user_journey" - name: "Get Docker logs ๐Ÿ”" if: ${{ failure() }} @@ -149,19 +145,17 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: "Install Dependencies ๐Ÿ“ฆ" run: | python -Im pip install hatch - sudo apt install wait-for-it -y + sudo apt update + sudo apt install -y wait-for-it - name: "Linting Checks ๐Ÿงน" run: | hatch env run -e lint lint - - name: "Build package ๐Ÿ“ฆ" - run: | - hatch build - - name: "Deploy docker-compose" run: | docker-compose up -d @@ -171,16 +165,17 @@ jobs: wait-for-it localhost:9000 # minio wait-for-it localhost:8080 # conda-store-server - - name: "Install built conda-store for tests ๐Ÿ“ฆ" + - name: "Build and install conda-store ๐Ÿ“ฆ" run: | + hatch build python -Im pip install dist/*.whl - - name: "Run basic tests - not authenticated" + - name: "Run basic tests - not authenticated ๐Ÿ”“" run: | sleep 20 ./tests/unauthenticated-tests.sh - - name: "Run basic tests - authenticated" + - name: "Run basic tests - authenticated ๐Ÿ”" run: | ./tests/authenticated-tests.sh @@ -198,7 +193,7 @@ jobs: docker-compose logs build-docker-image: - name: "Build docker images" + name: "Build Docker images ๐Ÿ‹" runs-on: ubuntu-latest strategy: matrix: @@ -228,7 +223,7 @@ jobs: tags: | type=sha - - name: "Build docker" + - name: "Build Docker image ๐Ÿ‹" uses: docker/build-push-action@v5 with: context: "${{ matrix.docker-image }}" From 696d61b50c4b063afc41241b8a22045cd13aee0e Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Fri, 22 Mar 2024 16:33:16 +0000 Subject: [PATCH 22/26] Pin build deps --- conda-store-server/environment-dev.yaml | 3 ++- conda-store-server/pyproject.toml | 8 +++----- conda-store/pyproject.toml | 3 ++- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/conda-store-server/environment-dev.yaml b/conda-store-server/environment-dev.yaml index 4b6d6c1a3..43c535f8c 100644 --- a/conda-store-server/environment-dev.yaml +++ b/conda-store-server/environment-dev.yaml @@ -20,7 +20,8 @@ dependencies: - docker-compose # build dependencies - hatch - - twine + - twine>5.0 + - pkginfo > 1.10 - build - pip - pip: diff --git a/conda-store-server/pyproject.toml b/conda-store-server/pyproject.toml index 4c12624fc..213e338aa 100644 --- a/conda-store-server/pyproject.toml +++ b/conda-store-server/pyproject.toml @@ -14,7 +14,7 @@ authors = [ { name = "Christopher Ostrouchov", email = "crhsi.ostrouchov@gmail.com" }, ] maintainers = [{ name = "Tania Allard", email = "trallard@bitsandchips.me" }, - {name= "Pavithra Eswaramoorthy", email = " pavithraes@outlook.com"}, + {name= "Pavithra Eswaramoorthy", email = "pavithraes@outlook.com"}, {name="Nikita Karetnikov", email="nkaretnikov@quansight.com"}, ] classifiers = [ @@ -75,9 +75,6 @@ Issues = "https://github.com/conda-incubator/conda-store/issues" [tool.hatch.version] path = "conda_store_server/__init__.py" -[project.optional-dependencies] -dev=["build", "twine"] - [tool.hatch.envs.dev] dependencies = [ "aiohttp>=3.8.1", @@ -91,7 +88,8 @@ dependencies = [ "pytest-celery", "pytest-mock", "pytest-playwright", - "twine" + "twine>5.0", + "pkginfo > 1.10", ] [tool.hatch.envs.lint] diff --git a/conda-store/pyproject.toml b/conda-store/pyproject.toml index c64117062..e569213d4 100644 --- a/conda-store/pyproject.toml +++ b/conda-store/pyproject.toml @@ -53,7 +53,8 @@ path = "conda_store/__init__.py" dependencies = [ "build", "pytest", - "twine", + "twine>5.0", + "pkginfo > 1.10", "pytest-mock", ] From 997faa2acb583f3948c6975a544220e7cb44848d Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Fri, 22 Mar 2024 16:50:08 +0000 Subject: [PATCH 23/26] Pin build deps --- .github/workflows/release.yaml | 12 ++++++++++++ conda-store-server/environment-dev.yaml | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e9b8f2545..98c1abaa0 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -38,6 +38,18 @@ jobs: - run: echo "Running on ${{ matrix.directory }}" + - name: "Check package build - ${{ matrix.directory }} ๐Ÿ“ฆ" + uses: hynek/build-and-inspect-python-package@v1 + with: + path: ${{ matrix.directory }} + continue-on-error: true + + # Quick hack - @trallard to come back and fix + - name: "Force reinstall" + run: | + /tmp/baipp/bin/python -Im pip install "pkginfo>=1.10" "twine>=5.0" + + - name: "Check package build - ${{ matrix.directory }} ๐Ÿ“ฆ" uses: hynek/build-and-inspect-python-package@v1 with: diff --git a/conda-store-server/environment-dev.yaml b/conda-store-server/environment-dev.yaml index 43c535f8c..1aa811f04 100644 --- a/conda-store-server/environment-dev.yaml +++ b/conda-store-server/environment-dev.yaml @@ -20,9 +20,9 @@ dependencies: - docker-compose # build dependencies - hatch - - twine>5.0 - - pkginfo > 1.10 - build - pip - pip: - pytest-playwright + - twine>5.0 + - pkginfo>=1.10 From afc2061790a79cf5f6d11707bc77012c18170cc0 Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Mon, 25 Mar 2024 12:02:57 +0000 Subject: [PATCH 24/26] :arrow_up: Use baipp from main --- .github/workflows/release.yaml | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 98c1abaa0..7fa1384bd 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -39,21 +39,10 @@ jobs: - run: echo "Running on ${{ matrix.directory }}" - name: "Check package build - ${{ matrix.directory }} ๐Ÿ“ฆ" - uses: hynek/build-and-inspect-python-package@v1 - with: - path: ${{ matrix.directory }} - continue-on-error: true - - # Quick hack - @trallard to come back and fix - - name: "Force reinstall" - run: | - /tmp/baipp/bin/python -Im pip install "pkginfo>=1.10" "twine>=5.0" - - - - name: "Check package build - ${{ matrix.directory }} ๐Ÿ“ฆ" - uses: hynek/build-and-inspect-python-package@v1 + uses: hynek/build-and-inspect-python-package@main with: path: ${{ matrix.directory }} + upload-name-suffix: "-${{ matrix.directory }}" # Adding a separate upload for publishing - name: "Upload build artefacts ๐Ÿ“ค" From 16bae5c4deabd403f71e295dccc5ef59ce14f7ef Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Mon, 25 Mar 2024 12:08:28 +0000 Subject: [PATCH 25/26] Fix twine version pin --- conda-store-server/environment-dev.yaml | 2 +- conda-store-server/pyproject.toml | 4 ++-- conda-store/pyproject.toml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/conda-store-server/environment-dev.yaml b/conda-store-server/environment-dev.yaml index 1aa811f04..c4e128a11 100644 --- a/conda-store-server/environment-dev.yaml +++ b/conda-store-server/environment-dev.yaml @@ -24,5 +24,5 @@ dependencies: - pip - pip: - pytest-playwright - - twine>5.0 + - twine>=5.0.0 - pkginfo>=1.10 diff --git a/conda-store-server/pyproject.toml b/conda-store-server/pyproject.toml index 213e338aa..92c1c5b56 100644 --- a/conda-store-server/pyproject.toml +++ b/conda-store-server/pyproject.toml @@ -88,8 +88,8 @@ dependencies = [ "pytest-celery", "pytest-mock", "pytest-playwright", - "twine>5.0", - "pkginfo > 1.10", + "twine>=5.0.0", + "pkginfo >= 1.10", ] [tool.hatch.envs.lint] diff --git a/conda-store/pyproject.toml b/conda-store/pyproject.toml index e569213d4..014a20661 100644 --- a/conda-store/pyproject.toml +++ b/conda-store/pyproject.toml @@ -53,8 +53,8 @@ path = "conda_store/__init__.py" dependencies = [ "build", "pytest", - "twine>5.0", - "pkginfo > 1.10", + "twine>=5.0.0", + "pkginfo >= 1.10", "pytest-mock", ] From 16911adaf09a82f1685b2d5461ab2113983351ba Mon Sep 17 00:00:00 2001 From: Tania Allard Date: Mon, 25 Mar 2024 17:26:11 +0000 Subject: [PATCH 26/26] Remove flower from dev --- .github/workflows/tests.yaml | 8 ++++---- conda-store-server/environment-dev.yaml | 5 ----- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 08996ba03..cd5f6364a 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -53,11 +53,11 @@ jobs: - name: "Build and install conda-store-server ๐Ÿ“ฆ" run: | hatch build - python -Im pip install dist/*.whl + python -m pip install dist/*.whl - name: "Unit tests โœ…" run: | - python -Im pytest -m "not extended_prefix and not user_journey" tests + python -m pytest -m "not extended_prefix and not user_journey" tests # https://github.com/actions/runner-images/issues/1052 - name: "Windows extended prefix unit tests โœ…" @@ -148,7 +148,7 @@ jobs: - name: "Install Dependencies ๐Ÿ“ฆ" run: | - python -Im pip install hatch + python -m pip install hatch sudo apt update sudo apt install -y wait-for-it @@ -168,7 +168,7 @@ jobs: - name: "Build and install conda-store ๐Ÿ“ฆ" run: | hatch build - python -Im pip install dist/*.whl + python -m pip install dist/*.whl - name: "Run basic tests - not authenticated ๐Ÿ”“" run: | diff --git a/conda-store-server/environment-dev.yaml b/conda-store-server/environment-dev.yaml index c4e128a11..1aa71d01f 100644 --- a/conda-store-server/environment-dev.yaml +++ b/conda-store-server/environment-dev.yaml @@ -1,21 +1,16 @@ name: conda-store-server-dev channels: - conda-forge - - microsoft - nodefaults dependencies: - python ==3.10 # conda builds - conda ==23.5.2 - - python-docker - # web server - - flower # dev dependencies - aiohttp>=3.8.1 - pytest - pytest-celery - pytest-mock - - playwright - docker-py<7 # for docker-compose - docker-compose # build dependencies