diff --git a/.github/workflows/build_docker_image.yaml b/.github/workflows/build_docker_image.yaml index fa5489081..e5d4d49b7 100644 --- a/.github/workflows/build_docker_image.yaml +++ b/.github/workflows/build_docker_image.yaml @@ -9,6 +9,9 @@ permissions: id-token: write # This is required for requesting the JWT contents: read # This is required for actions/checkout +env: + FORCE_COLOR: "1" # Make tools pretty. + jobs: build_and_push_docker_image: name: "Build Docker Images ๐Ÿ› " @@ -23,7 +26,7 @@ jobs: uses: actions/checkout@v4 - name: "Retrieve secret from Vault ๐Ÿ—" - uses: hashicorp/vault-action@v2 + uses: hashicorp/vault-action@v3 with: method: jwt url: "https://quansight-vault-public-vault-b2379fa7.d415e30e.z1.hashicorp.cloud:8200" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e9b8f2545..7fa1384bd 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -39,9 +39,10 @@ jobs: - run: echo "Running on ${{ matrix.directory }}" - name: "Check package build - ${{ matrix.directory }} ๐Ÿ“ฆ" - uses: hynek/build-and-inspect-python-package@v1 + uses: hynek/build-and-inspect-python-package@main with: path: ${{ matrix.directory }} + upload-name-suffix: "-${{ matrix.directory }}" # Adding a separate upload for publishing - name: "Upload build artefacts ๐Ÿ“ค" diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index c078552b3..cd5f6364a 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -2,6 +2,7 @@ name: Tests env: DEFAULT_PYTHON_VERSION: "3.10" + FORCE_COLOR: "1" # Make tools pretty. on: pull_request: @@ -20,13 +21,6 @@ jobs: strategy: matrix: os: ["ubuntu", "macos", "windows"] - include: - - os: ubuntu - environment-file: conda-store-server/environment-dev.yaml - - os: macos - environment-file: conda-store-server/environment-macos-dev.yaml - - os: windows - environment-file: conda-store-server/environment-windows-dev.yaml runs-on: ${{ matrix.os }}-latest defaults: run: @@ -37,9 +31,9 @@ jobs: uses: actions/checkout@v4 - name: "Set up env ${{ matrix.os }} ๐Ÿ" - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: - environment-file: ${{ matrix.environment-file }} + environment-file: conda-store-server/environment-dev.yaml miniforge-version: latest # This fixes a "DLL not found" issue importing ctypes from the hatch env @@ -54,15 +48,16 @@ jobs: - name: "Linting Checks ๐Ÿงน" run: | - hatch env run -e dev lint + hatch env run -e lint lint - - name: "Build package ๐Ÿ“ฆ" + - name: "Build and install conda-store-server ๐Ÿ“ฆ" run: | hatch build + python -m pip install dist/*.whl - name: "Unit tests โœ…" run: | - pytest -m "not extended_prefix and not user_journey" tests + python -m pytest -m "not extended_prefix and not user_journey" tests # https://github.com/actions/runner-images/issues/1052 - name: "Windows extended prefix unit tests โœ…" @@ -76,7 +71,6 @@ jobs: pytest -m "extended_prefix" tests if: matrix.os == 'windows' - integration-test-conda-store-server: name: "integration-test conda-store-server" runs-on: ubuntu-latest @@ -89,15 +83,15 @@ jobs: uses: actions/checkout@v4 - name: "Set up env ๐Ÿ" - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: environment-file: conda-store-server/environment-dev.yaml miniforge-version: latest - name: "Install build dependencies ๐Ÿ“ฆ" run: | - pip install hatch - sudo apt install wait-for-it -y + sudo apt update + sudo apt install -y wait-for-it - name: "Deploy docker-compose" run: | @@ -110,24 +104,23 @@ jobs: - name: "Run Playwright tests ๐ŸŽญ" run: | - playwright install - pytest --video on ../tests/test_playwright.py + playwright install --with-deps chromium + python -m pytest --video on ../tests/test_playwright.py - name: "Upload test results ๐Ÿ“ค" - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: ${{ always() }} with: name: playwright-tests - path: conda-store-server/test-results + path: conda-store-server/test-results/ - name: "Run integration tests โœ…" run: | - export PYTHONPATH=$PYTHONPATH:$PWD - pytest ../tests/test_api.py ../tests/test_metrics.py + python -m pytest ../tests/test_api.py ../tests/test_metrics.py - name: "Run user journey tests โœ…" run: | - pytest -m "user_journey" + python -m pytest -m "user_journey" - name: "Get Docker logs ๐Ÿ”" if: ${{ failure() }} @@ -142,28 +135,26 @@ jobs: working-directory: conda-store strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - name: "Checkout Repository ๐Ÿ›Ž" uses: actions/checkout@v4 - name: "Set up Python ๐Ÿ" - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: "Install Dependencies ๐Ÿ“ฆ" run: | - pip install hatch - sudo apt install wait-for-it -y + python -m pip install hatch + sudo apt update + sudo apt install -y wait-for-it - name: "Linting Checks ๐Ÿงน" run: | - hatch env run -e dev lint - - - name: "Build package ๐Ÿ“ฆ" - run: | - hatch build + hatch env run -e lint lint - name: "Deploy docker-compose" run: | @@ -174,16 +165,17 @@ jobs: wait-for-it localhost:9000 # minio wait-for-it localhost:8080 # conda-store-server - - name: "Install conda-store for tests ๐Ÿ“ฆ" + - name: "Build and install conda-store ๐Ÿ“ฆ" run: | - pip install . + hatch build + python -m pip install dist/*.whl - - name: "Run basic tests - not authenticated" + - name: "Run basic tests - not authenticated ๐Ÿ”“" run: | sleep 20 ./tests/unauthenticated-tests.sh - - name: "Run basic tests - authenticated" + - name: "Run basic tests - authenticated ๐Ÿ”" run: | ./tests/authenticated-tests.sh @@ -201,7 +193,7 @@ jobs: docker-compose logs build-docker-image: - name: "Build docker images" + name: "Build Docker images ๐Ÿ‹" runs-on: ubuntu-latest strategy: matrix: @@ -209,20 +201,20 @@ jobs: - conda-store - conda-store-server steps: - - name: "Checkout Repository" + - name: "Checkout Repository ๐Ÿ›Ž" uses: actions/checkout@v4 - - name: Set up Docker Buildx + - name: "Set up Docker Buildx" uses: docker/setup-buildx-action@v3 - - name: Lint Dockerfiles + - name: "Lint Dockerfiles ๐Ÿ”" uses: jbergstroem/hadolint-gh-action@v1 with: dockerfile: ${{ matrix.docker-image }}/Dockerfile output_format: tty error_level: 0 - - name: Docker Meta + - name: "Docker Meta" id: meta uses: docker/metadata-action@v5 with: @@ -231,7 +223,7 @@ jobs: tags: | type=sha - - name: Build docker + - name: "Build Docker image ๐Ÿ‹" uses: docker/build-push-action@v5 with: context: "${{ matrix.docker-image }}" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 164a8bf16..c44478ec2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,33 +8,27 @@ # - Register git hooks: pre-commit install --install-hooks # - Run on all files: pre-commit run --all-files +ci: + autoupdate_schedule: monthly + autofix_commit_msg: | + [pre-commit.ci] Apply automatic pre-commit fixes + repos: - repo: https://github.com/psf/black - rev: 23.9.1 + rev: 24.3.0 hooks: - id: black exclude: "examples|tests/assets" - repo: https://github.com/charliermarsh/ruff-pre-commit - # Ruff version. - rev: "v0.0.289" + rev: "v0.3.3" hooks: - id: ruff exclude: "examples|tests/assets" - args: ["--fix"] - - - repo: https://github.com/pycqa/isort - rev: 5.12.0 - hooks: - - id: isort - name: isort - additional_dependencies: [toml] - files: \.py$ - args: ["--profile", "black"] - exclude: "conda-store-server/conda_store_server/action/__init__.py" + args: [--fix, --exit-non-zero-on-fix] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: trailing-whitespace files: ".*\\.py" @@ -44,7 +38,7 @@ repos: - id: trailing-whitespace - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.3 + rev: v4.0.0-alpha.8 hooks: - id: prettier exclude: ^(examples/|templates/|) diff --git a/README.md b/README.md index 60ddef78a..8d2ec6f92 100644 --- a/README.md +++ b/README.md @@ -10,53 +10,46 @@ Flexible. Reproducible. Governable. --- -
- - - PyPi - - - - PyPi - - - - Conda - - - - Conda - - -
+| Information | Links| +| :---------- | :-----| +| Project | [![License](https://img.shields.io/badge/License-BSD%203--Clause-gray.svg?colorA=2b2d42&colorB=206532&style=flat.svg)](https://opensource.org/licenses/BSD-3-Clause) [![Read the docs](https://img.shields.io/badge/%F0%9F%93%96%20Read-the%20docs-gray.svg?colorA=2b2d42&colorB=206532&style=flat.svg)](https://conda.store)| +|Community | [![Open an issue](https://img.shields.io/badge/%F0%9F%93%9D%20Open-an%20issue-gray.svg?colorA=2b2d42&colorB=206532&style=flat.svg)](https://github.com/conda-incubator/conda-store/issues/new/choose) [![Community guidelines](https://img.shields.io/badge/๐Ÿค%20Community-guidelines-gray.svg?colorA=2b2d42&colorB=206532&style=flat.svg)](https://conda.store/community/introduction)| +|Releases | [![PyPI release conda-store](https://img.shields.io/pypi/v/conda-store)](https://badge.fury.io/py/conda-store?label=pypi%20conda-store&style=flat.svg) [![PyPI release conda-store-server](https://img.shields.io/pypi/v/conda-store-server?label=pypi%20conda-store-server)](https://badge.fury.io/py/conda-store-server) [![conda-forge release conda-store](https://img.shields.io/conda/vn/conda-forge/conda-store?label=conda-forge%20conda-store)]((https://anaconda.org/conda-forge/conda-store)) [![conda-forge release conda-store-server](https://img.shields.io/conda/vn/conda-forge/conda-store-server?label=conda-forge%20conda-store-server)]((https://anaconda.org/conda-forge/conda-store-server)) | --- -conda-store provides the familiarity and flexibility of conda environments, without compromising reliability for collaborative settings. +conda-store provides the familiarity and flexibility of conda environments, without compromising reliability for +collaborative settings. -conda-store is built to work for all team members from individual data scientists to administrators, while making sure your team follows best practices throughout the environment life cycle: from initial environment creation, to using environments in a production machine. +conda-store is built to work for all team members from individual data scientists to administrators, +while making sure your team follows best practices throughout the environment life cycle: +from initial environment creation to using environments in a production machine. ## Key features -- **Flexiblity**: +- **Flexibility**: - Users can create and update environments with the Graphical UI or a YAML editor. - The environments are automatically version-controlled and all versions are readily available. -- **Reproduciblity**: - - User can share environments quickly through the auto-generated artifacts including a lockfile, docker image, YAML file, and tarball. +- **Reproducibility**: + - Users can share environments quickly through the auto-generated artifacts including a lockfile, docker image, YAML file, and tarball. - conda-store pins exact versions of all packages and their dependencies in all the auto-generated artifacts. -- **Goverance**: - - Users have access to admin-approved packages and channels for their work, and can request new ones when needed. +- **Governance**: + - Users have access to admin-approved packages and channels for their work and can request new ones when needed. - Admins can insert or require certain packages and versions for organization-level compatibility. - - Admins can manage users' access-levels using "Namespaces", and allow users to share environments across (and only with) their team. + - Admins can manage users' access levels using "Namespaces", and allow users to share environments across (and only with) their team. ## Get started Learn more, including how to install, use, and contribute to conda-store in our documentation at [**conda.store**](https://conda.store/). -## Related repositories +## Contributing + +We welcome all types of contributions. Please read our [Contributing Guide](https://conda.store/community/contribute/) to get started. + +## Related repositories and projects -- We are working on a new UI for conda-store at: [`conda-incubator/conda-store-ui`](https://github.com/conda-incubator/conda-store-ui), and -- a JupyterLab extension at: [`conda-incubator/jupyterlab-conda-store`](https://github.com/conda-incubator/jupyterlab-conda-store). +- We are working on a new UI for conda-store at [`conda-incubator/conda-store-ui`](https://github.com/conda-incubator/conda-store-ui) and +- a JupyterLab extension at [`conda-incubator/jupyterlab-conda-store`](https://github.com/conda-incubator/jupyterlab-conda-store). ## Code of Conduct diff --git a/conda-store-server/conda_store_server/alembic/versions/03c839888c82_add_canceled_status.py b/conda-store-server/conda_store_server/alembic/versions/03c839888c82_add_canceled_status.py index 8c30d3b50..ac023d997 100644 --- a/conda-store-server/conda_store_server/alembic/versions/03c839888c82_add_canceled_status.py +++ b/conda-store-server/conda_store_server/alembic/versions/03c839888c82_add_canceled_status.py @@ -5,6 +5,7 @@ Create Date: 2024-01-29 03:56:36.889909 """ + import sqlalchemy as sa from alembic import op diff --git a/conda-store-server/conda_store_server/alembic/versions/0f7e23ff24ee_add_worker.py b/conda-store-server/conda_store_server/alembic/versions/0f7e23ff24ee_add_worker.py index ca086ebd0..19d0e4fd3 100644 --- a/conda-store-server/conda_store_server/alembic/versions/0f7e23ff24ee_add_worker.py +++ b/conda-store-server/conda_store_server/alembic/versions/0f7e23ff24ee_add_worker.py @@ -5,6 +5,7 @@ Create Date: 2023-12-13 21:01:45.546591 """ + import sqlalchemy as sa from alembic import op diff --git a/conda-store-server/conda_store_server/alembic/versions/16f65805dc8f_split_conda_package_into_conda_package_.py b/conda-store-server/conda_store_server/alembic/versions/16f65805dc8f_split_conda_package_into_conda_package_.py index dc6c07944..19d1e7067 100644 --- a/conda-store-server/conda_store_server/alembic/versions/16f65805dc8f_split_conda_package_into_conda_package_.py +++ b/conda-store-server/conda_store_server/alembic/versions/16f65805dc8f_split_conda_package_into_conda_package_.py @@ -5,6 +5,7 @@ Create Date: 2022-08-24 12:01:48.461989 """ + import sqlalchemy as sa from alembic import op diff --git a/conda-store-server/conda_store_server/alembic/versions/30b37e725c32_add_build_key_version.py b/conda-store-server/conda_store_server/alembic/versions/30b37e725c32_add_build_key_version.py index 0313a95f0..353a59555 100644 --- a/conda-store-server/conda_store_server/alembic/versions/30b37e725c32_add_build_key_version.py +++ b/conda-store-server/conda_store_server/alembic/versions/30b37e725c32_add_build_key_version.py @@ -5,6 +5,7 @@ Create Date: 2023-11-17 14:34:40.688865 """ + import sqlalchemy as sa from alembic import op diff --git a/conda-store-server/conda_store_server/alembic/versions/48be4072fe58_initial_schema.py b/conda-store-server/conda_store_server/alembic/versions/48be4072fe58_initial_schema.py index 7e3e15d87..bc83537c7 100644 --- a/conda-store-server/conda_store_server/alembic/versions/48be4072fe58_initial_schema.py +++ b/conda-store-server/conda_store_server/alembic/versions/48be4072fe58_initial_schema.py @@ -5,6 +5,7 @@ Create Date: 2022-06-01 18:37:12.396138 """ + import sqlalchemy as sa from alembic import op diff --git a/conda-store-server/conda_store_server/alembic/versions/57cd11b949d5_add_installer.py b/conda-store-server/conda_store_server/alembic/versions/57cd11b949d5_add_installer.py index db815fe1d..d4104a874 100644 --- a/conda-store-server/conda_store_server/alembic/versions/57cd11b949d5_add_installer.py +++ b/conda-store-server/conda_store_server/alembic/versions/57cd11b949d5_add_installer.py @@ -5,6 +5,7 @@ Create Date: 2024-01-28 14:31:35.723505 """ + import sqlalchemy as sa from alembic import op diff --git a/conda-store-server/conda_store_server/alembic/versions/5ad723de2abd_adding_container_registry_value_to_enum.py b/conda-store-server/conda_store_server/alembic/versions/5ad723de2abd_adding_container_registry_value_to_enum.py index 3398b7a36..282944eab 100644 --- a/conda-store-server/conda_store_server/alembic/versions/5ad723de2abd_adding_container_registry_value_to_enum.py +++ b/conda-store-server/conda_store_server/alembic/versions/5ad723de2abd_adding_container_registry_value_to_enum.py @@ -5,6 +5,7 @@ Create Date: 2022-08-05 22:14:34.110642 """ + import sqlalchemy as sa from alembic import op diff --git a/conda-store-server/conda_store_server/alembic/versions/771180018e1b_add_v2_role_mappings.py b/conda-store-server/conda_store_server/alembic/versions/771180018e1b_add_v2_role_mappings.py index a195b7a9f..02e859e0d 100644 --- a/conda-store-server/conda_store_server/alembic/versions/771180018e1b_add_v2_role_mappings.py +++ b/conda-store-server/conda_store_server/alembic/versions/771180018e1b_add_v2_role_mappings.py @@ -5,6 +5,7 @@ Create Date: 2023-11-29 09:02:35.835664 """ + import sqlalchemy as sa from alembic import op diff --git a/conda-store-server/conda_store_server/alembic/versions/8d63a091aff8_add_environment_description.py b/conda-store-server/conda_store_server/alembic/versions/8d63a091aff8_add_environment_description.py index 518362dbf..693bfa265 100644 --- a/conda-store-server/conda_store_server/alembic/versions/8d63a091aff8_add_environment_description.py +++ b/conda-store-server/conda_store_server/alembic/versions/8d63a091aff8_add_environment_description.py @@ -5,6 +5,7 @@ Create Date: 2022-07-15 14:22:00.351131 """ + import sqlalchemy as sa from alembic import op diff --git a/conda-store-server/conda_store_server/alembic/versions/abd7248d5327_adding_a_settings_table.py b/conda-store-server/conda_store_server/alembic/versions/abd7248d5327_adding_a_settings_table.py index 2fd34559d..0e1231dca 100644 --- a/conda-store-server/conda_store_server/alembic/versions/abd7248d5327_adding_a_settings_table.py +++ b/conda-store-server/conda_store_server/alembic/versions/abd7248d5327_adding_a_settings_table.py @@ -5,6 +5,7 @@ Create Date: 2023-05-11 16:38:12.210549 """ + import sqlalchemy as sa from alembic import op diff --git a/conda-store-server/conda_store_server/alembic/versions/b387747ca9b7_role_mapping.py b/conda-store-server/conda_store_server/alembic/versions/b387747ca9b7_role_mapping.py index 5f899a5bb..8a645298b 100644 --- a/conda-store-server/conda_store_server/alembic/versions/b387747ca9b7_role_mapping.py +++ b/conda-store-server/conda_store_server/alembic/versions/b387747ca9b7_role_mapping.py @@ -5,6 +5,7 @@ Create Date: 2023-07-04 14:35:48.177574 """ + import sqlalchemy as sa from alembic import op diff --git a/conda-store-server/conda_store_server/alembic/versions/d78e9889566a_add_status_info.py b/conda-store-server/conda_store_server/alembic/versions/d78e9889566a_add_status_info.py index d91938a11..e1cbdefc4 100644 --- a/conda-store-server/conda_store_server/alembic/versions/d78e9889566a_add_status_info.py +++ b/conda-store-server/conda_store_server/alembic/versions/d78e9889566a_add_status_info.py @@ -5,6 +5,7 @@ Create Date: 2023-11-07 12:25:04.416192 """ + import sqlalchemy as sa from alembic import op diff --git a/conda-store-server/conda_store_server/server/views/api.py b/conda-store-server/conda_store_server/server/views/api.py index 72c6826cf..ff4440b96 100644 --- a/conda-store-server/conda_store_server/server/views/api.py +++ b/conda-store-server/conda_store_server/server/views/api.py @@ -145,9 +145,11 @@ async def api_get_permissions( "status": "ok", "data": { "authenticated": authenticated, - "primary_namespace": entity.primary_namespace - if authenticated - else conda_store.default_namespace, + "primary_namespace": ( + entity.primary_namespace + if authenticated + else conda_store.default_namespace + ), "entity_permissions": entity_binding_permissions, "entity_roles": entity_binding_roles, "expiration": entity.exp if authenticated else None, diff --git a/conda-store-server/conda_store_server/server/views/registry.py b/conda-store-server/conda_store_server/server/views/registry.py index 7acd480c5..a43163d6e 100644 --- a/conda-store-server/conda_store_server/server/views/registry.py +++ b/conda-store-server/conda_store_server/server/views/registry.py @@ -159,9 +159,11 @@ def list_tags( try: auth.authorize_request( request, - image - if parts[0] != "conda-store-dynamic" - else "conda-store-dynamic/python", + ( + image + if parts[0] != "conda-store-dynamic" + else "conda-store-dynamic/python" + ), {Permissions.ENVIRONMENT_READ}, require=True, ) diff --git a/conda-store-server/environment-dev.yaml b/conda-store-server/environment-dev.yaml index b7dc836ad..1aa71d01f 100644 --- a/conda-store-server/environment-dev.yaml +++ b/conda-store-server/environment-dev.yaml @@ -1,63 +1,23 @@ name: conda-store-server-dev channels: - conda-forge - - microsoft - nodefaults dependencies: - python ==3.10 # conda builds - conda ==23.5.2 - - conda-docker >= 0.1.2 - - python-docker - - conda-pack - - conda-lock >=1.0.5 - - conda-package-handling - - conda-package-streaming - # web server - - celery - - flower - - redis-py - - sqlalchemy<=1.4.47 - - psycopg2 - - pymysql - - requests - - uvicorn - - fastapi - - pydantic < 2.0 - - pyyaml - - traitlets - - yarl - - pyjwt - - filelock - - itsdangerous - - jinja2 - - python-multipart - - alembic - # artifact storage - - minio - # installer - - constructor - # CLI - - typer - # dev dependencies - aiohttp>=3.8.1 - - hatch - pytest - pytest-celery - pytest-mock - - black ==22.3.0 - - flake8 - - ruff - - sphinx - - myst-parser - - sphinx-panels - - sphinx-copybutton - - pydata-sphinx-theme - - playwright - docker-py<7 # for docker-compose - docker-compose + # build dependencies + - hatch + - build - pip - - pip: - pytest-playwright + - twine>=5.0.0 + - pkginfo>=1.10 diff --git a/conda-store-server/environment-macos-dev.yaml b/conda-store-server/environment-macos-dev.yaml deleted file mode 100644 index 3cab8855d..000000000 --- a/conda-store-server/environment-macos-dev.yaml +++ /dev/null @@ -1,63 +0,0 @@ -name: conda-store-server-dev -channels: - - conda-forge - - microsoft - - nodefaults -dependencies: - - python ==3.10 - # conda builds - - conda ==23.5.2 - - python-docker - - conda-pack - - conda-lock >=1.0.5 - - mamba - - conda-package-handling - - conda-package-streaming - # web server - - celery - - flower - - redis-py - - sqlalchemy<=1.4.47 - - psycopg2 - - pymysql - - requests - - uvicorn - - fastapi - - pydantic < 2.0 - - pyyaml - - traitlets - - yarl - - pyjwt - - filelock - - itsdangerous - - jinja2 - - python-multipart - - alembic - # artifact storage - - minio - # installer - - constructor - # CLI - - typer - - # dev dependencies - - aiohttp>=3.8.1 - - hatch - - pytest - - pytest-celery - - pytest-mock - - black ==22.3.0 - - flake8 - - ruff - - sphinx - - myst-parser - - sphinx-panels - - sphinx-copybutton - - pydata-sphinx-theme - - playwright - - docker-py<7 # for docker-compose - - docker-compose - - pip - - - pip: - - pytest-playwright diff --git a/conda-store-server/environment-windows-dev.yaml b/conda-store-server/environment-windows-dev.yaml deleted file mode 100644 index 3cab8855d..000000000 --- a/conda-store-server/environment-windows-dev.yaml +++ /dev/null @@ -1,63 +0,0 @@ -name: conda-store-server-dev -channels: - - conda-forge - - microsoft - - nodefaults -dependencies: - - python ==3.10 - # conda builds - - conda ==23.5.2 - - python-docker - - conda-pack - - conda-lock >=1.0.5 - - mamba - - conda-package-handling - - conda-package-streaming - # web server - - celery - - flower - - redis-py - - sqlalchemy<=1.4.47 - - psycopg2 - - pymysql - - requests - - uvicorn - - fastapi - - pydantic < 2.0 - - pyyaml - - traitlets - - yarl - - pyjwt - - filelock - - itsdangerous - - jinja2 - - python-multipart - - alembic - # artifact storage - - minio - # installer - - constructor - # CLI - - typer - - # dev dependencies - - aiohttp>=3.8.1 - - hatch - - pytest - - pytest-celery - - pytest-mock - - black ==22.3.0 - - flake8 - - ruff - - sphinx - - myst-parser - - sphinx-panels - - sphinx-copybutton - - pydata-sphinx-theme - - playwright - - docker-py<7 # for docker-compose - - docker-compose - - pip - - - pip: - - pytest-playwright diff --git a/conda-store-server/environment.yaml b/conda-store-server/environment.yaml index f3b8e40f9..a865ffd32 100644 --- a/conda-store-server/environment.yaml +++ b/conda-store-server/environment.yaml @@ -6,33 +6,5 @@ dependencies: - python ==3.10 # conda environment builds - conda ==23.5.2 - - python-docker - - conda-docker >= 0.1.2 - - conda-pack - - conda-lock >=1.0.5 - - conda-package-handling - - conda-package-streaming # web server - - celery - flower - - redis-py - - sqlalchemy<=1.4.47 - - alembic - - psycopg2 - - pymysql - - requests - - pyyaml - - uvicorn - - fastapi - - pydantic < 2.0 - - traitlets - - yarl - - pyjwt - - filelock - - itsdangerous - - jinja2 - - python-multipart - # artifact storage - - minio - # installer - - constructor diff --git a/conda-store-server/pyproject.toml b/conda-store-server/pyproject.toml index c5c3e6093..92c1c5b56 100644 --- a/conda-store-server/pyproject.toml +++ b/conda-store-server/pyproject.toml @@ -9,10 +9,14 @@ description = "Conda Environment Management, Builds, and Serve" readme = "README.md" license = "BSD-3-Clause" requires-python = ">=3.8" -keywords = ["conda"] +keywords = ["conda", "dependency management", "environment management", "reproducibility"] authors = [ { name = "Christopher Ostrouchov", email = "crhsi.ostrouchov@gmail.com" }, ] +maintainers = [{ name = "Tania Allard", email = "trallard@bitsandchips.me" }, + {name= "Pavithra Eswaramoorthy", email = "pavithraes@outlook.com"}, + {name="Nikita Karetnikov", email="nkaretnikov@quansight.com"}, + ] classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", @@ -25,55 +29,77 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3 :: Only", ] + +dynamic = ["version"] + dependencies = [ + # env builds + # conda (which cannot be installed via pip see https://github.com/conda/conda/issues/11715) + "python-docker", + # we need platform dependent dependencies here "conda-docker; sys_platform == 'linux'", + "conda-lock >=1.0.5", "conda-pack", + "conda-package-handling", + "conda-package-streaming", + # web server + "alembic", "celery", - "sqlalchemy", - "requests", "fastapi", + "filelock", + "itsdangerous", + "jinja2", + "pyjwt", + "psycopg2-binary", + "pymysql", "pyyaml", "redis", + "requests", "pydantic<2", - "minio", + "python-multipart", + "sqlalchemy<=1.4.47", "traitlets", - "pyjwt", + "uvicorn", "yarl", - "filelock", - "itsdangerous", - "jinja2", - "python-multipart", - "python-docker" - # conda (which should not be included here) + # installer + "constructor", + # artifact storage + "minio", ] -dynamic = ["version"] [project.urls] Homepage = "https://conda.store/" Source = "https://github.com/conda-incubator/conda-store" +Issues = "https://github.com/conda-incubator/conda-store/issues" [tool.hatch.version] path = "conda_store_server/__init__.py" -[project.optional-dependencies] -dev = ["build", "twine"] - [tool.hatch.envs.dev] dependencies = [ + "aiohttp>=3.8.1", + "build", + "docker-compose", + "docker-py<7", + "flower", + "playwright", "pre-commit", - "sphinx", - "myst-parser", - "sphinx-panels", - "sphinx-copybutton", - "pydata-sphinx-theme", "pytest", + "pytest-celery", "pytest-mock", "pytest-playwright", + "twine>=5.0.0", + "pkginfo >= 1.10", ] -[tool.hatch.envs.dev.scripts] +[tool.hatch.envs.lint] +dependencies = ["pre-commit"] + +[tool.hatch.envs.lint.scripts] lint = ["pre-commit run --all"] -unit-test = "pytest tests -v" + +[tool.hatch.envs.dev.scripts] +unit-test = "pytest -m 'not extended_prefix and not user_journey' tests" playwright-test = [ "playwright install", "pytest --video on ../tests/test_playwright.py" @@ -92,7 +118,11 @@ conda-store-worker = "conda_store_server.worker.__main__:main" [tool.black] line-length = 88 -target-version = ['py37', 'py38', 'py39'] +target-version = ['py38', 'py39', 'py310', 'py311', 'py312'] + +[tool.ruff.lint.isort] +lines-between-types = 1 +lines-after-imports = 2 [tool.ruff] ignore = [ diff --git a/conda-store-server/tests/user_journeys/test_user_journeys.py b/conda-store-server/tests/user_journeys/test_user_journeys.py index 49584a5f0..a55e93b46 100644 --- a/conda-store-server/tests/user_journeys/test_user_journeys.py +++ b/conda-store-server/tests/user_journeys/test_user_journeys.py @@ -1,4 +1,5 @@ """User journey tests for the API.""" + import os import pytest diff --git a/conda-store-server/tests/user_journeys/utils/api_utils.py b/conda-store-server/tests/user_journeys/utils/api_utils.py index 0a50fd321..5030287de 100644 --- a/conda-store-server/tests/user_journeys/utils/api_utils.py +++ b/conda-store-server/tests/user_journeys/utils/api_utils.py @@ -1,4 +1,5 @@ """Helper functions for user journeys.""" + import time import uuid from enum import Enum diff --git a/conda-store/environment.yaml b/conda-store/environment.yaml index 122cb1d1b..aadebdf6a 100644 --- a/conda-store/environment.yaml +++ b/conda-store/environment.yaml @@ -2,19 +2,12 @@ name: conda-store channels: - conda-forge dependencies: - - pip - jupyterhub<2.0.0 - - jupyterlab>=3.0.0 + - jupyterlab>4.0 - nb_conda_kernels - # - nb_conda_store_kernels >=0.1.4 - - nodejs=16 + - nodejs=18 - yarn - # conda-store client dependencies - - yarl - - aiohttp>=3.8.1 - - rich - - click - - ruamel.yaml + - pip - pip: - https://github.com/yuvipanda/jupyter-launcher-shortcuts/archive/refs/heads/master.zip - jupyterlab-conda-store diff --git a/conda-store/pyproject.toml b/conda-store/pyproject.toml index 012a8dded..014a20661 100644 --- a/conda-store/pyproject.toml +++ b/conda-store/pyproject.toml @@ -9,12 +9,15 @@ description = "conda-store client" readme = "README.md" license = "BSD-3-Clause" requires-python = ">=3.8" -keywords = ["conda"] +keywords = ["conda", "dependency management", "environment management", "reproducibility"] authors = [ { name = "Christopher Ostrouchov", email = "chris.ostrouchov@gmail.com" }, ] -maintainers = [{ name = "Tania Allard", email = "trallard@bitsandchips.me" }] -# TODO: update maintainers +maintainers = [{ name = "Tania Allard", email = "trallard@bitsandchips.me" }, + {name= "Pavithra Eswaramoorthy", email = "pavithraes@outlook.com"}, + {name="Nikita Karetnikov", email="nkaretnikov@quansight.com"}, + ] + classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", @@ -31,33 +34,37 @@ classifiers = [ "Intended Audience :: Science/Research", "Intended Audience :: System Administrators", ] -dependencies = ["rich", "click", "yarl", "aiohttp", "ruamel.yaml"] dynamic = ["version"] +dependencies = ["rich", + "click", + "yarl", + "aiohttp>=3.8.1", + "ruamel.yaml"] [project.urls] Homepage = "https://conda.store/" Source = "https://github.com/conda-incubator/conda-store" +Issues = "https://github.com/conda-incubator/conda-store/issues" [tool.hatch.version] path = "conda_store/__init__.py" -[project.optional-dependencies] -dev = ["build", "twine"] - [tool.hatch.envs.dev] dependencies = [ - "pre-commit", - "sphinx", - "myst-parser", - "sphinx-panels", - "sphinx-copybutton", - "pydata-sphinx-theme", + "build", "pytest", + "twine>=5.0.0", + "pkginfo >= 1.10", "pytest-mock", ] -[tool.hatch.envs.dev.scripts] +[tool.hatch.envs.lint] +dependencies = ["pre-commit"] + +[tool.hatch.envs.lint.scripts] lint = ["pre-commit run --all"] + +[tool.hatch.envs.dev.scripts] test = "pytest tests" [project.scripts] @@ -65,10 +72,12 @@ conda-store = "conda_store.__main__:main" [tool.black] line-length = 88 -target-version = ['py38', 'py39', 'py310'] + +[tool.ruff.lint.isort] +lines-between-types = 1 +lines-after-imports = 2 [tool.ruff] ignore = [ "E501", # line-length - ] diff --git a/docusaurus-docs/community/contribute/contribute-code.md b/docusaurus-docs/community/contribute/contribute-code.md index bdfe2e687..6ac6d29b8 100644 --- a/docusaurus-docs/community/contribute/contribute-code.md +++ b/docusaurus-docs/community/contribute/contribute-code.md @@ -5,32 +5,50 @@ description: Contribute to conda-store's codebase # Contribute code :::warning -This page is in active development, content may be inaccurate and incomplete. +This page is in active development, content may be inaccurate and incomplete. If you encounter any content that needs improvement, please [create an issue](https://github.com/conda-incubator/conda-store/issues/new/choose) in our issue tracker. ::: -## Set up development environment - conda-store-core +## `conda-store-core` -### Docker (recommended) +Before setting up your `conda-store-core` development environment you will need to have a local copy of the `conda-store` repository. -Install the following dependencies before developing on conda-store. +If you are a first-time contributor: + +1. Go to [https://github.com/conda-incubator/conda-store/](https://github.com/conda-incubator/conda-store/) and click on the **Fork** button in the upper right corner to create your copy of the repository. +2. Clone the project to your local computer: + + ```bash + git clone https://github.com//conda-store/ + ``` + +Once you have a local copy of the `conda-store` repository, you can set up your development environment. +There are two main ways to set up your local environment for development: + +- Using [Docker and docker-compose(recommended)](#docker-based-development---conda-store-core) +- Local development [without Docker](#local-development-without-docker---conda-store-core) + +### Docker-based development - conda-store-core + +Install the following dependencies before developing on `conda-store`. - [Docker](https://docs.docker.com/engine/install/) - [docker-compose](https://docs.docker.com/compose/install/) -To deploy `conda-store` run the following command +To deploy `conda-store` run the following command: -```shell +```bash docker-compose up --build -d ``` :::important -Many of the conda-store docker images are built/tested for amd64(x86-64) +Many of the conda-store Docker images are built/tested for amd64(x86-64) there will be a performance impact when building and running on -arm architectures. Otherwise, this workflow has been shown to run and build on OSX. -Notice the `architecture: amd64` within the `docker-compose.yaml` files. +arm architectures. +Otherwise, this workflow has been shown to run and build on OSX. +**Notice** the `architecture: amd64` within the `docker-compose.yaml` files. ::: -The following resources will be available: +After running the docker-compose command, the following resources will be available: | Resource | Localhost port | username | password | |----------|----------------|----------|----------| @@ -40,7 +58,7 @@ The following resources will be available: | [PostgreSQL](https://www.postgresql.org/) (database: `conda-store`)| [localhost:5432](http://localhost:5432) | `admin` | `password` | | [Redis](https://www.redis.com/) | [localhost:6379](http://localhost:6379) | - | password | -On a fast machine this deployment should only take 10 or so seconds +On a fast machine, this deployment should only take 10 or so seconds assuming the docker images have been partially built before. If you are making any changes to `conda-store-server` and would like to see @@ -51,20 +69,26 @@ docker-compose down -v # not always necessary docker-compose up --build ``` -### Linux +### Local development without Docker - conda-store-core -1. Install the following dependencies before developing on conda-store: +You will need to install the following dependencies before developing on `conda-store`: - - [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/install/linux.html) +- [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html) -2. Install the development dependencies and activate the environment: +1. Install the development dependencies and activate the environment: -```shell -# replace this with environment-macos-dev.yaml or environment-windows-dev.yaml -# if you are on Mac or Windows -conda env create -f conda-store-server/environment-dev.yaml -conda activate conda-store-server-dev -``` + ```bash + # from the root of the repository + conda env create -f conda-store-server/environment-dev.yaml + conda activate conda-store-server-dev + ``` + +2. Install the package in editable mode: + + ```bash + # from the root of the repository + pip install -e ./conda-store-server + ``` 3. Running `conda-store` in `--standalone` mode launches celery as a subprocess of the web server. @@ -73,50 +97,106 @@ subprocess of the web server. python -m conda_store_server.server --standalone ``` -1. Visit [localhost:8080](http://localhost:8080/) +4. You should now be able to access the `conda-store` server at [localhost:8080](http://localhost:8080/) from your web browser. + +## `conda-store-ui` -## Set up development environment -- conda-store-ui +Before setting up your `conda-store-ui` development environment you will need to have a local copy of the +`conda-store-ui` repository. -To get started with conda-store-ui development, there are a couple of options. This guide will help you to set up your local development environment. +If you are a first-time contributor: -### Prerequisites +1. Go to [https://github.com/conda-incubator/conda-store-ui/](https://github.com/conda-incubator/conda-store-ui/) and click on the **Fork** button in the upper right corner to create your copy of the repository. +2. Clone the project to your local computer: + + ```bash + git clone https://github.com//conda-store-ui/ + ``` -Before setting up conda-store-ui, you must prepare your environment. +Once you have a local copy of the `conda-store` repository, you can set up your development environment. +There are two main ways to set up your local environment for development: -We use [Docker Compose](https://docs.docker.com/compose/) to set up the infrastructure before starting ensure that you have docker-compose installed. If you need to install docker-compose, please see their [installation documentation](https://docs.docker.com/compose/install/) +- Using [Docker and docker-compose(recommended)](#docker-based-development---conda-store-ui) +- Local development [without Docker](#local-development-without-docker---conda-store-ui) -1. Clone the [conda-store-ui](https://github.com/conda-incubator/conda-store-ui.git) repository. -2. Copy `.env.example` to `.env`. All default settings should work, but if you want to test against a different version of conda-store-server, you can specify it in the `.env` file by setting the `CONDA_STORE_SERVER_VERSION` variable to the desired version +### Pre-requisites -### Local Development with conda-store-ui running in Docker +- [Node.js](https://nodejs.org/en/download/) >= 18 +- [Yarn](https://classic.yarnpkg.com/lang/en/docs/) + +### Docker-based development - conda-store-ui Running conda-store-ui in Docker is the simplest way to set up your local development environment. -1. Run `yarn run start:docker` to start the entire development stack. -Open your local browser and go to [http://localhost:8000](http://localhost:8000) so see conda-store-ui. -3. You can then log in using the default username of `username` and default password of `password`. +We use [docker-compose](https://docs.docker.com/compose/) to set up the infrastructure before starting, +you must ensure you have docker-compose installed. +If you need to install docker-compose, please see their [installation documentation](https://docs.docker.com/compose/install/). + +1. After cloning the repository change to the project directory: + + ```bash + # from your command line or terminal + cd conda-store-ui + ``` + +2. Copy `.env.example` to `.env`. All default settings should work, but if you want to test against a different version + of `conda-store-server`, you can specify it in the `.env` file by setting the `CONDA_STORE_SERVER_VERSION` variable + to the desired version. +3. Run `yarn run start:docker` to start the entire development stack. +4. Open your local browser and go to [http://localhost:8000](http://localhost:8000) so see conda-store-ui. +5. You can then log in using the default username of `username` and default password of `password`. + +:::note + +Hot reloading is enabled, so when you make changes to source files, your browser will automatically reload +and reflect the changes. + +::: -**Note:** Hot reloading is enabled, so when you make changes to source files, your browser will reload and reflect the changes. +### Local development without Docker - conda-store-ui -### Local Development without running conda-store-ui in Docker +:::note This setup still uses Docker for supporting services but runs conda-store-ui locally. +::: + #### Set up your environment -This project uses [Conda](https://conda.io) for package management. To set up Conda, please see their [installation documentation](https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html). -1. Change to the project root ` cd conda-store-ui` -2. From the project root create the conda environment `conda env create -f environment_dev.yml` -3. Activate the development environment `conda activate cs-ui-dev-env` -4. Install yarn dependencies `yarn install` +This project uses [conda](https://conda.io) for package management. +To set up conda, please see their [installation documentation](https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html). + +1. After cloning the repository change to the project directory: + + ```bash + # from your command line or terminal + cd conda-store-ui + ``` + +2. From the project root create and activate a new conda environment: + + ```bash + conda env create -f environment_dev.yml + conda activate cs-ui-dev-env + ``` + +3. Install node dependencies: + + ```bash + yarn install + ``` #### Run the application -1. Run `yarn run start` and wait for the application to finish starting up -Open your local browser and go to [http://localhost:8000](http://localhost:8000) so see conda-store-ui. -3. You can then log in using the default username of `username` and default password of `password`. +1. Run `yarn run start` and wait for the application to spin up. +Open your local browser and go to [http://localhost:8000](http://localhost:8000). +2. You can then log in using the default username of `username` and default password of `password`. + +:::note + +Hot reloading is enabled, so when you make changes to source files, your browser will reload and reflect the changes. +::: -**Note:** Hot reloading is enabled, so when you make changes to source files, your browser will reload and reflect the changes. +The REST API is considered somewhat stable. If any changes are made to the API make sure the update the OpenAPI/Swagger +specification in `docs/_static/openapi.json`. This may be downloaded from the `/docs` endpoint when running conda-store. +Ensure that the `c.CondaStoreServer.url_prefix` is set to `/` when generating the endpoints. -## Workflows +### Adding new dependencies to the libraries + +### `conda-store-core` + +Runtime-required dependencies should be added to the `pyproject.toml` file: + +- `conda-store-server/pyproject.toml` +- `conda-store/pyproject.toml` + +Development dependencies should be added to both the `environment-dev.yaml` and `pyproject.toml` files. -### Changes to API +This will ensure that conda-store distributions are properly built and tested with the correct dependencies. + +### `conda-store-ui` + +Dependencies should be added to the [`package.json`](https://github.com/conda-incubator/conda-store-ui/blob/main/package.json) file. + +### Linting and formatting + +We use pre-commit hooks to ensure that code is formatted and linted before being committed. +To install the pre-commit hooks, run: + +```bash +pre-commit install --install-hooks +``` -The REST API is considered somewhat stable. If any changes are made to -the API make sure the update the OpenAPI/Swagger specification in -`docs/_static/openapi.json`. This may be downloaded from the `/docs` -endpoint when running conda-store. Ensure that the -`c.CondaStoreServer.url_prefix` is set to `/` when generating the -endpoints. +Now every time you commit, the pre-commit hooks will run and check your code. +We also use [pre-commit.ci](https://pre-commit.com/) to automatically run the pre-commit hooks on every Pull Requests. diff --git a/docusaurus-docs/conda-store/introduction.md b/docusaurus-docs/conda-store/introduction.md index dfbe5d7f0..59dbb4e87 100644 --- a/docusaurus-docs/conda-store/introduction.md +++ b/docusaurus-docs/conda-store/introduction.md @@ -9,11 +9,29 @@ description: Introduction to conda-store (core) documentation The [`conda-store` repository on GitHub][conda-store-repo] consists of two separate, yet related, packages: - **`conda-store-server`**: web server and workers that together provide the `conda-store` "service" through a REST API -- **`conda-store`** (client): a client which interacts with the service to offer user-facing command line interface +- **`conda-store`** (client): a client that interacts with the service to offer a user-facing command line interface -## Quickstart +## Pre-requisites -1. Install conda-store with `conda` or `pip`: +- Python `3.8` or later +- `conda`, if installing from scratch we recommend you install [`miniforge`](https://github.com/conda-forge/miniforge) or [`miniconda`](https://docs.anaconda.com/free/miniconda/miniconda-install/). + +:::warning + +`conda` is a hard requirement for `conda-store` and since it is not `pip` installable you need to have `conda` installed separately. + +::: + +## Quick start + +1. Create and activate a new `conda` environment: + +```bash +conda create -n conda-store-env python= +conda activate conda-store-env +``` + +2. Install `conda-store` with `conda` or `pip`: import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; @@ -25,26 +43,29 @@ import TabItem from '@theme/TabItem'; ```bash conda install -c conda-forge conda-store conda-store-server ``` + ```bash -pip install conda-store conda-store-server +# note that we still recommend you use conda to create an environment +python -m pip install conda-store conda-store-server ``` + -2. Start a standalone local instance and use the conda-store UI +3. Start a standalone local instance and use the `conda-store` UI ```bash conda-store-server --standalone ``` -You can then access conda-store at `localhost:8080` of the machine running it. +You can then access `conda-store` at `localhost:8080` of the machine running it. -3. Use the CLI +4. Use the CLI ```bash conda-store --help diff --git a/tests/test_api.py b/tests/test_api.py index 9583249b1..d5f4aded2 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -8,6 +8,7 @@ ordering for tests. """ + import asyncio import collections import datetime