Windows support

.github/workflows/tests.yaml

@@ -14,8 +14,7 @@ jobs:
     name: "unit-test conda-store-server"
     strategy:
       matrix:
-        # cannot run on windows due to needing fake-chroot for conda-docker
-        os: [ubuntu-latest, macos-latest]
+        os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     defaults:
       run:
@@ -29,7 +28,6 @@ jobs:
         if: matrix.os == 'ubuntu-latest'
         uses: conda-incubator/setup-miniconda@v2
         with:
-          mamba-version: "*"
           activate-environment: conda-store-server-dev
           environment-file: conda-store-server/environment-dev.yaml
           auto-activate-base: false
@@ -42,6 +40,24 @@ jobs:
           environment-file: conda-store-server/environment-macos-dev.yaml
           auto-activate-base: false
 
+      - name: Set up Python (Windows)
+        if: matrix.os == 'windows-latest'
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          activate-environment: conda-store-server-dev
+          environment-file: conda-store-server/environment-windows-dev.yaml
+          auto-activate-base: false
+
+      # This fixes a "DLL not found" issue importing ctypes from the hatch env
+      - name: Reinstall Python 3.10 on Windows runner
+        uses: nick-fields/[email protected]
+        with:
+          timeout_minutes: 9999
+          max_attempts: 6
+          command:
+            conda install --channel=conda-forge --quiet --yes python=${{ matrix.python }}
+        if: matrix.os == 'windows-latest'
+
       - name: Linting Checks
         run: |
           hatch env run -e dev lint
@@ -52,7 +68,7 @@ jobs:
 
       - name: Unit Tests
         run: |
-          pytest tests
+          pytest -vvv tests
 
   integration-test-conda-store-server:
     name: "integration-test conda-store-server"
@@ -68,7 +84,6 @@ jobs:
       - name: Set up Python
         uses: conda-incubator/setup-miniconda@v2
         with:
-          mamba-version: "*"
           activate-environment: conda-store-server-dev
           environment-file: conda-store-server/environment-dev.yaml
           auto-activate-base: false

conda-store-server/conda_store_server/action/base.py

@@ -18,7 +18,7 @@ def wrapper(*args, **kwargs):
             # redirect stdout -> action_context.stdout
             stack.enter_context(contextlib.redirect_stdout(action_context.stdout))
 
-            # redirect stderr -> action_context.stdout
+            # redirect stderr -> action_context.stderr
             stack.enter_context(contextlib.redirect_stderr(action_context.stdout))
 
             # create a temporary directory
@@ -38,20 +38,23 @@ class ActionContext:
     def __init__(self):
         self.id = str(uuid.uuid4())
         self.stdout = io.StringIO()
+        self.stderr = io.StringIO()
         self.log = logging.getLogger(f"conda_store_server.action.{self.id}")
         self.log.propagate = False
         self.log.addHandler(logging.StreamHandler(stream=self.stdout))
         self.log.setLevel(logging.INFO)
         self.result = None
         self.artifacts = {}
 
-    def run(self, *args, **kwargs):
+    def run(self, *args, redirect_stderr=True, **kwargs):
         result = subprocess.run(
             *args,
             **kwargs,
             stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
+            stderr=subprocess.STDOUT if redirect_stderr else subprocess.PIPE,
             encoding="utf-8",
         )
         self.stdout.write(result.stdout)
+        if not redirect_stderr:
+            self.stderr.write(result.stderr)
         return result

conda-store-server/conda_store_server/action/generate_conda_export.py

@@ -17,5 +17,7 @@ def action_generate_conda_export(
         "--json",
     ]
 
-    result = context.run(command, check=True)
+    result = context.run(command, check=True, redirect_stderr=False)
+    if result.stderr:
+        context.log.warning(f"conda env export stderr: {result.stderr}")
     return json.loads(result.stdout)

conda-store-server/conda_store_server/action/install_lockfile.py

@@ -1,5 +1,6 @@
 import json
 import pathlib
+import sys
 import typing
 
 from conda_store_server import action
@@ -16,7 +17,9 @@ def action_install_lockfile(
         json.dump(conda_lock_spec, f)
 
     command = [
-        "conda-lock",
+        sys.executable,
+        "-m",
+        "conda_lock",
         "install",
         "--validate-platform",
         "--log-level",

conda-store-server/conda_store_server/app.py

@@ -31,6 +31,8 @@
 )
 from traitlets.config import LoggingConfigurable
 
+ON_WIN = sys.platform.startswith("win")
+
 
 def conda_store_validate_specification(
     db: Session,
@@ -301,21 +303,24 @@ def _default_celery_results_backend(self):
     )
 
     default_uid = Integer(
-        os.getuid(),
+        None if ON_WIN else os.getuid(),
         help="default uid to assign to built environments",
         config=True,
+        allow_none=True,
     )
 
     default_gid = Integer(
-        os.getgid(),
+        None if ON_WIN else os.getgid(),
         help="default gid to assign to built environments",
         config=True,
+        allow_none=True,
     )
 
     default_permissions = Unicode(
-        "775",
+        None if ON_WIN else "775",
         help="default file permissions to assign to built environments",
         config=True,
+        allow_none=True,
     )
 
     default_docker_base_image = Union(

conda-store-server/conda_store_server/dbutil.py

@@ -1,6 +1,5 @@
 import os
 from contextlib import contextmanager
-from subprocess import check_call
 from tempfile import TemporaryDirectory
 
 from alembic import command
@@ -78,6 +77,8 @@ def upgrade(db_url, revision="head"):
     current_table_names = set(inspect(engine).get_table_names())
 
     with _temp_alembic_ini(db_url) as alembic_ini:
+        alembic_cfg = Config(alembic_ini)
+
         if (
             "alembic_version" not in current_table_names
             and len(current_table_names) > 0
@@ -86,10 +87,10 @@ def upgrade(db_url, revision="head"):
             # we stamp the revision at the first one, that introduces the alembic revisions.
             # I chose the leave the revision number hardcoded as it's not something
             # dynamic, not something we want to change, and tightly related to the codebase
-            command.stamp(Config(alembic_ini), "48be4072fe58")
+            command.stamp(alembic_cfg, "48be4072fe58")
             # After this point, whatever is in the database, Alembic will
             # believe it's at the first revision. If there are more upgrades/migrations
             # to run, they'll be at the next step :
 
         # run the upgrade.
-        check_call(["alembic", "-c", alembic_ini, "upgrade", revision])
+        command.upgrade(config=alembic_cfg, revision=revision)

conda-store-server/conda_store_server/orm.py

@@ -513,10 +513,11 @@ def update_packages(self, db, subdirs=None):
                 package_builds[package_key].append(new_package_build_dict)
             logger.info("CondaPackageBuild objects created")
 
-            batch_size = 1000
+            # sqlite3 has a max expression depth of 1000
+            batch_size = 990
             all_package_keys = list(package_builds.keys())
             for i in range(0, len(all_package_keys), batch_size):
-                logger.info(f"handling subset at index {i} (batch size {batch_size}")
+                logger.info(f"handling subset at index {i} (batch size {batch_size})")
                 subset_keys = all_package_keys[i : i + batch_size]
 
                 # retrieve the parent packages for the subset

conda-store-server/conda_store_server/schema.py

@@ -9,6 +9,8 @@
 from conda_store_server import conda_utils, utils
 from pydantic import BaseModel, Field, ValidationError, constr, validator
 
+ON_WIN = sys.platform.startswith("win")
+
 
 def _datetime_factory(offset: datetime.timedelta):
     """utcnow datetime + timezone as string"""
@@ -194,20 +196,20 @@ class Settings(BaseModel):
         metadata={"global": True},
     )
 
-    default_uid: int = Field(
-        os.getuid(),
+    default_uid: int | None = Field(
+        None if ON_WIN else os.getuid(),
         description="default uid to assign to built environments",
         metadata={"global": True},
     )
 
-    default_gid: int = Field(
-        os.getgid(),
+    default_gid: int | None = Field(
+        None if ON_WIN else os.getgid(),
         description="default gid to assign to built environments",
         metadata={"global": True},
     )
 
-    default_permissions: str = Field(
-        "775",
+    default_permissions: str | None = Field(
+        None if ON_WIN else "775",
         description="default file permissions to assign to built environments",
         metadata={"global": True},
     )

conda-store-server/conda_store_server/server/app.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import posixpath
 import sys
 
 import conda_store_server
@@ -198,9 +199,9 @@ def trim_slash(url):
         app = FastAPI(
             title="conda-store",
             version=__version__,
-            openapi_url=os.path.join(self.url_prefix, "openapi.json"),
-            docs_url=os.path.join(self.url_prefix, "docs"),
-            redoc_url=os.path.join(self.url_prefix, "redoc"),
+            openapi_url=posixpath.join(self.url_prefix, "openapi.json"),
+            docs_url=posixpath.join(self.url_prefix, "docs"),
+            redoc_url=posixpath.join(self.url_prefix, "redoc"),
             contact={
                 "name": "Quansight",
                 "url": "https://quansight.com",
@@ -335,6 +336,10 @@ def start(self):
 
         # start worker if in standalone mode
         if self.standalone:
+            address = "localhost" if self.address == "0.0.0.0" else self.address
+            print(
+                f"Starting standalone conda-store server at http://{address}:{self.port}"
+            )
             import multiprocessing
 
             multiprocessing.set_start_method("spawn")

conda-store-server/conda_store_server/server/auth.py

@@ -436,7 +436,9 @@ async def post_login_method(
             samesite="strict",
             domain=self.cookie_domain,
             # set cookie to expire at same time as jwt
-            max_age=(authentication_token.exp - datetime.datetime.utcnow()).seconds,
+            max_age=int(
+                (authentication_token.exp - datetime.datetime.utcnow()).total_seconds()
+            ),
         )
         return response
 

conda-store-server/conda_store_server/storage.py

@@ -1,5 +1,6 @@
 import io
 import os
+import posixpath
 import shutil
 
 import minio
@@ -223,7 +224,7 @@ def get(self, key):
             return f.read()
 
     def get_url(self, key):
-        return os.path.join(self.storage_url, key)
+        return posixpath.join(self.storage_url, key)
 
     def delete(self, db, build_id, key):
         filename = os.path.join(self.storage_path, key)

conda-store-server/conda_store_server/utils.py

@@ -50,13 +50,52 @@ def chdir(directory: pathlib.Path):
         os.chdir(current_directory)
 
 
+def du(path):
+    """
+    Pure Python equivalent of du -sb
+
+    Based on https://stackoverflow.com/a/55648984/161801
+    """
+    if os.path.islink(path):
+        return os.lstat(path).st_size
+    if os.path.isfile(path):
+        st = os.lstat(path)
+        return st.st_size
+    apparent_total_bytes = 0
+    have = set()
+    for dirpath, dirnames, filenames in os.walk(path):
+        apparent_total_bytes += os.lstat(dirpath).st_size
+        for f in filenames:
+            fp = os.path.join(dirpath, f)
+            if os.path.islink(fp):
+                apparent_total_bytes += os.lstat(fp).st_size
+                continue
+            st = os.lstat(fp)
+            if st.st_ino in have:
+                continue
+            have.add(st.st_ino)
+            apparent_total_bytes += st.st_size
+        for d in dirnames:
+            dp = os.path.join(dirpath, d)
+            if os.path.islink(dp):
+                apparent_total_bytes += os.lstat(dp).st_size
+
+    return apparent_total_bytes
+
+
 def disk_usage(path: pathlib.Path):
     if sys.platform == "darwin":
         cmd = ["du", "-sAB1", str(path)]
-    else:
+    elif sys.platform == "linux":
         cmd = ["du", "-sb", str(path)]
+    else:
+        return str(du(path))
 
-    return subprocess.check_output(cmd, encoding="utf-8").split()[0]
+    output = subprocess.check_output(cmd, encoding="utf-8").split()[0]
+    if sys.platform == "darwin":
+        # mac du does not have the -b option to return bytes
+        output = str(int(output) * 512)
+    return output
 
 
 @contextlib.contextmanager

conda-store-server/conda_store_server/worker/app.py

@@ -71,9 +71,18 @@ def start(self):
         argv = [
             "worker",
             "--loglevel=INFO",
-            "--beat",
         ]
 
+        # The default Celery pool requires this on Windows. See
+        # https://stackoverflow.com/questions/37255548/how-to-run-celery-on-windows
+        if sys.platform == "win32":
+            os.environ.setdefault("FORKED_BY_MULTIPROCESSING", "1")
+        else:
+            # --beat does not work on Windows
+            argv += [
+                "--beat",
+            ]
+
         if self.concurrency:
             argv.append(f"--concurrency={self.concurrency}")