Windows support

conda-store-server/conda_store_server/action/install_lockfile.py

@@ -1,3 +1,4 @@
+import sys
 import json
 import pathlib
 import typing
@@ -16,7 +17,9 @@ def action_install_lockfile(
         json.dump(conda_lock_spec, f)
 
     command = [
-        "conda-lock",
+        sys.executable,
+        "-m",
+        "conda_lock",
         "install",
         "--validate-platform",
         "--log-level",

conda-store-server/conda_store_server/app.py

@@ -32,6 +32,8 @@
 from traitlets.config import LoggingConfigurable
 
 
+ON_WIN = sys.platform.startswith("win")
+
 def conda_store_validate_specification(
     db: Session,
     conda_store: "CondaStore",
@@ -301,21 +303,24 @@ def _default_celery_results_backend(self):
     )
 
     default_uid = Integer(
-        os.getuid(),
+        None if ON_WIN else os.getuid(),
         help="default uid to assign to built environments",
         config=True,
+        allow_none=True,
     )
 
     default_gid = Integer(
-        os.getgid(),
+        None if ON_WIN else os.getgid(),
         help="default gid to assign to built environments",
         config=True,
+        allow_none=True,
     )
 
     default_permissions = Unicode(
-        "775",
+        None if ON_WIN else "775",
         help="default file permissions to assign to built environments",
         config=True,
+        allow_none=True,
     )
 
     default_docker_base_image = Union(

conda-store-server/conda_store_server/dbutil.py

@@ -1,6 +1,5 @@
 import os
 from contextlib import contextmanager
-from subprocess import check_call
 from tempfile import TemporaryDirectory
 
 from alembic import command
@@ -78,6 +77,9 @@ def upgrade(db_url, revision="head"):
     current_table_names = set(inspect(engine).get_table_names())
 
     with _temp_alembic_ini(db_url) as alembic_ini:
+
+        alembic_cfg = Config(alembic_ini)
+
         if (
             "alembic_version" not in current_table_names
             and len(current_table_names) > 0
@@ -86,10 +88,10 @@ def upgrade(db_url, revision="head"):
             # we stamp the revision at the first one, that introduces the alembic revisions.
             # I chose the leave the revision number hardcoded as it's not something
             # dynamic, not something we want to change, and tightly related to the codebase
-            command.stamp(Config(alembic_ini), "48be4072fe58")
+            command.stamp(alembic_cfg, "48be4072fe58")
             # After this point, whatever is in the database, Alembic will
             # believe it's at the first revision. If there are more upgrades/migrations
             # to run, they'll be at the next step :
 
         # run the upgrade.
-        check_call(["alembic", "-c", alembic_ini, "upgrade", revision])
+        command.upgrade(config=alembic_cfg, revision=revision)

conda-store-server/conda_store_server/orm.py

@@ -513,10 +513,11 @@ def update_packages(self, db, subdirs=None):
                 package_builds[package_key].append(new_package_build_dict)
             logger.info("CondaPackageBuild objects created")
 
-            batch_size = 1000
+            # sqlite3 has a max expression depth of 1000
+            batch_size = 990
             all_package_keys = list(package_builds.keys())
             for i in range(0, len(all_package_keys), batch_size):
-                logger.info(f"handling subset at index {i} (batch size {batch_size}")
+                logger.info(f"handling subset at index {i} (batch size {batch_size})")
                 subset_keys = all_package_keys[i : i + batch_size]
 
                 # retrieve the parent packages for the subset

conda-store-server/conda_store_server/schema.py

@@ -9,6 +9,7 @@
 from conda_store_server import conda_utils
 from pydantic import BaseModel, Field, constr, validator
 
+ON_WIN = sys.platform.startswith("win")
 
 def _datetime_factory(offset: datetime.timedelta):
     """utcnow datetime + timezone as string"""
@@ -194,20 +195,20 @@ class Settings(BaseModel):
         metadata={"global": True},
     )
 
-    default_uid: int = Field(
-        os.getuid(),
+    default_uid: int | None = Field(
+        None if ON_WIN else os.getuid(),
         description="default uid to assign to built environments",
         metadata={"global": True},
     )
 
-    default_gid: int = Field(
-        os.getgid(),
+    default_gid: int | None = Field(
+        None if ON_WIN else os.getgid(),
         description="default gid to assign to built environments",
         metadata={"global": True},
     )
 
-    default_permissions: str = Field(
-        "775",
+    default_permissions: str | None = Field(
+        None if ON_WIN else "775",
         description="default file permissions to assign to built environments",
         metadata={"global": True},
     )

conda-store-server/conda_store_server/server/app.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import posixpath
 import sys
 
 import conda_store_server
@@ -198,9 +199,9 @@ def trim_slash(url):
         app = FastAPI(
             title="conda-store",
             version=__version__,
-            openapi_url=os.path.join(self.url_prefix, "openapi.json"),
-            docs_url=os.path.join(self.url_prefix, "docs"),
-            redoc_url=os.path.join(self.url_prefix, "redoc"),
+            openapi_url=posixpath.join(self.url_prefix, "openapi.json"),
+            docs_url=posixpath.join(self.url_prefix, "docs"),
+            redoc_url=posixpath.join(self.url_prefix, "redoc"),
             contact={
                 "name": "Quansight",
                 "url": "https://quansight.com",

conda-store-server/conda_store_server/storage.py

@@ -1,5 +1,6 @@
 import io
 import os
+import posixpath
 import shutil
 
 import minio
@@ -223,7 +224,7 @@ def get(self, key):
             return f.read()
 
     def get_url(self, key):
-        return os.path.join(self.storage_url, key)
+        return posixpath.join(self.storage_url, key)
 
     def delete(self, db, build_id, key):
         filename = os.path.join(self.storage_path, key)

conda-store-server/conda_store_server/utils.py

@@ -50,11 +50,48 @@ def chdir(directory: pathlib.Path):
         os.chdir(current_directory)
 
 
+def du(path):
+    """
+    Pure Python equivalent of du -sb
+
+    Based on https://stackoverflow.com/a/55648984/161801
+    """
+    if os.path.islink(path):
+        return os.lstat(path).st_size
+    if os.path.isfile(path):
+        st = os.lstat(path)
+        return st.st_size
+    apparent_total_bytes = 0
+    have = set()
+    for dirpath, dirnames, filenames in os.walk(path):
+        apparent_total_bytes += os.lstat(dirpath).st_size
+        for f in filenames:
+            fp = os.path.join(dirpath, f)
+            if os.path.islink(fp):
+                apparent_total_bytes += os.lstat(fp).st_size
+                continue
+            st = os.lstat(fp)
+            if st.st_ino in have:
+                continue
+            have.add(st.st_ino)
+            apparent_total_bytes += st.st_size
+        for d in dirnames:
+            dp = os.path.join(dirpath, d)
+            if os.path.islink(dp):
+                apparent_total_bytes += os.lstat(dp).st_size
+
+    # Round up
+    n_blocks = (apparent_total_bytes + 511) // 512
+    return n_blocks
+
+
 def disk_usage(path: pathlib.Path):
     if sys.platform == "darwin":
         cmd = ["du", "-sAB1", str(path)]
-    else:
+    elif sys.platform == "linux":
         cmd = ["du", "-sb", str(path)]
+    else:
+        return str(du(path))
 
     return subprocess.check_output(cmd, encoding="utf-8").split()[0]
 

conda-store-server/conda_store_server/worker/app.py

@@ -71,9 +71,18 @@ def start(self):
         argv = [
             "worker",
             "--loglevel=INFO",
-            "--beat",
         ]
 
+        # The default Celery pool requires this on Windows. See
+        # https://stackoverflow.com/questions/37255548/how-to-run-celery-on-windows
+        if sys.platform == "win32":
+            os.environ.setdefault('FORKED_BY_MULTIPROCESSING', '1')
+        else:
+            # --beat does not work on Windows
+            argv += [
+                "--beat",
+            ]
+
         if self.concurrency:
             argv.append(f"--concurrency={self.concurrency}")
 

conda-store-server/hatch_build.py

@@ -61,10 +61,10 @@ def initialize(self, version: str, build_data: Dict[str, Any]) -> None:
             # main.js to enable easy configuration see
             # conda_store_server/server/templates/conda-store-ui.html
             # for global variable set
-            with (source_directory / "main.js").open("r") as source_f:
+            with (source_directory / "main.js").open("r", encoding='utf-8') as source_f:
                 content = source_f.read()
                 content = re.sub(
                     '"MISSING_ENV_VAR"', "GLOBAL_CONDA_STORE_STATE", content
                 )
-                with (destination_directory / "main.js").open("w") as dest_f:
+                with (destination_directory / "main.js").open("w", encoding='utf-8') as dest_f:
                     dest_f.write(content)