diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 6e30cfb58..6f3a51c1b 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -14,8 +14,7 @@ jobs: name: "unit-test conda-store-server" strategy: matrix: - # cannot run on windows due to needing fake-chroot for conda-docker - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.os }} defaults: run: @@ -29,7 +28,6 @@ jobs: if: matrix.os == 'ubuntu-latest' uses: conda-incubator/setup-miniconda@v2 with: - mamba-version: "*" activate-environment: conda-store-server-dev environment-file: conda-store-server/environment-dev.yaml auto-activate-base: false @@ -42,6 +40,24 @@ jobs: environment-file: conda-store-server/environment-macos-dev.yaml auto-activate-base: false + - name: Set up Python (Windows) + if: matrix.os == 'windows-latest' + uses: conda-incubator/setup-miniconda@v2 + with: + activate-environment: conda-store-server-dev + environment-file: conda-store-server/environment-windows-dev.yaml + auto-activate-base: false + + # This fixes a "DLL not found" issue importing ctypes from the hatch env + - name: Reinstall Python 3.10 on Windows runner + uses: nick-fields/retry@v2.8.3 + with: + timeout_minutes: 9999 + max_attempts: 6 + command: + conda install --channel=conda-forge --quiet --yes python=${{ matrix.python }} + if: matrix.os == 'windows-latest' + - name: Linting Checks run: | hatch env run -e dev lint @@ -52,7 +68,7 @@ jobs: - name: Unit Tests run: | - pytest tests + pytest -vvv tests integration-test-conda-store-server: name: "integration-test conda-store-server" @@ -68,7 +84,6 @@ jobs: - name: Set up Python uses: conda-incubator/setup-miniconda@v2 with: - mamba-version: "*" activate-environment: conda-store-server-dev environment-file: conda-store-server/environment-dev.yaml auto-activate-base: false diff --git a/conda-store-server/conda_store_server/action/base.py b/conda-store-server/conda_store_server/action/base.py index 43e0991a9..49e6e8e3b 100644 --- a/conda-store-server/conda_store_server/action/base.py +++ b/conda-store-server/conda_store_server/action/base.py @@ -18,7 +18,7 @@ def wrapper(*args, **kwargs): # redirect stdout -> action_context.stdout stack.enter_context(contextlib.redirect_stdout(action_context.stdout)) - # redirect stderr -> action_context.stdout + # redirect stderr -> action_context.stderr stack.enter_context(contextlib.redirect_stderr(action_context.stdout)) # create a temporary directory @@ -38,6 +38,7 @@ class ActionContext: def __init__(self): self.id = str(uuid.uuid4()) self.stdout = io.StringIO() + self.stderr = io.StringIO() self.log = logging.getLogger(f"conda_store_server.action.{self.id}") self.log.propagate = False self.log.addHandler(logging.StreamHandler(stream=self.stdout)) @@ -45,13 +46,15 @@ def __init__(self): self.result = None self.artifacts = {} - def run(self, *args, **kwargs): + def run(self, *args, redirect_stderr=True, **kwargs): result = subprocess.run( *args, **kwargs, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, + stderr=subprocess.STDOUT if redirect_stderr else subprocess.PIPE, encoding="utf-8", ) self.stdout.write(result.stdout) + if not redirect_stderr: + self.stderr.write(result.stderr) return result diff --git a/conda-store-server/conda_store_server/action/generate_conda_export.py b/conda-store-server/conda_store_server/action/generate_conda_export.py index 9cdedfeb8..cbcc0fe04 100644 --- a/conda-store-server/conda_store_server/action/generate_conda_export.py +++ b/conda-store-server/conda_store_server/action/generate_conda_export.py @@ -17,5 +17,7 @@ def action_generate_conda_export( "--json", ] - result = context.run(command, check=True) + result = context.run(command, check=True, redirect_stderr=False) + if result.stderr: + context.log.warning(f"conda env export stderr: {result.stderr}") return json.loads(result.stdout) diff --git a/conda-store-server/conda_store_server/action/install_lockfile.py b/conda-store-server/conda_store_server/action/install_lockfile.py index 60ac9f8dc..23d7d5c94 100644 --- a/conda-store-server/conda_store_server/action/install_lockfile.py +++ b/conda-store-server/conda_store_server/action/install_lockfile.py @@ -1,5 +1,6 @@ import json import pathlib +import sys import typing from conda_store_server import action @@ -16,7 +17,9 @@ def action_install_lockfile( json.dump(conda_lock_spec, f) command = [ - "conda-lock", + sys.executable, + "-m", + "conda_lock", "install", "--validate-platform", "--log-level", diff --git a/conda-store-server/conda_store_server/app.py b/conda-store-server/conda_store_server/app.py index aa97ac179..62b88b243 100644 --- a/conda-store-server/conda_store_server/app.py +++ b/conda-store-server/conda_store_server/app.py @@ -31,6 +31,8 @@ ) from traitlets.config import LoggingConfigurable +ON_WIN = sys.platform.startswith("win") + def conda_store_validate_specification( db: Session, @@ -301,21 +303,24 @@ def _default_celery_results_backend(self): ) default_uid = Integer( - os.getuid(), + None if ON_WIN else os.getuid(), help="default uid to assign to built environments", config=True, + allow_none=True, ) default_gid = Integer( - os.getgid(), + None if ON_WIN else os.getgid(), help="default gid to assign to built environments", config=True, + allow_none=True, ) default_permissions = Unicode( - "775", + None if ON_WIN else "775", help="default file permissions to assign to built environments", config=True, + allow_none=True, ) default_docker_base_image = Union( diff --git a/conda-store-server/conda_store_server/dbutil.py b/conda-store-server/conda_store_server/dbutil.py index 7f3eff640..b76b28ba6 100644 --- a/conda-store-server/conda_store_server/dbutil.py +++ b/conda-store-server/conda_store_server/dbutil.py @@ -1,6 +1,5 @@ import os from contextlib import contextmanager -from subprocess import check_call from tempfile import TemporaryDirectory from alembic import command @@ -78,6 +77,8 @@ def upgrade(db_url, revision="head"): current_table_names = set(inspect(engine).get_table_names()) with _temp_alembic_ini(db_url) as alembic_ini: + alembic_cfg = Config(alembic_ini) + if ( "alembic_version" not in current_table_names and len(current_table_names) > 0 @@ -86,10 +87,10 @@ def upgrade(db_url, revision="head"): # we stamp the revision at the first one, that introduces the alembic revisions. # I chose the leave the revision number hardcoded as it's not something # dynamic, not something we want to change, and tightly related to the codebase - command.stamp(Config(alembic_ini), "48be4072fe58") + command.stamp(alembic_cfg, "48be4072fe58") # After this point, whatever is in the database, Alembic will # believe it's at the first revision. If there are more upgrades/migrations # to run, they'll be at the next step : # run the upgrade. - check_call(["alembic", "-c", alembic_ini, "upgrade", revision]) + command.upgrade(config=alembic_cfg, revision=revision) diff --git a/conda-store-server/conda_store_server/orm.py b/conda-store-server/conda_store_server/orm.py index 5f2ae301c..5a9b11762 100644 --- a/conda-store-server/conda_store_server/orm.py +++ b/conda-store-server/conda_store_server/orm.py @@ -513,10 +513,11 @@ def update_packages(self, db, subdirs=None): package_builds[package_key].append(new_package_build_dict) logger.info("CondaPackageBuild objects created") - batch_size = 1000 + # sqlite3 has a max expression depth of 1000 + batch_size = 990 all_package_keys = list(package_builds.keys()) for i in range(0, len(all_package_keys), batch_size): - logger.info(f"handling subset at index {i} (batch size {batch_size}") + logger.info(f"handling subset at index {i} (batch size {batch_size})") subset_keys = all_package_keys[i : i + batch_size] # retrieve the parent packages for the subset diff --git a/conda-store-server/conda_store_server/schema.py b/conda-store-server/conda_store_server/schema.py index 173174249..66948d546 100644 --- a/conda-store-server/conda_store_server/schema.py +++ b/conda-store-server/conda_store_server/schema.py @@ -9,6 +9,8 @@ from conda_store_server import conda_utils, utils from pydantic import BaseModel, Field, ValidationError, constr, validator +ON_WIN = sys.platform.startswith("win") + def _datetime_factory(offset: datetime.timedelta): """utcnow datetime + timezone as string""" @@ -194,20 +196,20 @@ class Settings(BaseModel): metadata={"global": True}, ) - default_uid: int = Field( - os.getuid(), + default_uid: int | None = Field( + None if ON_WIN else os.getuid(), description="default uid to assign to built environments", metadata={"global": True}, ) - default_gid: int = Field( - os.getgid(), + default_gid: int | None = Field( + None if ON_WIN else os.getgid(), description="default gid to assign to built environments", metadata={"global": True}, ) - default_permissions: str = Field( - "775", + default_permissions: str | None = Field( + None if ON_WIN else "775", description="default file permissions to assign to built environments", metadata={"global": True}, ) diff --git a/conda-store-server/conda_store_server/server/app.py b/conda-store-server/conda_store_server/server/app.py index a96857b28..146d2f68b 100644 --- a/conda-store-server/conda_store_server/server/app.py +++ b/conda-store-server/conda_store_server/server/app.py @@ -1,5 +1,6 @@ import logging import os +import posixpath import sys import conda_store_server @@ -198,9 +199,9 @@ def trim_slash(url): app = FastAPI( title="conda-store", version=__version__, - openapi_url=os.path.join(self.url_prefix, "openapi.json"), - docs_url=os.path.join(self.url_prefix, "docs"), - redoc_url=os.path.join(self.url_prefix, "redoc"), + openapi_url=posixpath.join(self.url_prefix, "openapi.json"), + docs_url=posixpath.join(self.url_prefix, "docs"), + redoc_url=posixpath.join(self.url_prefix, "redoc"), contact={ "name": "Quansight", "url": "https://quansight.com", @@ -335,6 +336,10 @@ def start(self): # start worker if in standalone mode if self.standalone: + address = "localhost" if self.address == "0.0.0.0" else self.address + print( + f"Starting standalone conda-store server at http://{address}:{self.port}" + ) import multiprocessing multiprocessing.set_start_method("spawn") diff --git a/conda-store-server/conda_store_server/server/auth.py b/conda-store-server/conda_store_server/server/auth.py index ec0e2d0ae..86a072b74 100644 --- a/conda-store-server/conda_store_server/server/auth.py +++ b/conda-store-server/conda_store_server/server/auth.py @@ -436,7 +436,9 @@ async def post_login_method( samesite="strict", domain=self.cookie_domain, # set cookie to expire at same time as jwt - max_age=(authentication_token.exp - datetime.datetime.utcnow()).seconds, + max_age=int( + (authentication_token.exp - datetime.datetime.utcnow()).total_seconds() + ), ) return response diff --git a/conda-store-server/conda_store_server/storage.py b/conda-store-server/conda_store_server/storage.py index 3d014533f..5f349a676 100644 --- a/conda-store-server/conda_store_server/storage.py +++ b/conda-store-server/conda_store_server/storage.py @@ -1,5 +1,6 @@ import io import os +import posixpath import shutil import minio @@ -223,7 +224,7 @@ def get(self, key): return f.read() def get_url(self, key): - return os.path.join(self.storage_url, key) + return posixpath.join(self.storage_url, key) def delete(self, db, build_id, key): filename = os.path.join(self.storage_path, key) diff --git a/conda-store-server/conda_store_server/utils.py b/conda-store-server/conda_store_server/utils.py index f4ce9d90d..0bbac9226 100644 --- a/conda-store-server/conda_store_server/utils.py +++ b/conda-store-server/conda_store_server/utils.py @@ -50,13 +50,52 @@ def chdir(directory: pathlib.Path): os.chdir(current_directory) +def du(path): + """ + Pure Python equivalent of du -sb + + Based on https://stackoverflow.com/a/55648984/161801 + """ + if os.path.islink(path): + return os.lstat(path).st_size + if os.path.isfile(path): + st = os.lstat(path) + return st.st_size + apparent_total_bytes = 0 + have = set() + for dirpath, dirnames, filenames in os.walk(path): + apparent_total_bytes += os.lstat(dirpath).st_size + for f in filenames: + fp = os.path.join(dirpath, f) + if os.path.islink(fp): + apparent_total_bytes += os.lstat(fp).st_size + continue + st = os.lstat(fp) + if st.st_ino in have: + continue + have.add(st.st_ino) + apparent_total_bytes += st.st_size + for d in dirnames: + dp = os.path.join(dirpath, d) + if os.path.islink(dp): + apparent_total_bytes += os.lstat(dp).st_size + + return apparent_total_bytes + + def disk_usage(path: pathlib.Path): if sys.platform == "darwin": cmd = ["du", "-sAB1", str(path)] - else: + elif sys.platform == "linux": cmd = ["du", "-sb", str(path)] + else: + return str(du(path)) - return subprocess.check_output(cmd, encoding="utf-8").split()[0] + output = subprocess.check_output(cmd, encoding="utf-8").split()[0] + if sys.platform == "darwin": + # mac du does not have the -b option to return bytes + output = str(int(output) * 512) + return output @contextlib.contextmanager diff --git a/conda-store-server/conda_store_server/worker/app.py b/conda-store-server/conda_store_server/worker/app.py index 24c7a5470..b3278a395 100644 --- a/conda-store-server/conda_store_server/worker/app.py +++ b/conda-store-server/conda_store_server/worker/app.py @@ -71,9 +71,18 @@ def start(self): argv = [ "worker", "--loglevel=INFO", - "--beat", ] + # The default Celery pool requires this on Windows. See + # https://stackoverflow.com/questions/37255548/how-to-run-celery-on-windows + if sys.platform == "win32": + os.environ.setdefault("FORKED_BY_MULTIPROCESSING", "1") + else: + # --beat does not work on Windows + argv += [ + "--beat", + ] + if self.concurrency: argv.append(f"--concurrency={self.concurrency}") diff --git a/conda-store-server/environment-windows-dev.yaml b/conda-store-server/environment-windows-dev.yaml new file mode 100644 index 000000000..5f8db2786 --- /dev/null +++ b/conda-store-server/environment-windows-dev.yaml @@ -0,0 +1,56 @@ +name: conda-store-server-dev +channels: + - conda-forge + - Microsoft +dependencies: + - python ==3.10 + # conda builds + - conda ==23.5.2 + - python-docker + - conda-pack + - conda-lock >=1.0.5 + - mamba + - conda-package-handling + # web server + - celery + - flower + - redis-py + - sqlalchemy<=1.4.47 + - psycopg2 + - pymysql + - requests + - uvicorn + - fastapi + - pydantic < 2.0 + - pyyaml + - traitlets + - yarl + - pyjwt + - filelock + - itsdangerous + - jinja2 + - python-multipart + - alembic + # artifact storage + - minio + # CLI + - typer + + # dev dependencies + - hatch + - pytest + - pytest-celery + - pytest-mock + - black ==22.3.0 + - flake8 + - ruff + - sphinx + - myst-parser + - sphinx-panels + - sphinx-copybutton + - pydata-sphinx-theme + - playwright + - docker-compose + + - pip: + - pytest-playwright diff --git a/conda-store-server/hatch_build.py b/conda-store-server/hatch_build.py index 73c4ad9cb..02bcb9965 100644 --- a/conda-store-server/hatch_build.py +++ b/conda-store-server/hatch_build.py @@ -61,10 +61,12 @@ def initialize(self, version: str, build_data: Dict[str, Any]) -> None: # main.js to enable easy configuration see # conda_store_server/server/templates/conda-store-ui.html # for global variable set - with (source_directory / "main.js").open("r") as source_f: + with (source_directory / "main.js").open("r", encoding="utf-8") as source_f: content = source_f.read() content = re.sub( '"MISSING_ENV_VAR"', "GLOBAL_CONDA_STORE_STATE", content ) - with (destination_directory / "main.js").open("w") as dest_f: + with (destination_directory / "main.js").open( + "w", encoding="utf-8" + ) as dest_f: dest_f.write(content) diff --git a/conda-store-server/tests/test_actions.py b/conda-store-server/tests/test_actions.py index b6ea72a34..f795a61f6 100644 --- a/conda-store-server/tests/test_actions.py +++ b/conda-store-server/tests/test_actions.py @@ -15,9 +15,17 @@ def test_action_decorator(): def test_function(context): print("stdout") print("stderr", file=sys.stderr) - context.run(["echo", "subprocess"]) - context.run("echo subprocess_stdout", shell=True) - context.run("echo subprocess_stderr 1>&2", shell=True) + if sys.platform == "win32": + # echo is not a separate program on Windows + context.run(["cmd", "/c", "echo subprocess"]) + context.run("echo subprocess_stdout", shell=True) + context.run("echo subprocess_stderr>&2", shell=True) + context.run("echo subprocess_stderr_no_redirect>&2", shell=True, redirect_stderr=False) + else: + context.run(["echo", "subprocess"]) + context.run("echo subprocess_stdout", shell=True) + context.run("echo subprocess_stderr 1>&2", shell=True) + context.run("echo subprocess_stderr_no_redirect 1>&2", shell=True, redirect_stderr=False) context.log.info("log") return pathlib.Path.cwd() @@ -26,9 +34,10 @@ def test_function(context): context.stdout.getvalue() == "stdout\nstderr\nsubprocess\nsubprocess_stdout\nsubprocess_stderr\nlog\n" ) + assert context.stderr.getvalue() == "subprocess_stderr_no_redirect\n" # test that action direction is not the same as outside function assert context.result != pathlib.Path.cwd() - # test that temportary directory is cleaned up + # test that temporary directory is cleaned up assert not context.result.exists() @@ -155,6 +164,7 @@ def test_remove_conda_prefix(tmp_path, simple_conda_lock): assert not conda_prefix.exists() +@pytest.mark.skipif(sys.platform == "win32", reason="permissions are not supported on Windows") def test_set_conda_prefix_permissions(tmp_path, conda_store, simple_conda_lock): conda_prefix = tmp_path / "test" diff --git a/conda-store-server/tests/test_usage.py b/conda-store-server/tests/test_usage.py new file mode 100644 index 000000000..fcf3990a8 --- /dev/null +++ b/conda-store-server/tests/test_usage.py @@ -0,0 +1,32 @@ +from conda_store_server.utils import disk_usage, du + +# TODO: Add tests for the other functions in utils.py + +def test_disk_usage(tmp_path): + test_dir = tmp_path / "test_dir" + test_dir.mkdir() + + # This varies across OSes + dir_size = du(test_dir) + assert abs(dir_size - int(disk_usage(test_dir))) <= 1000 + + test_file = test_dir / "test_file" + test_file.write_text("a"*1000) + test_file2 = test_dir / "test_file2" + test_file2.write_text("b"*1000) + # Test hard links + test_file_hardlink = test_dir / "test_file_hardlink" + test_file_hardlink.hardlink_to(test_file) + + # The exact values depend on the block size and other details. Just check + # that it is in the right range. Note that disk_usage uses the du command + # on Mac and Linux but uses the Python du() function on Windows. + val = disk_usage(test_file) + assert isinstance(val, str) + assert 1000 <= int(val) <= 1500 + assert 1000 <= du(test_file) <= 1500 + + val = disk_usage(test_dir) + assert isinstance(val, str) + assert 2000 + dir_size <= int(val) <= 2700 + dir_size + assert 2000 + dir_size <= du(test_dir) <= 2700 + dir_size diff --git a/docs/administration.md b/docs/administration.md index a9356e2b6..0a0d37429 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -20,7 +20,7 @@ seen. When conda-store builds a given environment it has to locally install the environment in the directory specified in the [Traitlets](https://traitlets.readthedocs.io/en/stable/using_traitlets.html) -configuration `CondaStore.store_directroy`. Conda environments consist +configuration `CondaStore.store_directory`. Conda environments consist of many hardlinks to small files. This means that the `store_directory` is limited to the number of [IOPS](https://en.wikipedia.org/wiki/IOPS) the directory can @@ -217,17 +217,20 @@ filesystem. `CondaStore.default_uid` is the uid (user id) to assign to all files and directories in a given built environment. This setting is useful if you want to protect environments from modification from -certain users and groups. +certain users and groups. Note: this configuration option is not +supported on Windows. `CondaStore.default_gid` is the gid (group id) to assign to all files and directories in a given built environment. This setting is useful if you want to protect environments from modification from -certain users and groups. +certain users and groups. Note: this configuration option is not +supported on Windows. `CondaStore.default_permissions` is the filesystem permissions to assign to all files and directories in a given built environment. This setting is useful if you want to protect environments from -modification from certain users and groups. +modification from certain users and groups. Note: this configuration +option is not supported on Windows. `CondaStore.default_docker_base_image` default base image used for the Dockerized environments. Make sure to have a proper glibc within image @@ -542,9 +545,36 @@ in the BUILDING state and are not currently running on the workers. This feature only works for certain brokers e.g. redis. Database celery brokers are not supported. -This issue occurs when the worker spontaineously dies. This can happen +This issue occurs when the worker spontaneously dies. This can happen for several reasons: - worker is killed due to consuming too much memory (conda solver/builds can consume a lot of memory) - worker was killed for other reasons e.g. forced restart - bugs in conda-store + +### Long paths on Windows + +conda-store supports Windows in standalone mode. However, when creating +environments with certain packages, you may see errors like + +``` +ERROR:root:[WinError 206] The filename or extension is too long: 'C:\\...' +``` + +This error is due to the fact that Windows has a limitation that file paths +cannot be more than 260 characters. The fix is to set the registry key +`Computer\HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem\LongPathsEnabled +(Type: REG_DWORD)` to `1`, which removes this MAX_PATH limitation. See [this +Microsoft support +article](https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation) +for more details on how to set this registry key. + +If it is not possible to set this registry key, for instance, because you do +not have access to administrator privileges, you should configure the +conda-store `CondaStore.store_directory` to be as close to the filesystem root +as possible, so that the total length of the paths of package files is +minimized. + +See [conda-store issue +#588](https://github.com/conda-incubator/conda-store/issues/588) for more +details. diff --git a/docs/contributing.md b/docs/contributing.md index 613a2daee..36e819c8d 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -66,18 +66,20 @@ Install the following dependencies before developing on conda-store. Install the development dependencies and activate the environment. ```shell +# replace this with environment-macos-dev.yaml or environment-windows-dev.yaml +# if you are on Mac or Windows conda env create -f conda-store-server/environment-dev.yaml conda activate conda-store-server-dev ``` Running `conda-store`. `--standalone` mode launched celery as a -subprocess of the web server. +subprocess of the web server. Run -python -m conda_store_server.server --standalone tests/assets/conda_store_standalone_config.py - -```` +``` +python -m conda_store_server.server --standalone +``` -Visit [localhost:5000](http://localhost:5000/) +Then visit [localhost:5000](http://localhost:5000/). ### Changes to API @@ -99,6 +101,8 @@ To build the documentation install the development environment via Conda. ```shell +# replace this with environment-macos-dev.yaml or environment-windows-dev.yaml +# if you are on Mac or Windows conda env create -f conda-store-server/environment-dev.yaml conda activate conda-store-server-dev ```` diff --git a/docs/user_guide.md b/docs/user_guide.md index 8e6d728de..20c5c428a 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -73,6 +73,10 @@ conda-unpack ### Docker Registry +```{note} +Docker image creation is currently only supported on Linux. +``` + conda-store acts as a docker registry which allows for interesting ways to handle Conda environment. In addition this registry leverages [conda-docker](https://github.com/conda-incubator/conda-docker) which