Configuration of conda-store via traitlets (#87)

This is a change that should not have any impact on how conda-store
runs. However it makes conda-store significantly more configurable.

conda-store-server/conda_store_server/api.py

@@ -1,13 +1,10 @@
-import logging
 import datetime
 
 from sqlalchemy import and_, func
 
 from conda_store_server import orm
 from .conda import conda_platform
 
-logger = logging.getLogger(__name__)
-
 
 def list_environments(db, search=None):
     return db.query(orm.Environment).all()

conda-store-server/conda_store_server/app.py

@@ -1,43 +1,90 @@
 import os
 import pathlib
 import datetime
-import logging
 import shutil
 
 import yaml
+from traitlets import Type, Unicode, Integer
+from traitlets.config import LoggingConfigurable
 
 from conda_store_server import orm, utils, storage, schema
 
-logger = logging.getLogger(__name__)
 
+class CondaStore(LoggingConfigurable):
+    storage_class = Type(
+        default_value=storage.S3Storage,
+        klass=storage.Storage,
+        allow_none=False,
+        config=True,
+    )
+
+    store_directory = Unicode(
+        "conda-store-state",
+        help="directory for conda-store to build environments and store state",
+        config=True,
+    )
+
+    environment_directory = Unicode(
+        "conda-store-state/envs",
+        help="directory for symlinking conda environment builds",
+        config=True,
+    )
+
+    database_url = Unicode(
+        "sqlite:///conda-store.sqlite",
+        help="url for the database. e.g. 'sqlite:///conda-store.sqlite'",
+        config=True,
+    )
+
+    default_uid = Integer(
+        os.getuid(),
+        help="default uid to assign to built environments",
+        config=True,
+    )
+
+    default_gid = Integer(
+        os.getgid(),
+        help="default gid to assign to built environments",
+        config=True,
+    )
+
+    default_permissions = Unicode(
+        "775",
+        help="default file permissions to assign to built environments",
+        config=True,
+    )
 
-class CondaStore:
-    def __init__(self, store_directory, database_url=None, storage_backend="s3"):
-        self.store_directory = pathlib.Path(store_directory).resolve()
-        if not self.store_directory.is_dir():
-            logger.info(f"creating directory store_directory={store_directory}")
-            self.store_directory.mkdir(parents=True)
-
-        self.database_url = database_url or os.environ.get(
-            "CONDA_STORE_DB_URL",
-            f'sqlite:///{self.store_directory / "conda_store.sqlite"}',
-        )
+    @property
+    def session_factory(self):
+        if hasattr(self, "_session_factory"):
+            return self._session_factory
 
-        Session = orm.new_session_factory(url=self.database_url)
-        self.db = Session()
+        self._session_factory = orm.new_session_factory(url=self.database_url)
+        return self._session_factory
 
-        if storage_backend == schema.StorageBackend.FILESYSTEM:
-            storage_directory = self.store_directory / "storage"
-            self.storage = storage.LocalStorage(storage_directory)
-        elif storage_backend == schema.StorageBackend.S3:
-            self.storage = storage.S3Storage()
+    @property
+    def db(self):
+        if hasattr(self, "_db"):
+            return self._db
 
-        self.configuration.store_directory = str(self.store_directory)
+        self._db = self.session_factory()
+        return self._db
 
     @property
     def configuration(self):
         return orm.CondaStoreConfiguration.configuration(self.db)
 
+    @property
+    def storage(self):
+        if hasattr(self, "_storage"):
+            return self._storage
+        self._storage = self.storage_class(parent=self, log=self.log)
+        return self._storage
+
+    def ensure_directories(self):
+        os.makedirs(self.store_directory, exist_ok=True)
+        os.makedirs(self.environment_directory, exist_ok=True)
+
     def update_storage_metrics(self):
         configuration = self.configuration
         disk_usage = shutil.disk_usage(str(self.store_directory))
@@ -59,7 +106,9 @@ def update_conda_channels(self, channels=None, update_interval=60 * 60):
             + datetime.timedelta(seconds=update_interval)
             < datetime.datetime.now()
         ):
-            logger.info(f"packages were updated in the last seconds={update_interval}")
+            self.log.info(
+                f"packages were updated in the last seconds={update_interval}"
+            )
             return
 
         for channel in channels:
@@ -88,18 +137,18 @@ def register_environment(self, specification, namespace="library"):
             orm.Specification.sha256 == specification_sha256
         )
         if query.count() != 0:
-            logger.debug(
+            self.log.debug(
                 f"already registered specification name={specification.name} sha256={specification_sha256}"
             )
             return
 
-        logger.info(
+        self.log.info(
             f"registering specification name={specification.name} sha256={specification_sha256}"
         )
         specification = orm.Specification(specification.dict())
         self.db.add(specification)
         self.db.commit()
-        logger.info(
+        self.log.info(
             f"scheduling specification for build name={specification.name} sha256={specification.sha256}"
         )
         build = orm.Build(specification_id=specification.id)

conda-store-server/conda_store_server/build.py

@@ -1,6 +1,5 @@
 import os
 import shutil
-import logging
 import subprocess
 import pathlib
 import stat
@@ -20,9 +19,6 @@
 from conda_store_server.environment import discover_environments
 
 
-logger = logging.getLogger(__name__)
-
-
 def claim_build(conda_store):
     build = (
         conda_store.db.query(orm.Build)
@@ -55,7 +51,7 @@ def set_build_failed(conda_store, build, logs, reschedule=True):
             )
             .count()
         )
-        logger.info(
+        conda_store.log.info(
             f"specification name={build.specification.name} build has failed={num_failed_builds} times"
         )
         scheduled_on = datetime.datetime.utcnow() + datetime.timedelta(
@@ -66,7 +62,7 @@ def set_build_failed(conda_store, build, logs, reschedule=True):
                 specification_id=build.specification_id, scheduled_on=scheduled_on
             )
         )
-        logger.info(
+        conda_store.log.info(
             f"rescheduling specification name={build.specification.name} on {scheduled_on}"
         )
     conda_store.db.commit()
@@ -109,25 +105,27 @@ def package_query(package):
 
 
 def start_conda_build(conda_store, paths, storage_threshold, poll_interval):
-    logger.info(f"polling interval set to {poll_interval} seconds")
+    conda_store.log.info(f"polling interval set to {poll_interval} seconds")
     while True:
         environments = discover_environments(paths)
         for environment in environments:
             conda_store.register_environment(environment, namespace="filesystem")
 
         num_queued_builds = api.get_num_queued_builds(conda_store.db)
         if num_queued_builds > 0:
-            logger.info(f"number of queued conda builds {num_queued_builds}")
+            conda_store.log.info(f"number of queued conda builds {num_queued_builds}")
 
         disk_usage = conda_store.update_storage_metrics()
         if disk_usage.free < storage_threshold:
-            logger.warning(
+            conda_store.log.warning(
                 f"free disk space={storage_threshold:g} [bytes] below storage threshold"
             )
 
         num_schedulable_builds = api.get_num_schedulable_builds(conda_store.db)
         if num_schedulable_builds > 0:
-            logger.info(f"number of schedulable conda builds {num_schedulable_builds}")
+            conda_store.log.info(
+                f"number of schedulable conda builds {num_schedulable_builds}"
+            )
             conda_build(conda_store)
             time.sleep(1)
         else:
@@ -136,10 +134,8 @@ def start_conda_build(conda_store, paths, storage_threshold, poll_interval):
 
 def conda_build(conda_store):
     build = claim_build(conda_store)
-    store_directory = pathlib.Path(conda_store.configuration.store_directory)
-    environment_directory = pathlib.Path(
-        conda_store.configuration.environment_directory
-    )
+    store_directory = pathlib.Path(conda_store.store_directory)
+    environment_directory = pathlib.Path(conda_store.environment_directory)
     build_path = build.build_path(store_directory)
     environment_path = build.environment_path(environment_directory)
     try:
@@ -153,17 +149,21 @@ def conda_build(conda_store):
             and build_path.is_dir()
             and environment_path.resolve() == build_path
         ):
-            logger.debug(f"found cached {build_path} symlinked to {environment_path}")
+            conda_store.log.debug(
+                f"found cached {build_path} symlinked to {environment_path}"
+            )
         else:
-            logger.info(f"building {build_path} symlinked to {environment_path}")
+            conda_store.log.info(
+                f"building {build_path} symlinked to {environment_path}"
+            )
 
-            logger.info(
+            conda_store.log.info(
                 f"previously unfinished build of {build_path} cleaning directory"
             )
             if build_path.is_dir():
                 shutil.rmtree(str(build_path))
 
-            with utils.timer(logger, f"building {build_path}"):
+            with utils.timer(conda_store.log, f"building {build_path}"):
                 with tempfile.TemporaryDirectory() as tmpdir:
                     tmp_environment_filename = pathlib.Path(tmpdir) / "environment.yaml"
                     with tmp_environment_filename.open("w") as f:
@@ -180,28 +180,28 @@ def conda_build(conda_store):
 
         # modify permissions, uid, gid if they do not match
         stat_info = os.stat(build_path)
-        permissions = conda_store.configuration.default_permissions
-        uid = conda_store.configuration.default_uid
-        gid = conda_store.configuration.default_gid
+        permissions = conda_store.default_permissions
+        uid = conda_store.default_uid
+        gid = conda_store.default_gid
 
         if permissions is not None and oct(stat.S_IMODE(stat_info.st_mode))[-3:] != str(
             permissions
         ):
-            logger.info(
+            conda_store.log.info(
                 f"modifying permissions of {build_path} to permissions={permissions}"
             )
-            with utils.timer(logger, f"chmod of {build_path}"):
+            with utils.timer(conda_store.log, f"chmod of {build_path}"):
                 utils.chmod(build_path, permissions)
 
         if (
             uid is not None
             and gid is not None
             and (str(uid) != str(stat_info.st_uid) or str(gid) != str(stat_info.st_gid))
         ):
-            logger.info(
+            conda_store.log.info(
                 f"modifying permissions of {build_path} to uid={uid} and gid={gid}"
             )
-            with utils.timer(logger, f"chown of {build_path}"):
+            with utils.timer(conda_store.log, f"chown of {build_path}"):
                 utils.chown(build_path, uid, gid)
 
         packages = conda.conda_list(build_path)
@@ -213,10 +213,10 @@ def conda_build(conda_store):
 
         set_build_completed(conda_store, build, output.encode("utf-8"), packages)
     except Exception as e:
-        logger.exception(e)
+        conda_store.log.exception(e)
         set_build_failed(conda_store, build, traceback.format_exc().encode("utf-8"))
     except BaseException as e:
-        logger.error(
+        conda_store.log.error(
             f"exception {e.__class__.__name__} caught causing build={build.id} to be rescheduled"
         )
         set_build_failed(conda_store, build, traceback.format_exc().encode("utf-8"))
@@ -237,7 +237,7 @@ def build_conda_install(conda_store, build_path, environment_filename):
 
 
 def build_conda_pack(conda_store, conda_prefix, build):
-    logger.info(f"packaging archive of conda environment={conda_prefix}")
+    conda_store.log.info(f"packaging archive of conda environment={conda_prefix}")
     with tempfile.TemporaryDirectory() as tmpdir:
         output_filename = pathlib.Path(tmpdir) / "environment.tar.gz"
         conda.conda_pack(prefix=conda_prefix, output=output_filename)
@@ -255,7 +255,7 @@ def build_docker_image(conda_store, conda_prefix, build):
         fetch_precs,
     )
 
-    logger.info(f"creating docker archive of conda environment={conda_prefix}")
+    conda_store.log.info(f"creating docker archive of conda environment={conda_prefix}")
 
     user_conda = find_user_conda()
     info = conda_info(user_conda)
@@ -333,6 +333,6 @@ def build_docker_image(conda_store, conda_prefix, build):
         content_type="application/vnd.docker.distribution.manifest.v2+json",
     )
 
-    logger.info(
+    conda_store.log.info(
         f"built docker image: {image.name}:{image.tag} layers={len(image.layers)}"
     )