diff --git a/conda-store-server/conda_store_server/action/__init__.py b/conda-store-server/conda_store_server/action/__init__.py index 5cf70c697..613b5e14c 100644 --- a/conda-store-server/conda_store_server/action/__init__.py +++ b/conda-store-server/conda_store_server/action/__init__.py @@ -1,6 +1,7 @@ from conda_store_server.action.base import action # noqa from conda_store_server.action.generate_lockfile import action_solve_lockfile # noqa +from conda_store_server.action.generate_lockfile import action_save_lockfile # noqa from conda_store_server.action.download_packages import ( action_fetch_and_extract_conda_packages, # noqa ) diff --git a/conda-store-server/conda_store_server/action/generate_constructor_installer.py b/conda-store-server/conda_store_server/action/generate_constructor_installer.py index 0fbb6afd2..2a429d56d 100644 --- a/conda-store-server/conda_store_server/action/generate_constructor_installer.py +++ b/conda-store-server/conda_store_server/action/generate_constructor_installer.py @@ -3,6 +3,7 @@ import sys import tempfile import warnings +from typing import Union import yaml from conda_store_server import action, schema @@ -20,13 +21,20 @@ def get_installer_platform(): return context.subdir +# TODO: Ideally, this would just use the lockfile instead of reading data +# from the specification. But we currently also need to support legacy lockfiles +# generated by conda-store, which were not generated by conda-lock, and those +# don't have enough information for constructor to generate an installer. In +# particular, the channels are missing in that format. For more information on +# lockfile formats, see: api_get_build_lockfile and test_api_get_build_lockfile @action.action def action_generate_constructor_installer( context, conda_command: str, - specification: schema.CondaSpecification, + specification: Union[schema.CondaSpecification, schema.LockfileSpecification], installer_dir: pathlib.Path, version: str, + is_lockfile: bool = False, ): def write_file(filename, s): with open(filename, "w") as f: @@ -52,11 +60,28 @@ def write_file(filename, s): # conda and pip need to be in dependencies for the post_install script dependencies = ["conda", "pip"] pip_dependencies = [] - for d in specification.dependencies: - if type(d) is schema.CondaSpecificationPip: - pip_dependencies.extend(d.pip) - else: - dependencies.append(d) + channels = [] + + if is_lockfile: + # Adds channels + channels = [c.url for c in specification.lockfile.metadata.channels] + + # Adds dependencies + for p in specification.lockfile.package: + if p.manager == "pip": + pip_dependencies.append(f"{p.name}=={p.version}") + else: + dependencies.append(f"{p.name}=={p.version}") + else: + # Adds channels + channels = specification.channels + + # Adds dependencies + for d in specification.dependencies: + if type(d) is schema.CondaSpecificationPip: + pip_dependencies.extend(d.pip) + else: + dependencies.append(d) # Creates the construct.yaml file and post_install script ext = ".exe" if sys.platform == "win32" else ".sh" @@ -78,7 +103,7 @@ def write_file(filename, s): "installer_filename": str(installer_filename), "post_install": str(post_install_file), "name": specification.name, - "channels": specification.channels, + "channels": channels, "specs": dependencies, "version": version, } diff --git a/conda-store-server/conda_store_server/action/generate_lockfile.py b/conda-store-server/conda_store_server/action/generate_lockfile.py index 550012426..096ae8d44 100644 --- a/conda-store-server/conda_store_server/action/generate_lockfile.py +++ b/conda-store-server/conda_store_server/action/generate_lockfile.py @@ -5,7 +5,7 @@ import yaml from conda_lock.conda_lock import run_lock -from conda_store_server import action, conda_utils, schema +from conda_store_server import action, conda_utils, schema, utils from conda_store_server.action.utils import logged_command @@ -61,3 +61,17 @@ def action_solve_lockfile( with lockfile_filename.open() as f: return yaml.safe_load(f) + + +@action.action +def action_save_lockfile( + context, + specification: schema.LockfileSpecification, +): + lockfile = specification.dict()["lockfile"] + lockfile_filename = pathlib.Path.cwd() / "conda-lock.yaml" + + with lockfile_filename.open("w") as f: + json.dump(lockfile, f, cls=utils.CustomJSONEncoder) + + return lockfile diff --git a/conda-store-server/conda_store_server/alembic/versions/cd5b48ff57b5_lockfile_spec.py b/conda-store-server/conda_store_server/alembic/versions/cd5b48ff57b5_lockfile_spec.py new file mode 100644 index 000000000..c2b565f15 --- /dev/null +++ b/conda-store-server/conda_store_server/alembic/versions/cd5b48ff57b5_lockfile_spec.py @@ -0,0 +1,26 @@ +"""lockfile spec + +Revision ID: cd5b48ff57b5 +Revises: 03c839888c82 +Create Date: 2024-03-02 09:21:02.519805 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "cd5b48ff57b5" +down_revision = "03c839888c82" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "specification", + sa.Column("is_lockfile", sa.Boolean(), nullable=False, server_default="False"), + ) + + +def downgrade(): + op.drop_column("specification", "is_lockfile") diff --git a/conda-store-server/conda_store_server/api.py b/conda-store-server/conda_store_server/api.py index 01a6ec9d5..d748a917d 100644 --- a/conda-store-server/conda_store_server/api.py +++ b/conda-store-server/conda_store_server/api.py @@ -1,5 +1,5 @@ import re -from typing import Any, Dict, List +from typing import Any, Dict, List, Union from conda_store_server import conda_utils, orm, schema, utils from sqlalchemy import distinct, func, null, or_ @@ -378,19 +378,29 @@ def get_environment( return db.query(orm.Environment).join(orm.Namespace).filter(*filters).first() -def ensure_specification(db, specification: schema.CondaSpecification): +def ensure_specification( + db, + specification: Union[schema.CondaSpecification, schema.LockfileSpecification], + is_lockfile: bool = False, +): specification_sha256 = utils.datastructure_hash(specification.dict()) specification_orm = get_specification(db, sha256=specification_sha256) if specification_orm is None: - specification_orm = create_speficication(db, specification) + specification_orm = create_speficication( + db, specification, is_lockfile=is_lockfile + ) db.commit() return specification_orm -def create_speficication(db, specification: schema.CondaSpecification): - specification_orm = orm.Specification(specification.dict()) +def create_speficication( + db, + specification: Union[schema.CondaSpecification, schema.LockfileSpecification], + is_lockfile: bool = False, +): + specification_orm = orm.Specification(specification.dict(), is_lockfile=is_lockfile) db.add(specification_orm) return specification_orm diff --git a/conda-store-server/conda_store_server/app.py b/conda-store-server/conda_store_server/app.py index 8d68dd138..aabee9ae8 100644 --- a/conda-store-server/conda_store_server/app.py +++ b/conda-store-server/conda_store_server/app.py @@ -618,8 +618,10 @@ def register_environment( specification: dict, namespace: str = None, force: bool = True, + is_lockfile: bool = False, ): """Register a given specification to conda store with given namespace/name.""" + settings = self.get_settings(db) namespace = namespace or settings.default_namespace @@ -632,12 +634,18 @@ def register_environment( action=schema.Permissions.ENVIRONMENT_CREATE, ) - specification_model = self.validate_specification( - db=db, - conda_store=self, - namespace=namespace.name, - specification=schema.CondaSpecification.parse_obj(specification), - ) + if is_lockfile: + # It's a lockfile, do not do any validation in this case. If there + # are problems, these would be caught earlier during parsing or + # later when conda-lock attempts to install it. + specification_model = specification + else: + specification_model = self.validate_specification( + db=db, + conda_store=self, + namespace=namespace.name, + specification=schema.CondaSpecification.parse_obj(specification), + ) spec_sha256 = utils.datastructure_hash(specification_model.dict()) matching_specification = api.get_specification(db, sha256=spec_sha256) @@ -651,7 +659,9 @@ def register_environment( ): return None - specification = api.ensure_specification(db, specification_model) + specification = api.ensure_specification( + db, specification_model, is_lockfile=is_lockfile + ) environment_was_empty = ( api.get_environment(db, name=specification.name, namespace_id=namespace.id) is None @@ -663,7 +673,9 @@ def register_environment( description=specification.spec["description"], ) - build = self.create_build(db, environment.id, specification.sha256) + build = self.create_build( + db, environment.id, specification.sha256, is_lockfile=is_lockfile + ) if environment_was_empty: environment.current_build = build @@ -671,7 +683,13 @@ def register_environment( return build.id - def create_build(self, db: Session, environment_id: int, specification_sha256: str): + def create_build( + self, + db: Session, + environment_id: int, + specification_sha256: str, + is_lockfile: bool = False, + ): environment = api.get_environment(db, id=environment_id) self.validate_action( db=db, diff --git a/conda-store-server/conda_store_server/build.py b/conda-store-server/conda_store_server/build.py index f2b349b08..d12d4fa75 100644 --- a/conda-store-server/conda_store_server/build.py +++ b/conda-store-server/conda_store_server/build.py @@ -169,29 +169,45 @@ def build_conda_environment(db: Session, conda_store, build): environment_prefix = build.environment_path(conda_store) environment_prefix.parent.mkdir(parents=True, exist_ok=True) + is_lockfile = build.specification.is_lockfile + with utils.timer(conda_store.log, f"building conda_prefix={conda_prefix}"): - context = action.action_solve_lockfile( - settings.conda_command, - specification=schema.CondaSpecification.parse_obj( - build.specification.spec - ), - platforms=settings.conda_solve_platforms, - ) + if is_lockfile: + context = action.action_save_lockfile( + specification=schema.LockfileSpecification.parse_obj( + build.specification.spec + ), + ) + else: + context = action.action_solve_lockfile( + settings.conda_command, + specification=schema.CondaSpecification.parse_obj( + build.specification.spec + ), + platforms=settings.conda_solve_platforms, + ) conda_store.storage.set( db, build.id, build.conda_lock_key, - json.dumps(context.result, indent=4).encode("utf-8"), + json.dumps( + context.result, indent=4, cls=utils.CustomJSONEncoder + ).encode("utf-8"), content_type="application/json", artifact_type=schema.BuildArtifactType.LOCKFILE, ) + if is_lockfile: + lockfile_action_name = "action_save_lockfile" + else: + lockfile_action_name = "action_solve_lockfile" + append_to_logs( db, conda_store, build, - "::group::action_solve_lockfile\n" + f"::group::{lockfile_action_name}\n" + context.stdout.getvalue() + "\n::endgroup::\n", ) @@ -414,13 +430,23 @@ def build_constructor_installer(db: Session, conda_store, build: orm.Build): conda_store.log, f"creating installer for conda environment={conda_prefix}" ): with tempfile.TemporaryDirectory() as tmpdir: + is_lockfile = build.specification.is_lockfile + + if is_lockfile: + specification = schema.LockfileSpecification.parse_obj( + build.specification.spec + ) + else: + specification = schema.CondaSpecification.parse_obj( + build.specification.spec + ) + context = action.action_generate_constructor_installer( conda_command=settings.conda_command, - specification=schema.CondaSpecification.parse_obj( - build.specification.spec - ), + specification=specification, installer_dir=pathlib.Path(tmpdir), version=build.build_key, + is_lockfile=is_lockfile, ) output_filename = context.result append_to_logs( diff --git a/conda-store-server/conda_store_server/dbutil.py b/conda-store-server/conda_store_server/dbutil.py index b76b28ba6..c9305daab 100644 --- a/conda-store-server/conda_store_server/dbutil.py +++ b/conda-store-server/conda_store_server/dbutil.py @@ -1,9 +1,11 @@ +import json import os from contextlib import contextmanager from tempfile import TemporaryDirectory from alembic import command from alembic.config import Config +from conda_store_server import utils from sqlalchemy import create_engine, inspect _here = os.path.abspath(os.path.dirname(__file__)) @@ -71,7 +73,11 @@ def upgrade(db_url, revision="head"): revision: str [default: head] The alembic revision to upgrade to. """ - engine = create_engine(db_url) + engine = create_engine( + db_url, + # See the comment on the CustomJSONEncoder class on why this is needed + json_serializer=lambda x: json.dumps(x, cls=utils.CustomJSONEncoder), + ) # retrieves the names of tables in the DB current_table_names = set(inspect(engine).get_table_names()) diff --git a/conda-store-server/conda_store_server/orm.py b/conda-store-server/conda_store_server/orm.py index 3a626e82c..aaff273ee 100644 --- a/conda-store-server/conda_store_server/orm.py +++ b/conda-store-server/conda_store_server/orm.py @@ -1,4 +1,5 @@ import datetime +import json import logging import os import pathlib @@ -142,20 +143,22 @@ class Specification(Base): __tablename__ = "specification" - def __init__(self, specification): - if not validate_environment(specification): + def __init__(self, specification, is_lockfile: bool = False): + if not is_lockfile and not validate_environment(specification): raise ValueError( "specification={specification} is not valid conda environment.yaml" ) self.name = specification["name"] self.spec = specification self.sha256 = utils.datastructure_hash(self.spec) + self.is_lockfile = is_lockfile id = Column(Integer, primary_key=True) name = Column(Unicode(255), nullable=False) spec = Column(JSON, nullable=False) sha256 = Column(Unicode(255), unique=True, nullable=False) created_on = Column(DateTime, default=datetime.datetime.utcnow) + is_lockfile = Column(Boolean, default=False) builds = relationship("Build", back_populates="specification") solves = relationship("Solve", back_populates="specification") @@ -779,7 +782,12 @@ class KeyValueStore(Base): def new_session_factory(url="sqlite:///:memory:", reset=False, **kwargs): - engine = create_engine(url, **kwargs) + engine = create_engine( + url, + # See the comment on the CustomJSONEncoder class on why this is needed + json_serializer=lambda x: json.dumps(x, cls=utils.CustomJSONEncoder), + **kwargs, + ) session_factory = sessionmaker(bind=engine) return session_factory diff --git a/conda-store-server/conda_store_server/schema.py b/conda-store-server/conda_store_server/schema.py index 52142b4a9..5f27d5359 100644 --- a/conda-store-server/conda_store_server/schema.py +++ b/conda-store-server/conda_store_server/schema.py @@ -6,8 +6,10 @@ import sys from typing import Any, Callable, Dict, List, Optional, Union +from conda_lock.lockfile.v1.models import Lockfile from conda_store_server import conda_utils, utils from pydantic import BaseModel, Field, ValidationError, constr, validator +from pydantic.error_wrappers import ErrorWrapper def _datetime_factory(offset: datetime.timedelta): @@ -463,6 +465,49 @@ def parse_obj(cls, specification): raise utils.CondaStoreError(all_errors_hr) +class LockfileSpecification(BaseModel): + name: constr(regex=f"^[{ALLOWED_CHARACTERS}]+$") # noqa: F722 + description: Optional[str] = "" + lockfile: Lockfile + + @classmethod + def parse_obj(cls, specification): + # This uses pop because the version field must not be part of Lockfile + # input. Otherwise, the input will be rejected. The version field is + # hardcoded in the Lockfile schema and is only used when the output is + # printed. So the code below validates that the version is 1 if present + # and removes it to avoid the mentioned parsing error. + lockfile = specification.get("lockfile") + version = lockfile and lockfile.pop("version", None) + if version not in (None, 1): + # https://stackoverflow.com/questions/73968566/with-pydantic-how-can-i-create-my-own-validationerror-reason + raise ValidationError( + [ + ErrorWrapper( + ValueError("expected no version field or version equal to 1"), + "lockfile -> version", + ) + ], + LockfileSpecification, + ) + + return super().parse_obj(specification) + + def dict(self): + res = super().dict() + # The dict_for_output method includes the version field into the output + # and excludes unset fields. Without the version field present, + # conda-lock would reject a lockfile during parsing, so it wouldn't be + # installable, so we need to include the version + res["lockfile"] = self.lockfile.dict_for_output() + return res + + def __str__(self): + # This makes sure the format is suitable for output if this object is + # converted to a string, which can also happen implicitly + return str(self.dict()) + + ############################### # Docker Registry Schema ############################### diff --git a/conda-store-server/conda_store_server/server/templates/create.html b/conda-store-server/conda_store_server/server/templates/create.html index 5d37fe9f1..1a09acadf 100644 --- a/conda-store-server/conda_store_server/server/templates/create.html +++ b/conda-store-server/conda_store_server/server/templates/create.html @@ -26,6 +26,8 @@
{{ specification or "" }}
+ + @@ -64,6 +66,7 @@ let url = "{{ url_for('api_post_specification') }}"; let namespaceInput = document.querySelector('#namespace'); let specificationInput = document.querySelector('#specification'); + let isLockfileInput = document.querySelector('#is-lockfile'); let response = await fetch(url, { method: 'POST', @@ -73,6 +76,7 @@ body: JSON.stringify({ 'namespace': namespaceInput.value, 'specification': specificationInput.value, + 'is_lockfile': isLockfileInput.checked, }) }); diff --git a/conda-store-server/conda_store_server/server/views/api.py b/conda-store-server/conda_store_server/server/views/api.py index 72c6826cf..bcf49d983 100644 --- a/conda-store-server/conda_store_server/server/views/api.py +++ b/conda-store-server/conda_store_server/server/views/api.py @@ -790,6 +790,7 @@ async def api_post_specification( entity=Depends(dependencies.get_entity), specification: str = Body(""), namespace: Optional[str] = Body(None), + is_lockfile: Optional[bool] = Body(False, embed=True), ): with conda_store.get_db() as db: permissions = {Permissions.ENVIRONMENT_CREATE} @@ -805,7 +806,10 @@ async def api_post_specification( try: specification = yaml.safe_load(specification) - specification = schema.CondaSpecification.parse_obj(specification) + if is_lockfile: + specification = schema.LockfileSpecification.parse_obj(specification) + else: + specification = schema.CondaSpecification.parse_obj(specification) except yaml.error.YAMLError: raise HTTPException(status_code=400, detail="Unable to parse. Invalid YAML") except utils.CondaStoreError as e: @@ -822,7 +826,11 @@ async def api_post_specification( try: build_id = conda_store.register_environment( - db, specification, namespace_name, force=True + db, + specification, + namespace_name, + force=True, + is_lockfile=is_lockfile, ) except ValueError as e: raise HTTPException(status_code=400, detail=str(e.args[0])) diff --git a/conda-store-server/conda_store_server/utils.py b/conda-store-server/conda_store_server/utils.py index 10051de4a..d47e3275d 100644 --- a/conda-store-server/conda_store_server/utils.py +++ b/conda-store-server/conda_store_server/utils.py @@ -108,6 +108,16 @@ def timer(logger, prefix): logger.info(f"{prefix} took {time.time() - start_time:.3f} [s]") +# conda-lock defines Channel.used_env_vars as frozenset. But frozenset is not +# serializable to JSON, so this creates a custom JSON encoder to handle this +# type +class CustomJSONEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, frozenset): + return list(obj) + return super().default(obj) + + def recursive_sort(v): """Recursively sort a nested python objects of lists, dicts, strings, ints, and floats @@ -129,7 +139,7 @@ def sort_key(v): def datastructure_hash(v): - json_blob = json.dumps(recursive_sort(v)) + json_blob = json.dumps(recursive_sort(v), cls=CustomJSONEncoder) return hashlib.sha256(json_blob.encode("utf-8")).hexdigest()