diff --git a/repo2docker/__main__.py b/repo2docker/__main__.py index a52a734a9..1dd49ebe9 100644 --- a/repo2docker/__main__.py +++ b/repo2docker/__main__.py @@ -2,8 +2,8 @@ import sys import os import logging -import docker from .app import Repo2Docker +from .engine import BuildError, ImageLoadError from . import __version__ from .utils import validate_and_generate_port_mapping, is_valid_docker_image_name @@ -217,6 +217,8 @@ def get_argparser(): "--cache-from", action="append", default=[], help=Repo2Docker.cache_from.help ) + argparser.add_argument("--engine", help="Name of the container engine") + return argparser @@ -351,6 +353,9 @@ def make_r2d(argv=None): if args.cache_from: r2d.cache_from = args.cache_from + if args.engine: + r2d.engine = args.engine + r2d.environment = args.environment # if the source exists locally we don't want to delete it at the end @@ -371,12 +376,12 @@ def main(): r2d.initialize() try: r2d.start() - except docker.errors.BuildError as e: + except BuildError as e: # This is only raised by us if r2d.log_level == logging.DEBUG: r2d.log.exception(e) sys.exit(1) - except docker.errors.ImageLoadError as e: + except ImageLoadError as e: # This is only raised by us if r2d.log_level == logging.DEBUG: r2d.log.exception(e) diff --git a/repo2docker/app.py b/repo2docker/app.py index 8a804905c..470e9c0a9 100755 --- a/repo2docker/app.py +++ b/repo2docker/app.py @@ -11,15 +11,14 @@ import sys import logging import os +import entrypoints import getpass import shutil import tempfile import time -import docker +from .engine import BuildError, ContainerEngineException, ImageLoadError from urllib.parse import urlparse -from docker.utils import kwargs_from_env -from docker.errors import DockerException import escapism from pythonjsonlogger import jsonlogger @@ -382,6 +381,33 @@ def _user_name_default(self): config=True, ) + engine = Unicode( + "docker", + config=True, + help=""" + Name of the container engine. + + Defaults to 'docker'. + """, + ) + + def get_engine(self): + """Return an instance of the container engine. + + Currently no arguments are passed to the engine constructor. + """ + engines = entrypoints.get_group_named("repo2docker.engines") + try: + entry = engines[self.engine] + except KeyError: + raise ContainerEngineException( + "Container engine '{}' not found. Available engines: {}".format( + self.engine, ",".join(engines.keys()) + ) + ) + engine_class = entry.load() + return engine_class(parent=self) + def fetch(self, url, ref, checkout_path): """Fetch the contents of `url` and place it in `checkout_path`. @@ -474,13 +500,18 @@ def initialize(self): def push_image(self): """Push docker image to registry""" - client = docker.APIClient(version="auto", **kwargs_from_env()) + client = self.get_engine() # Build a progress setup for each layer, and only emit per-layer # info every 1.5s progress_layers = {} layers = {} last_emit_time = time.time() - for chunk in client.push(self.output_image_spec, stream=True): + for chunk in client.push(self.output_image_spec): + if client.string_output: + self.log.info(chunk, extra=dict(phase="pushing")) + continue + # else this is Docker output + # each chunk can be one or more lines of json events # split lines here in case multiple are delivered at once for line in chunk.splitlines(): @@ -492,7 +523,7 @@ def push_image(self): continue if "error" in progress: self.log.error(progress["error"], extra=dict(phase="failed")) - raise docker.errors.ImageLoadError(progress["error"]) + raise ImageLoadError(progress["error"]) if "id" not in progress: continue # deprecated truncated-progress data @@ -528,7 +559,7 @@ def start_container(self): Returns running container """ - client = docker.from_env(version="auto") + client = self.get_engine() docker_host = os.environ.get("DOCKER_HOST") if docker_host: @@ -565,11 +596,8 @@ def start_container(self): container_volumes = {} if self.volumes: - api_client = docker.APIClient( - version="auto", **docker.utils.kwargs_from_env() - ) - image = api_client.inspect_image(self.output_image_spec) - image_workdir = image["ContainerConfig"]["WorkingDir"] + image = client.inspect_image(self.output_image_spec) + image_workdir = image.config["WorkingDir"] for k, v in self.volumes.items(): container_volumes[os.path.abspath(k)] = { @@ -580,7 +608,6 @@ def start_container(self): run_kwargs = dict( publish_all_ports=self.all_ports, ports=ports, - detach=True, command=run_cmd, volumes=container_volumes, environment=self.environment, @@ -588,7 +615,7 @@ def start_container(self): run_kwargs.update(self.extra_run_kwargs) - container = client.containers.run(self.output_image_spec, **run_kwargs) + container = client.run(self.output_image_spec, **run_kwargs) while container.status == "created": time.sleep(0.5) @@ -611,7 +638,7 @@ def wait_for_container(self, container): if container.status == "running": self.log.info("Stopping container...\n", extra=dict(phase="running")) container.kill() - exit_code = container.attrs["State"]["ExitCode"] + exit_code = container.exitcode container.wait() @@ -645,12 +672,11 @@ def find_image(self): if self.dry_run: return False # check if we already have an image for this content - client = docker.APIClient(version="auto", **kwargs_from_env()) + client = self.get_engine() for image in client.images(): - if image["RepoTags"] is not None: - for tag in image["RepoTags"]: - if tag == self.output_image_spec + ":latest": - return True + for tag in image.tags: + if tag == self.output_image_spec + ":latest": + return True return False def build(self): @@ -660,12 +686,9 @@ def build(self): # Check if r2d can connect to docker daemon if not self.dry_run: try: - docker_client = docker.APIClient(version="auto", **kwargs_from_env()) - except DockerException as e: - self.log.error( - "\nDocker client initialization error: %s.\nCheck if docker is running on the host.\n", - e, - ) + docker_client = self.get_engine() + except ContainerEngineException as e: + self.log.error("\nContainer engine initialization error: %s\n", e) self.exit(1) # If the source to be executed is a directory, continue using the @@ -751,11 +774,14 @@ def build(self): self.cache_from, self.extra_build_kwargs, ): - if "stream" in l: + if docker_client.string_output: + self.log.info(l, extra=dict(phase="building")) + # else this is Docker output + elif "stream" in l: self.log.info(l["stream"], extra=dict(phase="building")) elif "error" in l: self.log.info(l["error"], extra=dict(phase="failure")) - raise docker.errors.BuildError(l["error"], build_log="") + raise BuildError(l["error"]) elif "status" in l: self.log.info( "Fetching base image...\r", extra=dict(phase="building") diff --git a/repo2docker/buildpacks/base.py b/repo2docker/buildpacks/base.py index 145d29648..46a407ad3 100644 --- a/repo2docker/buildpacks/base.py +++ b/repo2docker/buildpacks/base.py @@ -607,9 +607,6 @@ def _filter_tar(tar): tag=image_spec, custom_context=True, buildargs=build_args, - decode=True, - forcerm=True, - rm=True, container_limits=limits, cache_from=cache_from, ) diff --git a/repo2docker/buildpacks/docker.py b/repo2docker/buildpacks/docker.py index 5a0981655..8326aecfe 100644 --- a/repo2docker/buildpacks/docker.py +++ b/repo2docker/buildpacks/docker.py @@ -50,9 +50,6 @@ def build( dockerfile=self.binder_path(self.dockerfile), tag=image_spec, buildargs=build_args, - decode=True, - forcerm=True, - rm=True, container_limits=limits, cache_from=cache_from, ) diff --git a/repo2docker/docker.py b/repo2docker/docker.py new file mode 100644 index 000000000..d637674ad --- /dev/null +++ b/repo2docker/docker.py @@ -0,0 +1,119 @@ +""" +Docker container engine for repo2docker +""" + +import docker +from .engine import Container, ContainerEngine, ContainerEngineException, Image + + +class DockerContainer(Container): + def __init__(self, container): + self._c = container + + def reload(self): + return self._c.reload() + + def logs(self, *, stream=False): + return self._c.logs(stream=stream) + + def kill(self, *, signal="KILL"): + return self._c.kill(signal=signal) + + def remove(self): + return self._c.remove() + + def stop(self, *, timeout=10): + return self._c.stop(timeout=timeout) + + def wait(self): + return self._c.wait() + + @property + def exitcode(self): + return self._c.attrs["State"]["ExitCode"] + + @property + def status(self): + return self._c.status + + +class DockerEngine(ContainerEngine): + """ + https://docker-py.readthedocs.io/en/4.2.0/api.html#module-docker.api.build + """ + + string_output = False + + def __init__(self, *, parent): + super().__init__(parent=parent) + try: + self._apiclient = docker.APIClient( + version="auto", **docker.utils.kwargs_from_env() + ) + except docker.errors.DockerException as e: + raise ContainerEngineException("Check if docker is running on the host.", e) + + def build( + self, + *, + buildargs=None, + cache_from=None, + container_limits=None, + tag="", + custom_context=False, + dockerfile="", + fileobj=None, + path="", + **kwargs, + ): + return self._apiclient.build( + buildargs=buildargs, + cache_from=cache_from, + container_limits=container_limits, + forcerm=True, + rm=True, + tag=tag, + custom_context=custom_context, + decode=True, + dockerfile=dockerfile, + fileobj=fileobj, + path=path, + **kwargs, + ) + + def images(self): + images = self._apiclient.images() + return [Image(tags=image["RepoTags"]) for image in images] + + def inspect_image(self, image): + image = self._apiclient.inspect_image(image) + return Image(tags=image["RepoTags"], config=image["ContainerConfig"]) + + def push(self, image_spec): + return self._apiclient.push(image_spec, stream=True) + + def run( + self, + image_spec, + *, + command=None, + environment=None, + ports=None, + publish_all_ports=False, + remove=False, + volumes=None, + **kwargs, + ): + client = docker.from_env(version="auto") + container = client.containers.run( + image_spec, + command=command, + environment=(environment or []), + detach=True, + ports=(ports or {}), + publish_all_ports=publish_all_ports, + remove=remove, + volumes=(volumes or {}), + **kwargs, + ) + return DockerContainer(container) diff --git a/repo2docker/engine.py b/repo2docker/engine.py new file mode 100644 index 000000000..2db8bfca2 --- /dev/null +++ b/repo2docker/engine.py @@ -0,0 +1,314 @@ +""" +Interface for a repo2docker container engine +""" + +from abc import ABC, abstractmethod +from traitlets.config import LoggingConfigurable + + +# Based on https://docker-py.readthedocs.io/en/4.2.0/containers.html + + +class Container(ABC): + """ + Abstract container returned by repo2docker engines + """ + + @abstractmethod + def reload(self): + """ + Refresh container attributes + """ + + @abstractmethod + def logs(self, *, stream=False): + """ + Get the container logs. + + Parameters + ---------- + stream : bool + If `True` return an iterator over the log lines, otherwise return all logs + + Returns + ------- + str or generator of log strings + """ + + @abstractmethod + def kill(self, *, signal="KILL"): + """ + Send a signal to the container + + Parameters + ---------- + signal : str + The signal, default `KILL` + """ + + @abstractmethod + def remove(self): + """ + Remove the container + """ + + @abstractmethod + def stop(self, *, timeout=10): + """ + Stop the container + + Parameters + ---------- + timeout : If the container doesn't gracefully stop after this timeout kill it + """ + + @abstractmethod + def wait(self): + """ + Wait for the container to stop + """ + + @property + @abstractmethod + def exitcode(self): + """ + The container exit code if exited + """ + + @property + @abstractmethod + def status(self): + """ + The status of the container + + Returns + ------- + str : The status of the container. + Values include `created` `running` `exited`. + + Full list of statuses: + https://github.com/moby/moby/blob/v19.03.5/api/swagger.yaml#L4832 + """ + + +class Image: + """ + Information about a container image + """ + + def __init__(self, *, tags, config=None): + self._tags = tags or [] + self._config = config + + @property + def tags(self): + """ + A list of tags associated with an image. + + If locally built images have a localhost prefix this prefix should be removed or the image may not be recognised. + If there are no tags [] will be returned. + """ + return self._tags + + @property + def config(self): + """ + A dictionary of image configuration information + + If this is `None` the information has not been loaded. + If not `None` this must include the following fields: + - WorkingDir: The default working directory + """ + return self._config + + def __repr__(self): + return "Image(tags={},config={})".format(self.tags, self.config) + + +class ContainerEngine(LoggingConfigurable): + """ + Abstract container engine. + + Inherits from LoggingConfigurable, which means it has a log property. + Initialised with a reference to the parent so can also be configured using traitlets. + """ + + string_output = True + """ + Whether progress events should be strings or an object. + + Originally Docker was the only container engine supported by repo2docker. + Some operations including build() and push() would return generators of events in a Docker specific format. + This format of events is not easily constructable with other engines so the default is to return strings and raise an exception if an error occurs. + If an engine returns docker style events set this variable to False. + """ + + def __init__(self, *, parent): + """ + Initialise the container engine + + Parameters + ---------- + parent: Application + Reference to the parent application so that its configuration file can be used in this plugin. + """ + super().__init__(parent=parent) + + # Based on https://docker-py.readthedocs.io/en/4.2.0/api.html#module-docker.api.build + + def build( + self, + *, + buildargs={}, + cache_from=[], + container_limits={}, + tag="", + custom_context=False, + dockerfile="", + fileobj=None, + path="", + **kwargs, + ): + """ + Build a container + + Parameters + ---------- + buildargs : dict + Dictionary of build arguments + cache_from : list[str] + List of images to chech for caching + container_limits : dict + Dictionary of resources limits. These keys are supported: + - `cpusetcpus` + - `cpushares` + - `memory` + - `memswap` + tag : str + Tag to add to the image + + custom_context : bool + If `True` fileobj is a Tar file object containing the build context + dockerfile : str + Path to Dockerfile within the build context + fileobj : tarfile + A tar file-like object containing the build context + path : str + path to the Dockerfile + + Returns + ------- + A generator of strings. If an error occurs an exception must be thrown. + + If `string_output=True` this should instead be whatever Docker returns: + https://github.com/jupyter/repo2docker/blob/0.11.0/repo2docker/app.py#L725-L735 + """ + raise NotImplementedError("build not implemented") + + def images(self): + """ + List images + + Returns + ------- + list[Image] : List of Image objects. + """ + raise NotImplementedError("images not implemented") + + def inspect_image(self, image): + """ + Get information about an image + + TODO: This is specific to the engine, can we convert it to a standard format? + + Parameters + ---------- + image : str + The image + + Returns + ------- + Image object with .config dict. + """ + raise NotImplementedError("inspect_image not implemented") + + def push(self, image_spec): + """ + Push image to a registry + + Parameters + ---------- + image_spec : str + The repository spec to push to + + Returns + ------- + A generator of strings. If an error occurs an exception must be thrown. + + If `string_output=True` this should instead be whatever Docker returns: + https://github.com/jupyter/repo2docker/blob/0.11.0/repo2docker/app.py#L469-L495 + """ + raise NotImplementedError("push not implemented") + + # Note this is different from the Docker client which has Client.containers.run + def run( + self, + image_spec, + *, + command=[], + environment=[], + ports={}, + publish_all_ports=False, + remove=False, + volumes={}, + **kwargs, + ): + """ + Run a container + + Parameters + ---------- + image_spec : str + The image to run + command : list[str] + The command to run + environment : list[str] + List of environment variables in the form `ENVVAR=value` + ports : dict + Container port bindings in the form generated by `repo2docker.utils.validate_and_generate_port_mapping` + https://github.com/jupyter/repo2docker/blob/0.11.0/repo2docker/utils.py#L95 + publish_all_ports : bool + If `True` publish all ports to host + remove : bool + If `True` delete container when it completes + volumes : dict + Volume bindings in the form `{src : dest}` + + Returns + ------- + Container : the running container + + Raises + ------ + NotImplementedError + This engine does not support running containers + """ + raise NotImplementedError("Running containers not supported") + + +class ContainerEngineException(Exception): + """ + Base class for exceptions in the container engine + """ + + +class BuildError(ContainerEngineException): + """ + Container build error + """ + + +class ImageLoadError(ContainerEngineException): + """ + Container load/push error + """ diff --git a/setup.py b/setup.py index af0611497..27c19c93c 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ def get_identifier(json): version=versioneer.get_version(), install_requires=[ "docker", + "entrypoints", "escapism", "jinja2", "python-json-logger", @@ -89,6 +90,7 @@ def get_identifier(json): "console_scripts": [ "jupyter-repo2docker = repo2docker.__main__:main", "repo2docker = repo2docker.__main__:main", - ] + ], + "repo2docker.engines": ["docker = repo2docker.docker:DockerEngine"], }, ) diff --git a/tests/unit/test_app.py b/tests/unit/test_app.py index d45c28a4c..25b39cdb1 100644 --- a/tests/unit/test_app.py +++ b/tests/unit/test_app.py @@ -14,7 +14,7 @@ def test_find_image(): images = [{"RepoTags": ["some-org/some-repo:latest"]}] - with patch("repo2docker.app.docker.APIClient") as FakeDockerClient: + with patch("repo2docker.docker.docker.APIClient") as FakeDockerClient: instance = FakeDockerClient.return_value instance.images.return_value = images @@ -28,7 +28,7 @@ def test_find_image(): def test_dont_find_image(): images = [{"RepoTags": ["some-org/some-image-name:latest"]}] - with patch("repo2docker.app.docker.APIClient") as FakeDockerClient: + with patch("repo2docker.docker.docker.APIClient") as FakeDockerClient: instance = FakeDockerClient.return_value instance.images.return_value = images diff --git a/tests/unit/test_editable.py b/tests/unit/test_editable.py index a84e4ba2e..6d711e25f 100644 --- a/tests/unit/test_editable.py +++ b/tests/unit/test_editable.py @@ -44,12 +44,14 @@ def test_editable_by_host(): try: with tempfile.NamedTemporaryFile(dir=DIR, prefix="testfile", suffix=".txt"): - status, output = container.exec_run(["sh", "-c", "ls testfile????????.txt"]) + status, output = container._c.exec_run( + ["sh", "-c", "ls testfile????????.txt"] + ) assert status == 0 assert re.match(br"^testfile\w{8}\.txt\n$", output) is not None # After exiting the with block the file should stop existing # in the container as well as locally - status, output = container.exec_run(["sh", "-c", "ls testfile????????.txt"]) + status, output = container._c.exec_run(["sh", "-c", "ls testfile????????.txt"]) assert status == 2 assert re.match(br"^testfile\w{8}\.txt\n$", output) is None diff --git a/tests/unit/test_external_scripts.py b/tests/unit/test_external_scripts.py index 1265d3ac5..d03debe55 100644 --- a/tests/unit/test_external_scripts.py +++ b/tests/unit/test_external_scripts.py @@ -32,7 +32,7 @@ def get_build_script_files(self): assert container.status == "running" try: - status, output = container.exec_run(["sh", "-c", "cat /tmp/my_extra_script"]) + status, output = container._c.exec_run(["sh", "-c", "cat /tmp/my_extra_script"]) assert status == 0 assert output.decode("utf-8") == "Hello World of Absolute Paths!" finally: diff --git a/tests/unit/test_ports.py b/tests/unit/test_ports.py index 407a3e02c..edb71424a 100644 --- a/tests/unit/test_ports.py +++ b/tests/unit/test_ports.py @@ -77,7 +77,7 @@ def _cleanup(): container.reload() assert container.status == "running" - port_mapping = container.attrs["NetworkSettings"]["Ports"] + port_mapping = container._c.attrs["NetworkSettings"]["Ports"] if all_ports: port = port_mapping["8888/tcp"][0]["HostPort"]