diff --git a/goth/runner/container/compose.py b/goth/runner/container/compose.py index 7824023f..78a45344 100644 --- a/goth/runner/container/compose.py +++ b/goth/runner/container/compose.py @@ -8,6 +8,7 @@ from typing import AsyncIterator, ClassVar, Dict, List, Optional from docker import DockerClient +from docker.models.networks import Network import yaml from goth.runner.container import DockerContainer @@ -145,12 +146,53 @@ async def _wait_for_containers(self) -> None: await monitor.wait_for_entry(pattern, timeout=CONTAINER_READY_TIMEOUT) logger.info("Compose network ready") - async def stop_network(self): - """Stop the running compose network, removing its containers.""" + def _disconnect_containers(self, excluded_containers: List[str]) -> None: + """Disconnect containers from the default Docker compose network. + + This corresponds to executing the command + >>> docker network disconnect -f DEFAULT_NETWORK CONTAINER_NAME + for each container. + + All containers except those with names in `excluded_containers` are + disconnected. + """ + networks = self._docker_client.networks.list( + names=[DockerContainer.DEFAULT_NETWORK] + ) + if networks: + compose_network: Network = networks[0] + compose_network.reload() + connected_containers = [ + c["Name"] for c in compose_network.attrs["Containers"].values() + ] + yagna_containers = [ + name for name in connected_containers if name not in excluded_containers + ] + for container in yagna_containers: + logger.info( + "Disconnecting container %s from network %s...", + container, + DockerContainer.DEFAULT_NETWORK, + ) + compose_network.disconnect(container, force=True) + + async def stop_network(self, compose_containers: Optional[List[str]] = None): + """Stop the running compose network, removing its containers. + + Before the network is stopped, all yagna containers need to be disconnected, + due to https://github.com/moby/moby/issues/23302. + + To avoid explicitly disconnecting some containers -- for example, the ones + started by `docker-compose` itself that will be disconnected by + `docker-compose down` -- pass their names in `compose_containers`. + """ + for name, monitor in self._log_monitors.items(): logger.debug("stopping log monitor. name=%s", name) await monitor.stop() + self._disconnect_containers(compose_containers or []) + await run_command( ["docker-compose", "-f", str(self.config.file_path), "down", "-t", "0"] ) @@ -224,12 +266,14 @@ async def run_compose_network( Yields information on containers started in the compose network. """ + compose_containers = [] try: logger.debug( "Starting compose network. log_dir=%s, force_build=%s", log_dir, force_build ) containers = await compose_manager.start_network(log_dir, force_build) + compose_containers = list(containers.keys()) yield containers finally: logger.debug("Stopping compose network") - await compose_manager.stop_network() + await compose_manager.stop_network(compose_containers) diff --git a/poetry.lock b/poetry.lock index 660f32c2..fea776ed 100644 --- a/poetry.lock +++ b/poetry.lock @@ -92,25 +92,25 @@ typecheck = ["mypy"] [[package]] name = "black" -version = "20.8b1" +version = "21.7b0" description = "The uncompromising code formatter." category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.6.2" [package.dependencies] appdirs = "*" click = ">=7.1.2" mypy-extensions = ">=0.4.3" -pathspec = ">=0.6,<1" +pathspec = ">=0.8.1,<1" regex = ">=2020.1.8" -toml = ">=0.10.1" -typed-ast = ">=1.4.0" -typing-extensions = ">=3.7.4" +tomli = ">=0.2.6,<2.0.0" [package.extras] colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] +d = ["aiohttp (>=3.6.0)", "aiohttp-cors (>=0.4.0)"] +python2 = ["typed-ast (>=1.4.2)"] +uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "blinker" @@ -951,6 +951,14 @@ category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "tomli" +version = "1.2.1" +description = "A lil' TOML parser" +category = "dev" +optional = false +python-versions = ">=3.6" + [[package]] name = "tomlkit" version = "0.7.2" @@ -1089,7 +1097,7 @@ python-versions = "*" [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "c2278943c3f490082d08512f1045c0f722c86e4de3d7a417624c586374a7fcba" +content-hash = "525d476f2b1122a9aa7678cd468375d29334fcb9a03425f7ec45127076035625" [metadata.files] aiohttp = [ @@ -1165,7 +1173,8 @@ bcrypt = [ {file = "bcrypt-3.2.0.tar.gz", hash = "sha256:5b93c1726e50a93a033c36e5ca7fdcd29a5c7395af50a6892f5d9e7c6cfbfb29"}, ] black = [ - {file = "black-20.8b1.tar.gz", hash = "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"}, + {file = "black-21.7b0-py3-none-any.whl", hash = "sha256:1c7aa6ada8ee864db745b22790a32f94b2795c253a75d6d9b5e439ff10d23116"}, + {file = "black-21.7b0.tar.gz", hash = "sha256:c8373c6491de9362e39271630b65b964607bc5c79c83783547d76c839b3aa219"}, ] blinker = [ {file = "blinker-1.4.tar.gz", hash = "sha256:471aee25f3992bd325afa3772f1063dbdbbca947a041b8b89466dc00d606f8b6"}, @@ -1707,26 +1716,18 @@ pyyaml = [ {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"}, {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"}, {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"}, - {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347"}, - {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541"}, {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"}, {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"}, {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"}, {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"}, - {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa"}, - {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"}, {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"}, {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"}, {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"}, {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"}, - {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247"}, - {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc"}, {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"}, {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"}, {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"}, {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"}, - {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122"}, - {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6"}, {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"}, {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, @@ -1825,6 +1826,10 @@ toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] +tomli = [ + {file = "tomli-1.2.1-py3-none-any.whl", hash = "sha256:8dd0e9524d6f386271a36b41dbf6c57d8e32fd96fd22b6584679dc569d20899f"}, + {file = "tomli-1.2.1.tar.gz", hash = "sha256:a5b75cb6f3968abb47af1b40c1819dc519ea82bcc065776a866e8d74c5ca9442"}, +] tomlkit = [ {file = "tomlkit-0.7.2-py2.py3-none-any.whl", hash = "sha256:173ad840fa5d2aac140528ca1933c29791b79a374a0861a80347f42ec9328117"}, {file = "tomlkit-0.7.2.tar.gz", hash = "sha256:d7a454f319a7e9bd2e249f239168729327e4dd2d27b17dc68be264ad1ce36754"}, diff --git a/pyproject.toml b/pyproject.toml index fb7fd938..7abda98c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ ya-aioclient = "^0.6" ghapi = "^0.1.16" [tool.poetry.dev-dependencies] -black = "20.8b1" +black = "21.7b0" flake8 = "3.9.0" flake8-docstrings = "1.6.0" mypy = "^0.812"