From c73fc26824f2bffa791472a27039da0d2dd1ccbe Mon Sep 17 00:00:00 2001 From: Jun Guo Date: Wed, 4 Jan 2017 15:31:12 +0800 Subject: [PATCH 01/18] Fix 404 issue, change APIError to more accureate ImageNotFound Signed-off-by: Jun Guo --- compose/service.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/compose/service.py b/compose/service.py index e3862b6ea06..ee8c88a925a 100644 --- a/compose/service.py +++ b/compose/service.py @@ -10,6 +10,7 @@ import enum import six from docker.errors import APIError +from docker.errors import ImageNotFound from docker.errors import NotFound from docker.types import LogConfig from docker.utils.ports import build_port_bindings @@ -318,11 +319,8 @@ def ensure_image_exists(self, do_build=BuildAction.none): def image(self): try: return self.client.inspect_image(self.image_name) - except APIError as e: - if e.response.status_code == 404 and e.explanation and 'No such image' in str(e.explanation): - raise NoSuchImageError("Image '{}' not found".format(self.image_name)) - else: - raise + except ImageNotFound: + raise NoSuchImageError("Image '{}' not found".format(self.image_name)) @property def image_name(self): From 2648af6807f83f0dd85b236e89e4bc3ee5db15fc Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 22 Nov 2016 11:09:56 +0000 Subject: [PATCH 02/18] enable universal wheels Signed-off-by: Thomas Grainger --- Dockerfile.run | 2 +- script/build/image | 4 ++-- script/release/push-release | 6 +++--- setup.cfg | 2 ++ setup.py | 23 ++++++++++++++++++++++- 5 files changed, 30 insertions(+), 7 deletions(-) create mode 100644 setup.cfg diff --git a/Dockerfile.run b/Dockerfile.run index 4e76d64ffac..c6852af1483 100644 --- a/Dockerfile.run +++ b/Dockerfile.run @@ -7,7 +7,7 @@ RUN apk -U add \ COPY requirements.txt /code/requirements.txt RUN pip install -r /code/requirements.txt -ADD dist/docker-compose-release.tar.gz /code/docker-compose +ADD dist/docker-compose-release-py2.py3-none-any.whl /code/docker-compose RUN pip install --no-deps /code/docker-compose/docker-compose-* ENTRYPOINT ["/usr/bin/docker-compose"] diff --git a/script/build/image b/script/build/image index bdd98f03e76..28aa2047aeb 100755 --- a/script/build/image +++ b/script/build/image @@ -11,6 +11,6 @@ TAG=$1 VERSION="$(python setup.py --version)" ./script/build/write-git-sha -python setup.py sdist -cp dist/docker-compose-$VERSION.tar.gz dist/docker-compose-release.tar.gz +python setup.py sdist bdist_wheel +cp dist/docker-compose-$VERSION-py2.py3-none-any.whl dist/docker-compose-release-py2.py3-none-any.whl docker build -t docker/compose:$TAG -f Dockerfile.run . diff --git a/script/release/push-release b/script/release/push-release index d5ae3de9dab..d1a9e3f6648 100755 --- a/script/release/push-release +++ b/script/release/push-release @@ -54,13 +54,13 @@ git push $GITHUB_REPO $VERSION echo "Uploading the docker image" docker push docker/compose:$VERSION -echo "Uploading sdist to PyPI" +echo "Uploading package to PyPI" pandoc -f markdown -t rst README.md -o README.rst sed -i -e 's/logo.png?raw=true/https:\/\/github.com\/docker\/compose\/raw\/master\/logo.png?raw=true/' README.rst ./script/build/write-git-sha -python setup.py sdist +python setup.py sdist bdist_wheel if [ "$(command -v twine 2> /dev/null)" ]; then - twine upload ./dist/docker-compose-${VERSION/-/}.tar.gz + twine upload ./dist/docker-compose-${VERSION/-/}.tar.gz ./dist/docker-compose-${VERSION/-/}-py2.py3-none-any.whl else python setup.py upload fi diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000000..3c6e79cf31d --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 diff --git a/setup.py b/setup.py index 8b4cf709e02..00ca9f4c7b4 100644 --- a/setup.py +++ b/setup.py @@ -4,10 +4,12 @@ from __future__ import unicode_literals import codecs +import logging import os import re import sys +import pkg_resources from setuptools import find_packages from setuptools import setup @@ -49,7 +51,25 @@ def find_version(*file_paths): if sys.version_info[:2] < (3, 4): tests_require.append('mock >= 1.0.1') - install_requires.append('enum34 >= 1.0.4, < 2') + +extras_require = { + ':python_version < "3.4"': ['enum34 >= 1.0.4, < 2'] +} + + +try: + if 'bdist_wheel' not in sys.argv: + for key, value in extras_require.items(): + if key.startswith(':') and pkg_resources.evaluate_marker(key[1:]): + install_requires.extend(value) +except Exception: + logging.getLogger(__name__).exception( + 'Something went wrong calculating platform specific dependencies, so ' + "you're getting them all!" + ) + for key, value in extras_require.items(): + if key.startswith(':'): + install_requires.extend(value) setup( @@ -63,6 +83,7 @@ def find_version(*file_paths): include_package_data=True, test_suite='nose.collector', install_requires=install_requires, + extras_require=extras_require, tests_require=tests_require, entry_points=""" [console_scripts] From 534b4ed820ec2b2e2f7b296e0065c42ebf3489cd Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 4 Jan 2017 15:26:11 -0800 Subject: [PATCH 03/18] Falsy values in COMPOSE_CONVERT_WINDOWS_PATHS are properly recognized Signed-off-by: Joffrey F --- compose/config/config.py | 2 +- compose/config/environment.py | 11 ++++++++ tests/unit/config/environment_test.py | 40 +++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 tests/unit/config/environment_test.py diff --git a/compose/config/config.py b/compose/config/config.py index fd935591f6a..c11460fa593 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -712,7 +712,7 @@ def finalize_service(service_config, service_names, version, environment): if 'volumes' in service_dict: service_dict['volumes'] = [ VolumeSpec.parse( - v, environment.get('COMPOSE_CONVERT_WINDOWS_PATHS') + v, environment.get_boolean('COMPOSE_CONVERT_WINDOWS_PATHS') ) for v in service_dict['volumes'] ] diff --git a/compose/config/environment.py b/compose/config/environment.py index 5d6b5af690c..7b92693002c 100644 --- a/compose/config/environment.py +++ b/compose/config/environment.py @@ -105,3 +105,14 @@ def get(self, key, *args, **kwargs): super(Environment, self).get(key.upper(), *args, **kwargs) ) return super(Environment, self).get(key, *args, **kwargs) + + def get_boolean(self, key): + # Convert a value to a boolean using "common sense" rules. + # Unset, empty, "0" and "false" (i-case) yield False. + # All other values yield True. + value = self.get(key) + if not value: + return False + if value.lower() in ['0', 'false']: + return False + return True diff --git a/tests/unit/config/environment_test.py b/tests/unit/config/environment_test.py new file mode 100644 index 00000000000..20446d2bf2d --- /dev/null +++ b/tests/unit/config/environment_test.py @@ -0,0 +1,40 @@ +# encoding: utf-8 +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +from compose.config.environment import Environment +from tests import unittest + + +class EnvironmentTest(unittest.TestCase): + def test_get_simple(self): + env = Environment({ + 'FOO': 'bar', + 'BAR': '1', + 'BAZ': '' + }) + + assert env.get('FOO') == 'bar' + assert env.get('BAR') == '1' + assert env.get('BAZ') == '' + + def test_get_undefined(self): + env = Environment({ + 'FOO': 'bar' + }) + assert env.get('FOOBAR') is None + + def test_get_boolean(self): + env = Environment({ + 'FOO': '', + 'BAR': '0', + 'BAZ': 'FALSE', + 'FOOBAR': 'true', + }) + + assert env.get_boolean('FOO') is False + assert env.get_boolean('BAR') is False + assert env.get_boolean('BAZ') is False + assert env.get_boolean('FOOBAR') is True + assert env.get_boolean('UNDEFINED') is False From 2c157e8fa9a94d71637643a6ec807db8b21a9d29 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 6 Jan 2017 17:45:57 -0800 Subject: [PATCH 04/18] Use docker SDK 2.0.1 Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index bae5d9ea1e2..4b7c7b76050 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ PyYAML==3.11 backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.2.0 colorama==0.3.7 -docker==2.0.0 +docker==2.0.1 dockerpty==0.4.1 docopt==0.6.1 enum34==1.0.4; python_version < '3.4' diff --git a/setup.py b/setup.py index 8b4cf709e02..7954d92bba9 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ def find_version(*file_paths): 'requests >= 2.6.1, != 2.11.0, < 2.12', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 2.0.0, < 3.0', + 'docker >= 2.0.1, < 3.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From 52792b7a963af9c593e61c78c7f0c7f62550a85b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 10 Jan 2017 14:57:32 -0800 Subject: [PATCH 05/18] Update setup.py extra_requires Signed-off-by: Joffrey F --- setup.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 0ceb2a22a6e..2f2ba7429d1 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,9 @@ def find_version(*file_paths): tests_require.append('mock >= 1.0.1') extras_require = { - ':python_version < "3.4"': ['enum34 >= 1.0.4, < 2'] + ':python_version < "3.4"': ['enum34 >= 1.0.4, < 2'], + ':python_version < "3.5"': ['backports.ssl_match_hostname >= 3.5'], + ':python_version < "3.3"': ['ipaddress >= 1.0.16'], } @@ -64,8 +66,8 @@ def find_version(*file_paths): install_requires.extend(value) except Exception: logging.getLogger(__name__).exception( - 'Something went wrong calculating platform specific dependencies, so ' - "you're getting them all!" + 'Failed to compute platform dependencies. All dependencies will be ' + 'installed as a result.' ) for key, value in extras_require.items(): if key.startswith(':'): From 19190ea0df43978d1a9c9f0fefd644ca5b08aee3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 10 Jan 2017 16:43:26 -0800 Subject: [PATCH 06/18] Fix docker image build script when using universal wheels Signed-off-by: Joffrey F --- Dockerfile.run | 5 +++-- script/build/image | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile.run b/Dockerfile.run index c6852af1483..de46e35e5f5 100644 --- a/Dockerfile.run +++ b/Dockerfile.run @@ -1,5 +1,6 @@ FROM alpine:3.4 +ARG version RUN apk -U add \ python \ py-pip @@ -7,7 +8,7 @@ RUN apk -U add \ COPY requirements.txt /code/requirements.txt RUN pip install -r /code/requirements.txt -ADD dist/docker-compose-release-py2.py3-none-any.whl /code/docker-compose -RUN pip install --no-deps /code/docker-compose/docker-compose-* +COPY dist/docker_compose-${version}-py2.py3-none-any.whl /code/ +RUN pip install --no-deps /code/docker_compose-${version}-py2.py3-none-any.whl ENTRYPOINT ["/usr/bin/docker-compose"] diff --git a/script/build/image b/script/build/image index 28aa2047aeb..3590ce14e41 100755 --- a/script/build/image +++ b/script/build/image @@ -12,5 +12,4 @@ VERSION="$(python setup.py --version)" ./script/build/write-git-sha python setup.py sdist bdist_wheel -cp dist/docker-compose-$VERSION-py2.py3-none-any.whl dist/docker-compose-release-py2.py3-none-any.whl -docker build -t docker/compose:$TAG -f Dockerfile.run . +docker build --build-arg version=$VERSION -t docker/compose:$TAG -f Dockerfile.run . From 29b46d5b26055ade86a2e0e608342dd298e5a8c3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 11 Jan 2017 15:39:48 -0800 Subject: [PATCH 07/18] Use correct wheel file name in twine upload command Signed-off-by: Joffrey F --- script/release/push-release | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/release/push-release b/script/release/push-release index d1a9e3f6648..9db6f68941c 100755 --- a/script/release/push-release +++ b/script/release/push-release @@ -60,12 +60,13 @@ sed -i -e 's/logo.png?raw=true/https:\/\/github.com\/docker\/compose\/raw\/maste ./script/build/write-git-sha python setup.py sdist bdist_wheel if [ "$(command -v twine 2> /dev/null)" ]; then - twine upload ./dist/docker-compose-${VERSION/-/}.tar.gz ./dist/docker-compose-${VERSION/-/}-py2.py3-none-any.whl + twine upload ./dist/docker-compose-${VERSION/-/}.tar.gz ./dist/docker_compose-${VERSION/-/}-py2.py3-none-any.whl else python setup.py upload fi echo "Testing pip package" +deactivate || true virtualenv venv-test source venv-test/bin/activate pip install docker-compose==$VERSION From 2df31bb13c9a6820aba1d9b5a827329eded2b9cd Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 11 Jan 2017 16:25:40 -0800 Subject: [PATCH 08/18] Provide valid serialization of depends_on when format is not 2.1 Signed-off-by: Joffrey F --- compose/config/serialize.py | 9 ++++++++- tests/unit/config/config_test.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 768f3d4738c..05ac0d60df8 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -56,9 +56,16 @@ def denormalize_service_dict(service_dict, version): service_dict = service_dict.copy() if 'restart' in service_dict: - service_dict['restart'] = types.serialize_restart_spec(service_dict['restart']) + service_dict['restart'] = types.serialize_restart_spec( + service_dict['restart'] + ) if version == V1 and 'network_mode' not in service_dict: service_dict['network_mode'] = 'bridge' + if 'depends_on' in service_dict and version != V2_1: + service_dict['depends_on'] = sorted([ + svc for svc in service_dict['depends_on'].keys() + ]) + return service_dict diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 31a888ed007..ca7c61683b0 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -22,6 +22,7 @@ from compose.config.environment import Environment from compose.config.errors import ConfigurationError from compose.config.errors import VERSION_EXPLANATION +from compose.config.serialize import denormalize_service_dict from compose.config.types import VolumeSpec from compose.const import IS_WINDOWS_PLATFORM from compose.utils import nanoseconds_from_time_seconds @@ -3269,3 +3270,33 @@ def make_files(dirname, filenames): return os.path.basename(filename) finally: shutil.rmtree(project_dir) + + +class SerializeTest(unittest.TestCase): + def test_denormalize_depends_on_v3(self): + service_dict = { + 'image': 'busybox', + 'command': 'true', + 'depends_on': { + 'service2': {'condition': 'service_started'}, + 'service3': {'condition': 'service_started'}, + } + } + + assert denormalize_service_dict(service_dict, V3_0) == { + 'image': 'busybox', + 'command': 'true', + 'depends_on': ['service2', 'service3'] + } + + def test_denormalize_depends_on_v2_1(self): + service_dict = { + 'image': 'busybox', + 'command': 'true', + 'depends_on': { + 'service2': {'condition': 'service_started'}, + 'service3': {'condition': 'service_started'}, + } + } + + assert denormalize_service_dict(service_dict, V2_1) == service_dict From 56a1b02aac33d09ec7761729a8d6ddcb0fbdea0e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 17 Jan 2017 13:22:16 -0800 Subject: [PATCH 09/18] Catch healthcheck exceptions in parallel_execute Signed-off-by: Joffrey F --- compose/parallel.py | 40 ++++++++++++++++++------------- tests/integration/project_test.py | 4 ++-- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/compose/parallel.py b/compose/parallel.py index b2654dcfd91..e495410cff8 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -12,6 +12,8 @@ from six.moves.queue import Queue from compose.cli.signals import ShutdownException +from compose.errors import HealthCheckFailed +from compose.errors import NoHealthCheckConfigured from compose.errors import OperationFailedError from compose.utils import get_output_stream @@ -48,7 +50,7 @@ def parallel_execute(objects, func, get_name, msg, get_deps=None): elif isinstance(exception, APIError): errors[get_name(obj)] = exception.explanation writer.write(get_name(obj), 'error') - elif isinstance(exception, OperationFailedError): + elif isinstance(exception, (OperationFailedError, HealthCheckFailed, NoHealthCheckConfigured)): errors[get_name(obj)] = exception.msg writer.write(get_name(obj), 'error') elif isinstance(exception, UpstreamError): @@ -164,21 +166,27 @@ def feed_queue(objects, func, get_deps, results, state): for obj in pending: deps = get_deps(obj) - - if any(dep[0] in state.failed for dep in deps): - log.debug('{} has upstream errors - not processing'.format(obj)) - results.put((obj, None, UpstreamError())) - state.failed.add(obj) - elif all( - dep not in objects or ( - dep in state.finished and (not ready_check or ready_check(dep)) - ) for dep, ready_check in deps - ): - log.debug('Starting producer thread for {}'.format(obj)) - t = Thread(target=producer, args=(obj, func, results)) - t.daemon = True - t.start() - state.started.add(obj) + try: + if any(dep[0] in state.failed for dep in deps): + log.debug('{} has upstream errors - not processing'.format(obj)) + results.put((obj, None, UpstreamError())) + state.failed.add(obj) + elif all( + dep not in objects or ( + dep in state.finished and (not ready_check or ready_check(dep)) + ) for dep, ready_check in deps + ): + log.debug('Starting producer thread for {}'.format(obj)) + t = Thread(target=producer, args=(obj, func, results)) + t.daemon = True + t.start() + state.started.add(obj) + except (HealthCheckFailed, NoHealthCheckConfigured) as e: + log.debug( + 'Healthcheck for service(s) upstream of {} failed - ' + 'not processing'.format(obj) + ) + results.put((obj, None, e)) if state.is_done(): results.put(STOP) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index c5e3cf50ffb..ee2b7817bdc 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1443,7 +1443,7 @@ def test_project_up_unhealthy_dependency(self): project = Project.from_config( name='composetest', config_data=config_data, client=self.client ) - with pytest.raises(HealthCheckFailed): + with pytest.raises(ProjectError): project.up() containers = project.containers() assert len(containers) == 1 @@ -1479,7 +1479,7 @@ def test_project_up_no_healthcheck_dependency(self): project = Project.from_config( name='composetest', config_data=config_data, client=self.client ) - with pytest.raises(NoHealthCheckConfigured): + with pytest.raises(ProjectError): project.up() containers = project.containers() assert len(containers) == 1 From 1a02121ab55876f92bcceb62d1e81b7f114f0c79 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 18 Jan 2017 17:52:03 -0800 Subject: [PATCH 10/18] depends_on merge now retains condition information when present Signed-off-by: Joffrey F --- compose/config/config.py | 6 +++++- tests/unit/config/config_test.py | 34 ++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index c11460fa593..7e77421e520 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -818,6 +818,7 @@ def merge_service_dicts(base, override, version): md.merge_mapping('ulimits', parse_ulimits) md.merge_mapping('networks', parse_networks) md.merge_mapping('sysctls', parse_sysctls) + md.merge_mapping('depends_on', parse_depends_on) md.merge_sequence('links', ServiceLink.parse) for field in ['volumes', 'devices']: @@ -825,7 +826,7 @@ def merge_service_dicts(base, override, version): for field in [ 'ports', 'cap_add', 'cap_drop', 'expose', 'external_links', - 'security_opt', 'volumes_from', 'depends_on', + 'security_opt', 'volumes_from', ]: md.merge_field(field, merge_unique_items_lists, default=[]) @@ -920,6 +921,9 @@ def parse_dict_or_list(split_func, type_name, arguments): parse_labels = functools.partial(parse_dict_or_list, split_kv, 'labels') parse_networks = functools.partial(parse_dict_or_list, lambda k: (k, None), 'networks') parse_sysctls = functools.partial(parse_dict_or_list, split_kv, 'sysctls') +parse_depends_on = functools.partial( + parse_dict_or_list, lambda k: (k, {'condition': 'service_started'}), 'depends_on' +) def parse_ulimits(ulimits): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index ca7c61683b0..ab8bfcfcc46 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1713,6 +1713,40 @@ def test_merge_logging_v2_no_override(self): } } + def test_merge_depends_on_no_override(self): + base = { + 'image': 'busybox', + 'depends_on': { + 'app1': {'condition': 'service_started'}, + 'app2': {'condition': 'service_healthy'} + } + } + override = {} + actual = config.merge_service_dicts(base, override, V2_1) + assert actual == base + + def test_merge_depends_on_mixed_syntax(self): + base = { + 'image': 'busybox', + 'depends_on': { + 'app1': {'condition': 'service_started'}, + 'app2': {'condition': 'service_healthy'} + } + } + override = { + 'depends_on': ['app3'] + } + + actual = config.merge_service_dicts(base, override, V2_1) + assert actual == { + 'image': 'busybox', + 'depends_on': { + 'app1': {'condition': 'service_started'}, + 'app2': {'condition': 'service_healthy'}, + 'app3': {'condition': 'service_started'} + } + } + def test_external_volume_config(self): config_details = build_config_details({ 'version': '2', From 169289c8b66dcab760cdc7e1534fc0bece326d44 Mon Sep 17 00:00:00 2001 From: "Aaron.L.Xu" Date: Fri, 20 Jan 2017 00:52:13 +0800 Subject: [PATCH 11/18] find a fishbone Signed-off-by: Aaron.L.Xu --- script/test/versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/test/versions.py b/script/test/versions.py index 45ead14387a..0c3b8162dbb 100755 --- a/script/test/versions.py +++ b/script/test/versions.py @@ -5,7 +5,7 @@ The default release is the most recent non-RC version. -Recent is a list of unqiue major.minor versions, where each is the most +Recent is a list of unique major.minor versions, where each is the most recent version in the series. For example, if the list of versions is: From 1c46525c2baf8532434c320bf0443a520381431d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 19 Jan 2017 14:47:31 -0800 Subject: [PATCH 12/18] 1.11.0dev Signed-off-by: Joffrey F --- CHANGELOG.md | 48 +++++++++++++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c645ca2511..6699f880735 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,54 @@ Change log ========== +1.10.0 (2017-01-18) +------------------- + +### New Features + +#### Compose file version 3.0 + +- Introduced version 3.0 of the `docker-compose.yml` specification. This + version requires to be used with Docker Engine 1.13 or above and is + specifically designed to work with the `docker stack` commands. + +#### Compose file version 2.1 and up + +- Healthcheck configuration can now be done in the service definition using + the `healthcheck` parameter + +- Containers dependencies can now be set up to wait on positive healthchecks + when declared using `depends_on`. See the documentation for the updated + syntax. + **Note:** This feature will not be ported to version 3 Compose files. + +- Added support for the `sysctls` parameter in service definitions + +- Added support for the `userns_mode` parameter in service definitions + +- Compose now adds identifying labels to networks and volumes it creates + +#### Compose file version 2.0 and up + +- Added support for the `stop_grace_period` option in service definitions. + +### Bugfixes + +- Colored output now works properly on Windows. + +- Fixed a bug where docker-compose run would fail to set up link aliases + in interactive mode on Windows. + +- Networks created by Compose are now always made attachable + (Compose files v2.1 and up). + +- Fixed a bug where falsy values of `COMPOSE_CONVERT_WINDOWS_PATHS` + (`0`, `false`, empty value) were being interpreted as true. + +- Fixed a bug where forward slashes in some .dockerignore patterns weren't + being parsed correctly on Windows + + 1.9.0 (2016-11-16) ----------------- diff --git a/compose/__init__.py b/compose/__init__.py index 6f05b282f26..38417836476 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.10.0dev' +__version__ = '1.11.0dev' From 5c2165eaafbb625eb2058b199b571a228e86df03 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 19 Jan 2017 15:41:31 -0800 Subject: [PATCH 13/18] Fix volume definition in v3 schema Signed-off-by: Joffrey F --- compose/config/config_schema_v3.0.json | 4 ++-- tests/acceptance/cli_test.py | 8 +++++++- tests/fixtures/v3-full/docker-compose.yml | 4 ++++ tests/integration/testcases.py | 7 +++++-- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/compose/config/config_schema_v3.0.json b/compose/config/config_schema_v3.0.json index ae4c0530063..584b6ef5d8c 100644 --- a/compose/config/config_schema_v3.0.json +++ b/compose/config/config_schema_v3.0.json @@ -330,9 +330,9 @@ "name": {"type": "string"} }, "additionalProperties": false - } + }, + "labels": {"$ref": "#/definitions/list_or_dict"} }, - "labels": {"$ref": "#/definitions/list_or_dict"}, "additionalProperties": false }, diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index b9766226d9f..ce31dd18f3b 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -295,7 +295,13 @@ def test_config_v3(self): assert yaml.load(result.stdout) == { 'version': '3.0', 'networks': {}, - 'volumes': {}, + 'volumes': { + 'foobar': { + 'labels': { + 'com.docker.compose.test': 'true', + }, + }, + }, 'services': { 'web': { 'image': 'busybox', diff --git a/tests/fixtures/v3-full/docker-compose.yml b/tests/fixtures/v3-full/docker-compose.yml index b4d1b6422f3..a1661ab9363 100644 --- a/tests/fixtures/v3-full/docker-compose.yml +++ b/tests/fixtures/v3-full/docker-compose.yml @@ -35,3 +35,7 @@ services: retries: 5 stop_grace_period: 20s +volumes: + foobar: + labels: + com.docker.compose.test: 'true' diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index f6bc402bf7f..230bd2d9250 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -13,6 +13,7 @@ from compose.config.config import V1 from compose.config.config import V2_0 from compose.config.config import V2_1 +from compose.config.config import V3_0 from compose.config.environment import Environment from compose.const import API_VERSIONS from compose.const import LABEL_PROJECT @@ -36,13 +37,15 @@ def format_link(link): def engine_max_version(): if 'DOCKER_VERSION' not in os.environ: - return V2_1 + return V3_0 version = os.environ['DOCKER_VERSION'].partition('-')[0] if version_lt(version, '1.10'): return V1 elif version_lt(version, '1.12'): return V2_0 - return V2_1 + elif version_lt(version, '1.13'): + return V2_1 + return V3_0 def build_version_required_decorator(ignored_versions): From d83d31889ea937524db798aa8260638036503764 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 19 Jan 2017 16:05:13 -0800 Subject: [PATCH 14/18] Remove external_name from volume def in config output Signed-off-by: Joffrey F --- compose/config/serialize.py | 7 ++++++- tests/acceptance/cli_test.py | 14 ++++++++++++++ tests/fixtures/volumes/docker-compose.yml | 2 ++ tests/fixtures/volumes/external-volumes.yml | 16 ++++++++++++++++ 4 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/volumes/docker-compose.yml create mode 100644 tests/fixtures/volumes/external-volumes.yml diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 05ac0d60df8..9ea287a468a 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -32,6 +32,11 @@ def denormalize_config(config): if 'external_name' in net_conf: del net_conf['external_name'] + volumes = config.volumes.copy() + for vol_name, vol_conf in volumes.items(): + if 'external_name' in vol_conf: + del vol_conf['external_name'] + version = config.version if version == V1: version = V2_1 @@ -40,7 +45,7 @@ def denormalize_config(config): 'version': version, 'services': services, 'networks': networks, - 'volumes': config.volumes, + 'volumes': volumes, } diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index b9766226d9f..287c043c0b6 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -262,6 +262,20 @@ def test_config_external_network(self): } } + def test_config_external_volume(self): + self.base_dir = 'tests/fixtures/volumes' + result = self.dispatch(['-f', 'external-volumes.yml', 'config']) + json_result = yaml.load(result.stdout) + assert 'volumes' in json_result + assert json_result['volumes'] == { + 'foo': { + 'external': True + }, + 'bar': { + 'external': {'name': 'some_bar'} + } + } + def test_config_v1(self): self.base_dir = 'tests/fixtures/v1-config' result = self.dispatch(['config']) diff --git a/tests/fixtures/volumes/docker-compose.yml b/tests/fixtures/volumes/docker-compose.yml new file mode 100644 index 00000000000..da711ac42bb --- /dev/null +++ b/tests/fixtures/volumes/docker-compose.yml @@ -0,0 +1,2 @@ +version: '2.1' +services: {} diff --git a/tests/fixtures/volumes/external-volumes.yml b/tests/fixtures/volumes/external-volumes.yml new file mode 100644 index 00000000000..05c6c4844fe --- /dev/null +++ b/tests/fixtures/volumes/external-volumes.yml @@ -0,0 +1,16 @@ +version: "2.1" + +services: + web: + image: busybox + command: top + volumes: + - foo:/var/lib/ + - bar:/etc/ + +volumes: + foo: + external: true + bar: + external: + name: some_bar From 644e1716c33e0b3a3dcb4e5227b7dac0a289cffd Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 20 Jan 2017 12:55:59 -0500 Subject: [PATCH 15/18] Add missing network.internal to v3 schema. Signed-off-by: Daniel Nephin --- compose/config/config_schema_v3.0.json | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/config/config_schema_v3.0.json b/compose/config/config_schema_v3.0.json index 584b6ef5d8c..fbcd8bb859a 100644 --- a/compose/config/config_schema_v3.0.json +++ b/compose/config/config_schema_v3.0.json @@ -308,6 +308,7 @@ }, "additionalProperties": false }, + "internal": {"type": "boolean"}, "labels": {"$ref": "#/definitions/list_or_dict"} }, "additionalProperties": false From 20d6f450b5e37e5c634fdf517df32d7484a2b3ff Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 20 Jan 2017 15:05:53 -0800 Subject: [PATCH 16/18] Don't encode build context path on Windows Signed-off-by: Joffrey F --- compose/service.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/compose/service.py b/compose/service.py index 20a40c684f7..724e05652c0 100644 --- a/compose/service.py +++ b/compose/service.py @@ -22,6 +22,7 @@ from .config import merge_environment from .config.types import VolumeSpec from .const import DEFAULT_TIMEOUT +from .const import IS_WINDOWS_PLATFORM from .const import LABEL_CONFIG_HASH from .const import LABEL_CONTAINER_NUMBER from .const import LABEL_ONE_OFF @@ -769,9 +770,9 @@ def build(self, no_cache=False, pull=False, force_rm=False): build_opts = self.options.get('build', {}) path = build_opts.get('context') - # python2 os.path() doesn't support unicode, so we need to encode it to - # a byte string - if not six.PY3: + # python2 os.stat() doesn't support unicode on some UNIX, so we + # encode it to a bytestring to be safe + if not six.PY3 and not IS_WINDOWS_PLATFORM: path = path.encode('utf8') build_output = self.client.build( From e10d1140b95d33a589b1a971e0edda703bfb1e9b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Jan 2017 18:00:09 -0800 Subject: [PATCH 17/18] Convert time data back to string values when serializing config Signed-off-by: Joffrey F --- compose/config/serialize.py | 29 +++++++++++++++++++++++++ tests/acceptance/cli_test.py | 4 ++-- tests/unit/config/config_test.py | 36 ++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 2 deletions(-) diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 9ea287a468a..3745de82dbc 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -57,6 +57,25 @@ def serialize_config(config): width=80) +def serialize_ns_time_value(value): + result = (value, 'ns') + table = [ + (1000., 'us'), + (1000., 'ms'), + (1000., 's'), + (60., 'm'), + (60., 'h') + ] + for stage in table: + tmp = value / stage[0] + if tmp == int(value / stage[0]): + value = tmp + result = (int(value), stage[1]) + else: + break + return '{0}{1}'.format(*result) + + def denormalize_service_dict(service_dict, version): service_dict = service_dict.copy() @@ -73,4 +92,14 @@ def denormalize_service_dict(service_dict, version): svc for svc in service_dict['depends_on'].keys() ]) + if 'healthcheck' in service_dict: + if 'interval' in service_dict['healthcheck']: + service_dict['healthcheck']['interval'] = serialize_ns_time_value( + service_dict['healthcheck']['interval'] + ) + if 'timeout' in service_dict['healthcheck']: + service_dict['healthcheck']['timeout'] = serialize_ns_time_value( + service_dict['healthcheck']['timeout'] + ) + return service_dict diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 41a06c9502e..58160c80265 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -353,8 +353,8 @@ def test_config_v3(self): 'healthcheck': { 'test': 'cat /etc/passwd', - 'interval': 10000000000, - 'timeout': 1000000000, + 'interval': '10s', + 'timeout': '1s', 'retries': 5, }, diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index ab8bfcfcc46..d7947a4e802 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -23,6 +23,7 @@ from compose.config.errors import ConfigurationError from compose.config.errors import VERSION_EXPLANATION from compose.config.serialize import denormalize_service_dict +from compose.config.serialize import serialize_ns_time_value from compose.config.types import VolumeSpec from compose.const import IS_WINDOWS_PLATFORM from compose.utils import nanoseconds_from_time_seconds @@ -3334,3 +3335,38 @@ def test_denormalize_depends_on_v2_1(self): } assert denormalize_service_dict(service_dict, V2_1) == service_dict + + def test_serialize_time(self): + data = { + 9: '9ns', + 9000: '9us', + 9000000: '9ms', + 90000000: '90ms', + 900000000: '900ms', + 999999999: '999999999ns', + 1000000000: '1s', + 60000000000: '1m', + 60000000001: '60000000001ns', + 9000000000000: '150m', + 90000000000000: '25h', + } + + for k, v in data.items(): + assert serialize_ns_time_value(k) == v + + def test_denormalize_healthcheck(self): + service_dict = { + 'image': 'test', + 'healthcheck': { + 'test': 'exit 1', + 'interval': '1m40s', + 'timeout': '30s', + 'retries': 5 + } + } + processed_service = config.process_service(config.ServiceConfig( + '.', 'test', 'test', service_dict + )) + denormalized_service = denormalize_service_dict(processed_service, V2_1) + assert denormalized_service['healthcheck']['interval'] == '100s' + assert denormalized_service['healthcheck']['timeout'] == '30s' From 5895d8bbc9939524b449e296ac93d8e98aa70eb0 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 24 Jan 2017 15:02:27 -0800 Subject: [PATCH 18/18] Detect conflicting version of the docker python SDK and prevent execution until issue is fixed Signed-off-by: Joffrey F --- compose/cli/main.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/compose/cli/main.py b/compose/cli/main.py index c25ccbfa485..db068272a8e 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -14,6 +14,30 @@ from inspect import getdoc from operator import attrgetter + +# Attempt to detect https://github.com/docker/compose/issues/4344 +try: + # A regular import statement causes PyInstaller to freak out while + # trying to load pip. This way it is simply ignored. + pip = __import__('pip') + pip_packages = pip.get_installed_distributions() + if 'docker-py' in [pkg.project_name for pkg in pip_packages]: + from .colors import red + print( + red('ERROR:'), + "Dependency conflict: an older version of the 'docker-py' package " + "is polluting the namespace. " + "Run the following command to remedy the issue:\n" + "pip uninstall docker docker-py; pip install docker", + file=sys.stderr + ) + sys.exit(1) +except ImportError: + # pip is not available, which indicates it's probably the binary + # distribution of Compose which is not affected + pass + + from . import errors from . import signals from .. import __version__