From 427957ce13a980c6a3ab5e61255acdcd9a97f604 Mon Sep 17 00:00:00 2001 From: amine Date: Fri, 6 Aug 2021 16:37:50 +0100 Subject: [PATCH 01/18] add support for secrets and configs and DockerSwarmOperator --- .../providers/docker/operators/docker_swarm.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/airflow/providers/docker/operators/docker_swarm.py b/airflow/providers/docker/operators/docker_swarm.py index 00a158956c95f..fcfa48a5978de 100644 --- a/airflow/providers/docker/operators/docker_swarm.py +++ b/airflow/providers/docker/operators/docker_swarm.py @@ -15,10 +15,11 @@ # specific language governing permissions and limitations # under the License. """Run ephemeral Docker Swarm services""" -from typing import Optional +from typing import Optional, List import requests from docker import types +from docker.types.services import ConfigReference, SecretReference from airflow.exceptions import AirflowException from airflow.providers.docker.operators.docker import DockerOperator @@ -95,11 +96,21 @@ class DockerSwarmOperator(DockerOperator): :type enable_logging: bool """ - def __init__(self, *, image: str, enable_logging: bool = True, **kwargs) -> None: + def __init__( + self, + *, + image: str, + enable_logging: bool = True, + configs: List[ConfigReference] = None, + secrets: List[SecretReference] = None, + **kwargs + ) -> None: super().__init__(image=image, **kwargs) self.enable_logging = enable_logging self.service = None + self.configs = configs if configs is not None else [] + self.secrets = secrets if secrets is not None else [] def execute(self, context) -> None: self.cli = self._get_cli() @@ -121,6 +132,8 @@ def _run_service(self) -> None: env=self.environment, user=self.user, tty=self.tty, + configs=self.configs, + secrets=self.secrets, ), restart_policy=types.RestartPolicy(condition='none'), resources=types.Resources(mem_limit=self.mem_limit), From 91e56bc8a8180cfb9254cb020da0a22187a076f5 Mon Sep 17 00:00:00 2001 From: amine Date: Fri, 6 Aug 2021 17:05:11 +0100 Subject: [PATCH 02/18] update tests after adding secret and config support to DockerSwarmOperator --- tests/providers/docker/operators/test_docker_swarm.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/providers/docker/operators/test_docker_swarm.py b/tests/providers/docker/operators/test_docker_swarm.py index c41e1e2770537..adb415e4425a5 100644 --- a/tests/providers/docker/operators/test_docker_swarm.py +++ b/tests/providers/docker/operators/test_docker_swarm.py @@ -23,6 +23,7 @@ import requests from docker import APIClient from docker.types import Mount +from docker.types.services import ConfigReference, SecretReference from parameterized import parameterized from airflow.exceptions import AirflowException @@ -69,6 +70,8 @@ def _client_service_logs_effect(): mounts=[Mount(source='/host/path', target='/container/path', type='bind')], auto_remove=True, tty=True, + configs=[ConfigReference(config_id="dummy_cfg_id", config_name="dummy_cfg_name")], + secrets=[SecretReference(secret_id="dummy_secret_id", secret_name="dummy_secret_name")], ) operator.execute(None) @@ -82,6 +85,8 @@ def _client_service_logs_effect(): mounts=[Mount(source='/host/path', target='/container/path', type='bind')], tty=True, env={'UNIT': 'TEST', 'AIRFLOW_TMP_DIR': '/tmp/airflow'}, + configs=[ConfigReference(config_id="dummy_cfg_id", config_name="dummy_cfg_name")], + secrets=[SecretReference(secret_id="dummy_secret_id", secret_name="dummy_secret_name")], ) types_mock.RestartPolicy.assert_called_once_with(condition='none') types_mock.Resources.assert_called_once_with(mem_limit='128m') From 403d46ac729a7d436989f7a1c1406b94bee43601 Mon Sep 17 00:00:00 2001 From: amine Date: Fri, 6 Aug 2021 17:11:44 +0100 Subject: [PATCH 03/18] add replication option --- airflow/providers/docker/operators/docker_swarm.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/airflow/providers/docker/operators/docker_swarm.py b/airflow/providers/docker/operators/docker_swarm.py index fcfa48a5978de..658072b4e47e6 100644 --- a/airflow/providers/docker/operators/docker_swarm.py +++ b/airflow/providers/docker/operators/docker_swarm.py @@ -103,6 +103,8 @@ def __init__( enable_logging: bool = True, configs: List[ConfigReference] = None, secrets: List[SecretReference] = None, + mode: str = "replicated", + replicas: int = 1, **kwargs ) -> None: super().__init__(image=image, **kwargs) @@ -111,6 +113,8 @@ def __init__( self.service = None self.configs = configs if configs is not None else [] self.secrets = secrets if secrets is not None else [] + self.mode = mode + self.replicas = replicas def execute(self, context) -> None: self.cli = self._get_cli() @@ -140,6 +144,7 @@ def _run_service(self) -> None: ), name=f'airflow-{get_random_string()}', labels={'name': f'airflow__{self.dag_id}__{self.task_id}'}, + mode=types.ServiceMode(mode=self.mode, replicas=self.replicas), ) self.log.info('Service started: %s', str(self.service)) From edd1e0efaeff79d24675c897ed6a8e00e56d5c0f Mon Sep 17 00:00:00 2001 From: amine Date: Fri, 6 Aug 2021 17:19:08 +0100 Subject: [PATCH 04/18] update docstrings --- airflow/providers/docker/operators/docker_swarm.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/airflow/providers/docker/operators/docker_swarm.py b/airflow/providers/docker/operators/docker_swarm.py index 658072b4e47e6..be724b9afb843 100644 --- a/airflow/providers/docker/operators/docker_swarm.py +++ b/airflow/providers/docker/operators/docker_swarm.py @@ -94,6 +94,19 @@ class DockerSwarmOperator(DockerOperator): Supported only if the Docker engine is using json-file or journald logging drivers. The `tty` parameter should be set to use this with Python applications. :type enable_logging: bool + :param configs: List of docker configs to be exposed to the containers of the swarm service. + The configs are ConfigReference objects as per the docker api + [https://docker-py.readthedocs.io/en/stable/services.html#docker.models.services.ServiceCollection.create]_ + :type configs: list[ConfigReference] + :param secrets: List of docker secrets to be exposed to the containers of the swarm service. + The secrets are SecretReference objects as per the docker create_service api. + [https://docker-py.readthedocs.io/en/stable/services.html#docker.models.services.ServiceCollection.create]_ + :type secrets: list[SecretReference] + :param mode: Indicate whether a service should be deployed as a replicated or global service. + Can be either `replicated` or `global` + :type mode: str + :param replicas: Number of replicas. For replicated services only. + :type replicas: int """ def __init__( From 0b4775986006a4893670258f64ba52ebda15fa3a Mon Sep 17 00:00:00 2001 From: amine Date: Fri, 6 Aug 2021 17:24:50 +0100 Subject: [PATCH 05/18] update unittest with ServiceMode --- tests/providers/docker/operators/test_docker_swarm.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/providers/docker/operators/test_docker_swarm.py b/tests/providers/docker/operators/test_docker_swarm.py index adb415e4425a5..0079809bbbc4c 100644 --- a/tests/providers/docker/operators/test_docker_swarm.py +++ b/tests/providers/docker/operators/test_docker_swarm.py @@ -56,6 +56,7 @@ def _client_service_logs_effect(): types_mock.ContainerSpec.return_value = mock_obj types_mock.RestartPolicy.return_value = mock_obj types_mock.Resources.return_value = mock_obj + types_mock.ServiceMode.return_value = mock_obj client_class_mock.return_value = client_mock @@ -72,6 +73,8 @@ def _client_service_logs_effect(): tty=True, configs=[ConfigReference(config_id="dummy_cfg_id", config_name="dummy_cfg_name")], secrets=[SecretReference(secret_id="dummy_secret_id", secret_name="dummy_secret_name")], + mode="replicated", + replicas=3, ) operator.execute(None) @@ -90,6 +93,7 @@ def _client_service_logs_effect(): ) types_mock.RestartPolicy.assert_called_once_with(condition='none') types_mock.Resources.assert_called_once_with(mem_limit='128m') + types_mock.ServiceMode.assert_called_once_with(mode="replicated", replicas=3) client_class_mock.assert_called_once_with( base_url='unix://var/run/docker.sock', tls=None, version='1.19' From 5f57585b3f1e6d854f138c5387ab2c4d7c1c25b8 Mon Sep 17 00:00:00 2001 From: amine Date: Fri, 6 Aug 2021 17:37:27 +0100 Subject: [PATCH 06/18] add support for networks --- airflow/providers/docker/operators/docker_swarm.py | 13 +++++++++---- .../providers/docker/operators/test_docker_swarm.py | 3 ++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/airflow/providers/docker/operators/docker_swarm.py b/airflow/providers/docker/operators/docker_swarm.py index be724b9afb843..a58ba4aaa0214 100644 --- a/airflow/providers/docker/operators/docker_swarm.py +++ b/airflow/providers/docker/operators/docker_swarm.py @@ -15,11 +15,11 @@ # specific language governing permissions and limitations # under the License. """Run ephemeral Docker Swarm services""" -from typing import Optional, List +from typing import Optional, List, Union import requests from docker import types -from docker.types.services import ConfigReference, SecretReference +from docker.types.services import ConfigReference, SecretReference, NetworkAttachmentConfig from airflow.exceptions import AirflowException from airflow.providers.docker.operators.docker import DockerOperator @@ -107,6 +107,8 @@ class DockerSwarmOperator(DockerOperator): :type mode: str :param replicas: Number of replicas. For replicated services only. :type replicas: int + :param networks: List of network names or IDs or NetworkAttachmentConfig to attach the service to. + :type networks: List[Union[str, NetworkAttachmentConfig]] """ def __init__( @@ -118,16 +120,18 @@ def __init__( secrets: List[SecretReference] = None, mode: str = "replicated", replicas: int = 1, + networks: List[Union[str, NetworkAttachmentConfig]] = None, **kwargs ) -> None: super().__init__(image=image, **kwargs) self.enable_logging = enable_logging self.service = None - self.configs = configs if configs is not None else [] - self.secrets = secrets if secrets is not None else [] + self.configs = configs + self.secrets = secrets self.mode = mode self.replicas = replicas + self.networks = networks def execute(self, context) -> None: self.cli = self._get_cli() @@ -154,6 +158,7 @@ def _run_service(self) -> None: ), restart_policy=types.RestartPolicy(condition='none'), resources=types.Resources(mem_limit=self.mem_limit), + networks=self.networks, ), name=f'airflow-{get_random_string()}', labels={'name': f'airflow__{self.dag_id}__{self.task_id}'}, diff --git a/tests/providers/docker/operators/test_docker_swarm.py b/tests/providers/docker/operators/test_docker_swarm.py index 0079809bbbc4c..a4249dd8301dd 100644 --- a/tests/providers/docker/operators/test_docker_swarm.py +++ b/tests/providers/docker/operators/test_docker_swarm.py @@ -75,11 +75,12 @@ def _client_service_logs_effect(): secrets=[SecretReference(secret_id="dummy_secret_id", secret_name="dummy_secret_name")], mode="replicated", replicas=3, + networks=["dummy_network"], ) operator.execute(None) types_mock.TaskTemplate.assert_called_once_with( - container_spec=mock_obj, restart_policy=mock_obj, resources=mock_obj + container_spec=mock_obj, restart_policy=mock_obj, resources=mock_obj, networks=["dummy_network"], ) types_mock.ContainerSpec.assert_called_once_with( image='ubuntu:latest', From 7b7f3330404f2db1bf4cd1df4900ffcaa9ac6616 Mon Sep 17 00:00:00 2001 From: AmineB Date: Fri, 6 Aug 2021 23:27:07 +0200 Subject: [PATCH 07/18] black on docker_swarm.py --- airflow/providers/docker/operators/docker_swarm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airflow/providers/docker/operators/docker_swarm.py b/airflow/providers/docker/operators/docker_swarm.py index a58ba4aaa0214..2cee6c03e6c43 100644 --- a/airflow/providers/docker/operators/docker_swarm.py +++ b/airflow/providers/docker/operators/docker_swarm.py @@ -15,11 +15,11 @@ # specific language governing permissions and limitations # under the License. """Run ephemeral Docker Swarm services""" -from typing import Optional, List, Union +from typing import List, Optional, Union import requests from docker import types -from docker.types.services import ConfigReference, SecretReference, NetworkAttachmentConfig +from docker.types.services import ConfigReference, NetworkAttachmentConfig, SecretReference from airflow.exceptions import AirflowException from airflow.providers.docker.operators.docker import DockerOperator @@ -121,7 +121,7 @@ def __init__( mode: str = "replicated", replicas: int = 1, networks: List[Union[str, NetworkAttachmentConfig]] = None, - **kwargs + **kwargs, ) -> None: super().__init__(image=image, **kwargs) From 6c2e4bf403ec432422899467fcedf219201f34bf Mon Sep 17 00:00:00 2001 From: AmineB Date: Fri, 6 Aug 2021 23:28:34 +0200 Subject: [PATCH 08/18] Update test_docker_swarm.py --- tests/providers/docker/operators/test_docker_swarm.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/providers/docker/operators/test_docker_swarm.py b/tests/providers/docker/operators/test_docker_swarm.py index a4249dd8301dd..07d49f618704c 100644 --- a/tests/providers/docker/operators/test_docker_swarm.py +++ b/tests/providers/docker/operators/test_docker_swarm.py @@ -80,7 +80,10 @@ def _client_service_logs_effect(): operator.execute(None) types_mock.TaskTemplate.assert_called_once_with( - container_spec=mock_obj, restart_policy=mock_obj, resources=mock_obj, networks=["dummy_network"], + container_spec=mock_obj, + restart_policy=mock_obj, + resources=mock_obj, + networks=["dummy_network"], ) types_mock.ContainerSpec.assert_called_once_with( image='ubuntu:latest', From e50d9ac4620d35aa53cbbbe210893de2b3e190fe Mon Sep 17 00:00:00 2001 From: AmineB Date: Sat, 7 Aug 2021 09:03:07 +0100 Subject: [PATCH 09/18] Update docker_swarm.py --- airflow/providers/docker/operators/docker_swarm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/docker/operators/docker_swarm.py b/airflow/providers/docker/operators/docker_swarm.py index 2cee6c03e6c43..b1c3964c36480 100644 --- a/airflow/providers/docker/operators/docker_swarm.py +++ b/airflow/providers/docker/operators/docker_swarm.py @@ -154,7 +154,7 @@ def _run_service(self) -> None: user=self.user, tty=self.tty, configs=self.configs, - secrets=self.secrets, + secrets=self.secrets, ), restart_policy=types.RestartPolicy(condition='none'), resources=types.Resources(mem_limit=self.mem_limit), From 86e70c346754646500be53f179468a22a5723fc4 Mon Sep 17 00:00:00 2001 From: amine Date: Sat, 7 Aug 2021 19:16:19 +0100 Subject: [PATCH 10/18] black docker_swarm.py --- airflow/providers/docker/operators/docker_swarm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/docker/operators/docker_swarm.py b/airflow/providers/docker/operators/docker_swarm.py index b1c3964c36480..2cee6c03e6c43 100644 --- a/airflow/providers/docker/operators/docker_swarm.py +++ b/airflow/providers/docker/operators/docker_swarm.py @@ -154,7 +154,7 @@ def _run_service(self) -> None: user=self.user, tty=self.tty, configs=self.configs, - secrets=self.secrets, + secrets=self.secrets, ), restart_policy=types.RestartPolicy(condition='none'), resources=types.Resources(mem_limit=self.mem_limit), From ae6b1cb8440d0af350f431581c0ec58eb7f9453f Mon Sep 17 00:00:00 2001 From: amine Date: Sun, 8 Aug 2021 13:36:03 +0100 Subject: [PATCH 11/18] retrigger checks From b5352dfad32b05a0390dbae3792f1fb3ec37df02 Mon Sep 17 00:00:00 2001 From: AmineB Date: Mon, 9 Aug 2021 09:45:42 +0200 Subject: [PATCH 12/18] Improve typing for inputs in DockerSwarmOperator Co-authored-by: Tzu-ping Chung --- airflow/providers/docker/operators/docker_swarm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airflow/providers/docker/operators/docker_swarm.py b/airflow/providers/docker/operators/docker_swarm.py index 2cee6c03e6c43..37fb1ce77fa3e 100644 --- a/airflow/providers/docker/operators/docker_swarm.py +++ b/airflow/providers/docker/operators/docker_swarm.py @@ -116,11 +116,11 @@ def __init__( *, image: str, enable_logging: bool = True, - configs: List[ConfigReference] = None, - secrets: List[SecretReference] = None, + configs: Optional[List[ConfigReference]] = None, + secrets: Optional[List[SecretReference]] = None, mode: str = "replicated", replicas: int = 1, - networks: List[Union[str, NetworkAttachmentConfig]] = None, + networks: Optional[List[Union[str, NetworkAttachmentConfig]]] = None, **kwargs, ) -> None: super().__init__(image=image, **kwargs) From dc0484ae33daf6b288574f63f24e0e96591871cc Mon Sep 17 00:00:00 2001 From: amine Date: Mon, 9 Aug 2021 11:44:25 +0100 Subject: [PATCH 13/18] retrigger checks From ebfa0aef36291b2b94c2d54fc649c0f40798b13c Mon Sep 17 00:00:00 2001 From: amine Date: Mon, 9 Aug 2021 23:39:48 +0100 Subject: [PATCH 14/18] mode and replicas as one ServiceMode argument --- .../docker/operators/docker_swarm.py | 20 ++++++++----------- .../docker/operators/test_docker_swarm.py | 7 ++----- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/airflow/providers/docker/operators/docker_swarm.py b/airflow/providers/docker/operators/docker_swarm.py index 37fb1ce77fa3e..8f2ef940819ff 100644 --- a/airflow/providers/docker/operators/docker_swarm.py +++ b/airflow/providers/docker/operators/docker_swarm.py @@ -19,7 +19,7 @@ import requests from docker import types -from docker.types.services import ConfigReference, NetworkAttachmentConfig, SecretReference +from docker.types import ServiceMode, ConfigReference, NetworkAttachmentConfig, SecretReference from airflow.exceptions import AirflowException from airflow.providers.docker.operators.docker import DockerOperator @@ -97,16 +97,14 @@ class DockerSwarmOperator(DockerOperator): :param configs: List of docker configs to be exposed to the containers of the swarm service. The configs are ConfigReference objects as per the docker api [https://docker-py.readthedocs.io/en/stable/services.html#docker.models.services.ServiceCollection.create]_ - :type configs: list[ConfigReference] + :type configs: List[docker.types.ConfigReference] :param secrets: List of docker secrets to be exposed to the containers of the swarm service. The secrets are SecretReference objects as per the docker create_service api. [https://docker-py.readthedocs.io/en/stable/services.html#docker.models.services.ServiceCollection.create]_ - :type secrets: list[SecretReference] - :param mode: Indicate whether a service should be deployed as a replicated or global service. - Can be either `replicated` or `global` - :type mode: str - :param replicas: Number of replicas. For replicated services only. - :type replicas: int + :type secrets: List[docker.types.SecretReference] + :param mode: Indicate whether a service should be deployed as a replicated or global service, + and associated parameters + :type mode: docker.types.ServiceMode :param networks: List of network names or IDs or NetworkAttachmentConfig to attach the service to. :type networks: List[Union[str, NetworkAttachmentConfig]] """ @@ -118,8 +116,7 @@ def __init__( enable_logging: bool = True, configs: Optional[List[ConfigReference]] = None, secrets: Optional[List[SecretReference]] = None, - mode: str = "replicated", - replicas: int = 1, + mode: Optional[ServiceMode] = None, networks: Optional[List[Union[str, NetworkAttachmentConfig]]] = None, **kwargs, ) -> None: @@ -130,7 +127,6 @@ def __init__( self.configs = configs self.secrets = secrets self.mode = mode - self.replicas = replicas self.networks = networks def execute(self, context) -> None: @@ -162,7 +158,7 @@ def _run_service(self) -> None: ), name=f'airflow-{get_random_string()}', labels={'name': f'airflow__{self.dag_id}__{self.task_id}'}, - mode=types.ServiceMode(mode=self.mode, replicas=self.replicas), + mode=self.mode, ) self.log.info('Service started: %s', str(self.service)) diff --git a/tests/providers/docker/operators/test_docker_swarm.py b/tests/providers/docker/operators/test_docker_swarm.py index 07d49f618704c..fa8b5739845b1 100644 --- a/tests/providers/docker/operators/test_docker_swarm.py +++ b/tests/providers/docker/operators/test_docker_swarm.py @@ -23,7 +23,7 @@ import requests from docker import APIClient from docker.types import Mount -from docker.types.services import ConfigReference, SecretReference +from docker.types.services import ConfigReference, SecretReference, ServiceMode from parameterized import parameterized from airflow.exceptions import AirflowException @@ -56,7 +56,6 @@ def _client_service_logs_effect(): types_mock.ContainerSpec.return_value = mock_obj types_mock.RestartPolicy.return_value = mock_obj types_mock.Resources.return_value = mock_obj - types_mock.ServiceMode.return_value = mock_obj client_class_mock.return_value = client_mock @@ -73,8 +72,7 @@ def _client_service_logs_effect(): tty=True, configs=[ConfigReference(config_id="dummy_cfg_id", config_name="dummy_cfg_name")], secrets=[SecretReference(secret_id="dummy_secret_id", secret_name="dummy_secret_name")], - mode="replicated", - replicas=3, + mode=ServiceMode(mode="replicated", replicas=3), networks=["dummy_network"], ) operator.execute(None) @@ -97,7 +95,6 @@ def _client_service_logs_effect(): ) types_mock.RestartPolicy.assert_called_once_with(condition='none') types_mock.Resources.assert_called_once_with(mem_limit='128m') - types_mock.ServiceMode.assert_called_once_with(mode="replicated", replicas=3) client_class_mock.assert_called_once_with( base_url='unix://var/run/docker.sock', tls=None, version='1.19' From b2871e6db5231695af9ad7b03461b68129c1fc63 Mon Sep 17 00:00:00 2001 From: amine Date: Tue, 10 Aug 2021 09:44:59 +0100 Subject: [PATCH 15/18] uniformize imports --- .../providers/docker/operators/docker_swarm.py | 9 ++++----- .../docker/operators/test_docker_swarm.py | 18 ++++++++---------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/airflow/providers/docker/operators/docker_swarm.py b/airflow/providers/docker/operators/docker_swarm.py index 8f2ef940819ff..2d5373c840f17 100644 --- a/airflow/providers/docker/operators/docker_swarm.py +++ b/airflow/providers/docker/operators/docker_swarm.py @@ -19,7 +19,6 @@ import requests from docker import types -from docker.types import ServiceMode, ConfigReference, NetworkAttachmentConfig, SecretReference from airflow.exceptions import AirflowException from airflow.providers.docker.operators.docker import DockerOperator @@ -114,10 +113,10 @@ def __init__( *, image: str, enable_logging: bool = True, - configs: Optional[List[ConfigReference]] = None, - secrets: Optional[List[SecretReference]] = None, - mode: Optional[ServiceMode] = None, - networks: Optional[List[Union[str, NetworkAttachmentConfig]]] = None, + configs: Optional[List[types.ConfigReference]] = None, + secrets: Optional[List[types.SecretReference]] = None, + mode: Optional[types.ServiceMode] = None, + networks: Optional[List[Union[str, types.NetworkAttachmentConfig]]] = None, **kwargs, ) -> None: super().__init__(image=image, **kwargs) diff --git a/tests/providers/docker/operators/test_docker_swarm.py b/tests/providers/docker/operators/test_docker_swarm.py index fa8b5739845b1..1840561fd930d 100644 --- a/tests/providers/docker/operators/test_docker_swarm.py +++ b/tests/providers/docker/operators/test_docker_swarm.py @@ -21,9 +21,7 @@ import pytest import requests -from docker import APIClient -from docker.types import Mount -from docker.types.services import ConfigReference, SecretReference, ServiceMode +from docker import APIClient, types from parameterized import parameterized from airflow.exceptions import AirflowException @@ -67,12 +65,12 @@ def _client_service_logs_effect(): mem_limit='128m', user='unittest', task_id='unittest', - mounts=[Mount(source='/host/path', target='/container/path', type='bind')], + mounts=[types.Mount(source='/host/path', target='/container/path', type='bind')], auto_remove=True, tty=True, - configs=[ConfigReference(config_id="dummy_cfg_id", config_name="dummy_cfg_name")], - secrets=[SecretReference(secret_id="dummy_secret_id", secret_name="dummy_secret_name")], - mode=ServiceMode(mode="replicated", replicas=3), + configs=[types.ConfigReference(config_id="dummy_cfg_id", config_name="dummy_cfg_name")], + secrets=[types.SecretReference(secret_id="dummy_secret_id", secret_name="dummy_secret_name")], + mode=types.ServiceMode(mode="replicated", replicas=3), networks=["dummy_network"], ) operator.execute(None) @@ -87,11 +85,11 @@ def _client_service_logs_effect(): image='ubuntu:latest', command='env', user='unittest', - mounts=[Mount(source='/host/path', target='/container/path', type='bind')], + mounts=[types.Mount(source='/host/path', target='/container/path', type='bind')], tty=True, env={'UNIT': 'TEST', 'AIRFLOW_TMP_DIR': '/tmp/airflow'}, - configs=[ConfigReference(config_id="dummy_cfg_id", config_name="dummy_cfg_name")], - secrets=[SecretReference(secret_id="dummy_secret_id", secret_name="dummy_secret_name")], + configs=[types.ConfigReference(config_id="dummy_cfg_id", config_name="dummy_cfg_name")], + secrets=[types.SecretReference(secret_id="dummy_secret_id", secret_name="dummy_secret_name")], ) types_mock.RestartPolicy.assert_called_once_with(condition='none') types_mock.Resources.assert_called_once_with(mem_limit='128m') From de54f6980715577cb554da5078323ab0a9da0c4c Mon Sep 17 00:00:00 2001 From: amine Date: Mon, 16 Aug 2021 15:08:49 +0100 Subject: [PATCH 16/18] add mode on test_docker_swarm --- tests/providers/docker/operators/test_docker_swarm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/providers/docker/operators/test_docker_swarm.py b/tests/providers/docker/operators/test_docker_swarm.py index 1840561fd930d..8523644888de4 100644 --- a/tests/providers/docker/operators/test_docker_swarm.py +++ b/tests/providers/docker/operators/test_docker_swarm.py @@ -107,6 +107,7 @@ def _client_service_logs_effect(): assert csargs == (mock_obj,) assert cskwargs['labels'] == {'name': 'airflow__adhoc_airflow__unittest'} assert cskwargs['name'].startswith('airflow-') + assert cskwargs['mode'] == types.ServiceMode(mode="replicated", replicas=3) assert client_mock.tasks.call_count == 5 client_mock.remove_service.assert_called_once_with('some_id') From 6440a4bf88fcbe20141c87977e03c9a82dd8f7d4 Mon Sep 17 00:00:00 2001 From: amine Date: Mon, 16 Aug 2021 15:49:51 +0100 Subject: [PATCH 17/18] retrigger checks From d238bbcd28fe9924228aecdfdde5d62511679368 Mon Sep 17 00:00:00 2001 From: amine Date: Tue, 17 Aug 2021 08:30:15 +0100 Subject: [PATCH 18/18] retrigger tests