From 99afedcf54992ab5fe949eb6391778881e26255c Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Sat, 30 Jul 2022 00:05:34 +0800 Subject: [PATCH 01/58] tem store --- .../pydolphinscheduler/examples/tutorial.py | 4 ++++ .../resources_plugin/__init__.py | 0 .../resources_plugin/github.py | 18 ++++++++++++++++++ .../resources_plugin/gitlab.py | 0 .../resources_plugin/test.py | 0 .../src/pydolphinscheduler/tasks/shell.py | 4 ++++ 6 files changed, 26 insertions(+) create mode 100644 dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py create mode 100644 dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/github.py create mode 100644 dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/gitlab.py create mode 100644 dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/test.py diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py index 0478e685190f..8c10c699a39b 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py @@ -41,6 +41,10 @@ # [end package_import] # [start workflow_declare] +# @Resource( +# type="github", +# id="https://github.com/apache/dolphinscheduler/tree/dev/script" +# ) with ProcessDefinition( name="tutorial", schedule="0 0 0 * * ? *", diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/github.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/github.py new file mode 100644 index 000000000000..3e650ff4242e --- /dev/null +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/github.py @@ -0,0 +1,18 @@ + +class Github: + def __init__(self, path: str): + self.path = path + + def __call__(self, func): + def wrapper(): + print("github_resource __call__") + r = func() + return wrapper + +def func(): + printf("hello") + + +if __name__ == "__main__": + func() + diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/gitlab.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/gitlab.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/test.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/test.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/tasks/shell.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/tasks/shell.py index 9a73535c8cab..b253b7d0b8f3 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/tasks/shell.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/tasks/shell.py @@ -50,6 +50,10 @@ class Shell(Task): "raw_script", } + ext = ".sh" + ext_attr = "_raw_script" + def __init__(self, name: str, command: str, *args, **kwargs): + self._raw_script = command super().__init__(name, TaskType.SHELL, *args, **kwargs) self.raw_script = command From 1469f4059e4b47737434dc40477cee41c3432a59 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Sat, 30 Jul 2022 00:17:05 +0800 Subject: [PATCH 02/58] tem store --- .../resources_plugin/local.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py new file mode 100644 index 000000000000..11f186b25ad1 --- /dev/null +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py @@ -0,0 +1,24 @@ +import os.path + + +class Local: + + def __init__(self, prefix: str): + self._prefix = prefix + + @property + def prefix(self): + return self._prefix + + def read_file(self, suf: str): + try: + f = open(self._prefix + suf, 'r') + content = f.read() + f.close() + except FileNotFoundError: + print("{} is not found.".format(self.prefix + suf)) + except PermissionError: + print("You don't have permission to access {}.".format(self.prefix + suf)) + return content + + From 0eddb15c1891a8b7ace80d7aa04d5ce677dfbbbe Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Sat, 30 Jul 2022 00:18:56 +0800 Subject: [PATCH 03/58] tem store --- .../pydolphinscheduler/examples/tutorial.py | 48 +++++++++++++++---- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py index 8c10c699a39b..2a91245513bc 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py @@ -37,26 +37,58 @@ # Import task Shell object cause we would create some shell tasks later from pydolphinscheduler.tasks.shell import Shell +from pydolphinscheduler.constants import ResourcePluginType +from pydolphinscheduler.resources_plugin.__init__ import ResourcePlugin # [end package_import] # [start workflow_declare] -# @Resource( -# type="github", -# id="https://github.com/apache/dolphinscheduler/tree/dev/script" -# ) with ProcessDefinition( name="tutorial", schedule="0 0 0 * * ? *", start_time="2021-01-01", tenant="tenant_exists", + resource_plugin=ResourcePlugin( + type=ResourcePluginType.LOCAL, + prefix='/opt/', + ) ) as pd: # [end workflow_declare] # [start task_declare] - task_parent = Shell(name="task_parent", command="echo hello pydolphinscheduler") - task_child_one = Shell(name="task_child_one", command="echo 'child one'") - task_child_two = Shell(name="task_child_two", command="echo 'child two'") - task_union = Shell(name="task_union", command="echo union") + task_parent = Shell( + name="task_parent", + command="parent.sh", + resource_plugin=ResourcePlugin( + type=ResourcePluginType.LOCAL, + prefix='/opt/', + ) + ) + + task_child_one = Shell( + name="task_child_one", + command="child_one.sh", + resource_plugin=ResourcePlugin( + type=ResourcePluginType.LOCAL, + prefix='/opt/', + ) + ) + + task_child_two = Shell( + name="task_child_two", + command="child_two.sh", + resource_plugin=ResourcePlugin( + type=ResourcePluginType.LOCAL, + prefix='/opt/', + ) + ) + task_union = Shell( + name="task_union", + command="union.sh", + resource_plugin=ResourcePlugin( + type=ResourcePluginType.LOCAL, + prefix='/opt/', + ) + ) # [end task_declare] # [start task_relation_declare] From 0c3f063184290c8ea66fc481b0989a154f64fe4d Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Sat, 30 Jul 2022 00:34:38 +0800 Subject: [PATCH 04/58] temporary store --- .../src/main/resources/application.yaml | 4 +- .../src/main/resources/logback-spring.xml | 11 +++-- .../src/main/resources/common.properties | 7 +-- .../src/main/resources/application.yaml | 4 +- .../src/main/resources/logback-spring.xml | 11 +++-- .../src/pydolphinscheduler/constants.py | 8 ++++ .../core/process_definition.py | 5 +- .../src/pydolphinscheduler/core/task.py | 26 ++++++++++ .../resources_plugin/__init__.py | 47 +++++++++++++++++++ .../resources_plugin/github.py | 12 ----- .../resources_plugin/test.py | 21 +++++++++ .../src/main/resources/logback-spring.xml | 5 +- .../src/main/resources/application.yaml | 11 +++++ .../src/main/resources/logback-spring.xml | 11 +++-- pom.xml | 2 +- 15 files changed, 149 insertions(+), 36 deletions(-) diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml index 23b93028af40..d3588785af31 100644 --- a/dolphinscheduler-api/src/main/resources/application.yaml +++ b/dolphinscheduler-api/src/main/resources/application.yaml @@ -137,7 +137,9 @@ spring: on-profile: mysql datasource: driver-class-name: com.mysql.jdbc.Driver - url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 + url: jdbc:mysql://127.0.0.1:3306/chenrj?useUnicode=true&characterEncoding=UTF-8 + username: root + password: 4295 quartz: properties: org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate diff --git a/dolphinscheduler-api/src/main/resources/logback-spring.xml b/dolphinscheduler-api/src/main/resources/logback-spring.xml index bda628cc6053..975f50407d4f 100644 --- a/dolphinscheduler-api/src/main/resources/logback-spring.xml +++ b/dolphinscheduler-api/src/main/resources/logback-spring.xml @@ -51,11 +51,12 @@ - - - - - + + + + + + diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index 76c98d78ee35..72e4da84e5f8 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -19,7 +19,7 @@ data.basedir.path=/tmp/dolphinscheduler # resource storage type: HDFS, S3, NONE -resource.storage.type=NONE +resource.storage.type=HDFS # resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended resource.upload.path=/dolphinscheduler @@ -41,9 +41,10 @@ kerberos.expire.time=2 # resource view suffixs #resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js # if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path -hdfs.root.user=hdfs +hdfs.root.user=chenrj # if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir -fs.defaultFS=hdfs://mycluster:8020 +#fs.defaultFS=hdfs://mycluster:8020 +fs.defaultFS= file:/// aws.access.key.id=minioadmin aws.secret.access.key=minioadmin aws.region=us-east-1 diff --git a/dolphinscheduler-master/src/main/resources/application.yaml b/dolphinscheduler-master/src/main/resources/application.yaml index 0b6b3c811e54..b06e31ca493f 100644 --- a/dolphinscheduler-master/src/main/resources/application.yaml +++ b/dolphinscheduler-master/src/main/resources/application.yaml @@ -136,7 +136,9 @@ spring: on-profile: mysql datasource: driver-class-name: com.mysql.jdbc.Driver - url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 + url: jdbc:mysql://127.0.0.1:3306/chenrj?useUnicode=true&characterEncoding=UTF-8 + username: root + password: 4295 quartz: properties: org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate diff --git a/dolphinscheduler-master/src/main/resources/logback-spring.xml b/dolphinscheduler-master/src/main/resources/logback-spring.xml index 2e9cb45ada69..e8fa5d5decb2 100644 --- a/dolphinscheduler-master/src/main/resources/logback-spring.xml +++ b/dolphinscheduler-master/src/main/resources/logback-spring.xml @@ -64,11 +64,12 @@ - - - - - + + + + + + diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py index 3992917310cc..41358f911e25 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py @@ -107,3 +107,11 @@ class Time(str): FMT_STD_TIME = "%H:%M:%S" FMT_NO_COLON_TIME = "%H%M%S" + +class ResourcePluginType(str): + """Constants for resources plugin type, it will also show you which kind we support up to now.""" + LOCAL = "local" + GITHUB = "github" + GITLAB = "gitlab" + S3 = "S3" + OSS = "OSS" \ No newline at end of file diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py index dbf2c4179597..b10b23846945 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py @@ -21,13 +21,14 @@ from datetime import datetime from typing import Any, Dict, List, Optional, Set -from pydolphinscheduler.constants import ProcessDefinitionReleaseState, TaskType +from pydolphinscheduler.constants import ProcessDefinitionReleaseState, TaskType, ResourcePluginType from pydolphinscheduler.core import configuration from pydolphinscheduler.core.base import Base from pydolphinscheduler.exceptions import PyDSParamException, PyDSTaskNoFoundException from pydolphinscheduler.java_gateway import launch_gateway from pydolphinscheduler.side import Project, Tenant, User from pydolphinscheduler.utils.date import MAX_DATETIME, conv_from_str, conv_to_schedule +from pydolphinscheduler.resources_plugin.__init__ import ResourcePlugin class ProcessDefinitionContext: @@ -107,6 +108,7 @@ def __init__( timeout: Optional[int] = 0, release_state: Optional[str] = ProcessDefinitionReleaseState.ONLINE, param: Optional[Dict] = None, + resource_plugin: Optional[ResourcePlugin] = None, ): super().__init__(name, description) self.schedule = schedule @@ -129,6 +131,7 @@ def __init__( self.release_state = release_state self.param = param self.tasks: dict = {} + self.resource_plugin = resource_plugin # TODO how to fix circle import self._task_relations: set["TaskRelation"] = set() # noqa: F821 self._process_definition_code = None diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py index 599b97936982..9fcf603a276b 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py @@ -25,7 +25,9 @@ TaskFlag, TaskPriority, TaskTimeoutFlag, + ResourcePluginType, ) +from pydolphinscheduler.resources_plugin.__init__ import ResourcePlugin from pydolphinscheduler.core import configuration from pydolphinscheduler.core.base import Base from pydolphinscheduler.core.process_definition import ( @@ -97,6 +99,9 @@ class Task(Base): _task_custom_attr: set = set() + ext = None + ext_attr: str = None + DEFAULT_CONDITION_RESULT = {"successNode": [""], "failedNode": [""]} def __init__( @@ -119,6 +124,7 @@ def __init__( dependence: Optional[Dict] = None, wait_start_timeout: Optional[Dict] = None, condition_result: Optional[Dict] = None, + resource_plugin: Optional[ResourcePlugin] = None, ): super().__init__(name, description) @@ -159,6 +165,8 @@ def __init__( self.dependence = dependence or {} self.wait_start_timeout = wait_start_timeout or {} self._condition_result = condition_result or self.DEFAULT_CONDITION_RESULT + self.resource_plugin = resource_plugin + self.get_content() @property def process_definition(self) -> Optional[ProcessDefinition]: @@ -196,6 +204,24 @@ def task_params(self) -> Optional[Dict]: custom_attr |= self._task_custom_attr return self.get_define_custom(custom_attr=custom_attr) + def get_content(self): + """Get the file content according to the resource plugin""" + _ext_attr = getattr(self, str(self.ext_attr)) + if _ext_attr is not None: + if self.resource_plugin is None: + if self.process_definition.resources_plugin is not None: + res = self.process_definition.resource_plugin.resource + else: + return + else: + res = self.resource_plugin.resource + print(res) + + content = res.read_file(self.ext_attr) + setattr(self, _ext_attr[1:], content) + else: + raise ValueError('ext_attr is None') + def __hash__(self): return hash(self.code) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py index e69de29bb2d1..5d70f61875fd 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py @@ -0,0 +1,47 @@ +import importlib +import os + +from pydolphinscheduler.resources_plugin.local import Local + + +class ResourcePlugin: + + def __init__(self, type: str, prefix: str): + self._type = type + self._prefix = prefix + + def get_modules(self, package="."): + modules = [] + files = os.listdir(package) + for file in files: + if not file.startswith("__"): + name, ext = os.path.splitext(file) + modules.append(name) + return modules + + def import_module(self, script_name, script_path): + """Import module""" + spec = importlib.util.spec_from_file_location(script_name, script_path) + module = importlib.util.module_from_spec(spec) + setattr(module, self._type, self._type) + setattr(module, self._prefix, self._prefix) + spec.loader.exec_module(module) + return module + + @property + def resource(self): + # all_resource = importlib.import_module("") + # if self.type not in [str(i) for i in all_resource]: + # raise ValueError() + pwd = os.path.abspath(__file__) + parent_path = os.path.abspath(os.path.dirname(pwd) + os.path.sep + ".") + modules = self.get_modules(parent_path) + print(modules) + if self._type not in [module for module in modules]: + raise ValueError('{} type is not supported'.format(self._type)) + + script_name = self._type + '.py' + script_path = parent_path + '/' + script_name + + res = self.import_module(script_name, script_path) + return res diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/github.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/github.py index 3e650ff4242e..564ad4f83bf9 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/github.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/github.py @@ -3,16 +3,4 @@ class Github: def __init__(self, path: str): self.path = path - def __call__(self, func): - def wrapper(): - print("github_resource __call__") - r = func() - return wrapper - -def func(): - printf("hello") - - -if __name__ == "__main__": - func() diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/test.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/test.py index e69de29bb2d1..6d479726b92c 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/test.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/test.py @@ -0,0 +1,21 @@ +import importlib + + +def import_module(script_name, script_path): + """Import module""" + spec = importlib.util.spec_from_file_location(script_name, script_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +""""Debug only""" +if __name__ == "__main__": + # res = import_module('shell.py', '/home/chenrj/gitrep/dolphinscheduler/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/tasks/shell.py') + # print(res) + + # res = Local("/opt") + # print(res.prefix) + + res = importlib.import_module("local") + print(res) diff --git a/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml b/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml index d6ef76fa9eb7..a16188288b2a 100644 --- a/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml +++ b/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml @@ -73,11 +73,12 @@ - + + diff --git a/dolphinscheduler-worker/src/main/resources/application.yaml b/dolphinscheduler-worker/src/main/resources/application.yaml index f0be99cef139..78b722b279e2 100644 --- a/dolphinscheduler-worker/src/main/resources/application.yaml +++ b/dolphinscheduler-worker/src/main/resources/application.yaml @@ -89,3 +89,14 @@ management: metrics: enabled: true + +--- +spring: + config: + activate: + on-profile: mysql + datasource: + driver-class-name: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://127.0.0.1:3306/chenrj?useUnicode=true&characterEncoding=UTF-8 + username: root + password: 4295 \ No newline at end of file diff --git a/dolphinscheduler-worker/src/main/resources/logback-spring.xml b/dolphinscheduler-worker/src/main/resources/logback-spring.xml index c6d8ee415d96..27d5cc44c083 100644 --- a/dolphinscheduler-worker/src/main/resources/logback-spring.xml +++ b/dolphinscheduler-worker/src/main/resources/logback-spring.xml @@ -65,11 +65,12 @@ - - - - - + + + + + + diff --git a/pom.xml b/pom.xml index 9522d5eb860e..e650d6bda890 100644 --- a/pom.xml +++ b/pom.xml @@ -554,7 +554,7 @@ mysql mysql-connector-java ${mysql.connector.version} - test + compile com.h2database From 1d7b747430db84c7acad437c9b746988bf7c060c Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Thu, 4 Aug 2022 20:10:02 +0800 Subject: [PATCH 05/58] zancun --- .../src/pydolphinscheduler/core/task.py | 54 ++++++--- .../pydolphinscheduler/examples/tutorial.py | 8 +- .../src/pydolphinscheduler/exceptions.py | 4 + .../resources_plugin/__init__.py | 13 +- .../resources_plugin/local.py | 11 +- .../src/pydolphinscheduler/tasks/shell.py | 5 +- .../tests/core/test_task.py | 98 ++++++++++----- .../tests/resources_plugin/test_init.py | 112 ++++++++++++++++++ .../tests/resources_plugin/test_local.py | 55 +++++++++ .../tests/tasks/test_shell.py | 48 +++++++- .../tests/utils/test_file.py | 2 + 11 files changed, 340 insertions(+), 70 deletions(-) create mode 100644 dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py create mode 100644 dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py index 9fcf603a276b..ed7d4ac100a6 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py @@ -25,7 +25,6 @@ TaskFlag, TaskPriority, TaskTimeoutFlag, - ResourcePluginType, ) from pydolphinscheduler.resources_plugin.__init__ import ResourcePlugin from pydolphinscheduler.core import configuration @@ -99,7 +98,7 @@ class Task(Base): _task_custom_attr: set = set() - ext = None + ext: set = None ext_attr: str = None DEFAULT_CONDITION_RESULT = {"successNode": [""], "failedNode": [""]} @@ -204,23 +203,48 @@ def task_params(self) -> Optional[Dict]: custom_attr |= self._task_custom_attr return self.get_define_custom(custom_attr=custom_attr) + def get_res(self): + """ """ + if self.resource_plugin is None: + if self.process_definition.resource_plugin is not None: + res = self.process_definition.resource_plugin.resource + else: + raise ValueError("") + else: + res = self.resource_plugin.resource + return res + + def get_content(self): """Get the file content according to the resource plugin""" - _ext_attr = getattr(self, str(self.ext_attr)) + if self.ext_attr is None: + raise ValueError('ext_attr is None') + if self.ext is None: + raise ValueError('ext is None') + + _ext_attr = getattr(self, self.ext_attr) + if _ext_attr is not None: - if self.resource_plugin is None: - if self.process_definition.resources_plugin is not None: - res = self.process_definition.resource_plugin.resource + if _ext_attr.endswith(tuple(self.ext)): + + # res = self.get_res() + + if self.resource_plugin is None: + if self.process_definition.resource_plugin is not None: + res = self.process_definition.resource_plugin.resource + else: + return else: - return - else: - res = self.resource_plugin.resource - print(res) + res = self.resource_plugin.resource - content = res.read_file(self.ext_attr) - setattr(self, _ext_attr[1:], content) - else: - raise ValueError('ext_attr is None') + content = res.read_file(_ext_attr) + setattr(self, self.ext_attr[1:], content) + else: + index = _ext_attr.rfind('.') + if index != -1: + raise ValueError('This task does not support files with suffix {}, only supports {}'.format(_ext_attr[index:], ",".join(str(suf) for suf in self.ext))) + setattr(self, self.ext_attr[1:], _ext_attr) + # print(getattr(self, self.ext_attr[1:])) def __hash__(self): return hash(self.code) @@ -246,7 +270,7 @@ def __rlshift__(self, other: Union["Task", Sequence["Task"]]): return self def _set_deps( - self, tasks: Union["Task", Sequence["Task"]], upstream: bool = True + self, tasks: Union["Task", Sequence["Task"]], upstream: bool = True ) -> None: """ Set parameter tasks dependent to current task. diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py index 2a91245513bc..97482d65cdd2 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py @@ -57,7 +57,7 @@ # [start task_declare] task_parent = Shell( name="task_parent", - command="parent.sh", + command="parent.zsh", resource_plugin=ResourcePlugin( type=ResourcePluginType.LOCAL, prefix='/opt/', @@ -83,11 +83,7 @@ ) task_union = Shell( name="task_union", - command="union.sh", - resource_plugin=ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix='/opt/', - ) + command="echo union1 ab", ) # [end task_declare] diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/exceptions.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/exceptions.py index 4d70a5863763..5b0d1bb61f35 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/exceptions.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/exceptions.py @@ -40,3 +40,7 @@ class PyDSProcessDefinitionNotAssignException(PyDSBaseException): class PyDSConfException(PyDSBaseException): """Exception for pydolphinscheduler configuration error.""" + + +class PyResPluginException(PyDSBaseException): + """Exception for pydolphinscheduler resource plugin error.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py index 5d70f61875fd..ee18ad6eac07 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py @@ -1,7 +1,6 @@ import importlib import os - -from pydolphinscheduler.resources_plugin.local import Local +import importlib.util class ResourcePlugin: @@ -11,6 +10,7 @@ def __init__(self, type: str, prefix: str): self._prefix = prefix def get_modules(self, package="."): + """Get the names of all modules under a package""" modules = [] files = os.listdir(package) for file in files: @@ -23,20 +23,15 @@ def import_module(self, script_name, script_path): """Import module""" spec = importlib.util.spec_from_file_location(script_name, script_path) module = importlib.util.module_from_spec(spec) - setattr(module, self._type, self._type) - setattr(module, self._prefix, self._prefix) spec.loader.exec_module(module) - return module + plugin = getattr(module, self._type.capitalize()) + return plugin(self._prefix) @property def resource(self): - # all_resource = importlib.import_module("") - # if self.type not in [str(i) for i in all_resource]: - # raise ValueError() pwd = os.path.abspath(__file__) parent_path = os.path.abspath(os.path.dirname(pwd) + os.path.sep + ".") modules = self.get_modules(parent_path) - print(modules) if self._type not in [module for module in modules]: raise ValueError('{} type is not supported'.format(self._type)) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py index 11f186b25ad1..575863e5bbb3 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py @@ -1,6 +1,3 @@ -import os.path - - class Local: def __init__(self, prefix: str): @@ -15,10 +12,12 @@ def read_file(self, suf: str): f = open(self._prefix + suf, 'r') content = f.read() f.close() + return content except FileNotFoundError: - print("{} is not found.".format(self.prefix + suf)) + print("{} is not found".format(self.prefix + suf)) except PermissionError: - print("You don't have permission to access {}.".format(self.prefix + suf)) - return content + print("You don't have permission to access {}".format(self.prefix + suf)) + except Exception: + print("Unknown exception") diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/tasks/shell.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/tasks/shell.py index b253b7d0b8f3..36ec4e87d0da 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/tasks/shell.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/tasks/shell.py @@ -50,10 +50,9 @@ class Shell(Task): "raw_script", } - ext = ".sh" - ext_attr = "_raw_script" + ext: set = {".sh", ".zsh"} + ext_attr: str = "_raw_script" def __init__(self, name: str, command: str, *args, **kwargs): self._raw_script = command super().__init__(name, TaskType.SHELL, *args, **kwargs) - self.raw_script = command diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py index 7d4bbebdd33a..3fbc252b8795 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py @@ -18,9 +18,10 @@ """Test Task class function.""" import logging import re -from unittest.mock import patch +from unittest.mock import patch, PropertyMock import pytest +from pydolphinscheduler.constants import TaskType from pydolphinscheduler.core.process_definition import ProcessDefinition from pydolphinscheduler.core.task import Task, TaskRelation @@ -35,30 +36,30 @@ "attr, expect", [ ( - dict(), - { - "localParams": [], - "resourceList": [], - "dependence": {}, - "waitStartTimeout": {}, - "conditionResult": {"successNode": [""], "failedNode": [""]}, - }, + dict(), + { + "localParams": [], + "resourceList": [], + "dependence": {}, + "waitStartTimeout": {}, + "conditionResult": {"successNode": [""], "failedNode": [""]}, + }, ), ( - { - "local_params": ["foo", "bar"], - "resource_list": ["foo", "bar"], - "dependence": {"foo", "bar"}, - "wait_start_timeout": {"foo", "bar"}, - "condition_result": {"foo": ["bar"]}, - }, - { - "localParams": ["foo", "bar"], - "resourceList": ["foo", "bar"], - "dependence": {"foo", "bar"}, - "waitStartTimeout": {"foo", "bar"}, - "conditionResult": {"foo": ["bar"]}, - }, + { + "local_params": ["foo", "bar"], + "resource_list": ["foo", "bar"], + "dependence": {"foo", "bar"}, + "wait_start_timeout": {"foo", "bar"}, + "condition_result": {"foo": ["bar"]}, + }, + { + "localParams": ["foo", "bar"], + "resourceList": ["foo", "bar"], + "dependence": {"foo", "bar"}, + "waitStartTimeout": {"foo", "bar"}, + "conditionResult": {"foo": ["bar"]}, + }, ), ], ) @@ -160,8 +161,8 @@ def test_task_get_define(): "timeout": 0, } with patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(code, version), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(code, version), ): task = Task(name=name, task_type=task_type) assert task.get_define() == expect @@ -182,12 +183,12 @@ def test_two_tasks_shift(shift: str): else: assert False, f"Unexpect bit operator type {shift}." assert ( - 1 == len(upstream._downstream_task_codes) - and downstream.code in upstream._downstream_task_codes + 1 == len(upstream._downstream_task_codes) + and downstream.code in upstream._downstream_task_codes ), "Task downstream task attributes error, downstream codes size or specific code failed." assert ( - 1 == len(downstream._upstream_task_codes) - and upstream.code in downstream._upstream_task_codes + 1 == len(downstream._upstream_task_codes) + and upstream.code in downstream._upstream_task_codes ), "Task upstream task attributes error, upstream codes size or upstream code failed." @@ -241,3 +242,42 @@ def test_add_duplicate(caplog): re.findall("already in process definition", caplog.text), ] ) + +@pytest.mark.parametrize( + "val, expected", + [ + ("a.sh", "echo test command content"), + ("a.zsh", "echo test command content"), + # ("echo test command content", "echo test command content"), + ], +) +@patch( + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), +) +@patch( + "pydolphinscheduler.core.task.Task.ext", + new_callable=PropertyMock, + return_value={".sh"}, +) +@patch( + "pydolphinscheduler.core.task.Task.ext_attr", + new_callable=PropertyMock, + return_value="_raw_script", +) +@patch( + "pydolphinscheduler.core.task.Task.get_res", +) +def test_task_ext_attr(mock_res, mock_ext_attr, mock_ext, mock_code_version, val, expected): + """Test task shell task ext_attr.""" + with patch( + "pydolphinscheduler.core.task.Task._raw_script", + new_callable=PropertyMock, + create=True, + return_value=val, + ): + mock_res.return_value.read_file = expected + task = Task("test task ext_attr", TaskType.SHELL) + # assert expected == getattr(task, "raw_script") + + diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py new file mode 100644 index 000000000000..4c62018702a7 --- /dev/null +++ b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py @@ -0,0 +1,112 @@ +import os +import shutil + +import pytest +from pydolphinscheduler.constants import ResourcePluginType +from pydolphinscheduler.resources_plugin import ResourcePlugin +from pydolphinscheduler.exceptions import PyResPluginException + +modules_dir = 'modules' +modules_names = ["a", "b", "c"] +# res_plugin_prefix = Path(__file__).absolute().parent +pwd = os.path.abspath(__file__) +cur_path = os.path.abspath(os.path.dirname(pwd) + os.path.sep + ".") + "/" +modules_path = cur_path + modules_dir + + +@pytest.fixture +def setup_crt_first(): + """Set up and teardown about create folder first and then delete it.""" + os.mkdir(modules_dir) + yield + shutil.rmtree(modules_dir) + + +@pytest.fixture +def create_modules(setup_crt_first): + """Temporarily create an empty module based on module_names.""" + for modules_name in modules_names: + open(modules_path + "/" + "%s.py" % modules_name, "w") + yield + + +@pytest.mark.parametrize( + "attr, expect", + [ + ( + { + "type": ResourcePluginType.LOCAL, + "prefix": "/tmp/", + }, + modules_names + ) + ], +) +def test_resources_get_modules_name(attr, expect, create_modules): + """Test resource plugin to get all model names under a package""" + res = ResourcePlugin(**attr) + assert expect == res.get_modules(modules_path) + + +@pytest.mark.parametrize( + "attr, expect", + [ + ( + { + "type": ResourcePluginType.LOCAL, + "prefix": "/tmp/", + }, + "Local", + ) + ], +) +def test_resources_import_modules(attr, expect): + """Test resource plug-in to import model""" + res_plugin = ResourcePlugin(**attr) + script_name = "local.py" + script_path = "/home/chenrj/gitrep/dolphinscheduler/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py" + res = res_plugin.import_module(script_name, script_path) + assert expect == res.__class__.__name__ + + +@pytest.mark.parametrize( + "attr, expect", + [ + ( + { + "type": ResourcePluginType.LOCAL, + "prefix": "/tmp/", + }, + "Local", + ) + ], +) +def test_resources_resources(attr, expect): + """Test the factory mode of the resource plugin, and return the corresponding plugin according to the plugin type""" + res_plugin = ResourcePlugin(**attr) + res = res_plugin.resource + assert expect == res.__class__.__name__ + + +@pytest.mark.parametrize( + "attr", + [ + { + "type": "a", + "prefix": "/tmp/", + } + ], +) +def test_resources_unsupported_res(attr): + """Test unsupported plug-ins""" + with pytest.raises( + ValueError, match="{} type is not supported".format(attr.get("type")) + ): + res_plugin = ResourcePlugin(**attr) + res_plugin.resource + + + + + + diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py new file mode 100644 index 000000000000..69d37027abca --- /dev/null +++ b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py @@ -0,0 +1,55 @@ +import os + +import pytest + +from pydolphinscheduler.constants import ResourcePluginType +from pydolphinscheduler.resources_plugin import ResourcePlugin +from unittest.mock import patch +from pydolphinscheduler.tasks.shell import Shell +from pydolphinscheduler.utils import file +from tests.testing.file import delete_file + +file_path = 'local_res.sh' +file_content = "echo \"test res_local\"" +# res_plugin_prefix = Path(__file__).absolute().parent +pwd = os.path.abspath(__file__) +res_plugin_prefix = os.path.abspath(os.path.dirname(pwd) + os.path.sep + ".") + "/" + +@pytest.fixture +def setup_crt_first(): + """Set up and teardown about create file first and then delete it.""" + file.write(content=file_content, to_path=file_path) + yield + delete_file(file_path) + +@pytest.mark.parametrize( + "attr, expect", + [ + ( + { + "name": "test-local-res-command-content", + "command": file_path, + "resource_plugin": ResourcePlugin( + type=ResourcePluginType.LOCAL, + prefix=res_plugin_prefix, + ) + }, + file_content + ) + ], +) +@patch( + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), +) +def test_resources_local_shell_command_content(mock_code_version, attr, expect, setup_crt_first): + """Test task shell task command content.""" + task = Shell(**attr) + assert expect == getattr(task, "raw_script") + + + + + + + diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py index e42f6dc0fb9d..1151a69f10e9 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py @@ -17,12 +17,22 @@ """Test Task shell.""" - +import os from unittest.mock import patch import pytest - +from pydolphinscheduler.constants import ResourcePluginType +from pydolphinscheduler.resources_plugin import ResourcePlugin from pydolphinscheduler.tasks.shell import Shell +from pydolphinscheduler.utils import file + +from tests.testing.file import delete_file + +## 路径file +file_path = 'local_res.sh' +file_content = "echo \"test res_local\"" +pwd = os.path.abspath(__file__) +res_plugin_prefix = os.path.abspath(os.path.dirname(pwd) + os.path.sep + ".") + "/" @pytest.mark.parametrize( @@ -87,3 +97,37 @@ def test_shell_get_define(): ): shell = Shell(name, command) assert shell.get_define() == expect + + +@pytest.fixture +def setup_crt_first(): + """Set up and teardown about create file first and then delete it.""" + file.write(content=file_content, to_path=file_path) + yield + delete_file(file_path) + + +@pytest.mark.parametrize( + "attr, expect", + [ + ( + { + "name": "test-local-res-command-content", + "command": file_path, + "resource_plugin": ResourcePlugin( + type=ResourcePluginType.LOCAL, + prefix=res_plugin_prefix + ) + }, + file_content + ) + ], +) +@patch( + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), +) +def test_resources_local_shell_command_content(mock_code_version, attr, expect, setup_crt_first): + """Test task shell task command content through the local resource plug-in.""" + task = Shell(**attr) + assert expect == getattr(task, "raw_script") diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_file.py b/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_file.py index 4cc6df402f87..91f6ede8ce1d 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_file.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_file.py @@ -58,6 +58,7 @@ def test_write_not_create_parent(teardown_del_file): if file_test_dir.exists(): shutil.rmtree(str(file_test_dir)) assert not file_test_dir.exists() + # with pytest.raises( ValueError, match="Parent directory do not exists and set param `create` to `False`", @@ -79,6 +80,7 @@ def test_write_overwrite_error(setup_crt_first): assert Path(file_path).exists() new_content = f"new_{content}" + # with pytest.raises( FileExistsError, match=".*already exists and you choose not overwrite mode\\." ): From 6d68f9c646cfc70e565d27b8eb58618cd059844c Mon Sep 17 00:00:00 2001 From: Jiajie Zhong Date: Thu, 4 Aug 2022 20:50:19 +0800 Subject: [PATCH 06/58] fix test error --- .../src/pydolphinscheduler/core/task.py | 25 +++++++++---------- .../tests/core/test_task.py | 23 ++++++----------- .../tests/resources_plugin/__init__.py | 0 3 files changed, 20 insertions(+), 28 deletions(-) create mode 100644 dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/__init__.py diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py index ed7d4ac100a6..da35e4d368e5 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py @@ -214,6 +214,15 @@ def get_res(self): res = self.resource_plugin.resource return res + def get_plugin(self): + if self.resource_plugin is None: + if self.process_definition.resource_plugin is not None: + return self.process_definition.resource_plugin.resource + else: + raise ValueError + else: + return self.resource_plugin.resource + def get_content(self): """Get the file content according to the resource plugin""" @@ -226,24 +235,14 @@ def get_content(self): if _ext_attr is not None: if _ext_attr.endswith(tuple(self.ext)): - - # res = self.get_res() - - if self.resource_plugin is None: - if self.process_definition.resource_plugin is not None: - res = self.process_definition.resource_plugin.resource - else: - return - else: - res = self.resource_plugin.resource - + res = self.get_plugin() content = res.read_file(_ext_attr) - setattr(self, self.ext_attr[1:], content) + setattr(self, self.ext_attr.lstrip("_"), content) else: index = _ext_attr.rfind('.') if index != -1: raise ValueError('This task does not support files with suffix {}, only supports {}'.format(_ext_attr[index:], ",".join(str(suf) for suf in self.ext))) - setattr(self, self.ext_attr[1:], _ext_attr) + setattr(self, self.ext_attr.lstrip("_"), _ext_attr) # print(getattr(self, self.ext_attr[1:])) def __hash__(self): diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py index 3fbc252b8795..3a6dedf4916c 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py @@ -247,8 +247,7 @@ def test_add_duplicate(caplog): "val, expected", [ ("a.sh", "echo test command content"), - ("a.zsh", "echo test command content"), - # ("echo test command content", "echo test command content"), + ("echo test command content", "echo test command content"), ], ) @patch( @@ -265,19 +264,13 @@ def test_add_duplicate(caplog): new_callable=PropertyMock, return_value="_raw_script", ) -@patch( - "pydolphinscheduler.core.task.Task.get_res", -) -def test_task_ext_attr(mock_res, mock_ext_attr, mock_ext, mock_code_version, val, expected): +@patch("pydolphinscheduler.core.task.Task._raw_script", create=True, new_callable=PropertyMock,) +@patch("pydolphinscheduler.core.task.Task.get_plugin") +def test_task_ext_attr(m_plugin, m_raw_script, m_ext_attr, m_ext, m_code_version, val, expected): """Test task shell task ext_attr.""" - with patch( - "pydolphinscheduler.core.task.Task._raw_script", - new_callable=PropertyMock, - create=True, - return_value=val, - ): - mock_res.return_value.read_file = expected - task = Task("test task ext_attr", TaskType.SHELL) - # assert expected == getattr(task, "raw_script") + m_plugin.return_value.read_file.return_value = expected + m_raw_script.retrun_value = val + task = Task("test_task_ext_attr", "test_task_ext_attr") + assert expected == getattr(task, "raw_script") diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/__init__.py b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 From 3b8697a501133edb3766a7d724bde2633541951a Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Mon, 8 Aug 2022 19:11:21 +0800 Subject: [PATCH 07/58] local res --- .../pydolphinscheduler/docs/source/index.rst | 1 + .../resources_plugin/how-to-develop.rst | 38 ++++++++++++++ .../source/resources_plugin/how-to-use.rst | 35 +++++++++++++ .../docs/source/resources_plugin/index.rst | 31 +++++++++++ .../docs/source/resources_plugin/local.rst | 32 ++++++++++++ .../resources_plugin/resource-plugin.rst | 51 +++++++++++++++++++ .../resources_plugin/test.py | 21 -------- 7 files changed, 188 insertions(+), 21 deletions(-) create mode 100644 dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/how-to-develop.rst create mode 100644 dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/how-to-use.rst create mode 100644 dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/index.rst create mode 100644 dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/local.rst create mode 100644 dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/resource-plugin.rst delete mode 100644 dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/test.py diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/index.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/index.rst index 24ad107ad6dc..4dc0a949c90c 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/source/index.rst +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/index.rst @@ -36,6 +36,7 @@ then go and see :doc:`tutorial` for more detail. cli config api + resources_plugin/index Indices and tables ================== diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/how-to-develop.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/how-to-develop.rst new file mode 100644 index 000000000000..e68ba6d446d2 --- /dev/null +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/how-to-develop.rst @@ -0,0 +1,38 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +How to develop +============== + +How to develop a plugin +You need your plugin class created under dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin folder. + +Your plugin class needs two indispensable methods, one is the __init__ method, the parameter is the prefix of type str, the other is +the read_file function, the parameter is the file suffix of type str, the return value is the file content, if it is exists and is readable. + +In addition you need to add a constant with your plugin name to ResourcePluginType in dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py, eg LOCAL = "local". + +Example +------- +.. literalinclude:: ../../../src/pydolphinscheduler/resources_plugin/local.py + :start-after: [start local_res_definition] + :end-before: [end local_res_definition] + +.. literalinclude:: ../../../src/pydolphinscheduler/constants.py + :start-after: [start res_plugin_constants_definition] + :end-before: [end res_plugin_constants_definition] + diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/how-to-use.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/how-to-use.rst new file mode 100644 index 000000000000..79c9f65954c5 --- /dev/null +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/how-to-use.rst @@ -0,0 +1,35 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +How to use +========== +This article will show you how to use resource plug-ins. + +When you initialize the task subclass, you can add the parameter resource_plugin parameter, specify the type and +prefix of the resource plugin, the command at this time should be the file path, in other words, the prefix of the +resource plugin plus the command at this time is the command file in the specified Absolute paths in resource plugins. + +When the resource_plugin parameter is defined in both the task subclass and the workflow, the resource_plugin defined in the task subclass is used first. If the task subclass does not define resource_plugin, but the resource_plugin is defined in the workflow, the resource_plugin in the workflow is used. + +Of course, if neither the task subclass nor the workflow specifies resource_plugin, the command at this time will be executed as a script, in other words, we are forward compatible. + + +Example +------- +.. literalinclude:: ../../../src/pydolphinscheduler/examples/tutorial.py + :start-after: [start workflow_declare] + :end-before: [end task_relation_declare] diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/index.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/index.rst new file mode 100644 index 000000000000..c3437f9c9586 --- /dev/null +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/index.rst @@ -0,0 +1,31 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Resources_plugin +================ + +In this section + +.. toctree:: + :maxdepth: 1 + + how-to-use + how-to-develop + resource-plugin + local + + diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/local.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/local.rst new file mode 100644 index 000000000000..171b056bfea4 --- /dev/null +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/local.rst @@ -0,0 +1,32 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Local +===== + +Local is a local resource plugin for pydolphinscheduler. + +When using a local resource plugin, you do not need to use this class explicitly, you only need to add the +resource_plugin parameter in the task subclass or workflow definition. The resource_plugin parameter type is +ResourcePlugin. + +The use of specific resource plugins can be viewed: :doc:`./how-to-use` + +Dive Into +--------- + +.. automodule:: pydolphinscheduler.resources_plugin.local \ No newline at end of file diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/resource-plugin.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/resource-plugin.rst new file mode 100644 index 000000000000..24e3b233d31d --- /dev/null +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/resource-plugin.rst @@ -0,0 +1,51 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +ResourcePlugin +============== + +ResourcePlugin is the data type of the resources plugin parameter in task subclasses and workflow definitions. + + +Code +---- +.. literalinclude:: ../../../src/pydolphinscheduler/resources_plugin/__init__.py + :start-after: [start resource_plugin_definition] + :end-before: [end resource_plugin_definition] + +Dive Into +--------- +It has the following key functions. + +The __init__ function requires parameters. The first is the type of str type, which means the plugin type of the resource, +and the second is the prefix of the str type, which means the prefix of the resource. + +The get_all_modules function will return the absolute path of all resource plugins defined in the resource_plugin file. + +The import_module function has two parameters, script_name and script_path. +script_name is the name of the resource file without the suffix, such as the local resource plugin class local.py, its +script_name is local, and script_path is the absolute path of the resource plugin file + +The resource function will dynamically return the resource plugin object according to the type parameter of the __intit__ function, +for example, the type is ResourcePluginType.LOCAL, then the local plugin object is returned + + +.. automodule:: pydolphinscheduler.resources_plugin.__init__ + + + + diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/test.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/test.py deleted file mode 100644 index 6d479726b92c..000000000000 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/test.py +++ /dev/null @@ -1,21 +0,0 @@ -import importlib - - -def import_module(script_name, script_path): - """Import module""" - spec = importlib.util.spec_from_file_location(script_name, script_path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -""""Debug only""" -if __name__ == "__main__": - # res = import_module('shell.py', '/home/chenrj/gitrep/dolphinscheduler/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/tasks/shell.py') - # print(res) - - # res = Local("/opt") - # print(res.prefix) - - res = importlib.import_module("local") - print(res) From d5150e6462726023628396443e6b72305e30bb32 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Mon, 8 Aug 2022 19:17:23 +0800 Subject: [PATCH 08/58] local_res --- .../src/pydolphinscheduler/constants.py | 5 +- .../src/pydolphinscheduler/core/task.py | 28 ++-- .../resources_plugin/__init__.py | 49 ++++--- .../resources_plugin/gitlab.py | 3 + .../resources_plugin/local.py | 28 ++-- .../tests/core/test_task.py | 118 ++++++++++++++--- .../tests/resources_plugin/test_init.py | 114 ++++++++--------- .../tests/resources_plugin/test_local.py | 120 ++++++++++++++---- .../tests/tasks/test_shell.py | 22 ++-- 9 files changed, 312 insertions(+), 175 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py index 41358f911e25..d23197f2d3c8 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py @@ -108,10 +108,13 @@ class Time(str): FMT_STD_TIME = "%H:%M:%S" FMT_NO_COLON_TIME = "%H%M%S" + +# [start res_plugin_constants_definition] class ResourcePluginType(str): """Constants for resources plugin type, it will also show you which kind we support up to now.""" LOCAL = "local" GITHUB = "github" GITLAB = "gitlab" S3 = "S3" - OSS = "OSS" \ No newline at end of file + OSS = "OSS" +# [end res_plugin_constants_definition] diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py index ed7d4ac100a6..a0062dbefbe3 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py @@ -26,6 +26,7 @@ TaskPriority, TaskTimeoutFlag, ) +from pydolphinscheduler.exceptions import PyResPluginException from pydolphinscheduler.resources_plugin.__init__ import ResourcePlugin from pydolphinscheduler.core import configuration from pydolphinscheduler.core.base import Base @@ -203,16 +204,14 @@ def task_params(self) -> Optional[Dict]: custom_attr |= self._task_custom_attr return self.get_define_custom(custom_attr=custom_attr) - def get_res(self): - """ """ + def get_plugin(self): if self.resource_plugin is None: if self.process_definition.resource_plugin is not None: - res = self.process_definition.resource_plugin.resource + return self.process_definition.resource_plugin.resource else: - raise ValueError("") + raise PyResPluginException("The execution command of this task is a file, but the resource plugin is empty") else: - res = self.resource_plugin.resource - return res + return self.resource_plugin.resource def get_content(self): @@ -226,25 +225,14 @@ def get_content(self): if _ext_attr is not None: if _ext_attr.endswith(tuple(self.ext)): - - # res = self.get_res() - - if self.resource_plugin is None: - if self.process_definition.resource_plugin is not None: - res = self.process_definition.resource_plugin.resource - else: - return - else: - res = self.resource_plugin.resource - + res = self.get_plugin() content = res.read_file(_ext_attr) - setattr(self, self.ext_attr[1:], content) + setattr(self, self.ext_attr.lstrip("_"), content) else: index = _ext_attr.rfind('.') if index != -1: raise ValueError('This task does not support files with suffix {}, only supports {}'.format(_ext_attr[index:], ",".join(str(suf) for suf in self.ext))) - setattr(self, self.ext_attr[1:], _ext_attr) - # print(getattr(self, self.ext_attr[1:])) + setattr(self, self.ext_attr.lstrip("_"), _ext_attr) def __hash__(self): return hash(self.code) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py index ee18ad6eac07..522ed08da500 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py @@ -1,42 +1,37 @@ import importlib -import os import importlib.util +from pathlib import Path +from typing import Any, Generator -class ResourcePlugin: +from pydolphinscheduler.exceptions import PyDSConfException + +path_resources_plugin = Path(__file__).parent + +# [start resource_plugin_definition] +class ResourcePlugin: def __init__(self, type: str, prefix: str): - self._type = type - self._prefix = prefix - - def get_modules(self, package="."): - """Get the names of all modules under a package""" - modules = [] - files = os.listdir(package) - for file in files: - if not file.startswith("__"): - name, ext = os.path.splitext(file) - modules.append(name) - return modules + self.type = type + self.prefix = prefix + + def get_all_modules(self) -> Generator[Path, Any, None]: + """Get all res files path in resources_plugin directory.""" + return (ex for ex in path_resources_plugin.iterdir() if ex.is_file() and not ex.name.startswith("__")) def import_module(self, script_name, script_path): """Import module""" spec = importlib.util.spec_from_file_location(script_name, script_path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) - plugin = getattr(module, self._type.capitalize()) - return plugin(self._prefix) + plugin = getattr(module, self.type.capitalize()) + return plugin(self.prefix) @property def resource(self): - pwd = os.path.abspath(__file__) - parent_path = os.path.abspath(os.path.dirname(pwd) + os.path.sep + ".") - modules = self.get_modules(parent_path) - if self._type not in [module for module in modules]: - raise ValueError('{} type is not supported'.format(self._type)) - - script_name = self._type + '.py' - script_path = parent_path + '/' + script_name - - res = self.import_module(script_name, script_path) - return res + """Dynamically return resource plugin""" + for ex in self.get_all_modules(): + if ex.stem == self.type: + return self.import_module(ex.name, str(ex)) + raise PyDSConfException('{} type is not supported'.format(self.type)) +# [end resource_plugin_definition] diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/gitlab.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/gitlab.py index e69de29bb2d1..5b8dca91be96 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/gitlab.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/gitlab.py @@ -0,0 +1,3 @@ +class Gitlab: + def __init__(self, path: str): + self.path = path diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py index 575863e5bbb3..f4d15fd65602 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py @@ -1,3 +1,9 @@ +import os +from pathlib import Path +from pydolphinscheduler.exceptions import PyResPluginException + + +# [start local_res_definition] class Local: def __init__(self, prefix: str): @@ -5,19 +11,19 @@ def __init__(self, prefix: str): @property def prefix(self): + """Get the _prefix attribute""" return self._prefix def read_file(self, suf: str): - try: - f = open(self._prefix + suf, 'r') + """Get the content of the file, the address of the file is + the prefix of the resource plugin plus the parameter suf """ + path = Path(self.prefix).joinpath(suf) + if not path.exists(): + raise PyResPluginException("{} is not found".format(str(path))) + if not os.access(str(path), os.R_OK): + raise PyResPluginException("You don't have permission to access {}".format(self.prefix + suf)) + with open(path, 'r') as f: content = f.read() - f.close() - return content - except FileNotFoundError: - print("{} is not found".format(self.prefix + suf)) - except PermissionError: - print("You don't have permission to access {}".format(self.prefix + suf)) - except Exception: - print("Unknown exception") - + return content +# [end local_res_definition] diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py index 3fbc252b8795..967621048d8d 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py @@ -21,10 +21,12 @@ from unittest.mock import patch, PropertyMock import pytest -from pydolphinscheduler.constants import TaskType +from pydolphinscheduler.constants import TaskType, ResourcePluginType from pydolphinscheduler.core.process_definition import ProcessDefinition from pydolphinscheduler.core.task import Task, TaskRelation +from pydolphinscheduler.exceptions import PyResPluginException +from pydolphinscheduler.resources_plugin import ResourcePlugin from tests.testing.task import Task as testTask from tests.testing.task import TaskWithCode @@ -243,12 +245,13 @@ def test_add_duplicate(caplog): ] ) + @pytest.mark.parametrize( "val, expected", [ - ("a.sh", "echo test command content"), - ("a.zsh", "echo test command content"), - # ("echo test command content", "echo test command content"), + ("a.sh", "echo Test task attribute ext_attr"), + ("a.zsh", "echo Test task attribute ext_attr"), + ("echo Test task attribute ext_attr", "echo Test task attribute ext_attr"), ], ) @patch( @@ -258,7 +261,7 @@ def test_add_duplicate(caplog): @patch( "pydolphinscheduler.core.task.Task.ext", new_callable=PropertyMock, - return_value={".sh"}, + return_value={".sh", ".zsh"}, ) @patch( "pydolphinscheduler.core.task.Task.ext_attr", @@ -266,18 +269,99 @@ def test_add_duplicate(caplog): return_value="_raw_script", ) @patch( - "pydolphinscheduler.core.task.Task.get_res", + "pydolphinscheduler.core.task.Task._raw_script", + create=True, + new_callable=PropertyMock, ) -def test_task_ext_attr(mock_res, mock_ext_attr, mock_ext, mock_code_version, val, expected): - """Test task shell task ext_attr.""" - with patch( - "pydolphinscheduler.core.task.Task._raw_script", - new_callable=PropertyMock, - create=True, - return_value=val, - ): - mock_res.return_value.read_file = expected - task = Task("test task ext_attr", TaskType.SHELL) - # assert expected == getattr(task, "raw_script") +@patch("pydolphinscheduler.core.task.Task.get_plugin") +def test_task_ext_attr(m_plugin, m_raw_script, m_ext_attr, m_ext, m_code_version, val, expected): + """Test task attribute ext_attr.""" + m_plugin.return_value.read_file.return_value = expected + m_raw_script.return_value = val + task = Task("test_task_ext_attr", "test_task_ext_attr") + assert expected == getattr(task, "raw_script") +@pytest.mark.parametrize( + "attr, expected", + [ + ( + { + "name": "test_task_abtain_res_plugin", + "task_type": "TaskType", + "resource_plugin": ResourcePlugin( + type=ResourcePluginType.LOCAL, + prefix="prefix", + ), + "process_definition": ProcessDefinition( + name="process_definition", + resource_plugin=ResourcePlugin( + type=ResourcePluginType.GITHUB, + prefix="prefix", + ), + ) + }, + "Local" + ), + ( + { + "name": "test_task_abtain_res_plugin", + "task_type": "TaskType", + "resource_plugin": ResourcePlugin( + type=ResourcePluginType.GITLAB, + prefix="prefix", + ), + }, + "Gitlab" + ), + ( + { + "name": "test_task_abtain_res_plugin", + "task_type": "TaskType", + "process_definition": ProcessDefinition( + name="process_definition", + resource_plugin=ResourcePlugin( + type=ResourcePluginType.GITHUB, + prefix="prefix", + ), + ) + }, + "Github" + ), + ], +) +@patch( + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), +) +@patch("pydolphinscheduler.core.task.Task.get_content") +def test_task_obtain_res_plugin(m_get_content, m_code_version, attr, expected): + """Test task obtaining resource plug-in.""" + task = Task(**attr) + assert expected == task.get_plugin().__class__.__name__ + +@pytest.mark.parametrize( + "attr", + [ + { + "name": "test_task_abtain_res_plugin", + "task_type": "TaskType", + "process_definition": ProcessDefinition( + name="process_definition", + ) + }, + ], +) +@patch( + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), +) +@patch("pydolphinscheduler.core.task.Task.get_content") +def test_task_obtain_res_plugin_exception(m_get_content, m_code_version, attr): + """Test task obtaining resource plug-in exception.""" + with pytest.raises( + PyResPluginException, match="The execution command of this task is a file, but the resource plugin is empty" + ): + task = Task(**attr) + task.get_plugin() + diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py index 4c62018702a7..a9f736903b62 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py @@ -1,91 +1,89 @@ -import os -import shutil +from collections import Counter +from pathlib import Path import pytest + from pydolphinscheduler.constants import ResourcePluginType +from pydolphinscheduler.exceptions import PyDSConfException from pydolphinscheduler.resources_plugin import ResourcePlugin -from pydolphinscheduler.exceptions import PyResPluginException - -modules_dir = 'modules' -modules_names = ["a", "b", "c"] -# res_plugin_prefix = Path(__file__).absolute().parent -pwd = os.path.abspath(__file__) -cur_path = os.path.abspath(os.path.dirname(pwd) + os.path.sep + ".") + "/" -modules_path = cur_path + modules_dir - - -@pytest.fixture -def setup_crt_first(): - """Set up and teardown about create folder first and then delete it.""" - os.mkdir(modules_dir) - yield - shutil.rmtree(modules_dir) - -@pytest.fixture -def create_modules(setup_crt_first): - """Temporarily create an empty module based on module_names.""" - for modules_name in modules_names: - open(modules_path + "/" + "%s.py" % modules_name, "w") - yield +all_res = ["local", "github", "gitlab"] +project_root = Path(__file__).parent.parent.parent +resources_plugin_path = project_root.joinpath("src", "pydolphinscheduler", "resources_plugin") @pytest.mark.parametrize( - "attr, expect", + "attr, expected", [ ( { "type": ResourcePluginType.LOCAL, "prefix": "/tmp/", }, - modules_names + all_res ) ], ) -def test_resources_get_modules_name(attr, expect, create_modules): - """Test resource plugin to get all model names under a package""" +def test_resources_get_all_modules(attr, expected): + """Test resource plugin to get all res plugin names""" res = ResourcePlugin(**attr) - assert expect == res.get_modules(modules_path) + assert dict(Counter(expected)) == dict(Counter([ex.stem for ex in res.get_all_modules()])) @pytest.mark.parametrize( - "attr, expect", + "attrs, expected", [ ( { "type": ResourcePluginType.LOCAL, - "prefix": "/tmp/", + "module_attr": { + "script_name": "local.py", + "script_path": resources_plugin_path.joinpath("local.py") + } }, - "Local", - ) + "Local" + ), + ( + { + "type": ResourcePluginType.GITHUB, + "module_attr": { + "script_name": "github.py", + "script_path": resources_plugin_path.joinpath("github.py") + } + }, + "Github" + ), + ( + { + "type": ResourcePluginType.GITLAB, + "module_attr": { + "script_name": "gitlab.py", + "script_path": resources_plugin_path.joinpath("gitlab.py") + } + }, + "Gitlab" + ), ], ) -def test_resources_import_modules(attr, expect): +def test_resources_import_modules(attrs, expected): """Test resource plug-in to import model""" - res_plugin = ResourcePlugin(**attr) - script_name = "local.py" - script_path = "/home/chenrj/gitrep/dolphinscheduler/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py" - res = res_plugin.import_module(script_name, script_path) - assert expect == res.__class__.__name__ - + res_plugin = ResourcePlugin(attrs.get("type"), "plugin-prefix") + res = res_plugin.import_module(**attrs.get("module_attr")) + assert expected == res.__class__.__name__ @pytest.mark.parametrize( - "attr, expect", + "attr, expected", [ - ( - { - "type": ResourcePluginType.LOCAL, - "prefix": "/tmp/", - }, - "Local", - ) + (ResourcePluginType.LOCAL, "Local"), + (ResourcePluginType.GITLAB, "Gitlab"), + (ResourcePluginType.GITHUB, "Github"), ], ) -def test_resources_resources(attr, expect): - """Test the factory mode of the resource plugin, and return the corresponding plugin according to the plugin type""" - res_plugin = ResourcePlugin(**attr) +def test_resources_resources(attr, expected): + """Test resource plugin factory""" + res_plugin = ResourcePlugin(attr, "/tmp/") res = res_plugin.resource - assert expect == res.__class__.__name__ + assert expected == res.__class__.__name__ @pytest.mark.parametrize( @@ -100,13 +98,7 @@ def test_resources_resources(attr, expect): def test_resources_unsupported_res(attr): """Test unsupported plug-ins""" with pytest.raises( - ValueError, match="{} type is not supported".format(attr.get("type")) + PyDSConfException, match="{} type is not supported".format(attr.get("type")) ): res_plugin = ResourcePlugin(**attr) - res_plugin.resource - - - - - - + res_plugin.resource() diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py index 69d37027abca..4da6671f6410 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py @@ -1,55 +1,123 @@ import os +from pathlib import Path +from unittest.mock import patch, PropertyMock import pytest from pydolphinscheduler.constants import ResourcePluginType +from pydolphinscheduler.core import Task +from pydolphinscheduler.exceptions import PyDSConfException, PyResPluginException from pydolphinscheduler.resources_plugin import ResourcePlugin -from unittest.mock import patch -from pydolphinscheduler.tasks.shell import Shell +from pydolphinscheduler.resources_plugin.local import Local from pydolphinscheduler.utils import file from tests.testing.file import delete_file -file_path = 'local_res.sh' -file_content = "echo \"test res_local\"" -# res_plugin_prefix = Path(__file__).absolute().parent -pwd = os.path.abspath(__file__) -res_plugin_prefix = os.path.abspath(os.path.dirname(pwd) + os.path.sep + ".") + "/" +file_name = "local_res.sh" +file_content = "echo Test local res plugin" +res_plugin_prefix = Path(__file__).parent +file_path = res_plugin_prefix.joinpath(file_name) -@pytest.fixture + +@pytest.fixture() def setup_crt_first(): """Set up and teardown about create file first and then delete it.""" file.write(content=file_content, to_path=file_path) yield delete_file(file_path) + @pytest.mark.parametrize( - "attr, expect", + "val, expected", [ - ( - { - "name": "test-local-res-command-content", - "command": file_path, - "resource_plugin": ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix=res_plugin_prefix, - ) - }, - file_content - ) + (file_name, file_content), ], ) @patch( "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) -def test_resources_local_shell_command_content(mock_code_version, attr, expect, setup_crt_first): - """Test task shell task command content.""" - task = Shell(**attr) - assert expect == getattr(task, "raw_script") - - +@patch( + "pydolphinscheduler.core.task.Task.ext", + new_callable=PropertyMock, + return_value={".sh", }, +) +@patch( + "pydolphinscheduler.core.task.Task.ext_attr", + new_callable=PropertyMock, + return_value="_raw_script", +) +@patch( + "pydolphinscheduler.core.task.Task._raw_script", + create=True, + new_callable=PropertyMock, +) +def test_task_obtain_res_plugin(m_raw_script, m_ext_attr, m_ext, m_code_version, val, expected, setup_crt_first): + """Test task obtaining resource plug-in.""" + m_raw_script.return_value = val + task = Task( + name="test_task_ext_attr", + task_type=ResourcePluginType.LOCAL, + resource_plugin=ResourcePlugin( + type=ResourcePluginType.LOCAL, + prefix=str(res_plugin_prefix), + ) + ) + assert expected == getattr(task, "raw_script") +@pytest.mark.parametrize( + "attr, expected", + [ + ( + { + "prefix": res_plugin_prefix, + "file_name": file_name + }, + file_content + ) + ], +) +def test_local_res_read_file(attr, expected, setup_crt_first): + """Test the read_file function of the local resource plug-in""" + local = Local(str(attr.get("prefix"))) + local.read_file(attr.get("file_name")) + assert expected == local.read_file(file_name) +@pytest.mark.parametrize( + "attr", + [ + { + "prefix": res_plugin_prefix, + "file_name": file_name + }, + ], +) +def test_local_res_file_not_found(attr): + """test local resource plugin file does not exist""" + with pytest.raises( + PyResPluginException, + match="{} is not found".format(Path(attr.get("prefix")).joinpath(attr.get("file_name"))) + ): + local = Local(str(attr.get("prefix"))) + local.read_file(attr.get("file_name")) +@pytest.mark.parametrize( + "attr", + [ + { + "prefix": "/etc", + "file_name": "profile" + }, + ], +) +# +def test_local_res_file_no_permission(attr): + """test local resource plugin file does not have permission to access""" + with pytest.raises( + PyResPluginException, match="You don't have permission to access {}".format(Path(attr.get("prefix")).joinpath(attr.get("file_name"))) + ): + local = Local(str(attr.get("prefix"))) + path = Path(attr.get("prefix")).joinpath(attr.get("file_name")) + print(os.access(str(path), os.R_OK)) + local.read_file(attr.get("file_name")) diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py index 1151a69f10e9..8950abd709c5 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py @@ -17,7 +17,7 @@ """Test Task shell.""" -import os +from pathlib import Path from unittest.mock import patch import pytest @@ -28,11 +28,17 @@ from tests.testing.file import delete_file -## 路径file file_path = 'local_res.sh' file_content = "echo \"test res_local\"" -pwd = os.path.abspath(__file__) -res_plugin_prefix = os.path.abspath(os.path.dirname(pwd) + os.path.sep + ".") + "/" +res_plugin_prefix = Path(__file__).parent + + +@pytest.fixture +def setup_crt_first(): + """Set up and teardown about create file first and then delete it.""" + file.write(content=file_content, to_path=file_path) + yield + delete_file(file_path) @pytest.mark.parametrize( @@ -99,14 +105,6 @@ def test_shell_get_define(): assert shell.get_define() == expect -@pytest.fixture -def setup_crt_first(): - """Set up and teardown about create file first and then delete it.""" - file.write(content=file_content, to_path=file_path) - yield - delete_file(file_path) - - @pytest.mark.parametrize( "attr, expect", [ From bd7b506856a0672077d86fd4c7e1d5f5ba43ce7f Mon Sep 17 00:00:00 2001 From: chenrj <102030622+xdu-chenrj@users.noreply.github.com> Date: Tue, 9 Aug 2022 17:54:01 +0800 Subject: [PATCH 09/58] Update dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py Only local is reserved Co-authored-by: Jiajie Zhong --- .../pydolphinscheduler/src/pydolphinscheduler/constants.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py index 20a4c51e53e1..407e1543cea9 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py @@ -107,10 +107,6 @@ class Time(str): class ResourcePluginType(str): """Constants for resources plugin type, it will also show you which kind we support up to now.""" LOCAL = "local" - GITHUB = "github" - GITLAB = "gitlab" - S3 = "S3" - OSS = "OSS" # [end res_plugin_constants_definition] From 91416dbc536a9545e5dbf0031ccd64964d678c61 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Wed, 10 Aug 2022 17:04:19 +0800 Subject: [PATCH 10/58] fix conversation 1 --- .../src/main/resources/logback-spring.xml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/dolphinscheduler-api/src/main/resources/logback-spring.xml b/dolphinscheduler-api/src/main/resources/logback-spring.xml index dc44499570ea..bcea64734c28 100644 --- a/dolphinscheduler-api/src/main/resources/logback-spring.xml +++ b/dolphinscheduler-api/src/main/resources/logback-spring.xml @@ -53,13 +53,12 @@ - - - - - - + + + + + - + \ No newline at end of file From da119fa9c1c010e85322f1e5b31312b8f87e1ea8 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Wed, 10 Aug 2022 17:06:32 +0800 Subject: [PATCH 11/58] fix conversation 3 --- .../src/main/resources/logback-spring.xml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/dolphinscheduler-master/src/main/resources/logback-spring.xml b/dolphinscheduler-master/src/main/resources/logback-spring.xml index 34f75b206b83..b2d566af5e6a 100644 --- a/dolphinscheduler-master/src/main/resources/logback-spring.xml +++ b/dolphinscheduler-master/src/main/resources/logback-spring.xml @@ -66,13 +66,12 @@ - - - - - - + + + + + - + \ No newline at end of file From 99a17c272dee650aead1aa7600bb71c88c68b9ca Mon Sep 17 00:00:00 2001 From: chenrj <102030622+xdu-chenrj@users.noreply.github.com> Date: Wed, 10 Aug 2022 17:07:34 +0800 Subject: [PATCH 12/58] Update dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py fix Co-authored-by: Jiajie Zhong --- .../src/pydolphinscheduler/core/process_definition.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py index ae83a99829e8..5b6d8a827176 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py @@ -29,7 +29,7 @@ from pydolphinscheduler.java_gateway import JavaGate from pydolphinscheduler.models import Base, Project, Tenant, User from pydolphinscheduler.utils.date import MAX_DATETIME, conv_from_str, conv_to_schedule -from pydolphinscheduler.resources_plugin.__init__ import ResourcePlugin +from pydolphinscheduler.resources_plugin import ResourcePlugin class ProcessDefinitionContext: From a590f5589eab084752d86c4251fb8ce6e5ddcac9 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Wed, 10 Aug 2022 17:15:05 +0800 Subject: [PATCH 13/58] fix conversation 5 --- .../src/pydolphinscheduler/core/process_definition.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py index 5b6d8a827176..635cc23091cb 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py @@ -21,7 +21,6 @@ from datetime import datetime from typing import Any, Dict, List, Optional, Set - from pydolphinscheduler import configuration from pydolphinscheduler.constants import TaskType from pydolphinscheduler.core.resource import Resource From ccd0dea8f6a8160cdfa279dbd937933a2cf01c90 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Wed, 10 Aug 2022 17:17:21 +0800 Subject: [PATCH 14/58] fix conversation 6 --- .../pydolphinscheduler/src/pydolphinscheduler/constants.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py index 407e1543cea9..dfad10dfe325 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py @@ -115,5 +115,4 @@ class ResourcePluginType(str): class ResourceKey(str): """Constants for key of resource.""" - ID = "id" - + ID = "id" \ No newline at end of file From b6d00c42070138ba4a23aba4b8316def3f1ba066 Mon Sep 17 00:00:00 2001 From: chenrj <102030622+xdu-chenrj@users.noreply.github.com> Date: Wed, 10 Aug 2022 17:18:11 +0800 Subject: [PATCH 15/58] Update dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/index.rst fix Co-authored-by: Jiajie Zhong --- .../pydolphinscheduler/docs/source/resources_plugin/index.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/index.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/index.rst index c3437f9c9586..cb4fcc569124 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/index.rst +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/index.rst @@ -28,4 +28,3 @@ In this section resource-plugin local - From 242ebe8e319716ee5f189acf92b7b55be5cacb8e Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Wed, 10 Aug 2022 17:18:55 +0800 Subject: [PATCH 16/58] fix conversation 7 --- .../pydolphinscheduler/docs/source/resources_plugin/index.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/index.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/index.rst index c3437f9c9586..4edb14df3f01 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/index.rst +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/index.rst @@ -26,6 +26,4 @@ In this section how-to-use how-to-develop resource-plugin - local - - + local \ No newline at end of file From 7e080f12d4717c34e8cf098e86fba937d6160097 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Wed, 10 Aug 2022 17:19:52 +0800 Subject: [PATCH 17/58] fix conversation 8 --- .../pydolphinscheduler/src/pydolphinscheduler/core/task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py index 9f447ce70375..84ecc58d3f9f 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py @@ -29,7 +29,7 @@ TaskTimeoutFlag, ) from pydolphinscheduler.exceptions import PyResPluginException -from pydolphinscheduler.resources_plugin.__init__ import ResourcePlugin +from pydolphinscheduler.resources_plugin import ResourcePlugin from pydolphinscheduler.core.process_definition import ( ProcessDefinition, ProcessDefinitionContext, From 05fcedc23f02a990e55b9cd8a594d849389973c9 Mon Sep 17 00:00:00 2001 From: chenrj <102030622+xdu-chenrj@users.noreply.github.com> Date: Wed, 10 Aug 2022 17:20:03 +0800 Subject: [PATCH 18/58] Update dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py fix Co-authored-by: Jiajie Zhong --- .../pydolphinscheduler/src/pydolphinscheduler/core/task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py index 9f447ce70375..84ecc58d3f9f 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py @@ -29,7 +29,7 @@ TaskTimeoutFlag, ) from pydolphinscheduler.exceptions import PyResPluginException -from pydolphinscheduler.resources_plugin.__init__ import ResourcePlugin +from pydolphinscheduler.resources_plugin import ResourcePlugin from pydolphinscheduler.core.process_definition import ( ProcessDefinition, ProcessDefinitionContext, From bbaa7c2cb6215aabd80a8ff60e2adaac863e1b36 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Wed, 10 Aug 2022 17:21:32 +0800 Subject: [PATCH 19/58] fix conversation 9 --- .../src/pydolphinscheduler/core/task.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py index 84ecc58d3f9f..65abda555fb5 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py @@ -66,10 +66,10 @@ class TaskRelation(Base): } def __init__( - self, - pre_task_code: int, - post_task_code: int, - name: Optional[str] = None, + self, + pre_task_code: int, + post_task_code: int, + name: Optional[str] = None, ): super().__init__(name) self.pre_task_code = pre_task_code From 5f002d363fedecf1f28dd17bddc771fa4256f913 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Thu, 11 Aug 2022 15:46:41 +0800 Subject: [PATCH 20/58] fix conversation 10 --- .../examples/tutorial_resource_plugin.py | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py new file mode 100644 index 000000000000..4fd944eafce8 --- /dev/null +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +r""" +A tutorial example take you to experience pydolphinscheduler resource plugin. + +After tutorial_resource_plugin.py file submit to Apache DolphinScheduler server a DAG would be create, +and workflow DAG graph as below: + + --> task_child_one + / \ +task_parent --> --> task_union + \ / + --> task_child_two + +Resource plug-ins can be defined in workflows and tasks + +task_parent uses local resource plugin. +task_child_one uses local resource plugin. +task_child_two uses local resource plugin. +task_union does not use resource plug-ins + +it will instantiate and run all the task it have. +""" + +# [start tutorial] +# [start package_import] +# Import ProcessDefinition object to define your workflow attributes +from pydolphinscheduler.core.process_definition import ProcessDefinition + +# Import task Shell object cause we would create some shell tasks later +from pydolphinscheduler.tasks.shell import Shell +from pydolphinscheduler.constants import ResourcePluginType +from pydolphinscheduler.resources_plugin import ResourcePlugin + +# [end package_import] + +# [start workflow_declare] +with ProcessDefinition( + name="tutorial", + schedule="0 0 0 * * ? *", + start_time="2021-01-01", + tenant="tenant_exists", + resource_plugin=ResourcePlugin( + type=ResourcePluginType.LOCAL, + prefix='/opt/', + ) +) as pd: + # [end workflow_declare] + # [start task_declare] + task_parent = Shell( + name="task_parent", + command="parent.zsh", + resource_plugin=ResourcePlugin( + type=ResourcePluginType.LOCAL, + prefix='/opt/', + ) + ) + + task_child_one = Shell( + name="task_child_one", + command="child_one.sh", + resource_plugin=ResourcePlugin( + type=ResourcePluginType.LOCAL, + prefix='/opt/', + ) + ) + + task_child_two = Shell( + name="task_child_two", + command="child_two.sh", + resource_plugin=ResourcePlugin( + type=ResourcePluginType.LOCAL, + prefix='/opt/', + ) + ) + task_union = Shell( + name="task_union", + command="echo union", + ) + # [end task_declare] + + # [start task_relation_declare] + task_group = [task_child_one, task_child_two] + task_parent.set_downstream(task_group) + + task_union << task_group + # [end task_relation_declare] + + # [start submit_or_run] + pd.run() + # [end submit_or_run] +# [end tutorial] From cb7b9b8166a50c971194adb15b04e46f59b78bc0 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Thu, 11 Aug 2022 16:13:47 +0800 Subject: [PATCH 21/58] fix conversation 9 and 14 --- .../src/pydolphinscheduler/core/task.py | 40 +++++++++---------- .../resources_plugin/__init__.py | 29 +++++++++++++- .../resources_plugin/github.py | 6 --- .../resources_plugin/gitlab.py | 3 -- .../resources_plugin/local.py | 19 +++++++++ .../tests/resources_plugin/test_init.py | 22 ---------- 6 files changed, 66 insertions(+), 53 deletions(-) delete mode 100644 dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/github.py delete mode 100644 dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/gitlab.py diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py index 65abda555fb5..776f7e69edf2 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py @@ -108,26 +108,26 @@ class Task(Base): DEFAULT_CONDITION_RESULT = {"successNode": [""], "failedNode": [""]} def __init__( - self, - name: str, - task_type: str, - description: Optional[str] = None, - flag: Optional[str] = TaskFlag.YES, - task_priority: Optional[str] = TaskPriority.MEDIUM, - worker_group: Optional[str] = configuration.WORKFLOW_WORKER_GROUP, - delay_time: Optional[int] = 0, - fail_retry_times: Optional[int] = 0, - fail_retry_interval: Optional[int] = 1, - timeout_flag: Optional[int] = TaskTimeoutFlag.CLOSE, - timeout_notify_strategy: Optional = None, - timeout: Optional[int] = 0, - process_definition: Optional[ProcessDefinition] = None, - local_params: Optional[List] = None, - resource_list: Optional[List] = None, - dependence: Optional[Dict] = None, - wait_start_timeout: Optional[Dict] = None, - condition_result: Optional[Dict] = None, - resource_plugin: Optional[ResourcePlugin] = None, + self, + name: str, + task_type: str, + description: Optional[str] = None, + flag: Optional[str] = TaskFlag.YES, + task_priority: Optional[str] = TaskPriority.MEDIUM, + worker_group: Optional[str] = configuration.WORKFLOW_WORKER_GROUP, + delay_time: Optional[int] = 0, + fail_retry_times: Optional[int] = 0, + fail_retry_interval: Optional[int] = 1, + timeout_flag: Optional[int] = TaskTimeoutFlag.CLOSE, + timeout_notify_strategy: Optional = None, + timeout: Optional[int] = 0, + process_definition: Optional[ProcessDefinition] = None, + local_params: Optional[List] = None, + resource_list: Optional[List] = None, + dependence: Optional[Dict] = None, + wait_start_timeout: Optional[Dict] = None, + condition_result: Optional[Dict] = None, + resource_plugin: Optional[ResourcePlugin] = None, ): super().__init__(name, description) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py index 522ed08da500..d2432ef53d00 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py @@ -1,9 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Init resources_plugin package and dolphinScheduler ResourcePlugin object.""" + import importlib import importlib.util - from pathlib import Path from typing import Any, Generator - from pydolphinscheduler.exceptions import PyDSConfException path_resources_plugin = Path(__file__).parent @@ -11,6 +28,14 @@ # [start resource_plugin_definition] class ResourcePlugin: + """ResourcePlugin object, declare resource plugin for task and workflow to dolphinscheduler. + + :param type: A unique, meaningful string for the ResourcePlugin, + Its value should be taken from the constant of ResourceType in constants.py. + :param prefix: A string representing the prefix of ResourcePlugin. + + """ + def __init__(self, type: str, prefix: str): self.type = type self.prefix = prefix diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/github.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/github.py deleted file mode 100644 index 564ad4f83bf9..000000000000 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/github.py +++ /dev/null @@ -1,6 +0,0 @@ - -class Github: - def __init__(self, path: str): - self.path = path - - diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/gitlab.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/gitlab.py deleted file mode 100644 index 5b8dca91be96..000000000000 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/gitlab.py +++ /dev/null @@ -1,3 +0,0 @@ -class Gitlab: - def __init__(self, path: str): - self.path = path diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py index f4d15fd65602..9ba132aa3b45 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py @@ -1,3 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""DolphinScheduler local resource plugin.""" + import os from pathlib import Path from pydolphinscheduler.exceptions import PyResPluginException diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py index a9f736903b62..ecbaed3261e1 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py @@ -43,26 +43,6 @@ def test_resources_get_all_modules(attr, expected): }, "Local" ), - ( - { - "type": ResourcePluginType.GITHUB, - "module_attr": { - "script_name": "github.py", - "script_path": resources_plugin_path.joinpath("github.py") - } - }, - "Github" - ), - ( - { - "type": ResourcePluginType.GITLAB, - "module_attr": { - "script_name": "gitlab.py", - "script_path": resources_plugin_path.joinpath("gitlab.py") - } - }, - "Gitlab" - ), ], ) def test_resources_import_modules(attrs, expected): @@ -75,8 +55,6 @@ def test_resources_import_modules(attrs, expected): "attr, expected", [ (ResourcePluginType.LOCAL, "Local"), - (ResourcePluginType.GITLAB, "Gitlab"), - (ResourcePluginType.GITHUB, "Github"), ], ) def test_resources_resources(attr, expected): From ed8a0f120a155f705d8eaa89db9f8c4cd9027f21 Mon Sep 17 00:00:00 2001 From: chenrj <102030622+xdu-chenrj@users.noreply.github.com> Date: Thu, 11 Aug 2022 16:14:29 +0800 Subject: [PATCH 22/58] Update dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py fix Co-authored-by: Jiajie Zhong --- .../src/pydolphinscheduler/resources_plugin/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py index 522ed08da500..7192f30f43a4 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py @@ -15,7 +15,7 @@ def __init__(self, type: str, prefix: str): self.type = type self.prefix = prefix - def get_all_modules(self) -> Generator[Path, Any, None]: + def get_all_modules(self) -> Generator[Path]: """Get all res files path in resources_plugin directory.""" return (ex for ex in path_resources_plugin.iterdir() if ex.is_file() and not ex.name.startswith("__")) From 68dc2300e54155423c4557d2da02015f82b2dc2e Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Thu, 11 Aug 2022 16:15:33 +0800 Subject: [PATCH 23/58] fix conversation 15 --- .../src/pydolphinscheduler/resources_plugin/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py index d2432ef53d00..ca89aaf36a0d 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py @@ -40,7 +40,7 @@ def __init__(self, type: str, prefix: str): self.type = type self.prefix = prefix - def get_all_modules(self) -> Generator[Path, Any, None]: + def get_all_modules(self) -> Generator[Path]: """Get all res files path in resources_plugin directory.""" return (ex for ex in path_resources_plugin.iterdir() if ex.is_file() and not ex.name.startswith("__")) From 72291871f728093e100f700267483461fb82ec41 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Thu, 11 Aug 2022 16:23:20 +0800 Subject: [PATCH 24/58] fix conversation 14 and 17 --- .../src/pydolphinscheduler/core/task.py | 12 +++--------- .../pydolphinscheduler/resources_plugin/__init__.py | 2 +- .../src/pydolphinscheduler/resources_plugin/local.py | 3 ++- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py index 776f7e69edf2..fb39e1d0bae8 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py @@ -234,6 +234,9 @@ def task_params(self) -> Optional[Dict]: return self.get_define_custom(custom_attr=custom_attr) def get_plugin(self): + """Return the resource plug-in according to parameter resource_plugin and parameter + process_definition.resource_plugin. + """ if self.resource_plugin is None: if self.process_definition.resource_plugin is not None: return self.process_definition.resource_plugin.resource @@ -243,15 +246,6 @@ def get_plugin(self): else: return self.resource_plugin.resource - def get_plugin(self): - if self.resource_plugin is None: - if self.process_definition.resource_plugin is not None: - return self.process_definition.resource_plugin.resource - else: - raise ValueError - else: - return self.resource_plugin.resource - def get_content(self): """Get the file content according to the resource plugin""" if self.ext_attr is None: diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py index ca89aaf36a0d..32b21b2e46b2 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py @@ -34,7 +34,7 @@ class ResourcePlugin: Its value should be taken from the constant of ResourceType in constants.py. :param prefix: A string representing the prefix of ResourcePlugin. - """ + """ def __init__(self, type: str, prefix: str): self.type = type diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py index 9ba132aa3b45..6a834549b2e5 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py @@ -35,7 +35,8 @@ def prefix(self): def read_file(self, suf: str): """Get the content of the file, the address of the file is - the prefix of the resource plugin plus the parameter suf """ + the prefix of the resource plugin plus the parameter suf + """ path = Path(self.prefix).joinpath(suf) if not path.exists(): raise PyResPluginException("{} is not found".format(str(path))) From 9f00ee7af3793ba30e19a48d57f4ec93602554e9 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Thu, 11 Aug 2022 16:27:38 +0800 Subject: [PATCH 25/58] fix conversation 14 and 17 --- .../pydolphinscheduler/src/pydolphinscheduler/core/task.py | 4 ++-- .../src/pydolphinscheduler/resources_plugin/__init__.py | 3 ++- .../src/pydolphinscheduler/resources_plugin/local.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py index fb39e1d0bae8..abc49c18634d 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py @@ -28,16 +28,16 @@ TaskPriority, TaskTimeoutFlag, ) -from pydolphinscheduler.exceptions import PyResPluginException -from pydolphinscheduler.resources_plugin import ResourcePlugin from pydolphinscheduler.core.process_definition import ( ProcessDefinition, ProcessDefinitionContext, ) from pydolphinscheduler.core.resource import Resource from pydolphinscheduler.exceptions import PyDSParamException +from pydolphinscheduler.exceptions import PyResPluginException from pydolphinscheduler.java_gateway import JavaGate from pydolphinscheduler.models import Base +from pydolphinscheduler.resources_plugin import ResourcePlugin logger = getLogger(__name__) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py index 32b21b2e46b2..1419b411aacd 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py @@ -20,7 +20,8 @@ import importlib import importlib.util from pathlib import Path -from typing import Any, Generator +from typing import Generator + from pydolphinscheduler.exceptions import PyDSConfException path_resources_plugin = Path(__file__).parent diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py index 6a834549b2e5..88bea3a3e97c 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py @@ -19,6 +19,7 @@ import os from pathlib import Path + from pydolphinscheduler.exceptions import PyResPluginException @@ -46,4 +47,3 @@ def read_file(self, suf: str): content = f.read() return content # [end local_res_definition] - From 2333ff13cb1412c4cf7e9ad0f9b321f274e854a4 Mon Sep 17 00:00:00 2001 From: chenrj <102030622+xdu-chenrj@users.noreply.github.com> Date: Thu, 11 Aug 2022 16:41:21 +0800 Subject: [PATCH 26/58] Update dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/how-to-develop.rst fix Co-authored-by: Jiajie Zhong --- .../docs/source/resources_plugin/how-to-develop.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/how-to-develop.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/how-to-develop.rst index e68ba6d446d2..fcd519596290 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/how-to-develop.rst +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/how-to-develop.rst @@ -18,7 +18,6 @@ How to develop ============== -How to develop a plugin You need your plugin class created under dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin folder. Your plugin class needs two indispensable methods, one is the __init__ method, the parameter is the prefix of type str, the other is From e9c311b839daa9f998f5268dbd11e4deee6d48b1 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Thu, 11 Aug 2022 16:41:42 +0800 Subject: [PATCH 27/58] fix conversation 19 --- .../docs/source/resources_plugin/how-to-develop.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/how-to-develop.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/how-to-develop.rst index e68ba6d446d2..fcd519596290 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/how-to-develop.rst +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/how-to-develop.rst @@ -18,7 +18,6 @@ How to develop ============== -How to develop a plugin You need your plugin class created under dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin folder. Your plugin class needs two indispensable methods, one is the __init__ method, the parameter is the prefix of type str, the other is From ddc6e7791972438536f89e9ed7c94b99f9fa8e96 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Thu, 11 Aug 2022 16:49:24 +0800 Subject: [PATCH 28/58] revert tutorial.py --- .../{how-to-develop.rst => develop.rst} | 0 .../pydolphinscheduler/examples/tutorial.py | 42 +++---------------- 2 files changed, 5 insertions(+), 37 deletions(-) rename dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/{how-to-develop.rst => develop.rst} (100%) diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/how-to-develop.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/develop.rst similarity index 100% rename from dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/how-to-develop.rst rename to dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/develop.rst diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py index 97482d65cdd2..36ae78b73b53 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py @@ -37,8 +37,6 @@ # Import task Shell object cause we would create some shell tasks later from pydolphinscheduler.tasks.shell import Shell -from pydolphinscheduler.constants import ResourcePluginType -from pydolphinscheduler.resources_plugin.__init__ import ResourcePlugin # [end package_import] @@ -48,43 +46,13 @@ schedule="0 0 0 * * ? *", start_time="2021-01-01", tenant="tenant_exists", - resource_plugin=ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix='/opt/', - ) ) as pd: # [end workflow_declare] # [start task_declare] - task_parent = Shell( - name="task_parent", - command="parent.zsh", - resource_plugin=ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix='/opt/', - ) - ) - - task_child_one = Shell( - name="task_child_one", - command="child_one.sh", - resource_plugin=ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix='/opt/', - ) - ) - - task_child_two = Shell( - name="task_child_two", - command="child_two.sh", - resource_plugin=ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix='/opt/', - ) - ) - task_union = Shell( - name="task_union", - command="echo union1 ab", - ) + task_parent = Shell(name="task_parent", command="echo hello pydolphinscheduler") + task_child_one = Shell(name="task_child_one", command="echo 'child one'") + task_child_two = Shell(name="task_child_two", command="echo 'child two'") + task_union = Shell(name="task_union", command="echo union") # [end task_declare] # [start task_relation_declare] @@ -97,4 +65,4 @@ # [start submit_or_run] pd.run() # [end submit_or_run] -# [end tutorial] +# [end tutorial] \ No newline at end of file From 911632add855bd95222aed4f25b1200d48676b56 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Thu, 11 Aug 2022 16:53:00 +0800 Subject: [PATCH 29/58] fix local.rst --- .../pydolphinscheduler/docs/source/resources_plugin/local.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/local.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/local.rst index 171b056bfea4..4d23d248b3aa 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/local.rst +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/local.rst @@ -18,7 +18,7 @@ Local ===== -Local is a local resource plugin for pydolphinscheduler. +`Local` is a local resource plugin for pydolphinscheduler. When using a local resource plugin, you do not need to use this class explicitly, you only need to add the resource_plugin parameter in the task subclass or workflow definition. The resource_plugin parameter type is From 35d4b5863a1647eb6623d43c22c384afd9246ca4 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Thu, 11 Aug 2022 17:05:08 +0800 Subject: [PATCH 30/58] fix develop.rst --- .../docs/source/resources_plugin/develop.rst | 39 +++++++++++-------- .../src/pydolphinscheduler/constants.py | 4 +- .../resources_plugin/local.py | 9 +++-- 3 files changed, 31 insertions(+), 21 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/develop.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/develop.rst index fcd519596290..8be12281ba2f 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/develop.rst +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/develop.rst @@ -1,24 +1,24 @@ .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at .. http://www.apache.org/licenses/LICENSE-2.0 .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. How to develop ============== -You need your plugin class created under dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin folder. +When you want to create a new resource plugin, you need to add a new class in the module `mod`:`resources_plugin`. All you have to do is named your plugin, implement `__init__` and `read_file` method. Your plugin class needs two indispensable methods, one is the __init__ method, the parameter is the prefix of type str, the other is the read_file function, the parameter is the file suffix of type str, the return value is the file content, if it is exists and is readable. @@ -27,11 +27,18 @@ In addition you need to add a constant with your plugin name to ResourcePluginTy Example ------- +- Method `__init__`: Initiation method with `param`:`prefix` .. literalinclude:: ../../../src/pydolphinscheduler/resources_plugin/local.py - :start-after: [start local_res_definition] - :end-before: [end local_res_definition] + :start-after: [start init_method] + :end-before: [end init_method] +- Method `read_file `: Get content from the given URI, +.. literalinclude:: ../../../src/pydolphinscheduler/resources_plugin/local.py + :start-after: [start read_file_method] + :end-before: [end read_file_method] + +Last but not least, you should also add new resource plugin in constants.py .. literalinclude:: ../../../src/pydolphinscheduler/constants.py - :start-after: [start res_plugin_constants_definition] - :end-before: [end res_plugin_constants_definition] + :start-after: [start class_resource] + :end-before: [end class_resource] diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py index dfad10dfe325..6e9792b302f1 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py @@ -103,13 +103,13 @@ class Time(str): FMT_NO_COLON_TIME = "%H%M%S" -# [start res_plugin_constants_definition] +# [start class_resource] class ResourcePluginType(str): """Constants for resources plugin type, it will also show you which kind we support up to now.""" LOCAL = "local" -# [end res_plugin_constants_definition] +# [end class_resource] class ResourceKey(str): diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py index 88bea3a3e97c..7a2365d6f3e9 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py @@ -23,17 +23,19 @@ from pydolphinscheduler.exceptions import PyResPluginException -# [start local_res_definition] class Local: - + # [init_method] def __init__(self, prefix: str): self._prefix = prefix + # [end init_method] + @property def prefix(self): """Get the _prefix attribute""" return self._prefix + # [start read_file_method] def read_file(self, suf: str): """Get the content of the file, the address of the file is the prefix of the resource plugin plus the parameter suf @@ -46,4 +48,5 @@ def read_file(self, suf: str): with open(path, 'r') as f: content = f.read() return content -# [end local_res_definition] + + # [start read_file_method] From 2bbac8bf9d01355c071efb7a5558a150e79f80a9 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Thu, 11 Aug 2022 17:14:52 +0800 Subject: [PATCH 31/58] fix task.py --- .../pydolphinscheduler/src/pydolphinscheduler/core/task.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py index abc49c18634d..d7a76d23affa 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py @@ -248,10 +248,9 @@ def get_plugin(self): def get_content(self): """Get the file content according to the resource plugin""" - if self.ext_attr is None: - raise ValueError('ext_attr is None') - if self.ext is None: - raise ValueError('ext is None') + + if self.ext_attr is None and self.ext is None: + return _ext_attr = getattr(self, self.ext_attr) From a0ac32646b6e679f53403bcb4cc338aa106be057 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Thu, 11 Aug 2022 20:13:25 +0800 Subject: [PATCH 32/58] fix develop.rst,how-to-use.rst,index.rst,resource-plugin.rst and ResourcePlugin --- .../docs/source/resources_plugin/develop.rst | 4 +- .../source/resources_plugin/how-to-use.rst | 35 --------- .../docs/source/resources_plugin/index.rst | 3 +- .../resources_plugin/resource-plugin.rst | 77 ++++++++++++++----- .../resources_plugin/__init__.py | 11 +++ 5 files changed, 73 insertions(+), 57 deletions(-) delete mode 100644 dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/how-to-use.rst diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/develop.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/develop.rst index 8be12281ba2f..0340e6be4b53 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/develop.rst +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/develop.rst @@ -32,7 +32,9 @@ Example :start-after: [start init_method] :end-before: [end init_method] -- Method `read_file `: Get content from the given URI, +- Method `read_file `: Get content from the given URI, The function parameter is the suffix of the file path. +The file prefix has been initialized in init of the resource plug-in. The prefix plus suffix is the absolute +path of the file in this resource .. literalinclude:: ../../../src/pydolphinscheduler/resources_plugin/local.py :start-after: [start read_file_method] :end-before: [end read_file_method] diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/how-to-use.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/how-to-use.rst deleted file mode 100644 index 79c9f65954c5..000000000000 --- a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/how-to-use.rst +++ /dev/null @@ -1,35 +0,0 @@ -.. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - -.. http://www.apache.org/licenses/LICENSE-2.0 - -.. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -How to use -========== -This article will show you how to use resource plug-ins. - -When you initialize the task subclass, you can add the parameter resource_plugin parameter, specify the type and -prefix of the resource plugin, the command at this time should be the file path, in other words, the prefix of the -resource plugin plus the command at this time is the command file in the specified Absolute paths in resource plugins. - -When the resource_plugin parameter is defined in both the task subclass and the workflow, the resource_plugin defined in the task subclass is used first. If the task subclass does not define resource_plugin, but the resource_plugin is defined in the workflow, the resource_plugin in the workflow is used. - -Of course, if neither the task subclass nor the workflow specifies resource_plugin, the command at this time will be executed as a script, in other words, we are forward compatible. - - -Example -------- -.. literalinclude:: ../../../src/pydolphinscheduler/examples/tutorial.py - :start-after: [start workflow_declare] - :end-before: [end task_relation_declare] diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/index.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/index.rst index 4edb14df3f01..05a7ebd94ab1 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/index.rst +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/index.rst @@ -23,7 +23,6 @@ In this section .. toctree:: :maxdepth: 1 - how-to-use - how-to-develop + develop resource-plugin local \ No newline at end of file diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/resource-plugin.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/resource-plugin.rst index 24e3b233d31d..dd9da2cdc093 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/resource-plugin.rst +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/resource-plugin.rst @@ -1,19 +1,19 @@ .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at .. http://www.apache.org/licenses/LICENSE-2.0 .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. ResourcePlugin ============== @@ -31,21 +31,60 @@ Dive Into --------- It has the following key functions. -The __init__ function requires parameters. The first is the type of str type, which means the plugin type of the resource, -and the second is the prefix of the str type, which means the prefix of the resource. +- Method `__init__`: The `__init__` function requires parameters. The first is the `type` of str type, which means the plugin type of the resource, +and the second is the `prefix` of the str type, which means the prefix of the resource. -The get_all_modules function will return the absolute path of all resource plugins defined in the resource_plugin file. +.. literalinclude:: ../../../src/pydolphinscheduler/resources_plugin/__init__.py + :start-after: [start init_method] + :end-before: [end init_method] + +- Method `get_all_modules`: The `get_all_modules` function will return the absolute path of all resource plugins defined in the resource_plugin file. + +.. literalinclude:: ../../../src/pydolphinscheduler/resources_plugin/__init__.py + :start-after: [start get_all_modules] + :end-before: [end get_all_modules] + +- Method `import_module`: The `import_module` function has two parameters, `script_name` and `script_path`. +`script_name` is the name of the resource file without the suffix, such as the local resource plugin class local.py, its +`script_name` is local, and `script_path` is the absolute path of the resource plugin file -The import_module function has two parameters, script_name and script_path. -script_name is the name of the resource file without the suffix, such as the local resource plugin class local.py, its -script_name is local, and script_path is the absolute path of the resource plugin file +.. literalinclude:: ../../../src/pydolphinscheduler/resources_plugin/__init__.py + :start-after: [start import_module] + :end-before: [end import_module] -The resource function will dynamically return the resource plugin object according to the type parameter of the __intit__ function, -for example, the type is ResourcePluginType.LOCAL, then the local plugin object is returned +- Method `resource`: The `resource` function will dynamically return the resource plugin object according to the type parameter of the `__init__` function, +for example, the type is `ResourcePluginType.LOCAL`, then the local plugin object is returned +.. literalinclude:: ../../../src/pydolphinscheduler/resources_plugin/__init__.py + :start-after: [start resource] + :end-before: [end resource] .. automodule:: pydolphinscheduler.resources_plugin.__init__ +How to use +---------- +Resource plug-ins can be used in task subclasses and workflows. You can use the resource plug-ins by adding the `resource_plugin` parameter when they are initialized. + +Using resource plug-ins in workflows + +.. literalinclude:: ../../../src/pydolphinscheduler/examples/tutorial_resource_plugin.py + :start-after: [start workflow_declare] + :end-before: [end workflow_declare] + +Using resource plug-ins in sehll tasks + +.. literalinclude:: ../../../src/pydolphinscheduler/examples/tutorial_resource_plugin.py + :start-after: [start task_declare] + :end-before: [end task_declare] + +Use resource plug-ins in both tasks and workflows +.. literalinclude:: ../../../src/pydolphinscheduler/examples/tutorial_resource_plugin.py + :start-after: [start workflow_declare] + :end-before: [end task_declare] +When the resource_plugin parameter is defined in both the task subclass and the workflow, the resource_plugin defined in the task subclass is used first. +If the task subclass does not define resource_plugin, but the resource_plugin is defined in the workflow, the resource_plugin in the workflow is used. +Of course, if neither the task subclass nor the workflow specifies resource_plugin, the command at this time will be executed as a script, +in other words, we are forward compatible. \ No newline at end of file diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py index 1419b411aacd..074c18bd18c0 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py @@ -37,14 +37,21 @@ class ResourcePlugin: """ + # [start init] def __init__(self, type: str, prefix: str): self.type = type self.prefix = prefix + # [end init] + + # [start get_all_modules] def get_all_modules(self) -> Generator[Path]: """Get all res files path in resources_plugin directory.""" return (ex for ex in path_resources_plugin.iterdir() if ex.is_file() and not ex.name.startswith("__")) + # [end get_all_modules] + + # [start import_module] def import_module(self, script_name, script_path): """Import module""" spec = importlib.util.spec_from_file_location(script_name, script_path) @@ -53,11 +60,15 @@ def import_module(self, script_name, script_path): plugin = getattr(module, self.type.capitalize()) return plugin(self.prefix) + # [end import_module] + @property + # [start resource] def resource(self): """Dynamically return resource plugin""" for ex in self.get_all_modules(): if ex.stem == self.type: return self.import_module(ex.name, str(ex)) raise PyDSConfException('{} type is not supported'.format(self.type)) + # [end resource] # [end resource_plugin_definition] From ade9eb79c3c85dec5e03960a9a64f0beef0e190f Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Thu, 11 Aug 2022 20:41:27 +0800 Subject: [PATCH 33/58] fix ResourcePlugin and its test --- .../src/pydolphinscheduler/resources_plugin/__init__.py | 6 +++--- .../tests/resources_plugin/test_init.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py index 074c18bd18c0..db4a36f9beb0 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py @@ -20,7 +20,7 @@ import importlib import importlib.util from pathlib import Path -from typing import Generator +from typing import Generator, Any from pydolphinscheduler.exceptions import PyDSConfException @@ -45,7 +45,7 @@ def __init__(self, type: str, prefix: str): # [end init] # [start get_all_modules] - def get_all_modules(self) -> Generator[Path]: + def get_all_modules(self) -> Path: """Get all res files path in resources_plugin directory.""" return (ex for ex in path_resources_plugin.iterdir() if ex.is_file() and not ex.name.startswith("__")) @@ -69,6 +69,6 @@ def resource(self): for ex in self.get_all_modules(): if ex.stem == self.type: return self.import_module(ex.name, str(ex)) - raise PyDSConfException('{} type is not supported'.format(self.type)) + raise PyDSConfException('{} type is not supported.'.format(self.type)) # [end resource] # [end resource_plugin_definition] diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py index ecbaed3261e1..7e354ec9e69d 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py @@ -5,9 +5,9 @@ from pydolphinscheduler.constants import ResourcePluginType from pydolphinscheduler.exceptions import PyDSConfException -from pydolphinscheduler.resources_plugin import ResourcePlugin +from pydolphinscheduler.resources_plugin.__init__ import ResourcePlugin -all_res = ["local", "github", "gitlab"] +all_res = ["local"] project_root = Path(__file__).parent.parent.parent resources_plugin_path = project_root.joinpath("src", "pydolphinscheduler", "resources_plugin") @@ -17,8 +17,8 @@ [ ( { - "type": ResourcePluginType.LOCAL, - "prefix": "/tmp/", + "type": "res_type", + "prefix": "res_prefix", }, all_res ) From 6aec0ebd5bb177112ce87ccdab286a7fb49409c2 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Thu, 11 Aug 2022 20:44:26 +0800 Subject: [PATCH 34/58] fix ResourcePlugin and its test --- .../pydolphinscheduler/tests/resources_plugin/test_init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py index 7e354ec9e69d..7d9d730641b5 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py @@ -5,7 +5,7 @@ from pydolphinscheduler.constants import ResourcePluginType from pydolphinscheduler.exceptions import PyDSConfException -from pydolphinscheduler.resources_plugin.__init__ import ResourcePlugin +from pydolphinscheduler.resources_plugin import ResourcePlugin all_res = ["local"] project_root = Path(__file__).parent.parent.parent From e9aa2d4059de68939252ca969148cbcc1e5af024 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Thu, 11 Aug 2022 21:07:58 +0800 Subject: [PATCH 35/58] fix error exception --- .../resources_plugin/__init__.py | 5 ++--- .../pydolphinscheduler/tests/core/test_task.py | 10 +++++----- .../tests/resources_plugin/test_init.py | 4 ++-- .../tests/resources_plugin/test_local.py | 14 +++++++------- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py index db4a36f9beb0..4eb9cb627144 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py @@ -20,9 +20,8 @@ import importlib import importlib.util from pathlib import Path -from typing import Generator, Any -from pydolphinscheduler.exceptions import PyDSConfException +from pydolphinscheduler.exceptions import PyDSConfException, PyResPluginException path_resources_plugin = Path(__file__).parent @@ -69,6 +68,6 @@ def resource(self): for ex in self.get_all_modules(): if ex.stem == self.type: return self.import_module(ex.name, str(ex)) - raise PyDSConfException('{} type is not supported.'.format(self.type)) + raise PyResPluginException('{} type is not supported.'.format(self.type)) # [end resource] # [end resource_plugin_definition] diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py index 64836b6fd629..95ac454557cc 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py @@ -300,7 +300,7 @@ def test_task_ext_attr(m_plugin, m_raw_script, m_ext_attr, m_ext, m_code_version "process_definition": ProcessDefinition( name="process_definition", resource_plugin=ResourcePlugin( - type=ResourcePluginType.GITHUB, + type=ResourcePluginType.LOCAL, prefix="prefix", ), ) @@ -312,11 +312,11 @@ def test_task_ext_attr(m_plugin, m_raw_script, m_ext_attr, m_ext, m_code_version "name": "test_task_abtain_res_plugin", "task_type": "TaskType", "resource_plugin": ResourcePlugin( - type=ResourcePluginType.GITLAB, + type=ResourcePluginType.LOCAL, prefix="prefix", ), }, - "Gitlab" + "Local" ), ( { @@ -325,12 +325,12 @@ def test_task_ext_attr(m_plugin, m_raw_script, m_ext_attr, m_ext, m_code_version "process_definition": ProcessDefinition( name="process_definition", resource_plugin=ResourcePlugin( - type=ResourcePluginType.GITHUB, + type=ResourcePluginType.LOCAL, prefix="prefix", ), ) }, - "Github" + "Local" ), ], ) diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py index 7d9d730641b5..88263e0d4301 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py @@ -4,7 +4,7 @@ import pytest from pydolphinscheduler.constants import ResourcePluginType -from pydolphinscheduler.exceptions import PyDSConfException +from pydolphinscheduler.exceptions import PyDSConfException, PyResPluginException from pydolphinscheduler.resources_plugin import ResourcePlugin all_res = ["local"] @@ -76,7 +76,7 @@ def test_resources_resources(attr, expected): def test_resources_unsupported_res(attr): """Test unsupported plug-ins""" with pytest.raises( - PyDSConfException, match="{} type is not supported".format(attr.get("type")) + PyResPluginException, match="{} type is not supported".format(attr.get("type")) ): res_plugin = ResourcePlugin(**attr) res_plugin.resource() diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py index 4a12e15b4c38..dbb52fe815cf 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py @@ -69,11 +69,11 @@ def test_task_obtain_res_plugin(m_raw_script, m_ext_attr, m_ext, m_code_version, "attr, expected", [ ( - { - "prefix": res_plugin_prefix, - "file_name": file_name - }, - file_content + { + "prefix": res_plugin_prefix, + "file_name": file_name + }, + file_content ) ], ) @@ -96,8 +96,8 @@ def test_local_res_read_file(attr, expected, setup_crt_first): def test_local_res_file_not_found(attr): """test local resource plugin file does not exist""" with pytest.raises( - PyResPluginException, - match="{} is not found".format(Path(attr.get("prefix")).joinpath(attr.get("file_name"))) + PyResPluginException, + match="{} is not found".format(Path(attr.get("prefix")).joinpath(attr.get("file_name"))) ): local = Local(str(attr.get("prefix"))) local.read_file(attr.get("file_name")) From faf7860f7a11a817bd553f74351b9fa5a04034d1 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Thu, 11 Aug 2022 23:17:56 +0800 Subject: [PATCH 36/58] fix import order --- .../src/pydolphinscheduler/core/process_definition.py | 2 +- .../pydolphinscheduler/examples/tutorial_resource_plugin.py | 5 ++--- .../src/pydolphinscheduler/resources_plugin/__init__.py | 2 +- .../pydolphinscheduler/tests/resources_plugin/test_local.py | 4 +--- .../pydolphinscheduler/tests/tasks/test_shell.py | 2 +- 5 files changed, 6 insertions(+), 9 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py index 635cc23091cb..1d3dff693fd8 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py @@ -27,8 +27,8 @@ from pydolphinscheduler.exceptions import PyDSParamException, PyDSTaskNoFoundException from pydolphinscheduler.java_gateway import JavaGate from pydolphinscheduler.models import Base, Project, Tenant, User -from pydolphinscheduler.utils.date import MAX_DATETIME, conv_from_str, conv_to_schedule from pydolphinscheduler.resources_plugin import ResourcePlugin +from pydolphinscheduler.utils.date import MAX_DATETIME, conv_from_str, conv_to_schedule class ProcessDefinitionContext: diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py index 4fd944eafce8..79ab39948136 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py @@ -37,15 +37,14 @@ it will instantiate and run all the task it have. """ +from pydolphinscheduler.constants import ResourcePluginType # [start tutorial] # [start package_import] # Import ProcessDefinition object to define your workflow attributes from pydolphinscheduler.core.process_definition import ProcessDefinition - +from pydolphinscheduler.resources_plugin import ResourcePlugin # Import task Shell object cause we would create some shell tasks later from pydolphinscheduler.tasks.shell import Shell -from pydolphinscheduler.constants import ResourcePluginType -from pydolphinscheduler.resources_plugin import ResourcePlugin # [end package_import] diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py index 4eb9cb627144..a405c06e637d 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py @@ -21,7 +21,7 @@ import importlib.util from pathlib import Path -from pydolphinscheduler.exceptions import PyDSConfException, PyResPluginException +from pydolphinscheduler.exceptions import PyResPluginException path_resources_plugin = Path(__file__).parent diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py index dbb52fe815cf..dc4052505d5a 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py @@ -1,4 +1,3 @@ -import os from pathlib import Path from unittest.mock import patch, PropertyMock @@ -6,7 +5,7 @@ from pydolphinscheduler.constants import ResourcePluginType from pydolphinscheduler.core import Task -from pydolphinscheduler.exceptions import PyDSConfException, PyResPluginException +from pydolphinscheduler.exceptions import PyResPluginException from pydolphinscheduler.resources_plugin import ResourcePlugin from pydolphinscheduler.resources_plugin.local import Local from pydolphinscheduler.utils import file @@ -101,4 +100,3 @@ def test_local_res_file_not_found(attr): ): local = Local(str(attr.get("prefix"))) local.read_file(attr.get("file_name")) - diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py index 8950abd709c5..fb0328a505f7 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py @@ -21,11 +21,11 @@ from unittest.mock import patch import pytest + from pydolphinscheduler.constants import ResourcePluginType from pydolphinscheduler.resources_plugin import ResourcePlugin from pydolphinscheduler.tasks.shell import Shell from pydolphinscheduler.utils import file - from tests.testing.file import delete_file file_path = 'local_res.sh' From 93c51c5a15ac0b3ae57bde447379a8a6cddf40ed Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Thu, 11 Aug 2022 23:27:03 +0800 Subject: [PATCH 37/58] format code style --- .../pydolphinscheduler/src/pydolphinscheduler/core/task.py | 3 +-- .../pydolphinscheduler/examples/tutorial_resource_plugin.py | 2 ++ .../pydolphinscheduler/tests/core/test_task.py | 4 ++-- .../pydolphinscheduler/tests/resources_plugin/test_local.py | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py index d7a76d23affa..c57093722607 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py @@ -33,8 +33,7 @@ ProcessDefinitionContext, ) from pydolphinscheduler.core.resource import Resource -from pydolphinscheduler.exceptions import PyDSParamException -from pydolphinscheduler.exceptions import PyResPluginException +from pydolphinscheduler.exceptions import PyDSParamException, PyResPluginException from pydolphinscheduler.java_gateway import JavaGate from pydolphinscheduler.models import Base from pydolphinscheduler.resources_plugin import ResourcePlugin diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py index 79ab39948136..3e1de1a5e798 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py @@ -38,11 +38,13 @@ """ from pydolphinscheduler.constants import ResourcePluginType + # [start tutorial] # [start package_import] # Import ProcessDefinition object to define your workflow attributes from pydolphinscheduler.core.process_definition import ProcessDefinition from pydolphinscheduler.resources_plugin import ResourcePlugin + # Import task Shell object cause we would create some shell tasks later from pydolphinscheduler.tasks.shell import Shell diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py index 95ac454557cc..9f15b7015bae 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py @@ -18,11 +18,11 @@ """Test Task class function.""" import logging import re -from unittest.mock import patch, PropertyMock +from unittest.mock import PropertyMock, patch import pytest -from pydolphinscheduler.constants import TaskType, ResourcePluginType +from pydolphinscheduler.constants import ResourcePluginType, TaskType from pydolphinscheduler.core.process_definition import ProcessDefinition from pydolphinscheduler.core.task import Task, TaskRelation from pydolphinscheduler.exceptions import PyResPluginException diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py index dc4052505d5a..633237d75eae 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py @@ -1,5 +1,5 @@ from pathlib import Path -from unittest.mock import patch, PropertyMock +from unittest.mock import PropertyMock, patch import pytest From 20acc0aac00c7c059b051c8386383e6956ce19a8 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Thu, 11 Aug 2022 23:42:10 +0800 Subject: [PATCH 38/58] reformat files --- .../pydolphinscheduler/setup.py | 15 +-- .../src/pydolphinscheduler/constants.py | 3 +- .../src/pydolphinscheduler/core/engine.py | 4 +- .../core/process_definition.py | 7 +- .../src/pydolphinscheduler/core/resource.py | 5 +- .../src/pydolphinscheduler/core/task.py | 20 ++-- .../examples/task_condition_example.py | 7 +- .../examples/task_datax_example.py | 5 +- .../examples/task_dependent_example.py | 10 +- .../examples/task_sagemaker_example.py | 5 +- .../pydolphinscheduler/examples/tutorial.py | 2 +- .../examples/tutorial_resource_plugin.py | 25 +--- .../src/pydolphinscheduler/models/queue.py | 4 +- .../resources_plugin/__init__.py | 11 +- .../resources_plugin/local.py | 6 +- .../tests/cli/test_config.py | 7 +- .../tests/cli/test_version.py | 7 +- .../tests/core/test_configuration.py | 30 +---- .../tests/core/test_database.py | 11 +- .../tests/core/test_engine.py | 17 +-- .../tests/core/test_process_definition.py | 86 +++----------- .../tests/core/test_task.py | 112 ++++++++---------- .../integration/test_process_definition.py | 12 +- .../tests/resources_plugin/test_init.py | 39 ++---- .../tests/resources_plugin/test_local.py | 41 ++----- .../tests/tasks/test_condition.py | 71 ++--------- .../tests/tasks/test_dependent.py | 9 +- .../tests/tasks/test_flink.py | 4 +- .../tests/tasks/test_http.py | 6 +- .../tests/tasks/test_map_reduce.py | 4 +- .../tests/tasks/test_procedure.py | 6 +- .../tests/tasks/test_python.py | 12 +- .../tests/tasks/test_sagemaker.py | 3 +- .../tests/tasks/test_shell.py | 21 ++-- .../tests/tasks/test_spark.py | 4 +- .../tests/tasks/test_sql.py | 6 +- .../tests/tasks/test_sub_process.py | 3 +- .../tests/tasks/test_switch.py | 30 +---- .../tests/utils/test_yaml_parser.py | 49 +------- 39 files changed, 193 insertions(+), 526 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/setup.py b/dolphinscheduler-python/pydolphinscheduler/setup.py index 5a181223e43f..9b3248e3332c 100644 --- a/dolphinscheduler-python/pydolphinscheduler/setup.py +++ b/dolphinscheduler-python/pydolphinscheduler/setup.py @@ -127,12 +127,7 @@ def run(self) -> None: author_email="dev@dolphinscheduler.apache.org", url="https://dolphinscheduler.apache.org/", python_requires=">=3.6", - keywords=[ - "dolphinscheduler", - "workflow", - "scheduler", - "taskflow", - ], + keywords=["dolphinscheduler", "workflow", "scheduler", "taskflow",], project_urls={ "Homepage": "https://dolphinscheduler.apache.org", "Documentation": "https://dolphinscheduler.apache.org/python/index.html", @@ -146,9 +141,7 @@ def run(self) -> None: packages=find_packages(where="src"), package_dir={"": "src"}, include_package_data=True, - package_data={ - "pydolphinscheduler": ["default_config.yaml"], - }, + package_data={"pydolphinscheduler": ["default_config.yaml"],}, platforms=["any"], classifiers=[ # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers @@ -178,9 +171,7 @@ def run(self) -> None: "doc": doc, "build": build, }, - cmdclass={ - "pre_clean": CleanCommand, - }, + cmdclass={"pre_clean": CleanCommand,}, entry_points={ "console_scripts": [ "pydolphinscheduler = pydolphinscheduler.cli.commands:cli", diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py index 6e9792b302f1..3f4ae76ff8b1 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py @@ -106,6 +106,7 @@ class Time(str): # [start class_resource] class ResourcePluginType(str): """Constants for resources plugin type, it will also show you which kind we support up to now.""" + LOCAL = "local" @@ -115,4 +116,4 @@ class ResourcePluginType(str): class ResourceKey(str): """Constants for key of resource.""" - ID = "id" \ No newline at end of file + ID = "id" diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/engine.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/engine.py index 41021ed474e8..911538bd7d02 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/engine.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/engine.py @@ -86,9 +86,7 @@ def task_params(self, camel_attr: bool = True, custom_attr: set = None) -> Dict: custom_params = { "programType": self.program_type, "mainClass": self.main_class, - "mainJar": { - "id": self.get_jar_id(), - }, + "mainJar": {"id": self.get_jar_id(),}, } params.update(custom_params) return params diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py index 1d3dff693fd8..addd02522965 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py @@ -232,12 +232,7 @@ def param_json(self) -> Optional[List[Dict]]: if not self.param: return [] return [ - { - "prop": k, - "direct": "IN", - "type": "VARCHAR", - "value": v, - } + {"prop": k, "direct": "IN", "type": "VARCHAR", "value": v,} for k, v in self.param.items() ] diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/resource.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/resource.py index ea811915e280..bb1c86869237 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/resource.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/resource.py @@ -66,8 +66,5 @@ def create_or_update_resource(self): "`user_name` and `content` are required when create or update resource from python gate." ) JavaGate().create_or_update_resource( - self.user_name, - self.name, - self.content, - self.description, + self.user_name, self.name, self.content, self.description, ) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py index c57093722607..0f990e9257fe 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py @@ -65,10 +65,7 @@ class TaskRelation(Base): } def __init__( - self, - pre_task_code: int, - post_task_code: int, - name: Optional[str] = None, + self, pre_task_code: int, post_task_code: int, name: Optional[str] = None, ): super().__init__(name) self.pre_task_code = pre_task_code @@ -142,7 +139,7 @@ def __init__( self.timeout = timeout self._process_definition = None self.process_definition: ProcessDefinition = ( - process_definition or ProcessDefinitionContext.get() + process_definition or ProcessDefinitionContext.get() ) self._upstream_task_codes: Set[int] = set() self._downstream_task_codes: Set[int] = set() @@ -151,8 +148,8 @@ def __init__( self.code, self.version = self.gen_code_and_version() # Add task to process definition, maybe we could put into property process_definition latter if ( - self.process_definition is not None - and self.code not in self.process_definition.tasks + self.process_definition is not None + and self.code not in self.process_definition.tasks ): self.process_definition.add_task(self) else: @@ -241,7 +238,8 @@ def get_plugin(self): return self.process_definition.resource_plugin.resource else: raise PyResPluginException( - "The execution command of this task is a file, but the resource plugin is empty") + "The execution command of this task is a file, but the resource plugin is empty" + ) else: return self.resource_plugin.resource @@ -259,10 +257,10 @@ def get_content(self): content = res.read_file(_ext_attr) setattr(self, self.ext_attr.lstrip("_"), content) else: - index = _ext_attr.rfind('.') + index = _ext_attr.rfind(".") if index != -1: raise ValueError( - 'This task does not support files with suffix {}, only supports {}'.format( + "This task does not support files with suffix {}, only supports {}".format( _ext_attr[index:], ",".join(str(suf) for suf in self.ext) ) ) @@ -292,7 +290,7 @@ def __rlshift__(self, other: Union["Task", Sequence["Task"]]): return self def _set_deps( - self, tasks: Union["Task", Sequence["Task"]], upstream: bool = True + self, tasks: Union["Task", Sequence["Task"]], upstream: bool = True ) -> None: """ Set parameter tasks dependent to current task. diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_condition_example.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_condition_example.py index 2d73df4b40d5..7208a195cba6 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_condition_example.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_condition_example.py @@ -39,12 +39,7 @@ pre_task_1 = Shell(name="pre_task_1", command="echo pre_task_1") pre_task_2 = Shell(name="pre_task_2", command="echo pre_task_2") pre_task_3 = Shell(name="pre_task_3", command="echo pre_task_3") - cond_operator = And( - And( - SUCCESS(pre_task_1, pre_task_2), - FAILURE(pre_task_3), - ), - ) + cond_operator = And(And(SUCCESS(pre_task_1, pre_task_2), FAILURE(pre_task_3),),) success_branch = Shell(name="success_branch", command="echo success_branch") fail_branch = Shell(name="fail_branch", command="echo fail_branch") diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_datax_example.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_datax_example.py index 94bd449cf723..7850b294ee53 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_datax_example.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_datax_example.py @@ -72,10 +72,7 @@ } } -with ProcessDefinition( - name="task_datax_example", - tenant="tenant_exists", -) as pd: +with ProcessDefinition(name="task_datax_example", tenant="tenant_exists",) as pd: # This task synchronizes the data in `t_ds_project` # of `first_mysql` database to `target_project` of `second_mysql` database. # You have to make sure data source named `first_mysql` and `second_mysql` exists diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_dependent_example.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_dependent_example.py index db53bcc9f38e..497097e769b2 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_dependent_example.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_dependent_example.py @@ -40,19 +40,13 @@ from pydolphinscheduler.tasks.dependent import And, Dependent, DependentItem, Or from pydolphinscheduler.tasks.shell import Shell -with ProcessDefinition( - name="task_dependent_external", - tenant="tenant_exists", -) as pd: +with ProcessDefinition(name="task_dependent_external", tenant="tenant_exists",) as pd: task_1 = Shell(name="task_1", command="echo task 1") task_2 = Shell(name="task_2", command="echo task 2") task_3 = Shell(name="task_3", command="echo task 3") pd.submit() -with ProcessDefinition( - name="task_dependent_example", - tenant="tenant_exists", -) as pd: +with ProcessDefinition(name="task_dependent_example", tenant="tenant_exists",) as pd: task = Dependent( name="task_dependent", dependence=And( diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_sagemaker_example.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_sagemaker_example.py index b056f61a6384..b5c19620a830 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_sagemaker_example.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_sagemaker_example.py @@ -33,10 +33,7 @@ ], } -with ProcessDefinition( - name="task_sagemaker_example", - tenant="tenant_exists", -) as pd: +with ProcessDefinition(name="task_sagemaker_example", tenant="tenant_exists",) as pd: task_sagemaker = SageMaker( name="task_sagemaker", sagemaker_request_json=json.dumps(sagemaker_request_data, indent=2), diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py index 36ae78b73b53..0478e685190f 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py @@ -65,4 +65,4 @@ # [start submit_or_run] pd.run() # [end submit_or_run] -# [end tutorial] \ No newline at end of file +# [end tutorial] diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py index 3e1de1a5e798..8c126bf73752 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py @@ -56,43 +56,28 @@ schedule="0 0 0 * * ? *", start_time="2021-01-01", tenant="tenant_exists", - resource_plugin=ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix='/opt/', - ) + resource_plugin=ResourcePlugin(type=ResourcePluginType.LOCAL, prefix="/opt/",), ) as pd: # [end workflow_declare] # [start task_declare] task_parent = Shell( name="task_parent", command="parent.zsh", - resource_plugin=ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix='/opt/', - ) + resource_plugin=ResourcePlugin(type=ResourcePluginType.LOCAL, prefix="/opt/",), ) task_child_one = Shell( name="task_child_one", command="child_one.sh", - resource_plugin=ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix='/opt/', - ) + resource_plugin=ResourcePlugin(type=ResourcePluginType.LOCAL, prefix="/opt/",), ) task_child_two = Shell( name="task_child_two", command="child_two.sh", - resource_plugin=ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix='/opt/', - ) - ) - task_union = Shell( - name="task_union", - command="echo union", + resource_plugin=ResourcePlugin(type=ResourcePluginType.LOCAL, prefix="/opt/",), ) + task_union = Shell(name="task_union", command="echo union",) # [end task_declare] # [start task_relation_declare] diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/queue.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/queue.py index e6da2594c8ff..2478838e8025 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/queue.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/queue.py @@ -27,8 +27,6 @@ class Queue(BaseSide): """DolphinScheduler Queue object.""" def __init__( - self, - name: str = configuration.WORKFLOW_QUEUE, - description: Optional[str] = "", + self, name: str = configuration.WORKFLOW_QUEUE, description: Optional[str] = "", ): super().__init__(name, description) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py index a405c06e637d..39c21eb0c428 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py @@ -46,7 +46,11 @@ def __init__(self, type: str, prefix: str): # [start get_all_modules] def get_all_modules(self) -> Path: """Get all res files path in resources_plugin directory.""" - return (ex for ex in path_resources_plugin.iterdir() if ex.is_file() and not ex.name.startswith("__")) + return ( + ex + for ex in path_resources_plugin.iterdir() + if ex.is_file() and not ex.name.startswith("__") + ) # [end get_all_modules] @@ -68,6 +72,9 @@ def resource(self): for ex in self.get_all_modules(): if ex.stem == self.type: return self.import_module(ex.name, str(ex)) - raise PyResPluginException('{} type is not supported.'.format(self.type)) + raise PyResPluginException("{} type is not supported.".format(self.type)) + # [end resource] + + # [end resource_plugin_definition] diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py index 7a2365d6f3e9..e2bed6ee253e 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py @@ -44,8 +44,10 @@ def read_file(self, suf: str): if not path.exists(): raise PyResPluginException("{} is not found".format(str(path))) if not os.access(str(path), os.R_OK): - raise PyResPluginException("You don't have permission to access {}".format(self.prefix + suf)) - with open(path, 'r') as f: + raise PyResPluginException( + "You don't have permission to access {}".format(self.prefix + suf) + ) + with open(path, "r") as f: content = f.read() return content diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/cli/test_config.py b/dolphinscheduler-python/pydolphinscheduler/tests/cli/test_config.py index 516ad754a20c..a09cab80af89 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/cli/test_config.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/cli/test_config.py @@ -44,12 +44,7 @@ def teardown_file_env(): @pytest.mark.parametrize( - "home", - [ - None, - "/tmp/pydolphinscheduler", - "/tmp/test_abc", - ], + "home", [None, "/tmp/pydolphinscheduler", "/tmp/test_abc",], ) def test_config_init(teardown_file_env, home): """Test command line interface `config --init`.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/cli/test_version.py b/dolphinscheduler-python/pydolphinscheduler/tests/cli/test_version.py index f0dcb0e0639a..b97e04036216 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/cli/test_version.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/cli/test_version.py @@ -31,12 +31,7 @@ def test_version(): @pytest.mark.parametrize( - "part, idx", - [ - ("major", 0), - ("minor", 1), - ("micro", 2), - ], + "part, idx", [("major", 0), ("minor", 1), ("micro", 2),], ) def test_version_part(part: str, idx: int): """Test subcommand `version` option `--part`.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_configuration.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_configuration.py index b9dc8cb65645..1fa4934e339a 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_configuration.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_configuration.py @@ -48,13 +48,7 @@ def teardown_file_env(): @pytest.mark.parametrize( - "val, expect", - [ - ("1", 1), - ("123", 123), - ("4567", 4567), - (b"1234", 1234), - ], + "val, expect", [("1", 1), ("123", 123), ("4567", 4567), (b"1234", 1234),], ) def test_get_int(val: Any, expect: int): """Test function :func:`configuration.get_int`.""" @@ -62,13 +56,7 @@ def test_get_int(val: Any, expect: int): @pytest.mark.parametrize( - "val", - [ - "a", - "1a", - "1d2", - "1723-", - ], + "val", ["a", "1a", "1d2", "1723-",], ) def test_get_int_error(val: Any): """Test function :func:`configuration.get_int`.""" @@ -113,12 +101,7 @@ def test_config_path(home: Any, expect: str): @pytest.mark.parametrize( - "home", - [ - None, - "/tmp/pydolphinscheduler", - "/tmp/test_abc", - ], + "home", [None, "/tmp/pydolphinscheduler", "/tmp/test_abc",], ) def test_init_config_file(teardown_file_env, home: Any): """Test init config file.""" @@ -136,12 +119,7 @@ def test_init_config_file(teardown_file_env, home: Any): @pytest.mark.parametrize( - "home", - [ - None, - "/tmp/pydolphinscheduler", - "/tmp/test_abc", - ], + "home", [None, "/tmp/pydolphinscheduler", "/tmp/test_abc",], ) def test_init_config_file_duplicate(teardown_file_env, home: Any): """Test raise error with init config file which already exists.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_database.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_database.py index 1286a4a7f8b0..f0e2927f6d90 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_database.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_database.py @@ -30,17 +30,10 @@ @pytest.mark.parametrize( - "expect", - [ - { - TEST_DATABASE_TYPE_KEY: "mock_type", - TEST_DATABASE_KEY: 1, - } - ], + "expect", [{TEST_DATABASE_TYPE_KEY: "mock_type", TEST_DATABASE_KEY: 1,}], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) @patch( "pydolphinscheduler.core.database.Database.get_database_info", diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_engine.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_engine.py index e36c47ba1b64..fd5b35f77357 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_engine.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_engine.py @@ -31,8 +31,7 @@ @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) @patch( "pydolphinscheduler.core.engine.Engine.get_resource_info", @@ -64,9 +63,7 @@ def test_get_jar_detail(mock_resource, mock_code_version): }, { "mainClass": "org.apache.examples.mock.Mock", - "mainJar": { - "id": 1, - }, + "mainJar": {"id": 1,}, "programType": ProgramType.JAVA, "localParams": [], "resourceList": [], @@ -78,8 +75,7 @@ def test_get_jar_detail(mock_resource, mock_code_version): ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) @patch( "pydolphinscheduler.core.engine.Engine.get_resource_info", @@ -111,9 +107,7 @@ def test_property_task_params(mock_resource, mock_code_version, attr, expect): "taskType": "test-engine", "taskParams": { "mainClass": "org.apache.examples.mock.Mock", - "mainJar": { - "id": 1, - }, + "mainJar": {"id": 1,}, "programType": ProgramType.JAVA, "localParams": [], "resourceList": [], @@ -134,8 +128,7 @@ def test_property_task_params(mock_resource, mock_code_version, attr, expect): ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) @patch( "pydolphinscheduler.core.engine.Engine.get_resource_info", diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py index 30445bfbf356..a6510ac4d92f 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py @@ -108,11 +108,7 @@ def test_set_attr(name, cls, expect): @pytest.mark.parametrize( - "value,expect", - [ - ("online", 1), - ("offline", 0), - ], + "value,expect", [("online", 1), ("offline", 0),], ) def test_set_release_state(value, expect): """Test process definition set release_state attributes.""" @@ -123,14 +119,7 @@ def test_set_release_state(value, expect): @pytest.mark.parametrize( - "value", - [ - "oneline", - "offeline", - 1, - 0, - None, - ], + "value", ["oneline", "offeline", 1, 0, None,], ) def test_set_release_state_error(value): """Test process definition set release_state attributes with error.""" @@ -181,12 +170,7 @@ def test__parse_datetime(val, expect): @pytest.mark.parametrize( - "val", - [ - 20210101, - (2021, 1, 1), - {"year": "2021", "month": "1", "day": 1}, - ], + "val", [20210101, (2021, 1, 1), {"year": "2021", "month": "1", "day": 1},], ) def test__parse_datetime_not_support_type(val: Any): """Test process definition function _parse_datetime not support type error.""" @@ -196,11 +180,7 @@ def test__parse_datetime_not_support_type(val: Any): @pytest.mark.parametrize( - "val", - [ - "ALLL", - "nonee", - ], + "val", ["ALLL", "nonee",], ) def test_warn_type_not_support_type(val: str): """Test process definition param warning_type not support type error.""" @@ -213,43 +193,17 @@ def test_warn_type_not_support_type(val: str): @pytest.mark.parametrize( "param, expect", [ - ( - None, - [], - ), - ( - {}, - [], - ), + (None, [],), + ({}, [],), ( {"key1": "val1"}, - [ - { - "prop": "key1", - "direct": "IN", - "type": "VARCHAR", - "value": "val1", - } - ], + [{"prop": "key1", "direct": "IN", "type": "VARCHAR", "value": "val1",}], ), ( - { - "key1": "val1", - "key2": "val2", - }, + {"key1": "val1", "key2": "val2",}, [ - { - "prop": "key1", - "direct": "IN", - "type": "VARCHAR", - "value": "val1", - }, - { - "prop": "key2", - "direct": "IN", - "type": "VARCHAR", - "value": "val2", - }, + {"prop": "key1", "direct": "IN", "type": "VARCHAR", "value": "val1",}, + {"prop": "key2", "direct": "IN", "type": "VARCHAR", "value": "val2",}, ], ), ], @@ -261,8 +215,7 @@ def test_property_param_json(param, expect): @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) def test__pre_submit_check_switch_without_param(mock_code_version): """Test :func:`_pre_submit_check` if process definition with switch but without attribute param.""" @@ -286,8 +239,7 @@ def test__pre_submit_check_switch_without_param(mock_code_version): @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) def test__pre_submit_check_switch_with_local_params(mock_code_version): """Test :func:`_pre_submit_check` if process definition with switch with local params of task.""" @@ -381,9 +333,7 @@ def test_process_definition_simple_separate(): pd = ProcessDefinition(TEST_PROCESS_DEFINITION_NAME) for i in range(expect_tasks_num): curr_task = Task( - name=f"task-{i}", - task_type=f"type-{i}", - process_definition=pd, + name=f"task-{i}", task_type=f"type-{i}", process_definition=pd, ) # Set deps task i as i-1 parent if i > 0: @@ -394,10 +344,7 @@ def test_process_definition_simple_separate(): @pytest.mark.parametrize( - "user_attrs", - [ - {"tenant": "tenant_specific"}, - ], + "user_attrs", [{"tenant": "tenant_specific"},], ) def test_set_process_definition_user_attr(user_attrs): """Test user with correct attributes if we specific assigned to process definition object.""" @@ -420,10 +367,7 @@ def test_set_process_definition_user_attr(user_attrs): def test_schedule_json_none_schedule(): """Test function schedule_json with None as schedule.""" - with ProcessDefinition( - TEST_PROCESS_DEFINITION_NAME, - schedule=None, - ) as pd: + with ProcessDefinition(TEST_PROCESS_DEFINITION_NAME, schedule=None,) as pd: assert pd.schedule_json is None diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py index 9f15b7015bae..ce5fca6b94e1 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py @@ -38,14 +38,14 @@ "attr, expect", [ ( - dict(), - { - "localParams": [], - "resourceList": [], - "dependence": {}, - "waitStartTimeout": {}, - "conditionResult": {"successNode": [""], "failedNode": [""]}, - }, + dict(), + { + "localParams": [], + "resourceList": [], + "dependence": {}, + "waitStartTimeout": {}, + "conditionResult": {"successNode": [""], "failedNode": [""]}, + }, ), ( { @@ -66,20 +66,14 @@ ], ) @patch( - "pydolphinscheduler.core.resource.Resource.get_id_from_database", - return_value=1, + "pydolphinscheduler.core.resource.Resource.get_id_from_database", return_value=1, ) @patch( - "pydolphinscheduler.core.task.Task.user_name", - return_value="test_user", + "pydolphinscheduler.core.task.Task.user_name", return_value="test_user", ) def test_property_task_params(mock_resource, mock_user_name, attr, expect): """Test class task property.""" - task = testTask( - "test-property-task-params", - "test-task", - **attr, - ) + task = testTask("test-property-task-params", "test-task", **attr,) assert expect == task.task_params @@ -171,8 +165,8 @@ def test_task_get_define(): "timeout": 0, } with patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(code, version), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(code, version), ): task = Task(name=name, task_type=task_type) assert task.get_define() == expect @@ -193,12 +187,12 @@ def test_two_tasks_shift(shift: str): else: assert False, f"Unexpect bit operator type {shift}." assert ( - 1 == len(upstream._downstream_task_codes) - and downstream.code in upstream._downstream_task_codes + 1 == len(upstream._downstream_task_codes) + and downstream.code in upstream._downstream_task_codes ), "Task downstream task attributes error, downstream codes size or specific code failed." assert ( - 1 == len(downstream._upstream_task_codes) - and upstream.code in downstream._upstream_task_codes + 1 == len(downstream._upstream_task_codes) + and upstream.code in downstream._upstream_task_codes ), "Task upstream task attributes error, upstream codes size or upstream code failed." @@ -263,8 +257,7 @@ def test_add_duplicate(caplog): ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) @patch( "pydolphinscheduler.core.task.Task.ext", @@ -276,9 +269,15 @@ def test_add_duplicate(caplog): new_callable=PropertyMock, return_value="_raw_script", ) -@patch("pydolphinscheduler.core.task.Task._raw_script", create=True, new_callable=PropertyMock,) +@patch( + "pydolphinscheduler.core.task.Task._raw_script", + create=True, + new_callable=PropertyMock, +) @patch("pydolphinscheduler.core.task.Task.get_plugin") -def test_task_ext_attr(m_plugin, m_raw_script, m_ext_attr, m_ext, m_code_version, val, expected): +def test_task_ext_attr( + m_plugin, m_raw_script, m_ext_attr, m_ext, m_code_version, val, expected +): """Test task attribute ext_attr.""" m_plugin.return_value.read_file.return_value = expected m_raw_script.return_value = val @@ -294,29 +293,26 @@ def test_task_ext_attr(m_plugin, m_raw_script, m_ext_attr, m_ext, m_code_version "name": "test_task_abtain_res_plugin", "task_type": "TaskType", "resource_plugin": ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix="prefix", + type=ResourcePluginType.LOCAL, prefix="prefix", ), "process_definition": ProcessDefinition( name="process_definition", resource_plugin=ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix="prefix", + type=ResourcePluginType.LOCAL, prefix="prefix", ), - ) + ), }, - "Local" + "Local", ), ( { "name": "test_task_abtain_res_plugin", "task_type": "TaskType", "resource_plugin": ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix="prefix", + type=ResourcePluginType.LOCAL, prefix="prefix", ), }, - "Local" + "Local", ), ( { @@ -325,18 +321,16 @@ def test_task_ext_attr(m_plugin, m_raw_script, m_ext_attr, m_ext, m_code_version "process_definition": ProcessDefinition( name="process_definition", resource_plugin=ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix="prefix", + type=ResourcePluginType.LOCAL, prefix="prefix", ), - ) + ), }, - "Local" + "Local", ), ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) @patch("pydolphinscheduler.core.task.Task.get_content") def test_task_obtain_res_plugin(m_get_content, m_code_version, attr, expected): @@ -344,27 +338,26 @@ def test_task_obtain_res_plugin(m_get_content, m_code_version, attr, expected): task = Task(**attr) assert expected == task.get_plugin().__class__.__name__ + @pytest.mark.parametrize( "attr", [ { "name": "test_task_abtain_res_plugin", "task_type": "TaskType", - "process_definition": ProcessDefinition( - name="process_definition", - ) + "process_definition": ProcessDefinition(name="process_definition",), }, ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) @patch("pydolphinscheduler.core.task.Task.get_content") def test_task_obtain_res_plugin_exception(m_get_content, m_code_version, attr): """Test task obtaining resource plug-in exception.""" with pytest.raises( - PyResPluginException, match="The execution command of this task is a file, but the resource plugin is empty" + PyResPluginException, + match="The execution command of this task is a file, but the resource plugin is empty", ): task = Task(**attr) task.get_plugin() @@ -373,35 +366,24 @@ def test_task_obtain_res_plugin_exception(m_get_content, m_code_version, attr): @pytest.mark.parametrize( "resources, expect", [ - ( - ["/dev/test.py"], - [{"id": 1}], - ), - ( - ["/dev/test.py", {"id": 2}], - [{"id": 1}, {"id": 2}], - ), + (["/dev/test.py"], [{"id": 1}],), + (["/dev/test.py", {"id": 2}], [{"id": 1}, {"id": 2}],), ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) @patch( - "pydolphinscheduler.core.resource.Resource.get_id_from_database", - return_value=1, + "pydolphinscheduler.core.resource.Resource.get_id_from_database", return_value=1, ) @patch( - "pydolphinscheduler.core.task.Task.user_name", - return_value="test_user", + "pydolphinscheduler.core.task.Task.user_name", return_value="test_user", ) def test_python_resource_list( mock_code_version, mock_resource, mock_user_name, resources, expect ): """Test python task resource list.""" task = Task( - name="python_resource_list.", - task_type="PYTHON", - resource_list=resources, + name="python_resource_list.", task_type="PYTHON", resource_list=resources, ) assert task.resource_list == expect diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/integration/test_process_definition.py b/dolphinscheduler-python/pydolphinscheduler/tests/integration/test_process_definition.py index 1672bde53004..878926881b00 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/integration/test_process_definition.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/integration/test_process_definition.py @@ -29,17 +29,7 @@ @pytest.mark.parametrize( - "pre, post", - [ - ( - { - "user": "pre_user", - }, - { - "user": "post_user", - }, - ) - ], + "pre, post", [({"user": "pre_user",}, {"user": "post_user",},)], ) def test_change_process_definition_attr(pre: Dict, post: Dict): """Test whether process definition success when specific attribute change.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py index 88263e0d4301..a8e371bddc1e 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py @@ -9,25 +9,20 @@ all_res = ["local"] project_root = Path(__file__).parent.parent.parent -resources_plugin_path = project_root.joinpath("src", "pydolphinscheduler", "resources_plugin") +resources_plugin_path = project_root.joinpath( + "src", "pydolphinscheduler", "resources_plugin" +) @pytest.mark.parametrize( - "attr, expected", - [ - ( - { - "type": "res_type", - "prefix": "res_prefix", - }, - all_res - ) - ], + "attr, expected", [({"type": "res_type", "prefix": "res_prefix",}, all_res)], ) def test_resources_get_all_modules(attr, expected): """Test resource plugin to get all res plugin names""" res = ResourcePlugin(**attr) - assert dict(Counter(expected)) == dict(Counter([ex.stem for ex in res.get_all_modules()])) + assert dict(Counter(expected)) == dict( + Counter([ex.stem for ex in res.get_all_modules()]) + ) @pytest.mark.parametrize( @@ -38,10 +33,10 @@ def test_resources_get_all_modules(attr, expected): "type": ResourcePluginType.LOCAL, "module_attr": { "script_name": "local.py", - "script_path": resources_plugin_path.joinpath("local.py") - } + "script_path": resources_plugin_path.joinpath("local.py"), + }, }, - "Local" + "Local", ), ], ) @@ -51,11 +46,9 @@ def test_resources_import_modules(attrs, expected): res = res_plugin.import_module(**attrs.get("module_attr")) assert expected == res.__class__.__name__ + @pytest.mark.parametrize( - "attr, expected", - [ - (ResourcePluginType.LOCAL, "Local"), - ], + "attr, expected", [(ResourcePluginType.LOCAL, "Local"),], ) def test_resources_resources(attr, expected): """Test resource plugin factory""" @@ -65,13 +58,7 @@ def test_resources_resources(attr, expected): @pytest.mark.parametrize( - "attr", - [ - { - "type": "a", - "prefix": "/tmp/", - } - ], + "attr", [{"type": "a", "prefix": "/tmp/",}], ) def test_resources_unsupported_res(attr): """Test unsupported plug-ins""" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py index 633237d75eae..6ed376b928b8 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py @@ -26,19 +26,15 @@ def setup_crt_first(): @pytest.mark.parametrize( - "val, expected", - [ - (file_name, file_content), - ], + "val, expected", [(file_name, file_content),], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) @patch( "pydolphinscheduler.core.task.Task.ext", new_callable=PropertyMock, - return_value={".sh", }, + return_value={".sh",}, ) @patch( "pydolphinscheduler.core.task.Task.ext_attr", @@ -50,31 +46,24 @@ def setup_crt_first(): create=True, new_callable=PropertyMock, ) -def test_task_obtain_res_plugin(m_raw_script, m_ext_attr, m_ext, m_code_version, val, expected, setup_crt_first): +def test_task_obtain_res_plugin( + m_raw_script, m_ext_attr, m_ext, m_code_version, val, expected, setup_crt_first +): """Test task obtaining resource plug-in.""" m_raw_script.return_value = val task = Task( name="test_task_ext_attr", task_type=ResourcePluginType.LOCAL, resource_plugin=ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix=str(res_plugin_prefix), - ) + type=ResourcePluginType.LOCAL, prefix=str(res_plugin_prefix), + ), ) assert expected == getattr(task, "raw_script") @pytest.mark.parametrize( "attr, expected", - [ - ( - { - "prefix": res_plugin_prefix, - "file_name": file_name - }, - file_content - ) - ], + [({"prefix": res_plugin_prefix, "file_name": file_name}, file_content)], ) def test_local_res_read_file(attr, expected, setup_crt_first): """Test the read_file function of the local resource plug-in""" @@ -84,19 +73,15 @@ def test_local_res_read_file(attr, expected, setup_crt_first): @pytest.mark.parametrize( - "attr", - [ - { - "prefix": res_plugin_prefix, - "file_name": file_name - }, - ], + "attr", [{"prefix": res_plugin_prefix, "file_name": file_name},], ) def test_local_res_file_not_found(attr): """test local resource plugin file does not exist""" with pytest.raises( PyResPluginException, - match="{} is not found".format(Path(attr.get("prefix")).joinpath(attr.get("file_name"))) + match="{} is not found".format( + Path(attr.get("prefix")).joinpath(attr.get("file_name")) + ), ): local = Local(str(attr.get("prefix"))) local.read_file(attr.get("file_name")) diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_condition.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_condition.py index 523264034a85..fed9057ea829 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_condition.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_condition.py @@ -44,12 +44,7 @@ @pytest.mark.parametrize( - "obj, expect", - [ - (Status, "STATUS"), - (SUCCESS, "SUCCESS"), - (FAILURE, "FAILURE"), - ], + "obj, expect", [(Status, "STATUS"), (SUCCESS, "SUCCESS"), (FAILURE, "FAILURE"),], ) def test_class_status_status_name(obj: Status, expect: str): """Test class status and sub class property status_name.""" @@ -89,23 +84,13 @@ def test_class_status_depend_item_list_no_expect_type(obj: Status, tasks: Tuple) def test_class_status_depend_item_list(obj: Status, tasks: Tuple): """Test class status and sub class function :func:`depend_item_list`.""" status = obj.status_name() - expect = [ - { - "depTaskCode": i.code, - "status": status, - } - for i in tasks - ] + expect = [{"depTaskCode": i.code, "status": status,} for i in tasks] assert obj(*tasks).get_define() == expect @pytest.mark.parametrize( "obj, expect", - [ - (ConditionOperator, "CONDITIONOPERATOR"), - (And, "AND"), - (Or, "OR"), - ], + [(ConditionOperator, "CONDITIONOPERATOR"), (And, "AND"), (Or, "OR"),], ) def test_condition_operator_operator_name(obj: ConditionOperator, expect: str): """Test class ConditionOperator and sub class class function :func:`operator_name`.""" @@ -114,11 +99,7 @@ def test_condition_operator_operator_name(obj: ConditionOperator, expect: str): @pytest.mark.parametrize( "obj, expect", - [ - (ConditionOperator, "CONDITIONOPERATOR"), - (And, "AND"), - (Or, "OR"), - ], + [(ConditionOperator, "CONDITIONOPERATOR"), (And, "AND"), (Or, "OR"),], ) def test_condition_operator_relation(obj: ConditionOperator, expect: str): """Test class ConditionOperator and sub class class property `relation`.""" @@ -135,10 +116,7 @@ def test_condition_operator_relation(obj: ConditionOperator, expect: str): ), ( ConditionOperator, - [ - Status(Task("1", TEST_TYPE)), - 1.0, - ], + [Status(Task("1", TEST_TYPE)), 1.0,], ".*?operator parameter support ConditionTask and ConditionOperator.*?", ), ( @@ -262,10 +240,7 @@ def test_condition_operator_set_define_attr_operator( { "relation": obj.operator_name(), "dependItemList": [ - { - "depTaskCode": task.code, - "status": status.status_name(), - } + {"depTaskCode": task.code, "status": status.status_name(),} ], } for _ in range(task_num) @@ -304,10 +279,7 @@ def test_condition_operator_set_define_attr_mix_operator( { "relation": cond.operator_name(), "dependItemList": [ - { - "depTaskCode": task.code, - "status": status.status_name(), - } + {"depTaskCode": task.code, "status": status.status_name(),} ], } ) @@ -317,8 +289,7 @@ def test_condition_operator_set_define_attr_mix_operator( @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(12345, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(12345, 1), ) @patch( "pydolphinscheduler.tasks.condition.Condition.gen_code_and_version", @@ -328,14 +299,8 @@ def test_condition_get_define(mock_condition_code_version, mock_task_code_versio """Test task condition :func:`get_define`.""" common_task = Task(name="common_task", task_type="test_task_condition") cond_operator = And( - And( - SUCCESS(common_task, common_task), - FAILURE(common_task, common_task), - ), - Or( - SUCCESS(common_task, common_task), - FAILURE(common_task, common_task), - ), + And(SUCCESS(common_task, common_task), FAILURE(common_task, common_task),), + Or(SUCCESS(common_task, common_task), FAILURE(common_task, common_task),), ) name = "test_condition_get_define" @@ -395,8 +360,7 @@ def test_condition_get_define(mock_condition_code_version, mock_task_code_versio @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) def test_condition_set_dep_workflow(mock_task_code_version): """Test task condition set dependence in workflow level.""" @@ -404,12 +368,7 @@ def test_condition_set_dep_workflow(mock_task_code_version): pre_task_1 = Task(name="pre_task_1", task_type=TEST_TYPE) pre_task_2 = Task(name="pre_task_2", task_type=TEST_TYPE) pre_task_3 = Task(name="pre_task_3", task_type=TEST_TYPE) - cond_operator = And( - And( - SUCCESS(pre_task_1, pre_task_2), - FAILURE(pre_task_3), - ), - ) + cond_operator = And(And(SUCCESS(pre_task_1, pre_task_2), FAILURE(pre_task_3),),) success_branch = Task(name="success_branch", task_type=TEST_TYPE) fail_branch = Task(name="fail_branch", task_type=TEST_TYPE) @@ -451,10 +410,6 @@ def test_condition_set_dep_workflow(mock_task_code_version): assert all( [ child._downstream_task_codes == {condition.code} - for child in [ - pre_task_1, - pre_task_2, - pre_task_3, - ] + for child in [pre_task_1, pre_task_2, pre_task_3,] ] ) diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_dependent.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_dependent.py index f16e291c8231..2e237fefe0f9 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_dependent.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_dependent.py @@ -124,11 +124,7 @@ def test_dependent_item_date_error(): @pytest.mark.parametrize( - "task_name, result", - [ - ({"dependent_task_name": TEST_TASK}, TEST_TASK), - ({}, None), - ], + "task_name, result", [({"dependent_task_name": TEST_TASK}, TEST_TASK), ({}, None),], ) def test_dependent_item_code_parameter(task_name: dict, result: Optional[str]): """Test dependent item property code_parameter.""" @@ -710,8 +706,7 @@ def test_operator_dependent_task_list_multi_dependent_list( }, ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) def test_dependent_get_define(mock_code_version, mock_dep_code): """Test task dependent function get_define.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_flink.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_flink.py index 92ae3ba91fac..d32c8412d45f 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_flink.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_flink.py @@ -45,9 +45,7 @@ def test_flink_get_define(mock_resource): "taskType": "FLINK", "taskParams": { "mainClass": main_class, - "mainJar": { - "id": 1, - }, + "mainJar": {"id": 1,}, "programType": program_type, "deployMode": deploy_mode, "flinkVersion": FlinkVersion.LOW_VERSION, diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_http.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_http.py index 060cdec0b0d8..0788b043fafd 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_http.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_http.py @@ -68,8 +68,7 @@ def test_attr_exists(class_name, attrs): ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) def test_property_task_params(mock_code_version, attr, expect): """Test task http property.""" @@ -90,8 +89,7 @@ def test_property_task_params(mock_code_version, attr, expect): ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) def test_http_task_param_not_support_param(mock_code, param): """Test HttpTaskParams not support parameter.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_map_reduce.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_map_reduce.py index dbe9e513f5c3..6c26a18e8c9e 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_map_reduce.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_map_reduce.py @@ -45,9 +45,7 @@ def test_mr_get_define(mock_resource): "taskType": "MR", "taskParams": { "mainClass": main_class, - "mainJar": { - "id": 1, - }, + "mainJar": {"id": 1,}, "programType": program_type, "appName": None, "mainArgs": main_args, diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_procedure.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_procedure.py index 17825939555e..74bfd6970450 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_procedure.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_procedure.py @@ -52,8 +52,7 @@ ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) @patch( "pydolphinscheduler.core.database.Database.get_database_info", @@ -66,8 +65,7 @@ def test_property_task_params(mock_datasource, mock_code_version, attr, expect): @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) @patch( "pydolphinscheduler.core.database.Database.get_database_info", diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_python.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_python.py index 1cdd85d2cbfa..9fd143424879 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_python.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_python.py @@ -69,8 +69,7 @@ def foo(): # noqa: D103 ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) def test_property_task_params(mock_code_version, attr, expect): """Test task python property.""" @@ -79,15 +78,10 @@ def test_property_task_params(mock_code_version, attr, expect): @pytest.mark.parametrize( - "script_code", - [ - 123, - ("print", "hello world"), - ], + "script_code", [123, ("print", "hello world"),], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) def test_python_task_not_support_code(mock_code, script_code): """Test python task parameters.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sagemaker.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sagemaker.py index 8838eaf49766..1cf8bf820858 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sagemaker.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sagemaker.py @@ -55,8 +55,7 @@ ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) def test_property_task_params(mock_code_version, attr, expect): """Test task sagemaker task property.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py index fb0328a505f7..add509cad7c9 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py @@ -28,8 +28,8 @@ from pydolphinscheduler.utils import file from tests.testing.file import delete_file -file_path = 'local_res.sh' -file_content = "echo \"test res_local\"" +file_path = "local_res.sh" +file_content = 'echo "test res_local"' res_plugin_prefix = Path(__file__).parent @@ -58,8 +58,7 @@ def setup_crt_first(): ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) def test_property_task_params(mock_code_version, attr, expect): """Test task shell task property.""" @@ -113,19 +112,19 @@ def test_shell_get_define(): "name": "test-local-res-command-content", "command": file_path, "resource_plugin": ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix=res_plugin_prefix - ) + type=ResourcePluginType.LOCAL, prefix=res_plugin_prefix + ), }, - file_content + file_content, ) ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) -def test_resources_local_shell_command_content(mock_code_version, attr, expect, setup_crt_first): +def test_resources_local_shell_command_content( + mock_code_version, attr, expect, setup_crt_first +): """Test task shell task command content through the local resource plug-in.""" task = Shell(**attr) assert expect == getattr(task, "raw_script") diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_spark.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_spark.py index 3b0672f96345..efdb1c1414c6 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_spark.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_spark.py @@ -45,9 +45,7 @@ def test_spark_get_define(mock_resource): "taskType": "SPARK", "taskParams": { "mainClass": main_class, - "mainJar": { - "id": 1, - }, + "mainJar": {"id": 1,}, "programType": program_type, "deployMode": deploy_mode, "sparkVersion": SparkVersion.SPARK2, diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sql.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sql.py index ee0acc442e11..f7b5d78ad34e 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sql.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sql.py @@ -61,8 +61,7 @@ ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) @patch( "pydolphinscheduler.core.database.Database.get_database_info", @@ -103,8 +102,7 @@ def test_get_sql_type( ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) @patch( "pydolphinscheduler.core.database.Database.get_database_info", diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sub_process.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sub_process.py index 7f471a1b8b25..781db341b63e 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sub_process.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sub_process.py @@ -57,8 +57,7 @@ ), ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) def test_property_task_params(mock_code_version, mock_pd_info, attr, expect): """Test task sub process property.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_switch.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_switch.py index 1f6ff5bfa2dd..93028ecf6ec4 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_switch.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_switch.py @@ -48,12 +48,7 @@ def task_switch_arg_wrapper(obj, task: Task, exp: Optional[str] = None) -> Switc @pytest.mark.parametrize( - "obj", - [ - SwitchBranch, - Branch, - Default, - ], + "obj", [SwitchBranch, Branch, Default,], ) def test_switch_branch_attr_next_node(obj: SwitchBranch): """Test get attribute from class switch branch.""" @@ -63,11 +58,7 @@ def test_switch_branch_attr_next_node(obj: SwitchBranch): @pytest.mark.parametrize( - "obj", - [ - SwitchBranch, - Default, - ], + "obj", [SwitchBranch, Default,], ) def test_switch_branch_get_define_without_condition(obj: SwitchBranch): """Test function :func:`get_define` with None value of attribute condition from class switch branch.""" @@ -78,11 +69,7 @@ def test_switch_branch_get_define_without_condition(obj: SwitchBranch): @pytest.mark.parametrize( - "obj", - [ - SwitchBranch, - Branch, - ], + "obj", [SwitchBranch, Branch,], ) def test_switch_branch_get_define_condition(obj: SwitchBranch): """Test function :func:`get_define` with specific attribute condition from class switch branch.""" @@ -99,10 +86,7 @@ def test_switch_branch_get_define_condition(obj: SwitchBranch): @pytest.mark.parametrize( "args, msg", [ - ( - (1,), - ".*?parameter only support SwitchBranch but got.*?", - ), + ((1,), ".*?parameter only support SwitchBranch but got.*?",), ( (Default(Task(TEST_NAME, TEST_TYPE)), 2), ".*?parameter only support SwitchBranch but got.*?", @@ -213,8 +197,7 @@ def test_switch_condition_get_define_mix_branch_and_default(): @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) def test_switch_get_define(mock_task_code_version): """Test task switch :func:`get_define`.""" @@ -262,8 +245,7 @@ def test_switch_get_define(mock_task_code_version): @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", - return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), ) def test_switch_set_dep_workflow(mock_task_code_version): """Test task switch set dependence in workflow level.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_yaml_parser.py b/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_yaml_parser.py index ad3aaf7bd150..3c14f6a1166a 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_yaml_parser.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_yaml_parser.py @@ -84,18 +84,7 @@ @pytest.mark.parametrize( "src, delimiter, expect", - [ - ( - param[0], - "|", - expects[0], - ), - ( - param[1], - "/", - expects[1], - ), - ], + [(param[0], "|", expects[0],), (param[1], "/", expects[1],),], ) def test_yaml_parser_specific_delimiter(src: str, delimiter: str, expect: Dict): """Test specific delimiter for :class:`YamlParser`.""" @@ -121,17 +110,7 @@ def ch_dl(key): @pytest.mark.parametrize( - "src, expect", - [ - ( - param[0], - expects[0], - ), - ( - param[1], - expects[1], - ), - ], + "src, expect", [(param[0], expects[0],), (param[1], expects[1],),], ) def test_yaml_parser_contains(src: str, expect: Dict): """Test magic function :func:`YamlParser.__contain__` also with `key in obj` syntax.""" @@ -145,17 +124,7 @@ def test_yaml_parser_contains(src: str, expect: Dict): @pytest.mark.parametrize( - "src, expect", - [ - ( - param[0], - expects[0], - ), - ( - param[1], - expects[1], - ), - ], + "src, expect", [(param[0], expects[0],), (param[1], expects[1],),], ) def test_yaml_parser_get(src: str, expect: Dict): """Test magic function :func:`YamlParser.__getitem__` also with `obj[key]` syntax.""" @@ -177,17 +146,7 @@ def test_yaml_parser_get(src: str, expect: Dict): @pytest.mark.parametrize( - "src, expect", - [ - ( - param[0], - expects[0], - ), - ( - param[1], - expects[1], - ), - ], + "src, expect", [(param[0], expects[0],), (param[1], expects[1],),], ) def test_yaml_parser_set(src: str, expect: Dict): """Test magic function :func:`YamlParser.__setitem__` also with `obj[key] = val` syntax.""" From 4ba1c84c6a86687e2dc28bbf0be41af97b010e71 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Fri, 12 Aug 2022 00:06:01 +0800 Subject: [PATCH 39/58] fix tests/resource_plugin missing a valid license header --- .../tests/resources_plugin/__init__.py | 18 ++++++++++++++++++ .../tests/resources_plugin/test_init.py | 18 ++++++++++++++++++ .../tests/resources_plugin/test_local.py | 18 ++++++++++++++++++ 3 files changed, 54 insertions(+) diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/__init__.py b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/__init__.py index e69de29bb2d1..0b6bdf360bbc 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/__init__.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/__init__.py @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Init resources_plugin package tests.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py index a8e371bddc1e..5c24113a6835 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py @@ -1,3 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Test ResourcePlugin.""" from collections import Counter from pathlib import Path diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py index 6ed376b928b8..1fb3788df91c 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py @@ -1,3 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Test local resource plugin.""" from pathlib import Path from unittest.mock import PropertyMock, patch From af167a4d69aaca40c3791c670d542c2342d62b0b Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Sat, 13 Aug 2022 23:04:29 +0800 Subject: [PATCH 40/58] fix black code style --- .../pydolphinscheduler/setup.py | 15 +++- .../src/pydolphinscheduler/core/engine.py | 4 +- .../core/process_definition.py | 7 +- .../src/pydolphinscheduler/core/resource.py | 5 +- .../src/pydolphinscheduler/core/task.py | 5 +- .../examples/task_condition_example.py | 7 +- .../examples/task_datax_example.py | 5 +- .../examples/task_dependent_example.py | 10 ++- .../examples/task_sagemaker_example.py | 5 +- .../examples/tutorial_resource_plugin.py | 25 ++++-- .../src/pydolphinscheduler/models/queue.py | 4 +- .../resources_plugin/__init__.py | 6 +- .../tests/cli/test_config.py | 7 +- .../tests/core/test_configuration.py | 30 ++++++- .../tests/core/test_database.py | 11 ++- .../tests/core/test_engine.py | 17 ++-- .../tests/core/test_process_definition.py | 86 +++++++++++++++---- .../tests/core/test_task.py | 60 +++++++++---- .../integration/test_process_definition.py | 12 ++- .../tests/resources_plugin/test_init.py | 24 +++++- .../tests/resources_plugin/test_local.py | 20 +++-- .../tests/tasks/test_condition.py | 71 ++++++++++++--- .../tests/tasks/test_dependent.py | 9 +- .../tests/tasks/test_flink.py | 4 +- .../tests/tasks/test_http.py | 6 +- .../tests/tasks/test_map_reduce.py | 4 +- .../tests/tasks/test_procedure.py | 6 +- .../tests/tasks/test_python.py | 12 ++- .../tests/tasks/test_sagemaker.py | 3 +- .../tests/tasks/test_shell.py | 6 +- .../tests/tasks/test_spark.py | 4 +- .../tests/tasks/test_sql.py | 6 +- .../tests/tasks/test_sub_process.py | 3 +- .../tests/tasks/test_switch.py | 30 +++++-- .../tests/utils/test_yaml_parser.py | 49 ++++++++++- 35 files changed, 463 insertions(+), 115 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/setup.py b/dolphinscheduler-python/pydolphinscheduler/setup.py index e7f0555408d9..d6228a7693ba 100644 --- a/dolphinscheduler-python/pydolphinscheduler/setup.py +++ b/dolphinscheduler-python/pydolphinscheduler/setup.py @@ -131,7 +131,12 @@ def run(self) -> None: author_email="dev@dolphinscheduler.apache.org", url="https://dolphinscheduler.apache.org/", python_requires=">=3.6", - keywords=["dolphinscheduler", "workflow", "scheduler", "taskflow",], + keywords=[ + "dolphinscheduler", + "workflow", + "scheduler", + "taskflow", + ], project_urls={ "Homepage": "https://dolphinscheduler.apache.org", "Documentation": "https://dolphinscheduler.apache.org/python/index.html", @@ -145,7 +150,9 @@ def run(self) -> None: packages=find_packages(where="src"), package_dir={"": "src"}, include_package_data=True, - package_data={"pydolphinscheduler": ["default_config.yaml"],}, + package_data={ + "pydolphinscheduler": ["default_config.yaml"], + }, platforms=["any"], classifiers=[ # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers @@ -177,7 +184,9 @@ def run(self) -> None: "doc": doc, "build": build, }, - cmdclass={"pre_clean": CleanCommand,}, + cmdclass={ + "pre_clean": CleanCommand, + }, entry_points={ "console_scripts": [ "pydolphinscheduler = pydolphinscheduler.cli.commands:cli", diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/engine.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/engine.py index 911538bd7d02..41021ed474e8 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/engine.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/engine.py @@ -86,7 +86,9 @@ def task_params(self, camel_attr: bool = True, custom_attr: set = None) -> Dict: custom_params = { "programType": self.program_type, "mainClass": self.main_class, - "mainJar": {"id": self.get_jar_id(),}, + "mainJar": { + "id": self.get_jar_id(), + }, } params.update(custom_params) return params diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py index addd02522965..1d3dff693fd8 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py @@ -232,7 +232,12 @@ def param_json(self) -> Optional[List[Dict]]: if not self.param: return [] return [ - {"prop": k, "direct": "IN", "type": "VARCHAR", "value": v,} + { + "prop": k, + "direct": "IN", + "type": "VARCHAR", + "value": v, + } for k, v in self.param.items() ] diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/resource.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/resource.py index bb1c86869237..ea811915e280 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/resource.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/resource.py @@ -66,5 +66,8 @@ def create_or_update_resource(self): "`user_name` and `content` are required when create or update resource from python gate." ) JavaGate().create_or_update_resource( - self.user_name, self.name, self.content, self.description, + self.user_name, + self.name, + self.content, + self.description, ) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py index 0f990e9257fe..4659b80eb2d2 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py @@ -65,7 +65,10 @@ class TaskRelation(Base): } def __init__( - self, pre_task_code: int, post_task_code: int, name: Optional[str] = None, + self, + pre_task_code: int, + post_task_code: int, + name: Optional[str] = None, ): super().__init__(name) self.pre_task_code = pre_task_code diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_condition_example.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_condition_example.py index 7208a195cba6..2d73df4b40d5 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_condition_example.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_condition_example.py @@ -39,7 +39,12 @@ pre_task_1 = Shell(name="pre_task_1", command="echo pre_task_1") pre_task_2 = Shell(name="pre_task_2", command="echo pre_task_2") pre_task_3 = Shell(name="pre_task_3", command="echo pre_task_3") - cond_operator = And(And(SUCCESS(pre_task_1, pre_task_2), FAILURE(pre_task_3),),) + cond_operator = And( + And( + SUCCESS(pre_task_1, pre_task_2), + FAILURE(pre_task_3), + ), + ) success_branch = Shell(name="success_branch", command="echo success_branch") fail_branch = Shell(name="fail_branch", command="echo fail_branch") diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_datax_example.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_datax_example.py index 7850b294ee53..94bd449cf723 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_datax_example.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_datax_example.py @@ -72,7 +72,10 @@ } } -with ProcessDefinition(name="task_datax_example", tenant="tenant_exists",) as pd: +with ProcessDefinition( + name="task_datax_example", + tenant="tenant_exists", +) as pd: # This task synchronizes the data in `t_ds_project` # of `first_mysql` database to `target_project` of `second_mysql` database. # You have to make sure data source named `first_mysql` and `second_mysql` exists diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_dependent_example.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_dependent_example.py index 497097e769b2..db53bcc9f38e 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_dependent_example.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_dependent_example.py @@ -40,13 +40,19 @@ from pydolphinscheduler.tasks.dependent import And, Dependent, DependentItem, Or from pydolphinscheduler.tasks.shell import Shell -with ProcessDefinition(name="task_dependent_external", tenant="tenant_exists",) as pd: +with ProcessDefinition( + name="task_dependent_external", + tenant="tenant_exists", +) as pd: task_1 = Shell(name="task_1", command="echo task 1") task_2 = Shell(name="task_2", command="echo task 2") task_3 = Shell(name="task_3", command="echo task 3") pd.submit() -with ProcessDefinition(name="task_dependent_example", tenant="tenant_exists",) as pd: +with ProcessDefinition( + name="task_dependent_example", + tenant="tenant_exists", +) as pd: task = Dependent( name="task_dependent", dependence=And( diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_sagemaker_example.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_sagemaker_example.py index b5c19620a830..b056f61a6384 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_sagemaker_example.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_sagemaker_example.py @@ -33,7 +33,10 @@ ], } -with ProcessDefinition(name="task_sagemaker_example", tenant="tenant_exists",) as pd: +with ProcessDefinition( + name="task_sagemaker_example", + tenant="tenant_exists", +) as pd: task_sagemaker = SageMaker( name="task_sagemaker", sagemaker_request_json=json.dumps(sagemaker_request_data, indent=2), diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py index 8c126bf73752..374f83d80209 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py @@ -56,28 +56,43 @@ schedule="0 0 0 * * ? *", start_time="2021-01-01", tenant="tenant_exists", - resource_plugin=ResourcePlugin(type=ResourcePluginType.LOCAL, prefix="/opt/",), + resource_plugin=ResourcePlugin( + type=ResourcePluginType.LOCAL, + prefix="/opt/", + ), ) as pd: # [end workflow_declare] # [start task_declare] task_parent = Shell( name="task_parent", command="parent.zsh", - resource_plugin=ResourcePlugin(type=ResourcePluginType.LOCAL, prefix="/opt/",), + resource_plugin=ResourcePlugin( + type=ResourcePluginType.LOCAL, + prefix="/opt/", + ), ) task_child_one = Shell( name="task_child_one", command="child_one.sh", - resource_plugin=ResourcePlugin(type=ResourcePluginType.LOCAL, prefix="/opt/",), + resource_plugin=ResourcePlugin( + type=ResourcePluginType.LOCAL, + prefix="/opt/", + ), ) task_child_two = Shell( name="task_child_two", command="child_two.sh", - resource_plugin=ResourcePlugin(type=ResourcePluginType.LOCAL, prefix="/opt/",), + resource_plugin=ResourcePlugin( + type=ResourcePluginType.LOCAL, + prefix="/opt/", + ), + ) + task_union = Shell( + name="task_union", + command="echo union", ) - task_union = Shell(name="task_union", command="echo union",) # [end task_declare] # [start task_relation_declare] diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/queue.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/queue.py index 2478838e8025..e6da2594c8ff 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/queue.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/queue.py @@ -27,6 +27,8 @@ class Queue(BaseSide): """DolphinScheduler Queue object.""" def __init__( - self, name: str = configuration.WORKFLOW_QUEUE, description: Optional[str] = "", + self, + name: str = configuration.WORKFLOW_QUEUE, + description: Optional[str] = "", ): super().__init__(name, description) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py index 39c21eb0c428..fd0cb880e798 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py @@ -30,9 +30,9 @@ class ResourcePlugin: """ResourcePlugin object, declare resource plugin for task and workflow to dolphinscheduler. - :param type: A unique, meaningful string for the ResourcePlugin, - Its value should be taken from the constant of ResourceType in constants.py. - :param prefix: A string representing the prefix of ResourcePlugin. + :param type: A unique, meaningful string for the ResourcePlugin, + Its value should be taken from the constant of ResourceType in constants.py. + :param prefix: A string representing the prefix of ResourcePlugin. """ diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/cli/test_config.py b/dolphinscheduler-python/pydolphinscheduler/tests/cli/test_config.py index a09cab80af89..516ad754a20c 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/cli/test_config.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/cli/test_config.py @@ -44,7 +44,12 @@ def teardown_file_env(): @pytest.mark.parametrize( - "home", [None, "/tmp/pydolphinscheduler", "/tmp/test_abc",], + "home", + [ + None, + "/tmp/pydolphinscheduler", + "/tmp/test_abc", + ], ) def test_config_init(teardown_file_env, home): """Test command line interface `config --init`.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_configuration.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_configuration.py index 1fa4934e339a..b9dc8cb65645 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_configuration.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_configuration.py @@ -48,7 +48,13 @@ def teardown_file_env(): @pytest.mark.parametrize( - "val, expect", [("1", 1), ("123", 123), ("4567", 4567), (b"1234", 1234),], + "val, expect", + [ + ("1", 1), + ("123", 123), + ("4567", 4567), + (b"1234", 1234), + ], ) def test_get_int(val: Any, expect: int): """Test function :func:`configuration.get_int`.""" @@ -56,7 +62,13 @@ def test_get_int(val: Any, expect: int): @pytest.mark.parametrize( - "val", ["a", "1a", "1d2", "1723-",], + "val", + [ + "a", + "1a", + "1d2", + "1723-", + ], ) def test_get_int_error(val: Any): """Test function :func:`configuration.get_int`.""" @@ -101,7 +113,12 @@ def test_config_path(home: Any, expect: str): @pytest.mark.parametrize( - "home", [None, "/tmp/pydolphinscheduler", "/tmp/test_abc",], + "home", + [ + None, + "/tmp/pydolphinscheduler", + "/tmp/test_abc", + ], ) def test_init_config_file(teardown_file_env, home: Any): """Test init config file.""" @@ -119,7 +136,12 @@ def test_init_config_file(teardown_file_env, home: Any): @pytest.mark.parametrize( - "home", [None, "/tmp/pydolphinscheduler", "/tmp/test_abc",], + "home", + [ + None, + "/tmp/pydolphinscheduler", + "/tmp/test_abc", + ], ) def test_init_config_file_duplicate(teardown_file_env, home: Any): """Test raise error with init config file which already exists.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_database.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_database.py index f0e2927f6d90..1286a4a7f8b0 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_database.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_database.py @@ -30,10 +30,17 @@ @pytest.mark.parametrize( - "expect", [{TEST_DATABASE_TYPE_KEY: "mock_type", TEST_DATABASE_KEY: 1,}], + "expect", + [ + { + TEST_DATABASE_TYPE_KEY: "mock_type", + TEST_DATABASE_KEY: 1, + } + ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) @patch( "pydolphinscheduler.core.database.Database.get_database_info", diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_engine.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_engine.py index fd5b35f77357..e36c47ba1b64 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_engine.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_engine.py @@ -31,7 +31,8 @@ @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) @patch( "pydolphinscheduler.core.engine.Engine.get_resource_info", @@ -63,7 +64,9 @@ def test_get_jar_detail(mock_resource, mock_code_version): }, { "mainClass": "org.apache.examples.mock.Mock", - "mainJar": {"id": 1,}, + "mainJar": { + "id": 1, + }, "programType": ProgramType.JAVA, "localParams": [], "resourceList": [], @@ -75,7 +78,8 @@ def test_get_jar_detail(mock_resource, mock_code_version): ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) @patch( "pydolphinscheduler.core.engine.Engine.get_resource_info", @@ -107,7 +111,9 @@ def test_property_task_params(mock_resource, mock_code_version, attr, expect): "taskType": "test-engine", "taskParams": { "mainClass": "org.apache.examples.mock.Mock", - "mainJar": {"id": 1,}, + "mainJar": { + "id": 1, + }, "programType": ProgramType.JAVA, "localParams": [], "resourceList": [], @@ -128,7 +134,8 @@ def test_property_task_params(mock_resource, mock_code_version, attr, expect): ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) @patch( "pydolphinscheduler.core.engine.Engine.get_resource_info", diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py index a6510ac4d92f..30445bfbf356 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_process_definition.py @@ -108,7 +108,11 @@ def test_set_attr(name, cls, expect): @pytest.mark.parametrize( - "value,expect", [("online", 1), ("offline", 0),], + "value,expect", + [ + ("online", 1), + ("offline", 0), + ], ) def test_set_release_state(value, expect): """Test process definition set release_state attributes.""" @@ -119,7 +123,14 @@ def test_set_release_state(value, expect): @pytest.mark.parametrize( - "value", ["oneline", "offeline", 1, 0, None,], + "value", + [ + "oneline", + "offeline", + 1, + 0, + None, + ], ) def test_set_release_state_error(value): """Test process definition set release_state attributes with error.""" @@ -170,7 +181,12 @@ def test__parse_datetime(val, expect): @pytest.mark.parametrize( - "val", [20210101, (2021, 1, 1), {"year": "2021", "month": "1", "day": 1},], + "val", + [ + 20210101, + (2021, 1, 1), + {"year": "2021", "month": "1", "day": 1}, + ], ) def test__parse_datetime_not_support_type(val: Any): """Test process definition function _parse_datetime not support type error.""" @@ -180,7 +196,11 @@ def test__parse_datetime_not_support_type(val: Any): @pytest.mark.parametrize( - "val", ["ALLL", "nonee",], + "val", + [ + "ALLL", + "nonee", + ], ) def test_warn_type_not_support_type(val: str): """Test process definition param warning_type not support type error.""" @@ -193,17 +213,43 @@ def test_warn_type_not_support_type(val: str): @pytest.mark.parametrize( "param, expect", [ - (None, [],), - ({}, [],), + ( + None, + [], + ), + ( + {}, + [], + ), ( {"key1": "val1"}, - [{"prop": "key1", "direct": "IN", "type": "VARCHAR", "value": "val1",}], + [ + { + "prop": "key1", + "direct": "IN", + "type": "VARCHAR", + "value": "val1", + } + ], ), ( - {"key1": "val1", "key2": "val2",}, + { + "key1": "val1", + "key2": "val2", + }, [ - {"prop": "key1", "direct": "IN", "type": "VARCHAR", "value": "val1",}, - {"prop": "key2", "direct": "IN", "type": "VARCHAR", "value": "val2",}, + { + "prop": "key1", + "direct": "IN", + "type": "VARCHAR", + "value": "val1", + }, + { + "prop": "key2", + "direct": "IN", + "type": "VARCHAR", + "value": "val2", + }, ], ), ], @@ -215,7 +261,8 @@ def test_property_param_json(param, expect): @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) def test__pre_submit_check_switch_without_param(mock_code_version): """Test :func:`_pre_submit_check` if process definition with switch but without attribute param.""" @@ -239,7 +286,8 @@ def test__pre_submit_check_switch_without_param(mock_code_version): @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) def test__pre_submit_check_switch_with_local_params(mock_code_version): """Test :func:`_pre_submit_check` if process definition with switch with local params of task.""" @@ -333,7 +381,9 @@ def test_process_definition_simple_separate(): pd = ProcessDefinition(TEST_PROCESS_DEFINITION_NAME) for i in range(expect_tasks_num): curr_task = Task( - name=f"task-{i}", task_type=f"type-{i}", process_definition=pd, + name=f"task-{i}", + task_type=f"type-{i}", + process_definition=pd, ) # Set deps task i as i-1 parent if i > 0: @@ -344,7 +394,10 @@ def test_process_definition_simple_separate(): @pytest.mark.parametrize( - "user_attrs", [{"tenant": "tenant_specific"},], + "user_attrs", + [ + {"tenant": "tenant_specific"}, + ], ) def test_set_process_definition_user_attr(user_attrs): """Test user with correct attributes if we specific assigned to process definition object.""" @@ -367,7 +420,10 @@ def test_set_process_definition_user_attr(user_attrs): def test_schedule_json_none_schedule(): """Test function schedule_json with None as schedule.""" - with ProcessDefinition(TEST_PROCESS_DEFINITION_NAME, schedule=None,) as pd: + with ProcessDefinition( + TEST_PROCESS_DEFINITION_NAME, + schedule=None, + ) as pd: assert pd.schedule_json is None diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py index ce5fca6b94e1..c784810804ef 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py @@ -66,14 +66,20 @@ ], ) @patch( - "pydolphinscheduler.core.resource.Resource.get_id_from_database", return_value=1, + "pydolphinscheduler.core.resource.Resource.get_id_from_database", + return_value=1, ) @patch( - "pydolphinscheduler.core.task.Task.user_name", return_value="test_user", + "pydolphinscheduler.core.task.Task.user_name", + return_value="test_user", ) def test_property_task_params(mock_resource, mock_user_name, attr, expect): """Test class task property.""" - task = testTask("test-property-task-params", "test-task", **attr,) + task = testTask( + "test-property-task-params", + "test-task", + **attr, + ) assert expect == task.task_params @@ -257,7 +263,8 @@ def test_add_duplicate(caplog): ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) @patch( "pydolphinscheduler.core.task.Task.ext", @@ -293,12 +300,14 @@ def test_task_ext_attr( "name": "test_task_abtain_res_plugin", "task_type": "TaskType", "resource_plugin": ResourcePlugin( - type=ResourcePluginType.LOCAL, prefix="prefix", + type=ResourcePluginType.LOCAL, + prefix="prefix", ), "process_definition": ProcessDefinition( name="process_definition", resource_plugin=ResourcePlugin( - type=ResourcePluginType.LOCAL, prefix="prefix", + type=ResourcePluginType.LOCAL, + prefix="prefix", ), ), }, @@ -309,7 +318,8 @@ def test_task_ext_attr( "name": "test_task_abtain_res_plugin", "task_type": "TaskType", "resource_plugin": ResourcePlugin( - type=ResourcePluginType.LOCAL, prefix="prefix", + type=ResourcePluginType.LOCAL, + prefix="prefix", ), }, "Local", @@ -321,7 +331,8 @@ def test_task_ext_attr( "process_definition": ProcessDefinition( name="process_definition", resource_plugin=ResourcePlugin( - type=ResourcePluginType.LOCAL, prefix="prefix", + type=ResourcePluginType.LOCAL, + prefix="prefix", ), ), }, @@ -330,7 +341,8 @@ def test_task_ext_attr( ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) @patch("pydolphinscheduler.core.task.Task.get_content") def test_task_obtain_res_plugin(m_get_content, m_code_version, attr, expected): @@ -345,12 +357,15 @@ def test_task_obtain_res_plugin(m_get_content, m_code_version, attr, expected): { "name": "test_task_abtain_res_plugin", "task_type": "TaskType", - "process_definition": ProcessDefinition(name="process_definition",), + "process_definition": ProcessDefinition( + name="process_definition", + ), }, ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) @patch("pydolphinscheduler.core.task.Task.get_content") def test_task_obtain_res_plugin_exception(m_get_content, m_code_version, attr): @@ -366,24 +381,35 @@ def test_task_obtain_res_plugin_exception(m_get_content, m_code_version, attr): @pytest.mark.parametrize( "resources, expect", [ - (["/dev/test.py"], [{"id": 1}],), - (["/dev/test.py", {"id": 2}], [{"id": 1}, {"id": 2}],), + ( + ["/dev/test.py"], + [{"id": 1}], + ), + ( + ["/dev/test.py", {"id": 2}], + [{"id": 1}, {"id": 2}], + ), ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) @patch( - "pydolphinscheduler.core.resource.Resource.get_id_from_database", return_value=1, + "pydolphinscheduler.core.resource.Resource.get_id_from_database", + return_value=1, ) @patch( - "pydolphinscheduler.core.task.Task.user_name", return_value="test_user", + "pydolphinscheduler.core.task.Task.user_name", + return_value="test_user", ) def test_python_resource_list( mock_code_version, mock_resource, mock_user_name, resources, expect ): """Test python task resource list.""" task = Task( - name="python_resource_list.", task_type="PYTHON", resource_list=resources, + name="python_resource_list.", + task_type="PYTHON", + resource_list=resources, ) assert task.resource_list == expect diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/integration/test_process_definition.py b/dolphinscheduler-python/pydolphinscheduler/tests/integration/test_process_definition.py index 878926881b00..1672bde53004 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/integration/test_process_definition.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/integration/test_process_definition.py @@ -29,7 +29,17 @@ @pytest.mark.parametrize( - "pre, post", [({"user": "pre_user",}, {"user": "post_user",},)], + "pre, post", + [ + ( + { + "user": "pre_user", + }, + { + "user": "post_user", + }, + ) + ], ) def test_change_process_definition_attr(pre: Dict, post: Dict): """Test whether process definition success when specific attribute change.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py index 5c24113a6835..761c001fdad7 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py @@ -33,7 +33,16 @@ @pytest.mark.parametrize( - "attr, expected", [({"type": "res_type", "prefix": "res_prefix",}, all_res)], + "attr, expected", + [ + ( + { + "type": "res_type", + "prefix": "res_prefix", + }, + all_res, + ) + ], ) def test_resources_get_all_modules(attr, expected): """Test resource plugin to get all res plugin names""" @@ -66,7 +75,10 @@ def test_resources_import_modules(attrs, expected): @pytest.mark.parametrize( - "attr, expected", [(ResourcePluginType.LOCAL, "Local"),], + "attr, expected", + [ + (ResourcePluginType.LOCAL, "Local"), + ], ) def test_resources_resources(attr, expected): """Test resource plugin factory""" @@ -76,7 +88,13 @@ def test_resources_resources(attr, expected): @pytest.mark.parametrize( - "attr", [{"type": "a", "prefix": "/tmp/",}], + "attr", + [ + { + "type": "a", + "prefix": "/tmp/", + } + ], ) def test_resources_unsupported_res(attr): """Test unsupported plug-ins""" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py index 1fb3788df91c..bf5e43920350 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py @@ -44,15 +44,21 @@ def setup_crt_first(): @pytest.mark.parametrize( - "val, expected", [(file_name, file_content),], + "val, expected", + [ + (file_name, file_content), + ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) @patch( "pydolphinscheduler.core.task.Task.ext", new_callable=PropertyMock, - return_value={".sh",}, + return_value={ + ".sh", + }, ) @patch( "pydolphinscheduler.core.task.Task.ext_attr", @@ -73,7 +79,8 @@ def test_task_obtain_res_plugin( name="test_task_ext_attr", task_type=ResourcePluginType.LOCAL, resource_plugin=ResourcePlugin( - type=ResourcePluginType.LOCAL, prefix=str(res_plugin_prefix), + type=ResourcePluginType.LOCAL, + prefix=str(res_plugin_prefix), ), ) assert expected == getattr(task, "raw_script") @@ -91,7 +98,10 @@ def test_local_res_read_file(attr, expected, setup_crt_first): @pytest.mark.parametrize( - "attr", [{"prefix": res_plugin_prefix, "file_name": file_name},], + "attr", + [ + {"prefix": res_plugin_prefix, "file_name": file_name}, + ], ) def test_local_res_file_not_found(attr): """test local resource plugin file does not exist""" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_condition.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_condition.py index fed9057ea829..523264034a85 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_condition.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_condition.py @@ -44,7 +44,12 @@ @pytest.mark.parametrize( - "obj, expect", [(Status, "STATUS"), (SUCCESS, "SUCCESS"), (FAILURE, "FAILURE"),], + "obj, expect", + [ + (Status, "STATUS"), + (SUCCESS, "SUCCESS"), + (FAILURE, "FAILURE"), + ], ) def test_class_status_status_name(obj: Status, expect: str): """Test class status and sub class property status_name.""" @@ -84,13 +89,23 @@ def test_class_status_depend_item_list_no_expect_type(obj: Status, tasks: Tuple) def test_class_status_depend_item_list(obj: Status, tasks: Tuple): """Test class status and sub class function :func:`depend_item_list`.""" status = obj.status_name() - expect = [{"depTaskCode": i.code, "status": status,} for i in tasks] + expect = [ + { + "depTaskCode": i.code, + "status": status, + } + for i in tasks + ] assert obj(*tasks).get_define() == expect @pytest.mark.parametrize( "obj, expect", - [(ConditionOperator, "CONDITIONOPERATOR"), (And, "AND"), (Or, "OR"),], + [ + (ConditionOperator, "CONDITIONOPERATOR"), + (And, "AND"), + (Or, "OR"), + ], ) def test_condition_operator_operator_name(obj: ConditionOperator, expect: str): """Test class ConditionOperator and sub class class function :func:`operator_name`.""" @@ -99,7 +114,11 @@ def test_condition_operator_operator_name(obj: ConditionOperator, expect: str): @pytest.mark.parametrize( "obj, expect", - [(ConditionOperator, "CONDITIONOPERATOR"), (And, "AND"), (Or, "OR"),], + [ + (ConditionOperator, "CONDITIONOPERATOR"), + (And, "AND"), + (Or, "OR"), + ], ) def test_condition_operator_relation(obj: ConditionOperator, expect: str): """Test class ConditionOperator and sub class class property `relation`.""" @@ -116,7 +135,10 @@ def test_condition_operator_relation(obj: ConditionOperator, expect: str): ), ( ConditionOperator, - [Status(Task("1", TEST_TYPE)), 1.0,], + [ + Status(Task("1", TEST_TYPE)), + 1.0, + ], ".*?operator parameter support ConditionTask and ConditionOperator.*?", ), ( @@ -240,7 +262,10 @@ def test_condition_operator_set_define_attr_operator( { "relation": obj.operator_name(), "dependItemList": [ - {"depTaskCode": task.code, "status": status.status_name(),} + { + "depTaskCode": task.code, + "status": status.status_name(), + } ], } for _ in range(task_num) @@ -279,7 +304,10 @@ def test_condition_operator_set_define_attr_mix_operator( { "relation": cond.operator_name(), "dependItemList": [ - {"depTaskCode": task.code, "status": status.status_name(),} + { + "depTaskCode": task.code, + "status": status.status_name(), + } ], } ) @@ -289,7 +317,8 @@ def test_condition_operator_set_define_attr_mix_operator( @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(12345, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(12345, 1), ) @patch( "pydolphinscheduler.tasks.condition.Condition.gen_code_and_version", @@ -299,8 +328,14 @@ def test_condition_get_define(mock_condition_code_version, mock_task_code_versio """Test task condition :func:`get_define`.""" common_task = Task(name="common_task", task_type="test_task_condition") cond_operator = And( - And(SUCCESS(common_task, common_task), FAILURE(common_task, common_task),), - Or(SUCCESS(common_task, common_task), FAILURE(common_task, common_task),), + And( + SUCCESS(common_task, common_task), + FAILURE(common_task, common_task), + ), + Or( + SUCCESS(common_task, common_task), + FAILURE(common_task, common_task), + ), ) name = "test_condition_get_define" @@ -360,7 +395,8 @@ def test_condition_get_define(mock_condition_code_version, mock_task_code_versio @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) def test_condition_set_dep_workflow(mock_task_code_version): """Test task condition set dependence in workflow level.""" @@ -368,7 +404,12 @@ def test_condition_set_dep_workflow(mock_task_code_version): pre_task_1 = Task(name="pre_task_1", task_type=TEST_TYPE) pre_task_2 = Task(name="pre_task_2", task_type=TEST_TYPE) pre_task_3 = Task(name="pre_task_3", task_type=TEST_TYPE) - cond_operator = And(And(SUCCESS(pre_task_1, pre_task_2), FAILURE(pre_task_3),),) + cond_operator = And( + And( + SUCCESS(pre_task_1, pre_task_2), + FAILURE(pre_task_3), + ), + ) success_branch = Task(name="success_branch", task_type=TEST_TYPE) fail_branch = Task(name="fail_branch", task_type=TEST_TYPE) @@ -410,6 +451,10 @@ def test_condition_set_dep_workflow(mock_task_code_version): assert all( [ child._downstream_task_codes == {condition.code} - for child in [pre_task_1, pre_task_2, pre_task_3,] + for child in [ + pre_task_1, + pre_task_2, + pre_task_3, + ] ] ) diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_dependent.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_dependent.py index 2e237fefe0f9..f16e291c8231 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_dependent.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_dependent.py @@ -124,7 +124,11 @@ def test_dependent_item_date_error(): @pytest.mark.parametrize( - "task_name, result", [({"dependent_task_name": TEST_TASK}, TEST_TASK), ({}, None),], + "task_name, result", + [ + ({"dependent_task_name": TEST_TASK}, TEST_TASK), + ({}, None), + ], ) def test_dependent_item_code_parameter(task_name: dict, result: Optional[str]): """Test dependent item property code_parameter.""" @@ -706,7 +710,8 @@ def test_operator_dependent_task_list_multi_dependent_list( }, ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) def test_dependent_get_define(mock_code_version, mock_dep_code): """Test task dependent function get_define.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_flink.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_flink.py index d32c8412d45f..92ae3ba91fac 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_flink.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_flink.py @@ -45,7 +45,9 @@ def test_flink_get_define(mock_resource): "taskType": "FLINK", "taskParams": { "mainClass": main_class, - "mainJar": {"id": 1,}, + "mainJar": { + "id": 1, + }, "programType": program_type, "deployMode": deploy_mode, "flinkVersion": FlinkVersion.LOW_VERSION, diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_http.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_http.py index 0788b043fafd..060cdec0b0d8 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_http.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_http.py @@ -68,7 +68,8 @@ def test_attr_exists(class_name, attrs): ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) def test_property_task_params(mock_code_version, attr, expect): """Test task http property.""" @@ -89,7 +90,8 @@ def test_property_task_params(mock_code_version, attr, expect): ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) def test_http_task_param_not_support_param(mock_code, param): """Test HttpTaskParams not support parameter.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_map_reduce.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_map_reduce.py index 6c26a18e8c9e..dbe9e513f5c3 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_map_reduce.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_map_reduce.py @@ -45,7 +45,9 @@ def test_mr_get_define(mock_resource): "taskType": "MR", "taskParams": { "mainClass": main_class, - "mainJar": {"id": 1,}, + "mainJar": { + "id": 1, + }, "programType": program_type, "appName": None, "mainArgs": main_args, diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_procedure.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_procedure.py index 74bfd6970450..17825939555e 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_procedure.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_procedure.py @@ -52,7 +52,8 @@ ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) @patch( "pydolphinscheduler.core.database.Database.get_database_info", @@ -65,7 +66,8 @@ def test_property_task_params(mock_datasource, mock_code_version, attr, expect): @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) @patch( "pydolphinscheduler.core.database.Database.get_database_info", diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_python.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_python.py index 9fd143424879..1cdd85d2cbfa 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_python.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_python.py @@ -69,7 +69,8 @@ def foo(): # noqa: D103 ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) def test_property_task_params(mock_code_version, attr, expect): """Test task python property.""" @@ -78,10 +79,15 @@ def test_property_task_params(mock_code_version, attr, expect): @pytest.mark.parametrize( - "script_code", [123, ("print", "hello world"),], + "script_code", + [ + 123, + ("print", "hello world"), + ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) def test_python_task_not_support_code(mock_code, script_code): """Test python task parameters.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sagemaker.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sagemaker.py index 1cf8bf820858..8838eaf49766 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sagemaker.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sagemaker.py @@ -55,7 +55,8 @@ ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) def test_property_task_params(mock_code_version, attr, expect): """Test task sagemaker task property.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py index add509cad7c9..163261b2cf77 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py @@ -58,7 +58,8 @@ def setup_crt_first(): ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) def test_property_task_params(mock_code_version, attr, expect): """Test task shell task property.""" @@ -120,7 +121,8 @@ def test_shell_get_define(): ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) def test_resources_local_shell_command_content( mock_code_version, attr, expect, setup_crt_first diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_spark.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_spark.py index efdb1c1414c6..3b0672f96345 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_spark.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_spark.py @@ -45,7 +45,9 @@ def test_spark_get_define(mock_resource): "taskType": "SPARK", "taskParams": { "mainClass": main_class, - "mainJar": {"id": 1,}, + "mainJar": { + "id": 1, + }, "programType": program_type, "deployMode": deploy_mode, "sparkVersion": SparkVersion.SPARK2, diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sql.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sql.py index f7b5d78ad34e..ee0acc442e11 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sql.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sql.py @@ -61,7 +61,8 @@ ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) @patch( "pydolphinscheduler.core.database.Database.get_database_info", @@ -102,7 +103,8 @@ def test_get_sql_type( ], ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) @patch( "pydolphinscheduler.core.database.Database.get_database_info", diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sub_process.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sub_process.py index 781db341b63e..7f471a1b8b25 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sub_process.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_sub_process.py @@ -57,7 +57,8 @@ ), ) @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) def test_property_task_params(mock_code_version, mock_pd_info, attr, expect): """Test task sub process property.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_switch.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_switch.py index 93028ecf6ec4..1f6ff5bfa2dd 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_switch.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_switch.py @@ -48,7 +48,12 @@ def task_switch_arg_wrapper(obj, task: Task, exp: Optional[str] = None) -> Switc @pytest.mark.parametrize( - "obj", [SwitchBranch, Branch, Default,], + "obj", + [ + SwitchBranch, + Branch, + Default, + ], ) def test_switch_branch_attr_next_node(obj: SwitchBranch): """Test get attribute from class switch branch.""" @@ -58,7 +63,11 @@ def test_switch_branch_attr_next_node(obj: SwitchBranch): @pytest.mark.parametrize( - "obj", [SwitchBranch, Default,], + "obj", + [ + SwitchBranch, + Default, + ], ) def test_switch_branch_get_define_without_condition(obj: SwitchBranch): """Test function :func:`get_define` with None value of attribute condition from class switch branch.""" @@ -69,7 +78,11 @@ def test_switch_branch_get_define_without_condition(obj: SwitchBranch): @pytest.mark.parametrize( - "obj", [SwitchBranch, Branch,], + "obj", + [ + SwitchBranch, + Branch, + ], ) def test_switch_branch_get_define_condition(obj: SwitchBranch): """Test function :func:`get_define` with specific attribute condition from class switch branch.""" @@ -86,7 +99,10 @@ def test_switch_branch_get_define_condition(obj: SwitchBranch): @pytest.mark.parametrize( "args, msg", [ - ((1,), ".*?parameter only support SwitchBranch but got.*?",), + ( + (1,), + ".*?parameter only support SwitchBranch but got.*?", + ), ( (Default(Task(TEST_NAME, TEST_TYPE)), 2), ".*?parameter only support SwitchBranch but got.*?", @@ -197,7 +213,8 @@ def test_switch_condition_get_define_mix_branch_and_default(): @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) def test_switch_get_define(mock_task_code_version): """Test task switch :func:`get_define`.""" @@ -245,7 +262,8 @@ def test_switch_get_define(mock_task_code_version): @patch( - "pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(123, 1), + "pydolphinscheduler.core.task.Task.gen_code_and_version", + return_value=(123, 1), ) def test_switch_set_dep_workflow(mock_task_code_version): """Test task switch set dependence in workflow level.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_yaml_parser.py b/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_yaml_parser.py index 3c14f6a1166a..ad3aaf7bd150 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_yaml_parser.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_yaml_parser.py @@ -84,7 +84,18 @@ @pytest.mark.parametrize( "src, delimiter, expect", - [(param[0], "|", expects[0],), (param[1], "/", expects[1],),], + [ + ( + param[0], + "|", + expects[0], + ), + ( + param[1], + "/", + expects[1], + ), + ], ) def test_yaml_parser_specific_delimiter(src: str, delimiter: str, expect: Dict): """Test specific delimiter for :class:`YamlParser`.""" @@ -110,7 +121,17 @@ def ch_dl(key): @pytest.mark.parametrize( - "src, expect", [(param[0], expects[0],), (param[1], expects[1],),], + "src, expect", + [ + ( + param[0], + expects[0], + ), + ( + param[1], + expects[1], + ), + ], ) def test_yaml_parser_contains(src: str, expect: Dict): """Test magic function :func:`YamlParser.__contain__` also with `key in obj` syntax.""" @@ -124,7 +145,17 @@ def test_yaml_parser_contains(src: str, expect: Dict): @pytest.mark.parametrize( - "src, expect", [(param[0], expects[0],), (param[1], expects[1],),], + "src, expect", + [ + ( + param[0], + expects[0], + ), + ( + param[1], + expects[1], + ), + ], ) def test_yaml_parser_get(src: str, expect: Dict): """Test magic function :func:`YamlParser.__getitem__` also with `obj[key]` syntax.""" @@ -146,7 +177,17 @@ def test_yaml_parser_get(src: str, expect: Dict): @pytest.mark.parametrize( - "src, expect", [(param[0], expects[0],), (param[1], expects[1],),], + "src, expect", + [ + ( + param[0], + expects[0], + ), + ( + param[1], + expects[1], + ), + ], ) def test_yaml_parser_set(src: str, expect: Dict): """Test magic function :func:`YamlParser.__setitem__` also with `obj[key] = val` syntax.""" From 182df27ca3c07fa95994f2a8e3df0354b663ec31 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Sat, 13 Aug 2022 23:18:45 +0800 Subject: [PATCH 41/58] fix flake8 code style --- .../src/pydolphinscheduler/core/task.py | 7 ++++--- .../pydolphinscheduler/resources_plugin/__init__.py | 4 ++-- .../pydolphinscheduler/resources_plugin/local.py | 13 ++++++++++--- .../pydolphinscheduler/tests/core/test_task.py | 2 +- .../tests/resources_plugin/test_init.py | 10 +++++----- .../tests/resources_plugin/test_local.py | 4 ++-- 6 files changed, 24 insertions(+), 16 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py index 4659b80eb2d2..133cfbc83492 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py @@ -233,7 +233,9 @@ def task_params(self) -> Optional[Dict]: return self.get_define_custom(custom_attr=custom_attr) def get_plugin(self): - """Return the resource plug-in according to parameter resource_plugin and parameter + """Return the resource plug-in. + + according to parameter resource_plugin and parameter process_definition.resource_plugin. """ if self.resource_plugin is None: @@ -247,8 +249,7 @@ def get_plugin(self): return self.resource_plugin.resource def get_content(self): - """Get the file content according to the resource plugin""" - + """Get the file content according to the resource plugin.""" if self.ext_attr is None and self.ext is None: return diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py index fd0cb880e798..88f7dd6c6c1f 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py @@ -56,7 +56,7 @@ def get_all_modules(self) -> Path: # [start import_module] def import_module(self, script_name, script_path): - """Import module""" + """Import module.""" spec = importlib.util.spec_from_file_location(script_name, script_path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) @@ -68,7 +68,7 @@ def import_module(self, script_name, script_path): @property # [start resource] def resource(self): - """Dynamically return resource plugin""" + """Dynamically return resource plugin.""" for ex in self.get_all_modules(): if ex.stem == self.type: return self.import_module(ex.name, str(ex)) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py index e2bed6ee253e..bf5de7fbd559 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py @@ -24,6 +24,12 @@ class Local: + """Local object, declare local resource plugin for task and workflow to dolphinscheduler. + + :param prefix: A string representing the prefix of Local. + + """ + # [init_method] def __init__(self, prefix: str): self._prefix = prefix @@ -32,13 +38,14 @@ def __init__(self, prefix: str): @property def prefix(self): - """Get the _prefix attribute""" + """Get the _prefix attribute.""" return self._prefix # [start read_file_method] def read_file(self, suf: str): - """Get the content of the file, the address of the file is - the prefix of the resource plugin plus the parameter suf + """Get the content of the file. + + The address of the file is the prefix of the resource plugin plus the parameter suf. """ path = Path(self.prefix).joinpath(suf) if not path.exists(): diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py index c784810804ef..9f379b39c77f 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py @@ -22,7 +22,7 @@ import pytest -from pydolphinscheduler.constants import ResourcePluginType, TaskType +from pydolphinscheduler.constants import ResourcePluginType from pydolphinscheduler.core.process_definition import ProcessDefinition from pydolphinscheduler.core.task import Task, TaskRelation from pydolphinscheduler.exceptions import PyResPluginException diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py index 761c001fdad7..0882b7f91621 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py @@ -22,7 +22,7 @@ import pytest from pydolphinscheduler.constants import ResourcePluginType -from pydolphinscheduler.exceptions import PyDSConfException, PyResPluginException +from pydolphinscheduler.exceptions import PyResPluginException from pydolphinscheduler.resources_plugin import ResourcePlugin all_res = ["local"] @@ -45,7 +45,7 @@ ], ) def test_resources_get_all_modules(attr, expected): - """Test resource plugin to get all res plugin names""" + """Test resource plugin to get all res plugin names.""" res = ResourcePlugin(**attr) assert dict(Counter(expected)) == dict( Counter([ex.stem for ex in res.get_all_modules()]) @@ -68,7 +68,7 @@ def test_resources_get_all_modules(attr, expected): ], ) def test_resources_import_modules(attrs, expected): - """Test resource plug-in to import model""" + """Test resource plug-in to import model.""" res_plugin = ResourcePlugin(attrs.get("type"), "plugin-prefix") res = res_plugin.import_module(**attrs.get("module_attr")) assert expected == res.__class__.__name__ @@ -81,7 +81,7 @@ def test_resources_import_modules(attrs, expected): ], ) def test_resources_resources(attr, expected): - """Test resource plugin factory""" + """Test resource plugin factory.""" res_plugin = ResourcePlugin(attr, "/tmp/") res = res_plugin.resource assert expected == res.__class__.__name__ @@ -97,7 +97,7 @@ def test_resources_resources(attr, expected): ], ) def test_resources_unsupported_res(attr): - """Test unsupported plug-ins""" + """Test unsupported plug-ins.""" with pytest.raises( PyResPluginException, match="{} type is not supported".format(attr.get("type")) ): diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py index bf5e43920350..d7ac85673580 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py @@ -91,7 +91,7 @@ def test_task_obtain_res_plugin( [({"prefix": res_plugin_prefix, "file_name": file_name}, file_content)], ) def test_local_res_read_file(attr, expected, setup_crt_first): - """Test the read_file function of the local resource plug-in""" + """Test the read_file function of the local resource plug-in.""" local = Local(str(attr.get("prefix"))) local.read_file(attr.get("file_name")) assert expected == local.read_file(file_name) @@ -104,7 +104,7 @@ def test_local_res_read_file(attr, expected, setup_crt_first): ], ) def test_local_res_file_not_found(attr): - """test local resource plugin file does not exist""" + """Test local resource plugin file does not exist.""" with pytest.raises( PyResPluginException, match="{} is not found".format( From 45cad6d7d28629cab8fd9f5db40187b366250c15 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Sat, 13 Aug 2022 23:31:29 +0800 Subject: [PATCH 42/58] fix some test --- .../examples/tutorial_resource_plugin.py | 10 +++++----- .../pydolphinscheduler/tests/example/test_example.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py index 374f83d80209..67d6c090e3c9 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py @@ -39,7 +39,7 @@ from pydolphinscheduler.constants import ResourcePluginType -# [start tutorial] +# [start tutorial_resource_plugin] # [start package_import] # Import ProcessDefinition object to define your workflow attributes from pydolphinscheduler.core.process_definition import ProcessDefinition @@ -52,7 +52,7 @@ # [start workflow_declare] with ProcessDefinition( - name="tutorial", + name="tutorial_resource_plugin", schedule="0 0 0 * * ? *", start_time="2021-01-01", tenant="tenant_exists", @@ -60,7 +60,7 @@ type=ResourcePluginType.LOCAL, prefix="/opt/", ), -) as pd: +) as process_definition: # [end workflow_declare] # [start task_declare] task_parent = Shell( @@ -103,6 +103,6 @@ # [end task_relation_declare] # [start submit_or_run] - pd.run() + process_definition.run() # [end submit_or_run] -# [end tutorial] +# [end tutorial_resource_plugin] diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/example/test_example.py b/dolphinscheduler-python/pydolphinscheduler/tests/example/test_example.py index 70f367767cd4..e9389e609634 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/example/test_example.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/example/test_example.py @@ -97,7 +97,7 @@ def test_example_basic(): ), f"We expect all examples is python script, but get {ex.name}." # All except tutorial and __init__ is end with keyword "_example" - if ex.stem not in ("tutorial", "tutorial_decorator") and ex.stem != "__init__": + if ex.stem not in ("tutorial", "tutorial_decorator", "tutorial_resource_plugin") and ex.stem != "__init__": assert ex.stem.endswith( "_example" ), f"We expect all examples script end with keyword '_example', but get {ex.stem}." From 44c7b2274604188ab1b6ff5f278ea780a79a6d24 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Sun, 14 Aug 2022 00:26:15 +0800 Subject: [PATCH 43/58] fix tox e doc build --- .../docs/source/resources_plugin/develop.rst | 33 +++++++++++-------- .../docs/source/resources_plugin/local.rst | 6 ++-- .../resources_plugin/resource-plugin.rst | 32 ++++++++++-------- .../resources_plugin/__init__.py | 6 ++-- .../resources_plugin/local.py | 4 +-- .../tests/example/test_example.py | 6 +++- 6 files changed, 52 insertions(+), 35 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/develop.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/develop.rst index 0340e6be4b53..88d756aa0101 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/develop.rst +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/develop.rst @@ -1,19 +1,19 @@ .. Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at .. http://www.apache.org/licenses/LICENSE-2.0 .. Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. How to develop ============== @@ -28,18 +28,23 @@ In addition you need to add a constant with your plugin name to ResourcePluginTy Example ------- - Method `__init__`: Initiation method with `param`:`prefix` + .. literalinclude:: ../../../src/pydolphinscheduler/resources_plugin/local.py :start-after: [start init_method] :end-before: [end init_method] -- Method `read_file `: Get content from the given URI, The function parameter is the suffix of the file path. -The file prefix has been initialized in init of the resource plug-in. The prefix plus suffix is the absolute -path of the file in this resource +- Method `read_file`: Get content from the given URI, The function parameter is the suffix of the file path. + +The file prefix has been initialized in init of the resource plug-in. + +The prefix plus suffix is the absolute path of the file in this resource. + .. literalinclude:: ../../../src/pydolphinscheduler/resources_plugin/local.py :start-after: [start read_file_method] :end-before: [end read_file_method] Last but not least, you should also add new resource plugin in constants.py + .. literalinclude:: ../../../src/pydolphinscheduler/constants.py :start-after: [start class_resource] :end-before: [end class_resource] diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/local.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/local.rst index 4d23d248b3aa..afbe9a0540c7 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/local.rst +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/local.rst @@ -21,10 +21,10 @@ Local `Local` is a local resource plugin for pydolphinscheduler. When using a local resource plugin, you do not need to use this class explicitly, you only need to add the -resource_plugin parameter in the task subclass or workflow definition. The resource_plugin parameter type is -ResourcePlugin. +`resource_plugin` parameter in the task subclass or workflow definition. +The data type of parameter `resource_plugin` is `ResourcePlugin` -The use of specific resource plugins can be viewed: :doc:`./how-to-use` +For the specific use of resource plugins, you can see `How to use` in :doc:`./resource-plugin` Dive Into --------- diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/resource-plugin.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/resource-plugin.rst index dd9da2cdc093..7cd0b6f79887 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/resource-plugin.rst +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/resource-plugin.rst @@ -1,19 +1,19 @@ .. Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at .. http://www.apache.org/licenses/LICENSE-2.0 .. Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. ResourcePlugin ============== @@ -32,6 +32,7 @@ Dive Into It has the following key functions. - Method `__init__`: The `__init__` function requires parameters. The first is the `type` of str type, which means the plugin type of the resource, + and the second is the `prefix` of the str type, which means the prefix of the resource. .. literalinclude:: ../../../src/pydolphinscheduler/resources_plugin/__init__.py @@ -45,14 +46,17 @@ and the second is the `prefix` of the str type, which means the prefix of the re :end-before: [end get_all_modules] - Method `import_module`: The `import_module` function has two parameters, `script_name` and `script_path`. -`script_name` is the name of the resource file without the suffix, such as the local resource plugin class local.py, its -`script_name` is local, and `script_path` is the absolute path of the resource plugin file + +`script_name` is the name of the resource file without the suffix, such as the local resource plugin class local.py, + +its `script_name` is local, and `script_path` is the absolute path of the resource plugin file .. literalinclude:: ../../../src/pydolphinscheduler/resources_plugin/__init__.py :start-after: [start import_module] :end-before: [end import_module] - Method `resource`: The `resource` function will dynamically return the resource plugin object according to the type parameter of the `__init__` function, + for example, the type is `ResourcePluginType.LOCAL`, then the local plugin object is returned .. literalinclude:: ../../../src/pydolphinscheduler/resources_plugin/__init__.py @@ -84,7 +88,9 @@ Use resource plug-ins in both tasks and workflows :end-before: [end task_declare] When the resource_plugin parameter is defined in both the task subclass and the workflow, the resource_plugin defined in the task subclass is used first. + If the task subclass does not define resource_plugin, but the resource_plugin is defined in the workflow, the resource_plugin in the workflow is used. Of course, if neither the task subclass nor the workflow specifies resource_plugin, the command at this time will be executed as a script, + in other words, we are forward compatible. \ No newline at end of file diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py index 88f7dd6c6c1f..d545a2a3e0ae 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py @@ -31,17 +31,19 @@ class ResourcePlugin: """ResourcePlugin object, declare resource plugin for task and workflow to dolphinscheduler. :param type: A unique, meaningful string for the ResourcePlugin, + Its value should be taken from the constant of ResourceType in constants.py. + :param prefix: A string representing the prefix of ResourcePlugin. """ - # [start init] + # [start init_method] def __init__(self, type: str, prefix: str): self.type = type self.prefix = prefix - # [end init] + # [end init_method] # [start get_all_modules] def get_all_modules(self) -> Path: diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py index bf5de7fbd559..0d5e5bb0195b 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py @@ -30,7 +30,7 @@ class Local: """ - # [init_method] + # [start init_method] def __init__(self, prefix: str): self._prefix = prefix @@ -58,4 +58,4 @@ def read_file(self, suf: str): content = f.read() return content - # [start read_file_method] + # [end read_file_method] diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/example/test_example.py b/dolphinscheduler-python/pydolphinscheduler/tests/example/test_example.py index e9389e609634..319ad961f723 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/example/test_example.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/example/test_example.py @@ -97,7 +97,11 @@ def test_example_basic(): ), f"We expect all examples is python script, but get {ex.name}." # All except tutorial and __init__ is end with keyword "_example" - if ex.stem not in ("tutorial", "tutorial_decorator", "tutorial_resource_plugin") and ex.stem != "__init__": + if ( + ex.stem + not in ("tutorial", "tutorial_decorator", "tutorial_resource_plugin") + and ex.stem != "__init__" + ): assert ex.stem.endswith( "_example" ), f"We expect all examples script end with keyword '_example', but get {ex.stem}." From 220533633cac2e7367a4d206e8802d8e188aa0bb Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Sun, 14 Aug 2022 10:44:05 +0800 Subject: [PATCH 44/58] fix test_shell --- .../pydolphinscheduler/tests/tasks/test_shell.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py index 163261b2cf77..59a1685fe27d 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py @@ -28,9 +28,10 @@ from pydolphinscheduler.utils import file from tests.testing.file import delete_file -file_path = "local_res.sh" +file_name = "local_res.sh" file_content = 'echo "test res_local"' res_plugin_prefix = Path(__file__).parent +file_path = res_plugin_prefix.joinpath(file_name) @pytest.fixture @@ -111,7 +112,7 @@ def test_shell_get_define(): ( { "name": "test-local-res-command-content", - "command": file_path, + "command": file_name, "resource_plugin": ResourcePlugin( type=ResourcePluginType.LOCAL, prefix=res_plugin_prefix ), From 61176b5a93ab45a35cd0cdb6e4b72a6e5aa152b0 Mon Sep 17 00:00:00 2001 From: Jiajie Zhong Date: Sun, 14 Aug 2022 18:02:38 +0800 Subject: [PATCH 45/58] fix ci error --- .../examples/tutorial_resource_plugin.py | 67 ++++++------------- 1 file changed, 21 insertions(+), 46 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py index 67d6c090e3c9..83e93ee3122a 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py @@ -37,6 +37,8 @@ it will instantiate and run all the task it have. """ +import tempfile + from pydolphinscheduler.constants import ResourcePluginType # [start tutorial_resource_plugin] @@ -51,56 +53,29 @@ # [end package_import] # [start workflow_declare] -with ProcessDefinition( - name="tutorial_resource_plugin", - schedule="0 0 0 * * ? *", - start_time="2021-01-01", - tenant="tenant_exists", - resource_plugin=ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix="/opt/", - ), -) as process_definition: - # [end workflow_declare] - # [start task_declare] - task_parent = Shell( - name="task_parent", - command="parent.zsh", - resource_plugin=ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix="/opt/", - ), - ) - - task_child_one = Shell( - name="task_child_one", - command="child_one.sh", - resource_plugin=ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix="/opt/", - ), - ) +with tempfile.TemporaryDirectory() as tmpdir: - task_child_two = Shell( - name="task_child_two", - command="child_two.sh", + with ProcessDefinition( + name="tutorial_resource_plugin", + schedule="0 0 0 * * ? *", + start_time="2021-01-01", + tenant="tenant_exists", resource_plugin=ResourcePlugin( type=ResourcePluginType.LOCAL, - prefix="/opt/", + prefix=tmpdir, ), - ) - task_union = Shell( - name="task_union", - command="echo union", - ) - # [end task_declare] - - # [start task_relation_declare] - task_group = [task_child_one, task_child_two] - task_parent.set_downstream(task_group) - - task_union << task_group - # [end task_relation_declare] + ) as process_definition: + # [end workflow_declare] + # [start task_declare] + tmp = tempfile.NamedTemporaryFile(dir=tmpdir, suffix=".sh") + with open(tmp.name, "w") as f: + f.write("echo tutorial resource plugin") + task_parent = Shell( + name="local-resource-example", + command=tmp.name, + ) + print(task_parent.task_params) + # [end task_declare] # [start submit_or_run] process_definition.run() From 4167a0a7dd1f120376c4ddf78cdab4c8ea9eba3e Mon Sep 17 00:00:00 2001 From: Jiajie Zhong Date: Sun, 14 Aug 2022 18:43:59 +0800 Subject: [PATCH 46/58] fix ci error --- .../src/pydolphinscheduler/examples/tutorial_resource_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py index 83e93ee3122a..6dc0eed68ff0 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py @@ -67,7 +67,7 @@ ) as process_definition: # [end workflow_declare] # [start task_declare] - tmp = tempfile.NamedTemporaryFile(dir=tmpdir, suffix=".sh") + tmp = tempfile.NamedTemporaryFile(mode="w", dir=tmpdir, suffix=".sh") with open(tmp.name, "w") as f: f.write("echo tutorial resource plugin") task_parent = Shell( From 2726a2bff60336f31876d8ed18474053a92ce467 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Sun, 14 Aug 2022 23:08:46 +0800 Subject: [PATCH 47/58] fix windows tmp file cannot be written --- .../pydolphinscheduler/examples/tutorial_resource_plugin.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py index 6dc0eed68ff0..fff30c4dff53 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py @@ -36,8 +36,9 @@ it will instantiate and run all the task it have. """ - +import os import tempfile +from pathlib import Path from pydolphinscheduler.constants import ResourcePluginType @@ -67,7 +68,7 @@ ) as process_definition: # [end workflow_declare] # [start task_declare] - tmp = tempfile.NamedTemporaryFile(mode="w", dir=tmpdir, suffix=".sh") + tmp = tempfile.NamedTemporaryFile(delete=False, mode="w", dir=tmpdir, suffix=".sh") with open(tmp.name, "w") as f: f.write("echo tutorial resource plugin") task_parent = Shell( @@ -75,6 +76,7 @@ command=tmp.name, ) print(task_parent.task_params) + os.remove(tmp.name) # [end task_declare] # [start submit_or_run] From 3c9277783c477caca3621d7a25edd7fafa831e75 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Sun, 14 Aug 2022 23:08:46 +0800 Subject: [PATCH 48/58] fix windows tmp file cannot be written --- .../examples/tutorial_resource_plugin.py | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py index 6dc0eed68ff0..9e5c1a8d1e00 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py @@ -36,8 +36,8 @@ it will instantiate and run all the task it have. """ - -import tempfile +import os +from pathlib import Path from pydolphinscheduler.constants import ResourcePluginType @@ -53,29 +53,29 @@ # [end package_import] # [start workflow_declare] -with tempfile.TemporaryDirectory() as tmpdir: - - with ProcessDefinition( - name="tutorial_resource_plugin", - schedule="0 0 0 * * ? *", - start_time="2021-01-01", - tenant="tenant_exists", - resource_plugin=ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix=tmpdir, - ), - ) as process_definition: - # [end workflow_declare] - # [start task_declare] - tmp = tempfile.NamedTemporaryFile(mode="w", dir=tmpdir, suffix=".sh") - with open(tmp.name, "w") as f: - f.write("echo tutorial resource plugin") - task_parent = Shell( - name="local-resource-example", - command=tmp.name, - ) - print(task_parent.task_params) - # [end task_declare] +with ProcessDefinition( + name="tutorial_resource_plugin", + schedule="0 0 0 * * ? *", + start_time="2021-01-01", + tenant="tenant_exists", + resource_plugin=ResourcePlugin( + type=ResourcePluginType.LOCAL, + prefix="/tmp", + ), +) as process_definition: + # [end workflow_declare] + # [start task_declare] + file = "resource.sh" + path = Path("/tmp").joinpath(file) + with open(str(path), "w") as f: + f.write("echo tutorial resource plugin") + task_parent = Shell( + name="local-resource-example", + command=file, + ) + print(task_parent.task_params) + os.remove(path) + # [end task_declare] # [start submit_or_run] process_definition.run() From fd37037ae3d43f6f5c04bd19bf02d023119a112d Mon Sep 17 00:00:00 2001 From: Jiajie Zhong Date: Mon, 15 Aug 2022 15:41:51 +0800 Subject: [PATCH 49/58] try to fix error --- .../pydolphinscheduler/tests/resources_plugin/test_local.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py index d7ac85673580..0bf38bde7e44 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py @@ -107,9 +107,7 @@ def test_local_res_file_not_found(attr): """Test local resource plugin file does not exist.""" with pytest.raises( PyResPluginException, - match="{} is not found".format( - Path(attr.get("prefix")).joinpath(attr.get("file_name")) - ), + match=".* is not found", ): local = Local(str(attr.get("prefix"))) local.read_file(attr.get("file_name")) From 9c77bf4cdbfb7d46f318610367579b0ef4196ca9 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Tue, 16 Aug 2022 00:01:32 +0800 Subject: [PATCH 50/58] remove unnessnary change --- .../src/main/resources/logback-spring.xml | 7 +++---- .../src/main/resources/logback-spring.xml | 13 ++++++------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml b/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml index 7f82e52bad36..c2c41e5114dc 100644 --- a/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml +++ b/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml @@ -73,14 +73,13 @@ - - + - + \ No newline at end of file diff --git a/dolphinscheduler-worker/src/main/resources/logback-spring.xml b/dolphinscheduler-worker/src/main/resources/logback-spring.xml index 4096b4b1d882..612e4e46e3c4 100644 --- a/dolphinscheduler-worker/src/main/resources/logback-spring.xml +++ b/dolphinscheduler-worker/src/main/resources/logback-spring.xml @@ -67,13 +67,12 @@ - - - - - - + + + + + - + \ No newline at end of file From 5494243af90b08f8a45b244431383ba2ee698f93 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Tue, 16 Aug 2022 00:08:11 +0800 Subject: [PATCH 51/58] fix tutorial_resource_plugin describe --- .../examples/tutorial_resource_plugin.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py index 9e5c1a8d1e00..b95d1b4d66e4 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py @@ -18,22 +18,8 @@ r""" A tutorial example take you to experience pydolphinscheduler resource plugin. -After tutorial_resource_plugin.py file submit to Apache DolphinScheduler server a DAG would be create, -and workflow DAG graph as below: - - --> task_child_one - / \ -task_parent --> --> task_union - \ / - --> task_child_two - Resource plug-ins can be defined in workflows and tasks -task_parent uses local resource plugin. -task_child_one uses local resource plugin. -task_child_two uses local resource plugin. -task_union does not use resource plug-ins - it will instantiate and run all the task it have. """ import os From 8f48bfb16a10ca74ac2f438a9b87339f7d59c432 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Tue, 16 Aug 2022 11:07:07 +0800 Subject: [PATCH 52/58] remove unnessnary change --- dolphinscheduler-api/src/main/resources/logback-spring.xml | 2 +- dolphinscheduler-master/src/main/resources/logback-spring.xml | 2 +- .../src/main/resources/logback-spring.xml | 2 +- dolphinscheduler-worker/src/main/resources/application.yaml | 3 +-- dolphinscheduler-worker/src/main/resources/logback-spring.xml | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/dolphinscheduler-api/src/main/resources/logback-spring.xml b/dolphinscheduler-api/src/main/resources/logback-spring.xml index bcea64734c28..60fa20509ff3 100644 --- a/dolphinscheduler-api/src/main/resources/logback-spring.xml +++ b/dolphinscheduler-api/src/main/resources/logback-spring.xml @@ -61,4 +61,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-master/src/main/resources/logback-spring.xml b/dolphinscheduler-master/src/main/resources/logback-spring.xml index b2d566af5e6a..dd28aedc34b7 100644 --- a/dolphinscheduler-master/src/main/resources/logback-spring.xml +++ b/dolphinscheduler-master/src/main/resources/logback-spring.xml @@ -74,4 +74,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml b/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml index c2c41e5114dc..0cdeabde5ec3 100644 --- a/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml +++ b/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml @@ -82,4 +82,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-worker/src/main/resources/application.yaml b/dolphinscheduler-worker/src/main/resources/application.yaml index 174ddaa207b6..fb8a1cb704d3 100644 --- a/dolphinscheduler-worker/src/main/resources/application.yaml +++ b/dolphinscheduler-worker/src/main/resources/application.yaml @@ -110,7 +110,6 @@ management: metrics: enabled: true - # Override by profile --- @@ -122,4 +121,4 @@ spring: driver-class-name: com.mysql.cj.jdbc.Driver url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler username: root - password: root \ No newline at end of file + password: root diff --git a/dolphinscheduler-worker/src/main/resources/logback-spring.xml b/dolphinscheduler-worker/src/main/resources/logback-spring.xml index 612e4e46e3c4..6c00516efd73 100644 --- a/dolphinscheduler-worker/src/main/resources/logback-spring.xml +++ b/dolphinscheduler-worker/src/main/resources/logback-spring.xml @@ -75,4 +75,4 @@ - \ No newline at end of file + From e8f0a2ac6fd92cfd84e628db7787cbd12cec85c1 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Tue, 16 Aug 2022 11:07:07 +0800 Subject: [PATCH 53/58] remove unnessnary change --- dolphinscheduler-api/src/main/resources/logback-spring.xml | 2 +- dolphinscheduler-common/src/main/resources/common.properties | 1 + dolphinscheduler-master/src/main/resources/logback-spring.xml | 2 +- .../src/main/resources/logback-spring.xml | 2 +- dolphinscheduler-worker/src/main/resources/application.yaml | 3 +-- dolphinscheduler-worker/src/main/resources/logback-spring.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dolphinscheduler-api/src/main/resources/logback-spring.xml b/dolphinscheduler-api/src/main/resources/logback-spring.xml index bcea64734c28..60fa20509ff3 100644 --- a/dolphinscheduler-api/src/main/resources/logback-spring.xml +++ b/dolphinscheduler-api/src/main/resources/logback-spring.xml @@ -61,4 +61,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index 37bd725a8589..3b12d7c606a7 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -57,6 +57,7 @@ login.user.keytab.path=/opt/hdfs.headless.keytab # kerberos expire time, the unit is hour kerberos.expire.time=2 + # resourcemanager port, the default value is 8088 if not specified resource.manager.httpaddress.port=8088 # if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty diff --git a/dolphinscheduler-master/src/main/resources/logback-spring.xml b/dolphinscheduler-master/src/main/resources/logback-spring.xml index b2d566af5e6a..dd28aedc34b7 100644 --- a/dolphinscheduler-master/src/main/resources/logback-spring.xml +++ b/dolphinscheduler-master/src/main/resources/logback-spring.xml @@ -74,4 +74,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml b/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml index c2c41e5114dc..0cdeabde5ec3 100644 --- a/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml +++ b/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml @@ -82,4 +82,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-worker/src/main/resources/application.yaml b/dolphinscheduler-worker/src/main/resources/application.yaml index 174ddaa207b6..fb8a1cb704d3 100644 --- a/dolphinscheduler-worker/src/main/resources/application.yaml +++ b/dolphinscheduler-worker/src/main/resources/application.yaml @@ -110,7 +110,6 @@ management: metrics: enabled: true - # Override by profile --- @@ -122,4 +121,4 @@ spring: driver-class-name: com.mysql.cj.jdbc.Driver url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler username: root - password: root \ No newline at end of file + password: root diff --git a/dolphinscheduler-worker/src/main/resources/logback-spring.xml b/dolphinscheduler-worker/src/main/resources/logback-spring.xml index 612e4e46e3c4..6c00516efd73 100644 --- a/dolphinscheduler-worker/src/main/resources/logback-spring.xml +++ b/dolphinscheduler-worker/src/main/resources/logback-spring.xml @@ -75,4 +75,4 @@ - \ No newline at end of file + From 5c98bf70e9dd1a85b2d6f227ce297e7867e81e6e Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Tue, 16 Aug 2022 13:05:02 +0800 Subject: [PATCH 54/58] remove unnessary changes --- .../pydolphinscheduler/tests/utils/test_file.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_file.py b/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_file.py index 91f6ede8ce1d..4cc6df402f87 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_file.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/utils/test_file.py @@ -58,7 +58,6 @@ def test_write_not_create_parent(teardown_del_file): if file_test_dir.exists(): shutil.rmtree(str(file_test_dir)) assert not file_test_dir.exists() - # with pytest.raises( ValueError, match="Parent directory do not exists and set param `create` to `False`", @@ -80,7 +79,6 @@ def test_write_overwrite_error(setup_crt_first): assert Path(file_path).exists() new_content = f"new_{content}" - # with pytest.raises( FileExistsError, match=".*already exists and you choose not overwrite mode\\." ): From 31529fab1e04dc6385741468f63d8b13b1f0d8f0 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Thu, 18 Aug 2022 20:32:31 +0800 Subject: [PATCH 55/58] temporary storary --- .../src/pydolphinscheduler/core/task.py | 4 +- .../examples/tutorial_resource_plugin.py | 7 +- .../resources_plugin/__init__.py | 112 ++++++++++++------ .../tests/core/test_task.py | 20 +--- .../tests/tasks/test_shell.py | 5 +- 5 files changed, 85 insertions(+), 63 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py index 133cfbc83492..f6b145f996c5 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py @@ -240,13 +240,13 @@ def get_plugin(self): """ if self.resource_plugin is None: if self.process_definition.resource_plugin is not None: - return self.process_definition.resource_plugin.resource + return self.process_definition.resource_plugin else: raise PyResPluginException( "The execution command of this task is a file, but the resource plugin is empty" ) else: - return self.resource_plugin.resource + return self.resource_plugin def get_content(self): """Get the file content according to the resource plugin.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py index b95d1b4d66e4..1fd12a05e07e 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py @@ -44,17 +44,14 @@ schedule="0 0 0 * * ? *", start_time="2021-01-01", tenant="tenant_exists", - resource_plugin=ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix="/tmp", - ), + resource_plugin=ResourcePlugin.Local(prefix="/tmp"), ) as process_definition: # [end workflow_declare] # [start task_declare] file = "resource.sh" path = Path("/tmp").joinpath(file) with open(str(path), "w") as f: - f.write("echo tutorial resource plugin") + f.write('echo "tutorial resource plugin"') task_parent = Shell( name="local-resource-example", command=file, diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py index d545a2a3e0ae..05e3e9d48d8d 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py @@ -21,6 +21,9 @@ import importlib.util from pathlib import Path +import os +from pathlib import Path + from pydolphinscheduler.exceptions import PyResPluginException path_resources_plugin = Path(__file__).parent @@ -39,44 +42,81 @@ class ResourcePlugin: """ # [start init_method] - def __init__(self, type: str, prefix: str): - self.type = type - self.prefix = prefix - - # [end init_method] - - # [start get_all_modules] - def get_all_modules(self) -> Path: - """Get all res files path in resources_plugin directory.""" - return ( - ex - for ex in path_resources_plugin.iterdir() - if ex.is_file() and not ex.name.startswith("__") - ) - - # [end get_all_modules] - - # [start import_module] - def import_module(self, script_name, script_path): - """Import module.""" - spec = importlib.util.spec_from_file_location(script_name, script_path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - plugin = getattr(module, self.type.capitalize()) - return plugin(self.prefix) - - # [end import_module] - - @property - # [start resource] - def resource(self): - """Dynamically return resource plugin.""" - for ex in self.get_all_modules(): - if ex.stem == self.type: - return self.import_module(ex.name, str(ex)) - raise PyResPluginException("{} type is not supported.".format(self.type)) + # def __init__(self, type: str, prefix: str): + # self.type = type + # self.prefix = prefix + # + # # [end init_method] + # + # # [start get_all_modules] + # def get_all_modules(self) -> Path: + # """Get all res files path in resources_plugin directory.""" + # return ( + # ex + # for ex in path_resources_plugin.iterdir() + # if ex.is_file() and not ex.name.startswith("__") + # ) + # + # # [end get_all_modules] + # + # # [start import_module] + # def import_module(self, script_name, script_path): + # """Import module.""" + # spec = importlib.util.spec_from_file_location(script_name, script_path) + # module = importlib.util.module_from_spec(spec) + # spec.loader.exec_module(module) + # plugin = getattr(module, self.type.capitalize()) + # return plugin(self.prefix) + # + # # [end import_module] + # + # @property + # # [start resource] + # def resource(self): + # """Dynamically return resource plugin.""" + # for ex in self.get_all_modules(): + # if ex.stem == self.type: + # return self.import_module(ex.name, str(ex)) + # raise PyResPluginException("{} type is not supported.".format(self.type)) # [end resource] + class Local: + """Local object, declare local resource plugin for task and workflow to dolphinscheduler. + + :param prefix: A string representing the prefix of Local. + + """ + + # [start init_method] + def __init__(self, prefix: str): + self._prefix = prefix + + # [end init_method] + + @property + def prefix(self): + """Get the _prefix attribute.""" + return self._prefix + + # [start read_file_method] + def read_file(self, suf: str): + """Get the content of the file. + + The address of the file is the prefix of the resource plugin plus the parameter suf. + """ + path = Path(self.prefix).joinpath(suf) + if not path.exists(): + raise PyResPluginException("{} is not found".format(str(path))) + if not os.access(str(path), os.R_OK): + raise PyResPluginException( + "You don't have permission to access {}".format(self.prefix + suf) + ) + with open(path, "r") as f: + content = f.read() + return content + + # [end read_file_method] + # [end resource_plugin_definition] diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py index 9f379b39c77f..b731196c8ffb 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py @@ -299,16 +299,10 @@ def test_task_ext_attr( { "name": "test_task_abtain_res_plugin", "task_type": "TaskType", - "resource_plugin": ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix="prefix", - ), + "resource_plugin": ResourcePlugin.Local("prefix"), "process_definition": ProcessDefinition( name="process_definition", - resource_plugin=ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix="prefix", - ), + resource_plugin=ResourcePlugin.Local("prefix"), ), }, "Local", @@ -317,10 +311,7 @@ def test_task_ext_attr( { "name": "test_task_abtain_res_plugin", "task_type": "TaskType", - "resource_plugin": ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix="prefix", - ), + "resource_plugin": ResourcePlugin.Local("prefix"), }, "Local", ), @@ -330,10 +321,7 @@ def test_task_ext_attr( "task_type": "TaskType", "process_definition": ProcessDefinition( name="process_definition", - resource_plugin=ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix="prefix", - ), + resource_plugin=ResourcePlugin.Local("prefix"), ), }, "Local", diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py index 59a1685fe27d..7055fe476136 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py @@ -22,7 +22,6 @@ import pytest -from pydolphinscheduler.constants import ResourcePluginType from pydolphinscheduler.resources_plugin import ResourcePlugin from pydolphinscheduler.tasks.shell import Shell from pydolphinscheduler.utils import file @@ -113,9 +112,7 @@ def test_shell_get_define(): { "name": "test-local-res-command-content", "command": file_name, - "resource_plugin": ResourcePlugin( - type=ResourcePluginType.LOCAL, prefix=res_plugin_prefix - ), + "resource_plugin": ResourcePlugin.Local(res_plugin_prefix), }, file_content, ) From e76c1a7a854356da0c54686e70b47ae88092fd63 Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Thu, 18 Aug 2022 20:32:31 +0800 Subject: [PATCH 56/58] temporary storary --- .../src/pydolphinscheduler/core/task.py | 4 +- .../examples/tutorial_resource_plugin.py | 7 +- .../resources_plugin/__init__.py | 112 ++++++++++++------ .../tests/core/test_task.py | 20 +--- .../tests/tasks/test_shell.py | 5 +- 5 files changed, 85 insertions(+), 63 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py index 133cfbc83492..f6b145f996c5 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py @@ -240,13 +240,13 @@ def get_plugin(self): """ if self.resource_plugin is None: if self.process_definition.resource_plugin is not None: - return self.process_definition.resource_plugin.resource + return self.process_definition.resource_plugin else: raise PyResPluginException( "The execution command of this task is a file, but the resource plugin is empty" ) else: - return self.resource_plugin.resource + return self.resource_plugin def get_content(self): """Get the file content according to the resource plugin.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py index b95d1b4d66e4..990494c93346 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py @@ -44,17 +44,14 @@ schedule="0 0 0 * * ? *", start_time="2021-01-01", tenant="tenant_exists", - resource_plugin=ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix="/tmp", - ), + resource_plugin=ResourcePlugin.Local(prefix="/tmp"), ) as process_definition: # [end workflow_declare] # [start task_declare] file = "resource.sh" path = Path("/tmp").joinpath(file) with open(str(path), "w") as f: - f.write("echo tutorial resource plugin") + f.write('echo tutorial resource plugin') task_parent = Shell( name="local-resource-example", command=file, diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py index d545a2a3e0ae..05e3e9d48d8d 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py @@ -21,6 +21,9 @@ import importlib.util from pathlib import Path +import os +from pathlib import Path + from pydolphinscheduler.exceptions import PyResPluginException path_resources_plugin = Path(__file__).parent @@ -39,44 +42,81 @@ class ResourcePlugin: """ # [start init_method] - def __init__(self, type: str, prefix: str): - self.type = type - self.prefix = prefix - - # [end init_method] - - # [start get_all_modules] - def get_all_modules(self) -> Path: - """Get all res files path in resources_plugin directory.""" - return ( - ex - for ex in path_resources_plugin.iterdir() - if ex.is_file() and not ex.name.startswith("__") - ) - - # [end get_all_modules] - - # [start import_module] - def import_module(self, script_name, script_path): - """Import module.""" - spec = importlib.util.spec_from_file_location(script_name, script_path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - plugin = getattr(module, self.type.capitalize()) - return plugin(self.prefix) - - # [end import_module] - - @property - # [start resource] - def resource(self): - """Dynamically return resource plugin.""" - for ex in self.get_all_modules(): - if ex.stem == self.type: - return self.import_module(ex.name, str(ex)) - raise PyResPluginException("{} type is not supported.".format(self.type)) + # def __init__(self, type: str, prefix: str): + # self.type = type + # self.prefix = prefix + # + # # [end init_method] + # + # # [start get_all_modules] + # def get_all_modules(self) -> Path: + # """Get all res files path in resources_plugin directory.""" + # return ( + # ex + # for ex in path_resources_plugin.iterdir() + # if ex.is_file() and not ex.name.startswith("__") + # ) + # + # # [end get_all_modules] + # + # # [start import_module] + # def import_module(self, script_name, script_path): + # """Import module.""" + # spec = importlib.util.spec_from_file_location(script_name, script_path) + # module = importlib.util.module_from_spec(spec) + # spec.loader.exec_module(module) + # plugin = getattr(module, self.type.capitalize()) + # return plugin(self.prefix) + # + # # [end import_module] + # + # @property + # # [start resource] + # def resource(self): + # """Dynamically return resource plugin.""" + # for ex in self.get_all_modules(): + # if ex.stem == self.type: + # return self.import_module(ex.name, str(ex)) + # raise PyResPluginException("{} type is not supported.".format(self.type)) # [end resource] + class Local: + """Local object, declare local resource plugin for task and workflow to dolphinscheduler. + + :param prefix: A string representing the prefix of Local. + + """ + + # [start init_method] + def __init__(self, prefix: str): + self._prefix = prefix + + # [end init_method] + + @property + def prefix(self): + """Get the _prefix attribute.""" + return self._prefix + + # [start read_file_method] + def read_file(self, suf: str): + """Get the content of the file. + + The address of the file is the prefix of the resource plugin plus the parameter suf. + """ + path = Path(self.prefix).joinpath(suf) + if not path.exists(): + raise PyResPluginException("{} is not found".format(str(path))) + if not os.access(str(path), os.R_OK): + raise PyResPluginException( + "You don't have permission to access {}".format(self.prefix + suf) + ) + with open(path, "r") as f: + content = f.read() + return content + + # [end read_file_method] + # [end resource_plugin_definition] diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py index 9f379b39c77f..b731196c8ffb 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py @@ -299,16 +299,10 @@ def test_task_ext_attr( { "name": "test_task_abtain_res_plugin", "task_type": "TaskType", - "resource_plugin": ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix="prefix", - ), + "resource_plugin": ResourcePlugin.Local("prefix"), "process_definition": ProcessDefinition( name="process_definition", - resource_plugin=ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix="prefix", - ), + resource_plugin=ResourcePlugin.Local("prefix"), ), }, "Local", @@ -317,10 +311,7 @@ def test_task_ext_attr( { "name": "test_task_abtain_res_plugin", "task_type": "TaskType", - "resource_plugin": ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix="prefix", - ), + "resource_plugin": ResourcePlugin.Local("prefix"), }, "Local", ), @@ -330,10 +321,7 @@ def test_task_ext_attr( "task_type": "TaskType", "process_definition": ProcessDefinition( name="process_definition", - resource_plugin=ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix="prefix", - ), + resource_plugin=ResourcePlugin.Local("prefix"), ), }, "Local", diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py index 59a1685fe27d..7055fe476136 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py @@ -22,7 +22,6 @@ import pytest -from pydolphinscheduler.constants import ResourcePluginType from pydolphinscheduler.resources_plugin import ResourcePlugin from pydolphinscheduler.tasks.shell import Shell from pydolphinscheduler.utils import file @@ -113,9 +112,7 @@ def test_shell_get_define(): { "name": "test-local-res-command-content", "command": file_name, - "resource_plugin": ResourcePlugin( - type=ResourcePluginType.LOCAL, prefix=res_plugin_prefix - ), + "resource_plugin": ResourcePlugin.Local(res_plugin_prefix), }, file_content, ) From fed8ef9592f87d89d9a4b45cfa68808ee2422e7c Mon Sep 17 00:00:00 2001 From: "chenrj.xdu@gmail.com" Date: Sat, 20 Aug 2022 21:46:27 +0800 Subject: [PATCH 57/58] restructure --- .../docs/source/resources_plugin/develop.rst | 17 +-- .../docs/source/resources_plugin/local.rst | 6 +- .../resources_plugin/resource-plugin.rst | 55 +++------ .../src/pydolphinscheduler/constants.py | 10 -- .../core/process_definition.py | 2 +- .../core/resource_plugin.py | 49 ++++++++ .../src/pydolphinscheduler/core/task.py | 2 +- .../examples/tutorial_resource_plugin.py | 8 +- .../resources_plugin/__init__.py | 109 +----------------- .../resources_plugin/local.py | 12 +- .../tests/core/test_task.py | 11 +- .../tests/resources_plugin/test_init.py | 105 ----------------- .../tests/resources_plugin/test_local.py | 9 +- .../tests/tasks/test_shell.py | 4 +- 14 files changed, 98 insertions(+), 301 deletions(-) create mode 100644 dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/resource_plugin.py delete mode 100644 dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/develop.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/develop.rst index 88d756aa0101..a67b686eee2a 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/develop.rst +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/develop.rst @@ -18,12 +18,14 @@ How to develop ============== -When you want to create a new resource plugin, you need to add a new class in the module `mod`:`resources_plugin`. All you have to do is named your plugin, implement `__init__` and `read_file` method. +When you want to create a new resource plugin, you need to add a new class in the module `resources_plugin`. -Your plugin class needs two indispensable methods, one is the __init__ method, the parameter is the prefix of type str, the other is -the read_file function, the parameter is the file suffix of type str, the return value is the file content, if it is exists and is readable. +The resource plug-in class needs to inherit the abstract class `ResourcePlugin` and implement its abstract method `read_file` function. + +The parameter of the `__init__` function of `ResourcePlugin` is the prefix of STR type. You can override it when necessary. + +The `read_file` function parameter of `ResourcePlugin` is the file suffix of STR type, and its return value is the file content, if it exists and is readable. -In addition you need to add a constant with your plugin name to ResourcePluginType in dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py, eg LOCAL = "local". Example ------- @@ -42,10 +44,3 @@ The prefix plus suffix is the absolute path of the file in this resource. .. literalinclude:: ../../../src/pydolphinscheduler/resources_plugin/local.py :start-after: [start read_file_method] :end-before: [end read_file_method] - -Last but not least, you should also add new resource plugin in constants.py - -.. literalinclude:: ../../../src/pydolphinscheduler/constants.py - :start-after: [start class_resource] - :end-before: [end class_resource] - diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/local.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/local.rst index afbe9a0540c7..5da025a5c727 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/local.rst +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/local.rst @@ -20,9 +20,9 @@ Local `Local` is a local resource plugin for pydolphinscheduler. -When using a local resource plugin, you do not need to use this class explicitly, you only need to add the -`resource_plugin` parameter in the task subclass or workflow definition. -The data type of parameter `resource_plugin` is `ResourcePlugin` +When using a local resource plugin, you only need to add the `resource_plugin` parameter in the task subclass or workflow definition, +such as `resource_plugin=Local("/tmp")`. + For the specific use of resource plugins, you can see `How to use` in :doc:`./resource-plugin` diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/resource-plugin.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/resource-plugin.rst index 7cd0b6f79887..c84bd4bee5ba 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/resource-plugin.rst +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/resource-plugin.rst @@ -18,12 +18,12 @@ ResourcePlugin ============== -ResourcePlugin is the data type of the resources plugin parameter in task subclasses and workflow definitions. - +`ResourcePlugin` is an abstract class of resource plug-in parameters of task subclass and workflow. +All resource plugins need to inherit and override its abstract methods. Code ---- -.. literalinclude:: ../../../src/pydolphinscheduler/resources_plugin/__init__.py +.. literalinclude:: ../../../src/pydolphinscheduler/core/resource_plugin.py :start-after: [start resource_plugin_definition] :end-before: [end resource_plugin_definition] @@ -31,57 +31,36 @@ Dive Into --------- It has the following key functions. -- Method `__init__`: The `__init__` function requires parameters. The first is the `type` of str type, which means the plugin type of the resource, +- Method `__init__`: The `__init__` function has STR type parameter `prefix`, which means the plugin type of the resource. -and the second is the `prefix` of the str type, which means the prefix of the resource. +You can rewrite it if necessary. -.. literalinclude:: ../../../src/pydolphinscheduler/resources_plugin/__init__.py +.. literalinclude:: ../../../src/pydolphinscheduler/core/resource_plugin.py :start-after: [start init_method] :end-before: [end init_method] -- Method `get_all_modules`: The `get_all_modules` function will return the absolute path of all resource plugins defined in the resource_plugin file. - -.. literalinclude:: ../../../src/pydolphinscheduler/resources_plugin/__init__.py - :start-after: [start get_all_modules] - :end-before: [end get_all_modules] - -- Method `import_module`: The `import_module` function has two parameters, `script_name` and `script_path`. +- Method `read_file`: Get content from the given URI, The function parameter is the suffix of the file path. -`script_name` is the name of the resource file without the suffix, such as the local resource plugin class local.py, +The file prefix has been initialized in init of the resource plug-in. -its `script_name` is local, and `script_path` is the absolute path of the resource plugin file +The prefix plus suffix is the absolute path of the file in this resource. -.. literalinclude:: ../../../src/pydolphinscheduler/resources_plugin/__init__.py - :start-after: [start import_module] - :end-before: [end import_module] +It is an abstract function. You must rewrite it -- Method `resource`: The `resource` function will dynamically return the resource plugin object according to the type parameter of the `__init__` function, +.. literalinclude:: ../../../src/pydolphinscheduler/core/resource_plugin.py + :start-after: [start abstractmethod read_file] + :end-before: [end abstractmethod read_file] -for example, the type is `ResourcePluginType.LOCAL`, then the local plugin object is returned - -.. literalinclude:: ../../../src/pydolphinscheduler/resources_plugin/__init__.py - :start-after: [start resource] - :end-before: [end resource] - -.. automodule:: pydolphinscheduler.resources_plugin.__init__ +.. automodule:: pydolphinscheduler.core.resource_plugin How to use ---------- Resource plug-ins can be used in task subclasses and workflows. You can use the resource plug-ins by adding the `resource_plugin` parameter when they are initialized. +For example, local resource plug-ins, add `resource_plugin = Local("/tmp")`. -Using resource plug-ins in workflows - -.. literalinclude:: ../../../src/pydolphinscheduler/examples/tutorial_resource_plugin.py - :start-after: [start workflow_declare] - :end-before: [end workflow_declare] - -Using resource plug-ins in sehll tasks - -.. literalinclude:: ../../../src/pydolphinscheduler/examples/tutorial_resource_plugin.py - :start-after: [start task_declare] - :end-before: [end task_declare] +The resource plug-ins we currently support is `local`. -Use resource plug-ins in both tasks and workflows +Here is an example. .. literalinclude:: ../../../src/pydolphinscheduler/examples/tutorial_resource_plugin.py :start-after: [start workflow_declare] diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py index 3f4ae76ff8b1..4544a6989d56 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py @@ -103,16 +103,6 @@ class Time(str): FMT_NO_COLON_TIME = "%H%M%S" -# [start class_resource] -class ResourcePluginType(str): - """Constants for resources plugin type, it will also show you which kind we support up to now.""" - - LOCAL = "local" - - -# [end class_resource] - - class ResourceKey(str): """Constants for key of resource.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py index 1d3dff693fd8..2679d546cb0f 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/process_definition.py @@ -24,10 +24,10 @@ from pydolphinscheduler import configuration from pydolphinscheduler.constants import TaskType from pydolphinscheduler.core.resource import Resource +from pydolphinscheduler.core.resource_plugin import ResourcePlugin from pydolphinscheduler.exceptions import PyDSParamException, PyDSTaskNoFoundException from pydolphinscheduler.java_gateway import JavaGate from pydolphinscheduler.models import Base, Project, Tenant, User -from pydolphinscheduler.resources_plugin import ResourcePlugin from pydolphinscheduler.utils.date import MAX_DATETIME, conv_from_str, conv_to_schedule diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/resource_plugin.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/resource_plugin.py new file mode 100644 index 000000000000..457a7c27b7f4 --- /dev/null +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/resource_plugin.py @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""DolphinScheduler ResourcePlugin object.""" + + +from abc import ABCMeta, abstractmethod + + +# [start resource_plugin_definition] +class ResourcePlugin(object, metaclass=ABCMeta): + """ResourcePlugin object, declare resource plugin for task and workflow to dolphinscheduler. + + :param prefix: A string representing the prefix of ResourcePlugin. + + """ + + # [start init_method] + def __init__(self, prefix: str, *args, **kwargs): + self.prefix = prefix + + # [end init_method] + + # [start abstractmethod read_file] + @abstractmethod + def read_file(self, suf: str): + """Get the content of the file. + + The address of the file is the prefix of the resource plugin plus the parameter suf. + """ + + # [end abstractmethod read_file] + + +# [end resource_plugin_definition] diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py index f6b145f996c5..dcd6a7aea837 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/task.py @@ -33,10 +33,10 @@ ProcessDefinitionContext, ) from pydolphinscheduler.core.resource import Resource +from pydolphinscheduler.core.resource_plugin import ResourcePlugin from pydolphinscheduler.exceptions import PyDSParamException, PyResPluginException from pydolphinscheduler.java_gateway import JavaGate from pydolphinscheduler.models import Base -from pydolphinscheduler.resources_plugin import ResourcePlugin logger = getLogger(__name__) diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py index 990494c93346..5b02022ee9fc 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial_resource_plugin.py @@ -25,15 +25,13 @@ import os from pathlib import Path -from pydolphinscheduler.constants import ResourcePluginType - # [start tutorial_resource_plugin] # [start package_import] # Import ProcessDefinition object to define your workflow attributes from pydolphinscheduler.core.process_definition import ProcessDefinition -from pydolphinscheduler.resources_plugin import ResourcePlugin # Import task Shell object cause we would create some shell tasks later +from pydolphinscheduler.resources_plugin.local import Local from pydolphinscheduler.tasks.shell import Shell # [end package_import] @@ -44,14 +42,14 @@ schedule="0 0 0 * * ? *", start_time="2021-01-01", tenant="tenant_exists", - resource_plugin=ResourcePlugin.Local(prefix="/tmp"), + resource_plugin=Local("/tmp"), ) as process_definition: # [end workflow_declare] # [start task_declare] file = "resource.sh" path = Path("/tmp").joinpath(file) with open(str(path), "w") as f: - f.write('echo tutorial resource plugin') + f.write("echo tutorial resource plugin") task_parent = Shell( name="local-resource-example", command=file, diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py index 05e3e9d48d8d..b6bc7a5ffb5a 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/__init__.py @@ -15,108 +15,9 @@ # specific language governing permissions and limitations # under the License. -"""Init resources_plugin package and dolphinScheduler ResourcePlugin object.""" +"""Init resources_plugin package.""" +from pydolphinscheduler.resources_plugin.local import Local -import importlib -import importlib.util -from pathlib import Path - -import os -from pathlib import Path - -from pydolphinscheduler.exceptions import PyResPluginException - -path_resources_plugin = Path(__file__).parent - - -# [start resource_plugin_definition] -class ResourcePlugin: - """ResourcePlugin object, declare resource plugin for task and workflow to dolphinscheduler. - - :param type: A unique, meaningful string for the ResourcePlugin, - - Its value should be taken from the constant of ResourceType in constants.py. - - :param prefix: A string representing the prefix of ResourcePlugin. - - """ - - # [start init_method] - # def __init__(self, type: str, prefix: str): - # self.type = type - # self.prefix = prefix - # - # # [end init_method] - # - # # [start get_all_modules] - # def get_all_modules(self) -> Path: - # """Get all res files path in resources_plugin directory.""" - # return ( - # ex - # for ex in path_resources_plugin.iterdir() - # if ex.is_file() and not ex.name.startswith("__") - # ) - # - # # [end get_all_modules] - # - # # [start import_module] - # def import_module(self, script_name, script_path): - # """Import module.""" - # spec = importlib.util.spec_from_file_location(script_name, script_path) - # module = importlib.util.module_from_spec(spec) - # spec.loader.exec_module(module) - # plugin = getattr(module, self.type.capitalize()) - # return plugin(self.prefix) - # - # # [end import_module] - # - # @property - # # [start resource] - # def resource(self): - # """Dynamically return resource plugin.""" - # for ex in self.get_all_modules(): - # if ex.stem == self.type: - # return self.import_module(ex.name, str(ex)) - # raise PyResPluginException("{} type is not supported.".format(self.type)) - - # [end resource] - - class Local: - """Local object, declare local resource plugin for task and workflow to dolphinscheduler. - - :param prefix: A string representing the prefix of Local. - - """ - - # [start init_method] - def __init__(self, prefix: str): - self._prefix = prefix - - # [end init_method] - - @property - def prefix(self): - """Get the _prefix attribute.""" - return self._prefix - - # [start read_file_method] - def read_file(self, suf: str): - """Get the content of the file. - - The address of the file is the prefix of the resource plugin plus the parameter suf. - """ - path = Path(self.prefix).joinpath(suf) - if not path.exists(): - raise PyResPluginException("{} is not found".format(str(path))) - if not os.access(str(path), os.R_OK): - raise PyResPluginException( - "You don't have permission to access {}".format(self.prefix + suf) - ) - with open(path, "r") as f: - content = f.read() - return content - - # [end read_file_method] - - -# [end resource_plugin_definition] +__all__ = [ + "Local", +] diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py index 0d5e5bb0195b..8a20ed9737e6 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/resources_plugin/local.py @@ -20,10 +20,11 @@ import os from pathlib import Path +from pydolphinscheduler.core.resource_plugin import ResourcePlugin from pydolphinscheduler.exceptions import PyResPluginException -class Local: +class Local(ResourcePlugin): """Local object, declare local resource plugin for task and workflow to dolphinscheduler. :param prefix: A string representing the prefix of Local. @@ -31,16 +32,11 @@ class Local: """ # [start init_method] - def __init__(self, prefix: str): - self._prefix = prefix + def __init__(self, prefix: str, *args, **kwargs): + super().__init__(prefix, *args, **kwargs) # [end init_method] - @property - def prefix(self): - """Get the _prefix attribute.""" - return self._prefix - # [start read_file_method] def read_file(self, suf: str): """Get the content of the file. diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py index b731196c8ffb..de358b918574 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_task.py @@ -22,11 +22,10 @@ import pytest -from pydolphinscheduler.constants import ResourcePluginType from pydolphinscheduler.core.process_definition import ProcessDefinition from pydolphinscheduler.core.task import Task, TaskRelation from pydolphinscheduler.exceptions import PyResPluginException -from pydolphinscheduler.resources_plugin import ResourcePlugin +from pydolphinscheduler.resources_plugin import Local from tests.testing.task import Task as testTask from tests.testing.task import TaskWithCode @@ -299,10 +298,10 @@ def test_task_ext_attr( { "name": "test_task_abtain_res_plugin", "task_type": "TaskType", - "resource_plugin": ResourcePlugin.Local("prefix"), + "resource_plugin": Local("prefix"), "process_definition": ProcessDefinition( name="process_definition", - resource_plugin=ResourcePlugin.Local("prefix"), + resource_plugin=Local("prefix"), ), }, "Local", @@ -311,7 +310,7 @@ def test_task_ext_attr( { "name": "test_task_abtain_res_plugin", "task_type": "TaskType", - "resource_plugin": ResourcePlugin.Local("prefix"), + "resource_plugin": Local("prefix"), }, "Local", ), @@ -321,7 +320,7 @@ def test_task_ext_attr( "task_type": "TaskType", "process_definition": ProcessDefinition( name="process_definition", - resource_plugin=ResourcePlugin.Local("prefix"), + resource_plugin=Local("prefix"), ), }, "Local", diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py deleted file mode 100644 index 0882b7f91621..000000000000 --- a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_init.py +++ /dev/null @@ -1,105 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -"""Test ResourcePlugin.""" -from collections import Counter -from pathlib import Path - -import pytest - -from pydolphinscheduler.constants import ResourcePluginType -from pydolphinscheduler.exceptions import PyResPluginException -from pydolphinscheduler.resources_plugin import ResourcePlugin - -all_res = ["local"] -project_root = Path(__file__).parent.parent.parent -resources_plugin_path = project_root.joinpath( - "src", "pydolphinscheduler", "resources_plugin" -) - - -@pytest.mark.parametrize( - "attr, expected", - [ - ( - { - "type": "res_type", - "prefix": "res_prefix", - }, - all_res, - ) - ], -) -def test_resources_get_all_modules(attr, expected): - """Test resource plugin to get all res plugin names.""" - res = ResourcePlugin(**attr) - assert dict(Counter(expected)) == dict( - Counter([ex.stem for ex in res.get_all_modules()]) - ) - - -@pytest.mark.parametrize( - "attrs, expected", - [ - ( - { - "type": ResourcePluginType.LOCAL, - "module_attr": { - "script_name": "local.py", - "script_path": resources_plugin_path.joinpath("local.py"), - }, - }, - "Local", - ), - ], -) -def test_resources_import_modules(attrs, expected): - """Test resource plug-in to import model.""" - res_plugin = ResourcePlugin(attrs.get("type"), "plugin-prefix") - res = res_plugin.import_module(**attrs.get("module_attr")) - assert expected == res.__class__.__name__ - - -@pytest.mark.parametrize( - "attr, expected", - [ - (ResourcePluginType.LOCAL, "Local"), - ], -) -def test_resources_resources(attr, expected): - """Test resource plugin factory.""" - res_plugin = ResourcePlugin(attr, "/tmp/") - res = res_plugin.resource - assert expected == res.__class__.__name__ - - -@pytest.mark.parametrize( - "attr", - [ - { - "type": "a", - "prefix": "/tmp/", - } - ], -) -def test_resources_unsupported_res(attr): - """Test unsupported plug-ins.""" - with pytest.raises( - PyResPluginException, match="{} type is not supported".format(attr.get("type")) - ): - res_plugin = ResourcePlugin(**attr) - res_plugin.resource() diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py index 0bf38bde7e44..82b196f75a20 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/resources_plugin/test_local.py @@ -21,10 +21,8 @@ import pytest -from pydolphinscheduler.constants import ResourcePluginType from pydolphinscheduler.core import Task from pydolphinscheduler.exceptions import PyResPluginException -from pydolphinscheduler.resources_plugin import ResourcePlugin from pydolphinscheduler.resources_plugin.local import Local from pydolphinscheduler.utils import file from tests.testing.file import delete_file @@ -77,11 +75,8 @@ def test_task_obtain_res_plugin( m_raw_script.return_value = val task = Task( name="test_task_ext_attr", - task_type=ResourcePluginType.LOCAL, - resource_plugin=ResourcePlugin( - type=ResourcePluginType.LOCAL, - prefix=str(res_plugin_prefix), - ), + task_type="type", + resource_plugin=Local(str(res_plugin_prefix)), ) assert expected == getattr(task, "raw_script") diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py index 7055fe476136..d6f665f138b1 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/tasks/test_shell.py @@ -22,7 +22,7 @@ import pytest -from pydolphinscheduler.resources_plugin import ResourcePlugin +from pydolphinscheduler.resources_plugin import Local from pydolphinscheduler.tasks.shell import Shell from pydolphinscheduler.utils import file from tests.testing.file import delete_file @@ -112,7 +112,7 @@ def test_shell_get_define(): { "name": "test-local-res-command-content", "command": file_name, - "resource_plugin": ResourcePlugin.Local(res_plugin_prefix), + "resource_plugin": Local(str(res_plugin_prefix)), }, file_content, ) From e0bdcffe16af69a3781e9f9984de93e703ed246c Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Tue, 6 Sep 2022 17:11:37 +0800 Subject: [PATCH 58/58] Modified RST file: develop and resource-plugin --- .../docs/source/resources_plugin/develop.rst | 2 +- .../docs/source/resources_plugin/resource-plugin.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/develop.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/develop.rst index a67b686eee2a..9e112b240a09 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/develop.rst +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/develop.rst @@ -22,7 +22,7 @@ When you want to create a new resource plugin, you need to add a new class in th The resource plug-in class needs to inherit the abstract class `ResourcePlugin` and implement its abstract method `read_file` function. -The parameter of the `__init__` function of `ResourcePlugin` is the prefix of STR type. You can override it when necessary. +The parameter of the `__init__` function of `ResourcePlugin` is the prefix of STR type. You can override this function when necessary. The `read_file` function parameter of `ResourcePlugin` is the file suffix of STR type, and its return value is the file content, if it exists and is readable. diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/resource-plugin.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/resource-plugin.rst index c84bd4bee5ba..0b90eeecbfc5 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/resource-plugin.rst +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/resources_plugin/resource-plugin.rst @@ -31,9 +31,9 @@ Dive Into --------- It has the following key functions. -- Method `__init__`: The `__init__` function has STR type parameter `prefix`, which means the plugin type of the resource. +- Method `__init__`: The `__init__` function has STR type parameter `prefix`, which means the prefix of the resource. -You can rewrite it if necessary. +You can rewrite this function if necessary. .. literalinclude:: ../../../src/pydolphinscheduler/core/resource_plugin.py :start-after: [start init_method]