Source code for flytekit.clients.helpers
-
-from flytekit.clis.auth import credentials as _credentials_access
-
-
-
-[docs]def iterate_node_executions(
- client,
- workflow_execution_identifier=None,
- task_execution_identifier=None,
- limit=None,
- filters=None
+[docs]def iterate_node_executions(
+ client, workflow_execution_identifier=None, task_execution_identifier=None, limit=None, filters=None,
):
"""
This returns a generator for node executions.
@@ -192,14 +183,11 @@ Source code for flytekit.clients.helpers
workflow_execution_identifier=workflow_execution_identifier,
limit=num_to_fetch,
token=token,
- filters=filters
+ filters=filters,
)
else:
node_execs, next_token = client.list_node_executions_for_task_paginated(
- task_execution_identifier=task_execution_identifier,
- limit=num_to_fetch,
- token=token,
- filters=filters
+ task_execution_identifier=task_execution_identifier, limit=num_to_fetch, token=token, filters=filters,
)
for n in node_execs:
counter += 1
@@ -227,10 +215,7 @@ Source code for flytekit.clients.helpers
counter = 0
while True:
task_execs, next_token = client.list_task_executions_paginated(
- node_execution_identifier=node_execution_identifier,
- limit=num_to_fetch,
- token=token,
- filters=filters
+ node_execution_identifier=node_execution_identifier, limit=num_to_fetch, token=token, filters=filters,
)
for t in task_execs:
counter += 1
@@ -240,7 +225,6 @@ Source code for flytekit.clients.helpers
if not next_token:
break
token = next_token
-
diff --git a/_modules/flytekit/clients/raw.html b/_modules/flytekit/clients/raw.html
index be70a771a4..a6174900db 100644
--- a/_modules/flytekit/clients/raw.html
+++ b/_modules/flytekit/clients/raw.html
@@ -8,7 +8,7 @@
- flytekit.clients.raw — Flyte 0.6.0 documentation
+ flytekit.clients.raw — Flyte 0.7.0 documentation
@@ -162,22 +162,24 @@
Source code for flytekit.clients.raw
from __future__ import absolute_import
-from grpc import insecure_channel as _insecure_channel, secure_channel as _secure_channel, RpcError as _RpcError, \
- StatusCode as _GrpcStatusCode, ssl_channel_credentials as _ssl_channel_credentials
-from google.protobuf.json_format import MessageToJson as _MessageToJson
-from flyteidl.service import admin_pb2_grpc as _admin_service
-from flytekit.common.exceptions import user as _user_exceptions
-from flytekit.configuration.platform import AUTH as _AUTH
-from flytekit.configuration.creds import (
- CLIENT_ID as _CLIENT_ID,
- CLIENT_CREDENTIALS_SCOPE as _SCOPE,
-)
-from flytekit.clis.sdk_in_container import basic_auth as _basic_auth
import logging as _logging
+
import six as _six
-from flytekit.configuration import creds as _creds_config, platform as _platform_config
+from flyteidl.service import admin_pb2_grpc as _admin_service
+from google.protobuf.json_format import MessageToJson as _MessageToJson
+from grpc import RpcError as _RpcError
+from grpc import StatusCode as _GrpcStatusCode
+from grpc import insecure_channel as _insecure_channel
+from grpc import secure_channel as _secure_channel
+from grpc import ssl_channel_credentials as _ssl_channel_credentials
from flytekit.clis.auth import credentials as _credentials_access
+from flytekit.clis.sdk_in_container import basic_auth as _basic_auth
+from flytekit.common.exceptions import user as _user_exceptions
+from flytekit.configuration import creds as _creds_config
+from flytekit.configuration.creds import CLIENT_CREDENTIALS_SCOPE as _SCOPE
+from flytekit.configuration.creds import CLIENT_ID as _CLIENT_ID
+from flytekit.configuration.platform import AUTH as _AUTH
def _refresh_credentials_standard(flyte_client):
@@ -205,10 +207,10 @@ Source code for flytekit.clients.raw
auth_endpoints = _credentials_access.get_authorization_endpoints(flyte_client.url)
token_endpoint = auth_endpoints.token_endpoint
client_secret = _basic_auth.get_secret()
- _logging.debug('Basic authorization flow with client id {} scope {}'.format(_CLIENT_ID.get(), _SCOPE.get()))
+ _logging.debug("Basic authorization flow with client id {} scope {}".format(_CLIENT_ID.get(), _SCOPE.get()))
authorization_header = _basic_auth.get_basic_authorization_header(_CLIENT_ID.get(), client_secret)
token, expires_in = _basic_auth.get_token(token_endpoint, authorization_header, _SCOPE.get())
- _logging.info('Retrieved new token, expires in {}'.format(expires_in))
+ _logging.info("Retrieved new token, expires in {}".format(expires_in))
flyte_client.set_access_token(token)
@@ -223,7 +225,8 @@ Source code for flytekit.clients.raw
return _refresh_credentials_basic
else:
raise ValueError(
- "Invalid auth mode [{}] specified. Please update the creds config to use a valid value".format(auth_mode))
+ "Invalid auth mode [{}] specified. Please update the creds config to use a valid value".format(auth_mode)
+ )
def _handle_rpc_error(fn):
@@ -253,6 +256,7 @@ Source code for flytekit.clients.raw
raise _user_exceptions.FlyteEntityAlreadyExistsException(_six.text_type(e))
else:
raise
+
return handler
@@ -298,9 +302,7 @@ Source code for flytekit.clients.raw
self._channel = _insecure_channel(url, options=list((options or {}).items()))
else:
self._channel = _secure_channel(
- url,
- credentials or _ssl_channel_credentials(),
- options=list((options or {}).items())
+ url, credentials or _ssl_channel_credentials(), options=list((options or {}).items()),
)
self._stub = _admin_service.AdminServiceStub(self._channel)
self._metadata = None
@@ -314,7 +316,7 @@ Source code for flytekit.clients.raw
[docs] def set_access_token(self, access_token):
# Always set the header to lower-case regardless of what the config is. The grpc libraries that Admin uses
# to parse the metadata don't change the metadata, but they do automatically lower the key you're looking for.
- self._metadata = [(_creds_config.AUTHORIZATION_METADATA_KEY.get().lower(), "Bearer {}".format(access_token))]
+ self._metadata = [(_creds_config.AUTHORIZATION_METADATA_KEY.get().lower(), "Bearer {}".format(access_token),)]
[docs] def force_auth_flow(self):
refresh_handler_fn = _get_refresh_handler(_creds_config.AUTH_MODE.get())
@@ -766,20 +768,48 @@ Source code for flytekit.clients.raw
def update_project_domain_attributes(self, project_domain_attributes_update_request):
"""
This updates the attributes for a project and domain registered with the Flyte Admin Service
- :param flyteidl.admin..ProjectDomainAttributesUpdateRequest project_domain_attributes_update_request:
- :rtype: flyteidl.admin..ProjectDomainAttributesUpdateResponse
+ :param flyteidl.admin.ProjectDomainAttributesUpdateRequest project_domain_attributes_update_request:
+ :rtype: flyteidl.admin.ProjectDomainAttributesUpdateResponse
"""
- return self._stub.UpdateProjectDomainAttributes(project_domain_attributes_update_request,
- metadata=self._metadata)
+ return self._stub.UpdateProjectDomainAttributes(
+ project_domain_attributes_update_request, metadata=self._metadata
+ )
[docs] @_handle_rpc_error
def update_workflow_attributes(self, workflow_attributes_update_request):
"""
This updates the attributes for a project, domain, and workflow registered with the Flyte Admin Service
- :param flyteidl.admin..UpdateWorkflowAttributes workflow_attributes_update_request:
- :rtype: flyteidl.admin..workflow_attributes_update_requestResponse
+ :param flyteidl.admin.UpdateWorkflowAttributesRequest workflow_attributes_update_request:
+ :rtype: flyteidl.admin.WorkflowAttributesUpdateResponse
+ """
+ return self._stub.UpdateWorkflowAttributes(workflow_attributes_update_request, metadata=self._metadata)
+
+[docs] @_handle_rpc_error
+ def get_project_domain_attributes(self, project_domain_attributes_get_request):
+ """
+ This fetches the attributes for a project and domain registered with the Flyte Admin Service
+ :param flyteidl.admin.ProjectDomainAttributesGetRequest project_domain_attributes_get_request:
+ :rtype: flyteidl.admin.ProjectDomainAttributesGetResponse
+ """
+ return self._stub.GetProjectDomainAttributes(project_domain_attributes_get_request, metadata=self._metadata)
+
+[docs] @_handle_rpc_error
+ def get_workflow_attributes(self, workflow_attributes_get_request):
+ """
+ This fetches the attributes for a project, domain, and workflow registered with the Flyte Admin Service
+ :param flyteidl.admin.GetWorkflowAttributesAttributesRequest workflow_attributes_get_request:
+ :rtype: flyteidl.admin.WorkflowAttributesGetResponse
+ """
+ return self._stub.GetWorkflowAttributes(workflow_attributes_get_request, metadata=self._metadata)
+
+[docs] @_handle_rpc_error
+ def list_matchable_attributes(self, matchable_attributes_list_request):
+ """
+ This fetches the attributes for a specific resource type registered with the Flyte Admin Service
+ :param flyteidl.admin.ListMatchableAttributesRequest matchable_attributes_list_request:
+ :rtype: flyteidl.admin.ListMatchableAttributesResponse
"""
- return self._stub.UpdateWorkflowAttributes(workflow_attributes_update_request, metadata=self._metadata)
+ return self._stub.ListMatchableAttributes(matchable_attributes_list_request, metadata=self._metadata)
####################################################################################################################
#
diff --git a/_modules/flytekit/clis/auth/auth.html b/_modules/flytekit/clis/auth/auth.html
index 9fa01af6e4..e0385cadec 100644
--- a/_modules/flytekit/clis/auth/auth.html
+++ b/_modules/flytekit/clis/auth/auth.html
@@ -8,7 +8,7 @@
- flytekit.clis.auth.auth — Flyte 0.6.0 documentation
+ flytekit.clis.auth.auth — Flyte 0.7.0 documentation
@@ -162,13 +162,14 @@
Source code for flytekit.clis.auth.auth
import base64 as _base64
import hashlib as _hashlib
-import keyring as _keyring
import os as _os
import re as _re
-import requests as _requests
import webbrowser as _webbrowser
+from multiprocessing import Process as _Process
+from multiprocessing import Queue as _Queue
-from multiprocessing import Process as _Process, Queue as _Queue
+import keyring as _keyring
+import requests as _requests
try: # Python 3.5+
from http import HTTPStatus as _StatusCodes
@@ -186,12 +187,13 @@ Source code for flytekit.clis.auth.auth
import urllib.parse as _urlparse
from urllib.parse import urlencode as _urlencode
except ImportError: # Python 2
- import urlparse as _urlparse
from urllib import urlencode as _urlencode
+ import urlparse as _urlparse
+
_code_verifier_length = 64
_random_seed_length = 40
-_utf_8 = 'utf-8'
+_utf_8 = "utf-8"
# Identifies the service used for storing passwords in keyring
@@ -210,7 +212,7 @@ Source code for flytekit.clis.auth.auth
"""
code_verifier = _base64.urlsafe_b64encode(_os.urandom(_code_verifier_length)).decode(_utf_8)
# Eliminate invalid characters.
- code_verifier = _re.sub(r'[^a-zA-Z0-9_\-.~]+', '', code_verifier)
+ code_verifier = _re.sub(r"[^a-zA-Z0-9_\-.~]+", "", code_verifier)
if len(code_verifier) < 43:
raise ValueError("Verifier too short. number of bytes must be > 30.")
elif len(code_verifier) > 128:
@@ -221,7 +223,7 @@ Source code for flytekit.clis.auth.auth
def _generate_state_parameter():
state = _base64.urlsafe_b64encode(_os.urandom(_random_seed_length)).decode(_utf_8)
# Eliminate invalid characters.
- code_verifier = _re.sub('[^a-zA-Z0-9-_.,]+', '', state)
+ code_verifier = _re.sub("[^a-zA-Z0-9-_.,]+", "", state)
return code_verifier
@@ -234,7 +236,7 @@ Source code for flytekit.clis.auth.auth
code_challenge = _hashlib.sha256(code_verifier.encode(_utf_8)).digest()
code_challenge = _base64.urlsafe_b64encode(code_challenge).decode(_utf_8)
# Eliminate invalid characters
- code_challenge = code_challenge.replace('=', '')
+ code_challenge = code_challenge.replace("=", "")
return code_challenge
@@ -268,7 +270,7 @@ Source code for flytekit.clis.auth.auth
self.send_response(_StatusCodes.NOT_FOUND)
[docs] def handle_login(self, data):
- self.server.handle_authorization_code(AuthorizationCode(data['code'], data['state']))
+ self.server.handle_authorization_code(AuthorizationCode(data["code"], data["state"]))
[docs]class OAuthHTTPServer(_BaseHTTPServer.HTTPServer):
@@ -276,8 +278,10 @@ Source code for flytekit.clis.auth.auth
A simple wrapper around the BaseHTTPServer.HTTPServer implementation that binds an authorization_client for handling
authorization code callbacks.
"""
- def __init__(self, server_address, RequestHandlerClass, bind_and_activate=True,
- redirect_path=None, queue=None):
+
+ def __init__(
+ self, server_address, RequestHandlerClass, bind_and_activate=True, redirect_path=None, queue=None,
+ ):
_BaseHTTPServer.HTTPServer.__init__(self, server_address, RequestHandlerClass, bind_and_activate)
self._redirect_path = redirect_path
self._auth_code = None
@@ -313,7 +317,7 @@ Source code for flytekit.clis.auth.auth
self._state = state
self._credentials = None
self._refresh_token = None
- self._headers = {'content-type': "application/x-www-form-urlencoded"}
+ self._headers = {"content-type": "application/x-www-form-urlencoded"}
self._expired = False
self._params = {
@@ -387,22 +391,18 @@ Source code for flytekit.clis.auth.auth
[docs] def request_access_token(self, auth_code):
if self._state != auth_code.state:
raise ValueError("Unexpected state parameter [{}] passed".format(auth_code.state))
- self._params.update({
- "code": auth_code.code,
- "code_verifier": self._code_verifier,
- "grant_type": "authorization_code",
- })
+ self._params.update(
+ {"code": auth_code.code, "code_verifier": self._code_verifier, "grant_type": "authorization_code"}
+ )
resp = _requests.post(
- url=self._token_endpoint,
- data=self._params,
- headers=self._headers,
- allow_redirects=False
+ url=self._token_endpoint, data=self._params, headers=self._headers, allow_redirects=False,
)
if resp.status_code != _StatusCodes.OK:
# TODO: handle expected (?) error cases:
# https://auth0.com/docs/flows/guides/device-auth/call-api-device-auth#token-responses
- raise Exception('Failed to request access token with response: [{}] {}'.format(
- resp.status_code, resp.content))
+ raise Exception(
+ "Failed to request access token with response: [{}] {}".format(resp.status_code, resp.content)
+ )
self._initialize_credentials(resp)
[docs] def refresh_access_token(self):
@@ -411,11 +411,9 @@ Source code for flytekit.clis.auth.auth
resp = _requests.post(
url=self._token_endpoint,
- data={'grant_type': 'refresh_token',
- 'client_id': self._client_id,
- 'refresh_token': self._refresh_token},
+ data={"grant_type": "refresh_token", "client_id": self._client_id, "refresh_token": self._refresh_token},
headers=self._headers,
- allow_redirects=False
+ allow_redirects=False,
)
if resp.status_code != _StatusCodes.OK:
self._expired = True
diff --git a/_modules/flytekit/clis/auth/credentials.html b/_modules/flytekit/clis/auth/credentials.html
index 8a9cda8fa2..52881a401e 100644
--- a/_modules/flytekit/clis/auth/credentials.html
+++ b/_modules/flytekit/clis/auth/credentials.html
@@ -8,7 +8,7 @@
- flytekit.clis.auth.credentials — Flyte 0.6.0 documentation
+ flytekit.clis.auth.credentials — Flyte 0.7.0 documentation
@@ -163,18 +163,15 @@ Source code for flytekit.clis.auth.credentials
from __future__ import absolute_import
import logging as _logging
+import urllib.parse as _urlparse
from flytekit.clis.auth.auth import AuthorizationClient as _AuthorizationClient
from flytekit.clis.auth.discovery import DiscoveryClient as _DiscoveryClient
-
-from flytekit.configuration.creds import (
- REDIRECT_URI as _REDIRECT_URI,
- CLIENT_ID as _CLIENT_ID,
-)
-from flytekit.configuration.platform import URL as _URL, INSECURE as _INSECURE, HTTP_URL as _HTTP_URL
-
-
-import urllib.parse as _urlparse
+from flytekit.configuration.creds import CLIENT_ID as _CLIENT_ID
+from flytekit.configuration.creds import REDIRECT_URI as _REDIRECT_URI
+from flytekit.configuration.platform import HTTP_URL as _HTTP_URL
+from flytekit.configuration.platform import INSECURE as _INSECURE
+from flytekit.configuration.platform import URL as _URL
# Default, well known-URI string used for fetching JSON metadata. See https://tools.ietf.org/html/rfc8414#section-3.
discovery_endpoint_path = "./.well-known/oauth-authorization-server"
@@ -185,17 +182,17 @@
Source code for flytekit.clis.auth.credentials
if http_config_val:
scheme, netloc, path, _, _, _ = _urlparse.urlparse(http_config_val)
if not scheme:
- scheme = 'http' if insecure_val else 'https'
+ scheme = "http" if insecure_val else "https"
else: # Use the main _URL config object effectively
- scheme = 'http' if insecure_val else 'https'
+ scheme = "http" if insecure_val else "https"
netloc = platform_url_val
- path = ''
+ path = ""
computed_endpoint = _urlparse.urlunparse((scheme, netloc, path, None, None, None))
# The urljoin function needs a trailing slash in order to append things correctly. Also, having an extra slash
# at the end is okay, it just gets stripped out.
- computed_endpoint = _urlparse.urljoin(computed_endpoint + '/', discovery_endpoint_path)
- _logging.info('Using {} as discovery endpoint'.format(computed_endpoint))
+ computed_endpoint = _urlparse.urljoin(computed_endpoint + "/", discovery_endpoint_path)
+ _logging.info("Using {} as discovery endpoint".format(computed_endpoint))
return computed_endpoint
@@ -209,10 +206,12 @@
Source code for flytekit.clis.auth.credentials
return _authorization_client
authorization_endpoints = get_authorization_endpoints(flyte_client_url)
- _authorization_client =\
- _AuthorizationClient(redirect_uri=_REDIRECT_URI.get(), client_id=_CLIENT_ID.get(),
- auth_endpoint=authorization_endpoints.auth_endpoint,
- token_endpoint=authorization_endpoints.token_endpoint)
+ _authorization_client = _AuthorizationClient(
+ redirect_uri=_REDIRECT_URI.get(),
+ client_id=_CLIENT_ID.get(),
+ auth_endpoint=authorization_endpoints.auth_endpoint,
+ token_endpoint=authorization_endpoints.token_endpoint,
+ )
return _authorization_client
diff --git a/_modules/flytekit/clis/auth/discovery.html b/_modules/flytekit/clis/auth/discovery.html
index afcb097533..fe55efa50e 100644
--- a/_modules/flytekit/clis/auth/discovery.html
+++ b/_modules/flytekit/clis/auth/discovery.html
@@ -8,7 +8,7 @@
- flytekit.clis.auth.discovery — Flyte 0.6.0 documentation
+ flytekit.clis.auth.discovery — Flyte 0.7.0 documentation
@@ -160,16 +160,9 @@
Source code for flytekit.clis.auth.discovery
-import requests as _requests
-import logging
+import logging
-try: # Python 3.5+
- from http import HTTPStatus as _StatusCodes
-except ImportError:
- try: # Python 3
- from http import client as _StatusCodes
- except ImportError: # Python 2
- import httplib as _StatusCodes
+import requests as _requests
# These response keys are defined in https://tools.ietf.org/id/draft-ietf-oauth-discovery-08.html.
_authorization_endpoint_key = "authorization_endpoint"
@@ -180,6 +173,7 @@ Source code for flytekit.clis.auth.discovery
"""
A simple wrapper around commonly discovered endpoints used for the PKCE auth flow.
"""
+
def __init__(self, auth_endpoint=None, token_endpoint=None):
self._auth_endpoint = auth_endpoint
self._token_endpoint = token_endpoint
@@ -214,9 +208,7 @@ Source code for flytekit.clis.auth.discovery
[docs] def get_authorization_endpoints(self):
if self.authorization_endpoints is not None:
return self.authorization_endpoints
- resp = _requests.get(
- url=self._discovery_url,
- )
+ resp = _requests.get(url=self._discovery_url,)
response_body = resp.json()
@@ -224,19 +216,20 @@ Source code for flytekit.clis.auth.discovery
token_endpoint = response_body[_token_endpoint_key]
if authorization_endpoint is None:
- raise ValueError('Unable to discover authorization endpoint')
+ raise ValueError("Unable to discover authorization endpoint")
if token_endpoint is None:
- raise ValueError('Unable to discover token endpoint')
+ raise ValueError("Unable to discover token endpoint")
if authorization_endpoint.startswith("/"):
- authorization_endpoint= _requests.compat.urljoin(self._discovery_url, authorization_endpoint)
+ authorization_endpoint = _requests.compat.urljoin(self._discovery_url, authorization_endpoint)
if token_endpoint.startswith("/"):
token_endpoint = _requests.compat.urljoin(self._discovery_url, token_endpoint)
- self._authorization_endpoints = AuthorizationEndpoints(auth_endpoint=authorization_endpoint,
- token_endpoint=token_endpoint)
+ self._authorization_endpoints = AuthorizationEndpoints(
+ auth_endpoint=authorization_endpoint, token_endpoint=token_endpoint
+ )
return self.authorization_endpoints
diff --git a/_modules/flytekit/clis/helpers.html b/_modules/flytekit/clis/helpers.html
index d8a124374a..04daf524c7 100644
--- a/_modules/flytekit/clis/helpers.html
+++ b/_modules/flytekit/clis/helpers.html
@@ -8,7 +8,7 @@
- flytekit.clis.helpers — Flyte 0.6.0 documentation
+ flytekit.clis.helpers — Flyte 0.7.0 documentation
@@ -203,8 +203,7 @@ Source code for flytekit.clis.helpers
:rtype: dict[Text, Text]
"""
- return {split_arg[0]: split_arg[1] for split_arg in
- [input_arg.split('=', 1) for input_arg in input_arguments]}
+ return {split_arg[0]: split_arg[1] for split_arg in [input_arg.split("=", 1) for input_arg in input_arguments]}
[docs]def construct_literal_map_from_parameter_map(parameter_map, text_args):
@@ -227,7 +226,7 @@ Source code for flytekit.clis.helpers
if var_name in text_args and text_args[var_name] is not None:
inputs[var_name] = sdk_type.from_string(text_args[var_name])
else:
- raise Exception('Missing required parameter {}'.format(var_name))
+ raise Exception("Missing required parameter {}".format(var_name))
else:
if var_name in text_args and text_args[var_name] is not None:
inputs[var_name] = sdk_type.from_string(text_args[var_name])
@@ -243,7 +242,7 @@ Source code for flytekit.clis.helpers
:param Text str:
:rtype: bool
"""
- return not str.lower() in ['false', '0', 'off', 'no']
+ return not str.lower() in ["false", "0", "off", "no"]
diff --git a/_modules/flytekit/clis/sdk_in_container/basic_auth.html b/_modules/flytekit/clis/sdk_in_container/basic_auth.html
index 52cd1aafad..e6e1df2705 100644
--- a/_modules/flytekit/clis/sdk_in_container/basic_auth.html
+++ b/_modules/flytekit/clis/sdk_in_container/basic_auth.html
@@ -8,7 +8,7 @@
- flytekit.clis.sdk_in_container.basic_auth — Flyte 0.6.0 documentation
+ flytekit.clis.sdk_in_container.basic_auth — Flyte 0.7.0 documentation
@@ -168,11 +168,9 @@ Source code for flytekit.clis.sdk_in_container.basic_auth
import requests as _requests
from flytekit.common.exceptions.user import FlyteAuthenticationException as _FlyteAuthenticationException
-from flytekit.configuration.creds import (
- CLIENT_CREDENTIALS_SECRET as _CREDENTIALS_SECRET,
-)
+from flytekit.configuration.creds import CLIENT_CREDENTIALS_SECRET as _CREDENTIALS_SECRET
-_utf_8 = 'utf-8'
+_utf_8 = "utf-8"
[docs]def get_secret():
@@ -184,7 +182,7 @@ Source code for flytekit.clis.sdk_in_container.basic_auth
secret = _CREDENTIALS_SECRET.get()
if secret:
return secret
- raise _FlyteAuthenticationException('No secret could be found')
+ raise _FlyteAuthenticationException("No secret could be found")
[docs]def get_basic_authorization_header(client_id, client_secret):
@@ -208,23 +206,23 @@ Source code for flytekit.clis.sdk_in_container.basic_auth
in seconds
"""
headers = {
- 'Authorization': authorization_header,
- 'Cache-Control': 'no-cache',
- 'Accept': 'application/json',
- 'Content-Type': 'application/x-www-form-urlencoded'
+ "Authorization": authorization_header,
+ "Cache-Control": "no-cache",
+ "Accept": "application/json",
+ "Content-Type": "application/x-www-form-urlencoded",
}
body = {
- 'grant_type': 'client_credentials',
+ "grant_type": "client_credentials",
}
if scope is not None:
- body['scope'] = scope
+ body["scope"] = scope
response = _requests.post(token_endpoint, data=body, headers=headers)
if response.status_code != 200:
_logging.error("Non-200 ({}) received from IDP: {}".format(response.status_code, response.text))
- raise _FlyteAuthenticationException('Non-200 received from IDP')
+ raise _FlyteAuthenticationException("Non-200 received from IDP")
response = response.json()
- return response['access_token'], response['expires_in']
+ return response["access_token"], response["expires_in"]
diff --git a/_modules/flytekit/clis/sdk_in_container/launch_plan.html b/_modules/flytekit/clis/sdk_in_container/launch_plan.html
index f231561a7d..1d3bd779a5 100644
--- a/_modules/flytekit/clis/sdk_in_container/launch_plan.html
+++ b/_modules/flytekit/clis/sdk_in_container/launch_plan.html
@@ -8,7 +8,7 @@
- flytekit.clis.sdk_in_container.launch_plan — Flyte 0.6.0 documentation
+ flytekit.clis.sdk_in_container.launch_plan — Flyte 0.7.0 documentation
@@ -162,16 +162,17 @@
Source code for flytekit.clis.sdk_in_container.launch_plan
from __future__ import absolute_import
+import logging as _logging
+
import click
import six as _six
-import logging as _logging
from flytekit.clis.helpers import construct_literal_map_from_parameter_map as _construct_literal_map_from_parameter_map
from flytekit.clis.sdk_in_container import constants as _constants
from flytekit.common import utils as _utils
from flytekit.common.launch_plan import SdkLaunchPlan as _SdkLaunchPlan
-from flytekit.configuration.internal import look_up_version_from_image_tag as _look_up_version_from_image_tag, \
- IMAGE as _IMAGE
+from flytekit.configuration.internal import IMAGE as _IMAGE
+from flytekit.configuration.internal import look_up_version_from_image_tag as _look_up_version_from_image_tag
from flytekit.models import launch_plan as _launch_plan_model
from flytekit.models.core import identifier as _identifier
from flytekit.tools.module_loader import iterate_registerable_entities_in_order
@@ -192,13 +193,13 @@ Source code for flytekit.clis.sdk_in_container.launch_plan
pkgs = ctx.obj[_constants.CTX_PACKAGES]
# Discover all launch plans by loading the modules
for m, k, lp in iterate_registerable_entities_in_order(
- pkgs, include_entities={_SdkLaunchPlan},
- detect_unreferenced_entities=False):
+ pkgs, include_entities={_SdkLaunchPlan}, detect_unreferenced_entities=False
+ ):
safe_name = _utils.fqdn(m.__name__, k, entity_type=lp.resource_type)
commands.append(safe_name)
lps[safe_name] = lp
- ctx.obj['lps'] = lps
+ ctx.obj["lps"] = lps
commands.sort()
return commands
@@ -210,24 +211,25 @@ Source code for flytekit.clis.sdk_in_container.launch_plan
launch_plan = None
pkgs = ctx.obj[_constants.CTX_PACKAGES]
- if 'lps' in ctx.obj:
- launch_plan = ctx.obj['lps'][lp_argument]
+ if "lps" in ctx.obj:
+ launch_plan = ctx.obj["lps"][lp_argument]
else:
for m, k, lp in iterate_registerable_entities_in_order(
- pkgs, include_entities={_SdkLaunchPlan}, detect_unreferenced_entities=False):
+ pkgs, include_entities={_SdkLaunchPlan}, detect_unreferenced_entities=False,
+ ):
safe_name = _utils.fqdn(m.__name__, k, entity_type=lp.resource_type)
if lp_argument == safe_name:
launch_plan = lp
if launch_plan is None:
- raise Exception('Could not load launch plan {}'.format(lp_argument))
+ raise Exception("Could not load launch plan {}".format(lp_argument))
launch_plan._id = _identifier.Identifier(
_identifier.ResourceType.LAUNCH_PLAN,
ctx.obj[_constants.CTX_PROJECT],
ctx.obj[_constants.CTX_DOMAIN],
lp_argument,
- ctx.obj[_constants.CTX_VERSION]
+ ctx.obj[_constants.CTX_VERSION],
)
return self._get_command(ctx, launch_plan, lp_argument)
@@ -241,7 +243,6 @@ Source code for flytekit.clis.sdk_in_container.launch_plan
[docs]class LaunchPlanExecuteGroup(LaunchPlanAbstractGroup):
-
def _get_command(self, ctx, lp, cmd_name):
"""
This function returns the function that click will actually use to execute a specific launch plan. It also
@@ -262,10 +263,11 @@ Source code for flytekit.clis.sdk_in_container.launch_plan
ctx.obj[_constants.CTX_PROJECT],
ctx.obj[_constants.CTX_DOMAIN],
literal_inputs=inputs,
- notification_overrides=ctx.obj.get(_constants.CTX_NOTIFICATIONS, None)
+ notification_overrides=ctx.obj.get(_constants.CTX_NOTIFICATIONS, None),
+ )
+ click.echo(
+ click.style("Workflow scheduled, execution_id={}".format(_six.text_type(execution.id)), fg="blue",)
)
- click.echo(click.style("Workflow scheduled, execution_id={}".format(
- _six.text_type(execution.id)), fg='blue'))
command = click.Command(name=cmd_name, callback=_execute_lp)
@@ -274,15 +276,13 @@ Source code for flytekit.clis.sdk_in_container.launch_plan
param = lp.default_inputs.parameters[var_name]
# TODO: Figure out how to better handle the fact that we want strings to parse,
# but we probably shouldn't have click say that that's the type on the CLI.
- help_msg = '{} Type: {}'.format(
- _six.text_type(param.var.description),
- _six.text_type(param.var.type)
+ help_msg = "{} Type: {}".format(
+ _six.text_type(param.var.description), _six.text_type(param.var.type)
).strip()
if param.required:
# If it's a required input, add the required flag
- wrapper = click.option('--{}'.format(var_name), required=True, type=_six.text_type,
- help=help_msg)
+ wrapper = click.option("--{}".format(var_name), required=True, type=_six.text_type, help=help_msg,)
else:
# If it's not a required input, it should have a default
# Use to_python_std so that the text of the default ends up being parseable, if not, the click
@@ -290,16 +290,19 @@ Source code for flytekit.clis.sdk_in_container.launch_plan
# we'd get '11' and then we'd need annoying logic to differentiate between the default text
# and user text.
default = param.default.to_python_std()
- wrapper = click.option('--{}'.format(var_name), default='{}'.format(_six.text_type(default)),
- type=_six.text_type,
- help='{}. Default: {}'.format(help_msg, _six.text_type(default)))
+ wrapper = click.option(
+ "--{}".format(var_name),
+ default="{}".format(_six.text_type(default)),
+ type=_six.text_type,
+ help="{}. Default: {}".format(help_msg, _six.text_type(default)),
+ )
command = wrapper(command)
return command
-@click.group('lp')
+@click.group("lp")
@click.pass_context
def launch_plans(ctx):
"""
@@ -308,7 +311,7 @@ Source code for flytekit.clis.sdk_in_container.launch_plan
pass
-@click.group('execute', cls=LaunchPlanExecuteGroup)
+@click.group("execute", cls=LaunchPlanExecuteGroup)
@click.pass_context
def execute_launch_plan(ctx):
"""
@@ -331,16 +334,20 @@ Source code for flytekit.clis.sdk_in_container.launch_plan
project,
domain,
_utils.fqdn(m.__name__, k, entity_type=lp.resource_type),
- version
+ version,
)
if not (lp.is_scheduled and ignore_schedules):
_logging.info(f"Setting active {_utils.fqdn(m.__name__, k, entity_type=lp.resource_type)}")
lp.update(_launch_plan_model.LaunchPlanState.ACTIVE)
-@click.command('activate-all-schedules')
-@click.option('-v', '--version', type=str, help='Version to register tasks with. This is normally parsed from the'
- 'image, but you can override here.')
+@click.command("activate-all-schedules")
+@click.option(
+ "-v",
+ "--version",
+ type=str,
+ help="Version to register tasks with. This is normally parsed from the" "image, but you can override here.",
+)
@click.pass_context
def activate_all_schedules(ctx, version=None):
"""
@@ -348,7 +355,9 @@ Source code for flytekit.clis.sdk_in_container.launch_plan
The behavior of this command is identical to activate-all.
"""
- click.secho("activate-all-schedules is deprecated, please use activate-all instead.", color="yellow")
+ click.secho(
+ "activate-all-schedules is deprecated, please use activate-all instead.", color="yellow",
+ )
project = ctx.obj[_constants.CTX_PROJECT]
domain = ctx.obj[_constants.CTX_DOMAIN]
pkgs = ctx.obj[_constants.CTX_PACKAGES]
@@ -356,10 +365,16 @@ Source code for flytekit.clis.sdk_in_container.launch_plan
activate_all_impl(project, domain, version, pkgs)
-@click.command('activate-all')
-@click.option('-v', '--version', type=str, help='Version to register tasks with. This is normally parsed from the'
- 'image, but you can override here.')
-@click.option("--ignore-schedules", is_flag=True, help='Activate all except for launch plans with schedules.')
+@click.command("activate-all")
+@click.option(
+ "-v",
+ "--version",
+ type=str,
+ help="Version to register tasks with. This is normally parsed from the" "image, but you can override here.",
+)
+@click.option(
+ "--ignore-schedules", is_flag=True, help="Activate all except for launch plans with schedules.",
+)
@click.pass_context
def activate_all(ctx, version=None, ignore_schedules=False):
"""
diff --git a/_modules/flytekit/clis/sdk_in_container/pyflyte.html b/_modules/flytekit/clis/sdk_in_container/pyflyte.html
index 37f33b73e0..05c9d2159c 100644
--- a/_modules/flytekit/clis/sdk_in_container/pyflyte.html
+++ b/_modules/flytekit/clis/sdk_in_container/pyflyte.html
@@ -8,7 +8,7 @@
- flytekit.clis.sdk_in_container.pyflyte — Flyte 0.6.0 documentation
+ flytekit.clis.sdk_in_container.pyflyte — Flyte 0.7.0 documentation
@@ -160,11 +160,11 @@
Source code for flytekit.clis.sdk_in_container.pyflyte
-from __future__ import absolute_import
-from __future__ import print_function
+from __future__ import absolute_import, print_function
-import os as _os
import logging as _logging
+import os as _os
+
import click
try:
@@ -172,32 +172,49 @@ Source code for flytekit.clis.sdk_in_container.pyflyte
except ImportError:
from pathlib2 import Path # python 2 backport
-from flytekit.clis.sdk_in_container.constants import CTX_PACKAGES
+from flytekit.clis.sdk_in_container.constants import CTX_DOMAIN, CTX_PACKAGES, CTX_PROJECT, CTX_VERSION
+from flytekit.clis.sdk_in_container.launch_plan import launch_plans
from flytekit.clis.sdk_in_container.register import register
from flytekit.clis.sdk_in_container.serialize import serialize
-from flytekit.clis.sdk_in_container.constants import CTX_PROJECT, CTX_DOMAIN, CTX_VERSION
-from flytekit.clis.sdk_in_container.launch_plan import launch_plans
-from flytekit.configuration import internal as _internal_config, platform as _platform_config, sdk as _sdk_config
-
+from flytekit.configuration import internal as _internal_config
+from flytekit.configuration import platform as _platform_config
+from flytekit.configuration import sdk as _sdk_config
+from flytekit.configuration import set_flyte_config_file
from flytekit.configuration.internal import CONFIGURATION_PATH
+from flytekit.configuration.internal import IMAGE as _IMAGE
+from flytekit.configuration.internal import look_up_version_from_image_tag as _look_up_version_from_image_tag
from flytekit.configuration.platform import URL as _URL
-from flytekit.configuration import set_flyte_config_file
-from flytekit.configuration.internal import look_up_version_from_image_tag as _look_up_version_from_image_tag, \
- IMAGE as _IMAGE
from flytekit.configuration.sdk import WORKFLOW_PACKAGES as _WORKFLOW_PACKAGES
-@click.group('pyflyte', invoke_without_command=True)
-@click.option('-p', '--project', required=True, type=str,
- help='Flyte project to use. You can have more than one project per repo')
-@click.option('-d', '--domain', required=True, type=str, help='This is usually development, staging, or production')
-@click.option('-c', '--config', required=False, type=str, help='Path to config file for use within container')
-@click.option('-k', '--pkgs', required=False, multiple=True,
- help='Dot separated python packages to operate on. Multiple may be specified Please note that this '
- 'option will override the option specified in the configuration file, or environment variable')
-@click.option('-v', '--version', required=False, type=str, help='This is the version to apply globally for this '
- 'context')
-@click.option('-i', '--insecure', required=False, type=bool, help='Do not use SSL to connect to Admin')
+@click.group("pyflyte", invoke_without_command=True)
+@click.option(
+ "-p",
+ "--project",
+ required=True,
+ type=str,
+ help="Flyte project to use. You can have more than one project per repo",
+)
+@click.option(
+ "-d", "--domain", required=True, type=str, help="This is usually development, staging, or production",
+)
+@click.option(
+ "-c", "--config", required=False, type=str, help="Path to config file for use within container",
+)
+@click.option(
+ "-k",
+ "--pkgs",
+ required=False,
+ multiple=True,
+ help="Dot separated python packages to operate on. Multiple may be specified Please note that this "
+ "option will override the option specified in the configuration file, or environment variable",
+)
+@click.option(
+ "-v", "--version", required=False, type=str, help="This is the version to apply globally for this " "context",
+)
+@click.option(
+ "-i", "--insecure", required=False, type=bool, help="Do not use SSL to connect to Admin",
+)
@click.pass_context
def main(ctx, project, domain, config=None, pkgs=None, version=None, insecure=None):
"""
@@ -221,7 +238,7 @@ Source code for flytekit.clis.sdk_in_container.pyflyte
# says no, let's override the config object by overriding the environment variable.
if insecure and not _platform_config.INSECURE.get():
_platform_config.INSECURE.get()
- _os.environ[_platform_config.INSECURE.env_var] = 'True'
+ _os.environ[_platform_config.INSECURE.env_var] = "True"
# Handle package management - get from config if not specified on the command line
pkgs = pkgs or []
@@ -243,14 +260,19 @@ Source code for flytekit.clis.sdk_in_container.pyflyte
"""
configuration_file = Path(config_file_path or CONFIGURATION_PATH.get())
if configuration_file.is_file():
- click.secho('Using configuration file at {}'.format(configuration_file.absolute().as_posix()),
- fg='green')
+ click.secho(
+ "Using configuration file at {}".format(configuration_file.absolute().as_posix()), fg="green",
+ )
set_flyte_config_file(configuration_file.as_posix())
else:
- click.secho("Configuration file '{}' could not be loaded. Using values from environment.".format(CONFIGURATION_PATH.get()),
- color='yellow')
+ click.secho(
+ "Configuration file '{}' could not be loaded. Using values from environment.".format(
+ CONFIGURATION_PATH.get()
+ ),
+ color="yellow",
+ )
set_flyte_config_file(None)
- click.secho('Flyte Admin URL {}'.format(_URL.get()), fg='green')
+ click.secho("Flyte Admin URL {}".format(_URL.get()), fg="green")
main.add_command(register)
diff --git a/_modules/flytekit/clis/sdk_in_container/register.html b/_modules/flytekit/clis/sdk_in_container/register.html
index f2325f3406..a1e6fe9aba 100644
--- a/_modules/flytekit/clis/sdk_in_container/register.html
+++ b/_modules/flytekit/clis/sdk_in_container/register.html
@@ -8,7 +8,7 @@
- flytekit.clis.sdk_in_container.register — Flyte 0.6.0 documentation
+ flytekit.clis.sdk_in_container.register — Flyte 0.7.0 documentation
@@ -166,20 +166,23 @@ Source code for flytekit.clis.sdk_in_container.register
import click
-from flytekit.clis.sdk_in_container.constants import CTX_PROJECT, CTX_DOMAIN, CTX_TEST, CTX_PACKAGES, CTX_VERSION
+from flytekit.clis.sdk_in_container.constants import CTX_DOMAIN, CTX_PACKAGES, CTX_PROJECT, CTX_TEST, CTX_VERSION
from flytekit.common import utils as _utils
from flytekit.common.core import identifier as _identifier
from flytekit.common.tasks import task as _task
-from flytekit.configuration.internal import look_up_version_from_image_tag as _look_up_version_from_image_tag, \
- IMAGE as _IMAGE
+from flytekit.configuration.internal import IMAGE as _IMAGE
+from flytekit.configuration.internal import look_up_version_from_image_tag as _look_up_version_from_image_tag
from flytekit.tools.module_loader import iterate_registerable_entities_in_order
[docs]def register_all(project, domain, pkgs, test, version):
if test:
- click.echo('Test switch enabled, not doing anything...')
- click.echo('Running task, workflow, and launch plan registration for {}, {}, {} with version {}'.format(
- project, domain, pkgs, version))
+ click.echo("Test switch enabled, not doing anything...")
+ click.echo(
+ "Running task, workflow, and launch plan registration for {}, {}, {} with version {}".format(
+ project, domain, pkgs, version
+ )
+ )
# m = module (i.e. python file)
# k = value of dir(m), type str
@@ -188,13 +191,7 @@ Source code for flytekit.clis.sdk_in_container.register
for m, k, o in iterate_registerable_entities_in_order(pkgs):
name = _utils.fqdn(m.__name__, k, entity_type=o.resource_type)
_logging.debug("Found module {}\n K: {} Instantiated in {}".format(m, k, o._instantiated_in))
- o._id = _identifier.Identifier(
- o.resource_type,
- project,
- domain,
- name,
- version
- )
+ o._id = _identifier.Identifier(o.resource_type, project, domain, name, version)
loaded_entities.append(o)
for o in loaded_entities:
@@ -207,10 +204,9 @@ Source code for flytekit.clis.sdk_in_container.register
[docs]def register_tasks_only(project, domain, pkgs, test, version):
if test:
- click.echo('Test switch enabled, not doing anything...')
+ click.echo("Test switch enabled, not doing anything...")
- click.echo('Running task only registration for {}, {}, {} with version {}'.format(
- project, domain, pkgs, version))
+ click.echo("Running task only registration for {}, {}, {} with version {}".format(project, domain, pkgs, version))
# Discover all tasks by loading the module
for m, k, t in iterate_registerable_entities_in_order(pkgs, include_entities={_task.SdkTask}):
@@ -223,10 +219,12 @@ Source code for flytekit.clis.sdk_in_container.register
t.register(project, domain, name, version)
-@click.group('register')
+@click.group("register")
# --pkgs on the register group is DEPRECATED, use same arg on pyflyte.main instead
-@click.option('--pkgs', multiple=True, help="DEPRECATED. This arg can only be used before the 'register' keyword")
-@click.option('--test', is_flag=True, help='Dry run, do not actually register with Admin')
+@click.option(
+ "--pkgs", multiple=True, help="DEPRECATED. This arg can only be used before the 'register' keyword",
+)
+@click.option("--test", is_flag=True, help="Dry run, do not actually register with Admin")
@click.pass_context
def register(ctx, pkgs=None, test=None):
"""
@@ -241,9 +239,13 @@ Source code for flytekit.clis.sdk_in_container.register
ctx.obj[CTX_TEST] = test
-@click.command('tasks')
-@click.option('-v', '--version', type=str, help='Version to register tasks with. This is normally parsed from the'
- 'image, but you can override here.')
+@click.command("tasks")
+@click.option(
+ "-v",
+ "--version",
+ type=str,
+ help="Version to register tasks with. This is normally parsed from the" "image, but you can override here.",
+)
@click.pass_context
def tasks(ctx, version=None):
"""
@@ -258,9 +260,13 @@ Source code for flytekit.clis.sdk_in_container.register
register_tasks_only(project, domain, pkgs, test, version)
-@click.command('workflows')
-@click.option('-v', '--version', type=str, help='Version to register tasks with. This is normally parsed from the'
- 'image, but you can override here.')
+@click.command("workflows")
+@click.option(
+ "-v",
+ "--version",
+ type=str,
+ help="Version to register tasks with. This is normally parsed from the" "image, but you can override here.",
+)
@click.pass_context
def workflows(ctx, version=None):
"""
diff --git a/_modules/flytekit/clis/sdk_in_container/serialize.html b/_modules/flytekit/clis/sdk_in_container/serialize.html
index 4583d6de08..1b355e6a93 100644
--- a/_modules/flytekit/clis/sdk_in_container/serialize.html
+++ b/_modules/flytekit/clis/sdk_in_container/serialize.html
@@ -8,7 +8,7 @@
- flytekit.clis.sdk_in_container.serialize — Flyte 0.6.0 documentation
+ flytekit.clis.sdk_in_container.serialize — Flyte 0.7.0 documentation
@@ -160,8 +160,7 @@
Source code for flytekit.clis.sdk_in_container.serialize
-from __future__ import absolute_import
-from __future__ import print_function
+from __future__ import absolute_import, print_function
import logging as _logging
import math as _math
@@ -169,7 +168,7 @@ Source code for flytekit.clis.sdk_in_container.serialize
import click
-from flytekit.clis.sdk_in_container.constants import CTX_PACKAGES, CTX_PROJECT, CTX_DOMAIN, CTX_VERSION
+from flytekit.clis.sdk_in_container.constants import CTX_DOMAIN, CTX_PACKAGES, CTX_PROJECT, CTX_VERSION
from flytekit.common import utils as _utils
from flytekit.common.core import identifier as _identifier
from flytekit.common.exceptions.scopes import system_entry_point
@@ -198,26 +197,20 @@ Source code for flytekit.clis.sdk_in_container.serialize
for m, k, o in iterate_registerable_entities_in_order(pkgs, include_entities={_sdk_task.SdkTask}):
name = _utils.fqdn(m.__name__, k, entity_type=o.resource_type)
_logging.debug("Found module {}\n K: {} Instantiated in {}".format(m, k, o._instantiated_in))
- o._id = _identifier.Identifier(
- o.resource_type,
- project,
- domain,
- name,
- version
- )
+ o._id = _identifier.Identifier(o.resource_type, project, domain, name, version)
loaded_entities.append(o)
zero_padded_length = _determine_text_chars(len(loaded_entities))
for i, entity in enumerate(loaded_entities):
serialized = entity.serialize()
fname_index = str(i).zfill(zero_padded_length)
- fname = '{}_{}.pb'.format(fname_index, entity._id.name)
- click.echo(' Writing {} to\n {}'.format(entity._id, fname))
+ fname = "{}_{}.pb".format(fname_index, entity._id.name)
+ click.echo(" Writing {} to\n {}".format(entity._id, fname))
if folder:
fname = _os.path.join(folder, fname)
_write_proto_to_file(serialized, fname)
- identifier_fname = '{}_{}.identifier.pb'.format(fname_index, entity._id.name)
+ identifier_fname = "{}_{}.identifier.pb".format(fname_index, entity._id.name)
if folder:
identifier_fname = _os.path.join(folder, identifier_fname)
_write_proto_to_file(entity._id.to_flyte_idl(), identifier_fname)
@@ -254,21 +247,15 @@ Source code for flytekit.clis.sdk_in_container.serialize
for m, k, o in iterate_registerable_entities_in_order(pkgs):
name = _utils.fqdn(m.__name__, k, entity_type=o.resource_type)
_logging.debug("Found module {}\n K: {} Instantiated in {}".format(m, k, o._instantiated_in))
- o._id = _identifier.Identifier(
- o.resource_type,
- project,
- domain,
- name,
- version
- )
+ o._id = _identifier.Identifier(o.resource_type, project, domain, name, version)
loaded_entities.append(o)
zero_padded_length = _determine_text_chars(len(loaded_entities))
for i, entity in enumerate(loaded_entities):
serialized = entity.serialize()
fname_index = str(i).zfill(zero_padded_length)
- fname = '{}_{}.pb'.format(fname_index, entity._id.name)
- click.echo(' Writing {} to\n {}'.format(entity._id, fname))
+ fname = "{}_{}.pb".format(fname_index, entity._id.name)
+ click.echo(" Writing {} to\n {}".format(entity._id, fname))
if folder:
fname = _os.path.join(folder, fname)
_write_proto_to_file(serialized, fname)
@@ -278,7 +265,7 @@ Source code for flytekit.clis.sdk_in_container.serialize
# project/domain, etc.) made for this serialize call. We should not allow users to specify a different project
# for instance come registration time, to avoid mismatches between potential internal ids like the TaskTemplate
# and the registered entity.
- identifier_fname = '{}_{}.identifier.pb'.format(fname_index, entity._id.name)
+ identifier_fname = "{}_{}.identifier.pb".format(fname_index, entity._id.name)
if folder:
identifier_fname = _os.path.join(folder, identifier_fname)
_write_proto_to_file(entity._id.to_flyte_idl(), identifier_fname)
@@ -295,7 +282,7 @@ Source code for flytekit.clis.sdk_in_container.serialize
return _math.ceil(_math.log(length, 10))
-@click.group('serialize')
+@click.group("serialize")
@click.pass_context
def serialize(ctx):
"""
@@ -305,13 +292,17 @@ Source code for flytekit.clis.sdk_in_container.serialize
object contains the WorkflowTemplate, along with the relevant tasks for that workflow. In lieu of Admin,
this serialization step will set the URN of the tasks to the fully qualified name of the task function.
"""
- click.echo('Serializing Flyte elements with image {}'.format(_internal_configuration.IMAGE.get()))
+ click.echo("Serializing Flyte elements with image {}".format(_internal_configuration.IMAGE.get()))
-@click.command('tasks')
-@click.option('-v', '--version', type=str, help='Version to serialize tasks with. This is normally parsed from the'
- 'image, but you can override here.')
-@click.option('-f', '--folder', type=click.Path(exists=True))
+@click.command("tasks")
+@click.option(
+ "-v",
+ "--version",
+ type=str,
+ help="Version to serialize tasks with. This is normally parsed from the" "image, but you can override here.",
+)
+@click.option("-f", "--folder", type=click.Path(exists=True))
@click.pass_context
def tasks(ctx, version=None, folder=None):
project = ctx.obj[CTX_PROJECT]
@@ -321,32 +312,40 @@ Source code for flytekit.clis.sdk_in_container.serialize
if folder:
click.echo(f"Writing output to {folder}")
- version = version or ctx.obj[CTX_VERSION] or _internal_configuration.look_up_version_from_image_tag(
- _internal_configuration.IMAGE.get())
+ version = (
+ version
+ or ctx.obj[CTX_VERSION]
+ or _internal_configuration.look_up_version_from_image_tag(_internal_configuration.IMAGE.get())
+ )
internal_settings = {
- 'project': project,
- 'domain': domain,
- 'version': version,
+ "project": project,
+ "domain": domain,
+ "version": version,
}
# Populate internal settings for project/domain/version from the environment so that the file names are resolved
# with the correct strings. The file itself doesn't need to change though.
with TemporaryConfiguration(_internal_configuration.CONFIGURATION_PATH.get(), internal_settings):
- _logging.debug("Serializing with settings\n"
- "\n Project: {}"
- "\n Domain: {}"
- "\n Version: {}"
- "\n\nover the following packages {}".format(project, domain, version, pkgs)
- )
+ _logging.debug(
+ "Serializing with settings\n"
+ "\n Project: {}"
+ "\n Domain: {}"
+ "\n Version: {}"
+ "\n\nover the following packages {}".format(project, domain, version, pkgs)
+ )
serialize_tasks_only(project, domain, pkgs, version, folder)
-@click.command('workflows')
-@click.option('-v', '--version', type=str, help='Version to serialize tasks with. This is normally parsed from the'
- 'image, but you can override here.')
+@click.command("workflows")
+@click.option(
+ "-v",
+ "--version",
+ type=str,
+ help="Version to serialize tasks with. This is normally parsed from the" "image, but you can override here.",
+)
# For now let's just assume that the directory needs to exist. If you're docker run -v'ing, docker will create the
# directory for you so it shouldn't be a problem.
-@click.option('-f', '--folder', type=click.Path(exists=True))
+@click.option("-f", "--folder", type=click.Path(exists=True))
@click.pass_context
def workflows(ctx, version=None, folder=None):
_logging.getLogger().setLevel(_logging.DEBUG)
@@ -358,23 +357,27 @@ Source code for flytekit.clis.sdk_in_container.serialize
domain = ctx.obj[CTX_DOMAIN]
pkgs = ctx.obj[CTX_PACKAGES]
- version = version or ctx.obj[CTX_VERSION] or _internal_configuration.look_up_version_from_image_tag(
- _internal_configuration.IMAGE.get())
+ version = (
+ version
+ or ctx.obj[CTX_VERSION]
+ or _internal_configuration.look_up_version_from_image_tag(_internal_configuration.IMAGE.get())
+ )
internal_settings = {
- 'project': project,
- 'domain': domain,
- 'version': version,
+ "project": project,
+ "domain": domain,
+ "version": version,
}
# Populate internal settings for project/domain/version from the environment so that the file names are resolved
# with the correct strings. The file itself doesn't need to change though.
with TemporaryConfiguration(_internal_configuration.CONFIGURATION_PATH.get(), internal_settings):
- _logging.debug("Serializing with settings\n"
- "\n Project: {}"
- "\n Domain: {}"
- "\n Version: {}"
- "\n\nover the following packages {}".format(project, domain, version, pkgs)
- )
+ _logging.debug(
+ "Serializing with settings\n"
+ "\n Project: {}"
+ "\n Domain: {}"
+ "\n Version: {}"
+ "\n\nover the following packages {}".format(project, domain, version, pkgs)
+ )
serialize_all(project, domain, pkgs, version, folder)
diff --git a/_modules/flytekit/common/component_nodes.html b/_modules/flytekit/common/component_nodes.html
index 4e0e9687ff..89e726294d 100644
--- a/_modules/flytekit/common/component_nodes.html
+++ b/_modules/flytekit/common/component_nodes.html
@@ -8,7 +8,7 @@
- flytekit.common.component_nodes — Flyte 0.6.0 documentation
+ flytekit.common.component_nodes — Flyte 0.7.0 documentation
@@ -162,16 +162,16 @@
Source code for flytekit.common.component_nodes
from __future__ import absolute_import
-import six as _six
import logging as _logging
+import six as _six
+
from flytekit.common import sdk_bases as _sdk_bases
from flytekit.common.exceptions import system as _system_exceptions
from flytekit.models.core import workflow as _workflow_model
[docs]class SdkTaskNode(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _workflow_model.TaskNode)):
-
def __init__(self, sdk_task):
"""
:param flytekit.common.tasks.task.SdkTask sdk_task:
@@ -205,6 +205,7 @@ Source code for flytekit.common.component_nodes
<
:rtype: SdkTaskNode
"""
from flytekit.common.tasks import task as _task
+
if base_model.reference_id in tasks:
t = tasks[base_model.reference_id]
_logging.debug("Found existing task template for {}, will not retrieve from Admin".format(t.id))
@@ -228,9 +229,12 @@ Source code for flytekit.common.component_nodes
<
:param flytekit.common.launch_plan.SdkLaunchPlan sdk_launch_plan:
"""
if sdk_workflow and sdk_launch_plan:
- raise _system_exceptions.FlyteSystemException("SdkWorkflowNode cannot be called with both a workflow and "
- "a launchplan specified, please pick one. WF: {} LP: {}",
- sdk_workflow, sdk_launch_plan)
+ raise _system_exceptions.FlyteSystemException(
+ "SdkWorkflowNode cannot be called with both a workflow and "
+ "a launchplan specified, please pick one. WF: {} LP: {}",
+ sdk_workflow,
+ sdk_launch_plan,
+ )
self._sdk_workflow = sdk_workflow
self._sdk_launch_plan = sdk_launch_plan
@@ -286,7 +290,8 @@ Source code for flytekit.common.component_nodes
<
:rtype: SdkWorkflowNode
"""
# put the import statement here to prevent circular dependency error
- from flytekit.common import workflow as _workflow, launch_plan as _launch_plan
+ from flytekit.common import launch_plan as _launch_plan
+ from flytekit.common import workflow as _workflow
project = base_model.reference.project
domain = base_model.reference.domain
@@ -299,21 +304,22 @@ Source code for flytekit.common.component_nodes
<
# The workflow templates for sub-workflows should have been included in the original response
if base_model.reference in sub_workflows:
sw = sub_workflows[base_model.reference]
- promoted = _workflow.SdkWorkflow.promote_from_model(sw, sub_workflows=sub_workflows,
- tasks=tasks)
+ promoted = _workflow.SdkWorkflow.promote_from_model(sw, sub_workflows=sub_workflows, tasks=tasks)
return cls(sdk_workflow=promoted)
# If not found for some reason, fetch it from Admin again.
# The reason there is a warning here but not for tasks is because sub-workflows should always be passed
# along. Ideally subworkflows are never even registered with Admin, so fetching from Admin ideally doesn't
# return anything.
- _logging.warning("Your subworkflow with id {} is not included in the promote call.".format(
- base_model.reference))
+ _logging.warning(
+ "Your subworkflow with id {} is not included in the promote call.".format(base_model.reference)
+ )
sdk_workflow = _workflow.SdkWorkflow.fetch(project, domain, name, version)
return cls(sdk_workflow=sdk_workflow)
else:
- raise _system_exceptions.FlyteSystemException("Bad workflow node model, neither subworkflow nor "
- "launchplan specified.")
+ raise _system_exceptions.FlyteSystemException(
+ "Bad workflow node model, neither subworkflow nor " "launchplan specified."
+ )
diff --git a/_modules/flytekit/common/constants.html b/_modules/flytekit/common/constants.html
index e2b8c67072..a8f7c59d14 100644
--- a/_modules/flytekit/common/constants.html
+++ b/_modules/flytekit/common/constants.html
@@ -8,7 +8,7 @@
- flytekit.common.constants — Flyte 0.6.0 documentation
+ flytekit.common.constants — Flyte 0.7.0 documentation
@@ -162,10 +162,10 @@
Source code for flytekit.common.constants
from __future__ import absolute_import
-INPUT_FILE_NAME = 'inputs.pb'
-OUTPUT_FILE_NAME = 'outputs.pb'
-FUTURES_FILE_NAME = 'futures.pb'
-ERROR_FILE_NAME = 'error.pb'
+INPUT_FILE_NAME = "inputs.pb"
+OUTPUT_FILE_NAME = "outputs.pb"
+FUTURES_FILE_NAME = "futures.pb"
+ERROR_FILE_NAME = "error.pb"
[docs]class SdkTaskType(object):
@@ -187,9 +187,11 @@ Source code for flytekit.common.constants
# Raw container task is just a name, it defaults to using the regular container task (like python etc), but sets the data_config in the container
RAW_CONTAINER_TASK = "raw-container"
SAGEMAKER_TRAINING_JOB_TASK = "sagemaker_training_job_task"
+ SAGEMAKER_CUSTOM_TRAINING_JOB_TASK = "sagemaker_custom_training_job_task"
SAGEMAKER_HYPERPARAMETER_TUNING_JOB_TASK = "sagemaker_hyperparameter_tuning_job_task"
-GLOBAL_INPUT_NODE_ID = ''
+
+GLOBAL_INPUT_NODE_ID = ""
START_NODE_ID = "start-node"
END_NODE_ID = "end-node"
diff --git a/_modules/flytekit/common/core/identifier.html b/_modules/flytekit/common/core/identifier.html
index adb7a64c33..4c8cc10cb3 100644
--- a/_modules/flytekit/common/core/identifier.html
+++ b/_modules/flytekit/common/core/identifier.html
@@ -8,7 +8,7 @@
- flytekit.common.core.identifier — Flyte 0.6.0 documentation
+ flytekit.common.core.identifier — Flyte 0.7.0 documentation
@@ -161,10 +161,12 @@
Source code for flytekit.common.core.identifier
from __future__ import absolute_import
-from flytekit.models.core import identifier as _core_identifier
+
+import six as _six
+
from flytekit.common import sdk_bases as _sdk_bases
from flytekit.common.exceptions import user as _user_exceptions
-import six as _six
+from flytekit.models.core import identifier as _core_identifier
[docs]class Identifier(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _core_identifier.Identifier)):
@@ -183,11 +185,7 @@ Source code for flytekit.common.core.identifier
<
:rtype: Identifier
"""
return cls(
- base_model.resource_type,
- base_model.project,
- base_model.domain,
- base_model.name,
- base_model.version
+ base_model.resource_type, base_model.project, base_model.domain, base_model.name, base_model.version,
)
[docs] @classmethod
@@ -209,28 +207,19 @@ Source code for flytekit.common.core.identifier
<
if resource_type not in cls._STRING_TO_TYPE_MAP:
raise _user_exceptions.FlyteValueException(
"The provided string could not be parsed. The first element of an identifier must be one of: {}. "
- "Received: {}".format(
- list(cls._STRING_TO_TYPE_MAP.keys()),
- resource_type
- )
+ "Received: {}".format(list(cls._STRING_TO_TYPE_MAP.keys()), resource_type)
)
resource_type = cls._STRING_TO_TYPE_MAP[resource_type]
- return cls(
- resource_type,
- project,
- domain,
- name,
- version
- )
+ return cls(resource_type, project, domain, name, version)
def __str__(self):
return "{}:{}:{}:{}:{}".format(
- type(self)._TYPE_TO_STRING_MAP.get(self.resource_type, '<unknown>'),
+ type(self)._TYPE_TO_STRING_MAP.get(self.resource_type, "<unknown>"),
self.project,
self.domain,
self.name,
- self.version
+ self.version,
)
@@ -243,11 +232,7 @@ Source code for flytekit.common.core.identifier
<
:param flytekit.models.core.identifier.WorkflowExecutionIdentifier base_model:
:rtype: WorkflowExecutionIdentifier
"""
- return cls(
- base_model.project,
- base_model.domain,
- base_model.name,
- )
+ return cls(base_model.project, base_model.domain, base_model.name,)
[docs] @classmethod
def from_python_std(cls, string):
@@ -261,7 +246,7 @@ Source code for flytekit.common.core.identifier
<
raise _user_exceptions.FlyteValueException(
string,
"The provided string was not in a parseable format. The string for an identifier must be in the format"
- " ex:project:domain:name."
+ " ex:project:domain:name.",
)
resource_type, project, domain, name = segments
@@ -269,21 +254,13 @@ Source code for flytekit.common.core.identifier
<
if resource_type != "ex":
raise _user_exceptions.FlyteValueException(
resource_type,
- "The provided string could not be parsed. The first element of an execution identifier must be 'ex'."
+ "The provided string could not be parsed. The first element of an execution identifier must be 'ex'.",
)
- return cls(
- project,
- domain,
- name,
- )
+ return cls(project, domain, name,)
def __str__(self):
- return "ex:{}:{}:{}".format(
- self.project,
- self.domain,
- self.name
- )
+ return "ex:{}:{}:{}".format(self.project, self.domain, self.name)
[docs]class TaskExecutionIdentifier(
@@ -298,7 +275,7 @@ Source code for flytekit.common.core.identifier
<
return cls(
task_id=base_model.task_id,
node_execution_id=base_model.node_execution_id,
- retry_attempt=base_model.retry_attempt
+ retry_attempt=base_model.retry_attempt,
)
[docs] @classmethod
@@ -313,7 +290,7 @@ Source code for flytekit.common.core.identifier
<
raise _user_exceptions.FlyteValueException(
string,
"The provided string was not in a parseable format. The string for an identifier must be in the format"
- " te:exec_project:exec_domain:exec_name:node_id:task_project:task_domain:task_name:task_version:retry."
+ " te:exec_project:exec_domain:exec_name:node_id:task_project:task_domain:task_name:task_version:retry.",
)
resource_type, ep, ed, en, node_id, tp, td, tn, tv, retry = segments
@@ -321,14 +298,13 @@ Source code for flytekit.common.core.identifier
<
if resource_type != "te":
raise _user_exceptions.FlyteValueException(
resource_type,
- "The provided string could not be parsed. The first element of an execution identifier must be 'ex'."
+ "The provided string could not be parsed. The first element of an execution identifier must be 'ex'.",
)
return cls(
task_id=Identifier(_core_identifier.ResourceType.TASK, tp, td, tn, tv),
node_execution_id=_core_identifier.NodeExecutionIdentifier(
- node_id=node_id,
- execution_id=_core_identifier.WorkflowExecutionIdentifier(ep, ed, en),
+ node_id=node_id, execution_id=_core_identifier.WorkflowExecutionIdentifier(ep, ed, en),
),
retry_attempt=int(retry),
)
diff --git a/_modules/flytekit/common/exceptions/base.html b/_modules/flytekit/common/exceptions/base.html
index c18ceb2b66..686c632f00 100644
--- a/_modules/flytekit/common/exceptions/base.html
+++ b/_modules/flytekit/common/exceptions/base.html
@@ -8,7 +8,7 @@
- flytekit.common.exceptions.base — Flyte 0.6.0 documentation
+ flytekit.common.exceptions.base — Flyte 0.7.0 documentation
@@ -161,6 +161,7 @@
Source code for flytekit.common.exceptions.base
from __future__ import absolute_import
+
import six as _six
@@ -171,7 +172,7 @@ Source code for flytekit.common.exceptions.base
<
[docs]class FlyteException(_six.with_metaclass(_FlyteCodedExceptionMetaclass, Exception)):
- _ERROR_CODE = 'UnknownFlyteException'
+ _ERROR_CODE = "UnknownFlyteException"
[docs]class FlyteRecoverableException(FlyteException):
diff --git a/_modules/flytekit/common/exceptions/scopes.html b/_modules/flytekit/common/exceptions/scopes.html
index 7f918d3c71..2d137d64c3 100644
--- a/_modules/flytekit/common/exceptions/scopes.html
+++ b/_modules/flytekit/common/exceptions/scopes.html
@@ -8,7 +8,7 @@
- flytekit.common.exceptions.scopes — Flyte 0.6.0 documentation
+ flytekit.common.exceptions.scopes — Flyte 0.7.0 documentation
@@ -161,17 +161,20 @@
Source code for flytekit.common.exceptions.scopes
from __future__ import absolute_import
-from six import reraise as _reraise
-from wrapt import decorator as _decorator
from sys import exc_info as _exc_info
-from flytekit.common.exceptions import system as _system_exceptions, user as _user_exceptions, base as _base_exceptions
-from flytekit.models.core import errors as _error_model
from traceback import format_tb as _format_tb
+from six import reraise as _reraise
+from wrapt import decorator as _decorator
-[docs]class FlyteScopedException(Exception):
+from flytekit.common.exceptions import base as _base_exceptions
+from flytekit.common.exceptions import system as _system_exceptions
+from flytekit.common.exceptions import user as _user_exceptions
+from flytekit.models.core import errors as _error_model
+
+[docs]class FlyteScopedException(Exception):
def __init__(self, context, exc_type, exc_value, exc_tb, top_trim=0, bottom_trim=0, kind=None):
self._exc_type = exc_type
self._exc_value = exc_value
@@ -198,16 +201,10 @@ Source code for flytekit.common.exceptions.scopes
lines = _format_tb(top_tb, limit=limit)
lines = [line.rstrip() for line in lines]
- lines = ('\n'.join(lines).split('\n'))
- traceback_str = '\n '.join([""] + lines)
-
- format_str = (
- "Traceback (most recent call last):\n"
- "{traceback}\n"
- "\n"
- "Message:\n"
- "\n"
- " {message}")
+ lines = "\n".join(lines).split("\n")
+ traceback_str = "\n ".join([""] + lines)
+
+ format_str = "Traceback (most recent call last):\n" "{traceback}\n" "\n" "Message:\n" "\n" " {message}"
return format_str.format(traceback=traceback_str, message=str(self.value))
def __str__(self):
@@ -263,11 +260,8 @@ Source code for flytekit.common.exceptions.scopes
[docs]class FlyteScopedSystemException(FlyteScopedException):
-
def __init__(self, exc_type, exc_value, exc_tb, **kwargs):
- super(FlyteScopedSystemException, self).__init__(
- "SYSTEM", exc_type, exc_value, exc_tb, **kwargs
- )
+ super(FlyteScopedSystemException, self).__init__("SYSTEM", exc_type, exc_value, exc_tb, **kwargs)
@property
def verbose_message(self):
@@ -280,11 +274,8 @@ Source code for flytekit.common.exceptions.scopes
[docs]class FlyteScopedUserException(FlyteScopedException):
-
def __init__(self, exc_type, exc_value, exc_tb, **kwargs):
- super(FlyteScopedUserException, self).__init__(
- "USER", exc_type, exc_value, exc_tb, **kwargs
- )
+ super(FlyteScopedUserException, self).__init__("USER", exc_type, exc_value, exc_tb, **kwargs)
@property
def verbose_message(self):
@@ -332,15 +323,15 @@ Source code for flytekit.common.exceptions.scopes
except _user_exceptions.FlyteUserException:
# Re-raise from here.
_reraise(
- FlyteScopedUserException,
- FlyteScopedUserException(*_exc_info()),
- _exc_info()[2])
- except:
+ FlyteScopedUserException, FlyteScopedUserException(*_exc_info()), _exc_info()[2],
+ )
+ except Exception:
# System error, raise full stack-trace all the way up the chain.
_reraise(
FlyteScopedSystemException,
FlyteScopedSystemException(*_exc_info(), kind=_error_model.ContainerError.Kind.RECOVERABLE),
- _exc_info()[2])
+ _exc_info()[2],
+ )
finally:
_CONTEXT_STACK.pop()
@@ -371,21 +362,18 @@ Source code for flytekit.common.exceptions.scopes
_reraise(*_exc_info())
except _user_exceptions.FlyteUserException:
_reraise(
- FlyteScopedUserException,
- FlyteScopedUserException(*_exc_info()),
- _exc_info()[2])
+ FlyteScopedUserException, FlyteScopedUserException(*_exc_info()), _exc_info()[2],
+ )
except _system_exceptions.FlyteSystemException:
_reraise(
- FlyteScopedSystemException,
- FlyteScopedSystemException(*_exc_info()),
- _exc_info()[2])
- except:
+ FlyteScopedSystemException, FlyteScopedSystemException(*_exc_info()), _exc_info()[2],
+ )
+ except Exception:
# Any non-platform raised exception is a user exception.
# This will also catch FlyteUserException re-raised by the system_entry_point handler
_reraise(
- FlyteScopedUserException,
- FlyteScopedUserException(*_exc_info()),
- _exc_info()[2])
+ FlyteScopedUserException, FlyteScopedUserException(*_exc_info()), _exc_info()[2],
+ )
finally:
_CONTEXT_STACK.pop()
diff --git a/_modules/flytekit/common/exceptions/system.html b/_modules/flytekit/common/exceptions/system.html
index 74eb15bc6d..082ccd7b53 100644
--- a/_modules/flytekit/common/exceptions/system.html
+++ b/_modules/flytekit/common/exceptions/system.html
@@ -8,7 +8,7 @@
- flytekit.common.exceptions.system — Flyte 0.6.0 documentation
+ flytekit.common.exceptions.system — Flyte 0.7.0 documentation
@@ -161,6 +161,7 @@
Source code for flytekit.common.exceptions.system
from __future__ import absolute_import
+
from flytekit.common.exceptions import base as _base_exceptions
@@ -180,23 +181,21 @@ Source code for flytekit.common.exceptions.system
if task_name is None:
return "Entrypoint is not loadable! Could not load the module: '{task_module}'{additional_msg}".format(
task_module=task_module,
- additional_msg=" due to error: {}".format(additional_msg) if additional_msg is not None else "."
+ additional_msg=" due to error: {}".format(additional_msg) if additional_msg is not None else ".",
)
else:
- return "Entrypoint is not loadable! Could not find the task: '{task_name}' in '{task_module}'" \
- "{additional_msg}".format(
- task_module=task_module,
- task_name=task_name,
- additional_msg="." if additional_msg is None else " due to error: {}".format(additional_msg)
- )
+ return (
+ "Entrypoint is not loadable! Could not find the task: '{task_name}' in '{task_module}'"
+ "{additional_msg}".format(
+ task_module=task_module,
+ task_name=task_name,
+ additional_msg="." if additional_msg is None else " due to error: {}".format(additional_msg),
+ )
+ )
def __init__(self, task_module, task_name=None, additional_msg=None):
super(FlyteSystemException, self).__init__(
- self._create_verbose_message(
- task_module,
- task_name=task_name,
- additional_msg=additional_msg
- )
+ self._create_verbose_message(task_module, task_name=task_name, additional_msg=additional_msg)
)
diff --git a/_modules/flytekit/common/exceptions/user.html b/_modules/flytekit/common/exceptions/user.html
index 4c471426f2..8360db0740 100644
--- a/_modules/flytekit/common/exceptions/user.html
+++ b/_modules/flytekit/common/exceptions/user.html
@@ -8,7 +8,7 @@
- flytekit.common.exceptions.user — Flyte 0.6.0 documentation
+ flytekit.common.exceptions.user — Flyte 0.7.0 documentation
@@ -161,7 +161,9 @@
Source code for flytekit.common.exceptions.user
from __future__ import absolute_import
-from flytekit.common.exceptions.base import FlyteException as _FlyteException, FlyteRecoverableException as _Recoverable
+
+from flytekit.common.exceptions.base import FlyteException as _FlyteException
+from flytekit.common.exceptions.base import FlyteRecoverableException as _Recoverable
[docs]class FlyteUserException(_FlyteException):
@@ -181,23 +183,22 @@ Source code for flytekit.common.exceptions.user
<
return "Type error! Received: {} with value: {}, Expected{}: {}. {}".format(
received_type,
received_value,
- ' one of' if FlyteTypeException._is_a_container(expected_type) else '',
+ " one of" if FlyteTypeException._is_a_container(expected_type) else "",
expected_type,
- additional_msg or '')
+ additional_msg or "",
+ )
else:
return "Type error! Received: {}, Expected{}: {}. {}".format(
received_type,
- ' one of' if FlyteTypeException._is_a_container(expected_type) else '',
+ " one of" if FlyteTypeException._is_a_container(expected_type) else "",
expected_type,
- additional_msg or '')
+ additional_msg or "",
+ )
def __init__(self, received_type, expected_type, additional_msg=None, received_value=None):
super(FlyteTypeException, self).__init__(
self._create_verbose_message(
- received_type,
- expected_type,
- received_value=received_value,
- additional_msg=additional_msg
+ received_type, expected_type, received_value=received_value, additional_msg=additional_msg,
)
)
@@ -211,9 +212,7 @@ Source code for flytekit.common.exceptions.user
<
return "Value error! Received: {}. {}".format(received_value, error_message)
def __init__(self, received_value, error_message):
- super(FlyteValueException, self).__init__(
- self._create_verbose_message(received_value, error_message)
- )
+ super(FlyteValueException, self).__init__(self._create_verbose_message(received_value, error_message))
[docs]class FlyteAssertion(FlyteUserException, AssertionError):
diff --git a/_modules/flytekit/common/interface.html b/_modules/flytekit/common/interface.html
index c4a121e561..1bf8621652 100644
--- a/_modules/flytekit/common/interface.html
+++ b/_modules/flytekit/common/interface.html
@@ -8,7 +8,7 @@
- flytekit.common.interface — Flyte 0.6.0 documentation
+ flytekit.common.interface — Flyte 0.7.0 documentation
@@ -164,14 +164,17 @@ Source code for flytekit.common.interface
import six as _six
-from flytekit.common import sdk_bases as _sdk_bases, promise as _promise
+from flytekit.common import promise as _promise
+from flytekit.common import sdk_bases as _sdk_bases
from flytekit.common.exceptions import user as _user_exceptions
-from flytekit.common.types import helpers as _type_helpers, containers as _containers, primitives as _primitives
-from flytekit.models import interface as _interface_models, literals as _literal_models
+from flytekit.common.types import containers as _containers
+from flytekit.common.types import helpers as _type_helpers
+from flytekit.common.types import primitives as _primitives
+from flytekit.models import interface as _interface_models
+from flytekit.models import literals as _literal_models
[docs]class BindingData(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _literal_models.BindingData)):
-
@staticmethod
def _has_sub_bindings(m):
"""
@@ -191,9 +194,7 @@ Source code for flytekit.common.interface
:param flytekit.models.literals.BindingData model:
:rtype: BindingData
"""
- return cls(
- scalar=model.scalar, collection=model.collection, promise=model.promise, map=model.map
- )
+ return cls(scalar=model.scalar, collection=model.collection, promise=model.promise, map=model.map,)
[docs] @classmethod
def from_python_std(cls, literal_type, t_value, upstream_nodes=None):
@@ -214,7 +215,7 @@ Source code for flytekit.common.interface
_user_exceptions.FlyteTypeException(
t_value.sdk_type,
downstream_sdk_type,
- additional_msg="When binding workflow input: {}".format(t_value)
+ additional_msg="When binding workflow input: {}".format(t_value),
)
promise = t_value.promise
elif isinstance(t_value, _promise.NodeOutput):
@@ -222,7 +223,7 @@ Source code for flytekit.common.interface
_user_exceptions.FlyteTypeException(
t_value.sdk_type,
downstream_sdk_type,
- additional_msg="When binding node output: {}".format(t_value)
+ additional_msg="When binding node output: {}".format(t_value),
)
promise = t_value
if upstream_nodes is not None:
@@ -233,20 +234,19 @@ Source code for flytekit.common.interface
type(t_value),
downstream_sdk_type,
received_value=t_value,
- additional_msg="Cannot bind a list to a non-list type."
+ additional_msg="Cannot bind a list to a non-list type.",
)
collection = _literal_models.BindingDataCollection(
[
BindingData.from_python_std(
- downstream_sdk_type.sub_type.to_flyte_literal_type(),
- v,
- upstream_nodes=upstream_nodes
+ downstream_sdk_type.sub_type.to_flyte_literal_type(), v, upstream_nodes=upstream_nodes,
)
for v in t_value
]
)
- elif isinstance(t_value, dict) and \
- (not issubclass(downstream_sdk_type, _primitives.Generic) or BindingData._has_sub_bindings(t_value)):
+ elif isinstance(t_value, dict) and (
+ not issubclass(downstream_sdk_type, _primitives.Generic) or BindingData._has_sub_bindings(t_value)
+ ):
# TODO: This behavior should be embedded in the type engine. Someone should be able to alter behavior of
# TODO: binding logic by injecting their own type engine. The same goes for the list check above.
raise NotImplementedError("TODO: Cannot use map bindings at the moment")
@@ -259,7 +259,6 @@ Source code for flytekit.common.interface
[docs]class TypedInterface(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _interface_models.TypedInterface)):
-
[docs] @classmethod
def promote_from_model(cls, model):
"""
@@ -280,14 +279,10 @@ Source code for flytekit.common.interface
for k in sorted(self.inputs):
var = self.inputs[k]
if k not in map_of_bindings:
- raise _user_exceptions.FlyteAssertion(
- "Input was not specified for: {} of type {}".format(k, var.type)
- )
+ raise _user_exceptions.FlyteAssertion("Input was not specified for: {} of type {}".format(k, var.type))
binding_data[k] = BindingData.from_python_std(
- var.type,
- map_of_bindings[k],
- upstream_nodes=all_upstream_nodes
+ var.type, map_of_bindings[k], upstream_nodes=all_upstream_nodes
)
extra_inputs = set(binding_data.keys()) ^ set(map_of_bindings.keys())
@@ -303,7 +298,10 @@ Source code for flytekit.common.interface
seen_nodes.add(n)
min_upstream.append(n)
- return [_literal_models.Binding(k, bd) for k, bd in _six.iteritems(binding_data)], min_upstream
+ return (
+ [_literal_models.Binding(k, bd) for k, bd in _six.iteritems(binding_data)],
+ min_upstream,
+ )
def __repr__(self):
return "({inputs}) -> ({outputs})".format(
@@ -318,7 +316,7 @@ Source code for flytekit.common.interface
"{}: {}".format(k, _type_helpers.get_sdk_type_from_literal_type(v.type))
for k, v in _six.iteritems(self.outputs)
]
- )
+ ),
)
diff --git a/_modules/flytekit/common/launch_plan.html b/_modules/flytekit/common/launch_plan.html
index fb713a0187..70c912758e 100644
--- a/_modules/flytekit/common/launch_plan.html
+++ b/_modules/flytekit/common/launch_plan.html
@@ -8,7 +8,7 @@
- flytekit.common.launch_plan — Flyte 0.6.0 documentation
+ flytekit.common.launch_plan — Flyte 0.7.0 documentation
@@ -162,30 +162,40 @@
Source code for flytekit.common.launch_plan
from __future__ import absolute_import
-from flytekit.common import sdk_bases as _sdk_bases, promise as _promises, interface as _interface, nodes as _nodes, \
- workflow_execution as _workflow_execution
-from flytekit.common.core import identifier as _identifier
-from flytekit.common.exceptions import scopes as _exception_scopes, user as _user_exceptions
-
-from flytekit.common.mixins import registerable as _registerable, hash as _hash_mixin, launchable as _launchable_mixin
-from flytekit.common.types import helpers as _type_helpers
-from flytekit.configuration import sdk as _sdk_config, auth as _auth_config
-from flytekit.engines import loader as _engine_loader
-from flytekit.models import launch_plan as _launch_plan_models, schedule as _schedule_model, interface as \
- _interface_models, literals as _literal_models, common as _common_models
-from flytekit.models.core import identifier as _identifier_model, workflow as _workflow_models
import datetime as _datetime
-from deprecated import deprecated as _deprecated
import logging as _logging
-import six as _six
import uuid as _uuid
+import six as _six
+from deprecated import deprecated as _deprecated
+
+from flytekit.common import interface as _interface
+from flytekit.common import nodes as _nodes
+from flytekit.common import promise as _promises
+from flytekit.common import sdk_bases as _sdk_bases
+from flytekit.common import workflow_execution as _workflow_execution
+from flytekit.common.core import identifier as _identifier
+from flytekit.common.exceptions import scopes as _exception_scopes
+from flytekit.common.exceptions import user as _user_exceptions
+from flytekit.common.mixins import hash as _hash_mixin
+from flytekit.common.mixins import launchable as _launchable_mixin
+from flytekit.common.mixins import registerable as _registerable
+from flytekit.common.types import helpers as _type_helpers
+from flytekit.configuration import auth as _auth_config
+from flytekit.configuration import sdk as _sdk_config
+from flytekit.engines import loader as _engine_loader
+from flytekit.models import common as _common_models
+from flytekit.models import interface as _interface_models
+from flytekit.models import launch_plan as _launch_plan_models
+from flytekit.models import literals as _literal_models
+from flytekit.models import schedule as _schedule_model
+from flytekit.models.core import identifier as _identifier_model
+from flytekit.models.core import workflow as _workflow_models
+
[docs]class SdkLaunchPlan(
_six.with_metaclass(
- _sdk_bases.ExtendedSdkType,
- _launch_plan_models.LaunchPlanSpec,
- _launchable_mixin.LaunchableEntity,
+ _sdk_bases.ExtendedSdkType, _launch_plan_models.LaunchPlanSpec, _launchable_mixin.LaunchableEntity,
)
):
def __init__(self, *args, **kwargs):
@@ -215,6 +225,7 @@ Source code for flytekit.common.launch_plan
labels=model.labels,
annotations=model.annotations,
auth_role=model.auth_role,
+ raw_output_data_config=model.raw_output_data_config,
)
[docs] @classmethod
@@ -230,6 +241,7 @@ Source code for flytekit.common.launch_plan
:rtype: SdkLaunchPlan
"""
from flytekit.common import workflow as _workflow
+
launch_plan_id = _identifier.Identifier(
_identifier_model.ResourceType.LAUNCH_PLAN, project, domain, name, version
)
@@ -268,19 +280,22 @@ Source code for flytekit.common.launch_plan
:rtype: flytekit.models.common.AuthRole
"""
fixed_auth = super(SdkLaunchPlan, self).auth_role
- if fixed_auth is not None and\
- (fixed_auth.assumable_iam_role is not None or fixed_auth.kubernetes_service_account is not None):
- return fixed_auth
+ if fixed_auth is not None and (
+ fixed_auth.assumable_iam_role is not None or fixed_auth.kubernetes_service_account is not None
+ ):
+ return fixed_auth
assumable_iam_role = _auth_config.ASSUMABLE_IAM_ROLE.get()
kubernetes_service_account = _auth_config.KUBERNETES_SERVICE_ACCOUNT.get()
if not (assumable_iam_role or kubernetes_service_account):
- _logging.warning("Using deprecated `role` from config. "
- "Please update your config to use `assumable_iam_role` instead")
+ _logging.warning(
+ "Using deprecated `role` from config. " "Please update your config to use `assumable_iam_role` instead"
+ )
assumable_iam_role = _sdk_config.ROLE.get()
- return _common_models.AuthRole(assumable_iam_role=assumable_iam_role,
- kubernetes_service_account=kubernetes_service_account)
+ return _common_models.AuthRole(
+ assumable_iam_role=assumable_iam_role, kubernetes_service_account=kubernetes_service_account,
+ )
@property
def interface(self):
@@ -304,6 +319,18 @@ Source code for flytekit.common.launch_plan
"""
return "Launch Plan"
+ @property
+ def raw_output_data_config(self):
+ """
+ :rtype: flytekit.models.common.RawOutputDataConfig
+ """
+ raw_output_data_config = super(SdkLaunchPlan, self).raw_output_data_config
+ if raw_output_data_config is not None and raw_output_data_config.output_location_prefix != "":
+ return raw_output_data_config
+
+ # If it was not set explicitly then let's use the value found in the configuration.
+ return _common_models.RawOutputDataConfig(_auth_config.RAW_OUTPUT_DATA_PREFIX.get())
+
[docs] @_exception_scopes.system_entry_point
def validate(self):
# TODO: Validate workflow is satisfied
@@ -329,24 +356,38 @@ Source code for flytekit.common.launch_plan
"""
return _type_helpers.pack_python_std_map_to_literal_map(
inputs,
- {
- k: user_input.sdk_type
- for k, user_input in _six.iteritems(self.default_inputs.parameters) if k in inputs
- }
+ {k: user_input.sdk_type for k, user_input in _six.iteritems(self.default_inputs.parameters) if k in inputs},
)
-[docs] @_deprecated(reason="Use launch_with_literals instead", version='0.9.0')
- def execute_with_literals(self, project, domain, literal_inputs, name=None, notification_overrides=None,
- label_overrides=None, annotation_overrides=None):
+[docs] @_deprecated(reason="Use launch_with_literals instead", version="0.9.0")
+ def execute_with_literals(
+ self,
+ project,
+ domain,
+ literal_inputs,
+ name=None,
+ notification_overrides=None,
+ label_overrides=None,
+ annotation_overrides=None,
+ ):
"""
Deprecated.
"""
- return self.launch_with_literals(project, domain, literal_inputs, name, notification_overrides, label_overrides,
- annotation_overrides)
+ return self.launch_with_literals(
+ project, domain, literal_inputs, name, notification_overrides, label_overrides, annotation_overrides,
+ )
[docs] @_exception_scopes.system_entry_point
- def launch_with_literals(self, project, domain, literal_inputs, name=None, notification_overrides=None,
- label_overrides=None, annotation_overrides=None):
+ def launch_with_literals(
+ self,
+ project,
+ domain,
+ literal_inputs,
+ name=None,
+ notification_overrides=None,
+ label_overrides=None,
+ annotation_overrides=None,
+ ):
"""
Executes the launch plan and returns the execution identifier. This version of execution is meant for when
you already have a LiteralMap of inputs.
@@ -365,14 +406,18 @@ Source code for flytekit.common.launch_plan
"""
# Kubernetes requires names starting with an alphabet for some resources.
name = name or "f" + _uuid.uuid4().hex[:19]
- execution = _engine_loader.get_engine().get_launch_plan(self).launch(
- project,
- domain,
- name,
- literal_inputs,
- notification_overrides=notification_overrides,
- label_overrides=label_overrides,
- annotation_overrides=annotation_overrides,
+ execution = (
+ _engine_loader.get_engine()
+ .get_launch_plan(self)
+ .launch(
+ project,
+ domain,
+ name,
+ literal_inputs,
+ notification_overrides=notification_overrides,
+ label_overrides=label_overrides,
+ annotation_overrides=annotation_overrides,
+ )
)
return _workflow_execution.SdkWorkflowExecution.promote_from_model(execution)
@@ -386,14 +431,11 @@ Source code for flytekit.common.launch_plan
if len(args) > 0:
raise _user_exceptions.FlyteAssertion(
"When adding a launchplan as a node in a workflow, all inputs must be specified with kwargs only. We "
- "detected {} positional args.".format(self, len(args))
+ "detected {} positional args.".format(len(args))
)
# Take the default values from the launch plan
- default_inputs = {
- k: v.sdk_default
- for k, v in _six.iteritems(self.default_inputs.parameters) if not v.required
- }
+ default_inputs = {k: v.sdk_default for k, v in _six.iteritems(self.default_inputs.parameters) if not v.required}
default_inputs.update(input_map)
bindings, upstream_nodes = self.interface.create_bindings_for_inputs(default_inputs)
@@ -403,7 +445,7 @@ Source code for flytekit.common.launch_plan
metadata=_workflow_models.NodeMetadata("", _datetime.timedelta(), _literal_models.RetryStrategy(0)),
bindings=sorted(bindings, key=lambda b: b.var),
upstream_nodes=upstream_nodes,
- sdk_launch_plan=self
+ sdk_launch_plan=self,
)
def __repr__(self):
@@ -416,21 +458,20 @@ Source code for flytekit.common.launch_plan
# The difference between this and the SdkLaunchPlan class is that this runnable class is supposed to only be used for
# launch plans loaded alongside the current Python interpreter.
[docs]class SdkRunnableLaunchPlan(
- _hash_mixin.HashOnReferenceMixin,
- SdkLaunchPlan,
- _registerable.RegisterableEntity,
+ _hash_mixin.HashOnReferenceMixin, SdkLaunchPlan, _registerable.RegisterableEntity,
):
def __init__(
- self,
- sdk_workflow,
- default_inputs=None,
- fixed_inputs=None,
- role=None,
- schedule=None,
- notifications=None,
- labels=None,
- annotations=None,
- auth_role=None,
+ self,
+ sdk_workflow,
+ default_inputs=None,
+ fixed_inputs=None,
+ role=None,
+ schedule=None,
+ notifications=None,
+ labels=None,
+ annotations=None,
+ auth_role=None,
+ raw_output_data_config=None,
):
"""
:param flytekit.common.workflow.SdkWorkflow sdk_workflow:
@@ -446,6 +487,7 @@ Source code for flytekit.common.launch_plan
executed by this launch plan.
Any custom kubernetes annotations to apply to workflows executed by this launch plan.
:param flytekit.models.common.Authrole auth_role: The auth method with which to execute the workflow.
+ :param flytekit.models.common.RawOutputDataConfig raw_output_data_config: Config for offloading data
"""
if role and auth_role:
raise ValueError("Cannot set both role and auth. Role is deprecated, use auth instead.")
@@ -465,24 +507,24 @@ Source code for flytekit.common.launch_plan
super(SdkRunnableLaunchPlan, self).__init__(
None,
_launch_plan_models.LaunchPlanMetadata(
- schedule=schedule or _schedule_model.Schedule(''),
- notifications=notifications or []
+ schedule=schedule or _schedule_model.Schedule(""), notifications=notifications or [],
),
_interface_models.ParameterMap(default_inputs),
_type_helpers.pack_python_std_map_to_literal_map(
fixed_inputs,
{
k: _type_helpers.get_sdk_type_from_literal_type(var.type)
- for k, var in _six.iteritems(sdk_workflow.interface.inputs) if k in fixed_inputs
- }
+ for k, var in _six.iteritems(sdk_workflow.interface.inputs)
+ if k in fixed_inputs
+ },
),
labels or _common_models.Labels({}),
annotations or _common_models.Annotations({}),
auth_role,
+ raw_output_data_config or _common_models.RawOutputDataConfig(""),
)
self._interface = _interface.TypedInterface(
- {k: v.var for k, v in _six.iteritems(default_inputs)},
- sdk_workflow.interface.outputs
+ {k: v.var for k, v in _six.iteritems(default_inputs)}, sdk_workflow.interface.outputs,
)
self._upstream_entities = {sdk_workflow}
self._sdk_workflow = sdk_workflow
@@ -497,11 +539,7 @@ Source code for flytekit.common.launch_plan
"""
self.validate()
id_to_register = _identifier.Identifier(
- _identifier_model.ResourceType.LAUNCH_PLAN,
- project,
- domain,
- name,
- version
+ _identifier_model.ResourceType.LAUNCH_PLAN, project, domain, name, version
)
_engine_loader.get_engine().get_launch_plan(self).register(id_to_register)
self._id = id_to_register
diff --git a/_modules/flytekit/common/mixins/artifact.html b/_modules/flytekit/common/mixins/artifact.html
index 81ec93cba7..be8490a200 100644
--- a/_modules/flytekit/common/mixins/artifact.html
+++ b/_modules/flytekit/common/mixins/artifact.html
@@ -8,7 +8,7 @@
- flytekit.common.mixins.artifact — Flyte 0.6.0 documentation
+ flytekit.common.mixins.artifact — Flyte 0.7.0 documentation
@@ -161,16 +161,18 @@
Source code for flytekit.common.mixins.artifact
from __future__ import absolute_import
+
import abc as _abc
import datetime as _datetime
-import six as _six
import time as _time
-from flytekit.models import common as _common_models
+
+import six as _six
+
from flytekit.common.exceptions import user as _user_exceptions
+from flytekit.models import common as _common_models
[docs]class ExecutionArtifact(_six.with_metaclass(_common_models.FlyteABCMeta, object)):
-
@_abc.abstractproperty
def inputs(self):
"""
diff --git a/_modules/flytekit/common/mixins/hash.html b/_modules/flytekit/common/mixins/hash.html
index d1bdb20d78..29f9c25744 100644
--- a/_modules/flytekit/common/mixins/hash.html
+++ b/_modules/flytekit/common/mixins/hash.html
@@ -8,7 +8,7 @@
- flytekit.common.mixins.hash — Flyte 0.6.0 documentation
+ flytekit.common.mixins.hash — Flyte 0.7.0 documentation
diff --git a/_modules/flytekit/common/mixins/launchable.html b/_modules/flytekit/common/mixins/launchable.html
index 9775c264b4..b206f9572e 100644
--- a/_modules/flytekit/common/mixins/launchable.html
+++ b/_modules/flytekit/common/mixins/launchable.html
@@ -8,7 +8,7 @@
- flytekit.common.mixins.launchable — Flyte 0.6.0 documentation
+ flytekit.common.mixins.launchable — Flyte 0.7.0 documentation
@@ -161,15 +161,24 @@
Source code for flytekit.common.mixins.launchable
from __future__ import absolute_import
+
import abc as _abc
-import six as _six
+import six as _six
from deprecated import deprecated as _deprecated
[docs]class LaunchableEntity(_six.with_metaclass(_abc.ABCMeta, object)):
-[docs] def launch(self, project, domain, inputs=None, name=None, notification_overrides=None, label_overrides=None,
- annotation_overrides=None):
+[docs] def launch(
+ self,
+ project,
+ domain,
+ inputs=None,
+ name=None,
+ notification_overrides=None,
+ label_overrides=None,
+ annotation_overrides=None,
+ ):
"""
Creates a remote execution from the entity and returns the execution identifier.
This version of launch is meant for when inputs are specified as Python native types/structures.
@@ -198,9 +207,17 @@ Source code for flytekit.common.mixins.launchable
annotation_overrides=annotation_overrides,
)
-[docs] @_deprecated(reason="Use launch instead", version='0.9.0')
- def execute(self, project, domain, inputs=None, name=None, notification_overrides=None, label_overrides=None,
- annotation_overrides=None):
+[docs] @_deprecated(reason="Use launch instead", version="0.9.0")
+ def execute(
+ self,
+ project,
+ domain,
+ inputs=None,
+ name=None,
+ notification_overrides=None,
+ label_overrides=None,
+ annotation_overrides=None,
+ ):
"""
Deprecated.
"""
@@ -219,8 +236,16 @@ Source code for flytekit.common.mixins.launchable
pass
[docs] @_abc.abstractmethod
- def launch_with_literals(self, project, domain, literal_inputs, name=None, notification_overrides=None,
- label_overrides=None, annotation_overrides=None):
+ def launch_with_literals(
+ self,
+ project,
+ domain,
+ literal_inputs,
+ name=None,
+ notification_overrides=None,
+ label_overrides=None,
+ annotation_overrides=None,
+ ):
"""
Executes the entity and returns the execution identifier. This version of execution is meant for when
you already have a LiteralMap of inputs.
@@ -239,14 +264,23 @@ Source code for flytekit.common.mixins.launchable
"""
pass
-[docs] @_deprecated(reason="Use launch_with_literals instead", version='0.9.0')
- def execute_with_literals(self, project, domain, literal_inputs, name=None, notification_overrides=None,
- label_overrides=None, annotation_overrides=None):
+[docs] @_deprecated(reason="Use launch_with_literals instead", version="0.9.0")
+ def execute_with_literals(
+ self,
+ project,
+ domain,
+ literal_inputs,
+ name=None,
+ notification_overrides=None,
+ label_overrides=None,
+ annotation_overrides=None,
+ ):
"""
Deprecated.
"""
- return self.launch_with_literals(project, domain, literal_inputs, name, notification_overrides, label_overrides,
- annotation_overrides)
+ return self.launch_with_literals(
+ project, domain, literal_inputs, name, notification_overrides, label_overrides, annotation_overrides,
+ )
diff --git a/_modules/flytekit/common/mixins/registerable.html b/_modules/flytekit/common/mixins/registerable.html
index 58324a0466..06642718ed 100644
--- a/_modules/flytekit/common/mixins/registerable.html
+++ b/_modules/flytekit/common/mixins/registerable.html
@@ -8,7 +8,7 @@
- flytekit.common.mixins.registerable — Flyte 0.6.0 documentation
+ flytekit.common.mixins.registerable — Flyte 0.7.0 documentation
@@ -161,15 +161,17 @@
Source code for flytekit.common.mixins.registerable
from __future__ import absolute_import
+
import abc as _abc
-import inspect as _inspect
-import six as _six
import importlib as _importlib
+import inspect as _inspect
import logging as _logging
+import six as _six
+
from flytekit.common import sdk_bases as _sdk_bases
-from flytekit.common.exceptions import system as _system_exceptions
from flytekit.common import utils as _utils
+from flytekit.common.exceptions import system as _system_exceptions
class _InstanceTracker(_sdk_bases.ExtendedSdkType):
@@ -183,12 +185,13 @@ Source code for flytekit.common.mixins.registerable
like to only register a task once and do so with the name where it is defined. This metaclass allows us to do this
by inspecting the call stack when __call__ is called on the metaclass (thus instantiating an object).
"""
+
@staticmethod
def _find_instance_module():
frame = _inspect.currentframe()
while frame:
- if frame.f_code.co_name == '<module>':
- return frame.f_globals['__name__']
+ if frame.f_code.co_name == "<module>":
+ return frame.f_globals["__name__"]
frame = frame.f_back
return None
@@ -199,7 +202,6 @@ Source code for flytekit.common.mixins.registerable
[docs]class RegisterableEntity(_six.with_metaclass(_InstanceTracker, object)):
-
def __init__(self, *args, **kwargs):
self._platform_valid_name = None
super(RegisterableEntity, self).__init__(*args, **kwargs)
diff --git a/_modules/flytekit/common/nodes.html b/_modules/flytekit/common/nodes.html
index 6f5236eadc..edeb0b9579 100644
--- a/_modules/flytekit/common/nodes.html
+++ b/_modules/flytekit/common/nodes.html
@@ -8,7 +8,7 @@
- flytekit.common.nodes — Flyte 0.6.0 documentation
+ flytekit.common.nodes — Flyte 0.7.0 documentation
@@ -168,17 +168,23 @@ Source code for flytekit.common.nodes
import six as _six
from sortedcontainers import SortedDict as _SortedDict
+from flytekit.common import component_nodes as _component_nodes
from flytekit.common import constants as _constants
-from flytekit.common import sdk_bases as _sdk_bases, promise as _promise, component_nodes as _component_nodes
-from flytekit.common.exceptions import scopes as _exception_scopes, user as _user_exceptions
+from flytekit.common import promise as _promise
+from flytekit.common import sdk_bases as _sdk_bases
+from flytekit.common.exceptions import scopes as _exception_scopes
from flytekit.common.exceptions import system as _system_exceptions
-from flytekit.common.mixins import hash as _hash_mixin, artifact as _artifact_mixin
+from flytekit.common.exceptions import user as _user_exceptions
+from flytekit.common.mixins import artifact as _artifact_mixin
+from flytekit.common.mixins import hash as _hash_mixin
from flytekit.common.tasks import executions as _task_executions
from flytekit.common.types import helpers as _type_helpers
from flytekit.common.utils import _dnsify
from flytekit.engines import loader as _engine_loader
-from flytekit.models import common as _common_models, node_execution as _node_execution_models
-from flytekit.models.core import workflow as _workflow_model, execution as _execution_models
+from flytekit.models import common as _common_models
+from flytekit.models import node_execution as _node_execution_models
+from flytekit.models.core import execution as _execution_models
+from flytekit.models.core import workflow as _workflow_model
[docs]class ParameterMapper(_six.with_metaclass(_common_models.FlyteABCMeta, _SortedDict)):
@@ -224,7 +230,7 @@ Source code for flytekit.common.nodes
self._initialized = True
def __getattr__(self, key):
- if key == 'iteritems' and hasattr(super(ParameterMapper, self), 'items'):
+ if key == "iteritems" and hasattr(super(ParameterMapper, self), "items"):
return super(ParameterMapper, self).items
if hasattr(super(ParameterMapper, self), key):
return getattr(super(ParameterMapper, self), key)
@@ -233,7 +239,7 @@ Source code for flytekit.common.nodes
return self[key]
def __setattr__(self, key, value):
- if '_initialized' in self.__dict__:
+ if "_initialized" in self.__dict__:
raise _user_exceptions.FlyteAssertion("Parameters are immutable.")
else:
super(ParameterMapper, self).__setattr__(key, value)
@@ -262,18 +268,17 @@ Source code for flytekit.common.nodes
return _promise.NodeOutput(sdk_node, sdk_type, name)
-[docs]class SdkNode(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _hash_mixin.HashOnReferenceMixin, _workflow_model.Node)):
-
+[docs]class SdkNode(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _hash_mixin.HashOnReferenceMixin, _workflow_model.Node,)):
def __init__(
- self,
- id,
- upstream_nodes,
- bindings,
- metadata,
- sdk_task=None,
- sdk_workflow=None,
- sdk_launch_plan=None,
- sdk_branch=None
+ self,
+ id,
+ upstream_nodes,
+ bindings,
+ metadata,
+ sdk_task=None,
+ sdk_workflow=None,
+ sdk_launch_plan=None,
+ sdk_branch=None,
):
"""
:param Text id: A workflow-level unique identifier that identifies this node in the workflow. "inputs" and
@@ -292,15 +297,12 @@ Source code for flytekit.common.nodes
:param TODO sdk_branch: TODO
"""
non_none_entities = [
- entity
- for entity in [sdk_workflow, sdk_branch, sdk_launch_plan, sdk_task] if entity is not None
+ entity for entity in [sdk_workflow, sdk_branch, sdk_launch_plan, sdk_task] if entity is not None
]
if len(non_none_entities) != 1:
raise _user_exceptions.FlyteAssertion(
"An SDK node must have one underlying entity specified at once. Received the following "
- "entities: {}".format(
- non_none_entities
- )
+ "entities: {}".format(non_none_entities)
)
workflow_node = None
@@ -317,7 +319,7 @@ Source code for flytekit.common.nodes
output_aliases=[], # TODO: Are aliases a thing in SDK nodes
task_node=_component_nodes.SdkTaskNode(sdk_task) if sdk_task else None,
workflow_node=workflow_node,
- branch_node=sdk_branch.target if sdk_branch else None
+ branch_node=sdk_branch.target if sdk_branch else None,
)
self._upstream = upstream_nodes
self._executable_sdk_object = sdk_task or sdk_workflow or sdk_branch or sdk_launch_plan
@@ -349,7 +351,8 @@ Source code for flytekit.common.nodes
sdk_task_node = _component_nodes.SdkTaskNode.promote_from_model(model.task_node, tasks)
elif model.workflow_node is not None:
sdk_workflow_node = _component_nodes.SdkWorkflowNode.promote_from_model(
- model.workflow_node, sub_workflows, tasks)
+ model.workflow_node, sub_workflows, tasks
+ )
else:
raise _system_exceptions.FlyteSystemException("Bad Node model, neither task nor workflow detected")
@@ -386,7 +389,8 @@ Source code for flytekit.common.nodes
)
else:
raise _system_exceptions.FlyteSystemException(
- "Bad SdkWorkflowNode model, both lp and workflow are None")
+ "Bad SdkWorkflowNode model, both lp and workflow are None"
+ )
else:
raise _system_exceptions.FlyteSystemException("Bad SdkNode model, both task and workflow nodes are empty")
@@ -459,9 +463,7 @@ Source code for flytekit.common.nodes
[docs]class SdkNodeExecution(
_six.with_metaclass(
- _sdk_bases.ExtendedSdkType,
- _node_execution_models.NodeExecution,
- _artifact_mixin.ExecutionArtifact
+ _sdk_bases.ExtendedSdkType, _node_execution_models.NodeExecution, _artifact_mixin.ExecutionArtifact,
)
):
def __init__(self, *args, **kwargs):
@@ -515,8 +517,9 @@ Source code for flytekit.common.nodes
:rtype: dict[Text, T]
"""
if not self.is_complete:
- raise _user_exceptions.FlyteAssertion("Please what until the node execution has completed before "
- "requesting the outputs.")
+ raise _user_exceptions.FlyteAssertion(
+ "Please what until the node execution has completed before " "requesting the outputs."
+ )
if self.error:
raise _user_exceptions.FlyteAssertion("Outputs could not be found because the execution ended in failure.")
@@ -534,8 +537,9 @@ Source code for flytekit.common.nodes
:rtype: flytekit.models.core.execution.ExecutionError or None
"""
if not self.is_complete:
- raise _user_exceptions.FlyteAssertion("Please what until the node execution has completed before "
- "requesting error information.")
+ raise _user_exceptions.FlyteAssertion(
+ "Please what until the node execution has completed before " "requesting error information."
+ )
return self.closure.error
@property
@@ -558,11 +562,7 @@ Source code for flytekit.common.nodes
:param _node_execution_models.NodeExecution base_model:
:rtype: SdkNodeExecution
"""
- return cls(
- closure=base_model.closure,
- id=base_model.id,
- input_uri=base_model.input_uri
- )
+ return cls(closure=base_model.closure, id=base_model.id, input_uri=base_model.input_uri)
[docs] def sync(self):
"""
diff --git a/_modules/flytekit/common/notifications.html b/_modules/flytekit/common/notifications.html
index 987dc16f0e..1b94ff4355 100644
--- a/_modules/flytekit/common/notifications.html
+++ b/_modules/flytekit/common/notifications.html
@@ -8,7 +8,7 @@
- flytekit.common.notifications — Flyte 0.6.0 documentation
+ flytekit.common.notifications — Flyte 0.7.0 documentation
@@ -161,11 +161,13 @@
Source code for flytekit.common.notifications
from __future__ import absolute_import
-from flytekit.models import common as _common_model
-from flytekit.models.core import execution as _execution_model
+
+import six as _six
+
from flytekit.common import sdk_bases as _sdk_bases
from flytekit.common.exceptions import user as _user_exceptions
-import six as _six
+from flytekit.models import common as _common_model
+from flytekit.models.core import execution as _execution_model
[docs]class Notification(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _common_model.Notification)):
@@ -174,7 +176,7 @@ Source code for flytekit.common.notifications
_execution_model.WorkflowExecutionPhase.ABORTED,
_execution_model.WorkflowExecutionPhase.FAILED,
_execution_model.WorkflowExecutionPhase.SUCCEEDED,
- _execution_model.WorkflowExecutionPhase.TIMED_OUT
+ _execution_model.WorkflowExecutionPhase.TIMED_OUT,
}
def __init__(self, phases, email=None, pager_duty=None, slack=None):
@@ -196,7 +198,7 @@ Source code for flytekit.common.notifications
raise _user_exceptions.FlyteValueException(
phase,
self.VALID_PHASES,
- additional_message="Notifications can only be specified on terminal states."
+ additional_message="Notifications can only be specified on terminal states.",
)
[docs] @classmethod
@@ -226,10 +228,7 @@ Source code for flytekit.common.notifications
:param flytekit.models.common.Notification base_model:
:rtype: Notification
"""
- return cls(
- base_model.phases,
- base_model.pager_duty.recipients_email
- )
+ return cls(base_model.phases, base_model.pager_duty.recipients_email)
[docs]class Email(Notification):
@@ -247,10 +246,7 @@ Source code for flytekit.common.notifications
:param flytekit.models.common.Notification base_model:
:rtype: Notification
"""
- return cls(
- base_model.phases,
- base_model.email.recipients_email
- )
+ return cls(base_model.phases, base_model.email.recipients_email)
[docs]class Slack(Notification):
@@ -268,10 +264,7 @@ Source code for flytekit.common.notifications
:param flytekit.models.common.Notification base_model:
:rtype: Notification
"""
- return cls(
- base_model.phases,
- base_model.slack.recipients_email
- )
+ return cls(base_model.phases, base_model.slack.recipients_email)
diff --git a/_modules/flytekit/common/promise.html b/_modules/flytekit/common/promise.html
index 557ca495ee..8e2e34ceb7 100644
--- a/_modules/flytekit/common/promise.html
+++ b/_modules/flytekit/common/promise.html
@@ -8,7 +8,7 @@
- flytekit.common.promise — Flyte 0.6.0 documentation
+ flytekit.common.promise — Flyte 0.7.0 documentation
@@ -164,14 +164,15 @@ Source code for flytekit.common.promise
import six as _six
-from flytekit.common import constants as _constants, sdk_bases as _sdk_bases
+from flytekit.common import constants as _constants
+from flytekit.common import sdk_bases as _sdk_bases
from flytekit.common.exceptions import user as _user_exceptions
from flytekit.common.types import helpers as _type_helpers
-from flytekit.models import interface as _interface_models, types as _type_models
+from flytekit.models import interface as _interface_models
+from flytekit.models import types as _type_models
[docs]class Input(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _interface_models.Parameter)):
-
def __init__(self, name, sdk_type, help=None, **kwargs):
"""
:param Text name:
@@ -182,22 +183,22 @@ Source code for flytekit.common.promise
:param T default: If this is not a required input, the value will default to this value.
"""
param_default = None
- if 'required' not in kwargs and 'default' not in kwargs:
+ if "required" not in kwargs and "default" not in kwargs:
# Neither required or default is set so assume required
required = True
default = None
- elif kwargs.get('required', False) and 'default' in kwargs:
+ elif kwargs.get("required", False) and "default" in kwargs:
# Required cannot be set to True and have a default specified
raise _user_exceptions.FlyteAssertion("Default cannot be set when required is True")
- elif 'default' in kwargs:
+ elif "default" in kwargs:
# If default is specified, then required must be false and the value is whatever is specified
required = None
- default = kwargs['default']
+ default = kwargs["default"]
param_default = sdk_type.from_python_std(default)
else:
# If no default is set, but required is set, then the behavior is determined by required == True or False
default = None
- required = kwargs['required']
+ required = kwargs["required"]
if not required:
# If required == False, we assume default to be None
param_default = sdk_type.from_python_std(default)
@@ -210,9 +211,9 @@ Source code for flytekit.common.promise
self._promise = _type_models.OutputReference(_constants.GLOBAL_INPUT_NODE_ID, name)
self._name = name
super(Input, self).__init__(
- _interface_models.Variable(type=sdk_type.to_flyte_literal_type(), description=help or ''),
+ _interface_models.Variable(type=sdk_type.to_flyte_literal_type(), description=help or ""),
required=required,
- default=param_default
+ default=param_default,
)
[docs] def rename_and_return_reference(self, new_name):
@@ -274,24 +275,12 @@ Source code for flytekit.common.promise
if model.default is not None:
default_value = sdk_type.from_flyte_idl(model.default.to_flyte_idl()).to_python_std()
- return cls(
- "",
- sdk_type,
- help=model.var.description,
- required=False,
- default=default_value
- )
+ return cls("", sdk_type, help=model.var.description, required=False, default=default_value,)
else:
- return cls(
- "",
- sdk_type,
- help=model.var.description,
- required=True
- )
+ return cls("", sdk_type, help=model.var.description, required=True)
[docs]class NodeOutput(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _type_models.OutputReference)):
-
def __init__(self, sdk_node, sdk_type, var):
"""
:param flytekit.common.nodes.SdkNode sdk_node:
@@ -300,10 +289,7 @@ Source code for flytekit.common.promise
"""
self._node = sdk_node
self._type = sdk_type
- super(NodeOutput, self).__init__(
- self._node.id,
- var
- )
+ super(NodeOutput, self).__init__(self._node.id, var)
@property
def node_id(self):
@@ -319,8 +305,10 @@ Source code for flytekit.common.promise
:param flytekit.models.types.OutputReference model:
:rtype: NodeOutput
"""
- raise _user_exceptions.FlyteAssertion("A NodeOutput cannot be promoted from a protobuf because it must be "
- "contextualized by an existing SdkNode.")
+ raise _user_exceptions.FlyteAssertion(
+ "A NodeOutput cannot be promoted from a protobuf because it must be "
+ "contextualized by an existing SdkNode."
+ )
@property
def sdk_node(self):
diff --git a/_modules/flytekit/common/schedules.html b/_modules/flytekit/common/schedules.html
index a8f6339d07..e5eee3fd76 100644
--- a/_modules/flytekit/common/schedules.html
+++ b/_modules/flytekit/common/schedules.html
@@ -8,7 +8,7 @@
- flytekit.common.schedules — Flyte 0.6.0 documentation
+ flytekit.common.schedules — Flyte 0.7.0 documentation
@@ -161,13 +161,16 @@
Source code for flytekit.common.schedules
from __future__ import absolute_import, division
-from flytekit.models import schedule as _schedule_models
-from flytekit.common import sdk_bases as _sdk_bases
-from flytekit.common.exceptions import user as _user_exceptions
-import croniter as _croniter
+
import datetime as _datetime
+
+import croniter as _croniter
import six as _six
+from flytekit.common import sdk_bases as _sdk_bases
+from flytekit.common.exceptions import user as _user_exceptions
+from flytekit.models import schedule as _schedule_models
+
class _ExtendedSchedule(_schedule_models.Schedule):
@classmethod
@@ -180,7 +183,6 @@ Source code for flytekit.common.schedules
[docs]class CronSchedule(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _ExtendedSchedule)):
-
def __init__(self, cron_expression, kickoff_time_input_arg=None):
"""
:param Text cron_expression:
@@ -203,12 +205,10 @@ Source code for flytekit.common.schedules
if len(tokens) != 6:
raise _user_exceptions.FlyteAssertion(
"Cron expression is invalid. A cron expression must have 6 fields. Cron expressions are in the "
- "format of: `minute hour day-of-month month day-of-week year`. Received: `{}`".format(
- cron_expression
- )
+ "format of: `minute hour day-of-month month day-of-week year`. Received: `{}`".format(cron_expression)
)
- if tokens[2] != '?' and tokens[4] != '?':
+ if tokens[2] != "?" and tokens[4] != "?":
raise _user_exceptions.FlyteAssertion(
"Scheduled string is invalid. A cron expression must have a '?' for either day-of-month or "
"day-of-week. Please specify '?' for one of those fields. Cron expressions are in the format of: "
@@ -220,13 +220,11 @@ Source code for flytekit.common.schedules
try:
# Cut to 5 fields and just assume year field is good because croniter treats the 6th field as seconds.
# TODO: Parse this field ourselves and check
- _croniter.croniter(" ".join(cron_expression.replace('?', '*').split()[:5]))
- except:
+ _croniter.croniter(" ".join(cron_expression.replace("?", "*").split()[:5]))
+ except Exception:
raise _user_exceptions.FlyteAssertion(
"Scheduled string is invalid. The cron expression was found to be invalid."
- " Provided cron expr: {}".format(
- cron_expression
- )
+ " Provided cron expr: {}".format(cron_expression)
)
[docs] @classmethod
@@ -235,14 +233,10 @@ Source code for flytekit.common.schedules
:param flytekit.models.schedule.Schedule base_model:
:rtype: CronSchedule
"""
- return cls(
- base_model.cron_expression,
- kickoff_time_input_arg=base_model.kickoff_time_input_arg
- )
+ return cls(base_model.cron_expression, kickoff_time_input_arg=base_model.kickoff_time_input_arg,)
[docs]class FixedRate(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _ExtendedSchedule)):
-
def __init__(self, duration, kickoff_time_input_arg=None):
"""
:param datetime.timedelta duration:
@@ -268,18 +262,15 @@ Source code for flytekit.common.schedules
)
elif int(duration.total_seconds()) % _SECONDS_TO_DAYS == 0:
return _schedule_models.Schedule.FixedRate(
- int(duration.total_seconds() / _SECONDS_TO_DAYS),
- _schedule_models.Schedule.FixedRateUnit.DAY
+ int(duration.total_seconds() / _SECONDS_TO_DAYS), _schedule_models.Schedule.FixedRateUnit.DAY,
)
elif int(duration.total_seconds()) % _SECONDS_TO_HOURS == 0:
return _schedule_models.Schedule.FixedRate(
- int(duration.total_seconds() / _SECONDS_TO_HOURS),
- _schedule_models.Schedule.FixedRateUnit.HOUR
+ int(duration.total_seconds() / _SECONDS_TO_HOURS), _schedule_models.Schedule.FixedRateUnit.HOUR,
)
else:
return _schedule_models.Schedule.FixedRate(
- int(duration.total_seconds() / _SECONDS_TO_MINUTES),
- _schedule_models.Schedule.FixedRateUnit.MINUTE
+ int(duration.total_seconds() / _SECONDS_TO_MINUTES), _schedule_models.Schedule.FixedRateUnit.MINUTE,
)
[docs] @classmethod
@@ -295,10 +286,7 @@ Source code for flytekit.common.schedules
else:
duration = _datetime.timedelta(minutes=base_model.rate.value)
- return cls(
- duration,
- kickoff_time_input_arg=base_model.kickoff_time_input_arg
- )
+ return cls(duration, kickoff_time_input_arg=base_model.kickoff_time_input_arg)
diff --git a/_modules/flytekit/common/sdk_bases.html b/_modules/flytekit/common/sdk_bases.html
index ba8c48cd37..c32e3c3d29 100644
--- a/_modules/flytekit/common/sdk_bases.html
+++ b/_modules/flytekit/common/sdk_bases.html
@@ -8,7 +8,7 @@
- flytekit.common.sdk_bases — Flyte 0.6.0 documentation
+ flytekit.common.sdk_bases — Flyte 0.7.0 documentation
@@ -161,10 +161,13 @@
Source code for flytekit.common.sdk_bases
from __future__ import absolute_import
-from flytekit.models import common as _common
+
import abc as _abc
+
import six as _six
+from flytekit.models import common as _common
+
[docs]class ExtendedSdkType(_six.with_metaclass(_common.FlyteABCMeta, _common.FlyteType)):
"""
diff --git a/_modules/flytekit/common/tasks/executions.html b/_modules/flytekit/common/tasks/executions.html
index 679786338d..bd922331ac 100644
--- a/_modules/flytekit/common/tasks/executions.html
+++ b/_modules/flytekit/common/tasks/executions.html
@@ -8,7 +8,7 @@
- flytekit.common.tasks.executions — Flyte 0.6.0 documentation
+ flytekit.common.tasks.executions — Flyte 0.7.0 documentation
@@ -161,6 +161,9 @@
Source code for flytekit.common.tasks.executions
from __future__ import absolute_import
+
+import six as _six
+
from flytekit.common import sdk_bases as _sdk_bases
from flytekit.common.exceptions import user as _user_exceptions
from flytekit.common.mixins import artifact as _artifact_mixin
@@ -168,14 +171,11 @@ Source code for flytekit.common.tasks.executions
from flytekit.engines import loader as _engine_loader
from flytekit.models.admin import task_execution as _task_execution_model
from flytekit.models.core import execution as _execution_models
-import six as _six
[docs]class SdkTaskExecution(
_six.with_metaclass(
- _sdk_bases.ExtendedSdkType,
- _task_execution_model.TaskExecution,
- _artifact_mixin.ExecutionArtifact,
+ _sdk_bases.ExtendedSdkType, _task_execution_model.TaskExecution, _artifact_mixin.ExecutionArtifact,
)
):
def __init__(self, *args, **kwargs):
@@ -217,8 +217,9 @@ Source code for flytekit.common.tasks.executions
:rtype: dict[Text, T]
"""
if not self.is_complete:
- raise _user_exceptions.FlyteAssertion("Please what until the task execution has completed before "
- "requesting the outputs.")
+ raise _user_exceptions.FlyteAssertion(
+ "Please what until the task execution has completed before " "requesting the outputs."
+ )
if self.error:
raise _user_exceptions.FlyteAssertion("Outputs could not be found because the execution ended in failure.")
@@ -236,8 +237,9 @@ Source code for flytekit.common.tasks.executions
:rtype: flytekit.models.core.execution.ExecutionError or None
"""
if not self.is_complete:
- raise _user_exceptions.FlyteAssertion("Please what until the task execution has completed before "
- "requesting error information.")
+ raise _user_exceptions.FlyteAssertion(
+ "Please what until the task execution has completed before " "requesting error information."
+ )
return self.closure.error
[docs] def get_child_executions(self, filters=None):
@@ -246,13 +248,11 @@ Source code for flytekit.common.tasks.executions
:rtype: dict[Text, flytekit.common.nodes.SdkNodeExecution]
"""
from flytekit.common import nodes as _nodes
+
if not self.is_parent:
raise _user_exceptions.FlyteAssertion("Only task executions marked with 'is_parent' have child executions.")
models = _engine_loader.get_engine().get_task_execution(self).get_child_executions(filters=filters)
- return {
- k: _nodes.SdkNodeExecution.promote_from_model(v)
- for k, v in _six.iteritems(models)
- }
+ return {k: _nodes.SdkNodeExecution.promote_from_model(v) for k, v in _six.iteritems(models)}
[docs] @classmethod
def promote_from_model(cls, base_model):
diff --git a/_modules/flytekit/common/tasks/generic_spark_task.html b/_modules/flytekit/common/tasks/generic_spark_task.html
index 88c5d2dbe7..5feb2abb31 100644
--- a/_modules/flytekit/common/tasks/generic_spark_task.html
+++ b/_modules/flytekit/common/tasks/generic_spark_task.html
@@ -8,7 +8,7 @@
- flytekit.common.tasks.generic_spark_task — Flyte 0.6.0 documentation
+ flytekit.common.tasks.generic_spark_task — Flyte 0.7.0 documentation
@@ -162,55 +162,54 @@
Source code for flytekit.common.tasks.generic_spark_task
from __future__ import absolute_import
-try:
- from inspect import getfullargspec as _getargspec
-except ImportError:
- from inspect import getargspec as _getargspec
-
-from flytekit import __version__
import sys as _sys
-import six as _six
-from flytekit.common.tasks import task as _base_tasks
-from flytekit.common.types import helpers as _helpers, primitives as _primitives
-from flytekit.models import literals as _literal_models, task as _task_models
+import six as _six
from google.protobuf.json_format import MessageToDict as _MessageToDict
+
+from flytekit import __version__
from flytekit.common import interface as _interface
-from flytekit.common.exceptions import user as _user_exceptions
from flytekit.common.exceptions import scopes as _exception_scopes
-
+from flytekit.common.exceptions import user as _user_exceptions
+from flytekit.common.tasks import task as _base_tasks
+from flytekit.common.types import helpers as _helpers
+from flytekit.common.types import primitives as _primitives
from flytekit.configuration import internal as _internal_config
+from flytekit.models import literals as _literal_models
+from flytekit.models import task as _task_models
-input_types_supported = { _primitives.Integer,
- _primitives.Boolean,
- _primitives.Float,
- _primitives.String,
- _primitives.Datetime,
- _primitives.Timedelta,
- }
+input_types_supported = {
+ _primitives.Integer,
+ _primitives.Boolean,
+ _primitives.Float,
+ _primitives.String,
+ _primitives.Datetime,
+ _primitives.Timedelta,
+}
-[docs]class SdkGenericSparkTask( _base_tasks.SdkTask):
+[docs]class SdkGenericSparkTask(_base_tasks.SdkTask):
"""
This class includes the additional logic for building a task that executes as a Spark Job.
"""
+
def __init__(
- self,
- task_type,
- discovery_version,
- retries,
- interruptible,
- task_inputs,
- deprecated,
- discoverable,
- timeout,
- spark_type,
- main_class,
- main_application_file,
- spark_conf,
- hadoop_conf,
- environment,
+ self,
+ task_type,
+ discovery_version,
+ retries,
+ interruptible,
+ task_inputs,
+ deprecated,
+ discoverable,
+ timeout,
+ spark_type,
+ main_class,
+ main_application_file,
+ spark_conf,
+ hadoop_conf,
+ environment,
):
"""
:param Text task_type: string describing the task type
@@ -231,7 +230,7 @@ Source code for flytekit.common.tasks.generic_spark_task
spark_job = _task_models.SparkJob(
spark_conf=spark_conf,
hadoop_conf=hadoop_conf,
- spark_type = spark_type,
+ spark_type=spark_type,
application_file=main_application_file,
main_class=main_class,
executor_path=_sys.executable,
@@ -241,16 +240,12 @@ Source code for flytekit.common.tasks.generic_spark_task
task_type,
_task_models.TaskMetadata(
discoverable,
- _task_models.RuntimeMetadata(
- _task_models.RuntimeMetadata.RuntimeType.FLYTE_SDK,
- __version__,
- 'spark'
- ),
+ _task_models.RuntimeMetadata(_task_models.RuntimeMetadata.RuntimeType.FLYTE_SDK, __version__, "spark",),
timeout,
_literal_models.RetryStrategy(retries),
interruptible,
discovery_version,
- deprecated
+ deprecated,
),
_interface.TypedInterface({}, {}),
_MessageToDict(spark_job),
@@ -261,9 +256,7 @@ Source code for flytekit.common.tasks.generic_spark_task
task_inputs(self)
# Container after the Inputs have been updated.
- self._container = self._get_container_definition(
- environment=environment
- )
+ self._container = self._get_container_definition(environment=environment)
def _validate_inputs(self, inputs):
"""
@@ -271,10 +264,12 @@ Source code for flytekit.common.tasks.generic_spark_task
:raises: flytekit.common.exceptions.user.FlyteValidationException
"""
for k, v in _six.iteritems(inputs):
- sdk_type =_helpers.get_sdk_type_from_literal_type(v.type)
+ sdk_type = _helpers.get_sdk_type_from_literal_type(v.type)
if sdk_type not in input_types_supported:
raise _user_exceptions.FlyteValidationException(
- "Input Type '{}' not supported. Only Primitives are supported for Scala/Java Spark.".format(sdk_type)
+ "Input Type '{}' not supported. Only Primitives are supported for Scala/Java Spark.".format(
+ sdk_type
+ )
)
super(SdkGenericSparkTask, self)._validate_inputs(inputs)
@@ -290,8 +285,7 @@ Source code for flytekit.common.tasks.generic_spark_task
self.interface.inputs.update(inputs)
def _get_container_definition(
- self,
- environment=None,
+ self, environment=None,
):
"""
:rtype: Container
@@ -308,7 +302,7 @@ Source code for flytekit.common.tasks.generic_spark_task
args=args,
resources=_task_models.Resources([], []),
env=environment,
- config={}
+ config={},
)
diff --git a/_modules/flytekit/common/tasks/hive_task.html b/_modules/flytekit/common/tasks/hive_task.html
index 756a6015ed..ee98522d22 100644
--- a/_modules/flytekit/common/tasks/hive_task.html
+++ b/_modules/flytekit/common/tasks/hive_task.html
@@ -8,7 +8,7 @@
- flytekit.common.tasks.hive_task — Flyte 0.6.0 documentation
+ flytekit.common.tasks.hive_task — Flyte 0.7.0 documentation
@@ -167,19 +167,20 @@ Source code for flytekit.common.tasks.hive_task
<
import six as _six
from google.protobuf.json_format import MessageToDict as _MessageToDict
-from flytekit.common import constants as _constants, nodes as _nodes, interface as _interface
+from flytekit.common import constants as _constants
+from flytekit.common import interface as _interface
+from flytekit.common import nodes as _nodes
from flytekit.common.exceptions import scopes as _exception_scopes
-from flytekit.common.exceptions.user import FlyteTypeException as _FlyteTypeException, \
- FlyteValueException as _FlyteValueException
+from flytekit.common.exceptions.user import FlyteTypeException as _FlyteTypeException
+from flytekit.common.exceptions.user import FlyteValueException as _FlyteValueException
from flytekit.common.tasks import output as _task_output
-from flytekit.common.tasks import sdk_runnable as _sdk_runnable, task as _base_task
+from flytekit.common.tasks import sdk_runnable as _sdk_runnable
+from flytekit.common.tasks import task as _base_task
from flytekit.common.types import helpers as _type_helpers
-from flytekit.models import (
- qubole as _qubole,
- interface as _interface_model,
- literals as _literal_models,
- dynamic_job as _dynamic_job
-)
+from flytekit.models import dynamic_job as _dynamic_job
+from flytekit.models import interface as _interface_model
+from flytekit.models import literals as _literal_models
+from flytekit.models import qubole as _qubole
from flytekit.models.core import workflow as _workflow_model
ALLOWED_TAGS_COUNT = int(6)
@@ -192,26 +193,26 @@ Source code for flytekit.common.tasks.hive_task
<
"""
def __init__(
- self,
- task_function,
- task_type,
- discovery_version,
- retries,
- interruptible,
- deprecated,
- storage_request,
- cpu_request,
- gpu_request,
- memory_request,
- storage_limit,
- cpu_limit,
- gpu_limit,
- memory_limit,
- discoverable,
- timeout,
- cluster_label,
- tags,
- environment
+ self,
+ task_function,
+ task_type,
+ discovery_version,
+ retries,
+ interruptible,
+ deprecated,
+ storage_request,
+ cpu_request,
+ gpu_request,
+ memory_request,
+ storage_limit,
+ cpu_limit,
+ gpu_limit,
+ memory_limit,
+ discoverable,
+ timeout,
+ cluster_label,
+ tags,
+ environment,
):
"""
:param task_function: Function container user code. This will be executed via the SDK's engine.
@@ -234,9 +235,26 @@ Source code for flytekit.common.tasks.hive_task
<
:param dict[Text, Text] environment:
"""
self._task_function = task_function
- super(SdkHiveTask, self).__init__(task_function, task_type, discovery_version, retries, interruptible, deprecated,
- storage_request, cpu_request, gpu_request, memory_request, storage_limit,
- cpu_limit, gpu_limit, memory_limit, discoverable, timeout, environment, {})
+ super(SdkHiveTask, self).__init__(
+ task_function,
+ task_type,
+ discovery_version,
+ retries,
+ interruptible,
+ deprecated,
+ storage_request,
+ cpu_request,
+ gpu_request,
+ memory_request,
+ storage_limit,
+ cpu_limit,
+ gpu_limit,
+ memory_limit,
+ discoverable,
+ timeout,
+ environment,
+ {},
+ )
self._validate_task_parameters(cluster_label, tags)
self._cluster_label = cluster_label
self._tags = tags
@@ -256,8 +274,9 @@ Source code for flytekit.common.tasks.hive_task
<
plugin_objects = []
for q in queries_from_task:
- hive_query = _qubole.HiveQuery(query=q, timeout_sec=self.metadata.timeout.seconds,
- retry_count=self.metadata.retries.retries)
+ hive_query = _qubole.HiveQuery(
+ query=q, timeout_sec=self.metadata.timeout.seconds, retry_count=self.metadata.retries.retries,
+ )
# TODO: Remove this after all users of older SDK versions that did the single node, multi-query pattern are
# deprecated. This is only here for backwards compatibility - in addition to writing the query to the
@@ -265,8 +284,9 @@ Source code for flytekit.common.tasks.hive_task
<
# older plugin will continue to work.
query_collection = _qubole.HiveQueryCollection([hive_query])
- plugin_objects.append(_qubole.QuboleHiveJob(hive_query, self._cluster_label, self._tags,
- query_collection=query_collection))
+ plugin_objects.append(
+ _qubole.QuboleHiveJob(hive_query, self._cluster_label, self._tags, query_collection=query_collection,)
+ )
return plugin_objects
@@ -277,7 +297,7 @@ Source code for flytekit.common.tasks.hive_task
<
type(cluster_label),
{str, _six.text_type},
additional_msg="cluster_label for a hive task must be in text format",
- received_value=cluster_label
+ received_value=cluster_label,
)
if tags is not None:
if not (isinstance(tags, list) and all(isinstance(tag, (str, _six.text_type)) for tag in tags)):
@@ -285,12 +305,16 @@ Source code for flytekit.common.tasks.hive_task
<
type(tags),
[],
additional_msg="tags for a hive task must be in 'list of text' format",
- received_value=tags
+ received_value=tags,
)
if len(tags) > ALLOWED_TAGS_COUNT:
- raise _FlyteValueException(len(tags), "number of tags must be less than {}".format(ALLOWED_TAGS_COUNT))
+ raise _FlyteValueException(
+ len(tags), "number of tags must be less than {}".format(ALLOWED_TAGS_COUNT),
+ )
if not all(len(tag) for tag in tags):
- raise _FlyteValueException(tags, "length of a tag must be less than {} chars".format(MAX_TAG_LENGTH))
+ raise _FlyteValueException(
+ tags, "length of a tag must be less than {} chars".format(MAX_TAG_LENGTH),
+ )
@staticmethod
def _validate_queries(queries_from_task):
@@ -300,7 +324,7 @@ Source code for flytekit.common.tasks.hive_task
<
type(query_from_task),
{str, _six.text_type},
additional_msg="All queries returned from a Hive task must be in text format.",
- received_value=query_from_task
+ received_value=query_from_task,
)
def _produce_dynamic_job_spec(self, context, inputs):
@@ -310,9 +334,10 @@ Source code for flytekit.common.tasks.hive_task
<
:param flytekit.models.literals.LiteralMap literal_map inputs:
:rtype: flytekit.models.dynamic_job.DynamicJobSpec
"""
- inputs_dict = _type_helpers.unpack_literal_map_to_sdk_python_std(inputs, {
- k: _type_helpers.get_sdk_type_from_literal_type(v.type) for k, v in _six.iteritems(self.interface.inputs)
- })
+ inputs_dict = _type_helpers.unpack_literal_map_to_sdk_python_std(
+ inputs,
+ {k: _type_helpers.get_sdk_type_from_literal_type(v.type) for k, v in _six.iteritems(self.interface.inputs)},
+ )
outputs_dict = {
name: _task_output.OutputReference(_type_helpers.get_sdk_type_from_literal_type(variable.type))
for name, variable in _six.iteritems(self.interface.outputs)
@@ -327,27 +352,23 @@ Source code for flytekit.common.tasks.hive_task
<
generated_queries = self._generate_plugin_objects(context, inputs_dict)
# Create output bindings always - this has to happen after user code has run
- output_bindings = [_literal_models.Binding(var=name, binding=_interface.BindingData.from_python_std(
- b.sdk_type.to_flyte_literal_type(), b.value))
- for name, b in _six.iteritems(outputs_dict)]
+ output_bindings = [
+ _literal_models.Binding(
+ var=name, binding=_interface.BindingData.from_python_std(b.sdk_type.to_flyte_literal_type(), b.value),
+ )
+ for name, b in _six.iteritems(outputs_dict)
+ ]
i = 0
for quboleHiveJob in generated_queries:
- hive_job_node = _create_hive_job_node(
- "HiveQuery_{}".format(i),
- quboleHiveJob.to_flyte_idl(),
- self.metadata
- )
+ hive_job_node = _create_hive_job_node("HiveQuery_{}".format(i), quboleHiveJob.to_flyte_idl(), self.metadata)
nodes.append(hive_job_node)
tasks.append(hive_job_node.executable_sdk_object)
i += 1
dynamic_job_spec = _dynamic_job.DynamicJobSpec(
- min_successes=len(nodes),
- tasks=tasks,
- nodes=nodes,
- outputs=output_bindings,
- subworkflows=[])
+ min_successes=len(nodes), tasks=tasks, nodes=nodes, outputs=output_bindings, subworkflows=[],
+ )
return dynamic_job_spec
@@ -373,12 +394,11 @@ Source code for flytekit.common.tasks.hive_task
<
if len(spec.nodes) == 0:
return {
_constants.OUTPUT_FILE_NAME: _literal_models.LiteralMap(
- literals={binding.var: binding.binding.to_literal_model() for binding in spec.outputs})
+ literals={binding.var: binding.binding.to_literal_model() for binding in spec.outputs}
+ )
}
else:
- generated_files.update({
- _constants.FUTURES_FILE_NAME: spec
- })
+ generated_files.update({_constants.FUTURES_FILE_NAME: spec})
return generated_files
@@ -396,7 +416,7 @@ Source code for flytekit.common.tasks.hive_task
<
upstream_nodes=[],
bindings=[],
metadata=_workflow_model.NodeMetadata(name, metadata.timeout, _literal_models.RetryStrategy(0)),
- sdk_task=SdkHiveJob(hive_job, metadata)
+ sdk_task=SdkHiveJob(hive_job, metadata),
)
@@ -407,9 +427,7 @@ Source code for flytekit.common.tasks.hive_task
<
"""
def __init__(
- self,
- hive_job,
- metadata,
+ self, hive_job, metadata,
):
"""
:param _qubole.QuboleHiveJob hive_job: Hive job spec
diff --git a/_modules/flytekit/common/tasks/output.html b/_modules/flytekit/common/tasks/output.html
index 781ea3b434..c2ab8d2acb 100644
--- a/_modules/flytekit/common/tasks/output.html
+++ b/_modules/flytekit/common/tasks/output.html
@@ -8,7 +8,7 @@
- flytekit.common.tasks.output — Flyte 0.6.0 documentation
+ flytekit.common.tasks.output — Flyte 0.7.0 documentation
diff --git a/_modules/flytekit/common/tasks/presto_task.html b/_modules/flytekit/common/tasks/presto_task.html
index 9fd63e9db5..7d4ff0c281 100644
--- a/_modules/flytekit/common/tasks/presto_task.html
+++ b/_modules/flytekit/common/tasks/presto_task.html
@@ -8,7 +8,7 @@
- flytekit.common.tasks.presto_task — Flyte 0.6.0 documentation
+ flytekit.common.tasks.presto_task — Flyte 0.7.0 documentation
@@ -162,25 +162,22 @@
Source code for flytekit.common.tasks.presto_task
from __future__ import absolute_import
-import six as _six
-
+import datetime as _datetime
+import six as _six
from google.protobuf.json_format import MessageToDict as _MessageToDict
-from flytekit import __version__
+from flytekit import __version__
from flytekit.common import constants as _constants
-from flytekit.common.tasks import task as _base_task
-from flytekit.models import (
- interface as _interface_model
-)
-from flytekit.models import literals as _literals, types as _types, \
- task as _task_model
-
from flytekit.common import interface as _interface
-import datetime as _datetime
-from flytekit.models import presto as _presto_models
-from flytekit.common.types import helpers as _type_helpers
from flytekit.common.exceptions import scopes as _exception_scopes
+from flytekit.common.tasks import task as _base_task
+from flytekit.common.types import helpers as _type_helpers
+from flytekit.models import interface as _interface_model
+from flytekit.models import literals as _literals
+from flytekit.models import presto as _presto_models
+from flytekit.models import task as _task_model
+from flytekit.models import types as _types
[docs]class SdkPrestoTask(_base_task.SdkTask):
@@ -189,19 +186,19 @@ Source code for flytekit.common.tasks.presto_task
"""
def __init__(
- self,
- statement,
- output_schema,
- routing_group=None,
- catalog=None,
- schema=None,
- task_inputs=None,
- interruptible=False,
- discoverable=False,
- discovery_version=None,
- retries=1,
- timeout=None,
- deprecated=None
+ self,
+ statement,
+ output_schema,
+ routing_group=None,
+ catalog=None,
+ schema=None,
+ task_inputs=None,
+ interruptible=False,
+ discoverable=False,
+ discovery_version=None,
+ retries=1,
+ timeout=None,
+ deprecated=None,
):
"""
:param Text statement: Presto query specification
@@ -227,21 +224,16 @@ Source code for flytekit.common.tasks.presto_task
metadata = _task_model.TaskMetadata(
discoverable,
# This needs to have the proper version reflected in it
- _task_model.RuntimeMetadata(
- _task_model.RuntimeMetadata.RuntimeType.FLYTE_SDK, __version__,
- "python"),
+ _task_model.RuntimeMetadata(_task_model.RuntimeMetadata.RuntimeType.FLYTE_SDK, __version__, "python"),
timeout or _datetime.timedelta(seconds=0),
_literals.RetryStrategy(retries),
interruptible,
discovery_version,
- deprecated
+ deprecated,
)
presto_query = _presto_models.PrestoQuery(
- routing_group=routing_group or "",
- catalog=catalog or "",
- schema=schema or "",
- statement=statement
+ routing_group=routing_group or "", catalog=catalog or "", schema=schema or "", statement=statement,
)
# Here we set the routing_group, catalog, and schema as implicit
@@ -250,30 +242,28 @@ Source code for flytekit.common.tasks.presto_task
{
"__implicit_routing_group": _interface_model.Variable(
type=_types.LiteralType(simple=_types.SimpleType.STRING),
- description="The routing group set as an implicit input"
+ description="The routing group set as an implicit input",
),
"__implicit_catalog": _interface_model.Variable(
type=_types.LiteralType(simple=_types.SimpleType.STRING),
- description="The catalog set as an implicit input"
+ description="The catalog set as an implicit input",
),
"__implicit_schema": _interface_model.Variable(
type=_types.LiteralType(simple=_types.SimpleType.STRING),
- description="The schema set as an implicit input"
- )
+ description="The schema set as an implicit input",
+ ),
},
{
# Set the schema for the Presto query as an output
"results": _interface_model.Variable(
type=_types.LiteralType(schema=output_schema.schema_type),
- description="The schema for the Presto query"
+ description="The schema for the Presto query",
)
- })
+ },
+ )
super(SdkPrestoTask, self).__init__(
- _constants.SdkTaskType.PRESTO_TASK,
- metadata,
- i,
- _MessageToDict(presto_query.to_flyte_idl()),
+ _constants.SdkTaskType.PRESTO_TASK, metadata, i, _MessageToDict(presto_query.to_flyte_idl()),
)
# Set user provided inputs
@@ -294,9 +284,7 @@ Source code for flytekit.common.tasks.presto_task
def __call__(self, *args, **kwargs):
kwargs = self._add_implicit_inputs(kwargs)
- return super(SdkPrestoTask, self).__call__(
- *args, **kwargs
- )
+ return super(SdkPrestoTask, self).__call__(*args, **kwargs)
# Override method in order to set the implicit inputs
def _python_std_input_map_to_literal_map(self, inputs):
@@ -306,10 +294,10 @@ Source code for flytekit.common.tasks.presto_task
:rtype: flytekit.models.literals.LiteralMap
"""
inputs = self._add_implicit_inputs(inputs)
- return _type_helpers.pack_python_std_map_to_literal_map(inputs, {
- k: _type_helpers.get_sdk_type_from_literal_type(v.type)
- for k, v in _six.iteritems(self.interface.inputs)
- })
+ return _type_helpers.pack_python_std_map_to_literal_map(
+ inputs,
+ {k: _type_helpers.get_sdk_type_from_literal_type(v.type) for k, v in _six.iteritems(self.interface.inputs)},
+ )
[docs] @_exception_scopes.system_entry_point
def add_inputs(self, inputs):
diff --git a/_modules/flytekit/common/tasks/pytorch_task.html b/_modules/flytekit/common/tasks/pytorch_task.html
index ebe9cd7ce8..539b75bfe7 100644
--- a/_modules/flytekit/common/tasks/pytorch_task.html
+++ b/_modules/flytekit/common/tasks/pytorch_task.html
@@ -8,7 +8,7 @@
- flytekit.common.tasks.pytorch_task — Flyte 0.6.0 documentation
+ flytekit.common.tasks.pytorch_task — Flyte 0.7.0 documentation
@@ -162,22 +162,13 @@
Source code for flytekit.common.tasks.pytorch_task
from __future__ import absolute_import
-try:
- from inspect import getfullargspec as _getargspec
-except ImportError:
- from inspect import getargspec as _getargspec
-
-import six as _six
-from flytekit.common import constants as _constants
-from flytekit.common.exceptions import scopes as _exception_scopes
-from flytekit.common.tasks import output as _task_output, sdk_runnable as _sdk_runnable
-from flytekit.common.types import helpers as _type_helpers
-from flytekit.models import literals as _literal_models, task as _task_models
from google.protobuf.json_format import MessageToDict as _MessageToDict
+from flytekit.common.tasks import sdk_runnable as _sdk_runnable
+from flytekit.models import task as _task_models
-[docs]class SdkRunnablePytorchContainer(_sdk_runnable.SdkRunnableContainer):
+[docs]class SdkRunnablePytorchContainer(_sdk_runnable.SdkRunnableContainer):
@property
def args(self):
"""
@@ -186,31 +177,30 @@ Source code for flytekit.common.tasks.pytorch_task
"""
return self._args
+
[docs]class SdkPyTorchTask(_sdk_runnable.SdkRunnableTask):
def __init__(
- self,
- task_function,
- task_type,
- discovery_version,
- retries,
- interruptible,
- deprecated,
- discoverable,
- timeout,
- workers_count,
- per_replica_storage_request,
- per_replica_cpu_request,
- per_replica_gpu_request,
- per_replica_memory_request,
- per_replica_storage_limit,
- per_replica_cpu_limit,
- per_replica_gpu_limit,
- per_replica_memory_limit,
- environment
+ self,
+ task_function,
+ task_type,
+ discovery_version,
+ retries,
+ interruptible,
+ deprecated,
+ discoverable,
+ timeout,
+ workers_count,
+ per_replica_storage_request,
+ per_replica_cpu_request,
+ per_replica_gpu_request,
+ per_replica_memory_request,
+ per_replica_storage_limit,
+ per_replica_cpu_limit,
+ per_replica_gpu_limit,
+ per_replica_memory_limit,
+ environment,
):
- pytorch_job = _task_models.PyTorchJob(
- workers_count=workers_count
- ).to_flyte_idl()
+ pytorch_job = _task_models.PyTorchJob(workers_count=workers_count).to_flyte_idl()
super(SdkPyTorchTask, self).__init__(
task_function=task_function,
task_type=task_type,
@@ -229,13 +219,10 @@ Source code for flytekit.common.tasks.pytorch_task
discoverable=discoverable,
timeout=timeout,
environment=environment,
- custom=_MessageToDict(pytorch_job)
+ custom=_MessageToDict(pytorch_job),
)
- def _get_container_definition(
- self,
- **kwargs
- ):
+ def _get_container_definition(self, **kwargs):
"""
:rtype: SdkRunnablePytorchContainer
"""
diff --git a/_modules/flytekit/common/tasks/raw_container.html b/_modules/flytekit/common/tasks/raw_container.html
index 4132b31559..8ff49ca424 100644
--- a/_modules/flytekit/common/tasks/raw_container.html
+++ b/_modules/flytekit/common/tasks/raw_container.html
@@ -8,7 +8,7 @@
- flytekit.common.tasks.raw_container — Flyte 0.6.0 documentation
+ flytekit.common.tasks.raw_container — Flyte 0.7.0 documentation
@@ -172,7 +172,8 @@ Source code for flytekit.common.tasks.raw_container
from flytekit.common.tasks import task as _base_task
from flytekit.common.types.base_sdk_types import FlyteSdkType
from flytekit.configuration import resources as _resource_config
-from flytekit.models import literals as _literals, task as _task_models
+from flytekit.models import literals as _literals
+from flytekit.models import task as _task_models
from flytekit.models.interface import Variable
@@ -185,19 +186,19 @@ Source code for flytekit.common.tasks.raw_container
def _get_container_definition(
- image: str,
- command: List[str],
- args: List[str],
- data_loading_config: _task_models.DataLoadingConfig,
- storage_request: str = None,
- cpu_request: str = None,
- gpu_request: str = None,
- memory_request: str = None,
- storage_limit: str = None,
- cpu_limit: str = None,
- gpu_limit: str = None,
- memory_limit: str = None,
- environment: Dict[str, str] = None,
+ image: str,
+ command: List[str],
+ args: List[str],
+ data_loading_config: _task_models.DataLoadingConfig,
+ storage_request: str = None,
+ cpu_request: str = None,
+ gpu_request: str = None,
+ memory_request: str = None,
+ storage_limit: str = None,
+ cpu_limit: str = None,
+ gpu_limit: str = None,
+ memory_limit: str = None,
+ environment: Dict[str, str] = None,
) -> _task_models.Container:
storage_limit = storage_limit or _resource_config.DEFAULT_STORAGE_LIMIT.get()
storage_request = storage_request or _resource_config.DEFAULT_STORAGE_REQUEST.get()
@@ -211,62 +212,26 @@ Source code for flytekit.common.tasks.raw_container
requests = []
if storage_request:
requests.append(
- _task_models.Resources.ResourceEntry(
- _task_models.Resources.ResourceName.STORAGE,
- storage_request
- )
+ _task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.STORAGE, storage_request)
)
if cpu_request:
- requests.append(
- _task_models.Resources.ResourceEntry(
- _task_models.Resources.ResourceName.CPU,
- cpu_request
- )
- )
+ requests.append(_task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.CPU, cpu_request))
if gpu_request:
- requests.append(
- _task_models.Resources.ResourceEntry(
- _task_models.Resources.ResourceName.GPU,
- gpu_request
- )
- )
+ requests.append(_task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.GPU, gpu_request))
if memory_request:
requests.append(
- _task_models.Resources.ResourceEntry(
- _task_models.Resources.ResourceName.MEMORY,
- memory_request
- )
+ _task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.MEMORY, memory_request)
)
limits = []
if storage_limit:
- limits.append(
- _task_models.Resources.ResourceEntry(
- _task_models.Resources.ResourceName.STORAGE,
- storage_limit
- )
- )
+ limits.append(_task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.STORAGE, storage_limit))
if cpu_limit:
- limits.append(
- _task_models.Resources.ResourceEntry(
- _task_models.Resources.ResourceName.CPU,
- cpu_limit
- )
- )
+ limits.append(_task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.CPU, cpu_limit))
if gpu_limit:
- limits.append(
- _task_models.Resources.ResourceEntry(
- _task_models.Resources.ResourceName.GPU,
- gpu_limit
- )
- )
+ limits.append(_task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.GPU, gpu_limit))
if memory_limit:
- limits.append(
- _task_models.Resources.ResourceEntry(
- _task_models.Resources.ResourceName.MEMORY,
- memory_limit
- )
- )
+ limits.append(_task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.MEMORY, memory_limit))
if environment is None:
environment = {}
@@ -287,35 +252,36 @@ Source code for flytekit.common.tasks.raw_container
Use this task when you want to run an arbitrary container as a task (e.g. external tools, binaries compiled
separately as a container completely separate from the container where your Flyte workflow is defined.
"""
+
METADATA_FORMAT_JSON = _task_models.DataLoadingConfig.LITERALMAP_FORMAT_JSON
METADATA_FORMAT_YAML = _task_models.DataLoadingConfig.LITERALMAP_FORMAT_YAML
METADATA_FORMAT_PROTO = _task_models.DataLoadingConfig.LITERALMAP_FORMAT_PROTO
def __init__(
- self,
- inputs: Dict[str, FlyteSdkType],
- image: str,
- outputs: Dict[str, FlyteSdkType] = None,
- input_data_dir: str = None,
- output_data_dir: str = None,
- metadata_format: int = METADATA_FORMAT_JSON,
- io_strategy: _task_models.IOStrategy=None,
- command: List[str] = None,
- args: List[str] = None,
- storage_request: str = None,
- cpu_request: str = None,
- gpu_request: str = None,
- memory_request: str = None,
- storage_limit: str = None,
- cpu_limit: str = None,
- gpu_limit: str = None,
- memory_limit: str = None,
- environment: Dict[str, str] = None,
- interruptible: bool = False,
- discoverable: bool = False,
- discovery_version: str = None,
- retries: int = 1,
- timeout: _datetime.timedelta = None,
+ self,
+ inputs: Dict[str, FlyteSdkType],
+ image: str,
+ outputs: Dict[str, FlyteSdkType] = None,
+ input_data_dir: str = None,
+ output_data_dir: str = None,
+ metadata_format: int = METADATA_FORMAT_JSON,
+ io_strategy: _task_models.IOStrategy = None,
+ command: List[str] = None,
+ args: List[str] = None,
+ storage_request: str = None,
+ cpu_request: str = None,
+ gpu_request: str = None,
+ memory_request: str = None,
+ storage_limit: str = None,
+ cpu_limit: str = None,
+ gpu_limit: str = None,
+ memory_limit: str = None,
+ environment: Dict[str, str] = None,
+ interruptible: bool = False,
+ discoverable: bool = False,
+ discovery_version: str = None,
+ retries: int = 1,
+ timeout: _datetime.timedelta = None,
):
"""
:param inputs:
@@ -355,14 +321,12 @@ Source code for flytekit.common.tasks.raw_container
metadata = _task_models.TaskMetadata(
discoverable,
# This needs to have the proper version reflected in it
- _task_models.RuntimeMetadata(
- _task_models.RuntimeMetadata.RuntimeType.FLYTE_SDK, __version__,
- "python"),
+ _task_models.RuntimeMetadata(_task_models.RuntimeMetadata.RuntimeType.FLYTE_SDK, __version__, "python",),
timeout or _datetime.timedelta(seconds=0),
_literals.RetryStrategy(retries),
interruptible,
discovery_version,
- None
+ None,
)
# The interface is defined using the inputs and outputs
@@ -388,10 +352,9 @@ Source code for flytekit.common.tasks.raw_container
gpu_limit=gpu_limit,
memory_limit=memory_limit,
environment=environment,
- )
+ ),
)
-
[docs] @_exception_scopes.system_entry_point
def add_inputs(self, inputs: Dict[str, Variable]):
"""
diff --git a/_modules/flytekit/common/tasks/sagemaker/built_in_training_job_task.html b/_modules/flytekit/common/tasks/sagemaker/built_in_training_job_task.html
new file mode 100644
index 0000000000..6179f99c7e
--- /dev/null
+++ b/_modules/flytekit/common/tasks/sagemaker/built_in_training_job_task.html
@@ -0,0 +1,308 @@
+
+
+
+
+
+
+
+
+
+
+ flytekit.common.tasks.sagemaker.built_in_training_job_task — Flyte 0.7.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Source code for flytekit.common.tasks.sagemaker.built_in_training_job_task
+import datetime as _datetime
+
+from google.protobuf.json_format import MessageToDict
+
+from flytekit import __version__
+from flytekit.common import interface as _interface
+from flytekit.common.constants import SdkTaskType
+from flytekit.common.exceptions import user as _user_exceptions
+from flytekit.common.tasks import task as _sdk_task
+from flytekit.models import interface as _interface_model
+from flytekit.models import literals as _literal_models
+from flytekit.models import task as _task_models
+from flytekit.models import types as _idl_types
+from flytekit.models.core import types as _core_types
+from flytekit.models.sagemaker import training_job as _training_job_models
+
+
+def _content_type_to_blob_format(content_type: _training_job_models) -> str:
+ if content_type == _training_job_models.InputContentType.TEXT_CSV:
+ return "csv"
+ else:
+ raise _user_exceptions.FlyteValueException("Unsupported InputContentType: {}".format(content_type))
+
+
+[docs]class SdkBuiltinAlgorithmTrainingJobTask(_sdk_task.SdkTask):
+ def __init__(
+ self,
+ training_job_resource_config: _training_job_models.TrainingJobResourceConfig,
+ algorithm_specification: _training_job_models.AlgorithmSpecification,
+ retries: int = 0,
+ cacheable: bool = False,
+ cache_version: str = "",
+ ):
+ """
+
+ :param training_job_resource_config: The options to configure the training job
+ :param algorithm_specification: The options to configure the target algorithm of the training
+ :param retries: Number of retries to attempt
+ :param cacheable: The flag to set if the user wants the output of the task execution to be cached
+ :param cache_version: String describing the caching version for task discovery purposes
+ """
+ # Use the training job model as a measure of type checking
+ self._training_job_model = _training_job_models.TrainingJob(
+ algorithm_specification=algorithm_specification, training_job_resource_config=training_job_resource_config,
+ )
+
+ # Setting flyte-level timeout to 0, and let SageMaker takes the StoppingCondition and terminate the training
+ # job gracefully
+ timeout = _datetime.timedelta(seconds=0)
+
+ super(SdkBuiltinAlgorithmTrainingJobTask, self).__init__(
+ type=SdkTaskType.SAGEMAKER_TRAINING_JOB_TASK,
+ metadata=_task_models.TaskMetadata(
+ runtime=_task_models.RuntimeMetadata(
+ type=_task_models.RuntimeMetadata.RuntimeType.FLYTE_SDK, version=__version__, flavor="sagemaker",
+ ),
+ discoverable=cacheable,
+ timeout=timeout,
+ retries=_literal_models.RetryStrategy(retries=retries),
+ interruptible=False,
+ discovery_version=cache_version,
+ deprecated_error_message="",
+ ),
+ interface=_interface.TypedInterface(
+ inputs={
+ "static_hyperparameters": _interface_model.Variable(
+ type=_idl_types.LiteralType(simple=_idl_types.SimpleType.STRUCT), description="",
+ ),
+ "train": _interface_model.Variable(
+ type=_idl_types.LiteralType(
+ blob=_core_types.BlobType(
+ format=_content_type_to_blob_format(algorithm_specification.input_content_type),
+ dimensionality=_core_types.BlobType.BlobDimensionality.MULTIPART,
+ ),
+ ),
+ description="",
+ ),
+ "validation": _interface_model.Variable(
+ type=_idl_types.LiteralType(
+ blob=_core_types.BlobType(
+ format=_content_type_to_blob_format(algorithm_specification.input_content_type),
+ dimensionality=_core_types.BlobType.BlobDimensionality.MULTIPART,
+ ),
+ ),
+ description="",
+ ),
+ },
+ outputs={
+ "model": _interface_model.Variable(
+ type=_idl_types.LiteralType(
+ blob=_core_types.BlobType(
+ format="", dimensionality=_core_types.BlobType.BlobDimensionality.SINGLE,
+ )
+ ),
+ description="",
+ )
+ },
+ ),
+ custom=MessageToDict(self._training_job_model.to_flyte_idl()),
+ )
+
+ @property
+ def training_job_model(self) -> _training_job_models.TrainingJob:
+ return self._training_job_model
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/_modules/flytekit/common/tasks/sagemaker/custom_training_job_task.html b/_modules/flytekit/common/tasks/sagemaker/custom_training_job_task.html
new file mode 100644
index 0000000000..bdd149081a
--- /dev/null
+++ b/_modules/flytekit/common/tasks/sagemaker/custom_training_job_task.html
@@ -0,0 +1,286 @@
+
+
+
+
+
+
+
+
+
+
+ flytekit.common.tasks.sagemaker.custom_training_job_task — Flyte 0.7.0 documentation
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Source code for flytekit.common.tasks.sagemaker.custom_training_job_task
+from google.protobuf.json_format import MessageToDict
+
+from flytekit.common.constants import SdkTaskType
+from flytekit.common.tasks import sdk_runnable as _sdk_runnable
+from flytekit.models.sagemaker import training_job as _training_job_models
+
+
+[docs]class CustomTrainingJobTask(_sdk_runnable.SdkRunnableTask):
+ """
+ CustomTrainJobTask defines a python task that can run on SageMaker bring your own container.
+
+ """
+
+ def __init__(
+ self,
+ task_function,
+ cache_version,
+ retries,
+ deprecated,
+ storage_request,
+ cpu_request,
+ gpu_request,
+ memory_request,
+ storage_limit,
+ cpu_limit,
+ gpu_limit,
+ memory_limit,
+ cache,
+ timeout,
+ environment,
+ algorithm_specification: _training_job_models.AlgorithmSpecification,
+ training_job_resource_config: _training_job_models.TrainingJobResourceConfig,
+ ):
+ """
+ :param task_function: Function container user code. This will be executed via the SDK's engine.
+ :param Text cache_version: string describing the version for task discovery purposes
+ :param int retries: Number of retries to attempt
+ :param Text deprecated:
+ :param Text storage_request:
+ :param Text cpu_request:
+ :param Text gpu_request:
+ :param Text memory_request:
+ :param Text storage_limit:
+ :param Text cpu_limit:
+ :param Text gpu_limit:
+ :param Text memory_limit:
+ :param bool cache:
+ :param datetime.timedelta timeout:
+ :param dict[Text, Text] environment:
+ :param _training_job_models.AlgorithmSpecification algorithm_specification:
+ :param _training_job_models.TrainingJobResourceConfig training_job_resource_config:
+ """
+
+ # Use the training job model as a measure of type checking
+ self._training_job_model = _training_job_models.TrainingJob(
+ algorithm_specification=algorithm_specification, training_job_resource_config=training_job_resource_config
+ )
+
+ super().__init__(
+ task_function=task_function,
+ task_type=SdkTaskType.SAGEMAKER_CUSTOM_TRAINING_JOB_TASK,
+ discovery_version=cache_version,
+ retries=retries,
+ interruptible=False,
+ deprecated=deprecated,
+ storage_request=storage_request,
+ cpu_request=cpu_request,
+ gpu_request=gpu_request,
+ memory_request=memory_request,
+ storage_limit=storage_limit,
+ cpu_limit=cpu_limit,
+ gpu_limit=gpu_limit,
+ memory_limit=memory_limit,
+ discoverable=cache,
+ timeout=timeout,
+ environment=environment,
+ custom=MessageToDict(self._training_job_model.to_flyte_idl()),
+ )
+
+ @property
+ def training_job_model(self) -> _training_job_models.TrainingJob:
+ return self._training_job_model
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/_modules/flytekit/common/tasks/sagemaker/hpo_job_task.html b/_modules/flytekit/common/tasks/sagemaker/hpo_job_task.html
index ab448568d5..72b15da57b 100644
--- a/_modules/flytekit/common/tasks/sagemaker/hpo_job_task.html
+++ b/_modules/flytekit/common/tasks/sagemaker/hpo_job_task.html
@@ -8,7 +8,7 @@
- flytekit.common.tasks.sagemaker.hpo_job_task — Flyte 0.6.0 documentation
+ flytekit.common.tasks.sagemaker.hpo_job_task — Flyte 0.7.0 documentation
@@ -160,9 +160,8 @@
Source code for flytekit.common.tasks.sagemaker.hpo_job_task
-from __future__ import absolute_import
-
-import datetime as _datetime
+import datetime as _datetime
+import typing
from flyteidl.plugins.sagemaker import hyperparameter_tuning_job_pb2 as _pb2_hpo_job
from google.protobuf.json_format import MessageToDict
@@ -171,7 +170,8 @@ Source code for flytekit.common.tasks.sagemaker.hpo_job_task
from flytekit.common import interface as _interface
from flytekit.common.constants import SdkTaskType
from flytekit.common.tasks import task as _sdk_task
-from flytekit.common.tasks.sagemaker.training_job_task import SdkBuiltinAlgorithmTrainingJobTask
+from flytekit.common.tasks.sagemaker.built_in_training_job_task import SdkBuiltinAlgorithmTrainingJobTask
+from flytekit.common.tasks.sagemaker.custom_training_job_task import CustomTrainingJobTask
from flytekit.models import interface as _interface_model
from flytekit.models import literals as _literal_models
from flytekit.models import task as _task_models
@@ -182,15 +182,14 @@ Source code for flytekit.common.tasks.sagemaker.hpo_job_task
[docs]class SdkSimpleHyperparameterTuningJobTask(_sdk_task.SdkTask):
-
def __init__(
- self,
- max_number_of_training_jobs: int,
- max_parallel_training_jobs: int,
- training_job: SdkBuiltinAlgorithmTrainingJobTask,
- retries: int = 0,
- cacheable: bool = False,
- cache_version: str = "",
+ self,
+ max_number_of_training_jobs: int,
+ max_parallel_training_jobs: int,
+ training_job: typing.Union[SdkBuiltinAlgorithmTrainingJobTask, CustomTrainingJobTask],
+ retries: int = 0,
+ cacheable: bool = False,
+ cache_version: str = "",
):
"""
@@ -216,20 +215,17 @@ Source code for flytekit.common.tasks.sagemaker.hpo_job_task
timeout = _datetime.timedelta(seconds=0)
inputs = {
- "hyperparameter_tuning_job_config": _interface_model.Variable(
- _sdk_types.Types.Proto(
- _pb2_hpo_job.HyperparameterTuningJobConfig).to_flyte_literal_type(), ""
- ),
- }
+ "hyperparameter_tuning_job_config": _interface_model.Variable(
+ _sdk_types.Types.Proto(_pb2_hpo_job.HyperparameterTuningJobConfig).to_flyte_literal_type(), "",
+ ),
+ }
inputs.update(training_job.interface.inputs)
super(SdkSimpleHyperparameterTuningJobTask, self).__init__(
type=SdkTaskType.SAGEMAKER_HYPERPARAMETER_TUNING_JOB_TASK,
metadata=_task_models.TaskMetadata(
runtime=_task_models.RuntimeMetadata(
- type=_task_models.RuntimeMetadata.RuntimeType.FLYTE_SDK,
- version=__version__,
- flavor='sagemaker'
+ type=_task_models.RuntimeMetadata.RuntimeType.FLYTE_SDK, version=__version__, flavor="sagemaker",
),
discoverable=cacheable,
timeout=timeout,
@@ -244,13 +240,12 @@ Source code for flytekit.common.tasks.sagemaker.hpo_job_task
"model": _interface_model.Variable(
type=_types_models.LiteralType(
blob=_core_types.BlobType(
- format="",
- dimensionality=_core_types.BlobType.BlobDimensionality.SINGLE
+ format="", dimensionality=_core_types.BlobType.BlobDimensionality.SINGLE,
)
),
- description=""
+ description="",
)
- }
+ },
),
custom=MessageToDict(hpo_job),
)
diff --git a/_modules/flytekit/common/tasks/sdk_dynamic.html b/_modules/flytekit/common/tasks/sdk_dynamic.html
index a453b6dc84..c88f66ecb9 100644
--- a/_modules/flytekit/common/tasks/sdk_dynamic.html
+++ b/_modules/flytekit/common/tasks/sdk_dynamic.html
@@ -8,7 +8,7 @@
- flytekit.common.tasks.sdk_dynamic — Flyte 0.6.0 documentation
+ flytekit.common.tasks.sdk_dynamic — Flyte 0.7.0 documentation
@@ -162,22 +162,29 @@
Source code for flytekit.common.tasks.sdk_dynamic
from __future__ import absolute_import
-import os as _os
-
import itertools as _itertools
import math
+import os as _os
+
import six as _six
-from flytekit.common import constants as _constants, interface as _interface, sdk_bases as _sdk_bases, \
- launch_plan as _launch_plan, workflow as _workflow
+from flytekit.common import constants as _constants
+from flytekit.common import interface as _interface
+from flytekit.common import launch_plan as _launch_plan
+from flytekit.common import sdk_bases as _sdk_bases
+from flytekit.common import workflow as _workflow
from flytekit.common.core import identifier as _identifier
from flytekit.common.exceptions import scopes as _exception_scopes
from flytekit.common.mixins import registerable as _registerable
-from flytekit.common.tasks import output as _task_output, sdk_runnable as _sdk_runnable, task as _task
+from flytekit.common.tasks import output as _task_output
+from flytekit.common.tasks import sdk_runnable as _sdk_runnable
+from flytekit.common.tasks import task as _task
from flytekit.common.types import helpers as _type_helpers
from flytekit.common.utils import _dnsify
from flytekit.configuration import internal as _internal_config
-from flytekit.models import literals as _literal_models, dynamic_job as _dynamic_job, array_job as _array_job
+from flytekit.models import array_job as _array_job
+from flytekit.models import dynamic_job as _dynamic_job
+from flytekit.models import literals as _literal_models
[docs]class PromiseOutputReference(_task_output.OutputReference):
@@ -209,68 +216,23 @@ Source code for flytekit.common.tasks.sdk_dynamic
# Upload inputs to working directory under /array_job.input_ref/inputs.pb
input_path = _os.path.join(node.id, _constants.INPUT_FILE_NAME)
generated_files[input_path] = _literal_models.LiteralMap(
- literals={binding.var: binding.binding.to_literal_model() for binding in
- sub_task_node.inputs})
+ literals={binding.var: binding.binding.to_literal_model() for binding in sub_task_node.inputs}
+ )
+
+[docs]class SdkDynamicTaskMixin(object):
-[docs]class SdkDynamicTask(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _sdk_runnable.SdkRunnableTask)):
"""
- This class includes the additional logic for building a task that executes parent-child tasks in Python code. It
- has even more validation checks to ensure proper behavior than it's superclasses.
+ This mixin implements logic for building a task that executes
+ parent-child tasks in Python code.
- Since an SdkDynamicTask is assumed to run by hooking into Python code, we will provide additional shortcuts and
- methods on this object.
"""
- def __init__(
- self,
- task_function,
- task_type,
- discovery_version,
- retries,
- interruptible,
- deprecated,
- storage_request,
- cpu_request,
- gpu_request,
- memory_request,
- storage_limit,
- cpu_limit,
- gpu_limit,
- memory_limit,
- discoverable,
- timeout,
- allowed_failure_ratio,
- max_concurrency,
- environment,
- custom
- ):
+ def __init__(self, allowed_failure_ratio, max_concurrency):
"""
- :param task_function: Function container user code. This will be executed via the SDK's engine.
- :param Text task_type: string describing the task type
- :param Text discovery_version: string describing the version for task discovery purposes
- :param int retries: Number of retries to attempt
- :param bool interruptible: Whether or not task is interruptible
- :param Text deprecated:
- :param Text storage_request:
- :param Text cpu_request:
- :param Text gpu_request:
- :param Text memory_request:
- :param Text storage_limit:
- :param Text cpu_limit:
- :param Text gpu_limit:
- :param Text memory_limit:
- :param bool discoverable:
- :param datetime.timedelta timeout:
:param float allowed_failure_ratio:
:param int max_concurrency:
- :param dict[Text, Text] environment:
- :param dict[Text, T] custom:
"""
- super(SdkDynamicTask, self).__init__(
- task_function, task_type, discovery_version, retries, interruptible, deprecated,
- storage_request, cpu_request, gpu_request, memory_request, storage_limit,
- cpu_limit, gpu_limit, memory_limit, discoverable, timeout, environment, custom)
# These will only appear in the generated futures
self._allowed_failure_ratio = allowed_failure_ratio
@@ -282,9 +244,9 @@ Source code for flytekit.common.tasks.sdk_dynamic
:param str inputs_prefix:
:rtype: _array_job.ArrayJob
"""
- return _array_job.ArrayJob(parallelism=self._max_concurrency if self._max_concurrency else 0,
- size=1,
- min_successes=1)
+ return _array_job.ArrayJob(
+ parallelism=self._max_concurrency if self._max_concurrency else 0, size=1, min_successes=1,
+ )
@staticmethod
def _can_run_as_array(task_type):
@@ -316,9 +278,10 @@ Source code for flytekit.common.tasks.sdk_dynamic
:param flytekit.models.literals.LiteralMap literal_map inputs:
:rtype: (_dynamic_job.DynamicJobSpec, dict[Text, flytekit.models.common.FlyteIdlEntity])
"""
- inputs_dict = _type_helpers.unpack_literal_map_to_sdk_python_std(inputs, {
- k: _type_helpers.get_sdk_type_from_literal_type(v.type) for k, v in _six.iteritems(self.interface.inputs)
- })
+ inputs_dict = _type_helpers.unpack_literal_map_to_sdk_python_std(
+ inputs,
+ {k: _type_helpers.get_sdk_type_from_literal_type(v.type) for k, v in _six.iteritems(self.interface.inputs)},
+ )
outputs_dict = {
name: PromiseOutputReference(_type_helpers.get_sdk_type_from_literal_type(variable.type))
for name, variable in _six.iteritems(self.interface.outputs)
@@ -327,13 +290,18 @@ Source code for flytekit.common.tasks.sdk_dynamic
# Because users declare both inputs and outputs in their functions signatures, merge them together
# before calling user code
inputs_dict.update(outputs_dict)
- yielded_sub_tasks = [sub_task for sub_task in
- super(SdkDynamicTask, self)._execute_user_code(context, inputs_dict) or []]
+ yielded_sub_tasks = [sub_task for sub_task in self._execute_user_code(context, inputs_dict) or []]
upstream_nodes = list()
- output_bindings = [_literal_models.Binding(var=name, binding=_interface.BindingData.from_python_std(
- b.sdk_type.to_flyte_literal_type(), b.raw_value, upstream_nodes=upstream_nodes))
- for name, b in _six.iteritems(outputs_dict)]
+ output_bindings = [
+ _literal_models.Binding(
+ var=name,
+ binding=_interface.BindingData.from_python_std(
+ b.sdk_type.to_flyte_literal_type(), b.raw_value, upstream_nodes=upstream_nodes,
+ ),
+ )
+ for name, b in _six.iteritems(outputs_dict)
+ ]
upstream_nodes = set(upstream_nodes)
generated_files = {}
@@ -366,7 +334,7 @@ Source code for flytekit.common.tasks.sdk_dynamic
_internal_config.TASK_PROJECT.get() or _internal_config.PROJECT.get(),
_internal_config.TASK_DOMAIN.get() or _internal_config.DOMAIN.get(),
executable.platform_valid_name,
- _internal_config.TASK_VERSION.get() or _internal_config.VERSION.get()
+ _internal_config.TASK_VERSION.get() or _internal_config.VERSION.get(),
)
# Generate an id that's unique in the document (if the same task is used multiple times with
@@ -403,7 +371,10 @@ Source code for flytekit.common.tasks.sdk_dynamic
else:
array_job = self._create_array_job(inputs_prefix=unique_node_id)
node = sub_task_node.assign_id_and_return(unique_node_id)
- array_job_index[sub_task_node.executable_sdk_object] = (array_job, node)
+ array_job_index[sub_task_node.executable_sdk_object] = (
+ array_job,
+ node,
+ )
node_index = _six.text_type(array_job.size - 1)
for k, node_output in _six.iteritems(sub_task_node.outputs):
@@ -414,8 +385,8 @@ Source code for flytekit.common.tasks.sdk_dynamic
# Upload inputs to working directory under /array_job.input_ref/<index>/inputs.pb
input_path = _os.path.join(node.id, node_index, _constants.INPUT_FILE_NAME)
generated_files[input_path] = _literal_models.LiteralMap(
- literals={binding.var: binding.binding.to_literal_model() for binding in
- sub_task_node.inputs})
+ literals={binding.var: binding.binding.to_literal_model() for binding in sub_task_node.inputs}
+ )
else:
node = sub_task_node.assign_id_and_return(unique_node_id)
tasks.add(sub_task_node.executable_sdk_object)
@@ -424,8 +395,11 @@ Source code for flytekit.common.tasks.sdk_dynamic
# assign custom field to the ArrayJob properties computed.
for task, (array_job, _) in _six.iteritems(array_job_index):
# TODO: Reconstruct task template object instead of modifying an existing one?
- tasks.add(task.assign_custom_and_return(array_job.to_dict()).assign_type_and_return(
- _constants.SdkTaskType.CONTAINER_ARRAY_TASK))
+ tasks.add(
+ task.assign_custom_and_return(array_job.to_dict()).assign_type_and_return(
+ _constants.SdkTaskType.CONTAINER_ARRAY_TASK
+ )
+ )
# min_successes is absolute, it's computed as the reverse of allowed_failure_ratio and multiplied by the
# total length of tasks to get an absolute count.
@@ -435,11 +409,12 @@ Source code for flytekit.common.tasks.sdk_dynamic
tasks=list(tasks),
nodes=nodes,
outputs=output_bindings,
- subworkflows=list(sub_workflows))
+ subworkflows=list(sub_workflows),
+ )
return dynamic_job_spec, generated_files
-[docs] @_exception_scopes.system_entry_point
+[docs] @_exception_scopes.system_entry_point
def execute(self, context, inputs):
"""
Executes batch task's user code and produces futures file as well as all sub-task inputs.pb files.
@@ -459,14 +434,93 @@ Source code for flytekit.common.tasks.sdk_dynamic
if len(spec.nodes) == 0:
return {
_constants.OUTPUT_FILE_NAME: _literal_models.LiteralMap(
- literals={binding.var: binding.binding.to_literal_model() for binding in spec.outputs})
+ literals={binding.var: binding.binding.to_literal_model() for binding in spec.outputs}
+ )
}
else:
- generated_files.update({
- _constants.FUTURES_FILE_NAME: spec
- })
+ generated_files.update({_constants.FUTURES_FILE_NAME: spec})
return generated_files
+
+
+[docs]class SdkDynamicTask(
+ SdkDynamicTaskMixin, _sdk_runnable.SdkRunnableTask, metaclass=_sdk_bases.ExtendedSdkType,
+):
+
+ """
+ This class includes the additional logic for building a task that executes
+ parent-child tasks in Python code.
+
+ """
+
+ def __init__(
+ self,
+ task_function,
+ task_type,
+ discovery_version,
+ retries,
+ interruptible,
+ deprecated,
+ storage_request,
+ cpu_request,
+ gpu_request,
+ memory_request,
+ storage_limit,
+ cpu_limit,
+ gpu_limit,
+ memory_limit,
+ discoverable,
+ timeout,
+ allowed_failure_ratio,
+ max_concurrency,
+ environment,
+ custom,
+ ):
+ """
+ :param task_function: Function container user code. This will be executed via the SDK's engine.
+ :param Text task_type: string describing the task type
+ :param Text discovery_version: string describing the version for task discovery purposes
+ :param int retries: Number of retries to attempt
+ :param bool interruptible: Whether or not task is interruptible
+ :param Text deprecated:
+ :param Text storage_request:
+ :param Text cpu_request:
+ :param Text gpu_request:
+ :param Text memory_request:
+ :param Text storage_limit:
+ :param Text cpu_limit:
+ :param Text gpu_limit:
+ :param Text memory_limit:
+ :param bool discoverable:
+ :param datetime.timedelta timeout:
+ :param float allowed_failure_ratio:
+ :param int max_concurrency:
+ :param dict[Text, Text] environment:
+ :param dict[Text, T] custom:
+ """
+ _sdk_runnable.SdkRunnableTask.__init__(
+ self,
+ task_function,
+ task_type,
+ discovery_version,
+ retries,
+ interruptible,
+ deprecated,
+ storage_request,
+ cpu_request,
+ gpu_request,
+ memory_request,
+ storage_limit,
+ cpu_limit,
+ gpu_limit,
+ memory_limit,
+ discoverable,
+ timeout,
+ environment,
+ custom,
+ )
+
+ SdkDynamicTaskMixin.__init__(self, allowed_failure_ratio, max_concurrency)
diff --git a/_modules/flytekit/common/tasks/sdk_runnable.html b/_modules/flytekit/common/tasks/sdk_runnable.html
index 575a25c219..50077fdc96 100644
--- a/_modules/flytekit/common/tasks/sdk_runnable.html
+++ b/_modules/flytekit/common/tasks/sdk_runnable.html
@@ -8,7 +8,7 @@
- flytekit.common.tasks.sdk_runnable — Flyte 0.6.0 documentation
+ flytekit.common.tasks.sdk_runnable — Flyte 0.7.0 documentation
@@ -170,14 +170,21 @@ Source code for flytekit.common.tasks.sdk_runnable
import six as _six
from flytekit import __version__
-from flytekit.common import interface as _interface, constants as _constants, sdk_bases as _sdk_bases
-from flytekit.common.exceptions import user as _user_exceptions, scopes as _exception_scopes
-from flytekit.common.tasks import task as _base_task, output as _task_output
+from flytekit.common import constants as _constants
+from flytekit.common import interface as _interface
+from flytekit.common import sdk_bases as _sdk_bases
+from flytekit.common.core.identifier import WorkflowExecutionIdentifier
+from flytekit.common.exceptions import scopes as _exception_scopes
+from flytekit.common.exceptions import user as _user_exceptions
+from flytekit.common.tasks import output as _task_output
+from flytekit.common.tasks import task as _base_task
from flytekit.common.types import helpers as _type_helpers
-from flytekit.configuration import sdk as _sdk_config, internal as _internal_config, resources as _resource_config
+from flytekit.configuration import internal as _internal_config
+from flytekit.configuration import resources as _resource_config
+from flytekit.configuration import sdk as _sdk_config
from flytekit.engines import loader as _engine_loader
-from flytekit.models import literals as _literal_models, task as _task_models
-from flytekit.common.core.identifier import WorkflowExecutionIdentifier
+from flytekit.models import literals as _literal_models
+from flytekit.models import task as _task_models
[docs]class ExecutionParameters(object):
@@ -259,23 +266,10 @@ Source code for flytekit.common.tasks.sdk_runnable
[docs]class SdkRunnableContainer(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _task_models.Container)):
-
def __init__(
- self,
- command,
- args,
- resources,
- env,
- config,
+ self, command, args, resources, env, config,
):
- super(SdkRunnableContainer, self).__init__(
- "",
- command,
- args,
- resources,
- env or {},
- config
- )
+ super(SdkRunnableContainer, self).__init__("", command, args, resources, env or {}, config)
@property
def args(self):
@@ -321,25 +315,25 @@ Source code for flytekit.common.tasks.sdk_runnable
"""
def __init__(
- self,
- task_function,
- task_type,
- discovery_version,
- retries,
- interruptible,
- deprecated,
- storage_request,
- cpu_request,
- gpu_request,
- memory_request,
- storage_limit,
- cpu_limit,
- gpu_limit,
- memory_limit,
- discoverable,
- timeout,
- environment,
- custom
+ self,
+ task_function,
+ task_type,
+ discovery_version,
+ retries,
+ interruptible,
+ deprecated,
+ storage_request,
+ cpu_request,
+ gpu_request,
+ memory_request,
+ storage_limit,
+ cpu_limit,
+ gpu_limit,
+ memory_limit,
+ discoverable,
+ timeout,
+ environment,
+ custom,
):
"""
:param task_function: Function container user code. This will be executed via the SDK's engine.
@@ -368,15 +362,13 @@ Source code for flytekit.common.tasks.sdk_runnable
_task_models.TaskMetadata(
discoverable,
_task_models.RuntimeMetadata(
- _task_models.RuntimeMetadata.RuntimeType.FLYTE_SDK,
- __version__,
- 'python'
+ _task_models.RuntimeMetadata.RuntimeType.FLYTE_SDK, __version__, "python",
),
timeout,
_literal_models.RetryStrategy(retries),
interruptible,
discovery_version,
- deprecated
+ deprecated,
),
_interface.TypedInterface({}, {}),
custom,
@@ -389,8 +381,8 @@ Source code for flytekit.common.tasks.sdk_runnable
cpu_limit=cpu_limit,
gpu_limit=gpu_limit,
memory_limit=memory_limit,
- environment=environment
- )
+ environment=environment,
+ ),
)
self.id._name = "{}.{}".format(self.task_module, self.task_function_name)
@@ -407,7 +399,7 @@ Source code for flytekit.common.tasks.sdk_runnable
"""
self._validate_inputs(inputs)
self.interface.inputs.update(inputs)
-
+
[docs] @classmethod
def promote_from_model(cls, base_model):
# TODO: If the task exists in this container, we should be able to retrieve it.
@@ -438,10 +430,7 @@ Source code for flytekit.common.tasks.sdk_runnable
raise _user_exceptions.FlyteAssertion(
"The task {} is invalid because not all inputs and outputs in the "
"task function definition were specified in @outputs and @inputs. "
- "We are missing definitions for {}.".format(
- self,
- missing_args
- )
+ "We are missing definitions for {}.".format(self, missing_args)
)
[docs] @_exception_scopes.system_entry_point
@@ -451,11 +440,18 @@ Source code for flytekit.common.tasks.sdk_runnable
literals.
:returns: Depends on the behavior of the specific task in the unit engine.
"""
- return _engine_loader.get_engine('unit').get_task(self).execute(
- _type_helpers.pack_python_std_map_to_literal_map(input_map, {
- k: _type_helpers.get_sdk_type_from_literal_type(v.type)
- for k, v in _six.iteritems(self.interface.inputs)
- })
+ return (
+ _engine_loader.get_engine("unit")
+ .get_task(self)
+ .execute(
+ _type_helpers.pack_python_std_map_to_literal_map(
+ input_map,
+ {
+ k: _type_helpers.get_sdk_type_from_literal_type(v.type)
+ for k, v in _six.iteritems(self.interface.inputs)
+ },
+ )
+ )
)
[docs] @_exception_scopes.system_entry_point
@@ -466,11 +462,18 @@ Source code for flytekit.common.tasks.sdk_runnable
:rtype: dict[Text, T]
:returns: The output produced by this task in Python standard format.
"""
- return _engine_loader.get_engine('local').get_task(self).execute(
- _type_helpers.pack_python_std_map_to_literal_map(input_map, {
- k: _type_helpers.get_sdk_type_from_literal_type(v.type)
- for k, v in _six.iteritems(self.interface.inputs)
- })
+ return (
+ _engine_loader.get_engine("local")
+ .get_task(self)
+ .execute(
+ _type_helpers.pack_python_std_map_to_literal_map(
+ input_map,
+ {
+ k: _type_helpers.get_sdk_type_from_literal_type(v.type)
+ for k, v in _six.iteritems(self.interface.inputs)
+ },
+ )
+ )
)
def _execute_user_code(self, context, inputs):
@@ -496,7 +499,7 @@ Source code for flytekit.common.tasks.sdk_runnable
execution_id=_six.text_type(WorkflowExecutionIdentifier.promote_from_model(context.execution_id)),
stats=context.stats,
logging=context.logging,
- tmp_dir=context.working_directory
+ tmp_dir=context.working_directory,
),
**inputs
)
@@ -513,9 +516,10 @@ Source code for flytekit.common.tasks.sdk_runnable
working directory (with the names provided), which will in turn allow Flyte Propeller to push along the
workflow. Where as local engine will merely feed the outputs directly into the next node.
"""
- inputs_dict = _type_helpers.unpack_literal_map_to_sdk_python_std(inputs, {
- k: _type_helpers.get_sdk_type_from_literal_type(v.type) for k, v in _six.iteritems(self.interface.inputs)
- })
+ inputs_dict = _type_helpers.unpack_literal_map_to_sdk_python_std(
+ inputs,
+ {k: _type_helpers.get_sdk_type_from_literal_type(v.type) for k, v in _six.iteritems(self.interface.inputs)},
+ )
outputs_dict = {
name: _task_output.OutputReference(_type_helpers.get_sdk_type_from_literal_type(variable.type))
for name, variable in _six.iteritems(self.interface.outputs)
@@ -531,17 +535,17 @@ Source code for flytekit.common.tasks.sdk_runnable
}
def _get_container_definition(
- self,
- storage_request=None,
- cpu_request=None,
- gpu_request=None,
- memory_request=None,
- storage_limit=None,
- cpu_limit=None,
- gpu_limit=None,
- memory_limit=None,
- environment=None,
- cls=None,
+ self,
+ storage_request=None,
+ cpu_request=None,
+ gpu_request=None,
+ memory_request=None,
+ storage_limit=None,
+ cpu_limit=None,
+ gpu_limit=None,
+ memory_limit=None,
+ environment=None,
+ cls=None,
):
"""
:param Text storage_request:
@@ -568,61 +572,29 @@ Source code for flytekit.common.tasks.sdk_runnable
requests