Skip to content

Commit

Permalink
[DOM-57472] Add additional context to version hash calculation
Browse files Browse the repository at this point in the history
- This is necessary due to another pending PR to upstream flytekit:
  flyteorg#2428

  In case this PR is not likely to be merged, we have a plan to move away
  from this change, see the linked Doc in DOM-57472
  • Loading branch information
ddl-rliu authored and ddl-ebrown committed Aug 9, 2024
1 parent 9666f15 commit 4607240
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 1 deletion.
13 changes: 13 additions & 0 deletions flytekit/configuration/plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,11 @@

from click import Group
from importlib_metadata import entry_points
from typing import Any, Dict, List, Union

from flytekit.configuration import Config, get_config_file
from flytekit.core.python_auto_container import PythonAutoContainerTask
from flytekit.core.workflow import WorkflowBase
from flytekit.loggers import logger
from flytekit.remote import FlyteRemote

Expand Down Expand Up @@ -52,6 +55,11 @@ def get_default_image() -> Optional[str]:
def get_auth_success_html(endpoint: str) -> Optional[str]:
"""Get default success html for auth. Return None to use flytekit's default success html."""

@staticmethod
def get_additional_context_for_version_hash(entity: Union[PythonAutoContainerTask, WorkflowBase]) -> List[str]:
"""Get additional context to be used for calculating the version hash."""



class FlytekitPlugin:
@staticmethod
Expand Down Expand Up @@ -90,6 +98,11 @@ def get_auth_success_html(endpoint: str) -> Optional[str]:
"""Get default success html. Return None to use flytekit's default success html."""
return None

@staticmethod
def get_additional_context_for_version_hash(entity: Union[PythonAutoContainerTask, WorkflowBase]) -> List[str]:
"""Get additional context to be used for calculating the version hash."""
return []


def _get_plugin_from_entrypoint():
"""Get plugin from entrypoint."""
Expand Down
5 changes: 4 additions & 1 deletion flytekit/remote/remote.py
Original file line number Diff line number Diff line change
Expand Up @@ -1055,11 +1055,14 @@ def _get_image_names(entity: typing.Union[PythonAutoContainerTask, WorkflowBase]
if isinstance(entity, WorkflowBase):
default_inputs = entity.python_interface.default_inputs_as_kwargs

from flytekit.configuration.plugin import get_plugin
version_hash_additional_context = get_plugin().get_additional_context_for_version_hash(entity)

# The md5 version that we send to S3/GCS has to match the file contents exactly,
# but we don't have to use it when registering with the Flyte backend.
# For that add the hash of the compilation settings to hash of file
version = self._version_from_hash(
md5_bytes, serialization_settings, default_inputs, *_get_image_names(entity)
md5_bytes, serialization_settings, default_inputs, *_get_image_names(entity), *version_hash_additional_context
)

if isinstance(entity, PythonTask):
Expand Down

0 comments on commit 4607240

Please sign in to comment.