vdk-core,vdk-impala,vdk-lineage,vdk-trino: Support for pluggy 1.0 (#931)

The 1.0 release of `pluggy` introduced a breaking change by renaming its `callers` module to `_callers`. This has been fixed by using the `HookCallResult` constant from `vdk.api.plugin.plugin_registry` instead of `pluggy.callers._Result` everywhere necessary, and amending said constant to be either `pluggy.callers._Result` or `pluggy._callers._Result` dynamically based on the version of `pluggy` in the current Python env. Also fixed a test which had `hookwrapper` set to True for some reason inside its testing plugin. Testing done: ran tests locally, CICD Signed-off-by: Gabriel Georgiev <[email protected]>
vmware · Aug 4, 2022 · 219c6b3 · 219c6b3
1 parent 27d9ae6
commit 219c6b3
Show file tree

Hide file tree

Showing 10 changed files with 19 additions and 17 deletions.
diff --git a/projects/vdk-core/setup.cfg b/projects/vdk-core/setup.cfg
@@ -46,7 +46,7 @@ install_requires =
     # click 8 has some breaking changes that break vdk-control-cli
     # https://github.com/pallets/click/issues/1960
     click==8.*
-    pluggy==0.*
+    pluggy
     click_log
     click-plugins
     tenacity

diff --git a/projects/vdk-core/src/vdk/api/plugin/connection_hook_spec.py b/projects/vdk-core/src/vdk/api/plugin/connection_hook_spec.py
@@ -100,7 +100,7 @@ def db_connection_execute_operation(
                 # let's track duration of the query
                 start = time.time()
                 log.info(f"Starting query: {execution_cursor.get_managed_operation().get_operation()}")
-                outcome: pluggy.callers._Result
+                outcome: HookCallResult
                 outcome = yield # we yield the execution to other implementations (including default one)
                 is_success: bool = outcome.excinfo is None
                 log.info(f"Query finished. duration: {time.time() - start}. is_success: {is_success}")
@@ -112,7 +112,7 @@ def db_connection_execute_operation(
 
             @hookimpl(hookwrapper=True)
             db_connection_execute_operation(execution_cursor: ExecutionCursor) -> Optional[int]:
-                outcome: pluggy.callers._Result
+                outcome: HookCallResult
                 outcome = yield #
                 outcome.force_result(new_result)  # set new return result
 

diff --git a/projects/vdk-core/src/vdk/api/plugin/hook_markers.py b/projects/vdk-core/src/vdk/api/plugin/hook_markers.py
@@ -45,7 +45,7 @@
 If ``hookwrapper`` is ``True`` the hook implementations needs to execute exactly
 one ``yield``.  The code before the ``yield`` is run early before any non-hookwrapper
 function is run.  The code after the ``yield`` is run after all non-hookwrapper
-function have run.  The ``yield`` receives a :py:class:`.callers._Result` object
+function have run.  The ``yield`` receives a :py:class:`HookCallResult` object
 representing the exception or result outcome of the inner calls (including other
 hookwrapper calls).
 Example:

diff --git a/projects/vdk-core/src/vdk/api/plugin/plugin_registry.py b/projects/vdk-core/src/vdk/api/plugin/plugin_registry.py
@@ -43,7 +43,9 @@ class PluginException(Exception):
 """
 Alias for the type of plugin hook call result returned in hookWrapper=True types of plugin hooks
 """
-HookCallResult = pluggy.callers._Result
+HookCallResult = (
+    pluggy.callers._Result if pluggy.__version__ < "1.0" else pluggy._callers._Result
+)
 
 
 class IPluginRegistry(metaclass=ABCMeta):

diff --git a/projects/vdk-core/src/vdk/internal/builtin_plugins/run/execution_tracking.py b/projects/vdk-core/src/vdk/internal/builtin_plugins/run/execution_tracking.py
@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 import pluggy
 from vdk.api.plugin.hook_markers import hookimpl
+from vdk.api.plugin.plugin_registry import HookCallResult
 from vdk.internal.builtin_plugins.config import vdk_config
 from vdk.internal.builtin_plugins.run.execution_results import ExecutionResult
 from vdk.internal.builtin_plugins.run.execution_results import StepResult
@@ -38,7 +39,7 @@ def run_job(self, context: JobContext) -> None:
         It executes the provided steps starting from context.step_tree_root in sequential order (using BFS)
         using
         """
-        out: pluggy.callers._Result
+        out: HookCallResult
         out = yield
 
         result: ExecutionResult = out.get_result()
@@ -50,7 +51,7 @@ def run_step(self, context: JobContext, step: Step) -> None:
         state = context.core_context.state
 
         state.get(ExecutionStateStoreKeys.STEPS_STARTED).append(step.name)
-        out: pluggy.callers._Result
+        out: HookCallResult
         out = yield
         if out.excinfo:
             state.get(ExecutionStateStoreKeys.STEPS_FAILED).append(step.name)

diff --git a/projects/vdk-core/tests/functional/run/test_run_errors.py b/projects/vdk-core/tests/functional/run/test_run_errors.py
@@ -84,7 +84,7 @@ def test_run_job_plugin_fails(tmp_termination_msg_file):
 
     class RunJobFailsPlugin:
         @staticmethod
-        @hookimpl(hookwrapper=True)
+        @hookimpl()
         def run_job(context: JobContext) -> None:
             raise OverflowError("Overflow")
 

diff --git a/projects/vdk-core/tests/functional/run/test_run_sql_queries.py b/projects/vdk-core/tests/functional/run/test_run_sql_queries.py
@@ -7,10 +7,10 @@
 from typing import Optional
 from unittest import mock
 
-import pluggy
 from click.testing import Result
 from functional.run.util import job_path
 from vdk.api.plugin.hook_markers import hookimpl
+from vdk.api.plugin.plugin_registry import HookCallResult
 from vdk.internal.builtin_plugins.connection.execution_cursor import ExecutionCursor
 from vdk.internal.builtin_plugins.connection.pep249.interfaces import PEP249Connection
 from vdk.internal.builtin_plugins.connection.recovery_cursor import RecoveryCursor
@@ -257,7 +257,7 @@ def db_connection_execute_operation(
         self, execution_cursor: ExecutionCursor
     ) -> Optional[int]:
         self.log.append("start")
-        out: pluggy.callers._Result
+        out: HookCallResult
         out = yield
         self.log.append(("end", out.excinfo is None))
 

diff --git a/projects/vdk-plugins/vdk-impala/src/vdk/plugin/impala/impala_plugin.py b/projects/vdk-plugins/vdk-impala/src/vdk/plugin/impala/impala_plugin.py
@@ -6,9 +6,9 @@
 from typing import List
 
 import click
-import pluggy
 from tabulate import tabulate
 from vdk.api.plugin.hook_markers import hookimpl
+from vdk.api.plugin.plugin_registry import HookCallResult
 from vdk.api.plugin.plugin_registry import IPluginRegistry
 from vdk.internal.builtin_plugins.connection.decoration_cursor import DecorationCursor
 from vdk.internal.builtin_plugins.connection.recovery_cursor import RecoveryCursor
@@ -107,7 +107,7 @@ def initialize_job(self, context: JobContext) -> None:
     @staticmethod
     @hookimpl(hookwrapper=True, tryfirst=True)
     def run_step(context: JobContext, step: Step) -> None:
-        out: pluggy.callers._Result
+        out: HookCallResult
         out = yield
 
         if out.result.exception:

diff --git a/projects/vdk-plugins/vdk-lineage/src/vdk/plugin/lineage/plugin_lineage.py b/projects/vdk-plugins/vdk-lineage/src/vdk/plugin/lineage/plugin_lineage.py
@@ -6,12 +6,12 @@
 from typing import List
 from typing import Optional
 
-import pluggy
 from openlineage.client import OpenLineageClient
 from openlineage.client.facet import ParentRunFacet
 from openlineage.client.run import RunEvent
 from openlineage.client.run import RunState
 from vdk.api.plugin.hook_markers import hookimpl
+from vdk.api.plugin.plugin_registry import HookCallResult
 from vdk.internal.builtin_plugins.config.job_config import JobConfigKeys
 from vdk.internal.builtin_plugins.connection.decoration_cursor import DecorationCursor
 from vdk.internal.builtin_plugins.run.execution_results import ExecutionResult
@@ -94,7 +94,7 @@ def initialize_job(self, context: JobContext) -> None:
 
     @hookimpl(hookwrapper=True)
     def run_job(self, context: JobContext) -> Optional[ExecutionResult]:
-        out: pluggy.callers._Result
+        out: HookCallResult
         out = yield
         if self.__client:
             result: ExecutionResult = out.get_result()

diff --git a/projects/vdk-plugins/vdk-trino/src/vdk/plugin/trino/trino_plugin.py b/projects/vdk-plugins/vdk-trino/src/vdk/plugin/trino/trino_plugin.py
@@ -4,15 +4,14 @@
 import os
 import pathlib
 from typing import Callable
-from typing import Optional
 
 import click
-import pluggy
 import requests
 from tabulate import tabulate
 from trino.exceptions import TrinoUserError
 from vdk.api.lineage.model.logger.lineage_logger import ILineageLogger
 from vdk.api.plugin.hook_markers import hookimpl
+from vdk.api.plugin.plugin_registry import HookCallResult
 from vdk.internal.builtin_plugins.connection.pep249.interfaces import PEP249Connection
 from vdk.internal.builtin_plugins.run.execution_results import StepResult
 from vdk.internal.builtin_plugins.run.job_context import JobContext
@@ -94,7 +93,7 @@ def initialize_job(context: JobContext) -> None:
 
 @hookimpl(hookwrapper=True, tryfirst=True)
 def run_step(context: JobContext, step: Step) -> None:
-    out: pluggy.callers._Result
+    out: HookCallResult
     out = yield
     if out.excinfo:
         exc_type, exc_value, exc_traceback = out.excinfo