Dynamic modules

.gitmodules

@@ -4,9 +4,9 @@
 [submodule "repos/avalon-unreal-integration"]
 	path = repos/avalon-unreal-integration
 	url = https://github.com/pypeclub/avalon-unreal-integration.git
-[submodule "openpype/modules/ftrack/python2_vendor/ftrack-python-api"]
-	path = openpype/modules/ftrack/python2_vendor/ftrack-python-api
+[submodule "openpype/modules/default_modules/ftrack/python2_vendor/arrow"]
+	path = openpype/modules/default_modules/ftrack/python2_vendor/arrow
+	url = [email protected]:arrow-py/arrow.git
+[submodule "openpype/modules/default_modules/ftrack/python2_vendor/ftrack-python-api"]
+	path = openpype/modules/default_modules/ftrack/python2_vendor/ftrack-python-api
 	url = https://bitbucket.org/ftrack/ftrack-python-api.git
-[submodule "openpype/modules/ftrack/python2_vendor/arrow"]
-	path = openpype/modules/ftrack/python2_vendor/arrow
-	url = https://github.com/arrow-py/arrow.git

openpype/__init__.py

@@ -68,6 +68,10 @@ def patched_discover(superclass):
 def install():
     """Install Pype to Avalon."""
     from pyblish.lib import MessageHandler
+    from openpype.modules import load_modules
+
+    # Make sure modules are loaded
+    load_modules()
 
     def modified_emit(obj, record):
         """Method replacing `emit` in Pyblish's MessageHandler."""

openpype/lib/__init__.py

@@ -52,9 +52,11 @@
 )
 
 from .python_module_tools import (
+    import_filepath,
     modules_from_path,
     recursive_bases_from_class,
-    classes_from_module
+    classes_from_module,
+    import_module_from_dirpath
 )
 
 from .avalon_context import (
@@ -170,9 +172,11 @@
     "get_ffmpeg_tool_path",
     "ffprobe_streams",
 
+    "import_filepath",
     "modules_from_path",
     "recursive_bases_from_class",
     "classes_from_module",
+    "import_module_from_dirpath",
 
     "CURRENT_DOC_SCHEMAS",
     "PROJECT_NAME_ALLOWED_SYMBOLS",

openpype/lib/applications.py

@@ -1105,7 +1105,7 @@ def prepare_host_environments(data, implementation_envs=True):
     asset_doc = data.get("asset_doc")
     # Add tools environments
     groups_by_name = {}
-    tool_by_group_name = collections.defaultdict(list)
+    tool_by_group_name = collections.defaultdict(dict)
     if asset_doc:
         # Make sure each tool group can be added only once
         for key in asset_doc["data"].get("tools_env") or []:
@@ -1120,7 +1120,7 @@ def prepare_host_environments(data, implementation_envs=True):
             environments.append(group.environment)
             added_env_keys.add(group_name)
             for tool_name in sorted(tool_by_group_name[group_name].keys()):
-                tool = tool_by_group_name[tool_name]
+                tool = tool_by_group_name[group_name][tool_name]
                 environments.append(tool.environment)
                 added_env_keys.add(tool.name)
 

openpype/lib/python_module_tools.py

@@ -9,6 +9,38 @@
 PY3 = sys.version_info[0] == 3
 
 
+def import_filepath(filepath, module_name=None):
+    """Import python file as python module.
+
+    Python 2 and Python 3 compatibility.
+
+    Args:
+        filepath(str): Path to python file.
+        module_name(str): Name of loaded module. Only for Python 3. By default
+            is filled with filename of filepath.
+    """
+    if module_name is None:
+        module_name = os.path.splitext(os.path.basename(filepath))[0]
+
+    # Prepare module object where content of file will be parsed
+    module = types.ModuleType(module_name)
+
+    if PY3:
+        # Use loader so module has full specs
+        module_loader = importlib.machinery.SourceFileLoader(
+            module_name, filepath
+        )
+        module_loader.exec_module(module)
+    else:
+        # Execute module code and store content to module
+        with open(filepath) as _stream:
+            # Execute content and store it to module object
+            exec(_stream.read(), module.__dict__)
+
+        module.__file__ = filepath
+    return module
+
+
 def modules_from_path(folder_path):
     """Get python scripts as modules from a path.
 
@@ -55,23 +87,7 @@ def modules_from_path(folder_path):
             continue
 
         try:
-            # Prepare module object where content of file will be parsed
-            module = types.ModuleType(mod_name)
-
-            if PY3:
-                # Use loader so module has full specs
-                module_loader = importlib.machinery.SourceFileLoader(
-                    mod_name, full_path
-                )
-                module_loader.exec_module(module)
-            else:
-                # Execute module code and store content to module
-                with open(full_path) as _stream:
-                    # Execute content and store it to module object
-                    exec(_stream.read(), module.__dict__)
-
-                module.__file__ = full_path
-
+            module = import_filepath(full_path, mod_name)
             modules.append((full_path, module))
 
         except Exception:
@@ -127,3 +143,96 @@ def classes_from_module(superclass, module):
 
         classes.append(obj)
     return classes
+
+
+def _import_module_from_dirpath_py2(dirpath, module_name, dst_module_name):
+    """Import passed dirpath as python module using `imp`."""
+    if dst_module_name:
+        full_module_name = "{}.{}".format(dst_module_name, module_name)
+        dst_module = sys.modules[dst_module_name]
+    else:
+        full_module_name = module_name
+        dst_module = None
+
+    if full_module_name in sys.modules:
+        return sys.modules[full_module_name]
+
+    import imp
+
+    fp, pathname, description = imp.find_module(module_name, [dirpath])
+    module = imp.load_module(full_module_name, fp, pathname, description)
+    if dst_module is not None:
+        setattr(dst_module, module_name, module)
+
+    return module
+
+
+def _import_module_from_dirpath_py3(dirpath, module_name, dst_module_name):
+    """Import passed dirpath as python module using Python 3 modules."""
+    if dst_module_name:
+        full_module_name = "{}.{}".format(dst_module_name, module_name)
+        dst_module = sys.modules[dst_module_name]
+    else:
+        full_module_name = module_name
+        dst_module = None
+
+    # Skip import if is already imported
+    if full_module_name in sys.modules:
+        return sys.modules[full_module_name]
+
+    import importlib.util
+    from importlib._bootstrap_external import PathFinder
+
+    # Find loader for passed path and name
+    loader = PathFinder.find_module(full_module_name, [dirpath])
+
+    # Load specs of module
+    spec = importlib.util.spec_from_loader(
+        full_module_name, loader, origin=dirpath
+    )
+
+    # Create module based on specs
+    module = importlib.util.module_from_spec(spec)
+
+    # Store module to destination module and `sys.modules`
+    # WARNING this mus be done before module execution
+    if dst_module is not None:
+        setattr(dst_module, module_name, module)
+
+    sys.modules[full_module_name] = module
+
+    # Execute module import
+    loader.exec_module(module)
+
+    return module
+
+
+def import_module_from_dirpath(dirpath, folder_name, dst_module_name=None):
+    """Import passed directory as a python module.
+
+    Python 2 and 3 compatible.
+
+    Imported module can be assigned as a child attribute of already loaded
+    module from `sys.modules` if has support of `setattr`. That is not default
+    behavior of python modules so parent module must be a custom module with
+    that ability.
+
+    It is not possible to reimport already cached module. If you need to
+    reimport module you have to remove it from caches manually.
+
+    Args:
+        dirpath(str): Parent directory path of loaded folder.
+        folder_name(str): Folder name which should be imported inside passed
+            directory.
+        dst_module_name(str): Parent module name under which can be loaded
+            module added.
+    """
+    if PY3:
+        module = _import_module_from_dirpath_py3(
+            dirpath, folder_name, dst_module_name
+        )
+    else:
+        module = _import_module_from_dirpath_py2(
+            dirpath, folder_name, dst_module_name
+        )
+    return module

openpype/modules/README.md

@@ -1,7 +1,19 @@
-# Pype modules
-Pype modules should contain separated logic of specific kind of implementation, like Ftrack connection and usage code or Deadline farm rendering.
+# OpenPype modules/addons
+OpenPype modules should contain separated logic of specific kind of implementation, like Ftrack connection and usage code or Deadline farm rendering or may contain only special plugins. Addons work the same way currently there is no difference in module and addon.
 
-## Base class `PypeModule`
+## Modules concept
+- modules and addons are dynamically imported to virtual python module `openpype_modules` from which it is possible to import them no matter where is the modulo located
+- modules or addons should never be imported directly even if you know possible full import path
+    - it is because all of their content must be imported in specific order and should not be imported without defined functions as it may also break few implementation parts
+
+### TODOs
+- add module/addon manifest
+    - definition of module (not 100% defined content e.g. minimum require OpenPype version etc.)
+    - defying that folder is content of a module or an addon
+- module/addon have it's settings schemas and default values outside OpenPype
+- add general setting of paths to modules
+
+## Base class `OpenPypeModule`
 - abstract class as base for each module
 - implementation should be module's api withou GUI parts
 - may implement `get_global_environments` method which should return dictionary of environments that are globally appliable and value is the same for whole studio if launched at any workstation (except os specific paths)
@@ -17,6 +29,16 @@ Pype modules should contain separated logic of specific kind of implementation,
 - interface is class that has defined abstract methods to implement and may contain preimplemented helper methods
 - module that inherit from an interface must implement those abstract methods otherwise won't be initialized
 - it is easy to find which module object inherited from which interfaces withh 100% chance they have implemented required methods
+- interfaces can be defined in `interfaces.py` inside module directory
+    - the file can't use relative imports or import anything from other parts
+        of module itself at the header of file
+    - this is one of reasons why modules/addons can't be imported directly without using defined functions in OpenPype modules implementation
+
+## Base class `OpenPypeInterface`
+- has nothing implemented
+- has ABCMeta as metaclass
+- is defined to be able find out classes which inherit from this base to be
+    able tell this is an Interface
 
 ## Global interfaces
 - few interfaces are implemented for global usage
@@ -70,7 +92,7 @@ Pype modules should contain separated logic of specific kind of implementation,
 - Clockify has more inharitance it's class definition looks like
 ```
 class ClockifyModule(
-    PypeModule, # Says it's Pype module so ModulesManager will try to initialize.
+    OpenPypeModule, # Says it's Pype module so ModulesManager will try to initialize.
     ITrayModule, # Says has special implementation when used in tray.
     IPluginPaths, # Says has plugin paths that want to register (paths to clockify actions for launcher).
     IFtrackEventHandlerPaths, # Says has Ftrack actions/events for user/server.

openpype/modules/__init__.py

@@ -1,86 +1,21 @@
 # -*- coding: utf-8 -*-
 from .base import (
-    PypeModule,
-    ITrayModule,
-    ITrayAction,
-    ITrayService,
-    IPluginPaths,
-    ILaunchHookPaths,
+    OpenPypeModule,
+    OpenPypeInterface,
+
+    load_modules,
+
     ModulesManager,
     TrayModulesManager
 )
-from .settings_action import (
-    SettingsAction,
-    ISettingsChangeListener,
-    LocalSettingsAction
-)
-from .webserver import (
-    WebServerModule,
-    IWebServerRoutes
-)
-from .idle_manager import (
-    IdleManager,
-    IIdleManager
-)
-from .timers_manager import (
-    TimersManager,
-    ITimersManager
-)
-from .avalon_apps import AvalonModule
-from .launcher_action import LauncherAction
-from .ftrack import (
-    FtrackModule,
-    IFtrackEventHandlerPaths
-)
-from .clockify import ClockifyModule
-from .log_viewer import LogViewModule
-from .muster import MusterModule
-from .deadline import DeadlineModule
-from .project_manager_action import ProjectManagerAction
-from .standalonepublish_action import StandAlonePublishAction
-from .python_console_interpreter import PythonInterpreterAction
-from .sync_server import SyncServerModule
-from .slack import SlackIntegrationModule
 
 
 __all__ = (
-    "PypeModule",
-    "ITrayModule",
-    "ITrayAction",
-    "ITrayService",
-    "IPluginPaths",
-    "ILaunchHookPaths",
-    "ModulesManager",
-    "TrayModulesManager",
+    "OpenPypeModule",
+    "OpenPypeInterface",
 
-    "SettingsAction",
-    "LocalSettingsAction",
+    "load_modules",
 
-    "WebServerModule",
-    "IWebServerRoutes",
-
-    "IdleManager",
-    "IIdleManager",
-
-    "TimersManager",
-    "ITimersManager",
-
-    "AvalonModule",
-    "LauncherAction",
-
-    "FtrackModule",
-    "IFtrackEventHandlerPaths",
-
-    "ClockifyModule",
-    "IdleManager",
-    "LogViewModule",
-    "MusterModule",
-    "DeadlineModule",
-    "ProjectManagerAction",
-    "StandAlonePublishAction",
-    "PythonInterpreterAction",
-
-    "SyncServerModule",
-
-    "SlackIntegrationModule"
+    "ModulesManager",
+    "TrayModulesManager"
 )