Merge branch 'bugfix/fix-publish-plugin-loading2' into feature/911-new-traits-based-integrator

client/ayon_core/lib/python_module_tools.py

@@ -1,6 +1,8 @@
+"""Tools for working with python modules and classes."""
 import os
 import sys
 import types
+from typing import Optional
 import importlib
 import inspect
 import logging
@@ -8,13 +10,22 @@ import logging
 log = logging.getLogger(__name__)
 
 
-def import_filepath(filepath, module_name=None):
+def import_filepath(
+        filepath: str,
+        module_name: Optional[str] = None,
+        sys_module_name: Optional[str] = None) -> types.ModuleType:
     """Import python file as python module.
 
     Args:
         filepath (str): Path to python file.
         module_name (str): Name of loaded module. Only for Python 3. By default
             is filled with filename of filepath.
+        sys_module_name (str): Name of module in `sys.modules` where to store
+            loaded module. By default is None so module is not added to
+            `sys.modules`.
+
+    Todo (antirotor): We should add the module to the sys.modules always but
+        we need to be careful about it and test it properly.
 
     """
     if module_name is None:
@@ -28,6 +39,9 @@ def import_filepath(filepath, module_name=None):
     module_loader = importlib.machinery.SourceFileLoader(
         module_name, filepath
     )
+    # only add to sys.modules if requested
+    if sys_module_name:
+        sys.modules[sys_module_name] = module
     module_loader.exec_module(module)
     return module
 
@@ -126,7 +140,8 @@ def classes_from_module(superclass, module):
     return classes
 
 
-def import_module_from_dirpath(dirpath, folder_name, dst_module_name=None):
+def import_module_from_dirpath(
+        dirpath, folder_name, dst_module_name=None):
     """Import passed directory as a python module.
 
     Imported module can be assigned as a child attribute of already loaded
@@ -193,7 +208,7 @@ def is_func_signature_supported(func, *args, **kwargs):
     Notes:
         This does NOT check if the function would work with passed arguments
             only if they can be passed in. If function have *args, **kwargs
-            in paramaters, this will always return 'True'.
+            in parameters, this will always return 'True'.
 
     Example:
         >>> def my_function(my_number):

client/ayon_core/pipeline/farm/pyblish_functions.py

@@ -808,14 +808,14 @@ def _create_instances_for_aov(
             frames_to_render is not None
             and isinstance(collected_files, (list, tuple))  # not single file
         ):
-            frames_to_render = convert_frames_str_to_list(frames_to_render)
+            aov_frames_to_render = convert_frames_str_to_list(frames_to_render)
             collections, _ = clique.assemble(collected_files)
             collected_files = _get_real_files_to_render(
-                collections[0], frames_to_render)
+                collections[0], aov_frames_to_render)
         else:
             frame_start = int(skeleton.get("frameStartHandle"))
             frame_end = int(skeleton.get("frameEndHandle"))
-            frames_to_render = list(range(frame_start, frame_end + 1))
+            aov_frames_to_render = list(range(frame_start, frame_end + 1))
 
         dynamic_data = {
             "aov": aov,
@@ -937,8 +937,8 @@ def _create_instances_for_aov(
             "name": ext,
             "ext": ext,
             "files": collected_files,
-            "frameStart": frames_to_render[0],
-            "frameEnd": frames_to_render[-1],
+            "frameStart": aov_frames_to_render[0],
+            "frameEnd": aov_frames_to_render[-1],
             # If expectedFile are absolute, we need only filenames
             "stagingDir": staging_dir,
             "fps": new_instance.get("fps"),

client/ayon_core/pipeline/project_folders.py

@@ -1,6 +1,8 @@
+from __future__ import annotations
 import os
 import re
 import json
+from typing import Any, Union
 
 from ayon_core.settings import get_project_settings
 from ayon_core.lib import Logger
@@ -9,7 +11,7 @@ from .anatomy import Anatomy
 from .template_data import get_project_template_data
 
 
-def concatenate_splitted_paths(split_paths, anatomy):
+def concatenate_splitted_paths(split_paths, anatomy: Anatomy):
     log = Logger.get_logger("concatenate_splitted_paths")
     pattern_array = re.compile(r"\[.*\]")
     output = []
@@ -47,7 +49,7 @@ def concatenate_splitted_paths(split_paths, anatomy):
     return output
 
 
-def fill_paths(path_list, anatomy):
+def fill_paths(path_list: list[str], anatomy: Anatomy):
     format_data = get_project_template_data(project_name=anatomy.project_name)
     format_data["root"] = anatomy.roots
     filled_paths = []
@@ -59,7 +61,7 @@ def fill_paths(path_list, anatomy):
     return filled_paths
 
 
-def create_project_folders(project_name, basic_paths=None):
+def create_project_folders(project_name: str, basic_paths=None):
     log = Logger.get_logger("create_project_folders")
     anatomy = Anatomy(project_name)
     if basic_paths is None:
@@ -80,8 +82,19 @@ def create_project_folders(project_name, basic_paths=None):
             os.makedirs(path)
 
 
-def _list_path_items(folder_structure):
+def _list_path_items(
+        folder_structure: Union[dict[str, Any], list[str]]):
     output = []
+
+    # Allow leaf folders of the `project_folder_structure` to use a list of
+    # strings instead of a dictionary of keys with empty values.
+    if isinstance(folder_structure, list):
+        if not all(isinstance(item, str) for item in folder_structure):
+            raise ValueError(
+                f"List items must all be strings. Got: {folder_structure}")
+        return [[path] for path in folder_structure]
+
+    # Process key, value as key for folder names and value its subfolders
     for key, value in folder_structure.items():
         if not value:
             output.append(key)
@@ -99,7 +112,7 @@ def _list_path_items(folder_structure):
     return output
 
 
-def get_project_basic_paths(project_name):
+def get_project_basic_paths(project_name: str):
     project_settings = get_project_settings(project_name)
     folder_structure = (
         project_settings["core"]["project_folder_structure"]

client/ayon_core/pipeline/publish/lib.py

@@ -1,3 +1,5 @@
+"""Library functions for publishing."""
+from __future__ import annotations
 import os
 import sys
 import inspect
@@ -12,8 +14,8 @@ import pyblish.plugin
 import pyblish.api
 
 from ayon_core.lib import (
-    Logger,
     import_filepath,
+    Logger,
     filter_profiles,
 )
 from ayon_core.settings import get_project_settings
@@ -163,7 +165,7 @@ class HelpContent:
 
 def load_help_content_from_filepath(filepath):
     """Load help content from xml file.
-    Xml file may containt errors and warnings.
+    Xml file may contain errors and warnings.
     """
     errors = {}
     warnings = {}
@@ -208,8 +210,9 @@ def load_help_content_from_plugin(plugin):
     return load_help_content_from_filepath(filepath)
 
 
-def publish_plugins_discover(paths=None):
-    """Find and return available pyblish plug-ins
+def publish_plugins_discover(
+        paths: Optional[list[str]] = None) -> DiscoverResult:
+    """Find and return available pyblish plug-ins.
 
     Overridden function from `pyblish` module to be able to collect
         crashed files and reason of their crash.
@@ -252,17 +255,14 @@ def publish_plugins_discover(paths=None):
                 continue
 
             try:
-                module = import_filepath(abspath, mod_name)
+                module = import_filepath(
+                    abspath, mod_name, sys_module_name=mod_name)
 
-                # Store reference to original module, to avoid
-                # garbage collection from collecting it's global
-                # imports, such as `import os`.
-                sys.modules[abspath] = module
-
-            except Exception as err:
+            except Exception as err:  # noqa: BLE001
+                # we need broad exception to catch all possible errors.
                 result.crashed_file_paths[abspath] = sys.exc_info()
 
-                log.debug("Skipped: \"%s\" (%s)", mod_name, err)
+                log.debug('Skipped: "%s" (%s)', mod_name, err)
                 continue
 
             for plugin in pyblish.plugin.plugins_from_module(module):
@@ -280,9 +280,8 @@ def publish_plugins_discover(paths=None):
                     continue
 
                 plugin_names.append(plugin.__name__)
-
-                plugin.__module__ = module.__file__
-                key = "{0}.{1}".format(plugin.__module__, plugin.__name__)
+                plugin.__file__ = module.__file__
+                key = f"{module.__file__}.{plugin.__name__}"
                 plugins[key] = plugin
 
     # Include plug-ins from registration.
@@ -361,7 +360,7 @@ def get_plugin_settings(plugin, project_settings, log, category=None):
     # Settings category determined from path
     # - usually path is './<category>/plugins/publish/<plugin file>'
     # - category can be host name of addon name ('maya', 'deadline', ...)
-    filepath = os.path.normpath(inspect.getsourcefile(plugin))
+    filepath = os.path.normpath(inspect.getfile(plugin))
 
     split_path = filepath.rsplit(os.path.sep, 5)
     if len(split_path) < 4:
@@ -427,7 +426,7 @@ def filter_pyblish_plugins(plugins):
     log = Logger.get_logger("filter_pyblish_plugins")
 
     # TODO: Don't use host from 'pyblish.api' but from defined host by us.
-    #   - kept becau on farm is probably used host 'shell' which propably
+    #   - kept because on farm is probably used host 'shell' which probably
     #       affect how settings are applied there
     host_name = pyblish.api.current_host()
     project_name = os.environ.get("AYON_PROJECT_NAME")
@@ -529,7 +528,7 @@ def filter_instances_for_context_plugin(plugin, context):
 
     Args:
         plugin (pyblish.api.Plugin): Plugin with filters.
-        context (pyblish.api.Context): Pyblish context with insances.
+        context (pyblish.api.Context): Pyblish context with instances.
 
     Returns:
         Iterator[pyblish.lib.Instance]: Iteration of valid instances.

client/ayon_core/version.py

@@ -1,3 +1,3 @@
 # -*- coding: utf-8 -*-
 """Package declaring AYON addon 'core' version."""
-__version__ = "1.1.4+dev"
+__version__ = "1.1.5+dev"

package.py

@@ -1,6 +1,6 @@
 name = "core"
 title = "Core"
-version = "1.1.4+dev"
+version = "1.1.5+dev"
 
 client_dir = "ayon_core"
 

pyproject.toml

@@ -5,7 +5,7 @@
 
 [tool.poetry]
 name = "ayon-core"
-version = "1.1.4+dev"
+version = "1.1.5+dev"
 description = ""
 authors = ["Ynput Team <team@ynput.io>"]
 readme = "README.md"