Merge pull request #3527 from pypeclub/feature/OP-3593_Move-load-functions-into-pipeline

General: Move load related functions into pipeline

openpype/client/__init__.py

@@ -25,6 +25,8 @@ from .entities import (
     get_last_version_by_subset_name,
     get_output_link_versions,
 
+    version_is_latest,
+
     get_representation_by_id,
     get_representation_by_name,
     get_representations,
@@ -66,6 +68,8 @@ __all__ = (
     "get_last_version_by_subset_name",
     "get_output_link_versions",
 
+    "version_is_latest",
+
     "get_representation_by_id",
     "get_representation_by_name",
     "get_representations",

openpype/client/entities.py

@@ -561,6 +561,42 @@ def get_version_by_name(project_name, version, subset_id, fields=None):
     return conn.find_one(query_filter, _prepare_fields(fields))
 
 
+def version_is_latest(project_name, version_id):
+    """Is version the latest from it's subset.
+
+    Note:
+        Hero versions are considered as latest.
+
+    Todo:
+        Maybe raise exception when version was not found?
+
+    Args:
+        project_name (str):Name of project where to look for queried entities.
+        version_id (Union[str, ObjectId]): Version id which is checked.
+
+    Returns:
+        bool: True if is latest version from subset else False.
+    """
+
+    version_id = _convert_id(version_id)
+    if not version_id:
+        return False
+    version_doc = get_version_by_id(
+        project_name, version_id, fields=["_id", "type", "parent"]
+    )
+    # What to do when version is not found?
+    if not version_doc:
+        return False
+
+    if version_doc["type"] == "hero_version":
+        return True
+
+    last_version = get_last_version_by_subset_id(
+        project_name, version_doc["parent"], fields=["_id"]
+    )
+    return last_version["_id"] == version_id
+
+
 def _get_versions(
     project_name,
     subset_ids=None,

openpype/hosts/aftereffects/api/pipeline.py

@@ -1,5 +1,4 @@
 import os
-import sys
 
 from Qt import QtWidgets
 
@@ -15,6 +14,7 @@ from openpype.pipeline import (
     AVALON_CONTAINER_ID,
     legacy_io,
 )
+from openpype.pipeline.load import any_outdated_containers
 import openpype.hosts.aftereffects
 from openpype.lib import register_event_callback
 
@@ -136,7 +136,7 @@ def ls():
 
 def check_inventory():
     """Checks loaded containers if they are of highest version"""
-    if not lib.any_outdated():
+    if not any_outdated_containers():
         return
 
     # Warn about outdated containers.

openpype/hosts/harmony/api/pipeline.py

@@ -4,17 +4,15 @@ import logging
 
 import pyblish.api
 
-from openpype import lib
-from openpype.client import get_representation_by_id
 from openpype.lib import register_event_callback
 from openpype.pipeline import (
-    legacy_io,
     register_loader_plugin_path,
     register_creator_plugin_path,
     deregister_loader_plugin_path,
     deregister_creator_plugin_path,
     AVALON_CONTAINER_ID,
 )
+from openpype.pipeline.load import get_outdated_containers
 from openpype.pipeline.context_tools import get_current_project_asset
 import openpype.hosts.harmony
 import openpype.hosts.harmony.api as harmony
@@ -108,16 +106,7 @@ def check_inventory():
     in Harmony.
     """
 
-    project_name = legacy_io.active_project()
-    outdated_containers = []
-    for container in ls():
-        representation_id = container['representation']
-        representation_doc = get_representation_by_id(
-            project_name, representation_id, fields=["parent"]
-        )
-        if representation_doc and not lib.is_latest(representation_doc):
-            outdated_containers.append(container)
-
+    outdated_containers = get_outdated_containers()
     if not outdated_containers:
         return
 

openpype/hosts/harmony/plugins/load/load_background.py

@@ -5,8 +5,8 @@ from openpype.pipeline import (
     load,
     get_representation_path,
 )
+from openpype.pipeline.context_tools import is_representation_from_latest
 import openpype.hosts.harmony.api as harmony
-import openpype.lib
 
 
 copy_files = """function copyFile(srcFilename, dstFilename)
@@ -280,9 +280,7 @@ class BackgroundLoader(load.LoaderPlugin):
         )
 
     def update(self, container, representation):
-
         path = get_representation_path(representation)
-
         with open(path) as json_file:
             data = json.load(json_file)
 
@@ -300,10 +298,9 @@ class BackgroundLoader(load.LoaderPlugin):
 
         bg_folder = os.path.dirname(path)
 
-        path = get_representation_path(representation)
-
         print(container)
 
+        is_latest = is_representation_from_latest(representation)
         for layer in sorted(layers):
             file_to_import = [
                 os.path.join(bg_folder, layer).replace("\\", "/")
@@ -347,7 +344,7 @@ class BackgroundLoader(load.LoaderPlugin):
             }
             %s
             """ % (sig, sig)
-            if openpype.lib.is_latest(representation):
+            if is_latest:
                 harmony.send({"function": func, "args": [node, "green"]})
             else:
                 harmony.send({"function": func, "args": [node, "red"]})

openpype/hosts/harmony/plugins/load/load_imagesequence.py

@@ -10,8 +10,8 @@ from openpype.pipeline import (
     load,
     get_representation_path,
 )
+from openpype.pipeline.context_tools import is_representation_from_latest
 import openpype.hosts.harmony.api as harmony
-import openpype.lib
 
 
 class ImageSequenceLoader(load.LoaderPlugin):
@@ -109,7 +109,7 @@ class ImageSequenceLoader(load.LoaderPlugin):
         )
 
         # Colour node.
-        if openpype.lib.is_latest(representation):
+        if is_representation_from_latest(representation):
             harmony.send(
                 {
                     "function": "PypeHarmony.setColor",

openpype/hosts/harmony/plugins/load/load_template.py

@@ -10,8 +10,8 @@ from openpype.pipeline import (
     load,
     get_representation_path,
 )
+from openpype.pipeline.context_tools import is_representation_from_latest
 import openpype.hosts.harmony.api as harmony
-import openpype.lib
 
 
 class TemplateLoader(load.LoaderPlugin):
@@ -83,7 +83,7 @@ class TemplateLoader(load.LoaderPlugin):
         self_name = self.__class__.__name__
 
         update_and_replace = False
-        if openpype.lib.is_latest(representation):
+        if is_representation_from_latest(representation):
             self._set_green(node)
         else:
             self._set_red(node)

openpype/hosts/houdini/api/pipeline.py

@@ -12,13 +12,13 @@ from openpype.pipeline import (
     register_loader_plugin_path,
     AVALON_CONTAINER_ID,
 )
+from openpype.pipeline.load import any_outdated_containers
 import openpype.hosts.houdini
 from openpype.hosts.houdini.api import lib
 
 from openpype.lib import (
     register_event_callback,
     emit_event,
-    any_outdated,
 )
 
 from .lib import get_asset_fps
@@ -245,7 +245,7 @@ def on_open():
     # ensure it is using correct FPS for the asset
     lib.validate_fps()
 
-    if any_outdated():
+    if any_outdated_containers():
         from openpype.widgets import popup
 
         log.warning("Scene has outdated content.")

openpype/hosts/maya/api/pipeline.py

@@ -13,7 +13,6 @@ from openpype.host import HostBase, IWorkfileHost, ILoadHost
 import openpype.hosts.maya
 from openpype.tools.utils import host_tools
 from openpype.lib import (
-    any_outdated,
     register_event_callback,
     emit_event
 )
@@ -28,6 +27,7 @@ from openpype.pipeline import (
     deregister_creator_plugin_path,
     AVALON_CONTAINER_ID,
 )
+from openpype.pipeline.load import any_outdated_containers
 from openpype.hosts.maya.lib import copy_workspace_mel
 from . import menu, lib
 from .workio import (
@@ -470,7 +470,7 @@ def on_open():
     lib.validate_fps()
     lib.fix_incompatible_containers()
 
-    if any_outdated():
+    if any_outdated_containers():
         log.warning("Scene has outdated content.")
 
         # Find maya main window

openpype/hosts/photoshop/api/pipeline.py

@@ -1,6 +1,5 @@
 import os
 from Qt import QtWidgets
-from bson.objectid import ObjectId
 
 import pyblish.api
 
@@ -13,8 +12,8 @@ from openpype.pipeline import (
     deregister_loader_plugin_path,
     deregister_creator_plugin_path,
     AVALON_CONTAINER_ID,
-    registered_host,
 )
+from openpype.pipeline.load import any_outdated_containers
 import openpype.hosts.photoshop
 
 from . import lib
@@ -30,7 +29,7 @@ INVENTORY_PATH = os.path.join(PLUGINS_DIR, "inventory")
 
 
 def check_inventory():
-    if not lib.any_outdated():
+    if not any_outdated_containers():
         return
 
     # Warn about outdated containers.

openpype/lib/avalon_context.py

@@ -15,11 +15,10 @@ from openpype.client import (
     get_asset_by_name,
     get_subset_by_name,
     get_subsets,
-    get_version_by_id,
     get_last_versions,
     get_last_version_by_subset_id,
+    get_last_version_by_subset_name,
     get_representations,
-    get_representation_by_id,
     get_workfile_info,
 )
 from openpype.settings import (
@@ -180,7 +179,7 @@ def with_pipeline_io(func):
     return wrapped
 
 
-@with_pipeline_io
+@deprecated("openpype.pipeline.context_tools.is_representation_from_latest")
 def is_latest(representation):
     """Return whether the representation is from latest version
 
@@ -191,49 +190,18 @@ def is_latest(representation):
         bool: Whether the representation is of latest version.
     """
 
-    project_name = legacy_io.active_project()
-    version = get_version_by_id(
-        project_name,
-        representation["parent"],
-        fields=["_id", "type", "parent"]
-    )
-    if version["type"] == "hero_version":
-        return True
+    from openpype.pipeline.context_tools import is_representation_from_latest
 
-    # Get highest version under the parent
-    last_version = get_last_version_by_subset_id(
-        project_name, version["parent"], fields=["_id"]
-    )
-
-    return version["_id"] == last_version["_id"]
+    return is_representation_from_latest(representation)
 
 
-@with_pipeline_io
+@deprecated("openpype.pipeline.load.any_outdated_containers")
 def any_outdated():
     """Return whether the current scene has any outdated content"""
-    from openpype.pipeline import registered_host
 
-    project_name = legacy_io.active_project()
-    checked = set()
-    host = registered_host()
-    for container in host.ls():
-        representation = container['representation']
-        if representation in checked:
-            continue
+    from openpype.pipeline.load import any_outdated_containers
 
-        representation_doc = get_representation_by_id(
-            project_name, representation, fields=["parent"]
-        )
-        if representation_doc and not is_latest(representation_doc):
-            return True
-        elif not representation_doc:
-            log.debug("Container '{objectName}' has an invalid "
-                      "representation, it is missing in the "
-                      "database".format(**container))
-
-        checked.add(representation)
-
-    return False
+    return any_outdated_containers()
 
 
 @deprecated("openpype.pipeline.context_tools.get_current_project_asset")
@@ -313,7 +281,7 @@ def get_linked_assets(asset_doc):
     return list(get_assets(project_name, link_ids))
 
 
-@with_pipeline_io
+@deprecated("openpype.client.get_last_version_by_subset_name")
 def get_latest_version(asset_name, subset_name, dbcon=None, project_name=None):
     """Retrieve latest version from `asset_name`, and `subset_name`.
 
@@ -334,6 +302,8 @@ def get_latest_version(asset_name, subset_name, dbcon=None, project_name=None):
 
     if not project_name:
         if not dbcon:
+            from openpype.pipeline import legacy_io
+
             log.debug("Using `legacy_io` for query.")
             dbcon = legacy_io
             # Make sure is installed
@@ -341,37 +311,9 @@ def get_latest_version(asset_name, subset_name, dbcon=None, project_name=None):
 
         project_name = dbcon.active_project()
 
-    log.debug((
-        "Getting latest version for Project: \"{}\" Asset: \"{}\""
-        " and Subset: \"{}\""
-    ).format(project_name, asset_name, subset_name))
-
-    # Query asset document id by asset name
-    asset_doc = get_asset_by_name(project_name, asset_name, fields=["_id"])
-    if not asset_doc:
-        log.info(
-            "Asset \"{}\" was not found in Database.".format(asset_name)
-        )
-        return None
-
-    subset_doc = get_subset_by_name(
-        project_name, subset_name, asset_doc["_id"]
+    return get_last_version_by_subset_name(
+        project_name, subset_name, asset_name=asset_name
     )
-    if not subset_doc:
-        log.info(
-            "Subset \"{}\" was not found in Database.".format(subset_name)
-        )
-        return None
-
-    version_doc = get_last_version_by_subset_id(
-        project_name, subset_doc["_id"]
-    )
-    if not version_doc:
-        log.info(
-            "Subset \"{}\" does not have any version yet.".format(subset_name)
-        )
-        return None
-    return version_doc
 
 
 def get_workfile_template_key_from_context(

openpype/modules/deadline/plugins/publish/submit_publish_job.py

@@ -10,8 +10,10 @@ import clique
 
 import pyblish.api
 
-import openpype.api
-from openpype.client import get_representations
+from openpype.client import (
+    get_last_version_by_subset_name,
+    get_representations,
+)
 from openpype.pipeline import (
     get_representation_path,
     legacy_io,
@@ -343,8 +345,13 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
 
         # get latest version of subset
         # this will stop if subset wasn't published yet
-        version = openpype.api.get_latest_version(instance.data.get("asset"),
-                                                  instance.data.get("subset"))
+        project_name = legacy_io.active_project()
+        version = get_last_version_by_subset_name(
+            project_name,
+            instance.data.get("subset"),
+            asset_name=instance.data.get("asset")
+        )
+
         # get its files based on extension
         subset_resources = get_resources(
             project_name, version, representation.get("ext")
@@ -1025,9 +1032,12 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
         prev_start = None
         prev_end = None
 
-        version = openpype.api.get_latest_version(asset_name=asset,
-                                                  subset_name=subset
-                                                  )
+        project_name = legacy_io.active_project()
+        version = get_last_version_by_subset_name(
+            project_name,
+            subset,
+            asset_name=asset
+        )
 
         # Set prev start / end frames for comparison
         if not prev_start and not prev_end:
@@ -1072,7 +1082,12 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
                 based on 'publish' template
         """
         if not version:
-            version = openpype.api.get_latest_version(asset, subset)
+            project_name = legacy_io.active_project()
+            version = get_last_version_by_subset_name(
+                project_name,
+                subset,
+                asset_name=asset
+            )
             if version:
                 version = int(version["name"]) + 1
             else:

openpype/pipeline/context_tools.py

@@ -14,6 +14,7 @@ from openpype.client import (
     get_project,
     get_asset_by_id,
     get_asset_by_name,
+    version_is_latest,
 )
 from openpype.modules import load_modules, ModulesManager
 from openpype.settings import get_project_settings
@@ -334,3 +335,16 @@ def get_current_project_asset(asset_name=None, asset_id=None, fields=None):
         if not asset_name:
             return None
     return get_asset_by_name(project_name, asset_name, fields=fields)
+
+def is_representation_from_latest(representation):
+    """Return whether the representation is from latest version
+
+    Args:
+        representation (dict): The representation document from the database.
+
+    Returns:
+        bool: Whether the representation is of latest version.
+    """
+
+    project_name = legacy_io.active_project()
+    return version_is_latest(project_name, representation["parent"])

openpype/pipeline/load/__init__.py

@@ -24,6 +24,10 @@ from .utils import (
 
     loaders_from_repre_context,
     loaders_from_representation,
+
+    any_outdated_containers,
+    get_outdated_containers,
+    filter_containers,
 )
 
 from .plugins import (
@@ -66,6 +70,10 @@ __all__ = (
     "loaders_from_repre_context",
     "loaders_from_representation",
 
+    "any_outdated_containers",
+    "get_outdated_containers",
+    "filter_containers",
+
     # plugins.py
     "LoaderPlugin",
     "SubsetLoaderPlugin",

openpype/pipeline/load/utils.py

@@ -4,8 +4,10 @@ import copy
 import getpass
 import logging
 import inspect
+import collections
 import numbers
 
+from openpype.host import ILoadHost
 from openpype.client import (
     get_project,
     get_assets,
@@ -15,6 +17,7 @@ from openpype.client import (
     get_last_version_by_subset_id,
     get_hero_version_by_subset_id,
     get_version_by_name,
+    get_last_versions,
     get_representations,
     get_representation_by_id,
     get_representation_by_name,
@@ -28,6 +31,11 @@ from openpype.pipeline import (
 
 log = logging.getLogger(__name__)
 
+ContainersFilterResult = collections.namedtuple(
+    "ContainersFilterResult",
+    ["latest", "outdated", "not_foud", "invalid"]
+)
+
 
 class HeroVersionType(object):
     def __init__(self, version):
@@ -685,3 +693,164 @@ def loaders_from_representation(loaders, representation):
 
     context = get_representation_context(representation)
     return loaders_from_repre_context(loaders, context)
+
+
+def any_outdated_containers(host=None, project_name=None):
+    """Check if there are any outdated containers in scene."""
+
+    if get_outdated_containers(host, project_name):
+        return True
+    return False
+
+
+def get_outdated_containers(host=None, project_name=None):
+    """Collect outdated containers from host scene.
+
+    Currently registered host and project in global session are used if
+    arguments are not passed.
+
+    Args:
+        host (ModuleType): Host implementation with 'ls' function available.
+        project_name (str): Name of project in which context we are.
+    """
+
+    if host is None:
+        from openpype.pipeline import registered_host
+        host = registered_host()
+
+    if project_name is None:
+        project_name = legacy_io.active_project()
+
+    if isinstance(host, ILoadHost):
+        containers = host.get_containers()
+    else:
+        containers = host.ls()
+    return filter_containers(containers, project_name).outdated
+
+
+def filter_containers(containers, project_name):
+    """Filter containers and split them into 4 categories.
+
+    Categories are 'latest', 'outdated', 'invalid' and 'not_found'.
+    The 'lastest' containers are from last version, 'outdated' are not,
+    'invalid' are invalid containers (invalid content) and 'not_foud' has
+    some missing entity in database.
+
+    Args:
+        containers (Iterable[dict]): List of containers referenced into scene.
+        project_name (str): Name of project in which context shoud look for
+            versions.
+
+    Returns:
+        ContainersFilterResult: Named tuple with 'latest', 'outdated',
+            'invalid' and 'not_found' containers.
+    """
+
+    # Make sure containers is list that won't change
+    containers = list(containers)
+
+    outdated_containers = []
+    uptodate_containers = []
+    not_found_containers = []
+    invalid_containers = []
+    output = ContainersFilterResult(
+        uptodate_containers,
+        outdated_containers,
+        not_found_containers,
+        invalid_containers
+    )
+    # Query representation docs to get it's version ids
+    repre_ids = {
+        container["representation"]
+        for container in containers
+        if container["representation"]
+    }
+    if not repre_ids:
+        if containers:
+            invalid_containers.extend(containers)
+        return output
+
+    repre_docs = get_representations(
+        project_name,
+        representation_ids=repre_ids,
+        fields=["_id", "parent"]
+    )
+    # Store representations by stringified representation id
+    repre_docs_by_str_id = {}
+    repre_docs_by_version_id = collections.defaultdict(list)
+    for repre_doc in repre_docs:
+        repre_id = str(repre_doc["_id"])
+        version_id = repre_doc["parent"]
+        repre_docs_by_str_id[repre_id] = repre_doc
+        repre_docs_by_version_id[version_id].append(repre_doc)
+
+    # Query version docs to get it's subset ids
+    # - also query hero version to be able identify if representation
+    #   belongs to existing version
+    version_docs = get_versions(
+        project_name,
+        version_ids=repre_docs_by_version_id.keys(),
+        hero=True,
+        fields=["_id", "parent", "type"]
+    )
+    verisons_by_id = {}
+    versions_by_subset_id = collections.defaultdict(list)
+    hero_version_ids = set()
+    for version_doc in version_docs:
+        version_id = version_doc["_id"]
+        # Store versions by their ids
+        verisons_by_id[version_id] = version_doc
+        # There's no need to query subsets for hero versions
+        #   - they are considered as latest?
+        if version_doc["type"] == "hero_version":
+            hero_version_ids.add(version_id)
+            continue
+        subset_id = version_doc["parent"]
+        versions_by_subset_id[subset_id].append(version_doc)
+
+    last_versions = get_last_versions(
+        project_name,
+        subset_ids=versions_by_subset_id.keys(),
+        fields=["_id"]
+    )
+    # Figure out which versions are outdated
+    outdated_version_ids = set()
+    for subset_id, last_version_doc in last_versions.items():
+        for version_doc in versions_by_subset_id[subset_id]:
+            version_id = version_doc["_id"]
+            if version_id != last_version_doc["_id"]:
+                outdated_version_ids.add(version_id)
+
+    # Based on all collected data figure out which containers are outdated
+    #   - log out if there are missing representation or version documents
+    for container in containers:
+        container_name = container["objectName"]
+        repre_id = container["representation"]
+        if not repre_id:
+            invalid_containers.append(container)
+            continue
+
+        repre_doc = repre_docs_by_str_id.get(repre_id)
+        if not repre_doc:
+            log.debug((
+                "Container '{}' has an invalid representation."
+                " It is missing in the database."
+            ).format(container_name))
+            not_found_containers.append(container)
+            continue
+
+        version_id = repre_doc["parent"]
+        if version_id in outdated_version_ids:
+            outdated_containers.append(container)
+
+        elif version_id not in verisons_by_id:
+            log.debug((
+                "Representation on container '{}' has an invalid version."
+                " It is missing in the database."
+            ).format(container_name))
+            not_found_containers.append(container)
+
+        else:
+            uptodate_containers.append(container)
+
+    return output

openpype/plugins/publish/validate_containers.py

@@ -1,5 +1,5 @@
 import pyblish.api
-import openpype.lib
+from openpype.pipeline.load import any_outdated_containers
 
 
 class ShowInventory(pyblish.api.Action):
@@ -19,10 +19,10 @@ class ValidateContainers(pyblish.api.ContextPlugin):
 
     label = "Validate Containers"
     order = pyblish.api.ValidatorOrder
-    hosts = ["maya", "houdini", "nuke", "harmony", "photoshop"]
+    hosts = ["maya", "houdini", "nuke", "harmony", "photoshop", "aftereffects"]
     optional = True
     actions = [ShowInventory]
 
     def process(self, context):
-        if openpype.lib.any_outdated():
+        if any_outdated_containers():
             raise ValueError("There are outdated containers in the scene.")

openpype/tests/test_lib_restructuralization.py

@@ -22,7 +22,6 @@ def test_backward_compatibility(printer):
         from openpype.lib import any_outdated
         from openpype.lib import get_asset
         from openpype.lib import get_linked_assets
-        from openpype.lib import get_latest_version
         from openpype.lib import get_ffprobe_streams
 
         from openpype.hosts.fusion.lib import switch_item