Merge branch 'develop' into enhancement/fix_delivery_action

client/ayon_core/hosts/maya/api/lib.py

@@ -1519,24 +1519,30 @@ def extract_alembic(file,
 
 
 # region ID
-def get_id_required_nodes(referenced_nodes=False, nodes=None):
-    """Filter out any node which are locked (reference) or readOnly
+def get_id_required_nodes(referenced_nodes=False,
+                          nodes=None,
+                          existing_ids=True):
+    """Return nodes that should receive a `cbId` attribute.
+
+    This includes only mesh and curve nodes, parent transforms of the shape
+    nodes, file texture nodes and object sets (including shading engines).
+
+    This filters out any node which is locked, referenced, read-only,
+    intermediate object.
 
     Args:
-        referenced_nodes (bool): set True to filter out reference nodes
+        referenced_nodes (bool): set True to include referenced nodes
         nodes (list, Optional): nodes to consider
+        existing_ids (bool): set True to include nodes with `cbId` attribute
+
     Returns:
         nodes (set): list of filtered nodes
     """
 
-    lookup = None
-    if nodes is None:
-        # Consider all nodes
-        nodes = cmds.ls()
-    else:
-        # Build a lookup for the only allowed nodes in output based
-        # on `nodes` input of the function (+ ensure long names)
-        lookup = set(cmds.ls(nodes, long=True))
+    if nodes is not None and not nodes:
+        # User supplied an empty `nodes` list to check so all we can
+        # do is return the empty result
+        return set()
 
     def _node_type_exists(node_type):
         try:
@@ -1545,63 +1551,142 @@ def get_id_required_nodes(referenced_nodes=False, nodes=None):
         except RuntimeError:
             return False
 
+    def iterate(maya_iterator):
+        while not maya_iterator.isDone():
+            yield maya_iterator.thisNode()
+            maya_iterator.next()
+
     # `readOnly` flag is obsolete as of Maya 2016 therefore we explicitly
     # remove default nodes and reference nodes
-    camera_shapes = ["frontShape", "sideShape", "topShape", "perspShape"]
+    default_camera_shapes = {
+        "frontShape", "sideShape", "topShape", "perspShape"
+    }
 
-    ignore = set()
-    if not referenced_nodes:
-        ignore |= set(cmds.ls(long=True, referencedNodes=True))
-
-    # list all defaultNodes to filter out from the rest
-    ignore |= set(cmds.ls(long=True, defaultNodes=True))
-    ignore |= set(cmds.ls(camera_shapes, long=True))
-
-    # Remove Turtle from the result of `cmds.ls` if Turtle is loaded
-    # TODO: This should be a less specific check for a single plug-in.
-    if _node_type_exists("ilrBakeLayer"):
-        ignore |= set(cmds.ls(type="ilrBakeLayer", long=True))
-
-    # Establish set of nodes types to include
-    types = ["objectSet", "file", "mesh", "nurbsCurve", "nurbsSurface"]
+    # The filtered types do not include transforms because we only want the
+    # parent transforms that have a child shape that we filtered to, so we
+    # include the parents here
+    types = ["mesh", "nurbsCurve", "nurbsSurface", "file", "objectSet"]
 
     # Check if plugin nodes are available for Maya by checking if the plugin
     # is loaded
     if cmds.pluginInfo("pgYetiMaya", query=True, loaded=True):
         types.append("pgYetiMaya")
 
-    # We *always* ignore intermediate shapes, so we filter them out directly
-    nodes = cmds.ls(nodes, type=types, long=True, noIntermediate=True)
+    iterator_type = OpenMaya.MIteratorType()
+    # This tries to be closest matching API equivalents of `types` variable
+    iterator_type.filterList = [
+        OpenMaya.MFn.kMesh,  # mesh
+        OpenMaya.MFn.kNurbsSurface,  # nurbsSurface
+        OpenMaya.MFn.kNurbsCurve,  # nurbsCurve
+        OpenMaya.MFn.kFileTexture,  # file
+        OpenMaya.MFn.kSet,  # objectSet
+        OpenMaya.MFn.kPluginShape  # pgYetiMaya
+    ]
+    it = OpenMaya.MItDependencyNodes(iterator_type)
 
-    # The items which need to pass the id to their parent
-    # Add the collected transform to the nodes
-    dag = cmds.ls(nodes, type="dagNode", long=True)  # query only dag nodes
-    transforms = cmds.listRelatives(dag,
-                                    parent=True,
-                                    fullPath=True) or []
+    fn_dep = OpenMaya.MFnDependencyNode()
+    fn_dag = OpenMaya.MFnDagNode()
+    result = set()
 
-    nodes = set(nodes)
-    nodes |= set(transforms)
+    def _should_include_parents(obj):
+        """Whether to include parents of obj in output"""
+        if not obj.hasFn(OpenMaya.MFn.kShape):
+            return False
 
-    nodes -= ignore  # Remove the ignored nodes
-    if not nodes:
-        return nodes
+        fn_dag.setObject(obj)
+        if fn_dag.isIntermediateObject:
+            return False
 
-    # Ensure only nodes from the input `nodes` are returned when a
-    # filter was applied on function call because we also iterated
-    # to parents and alike
-    if lookup is not None:
-        nodes &= lookup
+        # Skip default cameras
+        if (
+            obj.hasFn(OpenMaya.MFn.kCamera) and
+            fn_dag.name() in default_camera_shapes
+        ):
+            return False
 
-    # Avoid locked nodes
-    nodes_list = list(nodes)
-    locked = cmds.lockNode(nodes_list, query=True, lock=True)
-    for node, lock in zip(nodes_list, locked):
-        if lock:
-            log.warning("Skipping locked node: %s" % node)
-            nodes.remove(node)
+        return True
 
-    return nodes
+    def _add_to_result_if_valid(obj):
+        """Add to `result` if the object should be included"""
+        fn_dep.setObject(obj)
+        if not existing_ids and fn_dep.hasAttribute("cbId"):
+            return
+
+        if not referenced_nodes and fn_dep.isFromReferencedFile:
+            return
+
+        if fn_dep.isDefaultNode:
+            return
+
+        if fn_dep.isLocked:
+            return
+
+        # Skip default cameras
+        if (
+            obj.hasFn(OpenMaya.MFn.kCamera) and
+            fn_dep.name() in default_camera_shapes
+        ):
+            return
+
+        if obj.hasFn(OpenMaya.MFn.kDagNode):
+            # DAG nodes
+            fn_dag.setObject(obj)
+
+            # Skip intermediate objects
+            if fn_dag.isIntermediateObject:
+                return
+
+            # DAG nodes can be instanced and thus may have multiple paths.
+            # We need to identify each path
+            paths = OpenMaya.MDagPath.getAllPathsTo(obj)
+            for dag in paths:
+                path = dag.fullPathName()
+                result.add(path)
+        else:
+            # Dependency node
+            path = fn_dep.name()
+            result.add(path)
+
+    for obj in iterate(it):
+        # For any non-intermediate shape node always include the parent
+        # even if we exclude the shape itself (e.g. when locked, default)
+        if _should_include_parents(obj):
+            fn_dag.setObject(obj)
+            parents = [
+                fn_dag.parent(index) for index in range(fn_dag.parentCount())
+            ]
+            for parent_obj in parents:
+                _add_to_result_if_valid(parent_obj)
+
+        _add_to_result_if_valid(obj)
+
+    if not result:
+        return result
+
+    # Exclude some additional types
+    exclude_types = []
+    if _node_type_exists("ilrBakeLayer"):
+        # Remove Turtle from the result if Turtle is loaded
+        exclude_types.append("ilrBakeLayer")
+
+    if exclude_types:
+        exclude_nodes = set(cmds.ls(nodes, long=True, type=exclude_types))
+        if exclude_nodes:
+            result -= exclude_nodes
+
+    # Filter to explicit input nodes if provided
+    if nodes is not None:
+        # The amount of input nodes to filter to can be large and querying
+        # many nodes can be slow in Maya. As such we want to try and reduce
+        # it as much as possible, so we include the type filter to try and
+        # reduce the result of `maya.cmds.ls` here.
+        nodes = set(cmds.ls(nodes, long=True, type=types + ["dagNode"]))
+        if nodes:
+            result &= nodes
+        else:
+            return set()
+
+    return result
 
 
 def get_id(node):

client/ayon_core/hosts/maya/api/pipeline.py

@@ -580,7 +580,8 @@ def on_save():
     _remove_workfile_lock()
 
     # Generate ids of the current context on nodes in the scene
-    nodes = lib.get_id_required_nodes(referenced_nodes=False)
+    nodes = lib.get_id_required_nodes(referenced_nodes=False,
+                                      existing_ids=False)
     for node, new_id in lib.generate_ids(nodes):
         lib.set_id(node, new_id, overwrite=False)
 

client/ayon_core/hosts/maya/plugins/publish/validate_node_ids.py

@@ -60,7 +60,8 @@ class ValidateNodeIDs(pyblish.api.InstancePlugin):
         # We do want to check the referenced nodes as it might be
         # part of the end product.
         id_nodes = lib.get_id_required_nodes(referenced_nodes=True,
-                                             nodes=instance[:])
-        invalid = [n for n in id_nodes if not lib.get_id(n)]
-
-        return invalid
+                                             nodes=instance[:],
+                                             # Exclude those with already
+                                             # existing ids
+                                             existing_ids=False)
+        return id_nodes

client/ayon_core/hosts/maya/plugins/publish/validate_node_ids_in_database.py

@@ -48,10 +48,10 @@ class ValidateNodeIdsInDatabase(pyblish.api.InstancePlugin):
             return
 
         # Get all id required nodes
-        id_required_nodes = lib.get_id_required_nodes(referenced_nodes=True,
+        id_required_nodes = lib.get_id_required_nodes(referenced_nodes=False,
                                                       nodes=nodes)
         if not id_required_nodes:
-            return []
+            return
 
         # check ids against database ids
         folder_ids = cls.get_project_folder_ids(context=instance.context)

client/ayon_core/hosts/nuke/plugins/load/load_clip.py

@@ -130,6 +130,18 @@ class LoadClip(plugin.NukeLoader):
             first = 1
             last = first + duration
 
+        # If a slate is present, the frame range is 1 frame longer for movies,
+        # but file sequences its the first frame that is 1 frame lower.
+        slate_frames = repre_entity["data"].get("slateFrames", 0)
+        extension = "." + repre_entity["context"]["ext"]
+
+        if extension in VIDEO_EXTENSIONS:
+            last += slate_frames
+
+        files_count = len(repre_entity["files"])
+        if extension in IMAGE_EXTENSIONS and files_count != 1:
+            first -= slate_frames
+
         # Fallback to folder name when namespace is None
         if namespace is None:
             namespace = context["folder"]["name"]
@@ -167,7 +179,9 @@ class LoadClip(plugin.NukeLoader):
                 repre_entity
             )
 
-            self._set_range_to_node(read_node, first, last, start_at_workfile)
+            self._set_range_to_node(
+                read_node, first, last, start_at_workfile, slate_frames
+            )
 
             version_name = version_entity["version"]
             if version_name < 0:
@@ -402,14 +416,21 @@ class LoadClip(plugin.NukeLoader):
             for member in members:
                 nuke.delete(member)
 
-    def _set_range_to_node(self, read_node, first, last, start_at_workfile):
+    def _set_range_to_node(
+        self, read_node, first, last, start_at_workfile, slate_frames=0
+    ):
         read_node['origfirst'].setValue(int(first))
         read_node['first'].setValue(int(first))
         read_node['origlast'].setValue(int(last))
         read_node['last'].setValue(int(last))
 
         # set start frame depending on workfile or version
-        self._loader_shift(read_node, start_at_workfile)
+        if start_at_workfile:
+            read_node['frame_mode'].setValue("start at")
+
+            start_frame = self.script_start - slate_frames
+
+            read_node['frame'].setValue(str(start_frame))
 
     def _make_retimes(self, parent_node, version_data):
         ''' Create all retime and timewarping nodes with copied animation '''
@@ -466,18 +487,6 @@ class LoadClip(plugin.NukeLoader):
                 for i, n in enumerate(dependent_nodes):
                     last_node.setInput(i, n)
 
-    def _loader_shift(self, read_node, workfile_start=False):
-        """ Set start frame of read node to a workfile start
-
-        Args:
-            read_node (nuke.Node): The nuke's read node
-            workfile_start (bool): set workfile start frame if true
-
-        """
-        if workfile_start:
-            read_node['frame_mode'].setValue("start at")
-            read_node['frame'].setValue(str(self.script_start))
-
     def _get_node_name(self, context):
         folder_entity = context["folder"]
         product_name = context["product"]["name"]

client/ayon_core/hosts/nuke/plugins/publish/extract_slate_frame.py

@@ -300,6 +300,10 @@ class ExtractSlateFrame(publish.Extractor):
             self.log.debug(
                 "__ matching_repre: {}".format(pformat(matching_repre)))
 
+        data = matching_repre.get("data", {})
+        data["slateFrames"] = 1
+        matching_repre["data"] = data
+
         self.log.info("Added slate frame to representation files")
 
     def add_comment_slate_node(self, instance, node):

client/ayon_core/plugins/publish/extract_review.py

@@ -32,6 +32,35 @@ from ayon_core.pipeline.publish import (
 from ayon_core.pipeline.publish.lib import add_repre_files_for_cleanup
 
 
+def frame_to_timecode(frame: int, fps: float) -> str:
+    """Convert a frame number and FPS to editorial timecode (HH:MM:SS:FF).
+
+    Unlike `ayon_core.pipeline.editorial.frames_to_timecode` this does not
+    rely on the `opentimelineio` package, so it can be used across hosts that
+    do not have it available.
+
+    Args:
+        frame (int): The frame number to be converted.
+        fps (float): The frames per second of the video.
+
+    Returns:
+        str: The timecode in HH:MM:SS:FF format.
+    """
+    # Calculate total seconds
+    total_seconds = frame / fps
+
+    # Extract hours, minutes, and seconds
+    hours = int(total_seconds // 3600)
+    minutes = int((total_seconds % 3600) // 60)
+    seconds = int(total_seconds % 60)
+
+    # Adjust for non-integer FPS by rounding the remaining frames appropriately
+    remaining_frames = round((total_seconds - int(total_seconds)) * fps)
+
+    # Format and return the timecode
+    return f"{hours:02d}:{minutes:02d}:{seconds:02d}:{remaining_frames:02d}"
+
+
 class ExtractReview(pyblish.api.InstancePlugin):
     """Extracting Review mov file for Ftrack
 
@@ -390,7 +419,16 @@ class ExtractReview(pyblish.api.InstancePlugin):
             # add outputName to anatomy format fill_data
             fill_data.update({
                 "output": output_name,
-                "ext": output_ext
+                "ext": output_ext,
+
+                # By adding `timecode` as data we can use it
+                # in the ffmpeg arguments for `--timecode` so that editorial
+                # like Resolve or Premiere can detect the start frame for e.g.
+                # review output files
+                "timecode": frame_to_timecode(
+                    frame=temp_data["frame_start_handle"],
+                    fps=float(instance.data["fps"])
+                )
             })
 
             try:  # temporary until oiiotool is supported cross platform

client/ayon_core/plugins/publish/extract_slate_data.py

@@ -0,0 +1,22 @@
+import pyblish.api
+
+from ayon_core.pipeline import publish
+
+
+class ExtractSlateData(publish.Extractor):
+    """Add slate data for integration."""
+
+    label = "Slate Data"
+    # Offset from ExtractReviewSlate and ExtractGenerateSlate.
+    order = pyblish.api.ExtractorOrder + 0.49
+    families = ["slate", "review"]
+    hosts = ["nuke", "shell"]
+
+    def process(self, instance):
+        for representation in instance.data.get("representations", []):
+            if "slate-frame" not in representation.get("tags", []):
+                continue
+
+            data = representation.get("data", {})
+            data["slateFrames"] = 1
+            representation["data"] = data

server/settings/tools.py

@@ -173,6 +173,7 @@ def _product_types_enum():
         "rig",
         "setdress",
         "take",
+        "usd",
         "usdShade",
         "vdbcache",
         "vrayproxy",