From 39399b1b269d63e4f0cf1cc9390a193c7e87cdc3 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Thu, 21 Mar 2024 16:32:37 +0000 Subject: [PATCH 1/6] Publish and load slate frame explicitly --- .../hosts/nuke/plugins/load/load_clip.py | 34 +++++++++++-------- .../plugins/publish/extract_slate_data.py | 22 ++++++++++++ 2 files changed, 41 insertions(+), 15 deletions(-) create mode 100644 client/ayon_core/plugins/publish/extract_slate_data.py diff --git a/client/ayon_core/hosts/nuke/plugins/load/load_clip.py b/client/ayon_core/hosts/nuke/plugins/load/load_clip.py index 8a41d854d9..b2044901a3 100644 --- a/client/ayon_core/hosts/nuke/plugins/load/load_clip.py +++ b/client/ayon_core/hosts/nuke/plugins/load/load_clip.py @@ -130,6 +130,11 @@ class LoadClip(plugin.NukeLoader): first = 1 last = first + duration + # If a slate is present, the frame range is 1 frame longer. + slate_frame = repre_entity["data"].get("slateFrame", False) + if slate_frame: + last += 1 + # Fallback to folder name when namespace is None if namespace is None: namespace = context["folder"]["name"] @@ -167,7 +172,9 @@ class LoadClip(plugin.NukeLoader): repre_entity ) - self._set_range_to_node(read_node, first, last, start_at_workfile) + self._set_range_to_node( + read_node, first, last, start_at_workfile, slate_frame + ) version_name = version_entity["version"] if version_name < 0: @@ -402,14 +409,23 @@ class LoadClip(plugin.NukeLoader): for member in members: nuke.delete(member) - def _set_range_to_node(self, read_node, first, last, start_at_workfile): + def _set_range_to_node( + self, read_node, first, last, start_at_workfile, slate_frame=False + ): read_node['origfirst'].setValue(int(first)) read_node['first'].setValue(int(first)) read_node['origlast'].setValue(int(last)) read_node['last'].setValue(int(last)) # set start frame depending on workfile or version - self._loader_shift(read_node, start_at_workfile) + if start_at_workfile: + read_node['frame_mode'].setValue("start at") + + start_frame = self.script_start + if slate_frame: + start_frame -= 1 + + read_node['frame'].setValue(str(start_frame)) def _make_retimes(self, parent_node, version_data): ''' Create all retime and timewarping nodes with copied animation ''' @@ -466,18 +482,6 @@ class LoadClip(plugin.NukeLoader): for i, n in enumerate(dependent_nodes): last_node.setInput(i, n) - def _loader_shift(self, read_node, workfile_start=False): - """ Set start frame of read node to a workfile start - - Args: - read_node (nuke.Node): The nuke's read node - workfile_start (bool): set workfile start frame if true - - """ - if workfile_start: - read_node['frame_mode'].setValue("start at") - read_node['frame'].setValue(str(self.script_start)) - def _get_node_name(self, context): folder_entity = context["folder"] product_name = context["product"]["name"] diff --git a/client/ayon_core/plugins/publish/extract_slate_data.py b/client/ayon_core/plugins/publish/extract_slate_data.py new file mode 100644 index 0000000000..0a28083f1e --- /dev/null +++ b/client/ayon_core/plugins/publish/extract_slate_data.py @@ -0,0 +1,22 @@ +import pyblish.api + +from ayon_core.pipeline import publish + + +class ExtractSlateData(publish.Extractor): + """Add slate data for integration.""" + + label = "Slate Data" + # Offset from ExtractReviewSlate and ExtractGenerateSlate. + order = pyblish.api.ExtractorOrder + 0.49 + families = ["slate", "review"] + hosts = ["nuke", "shell"] + + def process(self, instance): + for representation in instance.data.get("representations", []): + if "slate-frame" not in representation.get("tags", []): + continue + + data = representation.get("data", {}) + data["slateFrame"] = True + representation["data"] = data From d1e1f3942a7ed9cd88628b774f7421f14b07dd33 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Thu, 21 Mar 2024 17:38:43 +0000 Subject: [PATCH 2/6] Fix file sequences --- .../ayon_core/hosts/nuke/plugins/load/load_clip.py | 12 ++++++++++-- .../nuke/plugins/publish/extract_slate_frame.py | 4 ++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/hosts/nuke/plugins/load/load_clip.py b/client/ayon_core/hosts/nuke/plugins/load/load_clip.py index b2044901a3..86a6d359c3 100644 --- a/client/ayon_core/hosts/nuke/plugins/load/load_clip.py +++ b/client/ayon_core/hosts/nuke/plugins/load/load_clip.py @@ -130,10 +130,18 @@ class LoadClip(plugin.NukeLoader): first = 1 last = first + duration - # If a slate is present, the frame range is 1 frame longer. + # If a slate is present, the frame range is 1 frame longer for movies, + # but file sequences its the first frame that is 1 frame lower. slate_frame = repre_entity["data"].get("slateFrame", False) if slate_frame: - last += 1 + extension = "." + repre_entity["context"]["ext"] + + if extension in VIDEO_EXTENSIONS: + last += 1 + + files_count = len(repre_entity["files"]) + if extension in IMAGE_EXTENSIONS and files_count != 1: + first -= 1 # Fallback to folder name when namespace is None if namespace is None: diff --git a/client/ayon_core/hosts/nuke/plugins/publish/extract_slate_frame.py b/client/ayon_core/hosts/nuke/plugins/publish/extract_slate_frame.py index c013da84d2..67d2bd272c 100644 --- a/client/ayon_core/hosts/nuke/plugins/publish/extract_slate_frame.py +++ b/client/ayon_core/hosts/nuke/plugins/publish/extract_slate_frame.py @@ -300,6 +300,10 @@ class ExtractSlateFrame(publish.Extractor): self.log.debug( "__ matching_repre: {}".format(pformat(matching_repre))) + data = matching_repre.get("data", {}) + data["slateFrame"] = True + matching_repre["data"] = data + self.log.info("Added slate frame to representation files") def add_comment_slate_node(self, instance, node): From 312fd21b23f680c9f2032497ae9b857b38c52f99 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Thu, 28 Mar 2024 19:37:25 +0100 Subject: [PATCH 3/6] Expose `timecode` for formatting ffmpeg args --- .../plugins/publish/extract_review.py | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/plugins/publish/extract_review.py b/client/ayon_core/plugins/publish/extract_review.py index 905158c851..ac213b8669 100644 --- a/client/ayon_core/plugins/publish/extract_review.py +++ b/client/ayon_core/plugins/publish/extract_review.py @@ -32,6 +32,35 @@ from ayon_core.pipeline.publish import ( from ayon_core.pipeline.publish.lib import add_repre_files_for_cleanup +def frame_to_timecode(frame: int, fps: float) -> str: + """Convert a frame number and FPS to editorial timecode (HH:MM:SS:FF). + + Unlike `ayon_core.pipeline.editorial.frames_to_timecode` this does not + rely on the `opentimelineio` package, so it can be used across hosts that + do not have it available. + + Args: + frame (int): The frame number to be converted. + fps (float): The frames per second of the video. + + Returns: + str: The timecode in HH:MM:SS:FF format. + """ + # Calculate total seconds + total_seconds = frame / fps + + # Extract hours, minutes, and seconds + hours = int(total_seconds // 3600) + minutes = int((total_seconds % 3600) // 60) + seconds = int(total_seconds % 60) + + # Adjust for non-integer FPS by rounding the remaining frames appropriately + remaining_frames = round((total_seconds - int(total_seconds)) * fps) + + # Format and return the timecode + return f"{hours:02d}:{minutes:02d}:{seconds:02d}:{remaining_frames:02d}" + + class ExtractReview(pyblish.api.InstancePlugin): """Extracting Review mov file for Ftrack @@ -390,7 +419,16 @@ class ExtractReview(pyblish.api.InstancePlugin): # add outputName to anatomy format fill_data fill_data.update({ "output": output_name, - "ext": output_ext + "ext": output_ext, + + # By adding `timecode` as data we can use it + # in the ffmpeg arguments for `--timecode` so that editorial + # like Resolve or Premiere can detect the start frame for e.g. + # review output files + "timecode": frame_to_timecode( + frame=temp_data["frame_start_handle"], + fps=float(instance.data["fps"]) + ) }) try: # temporary until oiiotool is supported cross platform From 33fd95dd009d65097cc68f7f4f10251eff4ea874 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 29 Mar 2024 10:57:27 +0100 Subject: [PATCH 4/6] Add `usd` to product type enum for Loader Product Type Filter Used in `ayon+settings://core/tools/loader/product_type_filter_profiles` --- server/settings/tools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/settings/tools.py b/server/settings/tools.py index 488d27e8f1..fb8430a71c 100644 --- a/server/settings/tools.py +++ b/server/settings/tools.py @@ -173,6 +173,7 @@ def _product_types_enum(): "rig", "setdress", "take", + "usd", "usdShade", "vdbcache", "vrayproxy", From a4b0dbc4936da61f167912167959916138b8e2b4 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 29 Mar 2024 13:17:49 +0100 Subject: [PATCH 5/6] Optimize `cbId` logic for large scenes --- client/ayon_core/hosts/maya/api/lib.py | 191 +++++++++++++----- client/ayon_core/hosts/maya/api/pipeline.py | 3 +- .../maya/plugins/publish/validate_node_ids.py | 9 +- .../publish/validate_node_ids_in_database.py | 11 +- 4 files changed, 153 insertions(+), 61 deletions(-) diff --git a/client/ayon_core/hosts/maya/api/lib.py b/client/ayon_core/hosts/maya/api/lib.py index b18d3a0c33..4b9b80f0e7 100644 --- a/client/ayon_core/hosts/maya/api/lib.py +++ b/client/ayon_core/hosts/maya/api/lib.py @@ -1519,24 +1519,30 @@ def extract_alembic(file, # region ID -def get_id_required_nodes(referenced_nodes=False, nodes=None): - """Filter out any node which are locked (reference) or readOnly +def get_id_required_nodes(referenced_nodes=False, + nodes=None, + existing_ids=True): + """Return nodes that should receive a `cbId` attribute. + + This includes only mesh and curve nodes, parent transforms of the shape + nodes, file texture nodes and object sets (including shading engines). + + This filters out any node which is locked, referenced, read-only, + intermediate object. Args: - referenced_nodes (bool): set True to filter out reference nodes + referenced_nodes (bool): set True to include referenced nodes nodes (list, Optional): nodes to consider + existing_ids (bool): set True to include nodes with `cbId` attribute + Returns: nodes (set): list of filtered nodes """ - lookup = None - if nodes is None: - # Consider all nodes - nodes = cmds.ls() - else: - # Build a lookup for the only allowed nodes in output based - # on `nodes` input of the function (+ ensure long names) - lookup = set(cmds.ls(nodes, long=True)) + if nodes is not None and not nodes: + # User supplied an empty `nodes` list to check so all we can + # do is return the empty result + return set() def _node_type_exists(node_type): try: @@ -1545,63 +1551,142 @@ def get_id_required_nodes(referenced_nodes=False, nodes=None): except RuntimeError: return False + def iterate(maya_iterator): + while not maya_iterator.isDone(): + yield maya_iterator.thisNode() + maya_iterator.next() + # `readOnly` flag is obsolete as of Maya 2016 therefore we explicitly # remove default nodes and reference nodes - camera_shapes = ["frontShape", "sideShape", "topShape", "perspShape"] + default_camera_shapes = { + "frontShape", "sideShape", "topShape", "perspShape" + } - ignore = set() - if not referenced_nodes: - ignore |= set(cmds.ls(long=True, referencedNodes=True)) - - # list all defaultNodes to filter out from the rest - ignore |= set(cmds.ls(long=True, defaultNodes=True)) - ignore |= set(cmds.ls(camera_shapes, long=True)) - - # Remove Turtle from the result of `cmds.ls` if Turtle is loaded - # TODO: This should be a less specific check for a single plug-in. - if _node_type_exists("ilrBakeLayer"): - ignore |= set(cmds.ls(type="ilrBakeLayer", long=True)) - - # Establish set of nodes types to include - types = ["objectSet", "file", "mesh", "nurbsCurve", "nurbsSurface"] + # The filtered types do not include transforms because we only want the + # parent transforms that have a child shape that we filtered to, so we + # include the parents here + types = ["mesh", "nurbsCurve", "nurbsSurface", "file", "objectSet"] # Check if plugin nodes are available for Maya by checking if the plugin # is loaded if cmds.pluginInfo("pgYetiMaya", query=True, loaded=True): types.append("pgYetiMaya") - # We *always* ignore intermediate shapes, so we filter them out directly - nodes = cmds.ls(nodes, type=types, long=True, noIntermediate=True) + iterator_type = OpenMaya.MIteratorType() + # This tries to be closest matching API equivalents of `types` variable + iterator_type.filterList = [ + OpenMaya.MFn.kMesh, # mesh + OpenMaya.MFn.kNurbsSurface, # nurbsSurface + OpenMaya.MFn.kNurbsCurve, # nurbsCurve + OpenMaya.MFn.kFileTexture, # file + OpenMaya.MFn.kSet, # objectSet + OpenMaya.MFn.kPluginShape # pgYetiMaya + ] + it = OpenMaya.MItDependencyNodes(iterator_type) - # The items which need to pass the id to their parent - # Add the collected transform to the nodes - dag = cmds.ls(nodes, type="dagNode", long=True) # query only dag nodes - transforms = cmds.listRelatives(dag, - parent=True, - fullPath=True) or [] + fn_dep = OpenMaya.MFnDependencyNode() + fn_dag = OpenMaya.MFnDagNode() + result = set() - nodes = set(nodes) - nodes |= set(transforms) + def _should_include_parents(obj): + """Whether to include parents of obj in output""" + if not obj.hasFn(OpenMaya.MFn.kShape): + return False - nodes -= ignore # Remove the ignored nodes - if not nodes: - return nodes + fn_dag.setObject(obj) + if fn_dag.isIntermediateObject: + return False - # Ensure only nodes from the input `nodes` are returned when a - # filter was applied on function call because we also iterated - # to parents and alike - if lookup is not None: - nodes &= lookup + # Skip default cameras + if ( + obj.hasFn(OpenMaya.MFn.kCamera) and + fn_dag.name() in default_camera_shapes + ): + return False - # Avoid locked nodes - nodes_list = list(nodes) - locked = cmds.lockNode(nodes_list, query=True, lock=True) - for node, lock in zip(nodes_list, locked): - if lock: - log.warning("Skipping locked node: %s" % node) - nodes.remove(node) + return True - return nodes + def _add_to_result_if_valid(obj): + """Add to `result` if the object should be included""" + fn_dep.setObject(obj) + if not existing_ids and fn_dep.hasAttribute("cbId"): + return + + if not referenced_nodes and fn_dep.isFromReferencedFile: + return + + if fn_dep.isDefaultNode: + return + + if fn_dep.isLocked: + return + + # Skip default cameras + if ( + obj.hasFn(OpenMaya.MFn.kCamera) and + fn_dep.name() in default_camera_shapes + ): + return + + if obj.hasFn(OpenMaya.MFn.kDagNode): + # DAG nodes + fn_dag.setObject(obj) + + # Skip intermediate objects + if fn_dag.isIntermediateObject: + return + + # DAG nodes can be instanced and thus may have multiple paths. + # We need to identify each path + paths = OpenMaya.MDagPath.getAllPathsTo(obj) + for dag in paths: + path = dag.fullPathName() + result.add(path) + else: + # Dependency node + path = fn_dep.name() + result.add(path) + + for obj in iterate(it): + # For any non-intermediate shape node always include the parent + # even if we exclude the shape itself (e.g. when locked, default) + if _should_include_parents(obj): + fn_dag.setObject(obj) + parents = [ + fn_dag.parent(index) for index in range(fn_dag.parentCount()) + ] + for parent_obj in parents: + _add_to_result_if_valid(parent_obj) + + _add_to_result_if_valid(obj) + + if not result: + return result + + # Exclude some additional types + exclude_types = [] + if _node_type_exists("ilrBakeLayer"): + # Remove Turtle from the result if Turtle is loaded + exclude_types.append("ilrBakeLayer") + + if exclude_types: + exclude_nodes = set(cmds.ls(nodes, long=True, type=exclude_types)) + if exclude_nodes: + result -= exclude_nodes + + # Filter to explicit input nodes if provided + if nodes is not None: + # The amount of input nodes to filter to can be large and querying + # many nodes can be slow in Maya. As such we want to try and reduce + # it as much as possible, so we include the type filter to try and + # reduce the result of `maya.cmds.ls` here. + nodes = set(cmds.ls(nodes, long=True, type=types + ["dagNode"])) + if nodes: + result &= nodes + else: + return set() + + return result def get_id(node): diff --git a/client/ayon_core/hosts/maya/api/pipeline.py b/client/ayon_core/hosts/maya/api/pipeline.py index 2be452a22a..464e87814c 100644 --- a/client/ayon_core/hosts/maya/api/pipeline.py +++ b/client/ayon_core/hosts/maya/api/pipeline.py @@ -580,7 +580,8 @@ def on_save(): _remove_workfile_lock() # Generate ids of the current context on nodes in the scene - nodes = lib.get_id_required_nodes(referenced_nodes=False) + nodes = lib.get_id_required_nodes(referenced_nodes=False, + existing_ids=False) for node, new_id in lib.generate_ids(nodes): lib.set_id(node, new_id, overwrite=False) diff --git a/client/ayon_core/hosts/maya/plugins/publish/validate_node_ids.py b/client/ayon_core/hosts/maya/plugins/publish/validate_node_ids.py index ba748a4fc4..2d6f231cb5 100644 --- a/client/ayon_core/hosts/maya/plugins/publish/validate_node_ids.py +++ b/client/ayon_core/hosts/maya/plugins/publish/validate_node_ids.py @@ -60,7 +60,8 @@ class ValidateNodeIDs(pyblish.api.InstancePlugin): # We do want to check the referenced nodes as it might be # part of the end product. id_nodes = lib.get_id_required_nodes(referenced_nodes=True, - nodes=instance[:]) - invalid = [n for n in id_nodes if not lib.get_id(n)] - - return invalid + nodes=instance[:], + # Exclude those with already + # existing ids + existing_ids=False) + return id_nodes diff --git a/client/ayon_core/hosts/maya/plugins/publish/validate_node_ids_in_database.py b/client/ayon_core/hosts/maya/plugins/publish/validate_node_ids_in_database.py index 5ca9690fd7..58978fb538 100644 --- a/client/ayon_core/hosts/maya/plugins/publish/validate_node_ids_in_database.py +++ b/client/ayon_core/hosts/maya/plugins/publish/validate_node_ids_in_database.py @@ -43,11 +43,15 @@ class ValidateNodeIdsInDatabase(pyblish.api.InstancePlugin): @classmethod def get_invalid(cls, instance): - invalid = [] + nodes = instance[:] + if not nodes: + return # Get all id required nodes - id_required_nodes = lib.get_id_required_nodes(referenced_nodes=True, - nodes=instance[:]) + id_required_nodes = lib.get_id_required_nodes(referenced_nodes=False, + nodes=nodes) + if not id_required_nodes: + return # check ids against database ids project_name = instance.context.data["projectName"] @@ -58,6 +62,7 @@ class ValidateNodeIdsInDatabase(pyblish.api.InstancePlugin): } # Get all asset IDs + invalid = [] for node in id_required_nodes: cb_id = lib.get_id(node) From a08b1786b0891a3cd4e6bb5d8d557f3bf749b7c4 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Mon, 8 Apr 2024 08:47:50 +0100 Subject: [PATCH 6/6] Account for multiple slate frames. --- .../hosts/nuke/plugins/load/load_clip.py | 23 ++++++++----------- .../plugins/publish/extract_slate_frame.py | 2 +- .../plugins/publish/extract_slate_data.py | 2 +- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/client/ayon_core/hosts/nuke/plugins/load/load_clip.py b/client/ayon_core/hosts/nuke/plugins/load/load_clip.py index 86a6d359c3..062a5295ed 100644 --- a/client/ayon_core/hosts/nuke/plugins/load/load_clip.py +++ b/client/ayon_core/hosts/nuke/plugins/load/load_clip.py @@ -132,16 +132,15 @@ class LoadClip(plugin.NukeLoader): # If a slate is present, the frame range is 1 frame longer for movies, # but file sequences its the first frame that is 1 frame lower. - slate_frame = repre_entity["data"].get("slateFrame", False) - if slate_frame: - extension = "." + repre_entity["context"]["ext"] + slate_frames = repre_entity["data"].get("slateFrames", 0) + extension = "." + repre_entity["context"]["ext"] - if extension in VIDEO_EXTENSIONS: - last += 1 + if extension in VIDEO_EXTENSIONS: + last += slate_frames - files_count = len(repre_entity["files"]) - if extension in IMAGE_EXTENSIONS and files_count != 1: - first -= 1 + files_count = len(repre_entity["files"]) + if extension in IMAGE_EXTENSIONS and files_count != 1: + first -= slate_frames # Fallback to folder name when namespace is None if namespace is None: @@ -181,7 +180,7 @@ class LoadClip(plugin.NukeLoader): ) self._set_range_to_node( - read_node, first, last, start_at_workfile, slate_frame + read_node, first, last, start_at_workfile, slate_frames ) version_name = version_entity["version"] @@ -418,7 +417,7 @@ class LoadClip(plugin.NukeLoader): nuke.delete(member) def _set_range_to_node( - self, read_node, first, last, start_at_workfile, slate_frame=False + self, read_node, first, last, start_at_workfile, slate_frames=0 ): read_node['origfirst'].setValue(int(first)) read_node['first'].setValue(int(first)) @@ -429,9 +428,7 @@ class LoadClip(plugin.NukeLoader): if start_at_workfile: read_node['frame_mode'].setValue("start at") - start_frame = self.script_start - if slate_frame: - start_frame -= 1 + start_frame = self.script_start - slate_frames read_node['frame'].setValue(str(start_frame)) diff --git a/client/ayon_core/hosts/nuke/plugins/publish/extract_slate_frame.py b/client/ayon_core/hosts/nuke/plugins/publish/extract_slate_frame.py index 67d2bd272c..627888ac92 100644 --- a/client/ayon_core/hosts/nuke/plugins/publish/extract_slate_frame.py +++ b/client/ayon_core/hosts/nuke/plugins/publish/extract_slate_frame.py @@ -301,7 +301,7 @@ class ExtractSlateFrame(publish.Extractor): "__ matching_repre: {}".format(pformat(matching_repre))) data = matching_repre.get("data", {}) - data["slateFrame"] = True + data["slateFrames"] = 1 matching_repre["data"] = data self.log.info("Added slate frame to representation files") diff --git a/client/ayon_core/plugins/publish/extract_slate_data.py b/client/ayon_core/plugins/publish/extract_slate_data.py index 0a28083f1e..750fb5d60a 100644 --- a/client/ayon_core/plugins/publish/extract_slate_data.py +++ b/client/ayon_core/plugins/publish/extract_slate_data.py @@ -18,5 +18,5 @@ class ExtractSlateData(publish.Extractor): continue data = representation.get("data", {}) - data["slateFrame"] = True + data["slateFrames"] = 1 representation["data"] = data