From eddbc6c04511680439d551fc7f3f1291c21451a6 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Mon, 8 Jun 2020 16:10:12 +0100 Subject: [PATCH 1/6] Initial working editorial publishing. --- .../publish/integrate_ftrack_instances.py | 12 +- .../publish/extract_hierarchy_avalon.py | 2 +- .../plugins/harmony/publish/extract_render.py | 6 +- .../publish/collect_shots.py | 125 ++++++++++++++++++ 4 files changed, 138 insertions(+), 7 deletions(-) create mode 100644 pype/plugins/standalonepublisher/publish/collect_shots.py diff --git a/pype/plugins/ftrack/publish/integrate_ftrack_instances.py b/pype/plugins/ftrack/publish/integrate_ftrack_instances.py index 11b569fd12..f5d7689678 100644 --- a/pype/plugins/ftrack/publish/integrate_ftrack_instances.py +++ b/pype/plugins/ftrack/publish/integrate_ftrack_instances.py @@ -44,10 +44,14 @@ class IntegrateFtrackInstance(pyblish.api.InstancePlugin): family = instance.data['family'].lower() - asset_type = '' - asset_type = instance.data.get( - "ftrackFamily", self.family_mapping[family] - ) + asset_type = instance.data.get("ftrackFamily") + if not asset_type and family in self.family_mapping: + asset_type = self.family_mapping[family] + + # Ignore this instance if neither "ftrackFamily" or a family mapping is + # found. + if not asset_type: + return componentList = [] ft_session = instance.context.data["ftrackSession"] diff --git a/pype/plugins/global/publish/extract_hierarchy_avalon.py b/pype/plugins/global/publish/extract_hierarchy_avalon.py index ab8226f6ef..83cf03b042 100644 --- a/pype/plugins/global/publish/extract_hierarchy_avalon.py +++ b/pype/plugins/global/publish/extract_hierarchy_avalon.py @@ -7,7 +7,7 @@ class ExtractHierarchyToAvalon(pyblish.api.ContextPlugin): order = pyblish.api.ExtractorOrder - 0.01 label = "Extract Hierarchy To Avalon" - families = ["clip", "shot"] + families = ["clip", "shot", "editorial"] def process(self, context): if "hierarchyContext" not in context.data: diff --git a/pype/plugins/harmony/publish/extract_render.py b/pype/plugins/harmony/publish/extract_render.py index de6e8b9008..eb0850dc58 100644 --- a/pype/plugins/harmony/publish/extract_render.py +++ b/pype/plugins/harmony/publish/extract_render.py @@ -60,8 +60,10 @@ class ExtractRender(pyblish.api.InstancePlugin): harmony.save_scene() # Execute rendering. - output = pype.lib._subprocess([application_path, "-batch", scene_path]) - self.log.info(output) + import subprocess + subprocess.call([application_path, "-batch", scene_path]) + #output = pype.lib._subprocess([application_path, "-batch", scene_path]) + #self.log.info(output) # Collect rendered files. files = os.listdir(path) diff --git a/pype/plugins/standalonepublisher/publish/collect_shots.py b/pype/plugins/standalonepublisher/publish/collect_shots.py new file mode 100644 index 0000000000..fb566a15ee --- /dev/null +++ b/pype/plugins/standalonepublisher/publish/collect_shots.py @@ -0,0 +1,125 @@ +import os +from urllib.parse import unquote, urlparse + +import opentimelineio as otio +from bson import json_util + +import pyblish.api +from pype import lib +from avalon import io + + +class OTIO_View(pyblish.api.Action): + """Currently disabled because OTIO requires PySide2. Issue on Qt.py: + https://github.com/PixarAnimationStudios/OpenTimelineIO/issues/289 + """ + + label = "OTIO View" + icon = "wrench" + on = "failed" + + def process(self, context, plugin): + instance = context[0] + representation = instance.data["representations"][0] + file_path = os.path.join( + representation["stagingDir"], representation["files"] + ) + lib._subprocess(["otioview", file_path]) + + +class CollectShots(pyblish.api.InstancePlugin): + """Collect Anatomy object into Context""" + + order = pyblish.api.CollectorOrder + label = "Collect Shots" + hosts = ["standalonepublisher"] + families = ["editorial"] + actions = [] + + def process(self, instance): + representation = instance.data["representations"][0] + file_path = os.path.join( + representation["stagingDir"], representation["files"] + ) + timeline = otio.adapters.read_from_file(file_path) + tracks = timeline.each_child( + descended_from_type=otio.schema.track.Track + ) + asset_entity = instance.context.data["assetEntity"] + asset_name = asset_entity["name"] + + # Project specific prefix naming. This needs to be replaced with some + # options to be more flexible. + asset_name = asset_name.split("_")[0] + + shot_number = 10 + for track in tracks: + self.log.info(track) + + if "audio" in track.name.lower(): + continue + + instances = [] + for child in track.each_child(): + parse = urlparse(child.media_reference.target_url) + + # XML files from NukeStudio has extra "/" at the front of path. + path = os.path.normpath( + os.path.abspath(unquote(parse.path)[1:]) + ) + + frame_start = child.range_in_parent().start_time.value + frame_end = child.range_in_parent().end_time_inclusive().value + + name = f"{asset_name}_sh{shot_number:04}" + label = f"{name} (framerange: {frame_start}-{frame_end})" + instances.append( + instance.context.create_instance(**{ + "name": name, + "label": label, + "path": path, + "frameStart": frame_start, + "frameEnd": frame_end, + "family": "shot", + "asset": name, + "subset": "shotMain" + }) + ) + + shot_number += 10 + + visual_hierarchy = [asset_entity] + while True: + visual_parent = io.find_one( + {"_id": visual_hierarchy[-1]["data"]["visualParent"]} + ) + if visual_parent: + visual_hierarchy.append(visual_parent) + else: + visual_hierarchy.append(instance.context.data["projectEntity"]) + break + + context_hierarchy = None + for entity in visual_hierarchy: + childs = {} + if context_hierarchy: + name = context_hierarchy.pop("name") + childs = {name: context_hierarchy} + else: + for instance in instances: + childs[instance.data["name"]] = { + "childs": {}, "entity_type": "Shot" + } + + context_hierarchy = { + "entity_type": entity["data"]["entityType"], + "childs": childs, + "name": entity["name"] + } + + name = context_hierarchy.pop("name") + context_hierarchy = {name: context_hierarchy} + instance.context.data["hierarchyContext"] = context_hierarchy + self.log.info( + json_util.dumps(context_hierarchy, sort_keys=True, indent=4) + ) From 14b29fd5ac708c7b25c753f3908f7e7f74a8adba Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Mon, 8 Jun 2020 16:17:37 +0100 Subject: [PATCH 2/6] Fix accidental commit. --- pype/plugins/harmony/publish/extract_render.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pype/plugins/harmony/publish/extract_render.py b/pype/plugins/harmony/publish/extract_render.py index eb0850dc58..de6e8b9008 100644 --- a/pype/plugins/harmony/publish/extract_render.py +++ b/pype/plugins/harmony/publish/extract_render.py @@ -60,10 +60,8 @@ class ExtractRender(pyblish.api.InstancePlugin): harmony.save_scene() # Execute rendering. - import subprocess - subprocess.call([application_path, "-batch", scene_path]) - #output = pype.lib._subprocess([application_path, "-batch", scene_path]) - #self.log.info(output) + output = pype.lib._subprocess([application_path, "-batch", scene_path]) + self.log.info(output) # Collect rendered files. files = os.listdir(path) From cad378fca021e7e4db11bcd31658373090785922 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Sun, 14 Jun 2020 19:09:51 +0100 Subject: [PATCH 3/6] Extract shot mov and wav. --- .../plugins/harmony/publish/extract_render.py | 2 +- .../publish/collect_shots.py | 34 +++++---- .../publish/extract_review.py | 2 +- .../publish/extract_shot.py | 70 +++++++++++++++++++ .../publish/validate_editorial_resources.py | 28 ++++++++ 5 files changed, 122 insertions(+), 14 deletions(-) create mode 100644 pype/plugins/standalonepublisher/publish/extract_shot.py create mode 100644 pype/plugins/standalonepublisher/publish/validate_editorial_resources.py diff --git a/pype/plugins/harmony/publish/extract_render.py b/pype/plugins/harmony/publish/extract_render.py index de6e8b9008..56e7973f3a 100644 --- a/pype/plugins/harmony/publish/extract_render.py +++ b/pype/plugins/harmony/publish/extract_render.py @@ -87,7 +87,7 @@ class ExtractRender(pyblish.api.InstancePlugin): "frameEnd": frame_end, "fps": frame_rate, "preview": True, - "tags": ["review"] + "tags": ["review", "ftrackreview"] } instance.data["representations"] = [representation] self.log.info(frame_rate) diff --git a/pype/plugins/standalonepublisher/publish/collect_shots.py b/pype/plugins/standalonepublisher/publish/collect_shots.py index fb566a15ee..be7d5691ba 100644 --- a/pype/plugins/standalonepublisher/publish/collect_shots.py +++ b/pype/plugins/standalonepublisher/publish/collect_shots.py @@ -1,5 +1,4 @@ import os -from urllib.parse import unquote, urlparse import opentimelineio as otio from bson import json_util @@ -41,7 +40,16 @@ class CollectShots(pyblish.api.InstancePlugin): file_path = os.path.join( representation["stagingDir"], representation["files"] ) - timeline = otio.adapters.read_from_file(file_path) + instance.context.data["editorialPath"] = file_path + + extension = os.path.splitext(file_path)[1][1:] + kwargs = {} + if extension == "edl": + # EDL has no frame rate embedded so needs explicit frame rate else + # 24 is asssumed. + kwargs["rate"] = lib.get_asset()["data"]["fps"] + + timeline = otio.adapters.read_from_file(file_path, **kwargs) tracks = timeline.each_child( descended_from_type=otio.schema.track.Track ) @@ -61,13 +69,6 @@ class CollectShots(pyblish.api.InstancePlugin): instances = [] for child in track.each_child(): - parse = urlparse(child.media_reference.target_url) - - # XML files from NukeStudio has extra "/" at the front of path. - path = os.path.normpath( - os.path.abspath(unquote(parse.path)[1:]) - ) - frame_start = child.range_in_parent().start_time.value frame_end = child.range_in_parent().end_time_inclusive().value @@ -77,12 +78,15 @@ class CollectShots(pyblish.api.InstancePlugin): instance.context.create_instance(**{ "name": name, "label": label, - "path": path, "frameStart": frame_start, "frameEnd": frame_end, "family": "shot", + "families": ["review", "ftrack"], + "ftrackFamily": "review", "asset": name, - "subset": "shotMain" + "subset": "shotMain", + "representations": [], + "source": file_path }) ) @@ -108,7 +112,12 @@ class CollectShots(pyblish.api.InstancePlugin): else: for instance in instances: childs[instance.data["name"]] = { - "childs": {}, "entity_type": "Shot" + "childs": {}, + "entity_type": "Shot", + "custom_attributes": { + "frameStart": instance.data["frameStart"], + "frameEnd": instance.data["frameEnd"] + } } context_hierarchy = { @@ -121,5 +130,6 @@ class CollectShots(pyblish.api.InstancePlugin): context_hierarchy = {name: context_hierarchy} instance.context.data["hierarchyContext"] = context_hierarchy self.log.info( + "Hierarchy:\n" + json_util.dumps(context_hierarchy, sort_keys=True, indent=4) ) diff --git a/pype/plugins/standalonepublisher/publish/extract_review.py b/pype/plugins/standalonepublisher/publish/extract_review.py index 36793d4c62..0f845afcb1 100644 --- a/pype/plugins/standalonepublisher/publish/extract_review.py +++ b/pype/plugins/standalonepublisher/publish/extract_review.py @@ -42,7 +42,7 @@ class ExtractReviewSP(pyblish.api.InstancePlugin): self.log.debug("Families In: `{}`".format(instance.data["families"])) # get specific profile if was defined - specific_profiles = instance.data.get("repreProfiles") + specific_profiles = instance.data.get("repreProfiles", []) new_repres = [] # filter out mov and img sequences diff --git a/pype/plugins/standalonepublisher/publish/extract_shot.py b/pype/plugins/standalonepublisher/publish/extract_shot.py new file mode 100644 index 0000000000..34d2092d77 --- /dev/null +++ b/pype/plugins/standalonepublisher/publish/extract_shot.py @@ -0,0 +1,70 @@ +import os + +import pype.api +import pype.lib + + +class ExtractShot(pype.api.Extractor): + """Extract shot "mov" and "wav" files.""" + + label = "Extract Shot" + hosts = ["standalonepublisher"] + families = ["shot"] + + def process(self, instance): + staging_dir = self.staging_dir(instance) + self.log.info("Outputting shot to {}".format(staging_dir)) + + editorial_path = instance.context.data["editorialPath"] + basename = os.path.splitext(os.path.basename(editorial_path))[0] + + # Generate mov file. + fps = pype.lib.get_asset()["data"]["fps"] + input_path = os.path.join( + os.path.dirname(editorial_path), basename + ".mov" + ) + shot_mov = os.path.join(staging_dir, instance.data["name"] + ".mov") + args = [ + "ffmpeg", + "-ss", str(instance.data["frameStart"] / fps), + "-i", input_path, + "-t", str( + (instance.data["frameEnd"] - instance.data["frameStart"] + 1) / + fps + ), + "-crf", "18", + "-pix_fmt", "yuv420p", + shot_mov + ] + self.log.info(f"Processing: {args}") + output = pype.lib._subprocess(args) + self.log.info(output) + + instance.data["representations"].append({ + "name": "mov", + "ext": "mov", + "files": os.path.basename(shot_mov), + "stagingDir": staging_dir, + "frameStart": instance.data["frameStart"], + "frameEnd": instance.data["frameEnd"], + "fps": fps, + "thumbnail": True, + "tags": ["review", "ftrackreview"] + }) + + # Generate wav file. + shot_wav = os.path.join(staging_dir, instance.data["name"] + ".wav") + args = ["ffmpeg", "-i", shot_mov, shot_wav] + self.log.info(f"Processing: {args}") + output = pype.lib._subprocess(args) + self.log.info(output) + + instance.data["representations"].append({ + "name": "wav", + "ext": "wav", + "files": os.path.basename(shot_wav), + "stagingDir": staging_dir + }) + + # Required for extract_review plugin (L222 onwards). + instance.data["fps"] = fps diff --git a/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py b/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py new file mode 100644 index 0000000000..961641b8fa --- /dev/null +++ b/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py @@ -0,0 +1,28 @@ +import os + +import pyblish.api +import pype.api + + +class ValidateEditorialResources(pyblish.api.InstancePlugin): + """Validate there is a "mov" next to the editorial file.""" + + label = "Validate Editorial Resources" + hosts = ["standalonepublisher"] + families = ["editorial"] + order = pype.api.ValidateContentsOrder + + def process(self, instance): + representation = instance.data["representations"][0] + staging_dir = representation["stagingDir"] + basename = os.path.splitext( + os.path.basename(representation["files"]) + )[0] + + files = [x for x in os.listdir(staging_dir)] + + # Check for "mov" file. + filename = basename + ".mov" + filepath = os.path.join(staging_dir, filename) + msg = f"Missing \"{filepath}\"." + assert filename in files, msg From 6297607ff71c60f4ef6c50e3edbab6d388107a00 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Mon, 15 Jun 2020 09:36:07 +0100 Subject: [PATCH 4/6] Extract shot jpegs. --- .../publish/extract_shot.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/pype/plugins/standalonepublisher/publish/extract_shot.py b/pype/plugins/standalonepublisher/publish/extract_shot.py index 34d2092d77..f2fc2b74cf 100644 --- a/pype/plugins/standalonepublisher/publish/extract_shot.py +++ b/pype/plugins/standalonepublisher/publish/extract_shot.py @@ -1,5 +1,7 @@ import os +import clique + import pype.api import pype.lib @@ -52,6 +54,29 @@ class ExtractShot(pype.api.Extractor): "tags": ["review", "ftrackreview"] }) + # Generate jpegs. + shot_jpegs = os.path.join( + staging_dir, instance.data["name"] + ".%04d.jpeg" + ) + args = ["ffmpeg", "-i", shot_mov, shot_jpegs] + self.log.info(f"Processing: {args}") + output = pype.lib._subprocess(args) + self.log.info(output) + + collection = clique.Collection( + head=instance.data["name"] + ".", tail='.jpeg', padding=4 + ) + for f in os.listdir(staging_dir): + if collection.match(f): + collection.add(f) + + instance.data["representations"].append({ + "name": "jpeg", + "ext": "jpeg", + "files": list(collection), + "stagingDir": staging_dir + }) + # Generate wav file. shot_wav = os.path.join(staging_dir, instance.data["name"] + ".wav") args = ["ffmpeg", "-i", shot_mov, shot_wav] From 35e6e90db4452ca5a042311ec57de621dd1a16dc Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Mon, 15 Jun 2020 11:25:28 +0100 Subject: [PATCH 5/6] Ignore transitions. --- pype/plugins/standalonepublisher/publish/collect_shots.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pype/plugins/standalonepublisher/publish/collect_shots.py b/pype/plugins/standalonepublisher/publish/collect_shots.py index be7d5691ba..80ee875add 100644 --- a/pype/plugins/standalonepublisher/publish/collect_shots.py +++ b/pype/plugins/standalonepublisher/publish/collect_shots.py @@ -69,6 +69,12 @@ class CollectShots(pyblish.api.InstancePlugin): instances = [] for child in track.each_child(): + + # Transitions are ignored, because Clips have the full frame + # range. + if isinstance(child, otio.schema.transition.Transition): + continue + frame_start = child.range_in_parent().start_time.value frame_end = child.range_in_parent().end_time_inclusive().value From 5889f6d7a03ede295a2df821b4815b448896b363 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Mon, 15 Jun 2020 16:54:58 +0100 Subject: [PATCH 6/6] Integrate editorial files. --- pype/plugins/global/publish/integrate_new.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index f8429e8b58..d6111f95f5 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -83,7 +83,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): "fbx", "textures", "action", - "harmony.template" + "harmony.template", + "editorial" ] exclude_families = ["clip"] db_representation_context_keys = [