From 123fb5ff861fb5f344f5be8fe37b23c786f2d268 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 31 Jul 2020 18:50:05 +0200 Subject: [PATCH 01/30] feat(standalone): editorial wip --- .../publish/collect_clips.py | 230 +++++++++++ .../publish/collect_editorial.py | 76 ++++ .../publish/collect_frame_ranges.py | 56 +++ .../publish/collect_hierarchy.py | 358 ++++++++++++++++++ .../publish/collect_shot_names.py | 15 + .../publish/collect_shots.py | 147 ------- .../publish/extract_shot.py | 129 ++++--- .../publish/validate_clips.py | 24 ++ .../publish/validate_editorial_resources.py | 8 +- ...e_shots.py => validate_shot_duplicates.py} | 6 +- 10 files changed, 836 insertions(+), 213 deletions(-) create mode 100644 pype/plugins/standalonepublisher/publish/collect_clips.py create mode 100644 pype/plugins/standalonepublisher/publish/collect_editorial.py create mode 100644 pype/plugins/standalonepublisher/publish/collect_frame_ranges.py create mode 100644 pype/plugins/standalonepublisher/publish/collect_hierarchy.py create mode 100644 pype/plugins/standalonepublisher/publish/collect_shot_names.py delete mode 100644 pype/plugins/standalonepublisher/publish/collect_shots.py create mode 100644 pype/plugins/standalonepublisher/publish/validate_clips.py rename pype/plugins/standalonepublisher/publish/{validate_shots.py => validate_shot_duplicates.py} (77%) diff --git a/pype/plugins/standalonepublisher/publish/collect_clips.py b/pype/plugins/standalonepublisher/publish/collect_clips.py new file mode 100644 index 0000000000..e38a70f289 --- /dev/null +++ b/pype/plugins/standalonepublisher/publish/collect_clips.py @@ -0,0 +1,230 @@ +import os + +import opentimelineio as otio +from bson import json_util + +import pyblish.api +from pype import lib as plib +from avalon import io + + +class CollectClips(pyblish.api.InstancePlugin): + """Collect Clips instances from editorial's OTIO sequence""" + + order = pyblish.api.CollectorOrder + 0.01 + label = "Collect Shots" + hosts = ["standalonepublisher"] + families = ["editorial"] + + def process(self, instance): + # get context + context = instance.context + + # create asset_names conversion table + if not context.data.get("assetsShared"): + self.log.debug("Created `assetsShared` in context") + context.data["assetsShared"] = dict() + + # get timeline otio data + timeline = instance.data["otio_timeline"] + fps = plib.get_asset()["data"]["fps"] + + tracks = timeline.each_child( + descended_from_type=otio.schema.track.Track + ) + self.log.debug(f"__ tracks: `{tracks}`") + + # get data from avalon + asset_entity = instance.context.data["assetEntity"] + asset_data = asset_entity["data"] + asset_name = asset_entity["name"] + self.log.debug(f"__ asset_entity: `{asset_entity}`") + + # Project specific prefix naming. This needs to be replaced with some + # options to be more flexible. + asset_name = asset_name.split("_")[0] + + instances = [] + for track in tracks: + self.log.debug(f"__ track: `{track}`") + try: + track_start_frame = ( + abs(track.source_range.start_time.value) + ) + except AttributeError: + track_start_frame = 0 + + self.log.debug(f"__ track: `{track}`") + + for clip in track.each_child(): + # skip all generators like black ampty + if isinstance( + clip.media_reference, + otio.schema.GeneratorReference): + continue + + # Transitions are ignored, because Clips have the full frame + # range. + if isinstance(clip, otio.schema.transition.Transition): + continue + + if clip.name is None: + continue + + # Hardcoded to expect a shot name of "[name].[extension]" + clip_name = os.path.splitext(clip.name)[0].lower() + name = f"{asset_name}_{clip_name}" + + source_in = clip.trimmed_range().start_time.value + clip_in = clip.range_in_parent().start_time.value + clip_out = clip.range_in_parent().end_time_inclusive().value + clip_duration = clip.duration().value + self.log.debug(f"__ source_in: `{source_in}`") + self.log.debug(f"__ clip_in: `{clip_in}`") + self.log.debug(f"__ clip_out: `{clip_out}`") + self.log.debug(f"__ clip_duration: `{clip_duration}`") + + label = f"{name} (framerange: {clip_in}-{clip_out})" + + instances.append( + instance.context.create_instance(**{ + "name": name, + "label": label, + "asset": name, + "subset": "plateRef", + "item": clip, + + # timing properities + "trackStartFrame": track_start_frame, + "sourceIn": source_in, + "sourceOut": source_in + clip_duration, + "clipIn": clip_in, + "clipOut": clip_out, + "clipDuration": clip_duration, + "handleStart": asset_data["handleStart"], + "handleEnd": asset_data["handleEnd"], + "fps": fps, + + # instance properities + "family": "clip", + "families": ["review", "ftrack"], + "ftrackFamily": "review", + "representations": [] + }) + ) + # + # def process_old(self, instance): + # representation = instance.data["representations"][0] + # file_path = os.path.join( + # representation["stagingDir"], representation["files"] + # ) + # instance.context.data["editorialPath"] = file_path + # + # extension = os.path.splitext(file_path)[1][1:] + # kwargs = {} + # if extension == "edl": + # # EDL has no frame rate embedded so needs explicit frame rate else + # # 24 is asssumed. + # kwargs["rate"] = plib.get_asset()["data"]["fps"] + # + # timeline = otio.adapters.read_from_file(file_path, **kwargs) + # tracks = timeline.each_child( + # descended_from_type=otio.schema.track.Track + # ) + # asset_entity = instance.context.data["assetEntity"] + # asset_name = asset_entity["name"] + # + # # Ask user for sequence start. Usually 10:00:00:00. + # sequence_start_frame = 900000 + # + # # Project specific prefix naming. This needs to be replaced with some + # # options to be more flexible. + # asset_name = asset_name.split("_")[0] + # + # instances = [] + # for track in tracks: + # track_start_frame = ( + # abs(track.source_range.start_time.value) - sequence_start_frame + # ) + # for child in track.each_child(): + # # skip all generators like black ampty + # if isinstance( + # child.media_reference, + # otio.schema.GeneratorReference): + # continue + # + # # Transitions are ignored, because Clips have the full frame + # # range. + # if isinstance(child, otio.schema.transition.Transition): + # continue + # + # if child.name is None: + # continue + # + # # Hardcoded to expect a shot name of "[name].[extension]" + # child_name = os.path.splitext(child.name)[0].lower() + # name = f"{asset_name}_{child_name}" + # + # frame_start = track_start_frame + # frame_start += child.range_in_parent().start_time.value + # frame_end = track_start_frame + # frame_end += child.range_in_parent().end_time_inclusive().value + # + # label = f"{name} (framerange: {frame_start}-{frame_end})" + # instances.append( + # instance.context.create_instance(**{ + # "name": name, + # "label": label, + # "frameStart": frame_start, + # "frameEnd": frame_end, + # "family": "shot", + # "families": ["review", "ftrack"], + # "ftrackFamily": "review", + # "asset": name, + # "subset": "shotMain", + # "representations": [], + # "source": file_path + # }) + # ) + # + # visual_hierarchy = [asset_entity] + # while True: + # visual_parent = io.find_one( + # {"_id": visual_hierarchy[-1]["data"]["visualParent"]} + # ) + # if visual_parent: + # visual_hierarchy.append(visual_parent) + # else: + # visual_hierarchy.append(instance.context.data["projectEntity"]) + # break + # + # context_hierarchy = None + # for entity in visual_hierarchy: + # childs = {} + # if context_hierarchy: + # name = context_hierarchy.pop("name") + # childs = {name: context_hierarchy} + # else: + # for instance in instances: + # childs[instance.data["name"]] = { + # "childs": {}, + # "entity_type": "Shot", + # "custom_attributes": { + # "frameStart": instance.data["frameStart"], + # "frameEnd": instance.data["frameEnd"] + # } + # } + # + # context_hierarchy = { + # "entity_type": entity["data"]["entityType"], + # "childs": childs, + # "name": entity["name"] + # } + # + # name = context_hierarchy.pop("name") + # context_hierarchy = {name: context_hierarchy} + # instance.context.data["hierarchyContext"] = context_hierarchy + # self.log.info( + # "Hierarchy:\n" + + # json_util.dumps(context_hierarchy, sort_keys=True, indent=4) + # ) diff --git a/pype/plugins/standalonepublisher/publish/collect_editorial.py b/pype/plugins/standalonepublisher/publish/collect_editorial.py new file mode 100644 index 0000000000..52acead6cc --- /dev/null +++ b/pype/plugins/standalonepublisher/publish/collect_editorial.py @@ -0,0 +1,76 @@ +import os +import opentimelineio as otio +import pyblish.api +from pype import lib as plib +import pype.api + +class OTIO_View(pyblish.api.Action): + """Currently disabled because OTIO requires PySide2. Issue on Qt.py: + https://github.com/PixarAnimationStudios/OpenTimelineIO/issues/289 + """ + + label = "OTIO View" + icon = "wrench" + on = "failed" + + def process(self, context, plugin): + instance = context[0] + representation = instance.data["representations"][0] + file_path = os.path.join( + representation["stagingDir"], representation["files"] + ) + plib._subprocess(["otioview", file_path]) + + +class CollectEditorial(pyblish.api.InstancePlugin): + """Collect Editorial OTIO timeline""" + + order = pyblish.api.CollectorOrder + label = "Collect Editorial" + hosts = ["standalonepublisher"] + families = ["editorial"] + actions = [] + + # presets + extensions = [".mov", ".mp4"] + + def process(self, instance): + self.log.debug(f"__ instance: `{instance}`") + # get representation with editorial file + representation = instance.data["representations"][0] + + # make editorial sequence file path + staging_dir = representation["stagingDir"] + file_path = os.path.join( + staging_dir, representation["files"] + ) + + # get video file path + video_path = None + basename = os.path.splitext(os.path.basename(file_path))[0] + for f in os.listdir(staging_dir): + self.log.debug(f"__ test file: `{f}`") + # filter out by not sharing the same name + if os.path.splitext(f)[0] not in basename: + continue + # filter out by respected extensions + if os.path.splitext(f)[1] not in self.extensions: + continue + video_path = os.path.join( + staging_dir, f + ) + self.log.debug(f"__ video_path: `{video_path}`") + instance.context.data["editorialVideoPath"] = video_path + + # get editorial sequence file into otio timeline object + extension = os.path.splitext(file_path)[1] + kwargs = {} + if extension == ".edl": + # EDL has no frame rate embedded so needs explicit frame rate else + # 24 is asssumed. + kwargs["rate"] = plib.get_asset()["data"]["fps"] + + instance.data["otio_timeline"] = otio.adapters.read_from_file( + file_path, **kwargs) + + self.log.info(f"Added OTIO timeline from: `{file_path}`") diff --git a/pype/plugins/standalonepublisher/publish/collect_frame_ranges.py b/pype/plugins/standalonepublisher/publish/collect_frame_ranges.py new file mode 100644 index 0000000000..5e8292458f --- /dev/null +++ b/pype/plugins/standalonepublisher/publish/collect_frame_ranges.py @@ -0,0 +1,56 @@ +import pyblish.api + + +class CollectClipFrameRanges(pyblish.api.InstancePlugin): + """Collect all frame range data""" + + order = pyblish.api.CollectorOrder + 0.101 + label = "Collect Frame Ranges" + hosts = ["standalonepublisher"] + families = ["clip"] + + # presets + start_frame_offset = None # if 900000 for edl default then -900000 + custom_start_frame = None + + def process(self, instance): + + data = dict() + + # Timeline data. + handle_start = instance.data["handleStart"] + handle_end = instance.data["handleEnd"] + + source_in_h = instance.data("sourceInH", + instance.data("sourceIn") - handle_start) + source_out_h = instance.data("sourceOutH", + instance.data("sourceOut") + handle_end) + + timeline_in = instance.data["clipIn"] + timeline_out = instance.data["clipOut"] + + timeline_in_h = timeline_in - handle_start + timeline_out_h = timeline_out + handle_end + + # define starting frame for future shot + frame_start = self.custom_start_frame or timeline_in + + # add offset in case there is any + if self.start_frame_offset: + frame_start += self.start_frame_offset + + frame_end = frame_start + (timeline_out - timeline_in) + + data.update({ + "sourceInH": source_in_h, + "sourceOutH": source_out_h, + "frameStart": frame_start, + "frameEnd": frame_end, + "clipInH": timeline_in_h, + "clipOutH": timeline_out_h, + "clipDurationH": instance.data.get( + "clipDuration") + handle_start + handle_end + } + ) + self.log.debug("__ data: {}".format(data)) + instance.data.update(data) diff --git a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py new file mode 100644 index 0000000000..3a49b499da --- /dev/null +++ b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py @@ -0,0 +1,358 @@ +import pyblish.api +import avalon.api as avalon +import re +import os + + +class CollectHierarchyInstance(pyblish.api.InstancePlugin): + """Collecting hierarchy context from `parents` and `hierarchy` data + present in `clip` family instances coming from the request json data file + + It will add `hierarchical_context` into each instance for integrate + plugins to be able to create needed parents for the context if they + don't exist yet + """ + + label = "Collect Hierarchy Clip" + order = pyblish.api.CollectorOrder + 0.101 + hosts = ["standalonepublisher"] + families = ["clip"] + + # presets + search_patterns = { + "sequence": r"sc\d{3}", + "shot": r"sh\d{3}", + "episode": r"ep\d{2}" + } + shot_name_template = "{project[code]}{episode}{clip_name}" + shot_hierarchy = "{episode}{sequence}/{clip_name}" + shot_tasks = ["Animation", "Layout"] + + def convert_to_entity(self, key, value): + # ftrack compatible entity types + types = {"shot": "Shot", + "folder": "Folder", + "episode": "Episode", + "sequence": "Sequence", + "track": "Sequence", + } + # convert to entity type + entity_type = types.get(key, None) + + # return if any + if entity_type: + return {"entityType": entity_type, "entityName": value} + + def process(self, instance): + search_text = "" + context = instance.context + anatomy_data = context.data["anatomyData"] + asset_entity = context.data["assetEntity"] + asset_name = asset_entity["name"] + assets_shared = context.data.get("assetsShared") + + clip = instance.data["item"] + clip_name = os.path.splitext(clip.name)[0].lower() + asset = instance.data["asset"] + + clip_in = instance.data["clipIn"] + clip_out = instance.data["clipOut"] + fps = instance.data["fps"] + + hierarchy_data = dict(anatomy_data) + if self.search_patterns: + search_text += clip_name + asset_name + hierarchy_data.update({"clip_name": clip_name}) + for type, pattern in self.search_patterns.items(): + p = re.compile(pattern) + match = p.findall(search_text) + if not match: + continue + hierarchy_data[type] = match[-1] + + self.log.debug("__ hierarchy_data: {}".format(hierarchy_data)) + shot_name = self.shot_name_template.format(**hierarchy_data) + self.log.debug("__ shot_name: {}".format(shot_name)) + shot_hierarchy = self.shot_hierarchy.format(**hierarchy_data) + self.log.debug("__ shot_hierarchy: {}".format(shot_hierarchy)) + + # # build data for inner nukestudio project property + # data = { + # "sequence": ( + # context.data['activeSequence'].name().replace(' ', '_') + # ), + # "track": clip.parent().name().replace(' ', '_'), + # "clip": asset + # } + # self.log.debug("__ data: {}".format(data)) + # + # # Check for clips with the same range + # # this is for testing if any vertically neighbouring + # # clips has been already processed + # match = next(( + # k for k, v in assets_shared.items() + # if (v["_clipIn"] == clip_in) + # and (v["_clipOut"] == clip_out) + # ), False) + # + # self.log.debug( + # "__ assets_shared[match]: {}".format( + # assets_shared[match])) + # + # # check if hierarchy key is present in matched + # # vertically neighbouring clip + # if not assets_shared[match].get("hierarchy"): + # match = False + # + # # rise exception if multiple hierarchy tag found + # assert not match, ( + # "Two clips above each other with" + # " hierarchy tag are not allowed" + # " >> keep hierarchy tag only in one of them <<" + # ) + # + # d_metadata = dict() + # parents = list() + # + # # main template from Tag.note + # template = t_note + # + # # if shot in template then remove it + # if "shot" in template.lower(): + # instance.data["asset"] = [ + # t for t in template.split('/')][-1] + # template = "/".join( + # [t for t in template.split('/')][0:-1]) + # + # # take template from Tag.note and break it into parts + # template_split = template.split("/") + # patern = re.compile(r"\{([a-z]*?)\}") + # par_split = [patern.findall(t) + # for t in template.split("/")] + # + # # format all {} in two layers + # for k, v in t_metadata.items(): + # new_k = k.split(".")[1] + # + # # ignore all help strings + # if 'help' in k: + # continue + # # self.log.info("__ new_k: `{}`".format(new_k)) + # try: + # # first try all data and context data to + # # add to individual properties + # new_v = str(v).format( + # **dict(context.data, **data)) + # d_metadata[new_k] = new_v + # + # # create parents + # # find matching index of order + # p_match_i = [i for i, p in enumerate(par_split) + # if new_k in p] + # + # # if any is matching then convert to entity_types + # if p_match_i: + # parent = self.convert_to_entity( + # new_k, template_split[p_match_i[0]]) + # parents.insert(p_match_i[0], parent) + # except Exception: + # d_metadata[new_k] = v + # + # # create new shot asset name + # instance.data["asset"] = instance.data["asset"].format( + # **d_metadata) + # self.log.debug( + # "__ instance.data[asset]: " + # "{}".format(instance.data["asset"]) + # ) + # + # # lastly fill those individual properties itno + # # format the string with collected data + # parents = [{"entityName": p["entityName"].format( + # **d_metadata), "entityType": p["entityType"]} + # for p in parents] + # self.log.debug("__ parents: {}".format(parents)) + # + # hierarchy = template.format( + # **d_metadata) + # self.log.debug("__ hierarchy: {}".format(hierarchy)) + # + # # check if hierarchy attribute is already created + # # it should not be so return warning if it is + # hd = instance.data.get("hierarchy") + # assert not hd, ( + # "Only one Hierarchy Tag is allowed. " + # "Clip: `{}`".format(asset) + # ) + # + # # add formated hierarchy path into instance data + # instance.data["hierarchy"] = hierarchy + # instance.data["parents"] = parents + # + # self.log.info( + # "clip: {asset}[{clip_in}:{clip_out}]".format( + # **locals())) + # # adding to asset shared dict + # self.log.debug( + # "__ assets_shared: {}".format(assets_shared)) + # if assets_shared.get(asset): + # self.log.debug("Adding to shared assets: `{}`".format( + # asset)) + # asset_shared = assets_shared.get(asset) + # else: + # asset_shared = assets_shared[asset] + # + # asset_shared.update({ + # "asset": asset, + # "hierarchy": hierarchy, + # "parents": parents, + # "fps": fps, + # "tasks": instance.data["tasks"] + # }) + # + # # adding frame start if any on instance + # start_frame = instance.data.get("startingFrame") + # if start_frame: + # asset_shared.update({ + # "startingFrame": start_frame + # }) + # self.log.debug( + # "assets_shared: {assets_shared}".format(**locals())) + + +class CollectHierarchyContext(pyblish.api.ContextPlugin): + '''Collecting Hierarchy from instaces and building + context hierarchy tree + ''' + + label = "Collect Hierarchy Context" + order = pyblish.api.CollectorOrder + 0.102 + hosts = ["standalonepublisher"] + + def update_dict(self, ex_dict, new_dict): + for key in ex_dict: + if key in new_dict and isinstance(ex_dict[key], dict): + new_dict[key] = self.update_dict(ex_dict[key], new_dict[key]) + else: + if ex_dict.get(key) and new_dict.get(key): + continue + else: + new_dict[key] = ex_dict[key] + + return new_dict + + def process(self, context): + instances = context[:] + + # create hierarchyContext attr if context has none + + temp_context = {} + for instance in instances: + if 'projectfile' in instance.data.get('family', ''): + continue + + name = instance.data["asset"] + + # get handles + handle_start = int(instance.data["handleStart"]) + handle_end = int(instance.data["handleEnd"]) + + # inject assetsShared to other plates types + assets_shared = context.data.get("assetsShared") + + if assets_shared: + s_asset_data = assets_shared.get(name) + if s_asset_data: + self.log.debug("__ s_asset_data: {}".format(s_asset_data)) + name = instance.data["asset"] = s_asset_data["asset"] + instance.data["parents"] = s_asset_data["parents"] + instance.data["hierarchy"] = s_asset_data["hierarchy"] + instance.data["tasks"] = s_asset_data["tasks"] + instance.data["resolutionWidth"] = s_asset_data[ + "resolutionWidth"] + instance.data["resolutionHeight"] = s_asset_data[ + "resolutionHeight"] + instance.data["pixelAspect"] = s_asset_data["pixelAspect"] + instance.data["fps"] = s_asset_data["fps"] + + # adding frame start if any on instance + start_frame = s_asset_data.get("startingFrame") + if start_frame: + instance.data["frameStart"] = start_frame + instance.data["frameEnd"] = start_frame + ( + instance.data["clipOut"] - + instance.data["clipIn"]) + + + + self.log.debug( + "__ instance.data[parents]: {}".format( + instance.data["parents"] + ) + ) + self.log.debug( + "__ instance.data[hierarchy]: {}".format( + instance.data["hierarchy"] + ) + ) + self.log.debug( + "__ instance.data[name]: {}".format(instance.data["name"]) + ) + + in_info = {} + + in_info["inputs"] = [ + x["_id"] for x in instance.data.get("assetbuilds", []) + ] + + # suppose that all instances are Shots + in_info['entity_type'] = 'Shot' + + # get custom attributes of the shot + if instance.data.get("main"): + in_info['custom_attributes'] = { + "handleStart": handle_start, + "handleEnd": handle_end, + "frameStart": instance.data["frameStart"], + "frameEnd": instance.data["frameEnd"], + "clipIn": instance.data["clipIn"], + "clipOut": instance.data["clipOut"], + 'fps': instance.context.data["fps"] + } + + # adding SourceResolution if Tag was present + if instance.data.get("main"): + in_info['custom_attributes'].update({ + "resolutionWidth": instance.data["resolutionWidth"], + "resolutionHeight": instance.data["resolutionHeight"], + "pixelAspect": instance.data["pixelAspect"] + }) + + in_info['tasks'] = instance.data['tasks'] + + parents = instance.data.get('parents', []) + self.log.debug("__ in_info: {}".format(in_info)) + + actual = {name: in_info} + + for parent in reversed(parents): + next_dict = {} + parent_name = parent["entityName"] + next_dict[parent_name] = {} + next_dict[parent_name]["entity_type"] = parent["entityType"] + next_dict[parent_name]["childs"] = actual + actual = next_dict + + temp_context = self.update_dict(temp_context, actual) + + # TODO: 100% sure way of get project! Will be Name or Code? + project_name = avalon.Session["AVALON_PROJECT"] + final_context = {} + final_context[project_name] = {} + final_context[project_name]['entity_type'] = 'Project' + final_context[project_name]['childs'] = temp_context + + # adding hierarchy context to instance + context.data["hierarchyContext"] = final_context + self.log.debug("context.data[hierarchyContext] is: {}".format( + context.data["hierarchyContext"])) diff --git a/pype/plugins/standalonepublisher/publish/collect_shot_names.py b/pype/plugins/standalonepublisher/publish/collect_shot_names.py new file mode 100644 index 0000000000..7976c855d3 --- /dev/null +++ b/pype/plugins/standalonepublisher/publish/collect_shot_names.py @@ -0,0 +1,15 @@ +import pyblish.api +import re + + +class CollectShotNames(pyblish.api.InstancePlugin): + """ + Collecting shot names + """ + + label = "Collect shot names" + order = pyblish.api.CollectorOrder + 0.01 + hosts = ["standalonepublisher"] + + def process(self, instance): + self.log.info("Instance name: `{}`".format(instance.data["name"])) diff --git a/pype/plugins/standalonepublisher/publish/collect_shots.py b/pype/plugins/standalonepublisher/publish/collect_shots.py deleted file mode 100644 index 4f682bd808..0000000000 --- a/pype/plugins/standalonepublisher/publish/collect_shots.py +++ /dev/null @@ -1,147 +0,0 @@ -import os - -import opentimelineio as otio -from bson import json_util - -import pyblish.api -from pype import lib -from avalon import io - - -class OTIO_View(pyblish.api.Action): - """Currently disabled because OTIO requires PySide2. Issue on Qt.py: - https://github.com/PixarAnimationStudios/OpenTimelineIO/issues/289 - """ - - label = "OTIO View" - icon = "wrench" - on = "failed" - - def process(self, context, plugin): - instance = context[0] - representation = instance.data["representations"][0] - file_path = os.path.join( - representation["stagingDir"], representation["files"] - ) - lib._subprocess(["otioview", file_path]) - - -class CollectShots(pyblish.api.InstancePlugin): - """Collect Anatomy object into Context""" - - order = pyblish.api.CollectorOrder - label = "Collect Shots" - hosts = ["standalonepublisher"] - families = ["editorial"] - actions = [] - - def process(self, instance): - representation = instance.data["representations"][0] - file_path = os.path.join( - representation["stagingDir"], representation["files"] - ) - instance.context.data["editorialPath"] = file_path - - extension = os.path.splitext(file_path)[1][1:] - kwargs = {} - if extension == "edl": - # EDL has no frame rate embedded so needs explicit frame rate else - # 24 is asssumed. - kwargs["rate"] = lib.get_asset()["data"]["fps"] - - timeline = otio.adapters.read_from_file(file_path, **kwargs) - tracks = timeline.each_child( - descended_from_type=otio.schema.track.Track - ) - asset_entity = instance.context.data["assetEntity"] - asset_name = asset_entity["name"] - - # Ask user for sequence start. Usually 10:00:00:00. - sequence_start_frame = 900000 - - # Project specific prefix naming. This needs to be replaced with some - # options to be more flexible. - asset_name = asset_name.split("_")[0] - - instances = [] - for track in tracks: - track_start_frame = ( - abs(track.source_range.start_time.value) - sequence_start_frame - ) - for child in track.each_child(): - - # Transitions are ignored, because Clips have the full frame - # range. - if isinstance(child, otio.schema.transition.Transition): - continue - - if child.name is None: - continue - - # Hardcoded to expect a shot name of "[name].[extension]" - child_name = os.path.splitext(child.name)[0].lower() - name = f"{asset_name}_{child_name}" - - frame_start = track_start_frame - frame_start += child.range_in_parent().start_time.value - frame_end = track_start_frame - frame_end += child.range_in_parent().end_time_inclusive().value - - label = f"{name} (framerange: {frame_start}-{frame_end})" - instances.append( - instance.context.create_instance(**{ - "name": name, - "label": label, - "frameStart": frame_start, - "frameEnd": frame_end, - "family": "shot", - "families": ["review", "ftrack"], - "ftrackFamily": "review", - "asset": name, - "subset": "shotMain", - "representations": [], - "source": file_path - }) - ) - - visual_hierarchy = [asset_entity] - while True: - visual_parent = io.find_one( - {"_id": visual_hierarchy[-1]["data"]["visualParent"]} - ) - if visual_parent: - visual_hierarchy.append(visual_parent) - else: - visual_hierarchy.append(instance.context.data["projectEntity"]) - break - - context_hierarchy = None - for entity in visual_hierarchy: - childs = {} - if context_hierarchy: - name = context_hierarchy.pop("name") - childs = {name: context_hierarchy} - else: - for instance in instances: - childs[instance.data["name"]] = { - "childs": {}, - "entity_type": "Shot", - "custom_attributes": { - "frameStart": instance.data["frameStart"], - "frameEnd": instance.data["frameEnd"] - } - } - - context_hierarchy = { - "entity_type": entity["data"]["entityType"], - "childs": childs, - "name": entity["name"] - } - - name = context_hierarchy.pop("name") - context_hierarchy = {name: context_hierarchy} - instance.context.data["hierarchyContext"] = context_hierarchy - self.log.info( - "Hierarchy:\n" + - json_util.dumps(context_hierarchy, sort_keys=True, indent=4) - ) diff --git a/pype/plugins/standalonepublisher/publish/extract_shot.py b/pype/plugins/standalonepublisher/publish/extract_shot.py index d58ddfe8d5..e1e69d993b 100644 --- a/pype/plugins/standalonepublisher/publish/extract_shot.py +++ b/pype/plugins/standalonepublisher/publish/extract_shot.py @@ -1,9 +1,7 @@ import os - import clique - import pype.api -import pype.lib +import pype.lib as plib class ExtractShot(pype.api.Extractor): @@ -11,42 +9,52 @@ class ExtractShot(pype.api.Extractor): label = "Extract Shot" hosts = ["standalonepublisher"] - families = ["shot"] + families = ["clip"] def process(self, instance): - staging_dir = self.staging_dir(instance) - self.log.info("Outputting shot to {}".format(staging_dir)) + # get context + context = instance.context - editorial_path = instance.context.data["editorialPath"] - basename = os.path.splitext(os.path.basename(editorial_path))[0] + # get ffmpet path + ffmpeg_path = pype.lib.get_ffmpeg_tool_path("ffmpeg") + + # get staging dir + staging_dir = self.staging_dir(instance) + self.log.info("Staging dir set to: `{}`".format(staging_dir)) # Generate mov file. - fps = pype.lib.get_asset()["data"]["fps"] - input_path = os.path.join( - os.path.dirname(editorial_path), basename + ".mov" - ) - shot_mov = os.path.join(staging_dir, instance.data["name"] + ".mov") - ffmpeg_path = pype.lib.get_ffmpeg_tool_path("ffmpeg") + fps = instance.data["fps"] + video_file_path = context.data["editorialVideoPath"] + ext = os.path.splitext(os.path.basename(video_file_path))[-1] + + clip_trimed_path = os.path.join( + staging_dir, instance.data["name"] + ext) + + # check video file metadata + input_data = plib.ffprobe_streams(video_file_path)[0] + self.log.debug(f"__ input_data: `{input_data}`") + args = [ ffmpeg_path, - "-ss", str(instance.data["frameStart"] / fps), - "-i", input_path, + "-ss", str(instance.data["clipIn"] / fps), + "-i", video_file_path, "-t", str( - (instance.data["frameEnd"] - instance.data["frameStart"] + 1) / + (instance.data["clipOut"] - instance.data["clipIn"] + 1) / fps ), "-crf", "18", "-pix_fmt", "yuv420p", - shot_mov + clip_trimed_path ] self.log.info(f"Processing: {args}") - output = pype.lib._subprocess(args) + ffmpeg_args = " ".join(args) + output = pype.api.subprocess(ffmpeg_args) self.log.info(output) instance.data["representations"].append({ - "name": "mov", - "ext": "mov", - "files": os.path.basename(shot_mov), + "name": ext[1:], + "ext": ext[1:], + "files": os.path.basename(clip_trimed_path), "stagingDir": staging_dir, "frameStart": instance.data["frameStart"], "frameEnd": instance.data["frameEnd"], @@ -55,42 +63,41 @@ class ExtractShot(pype.api.Extractor): "tags": ["review", "ftrackreview"] }) - # Generate jpegs. - shot_jpegs = os.path.join( - staging_dir, instance.data["name"] + ".%04d.jpeg" - ) - args = [ffmpeg_path, "-i", shot_mov, shot_jpegs] - self.log.info(f"Processing: {args}") - output = pype.lib._subprocess(args) - self.log.info(output) - - collection = clique.Collection( - head=instance.data["name"] + ".", tail='.jpeg', padding=4 - ) - for f in os.listdir(staging_dir): - if collection.match(f): - collection.add(f) - - instance.data["representations"].append({ - "name": "jpeg", - "ext": "jpeg", - "files": list(collection), - "stagingDir": staging_dir - }) - - # Generate wav file. - shot_wav = os.path.join(staging_dir, instance.data["name"] + ".wav") - args = [ffmpeg_path, "-i", shot_mov, shot_wav] - self.log.info(f"Processing: {args}") - output = pype.lib._subprocess(args) - self.log.info(output) - - instance.data["representations"].append({ - "name": "wav", - "ext": "wav", - "files": os.path.basename(shot_wav), - "stagingDir": staging_dir - }) - - # Required for extract_review plugin (L222 onwards). - instance.data["fps"] = fps + # # Generate jpegs. + # clip_thumbnail = os.path.join( + # staging_dir, instance.data["name"] + ".%04d.jpeg" + # ) + # args = [ffmpeg_path, "-i", clip_trimed_path, clip_thumbnail] + # self.log.info(f"Processing: {args}") + # output = pype.lib._subprocess(args) + # self.log.info(output) + # + # # collect jpeg sequence if editorial data for publish + # # are image sequence + # collection = clique.Collection( + # head=instance.data["name"] + ".", tail='.jpeg', padding=4 + # ) + # for f in os.listdir(staging_dir): + # if collection.match(f): + # collection.add(f) + # + # instance.data["representations"].append({ + # "name": "jpeg", + # "ext": "jpeg", + # "files": list(collection), + # "stagingDir": staging_dir + # }) + # + # # Generate wav file. + # shot_wav = os.path.join(staging_dir, instance.data["name"] + ".wav") + # args = [ffmpeg_path, "-i", clip_trimed_path, shot_wav] + # self.log.info(f"Processing: {args}") + # output = pype.lib._subprocess(args) + # self.log.info(output) + # + # instance.data["representations"].append({ + # "name": "wav", + # "ext": "wav", + # "files": os.path.basename(shot_wav), + # "stagingDir": staging_dir + # }) diff --git a/pype/plugins/standalonepublisher/publish/validate_clips.py b/pype/plugins/standalonepublisher/publish/validate_clips.py new file mode 100644 index 0000000000..35b81da5c1 --- /dev/null +++ b/pype/plugins/standalonepublisher/publish/validate_clips.py @@ -0,0 +1,24 @@ + # Check for clips with the same range + # this is for testing if any vertically neighbouring + # clips has been already processed + clip_matching_with_range = next( + (k for k, v in context.data["assetsShared"].items() + if (v.get("_clipIn", 0) == clip_in) + and (v.get("_clipOut", 0) == clip_out) + ), False) + + # check if clip name is the same in matched + # vertically neighbouring clip + # if it is then it is correct and resent variable to False + # not to be rised wrong name exception + if asset in str(clip_matching_with_range): + clip_matching_with_range = False + + # rise wrong name exception if found one + assert (not clip_matching_with_range), ( + "matching clip: {asset}" + " timeline range ({clip_in}:{clip_out})" + " conflicting with {clip_matching_with_range}" + " >> rename any of clips to be the same as the other <<" + ).format( + **locals()) diff --git a/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py b/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py index 961641b8fa..e4d2279317 100644 --- a/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py +++ b/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py @@ -12,7 +12,11 @@ class ValidateEditorialResources(pyblish.api.InstancePlugin): families = ["editorial"] order = pype.api.ValidateContentsOrder + # presets + check_ext = None + def process(self, instance): + check_ext = self.check_ext or "mov" representation = instance.data["representations"][0] staging_dir = representation["stagingDir"] basename = os.path.splitext( @@ -21,8 +25,8 @@ class ValidateEditorialResources(pyblish.api.InstancePlugin): files = [x for x in os.listdir(staging_dir)] - # Check for "mov" file. - filename = basename + ".mov" + # Check for correct extansion in file name. + filename = basename + check_ext filepath = os.path.join(staging_dir, filename) msg = f"Missing \"{filepath}\"." assert filename in files, msg diff --git a/pype/plugins/standalonepublisher/publish/validate_shots.py b/pype/plugins/standalonepublisher/publish/validate_shot_duplicates.py similarity index 77% rename from pype/plugins/standalonepublisher/publish/validate_shots.py rename to pype/plugins/standalonepublisher/publish/validate_shot_duplicates.py index 3267af7685..04d2f3ea6c 100644 --- a/pype/plugins/standalonepublisher/publish/validate_shots.py +++ b/pype/plugins/standalonepublisher/publish/validate_shot_duplicates.py @@ -2,10 +2,10 @@ import pyblish.api import pype.api -class ValidateShots(pyblish.api.ContextPlugin): - """Validate there is a "mov" next to the editorial file.""" +class ValidateShotDuplicates(pyblish.api.ContextPlugin): + """Validating no duplicate names are in context.""" - label = "Validate Shots" + label = "Validate Shot Duplicates" hosts = ["standalonepublisher"] order = pype.api.ValidateContentsOrder From 9af892eceed4ad0aa943e1bf7d52ff28cd4c5905 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 4 Aug 2020 11:46:01 +0200 Subject: [PATCH 02/30] feat(sp): adding hierarchy and framerange colectors --- .../publish/collect_clips.py | 242 ++++++------- .../publish/collect_editorial.py | 66 ++-- .../publish/collect_frame_ranges.py | 2 +- .../publish/collect_hierarchy.py | 326 ++++++------------ .../publish/collect_shot_names.py | 15 - .../publish/extract_shot.py | 7 +- .../publish/validate_editorial_resources.py | 22 +- 7 files changed, 274 insertions(+), 406 deletions(-) delete mode 100644 pype/plugins/standalonepublisher/publish/collect_shot_names.py diff --git a/pype/plugins/standalonepublisher/publish/collect_clips.py b/pype/plugins/standalonepublisher/publish/collect_clips.py index e38a70f289..d8eb1d72c4 100644 --- a/pype/plugins/standalonepublisher/publish/collect_clips.py +++ b/pype/plugins/standalonepublisher/publish/collect_clips.py @@ -12,7 +12,7 @@ class CollectClips(pyblish.api.InstancePlugin): """Collect Clips instances from editorial's OTIO sequence""" order = pyblish.api.CollectorOrder + 0.01 - label = "Collect Shots" + label = "Collect Clips" hosts = ["standalonepublisher"] families = ["editorial"] @@ -40,8 +40,7 @@ class CollectClips(pyblish.api.InstancePlugin): asset_name = asset_entity["name"] self.log.debug(f"__ asset_entity: `{asset_entity}`") - # Project specific prefix naming. This needs to be replaced with some - # options to be more flexible. + # split selected context asset name asset_name = asset_name.split("_")[0] instances = [] @@ -71,7 +70,6 @@ class CollectClips(pyblish.api.InstancePlugin): if clip.name is None: continue - # Hardcoded to expect a shot name of "[name].[extension]" clip_name = os.path.splitext(clip.name)[0].lower() name = f"{asset_name}_{clip_name}" @@ -109,122 +107,124 @@ class CollectClips(pyblish.api.InstancePlugin): "family": "clip", "families": ["review", "ftrack"], "ftrackFamily": "review", - "representations": [] + "representations": [], + "editorialVideoPath": instance.data[ + "editorialVideoPath"] }) ) - # - # def process_old(self, instance): - # representation = instance.data["representations"][0] - # file_path = os.path.join( - # representation["stagingDir"], representation["files"] - # ) - # instance.context.data["editorialPath"] = file_path - # - # extension = os.path.splitext(file_path)[1][1:] - # kwargs = {} - # if extension == "edl": - # # EDL has no frame rate embedded so needs explicit frame rate else - # # 24 is asssumed. - # kwargs["rate"] = plib.get_asset()["data"]["fps"] - # - # timeline = otio.adapters.read_from_file(file_path, **kwargs) - # tracks = timeline.each_child( - # descended_from_type=otio.schema.track.Track - # ) - # asset_entity = instance.context.data["assetEntity"] - # asset_name = asset_entity["name"] - # - # # Ask user for sequence start. Usually 10:00:00:00. - # sequence_start_frame = 900000 - # - # # Project specific prefix naming. This needs to be replaced with some - # # options to be more flexible. - # asset_name = asset_name.split("_")[0] - # - # instances = [] - # for track in tracks: - # track_start_frame = ( - # abs(track.source_range.start_time.value) - sequence_start_frame - # ) - # for child in track.each_child(): - # # skip all generators like black ampty - # if isinstance( - # child.media_reference, - # otio.schema.GeneratorReference): - # continue - # - # # Transitions are ignored, because Clips have the full frame - # # range. - # if isinstance(child, otio.schema.transition.Transition): - # continue - # - # if child.name is None: - # continue - # - # # Hardcoded to expect a shot name of "[name].[extension]" - # child_name = os.path.splitext(child.name)[0].lower() - # name = f"{asset_name}_{child_name}" - # - # frame_start = track_start_frame - # frame_start += child.range_in_parent().start_time.value - # frame_end = track_start_frame - # frame_end += child.range_in_parent().end_time_inclusive().value - # - # label = f"{name} (framerange: {frame_start}-{frame_end})" - # instances.append( - # instance.context.create_instance(**{ - # "name": name, - # "label": label, - # "frameStart": frame_start, - # "frameEnd": frame_end, - # "family": "shot", - # "families": ["review", "ftrack"], - # "ftrackFamily": "review", - # "asset": name, - # "subset": "shotMain", - # "representations": [], - # "source": file_path - # }) - # ) - # - # visual_hierarchy = [asset_entity] - # while True: - # visual_parent = io.find_one( - # {"_id": visual_hierarchy[-1]["data"]["visualParent"]} - # ) - # if visual_parent: - # visual_hierarchy.append(visual_parent) - # else: - # visual_hierarchy.append(instance.context.data["projectEntity"]) - # break - # - # context_hierarchy = None - # for entity in visual_hierarchy: - # childs = {} - # if context_hierarchy: - # name = context_hierarchy.pop("name") - # childs = {name: context_hierarchy} - # else: - # for instance in instances: - # childs[instance.data["name"]] = { - # "childs": {}, - # "entity_type": "Shot", - # "custom_attributes": { - # "frameStart": instance.data["frameStart"], - # "frameEnd": instance.data["frameEnd"] - # } - # } - # - # context_hierarchy = { - # "entity_type": entity["data"]["entityType"], - # "childs": childs, - # "name": entity["name"] - # } - # - # name = context_hierarchy.pop("name") - # context_hierarchy = {name: context_hierarchy} - # instance.context.data["hierarchyContext"] = context_hierarchy - # self.log.info( - # "Hierarchy:\n" + - # json_util.dumps(context_hierarchy, sort_keys=True, indent=4) - # ) + + def process_old(self, instance): + representation = instance.data["representations"][0] + file_path = os.path.join( + representation["stagingDir"], representation["files"] + ) + instance.context.data["editorialPath"] = file_path + + extension = os.path.splitext(file_path)[1][1:] + kwargs = {} + if extension == "edl": + # EDL has no frame rate embedded so needs explicit frame rate else + # 24 is asssumed. + kwargs["rate"] = plib.get_asset()["data"]["fps"] + + timeline = otio.adapters.read_from_file(file_path, **kwargs) + tracks = timeline.each_child( + descended_from_type=otio.schema.track.Track + ) + asset_entity = instance.context.data["assetEntity"] + asset_name = asset_entity["name"] + + # Ask user for sequence start. Usually 10:00:00:00. + sequence_start_frame = 900000 + + # Project specific prefix naming. This needs to be replaced with some + # options to be more flexible. + asset_name = asset_name.split("_")[0] + + instances = [] + for track in tracks: + track_start_frame = ( + abs(track.source_range.start_time.value) - sequence_start_frame + ) + for child in track.each_child(): + # skip all generators like black ampty + if isinstance( + child.media_reference, + otio.schema.GeneratorReference): + continue + + # Transitions are ignored, because Clips have the full frame + # range. + if isinstance(child, otio.schema.transition.Transition): + continue + + if child.name is None: + continue + + # Hardcoded to expect a shot name of "[name].[extension]" + child_name = os.path.splitext(child.name)[0].lower() + name = f"{asset_name}_{child_name}" + + frame_start = track_start_frame + frame_start += child.range_in_parent().start_time.value + frame_end = track_start_frame + frame_end += child.range_in_parent().end_time_inclusive().value + + label = f"{name} (framerange: {frame_start}-{frame_end})" + instances.append( + instance.context.create_instance(**{ + "name": name, + "label": label, + "frameStart": frame_start, + "frameEnd": frame_end, + "family": "shot", + "families": ["review", "ftrack"], + "ftrackFamily": "review", + "asset": name, + "subset": "shotMain", + "representations": [], + "source": file_path + }) + ) + + visual_hierarchy = [asset_entity] + while True: + visual_parent = io.find_one( + {"_id": visual_hierarchy[-1]["data"]["visualParent"]} + ) + if visual_parent: + visual_hierarchy.append(visual_parent) + else: + visual_hierarchy.append(instance.context.data["projectEntity"]) + break + + context_hierarchy = None + for entity in visual_hierarchy: + childs = {} + if context_hierarchy: + name = context_hierarchy.pop("name") + childs = {name: context_hierarchy} + else: + for instance in instances: + childs[instance.data["name"]] = { + "childs": {}, + "entity_type": "Shot", + "custom_attributes": { + "frameStart": instance.data["frameStart"], + "frameEnd": instance.data["frameEnd"] + } + } + + context_hierarchy = { + "entity_type": entity["data"]["entityType"], + "childs": childs, + "name": entity["name"] + } + + name = context_hierarchy.pop("name") + context_hierarchy = {name: context_hierarchy} + instance.context.data["hierarchyContext"] = context_hierarchy + self.log.info( + "Hierarchy:\n" + + json_util.dumps(context_hierarchy, sort_keys=True, indent=4) + ) diff --git a/pype/plugins/standalonepublisher/publish/collect_editorial.py b/pype/plugins/standalonepublisher/publish/collect_editorial.py index 52acead6cc..baade27ba8 100644 --- a/pype/plugins/standalonepublisher/publish/collect_editorial.py +++ b/pype/plugins/standalonepublisher/publish/collect_editorial.py @@ -37,40 +37,40 @@ class CollectEditorial(pyblish.api.InstancePlugin): def process(self, instance): self.log.debug(f"__ instance: `{instance}`") # get representation with editorial file - representation = instance.data["representations"][0] - - # make editorial sequence file path - staging_dir = representation["stagingDir"] - file_path = os.path.join( - staging_dir, representation["files"] - ) - - # get video file path - video_path = None - basename = os.path.splitext(os.path.basename(file_path))[0] - for f in os.listdir(staging_dir): - self.log.debug(f"__ test file: `{f}`") - # filter out by not sharing the same name - if os.path.splitext(f)[0] not in basename: - continue - # filter out by respected extensions - if os.path.splitext(f)[1] not in self.extensions: - continue - video_path = os.path.join( - staging_dir, f + for representation in instance.data["representations"]: + self.log.debug(f"__ representation: `{representation}`") + # make editorial sequence file path + staging_dir = representation["stagingDir"] + file_path = os.path.join( + staging_dir, str(representation["files"]) ) - self.log.debug(f"__ video_path: `{video_path}`") - instance.context.data["editorialVideoPath"] = video_path - # get editorial sequence file into otio timeline object - extension = os.path.splitext(file_path)[1] - kwargs = {} - if extension == ".edl": - # EDL has no frame rate embedded so needs explicit frame rate else - # 24 is asssumed. - kwargs["rate"] = plib.get_asset()["data"]["fps"] + # get video file path + video_path = None + basename = os.path.splitext(os.path.basename(file_path))[0] + for f in os.listdir(staging_dir): + self.log.debug(f"__ test file: `{f}`") + # filter out by not sharing the same name + if os.path.splitext(f)[0] not in basename: + continue + # filter out by respected extensions + if os.path.splitext(f)[1] not in self.extensions: + continue + video_path = os.path.join( + staging_dir, f + ) + self.log.debug(f"__ video_path: `{video_path}`") + instance.data["editorialVideoPath"] = video_path - instance.data["otio_timeline"] = otio.adapters.read_from_file( - file_path, **kwargs) + # get editorial sequence file into otio timeline object + extension = os.path.splitext(file_path)[1] + kwargs = {} + if extension == ".edl": + # EDL has no frame rate embedded so needs explicit frame rate else + # 24 is asssumed. + kwargs["rate"] = plib.get_asset()["data"]["fps"] - self.log.info(f"Added OTIO timeline from: `{file_path}`") + instance.data["otio_timeline"] = otio.adapters.read_from_file( + file_path, **kwargs) + + self.log.info(f"Added OTIO timeline from: `{file_path}`") diff --git a/pype/plugins/standalonepublisher/publish/collect_frame_ranges.py b/pype/plugins/standalonepublisher/publish/collect_frame_ranges.py index 5e8292458f..c58cac3505 100644 --- a/pype/plugins/standalonepublisher/publish/collect_frame_ranges.py +++ b/pype/plugins/standalonepublisher/publish/collect_frame_ranges.py @@ -11,7 +11,7 @@ class CollectClipFrameRanges(pyblish.api.InstancePlugin): # presets start_frame_offset = None # if 900000 for edl default then -900000 - custom_start_frame = None + custom_start_frame = 1 def process(self, instance): diff --git a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py index 3a49b499da..7c968ce302 100644 --- a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py +++ b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py @@ -2,7 +2,7 @@ import pyblish.api import avalon.api as avalon import re import os - +from avalon import io class CollectHierarchyInstance(pyblish.api.InstancePlugin): """Collecting hierarchy context from `parents` and `hierarchy` data @@ -19,14 +19,19 @@ class CollectHierarchyInstance(pyblish.api.InstancePlugin): families = ["clip"] # presets - search_patterns = { - "sequence": r"sc\d{3}", - "shot": r"sh\d{3}", - "episode": r"ep\d{2}" + shot_rename_template = "{project[code]}{_episode_}{clip_name}" + shot_rename_search_patterns = { + "_sequence_": "sc\\d{3}", + "_shot_": "sh\\d{3}", + "_episode_": "ep\\d{2}" } - shot_name_template = "{project[code]}{episode}{clip_name}" - shot_hierarchy = "{episode}{sequence}/{clip_name}" - shot_tasks = ["Animation", "Layout"] + shot_add_hierarchy = { + "parents_path": "{sequence}", + "parents": { + "sequence": "{_episode_}{_sequence_}", + } + } + shot_add_tasks = ["Animation", "Layout"] def convert_to_entity(self, key, value): # ftrack compatible entity types @@ -43,181 +48,103 @@ class CollectHierarchyInstance(pyblish.api.InstancePlugin): if entity_type: return {"entityType": entity_type, "entityName": value} - def process(self, instance): + def rename_with_hierarchy(self, instance): search_text = "" - context = instance.context - anatomy_data = context.data["anatomyData"] - asset_entity = context.data["assetEntity"] - asset_name = asset_entity["name"] - assets_shared = context.data.get("assetsShared") - + parent_name = self.asset_entity["name"] clip = instance.data["item"] clip_name = os.path.splitext(clip.name)[0].lower() - asset = instance.data["asset"] - clip_in = instance.data["clipIn"] - clip_out = instance.data["clipOut"] - fps = instance.data["fps"] - - hierarchy_data = dict(anatomy_data) - if self.search_patterns: - search_text += clip_name + asset_name - hierarchy_data.update({"clip_name": clip_name}) - for type, pattern in self.search_patterns.items(): + if self.shot_rename_search_patterns: + search_text += parent_name + clip_name + self.hierarchy_data.update({"clip_name": clip_name}) + for type, pattern in self.shot_rename_search_patterns.items(): p = re.compile(pattern) match = p.findall(search_text) if not match: continue - hierarchy_data[type] = match[-1] + self.hierarchy_data[type] = match[-1] - self.log.debug("__ hierarchy_data: {}".format(hierarchy_data)) - shot_name = self.shot_name_template.format(**hierarchy_data) - self.log.debug("__ shot_name: {}".format(shot_name)) - shot_hierarchy = self.shot_hierarchy.format(**hierarchy_data) - self.log.debug("__ shot_hierarchy: {}".format(shot_hierarchy)) + self.log.debug("__ hierarchy_data: {}".format(self.hierarchy_data)) - # # build data for inner nukestudio project property - # data = { - # "sequence": ( - # context.data['activeSequence'].name().replace(' ', '_') - # ), - # "track": clip.parent().name().replace(' ', '_'), - # "clip": asset - # } - # self.log.debug("__ data: {}".format(data)) - # - # # Check for clips with the same range - # # this is for testing if any vertically neighbouring - # # clips has been already processed - # match = next(( - # k for k, v in assets_shared.items() - # if (v["_clipIn"] == clip_in) - # and (v["_clipOut"] == clip_out) - # ), False) - # - # self.log.debug( - # "__ assets_shared[match]: {}".format( - # assets_shared[match])) - # - # # check if hierarchy key is present in matched - # # vertically neighbouring clip - # if not assets_shared[match].get("hierarchy"): - # match = False - # - # # rise exception if multiple hierarchy tag found - # assert not match, ( - # "Two clips above each other with" - # " hierarchy tag are not allowed" - # " >> keep hierarchy tag only in one of them <<" - # ) - # - # d_metadata = dict() - # parents = list() - # - # # main template from Tag.note - # template = t_note - # - # # if shot in template then remove it - # if "shot" in template.lower(): - # instance.data["asset"] = [ - # t for t in template.split('/')][-1] - # template = "/".join( - # [t for t in template.split('/')][0:-1]) - # - # # take template from Tag.note and break it into parts - # template_split = template.split("/") - # patern = re.compile(r"\{([a-z]*?)\}") - # par_split = [patern.findall(t) - # for t in template.split("/")] - # - # # format all {} in two layers - # for k, v in t_metadata.items(): - # new_k = k.split(".")[1] - # - # # ignore all help strings - # if 'help' in k: - # continue - # # self.log.info("__ new_k: `{}`".format(new_k)) - # try: - # # first try all data and context data to - # # add to individual properties - # new_v = str(v).format( - # **dict(context.data, **data)) - # d_metadata[new_k] = new_v - # - # # create parents - # # find matching index of order - # p_match_i = [i for i, p in enumerate(par_split) - # if new_k in p] - # - # # if any is matching then convert to entity_types - # if p_match_i: - # parent = self.convert_to_entity( - # new_k, template_split[p_match_i[0]]) - # parents.insert(p_match_i[0], parent) - # except Exception: - # d_metadata[new_k] = v - # - # # create new shot asset name - # instance.data["asset"] = instance.data["asset"].format( - # **d_metadata) - # self.log.debug( - # "__ instance.data[asset]: " - # "{}".format(instance.data["asset"]) - # ) - # - # # lastly fill those individual properties itno - # # format the string with collected data - # parents = [{"entityName": p["entityName"].format( - # **d_metadata), "entityType": p["entityType"]} - # for p in parents] - # self.log.debug("__ parents: {}".format(parents)) - # - # hierarchy = template.format( - # **d_metadata) - # self.log.debug("__ hierarchy: {}".format(hierarchy)) - # - # # check if hierarchy attribute is already created - # # it should not be so return warning if it is - # hd = instance.data.get("hierarchy") - # assert not hd, ( - # "Only one Hierarchy Tag is allowed. " - # "Clip: `{}`".format(asset) - # ) - # - # # add formated hierarchy path into instance data - # instance.data["hierarchy"] = hierarchy - # instance.data["parents"] = parents - # - # self.log.info( - # "clip: {asset}[{clip_in}:{clip_out}]".format( - # **locals())) - # # adding to asset shared dict - # self.log.debug( - # "__ assets_shared: {}".format(assets_shared)) - # if assets_shared.get(asset): - # self.log.debug("Adding to shared assets: `{}`".format( - # asset)) - # asset_shared = assets_shared.get(asset) - # else: - # asset_shared = assets_shared[asset] - # - # asset_shared.update({ - # "asset": asset, - # "hierarchy": hierarchy, - # "parents": parents, - # "fps": fps, - # "tasks": instance.data["tasks"] - # }) - # - # # adding frame start if any on instance - # start_frame = instance.data.get("startingFrame") - # if start_frame: - # asset_shared.update({ - # "startingFrame": start_frame - # }) - # self.log.debug( - # "assets_shared: {assets_shared}".format(**locals())) + # format to new shot name + self.shot_name = self.shot_rename_template.format( + **self.hierarchy_data) + instance.data["asset"] = self.shot_name + self.log.debug("__ self.shot_name: {}".format(self.shot_name)) + + def create_hierarchy(self, instance): + parents = list() + hierarchy = "" + visual_hierarchy = [self.asset_entity] + while True: + visual_parent = io.find_one( + {"_id": visual_hierarchy[-1]["data"]["visualParent"]} + ) + if visual_parent: + visual_hierarchy.append(visual_parent) + else: + visual_hierarchy.append( + instance.context.data["projectEntity"]) + break + self.log.debug("__ visual_hierarchy: {}".format(visual_hierarchy)) + + # add current selection context hierarchy from standalonepublisher + for entity in reversed(visual_hierarchy): + parents.append({ + "entityType": entity["data"]["entityType"], + "entityName": entity["name"] + }) + + if self.shot_add_hierarchy: + # fill the parents parts from presets + for parent in self.shot_add_hierarchy["parents"]: + if not self.shot_add_hierarchy["parents"][parent]: + prnt = {"entity"} + else: + self.shot_add_hierarchy["parents"][parent] = self.shot_add_hierarchy[ + "parents"][parent].format(**self.hierarchy_data) + prnt = self.convert_to_entity( + parent, self.shot_add_hierarchy["parents"][parent]) + parents.append(prnt) + + hierarchy = self.shot_add_hierarchy[ + "parents_path"].format(**self.shot_add_hierarchy["parents"]) + + instance.data["hierarchy"] = hierarchy + instance.data["parents"] = parents + + if self.shot_add_tasks: + instance.data["tasks"] = self.shot_add_tasks + else: + instance.data["tasks"] = list() + + def process(self, instance): + assets_shared = instance.context.data.get("assetsShared") + context = instance.context + anatomy_data = context.data["anatomyData"] + + self.shot_name = instance.data["asset"] + self.hierarchy_data = dict(anatomy_data) + self.asset_entity = context.data["assetEntity"] + + frame_start = instance.data["frameStart"] + frame_end = instance.data["frameEnd"] + + if self.shot_rename_template: + self.rename_with_hierarchy(instance) + + self.create_hierarchy(instance) + + label = f"{self.shot_name} ({frame_start}-{frame_end})" + instance.data["label"] = label + + assets_shared[self.shot_name] = { + "asset": instance.data["asset"], + "hierarchy": instance.data["hierarchy"], + "parents": instance.data["parents"], + "fps": instance.data["fps"], + "tasks": instance.data["tasks"] + } class CollectHierarchyContext(pyblish.api.ContextPlugin): @@ -242,13 +169,12 @@ class CollectHierarchyContext(pyblish.api.ContextPlugin): return new_dict def process(self, context): - instances = context[:] - + instances = context # create hierarchyContext attr if context has none - temp_context = {} + final_context = {} for instance in instances: - if 'projectfile' in instance.data.get('family', ''): + if 'clip' not in instance.data.get('family', ''): continue name = instance.data["asset"] @@ -268,11 +194,6 @@ class CollectHierarchyContext(pyblish.api.ContextPlugin): instance.data["parents"] = s_asset_data["parents"] instance.data["hierarchy"] = s_asset_data["hierarchy"] instance.data["tasks"] = s_asset_data["tasks"] - instance.data["resolutionWidth"] = s_asset_data[ - "resolutionWidth"] - instance.data["resolutionHeight"] = s_asset_data[ - "resolutionHeight"] - instance.data["pixelAspect"] = s_asset_data["pixelAspect"] instance.data["fps"] = s_asset_data["fps"] # adding frame start if any on instance @@ -283,8 +204,6 @@ class CollectHierarchyContext(pyblish.api.ContextPlugin): instance.data["clipOut"] - instance.data["clipIn"]) - - self.log.debug( "__ instance.data[parents]: {}".format( instance.data["parents"] @@ -301,32 +220,20 @@ class CollectHierarchyContext(pyblish.api.ContextPlugin): in_info = {} - in_info["inputs"] = [ - x["_id"] for x in instance.data.get("assetbuilds", []) - ] - # suppose that all instances are Shots in_info['entity_type'] = 'Shot' # get custom attributes of the shot - if instance.data.get("main"): - in_info['custom_attributes'] = { - "handleStart": handle_start, - "handleEnd": handle_end, - "frameStart": instance.data["frameStart"], - "frameEnd": instance.data["frameEnd"], - "clipIn": instance.data["clipIn"], - "clipOut": instance.data["clipOut"], - 'fps': instance.context.data["fps"] - } - # adding SourceResolution if Tag was present - if instance.data.get("main"): - in_info['custom_attributes'].update({ - "resolutionWidth": instance.data["resolutionWidth"], - "resolutionHeight": instance.data["resolutionHeight"], - "pixelAspect": instance.data["pixelAspect"] - }) + in_info['custom_attributes'] = { + "handleStart": handle_start, + "handleEnd": handle_end, + "frameStart": instance.data["frameStart"], + "frameEnd": instance.data["frameEnd"], + "clipIn": instance.data["clipIn"], + "clipOut": instance.data["clipOut"], + 'fps': instance.data["fps"] + } in_info['tasks'] = instance.data['tasks'] @@ -343,14 +250,7 @@ class CollectHierarchyContext(pyblish.api.ContextPlugin): next_dict[parent_name]["childs"] = actual actual = next_dict - temp_context = self.update_dict(temp_context, actual) - - # TODO: 100% sure way of get project! Will be Name or Code? - project_name = avalon.Session["AVALON_PROJECT"] - final_context = {} - final_context[project_name] = {} - final_context[project_name]['entity_type'] = 'Project' - final_context[project_name]['childs'] = temp_context + final_context = self.update_dict(final_context, actual) # adding hierarchy context to instance context.data["hierarchyContext"] = final_context diff --git a/pype/plugins/standalonepublisher/publish/collect_shot_names.py b/pype/plugins/standalonepublisher/publish/collect_shot_names.py deleted file mode 100644 index 7976c855d3..0000000000 --- a/pype/plugins/standalonepublisher/publish/collect_shot_names.py +++ /dev/null @@ -1,15 +0,0 @@ -import pyblish.api -import re - - -class CollectShotNames(pyblish.api.InstancePlugin): - """ - Collecting shot names - """ - - label = "Collect shot names" - order = pyblish.api.CollectorOrder + 0.01 - hosts = ["standalonepublisher"] - - def process(self, instance): - self.log.info("Instance name: `{}`".format(instance.data["name"])) diff --git a/pype/plugins/standalonepublisher/publish/extract_shot.py b/pype/plugins/standalonepublisher/publish/extract_shot.py index e1e69d993b..724c22cb1a 100644 --- a/pype/plugins/standalonepublisher/publish/extract_shot.py +++ b/pype/plugins/standalonepublisher/publish/extract_shot.py @@ -12,9 +12,6 @@ class ExtractShot(pype.api.Extractor): families = ["clip"] def process(self, instance): - # get context - context = instance.context - # get ffmpet path ffmpeg_path = pype.lib.get_ffmpeg_tool_path("ffmpeg") @@ -24,7 +21,7 @@ class ExtractShot(pype.api.Extractor): # Generate mov file. fps = instance.data["fps"] - video_file_path = context.data["editorialVideoPath"] + video_file_path = instance.data["editorialVideoPath"] ext = os.path.splitext(os.path.basename(video_file_path))[-1] clip_trimed_path = os.path.join( @@ -60,7 +57,7 @@ class ExtractShot(pype.api.Extractor): "frameEnd": instance.data["frameEnd"], "fps": fps, "thumbnail": True, - "tags": ["review", "ftrackreview"] + "tags": ["review", "ftrackreview", "delete"] }) # # Generate jpegs. diff --git a/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py b/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py index e4d2279317..6033ed919b 100644 --- a/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py +++ b/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py @@ -9,24 +9,10 @@ class ValidateEditorialResources(pyblish.api.InstancePlugin): label = "Validate Editorial Resources" hosts = ["standalonepublisher"] - families = ["editorial"] + families = ["clip"] order = pype.api.ValidateContentsOrder - # presets - check_ext = None - def process(self, instance): - check_ext = self.check_ext or "mov" - representation = instance.data["representations"][0] - staging_dir = representation["stagingDir"] - basename = os.path.splitext( - os.path.basename(representation["files"]) - )[0] - - files = [x for x in os.listdir(staging_dir)] - - # Check for correct extansion in file name. - filename = basename + check_ext - filepath = os.path.join(staging_dir, filename) - msg = f"Missing \"{filepath}\"." - assert filename in files, msg + check_file = instance.data["editorialVideoPath"] + msg = f"Missing \"{check_file}\"." + assert check_file, msg From b41cca9972ec5ebac3047f60a69bd37c8be79e50 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 4 Aug 2020 12:53:13 +0200 Subject: [PATCH 03/30] feat(sp): refactory way the publish gui is started so preset are loaded to plugins --- pype/modules/standalonepublish/publish.py | 120 +++++++++------------- 1 file changed, 47 insertions(+), 73 deletions(-) diff --git a/pype/modules/standalonepublish/publish.py b/pype/modules/standalonepublish/publish.py index dd65030f7a..d2d9eb486f 100644 --- a/pype/modules/standalonepublish/publish.py +++ b/pype/modules/standalonepublish/publish.py @@ -6,13 +6,14 @@ import random import string from avalon import io, api -from avalon.tools import publish as av_publish - +import importlib import pype -from pype.api import execute +from pype.api import execute, Logger import pyblish.api -from . import PUBLISH_PATHS + + +log = Logger().get_logger("standalonepublisher") def set_context(project, asset, task, app): @@ -65,64 +66,12 @@ def publish(data, gui=True): # avalon_api_publish(data, gui) -def avalon_api_publish(data, gui=True): - ''' Launches Pyblish (GUI by default) - :param data: Should include data for pyblish and standalone collector - :type data: dict - :param gui: Pyblish will be launched in GUI mode if set to True - :type gui: bool - ''' - io.install() - - # Create hash name folder in temp - chars = "".join([random.choice(string.ascii_letters) for i in range(15)]) - staging_dir = tempfile.mkdtemp(chars) - - # create also json and fill with data - json_data_path = staging_dir + os.path.basename(staging_dir) + '.json' - with open(json_data_path, 'w') as outfile: - json.dump(data, outfile) - - args = [ - "-pp", os.pathsep.join(pyblish.api.registered_paths()) - ] - - envcopy = os.environ.copy() - envcopy["PYBLISH_HOSTS"] = "standalonepublisher" - envcopy["SAPUBLISH_INPATH"] = json_data_path - - if gui: - av_publish.show() - else: - returncode = execute([ - sys.executable, "-u", "-m", "pyblish" - ] + args, env=envcopy) - - io.uninstall() - - def cli_publish(data, gui=True): + from . import PUBLISH_PATHS + + PUBLISH_SCRIPT_PATH = os.path.join(os.path.dirname(__file__), "publish.py") io.install() - pyblish.api.deregister_all_plugins() - # Registers Global pyblish plugins - pype.install() - # Registers Standalone pyblish plugins - for path in PUBLISH_PATHS: - pyblish.api.register_plugin_path(path) - - project_plugins_paths = os.environ.get("PYPE_PROJECT_PLUGINS") - project_name = os.environ["AVALON_PROJECT"] - if project_plugins_paths and project_name: - for path in project_plugins_paths.split(os.pathsep): - if not path: - continue - plugin_path = os.path.join(path, project_name, "plugins") - if os.path.exists(plugin_path): - pyblish.api.register_plugin_path(plugin_path) - api.register_plugin_path(api.Loader, plugin_path) - api.register_plugin_path(api.Creator, plugin_path) - # Create hash name folder in temp chars = "".join([random.choice(string.ascii_letters) for i in range(15)]) staging_dir = tempfile.mkdtemp(chars) @@ -136,30 +85,55 @@ def cli_publish(data, gui=True): with open(json_data_path, 'w') as outfile: json.dump(data, outfile) - args = [ - "-pp", os.pathsep.join(pyblish.api.registered_paths()) - ] - - if gui: - args += ["gui"] - envcopy = os.environ.copy() envcopy["PYBLISH_HOSTS"] = "standalonepublisher" envcopy["SAPUBLISH_INPATH"] = json_data_path envcopy["SAPUBLISH_OUTPATH"] = return_data_path - envcopy["PYBLISH_GUI"] = "pyblish_pype" + envcopy["PYBLISHGUI"] = "pyblish_pype" + envcopy["PUBLISH_PATHS"] = os.pathsep.join(PUBLISH_PATHS) - returncode = execute([ - sys.executable, "-u", "-m", "pyblish" - ] + args, env=envcopy) + result = execute( + [sys.executable, PUBLISH_SCRIPT_PATH], + env=envcopy + ) result = {} if os.path.exists(json_data_path): with open(json_data_path, "r") as f: result = json.load(f) + log.info(f"Publish result: {result}") + io.uninstall() - # TODO: check if was pyblish successful - # if successful return True - print('Check result here') + return False + + +def main(env): + from avalon.tools import publish + # Registers pype's Global pyblish plugins + pype.install() + + # Register additional paths + addition_paths_str = env.get("PUBLISH_PATHS") or "" + addition_paths = addition_paths_str.split(os.pathsep) + for path in addition_paths: + path = os.path.normpath(path) + if not os.path.exists(path): + continue + pyblish.api.register_plugin_path(path) + + # Register project specific plugins + project_name = os.environ["AVALON_PROJECT"] + project_plugins_paths = env.get("PYPE_PROJECT_PLUGINS") or "" + for path in project_plugins_paths.split(os.pathsep): + plugin_path = os.path.join(path, project_name, "plugins") + if os.path.exists(plugin_path): + pyblish.api.register_plugin_path(plugin_path) + + return publish.show() + + +if __name__ == "__main__": + result = main(os.environ) + sys.exit(not bool(result)) From 04c4ba77626b3c89df66b053b6f511fd0b5a2926 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 4 Aug 2020 16:51:20 +0200 Subject: [PATCH 04/30] feat(global): moving to correct host folder --- .../publish/integrate_ftrack_component_overwrite.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pype/plugins/{premiere => ftrack}/publish/integrate_ftrack_component_overwrite.py (100%) diff --git a/pype/plugins/premiere/publish/integrate_ftrack_component_overwrite.py b/pype/plugins/ftrack/publish/integrate_ftrack_component_overwrite.py similarity index 100% rename from pype/plugins/premiere/publish/integrate_ftrack_component_overwrite.py rename to pype/plugins/ftrack/publish/integrate_ftrack_component_overwrite.py From c44873d2c1a56d8bbde6d80a788aa39e1c2fa333 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 4 Aug 2020 16:52:12 +0200 Subject: [PATCH 05/30] feat(global): adding standalonepublisher to hosts --- pype/plugins/global/publish/extract_burnin.py | 9 ++++++++- pype/plugins/global/publish/extract_review.py | 10 +++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py index 83ad4af1c2..e1508b9131 100644 --- a/pype/plugins/global/publish/extract_burnin.py +++ b/pype/plugins/global/publish/extract_burnin.py @@ -19,7 +19,14 @@ class ExtractBurnin(pype.api.Extractor): label = "Extract burnins" order = pyblish.api.ExtractorOrder + 0.03 families = ["review", "burnin"] - hosts = ["nuke", "maya", "shell", "nukestudio", "premiere"] + hosts = [ + "nuke", + "maya", + "shell", + "nukestudio", + "premiere", + "standalonepublisher" + ] optional = True positions = [ diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 30d1de8328..a16c3ce256 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -22,7 +22,15 @@ class ExtractReview(pyblish.api.InstancePlugin): label = "Extract Review" order = pyblish.api.ExtractorOrder + 0.02 families = ["review"] - hosts = ["nuke", "maya", "shell", "nukestudio", "premiere", "harmony"] + hosts = [ + "nuke", + "maya", + "shell", + "nukestudio", + "premiere", + "harmony", + "standalonepublisher" + ] # Supported extensions image_exts = ["exr", "jpg", "jpeg", "png", "dpx"] From 33d671edd9d0b6111aeb95e80c2aeac95d9ee46f Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 4 Aug 2020 16:55:02 +0200 Subject: [PATCH 06/30] feat(sp): updating extract_shot --- pype/plugins/standalonepublisher/publish/extract_shot.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pype/plugins/standalonepublisher/publish/extract_shot.py b/pype/plugins/standalonepublisher/publish/extract_shot.py index 724c22cb1a..7112bea15b 100644 --- a/pype/plugins/standalonepublisher/publish/extract_shot.py +++ b/pype/plugins/standalonepublisher/publish/extract_shot.py @@ -48,6 +48,10 @@ class ExtractShot(pype.api.Extractor): output = pype.api.subprocess(ffmpeg_args) self.log.info(output) + instance.data["families"].remove("review") + instance.data["families"].append("clip") + instance.data["family"] = "review" + instance.data["representations"].append({ "name": ext[1:], "ext": ext[1:], @@ -60,6 +64,8 @@ class ExtractShot(pype.api.Extractor): "tags": ["review", "ftrackreview", "delete"] }) + self.log.debug(f"Instance data: {instance.data}") + # # Generate jpegs. # clip_thumbnail = os.path.join( # staging_dir, instance.data["name"] + ".%04d.jpeg" From 83d35608a10ac392e6dc40c1dacc87b8eee0fca3 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 4 Aug 2020 16:55:52 +0200 Subject: [PATCH 07/30] clean(sp): make it nicer --- .../standalonepublisher/publish/collect_editorial.py | 6 +++--- .../standalonepublisher/publish/collect_hierarchy.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pype/plugins/standalonepublisher/publish/collect_editorial.py b/pype/plugins/standalonepublisher/publish/collect_editorial.py index baade27ba8..c710802ec0 100644 --- a/pype/plugins/standalonepublisher/publish/collect_editorial.py +++ b/pype/plugins/standalonepublisher/publish/collect_editorial.py @@ -2,7 +2,7 @@ import os import opentimelineio as otio import pyblish.api from pype import lib as plib -import pype.api + class OTIO_View(pyblish.api.Action): """Currently disabled because OTIO requires PySide2. Issue on Qt.py: @@ -66,8 +66,8 @@ class CollectEditorial(pyblish.api.InstancePlugin): extension = os.path.splitext(file_path)[1] kwargs = {} if extension == ".edl": - # EDL has no frame rate embedded so needs explicit frame rate else - # 24 is asssumed. + # EDL has no frame rate embedded so needs explicit + # frame rate else 24 is asssumed. kwargs["rate"] = plib.get_asset()["data"]["fps"] instance.data["otio_timeline"] = otio.adapters.read_from_file( diff --git a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py index 7c968ce302..4207b0824f 100644 --- a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py +++ b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py @@ -1,9 +1,9 @@ import pyblish.api -import avalon.api as avalon import re import os from avalon import io + class CollectHierarchyInstance(pyblish.api.InstancePlugin): """Collecting hierarchy context from `parents` and `hierarchy` data present in `clip` family instances coming from the request json data file From a07038d889751e29685f20353933630f2a4fbc6f Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 4 Aug 2020 16:56:24 +0200 Subject: [PATCH 08/30] feat(sp): adding required current file --- pype/plugins/standalonepublisher/publish/collect_editorial.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pype/plugins/standalonepublisher/publish/collect_editorial.py b/pype/plugins/standalonepublisher/publish/collect_editorial.py index c710802ec0..e11ea8c31f 100644 --- a/pype/plugins/standalonepublisher/publish/collect_editorial.py +++ b/pype/plugins/standalonepublisher/publish/collect_editorial.py @@ -44,6 +44,7 @@ class CollectEditorial(pyblish.api.InstancePlugin): file_path = os.path.join( staging_dir, str(representation["files"]) ) + instance.context.data["currentFile"] = file_path # get video file path video_path = None From 005e39b33c2679c4b182db6e295384a9e2de9704 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 4 Aug 2020 17:00:57 +0200 Subject: [PATCH 09/30] feat(sp): removing preset-able content --- .../publish/collect_hierarchy.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py index 4207b0824f..dcae3e7e87 100644 --- a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py +++ b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py @@ -19,19 +19,10 @@ class CollectHierarchyInstance(pyblish.api.InstancePlugin): families = ["clip"] # presets - shot_rename_template = "{project[code]}{_episode_}{clip_name}" - shot_rename_search_patterns = { - "_sequence_": "sc\\d{3}", - "_shot_": "sh\\d{3}", - "_episode_": "ep\\d{2}" - } - shot_add_hierarchy = { - "parents_path": "{sequence}", - "parents": { - "sequence": "{_episode_}{_sequence_}", - } - } - shot_add_tasks = ["Animation", "Layout"] + shot_rename_template = None + shot_rename_search_patterns = None + shot_add_hierarchy = None + shot_add_tasks = None def convert_to_entity(self, key, value): # ftrack compatible entity types From 1e73ba2397bb48a51921d0aa6edab76817f4d982 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 4 Aug 2020 17:04:06 +0200 Subject: [PATCH 10/30] feat(sp): removing preset-able content --- pype/plugins/standalonepublisher/publish/collect_editorial.py | 2 +- .../plugins/standalonepublisher/publish/collect_frame_ranges.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/plugins/standalonepublisher/publish/collect_editorial.py b/pype/plugins/standalonepublisher/publish/collect_editorial.py index e11ea8c31f..fa5b5f13a3 100644 --- a/pype/plugins/standalonepublisher/publish/collect_editorial.py +++ b/pype/plugins/standalonepublisher/publish/collect_editorial.py @@ -32,7 +32,7 @@ class CollectEditorial(pyblish.api.InstancePlugin): actions = [] # presets - extensions = [".mov", ".mp4"] + extensions = [".mov"] def process(self, instance): self.log.debug(f"__ instance: `{instance}`") diff --git a/pype/plugins/standalonepublisher/publish/collect_frame_ranges.py b/pype/plugins/standalonepublisher/publish/collect_frame_ranges.py index c58cac3505..5e8292458f 100644 --- a/pype/plugins/standalonepublisher/publish/collect_frame_ranges.py +++ b/pype/plugins/standalonepublisher/publish/collect_frame_ranges.py @@ -11,7 +11,7 @@ class CollectClipFrameRanges(pyblish.api.InstancePlugin): # presets start_frame_offset = None # if 900000 for edl default then -900000 - custom_start_frame = 1 + custom_start_frame = None def process(self, instance): From 17e3c19d7a3db4b2bafb9e6dd326179b07f034ab Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 4 Aug 2020 19:43:53 +0200 Subject: [PATCH 11/30] feat(sa): publishing clips wip --- .../standalonepublisher/publish/collect_clips.py | 4 ++-- .../publish/collect_hierarchy.py | 9 +++++++++ .../standalonepublisher/publish/extract_shot.py | 16 +++++++++++++--- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/pype/plugins/standalonepublisher/publish/collect_clips.py b/pype/plugins/standalonepublisher/publish/collect_clips.py index d8eb1d72c4..2fc44f8aeb 100644 --- a/pype/plugins/standalonepublisher/publish/collect_clips.py +++ b/pype/plugins/standalonepublisher/publish/collect_clips.py @@ -99,8 +99,8 @@ class CollectClips(pyblish.api.InstancePlugin): "clipIn": clip_in, "clipOut": clip_out, "clipDuration": clip_duration, - "handleStart": asset_data["handleStart"], - "handleEnd": asset_data["handleEnd"], + "handleStart": int(asset_data["handleStart"]), + "handleEnd": int(asset_data["handleEnd"]), "fps": fps, # instance properities diff --git a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py index dcae3e7e87..6d5ab4b94a 100644 --- a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py +++ b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py @@ -109,6 +109,12 @@ class CollectHierarchyInstance(pyblish.api.InstancePlugin): else: instance.data["tasks"] = list() + # updating hierarchy data + self.hierarchy_data.update({ + "asset": self.shot_name, + "task": "conform" + }) + def process(self, instance): assets_shared = instance.context.data.get("assetsShared") context = instance.context @@ -126,6 +132,9 @@ class CollectHierarchyInstance(pyblish.api.InstancePlugin): self.create_hierarchy(instance) + # adding anatomyData for burnins + instance.data["anatomyData"] = self.hierarchy_data + label = f"{self.shot_name} ({frame_start}-{frame_end})" instance.data["label"] = label diff --git a/pype/plugins/standalonepublisher/publish/extract_shot.py b/pype/plugins/standalonepublisher/publish/extract_shot.py index 7112bea15b..907773b90e 100644 --- a/pype/plugins/standalonepublisher/publish/extract_shot.py +++ b/pype/plugins/standalonepublisher/publish/extract_shot.py @@ -3,6 +3,8 @@ import clique import pype.api import pype.lib as plib +from pprint import pformat + class ExtractShot(pype.api.Extractor): """Extract shot "mov" and "wav" files.""" @@ -52,19 +54,27 @@ class ExtractShot(pype.api.Extractor): instance.data["families"].append("clip") instance.data["family"] = "review" + # frame ranges + frame_start = int(instance.data["frameStart"]) + frame_end = int(instance.data["frameEnd"]) + handle_start = int(instance.data["handleStart"]) + handle_end = int(instance.data["handleEnd"]) + instance.data["representations"].append({ "name": ext[1:], "ext": ext[1:], "files": os.path.basename(clip_trimed_path), "stagingDir": staging_dir, - "frameStart": instance.data["frameStart"], - "frameEnd": instance.data["frameEnd"], + "frameStart": frame_start, + "frameEnd": frame_end, + "frameStartFtrack": frame_start - handle_start, + "frameEndFtrack": frame_end - handle_end, "fps": fps, "thumbnail": True, "tags": ["review", "ftrackreview", "delete"] }) - self.log.debug(f"Instance data: {instance.data}") + self.log.debug(f"Instance data: {pformat(instance.data)}") # # Generate jpegs. # clip_thumbnail = os.path.join( From 8f57ae45fe70a72abceea9a4993e48d2e57e87c7 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 4 Aug 2020 21:02:44 +0200 Subject: [PATCH 12/30] feat(sp): wip publishing clips --- .../publish/collect_hierarchy_context.py | 1 + .../publish/collect_clips.py | 292 +++++++++--------- .../publish/collect_hierarchy.py | 15 +- .../publish/collect_shots.py | 37 +++ .../publish/extract_shot.py | 108 ++++--- 5 files changed, 273 insertions(+), 180 deletions(-) create mode 100644 pype/plugins/standalonepublisher/publish/collect_shots.py diff --git a/pype/plugins/nukestudio/publish/collect_hierarchy_context.py b/pype/plugins/nukestudio/publish/collect_hierarchy_context.py index 38040f8c51..a41e987bdb 100644 --- a/pype/plugins/nukestudio/publish/collect_hierarchy_context.py +++ b/pype/plugins/nukestudio/publish/collect_hierarchy_context.py @@ -210,6 +210,7 @@ class CollectHierarchyInstance(pyblish.api.ContextPlugin): self.log.debug( "assets_shared: {assets_shared}".format(**locals())) + class CollectHierarchyContext(pyblish.api.ContextPlugin): '''Collecting Hierarchy from instaces and building context hierarchy tree diff --git a/pype/plugins/standalonepublisher/publish/collect_clips.py b/pype/plugins/standalonepublisher/publish/collect_clips.py index 2fc44f8aeb..6a00675961 100644 --- a/pype/plugins/standalonepublisher/publish/collect_clips.py +++ b/pype/plugins/standalonepublisher/publish/collect_clips.py @@ -84,147 +84,159 @@ class CollectClips(pyblish.api.InstancePlugin): label = f"{name} (framerange: {clip_in}-{clip_out})" + new_instance_data = { + # shared attributes + "representations": [], + # timing properities + "trackStartFrame": track_start_frame, + "sourceIn": source_in, + "sourceOut": source_in + clip_duration, + "clipIn": clip_in, + "clipOut": clip_out, + "clipDuration": clip_duration, + "handleStart": int(asset_data["handleStart"]), + "handleEnd": int(asset_data["handleEnd"]), + "fps": fps + } + + # adding Review-able instance + shot_instance_data = new_instance_data.copy() + shot_instance_data.update({ + # unique attributes + "name": name, + "label": label, + "asset": name, + "subset": "plateRef", + "item": clip, + + # instance properities + "family": "clip", + "families": ["review", "ftrack"], + "ftrackFamily": "review", + "editorialVideoPath": instance.data[ + "editorialVideoPath"] + }) instances.append( - instance.context.create_instance(**{ - "name": name, - "label": label, - "asset": name, - "subset": "plateRef", - "item": clip, - - # timing properities - "trackStartFrame": track_start_frame, - "sourceIn": source_in, - "sourceOut": source_in + clip_duration, - "clipIn": clip_in, - "clipOut": clip_out, - "clipDuration": clip_duration, - "handleStart": int(asset_data["handleStart"]), - "handleEnd": int(asset_data["handleEnd"]), - "fps": fps, - - # instance properities - "family": "clip", - "families": ["review", "ftrack"], - "ftrackFamily": "review", - "representations": [], - "editorialVideoPath": instance.data[ - "editorialVideoPath"] - }) + instance.context.create_instance(**shot_instance_data) ) - def process_old(self, instance): - representation = instance.data["representations"][0] - file_path = os.path.join( - representation["stagingDir"], representation["files"] - ) - instance.context.data["editorialPath"] = file_path + context.data["assetsShared"][name] = { + "_clipIn": clip_in, + "_clipOut": clip_out + } - extension = os.path.splitext(file_path)[1][1:] - kwargs = {} - if extension == "edl": - # EDL has no frame rate embedded so needs explicit frame rate else - # 24 is asssumed. - kwargs["rate"] = plib.get_asset()["data"]["fps"] - - timeline = otio.adapters.read_from_file(file_path, **kwargs) - tracks = timeline.each_child( - descended_from_type=otio.schema.track.Track - ) - asset_entity = instance.context.data["assetEntity"] - asset_name = asset_entity["name"] - - # Ask user for sequence start. Usually 10:00:00:00. - sequence_start_frame = 900000 - - # Project specific prefix naming. This needs to be replaced with some - # options to be more flexible. - asset_name = asset_name.split("_")[0] - - instances = [] - for track in tracks: - track_start_frame = ( - abs(track.source_range.start_time.value) - sequence_start_frame - ) - for child in track.each_child(): - # skip all generators like black ampty - if isinstance( - child.media_reference, - otio.schema.GeneratorReference): - continue - - # Transitions are ignored, because Clips have the full frame - # range. - if isinstance(child, otio.schema.transition.Transition): - continue - - if child.name is None: - continue - - # Hardcoded to expect a shot name of "[name].[extension]" - child_name = os.path.splitext(child.name)[0].lower() - name = f"{asset_name}_{child_name}" - - frame_start = track_start_frame - frame_start += child.range_in_parent().start_time.value - frame_end = track_start_frame - frame_end += child.range_in_parent().end_time_inclusive().value - - label = f"{name} (framerange: {frame_start}-{frame_end})" - instances.append( - instance.context.create_instance(**{ - "name": name, - "label": label, - "frameStart": frame_start, - "frameEnd": frame_end, - "family": "shot", - "families": ["review", "ftrack"], - "ftrackFamily": "review", - "asset": name, - "subset": "shotMain", - "representations": [], - "source": file_path - }) - ) - - visual_hierarchy = [asset_entity] - while True: - visual_parent = io.find_one( - {"_id": visual_hierarchy[-1]["data"]["visualParent"]} - ) - if visual_parent: - visual_hierarchy.append(visual_parent) - else: - visual_hierarchy.append(instance.context.data["projectEntity"]) - break - - context_hierarchy = None - for entity in visual_hierarchy: - childs = {} - if context_hierarchy: - name = context_hierarchy.pop("name") - childs = {name: context_hierarchy} - else: - for instance in instances: - childs[instance.data["name"]] = { - "childs": {}, - "entity_type": "Shot", - "custom_attributes": { - "frameStart": instance.data["frameStart"], - "frameEnd": instance.data["frameEnd"] - } - } - - context_hierarchy = { - "entity_type": entity["data"]["entityType"], - "childs": childs, - "name": entity["name"] - } - - name = context_hierarchy.pop("name") - context_hierarchy = {name: context_hierarchy} - instance.context.data["hierarchyContext"] = context_hierarchy - self.log.info( - "Hierarchy:\n" + - json_util.dumps(context_hierarchy, sort_keys=True, indent=4) - ) + # def process_old(self, instance): + # representation = instance.data["representations"][0] + # file_path = os.path.join( + # representation["stagingDir"], representation["files"] + # ) + # instance.context.data["editorialPath"] = file_path + # + # extension = os.path.splitext(file_path)[1][1:] + # kwargs = {} + # if extension == "edl": + # # EDL has no frame rate embedded so needs explicit frame rate else + # # 24 is asssumed. + # kwargs["rate"] = plib.get_asset()["data"]["fps"] + # + # timeline = otio.adapters.read_from_file(file_path, **kwargs) + # tracks = timeline.each_child( + # descended_from_type=otio.schema.track.Track + # ) + # asset_entity = instance.context.data["assetEntity"] + # asset_name = asset_entity["name"] + # + # # Ask user for sequence start. Usually 10:00:00:00. + # sequence_start_frame = 900000 + # + # # Project specific prefix naming. This needs to be replaced with some + # # options to be more flexible. + # asset_name = asset_name.split("_")[0] + # + # instances = [] + # for track in tracks: + # track_start_frame = ( + # abs(track.source_range.start_time.value) - sequence_start_frame + # ) + # for child in track.each_child(): + # # skip all generators like black ampty + # if isinstance( + # child.media_reference, + # otio.schema.GeneratorReference): + # continue + # + # # Transitions are ignored, because Clips have the full frame + # # range. + # if isinstance(child, otio.schema.transition.Transition): + # continue + # + # if child.name is None: + # continue + # + # # Hardcoded to expect a shot name of "[name].[extension]" + # child_name = os.path.splitext(child.name)[0].lower() + # name = f"{asset_name}_{child_name}" + # + # frame_start = track_start_frame + # frame_start += child.range_in_parent().start_time.value + # frame_end = track_start_frame + # frame_end += child.range_in_parent().end_time_inclusive().value + # + # label = f"{name} (framerange: {frame_start}-{frame_end})" + # instances.append( + # instance.context.create_instance(**{ + # "name": name, + # "label": label, + # "frameStart": frame_start, + # "frameEnd": frame_end, + # "family": "shot", + # "families": ["review", "ftrack"], + # "ftrackFamily": "review", + # "asset": name, + # "subset": "shotMain", + # "representations": [], + # "source": file_path + # }) + # ) + # + # visual_hierarchy = [asset_entity] + # while True: + # visual_parent = io.find_one( + # {"_id": visual_hierarchy[-1]["data"]["visualParent"]} + # ) + # if visual_parent: + # visual_hierarchy.append(visual_parent) + # else: + # visual_hierarchy.append(instance.context.data["projectEntity"]) + # break + # + # context_hierarchy = None + # for entity in visual_hierarchy: + # childs = {} + # if context_hierarchy: + # name = context_hierarchy.pop("name") + # childs = {name: context_hierarchy} + # else: + # for instance in instances: + # childs[instance.data["name"]] = { + # "childs": {}, + # "entity_type": "Shot", + # "custom_attributes": { + # "frameStart": instance.data["frameStart"], + # "frameEnd": instance.data["frameEnd"] + # } + # } + # + # context_hierarchy = { + # "entity_type": entity["data"]["entityType"], + # "childs": childs, + # "name": entity["name"] + # } + # + # name = context_hierarchy.pop("name") + # context_hierarchy = {name: context_hierarchy} + # instance.context.data["hierarchyContext"] = context_hierarchy + # self.log.info( + # "Hierarchy:\n" + + # json_util.dumps(context_hierarchy, sort_keys=True, indent=4) + # ) diff --git a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py index 6d5ab4b94a..87878d560e 100644 --- a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py +++ b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py @@ -116,6 +116,7 @@ class CollectHierarchyInstance(pyblish.api.InstancePlugin): }) def process(self, instance): + asset = instance.data["asset"] assets_shared = instance.context.data.get("assetsShared") context = instance.context anatomy_data = context.data["anatomyData"] @@ -138,13 +139,23 @@ class CollectHierarchyInstance(pyblish.api.InstancePlugin): label = f"{self.shot_name} ({frame_start}-{frame_end})" instance.data["label"] = label - assets_shared[self.shot_name] = { + # dealing with shared attributes across instances + # with the same asset name + + if assets_shared.get(asset): + self.log.debug("Adding to shared assets: `{}`".format( + asset)) + asset_shared = assets_shared.get(asset) + else: + asset_shared = assets_shared[asset] + + asset_shared.update({ "asset": instance.data["asset"], "hierarchy": instance.data["hierarchy"], "parents": instance.data["parents"], "fps": instance.data["fps"], "tasks": instance.data["tasks"] - } + }) class CollectHierarchyContext(pyblish.api.ContextPlugin): diff --git a/pype/plugins/standalonepublisher/publish/collect_shots.py b/pype/plugins/standalonepublisher/publish/collect_shots.py new file mode 100644 index 0000000000..269a65fd86 --- /dev/null +++ b/pype/plugins/standalonepublisher/publish/collect_shots.py @@ -0,0 +1,37 @@ +from pyblish import api + + +class CollectShots(api.InstancePlugin): + """Collect Shot from Clip.""" + + # Run just before CollectClipSubsets + order = api.CollectorOrder + 0.1021 + label = "Collect Shots" + hosts = ["standalonepublisher"] + families = ["clip"] + + def process(self, instance): + + # Collect data. + data = {} + for key, value in instance.data.items(): + data[key] = value + + data["family"] = "shot" + data["families"] = [] + + data["subset"] = data["family"] + "Main" + + data["name"] = data["subset"] + "_" + data["asset"] + + data["label"] = ( + "{} - {} - tasks:{}".format( + data["asset"], + data["subset"], + data["tasks"] + ) + ) + + # Create instance. + self.log.debug("Creating instance with: {}".format(data["name"])) + instance.context.create_instance(**data) diff --git a/pype/plugins/standalonepublisher/publish/extract_shot.py b/pype/plugins/standalonepublisher/publish/extract_shot.py index 907773b90e..7c1486d7b6 100644 --- a/pype/plugins/standalonepublisher/publish/extract_shot.py +++ b/pype/plugins/standalonepublisher/publish/extract_shot.py @@ -13,6 +13,10 @@ class ExtractShot(pype.api.Extractor): hosts = ["standalonepublisher"] families = ["clip"] + # presets + add_representation = None # ".jpeg" + add_audio = True + def process(self, instance): # get ffmpet path ffmpeg_path = pype.lib.get_ffmpeg_tool_path("ffmpeg") @@ -76,41 +80,69 @@ class ExtractShot(pype.api.Extractor): self.log.debug(f"Instance data: {pformat(instance.data)}") - # # Generate jpegs. - # clip_thumbnail = os.path.join( - # staging_dir, instance.data["name"] + ".%04d.jpeg" - # ) - # args = [ffmpeg_path, "-i", clip_trimed_path, clip_thumbnail] - # self.log.info(f"Processing: {args}") - # output = pype.lib._subprocess(args) - # self.log.info(output) - # - # # collect jpeg sequence if editorial data for publish - # # are image sequence - # collection = clique.Collection( - # head=instance.data["name"] + ".", tail='.jpeg', padding=4 - # ) - # for f in os.listdir(staging_dir): - # if collection.match(f): - # collection.add(f) - # - # instance.data["representations"].append({ - # "name": "jpeg", - # "ext": "jpeg", - # "files": list(collection), - # "stagingDir": staging_dir - # }) - # - # # Generate wav file. - # shot_wav = os.path.join(staging_dir, instance.data["name"] + ".wav") - # args = [ffmpeg_path, "-i", clip_trimed_path, shot_wav] - # self.log.info(f"Processing: {args}") - # output = pype.lib._subprocess(args) - # self.log.info(output) - # - # instance.data["representations"].append({ - # "name": "wav", - # "ext": "wav", - # "files": os.path.basename(shot_wav), - # "stagingDir": staging_dir - # }) + if self.add_representation: + # Generate jpegs. + clip_img_sequence = os.path.join( + staging_dir, instance.data["name"] + ".%04d.jpeg" + ) + args = [ffmpeg_path, "-i", clip_trimed_path, clip_img_sequence] + self.log.info(f"Processing: {args}") + output = pype.lib._subprocess(args) + self.log.info(output) + + # collect jpeg sequence if editorial data for publish + # are image sequence + collection = clique.Collection( + head=instance.data["name"] + ".", tail='.jpeg', padding=4 + ) + for f in os.listdir(staging_dir): + if collection.match(f): + collection.add(f) + + instance.data["representations"].append({ + "name": "jpeg", + "ext": "jpeg", + "files": list(collection), + "stagingDir": staging_dir + }) + + if self.add_audio: + audio_ext = ".wav" + # Generate wav file. + shot_wav = os.path.join( + staging_dir, instance.data["name"] + audio_ext) + # Collect data. + data = {} + for key, value in instance.data.items(): + data[key] = value + + data["family"] = "audio" + data["families"] = ["ftrack"] + + data["subset"] = "audioMain" + + data["source"] = shot_wav + + data["name"] = data["subset"] + "_" + data["asset"] + + data["label"] = "{} - {} - ({})".format( + data['asset'], + data["subset"], + audio_ext + ) + + # Create instance. + self.log.debug("Creating instance with: {}".format(data["name"])) + instance = instance.context.create_instance(**data) + + args = [ffmpeg_path, "-i", clip_trimed_path, shot_wav] + self.log.info(f"Processing: {args}") + output = pype.lib._subprocess(args) + self.log.info(output) + + instance.data["representations"] = [{ + "name": "wav", + "ext": "wav", + "files": os.path.basename(shot_wav), + "stagingDir": staging_dir + }] From 8f9e0c9d28d8aa5c5042d417a7841065f72650d7 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 5 Aug 2020 14:05:16 +0200 Subject: [PATCH 13/30] feat(sp): adding audio and shot instancing --- .../publish/integrate_hierarchy_ftrack.py | 2 +- pype/plugins/global/publish/extract_burnin.py | 3 + .../publish/extract_hierarchy_avalon.py | 2 +- pype/plugins/global/publish/extract_review.py | 3 + .../publish/collect_clip_instances.py | 179 +++++++++++++ .../publish/collect_clips.py | 242 ------------------ .../publish/collect_editorial.py | 1 + .../publish/collect_frame_ranges.py | 56 ---- .../publish/collect_hierarchy.py | 54 ++-- .../publish/collect_shots.py | 37 --- .../publish/extract_shot.py | 148 ----------- .../publish/extract_shot_data.py | 104 ++++++++ .../publish/validate_editorial_resources.py | 2 +- 13 files changed, 318 insertions(+), 515 deletions(-) create mode 100644 pype/plugins/standalonepublisher/publish/collect_clip_instances.py delete mode 100644 pype/plugins/standalonepublisher/publish/collect_clips.py delete mode 100644 pype/plugins/standalonepublisher/publish/collect_frame_ranges.py delete mode 100644 pype/plugins/standalonepublisher/publish/collect_shots.py delete mode 100644 pype/plugins/standalonepublisher/publish/extract_shot.py create mode 100644 pype/plugins/standalonepublisher/publish/extract_shot_data.py diff --git a/pype/plugins/ftrack/publish/integrate_hierarchy_ftrack.py b/pype/plugins/ftrack/publish/integrate_hierarchy_ftrack.py index a0059c55a6..cc569ce2d1 100644 --- a/pype/plugins/ftrack/publish/integrate_hierarchy_ftrack.py +++ b/pype/plugins/ftrack/publish/integrate_hierarchy_ftrack.py @@ -35,7 +35,7 @@ class IntegrateHierarchyToFtrack(pyblish.api.ContextPlugin): order = pyblish.api.IntegratorOrder - 0.04 label = 'Integrate Hierarchy To Ftrack' - families = ["clip", "shot"] + families = ["shot"] optional = False def process(self, context): diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py index e1508b9131..08aa2c2d05 100644 --- a/pype/plugins/global/publish/extract_burnin.py +++ b/pype/plugins/global/publish/extract_burnin.py @@ -49,6 +49,9 @@ class ExtractBurnin(pype.api.Extractor): fields = None def process(self, instance): + representation = instance.data["representations"] + self.log.debug(f"_ representation: {representation}") + # ffmpeg doesn't support multipart exrs if instance.data.get("multipartExr") is True: instance_label = ( diff --git a/pype/plugins/global/publish/extract_hierarchy_avalon.py b/pype/plugins/global/publish/extract_hierarchy_avalon.py index 83cf03b042..ab8226f6ef 100644 --- a/pype/plugins/global/publish/extract_hierarchy_avalon.py +++ b/pype/plugins/global/publish/extract_hierarchy_avalon.py @@ -7,7 +7,7 @@ class ExtractHierarchyToAvalon(pyblish.api.ContextPlugin): order = pyblish.api.ExtractorOrder - 0.01 label = "Extract Hierarchy To Avalon" - families = ["clip", "shot", "editorial"] + families = ["clip", "shot"] def process(self, context): if "hierarchyContext" not in context.data: diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index a16c3ce256..46de467b2c 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -50,6 +50,9 @@ class ExtractReview(pyblish.api.InstancePlugin): to_height = 1080 def process(self, instance): + representation = instance.data["representations"] + self.log.debug(f"_ representation: {representation}") + # ffmpeg doesn't support multipart exrs if instance.data.get("multipartExr") is True: instance_label = ( diff --git a/pype/plugins/standalonepublisher/publish/collect_clip_instances.py b/pype/plugins/standalonepublisher/publish/collect_clip_instances.py new file mode 100644 index 0000000000..028a8b6aad --- /dev/null +++ b/pype/plugins/standalonepublisher/publish/collect_clip_instances.py @@ -0,0 +1,179 @@ +import os +import opentimelineio as otio +import tempfile +import pyblish.api +from pype import lib as plib + + +class CollectClipInstances(pyblish.api.InstancePlugin): + """Collect Clips instances from editorial's OTIO sequence""" + + order = pyblish.api.CollectorOrder + 0.01 + label = "Collect Clips" + hosts = ["standalonepublisher"] + families = ["editorial"] + + # presets + subsets = { + "referenceMain": { + "family": "review", + "families": ["review", "ftrack"], + "ftrackFamily": "review", + "extension": ".mp4" + }, + "audioMain": { + "family": "audio", + "families": ["ftrack"], + "ftrackFamily": "audio", + "extension": ".wav" + }, + "shotMain": { + "family": "shot", + "families": [] + } + } + start_frame_offset = None # if 900000 for edl default then -900000 + custom_start_frame = None + + def process(self, instance): + + staging_dir = os.path.normpath( + tempfile.mkdtemp(prefix="pyblish_tmp_") + ) + # get context + context = instance.context + + # create asset_names conversion table + if not context.data.get("assetsShared"): + self.log.debug("Created `assetsShared` in context") + context.data["assetsShared"] = dict() + + # get timeline otio data + timeline = instance.data["otio_timeline"] + fps = plib.get_asset()["data"]["fps"] + + tracks = timeline.each_child( + descended_from_type=otio.schema.track.Track + ) + self.log.debug(f"__ tracks: `{tracks}`") + + # get data from avalon + asset_entity = instance.context.data["assetEntity"] + asset_data = asset_entity["data"] + asset_name = asset_entity["name"] + self.log.debug(f"__ asset_entity: `{asset_entity}`") + + # Timeline data. + handle_start = int(asset_data["handleStart"]) + handle_end = int(asset_data["handleEnd"]) + + instances = [] + for track in tracks: + self.log.debug(f"__ track: `{track}`") + try: + track_start_frame = ( + abs(track.source_range.start_time.value) + ) + except AttributeError: + track_start_frame = 0 + + self.log.debug(f"__ track: `{track}`") + + for clip in track.each_child(): + # skip all generators like black ampty + if isinstance( + clip.media_reference, + otio.schema.GeneratorReference): + continue + + # Transitions are ignored, because Clips have the full frame + # range. + if isinstance(clip, otio.schema.transition.Transition): + continue + + if clip.name is None: + continue + + # basic unique asset name + clip_name = os.path.splitext(clip.name)[0].lower() + name = f"{asset_name.split('_')[0]}_{clip_name}" + + # frame ranges data + clip_in = clip.range_in_parent().start_time.value + clip_out = clip.range_in_parent().end_time_inclusive().value + clip_duration = clip.duration().value + + source_in = clip.trimmed_range().start_time.value + source_out = source_in + clip_duration + source_in_h = source_in - handle_start + source_out_h = source_out + handle_end + + clip_in_h = clip_in - handle_start + clip_out_h = clip_out + handle_end + + # define starting frame for future shot + frame_start = self.custom_start_frame or clip_in + + # add offset in case there is any + if self.start_frame_offset: + frame_start += self.start_frame_offset + + frame_end = frame_start + clip_duration + + # create shared new instance data + instance_data = { + "stagingDir": staging_dir, + + # shared attributes + "asset": name, + "assetShareName": name, + "editorialVideoPath": instance.data[ + "editorialVideoPath"], + "item": clip, + + # parent time properities + "trackStartFrame": track_start_frame, + "handleStart": handle_start, + "handleEnd": handle_end, + "fps": fps, + + # media source + "sourceIn": source_in, + "sourceOut": source_out, + "sourceInH": source_in_h, + "sourceOutH": source_out_h, + + # timeline + "clipIn": clip_in, + "clipOut": clip_out, + "clipDuration": clip_duration, + "clipInH": clip_in_h, + "clipOutH": clip_out_h, + "clipDurationH": clip_duration + handle_start + handle_end, + + # task + "frameStart": frame_start, + "frameEnd": frame_end, + "frameStartH": frame_start - handle_start, + "frameEndH": frame_end + handle_end + } + + # adding subsets to context as instances + for subset, properities in self.subsets.items(): + # adding Review-able instance + subset_instance_data = instance_data.copy() + subset_instance_data.update(properities) + subset_instance_data.update({ + # unique attributes + "name": f"{subset}_{name}", + "label": f"{subset} {name} ({clip_in}-{clip_out})", + "subset": subset + }) + instances.append(instance.context.create_instance( + **subset_instance_data)) + self.log.debug(instance_data) + + context.data["assetsShared"][name] = { + "_clipIn": clip_in, + "_clipOut": clip_out + } diff --git a/pype/plugins/standalonepublisher/publish/collect_clips.py b/pype/plugins/standalonepublisher/publish/collect_clips.py deleted file mode 100644 index 6a00675961..0000000000 --- a/pype/plugins/standalonepublisher/publish/collect_clips.py +++ /dev/null @@ -1,242 +0,0 @@ -import os - -import opentimelineio as otio -from bson import json_util - -import pyblish.api -from pype import lib as plib -from avalon import io - - -class CollectClips(pyblish.api.InstancePlugin): - """Collect Clips instances from editorial's OTIO sequence""" - - order = pyblish.api.CollectorOrder + 0.01 - label = "Collect Clips" - hosts = ["standalonepublisher"] - families = ["editorial"] - - def process(self, instance): - # get context - context = instance.context - - # create asset_names conversion table - if not context.data.get("assetsShared"): - self.log.debug("Created `assetsShared` in context") - context.data["assetsShared"] = dict() - - # get timeline otio data - timeline = instance.data["otio_timeline"] - fps = plib.get_asset()["data"]["fps"] - - tracks = timeline.each_child( - descended_from_type=otio.schema.track.Track - ) - self.log.debug(f"__ tracks: `{tracks}`") - - # get data from avalon - asset_entity = instance.context.data["assetEntity"] - asset_data = asset_entity["data"] - asset_name = asset_entity["name"] - self.log.debug(f"__ asset_entity: `{asset_entity}`") - - # split selected context asset name - asset_name = asset_name.split("_")[0] - - instances = [] - for track in tracks: - self.log.debug(f"__ track: `{track}`") - try: - track_start_frame = ( - abs(track.source_range.start_time.value) - ) - except AttributeError: - track_start_frame = 0 - - self.log.debug(f"__ track: `{track}`") - - for clip in track.each_child(): - # skip all generators like black ampty - if isinstance( - clip.media_reference, - otio.schema.GeneratorReference): - continue - - # Transitions are ignored, because Clips have the full frame - # range. - if isinstance(clip, otio.schema.transition.Transition): - continue - - if clip.name is None: - continue - - clip_name = os.path.splitext(clip.name)[0].lower() - name = f"{asset_name}_{clip_name}" - - source_in = clip.trimmed_range().start_time.value - clip_in = clip.range_in_parent().start_time.value - clip_out = clip.range_in_parent().end_time_inclusive().value - clip_duration = clip.duration().value - self.log.debug(f"__ source_in: `{source_in}`") - self.log.debug(f"__ clip_in: `{clip_in}`") - self.log.debug(f"__ clip_out: `{clip_out}`") - self.log.debug(f"__ clip_duration: `{clip_duration}`") - - label = f"{name} (framerange: {clip_in}-{clip_out})" - - new_instance_data = { - # shared attributes - "representations": [], - # timing properities - "trackStartFrame": track_start_frame, - "sourceIn": source_in, - "sourceOut": source_in + clip_duration, - "clipIn": clip_in, - "clipOut": clip_out, - "clipDuration": clip_duration, - "handleStart": int(asset_data["handleStart"]), - "handleEnd": int(asset_data["handleEnd"]), - "fps": fps - } - - # adding Review-able instance - shot_instance_data = new_instance_data.copy() - shot_instance_data.update({ - # unique attributes - "name": name, - "label": label, - "asset": name, - "subset": "plateRef", - "item": clip, - - # instance properities - "family": "clip", - "families": ["review", "ftrack"], - "ftrackFamily": "review", - "editorialVideoPath": instance.data[ - "editorialVideoPath"] - }) - instances.append( - instance.context.create_instance(**shot_instance_data) - ) - - context.data["assetsShared"][name] = { - "_clipIn": clip_in, - "_clipOut": clip_out - } - - # def process_old(self, instance): - # representation = instance.data["representations"][0] - # file_path = os.path.join( - # representation["stagingDir"], representation["files"] - # ) - # instance.context.data["editorialPath"] = file_path - # - # extension = os.path.splitext(file_path)[1][1:] - # kwargs = {} - # if extension == "edl": - # # EDL has no frame rate embedded so needs explicit frame rate else - # # 24 is asssumed. - # kwargs["rate"] = plib.get_asset()["data"]["fps"] - # - # timeline = otio.adapters.read_from_file(file_path, **kwargs) - # tracks = timeline.each_child( - # descended_from_type=otio.schema.track.Track - # ) - # asset_entity = instance.context.data["assetEntity"] - # asset_name = asset_entity["name"] - # - # # Ask user for sequence start. Usually 10:00:00:00. - # sequence_start_frame = 900000 - # - # # Project specific prefix naming. This needs to be replaced with some - # # options to be more flexible. - # asset_name = asset_name.split("_")[0] - # - # instances = [] - # for track in tracks: - # track_start_frame = ( - # abs(track.source_range.start_time.value) - sequence_start_frame - # ) - # for child in track.each_child(): - # # skip all generators like black ampty - # if isinstance( - # child.media_reference, - # otio.schema.GeneratorReference): - # continue - # - # # Transitions are ignored, because Clips have the full frame - # # range. - # if isinstance(child, otio.schema.transition.Transition): - # continue - # - # if child.name is None: - # continue - # - # # Hardcoded to expect a shot name of "[name].[extension]" - # child_name = os.path.splitext(child.name)[0].lower() - # name = f"{asset_name}_{child_name}" - # - # frame_start = track_start_frame - # frame_start += child.range_in_parent().start_time.value - # frame_end = track_start_frame - # frame_end += child.range_in_parent().end_time_inclusive().value - # - # label = f"{name} (framerange: {frame_start}-{frame_end})" - # instances.append( - # instance.context.create_instance(**{ - # "name": name, - # "label": label, - # "frameStart": frame_start, - # "frameEnd": frame_end, - # "family": "shot", - # "families": ["review", "ftrack"], - # "ftrackFamily": "review", - # "asset": name, - # "subset": "shotMain", - # "representations": [], - # "source": file_path - # }) - # ) - # - # visual_hierarchy = [asset_entity] - # while True: - # visual_parent = io.find_one( - # {"_id": visual_hierarchy[-1]["data"]["visualParent"]} - # ) - # if visual_parent: - # visual_hierarchy.append(visual_parent) - # else: - # visual_hierarchy.append(instance.context.data["projectEntity"]) - # break - # - # context_hierarchy = None - # for entity in visual_hierarchy: - # childs = {} - # if context_hierarchy: - # name = context_hierarchy.pop("name") - # childs = {name: context_hierarchy} - # else: - # for instance in instances: - # childs[instance.data["name"]] = { - # "childs": {}, - # "entity_type": "Shot", - # "custom_attributes": { - # "frameStart": instance.data["frameStart"], - # "frameEnd": instance.data["frameEnd"] - # } - # } - # - # context_hierarchy = { - # "entity_type": entity["data"]["entityType"], - # "childs": childs, - # "name": entity["name"] - # } - # - # name = context_hierarchy.pop("name") - # context_hierarchy = {name: context_hierarchy} - # instance.context.data["hierarchyContext"] = context_hierarchy - # self.log.info( - # "Hierarchy:\n" + - # json_util.dumps(context_hierarchy, sort_keys=True, indent=4) - # ) diff --git a/pype/plugins/standalonepublisher/publish/collect_editorial.py b/pype/plugins/standalonepublisher/publish/collect_editorial.py index fa5b5f13a3..406848628a 100644 --- a/pype/plugins/standalonepublisher/publish/collect_editorial.py +++ b/pype/plugins/standalonepublisher/publish/collect_editorial.py @@ -62,6 +62,7 @@ class CollectEditorial(pyblish.api.InstancePlugin): ) self.log.debug(f"__ video_path: `{video_path}`") instance.data["editorialVideoPath"] = video_path + instance.data["stagingDir"] = staging_dir # get editorial sequence file into otio timeline object extension = os.path.splitext(file_path)[1] diff --git a/pype/plugins/standalonepublisher/publish/collect_frame_ranges.py b/pype/plugins/standalonepublisher/publish/collect_frame_ranges.py deleted file mode 100644 index 5e8292458f..0000000000 --- a/pype/plugins/standalonepublisher/publish/collect_frame_ranges.py +++ /dev/null @@ -1,56 +0,0 @@ -import pyblish.api - - -class CollectClipFrameRanges(pyblish.api.InstancePlugin): - """Collect all frame range data""" - - order = pyblish.api.CollectorOrder + 0.101 - label = "Collect Frame Ranges" - hosts = ["standalonepublisher"] - families = ["clip"] - - # presets - start_frame_offset = None # if 900000 for edl default then -900000 - custom_start_frame = None - - def process(self, instance): - - data = dict() - - # Timeline data. - handle_start = instance.data["handleStart"] - handle_end = instance.data["handleEnd"] - - source_in_h = instance.data("sourceInH", - instance.data("sourceIn") - handle_start) - source_out_h = instance.data("sourceOutH", - instance.data("sourceOut") + handle_end) - - timeline_in = instance.data["clipIn"] - timeline_out = instance.data["clipOut"] - - timeline_in_h = timeline_in - handle_start - timeline_out_h = timeline_out + handle_end - - # define starting frame for future shot - frame_start = self.custom_start_frame or timeline_in - - # add offset in case there is any - if self.start_frame_offset: - frame_start += self.start_frame_offset - - frame_end = frame_start + (timeline_out - timeline_in) - - data.update({ - "sourceInH": source_in_h, - "sourceOutH": source_out_h, - "frameStart": frame_start, - "frameEnd": frame_end, - "clipInH": timeline_in_h, - "clipOutH": timeline_out_h, - "clipDurationH": instance.data.get( - "clipDuration") + handle_start + handle_end - } - ) - self.log.debug("__ data: {}".format(data)) - instance.data.update(data) diff --git a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py index 87878d560e..8380706457 100644 --- a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py +++ b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py @@ -16,7 +16,7 @@ class CollectHierarchyInstance(pyblish.api.InstancePlugin): label = "Collect Hierarchy Clip" order = pyblish.api.CollectorOrder + 0.101 hosts = ["standalonepublisher"] - families = ["clip"] + families = ["shot"] # presets shot_rename_template = None @@ -141,7 +141,6 @@ class CollectHierarchyInstance(pyblish.api.InstancePlugin): # dealing with shared attributes across instances # with the same asset name - if assets_shared.get(asset): self.log.debug("Adding to shared assets: `{}`".format( asset)) @@ -153,7 +152,6 @@ class CollectHierarchyInstance(pyblish.api.InstancePlugin): "asset": instance.data["asset"], "hierarchy": instance.data["hierarchy"], "parents": instance.data["parents"], - "fps": instance.data["fps"], "tasks": instance.data["tasks"] }) @@ -185,35 +183,22 @@ class CollectHierarchyContext(pyblish.api.ContextPlugin): final_context = {} for instance in instances: - if 'clip' not in instance.data.get('family', ''): + if 'editorial' in instance.data.get('family', ''): continue - - name = instance.data["asset"] - - # get handles - handle_start = int(instance.data["handleStart"]) - handle_end = int(instance.data["handleEnd"]) - - # inject assetsShared to other plates types + # inject assetsShared to other instances with + # the same `assetShareName` attribute in data assets_shared = context.data.get("assetsShared") + asset_shared_name = instance.data.get("assetShareName") + self.log.debug(f"_ assets_shared: {assets_shared}") + self.log.debug(f"_ asset_shared_name: {asset_shared_name}") - if assets_shared: - s_asset_data = assets_shared.get(name) - if s_asset_data: - self.log.debug("__ s_asset_data: {}".format(s_asset_data)) - name = instance.data["asset"] = s_asset_data["asset"] - instance.data["parents"] = s_asset_data["parents"] - instance.data["hierarchy"] = s_asset_data["hierarchy"] - instance.data["tasks"] = s_asset_data["tasks"] - instance.data["fps"] = s_asset_data["fps"] - - # adding frame start if any on instance - start_frame = s_asset_data.get("startingFrame") - if start_frame: - instance.data["frameStart"] = start_frame - instance.data["frameEnd"] = start_frame + ( - instance.data["clipOut"] - - instance.data["clipIn"]) + s_asset_data = assets_shared.get(asset_shared_name) + if s_asset_data: + self.log.debug("__ s_asset_data: {}".format(s_asset_data)) + instance.data["asset"] = s_asset_data["asset"] + instance.data["parents"] = s_asset_data["parents"] + instance.data["hierarchy"] = s_asset_data["hierarchy"] + instance.data["tasks"] = s_asset_data["tasks"] self.log.debug( "__ instance.data[parents]: {}".format( @@ -229,6 +214,17 @@ class CollectHierarchyContext(pyblish.api.ContextPlugin): "__ instance.data[name]: {}".format(instance.data["name"]) ) + # generate hierarchy data only on shot instances + if 'shot' not in instance.data.get('family', ''): + continue + + name = instance.data["asset"] + + # get handles + handle_start = int(instance.data["handleStart"]) + handle_end = int(instance.data["handleEnd"]) + + in_info = {} # suppose that all instances are Shots diff --git a/pype/plugins/standalonepublisher/publish/collect_shots.py b/pype/plugins/standalonepublisher/publish/collect_shots.py deleted file mode 100644 index 269a65fd86..0000000000 --- a/pype/plugins/standalonepublisher/publish/collect_shots.py +++ /dev/null @@ -1,37 +0,0 @@ -from pyblish import api - - -class CollectShots(api.InstancePlugin): - """Collect Shot from Clip.""" - - # Run just before CollectClipSubsets - order = api.CollectorOrder + 0.1021 - label = "Collect Shots" - hosts = ["standalonepublisher"] - families = ["clip"] - - def process(self, instance): - - # Collect data. - data = {} - for key, value in instance.data.items(): - data[key] = value - - data["family"] = "shot" - data["families"] = [] - - data["subset"] = data["family"] + "Main" - - data["name"] = data["subset"] + "_" + data["asset"] - - data["label"] = ( - "{} - {} - tasks:{}".format( - data["asset"], - data["subset"], - data["tasks"] - ) - ) - - # Create instance. - self.log.debug("Creating instance with: {}".format(data["name"])) - instance.context.create_instance(**data) diff --git a/pype/plugins/standalonepublisher/publish/extract_shot.py b/pype/plugins/standalonepublisher/publish/extract_shot.py deleted file mode 100644 index 7c1486d7b6..0000000000 --- a/pype/plugins/standalonepublisher/publish/extract_shot.py +++ /dev/null @@ -1,148 +0,0 @@ -import os -import clique -import pype.api -import pype.lib as plib - -from pprint import pformat - - -class ExtractShot(pype.api.Extractor): - """Extract shot "mov" and "wav" files.""" - - label = "Extract Shot" - hosts = ["standalonepublisher"] - families = ["clip"] - - # presets - add_representation = None # ".jpeg" - add_audio = True - - def process(self, instance): - # get ffmpet path - ffmpeg_path = pype.lib.get_ffmpeg_tool_path("ffmpeg") - - # get staging dir - staging_dir = self.staging_dir(instance) - self.log.info("Staging dir set to: `{}`".format(staging_dir)) - - # Generate mov file. - fps = instance.data["fps"] - video_file_path = instance.data["editorialVideoPath"] - ext = os.path.splitext(os.path.basename(video_file_path))[-1] - - clip_trimed_path = os.path.join( - staging_dir, instance.data["name"] + ext) - - # check video file metadata - input_data = plib.ffprobe_streams(video_file_path)[0] - self.log.debug(f"__ input_data: `{input_data}`") - - args = [ - ffmpeg_path, - "-ss", str(instance.data["clipIn"] / fps), - "-i", video_file_path, - "-t", str( - (instance.data["clipOut"] - instance.data["clipIn"] + 1) / - fps - ), - "-crf", "18", - "-pix_fmt", "yuv420p", - clip_trimed_path - ] - self.log.info(f"Processing: {args}") - ffmpeg_args = " ".join(args) - output = pype.api.subprocess(ffmpeg_args) - self.log.info(output) - - instance.data["families"].remove("review") - instance.data["families"].append("clip") - instance.data["family"] = "review" - - # frame ranges - frame_start = int(instance.data["frameStart"]) - frame_end = int(instance.data["frameEnd"]) - handle_start = int(instance.data["handleStart"]) - handle_end = int(instance.data["handleEnd"]) - - instance.data["representations"].append({ - "name": ext[1:], - "ext": ext[1:], - "files": os.path.basename(clip_trimed_path), - "stagingDir": staging_dir, - "frameStart": frame_start, - "frameEnd": frame_end, - "frameStartFtrack": frame_start - handle_start, - "frameEndFtrack": frame_end - handle_end, - "fps": fps, - "thumbnail": True, - "tags": ["review", "ftrackreview", "delete"] - }) - - self.log.debug(f"Instance data: {pformat(instance.data)}") - - if self.add_representation: - # Generate jpegs. - clip_img_sequence = os.path.join( - staging_dir, instance.data["name"] + ".%04d.jpeg" - ) - args = [ffmpeg_path, "-i", clip_trimed_path, clip_img_sequence] - self.log.info(f"Processing: {args}") - output = pype.lib._subprocess(args) - self.log.info(output) - - # collect jpeg sequence if editorial data for publish - # are image sequence - collection = clique.Collection( - head=instance.data["name"] + ".", tail='.jpeg', padding=4 - ) - for f in os.listdir(staging_dir): - if collection.match(f): - collection.add(f) - - instance.data["representations"].append({ - "name": "jpeg", - "ext": "jpeg", - "files": list(collection), - "stagingDir": staging_dir - }) - - if self.add_audio: - audio_ext = ".wav" - # Generate wav file. - shot_wav = os.path.join( - staging_dir, instance.data["name"] + audio_ext) - # Collect data. - data = {} - for key, value in instance.data.items(): - data[key] = value - - data["family"] = "audio" - data["families"] = ["ftrack"] - - data["subset"] = "audioMain" - - data["source"] = shot_wav - - data["name"] = data["subset"] + "_" + data["asset"] - - data["label"] = "{} - {} - ({})".format( - data['asset'], - data["subset"], - audio_ext - ) - - # Create instance. - self.log.debug("Creating instance with: {}".format(data["name"])) - instance = instance.context.create_instance(**data) - - args = [ffmpeg_path, "-i", clip_trimed_path, shot_wav] - self.log.info(f"Processing: {args}") - output = pype.lib._subprocess(args) - self.log.info(output) - - instance.data["representations"] = [{ - "name": "wav", - "ext": "wav", - "files": os.path.basename(shot_wav), - "stagingDir": staging_dir - }] diff --git a/pype/plugins/standalonepublisher/publish/extract_shot_data.py b/pype/plugins/standalonepublisher/publish/extract_shot_data.py new file mode 100644 index 0000000000..2732d0fa59 --- /dev/null +++ b/pype/plugins/standalonepublisher/publish/extract_shot_data.py @@ -0,0 +1,104 @@ +import os +import clique +import pype.api +import pype.lib as plib + +from pprint import pformat + + +class ExtractShotData(pype.api.Extractor): + """Extract shot "mov" and "wav" files.""" + + label = "Extract Shot Data" + hosts = ["standalonepublisher"] + families = ["review", "audio"] + + # presets + add_representation = None # ".jpeg" + + def process(self, instance): + representation = instance.data.get("representations") + self.log.debug(f"_ representation: {representation}") + + if not representation: + instance.data["representations"] = list() + + # get ffmpet path + ffmpeg_path = pype.lib.get_ffmpeg_tool_path("ffmpeg") + + # get staging dir + staging_dir = self.staging_dir(instance) + self.log.info("Staging dir set to: `{}`".format(staging_dir)) + + # Generate mov file. + fps = instance.data["fps"] + video_file_path = instance.data["editorialVideoPath"] + ext = instance.data.get("extension", ".mov") + + clip_trimed_path = os.path.join( + staging_dir, instance.data["name"] + ext) + # + # # check video file metadata + # input_data = plib.ffprobe_streams(video_file_path)[0] + # self.log.debug(f"__ input_data: `{input_data}`") + + args = [ + ffmpeg_path, + "-ss", str(instance.data["clipInH"] / fps), + "-i", video_file_path, + "-t", str(instance.data["clipDurationH"] / fps), + "-crf", "18", + "-pix_fmt", "yuv420p", + clip_trimed_path + ] + self.log.info(f"Processing: {args}") + ffmpeg_args = " ".join(args) + output = pype.api.subprocess(ffmpeg_args) + self.log.info(output) + + repr = { + "name": ext[1:], + "ext": ext[1:], + "files": os.path.basename(clip_trimed_path), + "stagingDir": staging_dir, + "frameStart": int(instance.data["frameStart"]), + "frameEnd": int(instance.data["frameEnd"]), + "frameStartFtrack": int(instance.data["frameStartH"]), + "frameEndFtrack": int(instance.data["frameEndH"]), + "fps": fps, + } + + if ext[1:] in ["mov", "mp4"]: + repr.update({ + "thumbnail": True, + "tags": ["review", "ftrackreview", "delete"]}) + + instance.data["representations"].append(repr) + + if self.add_representation: + # Generate jpegs. + clip_img_sequence = os.path.join( + staging_dir, instance.data["name"] + ".%04d.jpeg" + ) + args = [ffmpeg_path, "-i", clip_trimed_path, clip_img_sequence] + self.log.info(f"Processing: {args}") + output = pype.lib._subprocess(args) + self.log.info(output) + + # collect jpeg sequence if editorial data for publish + # are image sequence + collection = clique.Collection( + head=instance.data["name"] + ".", tail='.jpeg', padding=4 + ) + for f in os.listdir(staging_dir): + if collection.match(f): + collection.add(f) + + instance.data["representations"].append({ + "name": "jpeg", + "ext": "jpeg", + "files": list(collection), + "stagingDir": staging_dir + }) + + self.log.debug(f"Instance data: {pformat(instance.data)}") diff --git a/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py b/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py index 6033ed919b..ebc449c4ec 100644 --- a/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py +++ b/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py @@ -9,7 +9,7 @@ class ValidateEditorialResources(pyblish.api.InstancePlugin): label = "Validate Editorial Resources" hosts = ["standalonepublisher"] - families = ["clip"] + families = ["audio", "review"] order = pype.api.ValidateContentsOrder def process(self, instance): From ef2e4d507d783758a8edf388b2454d0fff68cc8f Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 5 Aug 2020 16:46:12 +0200 Subject: [PATCH 14/30] fix(sp): hierarchy was not generating proper parents --- .../publish/collect_hierarchy.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py index 8380706457..8e48d141f2 100644 --- a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py +++ b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py @@ -88,21 +88,22 @@ class CollectHierarchyInstance(pyblish.api.InstancePlugin): if self.shot_add_hierarchy: # fill the parents parts from presets - for parent in self.shot_add_hierarchy["parents"]: - if not self.shot_add_hierarchy["parents"][parent]: - prnt = {"entity"} - else: - self.shot_add_hierarchy["parents"][parent] = self.shot_add_hierarchy[ - "parents"][parent].format(**self.hierarchy_data) - prnt = self.convert_to_entity( - parent, self.shot_add_hierarchy["parents"][parent]) + shot_add_hierarchy = self.shot_add_hierarchy.copy() + hierarchy_parents = shot_add_hierarchy["parents"].copy() + for parent in hierarchy_parents: + hierarchy_parents[parent] = hierarchy_parents[parent].format( + **self.hierarchy_data) + prnt = self.convert_to_entity( + parent, hierarchy_parents[parent]) parents.append(prnt) - hierarchy = self.shot_add_hierarchy[ - "parents_path"].format(**self.shot_add_hierarchy["parents"]) + hierarchy = shot_add_hierarchy[ + "parents_path"].format(**hierarchy_parents) instance.data["hierarchy"] = hierarchy instance.data["parents"] = parents + self.log.debug(f"_>_ hierarchy: {hierarchy}") + self.log.debug(f"_>_ parents: {parents}") if self.shot_add_tasks: instance.data["tasks"] = self.shot_add_tasks @@ -180,14 +181,13 @@ class CollectHierarchyContext(pyblish.api.ContextPlugin): def process(self, context): instances = context # create hierarchyContext attr if context has none - + assets_shared = context.data.get("assetsShared") final_context = {} for instance in instances: if 'editorial' in instance.data.get('family', ''): continue # inject assetsShared to other instances with # the same `assetShareName` attribute in data - assets_shared = context.data.get("assetsShared") asset_shared_name = instance.data.get("assetShareName") self.log.debug(f"_ assets_shared: {assets_shared}") self.log.debug(f"_ asset_shared_name: {asset_shared_name}") From 7ed3c2686aa20e8b95a4f2d598ca69ea60ce3d19 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 6 Aug 2020 21:21:18 +0200 Subject: [PATCH 15/30] fix(sp): removing loggers for speedup --- .../publish/collect_hierarchy.py | 29 +------------------ 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py index 8e48d141f2..7433dc9b3a 100644 --- a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py +++ b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py @@ -55,13 +55,10 @@ class CollectHierarchyInstance(pyblish.api.InstancePlugin): continue self.hierarchy_data[type] = match[-1] - self.log.debug("__ hierarchy_data: {}".format(self.hierarchy_data)) - # format to new shot name self.shot_name = self.shot_rename_template.format( **self.hierarchy_data) instance.data["asset"] = self.shot_name - self.log.debug("__ self.shot_name: {}".format(self.shot_name)) def create_hierarchy(self, instance): parents = list() @@ -77,7 +74,6 @@ class CollectHierarchyInstance(pyblish.api.InstancePlugin): visual_hierarchy.append( instance.context.data["projectEntity"]) break - self.log.debug("__ visual_hierarchy: {}".format(visual_hierarchy)) # add current selection context hierarchy from standalonepublisher for entity in reversed(visual_hierarchy): @@ -102,8 +98,6 @@ class CollectHierarchyInstance(pyblish.api.InstancePlugin): instance.data["hierarchy"] = hierarchy instance.data["parents"] = parents - self.log.debug(f"_>_ hierarchy: {hierarchy}") - self.log.debug(f"_>_ parents: {parents}") if self.shot_add_tasks: instance.data["tasks"] = self.shot_add_tasks @@ -143,8 +137,6 @@ class CollectHierarchyInstance(pyblish.api.InstancePlugin): # dealing with shared attributes across instances # with the same asset name if assets_shared.get(asset): - self.log.debug("Adding to shared assets: `{}`".format( - asset)) asset_shared = assets_shared.get(asset) else: asset_shared = assets_shared[asset] @@ -189,31 +181,14 @@ class CollectHierarchyContext(pyblish.api.ContextPlugin): # inject assetsShared to other instances with # the same `assetShareName` attribute in data asset_shared_name = instance.data.get("assetShareName") - self.log.debug(f"_ assets_shared: {assets_shared}") - self.log.debug(f"_ asset_shared_name: {asset_shared_name}") s_asset_data = assets_shared.get(asset_shared_name) if s_asset_data: - self.log.debug("__ s_asset_data: {}".format(s_asset_data)) instance.data["asset"] = s_asset_data["asset"] instance.data["parents"] = s_asset_data["parents"] instance.data["hierarchy"] = s_asset_data["hierarchy"] instance.data["tasks"] = s_asset_data["tasks"] - self.log.debug( - "__ instance.data[parents]: {}".format( - instance.data["parents"] - ) - ) - self.log.debug( - "__ instance.data[hierarchy]: {}".format( - instance.data["hierarchy"] - ) - ) - self.log.debug( - "__ instance.data[name]: {}".format(instance.data["name"]) - ) - # generate hierarchy data only on shot instances if 'shot' not in instance.data.get('family', ''): continue @@ -245,7 +220,6 @@ class CollectHierarchyContext(pyblish.api.ContextPlugin): in_info['tasks'] = instance.data['tasks'] parents = instance.data.get('parents', []) - self.log.debug("__ in_info: {}".format(in_info)) actual = {name: in_info} @@ -261,5 +235,4 @@ class CollectHierarchyContext(pyblish.api.ContextPlugin): # adding hierarchy context to instance context.data["hierarchyContext"] = final_context - self.log.debug("context.data[hierarchyContext] is: {}".format( - context.data["hierarchyContext"])) + self.log.info("Hierarchy instance collected") From 4261972a8def9374f5205d68e86d29730efe444a Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 6 Aug 2020 21:22:23 +0200 Subject: [PATCH 16/30] feat(sp): editorial with multiple edl file instances --- pype/modules/standalonepublish/publish.py | 10 +- .../publish/collect_clip_instances.py | 17 ++-- .../publish/collect_context.py | 95 +++++++++++++++---- .../publish/collect_editorial.py | 4 + 4 files changed, 91 insertions(+), 35 deletions(-) diff --git a/pype/modules/standalonepublish/publish.py b/pype/modules/standalonepublish/publish.py index d2d9eb486f..0a30d5f2cb 100644 --- a/pype/modules/standalonepublish/publish.py +++ b/pype/modules/standalonepublish/publish.py @@ -5,8 +5,7 @@ import tempfile import random import string -from avalon import io, api -import importlib +from avalon import io import pype from pype.api import execute, Logger @@ -62,8 +61,6 @@ def set_context(project, asset, task, app): def publish(data, gui=True): # cli pyblish seems like better solution return cli_publish(data, gui) - # # this uses avalon pyblish launch tool - # avalon_api_publish(data, gui) def cli_publish(data, gui=True): @@ -76,10 +73,6 @@ def cli_publish(data, gui=True): chars = "".join([random.choice(string.ascii_letters) for i in range(15)]) staging_dir = tempfile.mkdtemp(chars) - # create json for return data - return_data_path = ( - staging_dir + os.path.basename(staging_dir) + 'return.json' - ) # create also json and fill with data json_data_path = staging_dir + os.path.basename(staging_dir) + '.json' with open(json_data_path, 'w') as outfile: @@ -88,7 +81,6 @@ def cli_publish(data, gui=True): envcopy = os.environ.copy() envcopy["PYBLISH_HOSTS"] = "standalonepublisher" envcopy["SAPUBLISH_INPATH"] = json_data_path - envcopy["SAPUBLISH_OUTPATH"] = return_data_path envcopy["PYBLISHGUI"] = "pyblish_pype" envcopy["PUBLISH_PATHS"] = os.pathsep.join(PUBLISH_PATHS) diff --git a/pype/plugins/standalonepublisher/publish/collect_clip_instances.py b/pype/plugins/standalonepublisher/publish/collect_clip_instances.py index 028a8b6aad..f99e56095c 100644 --- a/pype/plugins/standalonepublisher/publish/collect_clip_instances.py +++ b/pype/plugins/standalonepublisher/publish/collect_clip_instances.py @@ -36,16 +36,18 @@ class CollectClipInstances(pyblish.api.InstancePlugin): custom_start_frame = None def process(self, instance): - staging_dir = os.path.normpath( tempfile.mkdtemp(prefix="pyblish_tmp_") ) # get context context = instance.context + # attribute for checking duplicity during creation + if not context.data.get("assetNameCheck"): + context.data["assetNameCheck"] = list() + # create asset_names conversion table if not context.data.get("assetsShared"): - self.log.debug("Created `assetsShared` in context") context.data["assetsShared"] = dict() # get timeline otio data @@ -55,13 +57,11 @@ class CollectClipInstances(pyblish.api.InstancePlugin): tracks = timeline.each_child( descended_from_type=otio.schema.track.Track ) - self.log.debug(f"__ tracks: `{tracks}`") # get data from avalon asset_entity = instance.context.data["assetEntity"] asset_data = asset_entity["data"] asset_name = asset_entity["name"] - self.log.debug(f"__ asset_entity: `{asset_entity}`") # Timeline data. handle_start = int(asset_data["handleStart"]) @@ -69,7 +69,6 @@ class CollectClipInstances(pyblish.api.InstancePlugin): instances = [] for track in tracks: - self.log.debug(f"__ track: `{track}`") try: track_start_frame = ( abs(track.source_range.start_time.value) @@ -77,8 +76,6 @@ class CollectClipInstances(pyblish.api.InstancePlugin): except AttributeError: track_start_frame = 0 - self.log.debug(f"__ track: `{track}`") - for clip in track.each_child(): # skip all generators like black ampty if isinstance( @@ -98,6 +95,11 @@ class CollectClipInstances(pyblish.api.InstancePlugin): clip_name = os.path.splitext(clip.name)[0].lower() name = f"{asset_name.split('_')[0]}_{clip_name}" + if name not in context.data["assetNameCheck"]: + context.data["assetNameCheck"].append(name) + else: + self.log.warning(f"duplicate shot name: {name}") + # frame ranges data clip_in = clip.range_in_parent().start_time.value clip_out = clip.range_in_parent().end_time_inclusive().value @@ -171,7 +173,6 @@ class CollectClipInstances(pyblish.api.InstancePlugin): }) instances.append(instance.context.create_instance( **subset_instance_data)) - self.log.debug(instance_data) context.data["assetsShared"][name] = { "_clipIn": clip_in, diff --git a/pype/plugins/standalonepublisher/publish/collect_context.py b/pype/plugins/standalonepublisher/publish/collect_context.py index 8bd4e609ab..4dcb25f927 100644 --- a/pype/plugins/standalonepublisher/publish/collect_context.py +++ b/pype/plugins/standalonepublisher/publish/collect_context.py @@ -36,18 +36,6 @@ class CollectContextDataSAPublish(pyblish.api.ContextPlugin): def process(self, context): # get json paths from os and load them io.install() - input_json_path = os.environ.get("SAPUBLISH_INPATH") - output_json_path = os.environ.get("SAPUBLISH_OUTPATH") - - # context.data["stagingDir"] = os.path.dirname(input_json_path) - context.data["returnJsonPath"] = output_json_path - - with open(input_json_path, "r") as f: - in_data = json.load(f) - - asset_name = in_data["asset"] - family = in_data["family"] - subset = in_data["subset"] # Load presets presets = context.data.get("presets") @@ -57,19 +45,92 @@ class CollectContextDataSAPublish(pyblish.api.ContextPlugin): presets = config.get_presets() project = io.find_one({"type": "project"}) - asset = io.find_one({"type": "asset", "name": asset_name}) context.data["project"] = project + + # get json file context + input_json_path = os.environ.get("SAPUBLISH_INPATH") + + with open(input_json_path, "r") as f: + in_data = json.load(f) + self.log.debug(f"_ in_data: {in_data}") + + self.asset_name = in_data["asset"] + self.family = in_data["family"] + asset = io.find_one({"type": "asset", "name": self.asset_name}) context.data["asset"] = asset + # exception for editorial + if "editorial" in self.family: + # avoid subset name duplicity + if not context.data.get("subsetNamesCheck"): + context.data["subsetNamesCheck"] = list() + + in_data_list = list() + representations = in_data.pop("representations") + for repr in representations: + in_data_copy = in_data.copy() + ext = repr["ext"][1:] + subset = in_data_copy["subset"] + # filter out non editorial files + if ext not in ["edl", "xml"]: + in_data_copy["representations"] = [repr] + in_data_copy["subset"] = f"{ext}{subset}" + in_data_list.append(in_data_copy) + + files = repr.pop("files") + + # delete unneeded keys + delete_repr_keys = ["frameStart", "frameEnd"] + for k in delete_repr_keys: + if repr.get(k): + repr.pop(k) + + # convert files to list if it isnt + if not isinstance(files, list): + files = [files] + + self.log.debug(f"_ files: {files}") + for index, f in enumerate(files): + index += 1 + # copy dictionaries + in_data_copy = in_data_copy.copy() + repr_new = repr.copy() + + repr_new["files"] = f + repr_new["name"] = ext + in_data_copy["representations"] = [repr_new] + + # create subset Name + new_subset = f"{ext}{index}{subset}" + while new_subset in context.data["subsetNamesCheck"]: + index += 1 + new_subset = f"{ext}{index}{subset}" + + context.data["subsetNamesCheck"].append(new_subset) + in_data_copy["subset"] = new_subset + in_data_list.append(in_data_copy) + self.log.info(f"Creating subset: {ext}{index}{subset}") + else: + in_data_list = [in_data] + + self.log.debug(f"_ in_data_list: {in_data_list}") + + for in_data in in_data_list: + # create instance + self.create_instance(context, in_data) + + def create_instance(self, context, in_data): + subset = in_data["subset"] + instance = context.create_instance(subset) instance.data.update( { "subset": subset, - "asset": asset_name, + "asset": self.asset_name, "label": subset, "name": subset, - "family": family, + "family": self.family, "version": in_data.get("version", 1), "frameStart": in_data.get("representations", [None])[0].get( "frameStart", None @@ -77,7 +138,7 @@ class CollectContextDataSAPublish(pyblish.api.ContextPlugin): "frameEnd": in_data.get("representations", [None])[0].get( "frameEnd", None ), - "families": [family, "ftrack"], + "families": [self.family, "ftrack"], } ) self.log.info("collected instance: {}".format(instance.data)) @@ -105,5 +166,3 @@ class CollectContextDataSAPublish(pyblish.api.ContextPlugin): self.log.debug("Adding review family") instance.data["representations"].append(component) - - self.log.info(in_data) diff --git a/pype/plugins/standalonepublisher/publish/collect_editorial.py b/pype/plugins/standalonepublisher/publish/collect_editorial.py index 406848628a..a31125d9a8 100644 --- a/pype/plugins/standalonepublisher/publish/collect_editorial.py +++ b/pype/plugins/standalonepublisher/publish/collect_editorial.py @@ -35,6 +35,10 @@ class CollectEditorial(pyblish.api.InstancePlugin): extensions = [".mov"] def process(self, instance): + # remove context test attribute + if instance.context.data.get("subsetNamesCheck"): + instance.context.data.pop("subsetNamesCheck") + self.log.debug(f"__ instance: `{instance}`") # get representation with editorial file for representation in instance.data["representations"]: From 4bd0ab0b7b0cfb19f2441e362abb0a8af71f01ea Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 6 Aug 2020 21:22:47 +0200 Subject: [PATCH 17/30] fix(sp): improving data collection --- pype/modules/standalonepublish/widgets/widget_drop_frame.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/modules/standalonepublish/widgets/widget_drop_frame.py b/pype/modules/standalonepublish/widgets/widget_drop_frame.py index c91e906f45..57547a3d5f 100644 --- a/pype/modules/standalonepublish/widgets/widget_drop_frame.py +++ b/pype/modules/standalonepublish/widgets/widget_drop_frame.py @@ -357,7 +357,7 @@ class DropDataFrame(QtWidgets.QFrame): if data['name'] == item.in_data['name']: found = True break - paths = data['files'] + paths = list(data['files']) paths.extend(item.in_data['files']) c, r = clique.assemble(paths) if len(c) == 0: @@ -392,7 +392,7 @@ class DropDataFrame(QtWidgets.QFrame): else: if data['name'] != item.in_data['name']: continue - if data['files'] == item.in_data['files']: + if data['files'] == list(item.in_data['files']): found = True break a_name = 'merge' From d72f5a798e14da0fec26a371c38a20b7f469169c Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 6 Aug 2020 21:23:32 +0200 Subject: [PATCH 18/30] feat(sp): multi-threading hierarchy collection --- .../publish/collect_hierarchy.py | 77 +++++++++++++------ 1 file changed, 55 insertions(+), 22 deletions(-) diff --git a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py index 7433dc9b3a..f518813bfe 100644 --- a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py +++ b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py @@ -2,9 +2,11 @@ import pyblish.api import re import os from avalon import io +import queue +import threading -class CollectHierarchyInstance(pyblish.api.InstancePlugin): +class CollectHierarchyInstance(pyblish.api.ContextPlugin): """Collecting hierarchy context from `parents` and `hierarchy` data present in `clip` family instances coming from the request json data file @@ -18,6 +20,11 @@ class CollectHierarchyInstance(pyblish.api.InstancePlugin): hosts = ["standalonepublisher"] families = ["shot"] + # multiprocessing + num_worker_threads = 10 + queue = queue.Queue() + threads = [] + # presets shot_rename_template = None shot_rename_search_patterns = None @@ -41,29 +48,27 @@ class CollectHierarchyInstance(pyblish.api.InstancePlugin): def rename_with_hierarchy(self, instance): search_text = "" - parent_name = self.asset_entity["name"] + parent_name = instance.context.data["assetEntity"]["name"] clip = instance.data["item"] clip_name = os.path.splitext(clip.name)[0].lower() - if self.shot_rename_search_patterns: search_text += parent_name + clip_name - self.hierarchy_data.update({"clip_name": clip_name}) + instance.data["anatomyData"].update({"clip_name": clip_name}) for type, pattern in self.shot_rename_search_patterns.items(): p = re.compile(pattern) match = p.findall(search_text) if not match: continue - self.hierarchy_data[type] = match[-1] + instance.data["anatomyData"][type] = match[-1] # format to new shot name - self.shot_name = self.shot_rename_template.format( - **self.hierarchy_data) - instance.data["asset"] = self.shot_name + instance.data["asset"] = self.shot_rename_template.format( + **instance.data["anatomyData"]) def create_hierarchy(self, instance): parents = list() hierarchy = "" - visual_hierarchy = [self.asset_entity] + visual_hierarchy = [instance.context.data["assetEntity"]] while True: visual_parent = io.find_one( {"_id": visual_hierarchy[-1]["data"]["visualParent"]} @@ -88,7 +93,7 @@ class CollectHierarchyInstance(pyblish.api.InstancePlugin): hierarchy_parents = shot_add_hierarchy["parents"].copy() for parent in hierarchy_parents: hierarchy_parents[parent] = hierarchy_parents[parent].format( - **self.hierarchy_data) + **instance.data["anatomyData"]) prnt = self.convert_to_entity( parent, hierarchy_parents[parent]) parents.append(prnt) @@ -105,20 +110,49 @@ class CollectHierarchyInstance(pyblish.api.InstancePlugin): instance.data["tasks"] = list() # updating hierarchy data - self.hierarchy_data.update({ - "asset": self.shot_name, + instance.data["anatomyData"].update({ + "asset": instance.data["asset"], "task": "conform" }) - def process(self, instance): + def queue_worker(self, worker_index): + while True: + instance = self.queue.get() + if instance is None: + break + self.processing_instance(instance, worker_index) + self.queue.task_done() + + def process(self, context): + for i in range(self.num_worker_threads): + t = threading.Thread(target=self.queue_worker, args=(i,)) + t.start() + self.threads.append(t) + + for instance in context: + if instance.data["family"] in self.families: + self.queue.put(instance) + + # block until all tasks are done + self.queue.join() + + self.log.info('stopping workers!') + + # stop workers + for i in range(self.num_worker_threads): + self.queue.put(None) + + for t in self.threads: + t.join() + + def processing_instance(self, instance, worker_index): + self.log.info(f"_ worker_index: {worker_index}") + self.log.info(f"_ instance: {instance}") + # adding anatomyData for burnins + instance.data["anatomyData"] = instance.context.data["anatomyData"] + asset = instance.data["asset"] assets_shared = instance.context.data.get("assetsShared") - context = instance.context - anatomy_data = context.data["anatomyData"] - - self.shot_name = instance.data["asset"] - self.hierarchy_data = dict(anatomy_data) - self.asset_entity = context.data["assetEntity"] frame_start = instance.data["frameStart"] frame_end = instance.data["frameEnd"] @@ -128,10 +162,9 @@ class CollectHierarchyInstance(pyblish.api.InstancePlugin): self.create_hierarchy(instance) - # adding anatomyData for burnins - instance.data["anatomyData"] = self.hierarchy_data + shot_name = instance.data["asset"] - label = f"{self.shot_name} ({frame_start}-{frame_end})" + label = f"{shot_name} ({frame_start}-{frame_end})" instance.data["label"] = label # dealing with shared attributes across instances From d57f0faf5340979862a1bab795e6367ad0a73ae2 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 7 Aug 2020 16:31:52 +0200 Subject: [PATCH 19/30] fix(sp): one frame offset --- .../publish/collect_clip_instances.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/pype/plugins/standalonepublisher/publish/collect_clip_instances.py b/pype/plugins/standalonepublisher/publish/collect_clip_instances.py index f99e56095c..180e213342 100644 --- a/pype/plugins/standalonepublisher/publish/collect_clip_instances.py +++ b/pype/plugins/standalonepublisher/publish/collect_clip_instances.py @@ -32,7 +32,7 @@ class CollectClipInstances(pyblish.api.InstancePlugin): "families": [] } } - start_frame_offset = None # if 900000 for edl default then -900000 + timeline_frame_offset = None # if 900000 for edl default then -900000 custom_start_frame = None def process(self, instance): @@ -103,7 +103,14 @@ class CollectClipInstances(pyblish.api.InstancePlugin): # frame ranges data clip_in = clip.range_in_parent().start_time.value clip_out = clip.range_in_parent().end_time_inclusive().value - clip_duration = clip.duration().value + + # add offset in case there is any + if self.timeline_frame_offset: + clip_in += self.timeline_frame_offset + clip_out += self.timeline_frame_offset + + clip_duration = clip.duration().value - 1 + self.log.info(f"clip duration: {clip_duration}") source_in = clip.trimmed_range().start_time.value source_out = source_in + clip_duration @@ -114,13 +121,13 @@ class CollectClipInstances(pyblish.api.InstancePlugin): clip_out_h = clip_out + handle_end # define starting frame for future shot - frame_start = self.custom_start_frame or clip_in - - # add offset in case there is any - if self.start_frame_offset: - frame_start += self.start_frame_offset + if self.custom_start_frame is not None: + frame_start = self.custom_start_frame + else: + frame_start = clip_in frame_end = frame_start + clip_duration + self.log.info(f"frames: {frame_start}-{frame_end}") # create shared new instance data instance_data = { From b4070f5f4ecc1a70e075b70f049c8b5323ecea0c Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 7 Aug 2020 16:32:37 +0200 Subject: [PATCH 20/30] clean(sp): prints --- .../standalonepublisher/publish/collect_clip_instances.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pype/plugins/standalonepublisher/publish/collect_clip_instances.py b/pype/plugins/standalonepublisher/publish/collect_clip_instances.py index 180e213342..a8603a50b4 100644 --- a/pype/plugins/standalonepublisher/publish/collect_clip_instances.py +++ b/pype/plugins/standalonepublisher/publish/collect_clip_instances.py @@ -127,7 +127,6 @@ class CollectClipInstances(pyblish.api.InstancePlugin): frame_start = clip_in frame_end = frame_start + clip_duration - self.log.info(f"frames: {frame_start}-{frame_end}") # create shared new instance data instance_data = { From 059a0c6ae039f9a0f9aa1966346760364e1734b7 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 7 Aug 2020 18:09:57 +0200 Subject: [PATCH 21/30] feat(ps): audio main to version 1 only and fixing 0.5 frame offset --- .../publish/collect_clip_instances.py | 7 ++--- .../publish/extract_shot_data.py | 26 +++++++++++++++---- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/pype/plugins/standalonepublisher/publish/collect_clip_instances.py b/pype/plugins/standalonepublisher/publish/collect_clip_instances.py index a8603a50b4..f4d2810f2b 100644 --- a/pype/plugins/standalonepublisher/publish/collect_clip_instances.py +++ b/pype/plugins/standalonepublisher/publish/collect_clip_instances.py @@ -25,7 +25,8 @@ class CollectClipInstances(pyblish.api.InstancePlugin): "family": "audio", "families": ["ftrack"], "ftrackFamily": "audio", - "extension": ".wav" + "extension": ".wav", + "version": 1 }, "shotMain": { "family": "shot", @@ -109,7 +110,7 @@ class CollectClipInstances(pyblish.api.InstancePlugin): clip_in += self.timeline_frame_offset clip_out += self.timeline_frame_offset - clip_duration = clip.duration().value - 1 + clip_duration = clip.duration().value self.log.info(f"clip duration: {clip_duration}") source_in = clip.trimmed_range().start_time.value @@ -126,7 +127,7 @@ class CollectClipInstances(pyblish.api.InstancePlugin): else: frame_start = clip_in - frame_end = frame_start + clip_duration + frame_end = frame_start + (clip_duration - 1) # create shared new instance data instance_data = { diff --git a/pype/plugins/standalonepublisher/publish/extract_shot_data.py b/pype/plugins/standalonepublisher/publish/extract_shot_data.py index 2732d0fa59..5b9fb43ffa 100644 --- a/pype/plugins/standalonepublisher/publish/extract_shot_data.py +++ b/pype/plugins/standalonepublisher/publish/extract_shot_data.py @@ -42,15 +42,31 @@ class ExtractShotData(pype.api.Extractor): # input_data = plib.ffprobe_streams(video_file_path)[0] # self.log.debug(f"__ input_data: `{input_data}`") + start = float(instance.data["clipInH"]) + dur = float(instance.data["clipDurationH"]) + + if ext in ".wav": + start += 0.5 + dur += 0.5 + args = [ ffmpeg_path, - "-ss", str(instance.data["clipInH"] / fps), + "-ss", str(start / fps), "-i", video_file_path, - "-t", str(instance.data["clipDurationH"] / fps), - "-crf", "18", - "-pix_fmt", "yuv420p", - clip_trimed_path + "-t", str(dur / fps) ] + if ext in [".mov", ".mp4"]: + args.extend([ + "-crf", "18", + "-pix_fmt", "yuv420p"]) + elif ext in ".wav": + args.extend([ + "-vn -acodec pcm_s16le", + "-ar 44100 -ac 2"]) + + # add output path + args.append(clip_trimed_path) + self.log.info(f"Processing: {args}") ffmpeg_args = " ".join(args) output = pype.api.subprocess(ffmpeg_args) From 54d0c8f4b0dc52d20086e8cc9062404df688b060 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 7 Aug 2020 18:13:43 +0200 Subject: [PATCH 22/30] fix(sp): paths with space were not supported --- .../publish/collect_clip_instances.py | 6 +++--- .../standalonepublisher/publish/extract_shot_data.py | 10 +++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pype/plugins/standalonepublisher/publish/collect_clip_instances.py b/pype/plugins/standalonepublisher/publish/collect_clip_instances.py index f4d2810f2b..3d9773d0b2 100644 --- a/pype/plugins/standalonepublisher/publish/collect_clip_instances.py +++ b/pype/plugins/standalonepublisher/publish/collect_clip_instances.py @@ -78,6 +78,9 @@ class CollectClipInstances(pyblish.api.InstancePlugin): track_start_frame = 0 for clip in track.each_child(): + if clip.name is None: + continue + # skip all generators like black ampty if isinstance( clip.media_reference, @@ -89,9 +92,6 @@ class CollectClipInstances(pyblish.api.InstancePlugin): if isinstance(clip, otio.schema.transition.Transition): continue - if clip.name is None: - continue - # basic unique asset name clip_name = os.path.splitext(clip.name)[0].lower() name = f"{asset_name.split('_')[0]}_{clip_name}" diff --git a/pype/plugins/standalonepublisher/publish/extract_shot_data.py b/pype/plugins/standalonepublisher/publish/extract_shot_data.py index 5b9fb43ffa..e0b6196279 100644 --- a/pype/plugins/standalonepublisher/publish/extract_shot_data.py +++ b/pype/plugins/standalonepublisher/publish/extract_shot_data.py @@ -52,7 +52,7 @@ class ExtractShotData(pype.api.Extractor): args = [ ffmpeg_path, "-ss", str(start / fps), - "-i", video_file_path, + "-i", f"\"{video_file_path}\"", "-t", str(dur / fps) ] if ext in [".mov", ".mp4"]: @@ -65,7 +65,7 @@ class ExtractShotData(pype.api.Extractor): "-ar 44100 -ac 2"]) # add output path - args.append(clip_trimed_path) + args.append(f"\"{clip_trimed_path}\"") self.log.info(f"Processing: {args}") ffmpeg_args = " ".join(args) @@ -96,7 +96,11 @@ class ExtractShotData(pype.api.Extractor): clip_img_sequence = os.path.join( staging_dir, instance.data["name"] + ".%04d.jpeg" ) - args = [ffmpeg_path, "-i", clip_trimed_path, clip_img_sequence] + args = [ + ffmpeg_path, "-i", + f"\"{clip_trimed_path}\"", + f"\"{clip_img_sequence}\"" + ] self.log.info(f"Processing: {args}") output = pype.lib._subprocess(args) self.log.info(output) From 1a6a5e4b7fd89dc53513230781d96676dc31c16d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 7 Aug 2020 12:18:54 +0200 Subject: [PATCH 23/30] using -g 1 (same as -intra) to set right keyframes of burning output --- pype/scripts/otio_burnin.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pype/scripts/otio_burnin.py b/pype/scripts/otio_burnin.py index 104ff0255c..16e24757dd 100644 --- a/pype/scripts/otio_burnin.py +++ b/pype/scripts/otio_burnin.py @@ -528,6 +528,9 @@ def burnins_from_data( if pix_fmt: ffmpeg_args.append("-pix_fmt {}".format(pix_fmt)) + # Use group one (same as `-intra` argument, which is deprecated) + ffmpeg_args.append("-g 1") + ffmpeg_args_str = " ".join(ffmpeg_args) burnin.render( output_path, args=ffmpeg_args_str, overwrite=overwrite, **data From 72dc0246d7965f6911c123ecd205edb8a5982f7a Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Mon, 10 Aug 2020 10:04:23 +0200 Subject: [PATCH 24/30] custom path wasn't working for delivery --- .../modules/ftrack/actions/action_delivery.py | 70 +++++++++++++------ 1 file changed, 49 insertions(+), 21 deletions(-) diff --git a/pype/modules/ftrack/actions/action_delivery.py b/pype/modules/ftrack/actions/action_delivery.py index a2048222e5..231aebdf7a 100644 --- a/pype/modules/ftrack/actions/action_delivery.py +++ b/pype/modules/ftrack/actions/action_delivery.py @@ -11,7 +11,7 @@ from avalon.vendor import filelink from pype.api import Anatomy from pype.modules.ftrack.lib import BaseAction, statics_icon -from pype.modules.ftrack.lib.avalon_sync import CustAttrIdKey +from pype.modules.ftrack.lib.avalon_sync import CUST_ATTR_ID_KEY from pype.modules.ftrack.lib.io_nonsingleton import DbConnector @@ -81,13 +81,15 @@ class Delivery(BaseAction): anatomy = Anatomy(project_name) new_anatomies = [] first = None - for key in (anatomy.templates.get("delivery") or {}): - new_anatomies.append({ - "label": key, - "value": key - }) - if first is None: - first = key + for key, template in (anatomy.templates.get("delivery") or {}).items(): + # Use only keys with `{root}` or `{root[*]}` in value + if isinstance(template, str) and "{root" in template: + new_anatomies.append({ + "label": key, + "value": key + }) + if first is None: + first = key skipped = False # Add message if there are any common components @@ -243,7 +245,7 @@ class Delivery(BaseAction): version = entity["version"] parent = asset["parent"] - parent_mongo_id = parent["custom_attributes"].get(CustAttrIdKey) + parent_mongo_id = parent["custom_attributes"].get(CUST_ATTR_ID_KEY) if parent_mongo_id: parent_mongo_id = ObjectId(parent_mongo_id) else: @@ -293,6 +295,20 @@ class Delivery(BaseAction): repres_to_deliver.append(repre) anatomy = Anatomy(project_name) + + format_dict = {} + if location_path: + location_path = location_path.replace("\\", "/") + root_names = anatomy.root_names_from_templates( + anatomy.templates["delivery"] + ) + if root_names is None: + format_dict["root"] = location_path + else: + format_dict["root"] = {} + for name in root_names: + format_dict["root"][name] = location_path + for repre in repres_to_deliver: # Get destination repre path anatomy_data = copy.deepcopy(repre["context"]) @@ -339,25 +355,33 @@ class Delivery(BaseAction): repre_path = self.path_from_represenation(repre, anatomy) # TODO add backup solution where root of path from component # is repalced with root - if not frame: - self.process_single_file( - repre_path, anatomy, anatomy_name, anatomy_data - ) + args = ( + repre_path, + anatomy, + anatomy_name, + anatomy_data, + format_dict + ) + if not frame: + self.process_single_file(*args) else: - self.process_sequence( - repre_path, anatomy, anatomy_name, anatomy_data - ) + self.process_sequence(*args) self.db_con.uninstall() return self.report() def process_single_file( - self, repre_path, anatomy, anatomy_name, anatomy_data + self, repre_path, anatomy, anatomy_name, anatomy_data, format_dict ): anatomy_filled = anatomy.format(anatomy_data) - delivery_path = anatomy_filled["delivery"][anatomy_name] + if format_dict: + template_result = anatomy_filled["delivery"][anatomy_name] + delivery_path = template_result.rootless.format(**format_dict) + else: + delivery_path = anatomy_filled["delivery"][anatomy_name] + delivery_folder = os.path.dirname(delivery_path) if not os.path.exists(delivery_folder): os.makedirs(delivery_folder) @@ -365,7 +389,7 @@ class Delivery(BaseAction): self.copy_file(repre_path, delivery_path) def process_sequence( - self, repre_path, anatomy, anatomy_name, anatomy_data + self, repre_path, anatomy, anatomy_name, anatomy_data, format_dict ): dir_path, file_name = os.path.split(str(repre_path)) @@ -408,8 +432,12 @@ class Delivery(BaseAction): anatomy_data["frame"] = frame_indicator anatomy_filled = anatomy.format(anatomy_data) - delivery_path = anatomy_filled["delivery"][anatomy_name] - print(delivery_path) + if format_dict: + template_result = anatomy_filled["delivery"][anatomy_name] + delivery_path = template_result.rootless.format(**format_dict) + else: + delivery_path = anatomy_filled["delivery"][anatomy_name] + delivery_folder = os.path.dirname(delivery_path) dst_head, dst_tail = delivery_path.split(frame_indicator) dst_padding = src_collection.padding From 3f5c686a6a4b35b7cc854fbc4babdb40104c03e2 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 10 Aug 2020 11:08:15 +0200 Subject: [PATCH 25/30] fix(sp): shots were assigned to wrong parent --- .../publish/collect_hierarchy.py | 44 +++---------------- 1 file changed, 7 insertions(+), 37 deletions(-) diff --git a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py index f518813bfe..7c66f21966 100644 --- a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py +++ b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py @@ -2,9 +2,6 @@ import pyblish.api import re import os from avalon import io -import queue -import threading - class CollectHierarchyInstance(pyblish.api.ContextPlugin): """Collecting hierarchy context from `parents` and `hierarchy` data @@ -20,11 +17,6 @@ class CollectHierarchyInstance(pyblish.api.ContextPlugin): hosts = ["standalonepublisher"] families = ["shot"] - # multiprocessing - num_worker_threads = 10 - queue = queue.Queue() - threads = [] - # presets shot_rename_template = None shot_rename_search_patterns = None @@ -103,6 +95,7 @@ class CollectHierarchyInstance(pyblish.api.ContextPlugin): instance.data["hierarchy"] = hierarchy instance.data["parents"] = parents + self.log.debug(f"Hierarchy: {hierarchy}") if self.shot_add_tasks: instance.data["tasks"] = self.shot_add_tasks @@ -115,38 +108,12 @@ class CollectHierarchyInstance(pyblish.api.ContextPlugin): "task": "conform" }) - def queue_worker(self, worker_index): - while True: - instance = self.queue.get() - if instance is None: - break - self.processing_instance(instance, worker_index) - self.queue.task_done() - def process(self, context): - for i in range(self.num_worker_threads): - t = threading.Thread(target=self.queue_worker, args=(i,)) - t.start() - self.threads.append(t) - for instance in context: if instance.data["family"] in self.families: - self.queue.put(instance) + self.processing_instance(instance) - # block until all tasks are done - self.queue.join() - - self.log.info('stopping workers!') - - # stop workers - for i in range(self.num_worker_threads): - self.queue.put(None) - - for t in self.threads: - t.join() - - def processing_instance(self, instance, worker_index): - self.log.info(f"_ worker_index: {worker_index}") + def processing_instance(self, instance): self.log.info(f"_ instance: {instance}") # adding anatomyData for burnins instance.data["anatomyData"] = instance.context.data["anatomyData"] @@ -163,6 +130,10 @@ class CollectHierarchyInstance(pyblish.api.ContextPlugin): self.create_hierarchy(instance) shot_name = instance.data["asset"] + self.log.debug(f"Shot Name: {shot_name}") + + if instance.data["hierarchy"] not in shot_name: + self.log.warning("wrong parent") label = f"{shot_name} ({frame_start}-{frame_end})" instance.data["label"] = label @@ -232,7 +203,6 @@ class CollectHierarchyContext(pyblish.api.ContextPlugin): handle_start = int(instance.data["handleStart"]) handle_end = int(instance.data["handleEnd"]) - in_info = {} # suppose that all instances are Shots From 0931b067fd7da8a5598257dcf9a791cb460b19e7 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 10 Aug 2020 14:26:00 +0200 Subject: [PATCH 26/30] fix(ca): half frame extension --- pype/plugins/standalonepublisher/publish/extract_shot_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/plugins/standalonepublisher/publish/extract_shot_data.py b/pype/plugins/standalonepublisher/publish/extract_shot_data.py index e0b6196279..8c626b277c 100644 --- a/pype/plugins/standalonepublisher/publish/extract_shot_data.py +++ b/pype/plugins/standalonepublisher/publish/extract_shot_data.py @@ -47,7 +47,6 @@ class ExtractShotData(pype.api.Extractor): if ext in ".wav": start += 0.5 - dur += 0.5 args = [ ffmpeg_path, @@ -62,7 +61,8 @@ class ExtractShotData(pype.api.Extractor): elif ext in ".wav": args.extend([ "-vn -acodec pcm_s16le", - "-ar 44100 -ac 2"]) + "-ar 48000 -ac 2" + ]) # add output path args.append(f"\"{clip_trimed_path}\"") From a3e8828ad1759b8ed943d5edb1fcb2e45d79388e Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 10 Aug 2020 16:39:06 +0200 Subject: [PATCH 27/30] hound(sp): suggestions --- pype/plugins/global/publish/extract_burnin.py | 3 --- pype/plugins/global/publish/extract_review.py | 3 --- .../publish/collect_hierarchy.py | 1 + .../publish/extract_shot_data.py | 1 - .../publish/validate_clips.py | 24 ------------------- 5 files changed, 1 insertion(+), 31 deletions(-) delete mode 100644 pype/plugins/standalonepublisher/publish/validate_clips.py diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py index 08aa2c2d05..e1508b9131 100644 --- a/pype/plugins/global/publish/extract_burnin.py +++ b/pype/plugins/global/publish/extract_burnin.py @@ -49,9 +49,6 @@ class ExtractBurnin(pype.api.Extractor): fields = None def process(self, instance): - representation = instance.data["representations"] - self.log.debug(f"_ representation: {representation}") - # ffmpeg doesn't support multipart exrs if instance.data.get("multipartExr") is True: instance_label = ( diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 46de467b2c..a16c3ce256 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -50,9 +50,6 @@ class ExtractReview(pyblish.api.InstancePlugin): to_height = 1080 def process(self, instance): - representation = instance.data["representations"] - self.log.debug(f"_ representation: {representation}") - # ffmpeg doesn't support multipart exrs if instance.data.get("multipartExr") is True: instance_label = ( diff --git a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py index 7c66f21966..b5d37d0a6c 100644 --- a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py +++ b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py @@ -3,6 +3,7 @@ import re import os from avalon import io + class CollectHierarchyInstance(pyblish.api.ContextPlugin): """Collecting hierarchy context from `parents` and `hierarchy` data present in `clip` family instances coming from the request json data file diff --git a/pype/plugins/standalonepublisher/publish/extract_shot_data.py b/pype/plugins/standalonepublisher/publish/extract_shot_data.py index 8c626b277c..6cbc2c7882 100644 --- a/pype/plugins/standalonepublisher/publish/extract_shot_data.py +++ b/pype/plugins/standalonepublisher/publish/extract_shot_data.py @@ -1,7 +1,6 @@ import os import clique import pype.api -import pype.lib as plib from pprint import pformat diff --git a/pype/plugins/standalonepublisher/publish/validate_clips.py b/pype/plugins/standalonepublisher/publish/validate_clips.py deleted file mode 100644 index 35b81da5c1..0000000000 --- a/pype/plugins/standalonepublisher/publish/validate_clips.py +++ /dev/null @@ -1,24 +0,0 @@ - # Check for clips with the same range - # this is for testing if any vertically neighbouring - # clips has been already processed - clip_matching_with_range = next( - (k for k, v in context.data["assetsShared"].items() - if (v.get("_clipIn", 0) == clip_in) - and (v.get("_clipOut", 0) == clip_out) - ), False) - - # check if clip name is the same in matched - # vertically neighbouring clip - # if it is then it is correct and resent variable to False - # not to be rised wrong name exception - if asset in str(clip_matching_with_range): - clip_matching_with_range = False - - # rise wrong name exception if found one - assert (not clip_matching_with_range), ( - "matching clip: {asset}" - " timeline range ({clip_in}:{clip_out})" - " conflicting with {clip_matching_with_range}" - " >> rename any of clips to be the same as the other <<" - ).format( - **locals()) From 0b4e01eca72e4d95021ad5ea012a491eef0e251d Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Mon, 10 Aug 2020 17:16:50 +0200 Subject: [PATCH 28/30] bump version --- pype/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/version.py b/pype/version.py index ddcf716b76..d70304e62c 100644 --- a/pype/version.py +++ b/pype/version.py @@ -1 +1 @@ -__version__ = "2.11.3" +__version__ = "2.11.4" From 1db9d9e247d2fe1259350c8dc14b2446720a6909 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 10 Aug 2020 17:09:17 +0100 Subject: [PATCH 29/30] fix delivery action variable and create directories --- pype/modules/ftrack/actions/action_delivery.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/pype/modules/ftrack/actions/action_delivery.py b/pype/modules/ftrack/actions/action_delivery.py index 231aebdf7a..7dbb7c65e8 100644 --- a/pype/modules/ftrack/actions/action_delivery.py +++ b/pype/modules/ftrack/actions/action_delivery.py @@ -11,7 +11,7 @@ from avalon.vendor import filelink from pype.api import Anatomy from pype.modules.ftrack.lib import BaseAction, statics_icon -from pype.modules.ftrack.lib.avalon_sync import CUST_ATTR_ID_KEY +from pype.modules.ftrack.lib.avalon_sync import CustAttrIdKey from pype.modules.ftrack.lib.io_nonsingleton import DbConnector @@ -228,12 +228,7 @@ class Delivery(BaseAction): if location_path: location_path = os.path.normpath(location_path) if not os.path.exists(location_path): - return { - "success": False, - "message": ( - "Entered location path does not exists. \"{}\"" - ).format(location_path) - } + os.makedirs(location_path) self.db_con.install() self.db_con.Session["AVALON_PROJECT"] = project_name @@ -245,7 +240,7 @@ class Delivery(BaseAction): version = entity["version"] parent = asset["parent"] - parent_mongo_id = parent["custom_attributes"].get(CUST_ATTR_ID_KEY) + parent_mongo_id = parent["custom_attributes"].get(CustAttrIdKey) if parent_mongo_id: parent_mongo_id = ObjectId(parent_mongo_id) else: From 58cd5f8c7ef9dccca6967a913069344bf5734a02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= <33513211+antirotor@users.noreply.github.com> Date: Wed, 12 Aug 2020 12:06:40 +0200 Subject: [PATCH 30/30] fix invalid scope in validate scene settings --- pype/plugins/harmony/publish/validate_scene_settings.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pype/plugins/harmony/publish/validate_scene_settings.py b/pype/plugins/harmony/publish/validate_scene_settings.py index 3602f1ca22..d7895804bd 100644 --- a/pype/plugins/harmony/publish/validate_scene_settings.py +++ b/pype/plugins/harmony/publish/validate_scene_settings.py @@ -40,13 +40,11 @@ class ValidateSceneSettings(pyblish.api.InstancePlugin): expected_settings["frameEnd"] = frame_end - frame_start + 1 expected_settings["frameStart"] = 1 - - self.log.info(instance.context.data['anatomyData']['asset']) if any(string in instance.context.data['anatomyData']['asset'] - for string in frame_check_filter): - expected_settings.pop("frameEnd") + for string in self.frame_check_filter): + expected_settings.pop("frameEnd") func = """function func() {