From 2d375f092c1df316f633bcb3f68e8f4cab401e6b Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 6 Nov 2020 15:20:59 +0100 Subject: [PATCH] feat(SP): wip publishing editorial with dir of subsets and files --- .../publish/collect_clear_instances.py | 20 +++ .../publish/collect_instance_resources.py | 151 +++++++++++++++++- .../publish/collect_instances.py | 10 +- .../publish/validate_editorial_resources.py | 5 +- 4 files changed, 173 insertions(+), 13 deletions(-) create mode 100644 pype/plugins/standalonepublisher/publish/collect_clear_instances.py diff --git a/pype/plugins/standalonepublisher/publish/collect_clear_instances.py b/pype/plugins/standalonepublisher/publish/collect_clear_instances.py new file mode 100644 index 0000000000..0d64c57d3a --- /dev/null +++ b/pype/plugins/standalonepublisher/publish/collect_clear_instances.py @@ -0,0 +1,20 @@ +""" +Optional: + instance.data["remove"] -> mareker for removing +""" +import pyblish.api + + +class CollectClearInstances(pyblish.api.ContextPlugin): + """Clear all marked instances""" + + order = pyblish.api.CollectorOrder + 0.4999 + label = "Clear Instances" + hosts = ["standalonepublisher"] + + def process(self, context): + + for instance in context: + if instance.data.get("remove"): + self.log.info(f"Removing: {instance}") + context.remove(instance) diff --git a/pype/plugins/standalonepublisher/publish/collect_instance_resources.py b/pype/plugins/standalonepublisher/publish/collect_instance_resources.py index 63b98f2721..25bdffd422 100644 --- a/pype/plugins/standalonepublisher/publish/collect_instance_resources.py +++ b/pype/plugins/standalonepublisher/publish/collect_instance_resources.py @@ -2,6 +2,8 @@ import os import tempfile import pyblish.api from copy import deepcopy +import clique + class CollectInstanceResources(pyblish.api.InstancePlugin): """Collect instance's resources""" @@ -13,6 +15,10 @@ class CollectInstanceResources(pyblish.api.InstancePlugin): families = ["clip"] def process(self, instance): + context = instance.context + self.log.info(f"Processing instance: {instance}") + subset_files = dict() + subset_dirs = list() anatomy = instance.context.data["anatomy"] anatomy_data = deepcopy(instance.context.data["anatomyData"]) anatomy_data.update({"root": anatomy.roots}) @@ -23,6 +29,7 @@ class CollectInstanceResources(pyblish.api.InstancePlugin): editorial_source_root = instance.data["editorialSourceRoot"] editorial_source_path = instance.data["editorialSourcePath"] + # if `editorial_source_path` then loop trough if editorial_source_path: # add family if mov or mp4 found which is longer for # cutting `trimming` to enable `ExtractTrimmingVideoAudio` plugin @@ -33,23 +40,153 @@ class CollectInstanceResources(pyblish.api.InstancePlugin): instance.data["families"] += ["trimming"] return + # if template patern in path then fill it with `anatomy_data` if "{" in editorial_source_root: editorial_source_root = editorial_source_root.format( **anatomy_data) self.log.debug(f"root: {editorial_source_root}") - + # loop `editorial_source_root` and find clip name in folders + # and look for any subset name alternatives for root, dirs, files in os.walk(editorial_source_root): - if subset in root and clip_name in root: - staging_dir = root + correct_clip_dir = None + for d in dirs: + # avoid all non clip dirs + if d not in clip_name: + continue + # found correct dir for clip + correct_clip_dir = d - self.log.debug(f"staging_dir: {staging_dir}") + # continue if clip dir was not found + if not correct_clip_dir: + continue + clip_dir_path = os.path.join(root, correct_clip_dir) + subset_files_items = list() + # list content of clip dir and search for subset items + for subset_item in os.listdir(clip_dir_path): + # avoid all items which are not defined as subsets by name + if subset not in subset_item: + continue - # add `editorialSourceRoot` as staging dir + subset_item_path = os.path.join( + clip_dir_path, subset_item) + # if it is dir store it to `subset_dirs` list + if os.path.isdir(subset_item_path): + subset_dirs.append(subset_item_path) - # if `editorialSourcePath` is none then loop - # trough `editorialSourceRoot` + # if it is file then store it to `subset_files` list + if os.path.isfile(subset_item_path): + subset_files_items.append(subset_item_path) + + if subset_files_items: + subset_files.update({clip_dir_path: subset_files_items}) + if correct_clip_dir: + break + + if subset_dirs: + # look all dirs and check for subset name alternatives + copy_instance_data = deepcopy( + {_k: _v for _k, _v in instance.data.items()}) + + # find next available precise subset name with comprahantion + subset_dir_found = next( + (d for d in subset_dirs + if os.path.basename(d) in subset), + None) + + if not subset_dir_found: + instance.data["remove"] = True + + for _dir in subset_dirs: + sub_dir = os.path.basename(_dir) + instance_data = instance.data + # if subset name is only alternative then create new instance + if sub_dir != subset: + new_instance_data = dict() + for _key, _value in copy_instance_data.items(): + new_instance_data[_key] = _value + if not isinstance(_value, str): + continue + if subset in _value: + new_instance_data[_key] = _value.replace( + subset, sub_dir) + new_instance = context.create_instance( + new_instance_data["name"]) + new_instance.data.update(new_instance_data) + self.log.info(f"Creating new instance: {new_instance}") + instance_data = new_instance.data + + staging_dir = _dir + files = os.listdir(_dir) + collections, remainder = clique.assemble(files) + # self.log.debug(f"collections: {collections}") + # self.log.debug(f"remainder: {remainder}") + # self.log.debug(f"staging_dir: {staging_dir}") + + # add staging_dir to instance_data + instance_data["stagingDir"] = staging_dir + # add representations to instance_data + instance_data["representations"] = list() + + # loop trough collections and create representations + for _collection in collections: + ext = _collection.tail + repre_data = { + "name": ext[1:], + "ext": ext[1:], + "files": [item for item in _collection], + "stagingDir": staging_dir + } + instance_data["representations"].append(repre_data) + + # loop trough reminders and create representations + for _reminding_file in remainder: + ext = os.path.splitext(_reminding_file)[-1] + if ext not in instance_data["extensions"]: + continue + + repre_data = { + "name": ext[1:], + "ext": ext[1:], + "files": _reminding_file, + "stagingDir": staging_dir + } + + # exception for thumbnail + if "thumb" in _reminding_file: + repre_data.update({ + 'name': "thumbnail", + 'thumbnail': True + }) + + # exception for mp4 preview + if ".mp4" in _reminding_file: + frame_start = instance_data["frameStart"] + frame_end = instance_data["frameEnd"] + instance_data["families"].append("review") + repre_data.update({ + "frameStart": 0, + "frameEnd": (frame_end - frame_start) + 1, + "frameStartFtrack": 0, + "frameEndFtrack": (frame_end - frame_start) + 1, + "step": 1, + "fps": context.data.get("fps"), + "name": "review", + "tags": ["review", "ftrackreview"], + }) + + instance_data["representations"].append(repre_data) + + representations = instance_data["representations"] + self.log.debug(f">>>_<<< representations: {representations}") + + if subset_files: + staging_dir = list(subset_files.keys()).pop() + collections, remainder = clique.assemble(subset_files[staging_dir]) + # self.log.debug(f"collections: {collections}") + # self.log.debug(f"remainder: {remainder}") + # self.log.debug(f"staging_dir: {staging_dir}") # if image sequence then create representation > match # with subset name in dict diff --git a/pype/plugins/standalonepublisher/publish/collect_instances.py b/pype/plugins/standalonepublisher/publish/collect_instances.py index 9cd8d9f36c..3d577c1527 100644 --- a/pype/plugins/standalonepublisher/publish/collect_instances.py +++ b/pype/plugins/standalonepublisher/publish/collect_instances.py @@ -69,7 +69,6 @@ class CollectInstances(pyblish.api.InstancePlugin): handle_start = int(asset_data["handleStart"]) handle_end = int(asset_data["handleEnd"]) - instances = [] for track in tracks: try: track_start_frame = ( @@ -179,12 +178,13 @@ class CollectInstances(pyblish.api.InstancePlugin): subset_instance_data.update(properities) subset_instance_data.update({ # unique attributes - "name": f"{subset}_{name}", - "label": f"{subset} {name} ({clip_in}-{clip_out})", + "name": f"{name}_{subset}", + "label": f"{name} {subset} ({clip_in}-{clip_out})", "subset": subset }) - instances.append(instance.context.create_instance( - **subset_instance_data)) + # create new instance + instance.context.create_instance( + **subset_instance_data) context.data["assetsShared"][name] = { "_clipIn": clip_in, diff --git a/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py b/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py index 65581a6cdc..0dfca92f66 100644 --- a/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py +++ b/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py @@ -7,7 +7,10 @@ class ValidateEditorialResources(pyblish.api.InstancePlugin): label = "Validate Editorial Resources" hosts = ["standalonepublisher"] - families = ["clip"] + families = ["clip", "trimming"] + + # make sure it is enabled only if at least both families are available + match = pyblish.api.Subset order = pype.api.ValidateContentsOrder