From 37d81b09481d44b9f17a377a287e85b57872d0d4 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 2 Nov 2020 11:43:00 +0100 Subject: [PATCH 001/147] modified imports in loader --- pype/plugins/tvpaint/load/load_image.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/plugins/tvpaint/load/load_image.py b/pype/plugins/tvpaint/load/load_image.py index ec126adbee..ca4b7a4c86 100644 --- a/pype/plugins/tvpaint/load/load_image.py +++ b/pype/plugins/tvpaint/load/load_image.py @@ -1,5 +1,5 @@ from avalon import api -from avalon.tvpaint import CommunicatorWrapper +from avalon.tvpaint import CommunicationWrapper class ImportImage(api.Loader): @@ -35,4 +35,4 @@ class ImportImage(api.Loader): self.fname.replace("\\", "/"), layer_name ) - return CommunicatorWrapper.execute_george_through_file(george_script) + return CommunicationWrapper.execute_george_through_file(george_script) From 189c72e139bddd95fa2652eb5c5e7b43c198da09 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 2 Nov 2020 11:43:14 +0100 Subject: [PATCH 002/147] added review creator --- pype/plugins/tvpaint/create/create_review.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 pype/plugins/tvpaint/create/create_review.py diff --git a/pype/plugins/tvpaint/create/create_review.py b/pype/plugins/tvpaint/create/create_review.py new file mode 100644 index 0000000000..34c7e0c6ed --- /dev/null +++ b/pype/plugins/tvpaint/create/create_review.py @@ -0,0 +1,19 @@ +from avalon.tvpaint import pipeline, CommunicationWrapper + + +class CreateReview(pipeline.TVPaintCreator): + """Arnold Archive""" + + name = "review" + label = "Review" + family = "review" + icon = "cube" + defaults = ["Main"] + + def process(self): + instances = pipeline.list_instances() + for instance in instances: + if instance["family"] == self.family: + self.log.info("Review family is already Created.") + return + super(CreateReview, self).process() From 6bdace7a21540096d0439fb287c8c732c1f97215 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 3 Nov 2020 18:56:43 +0100 Subject: [PATCH 003/147] Initial beauty creator --- pype/plugins/tvpaint/create/create_beauty.py | 60 ++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 pype/plugins/tvpaint/create/create_beauty.py diff --git a/pype/plugins/tvpaint/create/create_beauty.py b/pype/plugins/tvpaint/create/create_beauty.py new file mode 100644 index 0000000000..08e1262943 --- /dev/null +++ b/pype/plugins/tvpaint/create/create_beauty.py @@ -0,0 +1,60 @@ +import avalon.io +from avalon.tvpaint import pipeline, CommunicationWrapper + + +class CreateBeauty(pipeline.TVPaintCreator): + """Arnold Archive""" + + name = "beauty" + label = "Beauty" + family = "renderLayer" + icon = "cube" + defaults = ["Main"] + + subset_template = "{family}{task}_{name}_{pass}" + + def process(self): + instances = pipeline.list_instances() + layers_data = CommunicationWrapper.layers_data() + group_ids = set() + for layer in layers_data: + if layer["selected"]: + group_ids.add(layer["group_id"]) + + if not group_ids: + raise AssertionError("Nothing is selected.") + + if len(group_ids) > 1: + raise AssertionError("More than one group is in selection.") + + group_id = tuple(group_ids)[0] + + existing_instance = None + existing_instance_idx = None + for idx, instance in enumerate(instances): + if ( + instance["family"] == self.family + and instance["group_id"] == group_id + ): + existing_instance = instance + existing_instance_idx = idx + break + + fill_data = { + "family": self.family, + "task": avalon.io.Session["AVALON_TASK"], + "name": "name", + "pass": "beauty" + } + subset_name = self.subset_template.format(**fill_data) + self.data["subset"] = subset_name + self.data["group_id"] = group_id + if existing_instance is not None: + if existing_instance == self.data: + self.log.info("Instance to create is same. Did nothing.") + return + instances[existing_instance_idx] = self.data + else: + instances.append(self.data) + + self.write_instances(instances) From 17044990936ac61744c494ea7275c262ff110ceb Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 3 Nov 2020 19:33:21 +0100 Subject: [PATCH 004/147] implemented basic collector --- .../tvpaint/publish/collect_instances.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 pype/plugins/tvpaint/publish/collect_instances.py diff --git a/pype/plugins/tvpaint/publish/collect_instances.py b/pype/plugins/tvpaint/publish/collect_instances.py new file mode 100644 index 0000000000..931d5b490b --- /dev/null +++ b/pype/plugins/tvpaint/publish/collect_instances.py @@ -0,0 +1,38 @@ +import json + +import pyblish.api +import avalon.io +from avalon.tvpaint import pipeline + + +class CollectInstances(pyblish.api.ContextPlugin): + label = "Collect Instances" + order = pyblish.api.CollectorOrder + hosts = ["tvpaint"] + + def process(self, context): + self.log.info("Collecting instance data from workfile") + instances_data = pipeline.list_instances() + self.log.debug("Collected ({}) instances: {}".format( + len(instances_data), instances_data + )) + + # TODO add validations of existing instances + # - layer id exists + for instance_data in instances_data: + asset_name = instance_data["asset"] + subset_name = instance_data["subset"] + family = instance_data["family"] + name = instance_data.get("name", subset_name) + active = instance_data.get("active", True) + + instance = context.create_instance( + name=name, + family=family, + families=[family], + subset=subset_name, + asset=asset_name, + active=active, + publish=active, + ) + self.log.debug(instance) From ceac453ed4e6223a27af1b3aff3a9dcae17ef100 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 3 Nov 2020 19:33:39 +0100 Subject: [PATCH 005/147] create beauty has more data --- pype/plugins/tvpaint/create/create_beauty.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pype/plugins/tvpaint/create/create_beauty.py b/pype/plugins/tvpaint/create/create_beauty.py index 08e1262943..c8303c8b11 100644 --- a/pype/plugins/tvpaint/create/create_beauty.py +++ b/pype/plugins/tvpaint/create/create_beauty.py @@ -40,15 +40,19 @@ class CreateBeauty(pipeline.TVPaintCreator): existing_instance_idx = idx break - fill_data = { + self.data["group_id"] = group_id + + name = self.data["subset"] + self.data["name"] = name + + subset_name = self.subset_template.format(**{ "family": self.family, "task": avalon.io.Session["AVALON_TASK"], - "name": "name", + "name": name, "pass": "beauty" - } - subset_name = self.subset_template.format(**fill_data) + }) self.data["subset"] = subset_name - self.data["group_id"] = group_id + if existing_instance is not None: if existing_instance == self.data: self.log.info("Instance to create is same. Did nothing.") From 3277c8c2b943c402dd51d579c1cda26ac0776766 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 4 Nov 2020 10:39:21 +0100 Subject: [PATCH 006/147] using tvpaint's lib instead of CommunicationWrapper --- pype/plugins/tvpaint/create/create_beauty.py | 4 ++-- pype/plugins/tvpaint/create/create_review.py | 2 +- pype/plugins/tvpaint/load/load_image.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pype/plugins/tvpaint/create/create_beauty.py b/pype/plugins/tvpaint/create/create_beauty.py index c8303c8b11..dd08c1a0d2 100644 --- a/pype/plugins/tvpaint/create/create_beauty.py +++ b/pype/plugins/tvpaint/create/create_beauty.py @@ -1,5 +1,5 @@ import avalon.io -from avalon.tvpaint import pipeline, CommunicationWrapper +from avalon.tvpaint import pipeline, lib class CreateBeauty(pipeline.TVPaintCreator): @@ -15,7 +15,7 @@ class CreateBeauty(pipeline.TVPaintCreator): def process(self): instances = pipeline.list_instances() - layers_data = CommunicationWrapper.layers_data() + layers_data = lib.layers_data() group_ids = set() for layer in layers_data: if layer["selected"]: diff --git a/pype/plugins/tvpaint/create/create_review.py b/pype/plugins/tvpaint/create/create_review.py index 34c7e0c6ed..567ab10f42 100644 --- a/pype/plugins/tvpaint/create/create_review.py +++ b/pype/plugins/tvpaint/create/create_review.py @@ -1,4 +1,4 @@ -from avalon.tvpaint import pipeline, CommunicationWrapper +from avalon.tvpaint import pipeline class CreateReview(pipeline.TVPaintCreator): diff --git a/pype/plugins/tvpaint/load/load_image.py b/pype/plugins/tvpaint/load/load_image.py index ca4b7a4c86..72bd23a720 100644 --- a/pype/plugins/tvpaint/load/load_image.py +++ b/pype/plugins/tvpaint/load/load_image.py @@ -1,5 +1,5 @@ from avalon import api -from avalon.tvpaint import CommunicationWrapper +from avalon.tvpaint import lib class ImportImage(api.Loader): @@ -35,4 +35,4 @@ class ImportImage(api.Loader): self.fname.replace("\\", "/"), layer_name ) - return CommunicationWrapper.execute_george_through_file(george_script) + return lib.execute_george_through_file(george_script) From 452c912a6265d043e4931c87e2abb4dbc7b5d28a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Wed, 4 Nov 2020 17:08:41 +0100 Subject: [PATCH 007/147] initial work on abstract collector --- pype/lib/abstract_collect_render.py | 242 ++++++++++++++++++++++++++++ pype/lib/abstract_expected_files.py | 53 ++++++ 2 files changed, 295 insertions(+) create mode 100644 pype/lib/abstract_collect_render.py create mode 100644 pype/lib/abstract_expected_files.py diff --git a/pype/lib/abstract_collect_render.py b/pype/lib/abstract_collect_render.py new file mode 100644 index 0000000000..38cdc1a7ce --- /dev/null +++ b/pype/lib/abstract_collect_render.py @@ -0,0 +1,242 @@ +# -*- coding: utf-8 -*- +"""Collect render template. + +TODO: use @dataclass when times come. + +""" +from abc import ABCMeta, abstractmethod + +import six +import attr + +from avalon import api +import pyblish.api + +from expected_files import ExpectedFiles + + +@attr.s +class RenderInstance(object): + """Data collected by collectors. + + This data class later on passed to collected instances. + Those attributes are required later on. + + """ + + # metadata + version = attr.ib() + time = attr.ib() + source = attr.ib() + label = attr.ib() + subset = attr.ib() + asset = attr.ib(init=False) + attachTo = attr.ib(init=False) + setMembers = attr.ib() + publish = attr.ib() + review = attr.ib(default=False) + renderer = attr.ib() + priority = attr.ib(default=50) + name = attr.ib() + + family = attr.ib(default="renderlayer") + families = attr.ib(default=["renderlayer"]) + + # format settings + resolutionWidth = attr.ib() + resolutionHeight = attr.ib() + pixelAspect = attr.ib() + multipartExr = attr.ib(default=False) + tileRendering = attr.ib() + tilesX = attr.ib() + tilesY = attr.ib() + convertToScanline = attr.ib(default=False) + + # time settings + frameStart = attr.ib() + frameEnd = attr.ib() + frameStep = attr.ib() + + @frameStart.validator + def check_frame_start(self, attribute, value): + """Validate if frame start is not larger then end.""" + if value >= self.frameEnd: + raise ValueError("frameStart must be smaller " + "or equal then frameEnd") + + @frameEnd.validator + def check_frame_end(self, attribute, value): + """Validate if frame end is not less then start.""" + if value <= self.frameStart: + raise ValueError("frameEnd must be smaller " + "or equal then frameStart") + + @tilesX.validator + def check_tiles_x(self, attribute, value): + """Validate if tile x isn't less then 1.""" + if not self.tileRendering: + return + if value < 1: + raise ValueError("tile X size cannot be less then 1") + + if value == 1 and self.tilesY == 1: + raise ValueError("both tiles X a Y sizes are set to 1") + + @tilesY.validator + def check_tiles_y(self, attribute, value): + """Validate if tile y isn't less then 1.""" + if not self.tileRendering: + return + if value < 1: + raise ValueError("tile Y size cannot be less then 1") + + if value == 1 and self.tilesX == 1: + raise ValueError("both tiles X a Y sizes are set to 1") + + +@six.add_metaclass(ABCMeta) +class CollectRender(pyblish.api.ContextPlugin): + """Gather all publishable render layers from renderSetup.""" + + order = pyblish.api.CollectorOrder + 0.01 + label = "Collect Render" + sync_workfile_version = False + + def process(self, context): + """Entry point to collector.""" + rendering_instance = None + for instance in context: + if "rendering" in instance.data["families"]: + rendering_instance = instance + rendering_instance.data["remove"] = True + + # make sure workfile instance publishing is enabled + if "workfile" in instance.data["families"]: + instance.data["publish"] = True + + if not rendering_instance: + self.log.info( + "No rendering instance found, skipping render " + "layer collection." + ) + return + + self._filepath = context.data["currentFile"].replace("\\", "/") + self._asset = api.Session["AVALON_ASSET"] + + render_instances = self.get_instances() + for render_instance in render_instances: + exp_files = self._get_expected_files(render_instance) + + frame_start_render = int(render_instance.frameStart) + frame_end_render = int(render_instance.frameEnd) + + if (int(context.data['frameStartHandle']) == frame_start_render + and int(context.data['frameEndHandle']) == frame_end_render): # noqa: W503, E501 + + handle_start = context.data['handleStart'] + handle_end = context.data['handleEnd'] + frame_start = context.data['frameStart'] + frame_end = context.data['frameEnd'] + frame_start_handle = context.data['frameStartHandle'] + frame_end_handle = context.data['frameEndHandle'] + else: + handle_start = 0 + handle_end = 0 + frame_start = frame_start_render + frame_end = frame_end_render + frame_start_handle = frame_start_render + frame_end_handle = frame_end_render + + data = { + "subset": render_instance.subset, + "attachTo": render_instance.attachTo, + "setMembers": render_instance.setMembers, + "multipartExr": exp_files.multipart, + "review": render_instance.review or False, + "publish": True, + + "handleStart": handle_start, + "handleEnd": handle_end, + "frameStart": frame_start, + "frameEnd": frame_end, + "frameStartHandle": frame_start_handle, + "frameEndHandle": frame_end_handle, + "byFrameStep": int(render_instance.frameStep), + "renderer": render_instance.renderer, + # instance subset + "family": render_instance.family, + "families": render_instance.families, + "asset": render_instance.asset, + "time": render_instance.time, + "author": context.data["user"], + # Add source to allow tracing back to the scene from + # which was submitted originally + "source": render_instance.source, + "expectedFiles": exp_files, + "resolutionWidth": render_instance.resolutionWidth, + "resolutionHeight": render_instance.resolutionHeight, + "pixelAspect": render_instance.pixelAspect, + "tileRendering": render_instance.tileRendering or False, + "tilesX": render_instance.tilesX or 2, + "tilesY": render_instance.tilesY or 2, + "priority": render_instance.priority, + "convertToScanline": render_instance.convertToScanline or False + } + if self.sync_workfile_version: + data["version"] = context.data["version"] + + # add additional data + data = self.add_additional_data(data) + + instance = context.create_instance(render_instance.name) + instance.data["label"] = render_instance.label + instance.data.update(data) + + self.post_collecting_action() + + @abstractmethod + def get_instances(self): + """Get all renderable instances and their data. + + Returns: + list of :class:`RenderInstance`: All collected renderable instances + (like render layers, write nodes, etc.) + + """ + pass + + def _get_expected_files(self, render_instance): + """Get list of expected files.""" + # return all expected files for all cameras and aovs in given + # frame range + ef = ExpectedFiles() + exp_files = ef.get(render_instance) + self.log.info("multipart: {}".format(ef.multipart)) + assert exp_files, "no file names were generated, this is bug" + + # if we want to attach render to subset, check if we have AOV's + # in expectedFiles. If so, raise error as we cannot attach AOV + # (considered to be subset on its own) to another subset + if render_instance.attachTo: + assert isinstance(exp_files, list), ( + "attaching multiple AOVs or renderable cameras to " + "subset is not supported" + ) + + def add_additional_data(self, data): + """Add additional data to collected instance. + + This can be overridden by host implementation to add custom + additional data. + + """ + return data + + def post_collecting_action(self): + """Execute some code after collection is done. + + This is useful for example for restoring current render layer. + + """ + pass diff --git a/pype/lib/abstract_expected_files.py b/pype/lib/abstract_expected_files.py new file mode 100644 index 0000000000..f493cdb751 --- /dev/null +++ b/pype/lib/abstract_expected_files.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +"""Abstract ExpectedFile class definition.""" +from abc import ABCMeta, abstractmethod +import six + + +@six.add_metaclass(ABCMeta) +class ExpectedFiles: + """Class grouping functionality for all supported renderers. + + Attributes: + multipart (bool): Flag if multipart exrs are used. + + """ + + multipart = False + + @abstractmethod + def get(self, render_instance): + """Get expected files for given renderer and render layer. + + This method should return dictionary of all files we are expecting + to be rendered from the host. Usually `render_instance` corresponds + to *render layer*. Result can be either flat list with the file + paths or it can be list of dictionaries. Each key corresponds to + for example AOV name or channel, etc. + + Example:: + + ['/path/to/file.001.exr', '/path/to/file.002.exr'] + + or as dictionary: + + [ + { + "beauty": ['/path/to/beauty.001.exr', ...], + "mask": ['/path/to/mask.001.exr'] + } + ] + + + Args: + renderer_instance (:class:`RenderInstance`): Data passed from + collector to determine files. This should be instance of + :class:`abstract_collect_render.RenderInstance` + + Returns: + list: Full paths to expected rendered files. + list of dict: Path to expected rendered files categorized by + AOVs, etc. + + """ + raise NotImplementedError() From 093fa1825c6d4a162e77fd3b6880c8cf8fe84b3b Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 5 Nov 2020 10:35:15 +0100 Subject: [PATCH 008/147] remove invalid docstring --- pype/plugins/tvpaint/create/create_beauty.py | 2 -- pype/plugins/tvpaint/create/create_review.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/pype/plugins/tvpaint/create/create_beauty.py b/pype/plugins/tvpaint/create/create_beauty.py index dd08c1a0d2..a838c2a5c9 100644 --- a/pype/plugins/tvpaint/create/create_beauty.py +++ b/pype/plugins/tvpaint/create/create_beauty.py @@ -3,8 +3,6 @@ from avalon.tvpaint import pipeline, lib class CreateBeauty(pipeline.TVPaintCreator): - """Arnold Archive""" - name = "beauty" label = "Beauty" family = "renderLayer" diff --git a/pype/plugins/tvpaint/create/create_review.py b/pype/plugins/tvpaint/create/create_review.py index 567ab10f42..279e03caea 100644 --- a/pype/plugins/tvpaint/create/create_review.py +++ b/pype/plugins/tvpaint/create/create_review.py @@ -2,8 +2,6 @@ from avalon.tvpaint import pipeline class CreateReview(pipeline.TVPaintCreator): - """Arnold Archive""" - name = "review" label = "Review" family = "review" From 8170ff5559e2a940ecd99da5d018e3f776593e2d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 5 Nov 2020 11:03:30 +0100 Subject: [PATCH 009/147] create beauty also renames group --- pype/plugins/tvpaint/create/create_beauty.py | 30 +++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/pype/plugins/tvpaint/create/create_beauty.py b/pype/plugins/tvpaint/create/create_beauty.py index a838c2a5c9..49ef2164d1 100644 --- a/pype/plugins/tvpaint/create/create_beauty.py +++ b/pype/plugins/tvpaint/create/create_beauty.py @@ -10,9 +10,14 @@ class CreateBeauty(pipeline.TVPaintCreator): defaults = ["Main"] subset_template = "{family}{task}_{name}_{pass}" + rename_script_template = ( + "tv_layercolor \"setcolor\"" + " {clip_id} {group_id} {r} {g} {b} \"{name}\"" + ) def process(self): instances = pipeline.list_instances() + groups_data = lib.groups_data() layers_data = lib.layers_data() group_ids = set() for layer in layers_data: @@ -26,6 +31,10 @@ class CreateBeauty(pipeline.TVPaintCreator): raise AssertionError("More than one group is in selection.") group_id = tuple(group_ids)[0] + if group_id == 0: + raise AssertionError( + "Selection is not in group. Can't mark selection as Beauty." + ) existing_instance = None existing_instance_idx = None @@ -39,7 +48,6 @@ class CreateBeauty(pipeline.TVPaintCreator): break self.data["group_id"] = group_id - name = self.data["subset"] self.data["name"] = name @@ -60,3 +68,23 @@ class CreateBeauty(pipeline.TVPaintCreator): instances.append(self.data) self.write_instances(instances) + + group = None + for group_data in groups_data: + if group_data["id"] == group_id: + group = group_data + + if not group: + return + + new_group_name = name.replace(" ", "_") + + rename_script = self.rename_script_template.format( + clip_id=group["clip_id"], + group_id=group["id"], + r=group["red"], + g=group["green"], + b=group["blue"], + name=new_group_name + ) + lib.execute_george_through_file(rename_script) From a2089fb2d283916dbce7eb5e3116ca0bfae3d568 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 5 Nov 2020 11:43:14 +0100 Subject: [PATCH 010/147] extract name from subset name --- pype/plugins/tvpaint/create/create_beauty.py | 27 +++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/pype/plugins/tvpaint/create/create_beauty.py b/pype/plugins/tvpaint/create/create_beauty.py index 49ef2164d1..07c66f8d02 100644 --- a/pype/plugins/tvpaint/create/create_beauty.py +++ b/pype/plugins/tvpaint/create/create_beauty.py @@ -36,19 +36,11 @@ class CreateBeauty(pipeline.TVPaintCreator): "Selection is not in group. Can't mark selection as Beauty." ) - existing_instance = None - existing_instance_idx = None - for idx, instance in enumerate(instances): - if ( - instance["family"] == self.family - and instance["group_id"] == group_id - ): - existing_instance = instance - existing_instance_idx = idx - break - - self.data["group_id"] = group_id + family = self.data["family"] name = self.data["subset"] + # Is this right way how to get name? + name = name[len(family):] + self.data["group_id"] = group_id self.data["name"] = name subset_name = self.subset_template.format(**{ @@ -59,6 +51,17 @@ class CreateBeauty(pipeline.TVPaintCreator): }) self.data["subset"] = subset_name + existing_instance = None + existing_instance_idx = None + for idx, instance in enumerate(instances): + if ( + instance["family"] == family + and instance["group_id"] == group_id + ): + existing_instance = instance + existing_instance_idx = idx + break + if existing_instance is not None: if existing_instance == self.data: self.log.info("Instance to create is same. Did nothing.") From c93a34dd14750b54bc35156c6add6da0b664b528 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 5 Nov 2020 12:03:41 +0100 Subject: [PATCH 011/147] added few logs --- pype/plugins/tvpaint/create/create_beauty.py | 45 ++++++++++++++------ 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/pype/plugins/tvpaint/create/create_beauty.py b/pype/plugins/tvpaint/create/create_beauty.py index 07c66f8d02..918c18070a 100644 --- a/pype/plugins/tvpaint/create/create_beauty.py +++ b/pype/plugins/tvpaint/create/create_beauty.py @@ -9,6 +9,8 @@ class CreateBeauty(pipeline.TVPaintCreator): icon = "cube" defaults = ["Main"] + rename_group = True + subset_template = "{family}{task}_{name}_{pass}" rename_script_template = ( "tv_layercolor \"setcolor\"" @@ -16,9 +18,12 @@ class CreateBeauty(pipeline.TVPaintCreator): ) def process(self): + self.log.debug("Query data from workfile.") instances = pipeline.list_instances() groups_data = lib.groups_data() layers_data = lib.layers_data() + + self.log.debug("Checking for selection groups.") group_ids = set() for layer in layers_data: if layer["selected"]: @@ -36,11 +41,14 @@ class CreateBeauty(pipeline.TVPaintCreator): "Selection is not in group. Can't mark selection as Beauty." ) + self.log.debug(f"Selected group id is \"{group_id}\".") + self.data["group_id"] = group_id + family = self.data["family"] name = self.data["subset"] # Is this right way how to get name? name = name[len(family):] - self.data["group_id"] = group_id + self.log.info(f"Extracted name form subset name \"{name}\".") self.data["name"] = name subset_name = self.subset_template.format(**{ @@ -49,6 +57,7 @@ class CreateBeauty(pipeline.TVPaintCreator): "name": name, "pass": "beauty" }) + self.log.info(f"New subset name \"{subset_name}\".") self.data["subset"] = subset_name existing_instance = None @@ -63,31 +72,41 @@ class CreateBeauty(pipeline.TVPaintCreator): break if existing_instance is not None: + self.log.info( + f"Beauty instance for group id {group_id} already exists." + ) if existing_instance == self.data: self.log.info("Instance to create is same. Did nothing.") return + self.log.debug("Overriding beauty instance with new data.") instances[existing_instance_idx] = self.data else: instances.append(self.data) self.write_instances(instances) - group = None - for group_data in groups_data: - if group_data["id"] == group_id: - group = group_data - - if not group: + if not self.rename_group: + self.log.info("Group rename function is turned off. Skipping") return - new_group_name = name.replace(" ", "_") + self.log.debug("Changing name of the group.") + selected_group = None + for group_data in groups_data: + if group_data["id"] == group_id: + selected_group = group_data + new_group_name = name.replace(" ", "_") rename_script = self.rename_script_template.format( - clip_id=group["clip_id"], - group_id=group["id"], - r=group["red"], - g=group["green"], - b=group["blue"], + clip_id=selected_group["clip_id"], + group_id=selected_group["id"], + r=selected_group["red"], + g=selected_group["green"], + b=selected_group["blue"], name=new_group_name ) lib.execute_george_through_file(rename_script) + + self.log.info( + f"Name of group with index {group_id}" + f" was changed to {new_group_name}." + ) From 917ca76bd36cdc79ac8e9583eca16b46b83d8a7a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 5 Nov 2020 12:07:45 +0100 Subject: [PATCH 012/147] typo fix --- pype/plugins/tvpaint/create/create_beauty.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/plugins/tvpaint/create/create_beauty.py b/pype/plugins/tvpaint/create/create_beauty.py index 918c18070a..43ae4232aa 100644 --- a/pype/plugins/tvpaint/create/create_beauty.py +++ b/pype/plugins/tvpaint/create/create_beauty.py @@ -48,7 +48,7 @@ class CreateBeauty(pipeline.TVPaintCreator): name = self.data["subset"] # Is this right way how to get name? name = name[len(family):] - self.log.info(f"Extracted name form subset name \"{name}\".") + self.log.info(f"Extracted name from subset name \"{name}\".") self.data["name"] = name subset_name = self.subset_template.format(**{ @@ -108,5 +108,5 @@ class CreateBeauty(pipeline.TVPaintCreator): self.log.info( f"Name of group with index {group_id}" - f" was changed to {new_group_name}." + f" was changed to \"{new_group_name}\"." ) From 9f704a5b276748073c366085c4919849fcdcb016 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Thu, 5 Nov 2020 12:20:53 +0100 Subject: [PATCH 013/147] wip on submit plugin --- pype/lib/abstract_collect_render.py | 4 +- pype/lib/abstract_submit_deadline.py | 80 ++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 pype/lib/abstract_submit_deadline.py diff --git a/pype/lib/abstract_collect_render.py b/pype/lib/abstract_collect_render.py index 38cdc1a7ce..bf14f6f850 100644 --- a/pype/lib/abstract_collect_render.py +++ b/pype/lib/abstract_collect_render.py @@ -12,7 +12,7 @@ import attr from avalon import api import pyblish.api -from expected_files import ExpectedFiles +from .abstract_expected_files import ExpectedFiles @attr.s @@ -95,7 +95,7 @@ class RenderInstance(object): @six.add_metaclass(ABCMeta) -class CollectRender(pyblish.api.ContextPlugin): +class AbstractCollectRender(pyblish.api.ContextPlugin): """Gather all publishable render layers from renderSetup.""" order = pyblish.api.CollectorOrder + 0.01 diff --git a/pype/lib/abstract_submit_deadline.py b/pype/lib/abstract_submit_deadline.py new file mode 100644 index 0000000000..8b8fa7bc1f --- /dev/null +++ b/pype/lib/abstract_submit_deadline.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +"""Abstract class for submitting jobs to Deadline.""" +import os +from abc import ABCMeta, abstractmethod + +import six +import attr +import requests + +import pyblish.api + + +@attr.s +class DeadlineJobInfo: + BatchName = attr.ib() + + +@attr.s +class DeadlinePluginInfo: + SceneFile = attr.ib() + + +@six.add_metaclass(ABCMeta) +class AbstractSubmitDeadline(pyblish.api.InstancePlugin): + + label = "Submit to Deadline" + order = pyblish.api.IntegratorOrder + 0.1 + use_published = True + asset_dependencies = False + + def submit(self, payload): + url = "{}/api/jobs".format(self._deadline_url) + response = self._requests_post(url, json=payload) + if not response.ok: + self.log.error("Submition failed!") + self.log.error(response.status_code) + self.log.error(response.content) + self.log.debug(payload) + raise RuntimeError(response.text) + + dependency = response.json() + return dependency["_id"] + + def _requests_post(self, *args, **kwargs): + """Wrap request post method. + + Disabling SSL certificate validation if ``DONT_VERIFY_SSL`` environment + variable is found. This is useful when Deadline or Muster server are + running with self-signed certificates and their certificate is not + added to trusted certificates on client machines. + + Warning: + Disabling SSL certificate validation is defeating one line + of defense SSL is providing and it is not recommended. + + """ + if 'verify' not in kwargs: + kwargs['verify'] = False if os.getenv("PYPE_DONT_VERIFY_SSL", True) else True # noqa + # add 10sec timeout before bailing out + kwargs['timeout'] = 10 + return requests.post(*args, **kwargs) + + def _requests_get(self, *args, **kwargs): + """Wrap request get method. + + Disabling SSL certificate validation if ``DONT_VERIFY_SSL`` environment + variable is found. This is useful when Deadline or Muster server are + running with self-signed certificates and their certificate is not + added to trusted certificates on client machines. + + Warning: + Disabling SSL certificate validation is defeating one line + of defense SSL is providing and it is not recommended. + + """ + if 'verify' not in kwargs: + kwargs['verify'] = False if os.getenv("PYPE_DONT_VERIFY_SSL", True) else True # noqa + # add 10sec timeout before bailing out + kwargs['timeout'] = 10 + return requests.get(*args, **kwargs) From 353c962617c47cefe63bf6c6e02aecb64792e1f5 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 5 Nov 2020 14:01:35 +0100 Subject: [PATCH 014/147] implemented render pass creator --- .../tvpaint/create/create_render_pass.py | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 pype/plugins/tvpaint/create/create_render_pass.py diff --git a/pype/plugins/tvpaint/create/create_render_pass.py b/pype/plugins/tvpaint/create/create_render_pass.py new file mode 100644 index 0000000000..fa51dcad26 --- /dev/null +++ b/pype/plugins/tvpaint/create/create_render_pass.py @@ -0,0 +1,85 @@ +import tempfile +import avalon.io +from avalon.tvpaint import pipeline, lib + + +class CreateRenderPass(pipeline.TVPaintCreator): + name = "render_pass" + label = "Render Pass" + family = "renderPass" + icon = "cube" + defaults = ["Main"] + + def process(self): + instances = pipeline.list_instances() + layers_data = lib.layers_data() + group_ids = set() + selected_layers = [] + for layer in layers_data: + if layer["selected"]: + selected_layers.append(layer) + group_ids.add(layer["group_id"]) + + if not selected_layers: + raise AssertionError("Nothing is selected.") + + if len(group_ids) != 1: + raise AssertionError("More than one group is in selection.") + + group_id = tuple(group_ids)[0] + + beauty_instance = None + for instance in instances: + if ( + instance["family"] == "renderLayer" + and instance["group_id"] == group_id + ): + beauty_instance = instance + break + + if beauty_instance is None: + raise AssertionError("Beauty pass does not exist yet.") + + family = self.data["family"] + name = self.data["subset"] + # Is this right way how to get name? + name = name[len(family):] + + self.data["group_id"] = group_id + self.data["name"] = name + + layer_ids = [layer["id"] for layer in selected_layers] + self.data["layer_ids"] = layer_ids + + beauty_subset_name = beauty_instance["subset"] + beauty_pass_name = "beauty" + if not beauty_subset_name.endswith(beauty_pass_name): + raise AssertionError( + "BUG: Beauty subset name does not end with \"{}\"".format( + beauty_pass_name + ) + ) + subset_name = beauty_subset_name[:-len(beauty_pass_name)] + name + self.data["subset"] = subset_name + + existing_instance = None + existing_instance_idx = None + for idx, instance in enumerate(instances): + if ( + instance["family"] == family + and instance["group_id"] == group_id + and instance["name"] == name + ): + existing_instance = instance + existing_instance_idx = idx + break + + if existing_instance is not None: + if existing_instance == self.data: + self.log.info("Instance to create is same. Did nothing.") + return + instances[existing_instance_idx] = self.data + else: + instances.append(self.data) + + self.write_instances(instances) From fe3310a2226959f6892d7a83239274b2ac81b967 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 5 Nov 2020 14:11:29 +0100 Subject: [PATCH 015/147] added few logs --- pype/plugins/tvpaint/create/create_beauty.py | 4 +-- .../tvpaint/create/create_render_pass.py | 25 ++++++++++++++++--- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/pype/plugins/tvpaint/create/create_beauty.py b/pype/plugins/tvpaint/create/create_beauty.py index 43ae4232aa..5701845288 100644 --- a/pype/plugins/tvpaint/create/create_beauty.py +++ b/pype/plugins/tvpaint/create/create_beauty.py @@ -45,14 +45,14 @@ class CreateBeauty(pipeline.TVPaintCreator): self.data["group_id"] = group_id family = self.data["family"] - name = self.data["subset"] # Is this right way how to get name? - name = name[len(family):] + name = self.data["subset"][len(family):] self.log.info(f"Extracted name from subset name \"{name}\".") self.data["name"] = name subset_name = self.subset_template.format(**{ "family": self.family, + # Should be task name capitalized? "task": avalon.io.Session["AVALON_TASK"], "name": name, "pass": "beauty" diff --git a/pype/plugins/tvpaint/create/create_render_pass.py b/pype/plugins/tvpaint/create/create_render_pass.py index fa51dcad26..ff60f299a3 100644 --- a/pype/plugins/tvpaint/create/create_render_pass.py +++ b/pype/plugins/tvpaint/create/create_render_pass.py @@ -10,9 +10,14 @@ class CreateRenderPass(pipeline.TVPaintCreator): icon = "cube" defaults = ["Main"] + beauty_pass_name = "beauty" + def process(self): + self.log.debug("Query data from workfile.") instances = pipeline.list_instances() layers_data = lib.layers_data() + + self.log.debug("Checking selection.") group_ids = set() selected_layers = [] for layer in layers_data: @@ -27,6 +32,7 @@ class CreateRenderPass(pipeline.TVPaintCreator): raise AssertionError("More than one group is in selection.") group_id = tuple(group_ids)[0] + self.log.debug(f"Selected group id is \"{group_id}\".") beauty_instance = None for instance in instances: @@ -44,6 +50,7 @@ class CreateRenderPass(pipeline.TVPaintCreator): name = self.data["subset"] # Is this right way how to get name? name = name[len(family):] + self.log.info(f"Extracted name from subset name \"{name}\".") self.data["group_id"] = group_id self.data["name"] = name @@ -52,15 +59,20 @@ class CreateRenderPass(pipeline.TVPaintCreator): self.data["layer_ids"] = layer_ids beauty_subset_name = beauty_instance["subset"] - beauty_pass_name = "beauty" - if not beauty_subset_name.endswith(beauty_pass_name): + self.log.info( + "New subset name will be created from " + f"beauty instance \"{beauty_subset_name}\"." + ) + + if not beauty_subset_name.endswith(self.beauty_pass_name): raise AssertionError( "BUG: Beauty subset name does not end with \"{}\"".format( - beauty_pass_name + self.beauty_pass_name ) ) - subset_name = beauty_subset_name[:-len(beauty_pass_name)] + name + subset_name = beauty_subset_name[:-len(self.beauty_pass_name)] + name self.data["subset"] = subset_name + self.log.info(f"New subset name is \"{subset_name}\".") existing_instance = None existing_instance_idx = None @@ -75,9 +87,14 @@ class CreateRenderPass(pipeline.TVPaintCreator): break if existing_instance is not None: + self.log.info( + f"Render pass instance for group id {group_id}" + f" and name \"{name}\" already exists." + ) if existing_instance == self.data: self.log.info("Instance to create is same. Did nothing.") return + self.log.debug("Overriding beauty instance with new data.") instances[existing_instance_idx] = self.data else: instances.append(self.data) From 89ccf0724b60d8a84933c2fc24ed89647dce838f Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 5 Nov 2020 14:34:45 +0100 Subject: [PATCH 016/147] added poor docstrings --- pype/plugins/tvpaint/create/create_beauty.py | 1 + pype/plugins/tvpaint/create/create_render_pass.py | 5 +++++ pype/plugins/tvpaint/create/create_review.py | 1 + 3 files changed, 7 insertions(+) diff --git a/pype/plugins/tvpaint/create/create_beauty.py b/pype/plugins/tvpaint/create/create_beauty.py index 5701845288..aaa51f156e 100644 --- a/pype/plugins/tvpaint/create/create_beauty.py +++ b/pype/plugins/tvpaint/create/create_beauty.py @@ -3,6 +3,7 @@ from avalon.tvpaint import pipeline, lib class CreateBeauty(pipeline.TVPaintCreator): + """Mark layer group as one instance.""" name = "beauty" label = "Beauty" family = "renderLayer" diff --git a/pype/plugins/tvpaint/create/create_render_pass.py b/pype/plugins/tvpaint/create/create_render_pass.py index ff60f299a3..37d7c501f9 100644 --- a/pype/plugins/tvpaint/create/create_render_pass.py +++ b/pype/plugins/tvpaint/create/create_render_pass.py @@ -4,6 +4,11 @@ from avalon.tvpaint import pipeline, lib class CreateRenderPass(pipeline.TVPaintCreator): + """Render pass is combination of one or more layers from same group. + + Requirement to create Render Pass is to have already created beauty + instance. Beauty instance is used as base for subset name. + """ name = "render_pass" label = "Render Pass" family = "renderPass" diff --git a/pype/plugins/tvpaint/create/create_review.py b/pype/plugins/tvpaint/create/create_review.py index 279e03caea..0163d26160 100644 --- a/pype/plugins/tvpaint/create/create_review.py +++ b/pype/plugins/tvpaint/create/create_review.py @@ -2,6 +2,7 @@ from avalon.tvpaint import pipeline class CreateReview(pipeline.TVPaintCreator): + """Review for global review of all layers.""" name = "review" label = "Review" family = "review" From 0b38b162259896dc33a550d97206783a5da6544c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 5 Nov 2020 14:51:32 +0100 Subject: [PATCH 017/147] few more comments --- pype/plugins/tvpaint/create/create_beauty.py | 14 ++++++++++++-- pype/plugins/tvpaint/create/create_render_pass.py | 10 ++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/pype/plugins/tvpaint/create/create_beauty.py b/pype/plugins/tvpaint/create/create_beauty.py index aaa51f156e..f68365a2d4 100644 --- a/pype/plugins/tvpaint/create/create_beauty.py +++ b/pype/plugins/tvpaint/create/create_beauty.py @@ -21,22 +21,25 @@ class CreateBeauty(pipeline.TVPaintCreator): def process(self): self.log.debug("Query data from workfile.") instances = pipeline.list_instances() - groups_data = lib.groups_data() layers_data = lib.layers_data() self.log.debug("Checking for selection groups.") + # Collect group ids from selection group_ids = set() for layer in layers_data: if layer["selected"]: group_ids.add(layer["group_id"]) + # Raise if there is no selection if not group_ids: raise AssertionError("Nothing is selected.") + # This creator should run only on one group if len(group_ids) > 1: raise AssertionError("More than one group is in selection.") group_id = tuple(group_ids)[0] + # If group id is `0` it is `default` group which is invalid if group_id == 0: raise AssertionError( "Selection is not in group. Can't mark selection as Beauty." @@ -46,11 +49,12 @@ class CreateBeauty(pipeline.TVPaintCreator): self.data["group_id"] = group_id family = self.data["family"] - # Is this right way how to get name? + # Extract entered name name = self.data["subset"][len(family):] self.log.info(f"Extracted name from subset name \"{name}\".") self.data["name"] = name + # Change subset name by template subset_name = self.subset_template.format(**{ "family": self.family, # Should be task name capitalized? @@ -61,6 +65,7 @@ class CreateBeauty(pipeline.TVPaintCreator): self.log.info(f"New subset name \"{subset_name}\".") self.data["subset"] = subset_name + # Check for instances for same group existing_instance = None existing_instance_idx = None for idx, instance in enumerate(instances): @@ -90,12 +95,17 @@ class CreateBeauty(pipeline.TVPaintCreator): self.log.info("Group rename function is turned off. Skipping") return + self.log.debug("Querying groups data from workfile.") + groups_data = lib.groups_data() + self.log.debug("Changing name of the group.") selected_group = None for group_data in groups_data: if group_data["id"] == group_id: selected_group = group_data + # Rename TVPaint group (keep color same) + # - groups can't contain spaces new_group_name = name.replace(" ", "_") rename_script = self.rename_script_template.format( clip_id=selected_group["clip_id"], diff --git a/pype/plugins/tvpaint/create/create_render_pass.py b/pype/plugins/tvpaint/create/create_render_pass.py index 37d7c501f9..a28348f46a 100644 --- a/pype/plugins/tvpaint/create/create_render_pass.py +++ b/pype/plugins/tvpaint/create/create_render_pass.py @@ -23,6 +23,7 @@ class CreateRenderPass(pipeline.TVPaintCreator): layers_data = lib.layers_data() self.log.debug("Checking selection.") + # Get all selected layers and their group ids group_ids = set() selected_layers = [] for layer in layers_data: @@ -30,15 +31,18 @@ class CreateRenderPass(pipeline.TVPaintCreator): selected_layers.append(layer) group_ids.add(layer["group_id"]) + # Raise if nothing is selected if not selected_layers: raise AssertionError("Nothing is selected.") + # Raise if layers from multiple groups are selected if len(group_ids) != 1: raise AssertionError("More than one group is in selection.") group_id = tuple(group_ids)[0] self.log.debug(f"Selected group id is \"{group_id}\".") + # Find beauty instance for selected layers beauty_instance = None for instance in instances: if ( @@ -48,9 +52,11 @@ class CreateRenderPass(pipeline.TVPaintCreator): beauty_instance = instance break + # Beauty is required for this creator so raise if was not found if beauty_instance is None: raise AssertionError("Beauty pass does not exist yet.") + # Extract entered name family = self.data["family"] name = self.data["subset"] # Is this right way how to get name? @@ -60,6 +66,7 @@ class CreateRenderPass(pipeline.TVPaintCreator): self.data["group_id"] = group_id self.data["name"] = name + # Collect selected layer ids to be stored into instance layer_ids = [layer["id"] for layer in selected_layers] self.data["layer_ids"] = layer_ids @@ -69,16 +76,19 @@ class CreateRenderPass(pipeline.TVPaintCreator): f"beauty instance \"{beauty_subset_name}\"." ) + # Beauty instance subset name should if not beauty_subset_name.endswith(self.beauty_pass_name): raise AssertionError( "BUG: Beauty subset name does not end with \"{}\"".format( self.beauty_pass_name ) ) + # Replace `beauty` in beauty's subset name with entered name subset_name = beauty_subset_name[:-len(self.beauty_pass_name)] + name self.data["subset"] = subset_name self.log.info(f"New subset name is \"{subset_name}\".") + # Check if same instance already exists existing_instance = None existing_instance_idx = None for idx, instance in enumerate(instances): From 5525113e219a2a224d9a6a293b0e6f79219a3ca5 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 5 Nov 2020 14:58:31 +0100 Subject: [PATCH 018/147] Instance collecting should work --- .../tvpaint/publish/collect_instances.py | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/pype/plugins/tvpaint/publish/collect_instances.py b/pype/plugins/tvpaint/publish/collect_instances.py index 931d5b490b..193302751d 100644 --- a/pype/plugins/tvpaint/publish/collect_instances.py +++ b/pype/plugins/tvpaint/publish/collect_instances.py @@ -1,5 +1,3 @@ -import json - import pyblish.api import avalon.io from avalon.tvpaint import pipeline @@ -20,19 +18,18 @@ class CollectInstances(pyblish.api.ContextPlugin): # TODO add validations of existing instances # - layer id exists for instance_data in instances_data: - asset_name = instance_data["asset"] - subset_name = instance_data["subset"] + # Fill families family = instance_data["family"] - name = instance_data.get("name", subset_name) - active = instance_data.get("active", True) + instance_data["families"] = [family] - instance = context.create_instance( - name=name, - family=family, - families=[family], - subset=subset_name, - asset=asset_name, - active=active, - publish=active, - ) - self.log.debug(instance) + # Instance name + subset_name = instance_data["subset"] + name = instance_data.get("name", subset_name) + instance_data["name"] = name + + active = instance_data.get("active", True) + instance_data["active"] = active + instance_data["publish"] = active + + instance = context.create_instance(**instance_data) + self.log.debug("Created instance: {}".format(instance)) From 727be359f2e0282b6f9875de133d535616e2aca3 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 5 Nov 2020 16:02:44 +0100 Subject: [PATCH 019/147] implemented collector where global data are collected --- .../tvpaint/publish/collect_workfile_data.py | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 pype/plugins/tvpaint/publish/collect_workfile_data.py diff --git a/pype/plugins/tvpaint/publish/collect_workfile_data.py b/pype/plugins/tvpaint/publish/collect_workfile_data.py new file mode 100644 index 0000000000..d81250c05b --- /dev/null +++ b/pype/plugins/tvpaint/publish/collect_workfile_data.py @@ -0,0 +1,57 @@ +import json + +import pyblish.api +from avalon.tvpaint import pipeline, lib + + +class CollectWorkfileData(pyblish.api.ContextPlugin): + label = "Collect Workfile Data" + order = pyblish.api.CollectorOrder - 0.01 + hosts = ["tvpaint"] + + def process(self, context): + self.log.info("Collecting instance data from workfile") + instance_data = pipeline.list_instances() + self.log.debug( + "Instance data:\"{}".format(json.dumps(instance_data, indent=4)) + ) + context.data["workfileInstances"] = instance_data + + self.log.info("Collecting layers data from workfile") + layers_data = lib.layers_data() + self.log.debug( + "Layers data:\"{}".format(json.dumps(layers_data, indent=4)) + ) + context.data["layersData"] = layers_data + + self.log.info("Collecting groups data from workfile") + group_data = lib.groups_data() + self.log.debug( + "Group data:\"{}".format(json.dumps(group_data, indent=4)) + ) + context.data["groupsData"] = group_data + + self.log.info("Collecting scene data from workfile") + workfile_info_parts = lib.execute_george("tv_projectinfo").split(" ") + + frame_start = int(workfile_info_parts.pop(-1)) + field_order = workfile_info_parts.pop(-1) + frame_rate = float(workfile_info_parts.pop(-1)) + pixel_apsect = float(workfile_info_parts.pop(-1)) + height = int(workfile_info_parts.pop(-1)) + width = int(workfile_info_parts.pop(-1)) + workfile_path = " ".join(workfile_info_parts).replace("\"", "") + + sceme_data = { + "currentFile": workfile_path, + "sceneWidth": width, + "sceneHeight": height, + "pixelAspect": pixel_apsect, + "frameStart": frame_start, + "frameRate": frame_rate, + "fieldOrder": field_order + } + self.log.debug( + "Scene data: {}".format(json.dumps(sceme_data,indent=4)) + ) + context.data.update(sceme_data) From 3b2a5a09e67cfb0f2c23e841a6ffa59d888ec482 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 5 Nov 2020 16:02:58 +0100 Subject: [PATCH 020/147] collect instances use globaly collected data --- .../tvpaint/publish/collect_instances.py | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/pype/plugins/tvpaint/publish/collect_instances.py b/pype/plugins/tvpaint/publish/collect_instances.py index 193302751d..caf6adee5f 100644 --- a/pype/plugins/tvpaint/publish/collect_instances.py +++ b/pype/plugins/tvpaint/publish/collect_instances.py @@ -1,5 +1,6 @@ +import json + import pyblish.api -import avalon.io from avalon.tvpaint import pipeline @@ -9,27 +10,28 @@ class CollectInstances(pyblish.api.ContextPlugin): hosts = ["tvpaint"] def process(self, context): - self.log.info("Collecting instance data from workfile") - instances_data = pipeline.list_instances() - self.log.debug("Collected ({}) instances: {}".format( - len(instances_data), instances_data + workfile_instances = context.data["workfileInstances"] + + self.log.debug("Collected ({}) instances:\n{}".format( + len(workfile_instances), + json.dumps(workfile_instances, indent=4) )) # TODO add validations of existing instances # - layer id exists - for instance_data in instances_data: + for workfile_instance in workfile_instances: # Fill families - family = instance_data["family"] - instance_data["families"] = [family] + family = workfile_instance["family"] + workfile_instance["families"] = [family] # Instance name - subset_name = instance_data["subset"] - name = instance_data.get("name", subset_name) - instance_data["name"] = name + subset_name = workfile_instance["subset"] + name = workfile_instance.get("name", subset_name) + workfile_instance["name"] = name - active = instance_data.get("active", True) - instance_data["active"] = active - instance_data["publish"] = active + active = workfile_instance.get("active", True) + workfile_instance["active"] = active + workfile_instance["publish"] = active - instance = context.create_instance(**instance_data) + instance = context.create_instance(**workfile_instance) self.log.debug("Created instance: {}".format(instance)) From 5b32de6d1043b5f5981f1e5f05ea2d2c89b22147 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 5 Nov 2020 16:05:29 +0100 Subject: [PATCH 021/147] using explicit id keys --- pype/plugins/tvpaint/create/create_beauty.py | 4 ++-- pype/plugins/tvpaint/create/create_render_pass.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pype/plugins/tvpaint/create/create_beauty.py b/pype/plugins/tvpaint/create/create_beauty.py index f68365a2d4..da7dd9aa82 100644 --- a/pype/plugins/tvpaint/create/create_beauty.py +++ b/pype/plugins/tvpaint/create/create_beauty.py @@ -101,7 +101,7 @@ class CreateBeauty(pipeline.TVPaintCreator): self.log.debug("Changing name of the group.") selected_group = None for group_data in groups_data: - if group_data["id"] == group_id: + if group_data["group_id"] == group_id: selected_group = group_data # Rename TVPaint group (keep color same) @@ -109,7 +109,7 @@ class CreateBeauty(pipeline.TVPaintCreator): new_group_name = name.replace(" ", "_") rename_script = self.rename_script_template.format( clip_id=selected_group["clip_id"], - group_id=selected_group["id"], + group_id=selected_group["group_id"], r=selected_group["red"], g=selected_group["green"], b=selected_group["blue"], diff --git a/pype/plugins/tvpaint/create/create_render_pass.py b/pype/plugins/tvpaint/create/create_render_pass.py index a28348f46a..9103bc735d 100644 --- a/pype/plugins/tvpaint/create/create_render_pass.py +++ b/pype/plugins/tvpaint/create/create_render_pass.py @@ -67,7 +67,7 @@ class CreateRenderPass(pipeline.TVPaintCreator): self.data["name"] = name # Collect selected layer ids to be stored into instance - layer_ids = [layer["id"] for layer in selected_layers] + layer_ids = [layer["layer_id"] for layer in selected_layers] self.data["layer_ids"] = layer_ids beauty_subset_name = beauty_instance["subset"] From 71236486f9e0be5fe83ce8d0da12d8a87398c535 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 5 Nov 2020 16:13:47 +0100 Subject: [PATCH 022/147] move order to "precollect" --- pype/plugins/tvpaint/publish/collect_instances.py | 2 +- pype/plugins/tvpaint/publish/collect_workfile_data.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/plugins/tvpaint/publish/collect_instances.py b/pype/plugins/tvpaint/publish/collect_instances.py index caf6adee5f..fcf02be154 100644 --- a/pype/plugins/tvpaint/publish/collect_instances.py +++ b/pype/plugins/tvpaint/publish/collect_instances.py @@ -6,7 +6,7 @@ from avalon.tvpaint import pipeline class CollectInstances(pyblish.api.ContextPlugin): label = "Collect Instances" - order = pyblish.api.CollectorOrder + order = pyblish.api.CollectorOrder - 1 hosts = ["tvpaint"] def process(self, context): diff --git a/pype/plugins/tvpaint/publish/collect_workfile_data.py b/pype/plugins/tvpaint/publish/collect_workfile_data.py index d81250c05b..8a3927d0f0 100644 --- a/pype/plugins/tvpaint/publish/collect_workfile_data.py +++ b/pype/plugins/tvpaint/publish/collect_workfile_data.py @@ -6,7 +6,7 @@ from avalon.tvpaint import pipeline, lib class CollectWorkfileData(pyblish.api.ContextPlugin): label = "Collect Workfile Data" - order = pyblish.api.CollectorOrder - 0.01 + order = pyblish.api.CollectorOrder - 1.01 hosts = ["tvpaint"] def process(self, context): From 07b0a692e822db4e43262af3b8edced421e93d78 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 5 Nov 2020 17:45:16 +0100 Subject: [PATCH 023/147] collect instances can collect more --- .../tvpaint/publish/collect_instances.py | 100 ++++++++++++++++-- .../tvpaint/publish/collect_workfile_data.py | 2 +- 2 files changed, 90 insertions(+), 12 deletions(-) diff --git a/pype/plugins/tvpaint/publish/collect_instances.py b/pype/plugins/tvpaint/publish/collect_instances.py index fcf02be154..76ec460091 100644 --- a/pype/plugins/tvpaint/publish/collect_instances.py +++ b/pype/plugins/tvpaint/publish/collect_instances.py @@ -19,19 +19,97 @@ class CollectInstances(pyblish.api.ContextPlugin): # TODO add validations of existing instances # - layer id exists - for workfile_instance in workfile_instances: + for instance_data in workfile_instances: + # Global instance data modifications # Fill families - family = workfile_instance["family"] - workfile_instance["families"] = [family] + family = instance_data["family"] + instance_data["families"] = [family] # Instance name - subset_name = workfile_instance["subset"] - name = workfile_instance.get("name", subset_name) - workfile_instance["name"] = name + subset_name = instance_data["subset"] + name = instance_data.get("name", subset_name) + instance_data["name"] = name - active = workfile_instance.get("active", True) - workfile_instance["active"] = active - workfile_instance["publish"] = active + active = instance_data.get("active", True) + instance_data["active"] = active + instance_data["publish"] = active - instance = context.create_instance(**workfile_instance) - self.log.debug("Created instance: {}".format(instance)) + # Different instance creation based on family + instance = None + if family == "review": + instance = context.create_instance(**instance_data) + elif family == "renderLayer": + instance = self.create_render_layer(context, instance_data) + elif family == "renderPass": + instance = self.create_render_pass(context, instance_data) + else: + raise AssertionError( + "Instance with unknown family \"{}\": {}" + ) + + if instance is not None: + self.log.debug("Created instance: {}\n{}".format( + instance, json.dumps(instance.data, indent=4) + )) + + def create_render_layer(self, context, instance_data): + layers_data = context.data["layersData"] + group_id = instance_data["group_id"] + + name = instance_data["name"] + instance_data["label"] = name + + group_layers = [] + for layer in layers_data: + if layer["group_id"] == group_id and layer["visible"]: + group_layers.append(layer) + + if not group_layers: + # Should be handled here? + self.log.warning( + f"Group with id {group_id} does not contain any layers." + f" Instance \"{name}\" not created." + ) + return None + + instance_data["layers"] = group_layers + return context.create_instance(**instance_data) + + def create_render_pass(self, context, instance_data): + # Change family to `renderLayer` + instance_data["family"] = "renderLayer" + instance_data["families"] = [instance_data["family"]] + + layers_data = context.data["layersData"] + layers_by_id = { + layer["layer_id"]: layer + for layer in layers_data + } + + group_id = instance_data["group_id"] + layer_ids = instance_data["layer_ids"] + render_pass_layers = [] + for layer_id in layer_ids: + layer = layers_by_id.get(layer_id) + if not layer: + self.log.warning(f"Layer with id {layer_id} was not found.") + continue + + # Move to validator? + if layer["group_id"] != group_id: + self.log.warning( + f"Layer with id {layer_id} is in different group." + ) + continue + render_pass_layers.append(layer) + + if not render_pass_layers: + name = instance_data["name"] + self.log.warning( + f"All layers from RenderPass \"{name}\" do not exist." + " Instance not created." + ) + return None + + instance_data["layers"] = render_pass_layers + return context.create_instance(**instance_data) diff --git a/pype/plugins/tvpaint/publish/collect_workfile_data.py b/pype/plugins/tvpaint/publish/collect_workfile_data.py index 8a3927d0f0..87076e978d 100644 --- a/pype/plugins/tvpaint/publish/collect_workfile_data.py +++ b/pype/plugins/tvpaint/publish/collect_workfile_data.py @@ -52,6 +52,6 @@ class CollectWorkfileData(pyblish.api.ContextPlugin): "fieldOrder": field_order } self.log.debug( - "Scene data: {}".format(json.dumps(sceme_data,indent=4)) + "Scene data: {}".format(json.dumps(sceme_data, indent=4)) ) context.data.update(sceme_data) From 9839e540252d77f51d243d3e5da94d7c2202495b Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 5 Nov 2020 18:26:49 +0100 Subject: [PATCH 024/147] added toggle callback to activate/deactivate instances --- pype/hosts/tvpaint/__init__.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/pype/hosts/tvpaint/__init__.py b/pype/hosts/tvpaint/__init__.py index cb02c87151..3df33fb52a 100644 --- a/pype/hosts/tvpaint/__init__.py +++ b/pype/hosts/tvpaint/__init__.py @@ -2,6 +2,7 @@ import os import logging from avalon.tvpaint.communication_server import register_localization_file +from avalon.tvpaint import pipeline import avalon.api import pyblish.api from pype import PLUGINS_DIR @@ -13,6 +14,23 @@ LOAD_PATH = os.path.join(PLUGINS_DIR, "tvpaint", "load") CREATE_PATH = os.path.join(PLUGINS_DIR, "tvpaint", "create") +def on_instance_toggle(instance, old_value, new_value): + instance_id = instance.data["id"] + found = False + current_instances = pipeline.list_instances() + for idx, workfile_instance in enumerate(current_instances): + if workfile_instance["id"] == instance_id: + found = True + break + + if not found: + return + + if "active" in current_instances[idx]: + current_instances[idx]["active"] = new_value + pipeline._write_instances(current_instances) + + def install(): log.info("Pype - Installing TVPaint integration") current_dir = os.path.dirname(os.path.abspath(__file__)) @@ -23,6 +41,12 @@ def install(): avalon.api.register_plugin_path(avalon.api.Loader, LOAD_PATH) avalon.api.register_plugin_path(avalon.api.Creator, CREATE_PATH) + registered_callbacks = ( + pyblish.api.registered_callbacks().get("instanceToggled") or [] + ) + if on_instance_toggle not in registered_callbacks: + pyblish.api.register_callback("instanceToggled", on_instance_toggle) + def uninstall(): log.info("Pype - Uninstalling TVPaint integration") From 25b87bca12b84d79192254353f919c84510d0864 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Thu, 5 Nov 2020 18:35:36 +0100 Subject: [PATCH 025/147] added Deadline JobInfo dataclass --- pype/lib/abstract_submit_deadline.py | 321 ++++++++++++++++++++++++++- 1 file changed, 320 insertions(+), 1 deletion(-) diff --git a/pype/lib/abstract_submit_deadline.py b/pype/lib/abstract_submit_deadline.py index 8b8fa7bc1f..dede4c19ba 100644 --- a/pype/lib/abstract_submit_deadline.py +++ b/pype/lib/abstract_submit_deadline.py @@ -2,6 +2,8 @@ """Abstract class for submitting jobs to Deadline.""" import os from abc import ABCMeta, abstractmethod +import platform +import getpass import six import attr @@ -12,7 +14,324 @@ import pyblish.api @attr.s class DeadlineJobInfo: - BatchName = attr.ib() + """Mapping of all Deadline *JobInfo* attributes. + + This contains all JobInfo attributes plus their default values. + Those attributes set to `None` shouldn't be posted to Deadline as + the only required one is `Plugin`. Their default values used by Deadline + are stated in + comments. + + ..seealso: + https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/manual-submission.html + + """ + + # Required + # ---------------------------------------------- + Plugin = attr.ib() + + # General + Frames = attr.ib(default=None) # default: 0 + Name = attr.ib(default="Untitled") + Comment = attr.ib(default=None) # default: empty + Department = attr.ib(default=None) # default: empty + BatchName = attr.ib(default=None) # default: empty + UserName = attr.ib(default=getpass.getuser()) + MachineName = attr.ib(default=platform.node()) + Pool = attr.ib(default=None) # default: "none" + SecondaryPool = attr.ib(default=None) + Group = attr.ib(default=None) # default: "none" + Priority = attr.ib(default=50) + ChunkSize = attr.ib(default=1) + ConcurrentTasks = attr.ib(default=1) + LimitConcurrentTasksToNumberOfCpus = attr.ib( + default=None) # default: "true" + OnJobComplete = attr.ib(default="Nothing") + SynchronizeAllAuxiliaryFiles = attr.ib(default=None) # default: false + ForceReloadPlugin = attr.ib(default=None) # default: false + Sequential = attr.ib(default=None) # default: false + SuppressEvents = attr.ib(default=None) # default: false + Protected = attr.ib(default=None) # default: false + InitialStatus = attr.ib(default="Active") + NetworkRoot = attr.ib(default=None) + + # Timeouts + # ---------------------------------------------- + MinRenderTimeSeconds = attr.ib(default=None) # Default: 0 + MinRenderTimeMinutes = attr.ib(default=None) # Default: 0 + TaskTimeoutSeconds = attr.ib(default=None) # Default: 0 + TaskTimeoutMinutes = attr.ib(default=None) # Default: 0 + StartJobTimeoutSeconds = attr.ib(default=None) # Default: 0 + StartJobTimeoutMinutes = attr.ib(default=None) # Default: 0 + InitializePluginTimeoutSeconds = attr.ib(default=None) # Default: 0 + # can be one of + OnTaskTimeout = attr.ib(default=None) # Default: Error + EnableTimeoutsForScriptTasks = attr.ib(default=None) # Default: false + EnableFrameTimeouts = attr.ib(default=None) # Default: false + EnableAutoTimeout = attr.ib(default=None) # Default: false + + # Interruptible + # ---------------------------------------------- + Interruptible = attr.ib(default=None) # Default: false + InterruptiblePercentage = attr.ib(default=None) + RemTimeThreshold = attr.ib(default=None) + + # Notifications + # ---------------------------------------------- + # can be comma separated list of users + NotificationTargets = attr.ib(default=None) # Default: blank + ClearNotificationTargets = attr.ib(default=None) # Default: false + # A comma separated list of additional email addresses + NotificationEmails = attr.ib(default=None) # Default: blank + OverrideNotificationMethod = attr.ib(default=None) # Default: false + EmailNotification = attr.ib(default=None) # Default: false + PopupNotification = attr.ib(default=None) # Default: false + # String with `[EOL]` used for end of line + NotificationNote = attr.ib(default=None) # Default: blank + + # Machine Limit + # ---------------------------------------------- + MachineLimit = attr.ib(default=None) # Default: 0 + MachineLimitProgress = attr.ib(default=None) # Default: -1.0 + Whitelist = attr.ib(default=None) # Default: blank + Blacklist = attr.ib(default=None) # Default: blank + + # Limits + # ---------------------------------------------- + # comma separated list of limit groups + LimitGroups = attr.ib(default=None) # Default: blank + + # Dependencies + # ---------------------------------------------- + # comma separated list of job IDs + JobDependencies = attr.ib(default=None) # Default: blank + JobDependencyPercentage = attr.ib(default=None) # Default: -1 + IsFrameDependent = attr.ib(default=None) # Default: false + FrameDependencyOffsetStart = attr.ib(default=None) # Default: 0 + FrameDependencyOffsetEnd = attr.ib(default=None) # Default: 0 + ResumeOnCompleteDependencies = attr.ib(default=None) # Default: true + ResumeOnDeletedDependencies = attr.ib(default=None) # Default: false + ResumeOnFailedDependencies = attr.ib(default=None) # Default: false + # comma separated list of asset paths + RequiredAssets = attr.ib(default=None) # Default: blank + # comma separated list of script paths + ScriptDependencies = attr.ib(default=None) # Default: blank + + # Failure Detection + # ---------------------------------------------- + OverrideJobFailureDetection = attr.ib(default=None) # Default: false + FailureDetectionJobErrors = attr.ib(default=None) # 0..x + OverrideTaskFailureDetection = attr.ib(default=None) # Default: false + FailureDetectionTaskErrors = attr.ib(default=None) # 0..x + IgnoreBadJobDetection = attr.ib(default=None) # Default: false + SendJobErrorWarning = attr.ib(default=None) # Default: false + + # Cleanup + # ---------------------------------------------- + DeleteOnComplete = attr.ib(default=None) # Default: false + ArchiveOnComplete = attr.ib(default=None) # Default: false + OverrideAutoJobCleanup = attr.ib(default=None) # Default: false + OverrideJobCleanup = attr.ib(default=None) + JobCleanupDays = attr.ib(default=None) # Default: false + # + OverrideJobCleanupType = attr.ib(default=None) + + # Scheduling + # ---------------------------------------------- + # + ScheduledType = attr.ib(default=None) # Default: None + #
+ ScheduledStartDateTime = attr.ib(default=None) + ScheduledDays = attr.ib(default=None) # Default: 1 + # + JobDelay = attr.ib(default=None) + # Time= + Scheduled = attr.ib(default=None) + + # Scripts + # ---------------------------------------------- + # all accept path to script + PreJobScript = attr.ib(default=None) # Default: blank + PostJobScript = attr.ib(default=None) # Default: blank + PreTaskScript = attr.ib(default=None) # Default: blank + PostTaskScript = attr.ib(default=None) # Default: blank + + # Event Opt-Ins + # ---------------------------------------------- + # comma separated list of plugins + EventOptIns = attr.ib(default=None) # Default: blank + + # Environment + # ---------------------------------------------- + _environmentKeyValue = attr.ib(factory=list) + + @property + def EnvironmentKeyValue(self): # noqa: N802 + """Return all environment key values formatted for Deadline. + + Returns: + list of tuples: as `[('EnvironmentKeyValue0', 'key=value')]` + + """ + out = [] + index = 0 + for v in self._environmentKeyValue: + out.append(("EnvironmentKeyValue{}".format(index), v)) + index += 1 + return out + + @EnvironmentKeyValue.setter + def EnvironmentKeyValue(self, val): # noqa: N802 + self._environmentKeyValue.append(val) + + IncludeEnvironment = attr.ib(default=None) # Default: false + UseJobEnvironmentOnly = attr.ib(default=None) # Default: false + CustomPluginDirectory = attr.ib(default=None) # Default: blank + + # Job Extra Info + # ---------------------------------------------- + _extraInfos = attr.ib(factory=list) + _extraInfoKeyValues = attr.ib(factory=list) + + @property + def ExtraInfo(self): # noqa: N802 + """Return all ExtraInfo values formatted for Deadline. + + Returns: + list of tuples: as `[('ExtraInfo0', 'value')]` + + """ + out = [] + index = 0 + for v in self._extraInfos: + out.append(("ExtraInfo{}".format(index), v)) + index += 1 + return out + + @ExtraInfo.setter + def ExtraInfo(self, val): # noqa: N802 + self._extraInfos.append(val) + + @property + def ExtraInfoKeyValue(self): # noqa: N802 + """Return all ExtraInfoKeyValue values formatted for Deadline. + + Returns: + list of tuples: as `[('ExtraInfoKeyValue0', 'key=value')]` + + """ + out = [] + index = 0 + for v in self._extraInfoKeyValues: + out.append(("ExtraInfoKeyValue{}".format(index), v)) + index += 1 + return out + + @ExtraInfoKeyValue.setter + def ExtraInfoKeyValue(self, val): # noqa: N802 + self._extraInfoKeyValues.append(val) + + # Task Extra Info Names + # ---------------------------------------------- + OverrideTaskExtraInfoNames = attr.ib(default=None) # Default: false + _taskExtraInfos = attr.ib(factory=list) + + @property + def TaskExtraInfoName(self): # noqa: N802 + """Return all TaskExtraInfoName values formatted for Deadline. + + Returns: + list of tuples: as `[('TaskExtraInfoName0', 'value')]` + + """ + out = [] + index = 0 + for v in self._taskExtraInfos: + out.append(("TaskExtraInfoName{}".format(index), v)) + index += 1 + return out + + @TaskExtraInfoName.setter + def TaskExtraInfoName(self, val): # noqa: N802 + self._taskExtraInfos.append(val) + + # Output + # ---------------------------------------------- + _outputFilename = attr.ib(factory=list) + _outputFilenameTile = attr.ib(factory=list) + _outputDirectory = attr.ib(factory=list) + + @property + def OutputFilename(self): # noqa: N802 + """Return all OutputFilename values formatted for Deadline. + + Returns: + list of tuples: as `[('OutputFilename0', 'filename')]` + + """ + out = [] + index = 0 + for v in self._outputFilename: + out.append(("OutputFilename{}".format(index), v)) + index += 1 + return out + + @OutputFilename.setter + def OutputFilename(self, val): # noqa: N802 + self._outputFilename.append(val) + + @property + def OutputFilenameTile(self): # noqa: N802 + """Return all OutputFilename#Tile values formatted for Deadline. + + Returns: + list of tuples: as `[('OutputFilename#Tile', 'tile')]` + + """ + out = [] + index = 0 + for v in self._outputFilenameTile: + out.append(("OutputFilename{}Tile".format(index), v)) + index += 1 + return out + + @OutputFilenameTile.setter + def OutputFilenameTile(self, val): # noqa: N802 + self._outputFilenameTile.append(val) + + @property + def OutputDirectory(self): # noqa: N802 + """Return all OutputDirectory values formatted for Deadline. + + Returns: + list of tuples: as `[('OutputDirectory0', 'dir')]` + + """ + out = [] + index = 0 + for v in self._outputDirectory: + out.append(("OutputDirectory{}".format(index), v)) + index += 1 + return out + + @OutputDirectory.setter + def OutputDirectory(self, val): # noqa: N802 + self._outputDirectory.append(val) + + # Tile Job + # ---------------------------------------------- + TileJob = attr.ib(default=None) # Default: false + TileJobFrame = attr.ib(default=None) # Default: 0 + TileJobTilesInX = attr.ib(default=None) # Default: 0 + TileJobTilesInY = attr.ib(default=None) # Default: 0 + TileJobTileCount = attr.ib(default=None) # Default: 0 + + # Maintenance Job + # ---------------------------------------------- + MaintenanceJob = attr.ib(default=None) # Default: false + MaintenanceJobStartFrame = attr.ib(default=None) # Default: 0 + MaintenanceJobEndFrame = attr.ib(default=None) # Default: 0 @attr.s From 1bb51aa0d01876297b187985ca2301a4d007c3a1 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 5 Nov 2020 18:59:51 +0100 Subject: [PATCH 026/147] always override instance --- pype/plugins/tvpaint/create/create_beauty.py | 7 ++----- pype/plugins/tvpaint/create/create_render_pass.py | 6 +----- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/pype/plugins/tvpaint/create/create_beauty.py b/pype/plugins/tvpaint/create/create_beauty.py index da7dd9aa82..30e0d4ccd8 100644 --- a/pype/plugins/tvpaint/create/create_beauty.py +++ b/pype/plugins/tvpaint/create/create_beauty.py @@ -79,12 +79,9 @@ class CreateBeauty(pipeline.TVPaintCreator): if existing_instance is not None: self.log.info( - f"Beauty instance for group id {group_id} already exists." + f"Beauty instance for group id {group_id} already exists" + ", overriding" ) - if existing_instance == self.data: - self.log.info("Instance to create is same. Did nothing.") - return - self.log.debug("Overriding beauty instance with new data.") instances[existing_instance_idx] = self.data else: instances.append(self.data) diff --git a/pype/plugins/tvpaint/create/create_render_pass.py b/pype/plugins/tvpaint/create/create_render_pass.py index 9103bc735d..3842c433e9 100644 --- a/pype/plugins/tvpaint/create/create_render_pass.py +++ b/pype/plugins/tvpaint/create/create_render_pass.py @@ -104,12 +104,8 @@ class CreateRenderPass(pipeline.TVPaintCreator): if existing_instance is not None: self.log.info( f"Render pass instance for group id {group_id}" - f" and name \"{name}\" already exists." + f" and name \"{name}\" already exists, overriding." ) - if existing_instance == self.data: - self.log.info("Instance to create is same. Did nothing.") - return - self.log.debug("Overriding beauty instance with new data.") instances[existing_instance_idx] = self.data else: instances.append(self.data) From 1a4f27291edc661c93b83ac070a0cb519e0534fa Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 5 Nov 2020 19:03:02 +0100 Subject: [PATCH 027/147] fix import after merge --- pype/plugins/tvpaint/load/load_image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/plugins/tvpaint/load/load_image.py b/pype/plugins/tvpaint/load/load_image.py index 5ab67e3df4..51a825a0c8 100644 --- a/pype/plugins/tvpaint/load/load_image.py +++ b/pype/plugins/tvpaint/load/load_image.py @@ -1,6 +1,6 @@ from avalon import api from avalon.vendor import qargparse -from avalon.tvpaint import CommunicatorWrapper +from avalon.tvpaint import lib class ImportImage(api.Loader): From 5a96915992bf97d7588cd9199ce2c1e175ac2744 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 5 Nov 2020 19:19:07 +0100 Subject: [PATCH 028/147] use variable Creator instead of TVPaintCreator --- pype/plugins/tvpaint/create/create_beauty.py | 2 +- pype/plugins/tvpaint/create/create_render_pass.py | 2 +- pype/plugins/tvpaint/create/create_review.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pype/plugins/tvpaint/create/create_beauty.py b/pype/plugins/tvpaint/create/create_beauty.py index 30e0d4ccd8..6512ad08b0 100644 --- a/pype/plugins/tvpaint/create/create_beauty.py +++ b/pype/plugins/tvpaint/create/create_beauty.py @@ -2,7 +2,7 @@ import avalon.io from avalon.tvpaint import pipeline, lib -class CreateBeauty(pipeline.TVPaintCreator): +class CreateBeauty(pipeline.Creator): """Mark layer group as one instance.""" name = "beauty" label = "Beauty" diff --git a/pype/plugins/tvpaint/create/create_render_pass.py b/pype/plugins/tvpaint/create/create_render_pass.py index 3842c433e9..9971109840 100644 --- a/pype/plugins/tvpaint/create/create_render_pass.py +++ b/pype/plugins/tvpaint/create/create_render_pass.py @@ -3,7 +3,7 @@ import avalon.io from avalon.tvpaint import pipeline, lib -class CreateRenderPass(pipeline.TVPaintCreator): +class CreateRenderPass(pipeline.Creator): """Render pass is combination of one or more layers from same group. Requirement to create Render Pass is to have already created beauty diff --git a/pype/plugins/tvpaint/create/create_review.py b/pype/plugins/tvpaint/create/create_review.py index 0163d26160..9f7ee1396e 100644 --- a/pype/plugins/tvpaint/create/create_review.py +++ b/pype/plugins/tvpaint/create/create_review.py @@ -1,7 +1,7 @@ from avalon.tvpaint import pipeline -class CreateReview(pipeline.TVPaintCreator): +class CreateReview(pipeline.Creator): """Review for global review of all layers.""" name = "review" label = "Review" From 08bce91cd920b271a562446a39fb69fbe7d46a07 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 5 Nov 2020 19:21:48 +0100 Subject: [PATCH 029/147] use TVPaint loader in ImportImage --- pype/plugins/tvpaint/load/load_image.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pype/plugins/tvpaint/load/load_image.py b/pype/plugins/tvpaint/load/load_image.py index 51a825a0c8..3a09a76c7b 100644 --- a/pype/plugins/tvpaint/load/load_image.py +++ b/pype/plugins/tvpaint/load/load_image.py @@ -1,9 +1,8 @@ -from avalon import api from avalon.vendor import qargparse -from avalon.tvpaint import lib +from avalon.tvpaint import lib, Loader -class ImportImage(api.Loader): +class ImportImage(Loader): """Load image or image sequence to TVPaint as new layer.""" families = ["render", "image", "background", "plate"] From 88ff7ad5866b0094380c33616d3b3c0a6e48e0ad Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 5 Nov 2020 19:24:48 +0100 Subject: [PATCH 030/147] removed unused --- pype/plugins/tvpaint/create/create_render_pass.py | 2 -- pype/plugins/tvpaint/load/load_image.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pype/plugins/tvpaint/create/create_render_pass.py b/pype/plugins/tvpaint/create/create_render_pass.py index 9971109840..e0e1b639e1 100644 --- a/pype/plugins/tvpaint/create/create_render_pass.py +++ b/pype/plugins/tvpaint/create/create_render_pass.py @@ -1,5 +1,3 @@ -import tempfile -import avalon.io from avalon.tvpaint import pipeline, lib diff --git a/pype/plugins/tvpaint/load/load_image.py b/pype/plugins/tvpaint/load/load_image.py index 3a09a76c7b..f77fab87f8 100644 --- a/pype/plugins/tvpaint/load/load_image.py +++ b/pype/plugins/tvpaint/load/load_image.py @@ -1,8 +1,8 @@ from avalon.vendor import qargparse -from avalon.tvpaint import lib, Loader +from avalon.tvpaint import lib, pipeline -class ImportImage(Loader): +class ImportImage(pipeline.Loader): """Load image or image sequence to TVPaint as new layer.""" families = ["render", "image", "background", "plate"] From c43a58efa9394bcf4d62575161fec9eedcd45889 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 5 Nov 2020 19:27:56 +0100 Subject: [PATCH 031/147] feat(SP): wip editorial expansion to image sequences --- .../publish/collect_editorial.py | 67 +++++++++--- .../publish/collect_hierarchy.py | 58 +++++++--- .../publish/collect_instance_data.py | 2 +- .../publish/collect_instance_resources.py | 57 ++++++++++ ...clip_instances.py => collect_instances.py} | 31 +++--- .../publish/extract_shot_data.py | 92 ---------------- .../publish/extract_trim_video_audio.py | 101 ++++++++++++++++++ .../publish/validate_editorial_resources.py | 2 +- 8 files changed, 277 insertions(+), 133 deletions(-) create mode 100644 pype/plugins/standalonepublisher/publish/collect_instance_resources.py rename pype/plugins/standalonepublisher/publish/{collect_clip_instances.py => collect_instances.py} (89%) delete mode 100644 pype/plugins/standalonepublisher/publish/extract_shot_data.py create mode 100644 pype/plugins/standalonepublisher/publish/extract_trim_video_audio.py diff --git a/pype/plugins/standalonepublisher/publish/collect_editorial.py b/pype/plugins/standalonepublisher/publish/collect_editorial.py index 5e6fd106e4..7e532c3741 100644 --- a/pype/plugins/standalonepublisher/publish/collect_editorial.py +++ b/pype/plugins/standalonepublisher/publish/collect_editorial.py @@ -1,3 +1,19 @@ +""" +Optional: + presets -> extensions ( + example of use: + [".mov", ".mp4"] + ) + presets -> source_dir ( + example of use: + "C:/pathToFolder" + "{root}/{project[name]}/inputs" + "{root[work]}/{project[name]}/inputs" + "./input" + "../input" + ) +""" + import os import opentimelineio as otio import pyblish.api @@ -33,8 +49,10 @@ class CollectEditorial(pyblish.api.InstancePlugin): # presets extensions = [".mov", ".mp4"] + source_dir = None def process(self, instance): + root_dir = None # remove context test attribute if instance.context.data.get("subsetNamesCheck"): instance.context.data.pop("subsetNamesCheck") @@ -53,19 +71,42 @@ class CollectEditorial(pyblish.api.InstancePlugin): # get video file path video_path = None basename = os.path.splitext(os.path.basename(file_path))[0] - for f in os.listdir(staging_dir): - self.log.debug(f"__ test file: `{f}`") - # filter out by not sharing the same name - if os.path.splitext(f)[0] not in basename: - continue - # filter out by respected extensions - if os.path.splitext(f)[1] not in self.extensions: - continue - video_path = os.path.join( - staging_dir, f - ) - self.log.debug(f"__ video_path: `{video_path}`") - instance.data["editorialVideoPath"] = video_path + + if self.source_dir: + source_dir = self.source_dir.replace("\\", "/") + if ("./" in source_dir) or ("../" in source_dir): + # get current working dir + cwd = os.getcwd() + # set cwd to staging dir for absolute path solving + os.chdir(staging_dir) + root_dir = os.path.abspath(source_dir) + # set back original cwd + os.chdir(cwd) + elif "{" in source_dir: + root_dir = source_dir + else: + root_dir = os.path.normpath(source_dir) + + if root_dir: + # search for source data will need to be done + instance.data["editorialSourceRoot"] = root_dir + instance.data["editorialSourcePath"] = None + else: + # source data are already found + for f in os.listdir(staging_dir): + # filter out by not sharing the same name + if os.path.splitext(f)[0] not in basename: + continue + # filter out by respected extensions + if os.path.splitext(f)[1] not in self.extensions: + continue + video_path = os.path.join( + staging_dir, f + ) + self.log.debug(f"__ video_path: `{video_path}`") + instance.data["editorialSourceRoot"] = staging_dir + instance.data["editorialSourcePath"] = video_path + instance.data["stagingDir"] = staging_dir # get editorial sequence file into otio timeline object diff --git a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py index ac7413706a..6ce6232943 100644 --- a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py +++ b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py @@ -2,7 +2,7 @@ import pyblish.api import re import os from avalon import io - +from copy import deepcopy class CollectHierarchyInstance(pyblish.api.ContextPlugin): """Collecting hierarchy context from `parents` and `hierarchy` data @@ -60,7 +60,7 @@ class CollectHierarchyInstance(pyblish.api.ContextPlugin): def create_hierarchy(self, instance): parents = list() - hierarchy = "" + hierarchy = list() visual_hierarchy = [instance.context.data["assetEntity"]] while True: visual_parent = io.find_one( @@ -81,22 +81,51 @@ class CollectHierarchyInstance(pyblish.api.ContextPlugin): }) if self.shot_add_hierarchy: + parent_template_patern = re.compile(r"\{([a-z]*?)\}") # fill the parents parts from presets shot_add_hierarchy = self.shot_add_hierarchy.copy() hierarchy_parents = shot_add_hierarchy["parents"].copy() - for parent in hierarchy_parents: - hierarchy_parents[parent] = hierarchy_parents[parent].format( - **instance.data["anatomyData"]) + + # fill parent keys data template from anatomy data + for parent_key in hierarchy_parents: + hierarchy_parents[parent_key] = hierarchy_parents[ + parent_key].format(**instance.data["anatomyData"]) + + for _index, _parent in enumerate( + shot_add_hierarchy["parents_path"].split("/")): + parent_filled = _parent.format(**hierarchy_parents) + parent_key = parent_template_patern.findall(_parent).pop() + + # in case SP context is set to the same folder + if (_index == 0) and ("folder" in parent_key) \ + and (parents[-1]["entityName"] == parent_filled): + self.log.debug(f" skiping : {parent_filled}") + continue + + # in case first parent is project then start parents from start + if (_index == 0) and ("project" in parent_key): + self.log.debug("rebuilding parents from scratch") + project_parent = parents[0] + parents = [project_parent] + self.log.debug(f"project_parent: {project_parent}") + self.log.debug(f"parents: {parents}") + continue + prnt = self.convert_to_entity( - parent, hierarchy_parents[parent]) + parent_key, parent_filled) parents.append(prnt) + hierarchy.append(parent_filled) - hierarchy = shot_add_hierarchy[ - "parents_path"].format(**hierarchy_parents) + # convert hierarchy to string + hierarchy = "/".join(hierarchy) + # assing to instance data instance.data["hierarchy"] = hierarchy instance.data["parents"] = parents + + # print self.log.debug(f"Hierarchy: {hierarchy}") + self.log.debug(f"parents: {parents}") if self.shot_add_tasks: instance.data["tasks"] = self.shot_add_tasks @@ -117,7 +146,8 @@ class CollectHierarchyInstance(pyblish.api.ContextPlugin): def processing_instance(self, instance): self.log.info(f"_ instance: {instance}") # adding anatomyData for burnins - instance.data["anatomyData"] = instance.context.data["anatomyData"] + instance.data["anatomyData"] = deepcopy( + instance.context.data["anatomyData"]) asset = instance.data["asset"] assets_shared = instance.context.data.get("assetsShared") @@ -133,9 +163,6 @@ class CollectHierarchyInstance(pyblish.api.ContextPlugin): shot_name = instance.data["asset"] self.log.debug(f"Shot Name: {shot_name}") - if instance.data["hierarchy"] not in shot_name: - self.log.warning("wrong parent") - label = f"{shot_name} ({frame_start}-{frame_end})" instance.data["label"] = label @@ -150,7 +177,8 @@ class CollectHierarchyInstance(pyblish.api.ContextPlugin): "asset": instance.data["asset"], "hierarchy": instance.data["hierarchy"], "parents": instance.data["parents"], - "tasks": instance.data["tasks"] + "tasks": instance.data["tasks"], + "anatomyData": instance.data["anatomyData"] }) @@ -194,6 +222,7 @@ class CollectHierarchyContext(pyblish.api.ContextPlugin): instance.data["parents"] = s_asset_data["parents"] instance.data["hierarchy"] = s_asset_data["hierarchy"] instance.data["tasks"] = s_asset_data["tasks"] + instance.data["anatomyData"] = s_asset_data["anatomyData"] # generate hierarchy data only on shot instances if 'shot' not in instance.data.get('family', ''): @@ -224,7 +253,9 @@ class CollectHierarchyContext(pyblish.api.ContextPlugin): in_info['tasks'] = instance.data['tasks'] + from pprint import pformat parents = instance.data.get('parents', []) + self.log.debug(f"parents: {pformat(parents)}") actual = {name: in_info} @@ -240,4 +271,5 @@ class CollectHierarchyContext(pyblish.api.ContextPlugin): # adding hierarchy context to instance context.data["hierarchyContext"] = final_context + self.log.debug(f"hierarchyContext: {pformat(final_context)}") self.log.info("Hierarchy instance collected") diff --git a/pype/plugins/standalonepublisher/publish/collect_instance_data.py b/pype/plugins/standalonepublisher/publish/collect_instance_data.py index 1b32ea9144..58b81324f5 100644 --- a/pype/plugins/standalonepublisher/publish/collect_instance_data.py +++ b/pype/plugins/standalonepublisher/publish/collect_instance_data.py @@ -22,7 +22,7 @@ class CollectInstanceData(pyblish.api.InstancePlugin): hosts = ["standalonepublisher"] def process(self, instance): - fps = instance.data["assetEntity"]["data"]["fps"] + fps = instance.context.data["assetEntity"]["data"]["fps"] instance.data.update({ "fps": fps }) diff --git a/pype/plugins/standalonepublisher/publish/collect_instance_resources.py b/pype/plugins/standalonepublisher/publish/collect_instance_resources.py new file mode 100644 index 0000000000..63b98f2721 --- /dev/null +++ b/pype/plugins/standalonepublisher/publish/collect_instance_resources.py @@ -0,0 +1,57 @@ +import os +import tempfile +import pyblish.api +from copy import deepcopy + +class CollectInstanceResources(pyblish.api.InstancePlugin): + """Collect instance's resources""" + + # must be after `CollectInstances` + order = pyblish.api.CollectorOrder + 0.011 + label = "Collect Instance Resources" + hosts = ["standalonepublisher"] + families = ["clip"] + + def process(self, instance): + anatomy = instance.context.data["anatomy"] + anatomy_data = deepcopy(instance.context.data["anatomyData"]) + anatomy_data.update({"root": anatomy.roots}) + + subset = instance.data["subset"] + clip_name = instance.data["clipName"] + + editorial_source_root = instance.data["editorialSourceRoot"] + editorial_source_path = instance.data["editorialSourcePath"] + + if editorial_source_path: + # add family if mov or mp4 found which is longer for + # cutting `trimming` to enable `ExtractTrimmingVideoAudio` plugin + staging_dir = os.path.normpath( + tempfile.mkdtemp(prefix="pyblish_tmp_") + ) + instance.data["stagingDir"] = staging_dir + instance.data["families"] += ["trimming"] + return + + if "{" in editorial_source_root: + editorial_source_root = editorial_source_root.format( + **anatomy_data) + + self.log.debug(f"root: {editorial_source_root}") + + for root, dirs, files in os.walk(editorial_source_root): + if subset in root and clip_name in root: + staging_dir = root + + self.log.debug(f"staging_dir: {staging_dir}") + + + # add `editorialSourceRoot` as staging dir + + # if `editorialSourcePath` is none then loop + # trough `editorialSourceRoot` + + # if image sequence then create representation > match + # with subset name in dict + + # idenfify as image sequence `isSequence` on instance data diff --git a/pype/plugins/standalonepublisher/publish/collect_clip_instances.py b/pype/plugins/standalonepublisher/publish/collect_instances.py similarity index 89% rename from pype/plugins/standalonepublisher/publish/collect_clip_instances.py rename to pype/plugins/standalonepublisher/publish/collect_instances.py index def0c13a78..9cd8d9f36c 100644 --- a/pype/plugins/standalonepublisher/publish/collect_clip_instances.py +++ b/pype/plugins/standalonepublisher/publish/collect_instances.py @@ -1,15 +1,14 @@ import os import opentimelineio as otio -import tempfile import pyblish.api from pype import lib as plib -class CollectClipInstances(pyblish.api.InstancePlugin): - """Collect Clips instances from editorial's OTIO sequence""" +class CollectInstances(pyblish.api.InstancePlugin): + """Collect instances from editorial's OTIO sequence""" order = pyblish.api.CollectorOrder + 0.01 - label = "Collect Clips" + label = "Collect Instances" hosts = ["standalonepublisher"] families = ["editorial"] @@ -19,13 +18,13 @@ class CollectClipInstances(pyblish.api.InstancePlugin): "family": "review", "families": ["clip", "ftrack"], # "ftrackFamily": "review", - "extension": ".mp4" + "extensions": [".mp4"] }, "audioMain": { "family": "audio", "families": ["clip", "ftrack"], # "ftrackFamily": "audio", - "extension": ".wav", + "extensions": [".wav"], # "version": 1 }, "shotMain": { @@ -37,12 +36,14 @@ class CollectClipInstances(pyblish.api.InstancePlugin): custom_start_frame = None def process(self, instance): - staging_dir = os.path.normpath( - tempfile.mkdtemp(prefix="pyblish_tmp_") - ) # get context context = instance.context + instance_data_filter = [ + "editorialSourceRoot", + "editorialSourcePath" + ] + # attribute for checking duplicity during creation if not context.data.get("assetNameCheck"): context.data["assetNameCheck"] = list() @@ -103,7 +104,10 @@ class CollectClipInstances(pyblish.api.InstancePlugin): # frame ranges data clip_in = clip.range_in_parent().start_time.value + clip_in += track_start_frame clip_out = clip.range_in_parent().end_time_inclusive().value + clip_out += track_start_frame + self.log.info(f"clip_in: {clip_in} | clip_out: {clip_out}") # add offset in case there is any if self.timeline_frame_offset: @@ -131,14 +135,11 @@ class CollectClipInstances(pyblish.api.InstancePlugin): # create shared new instance data instance_data = { - "stagingDir": staging_dir, - # shared attributes "asset": name, "assetShareName": name, - "editorialVideoPath": instance.data[ - "editorialVideoPath"], "item": clip, + "clipName": clip_name, # parent time properities "trackStartFrame": track_start_frame, @@ -167,6 +168,10 @@ class CollectClipInstances(pyblish.api.InstancePlugin): "frameEndH": frame_end + handle_end } + for data_key in instance_data_filter: + instance_data.update({ + data_key: instance.data.get(data_key)}) + # adding subsets to context as instances for subset, properities in self.subsets.items(): # adding Review-able instance diff --git a/pype/plugins/standalonepublisher/publish/extract_shot_data.py b/pype/plugins/standalonepublisher/publish/extract_shot_data.py deleted file mode 100644 index d5af7638ee..0000000000 --- a/pype/plugins/standalonepublisher/publish/extract_shot_data.py +++ /dev/null @@ -1,92 +0,0 @@ -import os -import clique -import pype.api - -from pprint import pformat - - -class ExtractShotData(pype.api.Extractor): - """Extract shot "mov" and "wav" files.""" - - label = "Extract Shot Data" - hosts = ["standalonepublisher"] - families = ["clip"] - - # presets - - def process(self, instance): - representation = instance.data.get("representations") - self.log.debug(f"_ representation: {representation}") - - if not representation: - instance.data["representations"] = list() - - # get ffmpet path - ffmpeg_path = pype.lib.get_ffmpeg_tool_path("ffmpeg") - - # get staging dir - staging_dir = self.staging_dir(instance) - self.log.info("Staging dir set to: `{}`".format(staging_dir)) - - # Generate mov file. - fps = instance.data["fps"] - video_file_path = instance.data["editorialVideoPath"] - ext = instance.data.get("extension", ".mov") - - clip_trimed_path = os.path.join( - staging_dir, instance.data["name"] + ext) - # - # # check video file metadata - # input_data = plib.ffprobe_streams(video_file_path)[0] - # self.log.debug(f"__ input_data: `{input_data}`") - - start = float(instance.data["clipInH"]) - dur = float(instance.data["clipDurationH"]) - - if ext in ".wav": - start += 0.5 - - args = [ - ffmpeg_path, - "-ss", str(start / fps), - "-i", f"\"{video_file_path}\"", - "-t", str(dur / fps) - ] - if ext in [".mov", ".mp4"]: - args.extend([ - "-crf", "18", - "-pix_fmt", "yuv420p"]) - elif ext in ".wav": - args.extend([ - "-vn -acodec pcm_s16le", - "-ar 48000 -ac 2" - ]) - - # add output path - args.append(f"\"{clip_trimed_path}\"") - - self.log.info(f"Processing: {args}") - ffmpeg_args = " ".join(args) - output = pype.api.subprocess(ffmpeg_args, shell=True) - self.log.info(output) - - repr = { - "name": ext[1:], - "ext": ext[1:], - "files": os.path.basename(clip_trimed_path), - "stagingDir": staging_dir, - "frameStart": int(instance.data["frameStart"]), - "frameEnd": int(instance.data["frameEnd"]), - "frameStartFtrack": int(instance.data["frameStartH"]), - "frameEndFtrack": int(instance.data["frameEndH"]), - "fps": fps, - } - - if ext[1:] in ["mov", "mp4"]: - repr.update({ - "thumbnail": True, - "tags": ["review", "ftrackreview", "delete"]}) - - instance.data["representations"].append(repr) - - self.log.debug(f"Instance data: {pformat(instance.data)}") diff --git a/pype/plugins/standalonepublisher/publish/extract_trim_video_audio.py b/pype/plugins/standalonepublisher/publish/extract_trim_video_audio.py new file mode 100644 index 0000000000..c955275b4f --- /dev/null +++ b/pype/plugins/standalonepublisher/publish/extract_trim_video_audio.py @@ -0,0 +1,101 @@ +import os +import pyblish.api +import pype.api + +from pprint import pformat + + +class ExtractTrimVideoAudio(pype.api.Extractor): + """Trim with ffmpeg "mov" and "wav" files.""" + + label = "Extract Trim Video/Audio" + hosts = ["standalonepublisher"] + families = ["clip", "trimming"] + + # make sure it is enabled only if at least both families are available + match = pyblish.api.Subset + + # presets + + def process(self, instance): + representation = instance.data.get("representations") + self.log.debug(f"_ representation: {representation}") + + if not representation: + instance.data["representations"] = list() + + # get ffmpet path + ffmpeg_path = pype.lib.get_ffmpeg_tool_path("ffmpeg") + + # get staging dir + staging_dir = self.staging_dir(instance) + self.log.info("Staging dir set to: `{}`".format(staging_dir)) + + # Generate mov file. + fps = instance.data["fps"] + video_file_path = instance.data["editorialSourcePath"] + extensions = instance.data.get("extensions", [".mov"]) + + for ext in extensions: + clip_trimed_path = os.path.join( + staging_dir, instance.data["name"] + ext) + # # check video file metadata + # input_data = plib.ffprobe_streams(video_file_path)[0] + # self.log.debug(f"__ input_data: `{input_data}`") + + start = float(instance.data["clipInH"]) + dur = float(instance.data["clipDurationH"]) + + if ext in ".wav": + # offset time as ffmpeg is having bug + start += 0.5 + # remove "review" from families + instance.data["families"] = [ + fml for fml in instance.data["families"] + if "trimming" not in fml + ] + + args = [ + ffmpeg_path, + "-ss", str(start / fps), + "-i", f"\"{video_file_path}\"", + "-t", str(dur / fps) + ] + if ext in [".mov", ".mp4"]: + args.extend([ + "-crf", "18", + "-pix_fmt", "yuv420p"]) + elif ext in ".wav": + args.extend([ + "-vn -acodec pcm_s16le", + "-ar 48000 -ac 2" + ]) + + # add output path + args.append(f"\"{clip_trimed_path}\"") + + self.log.info(f"Processing: {args}") + ffmpeg_args = " ".join(args) + output = pype.api.subprocess(ffmpeg_args, shell=True) + self.log.info(output) + + repr = { + "name": ext[1:], + "ext": ext[1:], + "files": os.path.basename(clip_trimed_path), + "stagingDir": staging_dir, + "frameStart": int(instance.data["frameStart"]), + "frameEnd": int(instance.data["frameEnd"]), + "frameStartFtrack": int(instance.data["frameStartH"]), + "frameEndFtrack": int(instance.data["frameEndH"]), + "fps": fps, + } + + if ext[1:] in ["mov", "mp4"]: + repr.update({ + "thumbnail": True, + "tags": ["review", "ftrackreview", "delete"]}) + + instance.data["representations"].append(repr) + + self.log.debug(f"Instance data: {pformat(instance.data)}") diff --git a/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py b/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py index 7e1694fbd1..65581a6cdc 100644 --- a/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py +++ b/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py @@ -15,6 +15,6 @@ class ValidateEditorialResources(pyblish.api.InstancePlugin): self.log.debug( f"Instance: {instance}, Families: " f"{[instance.data['family']] + instance.data['families']}") - check_file = instance.data["editorialVideoPath"] + check_file = instance.data["editorialSourcePath"] msg = f"Missing \"{check_file}\"." assert check_file, msg From 350fc4997847a34722fe6fe733a12a2b4b433ab0 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 5 Nov 2020 19:55:27 +0100 Subject: [PATCH 032/147] hound changes --- pype/hosts/tvpaint/__init__.py | 10 +++++----- pype/plugins/tvpaint/publish/collect_instances.py | 2 -- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/pype/hosts/tvpaint/__init__.py b/pype/hosts/tvpaint/__init__.py index 3df33fb52a..abda126ecd 100644 --- a/pype/hosts/tvpaint/__init__.py +++ b/pype/hosts/tvpaint/__init__.py @@ -16,18 +16,18 @@ CREATE_PATH = os.path.join(PLUGINS_DIR, "tvpaint", "create") def on_instance_toggle(instance, old_value, new_value): instance_id = instance.data["id"] - found = False + found_idx = None current_instances = pipeline.list_instances() for idx, workfile_instance in enumerate(current_instances): if workfile_instance["id"] == instance_id: - found = True + found_idx = idx break - if not found: + if found_idx is None: return - if "active" in current_instances[idx]: - current_instances[idx]["active"] = new_value + if "active" in current_instances[found_idx]: + current_instances[found_idx]["active"] = new_value pipeline._write_instances(current_instances) diff --git a/pype/plugins/tvpaint/publish/collect_instances.py b/pype/plugins/tvpaint/publish/collect_instances.py index 76ec460091..84fc7688f7 100644 --- a/pype/plugins/tvpaint/publish/collect_instances.py +++ b/pype/plugins/tvpaint/publish/collect_instances.py @@ -1,7 +1,5 @@ import json - import pyblish.api -from avalon.tvpaint import pipeline class CollectInstances(pyblish.api.ContextPlugin): From 23ebe56de69c26d1c80dddeb12e36cef8e252ddf Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 6 Nov 2020 09:55:29 +0100 Subject: [PATCH 033/147] instance toggle works with uuid key not with id --- pype/hosts/tvpaint/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/hosts/tvpaint/__init__.py b/pype/hosts/tvpaint/__init__.py index abda126ecd..7027f0fb55 100644 --- a/pype/hosts/tvpaint/__init__.py +++ b/pype/hosts/tvpaint/__init__.py @@ -15,11 +15,11 @@ CREATE_PATH = os.path.join(PLUGINS_DIR, "tvpaint", "create") def on_instance_toggle(instance, old_value, new_value): - instance_id = instance.data["id"] + instance_id = instance.data["uuid"] found_idx = None current_instances = pipeline.list_instances() for idx, workfile_instance in enumerate(current_instances): - if workfile_instance["id"] == instance_id: + if workfile_instance["uuid"] == instance_id: found_idx = idx break From f3e2354a5b2612f1e45c098a4381979a855fc5d9 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 6 Nov 2020 10:01:26 +0100 Subject: [PATCH 034/147] Renamed beauty to render layer --- pype/plugins/tvpaint/create/create_beauty.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/plugins/tvpaint/create/create_beauty.py b/pype/plugins/tvpaint/create/create_beauty.py index 6512ad08b0..10ac444a80 100644 --- a/pype/plugins/tvpaint/create/create_beauty.py +++ b/pype/plugins/tvpaint/create/create_beauty.py @@ -4,8 +4,8 @@ from avalon.tvpaint import pipeline, lib class CreateBeauty(pipeline.Creator): """Mark layer group as one instance.""" - name = "beauty" - label = "Beauty" + name = "render_layer" + label = "RenderLayer" family = "renderLayer" icon = "cube" defaults = ["Main"] From b3c94bbc4e06da961aa2a591d350b301bdc940fd Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 6 Nov 2020 10:34:07 +0100 Subject: [PATCH 035/147] renamed create_beauty file to create_render_layer --- .../tvpaint/create/{create_beauty.py => create_render_layer.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pype/plugins/tvpaint/create/{create_beauty.py => create_render_layer.py} (100%) diff --git a/pype/plugins/tvpaint/create/create_beauty.py b/pype/plugins/tvpaint/create/create_render_layer.py similarity index 100% rename from pype/plugins/tvpaint/create/create_beauty.py rename to pype/plugins/tvpaint/create/create_render_layer.py From 4ea18518939ac62041c7d1c0539b98a18ef269e2 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 6 Nov 2020 10:34:21 +0100 Subject: [PATCH 036/147] removed space from RenderPass label --- pype/plugins/tvpaint/create/create_render_pass.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/plugins/tvpaint/create/create_render_pass.py b/pype/plugins/tvpaint/create/create_render_pass.py index e0e1b639e1..f6343a2b0b 100644 --- a/pype/plugins/tvpaint/create/create_render_pass.py +++ b/pype/plugins/tvpaint/create/create_render_pass.py @@ -8,7 +8,7 @@ class CreateRenderPass(pipeline.Creator): instance. Beauty instance is used as base for subset name. """ name = "render_pass" - label = "Render Pass" + label = "RenderPass" family = "renderPass" icon = "cube" defaults = ["Main"] From 2d375f092c1df316f633bcb3f68e8f4cab401e6b Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 6 Nov 2020 15:20:59 +0100 Subject: [PATCH 037/147] feat(SP): wip publishing editorial with dir of subsets and files --- .../publish/collect_clear_instances.py | 20 +++ .../publish/collect_instance_resources.py | 151 +++++++++++++++++- .../publish/collect_instances.py | 10 +- .../publish/validate_editorial_resources.py | 5 +- 4 files changed, 173 insertions(+), 13 deletions(-) create mode 100644 pype/plugins/standalonepublisher/publish/collect_clear_instances.py diff --git a/pype/plugins/standalonepublisher/publish/collect_clear_instances.py b/pype/plugins/standalonepublisher/publish/collect_clear_instances.py new file mode 100644 index 0000000000..0d64c57d3a --- /dev/null +++ b/pype/plugins/standalonepublisher/publish/collect_clear_instances.py @@ -0,0 +1,20 @@ +""" +Optional: + instance.data["remove"] -> mareker for removing +""" +import pyblish.api + + +class CollectClearInstances(pyblish.api.ContextPlugin): + """Clear all marked instances""" + + order = pyblish.api.CollectorOrder + 0.4999 + label = "Clear Instances" + hosts = ["standalonepublisher"] + + def process(self, context): + + for instance in context: + if instance.data.get("remove"): + self.log.info(f"Removing: {instance}") + context.remove(instance) diff --git a/pype/plugins/standalonepublisher/publish/collect_instance_resources.py b/pype/plugins/standalonepublisher/publish/collect_instance_resources.py index 63b98f2721..25bdffd422 100644 --- a/pype/plugins/standalonepublisher/publish/collect_instance_resources.py +++ b/pype/plugins/standalonepublisher/publish/collect_instance_resources.py @@ -2,6 +2,8 @@ import os import tempfile import pyblish.api from copy import deepcopy +import clique + class CollectInstanceResources(pyblish.api.InstancePlugin): """Collect instance's resources""" @@ -13,6 +15,10 @@ class CollectInstanceResources(pyblish.api.InstancePlugin): families = ["clip"] def process(self, instance): + context = instance.context + self.log.info(f"Processing instance: {instance}") + subset_files = dict() + subset_dirs = list() anatomy = instance.context.data["anatomy"] anatomy_data = deepcopy(instance.context.data["anatomyData"]) anatomy_data.update({"root": anatomy.roots}) @@ -23,6 +29,7 @@ class CollectInstanceResources(pyblish.api.InstancePlugin): editorial_source_root = instance.data["editorialSourceRoot"] editorial_source_path = instance.data["editorialSourcePath"] + # if `editorial_source_path` then loop trough if editorial_source_path: # add family if mov or mp4 found which is longer for # cutting `trimming` to enable `ExtractTrimmingVideoAudio` plugin @@ -33,23 +40,153 @@ class CollectInstanceResources(pyblish.api.InstancePlugin): instance.data["families"] += ["trimming"] return + # if template patern in path then fill it with `anatomy_data` if "{" in editorial_source_root: editorial_source_root = editorial_source_root.format( **anatomy_data) self.log.debug(f"root: {editorial_source_root}") - + # loop `editorial_source_root` and find clip name in folders + # and look for any subset name alternatives for root, dirs, files in os.walk(editorial_source_root): - if subset in root and clip_name in root: - staging_dir = root + correct_clip_dir = None + for d in dirs: + # avoid all non clip dirs + if d not in clip_name: + continue + # found correct dir for clip + correct_clip_dir = d - self.log.debug(f"staging_dir: {staging_dir}") + # continue if clip dir was not found + if not correct_clip_dir: + continue + clip_dir_path = os.path.join(root, correct_clip_dir) + subset_files_items = list() + # list content of clip dir and search for subset items + for subset_item in os.listdir(clip_dir_path): + # avoid all items which are not defined as subsets by name + if subset not in subset_item: + continue - # add `editorialSourceRoot` as staging dir + subset_item_path = os.path.join( + clip_dir_path, subset_item) + # if it is dir store it to `subset_dirs` list + if os.path.isdir(subset_item_path): + subset_dirs.append(subset_item_path) - # if `editorialSourcePath` is none then loop - # trough `editorialSourceRoot` + # if it is file then store it to `subset_files` list + if os.path.isfile(subset_item_path): + subset_files_items.append(subset_item_path) + + if subset_files_items: + subset_files.update({clip_dir_path: subset_files_items}) + if correct_clip_dir: + break + + if subset_dirs: + # look all dirs and check for subset name alternatives + copy_instance_data = deepcopy( + {_k: _v for _k, _v in instance.data.items()}) + + # find next available precise subset name with comprahantion + subset_dir_found = next( + (d for d in subset_dirs + if os.path.basename(d) in subset), + None) + + if not subset_dir_found: + instance.data["remove"] = True + + for _dir in subset_dirs: + sub_dir = os.path.basename(_dir) + instance_data = instance.data + # if subset name is only alternative then create new instance + if sub_dir != subset: + new_instance_data = dict() + for _key, _value in copy_instance_data.items(): + new_instance_data[_key] = _value + if not isinstance(_value, str): + continue + if subset in _value: + new_instance_data[_key] = _value.replace( + subset, sub_dir) + new_instance = context.create_instance( + new_instance_data["name"]) + new_instance.data.update(new_instance_data) + self.log.info(f"Creating new instance: {new_instance}") + instance_data = new_instance.data + + staging_dir = _dir + files = os.listdir(_dir) + collections, remainder = clique.assemble(files) + # self.log.debug(f"collections: {collections}") + # self.log.debug(f"remainder: {remainder}") + # self.log.debug(f"staging_dir: {staging_dir}") + + # add staging_dir to instance_data + instance_data["stagingDir"] = staging_dir + # add representations to instance_data + instance_data["representations"] = list() + + # loop trough collections and create representations + for _collection in collections: + ext = _collection.tail + repre_data = { + "name": ext[1:], + "ext": ext[1:], + "files": [item for item in _collection], + "stagingDir": staging_dir + } + instance_data["representations"].append(repre_data) + + # loop trough reminders and create representations + for _reminding_file in remainder: + ext = os.path.splitext(_reminding_file)[-1] + if ext not in instance_data["extensions"]: + continue + + repre_data = { + "name": ext[1:], + "ext": ext[1:], + "files": _reminding_file, + "stagingDir": staging_dir + } + + # exception for thumbnail + if "thumb" in _reminding_file: + repre_data.update({ + 'name': "thumbnail", + 'thumbnail': True + }) + + # exception for mp4 preview + if ".mp4" in _reminding_file: + frame_start = instance_data["frameStart"] + frame_end = instance_data["frameEnd"] + instance_data["families"].append("review") + repre_data.update({ + "frameStart": 0, + "frameEnd": (frame_end - frame_start) + 1, + "frameStartFtrack": 0, + "frameEndFtrack": (frame_end - frame_start) + 1, + "step": 1, + "fps": context.data.get("fps"), + "name": "review", + "tags": ["review", "ftrackreview"], + }) + + instance_data["representations"].append(repre_data) + + representations = instance_data["representations"] + self.log.debug(f">>>_<<< representations: {representations}") + + if subset_files: + staging_dir = list(subset_files.keys()).pop() + collections, remainder = clique.assemble(subset_files[staging_dir]) + # self.log.debug(f"collections: {collections}") + # self.log.debug(f"remainder: {remainder}") + # self.log.debug(f"staging_dir: {staging_dir}") # if image sequence then create representation > match # with subset name in dict diff --git a/pype/plugins/standalonepublisher/publish/collect_instances.py b/pype/plugins/standalonepublisher/publish/collect_instances.py index 9cd8d9f36c..3d577c1527 100644 --- a/pype/plugins/standalonepublisher/publish/collect_instances.py +++ b/pype/plugins/standalonepublisher/publish/collect_instances.py @@ -69,7 +69,6 @@ class CollectInstances(pyblish.api.InstancePlugin): handle_start = int(asset_data["handleStart"]) handle_end = int(asset_data["handleEnd"]) - instances = [] for track in tracks: try: track_start_frame = ( @@ -179,12 +178,13 @@ class CollectInstances(pyblish.api.InstancePlugin): subset_instance_data.update(properities) subset_instance_data.update({ # unique attributes - "name": f"{subset}_{name}", - "label": f"{subset} {name} ({clip_in}-{clip_out})", + "name": f"{name}_{subset}", + "label": f"{name} {subset} ({clip_in}-{clip_out})", "subset": subset }) - instances.append(instance.context.create_instance( - **subset_instance_data)) + # create new instance + instance.context.create_instance( + **subset_instance_data) context.data["assetsShared"][name] = { "_clipIn": clip_in, diff --git a/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py b/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py index 65581a6cdc..0dfca92f66 100644 --- a/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py +++ b/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py @@ -7,7 +7,10 @@ class ValidateEditorialResources(pyblish.api.InstancePlugin): label = "Validate Editorial Resources" hosts = ["standalonepublisher"] - families = ["clip"] + families = ["clip", "trimming"] + + # make sure it is enabled only if at least both families are available + match = pyblish.api.Subset order = pype.api.ValidateContentsOrder From 739baf70f5707316bb8a007f0d21c2ab4e859076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Fri, 6 Nov 2020 16:19:29 +0100 Subject: [PATCH 038/147] submit to deadline functionality --- pype/lib/abstract_submit_deadline.py | 258 ++++++++++++++++++++++++--- 1 file changed, 238 insertions(+), 20 deletions(-) diff --git a/pype/lib/abstract_submit_deadline.py b/pype/lib/abstract_submit_deadline.py index dede4c19ba..26ea66f5ce 100644 --- a/pype/lib/abstract_submit_deadline.py +++ b/pype/lib/abstract_submit_deadline.py @@ -1,9 +1,14 @@ # -*- coding: utf-8 -*- -"""Abstract class for submitting jobs to Deadline.""" +"""Abstract package for submitting jobs to Deadline. + +It provides Deadline JobInfo data class. + +""" import os from abc import ABCMeta, abstractmethod import platform import getpass +from collections import OrderedDict import six import attr @@ -171,13 +176,13 @@ class DeadlineJobInfo: """Return all environment key values formatted for Deadline. Returns: - list of tuples: as `[('EnvironmentKeyValue0', 'key=value')]` + dict: as `{'EnvironmentKeyValue0', 'key=value'}` """ out = [] index = 0 for v in self._environmentKeyValue: - out.append(("EnvironmentKeyValue{}".format(index), v)) + out["EnvironmentKeyValue{}".format(index)] = v index += 1 return out @@ -199,13 +204,13 @@ class DeadlineJobInfo: """Return all ExtraInfo values formatted for Deadline. Returns: - list of tuples: as `[('ExtraInfo0', 'value')]` + dict: as `{'ExtraInfo0': 'value'}` """ out = [] index = 0 for v in self._extraInfos: - out.append(("ExtraInfo{}".format(index), v)) + out["ExtraInfo{}".format(index)] = v index += 1 return out @@ -218,13 +223,13 @@ class DeadlineJobInfo: """Return all ExtraInfoKeyValue values formatted for Deadline. Returns: - list of tuples: as `[('ExtraInfoKeyValue0', 'key=value')]` + dict: as {'ExtraInfoKeyValue0': 'key=value'}` """ out = [] index = 0 for v in self._extraInfoKeyValues: - out.append(("ExtraInfoKeyValue{}".format(index), v)) + out["ExtraInfoKeyValue{}".format(index)] = v index += 1 return out @@ -242,13 +247,13 @@ class DeadlineJobInfo: """Return all TaskExtraInfoName values formatted for Deadline. Returns: - list of tuples: as `[('TaskExtraInfoName0', 'value')]` + dict: as `{'TaskExtraInfoName0': 'value'}` """ out = [] index = 0 for v in self._taskExtraInfos: - out.append(("TaskExtraInfoName{}".format(index), v)) + out["TaskExtraInfoName{}".format(index)] = v index += 1 return out @@ -267,13 +272,13 @@ class DeadlineJobInfo: """Return all OutputFilename values formatted for Deadline. Returns: - list of tuples: as `[('OutputFilename0', 'filename')]` + dict: as `{'OutputFilename0': 'filename'}` """ out = [] index = 0 for v in self._outputFilename: - out.append(("OutputFilename{}".format(index), v)) + out["OutputFilename{}".format(index)] = v index += 1 return out @@ -286,13 +291,13 @@ class DeadlineJobInfo: """Return all OutputFilename#Tile values formatted for Deadline. Returns: - list of tuples: as `[('OutputFilename#Tile', 'tile')]` + dict: as `{'OutputFilenme#Tile': 'tile'}` """ out = [] index = 0 for v in self._outputFilenameTile: - out.append(("OutputFilename{}Tile".format(index), v)) + out["OutputFilename{}Tile".format(index)] = v index += 1 return out @@ -305,13 +310,13 @@ class DeadlineJobInfo: """Return all OutputDirectory values formatted for Deadline. Returns: - list of tuples: as `[('OutputDirectory0', 'dir')]` + dict: as `{'OutputDirectory0': 'dir'}` """ out = [] index = 0 for v in self._outputDirectory: - out.append(("OutputDirectory{}".format(index), v)) + out["OutputDirectory{}".format(index)] = v index += 1 return out @@ -333,21 +338,234 @@ class DeadlineJobInfo: MaintenanceJobStartFrame = attr.ib(default=None) # Default: 0 MaintenanceJobEndFrame = attr.ib(default=None) # Default: 0 + def render(self): + """Return all data serialized as dictionary. -@attr.s -class DeadlinePluginInfo: - SceneFile = attr.ib() + Returns: + OrderedDict: all serialized data. + + """ + def no_privates(a, _): + return not a.name.startswith("_") + + serialized = attr.asdict( + self, dict_factory=OrderedDict, filter=no_privates) + serialized.update(self.EnvironmentKeyValue) + serialized.update(self.ExtraInfo) + serialized.update(self.ExtraInfoKeyValue) + serialized.update(self.TaskExtraInfoName) + serialized.update(self.OutputFilename) + serialized.update(self.OutputFilenameTile) + serialized.update(self.OutputDirectory) + return serialized @six.add_metaclass(ABCMeta) class AbstractSubmitDeadline(pyblish.api.InstancePlugin): + """Class abstracting access to Deadline.""" label = "Submit to Deadline" order = pyblish.api.IntegratorOrder + 0.1 use_published = True asset_dependencies = False + def process(self, instance): + """Plugin entry point.""" + self._instance = instance + context = instance.context + self._deadline_url = os.environ.get( + "DEADLINE_REST_URL", "http://localhost:8082") + assert self._deadline_url, "Requires DEADLINE_REST_URL" + + file_path = None + if self.use_published: + file_path = self.from_published_scene() + + # fallback if nothing was set + if not file_path: + self.log.warning("Falling back to workfile") + file_path = context.data["currentFile"] + + self.scene_path = file_path + self.log.info("Using {} for render/export.".format(file_path)) + + self.job_info = self.get_job_info() + self.plugin_info = self.get_plugin_info() + self.aux_files = self.get_aux_files() + + def process_submission(self): + """Process data for submission. + + This takes Deadline JobInfo, PluginInfo, AuxFile, creates payload + from them and submit it do Deadline. + + Returns: + str: Deadline job ID + + """ + payload = self.assemble_payload() + return self.submit(payload) + + @abstractmethod + def get_job_info(self): + """Return filled Deadline JobInfo. + + This is host/plugin specific implementation of how to fill data in. + + See: + :class:`DeadlineJobInfo` + + Returns: + dict: Filled Deadline JobInfo. + + """ + pass + + @abstractmethod + def get_plugin_info(self): + """Return filled Deadline PluginInfo. + + This is host/plugin specific implementation of how to fill data in. + + See: + :class:`DeadlineJobInfo` + + Returns: + dict: Filled Deadline JobInfo. + + """ + pass + + def get_aux_files(self): + """Return list of auxiliary files for Deadline job. + + If needed this should be overriden, otherwise return empty list as + that field even empty must be present on Deadline submission. + + Returns: + list: List of files. + + """ + return [] + + def from_published_scene(self, replace_in_path=True): + """Switch work scene for published scene. + + If rendering/exporting from published scenes is enabled, this will + replace paths from working scene to published scene. + + Args: + replace_in_path (bool): if True, it will try to find + old scene name in path of expected files and replace it + with name of published scene. + + Returns: + str: Published scene path. + + Note: + Published scene path is actually determined from project Anatomy + as at the time this plugin is running scene can still no be + published. + + """ + anatomy = self._instance.context.data['anatomy'] + for i in self._instance.context: + if "workfile" in i.data["families"]: + # test if there is instance of workfile waiting + # to be published. + assert i.data["publish"] is True, ( + "Workfile (scene) must be published along") + # determine published path from Anatomy. + template_data = i.data.get("anatomyData") + rep = i.data.get("representations")[0].get("name") + template_data["representation"] = rep + template_data["ext"] = rep + template_data["comment"] = None + anatomy_filled = anatomy.format(template_data) + template_filled = anatomy_filled["publish"]["path"] + filepath = os.path.normpath(template_filled) + + self.log.info("Using published scene for render {}".format( + filepath)) + + if not os.path.exists(filepath): + self.log.error("published scene does not exist!") + raise + + if not replace_in_path: + return filepath + + # now we need to switch scene in expected files + # because token will now point to published + # scene file and that might differ from current one + new_scene = os.path.splitext( + os.path.basename(filepath))[0] + orig_scene = os.path.splitext( + os.path.basename( + self._instance.context.data["currentFile"]))[0] + exp = self._instance.data.get("expectedFiles") + + if isinstance(exp[0], dict): + # we have aovs and we need to iterate over them + new_exp = {} + for aov, files in exp[0].items(): + replaced_files = [] + for f in files: + replaced_files.append( + f.replace(orig_scene, new_scene) + ) + new_exp[aov] = replaced_files + self._instance.data["expectedFiles"] = [new_exp] + else: + new_exp = [] + for f in exp: + new_exp.append( + f.replace(orig_scene, new_scene) + ) + self._instance.data["expectedFiles"] = [new_exp] + self.log.info("Scene name was switched {} -> {}".format( + orig_scene, new_scene + )) + + return filepath + + def assemble_payload( + self, job_info=None, plugin_info=None, aux_files=None): + """Assemble payload data from its various parts. + + Args: + job_info (dict): Deadline JobInfo. You can use + :class:`DeadlineJobInfo` for it. + plugin_info (dict): Deadline PluginInfo. Plugin specific options. + aux_files (list, optional): List of auxiliary file to submit with + the job. + + Returns: + dict: Deadline Payload. + + """ + return { + "JobInfo": job_info or self.job_info, + "PluginInfo": plugin_info or self.plugin_info, + "AuxFiles": aux_files or self.aux_files + } + def submit(self, payload): + """Submit payload to Deadline API end-point. + + This takes payload in the form of JSON file and POST it to + Deadline jobs end-point. + + Args: + payload (str): string encoded json with job payload. + + Returns: + str: resulting Deadline job id. + + Throws: + RuntimeError: if submission fails. + + """ url = "{}/api/jobs".format(self._deadline_url) response = self._requests_post(url, json=payload) if not response.ok: @@ -357,8 +575,8 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): self.log.debug(payload) raise RuntimeError(response.text) - dependency = response.json() - return dependency["_id"] + result = response.json() + return result["_id"] def _requests_post(self, *args, **kwargs): """Wrap request post method. From cdfbb496900aef8afef47439dddd67815d22c8dc Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 6 Nov 2020 16:23:17 +0100 Subject: [PATCH 039/147] render layer's templete does not contain task and pass --- pype/plugins/tvpaint/create/create_render_layer.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pype/plugins/tvpaint/create/create_render_layer.py b/pype/plugins/tvpaint/create/create_render_layer.py index 10ac444a80..a9e20291ad 100644 --- a/pype/plugins/tvpaint/create/create_render_layer.py +++ b/pype/plugins/tvpaint/create/create_render_layer.py @@ -1,4 +1,3 @@ -import avalon.io from avalon.tvpaint import pipeline, lib @@ -12,7 +11,7 @@ class CreateBeauty(pipeline.Creator): rename_group = True - subset_template = "{family}{task}_{name}_{pass}" + subset_template = "{family}_{name}" rename_script_template = ( "tv_layercolor \"setcolor\"" " {clip_id} {group_id} {r} {g} {b} \"{name}\"" @@ -57,10 +56,7 @@ class CreateBeauty(pipeline.Creator): # Change subset name by template subset_name = self.subset_template.format(**{ "family": self.family, - # Should be task name capitalized? - "task": avalon.io.Session["AVALON_TASK"], - "name": name, - "pass": "beauty" + "name": name }) self.log.info(f"New subset name \"{subset_name}\".") self.data["subset"] = subset_name From eb0cc85dd85c54bddd269907ced4c1904af059cd Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 6 Nov 2020 16:24:56 +0100 Subject: [PATCH 040/147] render pass is not dependent on render layer subset name --- .../tvpaint/create/create_render_pass.py | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/pype/plugins/tvpaint/create/create_render_pass.py b/pype/plugins/tvpaint/create/create_render_pass.py index f6343a2b0b..cde8f3880e 100644 --- a/pype/plugins/tvpaint/create/create_render_pass.py +++ b/pype/plugins/tvpaint/create/create_render_pass.py @@ -13,7 +13,7 @@ class CreateRenderPass(pipeline.Creator): icon = "cube" defaults = ["Main"] - beauty_pass_name = "beauty" + subset_template = "{family}_{render_layer}_{pass}" def process(self): self.log.debug("Query data from workfile.") @@ -54,6 +54,8 @@ class CreateRenderPass(pipeline.Creator): if beauty_instance is None: raise AssertionError("Beauty pass does not exist yet.") + render_layer = beauty_instance["name"] + # Extract entered name family = self.data["family"] name = self.data["subset"] @@ -68,21 +70,12 @@ class CreateRenderPass(pipeline.Creator): layer_ids = [layer["layer_id"] for layer in selected_layers] self.data["layer_ids"] = layer_ids - beauty_subset_name = beauty_instance["subset"] - self.log.info( - "New subset name will be created from " - f"beauty instance \"{beauty_subset_name}\"." - ) - - # Beauty instance subset name should - if not beauty_subset_name.endswith(self.beauty_pass_name): - raise AssertionError( - "BUG: Beauty subset name does not end with \"{}\"".format( - self.beauty_pass_name - ) - ) # Replace `beauty` in beauty's subset name with entered name - subset_name = beauty_subset_name[:-len(self.beauty_pass_name)] + name + subset_name = self.subset_template.format(**{ + "family": family, + "render_layer": render_layer, + "pass": name + }) self.data["subset"] = subset_name self.log.info(f"New subset name is \"{subset_name}\".") From fd8d710f7eff7694cf8a4fee851df117358eedc8 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 6 Nov 2020 16:26:58 +0100 Subject: [PATCH 041/147] modified how instance labels are shown in publish gui --- pype/plugins/tvpaint/create/create_render_pass.py | 2 +- pype/plugins/tvpaint/publish/collect_instances.py | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pype/plugins/tvpaint/create/create_render_pass.py b/pype/plugins/tvpaint/create/create_render_pass.py index cde8f3880e..4756fe2413 100644 --- a/pype/plugins/tvpaint/create/create_render_pass.py +++ b/pype/plugins/tvpaint/create/create_render_pass.py @@ -64,7 +64,7 @@ class CreateRenderPass(pipeline.Creator): self.log.info(f"Extracted name from subset name \"{name}\".") self.data["group_id"] = group_id - self.data["name"] = name + self.data["pass"] = name # Collect selected layer ids to be stored into instance layer_ids = [layer["layer_id"] for layer in selected_layers] diff --git a/pype/plugins/tvpaint/publish/collect_instances.py b/pype/plugins/tvpaint/publish/collect_instances.py index 84fc7688f7..6dc934d6c6 100644 --- a/pype/plugins/tvpaint/publish/collect_instances.py +++ b/pype/plugins/tvpaint/publish/collect_instances.py @@ -51,12 +51,11 @@ class CollectInstances(pyblish.api.ContextPlugin): )) def create_render_layer(self, context, instance_data): + name = instance_data["name"] + instance_data["label"] = "{}_beauty".format(name) + layers_data = context.data["layersData"] group_id = instance_data["group_id"] - - name = instance_data["name"] - instance_data["label"] = name - group_layers = [] for layer in layers_data: if layer["group_id"] == group_id and layer["visible"]: @@ -74,6 +73,10 @@ class CollectInstances(pyblish.api.ContextPlugin): return context.create_instance(**instance_data) def create_render_pass(self, context, instance_data): + pass_name = instance_data["pass"] + render_layer = instance_data["render_layer"] + instance_data["label"] = "{}_{}".format(render_layer, pass_name) + # Change family to `renderLayer` instance_data["family"] = "renderLayer" instance_data["families"] = [instance_data["family"]] From 70a1ff08938ef350db61e07e01577a3fbca47c1a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 6 Nov 2020 16:27:09 +0100 Subject: [PATCH 042/147] add origin instance data to instance --- pype/plugins/tvpaint/publish/collect_instances.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pype/plugins/tvpaint/publish/collect_instances.py b/pype/plugins/tvpaint/publish/collect_instances.py index 6dc934d6c6..8491c77b87 100644 --- a/pype/plugins/tvpaint/publish/collect_instances.py +++ b/pype/plugins/tvpaint/publish/collect_instances.py @@ -1,4 +1,5 @@ import json +import copy import pyblish.api @@ -18,6 +19,8 @@ class CollectInstances(pyblish.api.ContextPlugin): # TODO add validations of existing instances # - layer id exists for instance_data in workfile_instances: + # Store workfile instance data to instance data + instance_data["originData"] = copy.deepcopy(instance_data) # Global instance data modifications # Fill families family = instance_data["family"] From 411b33f7bfecedafa7a70d8dbdac461f1864d1f8 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 6 Nov 2020 16:32:00 +0100 Subject: [PATCH 043/147] remove already done comment --- pype/plugins/tvpaint/publish/collect_instances.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pype/plugins/tvpaint/publish/collect_instances.py b/pype/plugins/tvpaint/publish/collect_instances.py index 8491c77b87..b5e8f941ba 100644 --- a/pype/plugins/tvpaint/publish/collect_instances.py +++ b/pype/plugins/tvpaint/publish/collect_instances.py @@ -16,8 +16,6 @@ class CollectInstances(pyblish.api.ContextPlugin): json.dumps(workfile_instances, indent=4) )) - # TODO add validations of existing instances - # - layer id exists for instance_data in workfile_instances: # Store workfile instance data to instance data instance_data["originData"] = copy.deepcopy(instance_data) From e3de8198d171313e637d87d54958ee5c41e5e91f Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 6 Nov 2020 17:04:15 +0100 Subject: [PATCH 044/147] fixed render pass creator --- pype/plugins/tvpaint/create/create_render_pass.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pype/plugins/tvpaint/create/create_render_pass.py b/pype/plugins/tvpaint/create/create_render_pass.py index 4756fe2413..585da6ec00 100644 --- a/pype/plugins/tvpaint/create/create_render_pass.py +++ b/pype/plugins/tvpaint/create/create_render_pass.py @@ -65,6 +65,7 @@ class CreateRenderPass(pipeline.Creator): self.data["group_id"] = group_id self.data["pass"] = name + self.data["render_layer"] = render_layer # Collect selected layer ids to be stored into instance layer_ids = [layer["layer_id"] for layer in selected_layers] @@ -86,7 +87,7 @@ class CreateRenderPass(pipeline.Creator): if ( instance["family"] == family and instance["group_id"] == group_id - and instance["name"] == name + and instance["pass"] == name ): existing_instance = instance existing_instance_idx = idx From ba5324fc2aef687a3fe6794b6235675552dc6530 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 6 Nov 2020 17:19:09 +0100 Subject: [PATCH 045/147] added check for same subset name existence --- .../tvpaint/create/create_render_layer.py | 44 ++++++++++++++++--- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/pype/plugins/tvpaint/create/create_render_layer.py b/pype/plugins/tvpaint/create/create_render_layer.py index a9e20291ad..f261466681 100644 --- a/pype/plugins/tvpaint/create/create_render_layer.py +++ b/pype/plugins/tvpaint/create/create_render_layer.py @@ -61,18 +61,33 @@ class CreateBeauty(pipeline.Creator): self.log.info(f"New subset name \"{subset_name}\".") self.data["subset"] = subset_name - # Check for instances for same group + # Check for instances of same group existing_instance = None existing_instance_idx = None + # Check if subset name is not already taken + same_subset_instance = None + same_subset_instance_idx = None for idx, instance in enumerate(instances): + if instance["family"] == family: + if instance["group_id"] == group_id: + existing_instance = instance + existing_instance_idx = idx + elif instance["subset"] == subset_name: + same_subset_instance = instance + same_subset_instance_idx = idx + if ( - instance["family"] == family - and instance["group_id"] == group_id + same_subset_instance_idx is not None + and existing_instance_idx is not None ): - existing_instance = instance - existing_instance_idx = idx break + if same_subset_instance_idx is not None: + if self._ask_user_subset_override(same_subset_instance): + instances.pop(same_subset_instance_idx) + else: + return + if existing_instance is not None: self.log.info( f"Beauty instance for group id {group_id} already exists" @@ -114,3 +129,22 @@ class CreateBeauty(pipeline.Creator): f"Name of group with index {group_id}" f" was changed to \"{new_group_name}\"." ) + + def _ask_user_subset_override(self, instance): + from Qt.QtWidgets import QMessageBox + + title = "Subset \"{}\" already exist".format(instance["subset"]) + text = ( + "Instance with subset name \"{}\" already exists." + "\n\nDo you want to override existing?" + ).format(instance["subset"]) + + dialog = QMessageBox() + dialog.setWindowTitle(title) + dialog.setText(text) + dialog.setStandardButtons(QMessageBox.Yes | QMessageBox.No) + dialog.setDefaultButton(QMessageBox.Yes) + dialog.exec_() + if dialog.result() == QMessageBox.Yes: + return True + return False From 79b1f5c1f82289b4a7adccebc0dc3afa8f304db8 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 6 Nov 2020 17:46:55 +0100 Subject: [PATCH 046/147] Added loader with containerization --- .../tvpaint/load/load_reference_image.py | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 pype/plugins/tvpaint/load/load_reference_image.py diff --git a/pype/plugins/tvpaint/load/load_reference_image.py b/pype/plugins/tvpaint/load/load_reference_image.py new file mode 100644 index 0000000000..3cba8d4a1a --- /dev/null +++ b/pype/plugins/tvpaint/load/load_reference_image.py @@ -0,0 +1,96 @@ +from avalon.vendor import qargparse +from avalon.tvpaint import lib, pipeline + + +class LoadImage(pipeline.Loader): + """Load image or image sequence to TVPaint as new layer.""" + + families = ["render", "image", "background", "plate"] + representations = ["*"] + + label = "Load Image" + order = 1 + icon = "image" + color = "white" + + import_script = ( + "filepath = \"{}\"\n" + "layer_name = \"{}\"\n" + "tv_loadsequence filepath {}PARSE layer_id\n" + "tv_layerrename layer_id layer_name" + ) + + defaults = { + "stretch": True, + "timestretch": True, + "preload": True + } + + options = [ + qargparse.Boolean( + "stretch", + label="Stretch to project size", + default=True, + help="Stretch loaded image/s to project resolution?" + ), + qargparse.Boolean( + "timestretch", + label="Stretch to timeline length", + default=True, + help="Clip loaded image/s to timeline length?" + ), + qargparse.Boolean( + "preload", + label="Preload loaded image/s", + default=True, + help="Preload image/s?" + ) + ] + + def load(self, context, name, namespace, options): + stretch = options.get("stretch", self.defaults["stretch"]) + timestretch = options.get("timestretch", self.defaults["timestretch"]) + preload = options.get("preload", self.defaults["preload"]) + + load_options = [] + if stretch: + load_options.append("\"STRETCH\"") + if timestretch: + load_options.append("\"TIMESTRETCH\"") + if preload: + load_options.append("\"PRELOAD\"") + + load_options_str = "" + for load_option in load_options: + load_options_str += (load_option + " ") + + # Prepare layer name + asset_name = context["asset"]["name"] + subset_name = context["subset"]["name"] + layer_name = self.get_unique_layer_name(asset_name, subset_name) + + # Fill import script with filename and layer name + # - filename mus not contain backwards slashes + george_script = self.import_script.format( + self.fname.replace("\\", "/"), + layer_name, + load_options_str + ) + + lib.execute_george_through_file(george_script) + + loaded_layer = None + layers = lib.layers_data() + for layer in layers: + if layer["name"] == layer_name: + loaded_layer = layer + break + + if loaded_layer is None: + raise AssertionError( + "Loading probably failed during execution of george script." + ) + + layer_ids = [loaded_layer["layer_id"]] + namespace = namespace or layer_name + return pipeline.containerise(name, namespace, layer_ids, context, self) From a9cd97f257d1bf58d2cd388190666af2cb90c003 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 6 Nov 2020 18:04:54 +0100 Subject: [PATCH 047/147] implemented remove method --- .../tvpaint/load/load_reference_image.py | 57 ++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/pype/plugins/tvpaint/load/load_reference_image.py b/pype/plugins/tvpaint/load/load_reference_image.py index 3cba8d4a1a..06bf1313f0 100644 --- a/pype/plugins/tvpaint/load/load_reference_image.py +++ b/pype/plugins/tvpaint/load/load_reference_image.py @@ -93,4 +93,59 @@ class LoadImage(pipeline.Loader): layer_ids = [loaded_layer["layer_id"]] namespace = namespace or layer_name - return pipeline.containerise(name, namespace, layer_ids, context, self) + return pipeline.containerise( + name=name, + namespace=namespace, + layer_ids=layer_ids, + context=context, + loader=self.__class__.__name__ + ) + + def _remove_layers(self, layers, layer_ids): + if not layer_ids: + return + + available_ids = set(layer["layer_id"] for layer in layers) + layer_ids_to_remove = [] + + for layer_id in layer_ids: + if layer_id in available_ids: + layer_ids_to_remove.append(layer_id) + + if not layer_ids_to_remove: + return + + george_script_lines = [] + for layer_id in layer_ids_to_remove: + line = "tv_layerkill {}".format(layer_id) + george_script_lines.append(line) + george_script = "\n".join(george_script_lines) + lib.execute_george_through_file(george_script) + + def remove(self, container): + layer_ids_str = container["objectName"] + layer_ids = [int(layer_id) for layer_id in layer_ids_str.split("|")] + + layers = lib.layers_data() + self._remove_layers(layers, layer_ids) + + current_containers = pipeline.ls() + pop_idx = None + for idx, cur_con in enumerate(current_containers): + if cur_con["objectName"] == layer_ids_str: + pop_idx = idx + break + + if pop_idx is None: + self.log.warning( + "Didn't found container in workfile containers. {}".format( + container + ) + ) + return + + current_containers.pop(pop_idx) + pipeline.write_workfile_metadata( + pipeline.SECTION_NAME_CONTAINERS, current_containers + ) + From f7337c9913523d511775892a71c8f26cc50bde97 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 6 Nov 2020 18:12:48 +0100 Subject: [PATCH 048/147] simplified logic --- pype/plugins/tvpaint/load/load_reference_image.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pype/plugins/tvpaint/load/load_reference_image.py b/pype/plugins/tvpaint/load/load_reference_image.py index 06bf1313f0..e3a224c283 100644 --- a/pype/plugins/tvpaint/load/load_reference_image.py +++ b/pype/plugins/tvpaint/load/load_reference_image.py @@ -101,10 +101,13 @@ class LoadImage(pipeline.Loader): loader=self.__class__.__name__ ) - def _remove_layers(self, layers, layer_ids): + def _remove_layers(self, layer_ids, layers=None): if not layer_ids: return + if layers is None: + layers = lib.layers_data() + available_ids = set(layer["layer_id"] for layer in layers) layer_ids_to_remove = [] @@ -123,16 +126,13 @@ class LoadImage(pipeline.Loader): lib.execute_george_through_file(george_script) def remove(self, container): - layer_ids_str = container["objectName"] - layer_ids = [int(layer_id) for layer_id in layer_ids_str.split("|")] - - layers = lib.layers_data() - self._remove_layers(layers, layer_ids) + layer_ids = self.layer_ids_from_container(container) + self._remove_layers(layer_ids) current_containers = pipeline.ls() pop_idx = None for idx, cur_con in enumerate(current_containers): - if cur_con["objectName"] == layer_ids_str: + if cur_con["objectName"] == container["objectName"]: pop_idx = idx break From e68ce589b5e076c351a81cf41ac5b74597ee405a Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 6 Nov 2020 18:41:36 +0100 Subject: [PATCH 049/147] feat(SP): final publishing editorial with image sequence and other --- .../publish/collect_hierarchy.py | 9 +- .../publish/collect_instance_resources.py | 241 ++++++++++-------- 2 files changed, 147 insertions(+), 103 deletions(-) diff --git a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py index 6ce6232943..3d9465cb1a 100644 --- a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py +++ b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py @@ -128,7 +128,14 @@ class CollectHierarchyInstance(pyblish.api.ContextPlugin): self.log.debug(f"parents: {parents}") if self.shot_add_tasks: - instance.data["tasks"] = self.shot_add_tasks + tasks_to_add = dict() + project_tasks = io.find_one({"type": "project"})["config"]["tasks"] + for task in self.shot_add_tasks: + for task_type in project_tasks.keys(): + if task_type.lower() in task.lower(): + tasks_to_add.update({task: {"type": task_type}}) + + instance.data["tasks"] = tasks_to_add else: instance.data["tasks"] = list() diff --git a/pype/plugins/standalonepublisher/publish/collect_instance_resources.py b/pype/plugins/standalonepublisher/publish/collect_instance_resources.py index 25bdffd422..6807e82193 100644 --- a/pype/plugins/standalonepublisher/publish/collect_instance_resources.py +++ b/pype/plugins/standalonepublisher/publish/collect_instance_resources.py @@ -1,4 +1,5 @@ import os +import re import tempfile import pyblish.api from copy import deepcopy @@ -15,12 +16,13 @@ class CollectInstanceResources(pyblish.api.InstancePlugin): families = ["clip"] def process(self, instance): - context = instance.context + self.context = instance.context + instance_data = instance.data self.log.info(f"Processing instance: {instance}") subset_files = dict() subset_dirs = list() - anatomy = instance.context.data["anatomy"] - anatomy_data = deepcopy(instance.context.data["anatomyData"]) + anatomy = self.context.data["anatomy"] + anatomy_data = deepcopy(self.context.data["anatomyData"]) anatomy_data.update({"root": anatomy.roots}) subset = instance.data["subset"] @@ -36,8 +38,8 @@ class CollectInstanceResources(pyblish.api.InstancePlugin): staging_dir = os.path.normpath( tempfile.mkdtemp(prefix="pyblish_tmp_") ) - instance.data["stagingDir"] = staging_dir - instance.data["families"] += ["trimming"] + instance_data["stagingDir"] = staging_dir + instance_data["families"] += ["trimming"] return # if template patern in path then fill it with `anatomy_data` @@ -86,109 +88,144 @@ class CollectInstanceResources(pyblish.api.InstancePlugin): if subset_dirs: # look all dirs and check for subset name alternatives - copy_instance_data = deepcopy( - {_k: _v for _k, _v in instance.data.items()}) - - # find next available precise subset name with comprahantion - subset_dir_found = next( - (d for d in subset_dirs - if os.path.basename(d) in subset), - None) - - if not subset_dir_found: - instance.data["remove"] = True - for _dir in subset_dirs: sub_dir = os.path.basename(_dir) - instance_data = instance.data # if subset name is only alternative then create new instance if sub_dir != subset: - new_instance_data = dict() - for _key, _value in copy_instance_data.items(): - new_instance_data[_key] = _value - if not isinstance(_value, str): - continue - if subset in _value: - new_instance_data[_key] = _value.replace( - subset, sub_dir) - new_instance = context.create_instance( - new_instance_data["name"]) - new_instance.data.update(new_instance_data) - self.log.info(f"Creating new instance: {new_instance}") - instance_data = new_instance.data + instance_data = self.duplicate_instance( + instance.data, subset, sub_dir) - staging_dir = _dir - files = os.listdir(_dir) - collections, remainder = clique.assemble(files) - # self.log.debug(f"collections: {collections}") - # self.log.debug(f"remainder: {remainder}") - # self.log.debug(f"staging_dir: {staging_dir}") - - # add staging_dir to instance_data - instance_data["stagingDir"] = staging_dir - # add representations to instance_data - instance_data["representations"] = list() - - # loop trough collections and create representations - for _collection in collections: - ext = _collection.tail - repre_data = { - "name": ext[1:], - "ext": ext[1:], - "files": [item for item in _collection], - "stagingDir": staging_dir - } - instance_data["representations"].append(repre_data) - - # loop trough reminders and create representations - for _reminding_file in remainder: - ext = os.path.splitext(_reminding_file)[-1] - if ext not in instance_data["extensions"]: - continue - - repre_data = { - "name": ext[1:], - "ext": ext[1:], - "files": _reminding_file, - "stagingDir": staging_dir - } - - # exception for thumbnail - if "thumb" in _reminding_file: - repre_data.update({ - 'name': "thumbnail", - 'thumbnail': True - }) - - # exception for mp4 preview - if ".mp4" in _reminding_file: - frame_start = instance_data["frameStart"] - frame_end = instance_data["frameEnd"] - instance_data["families"].append("review") - repre_data.update({ - "frameStart": 0, - "frameEnd": (frame_end - frame_start) + 1, - "frameStartFtrack": 0, - "frameEndFtrack": (frame_end - frame_start) + 1, - "step": 1, - "fps": context.data.get("fps"), - "name": "review", - "tags": ["review", "ftrackreview"], - }) - - instance_data["representations"].append(repre_data) - - representations = instance_data["representations"] - self.log.debug(f">>>_<<< representations: {representations}") + # create all representations + self.create_representations( + os.listdir(_dir), instance_data, _dir) if subset_files: - staging_dir = list(subset_files.keys()).pop() - collections, remainder = clique.assemble(subset_files[staging_dir]) - # self.log.debug(f"collections: {collections}") - # self.log.debug(f"remainder: {remainder}") - # self.log.debug(f"staging_dir: {staging_dir}") + unique_subset_names = list() + root_dir = list(subset_files.keys()).pop() + files_list = subset_files[root_dir] + search_patern = f"({subset}[A-Za-z0-9]+)(?=[\\._\\s])" + for _file in files_list: + patern = re.compile(search_patern) + match = patern.findall(_file) + if not match: + continue + match_subset = match.pop() + if match_subset in unique_subset_names: + continue + unique_subset_names.append(match_subset) - # if image sequence then create representation > match - # with subset name in dict + self.log.debug(f"unique_subset_names: {unique_subset_names}") - # idenfify as image sequence `isSequence` on instance data + for _un_subs in unique_subset_names: + instance_data = self.duplicate_instance( + instance.data, subset, _un_subs) + + # create all representations + self.create_representations( + [os.path.basename(f) for f in files_list], + instance_data, root_dir) + + # if the original subset name was not found in input folders + # then representations were nod added and it can be removed + if not instance.data.get("representations"): + instance.data["remove"] = True + + def duplicate_instance(self, instance_data, subset, new_subset): + + new_instance_data = dict() + for _key, _value in instance_data.items(): + new_instance_data[_key] = _value + if not isinstance(_value, str): + continue + if subset in _value: + new_instance_data[_key] = _value.replace( + subset, new_subset) + new_instance = self.context.create_instance( + new_instance_data["name"]) + new_instance.data.update(new_instance_data) + self.log.info(f"Creating new instance: {new_instance}") + return new_instance.data + + def create_representations( + self, files_list, instance_data, staging_dir): + """ Create representations from Collection object + """ + # collecting frames for later frame start/end reset + frames = list() + # break down Collection object to collections and reminders + collections, remainder = clique.assemble(files_list) + # add staging_dir to instance_data + instance_data["stagingDir"] = staging_dir + # add representations to instance_data + instance_data["representations"] = list() + + # loop trough collections and create representations + for _collection in collections: + ext = _collection.tail + frame_start = list(_collection.indexes)[0] + frame_end = list(_collection.indexes)[-1] + repre_data = { + "frameStart": frame_start, + "frameEnd": frame_end, + "name": ext[1:], + "ext": ext[1:], + "files": [item for item in _collection], + "stagingDir": staging_dir + } + instance_data["representations"].append(repre_data) + + # add to frames for frame range reset + frames.append(frame_start) + frames.append(frame_end) + + # loop trough reminders and create representations + for _reminding_file in remainder: + ext = os.path.splitext(_reminding_file)[-1] + if ext not in instance_data["extensions"]: + continue + + frame_start = 1 + frame_end = 1 + + repre_data = { + "name": ext[1:], + "ext": ext[1:], + "files": _reminding_file, + "stagingDir": staging_dir + } + + # exception for thumbnail + if "thumb" in _reminding_file: + repre_data.update({ + 'name': "thumbnail", + 'thumbnail': True + }) + + # exception for mp4 preview + if ".mp4" in _reminding_file: + frame_start = 0 + frame_end = ( + (instance_data["frameEnd"] - instance_data["frameStart"]) + + 1) + instance_data["families"] += ["review", "ftrack"] + repre_data.update({ + "frameStart": frame_start, + "frameEnd": frame_end, + "frameStartFtrack": frame_start, + "frameEndFtrack": frame_end, + "step": 1, + "fps": self.context.data.get("fps"), + "name": "review", + "tags": ["review", "ftrackreview", "delete"], + }) + + # add to frames for frame range reset only if no collection + if not collections: + frames.append(frame_start) + frames.append(frame_end) + + instance_data["representations"].append(repre_data) + + # reset frame start / end + instance_data["frameStart"] = min(frames) + instance_data["frameEnd"] = max(frames) From 79eb50c656b809bd40cc38763da243bd23dbcc48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Fri, 6 Nov 2020 18:54:05 +0100 Subject: [PATCH 050/147] few bugs and cosmetics --- pype/lib/abstract_collect_render.py | 47 ++++++++++----- pype/lib/abstract_expected_files.py | 2 +- pype/lib/abstract_submit_deadline.py | 90 +++++++++++++++------------- 3 files changed, 81 insertions(+), 58 deletions(-) diff --git a/pype/lib/abstract_collect_render.py b/pype/lib/abstract_collect_render.py index bf14f6f850..3c173a178d 100644 --- a/pype/lib/abstract_collect_render.py +++ b/pype/lib/abstract_collect_render.py @@ -34,45 +34,52 @@ class RenderInstance(object): attachTo = attr.ib(init=False) setMembers = attr.ib() publish = attr.ib() - review = attr.ib(default=False) renderer = attr.ib() - priority = attr.ib(default=50) name = attr.ib() - family = attr.ib(default="renderlayer") - families = attr.ib(default=["renderlayer"]) - # format settings resolutionWidth = attr.ib() resolutionHeight = attr.ib() pixelAspect = attr.ib() - multipartExr = attr.ib(default=False) + tileRendering = attr.ib() tilesX = attr.ib() tilesY = attr.ib() - convertToScanline = attr.ib(default=False) # time settings frameStart = attr.ib() frameEnd = attr.ib() frameStep = attr.ib() + # -------------------- + # With default values + # metadata + review = attr.ib(default=False) + priority = attr.ib(default=50) + + family = attr.ib(default="renderlayer") + families = attr.ib(default=["renderlayer"]) + + # format settings + multipartExr = attr.ib(default=False) + convertToScanline = attr.ib(default=False) + @frameStart.validator - def check_frame_start(self, attribute, value): + def check_frame_start(self, _, value): """Validate if frame start is not larger then end.""" if value >= self.frameEnd: raise ValueError("frameStart must be smaller " "or equal then frameEnd") @frameEnd.validator - def check_frame_end(self, attribute, value): + def check_frame_end(self, _, value): """Validate if frame end is not less then start.""" if value <= self.frameStart: raise ValueError("frameEnd must be smaller " "or equal then frameStart") @tilesX.validator - def check_tiles_x(self, attribute, value): + def check_tiles_x(self, _, value): """Validate if tile x isn't less then 1.""" if not self.tileRendering: return @@ -83,7 +90,7 @@ class RenderInstance(object): raise ValueError("both tiles X a Y sizes are set to 1") @tilesY.validator - def check_tiles_y(self, attribute, value): + def check_tiles_y(self, _, value): """Validate if tile y isn't less then 1.""" if not self.tileRendering: return @@ -102,6 +109,13 @@ class AbstractCollectRender(pyblish.api.ContextPlugin): label = "Collect Render" sync_workfile_version = False + def __init__(self, *args, **kwargs): + """Constructor.""" + super(AbstractCollectRender, self).__init__(*args, **kwargs) + self._file_path = None + self._asset = api.Session["AVALON_ASSET"] + + def process(self, context): """Entry point to collector.""" rendering_instance = None @@ -121,8 +135,7 @@ class AbstractCollectRender(pyblish.api.ContextPlugin): ) return - self._filepath = context.data["currentFile"].replace("\\", "/") - self._asset = api.Session["AVALON_ASSET"] + self._file_path = context.data["currentFile"].replace("\\", "/") render_instances = self.get_instances() for render_instance in render_instances: @@ -207,7 +220,12 @@ class AbstractCollectRender(pyblish.api.ContextPlugin): pass def _get_expected_files(self, render_instance): - """Get list of expected files.""" + """Get list of expected files. + + Returns: + list: expected files. + + """ # return all expected files for all cameras and aovs in given # frame range ef = ExpectedFiles() @@ -223,6 +241,7 @@ class AbstractCollectRender(pyblish.api.ContextPlugin): "attaching multiple AOVs or renderable cameras to " "subset is not supported" ) + return exp_files def add_additional_data(self, data): """Add additional data to collected instance. diff --git a/pype/lib/abstract_expected_files.py b/pype/lib/abstract_expected_files.py index f493cdb751..f9f3c17ef5 100644 --- a/pype/lib/abstract_expected_files.py +++ b/pype/lib/abstract_expected_files.py @@ -40,7 +40,7 @@ class ExpectedFiles: Args: - renderer_instance (:class:`RenderInstance`): Data passed from + render_instance (:class:`RenderInstance`): Data passed from collector to determine files. This should be instance of :class:`abstract_collect_render.RenderInstance` diff --git a/pype/lib/abstract_submit_deadline.py b/pype/lib/abstract_submit_deadline.py index 26ea66f5ce..869a011fd7 100644 --- a/pype/lib/abstract_submit_deadline.py +++ b/pype/lib/abstract_submit_deadline.py @@ -179,11 +179,9 @@ class DeadlineJobInfo: dict: as `{'EnvironmentKeyValue0', 'key=value'}` """ - out = [] - index = 0 - for v in self._environmentKeyValue: + out = {} + for index, v in enumerate(self._environmentKeyValue): out["EnvironmentKeyValue{}".format(index)] = v - index += 1 return out @EnvironmentKeyValue.setter @@ -207,11 +205,9 @@ class DeadlineJobInfo: dict: as `{'ExtraInfo0': 'value'}` """ - out = [] - index = 0 - for v in self._extraInfos: + out = {} + for index, v in enumerate(self._extraInfos): out["ExtraInfo{}".format(index)] = v - index += 1 return out @ExtraInfo.setter @@ -226,11 +222,9 @@ class DeadlineJobInfo: dict: as {'ExtraInfoKeyValue0': 'key=value'}` """ - out = [] - index = 0 - for v in self._extraInfoKeyValues: + out = {} + for index, v in enumerate(self._extraInfoKeyValues): out["ExtraInfoKeyValue{}".format(index)] = v - index += 1 return out @ExtraInfoKeyValue.setter @@ -250,11 +244,9 @@ class DeadlineJobInfo: dict: as `{'TaskExtraInfoName0': 'value'}` """ - out = [] - index = 0 - for v in self._taskExtraInfos: + out = {} + for index, v in enumerate(self._taskExtraInfos): out["TaskExtraInfoName{}".format(index)] = v - index += 1 return out @TaskExtraInfoName.setter @@ -275,11 +267,9 @@ class DeadlineJobInfo: dict: as `{'OutputFilename0': 'filename'}` """ - out = [] - index = 0 - for v in self._outputFilename: + out = {} + for index, v in enumerate(self._outputFilename): out["OutputFilename{}".format(index)] = v - index += 1 return out @OutputFilename.setter @@ -294,11 +284,9 @@ class DeadlineJobInfo: dict: as `{'OutputFilenme#Tile': 'tile'}` """ - out = [] - index = 0 - for v in self._outputFilenameTile: + out = {} + for index, v in enumerate(self._outputFilenameTile): out["OutputFilename{}Tile".format(index)] = v - index += 1 return out @OutputFilenameTile.setter @@ -313,11 +301,9 @@ class DeadlineJobInfo: dict: as `{'OutputDirectory0': 'dir'}` """ - out = [] - index = 0 - for v in self._outputDirectory: + out = {} + for index, v in enumerate(self._outputDirectory): out["OutputDirectory{}".format(index)] = v - index += 1 return out @OutputDirectory.setter @@ -338,18 +324,22 @@ class DeadlineJobInfo: MaintenanceJobStartFrame = attr.ib(default=None) # Default: 0 MaintenanceJobEndFrame = attr.ib(default=None) # Default: 0 - def render(self): + def serialize(self): """Return all data serialized as dictionary. Returns: OrderedDict: all serialized data. """ - def no_privates(a, _): - return not a.name.startswith("_") + def filter_data(a, v): + if a.name.startswith("_"): + return False + if v is None: + return False + return True serialized = attr.asdict( - self, dict_factory=OrderedDict, filter=no_privates) + self, dict_factory=OrderedDict, filter=filter_data) serialized.update(self.EnvironmentKeyValue) serialized.update(self.ExtraInfo) serialized.update(self.ExtraInfoKeyValue) @@ -369,6 +359,15 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): use_published = True asset_dependencies = False + def __init__(self, *args, **kwargs): + super(AbstractSubmitDeadline, self).__init__(*args, **kwargs) + self._instance = None + self._deadline_url = None + self.scene_path = None + self.job_info = None + self.plugin_info = None + self.aux_files = None + def process(self, instance): """Plugin entry point.""" self._instance = instance @@ -393,6 +392,8 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): self.plugin_info = self.get_plugin_info() self.aux_files = self.get_aux_files() + self.process_submission() + def process_submission(self): """Process data for submission. @@ -416,7 +417,7 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): :class:`DeadlineJobInfo` Returns: - dict: Filled Deadline JobInfo. + :class:`DeadlineJobInfo`: Filled Deadline JobInfo. """ pass @@ -461,6 +462,7 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): Returns: str: Published scene path. + None: if no published scene is found. Note: Published scene path is actually determined from project Anatomy @@ -469,6 +471,7 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): """ anatomy = self._instance.context.data['anatomy'] + file_path = None for i in self._instance.context: if "workfile" in i.data["families"]: # test if there is instance of workfile waiting @@ -483,23 +486,23 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): template_data["comment"] = None anatomy_filled = anatomy.format(template_data) template_filled = anatomy_filled["publish"]["path"] - filepath = os.path.normpath(template_filled) + file_path = os.path.normpath(template_filled) self.log.info("Using published scene for render {}".format( - filepath)) + file_path)) - if not os.path.exists(filepath): + if not os.path.exists(file_path): self.log.error("published scene does not exist!") raise if not replace_in_path: - return filepath + return file_path # now we need to switch scene in expected files # because token will now point to published # scene file and that might differ from current one new_scene = os.path.splitext( - os.path.basename(filepath))[0] + os.path.basename(file_path))[0] orig_scene = os.path.splitext( os.path.basename( self._instance.context.data["currentFile"]))[0] @@ -527,14 +530,14 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): orig_scene, new_scene )) - return filepath + return file_path def assemble_payload( self, job_info=None, plugin_info=None, aux_files=None): """Assemble payload data from its various parts. Args: - job_info (dict): Deadline JobInfo. You can use + job_info (DeadlineJobInfo): Deadline JobInfo. You can use :class:`DeadlineJobInfo` for it. plugin_info (dict): Deadline PluginInfo. Plugin specific options. aux_files (list, optional): List of auxiliary file to submit with @@ -544,8 +547,9 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): dict: Deadline Payload. """ + job = job_info or self.job_info return { - "JobInfo": job_info or self.job_info, + "JobInfo": job.serialize(), "PluginInfo": plugin_info or self.plugin_info, "AuxFiles": aux_files or self.aux_files } @@ -557,7 +561,7 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): Deadline jobs end-point. Args: - payload (str): string encoded json with job payload. + payload (dict): dict to become json in deadline submission. Returns: str: resulting Deadline job id. @@ -569,7 +573,7 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): url = "{}/api/jobs".format(self._deadline_url) response = self._requests_post(url, json=payload) if not response.ok: - self.log.error("Submition failed!") + self.log.error("Submission failed!") self.log.error(response.status_code) self.log.error(response.content) self.log.debug(payload) From ad216428f679354761c39cfb4bb390e8e3407130 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Fri, 6 Nov 2020 18:58:20 +0100 Subject: [PATCH 051/147] shut up hound --- pype/lib/abstract_collect_render.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pype/lib/abstract_collect_render.py b/pype/lib/abstract_collect_render.py index 3c173a178d..0f65a6ef90 100644 --- a/pype/lib/abstract_collect_render.py +++ b/pype/lib/abstract_collect_render.py @@ -115,7 +115,6 @@ class AbstractCollectRender(pyblish.api.ContextPlugin): self._file_path = None self._asset = api.Session["AVALON_ASSET"] - def process(self, context): """Entry point to collector.""" rendering_instance = None From b4ad6b68ea244d64387d4be2a0e9414d044f6d3c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 6 Nov 2020 19:06:31 +0100 Subject: [PATCH 052/147] implemented update method in laoder --- .../tvpaint/load/load_reference_image.py | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/pype/plugins/tvpaint/load/load_reference_image.py b/pype/plugins/tvpaint/load/load_reference_image.py index e3a224c283..0fa4cefc51 100644 --- a/pype/plugins/tvpaint/load/load_reference_image.py +++ b/pype/plugins/tvpaint/load/load_reference_image.py @@ -1,3 +1,4 @@ +from avalon.pipeline import get_representation_context from avalon.vendor import qargparse from avalon.tvpaint import lib, pipeline @@ -149,3 +150,95 @@ class LoadImage(pipeline.Loader): pipeline.SECTION_NAME_CONTAINERS, current_containers ) + def switch(self, container, representation): + self.update(container, representation) + + def update(self, container, representation): + """Replace container with different version. + + New layers are loaded as first step. Then is tried to change data in + new layers with data from old layers. When that is done old layers are + removed. + """ + # Create new containers first + context = get_representation_context(representation) + name = container["name"] + namespace = container["namespace"] + new_container = self.load(context, name, namespace, {}) + new_layer_ids = self.layer_ids_from_container(new_container) + + # Get layer ids from previous container + old_layer_ids = self.layer_ids_from_container(container) + + layers = lib.layers_data() + layers_by_id = { + layer["layer_id"]: layer + for layer in layers + } + + old_layers = [] + new_layers = [] + for layer_id in old_layer_ids: + layer = layers_by_id.get(layer_id) + if layer: + old_layers.append(layer) + + for layer_id in new_layer_ids: + layer = layers_by_id.get(layer_id) + if layer: + new_layers.append(layer) + + # Prepare few data + new_start_position = None + new_group_id = None + for layer in old_layers: + position = layer["position"] + group_id = layer["group_id"] + if new_start_position is None: + new_start_position = position + elif new_start_position > position: + new_start_position = position + + if new_group_id is None: + new_group_id = group_id + elif new_group_id < 0: + continue + elif new_group_id != group_id: + new_group_id = -1 + + george_script_lines = [] + # Group new layers to same group as previous container layers had + # - all old layers must be under same group + if new_group_id is not None and new_group_id > 0: + for layer in new_layers: + line = "tv_layercolor \"set\" {} {}".format( + layer["layer_id"], new_group_id + ) + george_script_lines.append(line) + + # Rename new layer to have same name + # - only if both old and new have one layer + if len(old_layers) == 1 and len(new_layers) == 1: + layer_name = old_layers[0]["name"] + george_script_lines.append( + "tv_layerrename {} \"{}\"".format( + new_layers[0]["layer_id"], layer_name + ) + ) + + # Change position of new layer + # - this must be done before remove old layers + if len(new_layers) == 1 and new_start_position is not None: + new_layer = new_layers[0] + george_script_lines.extend([ + "tv_layerset {}".format(new_layer["layer_id"]), + "tv_layermove {}".format(new_start_position) + ]) + + # Execute george scripts if there are any + if george_script_lines: + george_script = "\n".join(george_script_lines) + lib.execute_george_through_file(george_script) + + # Remove old container + self.remove(container) From e7b8717c594986984208bdf9df3ff4ae996216d1 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 6 Nov 2020 19:23:31 +0100 Subject: [PATCH 053/147] feat(PS): fixing original publishing with trimming --- .../standalonepublisher/publish/collect_hierarchy.py | 2 +- .../standalonepublisher/publish/collect_instances.py | 8 +++----- .../publish/extract_trim_video_audio.py | 8 ++++++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py index 3d9465cb1a..45d2fb4160 100644 --- a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py +++ b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py @@ -137,7 +137,7 @@ class CollectHierarchyInstance(pyblish.api.ContextPlugin): instance.data["tasks"] = tasks_to_add else: - instance.data["tasks"] = list() + instance.data["tasks"] = dict() # updating hierarchy data instance.data["anatomyData"].update({ diff --git a/pype/plugins/standalonepublisher/publish/collect_instances.py b/pype/plugins/standalonepublisher/publish/collect_instances.py index 3d577c1527..6e8bbea2e8 100644 --- a/pype/plugins/standalonepublisher/publish/collect_instances.py +++ b/pype/plugins/standalonepublisher/publish/collect_instances.py @@ -17,22 +17,20 @@ class CollectInstances(pyblish.api.InstancePlugin): "referenceMain": { "family": "review", "families": ["clip", "ftrack"], - # "ftrackFamily": "review", "extensions": [".mp4"] }, "audioMain": { "family": "audio", "families": ["clip", "ftrack"], - # "ftrackFamily": "audio", "extensions": [".wav"], - # "version": 1 }, "shotMain": { "family": "shot", "families": [] } } - timeline_frame_offset = None # if 900000 for edl default then -900000 + timeline_frame_start = 900000 # starndart edl default (01:00:00:00) + timeline_frame_offset = None custom_start_frame = None def process(self, instance): @@ -73,7 +71,7 @@ class CollectInstances(pyblish.api.InstancePlugin): try: track_start_frame = ( abs(track.source_range.start_time.value) - ) + ) - self.timeline_frame_start except AttributeError: track_start_frame = 0 diff --git a/pype/plugins/standalonepublisher/publish/extract_trim_video_audio.py b/pype/plugins/standalonepublisher/publish/extract_trim_video_audio.py index c955275b4f..193902a9f6 100644 --- a/pype/plugins/standalonepublisher/publish/extract_trim_video_audio.py +++ b/pype/plugins/standalonepublisher/publish/extract_trim_video_audio.py @@ -8,6 +8,8 @@ from pprint import pformat class ExtractTrimVideoAudio(pype.api.Extractor): """Trim with ffmpeg "mov" and "wav" files.""" + # must be before `ExtractThumbnailSP` + order = pyblish.api.ExtractorOrder - 0.01 label = "Extract Trim Video/Audio" hosts = ["standalonepublisher"] families = ["clip", "trimming"] @@ -37,6 +39,8 @@ class ExtractTrimVideoAudio(pype.api.Extractor): extensions = instance.data.get("extensions", [".mov"]) for ext in extensions: + self.log.info("Processing ext: `{}`".format(ext)) + clip_trimed_path = os.path.join( staging_dir, instance.data["name"] + ext) # # check video file metadata @@ -46,7 +50,7 @@ class ExtractTrimVideoAudio(pype.api.Extractor): start = float(instance.data["clipInH"]) dur = float(instance.data["clipDurationH"]) - if ext in ".wav": + if ext == ".wav": # offset time as ffmpeg is having bug start += 0.5 # remove "review" from families @@ -91,7 +95,7 @@ class ExtractTrimVideoAudio(pype.api.Extractor): "fps": fps, } - if ext[1:] in ["mov", "mp4"]: + if ext in [".mov", ".mp4"]: repr.update({ "thumbnail": True, "tags": ["review", "ftrackreview", "delete"]}) From 0f3f479bca4202a30463e880f7afa1b7015010d5 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 6 Nov 2020 20:03:05 +0100 Subject: [PATCH 054/147] feat(SP): default timeline frame start --- .../standalonepublisher/publish/collect_instances.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pype/plugins/standalonepublisher/publish/collect_instances.py b/pype/plugins/standalonepublisher/publish/collect_instances.py index 6e8bbea2e8..e99be5df38 100644 --- a/pype/plugins/standalonepublisher/publish/collect_instances.py +++ b/pype/plugins/standalonepublisher/publish/collect_instances.py @@ -29,7 +29,7 @@ class CollectInstances(pyblish.api.InstancePlugin): "families": [] } } - timeline_frame_start = 900000 # starndart edl default (01:00:00:00) + timeline_frame_start = 900000 # starndard edl default (10:00:00:00) timeline_frame_offset = None custom_start_frame = None @@ -68,13 +68,18 @@ class CollectInstances(pyblish.api.InstancePlugin): handle_end = int(asset_data["handleEnd"]) for track in tracks: + self.log.debug(f"track.name: {track.name}") try: track_start_frame = ( abs(track.source_range.start_time.value) - ) - self.timeline_frame_start + ) + self.log.debug(f"track_start_frame: {track_start_frame}") + track_start_frame -= self.timeline_frame_start except AttributeError: track_start_frame = 0 + self.log.debug(f"track_start_frame: {track_start_frame}") + for clip in track.each_child(): if clip.name is None: continue From 75b6401cdaf698afee815debefca49f835068a5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Mon, 9 Nov 2020 12:22:16 +0100 Subject: [PATCH 055/147] remove maya specific code --- pype/lib/abstract_collect_render.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/pype/lib/abstract_collect_render.py b/pype/lib/abstract_collect_render.py index 0f65a6ef90..b8ee107bd9 100644 --- a/pype/lib/abstract_collect_render.py +++ b/pype/lib/abstract_collect_render.py @@ -117,23 +117,11 @@ class AbstractCollectRender(pyblish.api.ContextPlugin): def process(self, context): """Entry point to collector.""" - rendering_instance = None for instance in context: - if "rendering" in instance.data["families"]: - rendering_instance = instance - rendering_instance.data["remove"] = True - # make sure workfile instance publishing is enabled if "workfile" in instance.data["families"]: instance.data["publish"] = True - if not rendering_instance: - self.log.info( - "No rendering instance found, skipping render " - "layer collection." - ) - return - self._file_path = context.data["currentFile"].replace("\\", "/") render_instances = self.get_instances() From 840ed3c104738ea4f1a54d79737d13dbcd480e3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Mon, 9 Nov 2020 15:34:49 +0100 Subject: [PATCH 056/147] solve multiple inheritance of meta classes --- pype/lib/abstract_collect_render.py | 4 ++-- pype/lib/abstract_metaplugins.py | 10 ++++++++++ pype/lib/abstract_submit_deadline.py | 5 +++-- 3 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 pype/lib/abstract_metaplugins.py diff --git a/pype/lib/abstract_collect_render.py b/pype/lib/abstract_collect_render.py index b8ee107bd9..89bbf6f024 100644 --- a/pype/lib/abstract_collect_render.py +++ b/pype/lib/abstract_collect_render.py @@ -13,6 +13,7 @@ from avalon import api import pyblish.api from .abstract_expected_files import ExpectedFiles +from .abstract_metaplugins import AbstractMetaContextPlugin @attr.s @@ -101,8 +102,7 @@ class RenderInstance(object): raise ValueError("both tiles X a Y sizes are set to 1") -@six.add_metaclass(ABCMeta) -class AbstractCollectRender(pyblish.api.ContextPlugin): +class AbstractCollectRender(AbstractMetaContextPlugin): """Gather all publishable render layers from renderSetup.""" order = pyblish.api.CollectorOrder + 0.01 diff --git a/pype/lib/abstract_metaplugins.py b/pype/lib/abstract_metaplugins.py new file mode 100644 index 0000000000..684d2ab19e --- /dev/null +++ b/pype/lib/abstract_metaplugins.py @@ -0,0 +1,10 @@ +from abc import ABCMeta +from pyblish.api import InstancePlugin, ContextPlugin + + +class AbstractMetaInstancePlugin(ABCMeta, InstancePlugin): + pass + + +class AbstractMetaContextPlugin(ABCMeta, ContextPlugin): + pass diff --git a/pype/lib/abstract_submit_deadline.py b/pype/lib/abstract_submit_deadline.py index 869a011fd7..e67e261cae 100644 --- a/pype/lib/abstract_submit_deadline.py +++ b/pype/lib/abstract_submit_deadline.py @@ -15,6 +15,7 @@ import attr import requests import pyblish.api +from .abstract_metaplugins import AbstractMetaInstancePlugin @attr.s @@ -350,8 +351,8 @@ class DeadlineJobInfo: return serialized -@six.add_metaclass(ABCMeta) -class AbstractSubmitDeadline(pyblish.api.InstancePlugin): +@six.add_metaclass(AbstractMetaInstancePlugin) +class AbstractSubmitDeadline: """Class abstracting access to Deadline.""" label = "Submit to Deadline" From 5510904127bf6b935e018f3867bd494bb2252fb7 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 9 Nov 2020 17:06:50 +0100 Subject: [PATCH 057/147] do not change family of renderPass to renderLayer --- pype/plugins/tvpaint/publish/collect_instances.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pype/plugins/tvpaint/publish/collect_instances.py b/pype/plugins/tvpaint/publish/collect_instances.py index b5e8f941ba..e751a18495 100644 --- a/pype/plugins/tvpaint/publish/collect_instances.py +++ b/pype/plugins/tvpaint/publish/collect_instances.py @@ -78,10 +78,6 @@ class CollectInstances(pyblish.api.ContextPlugin): render_layer = instance_data["render_layer"] instance_data["label"] = "{}_{}".format(render_layer, pass_name) - # Change family to `renderLayer` - instance_data["family"] = "renderLayer" - instance_data["families"] = [instance_data["family"]] - layers_data = context.data["layersData"] layers_by_id = { layer["layer_id"]: layer From 5bb609c512292fb803c0bf8168b5bbf37dc0eb13 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 10 Nov 2020 11:15:39 +0100 Subject: [PATCH 058/147] fix(SP): creating duplicity in instance data --- .../publish/collect_clear_instances.py | 15 ++-- .../publish/collect_instance_resources.py | 79 +++++++++++++------ .../publish/collect_instances.py | 7 +- .../publish/extract_thumbnail.py | 7 +- 4 files changed, 76 insertions(+), 32 deletions(-) diff --git a/pype/plugins/standalonepublisher/publish/collect_clear_instances.py b/pype/plugins/standalonepublisher/publish/collect_clear_instances.py index 0d64c57d3a..097e730251 100644 --- a/pype/plugins/standalonepublisher/publish/collect_clear_instances.py +++ b/pype/plugins/standalonepublisher/publish/collect_clear_instances.py @@ -5,16 +5,17 @@ Optional: import pyblish.api -class CollectClearInstances(pyblish.api.ContextPlugin): +class CollectClearInstances(pyblish.api.InstancePlugin): """Clear all marked instances""" order = pyblish.api.CollectorOrder + 0.4999 label = "Clear Instances" hosts = ["standalonepublisher"] - def process(self, context): - - for instance in context: - if instance.data.get("remove"): - self.log.info(f"Removing: {instance}") - context.remove(instance) + def process(self, instance): + self.log.debug( + f"Instance: `{instance}` | " + f"families: `{instance.data['families']}`") + if instance.data.get("remove"): + self.log.info(f"Removing: {instance}") + instance.context.remove(instance) diff --git a/pype/plugins/standalonepublisher/publish/collect_instance_resources.py b/pype/plugins/standalonepublisher/publish/collect_instance_resources.py index 6807e82193..3b2f121608 100644 --- a/pype/plugins/standalonepublisher/publish/collect_instance_resources.py +++ b/pype/plugins/standalonepublisher/publish/collect_instance_resources.py @@ -17,8 +17,8 @@ class CollectInstanceResources(pyblish.api.InstancePlugin): def process(self, instance): self.context = instance.context - instance_data = instance.data self.log.info(f"Processing instance: {instance}") + self.new_instances = [] subset_files = dict() subset_dirs = list() anatomy = self.context.data["anatomy"] @@ -38,8 +38,8 @@ class CollectInstanceResources(pyblish.api.InstancePlugin): staging_dir = os.path.normpath( tempfile.mkdtemp(prefix="pyblish_tmp_") ) - instance_data["stagingDir"] = staging_dir - instance_data["families"] += ["trimming"] + instance.data["stagingDir"] = staging_dir + instance.data["families"] += ["trimming"] return # if template patern in path then fill it with `anatomy_data` @@ -51,13 +51,14 @@ class CollectInstanceResources(pyblish.api.InstancePlugin): # loop `editorial_source_root` and find clip name in folders # and look for any subset name alternatives for root, dirs, files in os.walk(editorial_source_root): + # search only for directories related to clip name correct_clip_dir = None - for d in dirs: + for _d_search in dirs: # avoid all non clip dirs - if d not in clip_name: + if _d_search not in clip_name: continue # found correct dir for clip - correct_clip_dir = d + correct_clip_dir = _d_search # continue if clip dir was not found if not correct_clip_dir: @@ -83,22 +84,31 @@ class CollectInstanceResources(pyblish.api.InstancePlugin): if subset_files_items: subset_files.update({clip_dir_path: subset_files_items}) + + # break the loop if correct_clip_dir was captured + # no need to cary on if corect folder was found if correct_clip_dir: break if subset_dirs: # look all dirs and check for subset name alternatives for _dir in subset_dirs: + instance_data = deepcopy( + {k: v for k, v in instance.data.items()}) sub_dir = os.path.basename(_dir) # if subset name is only alternative then create new instance if sub_dir != subset: instance_data = self.duplicate_instance( - instance.data, subset, sub_dir) + instance_data, subset, sub_dir) # create all representations self.create_representations( os.listdir(_dir), instance_data, _dir) + if sub_dir == subset: + self.new_instances.append(instance_data) + # instance.data.update(instance_data) + if subset_files: unique_subset_names = list() root_dir = list(subset_files.keys()).pop() @@ -120,15 +130,21 @@ class CollectInstanceResources(pyblish.api.InstancePlugin): instance_data = self.duplicate_instance( instance.data, subset, _un_subs) - # create all representations - self.create_representations( - [os.path.basename(f) for f in files_list], - instance_data, root_dir) + # create all representations + self.create_representations( + [os.path.basename(f) for f in files_list + if _un_subs in f], + instance_data, root_dir) - # if the original subset name was not found in input folders - # then representations were nod added and it can be removed - if not instance.data.get("representations"): - instance.data["remove"] = True + # remove the original instance as it had been used only + # as template and is duplicated + self.context.remove(instance) + + # create all instances in self.new_instances into context + for new_instance in self.new_instances: + _new_instance = self.context.create_instance( + new_instance["name"]) + _new_instance.data.update(new_instance) def duplicate_instance(self, instance_data, subset, new_subset): @@ -140,11 +156,10 @@ class CollectInstanceResources(pyblish.api.InstancePlugin): if subset in _value: new_instance_data[_key] = _value.replace( subset, new_subset) - new_instance = self.context.create_instance( - new_instance_data["name"]) - new_instance.data.update(new_instance_data) - self.log.info(f"Creating new instance: {new_instance}") - return new_instance.data + + self.log.info(f"Creating new instance: {new_instance_data['name']}") + self.new_instances.append(new_instance_data) + return new_instance_data def create_representations( self, files_list, instance_data, staging_dir): @@ -159,9 +174,11 @@ class CollectInstanceResources(pyblish.api.InstancePlugin): # add representations to instance_data instance_data["representations"] = list() + collection_head_name = None # loop trough collections and create representations for _collection in collections: ext = _collection.tail + collection_head_name = _collection.head frame_start = list(_collection.indexes)[0] frame_end = list(_collection.indexes)[-1] repre_data = { @@ -172,6 +189,17 @@ class CollectInstanceResources(pyblish.api.InstancePlugin): "files": [item for item in _collection], "stagingDir": staging_dir } + + if "review" in instance_data["families"]: + repre_data.update({ + "thumbnail": True, + "frameStartFtrack": frame_start, + "frameEndFtrack": frame_end, + "step": 1, + "fps": self.context.data.get("fps"), + "name": "review", + "tags": ["review", "ftrackreview", "delete"], + }) instance_data["representations"].append(repre_data) # add to frames for frame range reset @@ -183,7 +211,11 @@ class CollectInstanceResources(pyblish.api.InstancePlugin): ext = os.path.splitext(_reminding_file)[-1] if ext not in instance_data["extensions"]: continue - + if collection_head_name and ( + (collection_head_name + ext[1:]) not in _reminding_file + ) and (ext in [".mp4", ".mov"]): + self.log.info(f"Skipping file: {_reminding_file}") + continue frame_start = 1 frame_end = 1 @@ -207,7 +239,10 @@ class CollectInstanceResources(pyblish.api.InstancePlugin): frame_end = ( (instance_data["frameEnd"] - instance_data["frameStart"]) + 1) - instance_data["families"] += ["review", "ftrack"] + # add review ftrack family into families + for _family in ["review", "ftrack"]: + if _family not in instance_data["families"]: + instance_data["families"].append(_family) repre_data.update({ "frameStart": frame_start, "frameEnd": frame_end, diff --git a/pype/plugins/standalonepublisher/publish/collect_instances.py b/pype/plugins/standalonepublisher/publish/collect_instances.py index e99be5df38..090ffe2cbb 100644 --- a/pype/plugins/standalonepublisher/publish/collect_instances.py +++ b/pype/plugins/standalonepublisher/publish/collect_instances.py @@ -186,10 +186,15 @@ class CollectInstances(pyblish.api.InstancePlugin): "subset": subset }) # create new instance - instance.context.create_instance( + _instance = instance.context.create_instance( **subset_instance_data) + self.log.debug( + f"Instance: `{_instance}` | " + f"families: `{subset_instance_data['families']}`") context.data["assetsShared"][name] = { "_clipIn": clip_in, "_clipOut": clip_out } + + self.log.debug("Instance: `{}` | families: `{}`") diff --git a/pype/plugins/standalonepublisher/publish/extract_thumbnail.py b/pype/plugins/standalonepublisher/publish/extract_thumbnail.py index fca4039d0e..f9932db695 100644 --- a/pype/plugins/standalonepublisher/publish/extract_thumbnail.py +++ b/pype/plugins/standalonepublisher/publish/extract_thumbnail.py @@ -46,6 +46,7 @@ class ExtractThumbnailSP(pyblish.api.InstancePlugin): files_len = 1 file = files + staging_dir = None is_jpeg = False if file.endswith(".jpeg") or file.endswith(".jpg"): is_jpeg = True @@ -56,7 +57,8 @@ class ExtractThumbnailSP(pyblish.api.InstancePlugin): elif is_jpeg: # use first frame as thumbnail if is sequence of jpegs - full_thumbnail_path = file + staging_dir = thumbnail_repre.get("stagingDir") + full_thumbnail_path = os.path.join(staging_dir, file) self.log.info( "For thumbnail is used file: {}".format(full_thumbnail_path) ) @@ -104,7 +106,7 @@ class ExtractThumbnailSP(pyblish.api.InstancePlugin): thumbnail_repre.pop("thumbnail") filename = os.path.basename(full_thumbnail_path) - staging_dir = os.path.dirname(full_thumbnail_path) + staging_dir = staging_dir or os.path.dirname(full_thumbnail_path) # create new thumbnail representation representation = { @@ -119,4 +121,5 @@ class ExtractThumbnailSP(pyblish.api.InstancePlugin): if not is_jpeg: representation["tags"].append("delete") + self.log.info(f"New representation {representation}") instance.data["representations"].append(representation) From 3c29bfb5bfad7acc2326ec88c1731e17965a5776 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 10 Nov 2020 11:51:44 +0100 Subject: [PATCH 059/147] hound(SP): suggestions --- .../standalonepublisher/publish/collect_instance_resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/plugins/standalonepublisher/publish/collect_instance_resources.py b/pype/plugins/standalonepublisher/publish/collect_instance_resources.py index 3b2f121608..565d066fd8 100644 --- a/pype/plugins/standalonepublisher/publish/collect_instance_resources.py +++ b/pype/plugins/standalonepublisher/publish/collect_instance_resources.py @@ -50,7 +50,7 @@ class CollectInstanceResources(pyblish.api.InstancePlugin): self.log.debug(f"root: {editorial_source_root}") # loop `editorial_source_root` and find clip name in folders # and look for any subset name alternatives - for root, dirs, files in os.walk(editorial_source_root): + for root, dirs, _files in os.walk(editorial_source_root): # search only for directories related to clip name correct_clip_dir = None for _d_search in dirs: From 4118093e2c36885876b21e79b99334253ad32f95 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 10 Nov 2020 13:01:36 +0100 Subject: [PATCH 060/147] fix(SP): after merge 2xDev balast --- .../publish/extract_shot_data.py | 92 ------------------- 1 file changed, 92 deletions(-) delete mode 100644 pype/plugins/standalonepublisher/publish/extract_shot_data.py diff --git a/pype/plugins/standalonepublisher/publish/extract_shot_data.py b/pype/plugins/standalonepublisher/publish/extract_shot_data.py deleted file mode 100644 index e4eb813bae..0000000000 --- a/pype/plugins/standalonepublisher/publish/extract_shot_data.py +++ /dev/null @@ -1,92 +0,0 @@ -import os -import clique -import pype.api - -from pprint import pformat - - -class ExtractShotData(pype.api.Extractor): - """Extract shot "mov" and "wav" files.""" - - label = "Extract Shot Data" - hosts = ["standalonepublisher"] - families = ["clip"] - - # presets - - def process(self, instance): - representation = instance.data.get("representations") - self.log.debug(f"_ representation: {representation}") - - if not representation: - instance.data["representations"] = list() - - # get ffmpet path - ffmpeg_path = pype.lib.get_ffmpeg_tool_path("ffmpeg") - - # get staging dir - staging_dir = self.staging_dir(instance) - self.log.info("Staging dir set to: `{}`".format(staging_dir)) - - # Generate mov file. - fps = instance.data["fps"] - video_file_path = instance.data["editorialVideoPath"] - ext = instance.data.get("extension", ".mov") - - clip_trimed_path = os.path.join( - staging_dir, instance.data["name"] + ext) - # - # # check video file metadata - # input_data = plib.ffprobe_streams(video_file_path)[0] - # self.log.debug(f"__ input_data: `{input_data}`") - - start = float(instance.data["clipInH"]) - dur = float(instance.data["clipDurationH"]) - - if ext in ".wav": - start += 0.5 - - args = [ - "\"{}\"".format(ffmpeg_path), - "-ss", str(start / fps), - "-i", f"\"{video_file_path}\"", - "-t", str(dur / fps) - ] - if ext in [".mov", ".mp4"]: - args.extend([ - "-crf", "18", - "-pix_fmt", "yuv420p"]) - elif ext in ".wav": - args.extend([ - "-vn -acodec pcm_s16le", - "-ar 48000 -ac 2" - ]) - - # add output path - args.append(f"\"{clip_trimed_path}\"") - - self.log.info(f"Processing: {args}") - ffmpeg_args = " ".join(args) - output = pype.api.subprocess(ffmpeg_args, shell=True) - self.log.info(output) - - repr = { - "name": ext[1:], - "ext": ext[1:], - "files": os.path.basename(clip_trimed_path), - "stagingDir": staging_dir, - "frameStart": int(instance.data["frameStart"]), - "frameEnd": int(instance.data["frameEnd"]), - "frameStartFtrack": int(instance.data["frameStartH"]), - "frameEndFtrack": int(instance.data["frameEndH"]), - "fps": fps, - } - - if ext[1:] in ["mov", "mp4"]: - repr.update({ - "thumbnail": True, - "tags": ["review", "ftrackreview", "delete"]}) - - instance.data["representations"].append(repr) - - self.log.debug(f"Instance data: {pformat(instance.data)}") From cd9d482763a9f06975eb42960934a7596164e85c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 10 Nov 2020 15:51:25 +0100 Subject: [PATCH 061/147] #698 - AE publish to deadline init --- pype/lib/__init__.py | 13 +++ .../publish/collect_current_file.py | 18 ++++ .../aftereffects/publish/collect_instances.py | 91 +++++++++++++++++++ .../aftereffects/publish/collect_render.py | 15 +++ .../publish/submit_aftereffects_deadline.py | 81 +++++++++++++++++ 5 files changed, 218 insertions(+) create mode 100644 pype/lib/__init__.py create mode 100644 pype/plugins/aftereffects/publish/collect_current_file.py create mode 100644 pype/plugins/aftereffects/publish/collect_instances.py create mode 100644 pype/plugins/aftereffects/publish/collect_render.py create mode 100644 pype/plugins/aftereffects/publish/submit_aftereffects_deadline.py diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py new file mode 100644 index 0000000000..989e94c5e9 --- /dev/null +++ b/pype/lib/__init__.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +"""Pype lib module.""" + + +from .abstract_submit_deadline import DeadlineJobInfo, AbstractSubmitDeadline +from .abstract_collect_render import RenderInstance, AbstractCollectRender + +__all__ = [ + "AbstractSubmitDeadline", + "DeadlineJobInfo", + "RenderInstance", + "AbstractCollectRender" +] diff --git a/pype/plugins/aftereffects/publish/collect_current_file.py b/pype/plugins/aftereffects/publish/collect_current_file.py new file mode 100644 index 0000000000..b59ff41a0e --- /dev/null +++ b/pype/plugins/aftereffects/publish/collect_current_file.py @@ -0,0 +1,18 @@ +import os + +import pyblish.api + +from avalon import aftereffects + + +class CollectCurrentFile(pyblish.api.ContextPlugin): + """Inject the current working file into context""" + + order = pyblish.api.CollectorOrder - 0.5 + label = "Current File" + hosts = ["aftereffects"] + + def process(self, context): + context.data["currentFile"] = os.path.normpath( + aftereffects.stub().get_active_document_full_name() + ).replace("\\", "/") diff --git a/pype/plugins/aftereffects/publish/collect_instances.py b/pype/plugins/aftereffects/publish/collect_instances.py new file mode 100644 index 0000000000..7a4d58d3af --- /dev/null +++ b/pype/plugins/aftereffects/publish/collect_instances.py @@ -0,0 +1,91 @@ +import os +from avalon import api +import pyblish.api + + +class CollectInstances(pyblish.api.ContextPlugin): + """ Adds the celaction render instances """ + + label = "Collect After Effects Instances" + order = pyblish.api.CollectorOrder + 0.1 + + def process(self, context): + task = api.Session["AVALON_TASK"] + current_file = context.data["currentFile"] + staging_dir = os.path.dirname(current_file) + scene_file = os.path.basename(current_file) + version = context.data["version"] + asset_entity = context.data["assetEntity"] + project_entity = context.data["projectEntity"] + + shared_instance_data = { + "asset": asset_entity["name"], + "frameStart": asset_entity["data"]["frameStart"], + "frameEnd": asset_entity["data"]["frameEnd"], + "handleStart": asset_entity["data"]["handleStart"], + "handleEnd": asset_entity["data"]["handleEnd"], + "fps": asset_entity["data"]["fps"], + "resolutionWidth": asset_entity["data"].get( + "resolutionWidth", + project_entity["data"]["resolutionWidth"]), + "resolutionHeight": asset_entity["data"].get( + "resolutionHeight", + project_entity["data"]["resolutionHeight"]), + "pixelAspect": 1, + "step": 1, + "version": version + } + + # workfile instance + family = "workfile" + subset = family + task.capitalize() + # Create instance + instance = context.create_instance(subset) + + # creating instance data + instance.data.update({ + "subset": subset, + "label": scene_file, + "family": family, + "families": [family, "ftrack"], + "representations": list() + }) + + # adding basic script data + instance.data.update(shared_instance_data) + + # creating representation + representation = { + 'name': 'aep', + 'ext': 'aep', + 'files': scene_file, + "stagingDir": staging_dir, + } + + instance.data["representations"].append(representation) + + self.log.info('Publishing After Effects workfile') + + # render instance + family = "render.farm" + subset = f"render{task}Main" + instance = context.create_instance(name=subset) + # getting instance state + instance.data["publish"] = True + + # add assetEntity data into instance + instance.data.update({ + "label": "{} - farm".format(subset), + "family": family, + "families": [family], + "subset": subset + }) + + # adding basic script data + instance.data.update(shared_instance_data) + + self.log.info('Publishing After Effects render instance') + self.log.debug(f"Instance data: `{instance.data}`") + + for i in context: + self.log.debug(f"{i.data['families']}") diff --git a/pype/plugins/aftereffects/publish/collect_render.py b/pype/plugins/aftereffects/publish/collect_render.py new file mode 100644 index 0000000000..080dd7e4b5 --- /dev/null +++ b/pype/plugins/aftereffects/publish/collect_render.py @@ -0,0 +1,15 @@ +from pype.lib import RenderInstance, AbstractCollectRender +import pyblish.api + +from avalon import aftereffects + +class CollectRender(AbstractCollectRender): + + order = pyblish.api.CollectorOrder + 0.01 + hosts = ["maya"] + label = "Collect Render Layers" + sync_workfile_version = False + + def get_instances(self): + print("hello") + return aftereffects.stub().get_metadata() \ No newline at end of file diff --git a/pype/plugins/aftereffects/publish/submit_aftereffects_deadline.py b/pype/plugins/aftereffects/publish/submit_aftereffects_deadline.py new file mode 100644 index 0000000000..e24a4fb470 --- /dev/null +++ b/pype/plugins/aftereffects/publish/submit_aftereffects_deadline.py @@ -0,0 +1,81 @@ +from pype.lib import AbstractSubmitDeadline, DeadlineJobInfo +from abc import ABCMeta, abstractmethod +import pyblish.api +import os +import attr +import six + +@attr.s +class DeadlinePluginInfo(): + SceneFile = attr.ib(default=None) + OutputFilePath = attr.ib(default=None) + StartupDirectory = attr.ib(default=None) + Arguments = attr.ib(default=None) + ProjectPath = attr.ib(default=None) + SceneFile = attr.ib(default=None) + AWSAssetFile0 = attr.ib(default=None) + + +@six.add_metaclass(ABCMeta) +class AfterEffectsSubmitDeadline(AbstractSubmitDeadline): + + label = "Submit to Deadline" + order = pyblish.api.IntegratorOrder + 0.1 + hosts = ["aftereffects"] + families = ["render.farm"] + + def get_job_info(self): + deadline_job_info = DeadlineJobInfo() + context = self._instance["context"] + + print("self._instance::{}".format(self._instance)) + print("context::{}".format(context)) + deadline_job_info.Name = "TestName" + deadline_job_info.Plugin = "AfterEffects" + deadline_job_info.UserName = "Test User" # context + deadline_job_info.Department = "Test department" + deadline_job_info.Priority = 50 + deadline_job_info.Group = "Test group" + deadline_job_info.Pool = "Test pool" + frame_range = "{}-{}".format(self._instance.data["frameStart"], + self._instance.data["frameEnd"]) + deadline_job_info.Frames = frame_range + deadline_job_info.Comment = "Test comment" # context + deadline_job_info.OutputFilename = "c:/projects/test.txt" + deadline_job_info.ScheduledType = "Once" + deadline_job_info.JobDelay = "00:00:00" + + print("deadline_job_info::{}".format(deadline_job_info)) + + return deadline_job_info + + def get_plugin_info(self): + deadline_plugin_info = DeadlinePluginInfo() + context = self._instance["context"] + script_path = context.data["currentFile"] + + render_path = self._instance.data['path'] + render_dir = os.path.normpath(os.path.dirname(render_path)) + + #renderer_path = "C:\\Program Files\\Adobe\\Adobe After Effects 2020\\Support Files\\aerender.exe" + + args = [ + "-s ", + "-e ", + f"-project {script_path}", + f"-output {render_dir}" + "-comp \"Comp\"" + ] + + deadline_plugin_info.SceneFile = script_path + deadline_plugin_info.OutputFilePath = render_dir.replace("\\", "/") + + deadline_plugin_info.StartupDirectory = "" + deadline_plugin_info.Arguments = " ".join(args) + + deadline_plugin_info.ProjectPath = script_path + deadline_plugin_info.AWSAssetFile0 = render_path + + print("deadline_plugin_info::{}".format(deadline_plugin_info)) + + return deadline_plugin_info From 2f615c9060b2bfd4f4b24c0230e7331b197199a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Tue, 10 Nov 2020 18:06:26 +0100 Subject: [PATCH 062/147] fix metaclass madness --- pype/lib/abstract_collect_render.py | 12 ++++++++---- pype/lib/abstract_metaplugins.py | 6 +++--- pype/lib/abstract_submit_deadline.py | 4 ++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/pype/lib/abstract_collect_render.py b/pype/lib/abstract_collect_render.py index 89bbf6f024..33202b6ac3 100644 --- a/pype/lib/abstract_collect_render.py +++ b/pype/lib/abstract_collect_render.py @@ -4,10 +4,10 @@ TODO: use @dataclass when times come. """ -from abc import ABCMeta, abstractmethod +from abc import abstractmethod -import six import attr +import six from avalon import api import pyblish.api @@ -102,7 +102,8 @@ class RenderInstance(object): raise ValueError("both tiles X a Y sizes are set to 1") -class AbstractCollectRender(AbstractMetaContextPlugin): +@six.add_metaclass(AbstractMetaContextPlugin) +class AbstractCollectRender(pyblish.api.ContextPlugin): """Gather all publishable render layers from renderSetup.""" order = pyblish.api.CollectorOrder + 0.01 @@ -196,9 +197,12 @@ class AbstractCollectRender(AbstractMetaContextPlugin): self.post_collecting_action() @abstractmethod - def get_instances(self): + def get_instances(self, context): """Get all renderable instances and their data. + Args: + context (pyblish.api.Context): Context object. + Returns: list of :class:`RenderInstance`: All collected renderable instances (like render layers, write nodes, etc.) diff --git a/pype/lib/abstract_metaplugins.py b/pype/lib/abstract_metaplugins.py index 684d2ab19e..f8163956ad 100644 --- a/pype/lib/abstract_metaplugins.py +++ b/pype/lib/abstract_metaplugins.py @@ -1,10 +1,10 @@ from abc import ABCMeta -from pyblish.api import InstancePlugin, ContextPlugin +from pyblish.plugin import MetaPlugin, ExplicitMetaPlugin -class AbstractMetaInstancePlugin(ABCMeta, InstancePlugin): +class AbstractMetaInstancePlugin(ABCMeta, MetaPlugin): pass -class AbstractMetaContextPlugin(ABCMeta, ContextPlugin): +class AbstractMetaContextPlugin(ABCMeta, ExplicitMetaPlugin): pass diff --git a/pype/lib/abstract_submit_deadline.py b/pype/lib/abstract_submit_deadline.py index e67e261cae..44ee9ce436 100644 --- a/pype/lib/abstract_submit_deadline.py +++ b/pype/lib/abstract_submit_deadline.py @@ -5,7 +5,7 @@ It provides Deadline JobInfo data class. """ import os -from abc import ABCMeta, abstractmethod +from abc import abstractmethod import platform import getpass from collections import OrderedDict @@ -352,7 +352,7 @@ class DeadlineJobInfo: @six.add_metaclass(AbstractMetaInstancePlugin) -class AbstractSubmitDeadline: +class AbstractSubmitDeadline(pyblish.api.InstancePlugin): """Class abstracting access to Deadline.""" label = "Submit to Deadline" From 98bcea52a554bc848558911cbc3b62130efa683a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 10 Nov 2020 18:29:55 +0100 Subject: [PATCH 063/147] instance.data always contain all layers for the instance --- .../tvpaint/publish/collect_instances.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/pype/plugins/tvpaint/publish/collect_instances.py b/pype/plugins/tvpaint/publish/collect_instances.py index e751a18495..bacc5984e8 100644 --- a/pype/plugins/tvpaint/publish/collect_instances.py +++ b/pype/plugins/tvpaint/publish/collect_instances.py @@ -37,21 +37,25 @@ class CollectInstances(pyblish.api.ContextPlugin): instance = None if family == "review": instance = context.create_instance(**instance_data) + instance.data["layers"] = context.data["layersData"] elif family == "renderLayer": - instance = self.create_render_layer(context, instance_data) + instance = self.create_render_layer_instance( + context, instance_data + ) elif family == "renderPass": - instance = self.create_render_pass(context, instance_data) + instance = self.create_render_pass_instance( + context, instance_data + ) else: raise AssertionError( "Instance with unknown family \"{}\": {}" ) - if instance is not None: - self.log.debug("Created instance: {}\n{}".format( - instance, json.dumps(instance.data, indent=4) - )) + self.log.debug("Created instance: {}\n{}".format( + instance, json.dumps(instance.data, indent=4) + )) - def create_render_layer(self, context, instance_data): + def create_render_layer_instance(self, context, instance_data): name = instance_data["name"] instance_data["label"] = "{}_beauty".format(name) @@ -73,7 +77,7 @@ class CollectInstances(pyblish.api.ContextPlugin): instance_data["layers"] = group_layers return context.create_instance(**instance_data) - def create_render_pass(self, context, instance_data): + def create_render_pass_instance(self, context, instance_data): pass_name = instance_data["pass"] render_layer = instance_data["render_layer"] instance_data["label"] = "{}_{}".format(render_layer, pass_name) From 62a2346ba6cbc38b112ae7744b637992c359df41 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 10 Nov 2020 19:22:36 +0100 Subject: [PATCH 064/147] renamed key `frameRate` to `fps` --- pype/plugins/tvpaint/publish/collect_workfile_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/plugins/tvpaint/publish/collect_workfile_data.py b/pype/plugins/tvpaint/publish/collect_workfile_data.py index 87076e978d..34709e34b0 100644 --- a/pype/plugins/tvpaint/publish/collect_workfile_data.py +++ b/pype/plugins/tvpaint/publish/collect_workfile_data.py @@ -48,7 +48,7 @@ class CollectWorkfileData(pyblish.api.ContextPlugin): "sceneHeight": height, "pixelAspect": pixel_apsect, "frameStart": frame_start, - "frameRate": frame_rate, + "fps": frame_rate, "fieldOrder": field_order } self.log.debug( From 434c684c791e85bc87883c1fb110cb63cbcce9d9 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 10 Nov 2020 19:23:50 +0100 Subject: [PATCH 065/147] frame end is set from last frame of all visible layers --- pype/plugins/tvpaint/publish/collect_workfile_data.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pype/plugins/tvpaint/publish/collect_workfile_data.py b/pype/plugins/tvpaint/publish/collect_workfile_data.py index 34709e34b0..b27eb2f693 100644 --- a/pype/plugins/tvpaint/publish/collect_workfile_data.py +++ b/pype/plugins/tvpaint/publish/collect_workfile_data.py @@ -42,12 +42,21 @@ class CollectWorkfileData(pyblish.api.ContextPlugin): width = int(workfile_info_parts.pop(-1)) workfile_path = " ".join(workfile_info_parts).replace("\"", "") + # TODO This is not porper way of getting last frame + # - but don't know better + last_frame = frame_start + for layer in layers_data: + frame_end = layer["frame_end"] + if frame_end > last_frame: + last_frame = frame_end + sceme_data = { "currentFile": workfile_path, "sceneWidth": width, "sceneHeight": height, "pixelAspect": pixel_apsect, "frameStart": frame_start, + "frameEnd": last_frame, "fps": frame_rate, "fieldOrder": field_order } From ac55207d11df3e6878a73a99979ab82cdf8ce36e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 10 Nov 2020 19:24:07 +0100 Subject: [PATCH 066/147] instances have also set frame start and end --- pype/plugins/tvpaint/publish/collect_instances.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pype/plugins/tvpaint/publish/collect_instances.py b/pype/plugins/tvpaint/publish/collect_instances.py index bacc5984e8..d75ea83a4c 100644 --- a/pype/plugins/tvpaint/publish/collect_instances.py +++ b/pype/plugins/tvpaint/publish/collect_instances.py @@ -17,6 +17,9 @@ class CollectInstances(pyblish.api.ContextPlugin): )) for instance_data in workfile_instances: + instance_data["frameStart"] = context.data["frameStart"] + instance_data["frameEnd"] = context.data["frameEnd"] + # Store workfile instance data to instance data instance_data["originData"] = copy.deepcopy(instance_data) # Global instance data modifications From f19e2479f8742dd1cdfd1f8a20bbe70ae9542518 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 10 Nov 2020 19:24:40 +0100 Subject: [PATCH 067/147] base implementation of tvpaint extractor --- .../plugins/tvpaint/publish/extract_output.py | 249 ++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 pype/plugins/tvpaint/publish/extract_output.py diff --git a/pype/plugins/tvpaint/publish/extract_output.py b/pype/plugins/tvpaint/publish/extract_output.py new file mode 100644 index 0000000000..ffb9143195 --- /dev/null +++ b/pype/plugins/tvpaint/publish/extract_output.py @@ -0,0 +1,249 @@ +import os +import shutil +import tempfile + +import pyblish.api +from avalon.tvpaint import pipeline, lib + + +class ExtractOutput(pyblish.api.Extractor): + label = "Extract Output" + hosts = ["tvpaint"] + families = ["review", "renderPass", "renderLayer"] + + save_mode_to_ext = { + "avi": ".avi", + "bmp": ".bmp", + "cin": ".cin", + "deep": ".dip", + "dps": ".dps", + "dpx": ".dpx", + "flc": ".fli", + "gif": ".gif", + "ilbm": ".iff", + "jpeg": ".jpg", + "pcx": ".pcx", + "png": ".png", + "psd": ".psd", + "qt": ".qt", + "rtv": ".rtv", + "sun": ".ras", + "tiff": ".tiff", + "tga": ".tga", + "vpb": ".vpb" + } + sequential_save_mode = { + "bmp", + "dpx", + "ilbm", + "jpeg", + "png", + "sun", + "tiff", + "tga" + } + + default_save_mode = "\"PNG\"" + save_mode_for_family = { + "review": "\"PNG\"", + "renderPass": "\"PNG\"", + "renderLayer": "\"PNG\"", + } + + def process(self, instance): + # Get all layers and filter out not visible + layers = instance.data["layers"] + filtered_layers = [ + layer + for layer in layers + if layer["visible"] + ] + + family_lowered = instance.data["family"].lower() + + self._prepare_save_modes() + + save_mode = self.save_mode_for_family.get( + family_lowered, self.default_save_mode + ) + filename_template = self._get_filename_template(save_mode) + ext = os.path.splitext(filename_template)[1].replace(".", "") + + self.log.debug( + "Using save mode > {} < and file template \"{}\"".format( + save_mode, filename_template + ) + ) + + tags = ["review"] + + # TODO: This should be already collected!!! + first_frame = int(lib.execute_george("tv_firstimage")) + last_frame = int(lib.execute_george("tv_lastimage")) + + # Save to temp + output_dir = tempfile.mkdtemp().replace("\\", "/") + self.log.debug( + "Files will be rendered to folder: {}".format(output_dir) + ) + output_files_by_frame = self.render( + save_mode, filename_template, output_dir, + filtered_layers, first_frame, last_frame + ) + self.fill_missing_frames( + output_files_by_frame, + first_frame, + last_frame, + filename_template + ) + + representations = instance.data.get("representations") or [] + repre_files = [ + os.path.basename(filepath) + for filepath in output_files_by_frame.values() + ] + new_repre = { + "name": ext, + "ext": ext, + "files": repre_files, + "stagingDir": output_dir, + "frameStart": first_frame, + "frameEnd": last_frame, + "tags": tags + } + self.log.debug("Creating new representation: {}".format(new_repre)) + representations.append(new_repre) + instance.data["representations"] = representations + + def _prepare_save_modes(self): + """Lower family names in keys and skip empty values.""" + new_specifications = {} + for key, value in self.save_mode_for_family.items(): + if value: + new_specifications[key.lower()] = value + else: + self.log.warning(( + "Save mode for family \"{}\" has empty value." + " The family will use default save mode: > {} <." + ).format(key, self.default_save_mode)) + self.save_mode_for_family = new_specifications + + def _get_filename_template(self, save_mode): + """Get filetemplate for rendered files. + + This is simple template contains `{frame}{ext}` for sequential outputs + and `single_file{ext}` for single file output. Output is rendered to + temporary folder so filename should not matter as integrator change + them. + """ + _save_mode = save_mode.lower() + _save_mode = _save_mode.split(" ")[0] + _save_mode = _save_mode.replace("\"", "") + self.log.info(_save_mode) + ext = self.save_mode_to_ext.get(_save_mode) + if ext is None: + raise AssertionError(( + "Couldn't find file extension for TVPaint's save mode: > {} <" + ).format(save_mode)) + + is_sequence = bool(_save_mode in self.sequential_save_mode) + if is_sequence: + template = "{frame}" + ext + else: + template = "single_file" + ext + return template + + def render( + self, save_mode, filename_template, output_dir, layers, + first_frame, last_frame + ): + """ Export images from TVPaint. + + Args: + save_mode (str): Argument for `tv_savemode` george script function. + More about save mode in documentation. + filename_template (str): Filename template of an output. Template + should already contain extension. Template may contain only + keyword argument `{frame}` or index argument (for same value). + Extension in template must match `save_mode`. + layers (list): List of layers to be exported. + first_frame (int): Starting frame from which export will begin. + last_frame (int): On which frame export will end. + + Retruns: + dict: Mapping frame to output filepath. + """ + + # Add save mode arguments to function + save_mode = "tv_SaveMode {}".format(save_mode) + + layers_by_position = { + layer["position"]: layer + for layer in layers + } + + sorted_positions = list(reversed(sorted(layers_by_position.keys()))) + if not sorted_positions: + return + + # Create temporary layer + new_layer_id = lib.execute_george("tv_layercreate _tmp_layer") + + # Merge layers to temp layer + george_script_lines = [] + # Set duplicated layer as current + george_script_lines.append("tv_layerset {}".format(new_layer_id)) + for position in sorted_positions: + layer = layers_by_position[position] + george_script_lines.append( + "tv_layermerge {}".format(layer["layer_id"]) + ) + + lib.execute_george_through_file("\n".join(george_script_lines)) + + # Frames with keyframe + exposure_frames = lib.get_exposure_frames( + new_layer_id, first_frame, last_frame + ) + + # Restart george script lines + george_script_lines = [] + george_script_lines.append(save_mode) + + all_output_files = {} + for frame in exposure_frames: + filename = filename_template.format(frame, frame=frame) + dst_path = "/".join([output_dir, filename]) + all_output_files[frame] = os.path.normpath(dst_path) + + # Go to frame + george_script_lines.append("tv_layerImage {}".format(frame)) + # Store image to output + george_script_lines.append("tv_saveimage \"{}\"".format(dst_path)) + + # Delete temporary layer + george_script_lines.append("tv_layerkill {}".format(new_layer_id)) + + lib.execute_george_through_file("\n".join(george_script_lines)) + + return all_output_files + + def fill_missing_frames( + self, filepaths_by_frame, first_frame, last_frame, filename_template + ): + output_dir = None + previous_frame_filepath = None + for frame in range(first_frame, last_frame + 1): + if frame in filepaths_by_frame: + previous_frame_filepath = filepaths_by_frame[frame] + continue + + if output_dir is None: + output_dir = os.path.dirname(previous_frame_filepath) + + filename = filename_template.format(frame=frame) + space_filepath = os.path.normpath( + os.path.join(output_dir, filename) + ) + filepaths_by_frame[frame] = space_filepath + shutil.copy(previous_frame_filepath, space_filepath) From 439f9557fb103075ac3566afeb1b5c0c2b2846bf Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 11 Nov 2020 11:17:47 +0100 Subject: [PATCH 068/147] extractor has more checks and can export only sequences --- .../plugins/tvpaint/publish/extract_output.py | 90 +++++++++++++------ 1 file changed, 65 insertions(+), 25 deletions(-) diff --git a/pype/plugins/tvpaint/publish/extract_output.py b/pype/plugins/tvpaint/publish/extract_output.py index ffb9143195..a302e8a6e8 100644 --- a/pype/plugins/tvpaint/publish/extract_output.py +++ b/pype/plugins/tvpaint/publish/extract_output.py @@ -51,22 +51,43 @@ class ExtractOutput(pyblish.api.Extractor): } def process(self, instance): + self.log.info( + "* Processing instance \"{}\"".format(instance.data["label"]) + ) + # Get all layers and filter out not visible + # TODO what to do if all are invisible? + # - skip without output? + # - still render but empty output? layers = instance.data["layers"] filtered_layers = [ layer for layer in layers if layer["visible"] ] - - family_lowered = instance.data["family"].lower() - + layer_ids = [layer["layer_id"] for layer in filtered_layers] + self.log.debug("Instance has {} layers with ids: {}".format( + len(filtered_layers), ", ".join(layer_ids) + )) + # This is plugin attribe cleanup method self._prepare_save_modes() + family_lowered = instance.data["family"].lower() save_mode = self.save_mode_for_family.get( family_lowered, self.default_save_mode ) - filename_template = self._get_filename_template(save_mode) + save_mode_type = self._get_save_mode_type(save_mode) + + is_sequence = bool(save_mode_type in self.sequential_save_mode) + if not is_sequence: + raise AssertionError(( + "Plugin can export only sequential frame output" + " but save mode for family \"{}\" is not for sequence > {} <" + ).format(instance.data["family"], save_mode)) + + filename_template = self._get_filename_template( + save_mode_type, save_mode + ) ext = os.path.splitext(filename_template)[1].replace(".", "") self.log.debug( @@ -75,21 +96,21 @@ class ExtractOutput(pyblish.api.Extractor): ) ) - tags = ["review"] - - # TODO: This should be already collected!!! - first_frame = int(lib.execute_george("tv_firstimage")) - last_frame = int(lib.execute_george("tv_lastimage")) - # Save to temp output_dir = tempfile.mkdtemp().replace("\\", "/") self.log.debug( "Files will be rendered to folder: {}".format(output_dir) ) + + first_frame = instance.data["frameStart"] + last_frame = instance.data["frameEnd"] + + # Render output output_files_by_frame = self.render( save_mode, filename_template, output_dir, filtered_layers, first_frame, last_frame ) + # Fill gaps in sequence self.fill_missing_frames( output_files_by_frame, first_frame, @@ -97,7 +118,14 @@ class ExtractOutput(pyblish.api.Extractor): filename_template ) - representations = instance.data.get("representations") or [] + # Add representation to instance's representations + if instance.data.get("representations") is None: + instance.data["representations"] = [] + + # Fill tags + # TODO where to find out which tags should be added? + tags = ["review"] + repre_files = [ os.path.basename(filepath) for filepath in output_files_by_frame.values() @@ -112,8 +140,8 @@ class ExtractOutput(pyblish.api.Extractor): "tags": tags } self.log.debug("Creating new representation: {}".format(new_repre)) - representations.append(new_repre) - instance.data["representations"] = representations + + instance.data["representations"].append(new_repre) def _prepare_save_modes(self): """Lower family names in keys and skip empty values.""" @@ -128,7 +156,20 @@ class ExtractOutput(pyblish.api.Extractor): ).format(key, self.default_save_mode)) self.save_mode_for_family = new_specifications - def _get_filename_template(self, save_mode): + def _get_save_mode_type(self, save_mode): + """Extract type of save mode. + + Helps to define output files extension. + """ + save_mode_type = ( + save_mode.lower() + .split(" ")[0] + .replace("\"", "") + ) + self.log.debug("Save mode type is \"{}\"".format(save_mode_type)) + return save_mode_type + + def _get_filename_template(self, save_mode_type, save_mode): """Get filetemplate for rendered files. This is simple template contains `{frame}{ext}` for sequential outputs @@ -136,22 +177,13 @@ class ExtractOutput(pyblish.api.Extractor): temporary folder so filename should not matter as integrator change them. """ - _save_mode = save_mode.lower() - _save_mode = _save_mode.split(" ")[0] - _save_mode = _save_mode.replace("\"", "") - self.log.info(_save_mode) - ext = self.save_mode_to_ext.get(_save_mode) + ext = self.save_mode_to_ext.get(save_mode_type) if ext is None: raise AssertionError(( "Couldn't find file extension for TVPaint's save mode: > {} <" ).format(save_mode)) - is_sequence = bool(_save_mode in self.sequential_save_mode) - if is_sequence: - template = "{frame}" + ext - else: - template = "single_file" + ext - return template + return "{frame}" + ext def render( self, save_mode, filename_template, output_dir, layers, @@ -177,11 +209,13 @@ class ExtractOutput(pyblish.api.Extractor): # Add save mode arguments to function save_mode = "tv_SaveMode {}".format(save_mode) + # Map layers by position layers_by_position = { layer["position"]: layer for layer in layers } + # Sort layer positions in reverse order sorted_positions = list(reversed(sorted(layers_by_position.keys()))) if not sorted_positions: return @@ -231,6 +265,12 @@ class ExtractOutput(pyblish.api.Extractor): def fill_missing_frames( self, filepaths_by_frame, first_frame, last_frame, filename_template ): + """Fill not rendered frames with previous frame. + + Extractor is rendering only frames with keyframes (exposure frames) to + get output faster which means there may be gaps between frames. + This function fill the missing frames. + """ output_dir = None previous_frame_filepath = None for frame in range(first_frame, last_frame + 1): From 36e914e16d1150273e3873d23e792e84d5998819 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 11 Nov 2020 11:40:32 +0100 Subject: [PATCH 069/147] add tvpaint to extract review --- pype/plugins/global/publish/extract_review.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 87afdd2b20..ffdfd3b91f 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -30,7 +30,8 @@ class ExtractReview(pyblish.api.InstancePlugin): "premiere", "harmony", "standalonepublisher", - "fusion" + "fusion", + "tvpaint" ] # Supported extensions From 88fbf6b10b0beae32024ff2aeca1c49fb099f5e2 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 11 Nov 2020 15:11:50 +0100 Subject: [PATCH 070/147] rename extract output to extract sequence --- .../publish/{extract_output.py => extract_sequence.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename pype/plugins/tvpaint/publish/{extract_output.py => extract_sequence.py} (99%) diff --git a/pype/plugins/tvpaint/publish/extract_output.py b/pype/plugins/tvpaint/publish/extract_sequence.py similarity index 99% rename from pype/plugins/tvpaint/publish/extract_output.py rename to pype/plugins/tvpaint/publish/extract_sequence.py index a302e8a6e8..1e63aab432 100644 --- a/pype/plugins/tvpaint/publish/extract_output.py +++ b/pype/plugins/tvpaint/publish/extract_sequence.py @@ -6,8 +6,8 @@ import pyblish.api from avalon.tvpaint import pipeline, lib -class ExtractOutput(pyblish.api.Extractor): - label = "Extract Output" +class ExtractSequence(pyblish.api.Extractor): + label = "Extract Sequence" hosts = ["tvpaint"] families = ["review", "renderPass", "renderLayer"] From 1aadb518cf9a4fc35a28f9e7ed08b89a54c043f4 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 11 Nov 2020 15:13:56 +0100 Subject: [PATCH 071/147] set tag properly --- pype/plugins/tvpaint/publish/extract_sequence.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pype/plugins/tvpaint/publish/extract_sequence.py b/pype/plugins/tvpaint/publish/extract_sequence.py index 1e63aab432..5021862a88 100644 --- a/pype/plugins/tvpaint/publish/extract_sequence.py +++ b/pype/plugins/tvpaint/publish/extract_sequence.py @@ -124,7 +124,10 @@ class ExtractSequence(pyblish.api.Extractor): # Fill tags # TODO where to find out which tags should be added? - tags = ["review"] + if family_lowered in ("review", "renderlayer"): + tags = ["review", "ftrack"] + else: + tags = [] repre_files = [ os.path.basename(filepath) From 1478f2559d8640275afac58075ba8e42a70b5925 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 11 Nov 2020 15:19:52 +0100 Subject: [PATCH 072/147] collect end frame for each instance individually --- pype/plugins/tvpaint/publish/collect_instances.py | 13 ++++++++++--- pype/plugins/tvpaint/publish/extract_sequence.py | 1 - 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/pype/plugins/tvpaint/publish/collect_instances.py b/pype/plugins/tvpaint/publish/collect_instances.py index d75ea83a4c..f1b3239f11 100644 --- a/pype/plugins/tvpaint/publish/collect_instances.py +++ b/pype/plugins/tvpaint/publish/collect_instances.py @@ -17,9 +17,6 @@ class CollectInstances(pyblish.api.ContextPlugin): )) for instance_data in workfile_instances: - instance_data["frameStart"] = context.data["frameStart"] - instance_data["frameEnd"] = context.data["frameEnd"] - # Store workfile instance data to instance data instance_data["originData"] = copy.deepcopy(instance_data) # Global instance data modifications @@ -54,6 +51,16 @@ class CollectInstances(pyblish.api.ContextPlugin): "Instance with unknown family \"{}\": {}" ) + frame_start = context.data["frameStart"] + frame_end = frame_start + for layer in instance.data["layers"]: + _frame_end = layer["frame_end"] + if _frame_end > frame_end: + frame_end = _frame_end + + instance.data["frameStart"] = frame_start + instance.data["frameEnd"] = frame_end + self.log.debug("Created instance: {}\n{}".format( instance, json.dumps(instance.data, indent=4) )) diff --git a/pype/plugins/tvpaint/publish/extract_sequence.py b/pype/plugins/tvpaint/publish/extract_sequence.py index 5021862a88..358e91931c 100644 --- a/pype/plugins/tvpaint/publish/extract_sequence.py +++ b/pype/plugins/tvpaint/publish/extract_sequence.py @@ -123,7 +123,6 @@ class ExtractSequence(pyblish.api.Extractor): instance.data["representations"] = [] # Fill tags - # TODO where to find out which tags should be added? if family_lowered in ("review", "renderlayer"): tags = ["review", "ftrack"] else: From 843084633c6feca4d8c609a867019310a7fb055f Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 11 Nov 2020 15:29:52 +0100 Subject: [PATCH 073/147] add pading to output template --- .../tvpaint/publish/extract_sequence.py | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/pype/plugins/tvpaint/publish/extract_sequence.py b/pype/plugins/tvpaint/publish/extract_sequence.py index 358e91931c..775e285852 100644 --- a/pype/plugins/tvpaint/publish/extract_sequence.py +++ b/pype/plugins/tvpaint/publish/extract_sequence.py @@ -85,8 +85,11 @@ class ExtractSequence(pyblish.api.Extractor): " but save mode for family \"{}\" is not for sequence > {} <" ).format(instance.data["family"], save_mode)) + frame_start = instance.data["frameStart"] + frame_end = instance.data["frameEnd"] + filename_template = self._get_filename_template( - save_mode_type, save_mode + save_mode_type, save_mode, frame_end ) ext = os.path.splitext(filename_template)[1].replace(".", "") @@ -102,19 +105,16 @@ class ExtractSequence(pyblish.api.Extractor): "Files will be rendered to folder: {}".format(output_dir) ) - first_frame = instance.data["frameStart"] - last_frame = instance.data["frameEnd"] - # Render output output_files_by_frame = self.render( save_mode, filename_template, output_dir, - filtered_layers, first_frame, last_frame + filtered_layers, frame_start, frame_end ) # Fill gaps in sequence self.fill_missing_frames( output_files_by_frame, - first_frame, - last_frame, + frame_start, + frame_end, filename_template ) @@ -137,8 +137,8 @@ class ExtractSequence(pyblish.api.Extractor): "ext": ext, "files": repre_files, "stagingDir": output_dir, - "frameStart": first_frame, - "frameEnd": last_frame, + "frameStart": frame_start, + "frameEnd": frame_end, "tags": tags } self.log.debug("Creating new representation: {}".format(new_repre)) @@ -171,7 +171,7 @@ class ExtractSequence(pyblish.api.Extractor): self.log.debug("Save mode type is \"{}\"".format(save_mode_type)) return save_mode_type - def _get_filename_template(self, save_mode_type, save_mode): + def _get_filename_template(self, save_mode_type, save_mode, frame_end): """Get filetemplate for rendered files. This is simple template contains `{frame}{ext}` for sequential outputs @@ -185,7 +185,12 @@ class ExtractSequence(pyblish.api.Extractor): "Couldn't find file extension for TVPaint's save mode: > {} <" ).format(save_mode)) - return "{frame}" + ext + frame_padding = 4 + frame_end_str_len = len(str(frame_end)) + if frame_end_str_len > frame_padding: + frame_padding = frame_end_str_len + + return "{{frame:0>{}}}".format(frame_padding) + ext def render( self, save_mode, filename_template, output_dir, layers, From 0d2ea16566e8b9ef1023021c269bf55a05dfa6d5 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 11 Nov 2020 15:47:12 +0100 Subject: [PATCH 074/147] added exporting of thumbnail and create of thumbnail representation --- .../tvpaint/publish/extract_sequence.py | 39 ++++++++++++++++++- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/pype/plugins/tvpaint/publish/extract_sequence.py b/pype/plugins/tvpaint/publish/extract_sequence.py index 775e285852..ffb79a853e 100644 --- a/pype/plugins/tvpaint/publish/extract_sequence.py +++ b/pype/plugins/tvpaint/publish/extract_sequence.py @@ -105,11 +105,17 @@ class ExtractSequence(pyblish.api.Extractor): "Files will be rendered to folder: {}".format(output_dir) ) + thumbnail_filename = "thumbnail" + # Render output output_files_by_frame = self.render( save_mode, filename_template, output_dir, - filtered_layers, frame_start, frame_end + filtered_layers, frame_start, frame_end, thumbnail_filename ) + thumbnail_fullpath = output_files_by_frame.pop( + output_files_by_frame, None + ) + # Fill gaps in sequence self.fill_missing_frames( output_files_by_frame, @@ -145,6 +151,19 @@ class ExtractSequence(pyblish.api.Extractor): instance.data["representations"].append(new_repre) + if not thumbnail_fullpath: + return + + # Create thumbnail representation + thumbnail_repre = { + "name": "thumbnail", + "ext": ext, + "files": [os.path.basename(thumbnail_fullpath)], + "stagingDir": output_dir, + "tags": ["thumbnail"] + } + instance.data["representations"].append(thumbnail_repre) + def _prepare_save_modes(self): """Lower family names in keys and skip empty values.""" new_specifications = {} @@ -194,7 +213,7 @@ class ExtractSequence(pyblish.api.Extractor): def render( self, save_mode, filename_template, output_dir, layers, - first_frame, last_frame + first_frame, last_frame, thumbnail_filename ): """ Export images from TVPaint. @@ -262,6 +281,22 @@ class ExtractSequence(pyblish.api.Extractor): # Store image to output george_script_lines.append("tv_saveimage \"{}\"".format(dst_path)) + # Export thumbnail + if thumbnail_filename: + basename, ext = os.path.splitext(thumbnail_filename) + if not ext: + ext = ".png" + thumbnail_fullpath = "/".join([output_dir, basename + ext]) + all_output_files[thumbnail_filename] = thumbnail_fullpath + # Force save mode to png for thumbnail + george_script_lines.append("tv_SaveMode \"PNG\"") + # Go to frame + george_script_lines.append("tv_layerImage {}".format(first_frame)) + # Store image to output + george_script_lines.append( + "tv_saveimage \"{}\"".format(thumbnail_fullpath) + ) + # Delete temporary layer george_script_lines.append("tv_layerkill {}".format(new_layer_id)) From 2780ce537765ca0a0a3e707f28dbb5ab484909fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Wed, 11 Nov 2020 16:05:59 +0100 Subject: [PATCH 075/147] make expected files abstract --- pype/lib/abstract_collect_render.py | 49 ++++++++++++++++++----------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/pype/lib/abstract_collect_render.py b/pype/lib/abstract_collect_render.py index 33202b6ac3..457c0906a4 100644 --- a/pype/lib/abstract_collect_render.py +++ b/pype/lib/abstract_collect_render.py @@ -127,7 +127,17 @@ class AbstractCollectRender(pyblish.api.ContextPlugin): render_instances = self.get_instances() for render_instance in render_instances: - exp_files = self._get_expected_files(render_instance) + exp_files = self.get_expected_files(render_instance) + assert exp_files, "no file names were generated, this is bug" + + # if we want to attach render to subset, check if we have AOV's + # in expectedFiles. If so, raise error as we cannot attach AOV + # (considered to be subset on its own) to another subset + if render_instance.attachTo: + assert isinstance(exp_files, list), ( + "attaching multiple AOVs or renderable cameras to " + "subset is not supported" + ) frame_start_render = int(render_instance.frameStart) frame_end_render = int(render_instance.frameEnd) @@ -210,29 +220,30 @@ class AbstractCollectRender(pyblish.api.ContextPlugin): """ pass - def _get_expected_files(self, render_instance): + @abstractmethod + def get_expected_files(self, render_instance): """Get list of expected files. Returns: - list: expected files. + list: expected files. This can be either simple list of files with + their paths, or list of dictionaries, where key is name of AOV + for example and value is list of files for that AOV. + + Example:: + + ['/path/to/file.001.exr', '/path/to/file.002.exr'] + + or as dictionary: + + [ + { + "beauty": ['/path/to/beauty.001.exr', ...], + "mask": ['/path/to/mask.001.exr'] + } + ] """ - # return all expected files for all cameras and aovs in given - # frame range - ef = ExpectedFiles() - exp_files = ef.get(render_instance) - self.log.info("multipart: {}".format(ef.multipart)) - assert exp_files, "no file names were generated, this is bug" - - # if we want to attach render to subset, check if we have AOV's - # in expectedFiles. If so, raise error as we cannot attach AOV - # (considered to be subset on its own) to another subset - if render_instance.attachTo: - assert isinstance(exp_files, list), ( - "attaching multiple AOVs or renderable cameras to " - "subset is not supported" - ) - return exp_files + pass def add_additional_data(self, data): """Add additional data to collected instance. From 8d1679205fb766747cfdeba84543ba9aa9f0fe37 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 11 Nov 2020 16:59:34 +0100 Subject: [PATCH 076/147] instances has prepared representations and fps --- pype/plugins/tvpaint/publish/collect_instances.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pype/plugins/tvpaint/publish/collect_instances.py b/pype/plugins/tvpaint/publish/collect_instances.py index f1b3239f11..3ac2fecd5f 100644 --- a/pype/plugins/tvpaint/publish/collect_instances.py +++ b/pype/plugins/tvpaint/publish/collect_instances.py @@ -17,6 +17,8 @@ class CollectInstances(pyblish.api.ContextPlugin): )) for instance_data in workfile_instances: + instance_data["fps"] = context.data["fps"] + # Store workfile instance data to instance data instance_data["originData"] = copy.deepcopy(instance_data) # Global instance data modifications @@ -32,6 +34,8 @@ class CollectInstances(pyblish.api.ContextPlugin): active = instance_data.get("active", True) instance_data["active"] = active instance_data["publish"] = active + # Add representations key + instance_data["representations"] = [] # Different instance creation based on family instance = None From 555c9382e7994e4a33200730c4ddad66fc76717e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 11 Nov 2020 17:00:32 +0100 Subject: [PATCH 077/147] few minor fixes --- .../tvpaint/publish/extract_sequence.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/pype/plugins/tvpaint/publish/extract_sequence.py b/pype/plugins/tvpaint/publish/extract_sequence.py index ffb79a853e..003ab673d2 100644 --- a/pype/plugins/tvpaint/publish/extract_sequence.py +++ b/pype/plugins/tvpaint/publish/extract_sequence.py @@ -65,10 +65,12 @@ class ExtractSequence(pyblish.api.Extractor): for layer in layers if layer["visible"] ] - layer_ids = [layer["layer_id"] for layer in filtered_layers] - self.log.debug("Instance has {} layers with ids: {}".format( - len(filtered_layers), ", ".join(layer_ids) - )) + layer_ids = [str(layer["layer_id"]) for layer in filtered_layers] + self.log.debug( + "Instance has {} layers with ids: {}".format( + len(layer_ids), ", ".join(layer_ids) + ) + ) # This is plugin attribe cleanup method self._prepare_save_modes() @@ -113,7 +115,7 @@ class ExtractSequence(pyblish.api.Extractor): filtered_layers, frame_start, frame_end, thumbnail_filename ) thumbnail_fullpath = output_files_by_frame.pop( - output_files_by_frame, None + thumbnail_filename, None ) # Fill gaps in sequence @@ -124,10 +126,6 @@ class ExtractSequence(pyblish.api.Extractor): filename_template ) - # Add representation to instance's representations - if instance.data.get("representations") is None: - instance.data["representations"] = [] - # Fill tags if family_lowered in ("review", "renderlayer"): tags = ["review", "ftrack"] @@ -158,7 +156,7 @@ class ExtractSequence(pyblish.api.Extractor): thumbnail_repre = { "name": "thumbnail", "ext": ext, - "files": [os.path.basename(thumbnail_fullpath)], + "files": os.path.basename(thumbnail_fullpath), "stagingDir": output_dir, "tags": ["thumbnail"] } From 601700e1caae22fd81e90207a3c14b68f95ceecf Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 11 Nov 2020 17:00:41 +0100 Subject: [PATCH 078/147] added warning message --- pype/plugins/tvpaint/publish/extract_sequence.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pype/plugins/tvpaint/publish/extract_sequence.py b/pype/plugins/tvpaint/publish/extract_sequence.py index 003ab673d2..906c183b90 100644 --- a/pype/plugins/tvpaint/publish/extract_sequence.py +++ b/pype/plugins/tvpaint/publish/extract_sequence.py @@ -318,6 +318,12 @@ class ExtractSequence(pyblish.api.Extractor): previous_frame_filepath = filepaths_by_frame[frame] continue + elif previous_frame_filepath is None: + self.log.warning( + "No frames to fill. Seems like nothing was exported." + ) + break + if output_dir is None: output_dir = os.path.dirname(previous_frame_filepath) From 4b5b725521fdc849b8bfcb3322a81cf326ffa625 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 11 Nov 2020 17:13:10 +0100 Subject: [PATCH 079/147] export first frame all the time --- pype/plugins/tvpaint/publish/extract_sequence.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pype/plugins/tvpaint/publish/extract_sequence.py b/pype/plugins/tvpaint/publish/extract_sequence.py index 906c183b90..de0a547540 100644 --- a/pype/plugins/tvpaint/publish/extract_sequence.py +++ b/pype/plugins/tvpaint/publish/extract_sequence.py @@ -264,6 +264,11 @@ class ExtractSequence(pyblish.api.Extractor): new_layer_id, first_frame, last_frame ) + # TODO what if there is not exposue frames? + # - this force to have first frame all the time + if first_frame not in exposure_frames: + exposure_frames.insert(0, first_frame) + # Restart george script lines george_script_lines = [] george_script_lines.append(save_mode) From 7c4f408e7777cd30d86d0ddd3e6a84fa2c4a1356 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 11 Nov 2020 17:39:00 +0100 Subject: [PATCH 080/147] added renderLayer and renderPass to integration --- pype/plugins/global/publish/integrate_new.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 4dc6006076..573228f01b 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -90,7 +90,9 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): "harmony.template", "harmony.palette", "editorial", - "background" + "background", + "renderLayer", + "renderPass" ] exclude_families = ["clip"] db_representation_context_keys = [ From 0525075a694af7c03b5d4b33ed8e0c93cb5b103e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 11 Nov 2020 17:52:24 +0100 Subject: [PATCH 081/147] use stagingDir on instance output --- pype/plugins/tvpaint/publish/extract_sequence.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pype/plugins/tvpaint/publish/extract_sequence.py b/pype/plugins/tvpaint/publish/extract_sequence.py index de0a547540..8eef84cc9c 100644 --- a/pype/plugins/tvpaint/publish/extract_sequence.py +++ b/pype/plugins/tvpaint/publish/extract_sequence.py @@ -101,8 +101,13 @@ class ExtractSequence(pyblish.api.Extractor): ) ) - # Save to temp - output_dir = tempfile.mkdtemp().replace("\\", "/") + # Save to staging dir + output_dir = instance.data.get("stagingDir") + if not output_dir: + # Create temp folder if staging dir is not set + output_dir = tempfile.mkdtemp().replace("\\", "/") + instance.data["stagingDir"] = output_dir + self.log.debug( "Files will be rendered to folder: {}".format(output_dir) ) From 4beb6d620f355563494fb8abbd079cab18bdab66 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 11 Nov 2020 18:10:46 +0100 Subject: [PATCH 082/147] #698 - added collect_render --- .../publish/collect_current_file.py | 2 -- .../aftereffects/publish/collect_render.py | 27 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 pype/plugins/aftereffects/publish/collect_render.py diff --git a/pype/plugins/aftereffects/publish/collect_current_file.py b/pype/plugins/aftereffects/publish/collect_current_file.py index f79b8a2876..b59ff41a0e 100644 --- a/pype/plugins/aftereffects/publish/collect_current_file.py +++ b/pype/plugins/aftereffects/publish/collect_current_file.py @@ -16,5 +16,3 @@ class CollectCurrentFile(pyblish.api.ContextPlugin): context.data["currentFile"] = os.path.normpath( aftereffects.stub().get_active_document_full_name() ).replace("\\", "/") - - #print(debug_pyblish_plugins_loading.discover()) diff --git a/pype/plugins/aftereffects/publish/collect_render.py b/pype/plugins/aftereffects/publish/collect_render.py new file mode 100644 index 0000000000..7fbeb37790 --- /dev/null +++ b/pype/plugins/aftereffects/publish/collect_render.py @@ -0,0 +1,27 @@ +from pype.lib import abstract_collect_render +import pyblish.api +from abc import ABCMeta +import pyblish.api +import six + +from avalon import aftereffects + +@six.add_metaclass(ABCMeta) +class CollectRender(abstract_collect_render.AbstractCollectRender): + + order = pyblish.api.CollectorOrder + 0.01 + hosts = ["maya"] + label = "Collect Render Layers" + sync_workfile_version = False + + def get_instances(self, context): + import web_pdb + web_pdb.set_trace() + print("hello {}".format(context)) + return aftereffects.stub().get_metadata() + + def get_expected_files(self, render_instance): + import web_pdb + web_pdb.set_trace() + + return [] \ No newline at end of file From 417e167c7e15c9705d45a045696ef0d07460f411 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 11 Nov 2020 18:16:12 +0100 Subject: [PATCH 083/147] added more families and removed tags --- .../tvpaint/publish/extract_sequence.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/pype/plugins/tvpaint/publish/extract_sequence.py b/pype/plugins/tvpaint/publish/extract_sequence.py index 8eef84cc9c..c4aba77f60 100644 --- a/pype/plugins/tvpaint/publish/extract_sequence.py +++ b/pype/plugins/tvpaint/publish/extract_sequence.py @@ -131,11 +131,21 @@ class ExtractSequence(pyblish.api.Extractor): filename_template ) - # Fill tags + # Fill tags and new families + tags = [] + new_families = ["review"] if family_lowered in ("review", "renderlayer"): - tags = ["review", "ftrack"] - else: - tags = [] + # QUESTION Thould this be set here or in collector? + # Add ftrack family + new_families.append("ftrack") + # Add `ftrackreview` tag + tags.append("ftrackreview") + # QUESTION we still use this? + instance.data["review"] = True + + for new_family in new_families: + if new_family not in instance.data["families"]: + instance.data["families"].append(new_family) repre_files = [ os.path.basename(filepath) From 1d35a8686be50496f475089d563f092e673f6913 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 11 Nov 2020 18:16:44 +0100 Subject: [PATCH 084/147] collect instances works more as should --- pype/plugins/tvpaint/publish/collect_instances.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pype/plugins/tvpaint/publish/collect_instances.py b/pype/plugins/tvpaint/publish/collect_instances.py index 3ac2fecd5f..87cf52e43e 100644 --- a/pype/plugins/tvpaint/publish/collect_instances.py +++ b/pype/plugins/tvpaint/publish/collect_instances.py @@ -42,6 +42,7 @@ class CollectInstances(pyblish.api.ContextPlugin): if family == "review": instance = context.create_instance(**instance_data) instance.data["layers"] = context.data["layersData"] + elif family == "renderLayer": instance = self.create_render_layer_instance( context, instance_data @@ -82,10 +83,10 @@ class CollectInstances(pyblish.api.ContextPlugin): if not group_layers: # Should be handled here? - self.log.warning( + self.log.warning(( f"Group with id {group_id} does not contain any layers." f" Instance \"{name}\" not created." - ) + )) return None instance_data["layers"] = group_layers @@ -93,6 +94,9 @@ class CollectInstances(pyblish.api.ContextPlugin): def create_render_pass_instance(self, context, instance_data): pass_name = instance_data["pass"] + self.log.info( + "Creating render pass instance. \"{}\"".format(pass_name) + ) render_layer = instance_data["render_layer"] instance_data["label"] = "{}_{}".format(render_layer, pass_name) @@ -113,9 +117,9 @@ class CollectInstances(pyblish.api.ContextPlugin): # Move to validator? if layer["group_id"] != group_id: - self.log.warning( - f"Layer with id {layer_id} is in different group." - ) + self.log.warning(( + "Layer \"{}\" with id < {} > is in different group." + ).format(layer["name"], layer_id)) continue render_pass_layers.append(layer) From de678a89842b470fcec2a6a47d04858a2a4a1e8c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 11 Nov 2020 19:14:25 +0100 Subject: [PATCH 085/147] removed unused import --- pype/plugins/tvpaint/publish/extract_sequence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/plugins/tvpaint/publish/extract_sequence.py b/pype/plugins/tvpaint/publish/extract_sequence.py index c4aba77f60..63a1e57695 100644 --- a/pype/plugins/tvpaint/publish/extract_sequence.py +++ b/pype/plugins/tvpaint/publish/extract_sequence.py @@ -3,7 +3,7 @@ import shutil import tempfile import pyblish.api -from avalon.tvpaint import pipeline, lib +from avalon.tvpaint import lib class ExtractSequence(pyblish.api.Extractor): From 0788f9ae52e3fde8497dfa46bb131cc82d095cde Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 12 Nov 2020 09:41:24 +0100 Subject: [PATCH 086/147] renamed beauty creator --- pype/plugins/tvpaint/create/create_render_layer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/plugins/tvpaint/create/create_render_layer.py b/pype/plugins/tvpaint/create/create_render_layer.py index f261466681..c2921cebbe 100644 --- a/pype/plugins/tvpaint/create/create_render_layer.py +++ b/pype/plugins/tvpaint/create/create_render_layer.py @@ -1,7 +1,7 @@ from avalon.tvpaint import pipeline, lib -class CreateBeauty(pipeline.Creator): +class CreateRenderlayer(pipeline.Creator): """Mark layer group as one instance.""" name = "render_layer" label = "RenderLayer" From dfece60dbc1313128b89dcd10679c37129ed90ed Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 12 Nov 2020 09:42:48 +0100 Subject: [PATCH 087/147] changed log message --- pype/plugins/tvpaint/publish/collect_instances.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/plugins/tvpaint/publish/collect_instances.py b/pype/plugins/tvpaint/publish/collect_instances.py index 87cf52e43e..68d0620253 100644 --- a/pype/plugins/tvpaint/publish/collect_instances.py +++ b/pype/plugins/tvpaint/publish/collect_instances.py @@ -126,8 +126,8 @@ class CollectInstances(pyblish.api.ContextPlugin): if not render_pass_layers: name = instance_data["name"] self.log.warning( - f"All layers from RenderPass \"{name}\" do not exist." - " Instance not created." + f"None of the layers from the RenderPass \"{name}\"" + " exist anymore. Instance not created." ) return None From 0bfe5b6e61b90acbbdd71a59dfd6ca4c75ae7af8 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 12 Nov 2020 09:43:16 +0100 Subject: [PATCH 088/147] fixed typo --- pype/plugins/tvpaint/publish/collect_workfile_data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pype/plugins/tvpaint/publish/collect_workfile_data.py b/pype/plugins/tvpaint/publish/collect_workfile_data.py index b27eb2f693..31fd97ced4 100644 --- a/pype/plugins/tvpaint/publish/collect_workfile_data.py +++ b/pype/plugins/tvpaint/publish/collect_workfile_data.py @@ -50,7 +50,7 @@ class CollectWorkfileData(pyblish.api.ContextPlugin): if frame_end > last_frame: last_frame = frame_end - sceme_data = { + scene_data = { "currentFile": workfile_path, "sceneWidth": width, "sceneHeight": height, @@ -61,6 +61,6 @@ class CollectWorkfileData(pyblish.api.ContextPlugin): "fieldOrder": field_order } self.log.debug( - "Scene data: {}".format(json.dumps(sceme_data, indent=4)) + "Scene data: {}".format(json.dumps(scene_data, indent=4)) ) - context.data.update(sceme_data) + context.data.update(scene_data) From df4ecca987bc9e0ff77be6c6a2ada29a735ccb4f Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 12 Nov 2020 09:49:25 +0100 Subject: [PATCH 089/147] moved family definition from extractor to collector --- pype/plugins/tvpaint/publish/collect_instances.py | 9 ++++++++- pype/plugins/tvpaint/publish/extract_sequence.py | 10 ---------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/pype/plugins/tvpaint/publish/collect_instances.py b/pype/plugins/tvpaint/publish/collect_instances.py index 68d0620253..169b969bc4 100644 --- a/pype/plugins/tvpaint/publish/collect_instances.py +++ b/pype/plugins/tvpaint/publish/collect_instances.py @@ -24,7 +24,8 @@ class CollectInstances(pyblish.api.ContextPlugin): # Global instance data modifications # Fill families family = instance_data["family"] - instance_data["families"] = [family] + # Add `review` family for thumbnail integration + instance_data["families"] = [family, "review"] # Instance name subset_name = instance_data["subset"] @@ -42,6 +43,8 @@ class CollectInstances(pyblish.api.ContextPlugin): if family == "review": instance = context.create_instance(**instance_data) instance.data["layers"] = context.data["layersData"] + # Add ftrack family + instance.data["families"].append("ftrack") elif family == "renderLayer": instance = self.create_render_layer_instance( @@ -90,6 +93,10 @@ class CollectInstances(pyblish.api.ContextPlugin): return None instance_data["layers"] = group_layers + + # Add ftrack family + instance_data["families"].append("ftrack") + return context.create_instance(**instance_data) def create_render_pass_instance(self, context, instance_data): diff --git a/pype/plugins/tvpaint/publish/extract_sequence.py b/pype/plugins/tvpaint/publish/extract_sequence.py index 63a1e57695..c899ecb9e6 100644 --- a/pype/plugins/tvpaint/publish/extract_sequence.py +++ b/pype/plugins/tvpaint/publish/extract_sequence.py @@ -133,19 +133,9 @@ class ExtractSequence(pyblish.api.Extractor): # Fill tags and new families tags = [] - new_families = ["review"] if family_lowered in ("review", "renderlayer"): - # QUESTION Thould this be set here or in collector? - # Add ftrack family - new_families.append("ftrack") # Add `ftrackreview` tag tags.append("ftrackreview") - # QUESTION we still use this? - instance.data["review"] = True - - for new_family in new_families: - if new_family not in instance.data["families"]: - instance.data["families"].append(new_family) repre_files = [ os.path.basename(filepath) From 426bc1e92bb739f7ee467c1d736b724ac34ef59d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 12 Nov 2020 09:51:31 +0100 Subject: [PATCH 090/147] skip instances without visible layers --- pype/plugins/tvpaint/publish/extract_sequence.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pype/plugins/tvpaint/publish/extract_sequence.py b/pype/plugins/tvpaint/publish/extract_sequence.py index c899ecb9e6..239dec5249 100644 --- a/pype/plugins/tvpaint/publish/extract_sequence.py +++ b/pype/plugins/tvpaint/publish/extract_sequence.py @@ -56,9 +56,6 @@ class ExtractSequence(pyblish.api.Extractor): ) # Get all layers and filter out not visible - # TODO what to do if all are invisible? - # - skip without output? - # - still render but empty output? layers = instance.data["layers"] filtered_layers = [ layer @@ -66,6 +63,13 @@ class ExtractSequence(pyblish.api.Extractor): if layer["visible"] ] layer_ids = [str(layer["layer_id"]) for layer in filtered_layers] + if not layer_ids: + self.log.info( + f"None of the layers from the instance" + " are visible. Extraction skipped." + ) + return + self.log.debug( "Instance has {} layers with ids: {}".format( len(layer_ids), ", ".join(layer_ids) From f179929662058f704092f957c2ea06744ec30cea Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 12 Nov 2020 10:24:22 +0100 Subject: [PATCH 091/147] moved validation of render pass group id to validator --- .../tvpaint/publish/collect_instances.py | 6 -- .../tvpaint/publish/validate_frame_range.py | 76 +++++++++++++++++++ 2 files changed, 76 insertions(+), 6 deletions(-) create mode 100644 pype/plugins/tvpaint/publish/validate_frame_range.py diff --git a/pype/plugins/tvpaint/publish/collect_instances.py b/pype/plugins/tvpaint/publish/collect_instances.py index 169b969bc4..3f5c1de367 100644 --- a/pype/plugins/tvpaint/publish/collect_instances.py +++ b/pype/plugins/tvpaint/publish/collect_instances.py @@ -122,12 +122,6 @@ class CollectInstances(pyblish.api.ContextPlugin): self.log.warning(f"Layer with id {layer_id} was not found.") continue - # Move to validator? - if layer["group_id"] != group_id: - self.log.warning(( - "Layer \"{}\" with id < {} > is in different group." - ).format(layer["name"], layer_id)) - continue render_pass_layers.append(layer) if not render_pass_layers: diff --git a/pype/plugins/tvpaint/publish/validate_frame_range.py b/pype/plugins/tvpaint/publish/validate_frame_range.py new file mode 100644 index 0000000000..cb9a103fa2 --- /dev/null +++ b/pype/plugins/tvpaint/publish/validate_frame_range.py @@ -0,0 +1,76 @@ +import collections +import pyblish.api + + +class ValidateLayersGroup(pyblish.api.InstancePlugin): + """Validate group ids of renderPass layers. + + Validates that all layers are in same group as they were during creation. + """ + + label = "Validate Layers Group" + order = pyblish.api.ValidatorOrder + families = ["renderPass"] + + def process(self, instance): + # Prepare layers + layers_data = instance.context.data["layersData"] + layers_by_id = { + layer["layer_id"]: layer + for layer in layers_data + } + + # Expected group id for instance layers + group_id = instance.data["group_id"] + # Layers ids of an instance + layer_ids = instance.data["layer_ids"] + # Check if all layers from render pass are in right group + invalid_layers_by_group_id = collections.defaultdict(list) + for layer_id in layer_ids: + layer = layers_by_id.get(layer_id) + _group_id = layer["group_id"] + if _group_id != group_id: + invalid_layers_by_group_id[_group_id].append(layer) + + # Everything is OK and skip exception + if not invalid_layers_by_group_id: + return + + # Exception message preparations + groups_data = instance.context.data["groupsData"] + groups_by_id = { + group["group_id"]: group + for group in groups_data + } + correct_group = groups_by_id[group_id] + + per_group_msgs = [] + for _group_id, layers in invalid_layers_by_group_id.items(): + _group = groups_by_id[_group_id] + layers_msgs = [] + for layer in layers: + layers_msgs.append( + "\"{}\" (id: {})".format(layer["name"], layer["layer_id"]) + ) + per_group_msgs.append( + "Group \"{}\" (id: {}) < {} >".format( + _group["name"], + _group["group_id"], + ", ".join(layers_msgs) + ) + ) + + # Raise an error + raise AssertionError(( + # Short message + "Layers in wrong group." + # Description what's wrong + " Layers from render pass \"{}\" must be in group {} (id: {})." + # Detailed message + " Layers in wrong group: {}" + ).format( + instance.data["label"], + correct_group["name"], + correct_group["group_id"], + " | ".join(per_group_msgs) + )) From 49f611701b3e41066578e0d15a507faa7f621792 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 12 Nov 2020 10:25:46 +0100 Subject: [PATCH 092/147] removed unused variable --- pype/plugins/tvpaint/publish/collect_instances.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pype/plugins/tvpaint/publish/collect_instances.py b/pype/plugins/tvpaint/publish/collect_instances.py index 3f5c1de367..9e8e5e7832 100644 --- a/pype/plugins/tvpaint/publish/collect_instances.py +++ b/pype/plugins/tvpaint/publish/collect_instances.py @@ -113,7 +113,6 @@ class CollectInstances(pyblish.api.ContextPlugin): for layer in layers_data } - group_id = instance_data["group_id"] layer_ids = instance_data["layer_ids"] render_pass_layers = [] for layer_id in layer_ids: From 86d1bededabd7fe0a7e15c8f7d40bea8edd2484c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 12 Nov 2020 10:54:42 +0100 Subject: [PATCH 093/147] change family to `render` in extractor --- pype/plugins/tvpaint/publish/extract_sequence.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pype/plugins/tvpaint/publish/extract_sequence.py b/pype/plugins/tvpaint/publish/extract_sequence.py index 239dec5249..aaad88fd5d 100644 --- a/pype/plugins/tvpaint/publish/extract_sequence.py +++ b/pype/plugins/tvpaint/publish/extract_sequence.py @@ -158,6 +158,10 @@ class ExtractSequence(pyblish.api.Extractor): instance.data["representations"].append(new_repre) + if family_lowered in ("renderpass", "renderlayer"): + # Change family to render + instance.data["family"] = "render" + if not thumbnail_fullpath: return From 3212a660c75ade81c5e12b245e078131fc768710 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 12 Nov 2020 11:00:59 +0100 Subject: [PATCH 094/147] remove `is_sequence` variable --- pype/plugins/tvpaint/publish/extract_sequence.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pype/plugins/tvpaint/publish/extract_sequence.py b/pype/plugins/tvpaint/publish/extract_sequence.py index aaad88fd5d..d173ac287d 100644 --- a/pype/plugins/tvpaint/publish/extract_sequence.py +++ b/pype/plugins/tvpaint/publish/extract_sequence.py @@ -84,8 +84,7 @@ class ExtractSequence(pyblish.api.Extractor): ) save_mode_type = self._get_save_mode_type(save_mode) - is_sequence = bool(save_mode_type in self.sequential_save_mode) - if not is_sequence: + if not bool(save_mode_type in self.sequential_save_mode): raise AssertionError(( "Plugin can export only sequential frame output" " but save mode for family \"{}\" is not for sequence > {} <" From 6f91253e4181f1857dfa4503e83ca1d02224b6bb Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 12 Nov 2020 11:09:08 +0100 Subject: [PATCH 095/147] fill data in assertion error --- pype/plugins/tvpaint/publish/collect_instances.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pype/plugins/tvpaint/publish/collect_instances.py b/pype/plugins/tvpaint/publish/collect_instances.py index 9e8e5e7832..8add2f01d9 100644 --- a/pype/plugins/tvpaint/publish/collect_instances.py +++ b/pype/plugins/tvpaint/publish/collect_instances.py @@ -56,7 +56,9 @@ class CollectInstances(pyblish.api.ContextPlugin): ) else: raise AssertionError( - "Instance with unknown family \"{}\": {}" + "Instance with unknown family \"{}\": {}".format( + family, instance_data + ) ) frame_start = context.data["frameStart"] From 655c5f0463cd8f0de55af9b47655f587256ac6db Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 12 Nov 2020 11:09:28 +0100 Subject: [PATCH 096/147] change subset name for renderPass and renderLayer in instance collector --- pype/plugins/tvpaint/publish/collect_instances.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pype/plugins/tvpaint/publish/collect_instances.py b/pype/plugins/tvpaint/publish/collect_instances.py index 8add2f01d9..9a3fd67110 100644 --- a/pype/plugins/tvpaint/publish/collect_instances.py +++ b/pype/plugins/tvpaint/publish/collect_instances.py @@ -32,6 +32,15 @@ class CollectInstances(pyblish.api.ContextPlugin): name = instance_data.get("name", subset_name) instance_data["name"] = name + # Replace family in subset name with `render` + # - only for `renderPass` and `renderLayer` + if family.lower() in ("renderlayer", "renderpass"): + new_subset_name = "render".join(subset_name.split(family)) + instance_data["subset"] = new_subset_name + self.log.debug("Changed subset name \"{}\"->\"{}\"".format( + subset_name, new_subset_name + )) + active = instance_data.get("active", True) instance_data["active"] = active instance_data["publish"] = active From aaf4fadf4766d1f178bf464608c40208ad9f82c1 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 12 Nov 2020 11:12:56 +0100 Subject: [PATCH 097/147] remove renderLayer and renderPass from integrate new families --- pype/plugins/global/publish/integrate_new.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 573228f01b..4dc6006076 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -90,9 +90,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): "harmony.template", "harmony.palette", "editorial", - "background", - "renderLayer", - "renderPass" + "background" ] exclude_families = ["clip"] db_representation_context_keys = [ From fae1294616b60f2e3993e24fbdd26c1b8d77e72e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 12 Nov 2020 11:18:46 +0100 Subject: [PATCH 098/147] render layer missed beauty in subset name --- .../tvpaint/publish/collect_instances.py | 35 +++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/pype/plugins/tvpaint/publish/collect_instances.py b/pype/plugins/tvpaint/publish/collect_instances.py index 9a3fd67110..fb8702149a 100644 --- a/pype/plugins/tvpaint/publish/collect_instances.py +++ b/pype/plugins/tvpaint/publish/collect_instances.py @@ -32,15 +32,6 @@ class CollectInstances(pyblish.api.ContextPlugin): name = instance_data.get("name", subset_name) instance_data["name"] = name - # Replace family in subset name with `render` - # - only for `renderPass` and `renderLayer` - if family.lower() in ("renderlayer", "renderpass"): - new_subset_name = "render".join(subset_name.split(family)) - instance_data["subset"] = new_subset_name - self.log.debug("Changed subset name \"{}\"->\"{}\"".format( - subset_name, new_subset_name - )) - active = instance_data.get("active", True) instance_data["active"] = active instance_data["publish"] = active @@ -86,8 +77,22 @@ class CollectInstances(pyblish.api.ContextPlugin): def create_render_layer_instance(self, context, instance_data): name = instance_data["name"] - instance_data["label"] = "{}_beauty".format(name) + subset_name = instance_data["subset"] + # Add beauty to subset name (and label) + beauty_template = "{}_Beauty" + subset_name = beauty_template.format(subset_name) + instance_data["label"] = beauty_template.format(name) + # Replace family in subset name with `render` family + # - this is because final output will be `render` family + family = instance_data["family"] + new_subset_name = "render".join(subset_name.split(family)) + instance_data["subset"] = new_subset_name + self.log.debug("Changed subset name \"{}\"->\"{}\"".format( + subset_name, new_subset_name + )) + + # Get all layers for the layer layers_data = context.data["layersData"] group_id = instance_data["group_id"] group_layers = [] @@ -118,6 +123,16 @@ class CollectInstances(pyblish.api.ContextPlugin): render_layer = instance_data["render_layer"] instance_data["label"] = "{}_{}".format(render_layer, pass_name) + # Replace family in subset name with `render` family + # - this is because final output will be `render` family + family = instance_data["family"] + subset_name = instance_data["subset"] + new_subset_name = "render".join(subset_name.split(family)) + instance_data["subset"] = new_subset_name + self.log.debug("Changed subset name \"{}\"->\"{}\"".format( + subset_name, new_subset_name + )) + layers_data = context.data["layersData"] layers_by_id = { layer["layer_id"]: layer From e3b06613e1af95c658afc586e514e05c5129334e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 12 Nov 2020 11:50:54 +0100 Subject: [PATCH 099/147] subset names are right now with task and right family --- .../tvpaint/publish/collect_instances.py | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/pype/plugins/tvpaint/publish/collect_instances.py b/pype/plugins/tvpaint/publish/collect_instances.py index fb8702149a..a9c19f9c0a 100644 --- a/pype/plugins/tvpaint/publish/collect_instances.py +++ b/pype/plugins/tvpaint/publish/collect_instances.py @@ -1,6 +1,7 @@ import json import copy import pyblish.api +from avalon import io class CollectInstances(pyblish.api.ContextPlugin): @@ -41,6 +42,11 @@ class CollectInstances(pyblish.api.ContextPlugin): # Different instance creation based on family instance = None if family == "review": + # Change subset name + task_name = io.Session["AVALON_TASK"] + new_subset_name = "{}{}".format(family, task_name.capitalize()) + instance_data["subset"] = new_subset_name + instance = context.create_instance(**instance_data) instance.data["layers"] = context.data["layersData"] # Add ftrack family @@ -77,16 +83,17 @@ class CollectInstances(pyblish.api.ContextPlugin): def create_render_layer_instance(self, context, instance_data): name = instance_data["name"] + # Change label subset_name = instance_data["subset"] - # Add beauty to subset name (and label) - beauty_template = "{}_Beauty" - subset_name = beauty_template.format(subset_name) - instance_data["label"] = beauty_template.format(name) + instance_data["label"] = "{}_Beauty".format(name) - # Replace family in subset name with `render` family - # - this is because final output will be `render` family - family = instance_data["family"] - new_subset_name = "render".join(subset_name.split(family)) + # Change subset name + # Final family of an instance will be `render` + new_family = "render" + task_name = io.Session["AVALON_TASK"] + new_subset_name = "{}{}_{}_Beauty".format( + new_family, task_name.capitalize(), name + ) instance_data["subset"] = new_subset_name self.log.debug("Changed subset name \"{}\"->\"{}\"".format( subset_name, new_subset_name @@ -120,17 +127,21 @@ class CollectInstances(pyblish.api.ContextPlugin): self.log.info( "Creating render pass instance. \"{}\"".format(pass_name) ) + # Change label render_layer = instance_data["render_layer"] instance_data["label"] = "{}_{}".format(render_layer, pass_name) - # Replace family in subset name with `render` family - # - this is because final output will be `render` family - family = instance_data["family"] - subset_name = instance_data["subset"] - new_subset_name = "render".join(subset_name.split(family)) + # Change subset name + # Final family of an instance will be `render` + new_family = "render" + old_subset_name = instance_data["subset"] + task_name = io.Session["AVALON_TASK"] + new_subset_name = "{}{}_{}_{}".format( + new_family, task_name.capitalize(), render_layer, pass_name + ) instance_data["subset"] = new_subset_name self.log.debug("Changed subset name \"{}\"->\"{}\"".format( - subset_name, new_subset_name + old_subset_name, new_subset_name )) layers_data = context.data["layersData"] From afa051b16eef118a5e6815e082202c06eca2d333 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Thu, 12 Nov 2020 18:50:54 +0100 Subject: [PATCH 100/147] add description, default values, context handling --- pype/lib/abstract_collect_render.py | 67 ++++++++++++++++------------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/pype/lib/abstract_collect_render.py b/pype/lib/abstract_collect_render.py index 457c0906a4..098788430b 100644 --- a/pype/lib/abstract_collect_render.py +++ b/pype/lib/abstract_collect_render.py @@ -12,7 +12,6 @@ import six from avalon import api import pyblish.api -from .abstract_expected_files import ExpectedFiles from .abstract_metaplugins import AbstractMetaContextPlugin @@ -26,44 +25,44 @@ class RenderInstance(object): """ # metadata - version = attr.ib() - time = attr.ib() - source = attr.ib() - label = attr.ib() - subset = attr.ib() - asset = attr.ib(init=False) - attachTo = attr.ib(init=False) - setMembers = attr.ib() - publish = attr.ib() - renderer = attr.ib() - name = attr.ib() + version = attr.ib() # instance version + time = attr.ib() # time of instance creation (avalon.api.time()) + source = attr.ib() # path to source scene file + label = attr.ib() # label to show in GUI + subset = attr.ib() # subset name + asset = attr.ib() # asset name (AVALON_ASSET) + attachTo = attr.ib() # subset name to attach render to + setMembers = attr.ib() # list of nodes/members producing render output + publish = attr.ib() # bool, True to publish instance + name = attr.ib() # instance name # format settings - resolutionWidth = attr.ib() - resolutionHeight = attr.ib() - pixelAspect = attr.ib() - - tileRendering = attr.ib() - tilesX = attr.ib() - tilesY = attr.ib() + resolutionWidth = attr.ib() # resolution width (1920) + resolutionHeight = attr.ib() # resolution height (1080) + pixelAspect = attr.ib() # pixel aspect (1.0) # time settings - frameStart = attr.ib() - frameEnd = attr.ib() - frameStep = attr.ib() + frameStart = attr.ib() # start frame + frameEnd = attr.ib() # start end + frameStep = attr.ib() # frame step # -------------------- # With default values # metadata - review = attr.ib(default=False) - priority = attr.ib(default=50) + renderer = attr.ib(default="") # renderer - can be used in Deadline + review = attr.ib(default=False) # genereate review from instance (bool) + priority = attr.ib(default=50) # job priority on farm family = attr.ib(default="renderlayer") - families = attr.ib(default=["renderlayer"]) + families = attr.ib(default=["renderlayer"]) # list of families # format settings - multipartExr = attr.ib(default=False) - convertToScanline = attr.ib(default=False) + multipartExr = attr.ib(default=False) # flag for multipart exrs + convertToScanline = attr.ib(default=False) # flag for exr conversion + + tileRendering = attr.ib(default=False) # bool: treat render as tiles + tilesX = attr.ib(default=0) # number of tiles in X + tilesY = attr.ib(default=0) # number of tiles in Y @frameStart.validator def check_frame_start(self, _, value): @@ -115,17 +114,23 @@ class AbstractCollectRender(pyblish.api.ContextPlugin): super(AbstractCollectRender, self).__init__(*args, **kwargs) self._file_path = None self._asset = api.Session["AVALON_ASSET"] + self._context = None def process(self, context): """Entry point to collector.""" + self._context = context for instance in context: # make sure workfile instance publishing is enabled - if "workfile" in instance.data["families"]: - instance.data["publish"] = True + try: + if "workfile" in instance.data["families"]: + instance.data["publish"] = True + except KeyError: + # be tolerant if 'families' is missing. + pass self._file_path = context.data["currentFile"].replace("\\", "/") - render_instances = self.get_instances() + render_instances = self.get_instances(context) for render_instance in render_instances: exp_files = self.get_expected_files(render_instance) assert exp_files, "no file names were generated, this is bug" @@ -163,7 +168,7 @@ class AbstractCollectRender(pyblish.api.ContextPlugin): "subset": render_instance.subset, "attachTo": render_instance.attachTo, "setMembers": render_instance.setMembers, - "multipartExr": exp_files.multipart, + "multipartExr": render_instance.multipartExr, "review": render_instance.review or False, "publish": True, From 8e5fe61294410dbe3d05f31d0d463803300f594c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 12 Nov 2020 18:54:44 +0100 Subject: [PATCH 101/147] Renamed file to highlight its workfile only --- ...llect_instances.py => collect_workfile.py} | 27 +++---------------- 1 file changed, 3 insertions(+), 24 deletions(-) rename pype/plugins/aftereffects/publish/{collect_instances.py => collect_workfile.py} (72%) diff --git a/pype/plugins/aftereffects/publish/collect_instances.py b/pype/plugins/aftereffects/publish/collect_workfile.py similarity index 72% rename from pype/plugins/aftereffects/publish/collect_instances.py rename to pype/plugins/aftereffects/publish/collect_workfile.py index 7a4d58d3af..2f8f9ae91b 100644 --- a/pype/plugins/aftereffects/publish/collect_instances.py +++ b/pype/plugins/aftereffects/publish/collect_workfile.py @@ -3,10 +3,10 @@ from avalon import api import pyblish.api -class CollectInstances(pyblish.api.ContextPlugin): - """ Adds the celaction render instances """ +class CollectWorkfile(pyblish.api.ContextPlugin): + """ Adds the AE render instances """ - label = "Collect After Effects Instances" + label = "Collect After Effects Workfile Instance" order = pyblish.api.CollectorOrder + 0.1 def process(self, context): @@ -66,26 +66,5 @@ class CollectInstances(pyblish.api.ContextPlugin): self.log.info('Publishing After Effects workfile') - # render instance - family = "render.farm" - subset = f"render{task}Main" - instance = context.create_instance(name=subset) - # getting instance state - instance.data["publish"] = True - - # add assetEntity data into instance - instance.data.update({ - "label": "{} - farm".format(subset), - "family": family, - "families": [family], - "subset": subset - }) - - # adding basic script data - instance.data.update(shared_instance_data) - - self.log.info('Publishing After Effects render instance') - self.log.debug(f"Instance data: `{instance.data}`") - for i in context: self.log.debug(f"{i.data['families']}") From 015ca1f9e1c32ea58253778aab6170f1ab3993ca Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 12 Nov 2020 19:17:45 +0100 Subject: [PATCH 102/147] Re-implemented collection of correct render instances --- .../aftereffects/publish/collect_render.py | 80 +++++++++++++++---- 1 file changed, 63 insertions(+), 17 deletions(-) diff --git a/pype/plugins/aftereffects/publish/collect_render.py b/pype/plugins/aftereffects/publish/collect_render.py index 7fbeb37790..f06e35de4f 100644 --- a/pype/plugins/aftereffects/publish/collect_render.py +++ b/pype/plugins/aftereffects/publish/collect_render.py @@ -1,27 +1,73 @@ -from pype.lib import abstract_collect_render +from pype.lib import abstract_collect_render, RenderInstance import pyblish.api -from abc import ABCMeta -import pyblish.api -import six +from avalon import api +import os +import copy from avalon import aftereffects -@six.add_metaclass(ABCMeta) -class CollectRender(abstract_collect_render.AbstractCollectRender): +class CollectAERender(abstract_collect_render.AbstractCollectRender): - order = pyblish.api.CollectorOrder + 0.01 - hosts = ["maya"] - label = "Collect Render Layers" - sync_workfile_version = False + order = pyblish.api.CollectorOrder + 0.498 + label = "Collect After Effects Render Layers" def get_instances(self, context): - import web_pdb - web_pdb.set_trace() - print("hello {}".format(context)) - return aftereffects.stub().get_metadata() + instances = [] + + current_file = context.data["currentFile"] + version = context.data["version"] + asset_entity = context.data["assetEntity"] + project_entity = context.data["projectEntity"] + + for inst in aftereffects.stub().get_metadata().values(): + if inst["family"] == "render" and inst["active"]: + instance = RenderInstance( + version=version, + time="", + source=current_file, + label="{} - farm".format(inst["subset"]), + subset=inst["subset"], + asset=context.data["assetEntity"]["name"], + attachTo=False, + setMembers='', + publish=True, + renderer='aerender', + name=inst["subset"], + resolutionWidth=asset_entity["data"].get( + "resolutionWidth", + project_entity["data"]["resolutionWidth"]), + resolutionHeight=asset_entity["data"].get( + "resolutionHeight", + project_entity["data"]["resolutionHeight"]), + pixelAspect=1, + tileRendering=False, + tilesX=0, + tilesY=0, + frameStart=asset_entity["data"]["frameStart"], + frameEnd=asset_entity["data"]["frameEnd"], + frameStep=1 + ) + instance._anatomy = context.data["anatomy"] + instance._anatomyData = context.data["anatomyData"] + instances.append(instance) + + return instances def get_expected_files(self, render_instance): - import web_pdb - web_pdb.set_trace() + anatomy = render_instance._anatomy + anatomy_data = copy.deepcopy(render_instance._anatomyData) + anatomy_data["family"] = render_instance.family + anatomy_data["version"] = render_instance.version + anatomy_data["subset"] = render_instance.subset + padding = anatomy.templates.get("frame_padding", 4) + anatomy_data.update({ + "frame": f"%0{padding}d", + "representation": "aif" + }) - return [] \ No newline at end of file + anatomy_filled = anatomy.format(anatomy_data) + import json + print("anatomy_filled::{}".format(json.dumps(anatomy_filled, indent=4))) + + render_path = anatomy_filled["render"]["path"] + return [render_path] From addeeaa4446a2ec9cd216a261ad3234541be0a3d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 13 Nov 2020 15:23:53 +0100 Subject: [PATCH 103/147] event can change value between hierarchical and nonhierarchical attributes with same key --- .../events/event_push_frame_values_to_task.py | 240 ++++++++++++------ 1 file changed, 157 insertions(+), 83 deletions(-) diff --git a/pype/modules/ftrack/events/event_push_frame_values_to_task.py b/pype/modules/ftrack/events/event_push_frame_values_to_task.py index 32993ef938..cd8b4e8c9d 100644 --- a/pype/modules/ftrack/events/event_push_frame_values_to_task.py +++ b/pype/modules/ftrack/events/event_push_frame_values_to_task.py @@ -10,15 +10,12 @@ class PushFrameValuesToTaskEvent(BaseEvent): cust_attrs_query = ( "select id, key, object_type_id, is_hierarchical, default" " from CustomAttributeConfiguration" - " where key in ({}) and object_type_id in ({})" + " where key in ({}) and" + " (object_type_id in ({}) or is_hierarchical is true)" ) interest_entity_types = {"Shot"} interest_attributes = {"frameStart", "frameEnd"} - interest_attr_mapping = { - "frameStart": "fstart", - "frameEnd": "fend" - } _cached_task_object_id = None _cached_interest_object_ids = None @@ -50,7 +47,9 @@ class PushFrameValuesToTaskEvent(BaseEvent): return cls._cached_interest_object_ids def launch(self, session, event): - interesting_data = self.extract_interesting_data(session, event) + interesting_data, changed_keys_by_object_id = ( + self.extract_interesting_data(session, event) + ) if not interesting_data: return @@ -66,92 +65,156 @@ class PushFrameValuesToTaskEvent(BaseEvent): if entity_id not in entities_by_id: interesting_data.pop(entity_id) - task_entities = self.get_task_entities(session, interesting_data) + attrs_by_obj_id, hier_attrs = self.attrs_configurations(session) + + task_object_id = self.task_object_id(session) + task_attrs = attrs_by_obj_id.get(task_object_id) + # Skip keys that are not both in hierachical and type specific + for object_id, keys in changed_keys_by_object_id.items(): + object_id_attrs = attrs_by_obj_id.get(object_id) + for key in keys: + if key not in hier_attrs: + attrs_by_obj_id[object_id].pop(key) + continue + + if ( + (not object_id_attrs or key not in object_id_attrs) + and (not task_attrs or key not in task_attrs) + ): + hier_attrs.pop(key) + + # Clean up empty values + for key, value in tuple(attrs_by_obj_id.items()): + if not value: + attrs_by_obj_id.pop(key) - attrs_by_obj_id = self.attrs_configurations(session) if not attrs_by_obj_id: self.log.warning(( - "There is not created Custom Attributes {}" - " for \"Task\" entity type." - ).format(self.join_keys(self.interest_attributes))) + "There is not created Custom Attributes {} " + " for entity types: {}" + ).format( + self.join_keys(self.interest_attributes), + self.join_keys(self.interest_entity_types) + )) return - task_entities_by_parent_id = collections.defaultdict(list) + # Prepare task entities + task_entities = [] + # If task entity does not contain changed attribute then skip + if task_attrs: + task_entities = self.get_task_entities(session, interesting_data) + + task_entities_by_id = {} + parent_id_by_task_id = {} for task_entity in task_entities: - task_entities_by_parent_id[task_entity["parent_id"]].append( - task_entity + task_entities_by_id[task_entity["id"]] = task_entity + parent_id_by_task_id[task_entity["id"]] = task_entity["parent_id"] + + changed_keys = set() + for keys in changed_keys_by_object_id.values(): + changed_keys |= set(keys) + + attr_id_to_key = {} + normal_attr_ids = set() + for attr_confs in attrs_by_obj_id.values(): + for key in changed_keys: + custom_attr_id = attr_confs.get(key) + if custom_attr_id: + normal_attr_ids.add(custom_attr_id) + attr_id_to_key[custom_attr_id] = key + + hier_attr_ids = set() + for key in changed_keys: + custom_attr_id = hier_attrs.get(key) + if custom_attr_id: + hier_attr_ids.add(custom_attr_id) + attr_id_to_key[custom_attr_id] = key + + entity_ids = ( + set(interesting_data.keys()) | set(task_entities_by_id.keys()) + ) + + joined_conf_ids = self.join_keys(normal_attr_ids | hier_attr_ids) + joined_entity_ids = self.join_keys(entity_ids) + + cust_attr_query = ( + "select value, entity_id from ContextCustomAttributeValue " + "where entity_id in ({}) and configuration_id in ({})" + ) + call_expr = [{ + "action": "query", + "expression": cust_attr_query.format( + joined_entity_ids, joined_conf_ids ) + }] + if hasattr(session, "call"): + [values] = session.call(call_expr) + else: + [values] = session._call(call_expr) - missing_keys_by_object_name = collections.defaultdict(set) - for parent_id, values in interesting_data.items(): - entities = task_entities_by_parent_id.get(parent_id) or [] - entities.append(entities_by_id[parent_id]) + current_values_by_id = {} + for item in values["data"]: + entity_id = item["entity_id"] + attr_id = item["configuration_id"] + if entity_id in task_entities_by_id and attr_id in hier_attrs: + continue - for hier_key, value in values.items(): - changed_ids = [] - for entity in entities: - key = self.interest_attr_mapping[hier_key] - entity_attrs_mapping = ( - attrs_by_obj_id.get(entity["object_type_id"]) + if entity_id not in current_values_by_id: + current_values_by_id[entity_id] = {} + current_values_by_id[entity_id][attr_id] = item["value"] + + for entity_id, current_values in current_values_by_id.items(): + parent_id = parent_id_by_task_id.get(entity_id) + if not parent_id: + parent_id = entity_id + values = interesting_data[parent_id] + + for attr_id, old_value in current_values.items(): + attr_key = attr_id_to_key.get(attr_id) + if not attr_key: + continue + + # Convert new value from string + new_value = values.get(attr_key) + if new_value is not None and old_value is not None: + try: + new_value = type(old_value)(new_value) + except Exception: + self.log.warning(( + "Couldn't convert from {} to {}." + " Skipping update values." + ).format(type(new_value), type(old_value))) + if new_value == old_value: + continue + + entity_key = collections.OrderedDict({ + "configuration_id": attr_id, + "entity_id": entity_id + }) + if new_value is None: + op = ftrack_api.operation.DeleteEntityOperation( + "CustomAttributeValue", + entity_key + ) + else: + op = ftrack_api.operation.UpdateEntityOperation( + "ContextCustomAttributeValue", + entity_key, + "value", + ftrack_api.symbol.NOT_SET, + new_value ) - if not entity_attrs_mapping: - missing_keys_by_object_name[entity.entity_type].add( - key - ) - continue - configuration_id = entity_attrs_mapping.get(key) - if not configuration_id: - missing_keys_by_object_name[entity.entity_type].add( - key - ) - continue - - changed_ids.append(entity["id"]) - entity_key = collections.OrderedDict({ - "configuration_id": configuration_id, - "entity_id": entity["id"] - }) - if value is None: - op = ftrack_api.operation.DeleteEntityOperation( - "CustomAttributeValue", - entity_key - ) - else: - op = ftrack_api.operation.UpdateEntityOperation( - "ContextCustomAttributeValue", - entity_key, - "value", - ftrack_api.symbol.NOT_SET, - value - ) - - session.recorded_operations.push(op) + session.recorded_operations.push(op) self.log.info(( "Changing Custom Attribute \"{}\" to value" - " \"{}\" on entities: {}" - ).format(key, value, self.join_keys(changed_ids))) - try: - session.commit() - except Exception: - session.rollback() - self.log.warning( - "Changing of values failed.", - exc_info=True - ) - if not missing_keys_by_object_name: - return - - msg_items = [] - for object_name, missing_keys in missing_keys_by_object_name.items(): - msg_items.append( - "{}: ({})".format(object_name, self.join_keys(missing_keys)) - ) - - self.log.warning(( - "Missing Custom Attribute configuration" - " per specific object types: {}" - ).format(", ".join(msg_items))) + " \"{}\" on entity: {}" + ).format(attr_key, new_value, entity_id)) + try: + session.commit() + except Exception: + session.rollback() + self.log.warning("Changing of values failed.", exc_info=True) def extract_interesting_data(self, session, event): # Filter if event contain relevant data @@ -160,6 +223,7 @@ class PushFrameValuesToTaskEvent(BaseEvent): return interesting_data = {} + changed_keys_by_object_id = {} for entity_info in entities_info: # Care only about tasks if entity_info.get("entityType") != "task": @@ -181,11 +245,17 @@ class PushFrameValuesToTaskEvent(BaseEvent): # Do not care about "Task" entity_type task_object_id = self.task_object_id(session) - if entity_info.get("objectTypeId") == task_object_id: + object_id = entity_info.get("objectTypeId") + if not object_id or object_id == task_object_id: continue interesting_data[entity_info["entityId"]] = entity_changes - return interesting_data + if object_id not in changed_keys_by_object_id: + changed_keys_by_object_id[object_id] = set() + + changed_keys_by_object_id[object_id] |= set(entity_changes.keys()) + + return interesting_data, changed_keys_by_object_id def get_entities(self, session, interesting_data): entities = session.query( @@ -213,17 +283,21 @@ class PushFrameValuesToTaskEvent(BaseEvent): object_ids.append(self.task_object_id(session)) attrs = session.query(self.cust_attrs_query.format( - self.join_keys(self.interest_attr_mapping.values()), + self.join_keys(self.interest_attributes), self.join_keys(object_ids) )).all() output = {} + hiearchical = {} for attr in attrs: + if attr["is_hierarchical"]: + hiearchical[attr["key"]] = attr["id"] + continue obj_id = attr["object_type_id"] if obj_id not in output: output[obj_id] = {} output[obj_id][attr["key"]] = attr["id"] - return output + return output, hiearchical def register(session, plugins_presets): From e8660954c91bd36eb9fc6ba840f270208ae92d55 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 13 Nov 2020 15:31:17 +0100 Subject: [PATCH 104/147] small cleanup --- .../events/event_push_frame_values_to_task.py | 70 +++++++++++-------- 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/pype/modules/ftrack/events/event_push_frame_values_to_task.py b/pype/modules/ftrack/events/event_push_frame_values_to_task.py index cd8b4e8c9d..ec9fcb29a0 100644 --- a/pype/modules/ftrack/events/event_push_frame_values_to_task.py +++ b/pype/modules/ftrack/events/event_push_frame_values_to_task.py @@ -14,6 +14,11 @@ class PushFrameValuesToTaskEvent(BaseEvent): " (object_type_id in ({}) or is_hierarchical is true)" ) + cust_attr_query = ( + "select value, entity_id from ContextCustomAttributeValue " + "where entity_id in ({}) and configuration_id in ({})" + ) + interest_entity_types = {"Shot"} interest_attributes = {"frameStart", "frameEnd"} _cached_task_object_id = None @@ -115,53 +120,25 @@ class PushFrameValuesToTaskEvent(BaseEvent): changed_keys |= set(keys) attr_id_to_key = {} - normal_attr_ids = set() for attr_confs in attrs_by_obj_id.values(): for key in changed_keys: custom_attr_id = attr_confs.get(key) if custom_attr_id: - normal_attr_ids.add(custom_attr_id) attr_id_to_key[custom_attr_id] = key - hier_attr_ids = set() for key in changed_keys: custom_attr_id = hier_attrs.get(key) if custom_attr_id: - hier_attr_ids.add(custom_attr_id) attr_id_to_key[custom_attr_id] = key entity_ids = ( set(interesting_data.keys()) | set(task_entities_by_id.keys()) ) + attr_ids = set(attr_id_to_key.keys()) - joined_conf_ids = self.join_keys(normal_attr_ids | hier_attr_ids) - joined_entity_ids = self.join_keys(entity_ids) - - cust_attr_query = ( - "select value, entity_id from ContextCustomAttributeValue " - "where entity_id in ({}) and configuration_id in ({})" + current_values_by_id = self.current_values( + session, attr_ids, entity_ids, task_entities_by_id, hier_attrs ) - call_expr = [{ - "action": "query", - "expression": cust_attr_query.format( - joined_entity_ids, joined_conf_ids - ) - }] - if hasattr(session, "call"): - [values] = session.call(call_expr) - else: - [values] = session._call(call_expr) - - current_values_by_id = {} - for item in values["data"]: - entity_id = item["entity_id"] - attr_id = item["configuration_id"] - if entity_id in task_entities_by_id and attr_id in hier_attrs: - continue - - if entity_id not in current_values_by_id: - current_values_by_id[entity_id] = {} - current_values_by_id[entity_id][attr_id] = item["value"] for entity_id, current_values in current_values_by_id.items(): parent_id = parent_id_by_task_id.get(entity_id) @@ -216,6 +193,37 @@ class PushFrameValuesToTaskEvent(BaseEvent): session.rollback() self.log.warning("Changing of values failed.", exc_info=True) + def current_values( + self, session, attr_ids, entity_ids, task_entities_by_id, hier_attrs + ): + current_values_by_id = {} + if not attr_ids or not entity_ids: + return current_values_by_id + joined_conf_ids = self.join_keys(attr_ids) + joined_entity_ids = self.join_keys(entity_ids) + + call_expr = [{ + "action": "query", + "expression": self.cust_attr_query.format( + joined_entity_ids, joined_conf_ids + ) + }] + if hasattr(session, "call"): + [values] = session.call(call_expr) + else: + [values] = session._call(call_expr) + + for item in values["data"]: + entity_id = item["entity_id"] + attr_id = item["configuration_id"] + if entity_id in task_entities_by_id and attr_id in hier_attrs: + continue + + if entity_id not in current_values_by_id: + current_values_by_id[entity_id] = {} + current_values_by_id[entity_id][attr_id] = item["value"] + return current_values_by_id + def extract_interesting_data(self, session, event): # Filter if event contain relevant data entities_info = event["data"].get("entities") From f45604e91ebd39b9880e7221da21c3d9f2c6ecb6 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 13 Nov 2020 16:11:14 +0100 Subject: [PATCH 105/147] event is caching it's own changes and skip them --- .../events/event_push_frame_values_to_task.py | 62 ++++++++++++++++++- 1 file changed, 59 insertions(+), 3 deletions(-) diff --git a/pype/modules/ftrack/events/event_push_frame_values_to_task.py b/pype/modules/ftrack/events/event_push_frame_values_to_task.py index ec9fcb29a0..00457c8bfc 100644 --- a/pype/modules/ftrack/events/event_push_frame_values_to_task.py +++ b/pype/modules/ftrack/events/event_push_frame_values_to_task.py @@ -1,4 +1,6 @@ import collections +import datetime + import ftrack_api from pype.modules.ftrack import BaseEvent @@ -19,10 +21,15 @@ class PushFrameValuesToTaskEvent(BaseEvent): "where entity_id in ({}) and configuration_id in ({})" ) - interest_entity_types = {"Shot"} - interest_attributes = {"frameStart", "frameEnd"} _cached_task_object_id = None _cached_interest_object_ids = None + _cached_user_id = None + _cached_changes = [] + _max_delta = 30 + + # Configrable (lists) + interest_entity_types = {"Shot"} + interest_attributes = {"frameStart", "frameEnd"} @staticmethod def join_keys(keys): @@ -51,6 +58,14 @@ class PushFrameValuesToTaskEvent(BaseEvent): ) return cls._cached_interest_object_ids + def session_user_id(self, session): + if self._cached_user_id is None: + user = session.query( + "User where username is \"{}\"".format(session.api_user) + ).one() + self._cached_user_id = user["id"] + return self._cached_user_id + def launch(self, session, event): interesting_data, changed_keys_by_object_id = ( self.extract_interesting_data(session, event) @@ -168,6 +183,12 @@ class PushFrameValuesToTaskEvent(BaseEvent): "configuration_id": attr_id, "entity_id": entity_id }) + self._cached_changes.append({ + "attr_key": attr_key, + "entity_id": entity_id, + "value": new_value, + "time": datetime.datetime.now() + }) if new_value is None: op = ftrack_api.operation.DeleteEntityOperation( "CustomAttributeValue", @@ -230,6 +251,16 @@ class PushFrameValuesToTaskEvent(BaseEvent): if not entities_info: return + # for key, value in event["data"].items(): + # self.log.info("{}: {}".format(key, value)) + session_user_id = self.session_user_id(session) + user_data = event["data"].get("user") + changed_by_session = False + if user_data and user_data.get("userid") == session_user_id: + changed_by_session = True + + current_time = datetime.datetime.now() + interesting_data = {} changed_keys_by_object_id = {} for entity_info in entities_info: @@ -248,6 +279,31 @@ class PushFrameValuesToTaskEvent(BaseEvent): if key in changes: entity_changes[key] = changes[key]["new"] + entity_id = entity_info["entityId"] + if changed_by_session: + for key, new_value in tuple(entity_changes.items()): + for cached in tuple(self._cached_changes): + if ( + cached["entity_id"] != entity_id + or cached["attr_key"] != key + ): + continue + + cached_value = cached["value"] + try: + new_value = type(cached_value)(new_value) + except Exception: + pass + + if cached_value == new_value: + self._cached_changes.remove(cached) + entity_changes.pop(key) + break + + delta = (current_time - cached["time"]).seconds + if delta > self._max_delta: + self._cached_changes.remove(cached) + if not entity_changes: continue @@ -257,7 +313,7 @@ class PushFrameValuesToTaskEvent(BaseEvent): if not object_id or object_id == task_object_id: continue - interesting_data[entity_info["entityId"]] = entity_changes + interesting_data[entity_id] = entity_changes if object_id not in changed_keys_by_object_id: changed_keys_by_object_id[object_id] = set() From 36e5782cffa0de23ee7fe6f2b12d6e4838cac367 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= <33513211+antirotor@users.noreply.github.com> Date: Fri, 13 Nov 2020 16:23:46 +0100 Subject: [PATCH 106/147] disable legacy override check for cameras legacy overrides doesn't work with render setup and make renderable cameras not renderable --- pype/hosts/maya/expected_files.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pype/hosts/maya/expected_files.py b/pype/hosts/maya/expected_files.py index dab4c18c2b..8d225bc13d 100644 --- a/pype/hosts/maya/expected_files.py +++ b/pype/hosts/maya/expected_files.py @@ -378,14 +378,8 @@ class AExpectedFiles: renderable = False if self.maya_is_true(cmds.getAttr("{}.renderable".format(cam))): renderable = True - - for override in self.get_layer_overrides( - "{}.renderable".format(cam), self.layer - ): - renderable = self.maya_is_true(override) - - if renderable: renderable_cameras.append(cam) + return renderable_cameras def maya_is_true(self, attr_val): From 63fc013c897ffe5f8986473df74aa3faccf7a686 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Tue, 17 Nov 2020 00:36:29 +0100 Subject: [PATCH 107/147] fix how render instance is created --- pype/lib/abstract_collect_render.py | 27 +++++---------------------- pype/lib/abstract_submit_deadline.py | 7 ++++--- 2 files changed, 9 insertions(+), 25 deletions(-) diff --git a/pype/lib/abstract_collect_render.py b/pype/lib/abstract_collect_render.py index 098788430b..cd53715763 100644 --- a/pype/lib/abstract_collect_render.py +++ b/pype/lib/abstract_collect_render.py @@ -124,6 +124,8 @@ class AbstractCollectRender(pyblish.api.ContextPlugin): try: if "workfile" in instance.data["families"]: instance.data["publish"] = True + if "renderFarm" in instance.data["families"]: + instance.data["remove"] = True except KeyError: # be tolerant if 'families' is missing. pass @@ -165,13 +167,6 @@ class AbstractCollectRender(pyblish.api.ContextPlugin): frame_end_handle = frame_end_render data = { - "subset": render_instance.subset, - "attachTo": render_instance.attachTo, - "setMembers": render_instance.setMembers, - "multipartExr": render_instance.multipartExr, - "review": render_instance.review or False, - "publish": True, - "handleStart": handle_start, "handleEnd": handle_end, "frameStart": frame_start, @@ -179,34 +174,22 @@ class AbstractCollectRender(pyblish.api.ContextPlugin): "frameStartHandle": frame_start_handle, "frameEndHandle": frame_end_handle, "byFrameStep": int(render_instance.frameStep), - "renderer": render_instance.renderer, - # instance subset - "family": render_instance.family, - "families": render_instance.families, - "asset": render_instance.asset, - "time": render_instance.time, + "author": context.data["user"], # Add source to allow tracing back to the scene from # which was submitted originally - "source": render_instance.source, "expectedFiles": exp_files, - "resolutionWidth": render_instance.resolutionWidth, - "resolutionHeight": render_instance.resolutionHeight, - "pixelAspect": render_instance.pixelAspect, - "tileRendering": render_instance.tileRendering or False, - "tilesX": render_instance.tilesX or 2, - "tilesY": render_instance.tilesY or 2, - "priority": render_instance.priority, - "convertToScanline": render_instance.convertToScanline or False } if self.sync_workfile_version: data["version"] = context.data["version"] # add additional data data = self.add_additional_data(data) + render_instance_dict = attr.asdict(render_instance) instance = context.create_instance(render_instance.name) instance.data["label"] = render_instance.label + instance.data.update(render_instance_dict) instance.data.update(data) self.post_collecting_action() diff --git a/pype/lib/abstract_submit_deadline.py b/pype/lib/abstract_submit_deadline.py index 44ee9ce436..3337860508 100644 --- a/pype/lib/abstract_submit_deadline.py +++ b/pype/lib/abstract_submit_deadline.py @@ -19,7 +19,7 @@ from .abstract_metaplugins import AbstractMetaInstancePlugin @attr.s -class DeadlineJobInfo: +class DeadlineJobInfo(object): """Mapping of all Deadline *JobInfo* attributes. This contains all JobInfo attributes plus their default values. @@ -474,14 +474,15 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): anatomy = self._instance.context.data['anatomy'] file_path = None for i in self._instance.context: - if "workfile" in i.data["families"]: + if "workfile" in i.data["families"] \ + or i.data["family"] == "workfile": # test if there is instance of workfile waiting # to be published. assert i.data["publish"] is True, ( "Workfile (scene) must be published along") # determine published path from Anatomy. template_data = i.data.get("anatomyData") - rep = i.data.get("representations")[0].get("name") + rep = i.data.get("representations")[0].get("ext") template_data["representation"] = rep template_data["ext"] = rep template_data["comment"] = None From 266dc08a8410eeb5ad9f93cfb33b4e697dbc46b1 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 17 Nov 2020 10:37:49 +0100 Subject: [PATCH 108/147] overview is default view --- pype/tools/pyblish_pype/settings.py | 2 +- pype/tools/pyblish_pype/window.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/tools/pyblish_pype/settings.py b/pype/tools/pyblish_pype/settings.py index fefdbea77f..5848cdf698 100644 --- a/pype/tools/pyblish_pype/settings.py +++ b/pype/tools/pyblish_pype/settings.py @@ -8,7 +8,7 @@ UseLabel = True # Customize which tab to start on. Possible choices are: "artist", "overview" # and "terminal". -InitialTab = "artist" +InitialTab = "overview" # Customize the window size. WindowSize = (430, 600) diff --git a/pype/tools/pyblish_pype/window.py b/pype/tools/pyblish_pype/window.py index 2f663cc08a..5b47a0d35e 100644 --- a/pype/tools/pyblish_pype/window.py +++ b/pype/tools/pyblish_pype/window.py @@ -520,7 +520,7 @@ class Window(QtWidgets.QDialog): ("terminal", terminal_page) ) - current_page = settings.InitialTab or "artist" + current_page = settings.InitialTab or "overview" self.comment_main_widget.setVisible( not current_page == "terminal" ) From d3f09293d3e85bd07015ce8162732719d142c561 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 17 Nov 2020 10:38:31 +0100 Subject: [PATCH 109/147] removed artist view, model and delagate --- pype/tools/pyblish_pype/delegate.py | 162 ---------------------------- pype/tools/pyblish_pype/model.py | 158 --------------------------- pype/tools/pyblish_pype/view.py | 55 ---------- pype/tools/pyblish_pype/window.py | 44 +------- 4 files changed, 2 insertions(+), 417 deletions(-) diff --git a/pype/tools/pyblish_pype/delegate.py b/pype/tools/pyblish_pype/delegate.py index cb9123bf3a..bf3fbc1853 100644 --- a/pype/tools/pyblish_pype/delegate.py +++ b/pype/tools/pyblish_pype/delegate.py @@ -522,168 +522,6 @@ class PluginDelegate(QtWidgets.QStyledItemDelegate): return QtCore.QSize(option.rect.width(), 20) -class ArtistDelegate(QtWidgets.QStyledItemDelegate): - """Delegate used on Artist page""" - - def paint(self, painter, option, index): - """Paint checkbox and text - - _______________________________________________ - | | label | duration |arrow| - |toggle |_____________________| | to | - | | families | |persp| - |_______|_____________________|___________|_____| - - """ - - # Layout - spacing = 10 - - body_rect = QtCore.QRectF(option.rect).adjusted(2, 2, -8, -2) - content_rect = body_rect.adjusted(5, 5, -5, -5) - - perspective_rect = QtCore.QRectF(body_rect) - perspective_rect.setWidth(35) - perspective_rect.setHeight(35) - perspective_rect.translate( - content_rect.width() - (perspective_rect.width() / 2) + 10, - (content_rect.height() / 2) - (perspective_rect.height() / 2) - ) - - toggle_rect = QtCore.QRectF(body_rect) - toggle_rect.setWidth(7) - toggle_rect.adjust(1, 1, 0, -1) - - icon_rect = QtCore.QRectF(content_rect) - icon_rect.translate(toggle_rect.width() + spacing, 3) - icon_rect.setWidth(35) - icon_rect.setHeight(35) - - duration_rect = QtCore.QRectF(content_rect) - duration_rect.translate(content_rect.width() - 50, 0) - - # Colors - check_color = colors["idle"] - - publish_states = index.data(Roles.PublishFlagsRole) - if publish_states is None: - return - if publish_states & InstanceStates.InProgress: - check_color = colors["active"] - - elif publish_states & InstanceStates.HasError: - check_color = colors["error"] - - elif publish_states & InstanceStates.HasWarning: - check_color = colors["warning"] - - elif publish_states & InstanceStates.HasFinished: - check_color = colors["ok"] - - elif not index.data(Roles.IsEnabledRole): - check_color = colors["inactive"] - - perspective_icon = icons["angle-right"] - - if not index.data(QtCore.Qt.CheckStateRole): - font_color = colors["inactive"] - else: - font_color = colors["idle"] - - if ( - option.state - & ( - QtWidgets.QStyle.State_MouseOver - or QtWidgets.QStyle.State_Selected - ) - ): - perspective_color = colors["idle"] - else: - perspective_color = colors["inactive"] - # Maintan reference to state, so we can restore it once we're done - painter.save() - - # Draw background - painter.fillRect(body_rect, colors["hover"]) - - # Draw icon - icon = index.data(QtCore.Qt.DecorationRole) - - painter.setFont(fonts["largeAwesome"]) - painter.setPen(QtGui.QPen(font_color)) - painter.drawText(icon_rect, icon) - - # Draw label - painter.setFont(fonts["h3"]) - label_rect = QtCore.QRectF(content_rect) - label_x_offset = icon_rect.width() + spacing - label_rect.translate( - label_x_offset, - 0 - ) - metrics = painter.fontMetrics() - label_rect.setHeight(metrics.lineSpacing()) - label_rect.setWidth( - content_rect.width() - - label_x_offset - - perspective_rect.width() - ) - # Elide label - label = index.data(QtCore.Qt.DisplayRole) - label = metrics.elidedText( - label, QtCore.Qt.ElideRight, label_rect.width() - ) - painter.drawText(label_rect, label) - - # Draw families - painter.setFont(fonts["h5"]) - painter.setPen(QtGui.QPen(colors["inactive"])) - - families = ", ".join(index.data(Roles.FamiliesRole)) - families = painter.fontMetrics().elidedText( - families, QtCore.Qt.ElideRight, label_rect.width() - ) - - families_rect = QtCore.QRectF(label_rect) - families_rect.translate(0, label_rect.height() + spacing) - - painter.drawText(families_rect, families) - - painter.setFont(fonts["largeAwesome"]) - painter.setPen(QtGui.QPen(perspective_color)) - painter.drawText(perspective_rect, perspective_icon) - - # Draw checkbox - pen = QtGui.QPen(check_color, 1) - painter.setPen(pen) - - if index.data(Roles.IsOptionalRole): - painter.drawRect(toggle_rect) - - if index.data(QtCore.Qt.CheckStateRole): - painter.fillRect(toggle_rect, check_color) - - elif ( - index.data(QtCore.Qt.CheckStateRole) - ): - painter.fillRect(toggle_rect, check_color) - - if option.state & QtWidgets.QStyle.State_MouseOver: - painter.fillRect(body_rect, colors["hover"]) - - if option.state & QtWidgets.QStyle.State_Selected: - painter.fillRect(body_rect, colors["selected"]) - - painter.setPen(colors["outline"]) - painter.drawRect(body_rect) - - # Ok, we're done, tidy up. - painter.restore() - - def sizeHint(self, option, index): - return QtCore.QSize(option.rect.width(), 80) - - class TerminalItem(QtWidgets.QStyledItemDelegate): """Delegate used exclusively for the Terminal""" diff --git a/pype/tools/pyblish_pype/model.py b/pype/tools/pyblish_pype/model.py index 1482ff85b0..ffcf9a6501 100644 --- a/pype/tools/pyblish_pype/model.py +++ b/pype/tools/pyblish_pype/model.py @@ -842,164 +842,6 @@ class InstanceModel(QtGui.QStandardItemModel): ) -class ArtistProxy(QtCore.QAbstractProxyModel): - def __init__(self, *args, **kwargs): - self.mapping_from = [] - self.mapping_to = [] - super(ArtistProxy, self).__init__(*args, **kwargs) - - def on_rows_inserted(self, parent_index, from_row, to_row): - if not parent_index.isValid(): - return - - parent_row = parent_index.row() - if parent_row >= len(self.mapping_from): - self.mapping_from.append(list()) - - new_from = None - new_to = None - for row_num in range(from_row, to_row + 1): - new_row = len(self.mapping_to) - new_to = new_row - if new_from is None: - new_from = new_row - - self.mapping_from[parent_row].insert(row_num, new_row) - self.mapping_to.insert(new_row, [parent_row, row_num]) - - self.rowsInserted.emit(self.parent(), new_from, new_to + 1) - - def _remove_rows(self, parent_row, from_row, to_row): - increment_num = self.mapping_from[parent_row][from_row] - - to_end_index = len(self.mapping_from[parent_row]) - 1 - for _idx in range(0, parent_row): - to_end_index += len(self.mapping_from[_idx]) - - removed_rows = 0 - _emit_last = None - for row_num in reversed(range(from_row, to_row + 1)): - row = self.mapping_from[parent_row].pop(row_num) - _emit_last = row - removed_rows += 1 - - _emit_first = int(increment_num) - mapping_from_len = len(self.mapping_from) - mapping_from_parent_len = len(self.mapping_from[parent_row]) - if parent_row < mapping_from_len: - for idx in range(from_row, mapping_from_parent_len): - self.mapping_from[parent_row][idx] = increment_num - increment_num += 1 - - if parent_row < mapping_from_len - 1: - for idx_i in range(parent_row + 1, mapping_from_len): - sub_values = self.mapping_from[idx_i] - if not sub_values: - continue - - for idx_j in range(0, len(sub_values)): - self.mapping_from[idx_i][idx_j] = increment_num - increment_num += 1 - - for idx in range(removed_rows): - self.mapping_to.pop(to_end_index - idx) - - return (_emit_first, _emit_last) - - def on_rows_removed(self, parent_index, from_row, to_row): - if parent_index.isValid(): - parent_row = parent_index.row() - _emit_first, _emit_last = self._remove_rows( - parent_row, from_row, to_row - ) - self.rowsRemoved.emit(self.parent(), _emit_first, _emit_last) - - else: - removed_rows = False - emit_first = None - emit_last = None - for row_num in reversed(range(from_row, to_row + 1)): - remaining_rows = self.mapping_from[row_num] - if remaining_rows: - removed_rows = True - _emit_first, _emit_last = self._remove_rows( - row_num, 0, len(remaining_rows) - 1 - ) - if emit_first is None: - emit_first = _emit_first - emit_last = _emit_last - - self.mapping_from.pop(row_num) - - diff = to_row - from_row + 1 - mapping_to_len = len(self.mapping_to) - if from_row < mapping_to_len: - for idx in range(from_row, mapping_to_len): - self.mapping_to[idx][0] -= diff - - if removed_rows: - self.rowsRemoved.emit(self.parent(), emit_first, emit_last) - - def on_reset(self): - self.modelReset.emit() - self.mapping_from = [] - self.mapping_to = [] - - def setSourceModel(self, source_model): - super(ArtistProxy, self).setSourceModel(source_model) - source_model.rowsInserted.connect(self.on_rows_inserted) - source_model.rowsRemoved.connect(self.on_rows_removed) - source_model.modelReset.connect(self.on_reset) - source_model.dataChanged.connect(self.on_data_changed) - - def on_data_changed(self, from_index, to_index, roles=None): - proxy_from_index = self.mapFromSource(from_index) - if from_index == to_index: - proxy_to_index = proxy_from_index - else: - proxy_to_index = self.mapFromSource(to_index) - - args = [proxy_from_index, proxy_to_index] - if Qt.__binding__ not in ("PyQt4", "PySide"): - args.append(roles or []) - self.dataChanged.emit(*args) - - def columnCount(self, parent=QtCore.QModelIndex()): - # This is not right for global proxy, but in this case it is enough - return self.sourceModel().columnCount() - - def rowCount(self, parent=QtCore.QModelIndex()): - if parent.isValid(): - return 0 - return len(self.mapping_to) - - def mapFromSource(self, index): - if not index.isValid(): - return QtCore.QModelIndex() - - parent_index = index.parent() - if not parent_index.isValid(): - return QtCore.QModelIndex() - - parent_idx = self.mapping_from[parent_index.row()] - my_row = parent_idx[index.row()] - return self.index(my_row, index.column()) - - def mapToSource(self, index): - if not index.isValid() or index.row() > len(self.mapping_to): - return self.sourceModel().index(index.row(), index.column()) - - parent_row, item_row = self.mapping_to[index.row()] - parent_index = self.sourceModel().index(parent_row, 0) - return self.sourceModel().index(item_row, 0, parent_index) - - def index(self, row, column, parent=QtCore.QModelIndex()): - return self.createIndex(row, column, QtCore.QModelIndex()) - - def parent(self, index=None): - return QtCore.QModelIndex() - - class TerminalDetailItem(QtGui.QStandardItem): key_label_record_map = ( ("instance", "Instance"), diff --git a/pype/tools/pyblish_pype/view.py b/pype/tools/pyblish_pype/view.py index b18738c9ab..777871ca8e 100644 --- a/pype/tools/pyblish_pype/view.py +++ b/pype/tools/pyblish_pype/view.py @@ -11,61 +11,6 @@ def _import_widgets(): from . import widgets -class ArtistView(QtWidgets.QListView): - # An item is requesting to be toggled, with optional forced-state - toggled = QtCore.Signal(QtCore.QModelIndex, object) - show_perspective = QtCore.Signal(QtCore.QModelIndex) - - def __init__(self, parent=None): - super(ArtistView, self).__init__(parent) - - self.horizontalScrollBar().hide() - self.viewport().setAttribute(QtCore.Qt.WA_Hover, True) - self.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) - self.setSelectionMode(QtWidgets.QAbstractItemView.ExtendedSelection) - self.setResizeMode(QtWidgets.QListView.Adjust) - self.setVerticalScrollMode(QtWidgets.QListView.ScrollPerPixel) - - def event(self, event): - if not event.type() == QtCore.QEvent.KeyPress: - return super(ArtistView, self).event(event) - - elif event.key() == QtCore.Qt.Key_Space: - for index in self.selectionModel().selectedIndexes(): - self.toggled.emit(index, None) - - return True - - elif event.key() == QtCore.Qt.Key_Backspace: - for index in self.selectionModel().selectedIndexes(): - self.toggled.emit(index, False) - - return True - - elif event.key() == QtCore.Qt.Key_Return: - for index in self.selectionModel().selectedIndexes(): - self.toggled.emit(index, True) - - return True - - return super(ArtistView, self).event(event) - - def focusOutEvent(self, event): - self.selectionModel().clear() - - def mouseReleaseEvent(self, event): - if event.button() == QtCore.Qt.LeftButton: - indexes = self.selectionModel().selectedIndexes() - if len(indexes) <= 1 and event.pos().x() < 20: - for index in indexes: - self.toggled.emit(index, None) - if len(indexes) == 1 and event.pos().x() > self.width() - 40: - for index in indexes: - self.show_perspective.emit(index) - - return super(ArtistView, self).mouseReleaseEvent(event) - - class OverviewView(QtWidgets.QTreeView): # An item is requesting to be toggled, with optional forced-state toggled = QtCore.Signal(QtCore.QModelIndex, object) diff --git a/pype/tools/pyblish_pype/window.py b/pype/tools/pyblish_pype/window.py index 5b47a0d35e..66015c4255 100644 --- a/pype/tools/pyblish_pype/window.py +++ b/pype/tools/pyblish_pype/window.py @@ -97,7 +97,6 @@ class Window(QtWidgets.QDialog): header_widget = QtWidgets.QWidget(parent=main_widget) header_tab_widget = QtWidgets.QWidget(header_widget) - header_tab_artist = QtWidgets.QRadioButton(header_tab_widget) header_tab_overview = QtWidgets.QRadioButton(header_tab_widget) header_tab_terminal = QtWidgets.QRadioButton(header_tab_widget) header_spacer = QtWidgets.QWidget(header_tab_widget) @@ -125,7 +124,6 @@ class Window(QtWidgets.QDialog): layout_tab = QtWidgets.QHBoxLayout(header_tab_widget) layout_tab.setContentsMargins(0, 0, 0, 0) layout_tab.setSpacing(0) - layout_tab.addWidget(header_tab_artist, 0) layout_tab.addWidget(header_tab_overview, 0) layout_tab.addWidget(header_tab_terminal, 0) layout_tab.addWidget(button_suspend_logs_widget, 0) @@ -141,37 +139,18 @@ class Window(QtWidgets.QDialog): header_widget.setLayout(layout) - # Artist Page - instance_model = model.InstanceModel(controller) - - artist_page = QtWidgets.QWidget() - - artist_view = view.ArtistView() - artist_view.show_perspective.connect(self.toggle_perspective_widget) - artist_proxy = model.ArtistProxy() - artist_proxy.setSourceModel(instance_model) - artist_view.setModel(artist_proxy) - - artist_delegate = delegate.ArtistDelegate() - artist_view.setItemDelegate(artist_delegate) - - layout = QtWidgets.QVBoxLayout(artist_page) - layout.addWidget(artist_view) - layout.setContentsMargins(5, 5, 5, 5) - layout.setSpacing(0) - - artist_page.setLayout(layout) - # Overview Page # TODO add parent overview_page = QtWidgets.QWidget() + instance_model = model.InstanceModel(controller) overview_instance_view = view.InstanceView( animated=settings.Animated, parent=overview_page ) overview_instance_delegate = delegate.InstanceDelegate( parent=overview_instance_view ) + overview_instance_view.setItemDelegate(overview_instance_delegate) overview_instance_view.setModel(instance_model) @@ -223,7 +202,6 @@ class Window(QtWidgets.QDialog): body_widget = QtWidgets.QWidget(main_widget) layout = QtWidgets.QHBoxLayout(body_widget) layout.setContentsMargins(5, 5, 5, 1) - layout.addWidget(artist_page) layout.addWidget(overview_page) layout.addWidget(terminal_page) @@ -361,12 +339,10 @@ class Window(QtWidgets.QDialog): "Footer": footer_widget, # Pages - "Artist": artist_page, "Overview": overview_page, "Terminal": terminal_page, # Tabs - "ArtistTab": header_tab_artist, "OverviewTab": header_tab_overview, "TerminalTab": header_tab_terminal, @@ -399,7 +375,6 @@ class Window(QtWidgets.QDialog): pages_widget, header_widget, body_widget, - artist_page, comment_box, overview_page, terminal_page, @@ -415,9 +390,6 @@ class Window(QtWidgets.QDialog): _widget.setAttribute(QtCore.Qt.WA_StyledBackground) # Signals - header_tab_artist.toggled.connect( - lambda: self.on_tab_changed("artist") - ) header_tab_overview.toggled.connect( lambda: self.on_tab_changed("overview") ) @@ -450,7 +422,6 @@ class Window(QtWidgets.QDialog): QtCore.Qt.DirectConnection ) - artist_view.toggled.connect(self.on_instance_toggle) overview_instance_view.toggled.connect(self.on_instance_toggle) overview_plugin_view.toggled.connect(self.on_plugin_toggle) @@ -491,9 +462,6 @@ class Window(QtWidgets.QDialog): self.plugin_proxy = plugin_proxy self.instance_model = instance_model - self.artist_proxy = artist_proxy - self.artist_view = artist_view - self.presets_button = presets_button self.animation_info_msg = animation_info_msg @@ -510,12 +478,10 @@ class Window(QtWidgets.QDialog): self.perspective_widget = perspective_widget self.tabs = { - "artist": header_tab_artist, "overview": header_tab_overview, "terminal": header_tab_terminal } self.pages = ( - ("artist", artist_page), ("overview", overview_page), ("terminal", terminal_page) ) @@ -1122,11 +1088,6 @@ class Window(QtWidgets.QDialog): for instance_id in existing_ids: self.instance_model.remove(instance_id) - if result.get("error"): - # Toggle from artist to overview tab on error - if self.tabs["artist"].isChecked(): - self.tabs["overview"].toggle() - result["records"] = self.terminal_model.prepare_records( result, self._suspend_logs @@ -1274,7 +1235,6 @@ class Window(QtWidgets.QDialog): self.terminal_proxy.deleteLater() self.plugin_proxy.deleteLater() - self.artist_view.setModel(None) self.overview_instance_view.setModel(None) self.overview_plugin_view.setModel(None) self.terminal_view.setModel(None) From 9c1343f80d0a42f2fc6a60cf88e38ca51fdaa780 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 17 Nov 2020 10:40:13 +0100 Subject: [PATCH 110/147] removed artist tab icon --- pype/tools/pyblish_pype/app.css | 4 ---- pype/tools/pyblish_pype/img/tab-home.png | Bin 313 -> 0 bytes 2 files changed, 4 deletions(-) delete mode 100644 pype/tools/pyblish_pype/img/tab-home.png diff --git a/pype/tools/pyblish_pype/app.css b/pype/tools/pyblish_pype/app.css index c51126e89f..33b6acbddb 100644 --- a/pype/tools/pyblish_pype/app.css +++ b/pype/tools/pyblish_pype/app.css @@ -222,10 +222,6 @@ QToolButton { background: #444; } -#Header #ArtistTab { - background-image: url("img/tab-home.png"); -} - #Header #TerminalTab { background-image: url("img/tab-terminal.png"); } diff --git a/pype/tools/pyblish_pype/img/tab-home.png b/pype/tools/pyblish_pype/img/tab-home.png deleted file mode 100644 index 9133d06edca1a1bc0560a007e1da2b98d03723df..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 313 zcmeAS@N?(olHy`uVBq!ia0vp^A|TAc1|)ksWqE-VTavfC3&Vd9T(EcfWT419PZ!4! zi{7i37J9iP3bZ{ePr1wL7-oD$Gp zqq^{WM@8UjzJ@B6o(A>@MvQVR#0{b)7sQ`m-m&A~>Sf;=L>pRNX2%E16&e!i?~ zTEVxK_DgG@@+KHYzA;vscRM_e)vxiX%8ym^f|I9Kz0rBxvuxgq_N=Qis*}#%;{4&C zU(C+asQ*>#;g^QnY;Tx$A2>VbDVu}&6~+y=>ers^oA7yZ{)}&#B@Hp%_CTLAc)I$z JtaD0e0st-zfV}_! From cdb48823f3476cdb335c18d831b71e3151ecbbc3 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 17 Nov 2020 17:37:59 +0100 Subject: [PATCH 111/147] allow sorting on InstanceView --- pype/tools/pyblish_pype/view.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pype/tools/pyblish_pype/view.py b/pype/tools/pyblish_pype/view.py index 777871ca8e..3bdb0c25ea 100644 --- a/pype/tools/pyblish_pype/view.py +++ b/pype/tools/pyblish_pype/view.py @@ -27,6 +27,8 @@ class OverviewView(QtWidgets.QTreeView): self.setVerticalScrollMode(QtWidgets.QTreeView.ScrollPerPixel) self.setHeaderHidden(True) self.setRootIsDecorated(False) + self.setSortingEnabled(True) + self.sortByColumn(0, QtCore.Qt.AscendingOrder) self.setIndentation(0) if animated: self.setAnimated(True) From 788bd3bcb9cf19fab8d477b43ad8682a7028486d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 17 Nov 2020 17:38:14 +0100 Subject: [PATCH 112/147] do not add context item under group item --- pype/tools/pyblish_pype/model.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/pype/tools/pyblish_pype/model.py b/pype/tools/pyblish_pype/model.py index ffcf9a6501..c00776e4fa 100644 --- a/pype/tools/pyblish_pype/model.py +++ b/pype/tools/pyblish_pype/model.py @@ -717,15 +717,18 @@ class InstanceModel(QtGui.QStandardItemModel): def append(self, instance): new_item = InstanceItem(instance) - families = new_item.data(Roles.FamiliesRole) - group_item = self.group_items.get(families[0]) - if not group_item: - group_item = GroupItem(families[0]) - self.appendRow(group_item) - self.group_items[families[0]] = group_item - self.group_created.emit(group_item.index()) + if new_item.is_context: + self.appendRow(new_item) + else: + families = new_item.data(Roles.FamiliesRole) + group_item = self.group_items.get(families[0]) + if not group_item: + group_item = GroupItem(families[0]) + self.appendRow(group_item) + self.group_items[families[0]] = group_item + self.group_created.emit(group_item.index()) - group_item.appendRow(new_item) + group_item.appendRow(new_item) instance_id = instance.id self.instance_items[instance_id] = new_item From 85105979427b2c5418f68629a0de385125b82874 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 17 Nov 2020 17:38:28 +0100 Subject: [PATCH 113/147] implemented sorting proxy model for instances --- pype/tools/pyblish_pype/model.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pype/tools/pyblish_pype/model.py b/pype/tools/pyblish_pype/model.py index c00776e4fa..ec9689381e 100644 --- a/pype/tools/pyblish_pype/model.py +++ b/pype/tools/pyblish_pype/model.py @@ -845,6 +845,22 @@ class InstanceModel(QtGui.QStandardItemModel): ) +class InstanceSortProxy(QtCore.QSortFilterProxyModel): + def __init__(self, *args, **kwargs): + super(InstanceSortProxy, self).__init__(*args, **kwargs) + # Do not care about lower/upper case + self.setSortCaseSensitivity(QtCore.Qt.CaseInsensitive) + + def lessThan(self, x_index, y_index): + x_type = x_index.data(Roles.TypeRole) + y_type = y_index.data(Roles.TypeRole) + if x_type != y_type: + if x_type == GroupType: + return False + return True + return super(InstanceSortProxy, self).lessThan(x_index, y_index) + + class TerminalDetailItem(QtGui.QStandardItem): key_label_record_map = ( ("instance", "Instance"), From e1b011c048e340f9e2869bdcada5684634fbc25b Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 17 Nov 2020 17:38:58 +0100 Subject: [PATCH 114/147] sorting proxy model used in pyblish gui --- pype/tools/pyblish_pype/window.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/pype/tools/pyblish_pype/window.py b/pype/tools/pyblish_pype/window.py index 66015c4255..df1d8687d5 100644 --- a/pype/tools/pyblish_pype/window.py +++ b/pype/tools/pyblish_pype/window.py @@ -143,16 +143,18 @@ class Window(QtWidgets.QDialog): # TODO add parent overview_page = QtWidgets.QWidget() - instance_model = model.InstanceModel(controller) overview_instance_view = view.InstanceView( animated=settings.Animated, parent=overview_page ) overview_instance_delegate = delegate.InstanceDelegate( parent=overview_instance_view ) + instance_model = model.InstanceModel(controller) + instance_sort_proxy = model.InstanceSortProxy() + instance_sort_proxy.setSourceModel(instance_model) overview_instance_view.setItemDelegate(overview_instance_delegate) - overview_instance_view.setModel(instance_model) + overview_instance_view.setModel(instance_sort_proxy) overview_plugin_view = view.PluginView( animated=settings.Animated, parent=overview_page @@ -437,9 +439,7 @@ class Window(QtWidgets.QDialog): self.on_plugin_action_menu_requested ) - instance_model.group_created.connect( - overview_instance_view.expand - ) + instance_model.group_created.connect(self.on_instance_group_created) self.main_widget = main_widget @@ -461,6 +461,7 @@ class Window(QtWidgets.QDialog): self.plugin_model = plugin_model self.plugin_proxy = plugin_proxy self.instance_model = instance_model + self.instance_sort_proxy = instance_sort_proxy self.presets_button = presets_button @@ -586,6 +587,10 @@ class Window(QtWidgets.QDialog): self.update_compatibility() + def on_instance_group_created(self, index): + _index = self.instance_sort_proxy.mapFromSource(index) + self.overview_instance_view.expand(_index) + def on_plugin_toggle(self, index, state=None): """An item is requesting to be toggled""" if not index.data(Roles.IsOptionalRole): @@ -982,11 +987,14 @@ class Window(QtWidgets.QDialog): def on_passed_group(self, order): for group_item in self.instance_model.group_items.values(): - if self.overview_instance_view.isExpanded(group_item.index()): + group_index = self.instance_sort_proxy.mapFromSource( + group_item.index() + ) + if self.overview_instance_view.isExpanded(group_index): continue if group_item.publish_states & GroupStates.HasError: - self.overview_instance_view.expand(group_item.index()) + self.overview_instance_view.expand(group_index) for group_item in self.plugin_model.group_items.values(): # TODO check only plugins from the group @@ -996,19 +1004,16 @@ class Window(QtWidgets.QDialog): if order != group_item.order: continue + group_index = self.plugin_proxy.mapFromSource(group_item.index()) if group_item.publish_states & GroupStates.HasError: - self.overview_plugin_view.expand( - self.plugin_proxy.mapFromSource(group_item.index()) - ) + self.overview_plugin_view.expand(group_index) continue group_item.setData( {GroupStates.HasFinished: True}, Roles.PublishFlagsRole ) - self.overview_plugin_view.collapse( - self.plugin_proxy.mapFromSource(group_item.index()) - ) + self.overview_plugin_view.collapse(group_index) def on_was_stopped(self): errored = self.controller.errored From d79078c4b9c3723b87e1780de81b4b005afc87b4 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 17 Nov 2020 19:30:37 +0100 Subject: [PATCH 115/147] sorting only in instance view --- pype/tools/pyblish_pype/view.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/tools/pyblish_pype/view.py b/pype/tools/pyblish_pype/view.py index 3bdb0c25ea..7245393714 100644 --- a/pype/tools/pyblish_pype/view.py +++ b/pype/tools/pyblish_pype/view.py @@ -27,8 +27,6 @@ class OverviewView(QtWidgets.QTreeView): self.setVerticalScrollMode(QtWidgets.QTreeView.ScrollPerPixel) self.setHeaderHidden(True) self.setRootIsDecorated(False) - self.setSortingEnabled(True) - self.sortByColumn(0, QtCore.Qt.AscendingOrder) self.setIndentation(0) if animated: self.setAnimated(True) @@ -107,6 +105,8 @@ class PluginView(OverviewView): class InstanceView(OverviewView): def __init__(self, *args, **kwargs): super(InstanceView, self).__init__(*args, **kwargs) + self.setSortingEnabled(True) + self.sortByColumn(0, QtCore.Qt.AscendingOrder) self.viewport().setMouseTracking(True) self._pressed_group_index = None self._pressed_expander = None From 25a293e0ecb2e02ba81cee4b039e041e38156ee2 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Tue, 17 Nov 2020 19:51:34 +0100 Subject: [PATCH 116/147] bump version --- pype/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/version.py b/pype/version.py index 4de2b4e35d..d0979fd030 100644 --- a/pype/version.py +++ b/pype/version.py @@ -1 +1 @@ -__version__ = "2.13.6" +__version__ = "2.14.0" From a08f4780292c7b1f4517966b3fcbcbbdf886b90d Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Tue, 17 Nov 2020 23:30:41 +0100 Subject: [PATCH 117/147] add changelog for past hotfixes --- .github_changelog_generator | 3 +- CHANGELOG.md | 68 +++++++++++++++++++++++++++++++++++-- 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/.github_changelog_generator b/.github_changelog_generator index cd09ebcbfa..da6c35cebf 100644 --- a/.github_changelog_generator +++ b/.github_changelog_generator @@ -1,7 +1,8 @@ pr-wo-labels=False exclude-labels=duplicate,question,invalid,wontfix,weekly-digest author=False -unreleased=False +unreleased=True since-tag=2.11.0 release-branch=master enhancement-label=**Enhancements:** +issues=False diff --git a/CHANGELOG.md b/CHANGELOG.md index 9349589f8f..b8b96fb4c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,69 @@ # Changelog +## [2.13.6](https://github.com/pypeclub/pype/tree/2.13.6) (2020-11-15) + +[Full Changelog](https://github.com/pypeclub/pype/compare/2.13.5...2.13.6) + +**Fixed bugs:** + +- Maya workfile version wasn't syncing with renders properly [\#711](https://github.com/pypeclub/pype/pull/711) +- Maya: Fix for publishing multiple cameras with review from the same scene [\#710](https://github.com/pypeclub/pype/pull/710) + +## [2.13.5](https://github.com/pypeclub/pype/tree/2.13.5) (2020-11-12) + +[Full Changelog](https://github.com/pypeclub/pype/compare/2.13.4...2.13.5) + +**Enhancements:** + +- 3.0 lib refactor [\#664](https://github.com/pypeclub/pype/issues/664) + +**Fixed bugs:** + +- Wrong thumbnail file was picked when publishing sequence in standalone publisher [\#703](https://github.com/pypeclub/pype/pull/703) +- Fix: Burnin data pass and FFmpeg tool check [\#701](https://github.com/pypeclub/pype/pull/701) + +## [2.13.4](https://github.com/pypeclub/pype/tree/2.13.4) (2020-11-09) + +[Full Changelog](https://github.com/pypeclub/pype/compare/2.13.3...2.13.4) + +**Enhancements:** + +- AfterEffects integration with Websocket [\#663](https://github.com/pypeclub/pype/issues/663) + +**Fixed bugs:** + +- Photoshop uhiding hidden layers [\#688](https://github.com/pypeclub/pype/issues/688) +- \#688 - Fix publishing hidden layers [\#692](https://github.com/pypeclub/pype/pull/692) + +**Closed issues:** + +- Nuke Favorite directories "shot dir" "project dir" - not working [\#684](https://github.com/pypeclub/pype/issues/684) + +**Merged pull requests:** + +- Nuke Favorite directories "shot dir" "project dir" - not working \#684 [\#685](https://github.com/pypeclub/pype/pull/685) + +## [2.13.3](https://github.com/pypeclub/pype/tree/2.13.3) (2020-11-03) + +[Full Changelog](https://github.com/pypeclub/pype/compare/2.13.2...2.13.3) + +**Enhancements:** + +- TV paint base integration [\#612](https://github.com/pypeclub/pype/issues/612) + +**Fixed bugs:** + +- Fix ffmpeg executable path with spaces [\#680](https://github.com/pypeclub/pype/pull/680) +- Hotfix: Added default version number [\#679](https://github.com/pypeclub/pype/pull/679) + +## [2.13.2](https://github.com/pypeclub/pype/tree/2.13.2) (2020-10-28) + +[Full Changelog](https://github.com/pypeclub/pype/compare/2.13.1...2.13.2) + +**Fixed bugs:** + +- Nuke: wrong conditions when fixing legacy write nodes [\#665](https://github.com/pypeclub/pype/pull/665) + ## [2.13.1](https://github.com/pypeclub/pype/tree/2.13.1) (2020-10-23) [Full Changelog](https://github.com/pypeclub/pype/compare/2.13.0...2.13.1) @@ -12,6 +76,7 @@ - Layer name is not propagating to metadata in Photoshop [\#654](https://github.com/pypeclub/pype/issues/654) - Loader in Photoshop fails with "can't set attribute" [\#650](https://github.com/pypeclub/pype/issues/650) +- Nuke Load mp4 wrong frame range [\#661](https://github.com/pypeclub/pype/issues/661) - Hiero: Review video file adding one frame to the end [\#659](https://github.com/pypeclub/pype/issues/659) ## [2.13.0](https://github.com/pypeclub/pype/tree/2.13.0) (2020-10-18) @@ -60,7 +125,6 @@ **Merged pull requests:** -- Audio file existence check [\#614](https://github.com/pypeclub/pype/pull/614) - Avalon module without Qt [\#581](https://github.com/pypeclub/pype/pull/581) - Ftrack module without Qt [\#577](https://github.com/pypeclub/pype/pull/577) @@ -135,6 +199,7 @@ **Merged pull requests:** +- Audio file existence check [\#614](https://github.com/pypeclub/pype/pull/614) - NKS small fixes [\#587](https://github.com/pypeclub/pype/pull/587) - Standalone publisher editorial plugins interfering [\#580](https://github.com/pypeclub/pype/pull/580) @@ -185,7 +250,6 @@ **Fixed bugs:** - Maya: Fix tile order for Draft Tile Assembler [\#511](https://github.com/pypeclub/pype/pull/511) -- NukeStudio: Fix comment tag collection and integration. [\#508](https://github.com/pypeclub/pype/pull/508) - Remove extra dash [\#501](https://github.com/pypeclub/pype/pull/501) - Fix: strip dot from repre names in single frame renders [\#498](https://github.com/pypeclub/pype/pull/498) - Better handling of destination during integrating [\#485](https://github.com/pypeclub/pype/pull/485) From 8f82aab6f1eacd7523259f1b8ae184f16a66b58a Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Tue, 17 Nov 2020 23:34:46 +0100 Subject: [PATCH 118/147] add camerarig family to maya --- pype/plugins/global/publish/integrate_new.py | 3 ++- pype/plugins/maya/create/create_camera.py | 11 ++++++++++- pype/plugins/maya/create/create_cameraRig.py | 11 +++++++++++ pype/plugins/maya/load/load_reference.py | 3 ++- pype/plugins/maya/publish/extract_maya_scene_raw.py | 3 ++- 5 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 pype/plugins/maya/create/create_cameraRig.py diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index d040c7167c..802be69ac5 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -90,7 +90,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): "harmony.template", "harmony.palette", "editorial", - "background" + "background", + "camerarig" ] exclude_families = ["clip"] db_representation_context_keys = [ diff --git a/pype/plugins/maya/create/create_camera.py b/pype/plugins/maya/create/create_camera.py index e40260401c..0841b0ff86 100644 --- a/pype/plugins/maya/create/create_camera.py +++ b/pype/plugins/maya/create/create_camera.py @@ -5,7 +5,7 @@ from pype.hosts.maya import lib class CreateCamera(avalon.maya.Creator): """Single baked camera""" - name = "cameraDefault" + name = "cameraMain" label = "Camera" family = "camera" icon = "video-camera" @@ -22,3 +22,12 @@ class CreateCamera(avalon.maya.Creator): # Bake to world space by default, when this is False it will also # include the parent hierarchy in the baked results self.data['bakeToWorldSpace'] = True + +class CreateCameraRig(avalon.maya.Creator): + """Complex hierarchy with camera.""" + + name = "camerarigMain" + label = "Camera Rig" + family = "camerarig" + icon = "video-camera" + defaults = ['Main'] diff --git a/pype/plugins/maya/create/create_cameraRig.py b/pype/plugins/maya/create/create_cameraRig.py new file mode 100644 index 0000000000..e7cc40dc24 --- /dev/null +++ b/pype/plugins/maya/create/create_cameraRig.py @@ -0,0 +1,11 @@ +import avalon.maya + + +class CreateMayaAscii(avalon.maya.Creator): + """Raw Maya Ascii file export""" + + name = "mayaAscii" + label = "Maya Ascii" + family = "mayaAscii" + icon = "file-archive-o" + defaults = ['Main'] diff --git a/pype/plugins/maya/load/load_reference.py b/pype/plugins/maya/load/load_reference.py index 3a9b75a2f5..dbb3cc98b2 100644 --- a/pype/plugins/maya/load/load_reference.py +++ b/pype/plugins/maya/load/load_reference.py @@ -15,7 +15,8 @@ class ReferenceLoader(pype.hosts.maya.plugin.ReferenceLoader): "setdress", "layout", "camera", - "rig"] + "rig", + "camerarig"] representations = ["ma", "abc", "fbx", "mb"] tool_names = ["loader"] diff --git a/pype/plugins/maya/publish/extract_maya_scene_raw.py b/pype/plugins/maya/publish/extract_maya_scene_raw.py index d273646af8..0e256bc69f 100644 --- a/pype/plugins/maya/publish/extract_maya_scene_raw.py +++ b/pype/plugins/maya/publish/extract_maya_scene_raw.py @@ -18,7 +18,8 @@ class ExtractMayaSceneRaw(pype.api.Extractor): hosts = ["maya"] families = ["mayaAscii", "setdress", - "layout"] + "layout", + "camerarig"] scene_type = "ma" def process(self, instance): From 0445938ef4d8f52f4e6a0d7a0775ea3178d3e0ef Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Tue, 17 Nov 2020 23:35:58 +0100 Subject: [PATCH 119/147] fix loading of vray proxies --- pype/plugins/maya/load/load_vrayproxy.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pype/plugins/maya/load/load_vrayproxy.py b/pype/plugins/maya/load/load_vrayproxy.py index 785488d923..c654f60e09 100644 --- a/pype/plugins/maya/load/load_vrayproxy.py +++ b/pype/plugins/maya/load/load_vrayproxy.py @@ -39,20 +39,20 @@ class VRayProxyLoader(api.Loader): with lib.maintained_selection(): cmds.namespace(addNamespace=namespace) with namespaced(namespace, new=False): - nodes = self.create_vray_proxy(name, + nodes, group_node = self.create_vray_proxy(name, filename=self.fname) self[:] = nodes if not nodes: return + # colour the group node presets = config.get_presets(project=os.environ['AVALON_PROJECT']) colors = presets['plugins']['maya']['load']['colors'] - c = colors.get(family) if c is not None: - cmds.setAttr("{0}_{1}.useOutlinerColor".format(name, "GRP"), 1) - cmds.setAttr("{0}_{1}.outlinerColor".format(name, "GRP"), + cmds.setAttr("{0}.useOutlinerColor".format(group_node), 1) + cmds.setAttr("{0}.outlinerColor".format(group_node), c[0], c[1], c[2]) return containerise( @@ -158,4 +158,5 @@ class VRayProxyLoader(api.Loader): cmds.refresh() cmds.setAttr("{}.geomType".format(vray_mesh), 2) - return nodes + + return nodes, group_node From 4a1fa9e8e3a9ac3f777e10750c4eca639d7f85d2 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Tue, 17 Nov 2020 23:42:52 +0100 Subject: [PATCH 120/147] shut up hound and remove obsolete creator --- pype/plugins/maya/create/create_camera.py | 1 + pype/plugins/maya/create/create_cameraRig.py | 11 ----------- 2 files changed, 1 insertion(+), 11 deletions(-) delete mode 100644 pype/plugins/maya/create/create_cameraRig.py diff --git a/pype/plugins/maya/create/create_camera.py b/pype/plugins/maya/create/create_camera.py index 0841b0ff86..acff93c03c 100644 --- a/pype/plugins/maya/create/create_camera.py +++ b/pype/plugins/maya/create/create_camera.py @@ -23,6 +23,7 @@ class CreateCamera(avalon.maya.Creator): # include the parent hierarchy in the baked results self.data['bakeToWorldSpace'] = True + class CreateCameraRig(avalon.maya.Creator): """Complex hierarchy with camera.""" diff --git a/pype/plugins/maya/create/create_cameraRig.py b/pype/plugins/maya/create/create_cameraRig.py deleted file mode 100644 index e7cc40dc24..0000000000 --- a/pype/plugins/maya/create/create_cameraRig.py +++ /dev/null @@ -1,11 +0,0 @@ -import avalon.maya - - -class CreateMayaAscii(avalon.maya.Creator): - """Raw Maya Ascii file export""" - - name = "mayaAscii" - label = "Maya Ascii" - family = "mayaAscii" - icon = "file-archive-o" - defaults = ['Main'] From 8fdb75b720b7649f0ee2b7c111a6d8ef93da21c0 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Tue, 17 Nov 2020 23:44:56 +0100 Subject: [PATCH 121/147] hound fixes --- pype/plugins/maya/load/load_vrayproxy.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pype/plugins/maya/load/load_vrayproxy.py b/pype/plugins/maya/load/load_vrayproxy.py index c654f60e09..894ec75c32 100644 --- a/pype/plugins/maya/load/load_vrayproxy.py +++ b/pype/plugins/maya/load/load_vrayproxy.py @@ -158,5 +158,4 @@ class VRayProxyLoader(api.Loader): cmds.refresh() cmds.setAttr("{}.geomType".format(vray_mesh), 2) - return nodes, group_node From c610071700e263d140df728f4e8c938aac660e5d Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Wed, 18 Nov 2020 09:38:09 +0100 Subject: [PATCH 122/147] mark RC version --- pype/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/version.py b/pype/version.py index d0979fd030..7509794b84 100644 --- a/pype/version.py +++ b/pype/version.py @@ -1 +1 @@ -__version__ = "2.14.0" +__version__ = "2.14.0-rc" From dbeed281cb1ddcea1923965cd51409b3e934bee4 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 18 Nov 2020 12:13:25 +0100 Subject: [PATCH 123/147] feat(SP): dictionary way of adding tasks from preset --- .../publish/collect_hierarchy.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py index 45d2fb4160..b2d71084e4 100644 --- a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py +++ b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py @@ -130,10 +130,17 @@ class CollectHierarchyInstance(pyblish.api.ContextPlugin): if self.shot_add_tasks: tasks_to_add = dict() project_tasks = io.find_one({"type": "project"})["config"]["tasks"] - for task in self.shot_add_tasks: + for task_name, task_data in self.shot_add_tasks.items(): for task_type in project_tasks.keys(): - if task_type.lower() in task.lower(): - tasks_to_add.update({task: {"type": task_type}}) + try: + if task_type in task_data["type"]: + tasks_to_add.update({task_name: task_data}) + except KeyError as error: + self.log.error( + "Wrong presets: `{}` \n" + "example: {\"task_name\": {\"type\":" + " \"FtrackTaskType\"}}".format(error) + ) instance.data["tasks"] = tasks_to_add else: From 03d11f368884262f0029bbad93bed549c09f5d50 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 18 Nov 2020 12:28:18 +0100 Subject: [PATCH 124/147] fix(SP): improving exception for nonexistent task type --- .../publish/collect_hierarchy.py | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py index b2d71084e4..be36f30f4b 100644 --- a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py +++ b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py @@ -131,16 +131,20 @@ class CollectHierarchyInstance(pyblish.api.ContextPlugin): tasks_to_add = dict() project_tasks = io.find_one({"type": "project"})["config"]["tasks"] for task_name, task_data in self.shot_add_tasks.items(): - for task_type in project_tasks.keys(): - try: - if task_type in task_data["type"]: - tasks_to_add.update({task_name: task_data}) - except KeyError as error: - self.log.error( - "Wrong presets: `{}` \n" - "example: {\"task_name\": {\"type\":" - " \"FtrackTaskType\"}}".format(error) - ) + try: + if task_data["type"] in project_tasks.keys(): + tasks_to_add.update({task_name: task_data}) + else: + raise KeyError( + "Wrong FtrackTaskType `{}` for `{}` is not" + " existing in `{}``".format( + task_data["type"], + task_name, + list(project_tasks.keys()))) + except KeyError as error: + raise KeyError( + "Wrong presets: `{0}`".format(error) + ) instance.data["tasks"] = tasks_to_add else: From 28af01c9e775b0e8bc320c52aac5e2e928523e1c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 18 Nov 2020 16:29:04 +0100 Subject: [PATCH 125/147] set_project will try to refresh model by it's own if project was not found --- pype/tools/launcher/widgets.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pype/tools/launcher/widgets.py b/pype/tools/launcher/widgets.py index 894dde3926..9344360c3e 100644 --- a/pype/tools/launcher/widgets.py +++ b/pype/tools/launcher/widgets.py @@ -49,6 +49,14 @@ class ProjectBar(QtWidgets.QWidget): def set_project(self, project_name): index = self.project_combobox.findText(project_name) + if index < 0: + # Try refresh combobox model + self.project_combobox.blockSignals(True) + self.model.refresh() + self.project_combobox.blockSignals(False) + + index = self.project_combobox.findText(project_name) + if index >= 0: self.project_combobox.setCurrentIndex(index) From b92501cc0a5adb899fdb2cf5491b0a9334f5d302 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 18 Nov 2020 16:29:19 +0100 Subject: [PATCH 126/147] do not refresh project bar of project change --- pype/tools/launcher/window.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pype/tools/launcher/window.py b/pype/tools/launcher/window.py index 7c680a927b..9940c77285 100644 --- a/pype/tools/launcher/window.py +++ b/pype/tools/launcher/window.py @@ -375,7 +375,6 @@ class LauncherWindow(QtWidgets.QDialog): def on_project_clicked(self, project_name): self.dbcon.Session["AVALON_PROJECT"] = project_name # Refresh projects - self.asset_panel.project_bar.refresh() self.asset_panel.set_project(project_name) self.set_page(1) self.refresh_actions() From 25e65a20a42b4d06fd2fe5037eb08545ad389a87 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 18 Nov 2020 16:29:48 +0100 Subject: [PATCH 127/147] trigger on_asset_changed when assets are done with refreshing --- pype/tools/launcher/window.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/tools/launcher/window.py b/pype/tools/launcher/window.py index 9940c77285..55635e2139 100644 --- a/pype/tools/launcher/window.py +++ b/pype/tools/launcher/window.py @@ -207,7 +207,7 @@ class AssetsPanel(QtWidgets.QWidget): self.assets_widget.refresh() # Force asset change callback to ensure tasks are correctly reset - tools_lib.schedule(self.on_asset_changed, 0.05, channel="assets") + self.assets_widget.refreshed.connect(self.on_asset_changed) def on_asset_changed(self): """Callback on asset selection changed From 45003e074ba4cdacc35838c5551a48f89ceed0a5 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 18 Nov 2020 17:28:20 +0100 Subject: [PATCH 128/147] #698 - expectedFiles generation reworked --- .../aftereffects/publish/collect_render.py | 107 ++++++++++++++---- 1 file changed, 87 insertions(+), 20 deletions(-) diff --git a/pype/plugins/aftereffects/publish/collect_render.py b/pype/plugins/aftereffects/publish/collect_render.py index f06e35de4f..c89c256afd 100644 --- a/pype/plugins/aftereffects/publish/collect_render.py +++ b/pype/plugins/aftereffects/publish/collect_render.py @@ -1,15 +1,30 @@ from pype.lib import abstract_collect_render, RenderInstance import pyblish.api -from avalon import api -import os import copy +import attr +import os from avalon import aftereffects + +@attr.s +class AERenderInstance(RenderInstance): + # extend generic, composition name is needed + comp_name = attr.ib(default=None) + toBeRenderedOn = attr.ib(default=None) + deadlineSubmissionJob = attr.ib(default=None) + anatomyData = attr.ib(default=None) + outputDir = attr.ib(default=None) + + class CollectAERender(abstract_collect_render.AbstractCollectRender): order = pyblish.api.CollectorOrder + 0.498 label = "Collect After Effects Render Layers" + hosts = ["aftereffects"] + + padding_width = 6 + rendered_extension = 'png' def get_instances(self, context): instances = [] @@ -19,9 +34,13 @@ class CollectAERender(abstract_collect_render.AbstractCollectRender): asset_entity = context.data["assetEntity"] project_entity = context.data["projectEntity"] - for inst in aftereffects.stub().get_metadata().values(): - if inst["family"] == "render" and inst["active"]: - instance = RenderInstance( + compositions = aftereffects.stub().get_items(True) + compositions_by_id = {item.id: item for item in compositions} + for item_id, inst in aftereffects.stub().get_metadata().items(): + if inst["family"] == "render.farm" and inst["active"]: + instance = AERenderInstance( + family=inst["family"], + families=[inst["family"]], version=version, time="", source=current_file, @@ -43,31 +62,79 @@ class CollectAERender(abstract_collect_render.AbstractCollectRender): tileRendering=False, tilesX=0, tilesY=0, - frameStart=asset_entity["data"]["frameStart"], - frameEnd=asset_entity["data"]["frameEnd"], - frameStep=1 - ) + frameStart=0, # asset_entity["data"]["frameStart"], + frameEnd=1, # asset_entity["data"]["frameEnd"], + frameStep=1, + toBeRenderedOn='deadline' + ) + + comp = compositions_by_id.get(int(item_id)) + if not comp: + raise ValueError("There is no composition for item {}". + format(item_id)) + instance.comp_name = comp.name instance._anatomy = context.data["anatomy"] - instance._anatomyData = context.data["anatomyData"] + instance.anatomyData = context.data["anatomyData"] + + instance.outputDir = self._get_output_dir(instance) + instances.append(instance) return instances def get_expected_files(self, render_instance): + """ + Returns list of rendered files that should be created by + Deadline. These are not published directly, they are source + for later 'submit_publish_job'. + + Args: + render_instance (RenderInstance): to pull anatomy and parts used + in url + + Returns: + (list) of absolut urls to rendered file + """ + start = render_instance.frameStart + end = render_instance.frameEnd + + # render to folder of workfile + base_dir = os.path.dirname(render_instance.source) + expected_files = [] + for frame in range(start, end + 1): + path = os.path.join(base_dir, "{}_{}_{}.{}.{}".format( + render_instance.asset, + render_instance.subset, + "v{:03d}".format(render_instance.version), + str(frame).zfill(self.padding_width), + self.rendered_extension + )) + expected_files.append(path) + + return expected_files + + def _get_output_dir(self, render_instance): + """ + Returns dir path of published asset. Required for + 'submit_publish_job'. + + It is different from rendered files (expectedFiles), these are + collected first in some 'staging' area, published later. + + Args: + render_instance (RenderInstance): to pull anatomy and parts used + in url + + Returns: + (str): absolute path to published files + """ anatomy = render_instance._anatomy - anatomy_data = copy.deepcopy(render_instance._anatomyData) + anatomy_data = copy.deepcopy(render_instance.anatomyData) anatomy_data["family"] = render_instance.family anatomy_data["version"] = render_instance.version anatomy_data["subset"] = render_instance.subset - padding = anatomy.templates.get("frame_padding", 4) - anatomy_data.update({ - "frame": f"%0{padding}d", - "representation": "aif" - }) anatomy_filled = anatomy.format(anatomy_data) - import json - print("anatomy_filled::{}".format(json.dumps(anatomy_filled, indent=4))) - render_path = anatomy_filled["render"]["path"] - return [render_path] + # for submit_publish_job + return anatomy_filled["render"]["folder"] From 0fcd304923ed0ebcc69721b0f6ac5e277624736b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 18 Nov 2020 17:30:23 +0100 Subject: [PATCH 129/147] #698 - reworked submit_aftereffects_deadline.py Added plugin for incrementing of workfile --- .../publish/increment_workfile.py | 30 ++++ .../publish/submit_aftereffects_deadline.py | 136 ++++++++++++------ 2 files changed, 122 insertions(+), 44 deletions(-) create mode 100644 pype/plugins/aftereffects/publish/increment_workfile.py diff --git a/pype/plugins/aftereffects/publish/increment_workfile.py b/pype/plugins/aftereffects/publish/increment_workfile.py new file mode 100644 index 0000000000..ef49d01280 --- /dev/null +++ b/pype/plugins/aftereffects/publish/increment_workfile.py @@ -0,0 +1,30 @@ +import pyblish.api +from pype.action import get_errored_plugins_from_data +from pype.lib import version_up + +from avalon import aftereffects + + +class IncrementWorkfile(pyblish.api.InstancePlugin): + """Increment the current workfile. + + Saves the current scene with an increased version number. + """ + + label = "Increment Workfile" + order = pyblish.api.IntegratorOrder + 9.0 + hosts = ["aftereffects"] + families = ["workfile"] + optional = True + + def process(self, instance): + errored_plugins = get_errored_plugins_from_data(instance.context) + if errored_plugins: + raise RuntimeError( + "Skipping incrementing current file because publishing failed." + ) + + scene_path = version_up(instance.context.data["currentFile"]) + aftereffects.stub().saveAs(scene_path, True) + + self.log.info("Incremented workfile to: {}".format(scene_path)) diff --git a/pype/plugins/aftereffects/publish/submit_aftereffects_deadline.py b/pype/plugins/aftereffects/publish/submit_aftereffects_deadline.py index 6378554d9e..c9e0e6323f 100644 --- a/pype/plugins/aftereffects/publish/submit_aftereffects_deadline.py +++ b/pype/plugins/aftereffects/publish/submit_aftereffects_deadline.py @@ -1,81 +1,129 @@ from pype.lib import abstract_submit_deadline, DeadlineJobInfo -from abc import ABCMeta, abstractmethod import pyblish.api import os import attr -import six +import json +import getpass +from avalon import api + @attr.s class DeadlinePluginInfo(): + Comp = attr.ib(default=None) SceneFile = attr.ib(default=None) OutputFilePath = attr.ib(default=None) + Output = attr.ib(default=None) StartupDirectory = attr.ib(default=None) Arguments = attr.ib(default=None) ProjectPath = attr.ib(default=None) SceneFile = attr.ib(default=None) AWSAssetFile0 = attr.ib(default=None) + Version = attr.ib(default=None) -@six.add_metaclass(ABCMeta) class AfterEffectsSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): - label = "Submit to Deadline" - order = pyblish.api.IntegratorOrder + 0.1 + label = "Submit AE to Deadline" + order = pyblish.api.IntegratorOrder hosts = ["aftereffects"] families = ["render.farm"] + use_published = False def get_job_info(self): - deadline_job_info = DeadlineJobInfo() - context = self._instance["context"] + dln_job_info = DeadlineJobInfo(Plugin="AfterEffects") - print("self._instance::{}".format(self._instance)) - print("context::{}".format(context)) - deadline_job_info.Name = "TestName" - deadline_job_info.Plugin = "AfterEffects" - deadline_job_info.UserName = "Test User" # context - deadline_job_info.Department = "Test department" - deadline_job_info.Priority = 50 - deadline_job_info.Group = "Test group" - deadline_job_info.Pool = "Test pool" - frame_range = "{}-{}".format(self._instance.data["frameStart"], - self._instance.data["frameEnd"]) - deadline_job_info.Frames = frame_range - deadline_job_info.Comment = "Test comment" # context - deadline_job_info.OutputFilename = "c:/projects/test.txt" - deadline_job_info.ScheduledType = "Once" - deadline_job_info.JobDelay = "00:00:00" + context = self._instance.context - print("deadline_job_info::{}".format(deadline_job_info)) + dln_job_info.Name = self._instance.data["name"] + dln_job_info.BatchName = os.path.basename(self._instance. + data["source"]) + dln_job_info.Plugin = "AfterEffects" + dln_job_info.UserName = context.data.get( + "deadlineUser", getpass.getuser()) + frame_range = "{}-{}".format(0, # self._instance.data["frameStart"], + 1) # self._instance.data["frameEnd"] + dln_job_info.Frames = frame_range + dln_job_info.OutputFilename = \ + os.path.basename(self._instance.data["expectedFiles"][0]) + dln_job_info.OutputDirectory = \ + os.path.dirname(self._instance.data["expectedFiles"][0]) + dln_job_info.JobDelay = "00:00:00" - return deadline_job_info + keys = [ + "FTRACK_API_KEY", + "FTRACK_API_USER", + "FTRACK_SERVER", + "AVALON_PROJECT", + "AVALON_ASSET", + "AVALON_TASK", + "PYPE_USERNAME", + "PYPE_DEV", + "PYPE_LOG_NO_COLORS" + ] + + environment = dict({key: os.environ[key] for key in keys + if key in os.environ}, **api.Session) + for key in keys: + val = environment.get(key) + if val: + dln_job_info.EnvironmentKeyValue = "{key}={value}".format( + key=key, + value=val) + + return dln_job_info def get_plugin_info(self): deadline_plugin_info = DeadlinePluginInfo() - context = self._instance["context"] + context = self._instance.context script_path = context.data["currentFile"] - render_path = self._instance.data['path'] - render_dir = os.path.normpath(os.path.dirname(render_path)) + render_path = self._instance.data["expectedFiles"][0] + # replace frame info ('000001') with Deadline's required '[#######]' + # expects filename in format project_asset_subset_version.FRAME.ext + render_dir = os.path.dirname(render_path) + file_name = os.path.basename(render_path) + arr = file_name.split('.') + assert len(arr) == 3, \ + "Unable to parse frames from {}".format(file_name) + hashed = '[{}]'.format(len(arr[1]) * "#") - #renderer_path = "C:\\Program Files\\Adobe\\Adobe After Effects 2020\\Support Files\\aerender.exe" - - args = [ - "-s ", - "-e ", - f"-project {script_path}", - f"-output {render_dir}" - "-comp \"Comp\"" - ] + render_path = os.path.join(render_dir, + '{}.{}.{}'.format(arr[0], hashed, arr[2])) + deadline_plugin_info.Comp = self._instance.data["comp_name"] + deadline_plugin_info.Version = "17.5" deadline_plugin_info.SceneFile = script_path - deadline_plugin_info.OutputFilePath = render_dir.replace("\\", "/") + deadline_plugin_info.Output = render_path.replace("\\", "/") - deadline_plugin_info.StartupDirectory = "" - deadline_plugin_info.Arguments = " ".join(args) + return attr.asdict(deadline_plugin_info) - deadline_plugin_info.ProjectPath = script_path - deadline_plugin_info.AWSAssetFile0 = render_path + # TODO temporary, probably should be done in abstract + # extends instance with Deadline submission for 'submit_publish_job' + def submit(self, payload): + """Submit payload to Deadline API end-point. - print("deadline_plugin_info::{}".format(deadline_plugin_info)) + This takes payload in the form of JSON file and POST it to + Deadline jobs end-point. - return deadline_plugin_info + Args: + payload (dict): dict to become json in deadline submission. + + Returns: + str: resulting Deadline job id. + + Throws: + RuntimeError: if submission fails. + + """ + url = "{}/api/jobs".format(self._deadline_url) + response = self._requests_post(url, json=payload) + if not response.ok: + self.log.error("Submission failed!") + self.log.error(response.status_code) + self.log.error(response.content) + self.log.debug(payload) + raise RuntimeError(response.text) + + result = response.json() + self._instance.data["deadlineSubmissionJob"] = result + return result["_id"] From 79f2041714afe6ed696d63ea6d0baf1ffbc8b0ad Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 18 Nov 2020 17:32:50 +0100 Subject: [PATCH 130/147] #698 - added new host --- pype/plugins/global/publish/submit_publish_job.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/plugins/global/publish/submit_publish_job.py b/pype/plugins/global/publish/submit_publish_job.py index 097761cf4e..d72d16f71f 100644 --- a/pype/plugins/global/publish/submit_publish_job.py +++ b/pype/plugins/global/publish/submit_publish_job.py @@ -128,7 +128,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): order = pyblish.api.IntegratorOrder + 0.2 icon = "tractor" - hosts = ["fusion", "maya", "nuke", "celaction"] + hosts = ["fusion", "maya", "nuke", "celaction", "aftereffects"] families = ["render.farm", "prerener", "renderlayer", "imagesequence", "vrayscene"] @@ -510,7 +510,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): """ representations = [] collections, remainders = clique.assemble(exp_files) - bake_render_path = instance.get("bakeRenderPath") + bake_render_path = instance.get("bakeRenderPath", []) # create representation for every collected sequence for collection in collections: From c5d64bd559796d4a0d51714db056792c92dfe8d5 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 18 Nov 2020 17:40:04 +0100 Subject: [PATCH 131/147] Added required attributes for submit_publish_job Added saving deadline response to instance --- pype/lib/abstract_collect_render.py | 6 ++++++ pype/lib/abstract_submit_deadline.py | 3 +++ 2 files changed, 9 insertions(+) diff --git a/pype/lib/abstract_collect_render.py b/pype/lib/abstract_collect_render.py index cd53715763..6bcef1ba90 100644 --- a/pype/lib/abstract_collect_render.py +++ b/pype/lib/abstract_collect_render.py @@ -64,6 +64,12 @@ class RenderInstance(object): tilesX = attr.ib(default=0) # number of tiles in X tilesY = attr.ib(default=0) # number of tiles in Y + # submit_publish_job + toBeRenderedOn = attr.ib(default=None) + deadlineSubmissionJob = attr.ib(default=None) + anatomyData = attr.ib(default=None) + outputDir = attr.ib(default=None) + @frameStart.validator def check_frame_start(self, _, value): """Validate if frame start is not larger then end.""" diff --git a/pype/lib/abstract_submit_deadline.py b/pype/lib/abstract_submit_deadline.py index 3337860508..09916523a4 100644 --- a/pype/lib/abstract_submit_deadline.py +++ b/pype/lib/abstract_submit_deadline.py @@ -582,6 +582,9 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): raise RuntimeError(response.text) result = response.json() + # for submit publish job + self._instance.data["deadlineSubmissionJob"] = result + return result["_id"] def _requests_post(self, *args, **kwargs): From b6bbb8de53e41c637b0929a6ef3b24e6c30bf328 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 18 Nov 2020 17:52:40 +0100 Subject: [PATCH 132/147] Cleanup --- pype/lib/__init__.py | 71 ++++++++++--------- .../aftereffects/publish/collect_render.py | 7 +- .../publish/submit_aftereffects_deadline.py | 34 +-------- 3 files changed, 42 insertions(+), 70 deletions(-) diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index 56eafc1c18..4e120afc9c 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -1,56 +1,54 @@ # -*- coding: utf-8 -*- """Pype lib module.""" - - -from .abstract_submit_deadline import DeadlineJobInfo, AbstractSubmitDeadline -from .abstract_collect_render import RenderInstance, AbstractCollectRender - from .deprecated import ( get_avalon_database, set_io_database ) +from .avalon_context import ( + is_latest, + any_outdated, + get_asset, + get_hierarchy, + get_linked_assets, + get_latest_version, + BuildWorkfile +) + from .hooks import PypeHook, execute_hook from .applications import ( ApplicationLaunchFailed, launch_application, - ApplicationAction + ApplicationAction, + _subprocess ) -from .plugin_tools import filter_pyblish_plugins +from .plugin_tools import filter_pyblish_plugins, source_hash -from .lib_old import ( - _subprocess, - get_paths_from_environ, - get_ffmpeg_tool_path, - get_hierarchy, - add_tool_to_environment, - modified_environ, - pairwise, - grouper, - is_latest, - any_outdated, - _rreplace, +from .path_tools import ( version_up, - switch_item, - _get_host_name, - get_asset, get_version_from_path, get_last_version_from_path, - get_subsets, - get_linked_assets, - BuildWorkfile, - ffprobe_streams, - source_hash, - get_latest_version + get_paths_from_environ, + get_ffmpeg_tool_path ) +from .ffmpeg_utils import ffprobe_streams + __all__ = [ "get_avalon_database", "set_io_database", + "is_latest", + "any_outdated", + "get_asset", + "get_hierarchy", + "get_linked_assets", + "get_latest_version", + "BuildWorkfile", + "PypeHook", "execute_hook", @@ -59,8 +57,15 @@ __all__ = [ "ApplicationAction", "filter_pyblish_plugins", - "AbstractSubmitDeadline", - "DeadlineJobInfo", - "RenderInstance", - "AbstractCollectRender" -] + + "version_up", + "get_version_from_path", + "get_last_version_from_path", + "get_paths_from_environ", + "get_ffmpeg_tool_path", + + "ffprobe_streams", + + "source_hash", + "_subprocess" +] \ No newline at end of file diff --git a/pype/plugins/aftereffects/publish/collect_render.py b/pype/plugins/aftereffects/publish/collect_render.py index c89c256afd..c119e4908d 100644 --- a/pype/plugins/aftereffects/publish/collect_render.py +++ b/pype/plugins/aftereffects/publish/collect_render.py @@ -1,4 +1,5 @@ -from pype.lib import abstract_collect_render, RenderInstance +from pype.lib import abstract_collect_render +from pype.lib.abstract_collect_render import RenderInstance import pyblish.api import copy import attr @@ -11,10 +12,6 @@ from avalon import aftereffects class AERenderInstance(RenderInstance): # extend generic, composition name is needed comp_name = attr.ib(default=None) - toBeRenderedOn = attr.ib(default=None) - deadlineSubmissionJob = attr.ib(default=None) - anatomyData = attr.ib(default=None) - outputDir = attr.ib(default=None) class CollectAERender(abstract_collect_render.AbstractCollectRender): diff --git a/pype/plugins/aftereffects/publish/submit_aftereffects_deadline.py b/pype/plugins/aftereffects/publish/submit_aftereffects_deadline.py index c9e0e6323f..25ff3e0bf1 100644 --- a/pype/plugins/aftereffects/publish/submit_aftereffects_deadline.py +++ b/pype/plugins/aftereffects/publish/submit_aftereffects_deadline.py @@ -1,4 +1,5 @@ -from pype.lib import abstract_submit_deadline, DeadlineJobInfo +from pype.lib import abstract_submit_deadline +from pype.lib.abstract_submit_deadline import DeadlineJobInfo import pyblish.api import os import attr @@ -96,34 +97,3 @@ class AfterEffectsSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline deadline_plugin_info.Output = render_path.replace("\\", "/") return attr.asdict(deadline_plugin_info) - - # TODO temporary, probably should be done in abstract - # extends instance with Deadline submission for 'submit_publish_job' - def submit(self, payload): - """Submit payload to Deadline API end-point. - - This takes payload in the form of JSON file and POST it to - Deadline jobs end-point. - - Args: - payload (dict): dict to become json in deadline submission. - - Returns: - str: resulting Deadline job id. - - Throws: - RuntimeError: if submission fails. - - """ - url = "{}/api/jobs".format(self._deadline_url) - response = self._requests_post(url, json=payload) - if not response.ok: - self.log.error("Submission failed!") - self.log.error(response.status_code) - self.log.error(response.content) - self.log.debug(payload) - raise RuntimeError(response.text) - - result = response.json() - self._instance.data["deadlineSubmissionJob"] = result - return result["_id"] From db5d9911564ddea71f542f9ff87eb8c77a5af004 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 18 Nov 2020 19:07:46 +0100 Subject: [PATCH 133/147] #698 - Fixed dummy values for frames Changed 'render' family in Creator to 'render.farm' --- pype/plugins/aftereffects/create/create_render.py | 4 ++-- pype/plugins/aftereffects/publish/collect_render.py | 8 ++++++-- .../aftereffects/publish/submit_aftereffects_deadline.py | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pype/plugins/aftereffects/create/create_render.py b/pype/plugins/aftereffects/create/create_render.py index f38a4766e4..858e2190c0 100644 --- a/pype/plugins/aftereffects/create/create_render.py +++ b/pype/plugins/aftereffects/create/create_render.py @@ -11,8 +11,8 @@ class CreateRender(api.Creator): """Render folder for publish.""" name = "renderDefault" - label = "Render" - family = "render" + label = "Render on Farm" + family = "render.farm" def process(self): # Photoshop can have multiple LayerSets with the same name, which does diff --git a/pype/plugins/aftereffects/publish/collect_render.py b/pype/plugins/aftereffects/publish/collect_render.py index c119e4908d..25273ac136 100644 --- a/pype/plugins/aftereffects/publish/collect_render.py +++ b/pype/plugins/aftereffects/publish/collect_render.py @@ -34,6 +34,10 @@ class CollectAERender(abstract_collect_render.AbstractCollectRender): compositions = aftereffects.stub().get_items(True) compositions_by_id = {item.id: item for item in compositions} for item_id, inst in aftereffects.stub().get_metadata().items(): + schema = inst.get('schema') + # loaded asset container skip it + if schema and 'container' in schema: + continue if inst["family"] == "render.farm" and inst["active"]: instance = AERenderInstance( family=inst["family"], @@ -59,8 +63,8 @@ class CollectAERender(abstract_collect_render.AbstractCollectRender): tileRendering=False, tilesX=0, tilesY=0, - frameStart=0, # asset_entity["data"]["frameStart"], - frameEnd=1, # asset_entity["data"]["frameEnd"], + frameStart=int(asset_entity["data"]["frameStart"]), + frameEnd=int(asset_entity["data"]["frameEnd"]), frameStep=1, toBeRenderedOn='deadline' ) diff --git a/pype/plugins/aftereffects/publish/submit_aftereffects_deadline.py b/pype/plugins/aftereffects/publish/submit_aftereffects_deadline.py index 25ff3e0bf1..d4b6f11653 100644 --- a/pype/plugins/aftereffects/publish/submit_aftereffects_deadline.py +++ b/pype/plugins/aftereffects/publish/submit_aftereffects_deadline.py @@ -41,8 +41,8 @@ class AfterEffectsSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline dln_job_info.Plugin = "AfterEffects" dln_job_info.UserName = context.data.get( "deadlineUser", getpass.getuser()) - frame_range = "{}-{}".format(0, # self._instance.data["frameStart"], - 1) # self._instance.data["frameEnd"] + frame_range = "{}-{}".format(self._instance.data["frameStart"], + self._instance.data["frameEnd"]) dln_job_info.Frames = frame_range dln_job_info.OutputFilename = \ os.path.basename(self._instance.data["expectedFiles"][0]) From 7cf75117d6f52cf34d604869b284be399051547a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 19 Nov 2020 17:54:55 +0100 Subject: [PATCH 134/147] implemented new event handler to pass status to task's parent in specific occations --- .../events/event_task_status_to_parent.py | 390 ++++++++++++++++++ 1 file changed, 390 insertions(+) create mode 100644 pype/modules/ftrack/events/event_task_status_to_parent.py diff --git a/pype/modules/ftrack/events/event_task_status_to_parent.py b/pype/modules/ftrack/events/event_task_status_to_parent.py new file mode 100644 index 0000000000..514eddb3af --- /dev/null +++ b/pype/modules/ftrack/events/event_task_status_to_parent.py @@ -0,0 +1,390 @@ +import collections +from pype.modules.ftrack import BaseEvent + + +class TaskStatusToParent(BaseEvent): + # Parent types where we care about changing of status + parent_types = ["shot", "asset build"] + + # All parent's tasks must have status name in `task_statuses` key to apply + # status name in `new_status` + parent_status_match_all_task_statuses = [ + { + "new_status": "approved", + "task_statuses": [ + "approved", "omitted" + ] + } + ] + + # Task's status was changed to something in `task_statuses` to apply + # `new_status` on it's parent + # - this is done only if `parent_status_match_all_task_statuses` filtering + # didn't found matching status + parent_status_match_task_statuse = [ + { + "new_status": "in progress", + "task_statuses": [ + "in progress" + ] + } + ] + + def register(self, *args, **kwargs): + result = super(TaskStatusToParent, self).register(*args, **kwargs) + # Clean up presetable attributes + _new_all_match = [] + if self.parent_status_match_all_task_statuses: + for item in self.parent_status_match_all_task_statuses: + _new_all_match.append({ + "new_status": item["new_status"].lower(), + "task_statuses": [ + status_name.lower() + for status_name in item["task_statuses"] + ] + }) + self.parent_status_match_all_task_statuses = _new_all_match + + _new_single_match = [] + if self.parent_status_match_task_statuse: + for item in self.parent_status_match_task_statuse: + _new_single_match.append({ + "new_status": item["new_status"].lower(), + "task_statuses": [ + status_name.lower() + for status_name in item["task_statuses"] + ] + }) + self.parent_status_match_task_statuse = _new_single_match + + self.parent_types = [ + parent_type.lower() + for parent_type in self.parent_types + ] + + return result + + def filter_entities_info(self, session, event): + # Filter if event contain relevant data + entities_info = event["data"].get("entities") + if not entities_info: + return + + filtered_entities = [] + for entity_info in entities_info: + # Care only about tasks + if entity_info.get("entityType") != "task": + continue + + # Care only about changes of status + changes = entity_info.get("changes") or {} + statusid_changes = changes.get("statusid") or {} + if ( + statusid_changes.get("new") is None + or statusid_changes.get("old") is None + ): + continue + + filtered_entities.append(entity_info) + + status_ids = [ + entity_info["changes"]["statusid"]["new"] + for entity_info in filtered_entities + ] + statuses_by_id = self.get_statuses_by_id( + session, status_ids=status_ids + ) + + # Care only about tasks having status with state `Done` + output = [] + for entity_info in filtered_entities: + status_id = entity_info["changes"]["statusid"]["new"] + entity_info["status_entity"] = statuses_by_id[status_id] + output.append(entity_info) + return output + + def get_parents_by_id(self, session, entities_info, object_types): + task_type_id = None + valid_object_type_ids = [] + for object_type in object_types: + object_name_low = object_type["name"].lower() + if object_name_low == "task": + task_type_id = object_type["id"] + + if object_name_low in self.parent_types: + valid_object_type_ids.append(object_type["id"]) + + parent_ids = [ + "\"{}\"".format(entity_info["parentId"]) + for entity_info in entities_info + if entity_info["objectTypeId"] == task_type_id + ] + if not parent_ids: + return {} + + parent_entities = session.query(( + "TypedContext where id in ({}) and object_type_id in ({})" + ).format( + ", ".join(parent_ids), ", ".join(valid_object_type_ids)) + ).all() + + return { + entity["id"]: entity + for entity in parent_entities + } + + def get_tasks_by_id(self, session, parent_ids): + joined_parent_ids = ",".join([ + "\"{}\"".format(parent_id) + for parent_id in parent_ids + ]) + task_entities = session.query( + "Task where parent_id in ({})".format(joined_parent_ids) + ).all() + + return { + entity["id"]: entity + for entity in task_entities + } + + def get_statuses_by_id(self, session, task_entities=None, status_ids=None): + if task_entities is None and status_ids is None: + return {} + + if status_ids is None: + status_ids = [] + for task_entity in task_entities: + status_ids.append(task_entity["status_id"]) + + if not status_ids: + return {} + + status_entities = session.query( + "Status where id in ({})".format(", ".join(status_ids)) + ).all() + + return { + entity["id"]: entity + for entity in status_entities + } + + def launch(self, session, event): + '''Propagates status from version to task when changed''' + + entities_info = self.filter_entities_info(session, event) + if not entities_info: + return + + object_types = session.query("select id, name from ObjectType").all() + parents_by_id = self.get_parents_by_id( + session, entities_info, object_types + ) + if not parents_by_id: + return + tasks_by_id = self.get_tasks_by_id( + session, tuple(parents_by_id.keys()) + ) + + # Just collect them in one variable + entities_by_id = {} + for entity_id, entity in parents_by_id.items(): + entities_by_id[entity_id] = entity + for entity_id, entity in tasks_by_id.items(): + entities_by_id[entity_id] = entity + + # Map task entities by their parents + tasks_by_parent_id = collections.defaultdict(list) + for task_entity in tasks_by_id.values(): + tasks_by_parent_id[task_entity["parent_id"]].append(task_entity) + + # Found status entities for all queried entities + statuses_by_id = self.get_statuses_by_id( + session, + entities_by_id.values() + ) + + # New status determination logic + new_statuses_by_parent_id = self.new_status_by_all_task_statuses( + parents_by_id.keys(), tasks_by_parent_id, statuses_by_id + ) + + # Check if there are remaining any parents that does not have + # determined new status yet + remainder_tasks_by_parent_id = collections.defaultdict(list) + for entity_info in entities_info: + parent_id = entity_info["parentId"] + if ( + # Skip if already has determined new status + parent_id in new_statuses_by_parent_id + # Skip if parent is not in parent mapping + # - if was not found or parent type is not interesting + or parent_id not in parents_by_id + ): + continue + + remainder_tasks_by_parent_id[parent_id].append( + entities_by_id[entity_info["entityId"]] + ) + + # Try to find new status for remained parents + new_statuses_by_parent_id.update( + self.new_status_by_remainders( + remainder_tasks_by_parent_id, + statuses_by_id + ) + ) + + # Make sure new_status is set to valid value + for parent_id in tuple(new_statuses_by_parent_id.keys()): + new_status_name = new_statuses_by_parent_id[parent_id] + if not new_status_name: + new_statuses_by_parent_id.pop(parent_id) + + # If there are not new statuses then just skip + if not new_statuses_by_parent_id: + return + + # Get project schema from any available entity + _entity = None + for _ent in entities_by_id.values(): + _entity = _ent + break + + project_entity = self.get_project_from_entity(_entity) + project_schema = project_entity["project_schema"] + + # Map type names by lowere type names + types_mapping = { + _type.lower(): _type + for _type in session.types + } + # Map object type id by lowered and modified object type name + object_type_mapping = {} + for object_type in object_types: + mapping_name = object_type["name"].lower().replace(" ", "") + object_type_mapping[object_type["id"]] = mapping_name + + statuses_by_obj_id = {} + for parent_id, new_status_name in new_statuses_by_parent_id.items(): + if not new_status_name: + continue + parent_entity = entities_by_id[parent_id] + obj_id = parent_entity["object_type_id"] + + # Find statuses for entity type by object type name + # in project's schema and cache them + if obj_id not in statuses_by_obj_id: + mapping_name = object_type_mapping[obj_id] + mapped_name = types_mapping.get(mapping_name) + statuses = project_schema.get_statuses(mapped_name) + statuses_by_obj_id[obj_id] = { + status["name"].lower(): status + for status in statuses + } + + statuses_by_name = statuses_by_obj_id[obj_id] + new_status = statuses_by_name.get(new_status_name) + ent_path = "/".join( + [ent["name"] for ent in parent_entity["link"]] + ) + if not new_status: + self.log.warning(( + "\"{}\" Couldn't change status to \"{}\"." + " Status is not available for entity type \"{}\"." + ).format( + new_status_name, ent_path, parent_entity.entity_type + )) + continue + + # Do nothing if status is already set + if new_status["name"].lower() == new_status_name: + continue + + try: + parent_entity["status"] = new_status + session.commit() + self.log.info( + "\"{}\" changed status to \"{}\"".format( + ent_path, new_status["name"] + ) + ) + except Exception: + session.rollback() + self.log.warning( + "\"{}\" status couldnt be set to \"{}\"".format( + ent_path, new_status["name"] + ), + exc_info=True + ) + + def new_status_by_all_task_statuses( + self, parent_ids, tasks_by_parent_id, statuses_by_id + ): + """All statuses of parent entity must match specific status names. + + Only if all task statuses match the condition parent's status name is + determined. + """ + output = {} + for parent_id in parent_ids: + task_statuses_lowered = set() + for task_entity in tasks_by_parent_id[parent_id]: + task_status = statuses_by_id[task_entity["status_id"]] + low_status_name = task_status["name"].lower() + task_statuses_lowered.add(low_status_name) + + new_status = None + for item in self.parent_status_match_all_task_statuses: + valid_item = True + for status_name_low in task_statuses_lowered: + if status_name_low not in item["task_statuses"]: + valid_item = False + break + + if valid_item: + new_status = item["new_status"] + break + + if new_status is not None: + output[parent_id] = new_status + + return output + + def new_status_by_remainders( + self, remainder_tasks_by_parent_id, statuses_by_id + ): + """By new task status can be determined new status of parent.""" + output = {} + if not remainder_tasks_by_parent_id: + return output + + for parent_id, task_entities in remainder_tasks_by_parent_id.items(): + if not task_entities: + continue + + # For cases there are multiple tasks in changes + # - task status which match any new status item by order in the + # list `parent_status_match_task_statuse` is preffered + best_order = len(self.parent_status_match_task_statuse) + best_order_status = None + for task_entity in task_entities: + task_status = statuses_by_id[task_entity["status_id"]] + low_status_name = task_status["name"].lower() + for order, item in enumerate( + self.parent_status_match_task_statuse + ): + if order >= best_order: + break + + if low_status_name in item["task_statuses"]: + best_order = order + best_order_status = item["new_status"] + break + + if best_order_status: + output[parent_id] = best_order_status + return output + + +def register(session, plugins_presets): + TaskStatusToParent(session, plugins_presets).register() From 2b56773410a65f36b07e757ffb2b61b462d3e0c7 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 19 Nov 2020 18:03:13 +0100 Subject: [PATCH 135/147] fixed filtering on next task update --- pype/modules/ftrack/events/event_next_task_update.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pype/modules/ftrack/events/event_next_task_update.py b/pype/modules/ftrack/events/event_next_task_update.py index 1f8407e559..deb789f981 100644 --- a/pype/modules/ftrack/events/event_next_task_update.py +++ b/pype/modules/ftrack/events/event_next_task_update.py @@ -27,6 +27,9 @@ class NextTaskUpdate(BaseEvent): first_filtered_entities.append(entity_info) + if not first_filtered_entities: + return first_filtered_entities + status_ids = [ entity_info["changes"]["statusid"]["new"] for entity_info in first_filtered_entities @@ -34,10 +37,16 @@ class NextTaskUpdate(BaseEvent): statuses_by_id = self.get_statuses_by_id( session, status_ids=status_ids ) + # Make sure `entity_type` is "Task" + task_object_type = session.query( + "select id, name from ObjectType where name is \"Task\"" + ).one() # Care only about tasks having status with state `Done` filtered_entities = [] for entity_info in first_filtered_entities: + if entity_info["objectTypeId"] != task_object_type["id"]: + continue status_id = entity_info["changes"]["statusid"]["new"] status_entity = statuses_by_id[status_id] if status_entity["state"]["name"].lower() == "done": From ac55c67fbf8e89a01a2d56a9613819ddc4480122 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 19 Nov 2020 18:16:46 +0100 Subject: [PATCH 136/147] change variable name --- .../ftrack/events/event_task_status_to_parent.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pype/modules/ftrack/events/event_task_status_to_parent.py b/pype/modules/ftrack/events/event_task_status_to_parent.py index 514eddb3af..b81e1f40bd 100644 --- a/pype/modules/ftrack/events/event_task_status_to_parent.py +++ b/pype/modules/ftrack/events/event_task_status_to_parent.py @@ -21,7 +21,7 @@ class TaskStatusToParent(BaseEvent): # `new_status` on it's parent # - this is done only if `parent_status_match_all_task_statuses` filtering # didn't found matching status - parent_status_match_task_statuse = [ + parent_status_by_task_status = [ { "new_status": "in progress", "task_statuses": [ @@ -46,8 +46,8 @@ class TaskStatusToParent(BaseEvent): self.parent_status_match_all_task_statuses = _new_all_match _new_single_match = [] - if self.parent_status_match_task_statuse: - for item in self.parent_status_match_task_statuse: + if self.parent_status_by_task_status: + for item in self.parent_status_by_task_status: _new_single_match.append({ "new_status": item["new_status"].lower(), "task_statuses": [ @@ -55,7 +55,7 @@ class TaskStatusToParent(BaseEvent): for status_name in item["task_statuses"] ] }) - self.parent_status_match_task_statuse = _new_single_match + self.parent_status_by_task_status = _new_single_match self.parent_types = [ parent_type.lower() @@ -364,14 +364,14 @@ class TaskStatusToParent(BaseEvent): # For cases there are multiple tasks in changes # - task status which match any new status item by order in the - # list `parent_status_match_task_statuse` is preffered - best_order = len(self.parent_status_match_task_statuse) + # list `parent_status_by_task_status` is preffered + best_order = len(self.parent_status_by_task_status) best_order_status = None for task_entity in task_entities: task_status = statuses_by_id[task_entity["status_id"]] low_status_name = task_status["name"].lower() for order, item in enumerate( - self.parent_status_match_task_statuse + self.parent_status_by_task_status ): if order >= best_order: break From bce9fd7572a743673f6099da29ceccdc00651474 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 19 Nov 2020 18:17:46 +0100 Subject: [PATCH 137/147] renamed event handler filename --- ...nt_task_status_to_parent.py => event_task_to_parent_status.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pype/modules/ftrack/events/{event_task_status_to_parent.py => event_task_to_parent_status.py} (100%) diff --git a/pype/modules/ftrack/events/event_task_status_to_parent.py b/pype/modules/ftrack/events/event_task_to_parent_status.py similarity index 100% rename from pype/modules/ftrack/events/event_task_status_to_parent.py rename to pype/modules/ftrack/events/event_task_to_parent_status.py From df1d3f323ddd3b9de6fbe7bc29c2a913739c7eae Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 20 Nov 2020 13:02:14 +0100 Subject: [PATCH 138/147] fixed status name check --- .../ftrack/events/event_task_to_parent_status.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pype/modules/ftrack/events/event_task_to_parent_status.py b/pype/modules/ftrack/events/event_task_to_parent_status.py index b81e1f40bd..f14c52e3a6 100644 --- a/pype/modules/ftrack/events/event_task_to_parent_status.py +++ b/pype/modules/ftrack/events/event_task_to_parent_status.py @@ -87,6 +87,9 @@ class TaskStatusToParent(BaseEvent): filtered_entities.append(entity_info) + if not filtered_entities: + return + status_ids = [ entity_info["changes"]["statusid"]["new"] for entity_info in filtered_entities @@ -292,12 +295,18 @@ class TaskStatusToParent(BaseEvent): "\"{}\" Couldn't change status to \"{}\"." " Status is not available for entity type \"{}\"." ).format( - new_status_name, ent_path, parent_entity.entity_type + ent_path, new_status_name, parent_entity.entity_type )) continue + current_status_name = parent_entity["status"]["name"] # Do nothing if status is already set - if new_status["name"].lower() == new_status_name: + if new_status["name"] == current_status_name: + self.log.debug( + "\"{}\" Status \"{}\" already set.".format( + ent_path, current_status_name + ) + ) continue try: From 825a31f20b510013903c45d22cb161487ee8c267 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 20 Nov 2020 14:15:55 +0100 Subject: [PATCH 139/147] #735 - fixes for After Effects Changed collect_render family to 'render' to get correct name from Creator tool Changed outputDir to point to rendered location for metadata.json file Pulling startFrame, endFrame from AE Added files for handling this to stub --- .../stubs/aftereffects_server_stub.py | 50 +++++++++++++++++++ .../aftereffects/create/create_render.py | 45 ++++++++--------- .../aftereffects/publish/collect_render.py | 45 +++++++++-------- .../publish/submit_aftereffects_deadline.py | 4 +- 4 files changed, 94 insertions(+), 50 deletions(-) diff --git a/pype/modules/websocket_server/stubs/aftereffects_server_stub.py b/pype/modules/websocket_server/stubs/aftereffects_server_stub.py index 697809363e..84dce39a41 100644 --- a/pype/modules/websocket_server/stubs/aftereffects_server_stub.py +++ b/pype/modules/websocket_server/stubs/aftereffects_server_stub.py @@ -205,6 +205,19 @@ class AfterEffectsServerStub(): item_id=item.id, path=path, item_name=item_name)) + def rename_item(self, item, item_name): + """ Replace item with item_name + + Args: + item (dict): + item_name (string): label on item in Project list + + """ + self.websocketserver.call(self.client.call + ('AfterEffects.rename_item', + item_id=item.id, + item_name=item_name)) + def delete_item(self, item): """ Deletes FootageItem with new file Args: @@ -234,6 +247,43 @@ class AfterEffectsServerStub(): color_idx=color_idx )) + def get_work_area(self, item_id): + """ Get work are information for render purposes + Args: + item_id (int): + + """ + res = self.websocketserver.call(self.client.call + ('AfterEffects.get_work_area', + item_id=item_id + )) + + records = self._to_records(res) + if records: + return records.pop() + + log.debug("Couldn't get work area") + + def set_work_area(self, item, start, duration, frame_rate): + """ + Set work area to predefined values (from Ftrack). + Work area directs what gets rendered. + Beware of rounding, AE expects seconds, not frames directly. + + Args: + item (dict): + start (float): workAreaStart in seconds + duration (float): in seconds + frame_rate (float): frames in seconds + """ + self.websocketserver.call(self.client.call + ('AfterEffects.set_work_area', + item_id=item.id, + start=start, + duration=duration, + frame_rate=frame_rate + )) + def save(self): """ Saves active document diff --git a/pype/plugins/aftereffects/create/create_render.py b/pype/plugins/aftereffects/create/create_render.py index 858e2190c0..1944cf9937 100644 --- a/pype/plugins/aftereffects/create/create_render.py +++ b/pype/plugins/aftereffects/create/create_render.py @@ -12,41 +12,36 @@ class CreateRender(api.Creator): name = "renderDefault" label = "Render on Farm" - family = "render.farm" + family = "render" def process(self): - # Photoshop can have multiple LayerSets with the same name, which does - # not work with Avalon. - txt = "Instance with name \"{}\" already exists.".format(self.name) stub = aftereffects.stub() # only after After Effects is up - for layer in stub.get_items(comps=True, - folders=False, - footages=False): - if self.name.lower() == layer.name.lower(): - msg = Qt.QtWidgets.QMessageBox() - msg.setIcon(Qt.QtWidgets.QMessageBox.Warning) - msg.setText(txt) - msg.exec_() - return False - log.debug("options:: {}".format(self.options)) - print("options:: {}".format(self.options)) if (self.options or {}).get("useSelection"): - log.debug("useSelection") - print("useSelection") items = stub.get_selected_items(comps=True, folders=False, footages=False) else: - items = stub.get_items(comps=True, - folders=False, - footages=False) - log.debug("items:: {}".format(items)) - print("items:: {}".format(items)) + self._show_msg("Please select only single composition at time.") + return False + if not items: - raise ValueError("Nothing to create. Select composition " + - "if 'useSelection' or create at least " + - "one composition.") + self._show_msg("Nothing to create. Select composition " + + "if 'useSelection' or create at least " + + "one composition.") + return False for item in items: + txt = "Instance with name \"{}\" already exists.".format(self.name) + if self.name.lower() == item.name.lower(): + self._show_msg(txt) + return False + stub.imprint(item, self.data) stub.set_label_color(item.id, 14) # Cyan options 0 - 16 + stub.rename_item(item, self.data["subset"]) + + def _show_msg(self, txt): + msg = Qt.QtWidgets.QMessageBox() + msg.setIcon(Qt.QtWidgets.QMessageBox.Warning) + msg.setText(txt) + msg.exec_() diff --git a/pype/plugins/aftereffects/publish/collect_render.py b/pype/plugins/aftereffects/publish/collect_render.py index 25273ac136..0e33a26806 100644 --- a/pype/plugins/aftereffects/publish/collect_render.py +++ b/pype/plugins/aftereffects/publish/collect_render.py @@ -1,7 +1,6 @@ from pype.lib import abstract_collect_render from pype.lib.abstract_collect_render import RenderInstance import pyblish.api -import copy import attr import os @@ -38,10 +37,20 @@ class CollectAERender(abstract_collect_render.AbstractCollectRender): # loaded asset container skip it if schema and 'container' in schema: continue - if inst["family"] == "render.farm" and inst["active"]: + + work_area_info = aftereffects.stub().get_work_area(int(item_id)) + frameStart = round(float(work_area_info.workAreaStart) * + float(work_area_info.frameRate)) + + frameEnd = round(float(work_area_info.workAreaStart) * + float(work_area_info.frameRate) + + float(work_area_info.workAreaDuration) * + float(work_area_info.frameRate)) + + if inst["family"] == "render" and inst["active"]: instance = AERenderInstance( - family=inst["family"], - families=[inst["family"]], + family="render.farm", # other way integrate would catch it + families=["render.farm"], version=version, time="", source=current_file, @@ -63,8 +72,8 @@ class CollectAERender(abstract_collect_render.AbstractCollectRender): tileRendering=False, tilesX=0, tilesY=0, - frameStart=int(asset_entity["data"]["frameStart"]), - frameEnd=int(asset_entity["data"]["frameEnd"]), + frameStart=frameStart, + frameEnd=frameEnd, frameStep=1, toBeRenderedOn='deadline' ) @@ -101,6 +110,7 @@ class CollectAERender(abstract_collect_render.AbstractCollectRender): # render to folder of workfile base_dir = os.path.dirname(render_instance.source) + base_dir = os.path.join(base_dir, 'renders', 'aftereffects') expected_files = [] for frame in range(start, end + 1): path = os.path.join(base_dir, "{}_{}_{}.{}.{}".format( @@ -116,26 +126,17 @@ class CollectAERender(abstract_collect_render.AbstractCollectRender): def _get_output_dir(self, render_instance): """ - Returns dir path of published asset. Required for - 'submit_publish_job'. - - It is different from rendered files (expectedFiles), these are - collected first in some 'staging' area, published later. + Returns dir path of rendered files, used in submit_publish_job + for metadata.json location Args: - render_instance (RenderInstance): to pull anatomy and parts used - in url + render_instance (RenderInstance): Returns: - (str): absolute path to published files + (str): absolute path to rendered files """ - anatomy = render_instance._anatomy - anatomy_data = copy.deepcopy(render_instance.anatomyData) - anatomy_data["family"] = render_instance.family - anatomy_data["version"] = render_instance.version - anatomy_data["subset"] = render_instance.subset - - anatomy_filled = anatomy.format(anatomy_data) + base_dir = os.path.dirname(render_instance.source) + base_dir = os.path.join(base_dir, 'renders', 'aftereffects') # for submit_publish_job - return anatomy_filled["render"]["folder"] + return base_dir diff --git a/pype/plugins/aftereffects/publish/submit_aftereffects_deadline.py b/pype/plugins/aftereffects/publish/submit_aftereffects_deadline.py index d4b6f11653..15d9e216fb 100644 --- a/pype/plugins/aftereffects/publish/submit_aftereffects_deadline.py +++ b/pype/plugins/aftereffects/publish/submit_aftereffects_deadline.py @@ -3,7 +3,6 @@ from pype.lib.abstract_submit_deadline import DeadlineJobInfo import pyblish.api import os import attr -import json import getpass from avalon import api @@ -17,7 +16,6 @@ class DeadlinePluginInfo(): StartupDirectory = attr.ib(default=None) Arguments = attr.ib(default=None) ProjectPath = attr.ib(default=None) - SceneFile = attr.ib(default=None) AWSAssetFile0 = attr.ib(default=None) Version = attr.ib(default=None) @@ -27,7 +25,7 @@ class AfterEffectsSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline label = "Submit AE to Deadline" order = pyblish.api.IntegratorOrder hosts = ["aftereffects"] - families = ["render.farm"] + families = ["render.farm"] # cannot be "render' as that is integrated use_published = False def get_job_info(self): From 6cd0e4651119458bb02421c51005802b994487f5 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 20 Nov 2020 18:29:45 +0100 Subject: [PATCH 140/147] #735 - added file name to render folder --- pype/plugins/aftereffects/publish/collect_render.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pype/plugins/aftereffects/publish/collect_render.py b/pype/plugins/aftereffects/publish/collect_render.py index 0e33a26806..13ffc3f208 100644 --- a/pype/plugins/aftereffects/publish/collect_render.py +++ b/pype/plugins/aftereffects/publish/collect_render.py @@ -108,9 +108,7 @@ class CollectAERender(abstract_collect_render.AbstractCollectRender): start = render_instance.frameStart end = render_instance.frameEnd - # render to folder of workfile - base_dir = os.path.dirname(render_instance.source) - base_dir = os.path.join(base_dir, 'renders', 'aftereffects') + base_dir = self._get_output_dir(render_instance) expected_files = [] for frame in range(start, end + 1): path = os.path.join(base_dir, "{}_{}_{}.{}.{}".format( @@ -127,7 +125,8 @@ class CollectAERender(abstract_collect_render.AbstractCollectRender): def _get_output_dir(self, render_instance): """ Returns dir path of rendered files, used in submit_publish_job - for metadata.json location + for metadata.json location. + Should be in separate folder inside of work area. Args: render_instance (RenderInstance): @@ -135,8 +134,11 @@ class CollectAERender(abstract_collect_render.AbstractCollectRender): Returns: (str): absolute path to rendered files """ + # render to folder of workfile base_dir = os.path.dirname(render_instance.source) - base_dir = os.path.join(base_dir, 'renders', 'aftereffects') + file_name, _ = os.path.splitext( + os.path.basename(render_instance.source)) + base_dir = os.path.join(base_dir, 'renders', 'aftereffects', file_name) # for submit_publish_job return base_dir From 88407488cd70c4830d3d5f0952deb1ea02a82d6b Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 20 Nov 2020 19:12:24 +0100 Subject: [PATCH 141/147] added skip logs to extract review --- pype/plugins/global/publish/extract_review.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index dd28e6fec3..857119176f 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -122,11 +122,24 @@ class ExtractReview(pyblish.api.InstancePlugin): # Loop through representations for repre in tuple(instance.data["representations"]): + repre_name = str(repre.get("name")) tags = repre.get("tags") or [] - if "review" not in tags or "thumbnail" in tags: + if "review" not in tags: + self.log.debug(( + "Repre: {} - Didn't found \"review\" in tags. Skipping" + ).format(repre_name)) + continue + + if "thumbnail" in tags: + self.log.debug(( + "Repre: {} - Found \"thumbnail\" in tags. Skipping" + ).format(repre_name)) continue if "passing" in tags: + self.log.debug(( + "Repre: {} - Found \"passing\" in tags. Skipping" + ).format(repre_name)) continue input_ext = repre["ext"] From a3184ac3412efc3e8b67b8e91cb84facc3ea40ca Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 20 Nov 2020 19:13:04 +0100 Subject: [PATCH 142/147] change tag ftrackreview to review --- pype/plugins/tvpaint/publish/extract_sequence.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pype/plugins/tvpaint/publish/extract_sequence.py b/pype/plugins/tvpaint/publish/extract_sequence.py index d173ac287d..c1c1d13872 100644 --- a/pype/plugins/tvpaint/publish/extract_sequence.py +++ b/pype/plugins/tvpaint/publish/extract_sequence.py @@ -137,8 +137,7 @@ class ExtractSequence(pyblish.api.Extractor): # Fill tags and new families tags = [] if family_lowered in ("review", "renderlayer"): - # Add `ftrackreview` tag - tags.append("ftrackreview") + tags.append("review") repre_files = [ os.path.basename(filepath) From de2b1c7df325842a1ae4e24f615517e3ecc265d5 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 20 Nov 2020 19:13:14 +0100 Subject: [PATCH 143/147] skip if instance was not created --- pype/plugins/tvpaint/publish/collect_instances.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pype/plugins/tvpaint/publish/collect_instances.py b/pype/plugins/tvpaint/publish/collect_instances.py index a9c19f9c0a..ddc447639a 100644 --- a/pype/plugins/tvpaint/publish/collect_instances.py +++ b/pype/plugins/tvpaint/publish/collect_instances.py @@ -67,6 +67,9 @@ class CollectInstances(pyblish.api.ContextPlugin): ) ) + if instance is None: + continue + frame_start = context.data["frameStart"] frame_end = frame_start for layer in instance.data["layers"]: From ac70c0515ad3438d3d12c0b1a787d34d77ac65d7 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 20 Nov 2020 20:04:21 +0100 Subject: [PATCH 144/147] #735 - changed resolving of review --- .../global/publish/submit_publish_job.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/pype/plugins/global/publish/submit_publish_job.py b/pype/plugins/global/publish/submit_publish_job.py index bc644987de..77a8816117 100644 --- a/pype/plugins/global/publish/submit_publish_job.py +++ b/pype/plugins/global/publish/submit_publish_job.py @@ -133,7 +133,9 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): families = ["render.farm", "prerener", "renderlayer", "imagesequence", "vrayscene"] - aov_filter = {"maya": ["beauty"]} + aov_filter = {"maya": [r".+(?:\.|_)([Bb]eauty)(?:\.|_).*"], + "aftereffects": [r".*"], # for everything from AE + "celaction": [r".*"]} enviro_filter = [ "FTRACK_API_USER", @@ -447,8 +449,13 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): preview = False if app in self.aov_filter.keys(): - if aov in self.aov_filter[app]: - preview = True + for aov_pattern in self.aov_filter[app]: + if re.match( + aov_pattern, + aov + ): + preview = True + break new_instance = copy(instance_data) new_instance["subset"] = subset_name @@ -513,29 +520,26 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): collections, remainders = clique.assemble(exp_files) bake_render_path = instance.get("bakeRenderPath", []) + print('@@@@ collections {}'.format(collections)) # create representation for every collected sequence for collection in collections: ext = collection.tail.lstrip(".") preview = False # if filtered aov name is found in filename, toggle it for # preview video rendering - for app in self.aov_filter: + for app in self.aov_filter.keys(): if os.environ.get("AVALON_APP", "") == app: for aov in self.aov_filter[app]: if re.match( - r".+(?:\.|_)({})(?:\.|_).*".format(aov), + aov, list(collection)[0] ): preview = True break - break if bake_render_path: preview = False - if "celaction" in pyblish.api.registered_hosts(): - preview = True - staging = os.path.dirname(list(collection)[0]) success, rootless_staging_dir = ( self.anatomy.find_root_template_from_path(staging) From ad321bb1e79330f839cf649a7cb69d40a8f96400 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 20 Nov 2020 20:07:36 +0100 Subject: [PATCH 145/147] Hound --- pype/plugins/global/publish/submit_publish_job.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pype/plugins/global/publish/submit_publish_job.py b/pype/plugins/global/publish/submit_publish_job.py index 77a8816117..5fce0af35a 100644 --- a/pype/plugins/global/publish/submit_publish_job.py +++ b/pype/plugins/global/publish/submit_publish_job.py @@ -450,8 +450,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): preview = False if app in self.aov_filter.keys(): for aov_pattern in self.aov_filter[app]: - if re.match( - aov_pattern, + if re.match(aov_pattern, aov ): preview = True @@ -520,7 +519,6 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): collections, remainders = clique.assemble(exp_files) bake_render_path = instance.get("bakeRenderPath", []) - print('@@@@ collections {}'.format(collections)) # create representation for every collected sequence for collection in collections: ext = collection.tail.lstrip(".") From a1bf6916a764a848a871dd613c5d106dc175bf67 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Mon, 23 Nov 2020 10:18:49 +0100 Subject: [PATCH 146/147] remove redundant 'review' data --- pype/plugins/global/publish/submit_publish_job.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pype/plugins/global/publish/submit_publish_job.py b/pype/plugins/global/publish/submit_publish_job.py index 5fce0af35a..256bf01665 100644 --- a/pype/plugins/global/publish/submit_publish_job.py +++ b/pype/plugins/global/publish/submit_publish_job.py @@ -559,7 +559,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): # If expectedFile are absolute, we need only filenames "stagingDir": staging, "fps": instance.get("fps"), - "tags": ["review", "preview"] if preview else [], + "tags": ["review"] if preview else [], } # poor man exclusion @@ -711,8 +711,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): "resolutionWidth": data.get("resolutionWidth", 1920), "resolutionHeight": data.get("resolutionHeight", 1080), "multipartExr": data.get("multipartExr", False), - "jobBatchName": data.get("jobBatchName", ""), - "review": data.get("review", True) + "jobBatchName": data.get("jobBatchName", "") } if "prerender" in instance.data["families"]: From dac6ccc1114f1b682b985255a5e912cf5cbb6748 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Mon, 23 Nov 2020 23:14:15 +0100 Subject: [PATCH 147/147] bump version --- pype/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/version.py b/pype/version.py index 4d8713b45c..d0979fd030 100644 --- a/pype/version.py +++ b/pype/version.py @@ -1 +1 @@ -__version__ = "2.14.0-rc" \ No newline at end of file +__version__ = "2.14.0"