From 1b4f452301436131c0024e80b5b80538861ceba8 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 3 May 2023 19:23:58 +0200 Subject: [PATCH] Extracted prepare_representations Various fixes. WIP --- .../plugins/publish/submit_publish_job.py | 149 ++--------------- .../publish/create_nuke_deadline_job.py | 48 +++++- .../publish/create_publish_royalrender_job.py | 32 +++- .../publish/submit_jobs_to_royalrender.py | 2 +- openpype/pipeline/farm/pyblish_functions.py | 156 +++++++++++++++++- 5 files changed, 230 insertions(+), 157 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index 86f647dd1b..58a0cd7219 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -21,10 +21,11 @@ from openpype.pipeline import ( from openpype.tests.lib import is_in_tests from openpype.pipeline.farm.patterning import match_aov_pattern from openpype.lib import is_running_from_build -from openpype.pipeline.farm.pyblish import ( +from openpype.pipeline.farm.pyblish_functions import ( create_skeleton_instance, create_instances_for_aov, - attach_instances_to_subset + attach_instances_to_subset, + prepare_representations ) @@ -332,140 +333,6 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): return deadline_publish_job_id - def _get_representations(self, instance, exp_files): - """Create representations for file sequences. - - This will return representations of expected files if they are not - in hierarchy of aovs. There should be only one sequence of files for - most cases, but if not - we create representation from each of them. - - Arguments: - instance (dict): instance data for which we are - setting representations - exp_files (list): list of expected files - - Returns: - list of representations - - """ - representations = [] - host_name = os.environ.get("AVALON_APP", "") - collections, remainders = clique.assemble(exp_files) - - # create representation for every collected sequence - for collection in collections: - ext = collection.tail.lstrip(".") - preview = False - # TODO 'useSequenceForReview' is temporary solution which does - # not work for 100% of cases. We must be able to tell what - # expected files contains more explicitly and from what - # should be review made. - # - "review" tag is never added when is set to 'False' - if instance["useSequenceForReview"]: - # toggle preview on if multipart is on - if instance.get("multipartExr", False): - self.log.debug( - "Adding preview tag because its multipartExr" - ) - preview = True - else: - render_file_name = list(collection)[0] - # if filtered aov name is found in filename, toggle it for - # preview video rendering - preview = match_aov_pattern( - host_name, self.aov_filter, render_file_name - ) - - staging = os.path.dirname(list(collection)[0]) - success, rootless_staging_dir = ( - self.anatomy.find_root_template_from_path(staging) - ) - if success: - staging = rootless_staging_dir - else: - self.log.warning(( - "Could not find root path for remapping \"{}\"." - " This may cause issues on farm." - ).format(staging)) - - frame_start = int(instance.get("frameStartHandle")) - if instance.get("slate"): - frame_start -= 1 - - rep = { - "name": ext, - "ext": ext, - "files": [os.path.basename(f) for f in list(collection)], - "frameStart": frame_start, - "frameEnd": int(instance.get("frameEndHandle")), - # If expectedFile are absolute, we need only filenames - "stagingDir": staging, - "fps": instance.get("fps"), - "tags": ["review"] if preview else [], - } - - # poor man exclusion - if ext in self.skip_integration_repre_list: - rep["tags"].append("delete") - - if instance.get("multipartExr", False): - rep["tags"].append("multipartExr") - - # support conversion from tiled to scanline - if instance.get("convertToScanline"): - self.log.info("Adding scanline conversion.") - rep["tags"].append("toScanline") - - representations.append(rep) - - self._solve_families(instance, preview) - - # add remainders as representations - for remainder in remainders: - ext = remainder.split(".")[-1] - - staging = os.path.dirname(remainder) - success, rootless_staging_dir = ( - self.anatomy.find_root_template_from_path(staging) - ) - if success: - staging = rootless_staging_dir - else: - self.log.warning(( - "Could not find root path for remapping \"{}\"." - " This may cause issues on farm." - ).format(staging)) - - rep = { - "name": ext, - "ext": ext, - "files": os.path.basename(remainder), - "stagingDir": staging, - } - - preview = match_aov_pattern( - host_name, self.aov_filter, remainder - ) - if preview: - rep.update({ - "fps": instance.get("fps"), - "tags": ["review"] - }) - self._solve_families(instance, preview) - - already_there = False - for repre in instance.get("representations", []): - # might be added explicitly before by publish_on_farm - already_there = repre.get("files") == rep["files"] - if already_there: - self.log.debug("repre {} already_there".format(repre)) - break - - if not already_there: - representations.append(rep) - - return representations - def _solve_families(self, instance, preview=False): families = instance.get("families") @@ -552,12 +419,16 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): if isinstance(instance.data.get("expectedFiles")[0], dict): instances = create_instances_for_aov( - instance, instance_skeleton_data, self.aov_filter) + instance, instance_skeleton_data, + self.aov_filter, self.skip_integration_repre_list) else: - representations = self._get_representations( + representations = prepare_representations( instance_skeleton_data, - instance.data.get("expectedFiles") + instance.data.get("expectedFiles"), + self.anatomy, + self.aov_filter, + self.skip_integration_repre_list ) if "representations" not in instance_skeleton_data.keys(): diff --git a/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py b/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py index 217ebed057..0472f2ea80 100644 --- a/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_nuke_deadline_job.py @@ -1,14 +1,13 @@ # -*- coding: utf-8 -*- """Submitting render job to RoyalRender.""" +import copy import os -import sys import re import platform from datetime import datetime from pyblish.api import InstancePlugin, IntegratorOrder, Context from openpype.tests.lib import is_in_tests -from openpype.lib import is_running_from_build from openpype.pipeline.publish.lib import get_published_workfile_instance from openpype.pipeline.publish import KnownPublishError from openpype.modules.royalrender.api import Api as rrApi @@ -34,6 +33,8 @@ class CreateNukeRoyalRenderJob(InstancePlugin, OpenPypePyblishPluginMixin): priority = 50 chunk_size = 1 concurrent_tasks = 1 + use_gpu = True + use_published = True @classmethod def get_attribute_defs(cls): @@ -69,6 +70,11 @@ class CreateNukeRoyalRenderJob(InstancePlugin, OpenPypePyblishPluginMixin): "suspend_publish", default=False, label="Suspend publish" + ), + BoolDef( + "use_published", + default=cls.use_published, + label="Use published workfile" ) ] @@ -81,6 +87,17 @@ class CreateNukeRoyalRenderJob(InstancePlugin, OpenPypePyblishPluginMixin): self.rr_api = None def process(self, instance): + # import json + # def _default_json(value): + # return str(value) + # filepath = "C:\\Users\\petrk\\PycharmProjects\\Pype3.0\\pype\\tests\\unit\\openpype\\modules\\royalrender\\plugins\\publish\\resources\\instance.json" + # with open(filepath, "w") as f: + # f.write(json.dumps(instance.data, indent=4, default=_default_json)) + # + # filepath = "C:\\Users\\petrk\\PycharmProjects\\Pype3.0\\pype\\tests\\unit\\openpype\\modules\\royalrender\\plugins\\publish\\resources\\context.json" + # with open(filepath, "w") as f: + # f.write(json.dumps(instance.context.data, indent=4, default=_default_json)) + if not instance.data.get("farm"): self.log.info("Skipping local instance.") return @@ -93,6 +110,7 @@ class CreateNukeRoyalRenderJob(InstancePlugin, OpenPypePyblishPluginMixin): "suspend_publish"] context = instance.context + self._instance = instance self._rr_root = self._resolve_rr_path(context, instance.data.get( "rrPathName")) # noqa @@ -218,7 +236,7 @@ class CreateNukeRoyalRenderJob(InstancePlugin, OpenPypePyblishPluginMixin): Renderer="", SeqStart=int(start_frame), SeqEnd=int(end_frame), - SeqStep=int(self._instance.data.get("byFrameStep"), 1), + SeqStep=int(self._instance.data.get("byFrameStep", 1)), SeqFileOffset=0, Version=nuke_version.group(), SceneName=script_path, @@ -298,7 +316,7 @@ class CreateNukeRoyalRenderJob(InstancePlugin, OpenPypePyblishPluginMixin): if "%" not in file: expected_files.append(path) - return + return expected_files if self._instance.data.get("slate"): start_frame -= 1 @@ -308,3 +326,25 @@ class CreateNukeRoyalRenderJob(InstancePlugin, OpenPypePyblishPluginMixin): for i in range(start_frame, (end_frame + 1)) ) return expected_files + + def preview_fname(self, path): + """Return output file path with #### for padding. + + Deadline requires the path to be formatted with # in place of numbers. + For example `/path/to/render.####.png` + + Args: + path (str): path to rendered images + + Returns: + str + + """ + self.log.debug("_ path: `{}`".format(path)) + if "%" in path: + search_results = re.search(r"(%0)(\d)(d.)", path).groups() + self.log.debug("_ search_results: `{}`".format(search_results)) + return int(search_results[1]) + if "#" in path: + self.log.debug("_ path: `{}`".format(path)) + return path diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index b1c84c87b9..6f0bc995d0 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -11,10 +11,11 @@ from openpype.modules.royalrender.rr_job import RRJob, RREnvList from openpype.pipeline.publish import KnownPublishError from openpype.lib.openpype_version import ( get_OpenPypeVersion, get_openpype_version) -from openpype.pipeline.farm.pyblish import ( +from openpype.pipeline.farm.pyblish_functions import ( create_skeleton_instance, create_instances_for_aov, - attach_instances_to_subset + attach_instances_to_subset, + prepare_representations ) @@ -31,6 +32,20 @@ class CreatePublishRoyalRenderJob(InstancePlugin): "harmony": [r".*"], # for everything from AE "celaction": [r".*"]} + skip_integration_repre_list = [] + + # mapping of instance properties to be transferred to new instance + # for every specified family + instance_transfer = { + "slate": ["slateFrames", "slate"], + "review": ["lutPath"], + "render2d": ["bakingNukeScripts", "version"], + "renderlayer": ["convertToScanline"] + } + + # list of family names to transfer to new family if present + families_transfer = ["render3d", "render2d", "ftrack", "slate"] + def process(self, instance): # data = instance.data.copy() context = instance.context @@ -46,15 +61,18 @@ class CreatePublishRoyalRenderJob(InstancePlugin): families_transfer=self.families_transfer, instance_transfer=self.instance_transfer) - instances = None if isinstance(instance.data.get("expectedFiles")[0], dict): instances = create_instances_for_aov( - instance, instance_skeleton_data, self.aov_filter) + instance, instance_skeleton_data, + self.aov_filter, self.skip_integration_repre_list) else: - representations = self._get_representations( + representations = prepare_representations( instance_skeleton_data, - instance.data.get("expectedFiles") + instance.data.get("expectedFiles"), + self.anatomy, + self.aov_filter, + self.skip_integration_repre_list ) if "representations" not in instance_skeleton_data.keys(): @@ -104,7 +122,7 @@ class CreatePublishRoyalRenderJob(InstancePlugin): RRJob: RoyalRender publish job. """ - data = instance.data.copy() + data = deepcopy(instance.data) subset = data["subset"] job_name = "Publish - {subset}".format(subset=subset) diff --git a/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py b/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py index 325fb36993..8546554372 100644 --- a/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py +++ b/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py @@ -11,7 +11,7 @@ from openpype.pipeline.publish import KnownPublishError class SubmitJobsToRoyalRender(ContextPlugin): """Find all jobs, create submission XML and submit it to RoyalRender.""" label = "Submit jobs to RoyalRender" - order = IntegratorOrder + 0.1 + order = IntegratorOrder + 0.3 targets = ["local"] def __init__(self): diff --git a/openpype/pipeline/farm/pyblish_functions.py b/openpype/pipeline/farm/pyblish_functions.py index 645b31b2de..792cc07f02 100644 --- a/openpype/pipeline/farm/pyblish_functions.py +++ b/openpype/pipeline/farm/pyblish_functions.py @@ -284,7 +284,150 @@ def _solve_families(families): return families -def create_instances_for_aov(instance, skeleton, aov_filter): +def prepare_representations(instance, exp_files, anatomy, aov_filter, + skip_integration_repre_list): + """Create representations for file sequences. + + This will return representations of expected files if they are not + in hierarchy of aovs. There should be only one sequence of files for + most cases, but if not - we create representation from each of them. + + Arguments: + instance (dict): instance data for which we are + setting representations + exp_files (list): list of expected files + anatomy (Anatomy): + aov_filter (dict): add review for specific aov names + skip_integration_repre_list (list): exclude specific extensions + + Returns: + list of representations + + """ + representations = [] + host_name = os.environ.get("AVALON_APP", "") + collections, remainders = clique.assemble(exp_files) + + log = Logger.get_logger("farm_publishing") + + # create representation for every collected sequence + for collection in collections: + ext = collection.tail.lstrip(".") + preview = False + # TODO 'useSequenceForReview' is temporary solution which does + # not work for 100% of cases. We must be able to tell what + # expected files contains more explicitly and from what + # should be review made. + # - "review" tag is never added when is set to 'False' + if instance["useSequenceForReview"]: + # toggle preview on if multipart is on + if instance.get("multipartExr", False): + log.debug( + "Adding preview tag because its multipartExr" + ) + preview = True + else: + render_file_name = list(collection)[0] + # if filtered aov name is found in filename, toggle it for + # preview video rendering + preview = match_aov_pattern( + host_name, aov_filter, render_file_name + ) + + staging = os.path.dirname(list(collection)[0]) + success, rootless_staging_dir = ( + anatomy.find_root_template_from_path(staging) + ) + if success: + staging = rootless_staging_dir + else: + log.warning(( + "Could not find root path for remapping \"{}\"." + " This may cause issues on farm." + ).format(staging)) + + frame_start = int(instance.get("frameStartHandle")) + if instance.get("slate"): + frame_start -= 1 + + rep = { + "name": ext, + "ext": ext, + "files": [os.path.basename(f) for f in list(collection)], + "frameStart": frame_start, + "frameEnd": int(instance.get("frameEndHandle")), + # If expectedFile are absolute, we need only filenames + "stagingDir": staging, + "fps": instance.get("fps"), + "tags": ["review"] if preview else [], + } + + # poor man exclusion + if ext in skip_integration_repre_list: + rep["tags"].append("delete") + + if instance.get("multipartExr", False): + rep["tags"].append("multipartExr") + + # support conversion from tiled to scanline + if instance.get("convertToScanline"): + log.info("Adding scanline conversion.") + rep["tags"].append("toScanline") + + representations.append(rep) + + if preview: + instance["families"] = _solve_families(instance["families"]) + + # add remainders as representations + for remainder in remainders: + ext = remainder.split(".")[-1] + + staging = os.path.dirname(remainder) + success, rootless_staging_dir = ( + anatomy.find_root_template_from_path(staging) + ) + if success: + staging = rootless_staging_dir + else: + log.warning(( + "Could not find root path for remapping \"{}\"." + " This may cause issues on farm." + ).format(staging)) + + rep = { + "name": ext, + "ext": ext, + "files": os.path.basename(remainder), + "stagingDir": staging, + } + + preview = match_aov_pattern( + host_name, aov_filter, remainder + ) + if preview: + rep.update({ + "fps": instance.get("fps"), + "tags": ["review"] + }) + instance["families"] = _solve_families(instance["families"]) + + already_there = False + for repre in instance.get("representations", []): + # might be added explicitly before by publish_on_farm + already_there = repre.get("files") == rep["files"] + if already_there: + log.debug("repre {} already_there".format(repre)) + break + + if not already_there: + representations.append(rep) + + return representations + + +def create_instances_for_aov(instance, skeleton, aov_filter, + skip_integration_repre_list): """Create instances from AOVs. This will create new pyblish.api.Instances by going over expected @@ -336,11 +479,13 @@ def create_instances_for_aov(instance, skeleton, aov_filter): instance, skeleton, aov_filter, - additional_color_data + additional_color_data, + skip_integration_repre_list ) -def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data): +def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data, + skip_integration_repre_list): """Create instance for each AOV found. This will create new instance for every AOV it can detect in expected @@ -427,7 +572,7 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data): render_file_name = os.path.basename(col[0]) else: render_file_name = os.path.basename(col) - aov_patterns = self.aov_filter + aov_patterns = aov_filter preview = match_aov_pattern(app, aov_patterns, render_file_name) # toggle preview on if multipart is on @@ -436,7 +581,6 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data): log.debug("Adding preview tag because its multipartExr") preview = True - new_instance = deepcopy(skeleton) new_instance["subsetGroup"] = group_name if preview: @@ -483,7 +627,7 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data): rep["tags"].append("toScanline") # poor man exclusion - if ext in self.skip_integration_repre_list: + if ext in skip_integration_repre_list: rep["tags"].append("delete") if preview: