From e2b3f7496c40b30322f0467c9505dab6c8e4c394 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Wed, 29 Jul 2020 14:47:17 +0200 Subject: [PATCH] submiter validation, respect render priority --- .../global/publish/submit_publish_job.py | 59 ++++++++++++---- pype/plugins/maya/publish/collect_render.py | 3 +- .../maya/publish/submit_maya_deadline.py | 4 ++ .../validate_deadline_tile_submission.py | 69 +++++++++++++++++++ .../maya/publish/validate_frame_range.py | 6 ++ 5 files changed, 125 insertions(+), 16 deletions(-) create mode 100644 pype/plugins/maya/publish/validate_deadline_tile_submission.py diff --git a/pype/plugins/global/publish/submit_publish_job.py b/pype/plugins/global/publish/submit_publish_job.py index 21dcf93cdb..43edc33cba 100644 --- a/pype/plugins/global/publish/submit_publish_job.py +++ b/pype/plugins/global/publish/submit_publish_job.py @@ -232,7 +232,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): return (metadata_path, roothless_mtdt_p) - def _submit_deadline_post_job(self, instance, job): + def _submit_deadline_post_job(self, instance, job, instances): """Submit publish job to Deadline. Deadline specific code separated from :meth:`process` for sake of @@ -281,24 +281,28 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): and we are just waiting for them. """ if instance.data.get("tileRendering"): + self.log.info("Adding tile assembly results as dependencies...") asset_index = 0 - for represenation in instance.data.get("representations", []): - if isinstance(represenation["files"], [list, tuple]): - for file in represenation["files"]: + for inst in instances: + for represenation in inst.get("representations", []): + self.log.debug( + "working on {}".format(represenation["name"])) + if isinstance(represenation["files"], (list, tuple)): + for file in represenation["files"]: + self.log.debug("adding {}".format(file)) + dependency = os.path.join(output_dir, file) + payload["JobInfo"]["AssetDependency{}".format(asset_index)] = dependency # noqa: E501 + else: dependency = os.path.join(output_dir, file) payload["JobInfo"]["AssetDependency{}".format(asset_index)] = dependency # noqa: E501 - asset_index += 1 - else: - dependency = os.path.join(output_dir, file) - payload["JobInfo"]["AssetDependency0"] = dependency + asset_index += 1 else: - payload["JobInfo"]["JobDependency0"] = job["_id"], + payload["JobInfo"]["JobDependency0"] = job["_id"] # Transfer the environment from the original job to this dependent # job so they use the same environment metadata_path, roothless_metadata_path = self._create_metadata_path( instance) - environment = job["Props"].get("Env", {}) environment["PYPE_METADATA_FILE"] = roothless_metadata_path environment["AVALON_PROJECT"] = io.Session["AVALON_PROJECT"] @@ -847,14 +851,15 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): r''' SUBMiT PUBLiSH JOB 2 D34DLiN3 ____ - ' ' .---. .---. .--. .---. .--..--..--..--. .---. - | | --= \ | . \/ _|/ \| . \ || || \ |/ _| - | JOB | --= / | | || __| .. | | | |;_ || \ || __| - | | |____./ \.__|._||_.|___./|_____|||__|\__|\.___| + ' ' .---. .---. .--. .---. .--..--..--..--. .---. + | | --= \ | . \/ _|/ \| . \ || || \ |/ _| + | JOB | --= / | | || __| .. | | | |;_ || \ || __| + | | |____./ \.__|._||_.|___./|_____|||__|\__|\.___| ._____. ''' + render_job = None if instance.data.get("toBeRenderedOn") == "deadline": render_job = data.pop("deadlineSubmissionJob", None) submission_type = "deadline" @@ -867,13 +872,37 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): raise AssertionError(("Cannot continue without valid Deadline " "or Muster submission.")) + if not render_job: + import getpass + + render_job = {} + self.log.info("Faking job data ...") + render_job["Props"] = {} + # Render job doesn't exist because we do not have prior submission. + # We still use data from it so lets fake it. + # + # Batch name reflect original scene name + render_job["Props"]["Batch"] = os.path.splitext(os.path.basename( + context.data.get("currentFile")))[0] + # User is deadline user + render_job["Props"]["User"] = context.data.get( + "deadlineUser", getpass.getuser()) + # Priority is now not handled at all + render_job["Props"]["Pri"] = instance.data.get("priority") + + render_job["Props"]["Env"] = { + "FTRACK_API_USER": os.environ.get("FTRACK_API_USER"), + "FTRACK_API_KEY": os.environ.get("FTRACK_API_KEY"), + "FTRACK_SERVER": os.environ.get("FTRACK_SERVER"), + } + if submission_type == "deadline": self.DEADLINE_REST_URL = os.environ.get( "DEADLINE_REST_URL", "http://localhost:8082" ) assert self.DEADLINE_REST_URL, "Requires DEADLINE_REST_URL" - self._submit_deadline_post_job(instance, render_job) + self._submit_deadline_post_job(instance, render_job, instances) # publish job file publish_job = { diff --git a/pype/plugins/maya/publish/collect_render.py b/pype/plugins/maya/publish/collect_render.py index 75567ae216..5ca9392080 100644 --- a/pype/plugins/maya/publish/collect_render.py +++ b/pype/plugins/maya/publish/collect_render.py @@ -242,7 +242,8 @@ class CollectMayaRender(pyblish.api.ContextPlugin): "resolutionWidth": cmds.getAttr("defaultResolution.width"), "resolutionHeight": cmds.getAttr("defaultResolution.height"), "pixelAspect": cmds.getAttr("defaultResolution.pixelAspect"), - "tileRendering": render_instance.data.get("tileRendering") or False # noqa: E501 + "tileRendering": render_instance.data.get("tileRendering") or False, # noqa: E501 + "priority": render_instance.data.get("priority") } # Apply each user defined attribute as data diff --git a/pype/plugins/maya/publish/submit_maya_deadline.py b/pype/plugins/maya/publish/submit_maya_deadline.py index 5c13f6e62d..5840a7b946 100644 --- a/pype/plugins/maya/publish/submit_maya_deadline.py +++ b/pype/plugins/maya/publish/submit_maya_deadline.py @@ -45,6 +45,7 @@ payload_skeleton = { "Plugin": "MayaPype", "Frames": "{start}-{end}x{step}", "Comment": None, + "Priority": 50, }, "PluginInfo": { "SceneFile": None, # Input @@ -302,6 +303,9 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): payload_skeleton["JobInfo"]["Name"] = jobname # Arbitrary username, for visualisation in Monitor payload_skeleton["JobInfo"]["UserName"] = deadline_user + # Set job priority + payload_skeleton["JobInfo"]["Priority"] = self._instance.data.get( + "priority", 50) # Optional, enable double-click to preview rendered # frames from Deadline Monitor payload_skeleton["JobInfo"]["OutputDirectory0"] = \ diff --git a/pype/plugins/maya/publish/validate_deadline_tile_submission.py b/pype/plugins/maya/publish/validate_deadline_tile_submission.py new file mode 100644 index 0000000000..b0b995de3e --- /dev/null +++ b/pype/plugins/maya/publish/validate_deadline_tile_submission.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +"""Validate settings from Deadline Submitter. + +This is useful mainly for tile rendering, where jobs on farm are created by +submitter script from Maya. + +Unfortunately Deadline doesn't expose frame number for tiles job so that +cannot be validated, even if it is important setting. Also we cannot +determine if 'Region Rendering' (tile rendering) is enabled or not because +of the same thing. + +""" +import os + +from maya import mel +from maya import cmds + +import pyblish.api +from pype.hosts.maya import lib + + +class ValidateDeadlineTileSubmission(pyblish.api.InstancePlugin): + """Validate Deadline Submission settings are OK for tile rendering.""" + + label = "Validate Deadline Tile Submission" + order = pyblish.api.ValidatorOrder + hosts = ["maya"] + families = ["renderlayer"] + if not os.environ.get("DEADLINE_REST_URL"): + active = False + + def process(self, instance): + """Entry point.""" + # try if Deadline submitter was loaded + if mel.eval("exists SubmitJobToDeadline") == 0: + # if not, try to load it manually + try: + mel.eval("source DeadlineMayaClient;") + except RuntimeError: + raise AssertionError("Deadline Maya client cannot be loaded") + mel.eval("DeadlineMayaClient();") + assert mel.eval("exists SubmitJobToDeadline") == 1, ( + "Deadline Submission script cannot be initialized.") + if instance.data.get("tileRendering"): + job_name = cmds.getAttr("defaultRenderGlobals.deadlineJobName") + scene_name = os.path.splitext(os.path.basename( + instance.context.data.get("currentFile")))[0] + if job_name != scene_name: + self.log.warning(("Job submitted through Deadline submitter " + "has different name then current scene " + "{} / {}").format(job_name, scene_name)) + if cmds.getAttr("defaultRenderGlobals.deadlineTileSingleJob") == 1: + layer = instance.data['setMembers'] + anim_override = lib.get_attr_in_layer( + "defaultRenderGlobals.animation", layer=layer) + assert anim_override, ( + "Animation must be enabled in " + "Render Settings even when rendering single frame." + ) + + start_frame = cmds.getAttr("defaultRenderGlobals.startFrame") + end_frame = cmds.getAttr("defaultRenderGlobals.endFrame") + assert start_frame == end_frame, ( + "Start frame and end frame are not equals. When " + "'Submit All Tles As A Single Job' is selected, only " + "single frame is expected to be rendered. It must match " + "the one specified in Deadline Submitter under " + "'Region Rendering'" + ) diff --git a/pype/plugins/maya/publish/validate_frame_range.py b/pype/plugins/maya/publish/validate_frame_range.py index 0d51a83cf5..1ee6e2bd25 100644 --- a/pype/plugins/maya/publish/validate_frame_range.py +++ b/pype/plugins/maya/publish/validate_frame_range.py @@ -29,6 +29,12 @@ class ValidateFrameRange(pyblish.api.InstancePlugin): def process(self, instance): context = instance.context + if instance.data.get("tileRendering"): + self.log.info(( + "Skipping frame range validation because " + "tile rendering is enabled." + )) + return frame_start_handle = int(context.data.get("frameStartHandle")) frame_end_handle = int(context.data.get("frameEndHandle"))