From 0c3513283993dcd0af468dceebf14e8e26c97f10 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 16 Jan 2020 15:56:02 +0100 Subject: [PATCH] feat(nuke): deadline submission with slate and dependend noninteractive job --- .../nuke/publish/submit_nuke_deadline.py | 131 +++++++++++------- 1 file changed, 78 insertions(+), 53 deletions(-) diff --git a/pype/plugins/nuke/publish/submit_nuke_deadline.py b/pype/plugins/nuke/publish/submit_nuke_deadline.py index 580a114562..71108189c0 100644 --- a/pype/plugins/nuke/publish/submit_nuke_deadline.py +++ b/pype/plugins/nuke/publish/submit_nuke_deadline.py @@ -26,37 +26,69 @@ class NukeSubmitDeadline(pyblish.api.InstancePlugin): def process(self, instance): node = instance[0] - # for x in instance: - # if x.Class() == "Write": - # node = x - # - # if node is None: - # return + context = instance.context DEADLINE_REST_URL = os.environ.get("DEADLINE_REST_URL", "http://localhost:8082") assert DEADLINE_REST_URL, "Requires DEADLINE_REST_URL" - context = instance.context + self.deadline_url = "{}/api/jobs".format(DEADLINE_REST_URL) + self._comment = context.data.get("comment", "") + self._ver = re.search(r"\d+\.\d+", context.data.get("hostVersion")) + self._deadline_user = context.data.get( + "deadlineUser", getpass.getuser()) + self._frame_start = int(instance.data["frameStart"]) + self._frame_end = int(instance.data["frameEnd"]) # get output path render_path = instance.data['path'] - render_dir = os.path.normpath(os.path.dirname(render_path)) - - # frame start definition - frame_start = int(instance.data["frameStart"]) - # exception for slate workflow - if "slate" in instance.data["families"]: - frame_start -= 1 - script_path = context.data["currentFile"] - script_name = os.path.basename(script_path) - comment = context.data.get("comment", "") + # exception for slate workflow + if "slate" in instance.data["families"]: + self._frame_start -= 1 - deadline_user = context.data.get("deadlineUser", getpass.getuser()) + response = self.payload_submit(instance, + script_path, + render_path, + node.name() + ) + # Store output dir for unified publisher (filesequence) + instance.data["deadlineSubmissionJob"] = response.json() + instance.data["publishJobState"] = "Active" + + if instance.data.get("bakeScriptPath"): + render_path = instance.data.get("bakeRenderPath") + script_path = instance.data.get("bakeScriptPath") + exe_node_name = instance.data.get("bakeWriteNodeName") + + # exception for slate workflow + if "slate" in instance.data["families"]: + self._frame_start += 1 + + resp = self.payload_submit(instance, + script_path, + render_path, + exe_node_name, + response.json() + ) + # Store output dir for unified publisher (filesequence) + instance.data["deadlineSubmissionJob"] = resp.json() + instance.data["publishJobState"] = "Suspended" + + def payload_submit(self, + instance, + script_path, + render_path, + exe_node_name, + responce_data=None + ): + render_dir = os.path.normpath(os.path.dirname(render_path)) + script_name = os.path.basename(script_path) jobname = "%s - %s" % (script_name, instance.name) - ver = re.search(r"\d+\.\d+", context.data.get("hostVersion")) + + if not responce_data: + responce_data = {} try: # Ensure render folder exists @@ -64,10 +96,6 @@ class NukeSubmitDeadline(pyblish.api.InstancePlugin): except OSError: pass - # Documentation for keys available at: - # https://docs.thinkboxsoftware.com - # /products/deadline/8.0/1_User%20Manual/manual - # /manual-submission.html#job-info-file-options payload = { "JobInfo": { # Top-level group name @@ -77,21 +105,20 @@ class NukeSubmitDeadline(pyblish.api.InstancePlugin): "Name": jobname, # Arbitrary username, for visualisation in Monitor - "UserName": deadline_user, + "UserName": self._deadline_user, + + "Priority": instance.data["deadlinePriority"], + + "Pool": "2d", + "SecondaryPool": "2d", "Plugin": "Nuke", "Frames": "{start}-{end}".format( - start=frame_start, - end=int(instance.data["frameEnd"]) + start=self._frame_start, + end=self._frame_end ), - "ChunkSize": instance.data["deadlineChunkSize"], - "Priority": instance.data["deadlinePriority"], + "Comment": self._comment, - "Comment": comment, - - # Optional, enable double-click to preview rendered - # frames from Deadline Monitor - # "OutputFilename0": output_filename_0.replace("\\", "/"), }, "PluginInfo": { # Input @@ -102,27 +129,29 @@ class NukeSubmitDeadline(pyblish.api.InstancePlugin): # "OutputFilePrefix": render_variables["filename_prefix"], # Mandatory for Deadline - "Version": ver.group(), + "Version": self._ver.group(), # Resolve relative references "ProjectPath": script_path, "AWSAssetFile0": render_path, # Only the specific write node is rendered. - "WriteNode": node.name() + "WriteNode": exe_node_name }, # Mandatory for Deadline, may be empty "AuxFiles": [] } + if responce_data.get("_id"): + payload["JobInfo"].update({ + "JobType": "Normal", + "BatchName": responce_data["Props"]["Batch"], + "JobDependency0": responce_data["_id"], + "ChunkSize": 99999999 + }) + # Include critical environment variables with submission keys = [ - # This will trigger `userSetup.py` on the slave - # such that proper initialisation happens the same - # way as it does on a local machine. - # TODO(marcus): This won't work if the slaves don't - # have accesss to these paths, such as if slaves are - # running Linux and the submitter is on Windows. "PYTHONPATH", "PATH", "AVALON_SCHEMA", @@ -168,11 +197,12 @@ class NukeSubmitDeadline(pyblish.api.InstancePlugin): if key == "PYTHONPATH": clean_path = clean_path.replace('python2', 'python3') + clean_path = clean_path.replace( - os.path.normpath( - environment['PYPE_STUDIO_CORE_MOUNT']), # noqa - os.path.normpath( - environment['PYPE_STUDIO_CORE_PATH'])) # noqa + os.path.normpath( + environment['PYPE_STUDIO_CORE_MOUNT']), # noqa + os.path.normpath( + environment['PYPE_STUDIO_CORE_PATH'])) # noqa clean_environment[key] = clean_path environment = clean_environment @@ -187,20 +217,15 @@ class NukeSubmitDeadline(pyblish.api.InstancePlugin): plugin = payload["JobInfo"]["Plugin"] self.log.info("using render plugin : {}".format(plugin)) - self.preflight_check(instance) - self.log.info("Submitting..") self.log.info(json.dumps(payload, indent=4, sort_keys=True)) - # E.g. http://192.168.0.1:8082/api/jobs - url = "{}/api/jobs".format(DEADLINE_REST_URL) - response = requests.post(url, json=payload) + response = requests.post(self.deadline_url, json=payload) + if not response.ok: raise Exception(response.text) - # Store output dir for unified publisher (filesequence) - instance.data["deadlineSubmissionJob"] = response.json() - instance.data["publishJobState"] = "Active" + return response def preflight_check(self, instance): """Ensure the startFrame, endFrame and byFrameStep are integers"""