From ef542916e348812df7267d0fe629cfa86e853c7e Mon Sep 17 00:00:00 2001 From: aardschok Date: Thu, 10 Aug 2017 16:47:52 +0200 Subject: [PATCH] added deadline plugins for configuration --- .../plugins/maya/publish/submit_deadline.py | 206 ++++++++++++++++++ .../plugins/publish/validate_deadline_done.py | 57 +++++ 2 files changed, 263 insertions(+) create mode 100644 colorbleed/plugins/maya/publish/submit_deadline.py create mode 100644 colorbleed/plugins/publish/validate_deadline_done.py diff --git a/colorbleed/plugins/maya/publish/submit_deadline.py b/colorbleed/plugins/maya/publish/submit_deadline.py new file mode 100644 index 0000000000..108bd562b3 --- /dev/null +++ b/colorbleed/plugins/maya/publish/submit_deadline.py @@ -0,0 +1,206 @@ +import pyblish.api + + +class MindbenderSubmitDeadline(pyblish.api.InstancePlugin): + """Submit available render layers to Deadline + + Renders are submitted to a Deadline Web Service as + supplied via the environment variable AVALON_DEADLINE + + """ + + label = "Submit to Deadline" + order = pyblish.api.IntegratorOrder + hosts = ["maya"] + families = ["mindbender.renderlayer"] + + def process(self, instance): + import os + import json + import shutil + import getpass + + from maya import cmds + + from avalon import api + from avalon.vendor import requests + + assert api.Session["AVALON_DEADLINE"], "Requires AVALON_DEADLINE" + + context = instance.context + workspace = context.data["workspaceDir"] + fpath = context.data["currentFile"] + fname = os.path.basename(fpath) + name, ext = os.path.splitext(fname) + comment = context.data.get("comment", "") + dirname = os.path.join(workspace, "renders", name) + + try: + os.makedirs(dirname) + except OSError: + pass + + # E.g. http://192.168.0.1:8082/api/jobs + url = api.Session["AVALON_DEADLINE"] + "/api/jobs" + + # Documentation for keys available at: + # https://docs.thinkboxsoftware.com + # /products/deadline/8.0/1_User%20Manual/manual + # /manual-submission.html#job-info-file-options + payload = { + "JobInfo": { + # Top-level group name + "BatchName": fname, + + # Job name, as seen in Monitor + "Name": "%s - %s" % (fname, instance.name), + + # Arbitrary username, for visualisation in Monitor + "UserName": getpass.getuser(), + + "Plugin": "MayaBatch", + "Frames": "{start}-{end}x{step}".format( + start=int(instance.data["startFrame"]), + end=int(instance.data["endFrame"]), + step=int(instance.data["byFrameStep"]), + ), + + "Comment": comment, + + # Optional, enable double-click to preview rendered + # frames from Deadline Monitor + "OutputFilename0": self.preview_fname(instance), + }, + "PluginInfo": { + # Input + "SceneFile": fpath, + + # Output directory and filename + "OutputFilePath": dirname, + "OutputFilePrefix": "/", + + # Mandatory for Deadline + "Version": cmds.about(version=True), + + # Only render layers are considered renderable in this pipeline + "UsingRenderLayers": True, + + # Render only this layer + "RenderLayer": instance.name, + + # Determine which renderer to use from the file itself + "Renderer": "file", + + # Resolve relative references + "ProjectPath": workspace, + }, + + # Mandatory for Deadline, may be empty + "AuxFiles": [] + } + + # Include critical variables with submission + environment = dict({ + # This will trigger `userSetup.py` on the slave + # such that proper initialisation happens the same + # way as it does on a local machine. + # TODO(marcus): This won't work if the slaves don't + # have accesss to these paths, such as if slaves are + # running Linux and the submitter is on Windows. + "PYTHONPATH": os.getenv("PYTHONPATH", ""), + + }, **api.Session) + + payload["JobInfo"].update({ + "EnvironmentKeyValue%d" % index: "{key}={value}".format( + key=key, + value=environment[key] + ) for index, key in enumerate(environment) + }) + + # Include optional render globals + payload["JobInfo"].update( + instance.data.get("renderGlobals", {}) + ) + + self.preflight_check(instance) + + self.log.info("Submitting..") + self.log.info(json.dumps( + payload, indent=4, sort_keys=True) + ) + + response = requests.post(url, json=payload) + + if response.ok: + # Write metadata for publish + fname = os.path.join(dirname, instance.name + ".json") + data = { + "submission": payload, + "session": api.Session, + "instance": instance.data, + "jobs": [ + response.json() + ], + } + + with open(fname, "w") as f: + json.dump(data, f, indent=4, sort_keys=True) + + else: + try: + shutil.rmtree(dirname) + except OSError: + # This is nice-to-have, but not critical to the operation + pass + + raise Exception(response.text) + + def preview_fname(self, instance): + """Return outputted filename with #### for padding + + Passing the absolute path to Deadline enables Deadline Monitor + to provide the user with a Job Output menu option. + + Deadline requires the path to be formatted with # in place of numbers. + + From + /path/to/render.0000.png + To + /path/to/render.####.png + + """ + + from maya import cmds + + # We'll need to take tokens into account + fname = cmds.renderSettings( + firstImageName=True, + fullPath=True, + layer=instance.name + )[0] + + try: + # Assume `c:/some/path/filename.0001.exr` + # TODO(marcus): Bulletproof this, the user may have + # chosen a different format for the outputted filename. + fname, padding, suffix = fname.rsplit(".", 2) + fname = ".".join([fname, "#" * len(padding), suffix]) + self.log.info("Assuming renders end up @ %s" % fname) + except ValueError: + fname = "" + self.log.info("Couldn't figure out where renders go") + + return fname + + def preflight_check(self, instance): + for key in ("startFrame", "endFrame", "byFrameStep"): + value = instance.data[key] + + if int(value) == value: + continue + + self.log.warning( + "%f=%d was rounded off to nearest integer" + % (value, int(value)) + ) diff --git a/colorbleed/plugins/publish/validate_deadline_done.py b/colorbleed/plugins/publish/validate_deadline_done.py new file mode 100644 index 0000000000..1ab539eefa --- /dev/null +++ b/colorbleed/plugins/publish/validate_deadline_done.py @@ -0,0 +1,57 @@ +import pyblish.api + + +class ValidateMindbenderDeadlineDone(pyblish.api.InstancePlugin): + """Ensure render is finished before publishing the resulting images""" + + label = "Rendered Successfully" + order = pyblish.api.ValidatorOrder + hosts = ["shell"] + families = ["mindbender.imagesequence"] + optional = True + + def process(self, instance): + from avalon import api + from avalon.vendor import requests + + # From Deadline documentation + # https://docs.thinkboxsoftware.com/products/deadline/8.0/ + # 1_User%20Manual/manual/rest-jobs.html#job-property-values + states = { + 0: "Unknown", + 1: "Active", + 2: "Suspended", + 3: "Completed", + 4: "Failed", + 6: "Pending", + } + + url = api.Session["AVALON_DEADLINE"] + "/api/jobs?JobID=%s" + + for job in instance.data["metadata"]["jobs"]: + response = requests.get(url % job["_id"]) + + if response.ok: + data = response.json()[0] + state = states.get(data["Stat"]) + + if state in (None, "Unknown"): + raise Exception("State of this render is unknown") + + elif state == "Active": + raise Exception("This render is still currently active") + + elif state == "Suspended": + raise Exception("This render is suspended") + + elif state == "Failed": + raise Exception("This render was not successful") + + elif state == "Pending": + raise Exception("This render is pending") + else: + self.log.info("%s was rendered successfully" % instance) + + else: + raise Exception("Could not determine the current status " + " of this render")