From 73cf209570fe0729afdbe90800c7ba6945a2a977 Mon Sep 17 00:00:00 2001 From: antirotor Date: Wed, 19 Jun 2019 22:53:01 +0200 Subject: [PATCH] feat(nuke): Deadline submitter. Minor fixes in connected collectors. --- pype/plugins/global/publish/integrate_new.py | 1 + .../global/publish/submit_publish_job.py | 16 +- pype/plugins/nuke/publish/collect_families.py | 8 +- pype/plugins/nuke/publish/collect_review.py | 14 +- .../extract_post_json.py | 28 +-- pype/plugins/nuke/publish/extract_review.py | 2 +- .../nuke/publish/submit_nuke_deadline.py | 206 ++++++++++++++++++ 7 files changed, 240 insertions(+), 35 deletions(-) rename pype/plugins/nuke/{_publish_unused => publish}/extract_post_json.py (75%) create mode 100644 pype/plugins/nuke/publish/submit_nuke_deadline.py diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 359bb9afe7..587add709e 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -60,6 +60,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): "nukescript", "render", "rendersetup", + "render.farm", "write", "rig", "plate", diff --git a/pype/plugins/global/publish/submit_publish_job.py b/pype/plugins/global/publish/submit_publish_job.py index 1b795157d7..f3ce4f7ec5 100644 --- a/pype/plugins/global/publish/submit_publish_job.py +++ b/pype/plugins/global/publish/submit_publish_job.py @@ -1,6 +1,8 @@ import os import json import re +from pprint import pprint +import logging from avalon import api, io from avalon.vendor import requests, clique @@ -215,7 +217,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): if not job: # No deadline job. Try Muster: musterSubmissionJob - job = instance.data.get("musterSubmissionJob") + job = data.pop("musterSubmissionJob") submission_type = "muster" if not job: raise RuntimeError("Can't continue without valid Deadline " @@ -362,7 +364,19 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): metadata["metadata"]["instance"]["endFrame"] = updated_end metadata_filename = "{}_metadata.json".format(subset) + metadata_path = os.path.join(output_dir, metadata_filename) + # convert log messages if they are `LogRecord` to their + # string format to allow serializing as JSON later on. + rendered_logs = [] + for log in metadata["metadata"]["instance"].get("_log", []): + if isinstance(log, logging.LogRecord): + rendered_logs.append(log.getMessage()) + else: + rendered_logs.append(log) + + metadata["metadata"]["instance"]["_log"] = rendered_logs + pprint(metadata) with open(metadata_path, "w") as f: json.dump(metadata, f, indent=4, sort_keys=True) diff --git a/pype/plugins/nuke/publish/collect_families.py b/pype/plugins/nuke/publish/collect_families.py index d7515f91ca..77388a9bd5 100644 --- a/pype/plugins/nuke/publish/collect_families.py +++ b/pype/plugins/nuke/publish/collect_families.py @@ -1,5 +1,5 @@ import pyblish.api - +import nuke @pyblish.api.log class CollectInstanceFamilies(pyblish.api.InstancePlugin): @@ -12,9 +12,10 @@ class CollectInstanceFamilies(pyblish.api.InstancePlugin): def process(self, instance): + # node = nuke.toNode(instance.data["name"]) node = instance[0] - self.log.info('processing {}'.format(node)) + self.log.info('processing {}'.format(node["name"].value())) families = [] if instance.data.get('families'): @@ -24,10 +25,13 @@ class CollectInstanceFamilies(pyblish.api.InstancePlugin): # instance.data["families"] = ["ftrack"] if node["render"].value(): + self.log.info("flagged for render") # dealing with local/farm rendering if node["render_farm"].value(): + self.log.info("adding render farm family") families.append("render.farm") else: + self.log.info("adding render to local") families.append("render.local") else: families.append("render.frames") diff --git a/pype/plugins/nuke/publish/collect_review.py b/pype/plugins/nuke/publish/collect_review.py index f75c675b8f..63d60b80fd 100644 --- a/pype/plugins/nuke/publish/collect_review.py +++ b/pype/plugins/nuke/publish/collect_review.py @@ -14,16 +14,12 @@ class CollectReview(pyblish.api.InstancePlugin): family_targets = [".local", ".frames"] def process(self, instance): - pass - families = [(f, search) for f in instance.data["families"] - for search in self.family_targets - if search in f][0] + families = [] + if "render.farm" not in instance.data["families"]: + families = [(f, search) for f in instance.data["families"] + for search in self.family_targets + if search in f][0] if families: - root_families = families[0].replace(families[1], "") - # instance.data["families"].append(".".join([ - # root_families, - # self.family - # ])) instance.data["families"].append("render.review") self.log.info("Review collected: `{}`".format(instance)) diff --git a/pype/plugins/nuke/_publish_unused/extract_post_json.py b/pype/plugins/nuke/publish/extract_post_json.py similarity index 75% rename from pype/plugins/nuke/_publish_unused/extract_post_json.py rename to pype/plugins/nuke/publish/extract_post_json.py index 6954abff3d..fe42781d52 100644 --- a/pype/plugins/nuke/_publish_unused/extract_post_json.py +++ b/pype/plugins/nuke/publish/extract_post_json.py @@ -4,6 +4,7 @@ import datetime import time import clique +from pprint import pformat import pyblish.api @@ -23,35 +24,18 @@ class ExtractJSON(pyblish.api.ContextPlugin): os.makedirs(workspace) context_data = context.data.copy() - out_data = dict(self.serialize(context_data)) + unwrapped_instance = [] + for i in context_data["instances"]: + unwrapped_instance.append(i.data) - instances_data = [] - for instance in context: - - data = {} - for key, value in instance.data.items(): - if isinstance(value, clique.Collection): - value = value.format() - - try: - json.dumps(value) - data[key] = value - except KeyError: - msg = "\"{0}\"".format(value) - msg += " in instance.data[\"{0}\"]".format(key) - msg += " could not be serialized." - self.log.debug(msg) - - instances_data.append(data) - - out_data["instances"] = instances_data + context_data["instances"] = unwrapped_instance timestamp = datetime.datetime.fromtimestamp( time.time()).strftime("%Y%m%d-%H%M%S") filename = timestamp + "_instances.json" with open(os.path.join(workspace, filename), "w") as outfile: - outfile.write(json.dumps(out_data, indent=4, sort_keys=True)) + outfile.write(pformat(context_data, depth=20)) def serialize(self, data): """ diff --git a/pype/plugins/nuke/publish/extract_review.py b/pype/plugins/nuke/publish/extract_review.py index 2eabbd4e87..368b9633aa 100644 --- a/pype/plugins/nuke/publish/extract_review.py +++ b/pype/plugins/nuke/publish/extract_review.py @@ -16,7 +16,7 @@ class ExtractDataForReview(pype.api.Extractor): label = "Extract Review" optional = True - families = ["render.review"] + families = ["render.review", "render.local"] hosts = ["nuke"] def process(self, instance): diff --git a/pype/plugins/nuke/publish/submit_nuke_deadline.py b/pype/plugins/nuke/publish/submit_nuke_deadline.py new file mode 100644 index 0000000000..37a5b632cb --- /dev/null +++ b/pype/plugins/nuke/publish/submit_nuke_deadline.py @@ -0,0 +1,206 @@ +import os +import json +import getpass + +import nuke + +from avalon import api +from avalon.vendor import requests + +import pyblish.api + + +class NukeSubmitDeadline(pyblish.api.InstancePlugin): + """Submit write to Deadline + + Renders are submitted to a Deadline Web Service as + supplied via the environment variable DEADLINE_REST_URL + + """ + + label = "Submit to Deadline" + order = pyblish.api.IntegratorOrder + 0.1 + hosts = ["nuke", "nukestudio"] + families = ["render.farm"] + optional = True + + def process(self, instance): + + # root = nuke.root() + # node_subset_name = instance.data.get("name", None) + node = instance[0] + + DEADLINE_REST_URL = os.environ.get("DEADLINE_REST_URL", + "http://localhost:8082") + assert DEADLINE_REST_URL, "Requires DEADLINE_REST_URL" + + context = instance.context + workspace = os.path.dirname(context.data["currentFile"]) + filepath = None + + # get path + path = nuke.filename(node) + output_dir = instance.data['outputDir'] + + filepath = context.data["currentFile"] + + self.log.debug(filepath) + + filename = os.path.basename(filepath) + comment = context.data.get("comment", "") + dirname = os.path.join(workspace, "renders") + deadline_user = context.data.get("deadlineUser", getpass.getuser()) + jobname = "%s - %s" % (filename, instance.name) + + try: + # Ensure render folder exists + os.makedirs(dirname) + except OSError: + pass + + # Documentation for keys available at: + # https://docs.thinkboxsoftware.com + # /products/deadline/8.0/1_User%20Manual/manual + # /manual-submission.html#job-info-file-options + payload = { + "JobInfo": { + # Top-level group name + "BatchName": filename, + + # Job name, as seen in Monitor + "Name": jobname, + + # Arbitrary username, for visualisation in Monitor + "UserName": deadline_user, + + "Plugin": "Nuke", + "Frames": "{start}-{end}".format( + start=int(instance.data["startFrame"]), + end=int(instance.data["endFrame"]) + ), + + "Comment": comment, + + # Optional, enable double-click to preview rendered + # frames from Deadline Monitor + # "OutputFilename0": output_filename_0.replace("\\", "/"), + }, + "PluginInfo": { + # Input + "SceneFile": filepath, + + # Output directory and filename + "OutputFilePath": dirname.replace("\\", "/"), + # "OutputFilePrefix": render_variables["filename_prefix"], + + # Mandatory for Deadline + "Version": context.data.get("hostVersion"), + + # Resolve relative references + "ProjectPath": workspace, + }, + + # Mandatory for Deadline, may be empty + "AuxFiles": [] + } + + # Include critical environment variables with submission + keys = [ + # This will trigger `userSetup.py` on the slave + # such that proper initialisation happens the same + # way as it does on a local machine. + # TODO(marcus): This won't work if the slaves don't + # have accesss to these paths, such as if slaves are + # running Linux and the submitter is on Windows. + "PYTHONPATH", + "PATH", + "AVALON_SCHEMA", + "FTRACK_API_KEY", + "FTRACK_API_USER", + "FTRACK_SERVER", + "PYBLISHPLUGINPATH", + "NUKE_PATH", + "TOOL_ENV" + ] + environment = dict({key: os.environ[key] for key in keys + if key in os.environ}, **api.Session) + # self.log.debug("enviro: {}".format(pprint(environment))) + for path in os.environ: + if path.lower().startswith('pype_'): + environment[path] = os.environ[path] + + environment["PATH"] = os.environ["PATH"] + # self.log.debug("enviro: {}".format(environment['PYPE_SCRIPTS'])) + clean_environment = {} + for key in environment: + clean_path = "" + self.log.debug("key: {}".format(key)) + to_process = environment[key] + if key == "PYPE_STUDIO_CORE_MOUNT": + clean_path = environment[key] + elif "://" in environment[key]: + clean_path = environment[key] + elif os.pathsep not in to_process: + try: + path = environment[key] + path.decode('UTF-8', 'strict') + clean_path = os.path.normpath(path) + except UnicodeDecodeError: + print('path contains non UTF characters') + else: + for path in environment[key].split(os.pathsep): + try: + path.decode('UTF-8', 'strict') + clean_path += os.path.normpath(path) + os.pathsep + except UnicodeDecodeError: + print('path contains non UTF characters') + + if key == "PYTHONPATH": + clean_path = clean_path.replace('python2', 'python3') + clean_path = clean_path.replace( + os.path.normpath( + environment['PYPE_STUDIO_CORE_MOUNT']), # noqa + os.path.normpath( + environment['PYPE_STUDIO_CORE_PATH'])) # noqa + clean_environment[key] = clean_path + + environment = clean_environment + + payload["JobInfo"].update({ + "EnvironmentKeyValue%d" % index: "{key}={value}".format( + key=key, + value=environment[key] + ) for index, key in enumerate(environment) + }) + + plugin = payload["JobInfo"]["Plugin"] + self.log.info("using render plugin : {}".format(plugin)) + + self.preflight_check(instance) + + self.log.info("Submitting..") + self.log.info(json.dumps(payload, indent=4, sort_keys=True)) + + # E.g. http://192.168.0.1:8082/api/jobs + url = "{}/api/jobs".format(DEADLINE_REST_URL) + response = requests.post(url, json=payload) + if not response.ok: + raise Exception(response.text) + + # Store output dir for unified publisher (filesequence) + instance.data["outputDir"] = os.path.dirname(output_dir) + instance.data["deadlineSubmissionJob"] = response.json() + + def preflight_check(self, instance): + """Ensure the startFrame, endFrame and byFrameStep are integers""" + + for key in ("startFrame", "endFrame"): + value = instance.data[key] + + if int(value) == value: + continue + + self.log.warning( + "%f=%d was rounded off to nearest integer" + % (value, int(value)) + )