From 1da91e6751c436aacecdd7ddf887cb0dc9730b65 Mon Sep 17 00:00:00 2001 From: aardschok Date: Thu, 10 Aug 2017 16:46:40 +0200 Subject: [PATCH 1/2] resolved issue with checking sets content --- colorbleed/maya/lib.py | 78 ++++++++++++++++++ .../plugins/maya/publish/collect_look.py | 80 ++----------------- .../maya/publish/validate_look_contents.py | 4 +- .../validate_look_no_default_shaders.py | 55 +++++++------ .../maya/publish/validate_look_sets.py | 73 +++++++++++++++++ 5 files changed, 188 insertions(+), 102 deletions(-) create mode 100644 colorbleed/plugins/maya/publish/validate_look_sets.py diff --git a/colorbleed/maya/lib.py b/colorbleed/maya/lib.py index 8b7d04ef87..2db57b32a6 100644 --- a/colorbleed/maya/lib.py +++ b/colorbleed/maya/lib.py @@ -931,3 +931,81 @@ def apply_shaders(relationships, shadernodes, nodes): # endregion apply_attributes(attributes, ns_nodes_by_id) + + +def get_isolate_view_sets(): + """ + + + """ + + view_sets = set() + for panel in cmds.getPanel(type="modelPanel"): + view_set = cmds.modelEditor(panel, query=True, viewObjects=True) + if view_set: + view_sets.add(view_set) + + return view_sets + + +def get_related_sets(node): + """Return objectSets that are relationships for a look for `node`. + + Filters out based on: + - id attribute is NOT `pyblish.avalon.container` + - shapes and deformer shapes (alembic creates meshShapeDeformed) + - set name ends with any from a predefined list + - set in not in viewport set (isolate selected for example) + + Args: + node (str): name of the current not to check + + Returns: + list: The related sets + + """ + + # Ignore specific suffices + ignore_suffices = ["out_SET", "controls_SET", "_INST", "_CON"] + + # Default nodes to ignore + defaults = ["initialShadingGroup", + "defaultLightSet", + "defaultObjectSet"] + + # Ids to ignore + ignored = ["pyblish.avalon.instance", + "pyblish.avalon.container"] + + view_sets = get_isolate_view_sets() + + related_sets = cmds.listSets(object=node, extendToShape=False) + if not related_sets: + return [] + + # Ignore `avalon.container` + sets = [s for s in related_sets if + not cmds.attributeQuery("id", node=s, exists=True) or + not cmds.getAttr("%s.id" % s) in ignored] + + # Exclude deformer sets + # Autodesk documentation on listSets command: + # type(uint) : Returns all sets in the scene of the given + # >>> type: + # >>> 1 - all rendering sets + # >>> 2 - all deformer sets + deformer_sets = cmds.listSets(object=node, + extendToShape=False, + type=2) or [] + deformer_sets = set(deformer_sets) # optimize lookup + sets = [s for s in sets if s not in deformer_sets] + + # Ignore when the set has a specific suffix + sets = [s for s in sets if not any(s.endswith(x) for x in ignore_suffices)] + + # Ignore viewport filter view sets (from isolate select and + # viewports) + sets = [s for s in sets if s not in view_sets] + sets = [s for s in sets if s not in defaults] + + return sets diff --git a/colorbleed/plugins/maya/publish/collect_look.py b/colorbleed/plugins/maya/publish/collect_look.py index d9f1b8067f..731fd127c8 100644 --- a/colorbleed/plugins/maya/publish/collect_look.py +++ b/colorbleed/plugins/maya/publish/collect_look.py @@ -66,9 +66,6 @@ class CollectLook(pyblish.api.InstancePlugin): label = "Collect Look" hosts = ["maya"] - # Ignore specifically named sets (check with endswith) - IGNORE = ["out_SET", "controls_SET", "_INST", "_CON"] - def process(self, instance): """Collect the Look in the instance with the correct layer settings""" @@ -90,16 +87,14 @@ class CollectLook(pyblish.api.InstancePlugin): sets = self.gather_sets(instance) # Lookup with absolute names (from root namespace) - instance_lookup = set([str(x) for x in cmds.ls(instance, - long=True, - absoluteName=True)]) + instance_lookup = set([str(x) for x in cmds.ls(instance, long=True)]) self.log.info("Gathering set relations..") for objset in sets: self.log.debug("From %s.." % objset) content = cmds.sets(objset, query=True) objset_members = sets[objset]["members"] - for member in cmds.ls(content, long=True, absoluteName=True): + for member in cmds.ls(content, long=True): member_data = self.collect_member_data(member, objset_members, instance_lookup, @@ -114,7 +109,7 @@ class CollectLook(pyblish.api.InstancePlugin): self.log.info("Gathering attribute changes to instance members..") attributes = self.collect_attributes_changed(instance) - looksets = cmds.ls(sets.keys(), absoluteName=True, long=True) + looksets = cmds.ls(sets.keys(), long=True) self.log.info("Found the following sets: {}".format(looksets)) @@ -152,19 +147,16 @@ class CollectLook(pyblish.api.InstancePlugin): dict """ - # Get view sets (so we can ignore those sets later) sets = dict() - view_sets = set() - for panel in cmds.getPanel(type="modelPanel"): - view_set = cmds.modelEditor(panel, query=True, viewObjects=True) - if view_set: - view_sets.add(view_set) for node in instance: - related_sets = self.get_related_sets(node, view_sets) + + related_sets = lib.get_related_sets(node) if not related_sets: continue + self.log.info("Found sets %s for %s", related_sets, node) + for objset in related_sets: if objset in sets: continue @@ -174,59 +166,6 @@ class CollectLook(pyblish.api.InstancePlugin): return sets - def get_related_sets(self, node, view_sets): - """Get the sets which do not belong to any specific group - - Filters out based on: - - id attribute is NOT `pyblish.avalon.container` - - shapes and deformer shapes (alembic creates meshShapeDeformed) - - set name ends with any from a predefined list - - set in not in viewport set (isolate selected for example) - - Args: - node (str): name of the current not to check - """ - defaults = ["initialShadingGroup", - "defaultLightSet", - "defaultObjectSet"] - - ignored = ["pyblish.avalon.instance", - "pyblish.avalon.container"] - - related_sets = cmds.listSets(object=node, extendToShape=False) - if not related_sets: - return [] - - # Ignore `avalon.container` - sets = [s for s in related_sets if - not cmds.attributeQuery("id", node=s, exists=True) or - not cmds.getAttr("%s.id" % s) in ignored] - - # Exclude deformer sets - # Autodesk documentation on listSets command: - # type(uint) : Returns all sets in the scene of the given - # >>> type: - # >>> 1 - all rendering sets - # >>> 2 - all deformer sets - deformer_sets = cmds.listSets(object=node, - extendToShape=False, - type=2) or [] - - deformer_sets = set(deformer_sets) # optimize lookup - sets = [s for s in sets if s not in deformer_sets] - - # Ignore specifically named sets - sets = [s for s in sets if not any(s.endswith(x) for x in self.IGNORE)] - - # Ignore viewport filter view sets (from isolate select and - # viewports) - sets = [s for s in sets if s not in view_sets] - sets = [s for s in sets if s not in defaults] - - self.log.info("Found sets %s for %s" % (sets, node)) - - return sets - def remove_sets_without_members(self, sets): """Remove any set which does not have any members @@ -270,11 +209,6 @@ class CollectLook(pyblish.api.InstancePlugin): if member in [m["name"] for m in objset_members]: return - # check node type, if mesh get parent! - if cmds.nodeType(node) == "mesh": - # A mesh will always have a transform node in Maya logic - node = cmds.listRelatives(node, parent=True, fullPath=True)[0] - if not cmds.attributeQuery("cbId", node=node, exists=True): self.log.error("Node '{}' has no attribute 'cbId'".format(node)) return diff --git a/colorbleed/plugins/maya/publish/validate_look_contents.py b/colorbleed/plugins/maya/publish/validate_look_contents.py index 1bdd3d0aef..1d9d192521 100644 --- a/colorbleed/plugins/maya/publish/validate_look_contents.py +++ b/colorbleed/plugins/maya/publish/validate_look_contents.py @@ -1,5 +1,3 @@ -import maya.cmds as cmds - import pyblish.api import colorbleed.api import colorbleed.maya.lib as lib @@ -67,7 +65,7 @@ class ValidateLookContents(pyblish.api.InstancePlugin): # Validate at least one single relationship is collected if not lookdata["relationships"]: cls.log.error("Look '{}' has no " - "`relationship`".format(instance.name)) + "`relationships`".format(instance.name)) invalid.add(instance.name) return invalid diff --git a/colorbleed/plugins/maya/publish/validate_look_no_default_shaders.py b/colorbleed/plugins/maya/publish/validate_look_no_default_shaders.py index 5042d01796..5fc8ea660f 100644 --- a/colorbleed/plugins/maya/publish/validate_look_no_default_shaders.py +++ b/colorbleed/plugins/maya/publish/validate_look_no_default_shaders.py @@ -31,41 +31,44 @@ class ValidateLookNoDefaultShaders(pyblish.api.InstancePlugin): @classmethod def get_invalid(cls, instance): - invalid = [] - disallowed = ["lambert1", - "initialShadingGroup", - "initialParticleSE", - "particleCloud1"] + disallowed = set(["lambert1", + "initialShadingGroup", + "initialParticleSE", + "particleCloud1"]) - members = cmds.listRelatives(instance, - allDescendents=True, - shapes=True, - noIntermediate=True) or [] - for member in members: + invalid = set() + for node in instance: # get connection # listConnections returns a list or None - shading_engine = cmds.listConnections(member, type="objectSet") - if not shading_engine: - cls.log.error("Detected shape without shading engine : " - "'{}'".format(member)) - invalid.append(member) - continue + object_sets = cmds.listConnections(node, type="objectSet") or [] - # retrieve the shading engine out of the list - shading_engine = shading_engine[0] - if shading_engine in disallowed: - cls.log.error("Member connected to a disallows objectSet: " - "'{}'".format(member)) - invalid.append(member) - else: - continue + # Ensure the shape in the instances have at least a single shader + # connected if it *can* have a shader, like a `surfaceShape` in + # Maya. + if (cmds.objectType(node, isAType="surfaceShape") and + not cmds.ls(object_sets, type="shadingEngine")): + cls.log.error("Detected shape without shading engine: " + "'{}'".format(node)) + invalid.add(node) - return invalid + # Check for any disallowed connections + if any(s in disallowed for s in object_sets): + + # Explicitly log each individual "wrong" connection. + for s in object_sets: + if s in disallowed: + cls.log.error("Node has unallowed connection to " + "'{}': {}".format(s, node)) + + invalid.add(node) + + return list(invalid) def process(self, instance): """Process all the nodes in the instance""" invalid = self.get_invalid(instance) if invalid: - raise RuntimeError("Invalid shaders found: {0}".format(invalid)) + raise RuntimeError("Invalid node relationships found: " + "{0}".format(invalid)) diff --git a/colorbleed/plugins/maya/publish/validate_look_sets.py b/colorbleed/plugins/maya/publish/validate_look_sets.py new file mode 100644 index 0000000000..56f104a18b --- /dev/null +++ b/colorbleed/plugins/maya/publish/validate_look_sets.py @@ -0,0 +1,73 @@ +from colorbleed.maya import lib + +import pyblish.api +import colorbleed.api + + +class ValidateLookSets(pyblish.api.InstancePlugin): + """Validate if any sets are missing from the instance and look data + + """ + + order = colorbleed.api.ValidateContentsOrder + families = ['colorbleed.lookdev'] + hosts = ['maya'] + label = 'Look Sets' + actions = [colorbleed.api.SelectInvalidAction] + + def process(self, instance): + """Process all the nodes in the instance""" + + if not instance[:]: + raise RuntimeError("Instance is empty") + + self.log.info("Validation '{}'".format(instance.name)) + invalid = self.get_invalid(instance) + if invalid: + raise RuntimeError("'{}' has invalid look " + "content".format(instance.name)) + + @classmethod + def get_invalid(cls, instance): + """Get all invalid nodes""" + + cls.log.info("Validating look content for " + "'{}'".format(instance.name)) + + lookdata = instance.data["lookData"] + relationships = lookdata["relationships"] + + invalid = [] + for node in instance: + sets = lib.get_related_sets(node) + if not sets: + continue + + missing_sets = [s for s in sets if s not in relationships] + if missing_sets: + # A set of this node is not coming along, this is wrong! + cls.log.error("Missing sets '{}' for node " + "'{}'".format(missing_sets, node)) + invalid.append(node) + continue + + # Ensure the node is in the sets that are collected + for shaderset, data in relationships.items(): + if shaderset not in sets: + # no need to check for a set if the node + # isn't in it anyway + continue + + member_nodes = [member['name'] for member in data['members']] + if node not in member_nodes: + # The node is not found in the collected set + # relationships + cls.log.error("Missing '{}' in collected set node " + "'{}'".format(node, shaderset)) + invalid.append(node) + + continue + + return invalid + + From ef542916e348812df7267d0fe629cfa86e853c7e Mon Sep 17 00:00:00 2001 From: aardschok Date: Thu, 10 Aug 2017 16:47:52 +0200 Subject: [PATCH 2/2] added deadline plugins for configuration --- .../plugins/maya/publish/submit_deadline.py | 206 ++++++++++++++++++ .../plugins/publish/validate_deadline_done.py | 57 +++++ 2 files changed, 263 insertions(+) create mode 100644 colorbleed/plugins/maya/publish/submit_deadline.py create mode 100644 colorbleed/plugins/publish/validate_deadline_done.py diff --git a/colorbleed/plugins/maya/publish/submit_deadline.py b/colorbleed/plugins/maya/publish/submit_deadline.py new file mode 100644 index 0000000000..108bd562b3 --- /dev/null +++ b/colorbleed/plugins/maya/publish/submit_deadline.py @@ -0,0 +1,206 @@ +import pyblish.api + + +class MindbenderSubmitDeadline(pyblish.api.InstancePlugin): + """Submit available render layers to Deadline + + Renders are submitted to a Deadline Web Service as + supplied via the environment variable AVALON_DEADLINE + + """ + + label = "Submit to Deadline" + order = pyblish.api.IntegratorOrder + hosts = ["maya"] + families = ["mindbender.renderlayer"] + + def process(self, instance): + import os + import json + import shutil + import getpass + + from maya import cmds + + from avalon import api + from avalon.vendor import requests + + assert api.Session["AVALON_DEADLINE"], "Requires AVALON_DEADLINE" + + context = instance.context + workspace = context.data["workspaceDir"] + fpath = context.data["currentFile"] + fname = os.path.basename(fpath) + name, ext = os.path.splitext(fname) + comment = context.data.get("comment", "") + dirname = os.path.join(workspace, "renders", name) + + try: + os.makedirs(dirname) + except OSError: + pass + + # E.g. http://192.168.0.1:8082/api/jobs + url = api.Session["AVALON_DEADLINE"] + "/api/jobs" + + # Documentation for keys available at: + # https://docs.thinkboxsoftware.com + # /products/deadline/8.0/1_User%20Manual/manual + # /manual-submission.html#job-info-file-options + payload = { + "JobInfo": { + # Top-level group name + "BatchName": fname, + + # Job name, as seen in Monitor + "Name": "%s - %s" % (fname, instance.name), + + # Arbitrary username, for visualisation in Monitor + "UserName": getpass.getuser(), + + "Plugin": "MayaBatch", + "Frames": "{start}-{end}x{step}".format( + start=int(instance.data["startFrame"]), + end=int(instance.data["endFrame"]), + step=int(instance.data["byFrameStep"]), + ), + + "Comment": comment, + + # Optional, enable double-click to preview rendered + # frames from Deadline Monitor + "OutputFilename0": self.preview_fname(instance), + }, + "PluginInfo": { + # Input + "SceneFile": fpath, + + # Output directory and filename + "OutputFilePath": dirname, + "OutputFilePrefix": "/", + + # Mandatory for Deadline + "Version": cmds.about(version=True), + + # Only render layers are considered renderable in this pipeline + "UsingRenderLayers": True, + + # Render only this layer + "RenderLayer": instance.name, + + # Determine which renderer to use from the file itself + "Renderer": "file", + + # Resolve relative references + "ProjectPath": workspace, + }, + + # Mandatory for Deadline, may be empty + "AuxFiles": [] + } + + # Include critical variables with submission + environment = dict({ + # This will trigger `userSetup.py` on the slave + # such that proper initialisation happens the same + # way as it does on a local machine. + # TODO(marcus): This won't work if the slaves don't + # have accesss to these paths, such as if slaves are + # running Linux and the submitter is on Windows. + "PYTHONPATH": os.getenv("PYTHONPATH", ""), + + }, **api.Session) + + payload["JobInfo"].update({ + "EnvironmentKeyValue%d" % index: "{key}={value}".format( + key=key, + value=environment[key] + ) for index, key in enumerate(environment) + }) + + # Include optional render globals + payload["JobInfo"].update( + instance.data.get("renderGlobals", {}) + ) + + self.preflight_check(instance) + + self.log.info("Submitting..") + self.log.info(json.dumps( + payload, indent=4, sort_keys=True) + ) + + response = requests.post(url, json=payload) + + if response.ok: + # Write metadata for publish + fname = os.path.join(dirname, instance.name + ".json") + data = { + "submission": payload, + "session": api.Session, + "instance": instance.data, + "jobs": [ + response.json() + ], + } + + with open(fname, "w") as f: + json.dump(data, f, indent=4, sort_keys=True) + + else: + try: + shutil.rmtree(dirname) + except OSError: + # This is nice-to-have, but not critical to the operation + pass + + raise Exception(response.text) + + def preview_fname(self, instance): + """Return outputted filename with #### for padding + + Passing the absolute path to Deadline enables Deadline Monitor + to provide the user with a Job Output menu option. + + Deadline requires the path to be formatted with # in place of numbers. + + From + /path/to/render.0000.png + To + /path/to/render.####.png + + """ + + from maya import cmds + + # We'll need to take tokens into account + fname = cmds.renderSettings( + firstImageName=True, + fullPath=True, + layer=instance.name + )[0] + + try: + # Assume `c:/some/path/filename.0001.exr` + # TODO(marcus): Bulletproof this, the user may have + # chosen a different format for the outputted filename. + fname, padding, suffix = fname.rsplit(".", 2) + fname = ".".join([fname, "#" * len(padding), suffix]) + self.log.info("Assuming renders end up @ %s" % fname) + except ValueError: + fname = "" + self.log.info("Couldn't figure out where renders go") + + return fname + + def preflight_check(self, instance): + for key in ("startFrame", "endFrame", "byFrameStep"): + value = instance.data[key] + + if int(value) == value: + continue + + self.log.warning( + "%f=%d was rounded off to nearest integer" + % (value, int(value)) + ) diff --git a/colorbleed/plugins/publish/validate_deadline_done.py b/colorbleed/plugins/publish/validate_deadline_done.py new file mode 100644 index 0000000000..1ab539eefa --- /dev/null +++ b/colorbleed/plugins/publish/validate_deadline_done.py @@ -0,0 +1,57 @@ +import pyblish.api + + +class ValidateMindbenderDeadlineDone(pyblish.api.InstancePlugin): + """Ensure render is finished before publishing the resulting images""" + + label = "Rendered Successfully" + order = pyblish.api.ValidatorOrder + hosts = ["shell"] + families = ["mindbender.imagesequence"] + optional = True + + def process(self, instance): + from avalon import api + from avalon.vendor import requests + + # From Deadline documentation + # https://docs.thinkboxsoftware.com/products/deadline/8.0/ + # 1_User%20Manual/manual/rest-jobs.html#job-property-values + states = { + 0: "Unknown", + 1: "Active", + 2: "Suspended", + 3: "Completed", + 4: "Failed", + 6: "Pending", + } + + url = api.Session["AVALON_DEADLINE"] + "/api/jobs?JobID=%s" + + for job in instance.data["metadata"]["jobs"]: + response = requests.get(url % job["_id"]) + + if response.ok: + data = response.json()[0] + state = states.get(data["Stat"]) + + if state in (None, "Unknown"): + raise Exception("State of this render is unknown") + + elif state == "Active": + raise Exception("This render is still currently active") + + elif state == "Suspended": + raise Exception("This render is suspended") + + elif state == "Failed": + raise Exception("This render was not successful") + + elif state == "Pending": + raise Exception("This render is pending") + else: + self.log.info("%s was rendered successfully" % instance) + + else: + raise Exception("Could not determine the current status " + " of this render")