From e4365b746b40d311c59b3305c39adc61dc1a0425 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 28 Jan 2022 18:18:06 +0100 Subject: [PATCH 01/17] Draft for allowing Arnold ROP render publishing to Deadline from Houdini --- .../plugins/create/create_arnold_rop.py | 50 ++++++ .../plugins/publish/collect_arnold_rop.py | 142 ++++++++++++++++++ .../plugins/publish/collect_instances.py | 2 +- .../plugins/publish/increment_current_file.py | 2 +- .../houdini/plugins/publish/save_scene.py | 3 +- .../publish/submit_houdini_render_deadline.py | 24 +-- 6 files changed, 208 insertions(+), 15 deletions(-) create mode 100644 openpype/hosts/houdini/plugins/create/create_arnold_rop.py create mode 100644 openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py diff --git a/openpype/hosts/houdini/plugins/create/create_arnold_rop.py b/openpype/hosts/houdini/plugins/create/create_arnold_rop.py new file mode 100644 index 0000000000..e0fad2a1cc --- /dev/null +++ b/openpype/hosts/houdini/plugins/create/create_arnold_rop.py @@ -0,0 +1,50 @@ +from avalon import houdini + + +class CreateArnoldRop(houdini.Creator): + """Arnold ROP""" + + label = "Arnold ROP" + family = "arnold_rop" + icon = "magic" + defaults = ["master"] + + def __init__(self, *args, **kwargs): + super(CreateArnoldRop, self).__init__(*args, **kwargs) + + # Clear the family prefix from the subset + subset = self.data["subset"] + subset_no_prefix = subset[len(self.family):] + subset_no_prefix = subset_no_prefix[0].lower() + subset_no_prefix[1:] + self.data["subset"] = subset_no_prefix + + # Add chunk size attribute + self.data["chunkSize"] = 1 + + # Remove the active, we are checking the bypass flag of the nodes + self.data.pop("active", None) + + self.data.update({"node_type": "arnold"}) + + def process(self): + instance = super(CreateArnoldRop, self).process() + + basename = instance.name() + instance.setName(basename + "_ROP", unique_name=True) + + prefix = '${HIP}/render/${HIPNAME}/`chs("subset")`.$F4.exr' + parms = { + # Render frame range + "trange": 1, + + # Arnold ROP settings + "ar_picture": prefix, + "ar_exr_half_precision": 1 # half precision + } + instance.setParms(parms) + + # Lock some Avalon attributes + to_lock = ["family", "id"] + for name in to_lock: + parm = instance.parm(name) + parm.lock(True) diff --git a/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py b/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py new file mode 100644 index 0000000000..e5abfccca0 --- /dev/null +++ b/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py @@ -0,0 +1,142 @@ +import re +import os + +import hou +import avalon.io as io +import avalon.api as api +import pyblish.api + + +def get_top_referenced_parm(parm): + + processed = set() # disallow infinite loop + while True: + if parm.path() in processed: + raise RuntimeError("Parameter references result in cycle.") + + processed.add(parm.path()) + + ref = parm.getReferencedParm() + if ref.path() == parm.path(): + # It returns itself when it doesn't reference + # another parameter + return ref + else: + parm = ref + + +def evalParmNoFrame(node, parm, pad_character="#"): + + parameter = node.parm(parm) + assert parameter, "Parameter does not exist: %s.%s" % (node, parm) + + # If the parameter has a parameter reference, then get that + # parameter instead as otherwise `unexpandedString()` fails. + parameter = get_top_referenced_parm(parameter) + + # Substitute out the frame numbering with padded characters + try: + raw = parameter.unexpandedString() + except hou.Error as exc: + print("Failed: %s" % parameter) + raise RuntimeError(exc) + + def replace(match): + padding = 1 + n = match.group(2) + if n and int(n): + padding = int(n) + return pad_character * padding + + expression = re.sub(r"(\$F([0-9]*))", replace, raw) + + with hou.ScriptEvalContext(parameter): + return hou.expandStringAtFrame(expression, 0) + + +class CollectArnoldROPRenderProducts(pyblish.api.InstancePlugin): + """Collect Arnold ROP Render Products + + Collects the instance.data["files"] for the render products. + + Provides: + instance -> files + + """ + + label = "Arnold ROP Render Products" + order = pyblish.api.CollectorOrder + 0.4 + hosts = ["houdini"] + families = ["arnold_rop"] + + def process(self, instance): + + rop = instance[0] + + # Collect chunkSize + chunk_size_parm = rop.parm("chunkSize") + if chunk_size_parm: + chunk_size = int(chunk_size_parm.eval()) + instance.data["chunkSize"] = chunk_size + self.log.debug("Chunk Size: %s" % chunk_size) + + default_prefix = evalParmNoFrame(rop, "ar_picture") + render_products = [] + + # Default beauty AOV + beauty_product = self.get_render_product_name(prefix=default_prefix, + suffix=None) + render_products.append(beauty_product) + + # TODO: Implement AOVS + # num_aovs = rop.evalParm("ar_aovs") + # for index in range(num_aovs): + # i = index + 1 + # + # # Skip disabled AOVs + # if not rop.evalParm("ar_enable_aov%s" % i): + # continue + # + # label = evalParmNoFrame(rop, "ar_aov_label%s" % i) + # if rop.evalParm("ar_aov_exr_enable_layer_name%s" % i): + # label = rop.evalParm("ar_aov_exr_layer_name%s" % i) + # + # aov_product = self.get_render_product_name(default_prefix, + # suffix=label) + # render_products.append(aov_product) + + for product in render_products: + self.log.debug("Found render product: %s" % product) + + filenames = list(render_products) + instance.data["files"] = filenames + + # For now by default do NOT try to publish the rendered output + instance.data["publishJobState"] = "Suspended" + + def get_render_product_name(self, prefix, suffix): + """Return the output filename using the AOV prefix and suffix""" + + # When AOV is explicitly defined in prefix we just swap it out + # directly with the AOV suffix to embed it. + # Note: ${AOV} seems to be evaluated in the parameter as %AOV% + has_aov_in_prefix = "%AOV%" in prefix + if has_aov_in_prefix: + # It seems that when some special separator characters are present + # before the %AOV% token that Redshift will secretly remove it if + # there is no suffix for the current product, for example: + # foo_%AOV% -> foo.exr + pattern = "%AOV%" if suffix else "[._-]?%AOV%" + product_name = re.sub(pattern, + suffix, + prefix, + flags=re.IGNORECASE) + else: + if suffix: + # Add ".{suffix}" before the extension + prefix_base, ext = os.path.splitext(prefix) + product_name = prefix_base + "." + suffix + ext + else: + product_name = prefix + + return product_name diff --git a/openpype/hosts/houdini/plugins/publish/collect_instances.py b/openpype/hosts/houdini/plugins/publish/collect_instances.py index 12d118f0cc..094f791a0b 100644 --- a/openpype/hosts/houdini/plugins/publish/collect_instances.py +++ b/openpype/hosts/houdini/plugins/publish/collect_instances.py @@ -109,6 +109,6 @@ class CollectInstances(pyblish.api.ContextPlugin): data["frameStart"] = node.evalParm("f1") data["frameEnd"] = node.evalParm("f2") - data["steps"] = node.evalParm("f3") + data["byFrameStep"] = node.evalParm("f3") return data diff --git a/openpype/hosts/houdini/plugins/publish/increment_current_file.py b/openpype/hosts/houdini/plugins/publish/increment_current_file.py index 31c2954ee7..7a1ad09249 100644 --- a/openpype/hosts/houdini/plugins/publish/increment_current_file.py +++ b/openpype/hosts/houdini/plugins/publish/increment_current_file.py @@ -15,7 +15,7 @@ class IncrementCurrentFile(pyblish.api.InstancePlugin): label = "Increment current file" order = pyblish.api.IntegratorOrder + 9.0 hosts = ["houdini"] - families = ["colorbleed.usdrender", "redshift_rop"] + families = ["colorbleed.usdrender", "redshift_rop", "arnold_rop"] targets = ["local"] def process(self, instance): diff --git a/openpype/hosts/houdini/plugins/publish/save_scene.py b/openpype/hosts/houdini/plugins/publish/save_scene.py index 1b12efa603..75ff4dcdfd 100644 --- a/openpype/hosts/houdini/plugins/publish/save_scene.py +++ b/openpype/hosts/houdini/plugins/publish/save_scene.py @@ -9,7 +9,8 @@ class SaveCurrentScene(pyblish.api.InstancePlugin): order = pyblish.api.IntegratorOrder - 0.49 hosts = ["houdini"] families = ["usdrender", - "redshift_rop"] + "redshift_rop", + "arnold_rop"] targets = ["local"] def process(self, instance): diff --git a/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_render_deadline.py b/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_render_deadline.py index fa146c0d30..9613bae7fe 100644 --- a/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_render_deadline.py +++ b/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_render_deadline.py @@ -27,13 +27,12 @@ class HoudiniSubmitRenderDeadline(pyblish.api.InstancePlugin): order = pyblish.api.IntegratorOrder hosts = ["houdini"] families = ["usdrender", - "redshift_rop"] - targets = ["local"] + "redshift_rop", + "arnold_rop"] def process(self, instance): context = instance.context - code = context.data["code"] filepath = context.data["currentFile"] filename = os.path.basename(filepath) comment = context.data.get("comment", "") @@ -42,16 +41,14 @@ class HoudiniSubmitRenderDeadline(pyblish.api.InstancePlugin): # Support code prefix label for batch name batch_name = filename - if code: - batch_name = "{0} - {1}".format(code, batch_name) # Output driver to render driver = instance[0] # StartFrame to EndFrame by byFrameStep frames = "{start}-{end}x{step}".format( - start=int(instance.data["startFrame"]), - end=int(instance.data["endFrame"]), + start=int(instance.data["frameStart"]), + end=int(instance.data["frameEnd"]), step=int(instance.data["byFrameStep"]), ) @@ -71,7 +68,7 @@ class HoudiniSubmitRenderDeadline(pyblish.api.InstancePlugin): "UserName": deadline_user, "Plugin": "Houdini", - "Pool": "houdini_redshift", # todo: remove hardcoded pool + #"Pool": "houdini", # todo(roy): Add pool options "Frames": frames, "ChunkSize": instance.data.get("chunkSize", 10), @@ -136,9 +133,12 @@ class HoudiniSubmitRenderDeadline(pyblish.api.InstancePlugin): def submit(self, instance, payload): - AVALON_DEADLINE = api.Session.get("AVALON_DEADLINE", - "http://localhost:8082") - assert AVALON_DEADLINE, "Requires AVALON_DEADLINE" + # get default deadline webservice url from deadline module + deadline_url = instance.context.data.get("defaultDeadline") + # if custom one is set in instance, use that + if instance.data.get("deadlineUrl"): + deadline_url = instance.data.get("deadlineUrl") + assert deadline_url, "Requires Deadline Webservice URL" plugin = payload["JobInfo"]["Plugin"] self.log.info("Using Render Plugin : {}".format(plugin)) @@ -147,7 +147,7 @@ class HoudiniSubmitRenderDeadline(pyblish.api.InstancePlugin): self.log.debug(json.dumps(payload, indent=4, sort_keys=True)) # E.g. http://192.168.0.1:8082/api/jobs - url = "{}/api/jobs".format(AVALON_DEADLINE) + url = "{}/api/jobs".format(deadline_url) response = requests.post(url, json=payload) if not response.ok: raise Exception(response.text) From 553518064efea3af8241a46164600d45b77cb3a7 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 28 Jan 2022 18:28:14 +0100 Subject: [PATCH 02/17] Shush the hound --- openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py | 4 +--- .../plugins/publish/submit_houdini_render_deadline.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py b/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py index e5abfccca0..93849502be 100644 --- a/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py +++ b/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py @@ -2,14 +2,12 @@ import re import os import hou -import avalon.io as io -import avalon.api as api import pyblish.api def get_top_referenced_parm(parm): - processed = set() # disallow infinite loop + processed = set() # disallow infinite loop while True: if parm.path() in processed: raise RuntimeError("Parameter references result in cycle.") diff --git a/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_render_deadline.py b/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_render_deadline.py index 9613bae7fe..996c5a5d46 100644 --- a/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_render_deadline.py +++ b/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_render_deadline.py @@ -68,7 +68,7 @@ class HoudiniSubmitRenderDeadline(pyblish.api.InstancePlugin): "UserName": deadline_user, "Plugin": "Houdini", - #"Pool": "houdini", # todo(roy): Add pool options + # "Pool": "houdini", # todo(roy): Add pool options "Frames": frames, "ChunkSize": instance.data.get("chunkSize", 10), From 976c708d59ecb860c7b2eccd270f99cb6e0fe1ae Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 23 Aug 2022 12:11:09 +0200 Subject: [PATCH 03/17] Refactor to new houdini Creator --- openpype/hosts/houdini/plugins/create/create_arnold_rop.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/openpype/hosts/houdini/plugins/create/create_arnold_rop.py b/openpype/hosts/houdini/plugins/create/create_arnold_rop.py index e0fad2a1cc..f9b5f0c53c 100644 --- a/openpype/hosts/houdini/plugins/create/create_arnold_rop.py +++ b/openpype/hosts/houdini/plugins/create/create_arnold_rop.py @@ -1,7 +1,7 @@ -from avalon import houdini +from openpype.hosts.houdini.api import plugin -class CreateArnoldRop(houdini.Creator): +class CreateArnoldRop(plugin.Creator): """Arnold ROP""" label = "Arnold ROP" @@ -26,8 +26,7 @@ class CreateArnoldRop(houdini.Creator): self.data.update({"node_type": "arnold"}) - def process(self): - instance = super(CreateArnoldRop, self).process() + def _process(self, instance): basename = instance.name() instance.setName(basename + "_ROP", unique_name=True) From 8bedec713d8d6e81bd6df43122ff6f8a780666e9 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 23 Aug 2022 12:11:30 +0200 Subject: [PATCH 04/17] Enable AOVs + fix collected files for publishing --- .../plugins/publish/collect_arnold_rop.py | 66 ++++++++++++++----- 1 file changed, 50 insertions(+), 16 deletions(-) diff --git a/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py b/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py index 93849502be..dfb8f0d5ee 100644 --- a/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py +++ b/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py @@ -86,22 +86,28 @@ class CollectArnoldROPRenderProducts(pyblish.api.InstancePlugin): suffix=None) render_products.append(beauty_product) - # TODO: Implement AOVS - # num_aovs = rop.evalParm("ar_aovs") - # for index in range(num_aovs): - # i = index + 1 - # - # # Skip disabled AOVs - # if not rop.evalParm("ar_enable_aov%s" % i): - # continue - # - # label = evalParmNoFrame(rop, "ar_aov_label%s" % i) - # if rop.evalParm("ar_aov_exr_enable_layer_name%s" % i): - # label = rop.evalParm("ar_aov_exr_layer_name%s" % i) - # - # aov_product = self.get_render_product_name(default_prefix, - # suffix=label) - # render_products.append(aov_product) + files_by_aov = { + "": self.generate_expected_files(instance, beauty_product) + } + + num_aovs = rop.evalParm("ar_aovs") + for index in range(num_aovs): + i = index + 1 + + # Skip disabled AOVs + if not rop.evalParm("ar_enable_aov%s" % i): + continue + + if rop.evalParm("ar_aov_exr_enable_layer_name%s" % i): + label = rop.evalParm("ar_aov_exr_layer_name%s" % i) + else: + label = evalParmNoFrame(rop, "ar_aov_label%s" % i) + + aov_product = self.get_render_product_name(default_prefix, + suffix=label) + render_products.append(aov_product) + files_by_aov[label] = self.generate_expected_files(instance, + aov_product) for product in render_products: self.log.debug("Found render product: %s" % product) @@ -111,6 +117,11 @@ class CollectArnoldROPRenderProducts(pyblish.api.InstancePlugin): # For now by default do NOT try to publish the rendered output instance.data["publishJobState"] = "Suspended" + instance.data["attachTo"] = [] # stub required data + + if "expectedFiles" not in instance.data: + instance.data["expectedFiles"] = list() + instance.data["expectedFiles"].append(files_by_aov) def get_render_product_name(self, prefix, suffix): """Return the output filename using the AOV prefix and suffix""" @@ -138,3 +149,26 @@ class CollectArnoldROPRenderProducts(pyblish.api.InstancePlugin): product_name = prefix return product_name + + def generate_expected_files(self, instance, path): + """Create expected files in instance data""" + + dir = os.path.dirname(path) + file = os.path.basename(path) + + if "#" in file: + pparts = file.split("#") + padding = "%0{}d".format(len(pparts) - 1) + file = pparts[0] + padding + pparts[-1] + + if "%" not in file: + return path + + expected_files = [] + start = instance.data["frameStart"] + end = instance.data["frameEnd"] + for i in range(int(start), (int(end) + 1)): + expected_files.append( + os.path.join(dir, (file % i)).replace("\\", "/")) + + return expected_files From 3f59f39847158975fed9467275729ba516507ce6 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 23 Aug 2022 12:16:23 +0200 Subject: [PATCH 05/17] Refactor submit deadline to use `AbstractSubmitDeadline` class --- .../publish/submit_houdini_render_deadline.py | 175 +++++++----------- 1 file changed, 71 insertions(+), 104 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py index 3cacc8dd16..8aabec6e30 100644 --- a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py @@ -1,16 +1,24 @@ -import os -import json -import getpass - -import requests -import pyblish.api - import hou +import os +import attr +import getpass +import pyblish.api + from openpype.pipeline import legacy_io +from openpype_modules.deadline import abstract_submit_deadline +from openpype_modules.deadline.abstract_submit_deadline import DeadlineJobInfo -class HoudiniSubmitRenderDeadline(pyblish.api.InstancePlugin): +@attr.s +class DeadlinePluginInfo(): + SceneFile = attr.ib(default=None) + OutputDriver = attr.ib(default=None) + Version = attr.ib(default=None) + IgnoreInputs = attr.ib(default=True) + + +class HoudiniSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): """Submit Solaris USD Render ROPs to Deadline. Renders are submitted to a Deadline Web Service as @@ -30,136 +38,95 @@ class HoudiniSubmitRenderDeadline(pyblish.api.InstancePlugin): "redshift_rop", "arnold_rop"] targets = ["local"] + use_published = True - def process(self, instance): + def get_job_info(self): + job_info = DeadlineJobInfo(Plugin="Houdini") + instance = self._instance context = instance.context + filepath = context.data["currentFile"] filename = os.path.basename(filepath) - comment = context.data.get("comment", "") - deadline_user = context.data.get("deadlineUser", getpass.getuser()) - jobname = "%s - %s" % (filename, instance.name) - # Support code prefix label for batch name - batch_name = filename + job_info.Name = "%s - %s" % (filename, instance.name) + job_info.BatchName = filename + job_info.Plugin = "Houdini" + job_info.UserName = context.data.get( + "deadlineUser", getpass.getuser()) - # Output driver to render - driver = instance[0] - - # StartFrame to EndFrame by byFrameStep + # Deadline requires integers in frame range frames = "{start}-{end}x{step}".format( start=int(instance.data["frameStart"]), end=int(instance.data["frameEnd"]), step=int(instance.data["byFrameStep"]), ) + job_info.Frames = frames - # Documentation for keys available at: - # https://docs.thinkboxsoftware.com - # /products/deadline/8.0/1_User%20Manual/manual - # /manual-submission.html#job-info-file-options - payload = { - "JobInfo": { - # Top-level group name - "BatchName": batch_name, + job_info.Pool = instance.data.get("primaryPool") + job_info.SecondaryPool = instance.data.get("secondaryPool") + job_info.ChunkSize = instance.data.get("chunkSize", 10) + job_info.Comment = context.data.get("comment") - # Job name, as seen in Monitor - "Name": jobname, - - # Arbitrary username, for visualisation in Monitor - "UserName": deadline_user, - - "Plugin": "Houdini", - "Pool": instance.data.get("primaryPool"), - "secondaryPool": instance.data.get("secondaryPool"), - "Frames": frames, - - "ChunkSize": instance.data.get("chunkSize", 10), - - "Comment": comment - }, - "PluginInfo": { - # Input - "SceneFile": filepath, - "OutputDriver": driver.path(), - - # Mandatory for Deadline - # Houdini version without patch number - "Version": hou.applicationVersionString().rsplit(".", 1)[0], - - "IgnoreInputs": True - }, - - # Mandatory for Deadline, may be empty - "AuxFiles": [] - } - - # Include critical environment variables with submission + api.Session keys = [ - # Submit along the current Avalon tool setup that we launched - # this application with so the Render Slave can build its own - # similar environment using it, e.g. "maya2018;vray4.x;yeti3.1.9" - "AVALON_TOOLS", + "FTRACK_API_KEY", + "FTRACK_API_USER", + "FTRACK_SERVER", + "OPENPYPE_SG_USER", + "AVALON_PROJECT", + "AVALON_ASSET", + "AVALON_TASK", + "AVALON_APP_NAME", + "OPENPYPE_DEV", + "OPENPYPE_LOG_NO_COLORS", "OPENPYPE_VERSION" ] # Add mongo url if it's enabled - if context.data.get("deadlinePassMongoUrl"): + if self._instance.context.data.get("deadlinePassMongoUrl"): keys.append("OPENPYPE_MONGO") environment = dict({key: os.environ[key] for key in keys if key in os.environ}, **legacy_io.Session) + for key in keys: + val = environment.get(key) + if val: + job_info.EnvironmentKeyValue = "{key}={value}".format( + key=key, + value=val) + # to recognize job from PYPE for turning Event On/Off + job_info.EnvironmentKeyValue = "OPENPYPE_RENDER_JOB=1" - payload["JobInfo"].update({ - "EnvironmentKeyValue%d" % index: "{key}={value}".format( - key=key, - value=environment[key] - ) for index, key in enumerate(environment) - }) - - # Include OutputFilename entries - # The first entry also enables double-click to preview rendered - # frames from Deadline Monitor - output_data = {} for i, filepath in enumerate(instance.data["files"]): dirname = os.path.dirname(filepath) fname = os.path.basename(filepath) - output_data["OutputDirectory%d" % i] = dirname.replace("\\", "/") - output_data["OutputFilename%d" % i] = fname + job_info.OutputDirectory = dirname.replace("\\", "/") + job_info.OutputFilename = fname - # For now ensure destination folder exists otherwise HUSK - # will fail to render the output image. This is supposedly fixed - # in new production builds of Houdini - # TODO Remove this workaround with Houdini 18.0.391+ - if not os.path.exists(dirname): - self.log.info("Ensuring output directory exists: %s" % - dirname) - os.makedirs(dirname) + return job_info - payload["JobInfo"].update(output_data) + def get_plugin_info(self): - self.submit(instance, payload) + instance = self._instance + context = instance.context - def submit(self, instance, payload): + # Output driver to render + driver = instance[0] + hou_major_minor = hou.applicationVersionString().rsplit(".", 1)[0] - # get default deadline webservice url from deadline module - deadline_url = instance.context.data.get("defaultDeadline") - # if custom one is set in instance, use that - if instance.data.get("deadlineUrl"): - deadline_url = instance.data.get("deadlineUrl") - assert deadline_url, "Requires Deadline Webservice URL" + plugin_info = DeadlinePluginInfo( + SceneFile=context.data["currentFile"], + OutputDriver=driver.path(), + Version=hou_major_minor, + IgnoreInputs=True + ) - plugin = payload["JobInfo"]["Plugin"] - self.log.info("Using Render Plugin : {}".format(plugin)) + return attr.asdict(plugin_info) - self.log.info("Submitting..") - self.log.debug(json.dumps(payload, indent=4, sort_keys=True)) - - # E.g. http://192.168.0.1:8082/api/jobs - url = "{}/api/jobs".format(deadline_url) - response = requests.post(url, json=payload) - if not response.ok: - raise Exception(response.text) + def process(self, instance): + super(HoudiniSubmitDeadline, self).process(instance) + # TODO: Avoid the need for this logic here, needed for submit publish # Store output dir for unified publisher (filesequence) output_dir = os.path.dirname(instance.data["files"][0]) instance.data["outputDir"] = output_dir - instance.data["deadlineSubmissionJob"] = response.json() + instance.data["toBeRenderedOn"] = "deadline" From 2b6ef60b654e383f9c17ec843747c30b3c1f0a76 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 23 Aug 2022 12:16:37 +0200 Subject: [PATCH 06/17] Submit publish job for houdini arnold rop --- .../modules/deadline/plugins/publish/submit_publish_job.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index 379953c9e4..552aa253f2 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -116,10 +116,12 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): deadline_plugin = "OpenPype" targets = ["local"] - hosts = ["fusion", "maya", "nuke", "celaction", "aftereffects", "harmony"] + hosts = ["fusion", "maya", "nuke", "celaction", "aftereffects", "harmony", + "houdini"] families = ["render.farm", "prerender.farm", - "renderlayer", "imagesequence", "vrayscene"] + "renderlayer", "imagesequence", "vrayscene", + "arnold_rop"] aov_filter = {"maya": [r".*([Bb]eauty).*"], "aftereffects": [r".*"], # for everything from AE From 58b524407b5035f592773004c312f970ca374bf1 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Tue, 23 Aug 2022 12:18:43 +0200 Subject: [PATCH 07/17] Fix style issue / shush hound --- openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py | 4 ++-- .../plugins/publish/submit_houdini_render_deadline.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py b/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py index dfb8f0d5ee..d90530d3a0 100644 --- a/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py +++ b/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py @@ -107,7 +107,7 @@ class CollectArnoldROPRenderProducts(pyblish.api.InstancePlugin): suffix=label) render_products.append(aov_product) files_by_aov[label] = self.generate_expected_files(instance, - aov_product) + aov_product) for product in render_products: self.log.debug("Found render product: %s" % product) @@ -120,7 +120,7 @@ class CollectArnoldROPRenderProducts(pyblish.api.InstancePlugin): instance.data["attachTo"] = [] # stub required data if "expectedFiles" not in instance.data: - instance.data["expectedFiles"] = list() + instance.data["expectedFiles"] = list() instance.data["expectedFiles"].append(files_by_aov) def get_render_product_name(self, prefix, suffix): diff --git a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py index 8aabec6e30..6b6eebba7e 100644 --- a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py @@ -91,8 +91,9 @@ class HoudiniSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): val = environment.get(key) if val: job_info.EnvironmentKeyValue = "{key}={value}".format( - key=key, - value=val) + key=key, + value=val + ) # to recognize job from PYPE for turning Event On/Off job_info.EnvironmentKeyValue = "OPENPYPE_RENDER_JOB=1" From 4ffef19e28d1d8f22d36eb6416228545721eff4e Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Wed, 25 Jan 2023 13:59:19 +0100 Subject: [PATCH 08/17] Refactor to use new style creator --- .../plugins/create/create_arnold_rop.py | 52 +++++++++++-------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/openpype/hosts/houdini/plugins/create/create_arnold_rop.py b/openpype/hosts/houdini/plugins/create/create_arnold_rop.py index f9b5f0c53c..c01586ef11 100644 --- a/openpype/hosts/houdini/plugins/create/create_arnold_rop.py +++ b/openpype/hosts/houdini/plugins/create/create_arnold_rop.py @@ -1,7 +1,7 @@ from openpype.hosts.houdini.api import plugin -class CreateArnoldRop(plugin.Creator): +class CreateArnoldRop(plugin.HoudiniCreator): """Arnold ROP""" label = "Arnold ROP" @@ -9,41 +9,47 @@ class CreateArnoldRop(plugin.Creator): icon = "magic" defaults = ["master"] - def __init__(self, *args, **kwargs): - super(CreateArnoldRop, self).__init__(*args, **kwargs) + # Default extension + ext = ".exr" - # Clear the family prefix from the subset - subset = self.data["subset"] - subset_no_prefix = subset[len(self.family):] - subset_no_prefix = subset_no_prefix[0].lower() + subset_no_prefix[1:] - self.data["subset"] = subset_no_prefix - - # Add chunk size attribute - self.data["chunkSize"] = 1 + def create(self, subset_name, instance_data, pre_create_data): + import hou # Remove the active, we are checking the bypass flag of the nodes - self.data.pop("active", None) + instance_data.pop("active", None) + instance_data.update({"node_type": "arnold"}) - self.data.update({"node_type": "arnold"}) + # Add chunk size attribute + instance_data["chunkSize"] = 1 - def _process(self, instance): + instance = super(CreateArnoldRop, self).create( + subset_name, + instance_data, + pre_create_data) # type: plugin.CreatedInstance - basename = instance.name() - instance.setName(basename + "_ROP", unique_name=True) + instance_node = hou.node(instance.get("instance_node")) - prefix = '${HIP}/render/${HIPNAME}/`chs("subset")`.$F4.exr' + # Hide Properties Tab on Arnold ROP since that's used + # for rendering instead of .ass Archive Export + parm_template_group = instance_node.parmTemplateGroup() + parm_template_group.hideFolder("Properties", True) + instance_node.setParmTemplateGroup(parm_template_group) + + filepath = "{}{}".format( + hou.text.expandString("$HIP/pyblish/"), + "{}.$F4{}".format(subset_name, self.ext) + ) parms = { # Render frame range "trange": 1, # Arnold ROP settings - "ar_picture": prefix, + "ar_picture": filepath, "ar_exr_half_precision": 1 # half precision } - instance.setParms(parms) - # Lock some Avalon attributes + instance_node.setParms(parms) + + # Lock any parameters in this list to_lock = ["family", "id"] - for name in to_lock: - parm = instance.parm(name) - parm.lock(True) + self.lock_parameters(instance_node, to_lock) From 56885f0eb3755fa4469854b94f7d8cf8be9a1d3f Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Wed, 25 Jan 2023 14:26:13 +0100 Subject: [PATCH 09/17] Add identifier --- openpype/hosts/houdini/plugins/create/create_arnold_rop.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openpype/hosts/houdini/plugins/create/create_arnold_rop.py b/openpype/hosts/houdini/plugins/create/create_arnold_rop.py index c01586ef11..41c4fcfaef 100644 --- a/openpype/hosts/houdini/plugins/create/create_arnold_rop.py +++ b/openpype/hosts/houdini/plugins/create/create_arnold_rop.py @@ -4,6 +4,7 @@ from openpype.hosts.houdini.api import plugin class CreateArnoldRop(plugin.HoudiniCreator): """Arnold ROP""" + identifier = "io.openpype.creators.houdini.arnold_rop" label = "Arnold ROP" family = "arnold_rop" icon = "magic" From 2b0f19a8a61ed2035781933686580ece30769119 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Wed, 25 Jan 2023 14:28:23 +0100 Subject: [PATCH 10/17] Fix arnold rop collector for new style creator --- openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py b/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py index d90530d3a0..4d82b74aa2 100644 --- a/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py +++ b/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py @@ -69,7 +69,7 @@ class CollectArnoldROPRenderProducts(pyblish.api.InstancePlugin): def process(self, instance): - rop = instance[0] + rop = hou.node(instance.data.get("instance_node")) # Collect chunkSize chunk_size_parm = rop.parm("chunkSize") From 6fdbf2b0e16f5c368b31427e01ccd25154982594 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Wed, 25 Jan 2023 14:30:57 +0100 Subject: [PATCH 11/17] The 'publish' key in instance data is optional and may not exist --- openpype/modules/deadline/abstract_submit_deadline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index 512ff800ee..51f2df19be 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -659,7 +659,7 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): # test if there is instance of workfile waiting # to be published. - assert i.data["publish"] is True, ( + assert i.data.get("publish", True) is True, ( "Workfile (scene) must be published along") return i From 4c50564d029dde1bfcce9a73a866f0d54ef0cfbe Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Wed, 25 Jan 2023 15:09:24 +0100 Subject: [PATCH 12/17] Fix node access --- .../deadline/plugins/publish/submit_houdini_render_deadline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py index 6b6eebba7e..a25bd0f0bb 100644 --- a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py @@ -111,7 +111,7 @@ class HoudiniSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): context = instance.context # Output driver to render - driver = instance[0] + driver = hou.node(instance.data["instance_node"]) hou_major_minor = hou.applicationVersionString().rsplit(".", 1)[0] plugin_info = DeadlinePluginInfo( From bbc20c16df3550bf0dcb340d88108c3bdb07a088 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Wed, 25 Jan 2023 15:09:38 +0100 Subject: [PATCH 13/17] Fix job info usage --- .../publish/submit_houdini_render_deadline.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py index a25bd0f0bb..d5c28994e0 100644 --- a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py @@ -88,20 +88,18 @@ class HoudiniSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): environment = dict({key: os.environ[key] for key in keys if key in os.environ}, **legacy_io.Session) for key in keys: - val = environment.get(key) - if val: - job_info.EnvironmentKeyValue = "{key}={value}".format( - key=key, - value=val - ) + value = environment.get(key) + if value: + job_info.EnvironmentKeyValue[key] = value + # to recognize job from PYPE for turning Event On/Off - job_info.EnvironmentKeyValue = "OPENPYPE_RENDER_JOB=1" + job_info.EnvironmentKeyValue["OPENPYPE_RENDER_JOB"] = "1" for i, filepath in enumerate(instance.data["files"]): dirname = os.path.dirname(filepath) fname = os.path.basename(filepath) - job_info.OutputDirectory = dirname.replace("\\", "/") - job_info.OutputFilename = fname + job_info.OutputDirectory += dirname.replace("\\", "/") + job_info.OutputFilename += fname return job_info From e2850d74e5e80bdeaae261ebbf2b2f0bfba8e47a Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Wed, 25 Jan 2023 15:12:50 +0100 Subject: [PATCH 14/17] Add dedicated collector for instance frame data --- .../publish/collect_instance_frame_data.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 openpype/hosts/houdini/plugins/publish/collect_instance_frame_data.py diff --git a/openpype/hosts/houdini/plugins/publish/collect_instance_frame_data.py b/openpype/hosts/houdini/plugins/publish/collect_instance_frame_data.py new file mode 100644 index 0000000000..584343cd64 --- /dev/null +++ b/openpype/hosts/houdini/plugins/publish/collect_instance_frame_data.py @@ -0,0 +1,56 @@ +import hou + +import pyblish.api + + +class CollectInstanceNodeFrameRange(pyblish.api.InstancePlugin): + """Collect time range frame data for the instance node.""" + + order = pyblish.api.CollectorOrder + 0.001 + label = "Instance Node Frame Range" + hosts = ["houdini"] + + def process(self, instance): + + node_path = instance.data.get("instance_node") + node = hou.node(node_path) if node_path else None + if not node_path or not node: + self.log.debug("No instance node found for instance: " + "{}".format(instance)) + return + + frame_data = self.get_frame_data(node) + if not frame_data: + return + + self.log.info("Collected time data: {}".format(frame_data)) + instance.data.update(frame_data) + + def get_frame_data(self, node): + """Get the frame data: start frame, end frame and steps + Args: + node(hou.Node) + + Returns: + dict + + """ + + data = {} + + if node.parm("trange") is None: + self.log.debug("Node has no 'trange' parameter: " + "{}".format(node.path())) + return data + + if node.evalParm("trange") == 0: + # Ignore 'render current frame' + self.log.debug("Node '{}' has 'Render current frame' set. " + "Time range data ignored.".format(node.path())) + return data + + data["frameStart"] = node.evalParm("f1") + data["frameEnd"] = node.evalParm("f2") + data["byFrameStep"] = node.evalParm("f3") + + return data From b3e4d8efd2a949986335d05c9f7a34abbc2f911c Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Wed, 25 Jan 2023 15:16:07 +0100 Subject: [PATCH 15/17] Fix increment current file plug-in --- .../plugins/publish/increment_current_file.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/openpype/hosts/houdini/plugins/publish/increment_current_file.py b/openpype/hosts/houdini/plugins/publish/increment_current_file.py index 8947530874..3b6496c38b 100644 --- a/openpype/hosts/houdini/plugins/publish/increment_current_file.py +++ b/openpype/hosts/houdini/plugins/publish/increment_current_file.py @@ -5,6 +5,7 @@ from openpype.pipeline import registered_host from openpype.action import get_errored_plugins_from_data from openpype.hosts.houdini.api import HoudiniHost + class IncrementCurrentFile(pyblish.api.ContextPlugin): """Increment the current file. @@ -18,18 +19,8 @@ class IncrementCurrentFile(pyblish.api.ContextPlugin): families = ["workfile", "redshift_rop", "arnold_rop", "usdrender"] optional = True - def process(self, instance): + def process(self, context): - # This should be a ContextPlugin, but this is a workaround - # for a bug in pyblish to run once for a family: issue #250 - context = instance.context - key = "__hasRun{}".format(self.__class__.__name__) - if context.data.get(key, False): - return - else: - context.data[key] = True - - context = instance.context errored_plugins = get_errored_plugins_from_data(context) if any( plugin.__name__ == "HoudiniSubmitPublishDeadline" From bd3183bcbc25b670255416f302bd1b11d261abc8 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Wed, 25 Jan 2023 15:17:07 +0100 Subject: [PATCH 16/17] Tweak error messages --- .../hosts/houdini/plugins/publish/increment_current_file.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/houdini/plugins/publish/increment_current_file.py b/openpype/hosts/houdini/plugins/publish/increment_current_file.py index 3b6496c38b..a307ee452c 100644 --- a/openpype/hosts/houdini/plugins/publish/increment_current_file.py +++ b/openpype/hosts/houdini/plugins/publish/increment_current_file.py @@ -4,6 +4,7 @@ from openpype.lib import version_up from openpype.pipeline import registered_host from openpype.action import get_errored_plugins_from_data from openpype.hosts.houdini.api import HoudiniHost +from openpype.pipeline.publish import KnownPublishError class IncrementCurrentFile(pyblish.api.ContextPlugin): @@ -26,7 +27,7 @@ class IncrementCurrentFile(pyblish.api.ContextPlugin): plugin.__name__ == "HoudiniSubmitPublishDeadline" for plugin in errored_plugins ): - raise RuntimeError( + raise KnownPublishError( "Skipping incrementing current file because " "submission to deadline failed." ) @@ -36,7 +37,7 @@ class IncrementCurrentFile(pyblish.api.ContextPlugin): current_file = host.current_file() assert ( context.data["currentFile"] == current_file - ), "Collected filename from current scene name." + ), "Collected filename mismatches from current scene name." new_filepath = version_up(current_file) host.save_workfile(new_filepath) From 5218b6550678cb3e3a5127a449a631652d62c249 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Sat, 11 Mar 2023 18:59:43 +0100 Subject: [PATCH 17/17] Shush hound --- .../deadline/plugins/publish/submit_houdini_render_deadline.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py index 3c7250e65b..d781496cde 100644 --- a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py @@ -12,6 +12,7 @@ from openpype_modules.deadline import abstract_submit_deadline from openpype_modules.deadline.abstract_submit_deadline import DeadlineJobInfo from openpype.lib import is_running_from_build + @attr.s class DeadlinePluginInfo(): SceneFile = attr.ib(default=None)