From 3841d298b2d1d3837a5a5e569cc120b3f87b9655 Mon Sep 17 00:00:00 2001 From: Fabia Serra Arrizabalaga Date: Mon, 7 Aug 2023 18:22:49 +0200 Subject: [PATCH 01/14] First draft at adding support in Houdini renders to split render in export + render tasks --- .../plugins/create/create_arnold_rop.py | 14 ++ .../plugins/create/create_mantra_rop.py | 11 + .../houdini/plugins/create/create_vray_rop.py | 15 ++ .../plugins/publish/collect_arnold_rop.py | 15 ++ .../plugins/publish/collect_mantra_rop.py | 15 ++ .../plugins/publish/collect_vray_rop.py | 16 ++ .../deadline/abstract_submit_deadline.py | 22 +- .../publish/submit_houdini_render_deadline.py | 205 ++++++++++++++++-- 8 files changed, 293 insertions(+), 20 deletions(-) diff --git a/openpype/hosts/houdini/plugins/create/create_arnold_rop.py b/openpype/hosts/houdini/plugins/create/create_arnold_rop.py index ca516619f6..d19795d3a9 100644 --- a/openpype/hosts/houdini/plugins/create/create_arnold_rop.py +++ b/openpype/hosts/houdini/plugins/create/create_arnold_rop.py @@ -14,6 +14,9 @@ class CreateArnoldRop(plugin.HoudiniCreator): # Default extension ext = "exr" + # Default to split export and render jobs + export_job = True + def create(self, subset_name, instance_data, pre_create_data): import hou @@ -49,6 +52,14 @@ class CreateArnoldRop(plugin.HoudiniCreator): "ar_exr_half_precision": 1 # half precision } + if pre_create_data.get("export_job"): + ass_filepath = "{export_dir}{subset_name}/{subset_name}.$F4.ass".format( + export_dir=hou.text.expandString("$HIP/pyblish/ass/"), + subset_name=subset_name, + ) + parms["ar_ass_export_enable"] = 1 + parms["ar_ass_file"] = ass_filepath + instance_node.setParms(parms) # Lock any parameters in this list @@ -67,6 +78,9 @@ class CreateArnoldRop(plugin.HoudiniCreator): BoolDef("farm", label="Submitting to Farm", default=True), + BoolDef("export_job", + label="Split export and render jobs", + default=self.export_job), EnumDef("image_format", image_format_enum, default=self.ext, diff --git a/openpype/hosts/houdini/plugins/create/create_mantra_rop.py b/openpype/hosts/houdini/plugins/create/create_mantra_rop.py index 5c29adb33f..41311a1e30 100644 --- a/openpype/hosts/houdini/plugins/create/create_mantra_rop.py +++ b/openpype/hosts/houdini/plugins/create/create_mantra_rop.py @@ -13,6 +13,9 @@ class CreateMantraROP(plugin.HoudiniCreator): icon = "magic" defaults = ["master"] + # Default to split export and render jobs + export_job = True + def create(self, subset_name, instance_data, pre_create_data): import hou # noqa @@ -45,6 +48,14 @@ class CreateMantraROP(plugin.HoudiniCreator): "vm_picture": filepath, } + if pre_create_data.get("export_job"): + ifd_filepath = "{export_dir}{subset_name}/{subset_name}.$F4.ifd".format( + export_dir=hou.text.expandString("$HIP/pyblish/ifd/"), + subset_name=subset_name, + ) + parms["soho_outputmode"] = 1 + parms["soho_diskfile"] = ifd_filepath + if self.selected_nodes: # If camera found in selection # we will use as render camera diff --git a/openpype/hosts/houdini/plugins/create/create_vray_rop.py b/openpype/hosts/houdini/plugins/create/create_vray_rop.py index 58748d4c34..29fab0c60c 100644 --- a/openpype/hosts/houdini/plugins/create/create_vray_rop.py +++ b/openpype/hosts/houdini/plugins/create/create_vray_rop.py @@ -18,6 +18,9 @@ class CreateVrayROP(plugin.HoudiniCreator): ext = "exr" + # Default to split export and render jobs + export_job = True + def create(self, subset_name, instance_data, pre_create_data): instance_data.pop("active", None) @@ -54,6 +57,15 @@ class CreateVrayROP(plugin.HoudiniCreator): "SettingsEXR_bits_per_channel": "16" # half precision } + if pre_create_data.get("export_job"): + scene_filepath = "{export_dir}{subset_name}/{subset_name}.$F4.vrscene".format( + export_dir=hou.text.expandString("$HIP/pyblish/vrscene/"), + subset_name=subset_name, + ) + # TODO: don't have VRay to check the names of these + parms["render_export_mode"] = 1 + parms["render_export_filepath"] = scene_filepath + if self.selected_nodes: # set up the render camera from the selected node camera = None @@ -142,6 +154,9 @@ class CreateVrayROP(plugin.HoudiniCreator): BoolDef("farm", label="Submitting to Farm", default=True), + BoolDef("export_job", + label="Split export and render jobs", + default=self.export_job), EnumDef("image_format", image_format_enum, default=self.ext, diff --git a/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py b/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py index 43b8428c60..063dd728e1 100644 --- a/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py +++ b/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py @@ -38,6 +38,21 @@ class CollectArnoldROPRenderProducts(pyblish.api.InstancePlugin): default_prefix = evalParmNoFrame(rop, "ar_picture") render_products = [] + # Store whether we are splitting the render job in an export + render + export_job = bool(rop.parm("ar_ass_export_enable").eval()) + instance.data["exportJob"] = export_job + export_prefix = None + export_products = [] + if export_job: + export_prefix = evalParmNoFrame(rop, "ar_ass_file", pad_character="0") + beauty_export_product = self.get_render_product_name( + prefix=export_prefix, + suffix=None) + export_products.append(beauty_export_product) + self.log.debug("Found export product: {}".format(beauty_export_product)) + instance.data["ifdFile"] = beauty_export_product + instance.data["exportFiles"] = list(export_products) + # Default beauty AOV beauty_product = self.get_render_product_name(prefix=default_prefix, suffix=None) diff --git a/openpype/hosts/houdini/plugins/publish/collect_mantra_rop.py b/openpype/hosts/houdini/plugins/publish/collect_mantra_rop.py index c4460f5350..b235e2c110 100644 --- a/openpype/hosts/houdini/plugins/publish/collect_mantra_rop.py +++ b/openpype/hosts/houdini/plugins/publish/collect_mantra_rop.py @@ -42,6 +42,21 @@ class CollectMantraROPRenderProducts(pyblish.api.InstancePlugin): default_prefix = evalParmNoFrame(rop, "vm_picture") render_products = [] + # Store whether we are splitting the render job in an export + render + export_job = bool(rop.parm("soho_outputmode").eval()) + instance.data["exportJob"] = export_job + export_prefix = None + export_products = [] + if export_job: + export_prefix = evalParmNoFrame(rop, "soho_diskfile", pad_character="0") + beauty_export_product = self.get_render_product_name( + prefix=export_prefix, + suffix=None) + export_products.append(beauty_export_product) + self.log.debug("Found export product: {}".format(beauty_export_product)) + instance.data["ifdFile"] = beauty_export_product + instance.data["exportFiles"] = list(export_products) + # Default beauty AOV beauty_product = self.get_render_product_name( prefix=default_prefix, suffix=None diff --git a/openpype/hosts/houdini/plugins/publish/collect_vray_rop.py b/openpype/hosts/houdini/plugins/publish/collect_vray_rop.py index d4fe37f993..89a7c8bf72 100644 --- a/openpype/hosts/houdini/plugins/publish/collect_vray_rop.py +++ b/openpype/hosts/houdini/plugins/publish/collect_vray_rop.py @@ -43,6 +43,22 @@ class CollectVrayROPRenderProducts(pyblish.api.InstancePlugin): render_products = [] # TODO: add render elements if render element + # Store whether we are splitting the render job in an export + render + # TODO: check names of VRay parms + export_job = bool(rop.parm("render_export_mode").eval()) + instance.data["exportJob"] = export_job + export_prefix = None + export_products = [] + if export_job: + export_prefix = evalParmNoFrame(rop, "render_export_filepath", pad_character="0") + beauty_export_product = self.get_render_product_name( + prefix=export_prefix, + suffix=None) + export_products.append(beauty_export_product) + self.log.debug("Found export product: {}".format(beauty_export_product)) + instance.data["ifdFile"] = beauty_export_product + instance.data["exportFiles"] = list(export_products) + beauty_product = self.get_beauty_render_product(default_prefix) render_products.append(beauty_product) files_by_aov = { diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index 3fa427204b..56d8afbe3e 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -446,11 +446,29 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin, self.scene_path = file_path self.log.info("Using {} for render/export.".format(file_path)) - self.job_info = self.get_job_info() + # Check whether we are splitting render job in export + render + # and if so, create a separate task for the render + # TODO: Find a cleaner way so `get_job_info` can take extra args depending + # on host + export_job = instance.data["exportJob"] + self.job_info = self.get_job_info(split_render_job=export_job, export_job=True) self.plugin_info = self.get_plugin_info() self.aux_files = self.get_aux_files() - self.process_submission() + job_id = self.process_submission() + if export_job: + self.log.info("Splitting export and render in two jobs") + self.log.info("Export job id: %s", job_id) + render_job_info = self.get_job_info( + split_render_job=True, dependency_job_ids=[job_id] + ) + render_plugin_info = self.get_plugin_info(split_render_job=True) + payload = self.assemble_payload( + job_info=render_job_info, + plugin_info=render_plugin_info + ) + render_job_id = self.submit(payload) + self.log.info("Render job id: %s", render_job_id) def process_submission(self): """Process data for submission. diff --git a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py index af341ca8e8..82f6876970 100644 --- a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py @@ -6,12 +6,15 @@ import getpass from datetime import datetime import pyblish.api -from openpype.pipeline import legacy_io +from openpype.pipeline import legacy_io, OpenPypePyblishPluginMixin from openpype.tests.lib import is_in_tests from openpype_modules.deadline import abstract_submit_deadline from openpype_modules.deadline.abstract_submit_deadline import DeadlineJobInfo -from openpype.lib import is_running_from_build - +from openpype.lib import ( + is_running_from_build, + BoolDef, + NumberDef +) @attr.s class DeadlinePluginInfo(): @@ -20,8 +23,27 @@ class DeadlinePluginInfo(): Version = attr.ib(default=None) IgnoreInputs = attr.ib(default=True) +@attr.s +class ArnoldRenderDeadlinePluginInfo(): + InputFile = attr.ib(default=None) + Verbose = attr.ib(default=4) -class HoudiniSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): + +@attr.s +class MantraRenderDeadlinePluginInfo(): + SceneFile = attr.ib(default=None) + Version = attr.ib(default=None) + + +@attr.s +class VrayRenderPluginInfo(): + InputFilename = attr.ib(default=None) + + +class HoudiniSubmitDeadline( + abstract_submit_deadline.AbstractSubmitDeadline, + OpenPypePyblishPluginMixin +): """Submit Solaris USD Render ROPs to Deadline. Renders are submitted to a Deadline Web Service as @@ -46,21 +68,117 @@ class HoudiniSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): targets = ["local"] use_published = True - def get_job_info(self): - job_info = DeadlineJobInfo(Plugin="Houdini") + # presets + priority = 50 + chunk_size = 1 + export_priority = 50 + export_chunk_size = 10 + group = "" + export_group = "" + department = "" + limit_groups = {} + env_allowed_keys = [] + env_search_replace_values = {} + + @classmethod + def apply_settings(cls, project_settings, system_settings): + settings = project_settings["deadline"]["publish"]["HoudiniSubmitDeadline"] # noqa + + # Take some defaults from settings + cls.use_published = settings.get( + "use_published", cls.use_published + ) + cls.priority = settings.get( + "priority", cls.priority + ) + cls.export_priority = settings.get( + "export_priority", cls.export_priority + ) + cls.chunk_size = settings.get("chunk_size", cls.chunk_size) + cls.export_chunk_size = settings.get( + "export_chunk_size", cls.export_chunk_size + ) + cls.group = settings.get("group", cls.group) + cls.export_group = settings.get("export_group", cls.export_group) + cls.department = settings.get("department", cls.department) + cls.env_allowed_keys = settings.get("env_allowed_keys", cls.env_allowed_keys) + cls.env_search_replace_values = settings.get( + "env_search_replace_values", cls.env_allowed_keys + ) + + @classmethod + def get_attribute_defs(cls): + return [ + NumberDef( + "priority", + label="Priority", + default=cls.priority, + decimals=0 + ), + NumberDef( + "chunk", + label="Frames Per Task", + default=cls.chunk_size, + decimals=0, + minimum=1, + maximum=1000 + ), + NumberDef( + "export_priority", + label="Export Priority", + default=cls.priority, + decimals=0 + ), + NumberDef( + "export_chunk", + label="Export Frames Per Task", + default=cls.export_chunk_size, + decimals=0, + minimum=1, + maximum=1000 + ), + BoolDef( + "suspend_publish", + default=False, + label="Suspend publish" + ) + ] + + def get_job_info(self, split_render_job=False, export_job=False, dependency_job_ids=None): instance = self._instance context = instance.context + attribute_values = self.get_attr_values_from_data(instance.data) + + if split_render_job and not export_job: + # Convert from family to Deadline plugin name + # i.e., arnold_rop -> Arnold + plugin = instance.data.get("family").replace("_rop", "").capitalize() + else: + plugin = "Houdini" + + job_info = DeadlineJobInfo(Plugin=plugin) + filepath = context.data["currentFile"] filename = os.path.basename(filepath) - job_info.Name = "{} - {}".format(filename, instance.name) job_info.BatchName = filename - job_info.Plugin = "Houdini" + job_info.UserName = context.data.get( "deadlineUser", getpass.getuser()) + if split_render_job and export_job: + job_info.Priority = attribute_values.get( + "export_priority", self.export_priority + ) + else: + job_info.Priority = attribute_values.get( + "priority", self.priority + ) + + job_info.Department = self.department + if is_in_tests(): job_info.BatchName += datetime.now().strftime("%d%m%Y%H%M%S") @@ -72,9 +190,23 @@ class HoudiniSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): ) job_info.Frames = frames + # Make sure we make job frame dependent so render tasks pick up a soon + # as export tasks are done + if split_render_job and not export_job: + job_info.IsFrameDependent = True + job_info.Pool = instance.data.get("primaryPool") job_info.SecondaryPool = instance.data.get("secondaryPool") - job_info.ChunkSize = instance.data.get("chunkSize", 10) + job_info.Group = self.group + if split_render_job and export_job: + job_info.ChunkSize = attribute_values.get( + "export_chunk", self.export_chunk_size + ) + else: + job_info.ChunkSize = attribute_values.get( + "chunk", self.chunk_size + ) + job_info.Comment = context.data.get("comment") keys = [ @@ -98,8 +230,19 @@ class HoudiniSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): if self._instance.context.data.get("deadlinePassMongoUrl"): keys.append("OPENPYPE_MONGO") + # add allowed keys from preset if any + if self.env_allowed_keys: + keys += self.env_allowed_keys + environment = dict({key: os.environ[key] for key in keys if key in os.environ}, **legacy_io.Session) + + # finally search replace in values of any key + if self.env_search_replace_values: + for key, value in environment.items(): + for _k, _v in self.env_search_replace_values.items(): + environment[key] = value.replace(_k, _v) + for key in keys: value = environment.get(key) if value: @@ -114,23 +257,49 @@ class HoudiniSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): job_info.OutputDirectory += dirname.replace("\\", "/") job_info.OutputFilename += fname + # Add dependencies if given + if dependency_job_ids: + job_info.JobDependencies = ",".join(dependency_job_ids) + return job_info - def get_plugin_info(self): + def get_plugin_info(self, split_render_job=False): instance = self._instance context = instance.context - # Output driver to render - driver = hou.node(instance.data["instance_node"]) hou_major_minor = hou.applicationVersionString().rsplit(".", 1)[0] - plugin_info = DeadlinePluginInfo( - SceneFile=context.data["currentFile"], - OutputDriver=driver.path(), - Version=hou_major_minor, - IgnoreInputs=True - ) + # Output driver to render + if split_render_job: + family = instance.data.get("family") + if family == "arnold_rop": + plugin_info = ArnoldRenderDeadlinePluginInfo( + InputFile=instance.data["ifdFile"] + ) + elif family == "mantra_rop": + plugin_info = MantraRenderDeadlinePluginInfo( + SceneFile=instance.data["ifdFile"], + Version=hou_major_minor, + ) + elif family == "vray_rop": + plugin_info = VrayRenderPluginInfo( + InputFilename=instance.data["ifdFile"], + ) + else: + self.log.error( + "Family %s not supported yet to split export and render job", + family + ) + return + else: + driver = hou.node(instance.data["instance_node"]) + plugin_info = DeadlinePluginInfo( + SceneFile=context.data["currentFile"], + OutputDriver=driver.path(), + Version=hou_major_minor, + IgnoreInputs=True + ) return attr.asdict(plugin_info) From fa261c6a3269815575824dfd035e8424195ecbec Mon Sep 17 00:00:00 2001 From: Fabia Serra Arrizabalaga Date: Mon, 7 Aug 2023 18:23:36 +0200 Subject: [PATCH 02/14] Add way to query export parm from Houdini and fix output parm --- openpype/hosts/houdini/api/lib.py | 121 ++++++++++++++++++++++++++---- 1 file changed, 106 insertions(+), 15 deletions(-) diff --git a/openpype/hosts/houdini/api/lib.py b/openpype/hosts/houdini/api/lib.py index b03f8c8fc1..b1673b746d 100644 --- a/openpype/hosts/houdini/api/lib.py +++ b/openpype/hosts/houdini/api/lib.py @@ -104,8 +104,8 @@ def get_id_required_nodes(): return list(nodes) -def get_output_parameter(node): - """Return the render output parameter name of the given node +def get_export_parameter(node): + """Return the export output parameter of the given node Example: root = hou.node("/obj") @@ -120,21 +120,112 @@ def get_output_parameter(node): hou.Parm """ + node_type = node.type().description() - node_type = node.type().name() - if node_type == "geometry": - return node.parm("sopoutput") - elif node_type == "alembic": - return node.parm("filename") - elif node_type == "comp": - return node.parm("copoutput") - elif node_type == "opengl": - return node.parm("picture") - elif node_type == "arnold": - if node.evalParm("ar_ass_export_enable"): - return node.parm("ar_ass_file") - elif node_type == "Redshift_Proxy_Output": + # Ensures the proper Take is selected for each ROP to retrieve the correct ifd + try: + rop_take = hou.takes.findTake(node.parm("take").eval()) + if rop_take is not None: + hou.takes.setCurrentTake(rop_take) + except AttributeError: + # hou object doesn't always have the 'takes' attribute + pass + + if node_type == "Mantra" and node.parm("soho_outputmode").eval(): + return node.parm("soho_diskfile") + elif node_type == "Alfred": + return node.parm("alf_diskfile") + elif (node_type == "RenderMan" or node_type == "RenderMan RIS"): + pre_ris22 = node.parm("rib_outputmode") and node.parm("rib_outputmode").eval() + ris22 = node.parm("diskfile") and node.parm("diskfile").eval() + if pre_ris22 or ris22: + return node.parm("soho_diskfile") + elif node_type == "Redshift" and node.parm("RS_archive_enable").eval(): return node.parm("RS_archive_file") + elif node_type == "Wedge" and node.parm("driver").eval(): + return get_export_parameter(node.node(node.parm("driver").eval())) + elif node_type == "Arnold": + return node.parm("ar_ass_file") + elif node_type == "Alembic" and node.parm("use_sop_path").eval(): + return node.parm("sop_path") + elif node_type == "Shotgun Mantra" and node.parm("soho_outputmode").eval(): + return node.parm("sgtk_soho_diskfile") + elif node_type == "Shotgun Alembic" and node.parm("use_sop_path").eval(): + return node.parm("sop_path") + elif node.type().nameWithCategory() == "Driver/vray_renderer": + return node.parm("render_export_filepath") + + raise TypeError("Node type '%s' not supported" % node_type) + + +def get_output_parameter(node): + """Return the render output parameter of the given node + + Example: + root = hou.node("/obj") + my_alembic_node = root.createNode("alembic") + get_output_parameter(my_alembic_node) + # Result: "output" + + Args: + node(hou.Node): node instance + + Returns: + hou.Parm + + """ + node_type = node.type().description() + category = node.type().category().name() + + # Figure out which type of node is being rendered + if node_type == "Geometry" or node_type == "Filmbox FBX" or (node_type == "ROP Output Driver" and category == "Sop"): + return node.parm("sopoutput") + elif node_type == "Composite": + return node.parm("copoutput") + elif node_type == "Channel": + return node.parm("chopoutput") + elif node_type == "Dynamics" or (node_type == "ROP Output Driver" and category == "Dop"): + return node.parm("dopoutput") + elif node_type == "Alfred": + return node.parm("alf_diskfile") + elif node_type == "RenderMan" or node_type == "RenderMan RIS": + return node.parm("ri_display") + elif node_type == "Redshift": + return node.parm("RS_returnmePrefix") + elif node_type == "Mantra": + return node.parm("vm_picture") + elif node_type == "Wedge": + driver_node = node.node(node.parm("driver").eval()) + if driver_node: + return get_output_parameter(driver_node) + elif node_type == "Arnold": + return node.parm("ar_picture") + elif node_type == "HQueue Simulation": + inner_node = node.node(node.parm("hq_driver").eval()) + if inner_node: + return get_output_parameter(inner_node) + elif node_type == "ROP Alembic Output": + return node.parm("filename") + elif node_type == "Redshift": + return node.parm("RS_returnmePrefix") + elif node_type == "Alembic": + return node.parm("filename") + elif node_type == "Shotgun Mantra": + return node.parm("sgtk_vm_picture") + elif node_type == "Shotgun Alembic": + return node.parm("filename") + elif node_type == "Bake Texture": + return node.parm("vm_uvoutputpicture1") + elif node_type == "OpenGL": + return node.parm("picture") + elif node_type == "Octane": + return node.parm("HO_img_fileName") + elif node_type == "Fetch": + inner_node = node.node(node.parm("source").eval()) + if inner_node: + return get_output_parameter(inner_node) + elif node.type().nameWithCategory() == "Driver/vray_renderer": + return node.parm("SettingsOutput_img_file_path") raise TypeError("Node type '%s' not supported" % node_type) From 2b2ad8f1ed8ff4922fa9216983958e79ea93e26a Mon Sep 17 00:00:00 2001 From: Fabia Serra Arrizabalaga Date: Mon, 7 Aug 2023 18:30:12 +0200 Subject: [PATCH 03/14] Add missing pre create attr for splitting render job to Mantra ROP --- openpype/hosts/houdini/plugins/create/create_mantra_rop.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openpype/hosts/houdini/plugins/create/create_mantra_rop.py b/openpype/hosts/houdini/plugins/create/create_mantra_rop.py index 41311a1e30..0601124ac0 100644 --- a/openpype/hosts/houdini/plugins/create/create_mantra_rop.py +++ b/openpype/hosts/houdini/plugins/create/create_mantra_rop.py @@ -90,6 +90,9 @@ class CreateMantraROP(plugin.HoudiniCreator): BoolDef("farm", label="Submitting to Farm", default=True), + BoolDef("export_job", + label="Split export and render jobs", + default=self.export_job), EnumDef("image_format", image_format_enum, default="exr", From aab913acbc7f33e0c79d2a966a52b22fd99c54fc Mon Sep 17 00:00:00 2001 From: Fabia Serra Arrizabalaga Date: Mon, 7 Aug 2023 18:32:09 +0200 Subject: [PATCH 04/14] Add Houdini deadline submission settings --- .../defaults/project_settings/deadline.json | 15 +++ .../schema_project_deadline.json | 91 +++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/openpype/settings/defaults/project_settings/deadline.json b/openpype/settings/defaults/project_settings/deadline.json index 1b8c8397d7..d088ef5747 100644 --- a/openpype/settings/defaults/project_settings/deadline.json +++ b/openpype/settings/defaults/project_settings/deadline.json @@ -19,6 +19,21 @@ "deadline" ] }, + "HoudiniSubmitDeadline": { + "enabled": true, + "active": true, + "use_published": true, + "priority": 50, + "render_chunk_size": 1, + "group": "none", + "export_priority": 50, + "export_chunk_size": 10, + "export_group": "none", + "department": "", + "env_allowed_keys": [], + "env_search_replace_values": {}, + "limit_groups": {} + }, "MayaSubmitDeadline": { "enabled": true, "optional": false, diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json b/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json index 6d59b5a92b..dfbcd8ef27 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json @@ -89,6 +89,97 @@ } ] }, + { + "type": "dict", + "collapsible": true, + "key": "HoudiniSubmitDeadline", + "label": "Houdini Submit to Deadline", + "checkbox_key": "enabled", + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + }, + { + "type": "boolean", + "key": "active", + "label": "Active" + }, + { + "type": "splitter" + }, + { + "type": "boolean", + "key": "use_published", + "label": "Use Published scene" + }, + { + "type": "number", + "key": "priority", + "label": "Priority" + }, + { + "type": "number", + "key": "render_chunk_size", + "label": "Render Chunk Size" + }, + { + "type": "text", + "key": "group", + "label": "Group" + }, + { + "type": "splitter" + }, + { + "type": "number", + "key": "export_priority", + "label": "Export Priority" + }, + { + "type": "number", + "key": "export_chunk_size", + "label": "Export Chunk Size" + }, + { + "type": "text", + "key": "export_group", + "label": "Export Group" + }, + { + "type": "splitter" + }, + { + "type": "text", + "key": "department", + "label": "Department" + }, + { + "type": "list", + "key": "env_allowed_keys", + "object_type": "text", + "label": "Allowed environment keys" + }, + { + "type": "dict-modifiable", + "key": "env_search_replace_values", + "label": "Search & replace in environment values", + "object_type": { + "type": "text" + } + }, + { + "type": "dict-modifiable", + "key": "limit_groups", + "label": "Limit Groups", + "object_type": { + "type": "list", + "object_type": "text" + } + } + ] + }, { "type": "dict", "collapsible": true, From 33dff766fe6777ff5e801beebe2b366ed5122c20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabi=C3=A0=20Serra=20Arrizabalaga?= Date: Mon, 7 Aug 2023 18:51:53 +0200 Subject: [PATCH 05/14] Shush Hound --- openpype/hosts/houdini/api/lib.py | 12 ++++++++---- .../houdini/plugins/create/create_arnold_rop.py | 9 +++++---- .../houdini/plugins/create/create_mantra_rop.py | 9 +++++---- .../hosts/houdini/plugins/create/create_vray_rop.py | 9 +++++---- .../houdini/plugins/publish/collect_arnold_rop.py | 10 +++++++--- .../houdini/plugins/publish/collect_mantra_rop.py | 10 +++++++--- .../houdini/plugins/publish/collect_vray_rop.py | 8 ++++++-- .../publish/submit_houdini_render_deadline.py | 11 ++++++++--- 8 files changed, 51 insertions(+), 27 deletions(-) diff --git a/openpype/hosts/houdini/api/lib.py b/openpype/hosts/houdini/api/lib.py index b1673b746d..94c4529ae2 100644 --- a/openpype/hosts/houdini/api/lib.py +++ b/openpype/hosts/houdini/api/lib.py @@ -122,7 +122,8 @@ def get_export_parameter(node): """ node_type = node.type().description() - # Ensures the proper Take is selected for each ROP to retrieve the correct ifd + # Ensures the proper Take is selected for each ROP to retrieve the correct + # ifd try: rop_take = hou.takes.findTake(node.parm("take").eval()) if rop_take is not None: @@ -136,7 +137,8 @@ def get_export_parameter(node): elif node_type == "Alfred": return node.parm("alf_diskfile") elif (node_type == "RenderMan" or node_type == "RenderMan RIS"): - pre_ris22 = node.parm("rib_outputmode") and node.parm("rib_outputmode").eval() + pre_ris22 = node.parm("rib_outputmode") and \ + node.parm("rib_outputmode").eval() ris22 = node.parm("diskfile") and node.parm("diskfile").eval() if pre_ris22 or ris22: return node.parm("soho_diskfile") @@ -178,13 +180,15 @@ def get_output_parameter(node): category = node.type().category().name() # Figure out which type of node is being rendered - if node_type == "Geometry" or node_type == "Filmbox FBX" or (node_type == "ROP Output Driver" and category == "Sop"): + if node_type == "Geometry" or node_type == "Filmbox FBX" or \ + (node_type == "ROP Output Driver" and category == "Sop"): return node.parm("sopoutput") elif node_type == "Composite": return node.parm("copoutput") elif node_type == "Channel": return node.parm("chopoutput") - elif node_type == "Dynamics" or (node_type == "ROP Output Driver" and category == "Dop"): + elif node_type == "Dynamics" or \ + (node_type == "ROP Output Driver" and category == "Dop"): return node.parm("dopoutput") elif node_type == "Alfred": return node.parm("alf_diskfile") diff --git a/openpype/hosts/houdini/plugins/create/create_arnold_rop.py b/openpype/hosts/houdini/plugins/create/create_arnold_rop.py index d19795d3a9..b2f04b64ec 100644 --- a/openpype/hosts/houdini/plugins/create/create_arnold_rop.py +++ b/openpype/hosts/houdini/plugins/create/create_arnold_rop.py @@ -53,10 +53,11 @@ class CreateArnoldRop(plugin.HoudiniCreator): } if pre_create_data.get("export_job"): - ass_filepath = "{export_dir}{subset_name}/{subset_name}.$F4.ass".format( - export_dir=hou.text.expandString("$HIP/pyblish/ass/"), - subset_name=subset_name, - ) + ass_filepath = \ + "{export_dir}{subset_name}/{subset_name}.$F4.ass".format( + export_dir=hou.text.expandString("$HIP/pyblish/ass/"), + subset_name=subset_name, + ) parms["ar_ass_export_enable"] = 1 parms["ar_ass_file"] = ass_filepath diff --git a/openpype/hosts/houdini/plugins/create/create_mantra_rop.py b/openpype/hosts/houdini/plugins/create/create_mantra_rop.py index 0601124ac0..62074fe432 100644 --- a/openpype/hosts/houdini/plugins/create/create_mantra_rop.py +++ b/openpype/hosts/houdini/plugins/create/create_mantra_rop.py @@ -49,10 +49,11 @@ class CreateMantraROP(plugin.HoudiniCreator): } if pre_create_data.get("export_job"): - ifd_filepath = "{export_dir}{subset_name}/{subset_name}.$F4.ifd".format( - export_dir=hou.text.expandString("$HIP/pyblish/ifd/"), - subset_name=subset_name, - ) + ifd_filepath = \ + "{export_dir}{subset_name}/{subset_name}.$F4.ifd".format( + export_dir=hou.text.expandString("$HIP/pyblish/ifd/"), + subset_name=subset_name, + ) parms["soho_outputmode"] = 1 parms["soho_diskfile"] = ifd_filepath diff --git a/openpype/hosts/houdini/plugins/create/create_vray_rop.py b/openpype/hosts/houdini/plugins/create/create_vray_rop.py index 29fab0c60c..de3cfc3858 100644 --- a/openpype/hosts/houdini/plugins/create/create_vray_rop.py +++ b/openpype/hosts/houdini/plugins/create/create_vray_rop.py @@ -58,10 +58,11 @@ class CreateVrayROP(plugin.HoudiniCreator): } if pre_create_data.get("export_job"): - scene_filepath = "{export_dir}{subset_name}/{subset_name}.$F4.vrscene".format( - export_dir=hou.text.expandString("$HIP/pyblish/vrscene/"), - subset_name=subset_name, - ) + scene_filepath = \ + "{export_dir}{subset_name}/{subset_name}.$F4.vrscene".format( + export_dir=hou.text.expandString("$HIP/pyblish/vrscene/"), + subset_name=subset_name, + ) # TODO: don't have VRay to check the names of these parms["render_export_mode"] = 1 parms["render_export_filepath"] = scene_filepath diff --git a/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py b/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py index 063dd728e1..4af1d80a2d 100644 --- a/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py +++ b/openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py @@ -38,18 +38,22 @@ class CollectArnoldROPRenderProducts(pyblish.api.InstancePlugin): default_prefix = evalParmNoFrame(rop, "ar_picture") render_products = [] - # Store whether we are splitting the render job in an export + render + # Store whether we are splitting the render job (export + render) export_job = bool(rop.parm("ar_ass_export_enable").eval()) instance.data["exportJob"] = export_job export_prefix = None export_products = [] if export_job: - export_prefix = evalParmNoFrame(rop, "ar_ass_file", pad_character="0") + export_prefix = evalParmNoFrame( + rop, "ar_ass_file", pad_character="0" + ) beauty_export_product = self.get_render_product_name( prefix=export_prefix, suffix=None) export_products.append(beauty_export_product) - self.log.debug("Found export product: {}".format(beauty_export_product)) + self.log.debug( + "Found export product: {}".format(beauty_export_product) + ) instance.data["ifdFile"] = beauty_export_product instance.data["exportFiles"] = list(export_products) diff --git a/openpype/hosts/houdini/plugins/publish/collect_mantra_rop.py b/openpype/hosts/houdini/plugins/publish/collect_mantra_rop.py index b235e2c110..7385a87679 100644 --- a/openpype/hosts/houdini/plugins/publish/collect_mantra_rop.py +++ b/openpype/hosts/houdini/plugins/publish/collect_mantra_rop.py @@ -42,18 +42,22 @@ class CollectMantraROPRenderProducts(pyblish.api.InstancePlugin): default_prefix = evalParmNoFrame(rop, "vm_picture") render_products = [] - # Store whether we are splitting the render job in an export + render + # Store whether we are splitting the render job (export + render) export_job = bool(rop.parm("soho_outputmode").eval()) instance.data["exportJob"] = export_job export_prefix = None export_products = [] if export_job: - export_prefix = evalParmNoFrame(rop, "soho_diskfile", pad_character="0") + export_prefix = evalParmNoFrame( + rop, "soho_diskfile", pad_character="0" + ) beauty_export_product = self.get_render_product_name( prefix=export_prefix, suffix=None) export_products.append(beauty_export_product) - self.log.debug("Found export product: {}".format(beauty_export_product)) + self.log.debug( + "Found export product: {}".format(beauty_export_product) + ) instance.data["ifdFile"] = beauty_export_product instance.data["exportFiles"] = list(export_products) diff --git a/openpype/hosts/houdini/plugins/publish/collect_vray_rop.py b/openpype/hosts/houdini/plugins/publish/collect_vray_rop.py index 89a7c8bf72..4259f19eb2 100644 --- a/openpype/hosts/houdini/plugins/publish/collect_vray_rop.py +++ b/openpype/hosts/houdini/plugins/publish/collect_vray_rop.py @@ -50,12 +50,16 @@ class CollectVrayROPRenderProducts(pyblish.api.InstancePlugin): export_prefix = None export_products = [] if export_job: - export_prefix = evalParmNoFrame(rop, "render_export_filepath", pad_character="0") + export_prefix = evalParmNoFrame( + rop, "render_export_filepath", pad_character="0" + ) beauty_export_product = self.get_render_product_name( prefix=export_prefix, suffix=None) export_products.append(beauty_export_product) - self.log.debug("Found export product: {}".format(beauty_export_product)) + self.log.debug( + "Found export product: {}".format(beauty_export_product) + ) instance.data["ifdFile"] = beauty_export_product instance.data["exportFiles"] = list(export_products) diff --git a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py index 82f6876970..d7d5ed906b 100644 --- a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py @@ -144,7 +144,12 @@ class HoudiniSubmitDeadline( ) ] - def get_job_info(self, split_render_job=False, export_job=False, dependency_job_ids=None): + def get_job_info( + self, + split_render_job=False, + export_job=False, + dependency_job_ids=None + ): instance = self._instance context = instance.context @@ -154,7 +159,7 @@ class HoudiniSubmitDeadline( if split_render_job and not export_job: # Convert from family to Deadline plugin name # i.e., arnold_rop -> Arnold - plugin = instance.data.get("family").replace("_rop", "").capitalize() + plugin = instance.data["family"].replace("_rop", "").capitalize() else: plugin = "Houdini" @@ -288,7 +293,7 @@ class HoudiniSubmitDeadline( ) else: self.log.error( - "Family %s not supported yet to split export and render job", + "Family '%s' not supported yet to split render job", family ) return From 2f62dd74ab9aed69f40d21b0ed3cc163c15d7bfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabi=C3=A0=20Serra=20Arrizabalaga?= Date: Mon, 7 Aug 2023 18:59:16 +0200 Subject: [PATCH 06/14] Shush Hound --- .../modules/deadline/abstract_submit_deadline.py | 8 +++++--- .../publish/submit_houdini_render_deadline.py | 15 +++++++++------ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index 56d8afbe3e..ae251f39b9 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -448,10 +448,12 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin, # Check whether we are splitting render job in export + render # and if so, create a separate task for the render - # TODO: Find a cleaner way so `get_job_info` can take extra args depending - # on host + # TODO: Find a cleaner way so `get_job_info` can take extra args + # depending on host export_job = instance.data["exportJob"] - self.job_info = self.get_job_info(split_render_job=export_job, export_job=True) + self.job_info = self.get_job_info( + split_render_job=export_job, export_job=True + ) self.plugin_info = self.get_plugin_info() self.aux_files = self.get_aux_files() diff --git a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py index d7d5ed906b..e5cd6467f7 100644 --- a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py @@ -23,6 +23,7 @@ class DeadlinePluginInfo(): Version = attr.ib(default=None) IgnoreInputs = attr.ib(default=True) + @attr.s class ArnoldRenderDeadlinePluginInfo(): InputFile = attr.ib(default=None) @@ -101,7 +102,9 @@ class HoudiniSubmitDeadline( cls.group = settings.get("group", cls.group) cls.export_group = settings.get("export_group", cls.export_group) cls.department = settings.get("department", cls.department) - cls.env_allowed_keys = settings.get("env_allowed_keys", cls.env_allowed_keys) + cls.env_allowed_keys = settings.get( + "env_allowed_keys", cls.env_allowed_keys + ) cls.env_search_replace_values = settings.get( "env_search_replace_values", cls.env_allowed_keys ) @@ -145,11 +148,11 @@ class HoudiniSubmitDeadline( ] def get_job_info( - self, - split_render_job=False, - export_job=False, - dependency_job_ids=None - ): + self, + split_render_job=False, + export_job=False, + dependency_job_ids=None + ): instance = self._instance context = instance.context From a0987388c9a3202e34a76f5057528f5f9f7d63ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabi=C3=A0=20Serra=20Arrizabalaga?= Date: Mon, 7 Aug 2023 22:18:13 +0200 Subject: [PATCH 07/14] Revert "Add Houdini deadline submission settings" This reverts commit aab913acbc7f33e0c79d2a966a52b22fd99c54fc. --- .../defaults/project_settings/deadline.json | 15 --- .../schema_project_deadline.json | 91 ------------------- 2 files changed, 106 deletions(-) diff --git a/openpype/settings/defaults/project_settings/deadline.json b/openpype/settings/defaults/project_settings/deadline.json index d088ef5747..1b8c8397d7 100644 --- a/openpype/settings/defaults/project_settings/deadline.json +++ b/openpype/settings/defaults/project_settings/deadline.json @@ -19,21 +19,6 @@ "deadline" ] }, - "HoudiniSubmitDeadline": { - "enabled": true, - "active": true, - "use_published": true, - "priority": 50, - "render_chunk_size": 1, - "group": "none", - "export_priority": 50, - "export_chunk_size": 10, - "export_group": "none", - "department": "", - "env_allowed_keys": [], - "env_search_replace_values": {}, - "limit_groups": {} - }, "MayaSubmitDeadline": { "enabled": true, "optional": false, diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json b/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json index dfbcd8ef27..6d59b5a92b 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json @@ -89,97 +89,6 @@ } ] }, - { - "type": "dict", - "collapsible": true, - "key": "HoudiniSubmitDeadline", - "label": "Houdini Submit to Deadline", - "checkbox_key": "enabled", - "children": [ - { - "type": "boolean", - "key": "enabled", - "label": "Enabled" - }, - { - "type": "boolean", - "key": "active", - "label": "Active" - }, - { - "type": "splitter" - }, - { - "type": "boolean", - "key": "use_published", - "label": "Use Published scene" - }, - { - "type": "number", - "key": "priority", - "label": "Priority" - }, - { - "type": "number", - "key": "render_chunk_size", - "label": "Render Chunk Size" - }, - { - "type": "text", - "key": "group", - "label": "Group" - }, - { - "type": "splitter" - }, - { - "type": "number", - "key": "export_priority", - "label": "Export Priority" - }, - { - "type": "number", - "key": "export_chunk_size", - "label": "Export Chunk Size" - }, - { - "type": "text", - "key": "export_group", - "label": "Export Group" - }, - { - "type": "splitter" - }, - { - "type": "text", - "key": "department", - "label": "Department" - }, - { - "type": "list", - "key": "env_allowed_keys", - "object_type": "text", - "label": "Allowed environment keys" - }, - { - "type": "dict-modifiable", - "key": "env_search_replace_values", - "label": "Search & replace in environment values", - "object_type": { - "type": "text" - } - }, - { - "type": "dict-modifiable", - "key": "limit_groups", - "label": "Limit Groups", - "object_type": { - "type": "list", - "object_type": "text" - } - } - ] - }, { "type": "dict", "collapsible": true, From d309df3b49b2d20cdac28a4150f1546f83ee5a8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabi=C3=A0=20Serra=20Arrizabalaga?= Date: Mon, 7 Aug 2023 22:22:31 +0200 Subject: [PATCH 08/14] Remove Houdini Deadline settings as it's not relevant to feature --- .../publish/submit_houdini_render_deadline.py | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py index e5cd6467f7..fdfc66b2c0 100644 --- a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py @@ -81,34 +81,6 @@ class HoudiniSubmitDeadline( env_allowed_keys = [] env_search_replace_values = {} - @classmethod - def apply_settings(cls, project_settings, system_settings): - settings = project_settings["deadline"]["publish"]["HoudiniSubmitDeadline"] # noqa - - # Take some defaults from settings - cls.use_published = settings.get( - "use_published", cls.use_published - ) - cls.priority = settings.get( - "priority", cls.priority - ) - cls.export_priority = settings.get( - "export_priority", cls.export_priority - ) - cls.chunk_size = settings.get("chunk_size", cls.chunk_size) - cls.export_chunk_size = settings.get( - "export_chunk_size", cls.export_chunk_size - ) - cls.group = settings.get("group", cls.group) - cls.export_group = settings.get("export_group", cls.export_group) - cls.department = settings.get("department", cls.department) - cls.env_allowed_keys = settings.get( - "env_allowed_keys", cls.env_allowed_keys - ) - cls.env_search_replace_values = settings.get( - "env_search_replace_values", cls.env_allowed_keys - ) - @classmethod def get_attribute_defs(cls): return [ From 20583173c663e84b84e0512692cf1495f51fb1ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabi=C3=A0=20Serra=20Arrizabalaga?= Date: Mon, 7 Aug 2023 22:26:02 +0200 Subject: [PATCH 09/14] Remove some other noise from PR feature --- .../publish/submit_houdini_render_deadline.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py index fdfc66b2c0..d6c658944a 100644 --- a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py @@ -76,10 +76,6 @@ class HoudiniSubmitDeadline( export_chunk_size = 10 group = "" export_group = "" - department = "" - limit_groups = {} - env_allowed_keys = [] - env_search_replace_values = {} @classmethod def get_attribute_defs(cls): @@ -157,8 +153,6 @@ class HoudiniSubmitDeadline( "priority", self.priority ) - job_info.Department = self.department - if is_in_tests(): job_info.BatchName += datetime.now().strftime("%d%m%Y%H%M%S") @@ -210,18 +204,9 @@ class HoudiniSubmitDeadline( if self._instance.context.data.get("deadlinePassMongoUrl"): keys.append("OPENPYPE_MONGO") - # add allowed keys from preset if any - if self.env_allowed_keys: - keys += self.env_allowed_keys - environment = dict({key: os.environ[key] for key in keys if key in os.environ}, **legacy_io.Session) - # finally search replace in values of any key - if self.env_search_replace_values: - for key, value in environment.items(): - for _k, _v in self.env_search_replace_values.items(): - environment[key] = value.replace(_k, _v) for key in keys: value = environment.get(key) From f58c4de0d4a284867eb0fb43b3d235abff9f26ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabi=C3=A0=20Serra=20Arrizabalaga?= Date: Mon, 7 Aug 2023 22:44:45 +0200 Subject: [PATCH 10/14] Move some of the logic to Houdini submitter so abstract functions are a bit more generic --- .../deadline/abstract_submit_deadline.py | 19 +++++------- .../publish/submit_houdini_render_deadline.py | 30 +++++++++++-------- 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index ae251f39b9..dddb4400c1 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -446,25 +446,20 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin, self.scene_path = file_path self.log.info("Using {} for render/export.".format(file_path)) - # Check whether we are splitting render job in export + render - # and if so, create a separate task for the render - # TODO: Find a cleaner way so `get_job_info` can take extra args - # depending on host - export_job = instance.data["exportJob"] - self.job_info = self.get_job_info( - split_render_job=export_job, export_job=True - ) + self.job_info = self.get_job_info() self.plugin_info = self.get_plugin_info() self.aux_files = self.get_aux_files() job_id = self.process_submission() + self.log.info("Submitted job to Deadline: {}.".format(job_id)) + + # TODO: Find a way that's more generic and not render type specific + export_job = instance.data["exportJob"] if export_job: self.log.info("Splitting export and render in two jobs") self.log.info("Export job id: %s", job_id) - render_job_info = self.get_job_info( - split_render_job=True, dependency_job_ids=[job_id] - ) - render_plugin_info = self.get_plugin_info(split_render_job=True) + render_job_info = self.get_job_info(dependency_job_ids=[job_id]) + render_plugin_info = self.get_plugin_info(job_type="render") payload = self.assemble_payload( job_info=render_job_info, plugin_info=render_plugin_info diff --git a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py index d6c658944a..83f6e81485 100644 --- a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py @@ -115,19 +115,24 @@ class HoudiniSubmitDeadline( ) ] - def get_job_info( - self, - split_render_job=False, - export_job=False, - dependency_job_ids=None - ): + def get_job_info(self, dependency_job_ids=None): instance = self._instance context = instance.context attribute_values = self.get_attr_values_from_data(instance.data) - if split_render_job and not export_job: + # Whether Deadline render submission is being split in two + # (extract + render) + split_render_job = instance.data["exportJob"] + + # If there's some dependency job ids we can assume this is a render job + # and not an export job + is_export_job = True + if dependency_job_ids: + is_export_job = False + + if split_render_job and not is_export_job: # Convert from family to Deadline plugin name # i.e., arnold_rop -> Arnold plugin = instance.data["family"].replace("_rop", "").capitalize() @@ -144,7 +149,7 @@ class HoudiniSubmitDeadline( job_info.UserName = context.data.get( "deadlineUser", getpass.getuser()) - if split_render_job and export_job: + if split_render_job and is_export_job: job_info.Priority = attribute_values.get( "export_priority", self.export_priority ) @@ -166,13 +171,13 @@ class HoudiniSubmitDeadline( # Make sure we make job frame dependent so render tasks pick up a soon # as export tasks are done - if split_render_job and not export_job: + if split_render_job and not is_export_job: job_info.IsFrameDependent = True job_info.Pool = instance.data.get("primaryPool") job_info.SecondaryPool = instance.data.get("secondaryPool") job_info.Group = self.group - if split_render_job and export_job: + if split_render_job and is_export_job: job_info.ChunkSize = attribute_values.get( "export_chunk", self.export_chunk_size ) @@ -207,7 +212,6 @@ class HoudiniSubmitDeadline( environment = dict({key: os.environ[key] for key in keys if key in os.environ}, **legacy_io.Session) - for key in keys: value = environment.get(key) if value: @@ -228,7 +232,7 @@ class HoudiniSubmitDeadline( return job_info - def get_plugin_info(self, split_render_job=False): + def get_plugin_info(self, job_type=None): instance = self._instance context = instance.context @@ -236,7 +240,7 @@ class HoudiniSubmitDeadline( hou_major_minor = hou.applicationVersionString().rsplit(".", 1)[0] # Output driver to render - if split_render_job: + if job_type == "render": family = instance.data.get("family") if family == "arnold_rop": plugin_info = ArnoldRenderDeadlinePluginInfo( From 12b2a74ea963a72737f9409f5bfd04acabb53f79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabi=C3=A0=20Serra=20Arrizabalaga?= Date: Mon, 7 Aug 2023 22:52:01 +0200 Subject: [PATCH 11/14] Fix docstring --- .../deadline/plugins/publish/submit_houdini_render_deadline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py index 83f6e81485..8434ddacd3 100644 --- a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py @@ -45,7 +45,7 @@ class HoudiniSubmitDeadline( abstract_submit_deadline.AbstractSubmitDeadline, OpenPypePyblishPluginMixin ): - """Submit Solaris USD Render ROPs to Deadline. + """Submit Render ROPs to Deadline. Renders are submitted to a Deadline Web Service as supplied via the environment variable AVALON_DEADLINE. From ad11a4d3d3c7ec8be0c605cf4da18deea3ab102e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabi=C3=A0=20Serra=20Arrizabalaga?= Date: Tue, 8 Aug 2023 16:08:45 +0200 Subject: [PATCH 12/14] Fix VRay submission parms --- openpype/hosts/houdini/plugins/create/create_vray_rop.py | 3 +-- openpype/hosts/houdini/plugins/publish/collect_vray_rop.py | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/openpype/hosts/houdini/plugins/create/create_vray_rop.py b/openpype/hosts/houdini/plugins/create/create_vray_rop.py index de3cfc3858..222088523a 100644 --- a/openpype/hosts/houdini/plugins/create/create_vray_rop.py +++ b/openpype/hosts/houdini/plugins/create/create_vray_rop.py @@ -63,8 +63,7 @@ class CreateVrayROP(plugin.HoudiniCreator): export_dir=hou.text.expandString("$HIP/pyblish/vrscene/"), subset_name=subset_name, ) - # TODO: don't have VRay to check the names of these - parms["render_export_mode"] = 1 + parms["render_export_mode"] = "1" parms["render_export_filepath"] = scene_filepath if self.selected_nodes: diff --git a/openpype/hosts/houdini/plugins/publish/collect_vray_rop.py b/openpype/hosts/houdini/plugins/publish/collect_vray_rop.py index 4259f19eb2..d7d53ff566 100644 --- a/openpype/hosts/houdini/plugins/publish/collect_vray_rop.py +++ b/openpype/hosts/houdini/plugins/publish/collect_vray_rop.py @@ -44,7 +44,6 @@ class CollectVrayROPRenderProducts(pyblish.api.InstancePlugin): # TODO: add render elements if render element # Store whether we are splitting the render job in an export + render - # TODO: check names of VRay parms export_job = bool(rop.parm("render_export_mode").eval()) instance.data["exportJob"] = export_job export_prefix = None @@ -63,7 +62,7 @@ class CollectVrayROPRenderProducts(pyblish.api.InstancePlugin): instance.data["ifdFile"] = beauty_export_product instance.data["exportFiles"] = list(export_products) - beauty_product = self.get_beauty_render_product(default_prefix) + beauty_product = self.get_render_product_name(default_prefix) render_products.append(beauty_product) files_by_aov = { "RGB Color": self.generate_expected_files(instance, @@ -97,7 +96,7 @@ class CollectVrayROPRenderProducts(pyblish.api.InstancePlugin): instance.data["colorspaceDisplay"] = colorspace_data["display"] instance.data["colorspaceView"] = colorspace_data["view"] - def get_beauty_render_product(self, prefix, suffix=""): + def get_render_product_name(self, prefix, suffix=""): """Return the beauty output filename if render element enabled """ aov_parm = ".{}".format(suffix) From cb7274bf1a79d5c5d315b5ae08e7f6042c598539 Mon Sep 17 00:00:00 2001 From: Fabia Serra Arrizabalaga Date: Wed, 9 Aug 2023 15:19:24 +0200 Subject: [PATCH 13/14] Fix render_export_mode for VRay so it only exports --- openpype/hosts/houdini/plugins/create/create_vray_rop.py | 4 +++- openpype/hosts/houdini/plugins/publish/collect_vray_rop.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/houdini/plugins/create/create_vray_rop.py b/openpype/hosts/houdini/plugins/create/create_vray_rop.py index 222088523a..cd5ef2013d 100644 --- a/openpype/hosts/houdini/plugins/create/create_vray_rop.py +++ b/openpype/hosts/houdini/plugins/create/create_vray_rop.py @@ -63,7 +63,9 @@ class CreateVrayROP(plugin.HoudiniCreator): export_dir=hou.text.expandString("$HIP/pyblish/vrscene/"), subset_name=subset_name, ) - parms["render_export_mode"] = "1" + # Setting render_export_mode to "2" because that's for + # "Export only" ("1" is for "Export & Render") + parms["render_export_mode"] = "2" parms["render_export_filepath"] = scene_filepath if self.selected_nodes: diff --git a/openpype/hosts/houdini/plugins/publish/collect_vray_rop.py b/openpype/hosts/houdini/plugins/publish/collect_vray_rop.py index d7d53ff566..184c644fd6 100644 --- a/openpype/hosts/houdini/plugins/publish/collect_vray_rop.py +++ b/openpype/hosts/houdini/plugins/publish/collect_vray_rop.py @@ -44,7 +44,7 @@ class CollectVrayROPRenderProducts(pyblish.api.InstancePlugin): # TODO: add render elements if render element # Store whether we are splitting the render job in an export + render - export_job = bool(rop.parm("render_export_mode").eval()) + export_job = rop.parm("render_export_mode").eval() == "2" instance.data["exportJob"] = export_job export_prefix = None export_products = [] From bfe8f88b241e4a71f3a9aabbfda2d87087f26e38 Mon Sep 17 00:00:00 2001 From: Fabia Serra Arrizabalaga Date: Thu, 10 Aug 2023 13:43:27 +0200 Subject: [PATCH 14/14] Add SeparateFilesPerFrame attribute to Vray plugin --- .../deadline/plugins/publish/submit_houdini_render_deadline.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py index 8434ddacd3..cc982cd2eb 100644 --- a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py @@ -39,6 +39,7 @@ class MantraRenderDeadlinePluginInfo(): @attr.s class VrayRenderPluginInfo(): InputFilename = attr.ib(default=None) + SeparateFilesPerFrame = attr.ib(default=True) class HoudiniSubmitDeadline(