diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 15025e47f2..062732c059 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -422,6 +422,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): assembly_job_info.Priority = instance.data.get( "tile_priority", self.tile_priority ) + assembly_job_info.TileJob = False pool = instance.context.data["project_settings"]["deadline"] pool = pool["publish"]["ProcessSubmittedJobOnFarm"]["deadline_pool"] @@ -450,15 +451,14 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): frame_assembly_job_info.ExtraInfo[0] = file_hash frame_assembly_job_info.ExtraInfo[1] = file frame_assembly_job_info.JobDependencies = tile_job_id + frame_assembly_job_info.Frames = frame # write assembly job config files - now = datetime.now() - config_file = os.path.join( output_dir, "{}_config_{}.txt".format( os.path.splitext(file)[0], - now.strftime("%Y_%m_%d_%H_%M_%S") + datetime.now().strftime("%Y_%m_%d_%H_%M_%S") ) ) try: @@ -469,6 +469,8 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): self.log.warning("Path is unreachable: " "`{}`".format(output_dir)) + assembly_plugin_info["ConfigFile"] = config_file + with open(config_file, "w") as cf: print("TileCount={}".format(tiles_count), file=cf) print("ImageFileName={}".format(file), file=cf) @@ -477,25 +479,30 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): print("ImageHeight={}".format( instance.data.get("resolutionHeight")), file=cf) + with open(config_file, "a") as cf: + # Need to reverse the order of the y tiles, because image + # coordinates are calculated from bottom left corner. tiles = _format_tiles( file, 0, instance.data.get("tilesX"), instance.data.get("tilesY"), instance.data.get("resolutionWidth"), instance.data.get("resolutionHeight"), - payload_plugin_info["OutputFilePrefix"] + payload_plugin_info["OutputFilePrefix"], + reversed_y=True )[1] for k, v in sorted(tiles.items()): print("{}={}".format(k, v), file=cf) - payload = self.assemble_payload( - job_info=frame_assembly_job_info, - plugin_info=assembly_plugin_info.copy(), - # todo: aux file transfers don't work with deadline webservice - # add config file as job auxFile - # aux_files=[config_file] + assembly_payloads.append( + self.assemble_payload( + job_info=frame_assembly_job_info, + plugin_info=assembly_plugin_info.copy(), + # This would fail if the client machine and webserice are + # using different storage paths. + aux_files=[config_file] + ) ) - assembly_payloads.append(payload) # Submit assembly jobs assembly_job_ids = [] @@ -505,6 +512,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): "submitting assembly job {} of {}".format(i + 1, num_assemblies) ) + self.log.info(payload) assembly_job_id = self.submit(payload) assembly_job_ids.append(assembly_job_id) @@ -764,8 +772,15 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): def _format_tiles( - filename, index, tiles_x, tiles_y, - width, height, prefix): + filename, + index, + tiles_x, + tiles_y, + width, + height, + prefix, + reversed_y=False +): """Generate tile entries for Deadline tile job. Returns two dictionaries - one that can be directly used in Deadline @@ -802,6 +817,7 @@ def _format_tiles( width (int): Width resolution of final image. height (int): Height resolution of final image. prefix (str): Image prefix. + reversed_y (bool): Reverses the order of the y tiles. Returns: (dict, dict): Tuple of two dictionaries - first can be used to @@ -824,12 +840,16 @@ def _format_tiles( cfg["TilesCropped"] = "False" tile = 0 + range_y = range(1, tiles_y + 1) + reversed_y_range = list(reversed(range_y)) for tile_x in range(1, tiles_x + 1): - for tile_y in reversed(range(1, tiles_y + 1)): + for i, tile_y in enumerate(range_y): + tile_y_index = tile_y + if reversed_y: + tile_y_index = reversed_y_range[i] + tile_prefix = "_tile_{}x{}_{}x{}_".format( - tile_x, tile_y, - tiles_x, - tiles_y + tile_x, tile_y_index, tiles_x, tiles_y ) new_filename = "{}/{}{}".format( @@ -844,11 +864,14 @@ def _format_tiles( right = (tile_x * w_space) - 1 # Job info - out["JobInfo"]["OutputFilename{}Tile{}".format(index, tile)] = new_filename # noqa: E501 + key = "OutputFilename{}".format(index) + out["JobInfo"][key] = new_filename # Plugin Info - out["PluginInfo"]["RegionPrefix{}".format(str(tile))] = \ - "/{}".format(tile_prefix).join(prefix.rsplit("/", 1)) + key = "RegionPrefix{}".format(str(tile)) + out["PluginInfo"][key] = "/{}".format( + tile_prefix + ).join(prefix.rsplit("/", 1)) out["PluginInfo"]["RegionTop{}".format(tile)] = top out["PluginInfo"]["RegionBottom{}".format(tile)] = bottom out["PluginInfo"]["RegionLeft{}".format(tile)] = left diff --git a/openpype/modules/deadline/plugins/publish/validate_expected_and_rendered_files.py b/openpype/modules/deadline/plugins/publish/validate_expected_and_rendered_files.py index f34f71d213..ff4be677e7 100644 --- a/openpype/modules/deadline/plugins/publish/validate_expected_and_rendered_files.py +++ b/openpype/modules/deadline/plugins/publish/validate_expected_and_rendered_files.py @@ -68,8 +68,15 @@ class ValidateExpectedFiles(pyblish.api.InstancePlugin): # files to be in the folder that we might not want to use. missing = expected_files - existing_files if missing: - raise RuntimeError("Missing expected files: {}".format( - sorted(missing))) + raise RuntimeError( + "Missing expected files: {}\n" + "Expected files: {}\n" + "Existing files: {}".format( + sorted(missing), + sorted(expected_files), + sorted(existing_files) + ) + ) def _get_frame_list(self, original_job_id): """Returns list of frame ranges from all render job. diff --git a/openpype/modules/deadline/repository/custom/plugins/OpenPypeTileAssembler/OpenPypeTileAssembler.py b/openpype/modules/deadline/repository/custom/plugins/OpenPypeTileAssembler/OpenPypeTileAssembler.py index 861f16518c..b51daffbc8 100644 --- a/openpype/modules/deadline/repository/custom/plugins/OpenPypeTileAssembler/OpenPypeTileAssembler.py +++ b/openpype/modules/deadline/repository/custom/plugins/OpenPypeTileAssembler/OpenPypeTileAssembler.py @@ -16,6 +16,10 @@ from Deadline.Scripting import ( FileUtils, RepositoryUtils, SystemUtils) +version_major = 1 +version_minor = 0 +version_patch = 0 +version_string = "{}.{}.{}".format(version_major, version_minor, version_patch) STRING_TAGS = { "format" } @@ -264,6 +268,7 @@ class OpenPypeTileAssembler(DeadlinePlugin): def initialize_process(self): """Initialization.""" + self.LogInfo("Plugin version: {}".format(version_string)) self.SingleFramesOnly = True self.StdoutHandling = True self.renderer = self.GetPluginInfoEntryWithDefault( @@ -320,12 +325,7 @@ class OpenPypeTileAssembler(DeadlinePlugin): output_file = data["ImageFileName"] output_file = RepositoryUtils.CheckPathMapping(output_file) output_file = self.process_path(output_file) - """ - _, ext = os.path.splitext(output_file) - if "exr" not in ext: - self.FailRender( - "[{}] Only EXR format is supported for now.".format(ext)) - """ + tile_info = [] for tile in range(int(data["TileCount"])): tile_info.append({ @@ -336,11 +336,6 @@ class OpenPypeTileAssembler(DeadlinePlugin): "width": int(data["Tile{}Width".format(tile)]) }) - # FFMpeg doesn't support tile coordinates at the moment. - # arguments = self.tile_completer_ffmpeg_args( - # int(data["ImageWidth"]), int(data["ImageHeight"]), - # tile_info, output_file) - arguments = self.tile_oiio_args( int(data["ImageWidth"]), int(data["ImageHeight"]), tile_info, output_file) @@ -362,20 +357,20 @@ class OpenPypeTileAssembler(DeadlinePlugin): def pre_render_tasks(self): """Load config file and do remapping.""" self.LogInfo("OpenPype Tile Assembler starting...") - scene_filename = self.GetDataFilename() + config_file = self.GetPluginInfoEntry("ConfigFile") temp_scene_directory = self.CreateTempDirectory( "thread" + str(self.GetThreadNumber())) - temp_scene_filename = Path.GetFileName(scene_filename) + temp_scene_filename = Path.GetFileName(config_file) self.config_file = Path.Combine( temp_scene_directory, temp_scene_filename) if SystemUtils.IsRunningOnWindows(): RepositoryUtils.CheckPathMappingInFileAndReplaceSeparator( - scene_filename, self.config_file, "/", "\\") + config_file, self.config_file, "/", "\\") else: RepositoryUtils.CheckPathMappingInFileAndReplaceSeparator( - scene_filename, self.config_file, "\\", "/") + config_file, self.config_file, "\\", "/") os.chmod(self.config_file, os.stat(self.config_file).st_mode) def post_render_tasks(self): @@ -459,75 +454,3 @@ class OpenPypeTileAssembler(DeadlinePlugin): args.append(output_path) return args - - def tile_completer_ffmpeg_args( - self, output_width, output_height, tiles_info, output_path): - """Generate ffmpeg arguments for tile assembly. - - Expected inputs are tiled images. - - Args: - output_width (int): Width of output image. - output_height (int): Height of output image. - tiles_info (list): List of tile items, each item must be - dictionary with `filepath`, `pos_x` and `pos_y` keys - representing path to file and x, y coordinates on output - image where top-left point of tile item should start. - output_path (str): Path to file where should be output stored. - - Returns: - (list): ffmpeg arguments. - - """ - previous_name = "base" - ffmpeg_args = [] - filter_complex_strs = [] - - filter_complex_strs.append("nullsrc=size={}x{}[{}]".format( - output_width, output_height, previous_name - )) - - new_tiles_info = {} - for idx, tile_info in enumerate(tiles_info): - # Add input and store input index - filepath = tile_info["filepath"] - ffmpeg_args.append("-i \"{}\"".format(filepath.replace("\\", "/"))) - - # Prepare initial filter complex arguments - index_name = "input{}".format(idx) - filter_complex_strs.append( - "[{}]setpts=PTS-STARTPTS[{}]".format(idx, index_name) - ) - tile_info["index"] = idx - new_tiles_info[index_name] = tile_info - - # Set frames to 1 - ffmpeg_args.append("-frames 1") - - # Concatenation filter complex arguments - global_index = 1 - total_index = len(new_tiles_info) - for index_name, tile_info in new_tiles_info.items(): - item_str = ( - "[{previous_name}][{index_name}]overlay={pos_x}:{pos_y}" - ).format( - previous_name=previous_name, - index_name=index_name, - pos_x=tile_info["pos_x"], - pos_y=tile_info["pos_y"] - ) - new_previous = "tmp{}".format(global_index) - if global_index != total_index: - item_str += "[{}]".format(new_previous) - filter_complex_strs.append(item_str) - previous_name = new_previous - global_index += 1 - - joined_parts = ";".join(filter_complex_strs) - filter_complex_str = "-filter_complex \"{}\"".format(joined_parts) - - ffmpeg_args.append(filter_complex_str) - ffmpeg_args.append("-y") - ffmpeg_args.append("\"{}\"".format(output_path)) - - return ffmpeg_args diff --git a/openpype/settings/defaults/project_settings/deadline.json b/openpype/settings/defaults/project_settings/deadline.json index 6b6f2d465b..0cbd323299 100644 --- a/openpype/settings/defaults/project_settings/deadline.json +++ b/openpype/settings/defaults/project_settings/deadline.json @@ -23,7 +23,7 @@ "enabled": true, "optional": false, "active": true, - "tile_assembler_plugin": "OpenPypeTileAssembler", + "tile_assembler_plugin": "DraftTileAssembler", "use_published": true, "import_reference": false, "asset_dependencies": true, diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json b/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json index bb5a65e1b7..9906939cc7 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json @@ -121,7 +121,7 @@ "DraftTileAssembler": "Draft Tile Assembler" }, { - "OpenPypeTileAssembler": "Open Image IO" + "OpenPypeTileAssembler": "OpenPype Tile Assembler" } ] }, diff --git a/website/docs/artist_hosts_maya.md b/website/docs/artist_hosts_maya.md index 41045935d5..0a551f0213 100644 --- a/website/docs/artist_hosts_maya.md +++ b/website/docs/artist_hosts_maya.md @@ -516,6 +516,22 @@ In the scene from where you want to publish your model create *Render subset*. P model subset (Maya set node) under corresponding `LAYER_` set under *Render instance*. During publish, it will submit this render to farm and after it is rendered, it will be attached to your model subset. +### Tile Rendering +:::note Deadline +This feature is only supported when using Deadline. See [here](module_deadline#openpypetileassembler-plugin) for setup. +::: +On the render instance objectset you'll find: + +* `Tile Rendering` - for enabling tile rendering. +* `Tile X` - number of tiles in the X axis. +* `Tile Y` - number of tiles in the Y axis. + +When submittig to Deadline, you'll get: + +- for each frame a tile rendering job, to render each from Maya. +- for each frame a tile assembly job, to assemble the rendered tiles. +- job to publish the assembled frames. + ## Render Setups ### Publishing Render Setups diff --git a/website/docs/module_deadline.md b/website/docs/module_deadline.md index ab1016788d..94b6a381c2 100644 --- a/website/docs/module_deadline.md +++ b/website/docs/module_deadline.md @@ -45,6 +45,10 @@ executable. It is recommended to use the `openpype_console` executable as it pro ![Configure plugin](assets/deadline_configure_plugin.png) +### OpenPypeTileAssembler Plugin +To setup tile rendering copy the `OpenPypeTileAssembler` plugin to the repository; +`[OpenPype]\openpype\modules\deadline\repository\custom\plugins\OpenPypeTileAssembler` > `[DeadlineRepository]\custom\plugins\OpenPypeTileAssembler` + ### Pools The main pools can be configured at `project_settings/deadline/publish/CollectDeadlinePools/primary_pool`, which is applied to the rendering jobs.