From 6d787cadd1e21d966332d02b3c6f0915b15633ee Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Fri, 18 Mar 2022 15:07:30 +0000 Subject: [PATCH 1/6] Implemented render publishing --- openpype/hosts/unreal/api/pipeline.py | 22 ++++ .../unreal/plugins/create/create_render.py | 113 ++++++++++++++++++ .../plugins/publish/collect_instances.py | 2 +- .../plugins/publish/collect_remove_marked.py | 24 ++++ .../publish/collect_render_instances.py | 106 ++++++++++++++++ .../publish/validate_sequence_frames.py | 45 +++++++ openpype/plugins/publish/extract_review.py | 3 +- 7 files changed, 313 insertions(+), 2 deletions(-) create mode 100644 openpype/hosts/unreal/plugins/create/create_render.py create mode 100644 openpype/hosts/unreal/plugins/publish/collect_remove_marked.py create mode 100644 openpype/hosts/unreal/plugins/publish/collect_render_instances.py create mode 100644 openpype/hosts/unreal/plugins/publish/validate_sequence_frames.py diff --git a/openpype/hosts/unreal/api/pipeline.py b/openpype/hosts/unreal/api/pipeline.py index 9ec11b942d..cf5ac6e4e0 100644 --- a/openpype/hosts/unreal/api/pipeline.py +++ b/openpype/hosts/unreal/api/pipeline.py @@ -47,6 +47,7 @@ def install(): print("installing OpenPype for Unreal ...") print("-=" * 40) logger.info("installing OpenPype for Unreal") + pyblish.api.register_host("unreal") pyblish.api.register_plugin_path(str(PUBLISH_PATH)) register_loader_plugin_path(str(LOAD_PATH)) api.register_plugin_path(LegacyCreator, str(CREATE_PATH)) @@ -416,3 +417,24 @@ def cast_map_to_str_dict(umap) -> dict: """ return {str(key): str(value) for (key, value) in umap.items()} + + +def get_subsequences(sequence: unreal.LevelSequence): + """Get list of subsequences from sequence. + + Args: + sequence (unreal.LevelSequence): Sequence + + Returns: + list(unreal.LevelSequence): List of subsequences + + """ + tracks = sequence.get_master_tracks() + subscene_track = None + for t in tracks: + if t.get_class() == unreal.MovieSceneSubTrack.static_class(): + subscene_track = t + break + if subscene_track is not None and subscene_track.get_sections(): + return subscene_track.get_sections() + return [] diff --git a/openpype/hosts/unreal/plugins/create/create_render.py b/openpype/hosts/unreal/plugins/create/create_render.py new file mode 100644 index 0000000000..49268c91f5 --- /dev/null +++ b/openpype/hosts/unreal/plugins/create/create_render.py @@ -0,0 +1,113 @@ +import unreal + +from openpype.hosts.unreal.api import pipeline +from openpype.hosts.unreal.api.plugin import Creator + + +class CreateRender(Creator): + """Create instance for sequence for rendering""" + + name = "unrealRender" + label = "Unreal - Render" + family = "render" + icon = "cube" + asset_types = ["LevelSequence"] + + root = "/Game/OpenPype/PublishInstances" + suffix = "_INS" + + def __init__(self, *args, **kwargs): + super(CreateRender, self).__init__(*args, **kwargs) + + def process(self): + subset = self.data["subset"] + + ar = unreal.AssetRegistryHelpers.get_asset_registry() + + # Get the master sequence and the master level. + # There should be only one sequence and one level in the directory. + filter = unreal.ARFilter( + class_names=["LevelSequence"], + package_paths=[f"/Game/OpenPype/{self.data['asset']}"], + recursive_paths=False) + sequences = ar.get_assets(filter) + ms = sequences[0].get_editor_property('object_path') + filter = unreal.ARFilter( + class_names=["World"], + package_paths=[f"/Game/OpenPype/{self.data['asset']}"], + recursive_paths=False) + levels = ar.get_assets(filter) + ml = levels[0].get_editor_property('object_path') + + selection = [] + if (self.options or {}).get("useSelection"): + sel_objects = unreal.EditorUtilityLibrary.get_selected_assets() + selection = [ + a.get_path_name() for a in sel_objects + if a.get_class().get_name() in self.asset_types] + else: + selection.append(self.data['sequence']) + + unreal.log(f"selection: {selection}") + + path = f"{self.root}" + unreal.EditorAssetLibrary.make_directory(path) + + ar = unreal.AssetRegistryHelpers.get_asset_registry() + + for a in selection: + ms_obj = ar.get_asset_by_object_path(ms).get_asset() + + seq_data = None + + if a == ms: + seq_data = { + "sequence": ms_obj, + "output": f"{ms_obj.get_name()}", + "frame_range": ( + ms_obj.get_playback_start(), ms_obj.get_playback_end()) + } + else: + seq_data_list = [{ + "sequence": ms_obj, + "output": f"{ms_obj.get_name()}", + "frame_range": ( + ms_obj.get_playback_start(), ms_obj.get_playback_end()) + }] + + for s in seq_data_list: + subscenes = pipeline.get_subsequences(s.get('sequence')) + + for ss in subscenes: + curr_data = { + "sequence": ss.get_sequence(), + "output": (f"{s.get('output')}/" + f"{ss.get_sequence().get_name()}"), + "frame_range": ( + ss.get_start_frame(), ss.get_end_frame() - 1) + } + + if ss.get_sequence().get_path_name() == a: + seq_data = curr_data + break + seq_data_list.append(curr_data) + + if seq_data is not None: + break + + if not seq_data: + continue + + d = self.data.copy() + d["members"] = [a] + d["sequence"] = a + d["master_sequence"] = ms + d["master_level"] = ml + d["output"] = seq_data.get('output') + d["frameStart"] = seq_data.get('frame_range')[0] + d["frameEnd"] = seq_data.get('frame_range')[1] + + container_name = f"{subset}{self.suffix}" + pipeline.create_publish_instance( + instance=container_name, path=path) + pipeline.imprint(f"{path}/{container_name}", d) diff --git a/openpype/hosts/unreal/plugins/publish/collect_instances.py b/openpype/hosts/unreal/plugins/publish/collect_instances.py index 94e732d728..2f604cb322 100644 --- a/openpype/hosts/unreal/plugins/publish/collect_instances.py +++ b/openpype/hosts/unreal/plugins/publish/collect_instances.py @@ -17,7 +17,7 @@ class CollectInstances(pyblish.api.ContextPlugin): """ label = "Collect Instances" - order = pyblish.api.CollectorOrder + order = pyblish.api.CollectorOrder - 0.1 hosts = ["unreal"] def process(self, context): diff --git a/openpype/hosts/unreal/plugins/publish/collect_remove_marked.py b/openpype/hosts/unreal/plugins/publish/collect_remove_marked.py new file mode 100644 index 0000000000..69e69f6630 --- /dev/null +++ b/openpype/hosts/unreal/plugins/publish/collect_remove_marked.py @@ -0,0 +1,24 @@ +import pyblish.api + + +class CollectRemoveMarked(pyblish.api.ContextPlugin): + """Remove marked data + + Remove instances that have 'remove' in their instance.data + + """ + + order = pyblish.api.CollectorOrder + 0.499 + label = 'Remove Marked Instances' + + def process(self, context): + + self.log.debug(context) + # make ftrack publishable + instances_to_remove = [] + for instance in context: + if instance.data.get('remove'): + instances_to_remove.append(instance) + + for instance in instances_to_remove: + context.remove(instance) diff --git a/openpype/hosts/unreal/plugins/publish/collect_render_instances.py b/openpype/hosts/unreal/plugins/publish/collect_render_instances.py new file mode 100644 index 0000000000..6eb51517c6 --- /dev/null +++ b/openpype/hosts/unreal/plugins/publish/collect_render_instances.py @@ -0,0 +1,106 @@ +from pathlib import Path +from tkinter.font import families +import unreal + +import pyblish.api +from openpype import lib +from openpype.pipeline import legacy_create +from openpype.hosts.unreal.api import pipeline + + +class CollectRenderInstances(pyblish.api.InstancePlugin): + """ This collector will try to find all the rendered frames. + + """ + order = pyblish.api.CollectorOrder + hosts = ["unreal"] + families = ["render"] + label = "Collect Render Instances" + + def process(self, instance): + self.log.debug("Preparing Rendering Instances") + + context = instance.context + + data = instance.data + data['remove'] = True + + ar = unreal.AssetRegistryHelpers.get_asset_registry() + + sequence = ar.get_asset_by_object_path( + data.get('sequence')).get_asset() + + sequences = [{ + "sequence": sequence, + "output": data.get('output'), + "frame_range": ( + data.get('frameStart'), data.get('frameEnd')) + }] + + for s in sequences: + self.log.debug(f"Processing: {s.get('sequence').get_name()}") + subscenes = pipeline.get_subsequences(s.get('sequence')) + + if subscenes: + for ss in subscenes: + sequences.append({ + "sequence": ss.get_sequence(), + "output": (f"{s.get('output')}/" + f"{ss.get_sequence().get_name()}"), + "frame_range": ( + ss.get_start_frame(), ss.get_end_frame() - 1) + }) + else: + # Avoid creating instances for camera sequences + if "_camera" not in s.get('sequence').get_name(): + seq = s.get('sequence') + seq_name = seq.get_name() + + new_instance = context.create_instance( + f"{data.get('subset')}_" + f"{seq_name}") + new_instance[:] = seq_name + + new_data = new_instance.data + + new_data["asset"] = seq_name + new_data["setMembers"] = seq_name + new_data["family"] = "render" + new_data["families"] = ["render", "review"] + new_data["parent"] = data.get("parent") + new_data["subset"] = f"{data.get('subset')}_{seq_name}" + new_data["level"] = data.get("level") + new_data["output"] = s.get('output') + new_data["fps"] = seq.get_display_rate().numerator + new_data["frameStart"] = s.get('frame_range')[0] + new_data["frameEnd"] = s.get('frame_range')[1] + new_data["sequence"] = seq.get_path_name() + new_data["master_sequence"] = data["master_sequence"] + new_data["master_level"] = data["master_level"] + + self.log.debug(f"new instance data: {new_data}") + + project_dir = unreal.Paths.project_dir() + render_dir = (f"{project_dir}/Saved/MovieRenders/" + f"{s.get('output')}") + render_path = Path(render_dir) + + frames = [] + + for x in render_path.iterdir(): + if x.is_file() and x.suffix == '.png': + frames.append(str(x.name)) + + if "representations" not in new_instance.data: + new_instance.data["representations"] = [] + + repr = { + 'frameStart': s.get('frame_range')[0], + 'frameEnd': s.get('frame_range')[1], + 'name': 'png', + 'ext': 'png', + 'files': frames, + 'stagingDir': render_dir, + 'tags': ['review'] + } + new_instance.data["representations"].append(repr) diff --git a/openpype/hosts/unreal/plugins/publish/validate_sequence_frames.py b/openpype/hosts/unreal/plugins/publish/validate_sequence_frames.py new file mode 100644 index 0000000000..0a77281d16 --- /dev/null +++ b/openpype/hosts/unreal/plugins/publish/validate_sequence_frames.py @@ -0,0 +1,45 @@ +from pathlib import Path + +import unreal + +import pyblish.api + + +class ValidateSequenceFrames(pyblish.api.InstancePlugin): + """Ensure the sequence of frames is complete + + The files found in the folder are checked against the frameStart and + frameEnd of the instance. If the first or last file is not + corresponding with the first or last frame it is flagged as invalid. + """ + + order = pyblish.api.ValidatorOrder + label = "Validate Sequence Frames" + families = ["render"] + hosts = ["unreal"] + optional = True + + def process(self, instance): + self.log.debug(instance.data) + + representations = instance.data.get("representations") + for repr in representations: + frames = [] + for x in repr.get("files"): + # Get frame number. The last one contains the file extension, + # while the one before that is the frame number. + # `lstrip` removes any leading zeros. `or "0"` is to tackle + # the case where the frame number is "00". + frame = int(str(x).split('.')[-2]) + frames.append(frame) + frames.sort() + current_range = (frames[0], frames[-1]) + required_range = (instance.data["frameStart"], + instance.data["frameEnd"]) + + if current_range != required_range: + raise ValueError(f"Invalid frame range: {current_range} - " + f"expected: {required_range}") + + assert len(frames) == int(frames[-1]) - int(frames[0]) + 1, \ + "Missing frames" diff --git a/openpype/plugins/publish/extract_review.py b/openpype/plugins/publish/extract_review.py index 3ecea1f8bd..35ad6270cf 100644 --- a/openpype/plugins/publish/extract_review.py +++ b/openpype/plugins/publish/extract_review.py @@ -51,7 +51,8 @@ class ExtractReview(pyblish.api.InstancePlugin): "resolve", "webpublisher", "aftereffects", - "flame" + "flame", + "unreal" ] # Supported extensions From b9f387dc505664f0c52c6a45330cbc2c6786a611 Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Fri, 18 Mar 2022 16:09:53 +0000 Subject: [PATCH 2/6] Hound fix --- .../hosts/unreal/plugins/publish/collect_render_instances.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/openpype/hosts/unreal/plugins/publish/collect_render_instances.py b/openpype/hosts/unreal/plugins/publish/collect_render_instances.py index 6eb51517c6..9d60b65d08 100644 --- a/openpype/hosts/unreal/plugins/publish/collect_render_instances.py +++ b/openpype/hosts/unreal/plugins/publish/collect_render_instances.py @@ -1,10 +1,7 @@ from pathlib import Path -from tkinter.font import families import unreal import pyblish.api -from openpype import lib -from openpype.pipeline import legacy_create from openpype.hosts.unreal.api import pipeline From 9fab478edf926bbe45dcfa294d2b53c767ebf086 Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Fri, 18 Mar 2022 16:54:20 +0000 Subject: [PATCH 3/6] Improvements and more consistency in validator for rendered frames --- .../publish/validate_sequence_frames.py | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/openpype/hosts/unreal/plugins/publish/validate_sequence_frames.py b/openpype/hosts/unreal/plugins/publish/validate_sequence_frames.py index 0a77281d16..2684581e9d 100644 --- a/openpype/hosts/unreal/plugins/publish/validate_sequence_frames.py +++ b/openpype/hosts/unreal/plugins/publish/validate_sequence_frames.py @@ -1,4 +1,5 @@ from pathlib import Path +import clique import unreal @@ -20,19 +21,17 @@ class ValidateSequenceFrames(pyblish.api.InstancePlugin): optional = True def process(self, instance): - self.log.debug(instance.data) - representations = instance.data.get("representations") for repr in representations: - frames = [] - for x in repr.get("files"): - # Get frame number. The last one contains the file extension, - # while the one before that is the frame number. - # `lstrip` removes any leading zeros. `or "0"` is to tackle - # the case where the frame number is "00". - frame = int(str(x).split('.')[-2]) - frames.append(frame) - frames.sort() + patterns = [clique.PATTERNS["frames"]] + collections, remainder = clique.assemble( + repr["files"], minimum_items=1, patterns=patterns) + + assert not remainder, "Must not have remainder" + assert len(collections) == 1, "Must detect single collection" + collection = collections[0] + frames = list(collection.indexes) + current_range = (frames[0], frames[-1]) required_range = (instance.data["frameStart"], instance.data["frameEnd"]) @@ -41,5 +40,5 @@ class ValidateSequenceFrames(pyblish.api.InstancePlugin): raise ValueError(f"Invalid frame range: {current_range} - " f"expected: {required_range}") - assert len(frames) == int(frames[-1]) - int(frames[0]) + 1, \ - "Missing frames" + missing = collection.holes().indexes + assert not missing, "Missing frames: %s" % (missing,) From 541f44988dc654bd2609d865875a1ed47908014e Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Fri, 18 Mar 2022 16:55:54 +0000 Subject: [PATCH 4/6] More hound fixes --- .../hosts/unreal/plugins/publish/validate_sequence_frames.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/openpype/hosts/unreal/plugins/publish/validate_sequence_frames.py b/openpype/hosts/unreal/plugins/publish/validate_sequence_frames.py index 2684581e9d..87f1338ee8 100644 --- a/openpype/hosts/unreal/plugins/publish/validate_sequence_frames.py +++ b/openpype/hosts/unreal/plugins/publish/validate_sequence_frames.py @@ -1,8 +1,5 @@ -from pathlib import Path import clique -import unreal - import pyblish.api From 0274be6bed9c7bf92c862ae33c0eab0db3b88eb4 Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Fri, 18 Mar 2022 17:22:52 +0000 Subject: [PATCH 5/6] Added rendering --- openpype/hosts/unreal/api/rendering.py | 125 +++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 openpype/hosts/unreal/api/rendering.py diff --git a/openpype/hosts/unreal/api/rendering.py b/openpype/hosts/unreal/api/rendering.py new file mode 100644 index 0000000000..376e1b75ce --- /dev/null +++ b/openpype/hosts/unreal/api/rendering.py @@ -0,0 +1,125 @@ +import unreal + +from openpype.hosts.unreal.api import pipeline + + +queue = None +executor = None + + +def _queue_finish_callback(exec, success): + unreal.log("Render completed. Success: " + str(success)) + + # Delete our reference so we don't keep it alive. + global executor + global queue + del executor + del queue + + +def _job_finish_callback(job, success): + # You can make any edits you want to the editor world here, and the world + # will be duplicated when the next render happens. Make sure you undo your + # edits in OnQueueFinishedCallback if you don't want to leak state changes + # into the editor world. + unreal.log("Individual job completed.") + + +def start_rendering(): + """ + Start the rendering process. + """ + print("Starting rendering...") + + # Get selected sequences + assets = unreal.EditorUtilityLibrary.get_selected_assets() + + # instances = pipeline.ls_inst() + instances = [ + a for a in assets + if a.get_class().get_name() == "OpenPypePublishInstance"] + + inst_data = [] + + for i in instances: + data = pipeline.parse_container(i.get_path_name()) + if data["family"] == "render": + inst_data.append(data) + + # subsystem = unreal.get_editor_subsystem( + # unreal.MoviePipelineQueueSubsystem) + # queue = subsystem.get_queue() + global queue + queue = unreal.MoviePipelineQueue() + + ar = unreal.AssetRegistryHelpers.get_asset_registry() + + for i in inst_data: + sequence = ar.get_asset_by_object_path(i["sequence"]).get_asset() + + sequences = [{ + "sequence": sequence, + "output": f"{i['output']}", + "frame_range": ( + int(float(i["frameStart"])), + int(float(i["frameEnd"])) + 1) + }] + render_list = [] + + # Get all the sequences to render. If there are subsequences, + # add them and their frame ranges to the render list. We also + # use the names for the output paths. + for s in sequences: + subscenes = pipeline.get_subsequences(s.get('sequence')) + + if subscenes: + for ss in subscenes: + sequences.append({ + "sequence": ss.get_sequence(), + "output": (f"{s.get('output')}/" + f"{ss.get_sequence().get_name()}"), + "frame_range": ( + ss.get_start_frame(), ss.get_end_frame()) + }) + else: + # Avoid rendering camera sequences + if "_camera" not in s.get('sequence').get_name(): + render_list.append(s) + + # Create the rendering jobs and add them to the queue. + for r in render_list: + job = queue.allocate_new_job(unreal.MoviePipelineExecutorJob) + job.sequence = unreal.SoftObjectPath(i["master_sequence"]) + job.map = unreal.SoftObjectPath(i["master_level"]) + job.author = "OpenPype" + + # User data could be used to pass data to the job, that can be + # read in the job's OnJobFinished callback. We could, + # for instance, pass the AvalonPublishInstance's path to the job. + # job.user_data = "" + + settings = job.get_configuration().find_or_add_setting_by_class( + unreal.MoviePipelineOutputSetting) + settings.output_resolution = unreal.IntPoint(1920, 1080) + settings.custom_start_frame = r.get("frame_range")[0] + settings.custom_end_frame = r.get("frame_range")[1] + settings.use_custom_playback_range = True + settings.file_name_format = "{sequence_name}.{frame_number}" + settings.output_directory.path += r.get('output') + + renderPass = job.get_configuration().find_or_add_setting_by_class( + unreal.MoviePipelineDeferredPassBase) + renderPass.disable_multisample_effects = True + + job.get_configuration().find_or_add_setting_by_class( + unreal.MoviePipelineImageSequenceOutput_PNG) + + # If there are jobs in the queue, start the rendering process. + if queue.get_jobs(): + global executor + executor = unreal.MoviePipelinePIEExecutor() + executor.on_executor_finished_delegate.add_callable_unique( + _queue_finish_callback) + executor.on_individual_job_finished_delegate.add_callable_unique( + _job_finish_callback) # Only available on PIE Executor + executor.execute(queue) From a8680e9f23e885bd3a1957876198b9c249f23fdb Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Fri, 18 Mar 2022 17:25:03 +0000 Subject: [PATCH 6/6] Code cleanup --- openpype/hosts/unreal/plugins/create/create_render.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/openpype/hosts/unreal/plugins/create/create_render.py b/openpype/hosts/unreal/plugins/create/create_render.py index 49268c91f5..77fc98bcec 100644 --- a/openpype/hosts/unreal/plugins/create/create_render.py +++ b/openpype/hosts/unreal/plugins/create/create_render.py @@ -16,9 +16,6 @@ class CreateRender(Creator): root = "/Game/OpenPype/PublishInstances" suffix = "_INS" - def __init__(self, *args, **kwargs): - super(CreateRender, self).__init__(*args, **kwargs) - def process(self): subset = self.data["subset"]