From 8dcc2eabff6a39c47f4fae519cc9de658039312d Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 18 Feb 2022 22:24:37 +0100 Subject: [PATCH 001/126] flame: adding batch utils for creating batch in desktop --- openpype/hosts/flame/api/batch_utils.py | 37 +++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 openpype/hosts/flame/api/batch_utils.py diff --git a/openpype/hosts/flame/api/batch_utils.py b/openpype/hosts/flame/api/batch_utils.py new file mode 100644 index 0000000000..3a155c4b8d --- /dev/null +++ b/openpype/hosts/flame/api/batch_utils.py @@ -0,0 +1,37 @@ +import flame + + +def create_batch(name, frame_start, frame_end, **kwargs): + schematicReels = ['LoadedReel1'] + shelfReels = ['ShelfReel1'] + + handle_start = kwargs.get("handleStart") + handle_end = kwargs.get("handleEnd") + + if handle_start: + frame_start -= handle_start + if handle_end: + frame_end += handle_end + + # Create batch group with name, start_frame value, duration value, + # set of schematic reel names, set of shelf reel names + flame.batch.create_batch_group( + name, + start_frame=frame_start, + duration=frame_end, + reels=schematicReels, + shelf_reels=shelfReels + ) + + if kwargs.get("switch_batch_tab"): + # use this command to switch to the batch tab + flame.batch.go_to() + + comp = flame.batch.create_node("Comp") + writeFile = flame.batch.create_node("Write File") + + # connect nodes + flame.batch.connect_nodes(comp, "Result", writeFile, "Front") + + # sort batch nodes + flame.batch.organize() From 5391f1fff3abe36dde6289778ab2823e79616ab7 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 18 Feb 2022 22:46:27 +0100 Subject: [PATCH 002/126] flame: adding write node to batch utils --- openpype/hosts/flame/api/batch_utils.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/flame/api/batch_utils.py b/openpype/hosts/flame/api/batch_utils.py index 3a155c4b8d..2c80834928 100644 --- a/openpype/hosts/flame/api/batch_utils.py +++ b/openpype/hosts/flame/api/batch_utils.py @@ -5,6 +5,7 @@ def create_batch(name, frame_start, frame_end, **kwargs): schematicReels = ['LoadedReel1'] shelfReels = ['ShelfReel1'] + write_pref = kwargs["write_pref"] handle_start = kwargs.get("handleStart") handle_end = kwargs.get("handleEnd") @@ -27,11 +28,23 @@ def create_batch(name, frame_start, frame_end, **kwargs): # use this command to switch to the batch tab flame.batch.go_to() - comp = flame.batch.create_node("Comp") - writeFile = flame.batch.create_node("Write File") + comp_node = flame.batch.create_node("Comp") + + # create write node + write_node = flame.batch.create_node('Write File') + write_node.media_path = write_pref["media_path"] + write_node.media_path_pattern = write_pref["media_path_pattern"] + write_node.create_clip = write_pref["create_clip"] + write_node.include_setup = write_pref["include_setup"] + write_node.create_clip_path = write_pref["create_clip_path"] + write_node.include_setup_path = write_pref["include_setup_path"] + write_node.file_type = write_pref["file_type"] + write_node.bit_depth = write_pref["bit_depth"] + write_node.frame_index_mode = write_pref["frame_index_mode"] + write_node.frame_padding = int(write_pref["frame_padding"]) # connect nodes - flame.batch.connect_nodes(comp, "Result", writeFile, "Front") + flame.batch.connect_nodes(comp_node, "Result", write_node, "Front") # sort batch nodes flame.batch.organize() From cdc3d0be792e718b17fb1290f3f05a5ea8c4380f Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 22 Feb 2022 12:49:11 +0100 Subject: [PATCH 003/126] flame: batch utils to api --- openpype/hosts/flame/api/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/flame/api/__init__.py b/openpype/hosts/flame/api/__init__.py index 56bbadd2fc..98a1a23e89 100644 --- a/openpype/hosts/flame/api/__init__.py +++ b/openpype/hosts/flame/api/__init__.py @@ -70,6 +70,9 @@ from .render_utils import ( export_clip, get_preset_path_by_xml_name ) +from .batch_utils import ( + create_batch +) __all__ = [ # constants @@ -140,5 +143,8 @@ __all__ = [ # render utils "export_clip", - "get_preset_path_by_xml_name" + "get_preset_path_by_xml_name", + + # batch utils + "create_batch" ] From 5d8e3e293f46860f9de52f93736f9646023de501 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 22 Feb 2022 12:49:33 +0100 Subject: [PATCH 004/126] flame: adding docstrigs to create_batch --- openpype/hosts/flame/api/batch_utils.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/openpype/hosts/flame/api/batch_utils.py b/openpype/hosts/flame/api/batch_utils.py index 2c80834928..a1fe7961c4 100644 --- a/openpype/hosts/flame/api/batch_utils.py +++ b/openpype/hosts/flame/api/batch_utils.py @@ -2,8 +2,15 @@ import flame def create_batch(name, frame_start, frame_end, **kwargs): - schematicReels = ['LoadedReel1'] - shelfReels = ['ShelfReel1'] + """Create Batch Group in active project's Desktop + + Args: + name (str): name of batch group to be created + frame_start (int): start frame of batch + frame_end (int): end frame of batch + """ + schematic_reels = kwargs.get("shematic_reels") or ['LoadedReel1'] + shelf_reels = kwargs.get("shelf_reels") or ['ShelfReel1'] write_pref = kwargs["write_pref"] handle_start = kwargs.get("handleStart") @@ -20,8 +27,8 @@ def create_batch(name, frame_start, frame_end, **kwargs): name, start_frame=frame_start, duration=frame_end, - reels=schematicReels, - shelf_reels=shelfReels + reels=schematic_reels, + shelf_reels=shelf_reels ) if kwargs.get("switch_batch_tab"): From 162df8c0aca975da19778cd05e22771f651458a9 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 22 Feb 2022 13:00:14 +0100 Subject: [PATCH 005/126] flame: itegrator wip --- .../flame/plugins/publish/integrate_batch_group.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 openpype/hosts/flame/plugins/publish/integrate_batch_group.py diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py new file mode 100644 index 0000000000..fd88ed318e --- /dev/null +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -0,0 +1,14 @@ +import pyblish +import openpype.hosts.flame.api as opfapi + +@pyblish.api.log +class IntegrateBatchGroup(pyblish.api.InstancePlugin): + """Integrate published shot to batch group""" + + order = pyblish.api.IntegratorOrder + 0.45 + label = "Integrate Batch Groups" + hosts = ["flame"] + families = ["clip"] + + def process(self, instance): + opfapi.create_batch \ No newline at end of file From c120135f1fa467587a1b41f0e3c6282a2285b200 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 16 Mar 2022 21:08:35 +0100 Subject: [PATCH 006/126] Removed submodule openpype/modules/default_modules/ftrack/python2_vendor/arrow --- openpype/modules/default_modules/ftrack/python2_vendor/arrow | 1 - 1 file changed, 1 deletion(-) delete mode 160000 openpype/modules/default_modules/ftrack/python2_vendor/arrow diff --git a/openpype/modules/default_modules/ftrack/python2_vendor/arrow b/openpype/modules/default_modules/ftrack/python2_vendor/arrow deleted file mode 160000 index b746fedf72..0000000000 --- a/openpype/modules/default_modules/ftrack/python2_vendor/arrow +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b746fedf7286c3755a46f07ab72f4c414cd41fc0 From d1a733cf885e8955746ef6e71db01b76fe7c96be Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 16 Mar 2022 21:08:43 +0100 Subject: [PATCH 007/126] Removed submodule openpype/modules/default_modules/ftrack/python2_vendor/ftrack-python-api --- .../default_modules/ftrack/python2_vendor/ftrack-python-api | 1 - 1 file changed, 1 deletion(-) delete mode 160000 openpype/modules/default_modules/ftrack/python2_vendor/ftrack-python-api diff --git a/openpype/modules/default_modules/ftrack/python2_vendor/ftrack-python-api b/openpype/modules/default_modules/ftrack/python2_vendor/ftrack-python-api deleted file mode 160000 index d277f474ab..0000000000 --- a/openpype/modules/default_modules/ftrack/python2_vendor/ftrack-python-api +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d277f474ab016e7b53479c36af87cb861d0cc53e From fdb9f0da77f1a996be4160f52acdc1036238bc39 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 16 Mar 2022 21:08:50 +0100 Subject: [PATCH 008/126] Removed submodule repos/avalon-unreal-integration --- repos/avalon-unreal-integration | 1 - 1 file changed, 1 deletion(-) delete mode 160000 repos/avalon-unreal-integration diff --git a/repos/avalon-unreal-integration b/repos/avalon-unreal-integration deleted file mode 160000 index 43f6ea9439..0000000000 --- a/repos/avalon-unreal-integration +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 43f6ea943980b29c02a170942b566ae11f2b7080 From db03b47b8ee5615bb19ee90f1dc52f8f05cf379b Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 22 Mar 2022 15:14:38 +0100 Subject: [PATCH 009/126] hound fix --- openpype/hosts/flame/api/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/hosts/flame/api/__init__.py b/openpype/hosts/flame/api/__init__.py index 7c03186ff0..97f83ccf07 100644 --- a/openpype/hosts/flame/api/__init__.py +++ b/openpype/hosts/flame/api/__init__.py @@ -146,7 +146,6 @@ __all__ = [ "export_clip", "get_preset_path_by_xml_name", "modify_preset_file", - # batch utils "create_batch" From aab2ed17f8d582056f5835613845d13ea7205b24 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 22 Mar 2022 15:15:04 +0100 Subject: [PATCH 010/126] flame: ingegrate batch wip --- .../plugins/publish/integrate_batch_group.py | 66 ++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py index fd88ed318e..aaa405343c 100644 --- a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -1,6 +1,7 @@ import pyblish import openpype.hosts.flame.api as opfapi + @pyblish.api.log class IntegrateBatchGroup(pyblish.api.InstancePlugin): """Integrate published shot to batch group""" @@ -11,4 +12,67 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): families = ["clip"] def process(self, instance): - opfapi.create_batch \ No newline at end of file + frame_start = instance.data["frameStart"] + frame_end = instance.data["frameEnd"] + handle_start = instance.data["handleStart"] + handle_end = instance.data["handleEnd"] + + asset_name = instance.data["asset"] + write_pref_data = self._get_write_prefs(instance) + + batch_data = { + "shematic_reels": [ + "OP_LoadedReel" + ], + "write_pref": write_pref_data, + "handleStart": handle_start, + "handleEnd": handle_end + } + + opfapi.create_batch(asset_name, frame_start, frame_end, batch_data) + + def _get_write_prefs(self, instance): + # The path attribute where the rendered clip is exported + # /path/to/file.[0001-0010].exr + media_path = "{render_path}".format() + # name of file represented by tokens + media_path_pattern = "_v." + # The Create Open Clip attribute of the Write File node. \ + # Determines if an Open Clip is created by the Write File node. + create_clip = True + # The Include Setup attribute of the Write File node. + # Determines if a Batch Setup file is created by the Write File node. + include_setup = True + # The path attribute where the Open Clip file is exported by + # the Write File node. + create_clip_path = "" + include_setup_path = None + # The file type for the files written by the Write File node. + # Setting this attribute also overwrites format_extension, + # bit_depth and compress_mode to match the defaults for + # this file type. + file_type = "OpenEXR" + # The bit depth for the files written by the Write File node. + # This attribute resets to match file_type whenever file_type is set. + bit_depth = "16" + frame_index_mode = None + frame_padding = 0 + # The versioning mode of the Open Clip exported by the Write File node. + # Only available if create_clip = True. + version_mode = "Follow Iteration" + version_name = "v" + + return { + "media_path": media_path, + "media_path_pattern": media_path_pattern, + "create_clip": create_clip, + "include_setup": include_setup, + "create_clip_path": create_clip_path, + "include_setup_path": include_setup_path, + "file_type": file_type, + "bit_depth": bit_depth, + "frame_index_mode": frame_index_mode, + "frame_padding": frame_padding, + "version_mode": version_mode, + "version_name": version_name + } From 12aeb88e0a2cd7ccfa6dae9dd7f20d83bf4577b5 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 22 Mar 2022 17:13:11 +0100 Subject: [PATCH 011/126] flame: integrate batch [wip] --- .../plugins/publish/integrate_batch_group.py | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py index aaa405343c..780531287b 100644 --- a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -1,3 +1,4 @@ +import os import pyblish import openpype.hosts.flame.api as opfapi @@ -32,9 +33,12 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): opfapi.create_batch(asset_name, frame_start, frame_end, batch_data) def _get_write_prefs(self, instance): + shot_path = instance.data[""] + render_dir_path = os.path.join( + shot_path, "work", task, "render", "flame") # The path attribute where the rendered clip is exported # /path/to/file.[0001-0010].exr - media_path = "{render_path}".format() + media_path = render_dir_path # name of file represented by tokens media_path_pattern = "_v." # The Create Open Clip attribute of the Write File node. \ @@ -46,17 +50,33 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): # The path attribute where the Open Clip file is exported by # the Write File node. create_clip_path = "" - include_setup_path = None + # The path attribute where the Batch setup file + # is exported by the Write File node. + include_setup_path = "./_v" # The file type for the files written by the Write File node. # Setting this attribute also overwrites format_extension, # bit_depth and compress_mode to match the defaults for # this file type. file_type = "OpenEXR" + # The file extension for the files written by the Write File node. + # This attribute resets to match file_type whenever file_type + # is set. If you require a specific extension, you must + # set format_extension after setting file_type. + format_extension = "exr" # The bit depth for the files written by the Write File node. # This attribute resets to match file_type whenever file_type is set. bit_depth = "16" - frame_index_mode = None - frame_padding = 0 + # The compressing attribute for the files exported by the Write + # File node. Only relevant when file_type in 'OpenEXR', 'Sgi', 'Tiff' + compress = True + # The compression format attribute for the specific File Types + # export by the Write File node. You must set compress_mode + # after setting file_type. + compress_mode = "DWAB" + # The frame index mode attribute of the Write File node. + # Value range: `Use Timecode` or `Use Start Frame` + frame_index_mode = "Use Start Frame" + frame_padding = 6 # The versioning mode of the Open Clip exported by the Write File node. # Only available if create_clip = True. version_mode = "Follow Iteration" @@ -70,7 +90,10 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): "create_clip_path": create_clip_path, "include_setup_path": include_setup_path, "file_type": file_type, + "format_extension": format_extension, "bit_depth": bit_depth, + "compress": compress, + "compress_mode": compress_mode, "frame_index_mode": frame_index_mode, "frame_padding": frame_padding, "version_mode": version_mode, From 7dd0c86a17f0a95ac6e16416473978169cf793a7 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 23 Mar 2022 17:11:48 +0100 Subject: [PATCH 012/126] flame: collect timeline instances settings --- .../defaults/project_settings/flame.json | 31 +++++++ .../projects_schema/schema_project_flame.json | 81 +++++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/openpype/settings/defaults/project_settings/flame.json b/openpype/settings/defaults/project_settings/flame.json index c7188b10b5..939752c778 100644 --- a/openpype/settings/defaults/project_settings/flame.json +++ b/openpype/settings/defaults/project_settings/flame.json @@ -20,6 +20,37 @@ } }, "publish": { + "CollectTimelineInstances": { + "xml_preset_attrs_from_comments": [ + { + "name": "width", + "type": "number" + }, + { + "name": "height", + "type": "number" + }, + { + "name": "pixelRatio", + "type": "float" + }, + { + "name": "resizeType", + "type": "string" + }, + { + "name": "resizeFilter", + "type": "string" + } + ], + "add_tasks": [ + { + "name": "compositing", + "type": "Compositing", + "create_batch_group": true + } + ] + }, "ExtractSubsetResources": { "keep_original_representation": false, "export_presets_mapping": { diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_flame.json b/openpype/settings/entities/schemas/projects_schema/schema_project_flame.json index e352f8b132..8057b07d9c 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_flame.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_flame.json @@ -136,6 +136,87 @@ "key": "publish", "label": "Publish plugins", "children": [ + { + "type": "dict", + "collapsible": true, + "key": "CollectTimelineInstances", + "label": "Collect Timeline Instances", + "is_group": true, + "children": [ + { + "type": "collapsible-wrap", + "label": "XML presets attributes parsable from segment comments", + "collapsible": true, + "collapsed": true, + "children": [ + { + "type": "list", + "key": "xml_preset_attrs_from_comments", + "object_type": { + "type": "dict", + "children": [ + { + "type": "text", + "key": "name", + "label": "Attribute name" + }, + { + "key": "type", + "label": "Attribute type", + "type": "enum", + "default": "number", + "enum_items": [ + { + "number": "number" + }, + { + "float": "float" + }, + { + "string": "string" + } + ] + } + ] + } + } + ] + }, + { + "type": "collapsible-wrap", + "label": "Add tasks", + "collapsible": true, + "collapsed": true, + "children": [ + { + "type": "list", + "key": "add_tasks", + "object_type": { + "type": "dict", + "children": [ + { + "type": "text", + "key": "name", + "label": "Task name" + }, + { + "key": "type", + "label": "Task type", + "multiselection": false, + "type": "task-types-enum" + }, + { + "type": "boolean", + "key": "create_batch_group", + "label": "Create batch group" + } + ] + } + } + ] + } + ] + }, { "type": "dict", "collapsible": true, From 074703f8cff5b4432a98d1fb28f9a6f42943c694 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 23 Mar 2022 17:13:09 +0100 Subject: [PATCH 013/126] flame: use settings in collect timeline instances --- .../publish/collect_timeline_instances.py | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/openpype/hosts/flame/plugins/publish/collect_timeline_instances.py b/openpype/hosts/flame/plugins/publish/collect_timeline_instances.py index 70340ad7a2..94348601b2 100644 --- a/openpype/hosts/flame/plugins/publish/collect_timeline_instances.py +++ b/openpype/hosts/flame/plugins/publish/collect_timeline_instances.py @@ -21,15 +21,9 @@ class CollectTimelineInstances(pyblish.api.ContextPlugin): audio_track_items = [] - # TODO: add to settings # settings - xml_preset_attrs_from_comments = { - "width": "number", - "height": "number", - "pixelRatio": "float", - "resizeType": "string", - "resizeFilter": "string" - } + xml_preset_attrs_from_comments = [] + add_tasks = [] def process(self, context): project = context.data["flameProject"] @@ -106,7 +100,11 @@ class CollectTimelineInstances(pyblish.api.ContextPlugin): "fps": self.fps, "flameSourceClip": source_clip, "sourceFirstFrame": int(first_frame), - "path": file_path + "path": file_path, + "flameAddTasks": self.add_tasks, + "tasks": { + task["name"]: {"type": task["type"]} + for task in self.add_tasks} }) # get otio clip data @@ -181,14 +179,17 @@ class CollectTimelineInstances(pyblish.api.ContextPlugin): # split to key and value key, value = split.split(":") - for a_name, a_type in self.xml_preset_attrs_from_comments.items(): + for attr_data in self.xml_preset_attrs_from_comments: + a_name = attr_data["name"] + a_type = attr_data["type"] + # exclude all not related attributes if a_name.lower() not in key.lower(): continue # get pattern defined by type pattern = TXT_PATERN - if a_type in ("number" , "float"): + if a_type in ("number", "float"): pattern = NUM_PATERN res_goup = pattern.findall(value) From 029135acf5513756ffc3e01fafa66e8ddd565d32 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 23 Mar 2022 20:38:46 +0100 Subject: [PATCH 014/126] flame: integrator bath group [finishing] --- openpype/hosts/flame/api/batch_utils.py | 12 +++- .../plugins/publish/integrate_batch_group.py | 65 +++++++++++++++---- 2 files changed, 61 insertions(+), 16 deletions(-) diff --git a/openpype/hosts/flame/api/batch_utils.py b/openpype/hosts/flame/api/batch_utils.py index a1fe7961c4..d309c5985d 100644 --- a/openpype/hosts/flame/api/batch_utils.py +++ b/openpype/hosts/flame/api/batch_utils.py @@ -37,8 +37,12 @@ def create_batch(name, frame_start, frame_end, **kwargs): comp_node = flame.batch.create_node("Comp") + # TODO: convert this to iterational processing, + # so it could be driven from `imageio` settigns # create write node write_node = flame.batch.create_node('Write File') + # assign attrs + write_node.name = write_pref["name"] write_node.media_path = write_pref["media_path"] write_node.media_path_pattern = write_pref["media_path_pattern"] write_node.create_clip = write_pref["create_clip"] @@ -46,11 +50,15 @@ def create_batch(name, frame_start, frame_end, **kwargs): write_node.create_clip_path = write_pref["create_clip_path"] write_node.include_setup_path = write_pref["include_setup_path"] write_node.file_type = write_pref["file_type"] + write_node.format_extension = write_pref["format_extension"] write_node.bit_depth = write_pref["bit_depth"] + write_node.compress = write_pref["compress"] + write_node.compress_mode = write_pref["compress_mode"] write_node.frame_index_mode = write_pref["frame_index_mode"] - write_node.frame_padding = int(write_pref["frame_padding"]) + write_node.frame_padding = write_pref["frame_padding"] + write_node.version_mode = write_pref["version_mode"] + write_node.version_name = write_pref["version_name"] - # connect nodes flame.batch.connect_nodes(comp_node, "Result", write_node, "Front") # sort batch nodes diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py index 780531287b..808c059816 100644 --- a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -1,5 +1,7 @@ import os +from pprint import pformat import pyblish +from openpype.lib import get_workdir import openpype.hosts.flame.api as opfapi @@ -17,25 +19,52 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): frame_end = instance.data["frameEnd"] handle_start = instance.data["handleStart"] handle_end = instance.data["handleEnd"] - asset_name = instance.data["asset"] - write_pref_data = self._get_write_prefs(instance) + add_tasks = instance.data["flameAddTasks"] - batch_data = { - "shematic_reels": [ - "OP_LoadedReel" - ], - "write_pref": write_pref_data, - "handleStart": handle_start, - "handleEnd": handle_end - } + # iterate all tasks from settings + for task_data in add_tasks: + # exclude batch group + if not task_data["create_batch_group"]: + continue + task_name = task_data["name"] + batchgroup_name = "{}_{}".format(asset_name, task_name) + write_pref_data = self._get_write_prefs(instance, task_data) - opfapi.create_batch(asset_name, frame_start, frame_end, batch_data) + batch_data = { + "shematic_reels": [ + "OP_LoadedReel" + ], + "write_pref": write_pref_data, + "handleStart": handle_start, + "handleEnd": handle_end + } + self.log.debug( + "__ batch_data: {}".format(pformat(batch_data))) - def _get_write_prefs(self, instance): - shot_path = instance.data[""] + # create batch with utils + opfapi.create_batch( + batchgroup_name, + frame_start, + frame_end, + batch_data + ) + + def _get_write_prefs(self, instance, task_data): + anatomy_data = instance.data["anatomyData"] + + task_workfile_path = self._get_shot_task_dir_path(instance, task_data) + self.log.debug("__ task_workfile_path: {}".format(task_workfile_path)) + + # TODO: this might be done with template in settings render_dir_path = os.path.join( - shot_path, "work", task, "render", "flame") + task_workfile_path, "render", "flame") + + # TODO: add most of these to `imageio/flame/batch/write_node` + name = "{project[code]}_{asset}_{task[name]}".format( + **anatomy_data + ) + # The path attribute where the rendered clip is exported # /path/to/file.[0001-0010].exr media_path = render_dir_path @@ -83,6 +112,7 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): version_name = "v" return { + "name": name, "media_path": media_path, "media_path_pattern": media_path_pattern, "create_clip": create_clip, @@ -99,3 +129,10 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): "version_mode": version_mode, "version_name": version_name } + + def _get_shot_task_dir_path(self, instance, task_data): + project_doc = instance.data["projectEntity"] + asset_entity = instance.data["assetEntity"] + + return get_workdir( + project_doc, asset_entity, task_data["name"], "flame") From 9fb6d7a7230149b9d1c7b273eb8fcc532a84a3fb Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Mar 2022 12:04:29 +0100 Subject: [PATCH 015/126] flame: batchgroup uses duration --- openpype/hosts/flame/api/batch_utils.py | 14 ++++++-------- .../flame/plugins/publish/integrate_batch_group.py | 5 +++-- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/openpype/hosts/flame/api/batch_utils.py b/openpype/hosts/flame/api/batch_utils.py index d309c5985d..a47d62a10e 100644 --- a/openpype/hosts/flame/api/batch_utils.py +++ b/openpype/hosts/flame/api/batch_utils.py @@ -1,7 +1,7 @@ import flame -def create_batch(name, frame_start, frame_end, **kwargs): +def create_batch(name, frame_start, frame_duration, **kwargs): """Create Batch Group in active project's Desktop Args: @@ -13,20 +13,18 @@ def create_batch(name, frame_start, frame_end, **kwargs): shelf_reels = kwargs.get("shelf_reels") or ['ShelfReel1'] write_pref = kwargs["write_pref"] - handle_start = kwargs.get("handleStart") - handle_end = kwargs.get("handleEnd") + handle_start = kwargs.get("handleStart") or 0 + handle_end = kwargs.get("handleEnd") or 0 - if handle_start: - frame_start -= handle_start - if handle_end: - frame_end += handle_end + frame_start -= handle_start + frame_duration += handle_start + handle_end # Create batch group with name, start_frame value, duration value, # set of schematic reel names, set of shelf reel names flame.batch.create_batch_group( name, start_frame=frame_start, - duration=frame_end, + duration=frame_duration, reels=schematic_reels, shelf_reels=shelf_reels ) diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py index 808c059816..0a21d6ca2d 100644 --- a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -19,6 +19,7 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): frame_end = instance.data["frameEnd"] handle_start = instance.data["handleStart"] handle_end = instance.data["handleEnd"] + frame_duration = (frame_end - frame_start) + 1 asset_name = instance.data["asset"] add_tasks = instance.data["flameAddTasks"] @@ -46,8 +47,8 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): opfapi.create_batch( batchgroup_name, frame_start, - frame_end, - batch_data + frame_duration, + **batch_data ) def _get_write_prefs(self, instance, task_data): From a87f778f1e95d16d09097d923ca3f1d519e86126 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Mar 2022 12:56:18 +0100 Subject: [PATCH 016/126] flame: reuse batch groups --- openpype/hosts/flame/api/__init__.py | 4 ++- openpype/hosts/flame/api/lib.py | 9 +++++++ .../plugins/publish/integrate_batch_group.py | 26 ++++++++++++++----- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/openpype/hosts/flame/api/__init__.py b/openpype/hosts/flame/api/__init__.py index 97f83ccf07..561aaab3de 100644 --- a/openpype/hosts/flame/api/__init__.py +++ b/openpype/hosts/flame/api/__init__.py @@ -29,7 +29,8 @@ from .lib import ( get_frame_from_filename, get_padding_from_filename, maintained_object_duplication, - get_clip_segment + get_clip_segment, + get_batch_group_from_desktop ) from .utils import ( setup, @@ -105,6 +106,7 @@ __all__ = [ "get_padding_from_filename", "maintained_object_duplication", "get_clip_segment", + "get_batch_group_from_desktop", # pipeline "install", diff --git a/openpype/hosts/flame/api/lib.py b/openpype/hosts/flame/api/lib.py index 74d9e7607a..9a6b86209d 100644 --- a/openpype/hosts/flame/api/lib.py +++ b/openpype/hosts/flame/api/lib.py @@ -708,3 +708,12 @@ def get_clip_segment(flame_clip): raise ValueError("Clip `{}` has too many segments!".format(name)) return segments[0] + + +def get_batch_group_from_desktop(name): + project = get_current_project() + project_desktop = project.current_workspace.desktop + + for bgroup in project_desktop.batch_groups: + if bgroup.name.get_value() == name: + return bgroup diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py index 0a21d6ca2d..3a8173791a 100644 --- a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -43,13 +43,25 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): self.log.debug( "__ batch_data: {}".format(pformat(batch_data))) - # create batch with utils - opfapi.create_batch( - batchgroup_name, - frame_start, - frame_duration, - **batch_data - ) + # check if the batch group already exists + bgroup = opfapi.get_batch_group_from_desktop(batchgroup_name) + + if not bgroup: + self.log.info( + "Creating new batch group: {}".format(batchgroup_name)) + # create batch with utils + opfapi.create_batch( + batchgroup_name, + frame_start, + frame_duration, + **batch_data + ) + else: + self.log.info( + "Updating batch group: {}".format(batchgroup_name)) + # update already created batch group + bgroup.start_frame = frame_start + bgroup.duration = frame_duration def _get_write_prefs(self, instance, task_data): anatomy_data = instance.data["anatomyData"] From c26ff2ab544fadc7121d54ec49a2b35433e6122a Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Mar 2022 12:56:36 +0100 Subject: [PATCH 017/126] flame: fix task name on write file node --- openpype/hosts/flame/plugins/publish/integrate_batch_group.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py index 3a8173791a..af2b0fad65 100644 --- a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -65,6 +65,8 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): def _get_write_prefs(self, instance, task_data): anatomy_data = instance.data["anatomyData"] + # update task data in anatomy data + anatomy_data.update(task_data) task_workfile_path = self._get_shot_task_dir_path(instance, task_data) self.log.debug("__ task_workfile_path: {}".format(task_workfile_path)) From 590e966a7d18a4c1f7dea0e08a3056a202607670 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Mar 2022 14:29:49 +0100 Subject: [PATCH 018/126] flame: updating anatomy data with correct task data --- .../plugins/publish/integrate_batch_group.py | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py index af2b0fad65..c54eeec05c 100644 --- a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -1,4 +1,5 @@ import os +import copy from pprint import pformat import pyblish from openpype.lib import get_workdir @@ -63,10 +64,28 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): bgroup.start_frame = frame_start bgroup.duration = frame_duration - def _get_write_prefs(self, instance, task_data): - anatomy_data = instance.data["anatomyData"] + def _get_anamoty_data_with_current_task(self, instance, task_data): + anatomy_data = copy.deepcopy(instance.data["anatomyData"]) + task_name = task_data["name"] + task_type = task_data["type"] + anatomy_obj = instance.context.data["anatomy"] + # update task data in anatomy data - anatomy_data.update(task_data) + project_task_types = anatomy_obj["tasks"] + task_code = project_task_types.get(task_type, {}).get("short_name") + anatomy_data.update({ + "task": { + "name": task_name, + "type": task_type, + "short": task_code + } + }) + return anatomy_data + + def _get_write_prefs(self, instance, task_data): + # update task in anatomy data + anatomy_data = self._get_anamoty_data_with_current_task( + instance, task_data) task_workfile_path = self._get_shot_task_dir_path(instance, task_data) self.log.debug("__ task_workfile_path: {}".format(task_workfile_path)) From d5521ae8407a9f9bb4d1f05e8a1ef048700acf45 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Mar 2022 15:37:05 +0100 Subject: [PATCH 019/126] flame: adding loading of plate into integrator --- openpype/hosts/flame/api/batch_utils.py | 4 +- .../plugins/publish/integrate_batch_group.py | 87 +++++++++++-------- 2 files changed, 53 insertions(+), 38 deletions(-) diff --git a/openpype/hosts/flame/api/batch_utils.py b/openpype/hosts/flame/api/batch_utils.py index a47d62a10e..99e053faf1 100644 --- a/openpype/hosts/flame/api/batch_utils.py +++ b/openpype/hosts/flame/api/batch_utils.py @@ -21,7 +21,7 @@ def create_batch(name, frame_start, frame_duration, **kwargs): # Create batch group with name, start_frame value, duration value, # set of schematic reel names, set of shelf reel names - flame.batch.create_batch_group( + bgroup = flame.batch.create_batch_group( name, start_frame=frame_start, duration=frame_duration, @@ -61,3 +61,5 @@ def create_batch(name, frame_start, frame_duration, **kwargs): # sort batch nodes flame.batch.organize() + + return bgroup diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py index c54eeec05c..97b456c18c 100644 --- a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -16,12 +16,6 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): families = ["clip"] def process(self, instance): - frame_start = instance.data["frameStart"] - frame_end = instance.data["frameEnd"] - handle_start = instance.data["handleStart"] - handle_end = instance.data["handleEnd"] - frame_duration = (frame_end - frame_start) + 1 - asset_name = instance.data["asset"] add_tasks = instance.data["flameAddTasks"] # iterate all tasks from settings @@ -29,40 +23,59 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): # exclude batch group if not task_data["create_batch_group"]: continue - task_name = task_data["name"] - batchgroup_name = "{}_{}".format(asset_name, task_name) - write_pref_data = self._get_write_prefs(instance, task_data) - batch_data = { - "shematic_reels": [ - "OP_LoadedReel" - ], - "write_pref": write_pref_data, - "handleStart": handle_start, - "handleEnd": handle_end - } - self.log.debug( - "__ batch_data: {}".format(pformat(batch_data))) + # create or get already created batch group + bgroup = self._get_batch_group(instance, task_data) - # check if the batch group already exists - bgroup = opfapi.get_batch_group_from_desktop(batchgroup_name) + # load plate to batch group + self.log.info("Loading subset `{}` into batch `{}`".format( + instance.data["subset"], bgroup.name.get_value() + )) - if not bgroup: - self.log.info( - "Creating new batch group: {}".format(batchgroup_name)) - # create batch with utils - opfapi.create_batch( - batchgroup_name, - frame_start, - frame_duration, - **batch_data - ) - else: - self.log.info( - "Updating batch group: {}".format(batchgroup_name)) - # update already created batch group - bgroup.start_frame = frame_start - bgroup.duration = frame_duration + def _get_batch_group(self, instance, task_data): + frame_start = instance.data["frameStart"] + frame_end = instance.data["frameEnd"] + handle_start = instance.data["handleStart"] + handle_end = instance.data["handleEnd"] + frame_duration = (frame_end - frame_start) + 1 + asset_name = instance.data["asset"] + + task_name = task_data["name"] + batchgroup_name = "{}_{}".format(asset_name, task_name) + write_pref_data = self._get_write_prefs(instance, task_data) + + batch_data = { + "shematic_reels": [ + "OP_LoadedReel" + ], + "write_pref": write_pref_data, + "handleStart": handle_start, + "handleEnd": handle_end + } + self.log.debug( + "__ batch_data: {}".format(pformat(batch_data))) + + # check if the batch group already exists + bgroup = opfapi.get_batch_group_from_desktop(batchgroup_name) + + if not bgroup: + self.log.info( + "Creating new batch group: {}".format(batchgroup_name)) + # create batch with utils + bgroup = opfapi.create_batch( + batchgroup_name, + frame_start, + frame_duration, + **batch_data + ) + else: + self.log.info( + "Updating batch group: {}".format(batchgroup_name)) + # update already created batch group + bgroup.start_frame = frame_start + bgroup.duration = frame_duration + + return bgroup def _get_anamoty_data_with_current_task(self, instance, task_data): anatomy_data = copy.deepcopy(instance.data["anatomyData"]) From 38268bc83102964c22db15129516a32ba5d5f455 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Mar 2022 16:28:46 +0100 Subject: [PATCH 020/126] flame: let extractor drive loading to batch group --- .../settings/defaults/project_settings/flame.json | 4 +++- .../projects_schema/schema_project_flame.json | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/openpype/settings/defaults/project_settings/flame.json b/openpype/settings/defaults/project_settings/flame.json index 939752c778..a2b9bef103 100644 --- a/openpype/settings/defaults/project_settings/flame.json +++ b/openpype/settings/defaults/project_settings/flame.json @@ -62,7 +62,9 @@ "ignore_comment_attrs": false, "colorspace_out": "ACES - ACEScg", "representation_add_range": true, - "representation_tags": [] + "representation_tags": [], + "load_to_batch_group": true, + "batch_group_loader_name": "LoadClip" } } } diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_flame.json b/openpype/settings/entities/schemas/projects_schema/schema_project_flame.json index 8057b07d9c..c991577799 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_flame.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_flame.json @@ -302,6 +302,20 @@ "type": "text", "multiline": false } + }, + { + "type": "separator" + }, + { + "type": "boolean", + "key": "load_to_batch_group", + "label": "Load to batch group reel", + "default": false + }, + { + "type": "text", + "key": "batch_group_loader_name", + "label": "Use loader name" } ] } From 0407465ee1a2438d8a84d9d0704bb38dd56c1a2c Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Mar 2022 17:24:21 +0100 Subject: [PATCH 021/126] flame: add loadable arguments to extracted repres --- .../hosts/flame/plugins/publish/extract_subset_resources.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/flame/plugins/publish/extract_subset_resources.py b/openpype/hosts/flame/plugins/publish/extract_subset_resources.py index 32f6b9508f..7c29bcf944 100644 --- a/openpype/hosts/flame/plugins/publish/extract_subset_resources.py +++ b/openpype/hosts/flame/plugins/publish/extract_subset_resources.py @@ -211,7 +211,11 @@ class ExtractSubsetResources(openpype.api.Extractor): "tags": repre_tags, "data": { "colorspace": color_out - } + }, + "load_to_batch_group": preset_config.get( + "load_to_batch_group"), + "batch_group_loader_name": preset_config.get( + "batch_group_loader_name") } # collect all available content of export dir From 638864150493e0f46b2fd41a6fbe0609434dc536 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Mar 2022 17:24:43 +0100 Subject: [PATCH 022/126] flame: finalize loading procedure in batch integrator --- .../plugins/publish/integrate_batch_group.py | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py index 97b456c18c..62211d7ace 100644 --- a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -4,6 +4,7 @@ from pprint import pformat import pyblish from openpype.lib import get_workdir import openpype.hosts.flame.api as opfapi +import openpype.pipeline as op_pipeline @pyblish.api.log @@ -15,6 +16,9 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): hosts = ["flame"] families = ["clip"] + # settings + default_loader = "LoadClip" + def process(self, instance): add_tasks = instance.data["flameAddTasks"] @@ -31,6 +35,77 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): self.log.info("Loading subset `{}` into batch `{}`".format( instance.data["subset"], bgroup.name.get_value() )) + self._load_clip_to_context(instance, bgroup) + + def _load_clip_to_context(self, instance, bgroup): + # get all loaders for host + loaders = op_pipeline.discover_loader_plugins() + + # get all published representations + published_representations = instance.data["published_representations"] + + # get all loadable representations + representations = instance.data["representations"] + + # get repre_id for the loadable representations + loadable_representations = [ + { + "name": _repr["name"], + "loader": _repr.get("batch_group_loader_name"), + # match loader to the loadable representation + "_id": next( + ( + id + for id, repr in published_representations.items() + if repr["representation"]["name"] == _repr["name"] + ), + None + ) + } + for _repr in representations + if _repr.get("load_to_batch_group") is not None + ] + + # get representation context from the repre_id + representation_ids = [ + repre["_id"] + for repre in loadable_representations + if repre["_id"] is not None + ] + repre_contexts = op_pipeline.load.get_repres_contexts( + representation_ids) + + # loop all returned repres from repre_context dict + for repre_id, repre_context in repre_contexts.items(): + # get loader name by representation id + loader_name = next( + ( + repr["loader"] + for repr in loadable_representations + if repr["_id"] == repre_id + ), + self.default_loader + ) + # get loader plugin + Loader = next( + ( + loader_plugin + for loader_plugin in loaders + if loader_plugin.__name__ == loader_name + ), + None + ) + if Loader: + # load to flame by representation context + op_pipeline.load.load_with_repre_context(Loader, repre_context) + else: + self.log.warning( + "Something got wrong and there is not Loader found for " + "following data: {}".format( + pformat(loadable_representations)) + ) + + def _get_batch_group(self, instance, task_data): frame_start = instance.data["frameStart"] From cde1caaa9180fcc7e4165995e9760429f2a55e07 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Mar 2022 17:40:27 +0100 Subject: [PATCH 023/126] flame: clean args types --- .../publish/extract_subset_resources.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/openpype/hosts/flame/plugins/publish/extract_subset_resources.py b/openpype/hosts/flame/plugins/publish/extract_subset_resources.py index 7c29bcf944..00b87c05a0 100644 --- a/openpype/hosts/flame/plugins/publish/extract_subset_resources.py +++ b/openpype/hosts/flame/plugins/publish/extract_subset_resources.py @@ -108,6 +108,18 @@ class ExtractSubsetResources(openpype.api.Extractor): ignore_comment_attrs = preset_config["ignore_comment_attrs"] color_out = preset_config["colorspace_out"] + # get attribures related loading in integrate_batch_group + load_to_batch_group = preset_config.get( + "load_to_batch_group") + batch_group_loader_name = preset_config.get( + "batch_group_loader_name") + + # convert to None if empty string + if batch_group_loader_name: + batch_group_loader_name = str(batch_group_loader_name) + if batch_group_loader_name == "": + batch_group_loader_name = None + # get frame range with handles for representation range frame_start_handle = frame_start - handle_start source_duration_handles = ( @@ -212,10 +224,8 @@ class ExtractSubsetResources(openpype.api.Extractor): "data": { "colorspace": color_out }, - "load_to_batch_group": preset_config.get( - "load_to_batch_group"), - "batch_group_loader_name": preset_config.get( - "batch_group_loader_name") + "load_to_batch_group": load_to_batch_group, + "batch_group_loader_name": batch_group_loader_name } # collect all available content of export dir From 4cfd22b6393b3a4d7e5c18046f7d0340ce124e27 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Mar 2022 17:47:06 +0100 Subject: [PATCH 024/126] flame: improving loading with exception --- .../hosts/flame/plugins/publish/integrate_batch_group.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py index 62211d7ace..08632c3018 100644 --- a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -97,7 +97,13 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): ) if Loader: # load to flame by representation context - op_pipeline.load.load_with_repre_context(Loader, repre_context) + try: + op_pipeline.load.load_with_repre_context( + Loader, repre_context) + except op_pipeline.load.IncompatibleLoaderError as msg: + self.log.error( + "Check allowed representations for Loader `{}` " + "in settings > error: {}".format(Loader.__name__, msg)) else: self.log.warning( "Something got wrong and there is not Loader found for " From a4f8cdb76962f6f4c9e4efef49079c3926e486e6 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Mar 2022 17:51:22 +0100 Subject: [PATCH 025/126] flame: better logging for loading fail --- .../hosts/flame/plugins/publish/integrate_batch_group.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py index 08632c3018..f1049e4697 100644 --- a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -103,7 +103,14 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): except op_pipeline.load.IncompatibleLoaderError as msg: self.log.error( "Check allowed representations for Loader `{}` " - "in settings > error: {}".format(Loader.__name__, msg)) + "in settings > error: {}".format( + Loader.__name__, msg)) + self.log.error( + "Representaton context >>{}<< is not compatible " + "with loader `{}`".format( + pformat(repre_context), Loader.__name__ + ) + ) else: self.log.warning( "Something got wrong and there is not Loader found for " From 585d53deee223f359e6732620ea0188f8d00ec5c Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 25 Mar 2022 15:21:22 +0100 Subject: [PATCH 026/126] flame: improving loading in integrate batch plugin --- .../plugins/publish/extract_subset_resources.py | 6 ++---- .../flame/plugins/publish/integrate_batch_group.py | 14 +++++++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/openpype/hosts/flame/plugins/publish/extract_subset_resources.py b/openpype/hosts/flame/plugins/publish/extract_subset_resources.py index 00b87c05a0..31f7b6d574 100644 --- a/openpype/hosts/flame/plugins/publish/extract_subset_resources.py +++ b/openpype/hosts/flame/plugins/publish/extract_subset_resources.py @@ -115,10 +115,8 @@ class ExtractSubsetResources(openpype.api.Extractor): "batch_group_loader_name") # convert to None if empty string - if batch_group_loader_name: - batch_group_loader_name = str(batch_group_loader_name) - if batch_group_loader_name == "": - batch_group_loader_name = None + if batch_group_loader_name == "": + batch_group_loader_name = None # get frame range with handles for representation range frame_start_handle = frame_start - handle_start diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py index f1049e4697..81b304ff0b 100644 --- a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -66,6 +66,9 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): if _repr.get("load_to_batch_group") is not None ] + self.log.debug("__ loadable_representations: {}".format(pformat( + loadable_representations))) + # get representation context from the repre_id representation_ids = [ repre["_id"] @@ -75,17 +78,20 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): repre_contexts = op_pipeline.load.get_repres_contexts( representation_ids) + self.log.debug("__ repre_contexts: {}".format(pformat( + repre_contexts))) + # loop all returned repres from repre_context dict for repre_id, repre_context in repre_contexts.items(): + self.log.debug("__ repre_id: {}".format(repre_id)) # get loader name by representation id loader_name = next( ( repr["loader"] for repr in loadable_representations if repr["_id"] == repre_id - ), - self.default_loader - ) + )) or self.default_loader + # get loader plugin Loader = next( ( @@ -118,8 +124,6 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): pformat(loadable_representations)) ) - - def _get_batch_group(self, instance, task_data): frame_start = instance.data["frameStart"] frame_end = instance.data["frameEnd"] From ed4388184ad768dbf00ce050efff8eaf11d3cf7c Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 25 Mar 2022 15:56:12 +0100 Subject: [PATCH 027/126] flame: adding clip loader to current batch --- .../flame/plugins/load/load_clip_batch.py | 135 ++++++++++++++++++ .../defaults/project_settings/flame.json | 22 +++ .../projects_schema/schema_project_flame.json | 42 ++++++ 3 files changed, 199 insertions(+) create mode 100644 openpype/hosts/flame/plugins/load/load_clip_batch.py diff --git a/openpype/hosts/flame/plugins/load/load_clip_batch.py b/openpype/hosts/flame/plugins/load/load_clip_batch.py new file mode 100644 index 0000000000..81af34744e --- /dev/null +++ b/openpype/hosts/flame/plugins/load/load_clip_batch.py @@ -0,0 +1,135 @@ +import os +import flame +from pprint import pformat +import openpype.hosts.flame.api as opfapi + + +class LoadClipBatch(opfapi.ClipLoader): + """Load a subset to timeline as clip + + Place clip to timeline on its asset origin timings collected + during conforming to project + """ + + families = ["render2d", "source", "plate", "render", "review"] + representations = ["exr", "dpx", "jpg", "jpeg", "png", "h264"] + + label = "Load as clip to current batch" + order = -10 + icon = "code-fork" + color = "orange" + + # settings + reel_name = "OP_LoadedReel" + clip_name_template = "{asset}_{subset}_{representation}" + + def load(self, context, name, namespace, options): + + # get flame objects + self.batch = flame.batch + + # load clip to timeline and get main variables + namespace = namespace + version = context['version'] + version_data = version.get("data", {}) + version_name = version.get("name", None) + colorspace = version_data.get("colorspace", None) + clip_name = self.clip_name_template.format( + **context["representation"]["context"]) + + # todo: settings in imageio + # convert colorspace with ocio to flame mapping + # in imageio flame section + colorspace = colorspace + + # create workfile path + workfile_dir = os.environ["AVALON_WORKDIR"] + openclip_dir = os.path.join( + workfile_dir, clip_name + ) + openclip_path = os.path.join( + openclip_dir, clip_name + ".clip" + ) + if not os.path.exists(openclip_dir): + os.makedirs(openclip_dir) + + # prepare clip data from context ad send it to openClipLoader + loading_context = { + "path": self.fname.replace("\\", "/"), + "colorspace": colorspace, + "version": "v{:0>3}".format(version_name), + "logger": self.log + + } + self.log.debug(pformat( + loading_context + )) + self.log.debug(openclip_path) + + # make openpype clip file + opfapi.OpenClipSolver(openclip_path, loading_context).make() + + # prepare Reel group in actual desktop + opc = self._get_clip( + clip_name, + openclip_path + ) + + # add additional metadata from the version to imprint Avalon knob + add_keys = [ + "frameStart", "frameEnd", "source", "author", + "fps", "handleStart", "handleEnd" + ] + + # move all version data keys to tag data + data_imprint = { + key: version_data.get(key, str(None)) + for key in add_keys + } + # add variables related to version context + data_imprint.update({ + "version": version_name, + "colorspace": colorspace, + "objectName": clip_name + }) + + # TODO: finish the containerisation + # opc_segment = opfapi.get_clip_segment(opc) + + # return opfapi.containerise( + # opc_segment, + # name, namespace, context, + # self.__class__.__name__, + # data_imprint) + + return opc + + def _get_clip(self, name, clip_path): + reel = self._get_reel() + + # with maintained openclip as opc + matching_clip = next( + ( + cl for cl in reel.clips + if cl.name.get_value() == name + ) + ) + + if not matching_clip: + created_clips = flame.import_clips(str(clip_path), reel) + return created_clips.pop() + + return matching_clip + + def _get_reel(self): + + matching_reel = [ + rg for rg in self.batch.reels + if rg.name.get_value() == self.reel_name + ] + + return ( + matching_reel.pop() + if matching_reel + else self.batch.create_reel(str(self.reel_name)) + ) diff --git a/openpype/settings/defaults/project_settings/flame.json b/openpype/settings/defaults/project_settings/flame.json index a2b9bef103..afd0834c9d 100644 --- a/openpype/settings/defaults/project_settings/flame.json +++ b/openpype/settings/defaults/project_settings/flame.json @@ -92,6 +92,28 @@ "reel_group_name": "OpenPype_Reels", "reel_name": "Loaded", "clip_name_template": "{asset}_{subset}_{representation}" + }, + "LoadClipBatch": { + "enabled": true, + "families": [ + "render2d", + "source", + "plate", + "render", + "review" + ], + "representations": [ + "exr", + "dpx", + "jpg", + "jpeg", + "png", + "h264", + "mov", + "mp4" + ], + "reel_name": "OP_LoadedReel", + "clip_name_template": "{asset}_{subset}_{representation}" } } } \ No newline at end of file diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_flame.json b/openpype/settings/entities/schemas/projects_schema/schema_project_flame.json index c991577799..fe11d63ac2 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_flame.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_flame.json @@ -376,6 +376,48 @@ "label": "Clip name template" } ] + }, + { + "type": "dict", + "collapsible": true, + "key": "LoadClipBatch", + "label": "Load as clip to current batch", + "checkbox_key": "enabled", + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + }, + { + "type": "list", + "key": "families", + "label": "Families", + "object_type": "text" + }, + { + "type": "list", + "key": "representations", + "label": "Representations", + "object_type": "text" + }, + { + "type": "separator" + }, + { + "type": "text", + "key": "reel_name", + "label": "Reel name" + }, + { + "type": "separator" + }, + { + "type": "text", + "key": "clip_name_template", + "label": "Clip name template" + } + ] } ] } From 267a3e04ed4fda8c3837af28379e3a8812312fb2 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 25 Mar 2022 16:52:03 +0100 Subject: [PATCH 028/126] flame: improving batch attributes --- openpype/hosts/flame/api/batch_utils.py | 1 + openpype/hosts/flame/plugins/load/load_clip_batch.py | 11 +++++------ .../flame/plugins/publish/integrate_batch_group.py | 4 +++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/openpype/hosts/flame/api/batch_utils.py b/openpype/hosts/flame/api/batch_utils.py index 99e053faf1..43742c6e4f 100644 --- a/openpype/hosts/flame/api/batch_utils.py +++ b/openpype/hosts/flame/api/batch_utils.py @@ -56,6 +56,7 @@ def create_batch(name, frame_start, frame_duration, **kwargs): write_node.frame_padding = write_pref["frame_padding"] write_node.version_mode = write_pref["version_mode"] write_node.version_name = write_pref["version_name"] + write_node.version_padding = write_pref["version_padding"] flame.batch.connect_nodes(comp_node, "Result", write_node, "Front") diff --git a/openpype/hosts/flame/plugins/load/load_clip_batch.py b/openpype/hosts/flame/plugins/load/load_clip_batch.py index 81af34744e..bf0bbb5168 100644 --- a/openpype/hosts/flame/plugins/load/load_clip_batch.py +++ b/openpype/hosts/flame/plugins/load/load_clip_batch.py @@ -108,12 +108,11 @@ class LoadClipBatch(opfapi.ClipLoader): reel = self._get_reel() # with maintained openclip as opc - matching_clip = next( - ( - cl for cl in reel.clips - if cl.name.get_value() == name - ) - ) + matching_clip = None + for cl in reel.clips: + if cl.name.get_value() != name: + continue + matching_clip = cl if not matching_clip: created_clips = flame.import_clips(str(clip_path), reel) diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py index 81b304ff0b..536bf0d807 100644 --- a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -249,6 +249,7 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): # Only available if create_clip = True. version_mode = "Follow Iteration" version_name = "v" + version_padding = 3 return { "name": name, @@ -266,7 +267,8 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): "frame_index_mode": frame_index_mode, "frame_padding": frame_padding, "version_mode": version_mode, - "version_name": version_name + "version_name": version_name, + "version_padding": version_padding } def _get_shot_task_dir_path(self, instance, task_data): From f8e99f38c97cf37b8001e4c5848d93e00a7f9107 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 25 Mar 2022 16:53:47 +0100 Subject: [PATCH 029/126] flame: make dirs for batch renders add one more directory layer for renders --- .../hosts/flame/plugins/publish/integrate_batch_group.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py index 536bf0d807..eaab429111 100644 --- a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -199,6 +199,9 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): render_dir_path = os.path.join( task_workfile_path, "render", "flame") + if not os.path.exists(render_dir_path): + os.makedirs(render_dir_path, mode=0o777) + # TODO: add most of these to `imageio/flame/batch/write_node` name = "{project[code]}_{asset}_{task[name]}".format( **anatomy_data @@ -208,7 +211,7 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): # /path/to/file.[0001-0010].exr media_path = render_dir_path # name of file represented by tokens - media_path_pattern = "_v." + media_path_pattern = "_v/_v." # The Create Open Clip attribute of the Write File node. \ # Determines if an Open Clip is created by the Write File node. create_clip = True From 08f80ecf15911f1e96808fd8c6032b55d4f596e7 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 28 Mar 2022 13:29:16 +0200 Subject: [PATCH 030/126] flame: make sure only one clip in xml --- openpype/hosts/flame/api/plugin.py | 59 ++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 16 deletions(-) diff --git a/openpype/hosts/flame/api/plugin.py b/openpype/hosts/flame/api/plugin.py index 4c9d3c5383..3a322e5208 100644 --- a/openpype/hosts/flame/api/plugin.py +++ b/openpype/hosts/flame/api/plugin.py @@ -1,24 +1,22 @@ +import itertools import os import re import shutil import sys -from xml.etree import ElementTree as ET -import six -import qargparse -from Qt import QtWidgets, QtCore -import openpype.api as openpype -from openpype.pipeline import ( - LegacyCreator, - LoaderPlugin, -) -from openpype import style -from . import ( - lib as flib, - pipeline as fpipeline, - constants -) - +import xml.etree.cElementTree as cET from copy import deepcopy +from xml.etree import ElementTree as ET + +import openpype.api as openpype +import qargparse +import six +from openpype import style +from openpype.pipeline import LegacyCreator, LoaderPlugin +from Qt import QtCore, QtWidgets + +from . import constants +from . import lib as flib +from . import pipeline as fpipeline log = openpype.Logger.get_logger(__name__) @@ -749,10 +747,39 @@ class OpenClipSolver: # execute creation of clip xml template data try: openpype.run_subprocess(cmd_args) + self._make_single_clip_media_info() except TypeError: self.log.error("Error creating self.tmp_file") six.reraise(*sys.exc_info()) + def _make_single_clip_media_info(self): + with open(self.tmp_file) as f: + lines = f.readlines() + _added_root = itertools.chain( + "", deepcopy(lines)[1:], "") + new_root = ET.fromstringlist(_added_root) + + # find the clip which is matching to my input name + xml_clips = new_root.findall("clip") + matching_clip = None + for xml_clip in xml_clips: + if xml_clip.find("name").text == self.feed_basename: + matching_clip = xml_clip + + if not matching_clip: + # return warning there is missing clip + raise ET.ParseError( + "Missing clip in `{}`. Available clips {}".format( + self.feed_basename, [ + xml_clip.find("name").text + for xml_clip in xml_clips + ] + )) + # save it as new file + tree = cET.ElementTree(matching_clip) + tree.write(self.tmp_file, xml_declaration=True, + method='xml', encoding='UTF-8') + def _clear_tmp_file(self): if os.path.isfile(self.tmp_file): os.remove(self.tmp_file) From 34a65cb646e5d267899f3f3df5eed4de72ac2074 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 28 Mar 2022 13:29:40 +0200 Subject: [PATCH 031/126] flame: ignore clip file with zero lines --- openpype/hosts/flame/api/plugin.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/flame/api/plugin.py b/openpype/hosts/flame/api/plugin.py index 3a322e5208..949e8ad406 100644 --- a/openpype/hosts/flame/api/plugin.py +++ b/openpype/hosts/flame/api/plugin.py @@ -708,19 +708,32 @@ class OpenClipSolver: self.feed_dir = os.path.dirname(feed_path) self.feed_ext = os.path.splitext(self.feed_basename)[1][1:].lower() - if not os.path.isfile(openclip_file_path): + if not self._is_valid_tmp_file(openclip_file_path): # openclip does not exist yet and will be created self.tmp_file = self.out_file = openclip_file_path self.create_new_clip = True else: + # update already created clip # output a temp file self.out_file = openclip_file_path self.tmp_file = os.path.join(self.feed_dir, self.tmp_name) + + # remove previously generated temp files + # it will be regenerated self._clear_tmp_file() self.log.info("Temp File: {}".format(self.tmp_file)) + def _is_valid_tmp_file(self, file): + # check if file exists + if os.path.isfile(file): + with open(self.tmp_file) as f: + lines = f.readlines() + if len(lines) < 1: + self._clear_tmp_file() + return False + def make(self): self._generate_media_info_file() From 1c6ab37f351e87a0a4a01a93eb14de47668333f6 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 28 Mar 2022 20:02:58 +0200 Subject: [PATCH 032/126] flame: improving tmp file validation --- openpype/hosts/flame/api/plugin.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/flame/api/plugin.py b/openpype/hosts/flame/api/plugin.py index 949e8ad406..ab60bbad11 100644 --- a/openpype/hosts/flame/api/plugin.py +++ b/openpype/hosts/flame/api/plugin.py @@ -730,9 +730,12 @@ class OpenClipSolver: if os.path.isfile(file): with open(self.tmp_file) as f: lines = f.readlines() - if len(lines) < 1: - self._clear_tmp_file() - return False + if len(lines) > 2: + return True + + # file is probably corrupted + self._clear_tmp_file() + return False def make(self): self._generate_media_info_file() From 0abc8ae61367a3ee03a896704557b324ffc1e1bd Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 28 Mar 2022 20:18:59 +0200 Subject: [PATCH 033/126] flame: rework xml write file --- openpype/hosts/flame/api/plugin.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/openpype/hosts/flame/api/plugin.py b/openpype/hosts/flame/api/plugin.py index ab60bbad11..0eba06a86d 100644 --- a/openpype/hosts/flame/api/plugin.py +++ b/openpype/hosts/flame/api/plugin.py @@ -791,10 +791,8 @@ class OpenClipSolver: for xml_clip in xml_clips ] )) - # save it as new file - tree = cET.ElementTree(matching_clip) - tree.write(self.tmp_file, xml_declaration=True, - method='xml', encoding='UTF-8') + + self._write_result_xml_to_file(self.tmp_file, matching_clip) def _clear_tmp_file(self): if os.path.isfile(self.tmp_file): @@ -901,7 +899,7 @@ class OpenClipSolver: self.log.info("Adding feed version: {}".format( self.feed_version_name)) - self._write_result_xml_to_file(xml_data) + self._write_result_xml_to_file(self.out_file, xml_data) self.log.info("openClip Updated: {}".format(self.out_file)) @@ -940,9 +938,11 @@ class OpenClipSolver: self._clear_handler(xml_root) return ET.tostring(xml_root).decode('utf-8') - def _write_result_xml_to_file(self, xml_data): - with open(self.out_file, "w") as f: - f.write(xml_data) + def _write_result_xml_to_file(self, file, xml_data): + # save it as new file + tree = cET.ElementTree(xml_data) + tree.write(file, xml_declaration=True, + method='xml', encoding='UTF-8') def _create_openclip_backup_file(self, file): bck_file = "{}.bak".format(file) From ae36d089690f9acb078cc185e5315667523669dc Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 28 Mar 2022 20:54:12 +0200 Subject: [PATCH 034/126] flame: little fixes of loading --- openpype/hosts/flame/api/plugin.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/flame/api/plugin.py b/openpype/hosts/flame/api/plugin.py index 0eba06a86d..3673dc6671 100644 --- a/openpype/hosts/flame/api/plugin.py +++ b/openpype/hosts/flame/api/plugin.py @@ -728,13 +728,14 @@ class OpenClipSolver: def _is_valid_tmp_file(self, file): # check if file exists if os.path.isfile(file): - with open(self.tmp_file) as f: + # test also if file is not empty + with open(file) as f: lines = f.readlines() if len(lines) > 2: return True # file is probably corrupted - self._clear_tmp_file() + os.remove(file) return False def make(self): @@ -779,7 +780,7 @@ class OpenClipSolver: xml_clips = new_root.findall("clip") matching_clip = None for xml_clip in xml_clips: - if xml_clip.find("name").text == self.feed_basename: + if xml_clip.find("name").text in self.feed_basename: matching_clip = xml_clip if not matching_clip: From 2bf75d270a3fbfa0054d750159439a52e1f0369f Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 28 Mar 2022 21:05:35 +0200 Subject: [PATCH 035/126] flame: fix loaded name to `output` make condition for fix if output is not in context data --- openpype/hosts/flame/plugins/load/load_clip.py | 2 +- openpype/hosts/flame/plugins/load/load_clip_batch.py | 7 ++++++- openpype/settings/defaults/project_settings/flame.json | 4 ++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/openpype/hosts/flame/plugins/load/load_clip.py b/openpype/hosts/flame/plugins/load/load_clip.py index 8980f72cb8..b27600db1f 100644 --- a/openpype/hosts/flame/plugins/load/load_clip.py +++ b/openpype/hosts/flame/plugins/load/load_clip.py @@ -22,7 +22,7 @@ class LoadClip(opfapi.ClipLoader): # settings reel_group_name = "OpenPype_Reels" reel_name = "Loaded" - clip_name_template = "{asset}_{subset}_{representation}" + clip_name_template = "{asset}_{subset}_{output}" def load(self, context, name, namespace, options): diff --git a/openpype/hosts/flame/plugins/load/load_clip_batch.py b/openpype/hosts/flame/plugins/load/load_clip_batch.py index bf0bbb5168..1f87f94cc6 100644 --- a/openpype/hosts/flame/plugins/load/load_clip_batch.py +++ b/openpype/hosts/flame/plugins/load/load_clip_batch.py @@ -21,7 +21,7 @@ class LoadClipBatch(opfapi.ClipLoader): # settings reel_name = "OP_LoadedReel" - clip_name_template = "{asset}_{subset}_{representation}" + clip_name_template = "{asset}_{subset}_{output}" def load(self, context, name, namespace, options): @@ -34,6 +34,11 @@ class LoadClipBatch(opfapi.ClipLoader): version_data = version.get("data", {}) version_name = version.get("name", None) colorspace = version_data.get("colorspace", None) + + # in case output is not in context replace key to representation + if not context["representation"]["context"].get("output"): + self.clip_name_template.replace("output", "representation") + clip_name = self.clip_name_template.format( **context["representation"]["context"]) diff --git a/openpype/settings/defaults/project_settings/flame.json b/openpype/settings/defaults/project_settings/flame.json index afd0834c9d..ef7a2a4467 100644 --- a/openpype/settings/defaults/project_settings/flame.json +++ b/openpype/settings/defaults/project_settings/flame.json @@ -91,7 +91,7 @@ ], "reel_group_name": "OpenPype_Reels", "reel_name": "Loaded", - "clip_name_template": "{asset}_{subset}_{representation}" + "clip_name_template": "{asset}_{subset}_{output}" }, "LoadClipBatch": { "enabled": true, @@ -113,7 +113,7 @@ "mp4" ], "reel_name": "OP_LoadedReel", - "clip_name_template": "{asset}_{subset}_{representation}" + "clip_name_template": "{asset}_{subset}_{output}" } } } \ No newline at end of file From eda39b5de29e9bce283a3326427db8508d2cfb05 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 28 Mar 2022 21:12:56 +0200 Subject: [PATCH 036/126] flame: fix write to xml file input args --- openpype/hosts/flame/api/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/flame/api/plugin.py b/openpype/hosts/flame/api/plugin.py index 3673dc6671..750609f7d6 100644 --- a/openpype/hosts/flame/api/plugin.py +++ b/openpype/hosts/flame/api/plugin.py @@ -829,7 +829,7 @@ class OpenClipSolver: xml_data = self._fix_xml_data(tmp_xml) self.log.info("Adding feed version: {}".format(self.feed_basename)) - self._write_result_xml_to_file(xml_data) + self._write_result_xml_to_file(self.out_file, xml_data) self.log.info("openClip Updated: {}".format(self.tmp_file)) From 44257be4863cc0eb0522ef14aa431bb10344c14c Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 29 Mar 2022 08:43:38 +0200 Subject: [PATCH 037/126] flame: fix utf8 error `'unicode' object has no attribute 'getiterator'` --- openpype/hosts/flame/api/plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/flame/api/plugin.py b/openpype/hosts/flame/api/plugin.py index 750609f7d6..d5790d2f10 100644 --- a/openpype/hosts/flame/api/plugin.py +++ b/openpype/hosts/flame/api/plugin.py @@ -783,7 +783,7 @@ class OpenClipSolver: if xml_clip.find("name").text in self.feed_basename: matching_clip = xml_clip - if not matching_clip: + if matching_clip is not None: # return warning there is missing clip raise ET.ParseError( "Missing clip in `{}`. Available clips {}".format( @@ -937,7 +937,7 @@ class OpenClipSolver: def _fix_xml_data(self, xml_data): xml_root = xml_data.getroot() self._clear_handler(xml_root) - return ET.tostring(xml_root).decode('utf-8') + return xml_root def _write_result_xml_to_file(self, file, xml_data): # save it as new file From 69f5ace08485f0aea46a586602629a21416b779c Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 29 Mar 2022 08:46:50 +0200 Subject: [PATCH 038/126] flame: fix condition direction --- openpype/hosts/flame/api/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/flame/api/plugin.py b/openpype/hosts/flame/api/plugin.py index d5790d2f10..464f5ce89b 100644 --- a/openpype/hosts/flame/api/plugin.py +++ b/openpype/hosts/flame/api/plugin.py @@ -783,7 +783,7 @@ class OpenClipSolver: if xml_clip.find("name").text in self.feed_basename: matching_clip = xml_clip - if matching_clip is not None: + if matching_clip is None: # return warning there is missing clip raise ET.ParseError( "Missing clip in `{}`. Available clips {}".format( From 3459cec3a9adf5537d11c5963e7b33ec9b5d5c2b Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 29 Mar 2022 10:29:17 +0200 Subject: [PATCH 039/126] flame: task workdir for .clip when integrating batch --- openpype/hosts/flame/plugins/load/load_clip_batch.py | 2 +- .../flame/plugins/publish/integrate_batch_group.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/openpype/hosts/flame/plugins/load/load_clip_batch.py b/openpype/hosts/flame/plugins/load/load_clip_batch.py index 1f87f94cc6..252c92516d 100644 --- a/openpype/hosts/flame/plugins/load/load_clip_batch.py +++ b/openpype/hosts/flame/plugins/load/load_clip_batch.py @@ -48,7 +48,7 @@ class LoadClipBatch(opfapi.ClipLoader): colorspace = colorspace # create workfile path - workfile_dir = os.environ["AVALON_WORKDIR"] + workfile_dir = options.get("workdir") or os.environ["AVALON_WORKDIR"] openclip_dir = os.path.join( workfile_dir, clip_name ) diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py index eaab429111..7c61ed62b5 100644 --- a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -105,7 +105,9 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): # load to flame by representation context try: op_pipeline.load.load_with_repre_context( - Loader, repre_context) + Loader, repre_context, **{ + "data": {"workdir": self.task_workdir} + }) except op_pipeline.load.IncompatibleLoaderError as msg: self.log.error( "Check allowed representations for Loader `{}` " @@ -192,12 +194,14 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): anatomy_data = self._get_anamoty_data_with_current_task( instance, task_data) - task_workfile_path = self._get_shot_task_dir_path(instance, task_data) - self.log.debug("__ task_workfile_path: {}".format(task_workfile_path)) + self.task_workdir = self._get_shot_task_dir_path( + instance, task_data) + self.log.debug("__ task_workdir: {}".format( + self.task_workdir)) # TODO: this might be done with template in settings render_dir_path = os.path.join( - task_workfile_path, "render", "flame") + self.task_workdir, "render", "flame") if not os.path.exists(render_dir_path): os.makedirs(render_dir_path, mode=0o777) From fe11ad9868cca57c6bc9ff34e8011beaba7989f4 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Wed, 30 Mar 2022 10:06:50 +0200 Subject: [PATCH 040/126] Remove unused website docs pages --- website/docs/api.md | 7 ------- website/docs/artist_hosts.md | 17 ----------------- website/docs/hosts-maya.md | 33 --------------------------------- 3 files changed, 57 deletions(-) delete mode 100644 website/docs/api.md delete mode 100644 website/docs/artist_hosts.md delete mode 100644 website/docs/hosts-maya.md diff --git a/website/docs/api.md b/website/docs/api.md deleted file mode 100644 index 7cad92d603..0000000000 --- a/website/docs/api.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -id: api -title: Pype API -sidebar_label: API ---- - -Work in progress diff --git a/website/docs/artist_hosts.md b/website/docs/artist_hosts.md deleted file mode 100644 index 609f6d97c8..0000000000 --- a/website/docs/artist_hosts.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -id: artist_hosts -title: Hosts -sidebar_label: Hosts ---- - -## Maya - -## Houdini - -## Nuke - -## Fusion - -## Unreal - -## System diff --git a/website/docs/hosts-maya.md b/website/docs/hosts-maya.md deleted file mode 100644 index 0ee0c2d86b..0000000000 --- a/website/docs/hosts-maya.md +++ /dev/null @@ -1,33 +0,0 @@ -### Tools -Creator -Publisher -Loader -Scene Inventory -Look assigner -Workfiles - -### Plugins -Deadline -Muster -Yeti -Arnold -Vray -Redshift - -### Families -Model -Look -Rig -Animation -Cache -Camera -Assembly -MayaAscii (generic scene) -Setdress -RenderSetup -Review -arnoldStandin -vrayProxy -vrayScene -yetiCache -yetiRig From ad7578fc7339ad2aaa885d616dc3e07a7a2df937 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Wed, 30 Mar 2022 10:10:49 +0200 Subject: [PATCH 041/126] Remove unused `manager_naming.md` --- website/docs/manager_naming.md | 56 ---------------------------------- 1 file changed, 56 deletions(-) delete mode 100644 website/docs/manager_naming.md diff --git a/website/docs/manager_naming.md b/website/docs/manager_naming.md deleted file mode 100644 index bf822fbeb4..0000000000 --- a/website/docs/manager_naming.md +++ /dev/null @@ -1,56 +0,0 @@ ---- -id: manager_naming -title: Naming Conventions -sidebar_label: Naming Conventions ---- - -:::note -This naming convention holds true for most of our pipeline. Please match it as close as possible even for projects and files that might be outside of pipeline scope at this point. Small errors count! The reason for given formatting is to allow people to understand the file at glance and that a script or a program can easily get meaningful information from your files without errors. -::: - -## General rules - -For more detailed rules and different file types, have a look at naming conventions for scenes and assets - -- Every file starts with file code based on a project it belongs to e.g. ‘tst_’, ‘drm_’ -- Optional subversion and comment always comes after the major version. v##.subversion_comment. -- File names can only be composed of letters, numbers, underscores `_` and dots “.” -- You can use snakeCase or CamelCase if you need more words in a section.  thisIsLongerSentenceInComment -- No spaces in filenames. Ever! -- Frame numbers are always separated by a period ”.” -- If you're not sure use this template: - -## Work files - -**`{code}_{shot}_{task}_v001.ext`** - -**`{code}_{asset}_{task}_v001.ext`** - -**Examples:** - - prj_sh010_enviro_v001.ma - prj_sh010_animation_v001.ma - prj_sh010_comp_v001.nk - - prj_bob_modelling_v001.ma - prj_bob_rigging_v001.ma - prj_bob_lookdev_v001.ma - -:::info -In all of the examples anything enclosed in curly brackets  { } is compulsory in the name. -Anything in square brackets [ ] is optional. -::: - -## Published Assets - -**`{code}_{asset}_{family}_{subset}_{version}_[comment].ext`** - -**Examples:** - - prj_bob_model_main_v01.ma - prj_bob_model_hires_v01.ma - prj_bob_model_main_v01_clothes.ma - prj_bob_model_main_v01_body.ma - prj_bob_rig_main_v01.ma - Prj_bob_look_main_v01.ma - Prj_bob_look_wet_v01.ma From df6499868bca5b0a3eea579591f309b3e78e1f59 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 30 Mar 2022 12:22:43 +0200 Subject: [PATCH 042/126] flame: cleaning code --- openpype/hosts/flame/api/__init__.py | 4 ---- openpype/hosts/flame/api/lib.py | 17 +---------------- openpype/hosts/flame/api/scripts/wiretap_com.py | 2 +- openpype/hosts/flame/plugins/load/load_clip.py | 2 +- .../hosts/flame/plugins/load/load_clip_batch.py | 2 +- .../plugins/publish/integrate_batch_group.py | 3 +++ 6 files changed, 7 insertions(+), 23 deletions(-) diff --git a/openpype/hosts/flame/api/__init__.py b/openpype/hosts/flame/api/__init__.py index 561aaab3de..28511458c2 100644 --- a/openpype/hosts/flame/api/__init__.py +++ b/openpype/hosts/flame/api/__init__.py @@ -11,10 +11,8 @@ from .constants import ( from .lib import ( CTX, FlameAppFramework, - get_project_manager, get_current_project, get_current_sequence, - create_bin, create_segment_data_marker, get_segment_data_marker, set_segment_data_marker, @@ -87,10 +85,8 @@ __all__ = [ # lib "CTX", "FlameAppFramework", - "get_project_manager", "get_current_project", "get_current_sequence", - "create_bin", "create_segment_data_marker", "get_segment_data_marker", "set_segment_data_marker", diff --git a/openpype/hosts/flame/api/lib.py b/openpype/hosts/flame/api/lib.py index dd91252a00..7316fa1c5b 100644 --- a/openpype/hosts/flame/api/lib.py +++ b/openpype/hosts/flame/api/lib.py @@ -227,16 +227,6 @@ class FlameAppFramework(object): return True -def get_project_manager(): - # TODO: get_project_manager - return - - -def get_media_storage(): - # TODO: get_media_storage - return - - def get_current_project(): import flame return flame.project.current_project @@ -266,11 +256,6 @@ def get_current_sequence(selection): return process_timeline -def create_bin(name, root=None): - # TODO: create_bin - return - - def rescan_hooks(): import flame try: @@ -724,5 +709,5 @@ def get_batch_group_from_desktop(name): project_desktop = project.current_workspace.desktop for bgroup in project_desktop.batch_groups: - if bgroup.name.get_value() == name: + if bgroup.name.get_value() in name: return bgroup diff --git a/openpype/hosts/flame/api/scripts/wiretap_com.py b/openpype/hosts/flame/api/scripts/wiretap_com.py index 54993d34eb..14fbcec954 100644 --- a/openpype/hosts/flame/api/scripts/wiretap_com.py +++ b/openpype/hosts/flame/api/scripts/wiretap_com.py @@ -254,7 +254,7 @@ class WireTapCom(object): filtered_users = [user for user in used_names if user_name in user] if filtered_users: - # todo: need to find lastly created following regex pattern for + # TODO: need to find lastly created following regex pattern for # date used in name return filtered_users.pop() diff --git a/openpype/hosts/flame/plugins/load/load_clip.py b/openpype/hosts/flame/plugins/load/load_clip.py index b27600db1f..e0a7297381 100644 --- a/openpype/hosts/flame/plugins/load/load_clip.py +++ b/openpype/hosts/flame/plugins/load/load_clip.py @@ -39,7 +39,7 @@ class LoadClip(opfapi.ClipLoader): clip_name = self.clip_name_template.format( **context["representation"]["context"]) - # todo: settings in imageio + # TODO: settings in imageio # convert colorspace with ocio to flame mapping # in imageio flame section colorspace = colorspace diff --git a/openpype/hosts/flame/plugins/load/load_clip_batch.py b/openpype/hosts/flame/plugins/load/load_clip_batch.py index 252c92516d..3c13d88d3a 100644 --- a/openpype/hosts/flame/plugins/load/load_clip_batch.py +++ b/openpype/hosts/flame/plugins/load/load_clip_batch.py @@ -42,7 +42,7 @@ class LoadClipBatch(opfapi.ClipLoader): clip_name = self.clip_name_template.format( **context["representation"]["context"]) - # todo: settings in imageio + # TODO: settings in imageio # convert colorspace with ocio to flame mapping # in imageio flame section colorspace = colorspace diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py index 7c61ed62b5..253a1d6192 100644 --- a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -166,8 +166,11 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): self.log.info( "Updating batch group: {}".format(batchgroup_name)) # update already created batch group + bgroup.name = batchgroup_name bgroup.start_frame = frame_start bgroup.duration = frame_duration + # TODO: also update write node if there is any + # TODO: also update loaders to start from correct frameStart return bgroup From 5580ef083bd51bba96e11e1d68156d9dbedc4809 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 30 Mar 2022 12:24:12 +0200 Subject: [PATCH 043/126] hound catch --- openpype/hosts/flame/plugins/publish/integrate_batch_group.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py index 253a1d6192..4dd6081170 100644 --- a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -218,7 +218,8 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): # /path/to/file.[0001-0010].exr media_path = render_dir_path # name of file represented by tokens - media_path_pattern = "_v/_v." + media_path_pattern = ( + "_v/_v.") # The Create Open Clip attribute of the Write File node. \ # Determines if an Open Clip is created by the Write File node. create_clip = True From 2520ceca630d5985a9646e0002aaea352445cfd0 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Wed, 30 Mar 2022 22:12:00 +0200 Subject: [PATCH 044/126] Fix #2946: Avoid ImportError on `hdefereval` when Houdini runs without UI --- openpype/hosts/houdini/api/pipeline.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/houdini/api/pipeline.py b/openpype/hosts/houdini/api/pipeline.py index d079c9ea81..31c82b1cfd 100644 --- a/openpype/hosts/houdini/api/pipeline.py +++ b/openpype/hosts/houdini/api/pipeline.py @@ -4,7 +4,6 @@ import logging import contextlib import hou -import hdefereval import pyblish.api import avalon.api @@ -305,7 +304,13 @@ def on_new(): start = hou.playbar.playbackRange()[0] hou.setFrame(start) - hdefereval.executeDeferred(_enforce_start_frame) + if hou.isUIAvailable(): + import hdefereval + hdefereval.executeDeferred(_enforce_start_frame) + else: + # Run without execute deferred when no UI is available because + # without UI `hdefereval` is not available to import + _enforce_start_frame() def _set_context_settings(): From 1926e107659790d10a63770f70b72a6f7cf88ef1 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 4 Apr 2022 20:50:58 +0200 Subject: [PATCH 045/126] flame: redundant code --- openpype/hosts/flame/plugins/publish/integrate_batch_group.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py index 4dd6081170..fc5f4cfcd0 100644 --- a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -7,7 +7,7 @@ import openpype.hosts.flame.api as opfapi import openpype.pipeline as op_pipeline -@pyblish.api.log + class IntegrateBatchGroup(pyblish.api.InstancePlugin): """Integrate published shot to batch group""" From 54897163caee3dbb783bcafd68b7648c99434c9d Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 4 Apr 2022 20:52:09 +0200 Subject: [PATCH 046/126] haunch catch --- openpype/hosts/flame/plugins/publish/integrate_batch_group.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py index fc5f4cfcd0..a9ccd6b4a1 100644 --- a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -7,7 +7,6 @@ import openpype.hosts.flame.api as opfapi import openpype.pipeline as op_pipeline - class IntegrateBatchGroup(pyblish.api.InstancePlugin): """Integrate published shot to batch group""" From 0dfca2ff4589e996708c9d27108e282d9107d847 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 5 Apr 2022 11:38:00 +0200 Subject: [PATCH 047/126] Flame: refining the code for better understanding of flow --- .../plugins/publish/integrate_batch_group.py | 78 ++++++++----------- 1 file changed, 33 insertions(+), 45 deletions(-) diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py index a9ccd6b4a1..979134bbfe 100644 --- a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -38,44 +38,40 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): def _load_clip_to_context(self, instance, bgroup): # get all loaders for host - loaders = op_pipeline.discover_loader_plugins() + loaders_by_name = { + loader.__name__: loader + for loader in op_pipeline.discover_loader_plugins() + } # get all published representations published_representations = instance.data["published_representations"] + repres_db_id_by_name = { + repre_info["representation"]["name"]: repre_id + for repre_id, repre_info in published_representations.items() + } # get all loadable representations - representations = instance.data["representations"] + repres_by_name = { + repre["name"]: repre for repre in instance.data["representations"] + } # get repre_id for the loadable representations - loadable_representations = [ - { - "name": _repr["name"], - "loader": _repr.get("batch_group_loader_name"), - # match loader to the loadable representation - "_id": next( - ( - id - for id, repr in published_representations.items() - if repr["representation"]["name"] == _repr["name"] - ), - None - ) + loader_name_by_repre_id = { + repres_db_id_by_name[repr_name]: { + "loader": repr_data["batch_group_loader_name"], + # add repre data for exception logging + "_repre_data": repr_data } - for _repr in representations - if _repr.get("load_to_batch_group") is not None - ] + for repr_name, repr_data in repres_by_name.items() + if repr_data.get("load_to_batch_group") + } - self.log.debug("__ loadable_representations: {}".format(pformat( - loadable_representations))) + self.log.debug("__ loader_name_by_repre_id: {}".format(pformat( + loader_name_by_repre_id))) # get representation context from the repre_id - representation_ids = [ - repre["_id"] - for repre in loadable_representations - if repre["_id"] is not None - ] repre_contexts = op_pipeline.load.get_repres_contexts( - representation_ids) + loader_name_by_repre_id.keys()) self.log.debug("__ repre_contexts: {}".format(pformat( repre_contexts))) @@ -84,45 +80,37 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): for repre_id, repre_context in repre_contexts.items(): self.log.debug("__ repre_id: {}".format(repre_id)) # get loader name by representation id - loader_name = next( - ( - repr["loader"] - for repr in loadable_representations - if repr["_id"] == repre_id - )) or self.default_loader + loader_name = ( + loader_name_by_repre_id[repre_id]["loader"] + # if nothing was added to settings fallback to default + or self.default_loader + ) # get loader plugin - Loader = next( - ( - loader_plugin - for loader_plugin in loaders - if loader_plugin.__name__ == loader_name - ), - None - ) - if Loader: + loader_plugin = loaders_by_name.get(loader_name) + if loader_plugin: # load to flame by representation context try: op_pipeline.load.load_with_repre_context( - Loader, repre_context, **{ + loader_plugin, repre_context, **{ "data": {"workdir": self.task_workdir} }) except op_pipeline.load.IncompatibleLoaderError as msg: self.log.error( "Check allowed representations for Loader `{}` " "in settings > error: {}".format( - Loader.__name__, msg)) + loader_plugin.__name__, msg)) self.log.error( "Representaton context >>{}<< is not compatible " "with loader `{}`".format( - pformat(repre_context), Loader.__name__ + pformat(repre_context), loader_plugin.__name__ ) ) else: self.log.warning( "Something got wrong and there is not Loader found for " "following data: {}".format( - pformat(loadable_representations)) + pformat(loader_name_by_repre_id)) ) def _get_batch_group(self, instance, task_data): From 5b260afc6af67bbcda403e7ce35922e7a8350ff6 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 5 Apr 2022 15:47:57 +0200 Subject: [PATCH 048/126] flame: avoid hidden segment processing --- openpype/hosts/flame/otio/flame_export.py | 43 +++++++++++------------ 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/openpype/hosts/flame/otio/flame_export.py b/openpype/hosts/flame/otio/flame_export.py index 8c240fc9d5..f9dbe68421 100644 --- a/openpype/hosts/flame/otio/flame_export.py +++ b/openpype/hosts/flame/otio/flame_export.py @@ -401,8 +401,10 @@ def get_clips_in_reels(project): version = clip.versions[-1] track = version.tracks[-1] + # each reel clip is also having one segment for segment in track.segments: - segment_data = _get_segment_attributes(segment) + segment_data = _get_segment_attributes( + segment, from_clip=True) clip_data.update(segment_data) output_clips.append(clip_data) @@ -489,12 +491,14 @@ def add_otio_metadata(otio_item, item, **kwargs): otio_item.metadata.update({key: value}) -def _get_shot_tokens_values(clip, tokens): +def _get_shot_tokens_values(clip, tokens, from_clip=False): old_value = None output = {} - if not clip.shot_name: - return output + # in case it is segment from reel clip + # avoiding duplicity of segement data + if from_clip: + return {} old_value = clip.shot_name.get_value() @@ -512,16 +516,19 @@ def _get_shot_tokens_values(clip, tokens): return output -def _get_segment_attributes(segment): +def _get_segment_attributes(segment, from_clip=False): # log.debug(dir(segment)) - - if str(segment.name)[1:-1] == "": + if ( + segment.name.get_value() == "" + or segment.hidden + ): return None # Add timeline segment to tree clip_data = { "segment_name": segment.name.get_value(), "segment_comment": segment.comment.get_value(), + "shot_name": segment.shot_name.get_value(), "tape_name": segment.tape_name, "source_name": segment.source_name, "fpath": segment.file_path, @@ -531,7 +538,7 @@ def _get_segment_attributes(segment): # add all available shot tokens shot_tokens = _get_shot_tokens_values(segment, [ "", "", "", "", - ]) + ], from_clip) clip_data.update(shot_tokens) # populate shot source metadata @@ -597,11 +604,7 @@ def create_otio_timeline(sequence): continue all_segments.append(clip_data) - segments_ordered = { - itemindex: clip_data - for itemindex, clip_data in enumerate( - all_segments) - } + segments_ordered = dict(enumerate(all_segments)) log.debug("_ segments_ordered: {}".format( pformat(segments_ordered) )) @@ -612,15 +615,11 @@ def create_otio_timeline(sequence): log.debug("_ itemindex: {}".format(itemindex)) # Add Gap if needed - if itemindex == 0: - # if it is first track item at track then add - # it to previous item - prev_item = segment_data - - else: - # get previous item - prev_item = segments_ordered[itemindex - 1] - + prev_item = ( + segment_data + if itemindex == 0 + else segments_ordered[itemindex - 1] + ) log.debug("_ segment_data: {}".format(segment_data)) # calculate clip frame range difference from each other From 575898490f4aad68256207da0dea4f9960e2948f Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 5 Apr 2022 15:57:16 +0200 Subject: [PATCH 049/126] flame: fixing broken get_clips_in_reels --- openpype/hosts/flame/otio/flame_export.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/flame/otio/flame_export.py b/openpype/hosts/flame/otio/flame_export.py index f9dbe68421..78e5ceecb6 100644 --- a/openpype/hosts/flame/otio/flame_export.py +++ b/openpype/hosts/flame/otio/flame_export.py @@ -405,7 +405,8 @@ def get_clips_in_reels(project): for segment in track.segments: segment_data = _get_segment_attributes( segment, from_clip=True) - clip_data.update(segment_data) + if segment_data: + clip_data.update(segment_data) output_clips.append(clip_data) From 8d4541d68da458f5121029353119ab8ae7ff4791 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 5 Apr 2022 16:29:04 +0200 Subject: [PATCH 050/126] flame: hidden attribute is PyAttribute so need to get value --- openpype/hosts/flame/otio/flame_export.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/flame/otio/flame_export.py b/openpype/hosts/flame/otio/flame_export.py index 78e5ceecb6..1b5980b40a 100644 --- a/openpype/hosts/flame/otio/flame_export.py +++ b/openpype/hosts/flame/otio/flame_export.py @@ -518,10 +518,13 @@ def _get_shot_tokens_values(clip, tokens, from_clip=False): def _get_segment_attributes(segment, from_clip=False): - # log.debug(dir(segment)) + + log.debug("Segment name|hidden: {}|{}".format( + segment.name.get_value(), segment.hidden + )) if ( segment.name.get_value() == "" - or segment.hidden + or segment.hidden.get_value() ): return None @@ -591,7 +594,12 @@ def create_otio_timeline(sequence): # create otio tracks and clips for ver in sequence.versions: for track in ver.tracks: - if len(track.segments) == 0 and track.hidden: + # avoid all empty tracks + # or hidden tracks + if ( + len(track.segments) == 0 + or track.hidden.get_value() + ): return None # convert track to otio From 60118298b6c3dcf2f41488624a3b6d3bf9166990 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 5 Apr 2022 16:29:31 +0200 Subject: [PATCH 051/126] flame: make reel clip validation optional --- openpype/hosts/flame/plugins/publish/validate_source_clip.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openpype/hosts/flame/plugins/publish/validate_source_clip.py b/openpype/hosts/flame/plugins/publish/validate_source_clip.py index 9ff015f628..345c00e05a 100644 --- a/openpype/hosts/flame/plugins/publish/validate_source_clip.py +++ b/openpype/hosts/flame/plugins/publish/validate_source_clip.py @@ -9,6 +9,8 @@ class ValidateSourceClip(pyblish.api.InstancePlugin): label = "Validate Source Clip" hosts = ["flame"] families = ["clip"] + optional = True + active = False def process(self, instance): flame_source_clip = instance.data["flameSourceClip"] From 818c3fe91f420de9b08ff4ebfc1bdae301a42927 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 5 Apr 2022 16:58:25 +0200 Subject: [PATCH 052/126] flame: fallback if reel clip is not available --- .../publish/extract_subset_resources.py | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/flame/plugins/publish/extract_subset_resources.py b/openpype/hosts/flame/plugins/publish/extract_subset_resources.py index 31f7b6d574..341f12be16 100644 --- a/openpype/hosts/flame/plugins/publish/extract_subset_resources.py +++ b/openpype/hosts/flame/plugins/publish/extract_subset_resources.py @@ -63,7 +63,10 @@ class ExtractSubsetResources(openpype.api.Extractor): segment = instance.data["item"] sequence_clip = instance.context.data["flameSequence"] clip_data = instance.data["flameSourceClip"] - clip = clip_data["PyClip"] + + reel_clip = None + if clip_data: + reel_clip = clip_data["PyClip"] # segment's parent track name s_track_name = segment.parent.name.get_value() @@ -127,8 +130,20 @@ class ExtractSubsetResources(openpype.api.Extractor): in_mark = (source_start_handles - source_first_frame) + 1 out_mark = in_mark + source_duration_handles + # make test for type of preset and available reel_clip + if ( + not reel_clip + and export_type != "Sequence Publish" + ): + self.log.warning(( + "Skipping preset {}. Not available " + "reel clip for {}").format( + preset_file, segment.name.get_value() + )) + continue + # by default export source clips - exporting_clip = clip + exporting_clip = reel_clip if export_type == "Sequence Publish": # change export clip to sequence @@ -344,7 +359,7 @@ class ExtractSubsetResources(openpype.api.Extractor): # create otio tracks and clips for ver in sequence_clip.versions: for track in ver.tracks: - if len(track.segments) == 0 and track.hidden: + if len(track.segments) == 0 and track.hidden.get_value(): continue if track.name.get_value() != track_name: From 246127c73b7d06bf5e7a2b68a7fb0bd31e949b22 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 5 Apr 2022 17:20:41 +0200 Subject: [PATCH 053/126] flame: hidding all unrelated segments and tracks --- .../plugins/publish/extract_subset_resources.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/flame/plugins/publish/extract_subset_resources.py b/openpype/hosts/flame/plugins/publish/extract_subset_resources.py index 341f12be16..a780f8c9e5 100644 --- a/openpype/hosts/flame/plugins/publish/extract_subset_resources.py +++ b/openpype/hosts/flame/plugins/publish/extract_subset_resources.py @@ -61,6 +61,7 @@ class ExtractSubsetResources(openpype.api.Extractor): # flame objects segment = instance.data["item"] + segment_name = segment.name.get_value() sequence_clip = instance.context.data["flameSequence"] clip_data = instance.data["flameSourceClip"] @@ -138,7 +139,7 @@ class ExtractSubsetResources(openpype.api.Extractor): self.log.warning(( "Skipping preset {}. Not available " "reel clip for {}").format( - preset_file, segment.name.get_value() + preset_file, segment_name )) continue @@ -175,7 +176,7 @@ class ExtractSubsetResources(openpype.api.Extractor): if export_type == "Sequence Publish": # only keep visible layer where instance segment is child - self.hide_other_tracks(duplclip, s_track_name) + self.hide_others(duplclip, segment_name, s_track_name) # validate xml preset file is filled if preset_file == "": @@ -349,11 +350,12 @@ class ExtractSubsetResources(openpype.api.Extractor): return new_stage_dir, new_files_list - def hide_other_tracks(self, sequence_clip, track_name): + def hide_others(self, sequence_clip, segment_name, track_name): """Helper method used only if sequence clip is used Args: sequence_clip (flame.Clip): sequence clip + segment_name (str): segment name track_name (str): track name """ # create otio tracks and clips @@ -362,5 +364,12 @@ class ExtractSubsetResources(openpype.api.Extractor): if len(track.segments) == 0 and track.hidden.get_value(): continue + # hide tracks which are not parent track if track.name.get_value() != track_name: track.hidden = True + continue + + # hidde all other segments + for segment in track.segments: + if segment.name.get_value() != segment_name: + segment.hidden = True From 312d0309ab92de834629c58587f1a758d1d1e90c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 6 Apr 2022 12:36:09 +0200 Subject: [PATCH 054/126] Fix - reworked alternative sites Implements recursive relationship between alternative sites --- openpype/plugins/publish/integrate_new.py | 80 +++++++++++++++++------ 1 file changed, 61 insertions(+), 19 deletions(-) diff --git a/openpype/plugins/publish/integrate_new.py b/openpype/plugins/publish/integrate_new.py index 959fd3bbee..ed1c02b825 100644 --- a/openpype/plugins/publish/integrate_new.py +++ b/openpype/plugins/publish/integrate_new.py @@ -1116,18 +1116,17 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): rec["sites"].append(meta) already_attached_sites[meta["name"]] = None + # add alternative sites + rec, already_attached_sites = self._add_alternative_sites( + system_sync_server_presets, already_attached_sites, rec) + # add skeleton for site where it should be always synced to - for always_on_site in always_accesible: + for always_on_site in set(always_accesible): if always_on_site not in already_attached_sites.keys(): meta = {"name": always_on_site.strip()} rec["sites"].append(meta) already_attached_sites[meta["name"]] = None - # add alternative sites - rec = self._add_alternative_sites(system_sync_server_presets, - already_attached_sites, - rec) - log.debug("final sites:: {}".format(rec["sites"])) return rec @@ -1158,22 +1157,65 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): """ conf_sites = system_sync_server_presets.get("sites", {}) + alt_site_pairs = self._get_alt_site_pairs(conf_sites) + + already_attached_keys = list(already_attached_sites.keys()) + for added_site in already_attached_keys: + real_created = already_attached_sites[added_site] + for alt_site in alt_site_pairs.get(added_site, []): + if alt_site in already_attached_sites.keys(): + continue + meta = {"name": alt_site} + # alt site inherits state of 'created_dt' + if real_created: + meta["created_dt"] = real_created + rec["sites"].append(meta) + already_attached_sites[meta["name"]] = real_created + + return rec, already_attached_sites + + def _get_alt_site_pairs(self, conf_sites): + """Returns dict of site and its alternative sites. + + If `site` has alternative site, it means that alt_site has 'site' as + alternative site + Args: + conf_sites (dict) + Returns: + (dict): {'site': [alternative sites]...} + """ + alt_site_pairs = {} for site_name, site_info in conf_sites.items(): alt_sites = set(site_info.get("alternative_sites", [])) - already_attached_keys = list(already_attached_sites.keys()) - for added_site in already_attached_keys: - if added_site in alt_sites: - if site_name in already_attached_keys: - continue - meta = {"name": site_name} - real_created = already_attached_sites[added_site] - # alt site inherits state of 'created_dt' - if real_created: - meta["created_dt"] = real_created - rec["sites"].append(meta) - already_attached_sites[meta["name"]] = real_created + if not alt_site_pairs.get(site_name): + alt_site_pairs[site_name] = [] - return rec + alt_site_pairs[site_name].extend(alt_sites) + + for alt_site in alt_sites: + if not alt_site_pairs.get(alt_site): + alt_site_pairs[alt_site] = [] + alt_site_pairs[alt_site].extend([site_name]) + + # transitive relationship, eg site is alternative to another which is + # alternative to nex site + loop = True + while loop: + loop = False + for site_name, alt_sites in alt_site_pairs.items(): + for alt_site in alt_sites: + # safety against wrong config + # {"SFTP": {"alternative_site": "SFTP"} + if alt_site == site_name: + continue + + for alt_alt_site in alt_site_pairs.get(alt_site, []): + if ( alt_alt_site != site_name + and alt_alt_site not in alt_sites): + alt_site_pairs[site_name].append(alt_alt_site) + loop = True + + return alt_site_pairs def handle_destination_files(self, integrated_file_sizes, mode): """ Clean destination files From 9f9f47145b0a1e88c5e28c1b2ade7c842191e14c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 6 Apr 2022 12:45:49 +0200 Subject: [PATCH 055/126] Fix - added active site from settings if same as local id Without this Tray configuring background process will not show proper site in LS dropdown --- openpype/modules/sync_server/sync_server_module.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/openpype/modules/sync_server/sync_server_module.py b/openpype/modules/sync_server/sync_server_module.py index caf58503f1..ddcf16a410 100644 --- a/openpype/modules/sync_server/sync_server_module.py +++ b/openpype/modules/sync_server/sync_server_module.py @@ -848,6 +848,11 @@ class SyncServerModule(OpenPypeModule, ITrayModule): if self.enabled and sync_settings.get('enabled'): sites.append(self.LOCAL_SITE) + active_site = sync_settings["config"]["active_site"] + # for Tray running background process + if active_site == get_local_site_id() and active_site not in sites: + sites.append(active_site) + return sites def tray_init(self): From 87f40d53e098ce3cef8bc31fc733e73df36f9cc3 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 6 Apr 2022 14:44:03 +0200 Subject: [PATCH 056/126] flame: adding media info getter obj --- openpype/hosts/flame/api/lib.py | 116 +++++++++++++++++++++++++++++++- 1 file changed, 114 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/flame/api/lib.py b/openpype/hosts/flame/api/lib.py index 7316fa1c5b..4e989abb9e 100644 --- a/openpype/hosts/flame/api/lib.py +++ b/openpype/hosts/flame/api/lib.py @@ -1,9 +1,14 @@ import sys import os import re +import six import json import pickle +import itertools import contextlib +import xml.etree.cElementTree as cET +from copy import deepcopy +from xml.etree import ElementTree as ET from pprint import pformat from .constants import ( MARKER_COLOR, @@ -12,9 +17,10 @@ from .constants import ( COLOR_MAP, MARKER_PUBLISH_DEFAULT ) -from openpype.api import Logger -log = Logger.get_logger(__name__) +import openpype.api as openpype + +log = openpype.Logger.get_logger(__name__) FRAME_PATTERN = re.compile(r"[\._](\d+)[\.]") @@ -711,3 +717,109 @@ def get_batch_group_from_desktop(name): for bgroup in project_desktop.batch_groups: if bgroup.name.get_value() in name: return bgroup + + +class MediaInfoFile: + media_script_path = "/opt/Autodesk/mio/current/dl_get_media_info" + tmp_name = "_tmp.clip" + tmp_file = None + + out_feed_nb_ticks = None + out_feed_fps = None + out_feed_drop_mode = None + + log = log + + def __init__(self, path): + # test if media script paht exists + self._validate_media_script_path() + + # derivate other feed variables + self.feed_basename = os.path.basename(path) + self.feed_dir = os.path.dirname(path) + self.feed_ext = os.path.splitext(self.feed_basename)[1][1:].lower() + + self.tmp_file = os.path.join(self.feed_dir, self.tmp_name) + + # remove previously generated temp files + # it will be regenerated + self._clear_tmp_file() + + self.log.info("Temp File: {}".format(self.tmp_file)) + + def _validate_media_script_path(self): + if not os.path.isfile(self.media_script_path): + raise IOError("Media Scirpt does not exist: `{}`".format( + self.media_script_path)) + + def _generate_media_info_file(self): + # Create cmd arguments for gettig xml file info file + cmd_args = [ + self.media_script_path, + "-e", self.feed_ext, + "-o", self.tmp_file, + self.feed_dir + ] + + # execute creation of clip xml template data + try: + openpype.run_subprocess(cmd_args) + self._make_single_clip_media_info() + except TypeError: + self.log.error("Error creating self.tmp_file") + six.reraise(*sys.exc_info()) + + def _make_single_clip_media_info(self): + with open(self.tmp_file) as f: + lines = f.readlines() + _added_root = itertools.chain( + "", deepcopy(lines)[1:], "") + new_root = ET.fromstringlist(_added_root) + + # find the clip which is matching to my input name + xml_clips = new_root.findall("clip") + matching_clip = None + for xml_clip in xml_clips: + if xml_clip.find("name").text in self.feed_basename: + matching_clip = xml_clip + + if matching_clip is None: + # return warning there is missing clip + raise ET.ParseError( + "Missing clip in `{}`. Available clips {}".format( + self.feed_basename, [ + xml_clip.find("name").text + for xml_clip in xml_clips + ] + )) + + self._write_result_xml_to_file(self.tmp_file, matching_clip) + + def _clear_tmp_file(self): + if os.path.isfile(self.tmp_file): + os.remove(self.tmp_file) + + def _get_time_info_from_origin(self, xml_data): + try: + for out_track in xml_data.iter('track'): + for out_feed in out_track.iter('feeds'): + out_feed_nb_ticks_obj = out_feed.find( + 'startTimecode/nbTicks') + self.out_feed_nb_ticks = out_feed_nb_ticks_obj.text + out_feed_fps_obj = out_feed.find( + 'startTimecode/rate') + self.out_feed_fps = out_feed_fps_obj.text + out_feed_drop_mode_obj = out_feed.find( + 'startTimecode/dropMode') + self.out_feed_drop_mode = out_feed_drop_mode_obj.text + break + else: + continue + except Exception as msg: + self.log.warning(msg) + + def _write_result_xml_to_file(self, file, xml_data): + # save it as new file + tree = cET.ElementTree(xml_data) + tree.write(file, xml_declaration=True, + method='xml', encoding='UTF-8') \ No newline at end of file From 7dc2c618a0b6ffd80529620d576f929afaa01cf0 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 6 Apr 2022 14:53:13 +0200 Subject: [PATCH 057/126] flame: update media info file generator --- openpype/hosts/flame/api/lib.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/flame/api/lib.py b/openpype/hosts/flame/api/lib.py index 4e989abb9e..633854168a 100644 --- a/openpype/hosts/flame/api/lib.py +++ b/openpype/hosts/flame/api/lib.py @@ -724,6 +724,7 @@ class MediaInfoFile: tmp_name = "_tmp.clip" tmp_file = None + clip_data = None out_feed_nb_ticks = None out_feed_fps = None out_feed_drop_mode = None @@ -752,7 +753,7 @@ class MediaInfoFile: raise IOError("Media Scirpt does not exist: `{}`".format( self.media_script_path)) - def _generate_media_info_file(self): + def generate_media_info_file(self): # Create cmd arguments for gettig xml file info file cmd_args = [ self.media_script_path, @@ -793,6 +794,8 @@ class MediaInfoFile: ] )) + self._get_time_info_from_origin(matching_clip) + self.clip_data = matching_clip self._write_result_xml_to_file(self.tmp_file, matching_clip) def _clear_tmp_file(self): From d79773e801b428ee629521a65374406cda8b8bfc Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 6 Apr 2022 15:42:58 +0200 Subject: [PATCH 058/126] flame: fixing feeds to feed tag --- openpype/hosts/flame/api/lib.py | 2 +- openpype/hosts/flame/api/plugin.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/flame/api/lib.py b/openpype/hosts/flame/api/lib.py index 633854168a..508dc0155f 100644 --- a/openpype/hosts/flame/api/lib.py +++ b/openpype/hosts/flame/api/lib.py @@ -805,7 +805,7 @@ class MediaInfoFile: def _get_time_info_from_origin(self, xml_data): try: for out_track in xml_data.iter('track'): - for out_feed in out_track.iter('feeds'): + for out_feed in out_track.iter('feed'): out_feed_nb_ticks_obj = out_feed.find( 'startTimecode/nbTicks') self.out_feed_nb_ticks = out_feed_nb_ticks_obj.text diff --git a/openpype/hosts/flame/api/plugin.py b/openpype/hosts/flame/api/plugin.py index 464f5ce89b..bd0f9f1a81 100644 --- a/openpype/hosts/flame/api/plugin.py +++ b/openpype/hosts/flame/api/plugin.py @@ -679,6 +679,7 @@ class ClipLoader(LoaderPlugin): ] +# TODO: inheritance from flame.api.lib.MediaInfoFile class OpenClipSolver: media_script_path = "/opt/Autodesk/mio/current/dl_get_media_info" tmp_name = "_tmp.clip" From 67c759ae2d72558c2caa5cfcc30266a1d16cddb9 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 6 Apr 2022 15:45:42 +0200 Subject: [PATCH 059/126] flame: add media info file class to api --- openpype/hosts/flame/api/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/flame/api/__init__.py b/openpype/hosts/flame/api/__init__.py index 28511458c2..1308b04a7d 100644 --- a/openpype/hosts/flame/api/__init__.py +++ b/openpype/hosts/flame/api/__init__.py @@ -28,7 +28,8 @@ from .lib import ( get_padding_from_filename, maintained_object_duplication, get_clip_segment, - get_batch_group_from_desktop + get_batch_group_from_desktop, + MediaInfoFile ) from .utils import ( setup, @@ -103,6 +104,7 @@ __all__ = [ "maintained_object_duplication", "get_clip_segment", "get_batch_group_from_desktop", + "MediaInfoFile", # pipeline "install", From 6c9f9c18119cc286078cd109d3253073a3d273aa Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 6 Apr 2022 16:08:31 +0200 Subject: [PATCH 060/126] flame: add generator into init of class --- openpype/hosts/flame/api/lib.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/flame/api/lib.py b/openpype/hosts/flame/api/lib.py index 508dc0155f..998d7dfa7e 100644 --- a/openpype/hosts/flame/api/lib.py +++ b/openpype/hosts/flame/api/lib.py @@ -748,12 +748,14 @@ class MediaInfoFile: self.log.info("Temp File: {}".format(self.tmp_file)) + self._generate_media_info_file() + def _validate_media_script_path(self): if not os.path.isfile(self.media_script_path): raise IOError("Media Scirpt does not exist: `{}`".format( self.media_script_path)) - def generate_media_info_file(self): + def _generate_media_info_file(self): # Create cmd arguments for gettig xml file info file cmd_args = [ self.media_script_path, From f43e6f0fcdfdb82876c06f6631654a3318edf053 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 6 Apr 2022 16:08:48 +0200 Subject: [PATCH 061/126] flame: get real source_in --- openpype/hosts/flame/otio/flame_export.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/flame/otio/flame_export.py b/openpype/hosts/flame/otio/flame_export.py index 1b5980b40a..3e76968963 100644 --- a/openpype/hosts/flame/otio/flame_export.py +++ b/openpype/hosts/flame/otio/flame_export.py @@ -7,6 +7,7 @@ import json import logging import opentimelineio as otio from . import utils +from openpype.hosts.flame.api import MediaInfoFile import flame from pprint import pformat @@ -345,7 +346,13 @@ def create_otio_clip(clip_data): media_reference = create_otio_reference(clip_data) # calculate source in - first_frame = utils.get_frame_from_filename(clip_data["fpath"]) or 0 + media_info = MediaInfoFile(clip_data["fpath"]) + xml_timecode_ticks = media_info.out_feed_nb_ticks + if xml_timecode_ticks: + first_frame = int(xml_timecode_ticks) + else: + first_frame = utils.get_frame_from_filename(clip_data["fpath"]) or 0 + source_in = int(clip_data["source_in"]) - int(first_frame) # creatae source range From c2788070a370a39536e134acc13a5f5243b2530f Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 7 Apr 2022 12:22:34 +0200 Subject: [PATCH 062/126] flame: refactoring batch group creation --- openpype/hosts/flame/api/__init__.py | 6 +- openpype/hosts/flame/api/batch_utils.py | 146 +++++++++++++----- .../plugins/publish/integrate_batch_group.py | 98 ++++++++---- 3 files changed, 182 insertions(+), 68 deletions(-) diff --git a/openpype/hosts/flame/api/__init__.py b/openpype/hosts/flame/api/__init__.py index 1308b04a7d..a0c40904ed 100644 --- a/openpype/hosts/flame/api/__init__.py +++ b/openpype/hosts/flame/api/__init__.py @@ -72,7 +72,8 @@ from .render_utils import ( modify_preset_file ) from .batch_utils import ( - create_batch + create_batch_group, + create_batch_group_conent ) __all__ = [ @@ -148,5 +149,6 @@ __all__ = [ "modify_preset_file", # batch utils - "create_batch" + "create_batch_group", + "create_batch_group_conent" ] diff --git a/openpype/hosts/flame/api/batch_utils.py b/openpype/hosts/flame/api/batch_utils.py index 43742c6e4f..26f324090b 100644 --- a/openpype/hosts/flame/api/batch_utils.py +++ b/openpype/hosts/flame/api/batch_utils.py @@ -1,66 +1,134 @@ import flame -def create_batch(name, frame_start, frame_duration, **kwargs): +def create_batch_group( + name, + frame_start, + frame_duration, + update_batch_group=None, + **kwargs +): """Create Batch Group in active project's Desktop Args: name (str): name of batch group to be created frame_start (int): start frame of batch frame_end (int): end frame of batch + update_batch_group (PyBatch)[optional]: batch group to update + + Return: + PyBatch: active flame batch group """ + # make sure some batch obj is present + batch_group = update_batch_group or flame.batch + schematic_reels = kwargs.get("shematic_reels") or ['LoadedReel1'] shelf_reels = kwargs.get("shelf_reels") or ['ShelfReel1'] - write_pref = kwargs["write_pref"] handle_start = kwargs.get("handleStart") or 0 handle_end = kwargs.get("handleEnd") or 0 frame_start -= handle_start frame_duration += handle_start + handle_end - # Create batch group with name, start_frame value, duration value, - # set of schematic reel names, set of shelf reel names - bgroup = flame.batch.create_batch_group( - name, - start_frame=frame_start, - duration=frame_duration, - reels=schematic_reels, - shelf_reels=shelf_reels - ) + if not update_batch_group: + # Create batch group with name, start_frame value, duration value, + # set of schematic reel names, set of shelf reel names + batch_group = batch_group.create_batch_group( + name, + start_frame=frame_start, + duration=frame_duration, + reels=schematic_reels, + shelf_reels=shelf_reels + ) + else: + batch_group.name = name + batch_group.start_frame = frame_start + batch_group.duration = frame_duration + + # add reels to batch group + _add_reels_to_batch_group( + batch_group, schematic_reels, shelf_reels) + + # TODO: also update write node if there is any + # TODO: also update loaders to start from correct frameStart if kwargs.get("switch_batch_tab"): # use this command to switch to the batch tab - flame.batch.go_to() + batch_group.go_to() - comp_node = flame.batch.create_node("Comp") + return batch_group - # TODO: convert this to iterational processing, - # so it could be driven from `imageio` settigns - # create write node - write_node = flame.batch.create_node('Write File') - # assign attrs - write_node.name = write_pref["name"] - write_node.media_path = write_pref["media_path"] - write_node.media_path_pattern = write_pref["media_path_pattern"] - write_node.create_clip = write_pref["create_clip"] - write_node.include_setup = write_pref["include_setup"] - write_node.create_clip_path = write_pref["create_clip_path"] - write_node.include_setup_path = write_pref["include_setup_path"] - write_node.file_type = write_pref["file_type"] - write_node.format_extension = write_pref["format_extension"] - write_node.bit_depth = write_pref["bit_depth"] - write_node.compress = write_pref["compress"] - write_node.compress_mode = write_pref["compress_mode"] - write_node.frame_index_mode = write_pref["frame_index_mode"] - write_node.frame_padding = write_pref["frame_padding"] - write_node.version_mode = write_pref["version_mode"] - write_node.version_name = write_pref["version_name"] - write_node.version_padding = write_pref["version_padding"] - flame.batch.connect_nodes(comp_node, "Result", write_node, "Front") +def _add_reels_to_batch_group(batch_group, reels, shelf_reels): + # update or create defined reels + # helper variables + reel_names = [ + r.name.get_value() + for r in batch_group.reels + ] + shelf_reel_names = [ + r.name.get_value() + for r in batch_group.shelf_reels + ] + # add schematic reels + for _r in reels: + if _r in reel_names: + continue + batch_group.create_reel(_r) + + # add shelf reels + for _sr in shelf_reels: + if _sr in shelf_reel_names: + continue + batch_group.create_shelf_reel(_sr) + + +def create_batch_group_conent(batch_nodes, batch_links, batch_group=None): + """Creating batch group with links + + Args: + batch_nodes (list of dict): each dict is node definition + batch_links (list of dict): each dict is link definition + batch_group (PyBatch, optional): batch group. Defaults to None. + """ + # make sure some batch obj is present + batch_group = batch_group or flame.batch + + created_nodes = {} + for node in batch_nodes: + # NOTE: node_props needs to be ideally OrederDict type + node_id, node_type, node_props = ( + node["id"], node["type"], node["properties"]) + + # create batch node + batch_node = batch_group.create_node(node_type) + + # set attributes found in node props + for key, value in node_props.items(): + if not hasattr(batch_node, key): + continue + setattr(batch_node, key, value) + + # add created node for possible linking + created_nodes[node_id] = batch_node + + # link nodes to each other + for link in batch_links: + _from_n, _to_n = link["from_node"], link["to_node"] + + # check if all linking nodes are available + if not all([ + created_nodes.get(_from_n["id"]), + created_nodes.get(_to_n["id"]) + ]): + continue + + # link nodes in defined link + batch_group.connect_nodes( + created_nodes[_from_n["id"]], _from_n["connector"], + created_nodes[_to_n["id"]], _to_n["connector"] + ) # sort batch nodes - flame.batch.organize() - - return bgroup + batch_group.organize() diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py index 979134bbfe..524d9b1ac2 100644 --- a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -1,5 +1,6 @@ import os import copy +from collections import OrderedDict from pprint import pformat import pyblish from openpype.lib import get_workdir @@ -30,12 +31,48 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): # create or get already created batch group bgroup = self._get_batch_group(instance, task_data) + # add batch group content + self._add_nodes_to_batch_with_links(instance, task_data, bgroup) + # load plate to batch group self.log.info("Loading subset `{}` into batch `{}`".format( instance.data["subset"], bgroup.name.get_value() )) self._load_clip_to_context(instance, bgroup) + def _add_nodes_to_batch_with_links(self, instance, task_data, batch_group): + # get write file node properties > OrederDict because order does mater + write_pref_data = self._get_write_prefs(instance, task_data) + + batch_nodes = [ + { + "type": "comp", + "properties": {}, + "id": "comp_node01" + }, + { + "type": "Write File", + "properties": write_pref_data, + "id": "write_file_node01" + } + ] + batch_links = [ + { + "from_node": { + "id": "comp_node01", + "connector": "Result" + }, + "to_node": { + "id": "write_file_node01", + "connector": "Front" + } + } + ] + + # add nodes into batch group + opfapi.create_batch_group_conent( + batch_nodes, batch_links, batch_group) + def _load_clip_to_context(self, instance, bgroup): # get all loaders for host loaders_by_name = { @@ -123,13 +160,11 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): task_name = task_data["name"] batchgroup_name = "{}_{}".format(asset_name, task_name) - write_pref_data = self._get_write_prefs(instance, task_data) batch_data = { "shematic_reels": [ "OP_LoadedReel" ], - "write_pref": write_pref_data, "handleStart": handle_start, "handleEnd": handle_end } @@ -143,21 +178,24 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): self.log.info( "Creating new batch group: {}".format(batchgroup_name)) # create batch with utils - bgroup = opfapi.create_batch( + bgroup = opfapi.create_batch_group( batchgroup_name, frame_start, frame_duration, **batch_data ) + else: self.log.info( "Updating batch group: {}".format(batchgroup_name)) # update already created batch group - bgroup.name = batchgroup_name - bgroup.start_frame = frame_start - bgroup.duration = frame_duration - # TODO: also update write node if there is any - # TODO: also update loaders to start from correct frameStart + bgroup = opfapi.create_batch_group( + batchgroup_name, + frame_start, + frame_duration, + update_batch_group=bgroup, + **batch_data + ) return bgroup @@ -249,25 +287,31 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): version_name = "v" version_padding = 3 - return { - "name": name, - "media_path": media_path, - "media_path_pattern": media_path_pattern, - "create_clip": create_clip, - "include_setup": include_setup, - "create_clip_path": create_clip_path, - "include_setup_path": include_setup_path, - "file_type": file_type, - "format_extension": format_extension, - "bit_depth": bit_depth, - "compress": compress, - "compress_mode": compress_mode, - "frame_index_mode": frame_index_mode, - "frame_padding": frame_padding, - "version_mode": version_mode, - "version_name": version_name, - "version_padding": version_padding - } + # return it as ordered dict + reutrn_dict = OrderedDict() + # need to make sure the order of keys is correct + for item in ( + ("name", name), + ("media_path", media_path), + ("media_path_pattern", media_path_pattern), + ("create_clip", create_clip), + ("include_setup", include_setup), + ("create_clip_path", create_clip_path), + ("include_setup_path", include_setup_path), + ("file_type", file_type), + ("format_extension", format_extension), + ("bit_depth", bit_depth), + ("compress", compress), + ("compress_mode", compress_mode), + ("frame_index_mode", frame_index_mode), + ("frame_padding", frame_padding), + ("version_mode", version_mode), + ("version_name", version_name), + ("version_padding", version_padding) + ): + reutrn_dict.update({item[0]: item[1]}) + + return reutrn_dict def _get_shot_task_dir_path(self, instance, task_data): project_doc = instance.data["projectEntity"] From c2a41760676afceb065351520db9cfeec62b9d33 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 7 Apr 2022 13:30:43 +0200 Subject: [PATCH 063/126] flame: fix integrating batch group to loader with bgroup --- openpype/hosts/flame/plugins/load/load_clip_batch.py | 2 +- .../hosts/flame/plugins/publish/integrate_batch_group.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/flame/plugins/load/load_clip_batch.py b/openpype/hosts/flame/plugins/load/load_clip_batch.py index 3c13d88d3a..5de3226035 100644 --- a/openpype/hosts/flame/plugins/load/load_clip_batch.py +++ b/openpype/hosts/flame/plugins/load/load_clip_batch.py @@ -26,7 +26,7 @@ class LoadClipBatch(opfapi.ClipLoader): def load(self, context, name, namespace, options): # get flame objects - self.batch = flame.batch + self.batch = options.get("batch") or flame.batch # load clip to timeline and get main variables namespace = namespace diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py index 524d9b1ac2..c70c2baa4a 100644 --- a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -130,7 +130,10 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): try: op_pipeline.load.load_with_repre_context( loader_plugin, repre_context, **{ - "data": {"workdir": self.task_workdir} + "data": { + "workdir": self.task_workdir, + "batch": bgroup + } }) except op_pipeline.load.IncompatibleLoaderError as msg: self.log.error( From 614c49b57f4acd32d9d6db9cf97f6e0c628ce33d Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 7 Apr 2022 14:27:00 +0200 Subject: [PATCH 064/126] flame: fix wiretap with umask --- openpype/hosts/flame/api/scripts/wiretap_com.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/flame/api/scripts/wiretap_com.py b/openpype/hosts/flame/api/scripts/wiretap_com.py index 14fbcec954..d904e7850a 100644 --- a/openpype/hosts/flame/api/scripts/wiretap_com.py +++ b/openpype/hosts/flame/api/scripts/wiretap_com.py @@ -185,7 +185,9 @@ class WireTapCom(object): exit_code = subprocess.call( project_create_cmd, - cwd=os.path.expanduser('~')) + cwd=os.path.expanduser('~'), + preexec_fn=_subprocess_preexec_fn + ) if exit_code != 0: RuntimeError("Cannot create project in flame db") @@ -448,7 +450,9 @@ class WireTapCom(object): exit_code = subprocess.call( project_colorspace_cmd, - cwd=os.path.expanduser('~')) + cwd=os.path.expanduser('~'), + preexec_fn=_subprocess_preexec_fn + ) if exit_code != 0: RuntimeError("Cannot set colorspace {} on project {}".format( @@ -456,6 +460,11 @@ class WireTapCom(object): )) +def _subprocess_preexec_fn(): + os.setpgrp() + os.umask(0o022) + + if __name__ == "__main__": # get json exchange data json_path = sys.argv[-1] From ce4aa40f217857ae527800667d243e048ff40159 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 7 Apr 2022 14:52:13 +0200 Subject: [PATCH 065/126] flame: fixing umask to 0o000 to reflect permissions to 0777 --- openpype/hosts/flame/api/scripts/wiretap_com.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/flame/api/scripts/wiretap_com.py b/openpype/hosts/flame/api/scripts/wiretap_com.py index d904e7850a..f78102c0a1 100644 --- a/openpype/hosts/flame/api/scripts/wiretap_com.py +++ b/openpype/hosts/flame/api/scripts/wiretap_com.py @@ -462,7 +462,7 @@ class WireTapCom(object): def _subprocess_preexec_fn(): os.setpgrp() - os.umask(0o022) + os.umask(0o000) if __name__ == "__main__": From cd59b3af66e6bd39e20cce1ec2d1195a26225e9e Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 7 Apr 2022 14:53:29 +0200 Subject: [PATCH 066/126] flame: make sure nodes are not duplicated and update --- openpype/hosts/flame/api/batch_utils.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/flame/api/batch_utils.py b/openpype/hosts/flame/api/batch_utils.py index 26f324090b..23d16f8d1a 100644 --- a/openpype/hosts/flame/api/batch_utils.py +++ b/openpype/hosts/flame/api/batch_utils.py @@ -94,15 +94,25 @@ def create_batch_group_conent(batch_nodes, batch_links, batch_group=None): """ # make sure some batch obj is present batch_group = batch_group or flame.batch - + all_batch_nodes = { + b.name.get_value(): b + for b in batch_group.nodes + } created_nodes = {} for node in batch_nodes: # NOTE: node_props needs to be ideally OrederDict type node_id, node_type, node_props = ( node["id"], node["type"], node["properties"]) - # create batch node - batch_node = batch_group.create_node(node_type) + # get node name for checking if exists + node_name = node_props.get("name") or node_id + + if all_batch_nodes.get(node_name): + # update existing batch node + batch_node = all_batch_nodes[node_name] + else: + # create new batch node + batch_node = batch_group.create_node(node_type) # set attributes found in node props for key, value in node_props.items(): From 4ed6c0257ece24cc254afb4edc9a4cc698c4dd53 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 7 Apr 2022 15:05:46 +0200 Subject: [PATCH 067/126] flame: returning all batch nodes --- openpype/hosts/flame/api/batch_utils.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/openpype/hosts/flame/api/batch_utils.py b/openpype/hosts/flame/api/batch_utils.py index 23d16f8d1a..20118c249c 100644 --- a/openpype/hosts/flame/api/batch_utils.py +++ b/openpype/hosts/flame/api/batch_utils.py @@ -91,6 +91,9 @@ def create_batch_group_conent(batch_nodes, batch_links, batch_group=None): batch_nodes (list of dict): each dict is node definition batch_links (list of dict): each dict is link definition batch_group (PyBatch, optional): batch group. Defaults to None. + + Return: + dict: all batch nodes {name or id: PyNode} """ # make sure some batch obj is present batch_group = batch_group or flame.batch @@ -98,7 +101,6 @@ def create_batch_group_conent(batch_nodes, batch_links, batch_group=None): b.name.get_value(): b for b in batch_group.nodes } - created_nodes = {} for node in batch_nodes: # NOTE: node_props needs to be ideally OrederDict type node_id, node_type, node_props = ( @@ -121,7 +123,7 @@ def create_batch_group_conent(batch_nodes, batch_links, batch_group=None): setattr(batch_node, key, value) # add created node for possible linking - created_nodes[node_id] = batch_node + all_batch_nodes[node_id] = batch_node # link nodes to each other for link in batch_links: @@ -129,16 +131,18 @@ def create_batch_group_conent(batch_nodes, batch_links, batch_group=None): # check if all linking nodes are available if not all([ - created_nodes.get(_from_n["id"]), - created_nodes.get(_to_n["id"]) + all_batch_nodes.get(_from_n["id"]), + all_batch_nodes.get(_to_n["id"]) ]): continue # link nodes in defined link batch_group.connect_nodes( - created_nodes[_from_n["id"]], _from_n["connector"], - created_nodes[_to_n["id"]], _to_n["connector"] + all_batch_nodes[_from_n["id"]], _from_n["connector"], + all_batch_nodes[_to_n["id"]], _to_n["connector"] ) # sort batch nodes batch_group.organize() + + return all_batch_nodes From 4da106cf3def8ee5d2dfb91076f0fecb7f05bc0e Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 7 Apr 2022 15:06:14 +0200 Subject: [PATCH 068/126] flame: debug log nodes attrs --- .../flame/plugins/publish/integrate_batch_group.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py index c70c2baa4a..350acdfa90 100644 --- a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -32,7 +32,16 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): bgroup = self._get_batch_group(instance, task_data) # add batch group content - self._add_nodes_to_batch_with_links(instance, task_data, bgroup) + all_batch_nodes = self._add_nodes_to_batch_with_links( + instance, task_data, bgroup) + + for name, node in all_batch_nodes: + self.log.debug("name: {}, dir: {}".format( + name, dir(node) + )) + self.log.debug("__ node.attributes: {}".format( + node.attributes + )) # load plate to batch group self.log.info("Loading subset `{}` into batch `{}`".format( @@ -70,7 +79,7 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): ] # add nodes into batch group - opfapi.create_batch_group_conent( + return opfapi.create_batch_group_conent( batch_nodes, batch_links, batch_group) def _load_clip_to_context(self, instance, bgroup): From d0a2a781aea6009a2b928c7752ef64ae87c47fab Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 7 Apr 2022 15:08:45 +0200 Subject: [PATCH 069/126] flame: fixing head and tail --- .../hosts/flame/plugins/publish/collect_timeline_instances.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/flame/plugins/publish/collect_timeline_instances.py b/openpype/hosts/flame/plugins/publish/collect_timeline_instances.py index fe9bce5232..0af769a380 100644 --- a/openpype/hosts/flame/plugins/publish/collect_timeline_instances.py +++ b/openpype/hosts/flame/plugins/publish/collect_timeline_instances.py @@ -72,9 +72,9 @@ class CollectTimelineInstances(pyblish.api.ContextPlugin): # solve handles length marker_data["handleStart"] = min( - marker_data["handleStart"], head) + marker_data["handleStart"], abs(head)) marker_data["handleEnd"] = min( - marker_data["handleEnd"], tail) + marker_data["handleEnd"], abs(tail)) with_audio = bool(marker_data.pop("audio")) From 1535b4712409a57bbbbddb73a65261106423ec7b Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 7 Apr 2022 15:10:23 +0200 Subject: [PATCH 070/126] flame: add hack comment --- .../hosts/flame/plugins/publish/collect_timeline_instances.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openpype/hosts/flame/plugins/publish/collect_timeline_instances.py b/openpype/hosts/flame/plugins/publish/collect_timeline_instances.py index 0af769a380..95c2002bd9 100644 --- a/openpype/hosts/flame/plugins/publish/collect_timeline_instances.py +++ b/openpype/hosts/flame/plugins/publish/collect_timeline_instances.py @@ -247,6 +247,7 @@ class CollectTimelineInstances(pyblish.api.ContextPlugin): head = clip_data.get("segment_head") tail = clip_data.get("segment_tail") + # HACK: it is here to serve for versions bellow 2021.1 if not head: head = int(clip_data["source_in"]) - int(first_frame) if not tail: From 304584573b7c67aac46f6b255f1ceb09fe1b4d7f Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 7 Apr 2022 15:14:32 +0200 Subject: [PATCH 071/126] flame: missing dict items function call --- openpype/hosts/flame/plugins/publish/integrate_batch_group.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py index 350acdfa90..cac99a25ac 100644 --- a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -35,7 +35,7 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): all_batch_nodes = self._add_nodes_to_batch_with_links( instance, task_data, bgroup) - for name, node in all_batch_nodes: + for name, node in all_batch_nodes.items(): self.log.debug("name: {}, dir: {}".format( name, dir(node) )) From 16bd11083da52c5836e183e8895524279378f08e Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 7 Apr 2022 15:22:09 +0200 Subject: [PATCH 072/126] flame: set node name if it doesn't exists in node props then set it from node_id --- openpype/hosts/flame/api/batch_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/flame/api/batch_utils.py b/openpype/hosts/flame/api/batch_utils.py index 20118c249c..d4c8294466 100644 --- a/openpype/hosts/flame/api/batch_utils.py +++ b/openpype/hosts/flame/api/batch_utils.py @@ -107,7 +107,7 @@ def create_batch_group_conent(batch_nodes, batch_links, batch_group=None): node["id"], node["type"], node["properties"]) # get node name for checking if exists - node_name = node_props.get("name") or node_id + node_name = node_props.pop("name", None) or node_id if all_batch_nodes.get(node_name): # update existing batch node @@ -116,6 +116,9 @@ def create_batch_group_conent(batch_nodes, batch_links, batch_group=None): # create new batch node batch_node = batch_group.create_node(node_type) + # set name + setattr(batch_node, "name", node_name) + # set attributes found in node props for key, value in node_props.items(): if not hasattr(batch_node, key): From aa0176aeb71af90fd8808d30ffed0098ac55a8b9 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 7 Apr 2022 15:58:01 +0200 Subject: [PATCH 073/126] modified extract slate frame to add values based on templates --- .../plugins/publish/extract_slate_frame.py | 58 +++++++++++++++---- 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/openpype/hosts/nuke/plugins/publish/extract_slate_frame.py b/openpype/hosts/nuke/plugins/publish/extract_slate_frame.py index e917a28046..9737d4d5f8 100644 --- a/openpype/hosts/nuke/plugins/publish/extract_slate_frame.py +++ b/openpype/hosts/nuke/plugins/publish/extract_slate_frame.py @@ -1,6 +1,8 @@ import os import nuke +import six import pyblish.api + import openpype from openpype.hosts.nuke.api.lib import maintained_selection @@ -18,6 +20,10 @@ class ExtractSlateFrame(openpype.api.Extractor): families = ["slate"] hosts = ["nuke"] + key_value_mapping = { + "f_submission_note": [True, "{comment}"], + "f_submitting_for": [True, "{intent[value]}"] + } def process(self, instance): if hasattr(self, "viewer_lut_raw"): @@ -129,9 +135,7 @@ class ExtractSlateFrame(openpype.api.Extractor): for node in temporary_nodes: nuke.delete(node) - def get_view_process_node(self): - # Select only the target node if nuke.selectedNodes(): [n.setSelected(False) for n in nuke.selectedNodes()] @@ -162,13 +166,45 @@ class ExtractSlateFrame(openpype.api.Extractor): return comment = instance.context.data.get("comment") - intent_value = instance.context.data.get("intent") - if intent_value and isinstance(intent_value, dict): - intent_value = intent_value.get("value") + intent = instance.context.data.get("intent") + if not isinstance(intent, dict): + intent = { + "label": intent, + "value": intent + } - try: - node["f_submission_note"].setValue(comment) - node["f_submitting_for"].setValue(intent_value or "") - except NameError: - return - instance.data.pop("slateNode") + fill_data = { + "comment": comment, + "intent": intent + } + + for key, value in self.key_value_mapping.items(): + enabled, template = value + if not enabled: + continue + + try: + value = template.format(**fill_data) + + except ValueError: + self.log.warning( + "Couldn't fill template \"{}\" with data: {}".format( + template, fill_data + ), + exc_info=True + ) + continue + + except KeyError: + self.log.warning( + "Template contains unknown key", + exc_info=True + ) + continue + + try: + node[key].setValue(value) + except NameError: + self.log.warning( + "Failed to set value \"{}\" on node attribute \"{}\"" + ).format(value)) From 18c53a527a22c1a52d12c181d4d6302715ddf148 Mon Sep 17 00:00:00 2001 From: jrsndlr Date: Thu, 7 Apr 2022 16:06:20 +0200 Subject: [PATCH 074/126] Nuke Tut doc fix remove artist_host_nuke.md, fix in-page link to artist_hosts_nuke.md --- website/docs/artist_hosts_nuke.md | 145 -------------------------- website/docs/artist_hosts_nuke_tut.md | 2 +- 2 files changed, 1 insertion(+), 146 deletions(-) delete mode 100644 website/docs/artist_hosts_nuke.md diff --git a/website/docs/artist_hosts_nuke.md b/website/docs/artist_hosts_nuke.md deleted file mode 100644 index 1e02599570..0000000000 --- a/website/docs/artist_hosts_nuke.md +++ /dev/null @@ -1,145 +0,0 @@ ---- -id: artist_hosts_nuke -title: Nuke -sidebar_label: Nuke ---- - -:::important -After Nuke starts it will automatically **Apply All Settings** for you. If you are sure the settings are wrong just contact your supervisor and he will set them correctly for you in project database. -::: - -:::note -The workflows are identical for both. We are supporting versions **`11.0`** and above. -::: - -## OpenPype global tools - -- [Set Context](artist_tools.md#set-context) -- [Work Files](artist_tools.md#workfiles) -- [Create](artist_tools.md#creator) -- [Load](artist_tools.md#loader) -- [Manage (Inventory)](artist_tools.md#inventory) -- [Publish](artist_tools.md#publisher) -- [Library Loader](artist_tools.md#library-loader) - -## Nuke specific tools - -
-
- -### Set Frame Ranges - -Use this feature in case you are not sure the frame range is correct. - -##### Result - -- setting Frame Range in script settings -- setting Frame Range in viewers (timeline) - -
-
- -![Set Frame Ranges](assets/nuke_setFrameRanges.png) - -
-
- - -
- -![Set Frame Ranges Timeline](assets/nuke_setFrameRanges_timeline.png) - -
- -1. limiting to Frame Range without handles -2. **Input** handle on start -3. **Output** handle on end - -
-
- -### Set Resolution - -
-
- - -This menu item will set correct resolution format for you defined by your production. - -##### Result - -- creates new item in formats with project name -- sets the new format as used - -
-
- -![Set Resolution](assets/nuke_setResolution.png) - -
-
- - -### Set Colorspace - -
-
- -This menu item will set correct Colorspace definitions for you. All has to be configured by your production (Project coordinator). - -##### Result - -- set Colorspace in your script settings -- set preview LUT to your viewers -- set correct colorspace to all discovered Read nodes (following expression set in settings) - -
-
- -![Set Colorspace](assets/nuke_setColorspace.png) - -
-
- - -### Apply All Settings - -
-
- -It is usually enough if you once per while use this option just to make yourself sure the workfile is having set correct properties. - -##### Result - -- set Frame Ranges -- set Colorspace -- set Resolution - -
-
- -![Apply All Settings](assets/nuke_applyAllSettings.png) - -
-
- -### Build Workfile - -
-
- -This tool will append all available subsets into an actual node graph. It will look into database and get all last [versions](artist_concepts.md#version) of available [subsets](artist_concepts.md#subset). - - -##### Result - -- adds all last versions of subsets (rendered image sequences) as read nodes -- adds publishable write node as `renderMain` subset - -
-
- -![Build First Work File](assets/nuke_buildFirstWorkfile.png) - -
-
\ No newline at end of file diff --git a/website/docs/artist_hosts_nuke_tut.md b/website/docs/artist_hosts_nuke_tut.md index 4b0ef7a78a..eefb213dd2 100644 --- a/website/docs/artist_hosts_nuke_tut.md +++ b/website/docs/artist_hosts_nuke_tut.md @@ -161,7 +161,7 @@ Nuke OpenPype menu shows the current context Launching Nuke with context stops your timer, and starts the clock on the shot and task you picked. -Openpype makes initial setup for your Nuke script. It is the same as running [Apply All Settings](artist_hosts_nuke.md#apply-all-settings) from the OpenPype menu. +Openpype makes initial setup for your Nuke script. It is the same as running [Apply All Settings](artist_hosts_nuke_tut.md#apply-all-settings) from the OpenPype menu. - Reads frame range and resolution from Avalon database, sets it in Nuke Project Settings, Creates Viewer node, sets it’s range and indicates handles by In and Out points. From 5bc3516baf676d115a992a6f63828a5c110c2556 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 7 Apr 2022 16:43:46 +0200 Subject: [PATCH 075/126] added settings for slate templates --- .../plugins/publish/extract_slate_frame.py | 15 ++++-- .../defaults/project_settings/nuke.json | 14 +++++- .../schemas/schema_nuke_publish.json | 47 +++++++++++++++++++ 3 files changed, 70 insertions(+), 6 deletions(-) diff --git a/openpype/hosts/nuke/plugins/publish/extract_slate_frame.py b/openpype/hosts/nuke/plugins/publish/extract_slate_frame.py index 9737d4d5f8..f71d3ffff5 100644 --- a/openpype/hosts/nuke/plugins/publish/extract_slate_frame.py +++ b/openpype/hosts/nuke/plugins/publish/extract_slate_frame.py @@ -1,6 +1,7 @@ import os import nuke -import six +import copy + import pyblish.api import openpype @@ -20,9 +21,12 @@ class ExtractSlateFrame(openpype.api.Extractor): families = ["slate"] hosts = ["nuke"] + # Settings values + # - can be extended by other attributes from node in the future key_value_mapping = { "f_submission_note": [True, "{comment}"], - "f_submitting_for": [True, "{intent[value]}"] + "f_submitting_for": [True, "{intent[value]}"], + "f_vfx_scope_of_work": [False, ""] } def process(self, instance): @@ -173,10 +177,11 @@ class ExtractSlateFrame(openpype.api.Extractor): "value": intent } - fill_data = { + fill_data = copy.deepcopy(instance.data["anatomyData"]) + fill_data.update({ "comment": comment, "intent": intent - } + }) for key, value in self.key_value_mapping.items(): enabled, template = value @@ -205,6 +210,6 @@ class ExtractSlateFrame(openpype.api.Extractor): try: node[key].setValue(value) except NameError: - self.log.warning( + self.log.warning(( "Failed to set value \"{}\" on node attribute \"{}\"" ).format(value)) diff --git a/openpype/settings/defaults/project_settings/nuke.json b/openpype/settings/defaults/project_settings/nuke.json index 44d7f2d9d0..bdccb9b38e 100644 --- a/openpype/settings/defaults/project_settings/nuke.json +++ b/openpype/settings/defaults/project_settings/nuke.json @@ -160,7 +160,19 @@ } }, "ExtractSlateFrame": { - "viewer_lut_raw": false + "viewer_lut_raw": false, + "f_submission_note": [ + true, + "{comment}" + ], + "f_submitting_for": [ + true, + "{intent[value]}" + ], + "f_vfx_scope_of_work": [ + false, + "" + ] }, "IncrementScriptVersion": { "enabled": true, diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_nuke_publish.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_nuke_publish.json index 27e8957786..8dbf224ce5 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_nuke_publish.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_nuke_publish.json @@ -389,6 +389,53 @@ "type": "boolean", "key": "viewer_lut_raw", "label": "Viewer LUT raw" + }, + { + "type": "separator" + }, + { + "type": "label", + "label": "Fill specific slate node values with templates. Uncheck the checkbox to not change the value.", + "word_wrap": true + }, + { + "type": "list-strict", + "key": "f_submission_note", + "label": "Submission Note", + "object_types": [ + { + "type": "boolean" + }, + { + "type": "text" + } + ] + }, + { + "type": "list-strict", + "key": "f_submitting_for", + "label": "Submission For", + "object_types": [ + { + "type": "boolean" + }, + { + "type": "text" + } + ] + }, + { + "type": "list-strict", + "key": "f_vfx_scope_of_work", + "label": "VFX Scope Of Work", + "object_types": [ + { + "type": "boolean" + }, + { + "type": "text" + } + ] } ] }, From afbacb1944b690f03e31fb129d39476880c3934a Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 7 Apr 2022 17:14:33 +0200 Subject: [PATCH 076/126] flame: fixing setter for node name --- openpype/hosts/flame/api/batch_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/flame/api/batch_utils.py b/openpype/hosts/flame/api/batch_utils.py index d4c8294466..9d419a4a90 100644 --- a/openpype/hosts/flame/api/batch_utils.py +++ b/openpype/hosts/flame/api/batch_utils.py @@ -117,7 +117,7 @@ def create_batch_group_conent(batch_nodes, batch_links, batch_group=None): batch_node = batch_group.create_node(node_type) # set name - setattr(batch_node, "name", node_name) + batch_node.name.set_value(node_name) # set attributes found in node props for key, value in node_props.items(): From fc6d01d0043df4e019ab89cf44d8c592e71be8e3 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 7 Apr 2022 17:15:44 +0200 Subject: [PATCH 077/126] hound catch --- openpype/hosts/flame/api/lib.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/flame/api/lib.py b/openpype/hosts/flame/api/lib.py index 998d7dfa7e..a4d8a7f9f0 100644 --- a/openpype/hosts/flame/api/lib.py +++ b/openpype/hosts/flame/api/lib.py @@ -826,5 +826,7 @@ class MediaInfoFile: def _write_result_xml_to_file(self, file, xml_data): # save it as new file tree = cET.ElementTree(xml_data) - tree.write(file, xml_declaration=True, - method='xml', encoding='UTF-8') \ No newline at end of file + tree.write( + file, xml_declaration=True, + method='xml', encoding='UTF-8' + ) From 81b2be514f970e19ccc292bee0afba80f893cb83 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 7 Apr 2022 18:47:37 +0200 Subject: [PATCH 078/126] changed order of ftrack collectors --- openpype/modules/ftrack/plugins/publish/collect_ftrack_api.py | 2 +- .../modules/ftrack/plugins/publish/collect_ftrack_family.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/modules/ftrack/plugins/publish/collect_ftrack_api.py b/openpype/modules/ftrack/plugins/publish/collect_ftrack_api.py index 07af217fb6..436a61cc18 100644 --- a/openpype/modules/ftrack/plugins/publish/collect_ftrack_api.py +++ b/openpype/modules/ftrack/plugins/publish/collect_ftrack_api.py @@ -6,7 +6,7 @@ import avalon.api class CollectFtrackApi(pyblish.api.ContextPlugin): """ Collects an ftrack session and the current task id. """ - order = pyblish.api.CollectorOrder + 0.4999 + order = pyblish.api.CollectorOrder + 0.4991 label = "Collect Ftrack Api" def process(self, context): diff --git a/openpype/modules/ftrack/plugins/publish/collect_ftrack_family.py b/openpype/modules/ftrack/plugins/publish/collect_ftrack_family.py index 70030acad9..95987fe42e 100644 --- a/openpype/modules/ftrack/plugins/publish/collect_ftrack_family.py +++ b/openpype/modules/ftrack/plugins/publish/collect_ftrack_family.py @@ -25,7 +25,7 @@ class CollectFtrackFamily(pyblish.api.InstancePlugin): based on 'families' (editorial drives it by presence of 'review') """ label = "Collect Ftrack Family" - order = pyblish.api.CollectorOrder + 0.4998 + order = pyblish.api.CollectorOrder + 0.4990 profiles = None From a34f279685d087dda82c99a6b0c32f9ceb9cc907 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 7 Apr 2022 18:47:53 +0200 Subject: [PATCH 079/126] modified labels to contain colons --- .../projects_schema/schemas/schema_nuke_publish.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_nuke_publish.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_nuke_publish.json index 8dbf224ce5..3bf0eb3214 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_nuke_publish.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_nuke_publish.json @@ -401,7 +401,7 @@ { "type": "list-strict", "key": "f_submission_note", - "label": "Submission Note", + "label": "Submission Note:", "object_types": [ { "type": "boolean" @@ -414,7 +414,7 @@ { "type": "list-strict", "key": "f_submitting_for", - "label": "Submission For", + "label": "Submission For:", "object_types": [ { "type": "boolean" @@ -427,7 +427,7 @@ { "type": "list-strict", "key": "f_vfx_scope_of_work", - "label": "VFX Scope Of Work", + "label": "VFX Scope Of Work:", "object_types": [ { "type": "boolean" From 2ce4704a981f621d4bdfbdc7af27840edf0cb409 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 7 Apr 2022 18:55:43 +0200 Subject: [PATCH 080/126] added collector looking for custom attribute values and it's settings --- .../modules/ftrack/lib/custom_attributes.py | 2 +- .../publish/collect_custom_attributes_data.py | 138 ++++++++++++++++++ .../defaults/project_settings/ftrack.json | 4 + .../schema_project_ftrack.json | 25 ++++ 4 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 openpype/modules/ftrack/plugins/publish/collect_custom_attributes_data.py diff --git a/openpype/modules/ftrack/lib/custom_attributes.py b/openpype/modules/ftrack/lib/custom_attributes.py index 29c6b5e7f8..2f53815368 100644 --- a/openpype/modules/ftrack/lib/custom_attributes.py +++ b/openpype/modules/ftrack/lib/custom_attributes.py @@ -135,7 +135,7 @@ def query_custom_attributes( output.extend( session.query( ( - "select value, entity_id from {}" + "select value, entity_id, configuration_id from {}" " where entity_id in ({}) and configuration_id in ({})" ).format( table_name, diff --git a/openpype/modules/ftrack/plugins/publish/collect_custom_attributes_data.py b/openpype/modules/ftrack/plugins/publish/collect_custom_attributes_data.py new file mode 100644 index 0000000000..f04c7c7954 --- /dev/null +++ b/openpype/modules/ftrack/plugins/publish/collect_custom_attributes_data.py @@ -0,0 +1,138 @@ +""" +Requires: + context > ftrackSession + context > ftrackEntity + instance > ftrackEntity + +Provides: + instance > customData > ftrack +""" +import copy + +import pyblish.api + + +class CollectFtrackCustomAttributeData(pyblish.api.ContextPlugin): + """Collect custom attribute values and store them to customData. + + Data are stored into each instance in context under + instance.data["customData"]["ftrack"]. + """ + + order = pyblish.api.CollectorOrder + 0.4992 + label = "Collect Ftrack Custom Attribute Data" + + # Name of custom attributes for which will be look for + custom_attribute_keys = [] + + def process(self, context): + if not self.custom_attribute_keys: + self.log.info("Custom attribute keys are not set. Skipping") + return + + ftrack_entities_by_id = {} + default_entity_id = None + + context_entity = context.data.get("ftrackEntity") + if context_entity: + entity_id = context_entity["id"] + default_entity_id = entity_id + ftrack_entities_by_id[entity_id] = context_entity + + instances_by_entity_id = { + default_entity_id: [] + } + for instance in context: + entity = instance.data.get("ftrackEntity") + if not entity: + instances_by_entity_id[default_entity_id].append(instance) + continue + + entity_id = entity["id"] + ftrack_entities_by_id[entity_id] = entity + if entity_id not in instances_by_entity_id: + instances_by_entity_id[entity_id] = [] + instances_by_entity_id[entity_id].append(instance) + + if not ftrack_entities_by_id: + self.log.info("Ftrack entities are not set. Skipping") + return + + session = context.data["ftrackSession"] + custom_attr_key_by_id = self.query_attr_confs(session) + if not custom_attr_key_by_id: + self.log.info(( + "Didn't find any of defined custom attributes {}" + ).format(", ".join(self.custom_attribute_keys))) + return + + entity_ids = list(instances_by_entity_id.keys()) + values_by_entity_id = self.query_attr_values( + session, entity_ids, custom_attr_key_by_id + ) + + for entity_id, instances in instances_by_entity_id.items(): + if entity_id not in values_by_entity_id: + # Use defaut empty values + entity_id = None + + value = values_by_entity_id[entity_id] + if "customData" not in instance.data: + instance.data["customData"] = {} + instance.data["customData"]["ftrack"] = copy.deepcopy(value) + + def query_attr_values(self, session, entity_ids, custom_attr_key_by_id): + # Prepare values for query + entity_ids_joined = ",".join([ + '"{}"'.format(entity_id) + for entity_id in entity_ids + ]) + conf_ids_joined = ",".join([ + '"{}"'.format(conf_id) + for conf_id in custom_attr_key_by_id.keys() + ]) + # Query custom attribute values + value_items = session.query( + ( + "select value, entity_id, configuration_id" + " from CustomAttributeValue" + " where entity_id in ({}) and configuration_id in ({})" + ).format( + entity_ids_joined, + conf_ids_joined + ) + ).all() + + # Prepare default value output per entity id + values_by_key = { + key: None for key in self.custom_attribute_keys + } + # Prepare all entity ids that were queried + values_by_entity_id = { + entity_id: copy.deepcopy(values_by_key) + for entity_id in entity_ids + } + # Add none entity id which is used as default value + values_by_entity_id[None] = copy.deepcopy(values_by_key) + # Go through queried data and store them + for item in value_items: + conf_id = item["configuration_id"] + conf_key = custom_attr_key_by_id[conf_id] + entity_id = item["entity_id"] + values_by_entity_id[entity_id][conf_key] = item["value"] + return values_by_entity_id + + def query_attr_confs(self, session): + custom_attributes = set(self.custom_attribute_keys) + cust_attrs_query = ( + "select id, key from CustomAttributeConfiguration" + " where key in ({})" + ).format(", ".join( + ["\"{}\"".format(attr_name) for attr_name in custom_attributes] + )) + + custom_attr_confs = session.query(cust_attrs_query).all() + return { + conf["id"]: conf["key"] + for conf in custom_attr_confs + } diff --git a/openpype/settings/defaults/project_settings/ftrack.json b/openpype/settings/defaults/project_settings/ftrack.json index 31d6a70ac7..deade08c0b 100644 --- a/openpype/settings/defaults/project_settings/ftrack.json +++ b/openpype/settings/defaults/project_settings/ftrack.json @@ -352,6 +352,10 @@ } ] }, + "CollectFtrackCustomAttributeData": { + "enabled": false, + "custom_attribute_keys": [] + }, "IntegrateFtrackNote": { "enabled": true, "note_template": "{intent}: {comment}", diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_ftrack.json b/openpype/settings/entities/schemas/projects_schema/schema_project_ftrack.json index 5ce9b24b4b..47effb3dbd 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_ftrack.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_ftrack.json @@ -725,6 +725,31 @@ } ] }, + { + "type": "dict", + "collapsible": true, + "checkbox_key": "enabled", + "key": "CollectFtrackCustomAttributeData", + "label": "Collect Custom Attribute Data", + "is_group": true, + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + }, + { + "type": "label", + "label": "Collect custom attributes from ftrack for ftrack entities that can be used in some templates during publishing." + }, + { + "type": "list", + "key": "custom_attribute_keys", + "label": "Custom attribute keys", + "object_type": "text" + } + ] + }, { "type": "dict", "collapsible": true, From ab252ec7c9b7af96835ff248bce0cc75d6df6bd9 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 7 Apr 2022 18:56:00 +0200 Subject: [PATCH 081/126] added ability to define word wrap of labels --- openpype/settings/entities/schemas/README.md | 1 + openpype/tools/settings/settings/base.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/openpype/settings/entities/schemas/README.md b/openpype/settings/entities/schemas/README.md index fbfd699937..b4bfef2972 100644 --- a/openpype/settings/entities/schemas/README.md +++ b/openpype/settings/entities/schemas/README.md @@ -745,6 +745,7 @@ How output of the schema could look like on save: ### label - add label with note or explanations - it is possible to use html tags inside the label +- set `work_wrap` to `true`/`false` if you want to enable word wrapping in UI (default: `false`) ``` { diff --git a/openpype/tools/settings/settings/base.py b/openpype/tools/settings/settings/base.py index bd48b3a966..44ec09b2ca 100644 --- a/openpype/tools/settings/settings/base.py +++ b/openpype/tools/settings/settings/base.py @@ -567,7 +567,9 @@ class GUIWidget(BaseWidget): def _create_label_ui(self): label = self.entity["label"] + word_wrap = self.entity.schema_data.get("word_wrap", False) label_widget = QtWidgets.QLabel(label, self) + label_widget.setWordWrap(word_wrap) label_widget.setTextInteractionFlags(QtCore.Qt.TextBrowserInteraction) label_widget.setObjectName("SettingsLabel") label_widget.linkActivated.connect(self._on_link_activate) From 486317cb96826185cd1257c880ee04681dd6a264 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 7 Apr 2022 19:06:25 +0200 Subject: [PATCH 082/126] added custom data to slate text formatting data --- openpype/hosts/nuke/plugins/publish/extract_slate_frame.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openpype/hosts/nuke/plugins/publish/extract_slate_frame.py b/openpype/hosts/nuke/plugins/publish/extract_slate_frame.py index f71d3ffff5..6935afe144 100644 --- a/openpype/hosts/nuke/plugins/publish/extract_slate_frame.py +++ b/openpype/hosts/nuke/plugins/publish/extract_slate_frame.py @@ -179,6 +179,9 @@ class ExtractSlateFrame(openpype.api.Extractor): fill_data = copy.deepcopy(instance.data["anatomyData"]) fill_data.update({ + "custom": copy.deepcopy( + instance.data.get("customData") or {} + ), "comment": comment, "intent": intent }) From bedccd6f69540117379f7db680260ebd3bfdcea2 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 7 Apr 2022 19:06:35 +0200 Subject: [PATCH 083/126] added custom data to burnin custm data --- openpype/plugins/publish/extract_burnin.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/openpype/plugins/publish/extract_burnin.py b/openpype/plugins/publish/extract_burnin.py index b2ca8850b6..a543083a87 100644 --- a/openpype/plugins/publish/extract_burnin.py +++ b/openpype/plugins/publish/extract_burnin.py @@ -221,11 +221,17 @@ class ExtractBurnin(openpype.api.Extractor): filled_anatomy = anatomy.format_all(burnin_data) burnin_data["anatomy"] = filled_anatomy.get_solved() - # Add context data burnin_data. - burnin_data["custom"] = ( + custom_data = copy.deepcopy( + instance.data.get("customData") or {} + ) + # Backwards compatibility + custom_data.update( instance.data.get("custom_burnin_data") or {} ) + # Add context data burnin_data. + burnin_data["custom"] = custom_data + # Add source camera name to burnin data camera_name = repre.get("camera_name") if camera_name: From f9c2f87f903d4f411b8b950997e258756a6aa0f3 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 7 Apr 2022 19:09:29 +0200 Subject: [PATCH 084/126] fixed adding data to instance --- .../publish/collect_custom_attributes_data.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/openpype/modules/ftrack/plugins/publish/collect_custom_attributes_data.py b/openpype/modules/ftrack/plugins/publish/collect_custom_attributes_data.py index f04c7c7954..ef0e4a9ccb 100644 --- a/openpype/modules/ftrack/plugins/publish/collect_custom_attributes_data.py +++ b/openpype/modules/ftrack/plugins/publish/collect_custom_attributes_data.py @@ -76,10 +76,17 @@ class CollectFtrackCustomAttributeData(pyblish.api.ContextPlugin): # Use defaut empty values entity_id = None - value = values_by_entity_id[entity_id] - if "customData" not in instance.data: - instance.data["customData"] = {} - instance.data["customData"]["ftrack"] = copy.deepcopy(value) + for instance in instances: + value = copy.deepcopy(values_by_entity_id[entity_id]) + if "customData" not in instance.data: + instance.data["customData"] = {} + instance.data["customData"]["ftrack"] = value + instance_label = ( + instance.data.get("label") or instance.data["name"] + ) + self.log.debug(( + "Added ftrack custom data to instance \"{}\": {}" + ).format(instance_label, value)) def query_attr_values(self, session, entity_ids, custom_attr_key_by_id): # Prepare values for query From a2eb78a3a7d3c275fb650c70cd3d67a275d80b9c Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 7 Apr 2022 19:17:05 +0200 Subject: [PATCH 085/126] added few more comments --- .../ftrack/plugins/publish/collect_custom_attributes_data.py | 3 +++ openpype/plugins/publish/extract_burnin.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/openpype/modules/ftrack/plugins/publish/collect_custom_attributes_data.py b/openpype/modules/ftrack/plugins/publish/collect_custom_attributes_data.py index ef0e4a9ccb..43fa3bc3f8 100644 --- a/openpype/modules/ftrack/plugins/publish/collect_custom_attributes_data.py +++ b/openpype/modules/ftrack/plugins/publish/collect_custom_attributes_data.py @@ -17,6 +17,9 @@ class CollectFtrackCustomAttributeData(pyblish.api.ContextPlugin): Data are stored into each instance in context under instance.data["customData"]["ftrack"]. + + Hierarchical attributes are not looked up properly for that functionality + custom attribute values lookup must be extended. """ order = pyblish.api.CollectorOrder + 0.4992 diff --git a/openpype/plugins/publish/extract_burnin.py b/openpype/plugins/publish/extract_burnin.py index a543083a87..41c84103a6 100644 --- a/openpype/plugins/publish/extract_burnin.py +++ b/openpype/plugins/publish/extract_burnin.py @@ -224,7 +224,7 @@ class ExtractBurnin(openpype.api.Extractor): custom_data = copy.deepcopy( instance.data.get("customData") or {} ) - # Backwards compatibility + # Backwards compatibility (since 2022/04/07) custom_data.update( instance.data.get("custom_burnin_data") or {} ) From 5f1940a9982ffee2266776d0f70812853241c941 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 8 Apr 2022 11:17:22 +0200 Subject: [PATCH 086/126] flame: adding maintainable temp file path --- openpype/hosts/flame/api/__init__.py | 2 ++ openpype/hosts/flame/api/lib.py | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/openpype/hosts/flame/api/__init__.py b/openpype/hosts/flame/api/__init__.py index a0c40904ed..6744a7ff11 100644 --- a/openpype/hosts/flame/api/__init__.py +++ b/openpype/hosts/flame/api/__init__.py @@ -27,6 +27,7 @@ from .lib import ( get_frame_from_filename, get_padding_from_filename, maintained_object_duplication, + maintained_temp_file_path, get_clip_segment, get_batch_group_from_desktop, MediaInfoFile @@ -103,6 +104,7 @@ __all__ = [ "get_frame_from_filename", "get_padding_from_filename", "maintained_object_duplication", + "maintained_temp_file_path", "get_clip_segment", "get_batch_group_from_desktop", "MediaInfoFile", diff --git a/openpype/hosts/flame/api/lib.py b/openpype/hosts/flame/api/lib.py index a4d8a7f9f0..51d48becf1 100644 --- a/openpype/hosts/flame/api/lib.py +++ b/openpype/hosts/flame/api/lib.py @@ -4,6 +4,7 @@ import re import six import json import pickle +import tempfile import itertools import contextlib import xml.etree.cElementTree as cET @@ -695,6 +696,25 @@ def maintained_object_duplication(item): flame.delete(duplicate) +@contextlib.contextmanager +def maintained_temp_file_path(suffix=None): + _suffix = suffix or "" + + try: + # Store dumped json to temporary file + temporary_file = tempfile.mktemp( + suffix=_suffix, prefix="flame_maintained_") + yield temporary_file.name.replace("\\", "/") + + except IOError as _error: + raise IOError( + "Not able to create temp json file: {}".format(_error)) from _error + + finally: + # Remove the temporary json + os.remove(temporary_file) + + def get_clip_segment(flame_clip): name = flame_clip.name.get_value() version = flame_clip.versions[0] From 03bf240816d98bd47e0944eac0a57706c7cc0869 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 8 Apr 2022 11:18:10 +0200 Subject: [PATCH 087/126] flame: refactoring MediaInfoFile class so it is parentable --- openpype/hosts/flame/api/lib.py | 161 +++++++++++++++++++++++--------- 1 file changed, 116 insertions(+), 45 deletions(-) diff --git a/openpype/hosts/flame/api/lib.py b/openpype/hosts/flame/api/lib.py index 51d48becf1..de3467aa76 100644 --- a/openpype/hosts/flame/api/lib.py +++ b/openpype/hosts/flame/api/lib.py @@ -1,7 +1,6 @@ import sys import os import re -import six import json import pickle import tempfile @@ -740,19 +739,34 @@ def get_batch_group_from_desktop(name): class MediaInfoFile: - media_script_path = "/opt/Autodesk/mio/current/dl_get_media_info" - tmp_name = "_tmp.clip" - tmp_file = None + """Class to get media info file clip data - clip_data = None - out_feed_nb_ticks = None - out_feed_fps = None - out_feed_drop_mode = None + Raises: + IOError: MEDIA_SCRIPT_PATH path doesn't exists + TypeError: Not able to generate clip xml data file + ET.ParseError: Missing clip in xml clip data + IOError: Not able to save xml clip data to file + + Attributes: + str: `MEDIA_SCRIPT_PATH` path to flame binary + logging.Logger: `log` logger + """ + MEDIA_SCRIPT_PATH = "/opt/Autodesk/mio/current/dl_get_media_info" log = log - def __init__(self, path): - # test if media script paht exists + _clip_data = None + _start_frame = None + _fps = None + _drop_mode = None + + def __init__(self, path, **kwargs): + + # replace log if any + if kwargs.get("log"): + self.log = kwargs["log"] + + # test if `dl_get_media_info` paht exists self._validate_media_script_path() # derivate other feed variables @@ -760,40 +774,93 @@ class MediaInfoFile: self.feed_dir = os.path.dirname(path) self.feed_ext = os.path.splitext(self.feed_basename)[1][1:].lower() - self.tmp_file = os.path.join(self.feed_dir, self.tmp_name) + with maintained_temp_file_path(".clip") as tmp_path: + self.log.info("Temp File: {}".format(tmp_path)) + self._generate_media_info_file(tmp_path) - # remove previously generated temp files - # it will be regenerated - self._clear_tmp_file() + # get clip data and make them single if there is multiple + # clips data + xml_data = self._make_single_clip_media_info(tmp_path) - self.log.info("Temp File: {}".format(self.tmp_file)) + # get all time related data and assign them + self._get_time_info_from_origin(xml_data) + self.set_clip_data(xml_data) - self._generate_media_info_file() + @property + def clip_data(self): + """Clip's xml clip data + + Returns: + xml.etree.ElementTree: xml data + """ + return self._clip_data + + @clip_data.setter + def clip_data(self, data): + self._clip_data = data + + @property + def start_frame(self): + """ Clip's starting frame found in timecode + + Returns: + int: number of frames + """ + return self._start_frame + + @start_frame.setter + def start_frame(self, number): + self._start_frame = int(number) + + @property + def fps(self): + """ Clip's frame rate + + Returns: + float: frame rate + """ + return self._fps + + @fps.setter + def fps(self, fl_number): + self._fps = float(fl_number) + + @property + def drop_mode(self): + """ Clip's drop frame mode + + Returns: + str: drop frame flag + """ + return self._drop_mode + + @drop_mode.setter + def drop_mode(self, text): + self._drop_mode = str(text) def _validate_media_script_path(self): - if not os.path.isfile(self.media_script_path): + if not os.path.isfile(self.MEDIA_SCRIPT_PATH): raise IOError("Media Scirpt does not exist: `{}`".format( - self.media_script_path)) + self.MEDIA_SCRIPT_PATH)) - def _generate_media_info_file(self): + def _generate_media_info_file(self, fpath): # Create cmd arguments for gettig xml file info file cmd_args = [ - self.media_script_path, + self.MEDIA_SCRIPT_PATH, "-e", self.feed_ext, - "-o", self.tmp_file, + "-o", fpath, self.feed_dir ] - # execute creation of clip xml template data try: + # execute creation of clip xml template data openpype.run_subprocess(cmd_args) - self._make_single_clip_media_info() - except TypeError: - self.log.error("Error creating self.tmp_file") - six.reraise(*sys.exc_info()) + except TypeError as error: + raise TypeError( + "Error creating `{}` due: {}".format(fpath, error)) from error - def _make_single_clip_media_info(self): - with open(self.tmp_file) as f: + def _make_single_clip_media_info(self, fpath): + with open(fpath) as f: lines = f.readlines() _added_root = itertools.chain( "", deepcopy(lines)[1:], "") @@ -816,37 +883,41 @@ class MediaInfoFile: ] )) - self._get_time_info_from_origin(matching_clip) - self.clip_data = matching_clip - self._write_result_xml_to_file(self.tmp_file, matching_clip) - - def _clear_tmp_file(self): - if os.path.isfile(self.tmp_file): - os.remove(self.tmp_file) + return matching_clip def _get_time_info_from_origin(self, xml_data): try: for out_track in xml_data.iter('track'): for out_feed in out_track.iter('feed'): + # start frame out_feed_nb_ticks_obj = out_feed.find( 'startTimecode/nbTicks') - self.out_feed_nb_ticks = out_feed_nb_ticks_obj.text + self.start_frame(out_feed_nb_ticks_obj.text) + + # fps out_feed_fps_obj = out_feed.find( 'startTimecode/rate') - self.out_feed_fps = out_feed_fps_obj.text + self.fps(out_feed_fps_obj.text) + + # drop frame mode out_feed_drop_mode_obj = out_feed.find( 'startTimecode/dropMode') - self.out_feed_drop_mode = out_feed_drop_mode_obj.text + self.drop_mode(out_feed_drop_mode_obj.text) break else: continue except Exception as msg: self.log.warning(msg) - def _write_result_xml_to_file(self, file, xml_data): - # save it as new file - tree = cET.ElementTree(xml_data) - tree.write( - file, xml_declaration=True, - method='xml', encoding='UTF-8' - ) + @staticmethod + def write_clip_data_to_file(fpath, xml_data): + try: + # save it as new file + tree = cET.ElementTree(xml_data) + tree.write( + fpath, xml_declaration=True, + method='xml', encoding='UTF-8' + ) + except IOError as error: + raise IOError( + "Not able to write data to file: {}".format(error)) from error From 1848513fb90943266c882a1fd55840e2c069f24d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 8 Apr 2022 11:49:35 +0200 Subject: [PATCH 088/126] OP-3072 - check enablement as a first step for GDrive --- openpype/modules/sync_server/providers/gdrive.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/openpype/modules/sync_server/providers/gdrive.py b/openpype/modules/sync_server/providers/gdrive.py index b783f7958b..aa7329b104 100644 --- a/openpype/modules/sync_server/providers/gdrive.py +++ b/openpype/modules/sync_server/providers/gdrive.py @@ -73,6 +73,11 @@ class GDriveHandler(AbstractProvider): format(site_name)) return + if not self.presets["enabled"]: + log.debug("Sync Server: Site {} not enabled for {}.". + format(site_name, project_name)) + return + current_platform = platform.system().lower() cred_path = self.presets.get("credentials_url", {}). \ get(current_platform) or '' @@ -97,11 +102,10 @@ class GDriveHandler(AbstractProvider): return self.service = None - if self.presets["enabled"]: - self.service = self._get_gd_service(cred_path) + self.service = self._get_gd_service(cred_path) - self._tree = tree - self.active = True + self._tree = tree + self.active = True def is_active(self): """ From fc49311937375908301845196e8677967476a7c1 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 8 Apr 2022 11:50:32 +0200 Subject: [PATCH 089/126] OP-3072 - check enablement as a first step for Dropbox --- .../modules/sync_server/providers/dropbox.py | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/openpype/modules/sync_server/providers/dropbox.py b/openpype/modules/sync_server/providers/dropbox.py index f5910299e5..dfc42fed75 100644 --- a/openpype/modules/sync_server/providers/dropbox.py +++ b/openpype/modules/sync_server/providers/dropbox.py @@ -17,6 +17,7 @@ class DropboxHandler(AbstractProvider): self.active = False self.site_name = site_name self.presets = presets + self.dbx = None if not self.presets: log.info( @@ -24,6 +25,11 @@ class DropboxHandler(AbstractProvider): ) return + if not self.presets["enabled"]: + log.debug("Sync Server: Site {} not enabled for {}.". + format(site_name, project_name)) + return + token = self.presets.get("token", "") if not token: msg = "Sync Server: No access token for dropbox provider" @@ -44,16 +50,13 @@ class DropboxHandler(AbstractProvider): log.info(msg) return - self.dbx = None - - if self.presets["enabled"]: - try: - self.dbx = self._get_service( - token, acting_as_member, team_folder_name - ) - except Exception as e: - log.info("Could not establish dropbox object: {}".format(e)) - return + try: + self.dbx = self._get_service( + token, acting_as_member, team_folder_name + ) + except Exception as e: + log.info("Could not establish dropbox object: {}".format(e)) + return super(AbstractProvider, self).__init__() From adbd3593424fe9ea981705505038bcf89b9d81b9 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 8 Apr 2022 11:54:36 +0200 Subject: [PATCH 090/126] flame: otio removing reel clip dependency --- openpype/hosts/flame/otio/flame_export.py | 70 ++++------------------- 1 file changed, 11 insertions(+), 59 deletions(-) diff --git a/openpype/hosts/flame/otio/flame_export.py b/openpype/hosts/flame/otio/flame_export.py index 3e76968963..25be310d01 100644 --- a/openpype/hosts/flame/otio/flame_export.py +++ b/openpype/hosts/flame/otio/flame_export.py @@ -261,24 +261,15 @@ def create_otio_markers(otio_item, item): otio_item.markers.append(otio_marker) -def create_otio_reference(clip_data): +def create_otio_reference(clip_data, fps=None): metadata = _get_metadata(clip_data) # get file info for path and start frame frame_start = 0 - fps = CTX.get_fps() + fps = fps or CTX.get_fps() path = clip_data["fpath"] - reel_clip = None - match_reel_clip = [ - clip for clip in CTX.clips - if clip["fpath"] == path - ] - if match_reel_clip: - reel_clip = match_reel_clip.pop() - fps = reel_clip["fps"] - file_name = os.path.basename(path) file_head, extension = os.path.splitext(file_name) @@ -342,16 +333,17 @@ def create_otio_reference(clip_data): def create_otio_clip(clip_data): segment = clip_data["PySegment"] - # create media reference - media_reference = create_otio_reference(clip_data) - # calculate source in media_info = MediaInfoFile(clip_data["fpath"]) - xml_timecode_ticks = media_info.out_feed_nb_ticks - if xml_timecode_ticks: - first_frame = int(xml_timecode_ticks) - else: - first_frame = utils.get_frame_from_filename(clip_data["fpath"]) or 0 + media_timecode_start = media_info.start_frame + media_fps = media_info.fps + + # create media reference + media_reference = create_otio_reference(clip_data, media_fps) + + # define first frame + first_frame = media_timecode_start or utils.get_frame_from_filename( + clip_data["fpath"]) or 0 source_in = int(clip_data["source_in"]) - int(first_frame) @@ -385,41 +377,6 @@ def create_otio_gap(gap_start, clip_start, tl_start_frame, fps): ) -def get_clips_in_reels(project): - output_clips = [] - project_desktop = project.current_workspace.desktop - - for reel_group in project_desktop.reel_groups: - for reel in reel_group.reels: - for clip in reel.clips: - clip_data = { - "PyClip": clip, - "fps": float(str(clip.frame_rate)[:-4]) - } - - attrs = [ - "name", "width", "height", - "ratio", "sample_rate", "bit_depth" - ] - - for attr in attrs: - val = getattr(clip, attr) - clip_data[attr] = val - - version = clip.versions[-1] - track = version.tracks[-1] - # each reel clip is also having one segment - for segment in track.segments: - segment_data = _get_segment_attributes( - segment, from_clip=True) - if segment_data: - clip_data.update(segment_data) - - output_clips.append(clip_data) - - return output_clips - - def _get_colourspace_policy(): output = {} @@ -579,11 +536,6 @@ def create_otio_timeline(sequence): log.info(sequence.attributes) CTX.project = get_current_flame_project() - CTX.clips = get_clips_in_reels(CTX.project) - - log.debug(pformat( - CTX.clips - )) # get current timeline CTX.set_fps( From 4797f584981bd5d07370f417ebe073c6cedbd76f Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 8 Apr 2022 11:59:34 +0200 Subject: [PATCH 091/126] flame: add todos for metadata feature --- openpype/hosts/flame/api/lib.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openpype/hosts/flame/api/lib.py b/openpype/hosts/flame/api/lib.py index de3467aa76..e9e0130401 100644 --- a/openpype/hosts/flame/api/lib.py +++ b/openpype/hosts/flame/api/lib.py @@ -271,6 +271,7 @@ def rescan_hooks(): def get_metadata(project_name, _log=None): + # TODO: can be replaced by MediaInfoFile class method from adsk.libwiretapPythonClientAPI import ( WireTapClient, WireTapServerHandle, @@ -750,6 +751,8 @@ class MediaInfoFile: Attributes: str: `MEDIA_SCRIPT_PATH` path to flame binary logging.Logger: `log` logger + + TODO: add method for getting metadata to dict """ MEDIA_SCRIPT_PATH = "/opt/Autodesk/mio/current/dl_get_media_info" From 7f0b4710f2f1fba3f185e1ee47ec85fa03187662 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 8 Apr 2022 12:13:47 +0200 Subject: [PATCH 092/126] Refactor - faster resolution of query --- openpype/modules/sync_server/sync_server_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/sync_server/sync_server_module.py b/openpype/modules/sync_server/sync_server_module.py index ddcf16a410..2c27571f9f 100644 --- a/openpype/modules/sync_server/sync_server_module.py +++ b/openpype/modules/sync_server/sync_server_module.py @@ -850,7 +850,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): active_site = sync_settings["config"]["active_site"] # for Tray running background process - if active_site == get_local_site_id() and active_site not in sites: + if active_site not in sites and active_site == get_local_site_id(): sites.append(active_site) return sites From bfae95a8475981e014561c1eec8b9b54de5a8424 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 8 Apr 2022 12:23:16 +0200 Subject: [PATCH 093/126] flame: OpenClipSolver inheriting from MediaInfoFile class --- openpype/hosts/flame/api/lib.py | 4 +- openpype/hosts/flame/api/plugin.py | 162 ++++++----------------------- 2 files changed, 36 insertions(+), 130 deletions(-) diff --git a/openpype/hosts/flame/api/lib.py b/openpype/hosts/flame/api/lib.py index e9e0130401..91f5c26562 100644 --- a/openpype/hosts/flame/api/lib.py +++ b/openpype/hosts/flame/api/lib.py @@ -766,8 +766,8 @@ class MediaInfoFile: def __init__(self, path, **kwargs): # replace log if any - if kwargs.get("log"): - self.log = kwargs["log"] + if kwargs.get("logger"): + self.log = kwargs["logger"] # test if `dl_get_media_info` paht exists self._validate_media_script_path() diff --git a/openpype/hosts/flame/api/plugin.py b/openpype/hosts/flame/api/plugin.py index bd0f9f1a81..a23be946ba 100644 --- a/openpype/hosts/flame/api/plugin.py +++ b/openpype/hosts/flame/api/plugin.py @@ -679,53 +679,39 @@ class ClipLoader(LoaderPlugin): ] -# TODO: inheritance from flame.api.lib.MediaInfoFile -class OpenClipSolver: - media_script_path = "/opt/Autodesk/mio/current/dl_get_media_info" - tmp_name = "_tmp.clip" - tmp_file = None +class OpenClipSolver(flib.MediaInfoFile): create_new_clip = False - out_feed_nb_ticks = None - out_feed_fps = None - out_feed_drop_mode = None - log = log def __init__(self, openclip_file_path, feed_data): - # test if media script paht exists - self._validate_media_script_path() + self.out_file = openclip_file_path # new feed variables: - feed_path = feed_data["path"] - self.feed_version_name = feed_data["version"] - self.feed_colorspace = feed_data.get("colorspace") + feed_path = feed_data.pop("path") + # initialize parent class + super(OpenClipSolver).__init__( + feed_path, + **feed_data + ) + + # get logger if any if feed_data.get("logger"): self.log = feed_data["logger"] + # get other metadata + self.feed_version_name = feed_data["version"] + self.feed_colorspace = feed_data.get("colorspace") + # derivate other feed variables self.feed_basename = os.path.basename(feed_path) self.feed_dir = os.path.dirname(feed_path) self.feed_ext = os.path.splitext(self.feed_basename)[1][1:].lower() - if not self._is_valid_tmp_file(openclip_file_path): - # openclip does not exist yet and will be created - self.tmp_file = self.out_file = openclip_file_path + if not self._is_valid_tmp_file(self.out_file): self.create_new_clip = True - else: - # update already created clip - # output a temp file - self.out_file = openclip_file_path - self.tmp_file = os.path.join(self.feed_dir, self.tmp_name) - - # remove previously generated temp files - # it will be regenerated - self._clear_tmp_file() - - self.log.info("Temp File: {}".format(self.tmp_file)) - def _is_valid_tmp_file(self, file): # check if file exists if os.path.isfile(file): @@ -740,7 +726,6 @@ class OpenClipSolver: return False def make(self): - self._generate_media_info_file() if self.create_new_clip: # New openClip @@ -748,58 +733,6 @@ class OpenClipSolver: else: self._update_open_clip() - def _validate_media_script_path(self): - if not os.path.isfile(self.media_script_path): - raise IOError("Media Scirpt does not exist: `{}`".format( - self.media_script_path)) - - def _generate_media_info_file(self): - # Create cmd arguments for gettig xml file info file - cmd_args = [ - self.media_script_path, - "-e", self.feed_ext, - "-o", self.tmp_file, - self.feed_dir - ] - - # execute creation of clip xml template data - try: - openpype.run_subprocess(cmd_args) - self._make_single_clip_media_info() - except TypeError: - self.log.error("Error creating self.tmp_file") - six.reraise(*sys.exc_info()) - - def _make_single_clip_media_info(self): - with open(self.tmp_file) as f: - lines = f.readlines() - _added_root = itertools.chain( - "", deepcopy(lines)[1:], "") - new_root = ET.fromstringlist(_added_root) - - # find the clip which is matching to my input name - xml_clips = new_root.findall("clip") - matching_clip = None - for xml_clip in xml_clips: - if xml_clip.find("name").text in self.feed_basename: - matching_clip = xml_clip - - if matching_clip is None: - # return warning there is missing clip - raise ET.ParseError( - "Missing clip in `{}`. Available clips {}".format( - self.feed_basename, [ - xml_clip.find("name").text - for xml_clip in xml_clips - ] - )) - - self._write_result_xml_to_file(self.tmp_file, matching_clip) - - def _clear_tmp_file(self): - if os.path.isfile(self.tmp_file): - os.remove(self.tmp_file) - def _clear_handler(self, xml_object): for handler in xml_object.findall("./handler"): self.log.debug("Handler found") @@ -808,9 +741,8 @@ class OpenClipSolver: def _create_new_open_clip(self): self.log.info("Building new openClip") - tmp_xml = ET.parse(self.tmp_file) - - tmp_xml_feeds = tmp_xml.find('tracks/track/feeds') + # clip data comming from MediaInfoFile + tmp_xml_feeds = self.clip_data.find('tracks/track/feeds') tmp_xml_feeds.set('currentVersion', self.feed_version_name) for tmp_feed in tmp_xml_feeds: tmp_feed.set('vuid', self.feed_version_name) @@ -821,46 +753,47 @@ class OpenClipSolver: self._clear_handler(tmp_feed) - tmp_xml_versions_obj = tmp_xml.find('versions') + tmp_xml_versions_obj = self.clip_data.find('versions') tmp_xml_versions_obj.set('currentVersion', self.feed_version_name) for xml_new_version in tmp_xml_versions_obj: xml_new_version.set('uid', self.feed_version_name) xml_new_version.set('type', 'version') - xml_data = self._fix_xml_data(tmp_xml) + xml_data = self._fix_xml_data(self.clip_data) self.log.info("Adding feed version: {}".format(self.feed_basename)) - self._write_result_xml_to_file(self.out_file, xml_data) - - self.log.info("openClip Updated: {}".format(self.tmp_file)) + self.write_clip_data_to_file(self.out_file, xml_data) def _update_open_clip(self): self.log.info("Updating openClip ..") out_xml = ET.parse(self.out_file) - tmp_xml = ET.parse(self.tmp_file) self.log.debug(">> out_xml: {}".format(out_xml)) - self.log.debug(">> tmp_xml: {}".format(tmp_xml)) + self.log.debug(">> self.clip_data: {}".format(self.clip_data)) # Get new feed from tmp file - tmp_xml_feed = tmp_xml.find('tracks/track/feeds/feed') + tmp_xml_feed = self.clip_data.find('tracks/track/feeds/feed') self._clear_handler(tmp_xml_feed) - self._get_time_info_from_origin(out_xml) - if self.out_feed_fps: + # update fps from MediaInfoFile class + if self.fps: tmp_feed_fps_obj = tmp_xml_feed.find( "startTimecode/rate") - tmp_feed_fps_obj.text = self.out_feed_fps - if self.out_feed_nb_ticks: + tmp_feed_fps_obj.text = self.fps + + # update start_frame from MediaInfoFile class + if self.start_frame: tmp_feed_nb_ticks_obj = tmp_xml_feed.find( "startTimecode/nbTicks") - tmp_feed_nb_ticks_obj.text = self.out_feed_nb_ticks - if self.out_feed_drop_mode: + tmp_feed_nb_ticks_obj.text = self.start_frame + + # update drop_mode from MediaInfoFile class + if self.drop_mode: tmp_feed_drop_mode_obj = tmp_xml_feed.find( "startTimecode/dropMode") - tmp_feed_drop_mode_obj.text = self.out_feed_drop_mode + tmp_feed_drop_mode_obj.text = self.drop_mode new_path_obj = tmp_xml_feed.find( "spans/span/path") @@ -901,31 +834,10 @@ class OpenClipSolver: self.log.info("Adding feed version: {}".format( self.feed_version_name)) - self._write_result_xml_to_file(self.out_file, xml_data) + self.write_clip_data_to_file(self.out_file, xml_data) self.log.info("openClip Updated: {}".format(self.out_file)) - self._clear_tmp_file() - - def _get_time_info_from_origin(self, xml_data): - try: - for out_track in xml_data.iter('track'): - for out_feed in out_track.iter('feed'): - out_feed_nb_ticks_obj = out_feed.find( - 'startTimecode/nbTicks') - self.out_feed_nb_ticks = out_feed_nb_ticks_obj.text - out_feed_fps_obj = out_feed.find( - 'startTimecode/rate') - self.out_feed_fps = out_feed_fps_obj.text - out_feed_drop_mode_obj = out_feed.find( - 'startTimecode/dropMode') - self.out_feed_drop_mode = out_feed_drop_mode_obj.text - break - else: - continue - except Exception as msg: - self.log.warning(msg) - def _feed_exists(self, xml_data, path): # loop all available feed paths and check if # the path is not already in file @@ -940,12 +852,6 @@ class OpenClipSolver: self._clear_handler(xml_root) return xml_root - def _write_result_xml_to_file(self, file, xml_data): - # save it as new file - tree = cET.ElementTree(xml_data) - tree.write(file, xml_declaration=True, - method='xml', encoding='UTF-8') - def _create_openclip_backup_file(self, file): bck_file = "{}.bak".format(file) # if backup does not exist From 507f3615ab8f42f5664afcac01d339e0517afdf5 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 8 Apr 2022 12:24:04 +0200 Subject: [PATCH 094/126] Refactor - changed logic to loop through alt sites --- openpype/plugins/publish/integrate_new.py | 31 +++++++++++++---------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/openpype/plugins/publish/integrate_new.py b/openpype/plugins/publish/integrate_new.py index ed1c02b825..3eca460ba3 100644 --- a/openpype/plugins/publish/integrate_new.py +++ b/openpype/plugins/publish/integrate_new.py @@ -8,6 +8,7 @@ import errno import six import re import shutil +from collections import deque from bson.objectid import ObjectId from pymongo import DeleteOne, InsertOne @@ -1199,21 +1200,23 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): # transitive relationship, eg site is alternative to another which is # alternative to nex site - loop = True - while loop: - loop = False - for site_name, alt_sites in alt_site_pairs.items(): - for alt_site in alt_sites: - # safety against wrong config - # {"SFTP": {"alternative_site": "SFTP"} - if alt_site == site_name: - continue + for site_name, alt_sites in alt_site_pairs.items(): + sites_queue = deque(alt_sites) + while sites_queue: + alt_site = sites_queue.popleft() - for alt_alt_site in alt_site_pairs.get(alt_site, []): - if ( alt_alt_site != site_name - and alt_alt_site not in alt_sites): - alt_site_pairs[site_name].append(alt_alt_site) - loop = True + # safety against wrong config + # {"SFTP": {"alternative_site": "SFTP"} + if alt_site == site_name or alt_site not in alt_site_pairs: + continue + + for alt_alt_site in alt_site_pairs[alt_site]: + if ( + alt_alt_site != site_name + and alt_alt_site not in alt_sites + ): + alt_sites.append(alt_alt_site) + sites_queue.append(alt_alt_site) return alt_site_pairs From 72ecb6192a0ac609b4ad951dbd35712144b012a7 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 8 Apr 2022 12:52:27 +0200 Subject: [PATCH 095/126] flame: fixing flame compatibility and python2 --- openpype/hosts/flame/api/__init__.py | 1 - openpype/hosts/flame/api/lib.py | 6 +++--- openpype/hosts/flame/api/plugin.py | 5 +---- openpype/hosts/flame/otio/flame_export.py | 5 ++--- 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/openpype/hosts/flame/api/__init__.py b/openpype/hosts/flame/api/__init__.py index 6744a7ff11..2c461e5f16 100644 --- a/openpype/hosts/flame/api/__init__.py +++ b/openpype/hosts/flame/api/__init__.py @@ -57,7 +57,6 @@ from .plugin import ( PublishableClip, ClipLoader, OpenClipSolver - ) from .workio import ( open_file, diff --git a/openpype/hosts/flame/api/lib.py b/openpype/hosts/flame/api/lib.py index 91f5c26562..6d93018fef 100644 --- a/openpype/hosts/flame/api/lib.py +++ b/openpype/hosts/flame/api/lib.py @@ -708,7 +708,7 @@ def maintained_temp_file_path(suffix=None): except IOError as _error: raise IOError( - "Not able to create temp json file: {}".format(_error)) from _error + "Not able to create temp json file: {}".format(_error)) finally: # Remove the temporary json @@ -860,7 +860,7 @@ class MediaInfoFile: openpype.run_subprocess(cmd_args) except TypeError as error: raise TypeError( - "Error creating `{}` due: {}".format(fpath, error)) from error + "Error creating `{}` due: {}".format(fpath, error)) def _make_single_clip_media_info(self, fpath): with open(fpath) as f: @@ -923,4 +923,4 @@ class MediaInfoFile: ) except IOError as error: raise IOError( - "Not able to write data to file: {}".format(error)) from error + "Not able to write data to file: {}".format(error)) diff --git a/openpype/hosts/flame/api/plugin.py b/openpype/hosts/flame/api/plugin.py index a23be946ba..ab74bb4605 100644 --- a/openpype/hosts/flame/api/plugin.py +++ b/openpype/hosts/flame/api/plugin.py @@ -1,15 +1,11 @@ -import itertools import os import re import shutil -import sys -import xml.etree.cElementTree as cET from copy import deepcopy from xml.etree import ElementTree as ET import openpype.api as openpype import qargparse -import six from openpype import style from openpype.pipeline import LegacyCreator, LoaderPlugin from Qt import QtCore, QtWidgets @@ -740,6 +736,7 @@ class OpenClipSolver(flib.MediaInfoFile): def _create_new_open_clip(self): self.log.info("Building new openClip") + self.log.debug(">> self.clip_data: {}".format(self.clip_data)) # clip data comming from MediaInfoFile tmp_xml_feeds = self.clip_data.find('tracks/track/feeds') diff --git a/openpype/hosts/flame/otio/flame_export.py b/openpype/hosts/flame/otio/flame_export.py index 25be310d01..0b9c9ce817 100644 --- a/openpype/hosts/flame/otio/flame_export.py +++ b/openpype/hosts/flame/otio/flame_export.py @@ -7,13 +7,10 @@ import json import logging import opentimelineio as otio from . import utils -from openpype.hosts.flame.api import MediaInfoFile import flame from pprint import pformat -reload(utils) # noqa - log = logging.getLogger(__name__) @@ -331,6 +328,8 @@ def create_otio_reference(clip_data, fps=None): def create_otio_clip(clip_data): + from openpype.hosts.flame.api import MediaInfoFile + segment = clip_data["PySegment"] # calculate source in From 9256e022909bb363d8e4cf4be532248363579d40 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 8 Apr 2022 13:09:29 +0200 Subject: [PATCH 096/126] flame: setter getter error --- openpype/hosts/flame/api/lib.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/openpype/hosts/flame/api/lib.py b/openpype/hosts/flame/api/lib.py index 6d93018fef..d68dd2a886 100644 --- a/openpype/hosts/flame/api/lib.py +++ b/openpype/hosts/flame/api/lib.py @@ -704,7 +704,7 @@ def maintained_temp_file_path(suffix=None): # Store dumped json to temporary file temporary_file = tempfile.mktemp( suffix=_suffix, prefix="flame_maintained_") - yield temporary_file.name.replace("\\", "/") + yield temporary_file.replace("\\", "/") except IOError as _error: raise IOError( @@ -787,7 +787,7 @@ class MediaInfoFile: # get all time related data and assign them self._get_time_info_from_origin(xml_data) - self.set_clip_data(xml_data) + self.clip_data = xml_data @property def clip_data(self): @@ -895,17 +895,17 @@ class MediaInfoFile: # start frame out_feed_nb_ticks_obj = out_feed.find( 'startTimecode/nbTicks') - self.start_frame(out_feed_nb_ticks_obj.text) + self.start_frame = out_feed_nb_ticks_obj.text # fps out_feed_fps_obj = out_feed.find( 'startTimecode/rate') - self.fps(out_feed_fps_obj.text) + self.fps = out_feed_fps_obj.text # drop frame mode out_feed_drop_mode_obj = out_feed.find( 'startTimecode/dropMode') - self.drop_mode(out_feed_drop_mode_obj.text) + self.drop_mode = out_feed_drop_mode_obj.text break else: continue From 3f9b06139d44448416deae9d251dabffdcfb9506 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 8 Apr 2022 14:13:43 +0200 Subject: [PATCH 097/126] flame: fix getroot --- openpype/hosts/flame/api/lib.py | 2 +- openpype/hosts/flame/api/plugin.py | 11 +++-------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/openpype/hosts/flame/api/lib.py b/openpype/hosts/flame/api/lib.py index d68dd2a886..6ee0eb6a82 100644 --- a/openpype/hosts/flame/api/lib.py +++ b/openpype/hosts/flame/api/lib.py @@ -739,7 +739,7 @@ def get_batch_group_from_desktop(name): return bgroup -class MediaInfoFile: +class MediaInfoFile(object): """Class to get media info file clip data Raises: diff --git a/openpype/hosts/flame/api/plugin.py b/openpype/hosts/flame/api/plugin.py index ab74bb4605..6136c4922c 100644 --- a/openpype/hosts/flame/api/plugin.py +++ b/openpype/hosts/flame/api/plugin.py @@ -687,7 +687,7 @@ class OpenClipSolver(flib.MediaInfoFile): feed_path = feed_data.pop("path") # initialize parent class - super(OpenClipSolver).__init__( + super(OpenClipSolver, self).__init__( feed_path, **feed_data ) @@ -756,7 +756,7 @@ class OpenClipSolver(flib.MediaInfoFile): xml_new_version.set('uid', self.feed_version_name) xml_new_version.set('type', 'version') - xml_data = self._fix_xml_data(self.clip_data) + xml_data = self._clear_handler(self.clip_data) self.log.info("Adding feed version: {}".format(self.feed_basename)) self.write_clip_data_to_file(self.out_file, xml_data) @@ -823,7 +823,7 @@ class OpenClipSolver(flib.MediaInfoFile): "version", {"type": "version", "uid": self.feed_version_name}) out_xml_versions_obj.insert(0, new_version_obj) - xml_data = self._fix_xml_data(out_xml) + xml_data = self._clear_handler(out_xml) # fist create backup self._create_openclip_backup_file(self.out_file) @@ -844,11 +844,6 @@ class OpenClipSolver(flib.MediaInfoFile): "Not appending file as it already is in .clip file") return True - def _fix_xml_data(self, xml_data): - xml_root = xml_data.getroot() - self._clear_handler(xml_root) - return xml_root - def _create_openclip_backup_file(self, file): bck_file = "{}.bak".format(file) # if backup does not exist From 9a8c41371bc3c7a7c3be37e655021005547b9a30 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 8 Apr 2022 15:12:16 +0200 Subject: [PATCH 098/126] use current project for query asset --- openpype/tools/utils/lib.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/openpype/tools/utils/lib.py b/openpype/tools/utils/lib.py index 422d0f5389..efaf671915 100644 --- a/openpype/tools/utils/lib.py +++ b/openpype/tools/utils/lib.py @@ -409,6 +409,7 @@ class FamilyConfigCache: project_name = os.environ.get("AVALON_PROJECT") asset_name = os.environ.get("AVALON_ASSET") task_name = os.environ.get("AVALON_TASK") + host_name = os.environ.get("AVALON_APP") if not all((project_name, asset_name, task_name)): return @@ -422,15 +423,18 @@ class FamilyConfigCache: ["family_filter_profiles"] ) if profiles: - asset_doc = self.dbcon.find_one( + # Make sure connection is installed + # - accessing attribute which does not have auto-install + self.dbcon.install() + asset_doc = self.dbcon.database[project_name].find_one( {"type": "asset", "name": asset_name}, {"data.tasks": True} - ) + ) or {} tasks_info = asset_doc.get("data", {}).get("tasks") or {} task_type = tasks_info.get(task_name, {}).get("type") profiles_filter = { "task_types": task_type, - "hosts": os.environ["AVALON_APP"] + "hosts": host_name } matching_item = filter_profiles(profiles, profiles_filter) From 8420a164c2c5e7537c584f30e1e3b8469417cacf Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 8 Apr 2022 15:13:05 +0200 Subject: [PATCH 099/126] flame: temp reverse commit bfae95a8475981e014561c1eec8b9b54de5a8424 --- openpype/hosts/flame/api/plugin.py | 176 +++++++++++++++++++++++------ 1 file changed, 139 insertions(+), 37 deletions(-) diff --git a/openpype/hosts/flame/api/plugin.py b/openpype/hosts/flame/api/plugin.py index 6136c4922c..bd0f9f1a81 100644 --- a/openpype/hosts/flame/api/plugin.py +++ b/openpype/hosts/flame/api/plugin.py @@ -1,11 +1,15 @@ +import itertools import os import re import shutil +import sys +import xml.etree.cElementTree as cET from copy import deepcopy from xml.etree import ElementTree as ET import openpype.api as openpype import qargparse +import six from openpype import style from openpype.pipeline import LegacyCreator, LoaderPlugin from Qt import QtCore, QtWidgets @@ -675,39 +679,53 @@ class ClipLoader(LoaderPlugin): ] -class OpenClipSolver(flib.MediaInfoFile): +# TODO: inheritance from flame.api.lib.MediaInfoFile +class OpenClipSolver: + media_script_path = "/opt/Autodesk/mio/current/dl_get_media_info" + tmp_name = "_tmp.clip" + tmp_file = None create_new_clip = False + out_feed_nb_ticks = None + out_feed_fps = None + out_feed_drop_mode = None + log = log def __init__(self, openclip_file_path, feed_data): - self.out_file = openclip_file_path + # test if media script paht exists + self._validate_media_script_path() # new feed variables: - feed_path = feed_data.pop("path") - - # initialize parent class - super(OpenClipSolver, self).__init__( - feed_path, - **feed_data - ) - - # get logger if any - if feed_data.get("logger"): - self.log = feed_data["logger"] - - # get other metadata + feed_path = feed_data["path"] self.feed_version_name = feed_data["version"] self.feed_colorspace = feed_data.get("colorspace") + if feed_data.get("logger"): + self.log = feed_data["logger"] + # derivate other feed variables self.feed_basename = os.path.basename(feed_path) self.feed_dir = os.path.dirname(feed_path) self.feed_ext = os.path.splitext(self.feed_basename)[1][1:].lower() - if not self._is_valid_tmp_file(self.out_file): + if not self._is_valid_tmp_file(openclip_file_path): + # openclip does not exist yet and will be created + self.tmp_file = self.out_file = openclip_file_path self.create_new_clip = True + else: + # update already created clip + # output a temp file + self.out_file = openclip_file_path + self.tmp_file = os.path.join(self.feed_dir, self.tmp_name) + + # remove previously generated temp files + # it will be regenerated + self._clear_tmp_file() + + self.log.info("Temp File: {}".format(self.tmp_file)) + def _is_valid_tmp_file(self, file): # check if file exists if os.path.isfile(file): @@ -722,6 +740,7 @@ class OpenClipSolver(flib.MediaInfoFile): return False def make(self): + self._generate_media_info_file() if self.create_new_clip: # New openClip @@ -729,6 +748,58 @@ class OpenClipSolver(flib.MediaInfoFile): else: self._update_open_clip() + def _validate_media_script_path(self): + if not os.path.isfile(self.media_script_path): + raise IOError("Media Scirpt does not exist: `{}`".format( + self.media_script_path)) + + def _generate_media_info_file(self): + # Create cmd arguments for gettig xml file info file + cmd_args = [ + self.media_script_path, + "-e", self.feed_ext, + "-o", self.tmp_file, + self.feed_dir + ] + + # execute creation of clip xml template data + try: + openpype.run_subprocess(cmd_args) + self._make_single_clip_media_info() + except TypeError: + self.log.error("Error creating self.tmp_file") + six.reraise(*sys.exc_info()) + + def _make_single_clip_media_info(self): + with open(self.tmp_file) as f: + lines = f.readlines() + _added_root = itertools.chain( + "", deepcopy(lines)[1:], "") + new_root = ET.fromstringlist(_added_root) + + # find the clip which is matching to my input name + xml_clips = new_root.findall("clip") + matching_clip = None + for xml_clip in xml_clips: + if xml_clip.find("name").text in self.feed_basename: + matching_clip = xml_clip + + if matching_clip is None: + # return warning there is missing clip + raise ET.ParseError( + "Missing clip in `{}`. Available clips {}".format( + self.feed_basename, [ + xml_clip.find("name").text + for xml_clip in xml_clips + ] + )) + + self._write_result_xml_to_file(self.tmp_file, matching_clip) + + def _clear_tmp_file(self): + if os.path.isfile(self.tmp_file): + os.remove(self.tmp_file) + def _clear_handler(self, xml_object): for handler in xml_object.findall("./handler"): self.log.debug("Handler found") @@ -736,10 +807,10 @@ class OpenClipSolver(flib.MediaInfoFile): def _create_new_open_clip(self): self.log.info("Building new openClip") - self.log.debug(">> self.clip_data: {}".format(self.clip_data)) - # clip data comming from MediaInfoFile - tmp_xml_feeds = self.clip_data.find('tracks/track/feeds') + tmp_xml = ET.parse(self.tmp_file) + + tmp_xml_feeds = tmp_xml.find('tracks/track/feeds') tmp_xml_feeds.set('currentVersion', self.feed_version_name) for tmp_feed in tmp_xml_feeds: tmp_feed.set('vuid', self.feed_version_name) @@ -750,47 +821,46 @@ class OpenClipSolver(flib.MediaInfoFile): self._clear_handler(tmp_feed) - tmp_xml_versions_obj = self.clip_data.find('versions') + tmp_xml_versions_obj = tmp_xml.find('versions') tmp_xml_versions_obj.set('currentVersion', self.feed_version_name) for xml_new_version in tmp_xml_versions_obj: xml_new_version.set('uid', self.feed_version_name) xml_new_version.set('type', 'version') - xml_data = self._clear_handler(self.clip_data) + xml_data = self._fix_xml_data(tmp_xml) self.log.info("Adding feed version: {}".format(self.feed_basename)) - self.write_clip_data_to_file(self.out_file, xml_data) + self._write_result_xml_to_file(self.out_file, xml_data) + + self.log.info("openClip Updated: {}".format(self.tmp_file)) def _update_open_clip(self): self.log.info("Updating openClip ..") out_xml = ET.parse(self.out_file) + tmp_xml = ET.parse(self.tmp_file) self.log.debug(">> out_xml: {}".format(out_xml)) - self.log.debug(">> self.clip_data: {}".format(self.clip_data)) + self.log.debug(">> tmp_xml: {}".format(tmp_xml)) # Get new feed from tmp file - tmp_xml_feed = self.clip_data.find('tracks/track/feeds/feed') + tmp_xml_feed = tmp_xml.find('tracks/track/feeds/feed') self._clear_handler(tmp_xml_feed) + self._get_time_info_from_origin(out_xml) - # update fps from MediaInfoFile class - if self.fps: + if self.out_feed_fps: tmp_feed_fps_obj = tmp_xml_feed.find( "startTimecode/rate") - tmp_feed_fps_obj.text = self.fps - - # update start_frame from MediaInfoFile class - if self.start_frame: + tmp_feed_fps_obj.text = self.out_feed_fps + if self.out_feed_nb_ticks: tmp_feed_nb_ticks_obj = tmp_xml_feed.find( "startTimecode/nbTicks") - tmp_feed_nb_ticks_obj.text = self.start_frame - - # update drop_mode from MediaInfoFile class - if self.drop_mode: + tmp_feed_nb_ticks_obj.text = self.out_feed_nb_ticks + if self.out_feed_drop_mode: tmp_feed_drop_mode_obj = tmp_xml_feed.find( "startTimecode/dropMode") - tmp_feed_drop_mode_obj.text = self.drop_mode + tmp_feed_drop_mode_obj.text = self.out_feed_drop_mode new_path_obj = tmp_xml_feed.find( "spans/span/path") @@ -823,7 +893,7 @@ class OpenClipSolver(flib.MediaInfoFile): "version", {"type": "version", "uid": self.feed_version_name}) out_xml_versions_obj.insert(0, new_version_obj) - xml_data = self._clear_handler(out_xml) + xml_data = self._fix_xml_data(out_xml) # fist create backup self._create_openclip_backup_file(self.out_file) @@ -831,10 +901,31 @@ class OpenClipSolver(flib.MediaInfoFile): self.log.info("Adding feed version: {}".format( self.feed_version_name)) - self.write_clip_data_to_file(self.out_file, xml_data) + self._write_result_xml_to_file(self.out_file, xml_data) self.log.info("openClip Updated: {}".format(self.out_file)) + self._clear_tmp_file() + + def _get_time_info_from_origin(self, xml_data): + try: + for out_track in xml_data.iter('track'): + for out_feed in out_track.iter('feed'): + out_feed_nb_ticks_obj = out_feed.find( + 'startTimecode/nbTicks') + self.out_feed_nb_ticks = out_feed_nb_ticks_obj.text + out_feed_fps_obj = out_feed.find( + 'startTimecode/rate') + self.out_feed_fps = out_feed_fps_obj.text + out_feed_drop_mode_obj = out_feed.find( + 'startTimecode/dropMode') + self.out_feed_drop_mode = out_feed_drop_mode_obj.text + break + else: + continue + except Exception as msg: + self.log.warning(msg) + def _feed_exists(self, xml_data, path): # loop all available feed paths and check if # the path is not already in file @@ -844,6 +935,17 @@ class OpenClipSolver(flib.MediaInfoFile): "Not appending file as it already is in .clip file") return True + def _fix_xml_data(self, xml_data): + xml_root = xml_data.getroot() + self._clear_handler(xml_root) + return xml_root + + def _write_result_xml_to_file(self, file, xml_data): + # save it as new file + tree = cET.ElementTree(xml_data) + tree.write(file, xml_declaration=True, + method='xml', encoding='UTF-8') + def _create_openclip_backup_file(self, file): bck_file = "{}.bak".format(file) # if backup does not exist From 2a618081fdf9a6f81e6b63b8c020fefa84a30586 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 8 Apr 2022 15:58:59 +0200 Subject: [PATCH 100/126] added settings for white list of environment variables --- .../settings/defaults/system_settings/general.json | 1 + .../entities/schemas/system_schema/schema_general.json | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/openpype/settings/defaults/system_settings/general.json b/openpype/settings/defaults/system_settings/general.json index 5a3e39e5b6..e1785f8709 100644 --- a/openpype/settings/defaults/system_settings/general.json +++ b/openpype/settings/defaults/system_settings/general.json @@ -12,6 +12,7 @@ "linux": [], "darwin": [] }, + "local_env_white_list": [], "openpype_path": { "windows": [], "darwin": [], diff --git a/openpype/settings/entities/schemas/system_schema/schema_general.json b/openpype/settings/entities/schemas/system_schema/schema_general.json index 6306317df8..997404b2e6 100644 --- a/openpype/settings/entities/schemas/system_schema/schema_general.json +++ b/openpype/settings/entities/schemas/system_schema/schema_general.json @@ -110,6 +110,16 @@ { "type": "splitter" }, + { + "type": "list", + "key": "local_env_white_list", + "label": "White list of local environment variables", + "use_label_wrap": true, + "object_type": "text" + }, + { + "type": "splitter" + }, { "type": "collapsible-wrap", "label": "OpenPype deployment control", From 29dca65202d45a79e66c619b95d3408e227a9c05 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 8 Apr 2022 16:59:34 +0200 Subject: [PATCH 101/126] Refactor - changed to defaultdict --- openpype/plugins/publish/integrate_new.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/openpype/plugins/publish/integrate_new.py b/openpype/plugins/publish/integrate_new.py index 3eca460ba3..5dcbb8fabd 100644 --- a/openpype/plugins/publish/integrate_new.py +++ b/openpype/plugins/publish/integrate_new.py @@ -8,7 +8,7 @@ import errno import six import re import shutil -from collections import deque +from collections import deque, defaultdict from bson.objectid import ObjectId from pymongo import DeleteOne, InsertOne @@ -1185,21 +1185,14 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): Returns: (dict): {'site': [alternative sites]...} """ - alt_site_pairs = {} + alt_site_pairs = defaultdict(list) for site_name, site_info in conf_sites.items(): alt_sites = set(site_info.get("alternative_sites", [])) - if not alt_site_pairs.get(site_name): - alt_site_pairs[site_name] = [] - alt_site_pairs[site_name].extend(alt_sites) for alt_site in alt_sites: - if not alt_site_pairs.get(alt_site): - alt_site_pairs[alt_site] = [] - alt_site_pairs[alt_site].extend([site_name]) + alt_site_pairs[alt_site].append(site_name) - # transitive relationship, eg site is alternative to another which is - # alternative to nex site for site_name, alt_sites in alt_site_pairs.items(): sites_queue = deque(alt_sites) while sites_queue: From 0ab101a5b12b755f07551d32203779103ee059f7 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 8 Apr 2022 17:11:16 +0200 Subject: [PATCH 102/126] added widgets for environments settings --- .../settings/local_settings/constants.py | 1 + .../local_settings/environments_widget.py | 80 +++++++++++++++++++ .../tools/settings/local_settings/window.py | 31 ++++++- 3 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 openpype/tools/settings/local_settings/environments_widget.py diff --git a/openpype/tools/settings/local_settings/constants.py b/openpype/tools/settings/local_settings/constants.py index 1836c579af..16f87b6f05 100644 --- a/openpype/tools/settings/local_settings/constants.py +++ b/openpype/tools/settings/local_settings/constants.py @@ -9,6 +9,7 @@ LABEL_DISCARD_CHANGES = "Discard changes" # TODO move to settings constants LOCAL_GENERAL_KEY = "general" LOCAL_PROJECTS_KEY = "projects" +LOCAL_ENV_KEY = "environments" LOCAL_APPS_KEY = "applications" # Roots key constant diff --git a/openpype/tools/settings/local_settings/environments_widget.py b/openpype/tools/settings/local_settings/environments_widget.py new file mode 100644 index 0000000000..70631e8f29 --- /dev/null +++ b/openpype/tools/settings/local_settings/environments_widget.py @@ -0,0 +1,80 @@ +from Qt import QtWidgets + +from openpype.tools.utils import PlaceholderLineEdit + + +class LocalEnvironmentsWidgets(QtWidgets.QWidget): + def __init__(self, system_settings_entity, parent): + super(LocalEnvironmentsWidgets, self).__init__(parent) + + self._widgets_by_env_key = {} + self.system_settings_entity = system_settings_entity + + content_widget = QtWidgets.QWidget(self) + content_layout = QtWidgets.QGridLayout(content_widget) + content_layout.setContentsMargins(0, 0, 0, 0) + + layout = QtWidgets.QVBoxLayout(self) + layout.setContentsMargins(0, 0, 0, 0) + + self._layout = layout + self._content_layout = content_layout + self._content_widget = content_widget + + def _clear_layout(self, layout): + while layout.count() > 0: + item = layout.itemAt(0) + widget = item.widget() + layout.removeItem(item) + if widget is not None: + widget.setVisible(False) + widget.deleteLater() + + def _reset_env_widgets(self): + self._clear_layout(self._content_layout) + self._clear_layout(self._layout) + + content_widget = QtWidgets.QWidget(self) + content_layout = QtWidgets.QGridLayout(content_widget) + content_layout.setContentsMargins(0, 0, 0, 0) + content_layout.setColumnStretch(0, 0) + content_layout.setColumnStretch(1, 1) + white_list_entity = ( + self.system_settings_entity["general"]["local_env_white_list"] + ) + + for row, item in enumerate(white_list_entity): + key = item.value + label_widget = QtWidgets.QLabel(key, self) + input_widget = PlaceholderLineEdit(self) + input_widget.setPlaceholderText("< Keep studio value >") + + content_layout.addWidget(label_widget, row, 0) + content_layout.addWidget(input_widget, row, 1) + + self._widgets_by_env_key[key] = input_widget + + self._layout.addWidget(content_widget, 1) + + self._content_layout = content_layout + self._content_widget = content_widget + + def update_local_settings(self, value): + if not value: + value = {} + + self._reset_env_widgets() + + for env_key, widget in self._widgets_by_env_key.items(): + env_value = value.get(env_key) or "" + widget.setText(env_value) + + def settings_value(self): + output = {} + for env_key, widget in self._widgets_by_env_key.items(): + value = widget.text() + if value: + output[env_key] = value + if not output: + return None + return output diff --git a/openpype/tools/settings/local_settings/window.py b/openpype/tools/settings/local_settings/window.py index fb47e69a17..4db0e01476 100644 --- a/openpype/tools/settings/local_settings/window.py +++ b/openpype/tools/settings/local_settings/window.py @@ -25,11 +25,13 @@ from .experimental_widget import ( LOCAL_EXPERIMENTAL_KEY ) from .apps_widget import LocalApplicationsWidgets +from .environments_widget import LocalEnvironmentsWidgets from .projects_widget import ProjectSettingsWidget from .constants import ( LOCAL_GENERAL_KEY, LOCAL_PROJECTS_KEY, + LOCAL_ENV_KEY, LOCAL_APPS_KEY ) @@ -49,18 +51,20 @@ class LocalSettingsWidget(QtWidgets.QWidget): self.pype_mongo_widget = None self.general_widget = None self.experimental_widget = None + self.envs_widget = None self.apps_widget = None self.projects_widget = None - self._create_pype_mongo_ui() + self._create_mongo_url_ui() self._create_general_ui() self._create_experimental_ui() + self._create_environments_ui() self._create_app_ui() self._create_project_ui() self.main_layout.addStretch(1) - def _create_pype_mongo_ui(self): + def _create_mongo_url_ui(self): pype_mongo_expand_widget = ExpandingWidget("OpenPype Mongo URL", self) pype_mongo_content = QtWidgets.QWidget(self) pype_mongo_layout = QtWidgets.QVBoxLayout(pype_mongo_content) @@ -110,6 +114,22 @@ class LocalSettingsWidget(QtWidgets.QWidget): self.experimental_widget = experimental_widget + def _create_environments_ui(self): + envs_expand_widget = ExpandingWidget("Environments", self) + envs_content = QtWidgets.QWidget(self) + envs_layout = QtWidgets.QVBoxLayout(envs_content) + envs_layout.setContentsMargins(CHILD_OFFSET, 5, 0, 0) + envs_expand_widget.set_content_widget(envs_content) + + envs_widget = LocalEnvironmentsWidgets( + self.system_settings, envs_content + ) + envs_layout.addWidget(envs_widget) + + self.main_layout.addWidget(envs_expand_widget) + + self.envs_widget = envs_widget + def _create_app_ui(self): # Applications app_expand_widget = ExpandingWidget("Applications", self) @@ -154,6 +174,9 @@ class LocalSettingsWidget(QtWidgets.QWidget): self.general_widget.update_local_settings( value.get(LOCAL_GENERAL_KEY) ) + self.envs_widget.update_local_settings( + value.get(LOCAL_ENV_KEY) + ) self.app_widget.update_local_settings( value.get(LOCAL_APPS_KEY) ) @@ -170,6 +193,10 @@ class LocalSettingsWidget(QtWidgets.QWidget): if general_value: output[LOCAL_GENERAL_KEY] = general_value + envs_value = self.envs_widget.settings_value() + if envs_value: + output[LOCAL_ENV_KEY] = envs_value + app_value = self.app_widget.settings_value() if app_value: output[LOCAL_APPS_KEY] = app_value From d7262446f82d6adc08650853c7649cf133d0895c Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 8 Apr 2022 17:11:48 +0200 Subject: [PATCH 103/126] use local settings to override environments during launching of application --- openpype/lib/applications.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/openpype/lib/applications.py b/openpype/lib/applications.py index 5821c863d7..049658a548 100644 --- a/openpype/lib/applications.py +++ b/openpype/lib/applications.py @@ -13,7 +13,8 @@ import six from openpype.settings import ( get_system_settings, - get_project_settings + get_project_settings, + get_local_settings ) from openpype.settings.constants import ( METADATA_KEYS, @@ -1272,6 +1273,9 @@ class EnvironmentPrepData(dict): if data.get("env") is None: data["env"] = os.environ.copy() + if "system_settings" not in data: + data["system_settings"] = get_system_settings() + super(EnvironmentPrepData, self).__init__(data) @@ -1434,6 +1438,19 @@ def prepare_app_environments(data, env_group=None, implementation_envs=True): ) ) + # Use environments from local settings + filtered_local_envs = {} + system_settings = data["system_settings"] + whitelist_envs = system_settings["general"].get("local_env_white_list") + if whitelist_envs: + local_settings = get_local_settings() + local_envs = local_settings.get("environments") or {} + filtered_local_envs = { + key: value + for key, value in local_envs.items() + if key in whitelist_envs + } + env_values = {} for _env_values in environments: if not _env_values: @@ -1441,6 +1458,10 @@ def prepare_app_environments(data, env_group=None, implementation_envs=True): # Choose right platform tool_env = parse_environments(_env_values, env_group) + for key, value in filtered_local_envs.items(): + if key in tool_env: + tool_env[key] = value + # Merge dictionaries env_values = _merge_env(tool_env, env_values) @@ -1611,7 +1632,6 @@ def _prepare_last_workfile(data, workdir): result will be stored. workdir (str): Path to folder where workfiles should be stored. """ - import avalon.api from openpype.pipeline import HOST_WORKFILE_EXTENSIONS log = data["log"] From 483d97f71b4655a6c0391b52eedf2b5952780a35 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 8 Apr 2022 17:37:01 +0200 Subject: [PATCH 104/126] OP-3073 - fix removed wrong hardcoded family --- .../webpublisher/plugins/publish/collect_published_files.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py index 65cef14703..56b2ef6e20 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -209,7 +209,6 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): msg = "No family found for combination of " +\ "task_type: {}, is_sequence:{}, extension: {}".format( task_type, is_sequence, extension) - found_family = "render" assert found_family, msg return (found_family, From 99c3cb50330decc5cdb8666b74eb6a04dc7577fa Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 8 Apr 2022 17:48:58 +0200 Subject: [PATCH 105/126] fix dicionary loop --- .../modules/ftrack/plugins/publish/integrate_ftrack_api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_api.py b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_api.py index 7ebf807f55..650c59fae8 100644 --- a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_api.py +++ b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_api.py @@ -263,7 +263,9 @@ class IntegrateFtrackApi(pyblish.api.InstancePlugin): self.log.info("Creating asset types with short names: {}".format( ", ".join(asset_type_names_by_missing_shorts.keys()) )) - for missing_short, type_name in asset_type_names_by_missing_shorts: + for missing_short, type_name in ( + asset_type_names_by_missing_shorts.items() + ): # Use short for name if name is not defined if not type_name: type_name = missing_short From d4182e5cea64f6374b2a8ef443215f678b0c40bd Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 8 Apr 2022 18:02:48 +0200 Subject: [PATCH 106/126] dev_test_plugin --- openpype/hosts/flame/api/test_plugin.py | 428 ++++++++++++++++++++++++ openpype/version.py | 9 +- 2 files changed, 436 insertions(+), 1 deletion(-) create mode 100644 openpype/hosts/flame/api/test_plugin.py diff --git a/openpype/hosts/flame/api/test_plugin.py b/openpype/hosts/flame/api/test_plugin.py new file mode 100644 index 0000000000..d75819a9e3 --- /dev/null +++ b/openpype/hosts/flame/api/test_plugin.py @@ -0,0 +1,428 @@ +import os +import tempfile +import itertools +import contextlib +import xml.etree.cElementTree as cET +from copy import deepcopy +import shutil +from xml.etree import ElementTree as ET + +import openpype.api as openpype + +import logging + +log = logging.getLogger(__name__) + + +@contextlib.contextmanager +def maintained_temp_file_path(suffix=None): + _suffix = suffix or "" + + try: + # Store dumped json to temporary file + temporary_file = tempfile.mktemp( + suffix=_suffix, prefix="flame_maintained_") + yield temporary_file.replace("\\", "/") + + except IOError as _error: + raise IOError( + "Not able to create temp json file: {}".format(_error)) + + finally: + # Remove the temporary json + os.remove(temporary_file) + + +class MediaInfoFile(object): + """Class to get media info file clip data + + Raises: + IOError: MEDIA_SCRIPT_PATH path doesn't exists + TypeError: Not able to generate clip xml data file + ET.ParseError: Missing clip in xml clip data + IOError: Not able to save xml clip data to file + + Attributes: + str: `MEDIA_SCRIPT_PATH` path to flame binary + logging.Logger: `log` logger + + TODO: add method for getting metadata to dict + """ + MEDIA_SCRIPT_PATH = "/opt/Autodesk/mio/current/dl_get_media_info" + + log = log + + _clip_data = None + _start_frame = None + _fps = None + _drop_mode = None + + def __init__(self, path, **kwargs): + + # replace log if any + if kwargs.get("logger"): + self.log = kwargs["logger"] + + # test if `dl_get_media_info` paht exists + self._validate_media_script_path() + + # derivate other feed variables + self.feed_basename = os.path.basename(path) + self.feed_dir = os.path.dirname(path) + self.feed_ext = os.path.splitext(self.feed_basename)[1][1:].lower() + + with maintained_temp_file_path(".clip") as tmp_path: + self.log.info("Temp File: {}".format(tmp_path)) + self._generate_media_info_file(tmp_path) + + # get clip data and make them single if there is multiple + # clips data + xml_data = self._make_single_clip_media_info(tmp_path) + self.log.info("xml_data: {}".format(xml_data)) + self.log.info("type: {}".format(type(xml_data))) + + # get all time related data and assign them + self._get_time_info_from_origin(xml_data) + self.log.info("start_frame: {}".format(self.start_frame)) + self.log.info("fps: {}".format(self.fps)) + self.log.info("drop frame: {}".format(self.drop_mode)) + self.clip_data = xml_data + + @property + def clip_data(self): + """Clip's xml clip data + + Returns: + xml.etree.ElementTree: xml data + """ + return self._clip_data + + @clip_data.setter + def clip_data(self, data): + self._clip_data = data + + @property + def start_frame(self): + """ Clip's starting frame found in timecode + + Returns: + int: number of frames + """ + return self._start_frame + + @start_frame.setter + def start_frame(self, number): + self._start_frame = int(number) + + @property + def fps(self): + """ Clip's frame rate + + Returns: + float: frame rate + """ + return self._fps + + @fps.setter + def fps(self, fl_number): + self._fps = float(fl_number) + + @property + def drop_mode(self): + """ Clip's drop frame mode + + Returns: + str: drop frame flag + """ + return self._drop_mode + + @drop_mode.setter + def drop_mode(self, text): + self._drop_mode = str(text) + + def _validate_media_script_path(self): + if not os.path.isfile(self.MEDIA_SCRIPT_PATH): + raise IOError("Media Scirpt does not exist: `{}`".format( + self.MEDIA_SCRIPT_PATH)) + + def _generate_media_info_file(self, fpath): + # Create cmd arguments for gettig xml file info file + cmd_args = [ + self.MEDIA_SCRIPT_PATH, + "-e", self.feed_ext, + "-o", fpath, + self.feed_dir + ] + + try: + # execute creation of clip xml template data + openpype.run_subprocess(cmd_args) + except TypeError as error: + raise TypeError( + "Error creating `{}` due: {}".format(fpath, error)) + + def _make_single_clip_media_info(self, fpath): + with open(fpath) as f: + lines = f.readlines() + _added_root = itertools.chain( + "", deepcopy(lines)[1:], "") + new_root = ET.fromstringlist(_added_root) + + # find the clip which is matching to my input name + xml_clips = new_root.findall("clip") + matching_clip = None + for xml_clip in xml_clips: + if xml_clip.find("name").text in self.feed_basename: + matching_clip = xml_clip + + if matching_clip is None: + # return warning there is missing clip + raise ET.ParseError( + "Missing clip in `{}`. Available clips {}".format( + self.feed_basename, [ + xml_clip.find("name").text + for xml_clip in xml_clips + ] + )) + + return matching_clip + + def _get_time_info_from_origin(self, xml_data): + try: + for out_track in xml_data.iter('track'): + for out_feed in out_track.iter('feed'): + # start frame + out_feed_nb_ticks_obj = out_feed.find( + 'startTimecode/nbTicks') + self.start_frame = out_feed_nb_ticks_obj.text + + # fps + out_feed_fps_obj = out_feed.find( + 'startTimecode/rate') + self.fps = out_feed_fps_obj.text + + # drop frame mode + out_feed_drop_mode_obj = out_feed.find( + 'startTimecode/dropMode') + self.drop_mode = out_feed_drop_mode_obj.text + break + else: + continue + except Exception as msg: + self.log.warning(msg) + + @staticmethod + def write_clip_data_to_file(fpath, xml_data): + log.info(">>> type of xml_data: {}".format(type(xml_data))) + if isinstance(xml_data, ET.ElementTree): + xml_data = xml_data.getroot() + try: + # save it as new file + tree = cET.ElementTree(xml_data) + tree.write( + fpath, xml_declaration=True, + method='xml', encoding='UTF-8' + ) + except IOError as error: + raise IOError( + "Not able to write data to file: {}".format(error)) + + +class OpenClipSolver(MediaInfoFile): + create_new_clip = False + + log = log + + def __init__(self, openclip_file_path, feed_data): + self.out_file = openclip_file_path + + # new feed variables: + feed_path = feed_data.pop("path") + + # initialize parent class + super(OpenClipSolver, self).__init__( + feed_path, + **feed_data + ) + + # get other metadata + self.feed_version_name = feed_data["version"] + self.feed_colorspace = feed_data.get("colorspace") + self.log.info("feed_version_name: {}".format(self.feed_version_name)) + + # derivate other feed variables + self.feed_basename = os.path.basename(feed_path) + self.feed_dir = os.path.dirname(feed_path) + self.feed_ext = os.path.splitext(self.feed_basename)[1][1:].lower() + self.log.info("feed_ext: {}".format(self.feed_ext)) + self.log.info("out_file: {}".format(self.out_file)) + if not self._is_valid_tmp_file(self.out_file): + self.create_new_clip = True + + def _is_valid_tmp_file(self, file): + # check if file exists + if os.path.isfile(file): + # test also if file is not empty + with open(file) as f: + lines = f.readlines() + if len(lines) > 2: + return True + + # file is probably corrupted + os.remove(file) + return False + + def make(self): + + if self.create_new_clip: + # New openClip + self._create_new_open_clip() + else: + self._update_open_clip() + + def _clear_handler(self, xml_object): + for handler in xml_object.findall("./handler"): + self.log.info("Handler found") + xml_object.remove(handler) + + def _create_new_open_clip(self): + self.log.info("Building new openClip") + self.log.info(">> self.clip_data: {}".format(self.clip_data)) + + # clip data comming from MediaInfoFile + tmp_xml_feeds = self.clip_data.find('tracks/track/feeds') + tmp_xml_feeds.set('currentVersion', self.feed_version_name) + for tmp_feed in tmp_xml_feeds: + tmp_feed.set('vuid', self.feed_version_name) + + # add colorspace if any is set + if self.feed_colorspace: + self._add_colorspace(tmp_feed, self.feed_colorspace) + + self._clear_handler(tmp_feed) + + tmp_xml_versions_obj = self.clip_data.find('versions') + tmp_xml_versions_obj.set('currentVersion', self.feed_version_name) + for xml_new_version in tmp_xml_versions_obj: + xml_new_version.set('uid', self.feed_version_name) + xml_new_version.set('type', 'version') + + self._clear_handler(self.clip_data) + self.log.info("Adding feed version: {}".format(self.feed_basename)) + + self.write_clip_data_to_file(self.out_file, self.clip_data) + + def _update_open_clip(self): + self.log.info("Updating openClip ..") + + out_xml = ET.parse(self.out_file) + + self.log.info(">> out_xml: {}".format(out_xml)) + self.log.info(">> self.clip_data: {}".format(self.clip_data)) + + # Get new feed from tmp file + tmp_xml_feed = self.clip_data.find('tracks/track/feeds/feed') + + self._clear_handler(tmp_xml_feed) + + # update fps from MediaInfoFile class + if self.fps: + tmp_feed_fps_obj = tmp_xml_feed.find( + "startTimecode/rate") + tmp_feed_fps_obj.text = str(self.fps) + + # update start_frame from MediaInfoFile class + if self.start_frame: + tmp_feed_nb_ticks_obj = tmp_xml_feed.find( + "startTimecode/nbTicks") + tmp_feed_nb_ticks_obj.text = str(self.start_frame) + + # update drop_mode from MediaInfoFile class + if self.drop_mode: + tmp_feed_drop_mode_obj = tmp_xml_feed.find( + "startTimecode/dropMode") + tmp_feed_drop_mode_obj.text = str(self.drop_mode) + + new_path_obj = tmp_xml_feed.find( + "spans/span/path") + new_path = new_path_obj.text + + feed_added = False + if not self._feed_exists(out_xml, new_path): + tmp_xml_feed.set('vuid', self.feed_version_name) + # Append new temp file feed to .clip source out xml + out_track = out_xml.find("tracks/track") + # add colorspace if any is set + if self.feed_colorspace: + self._add_colorspace(tmp_xml_feed, self.feed_colorspace) + + out_feeds = out_track.find('feeds') + out_feeds.set('currentVersion', self.feed_version_name) + out_feeds.append(tmp_xml_feed) + + self.log.info( + "Appending new feed: {}".format( + self.feed_version_name)) + feed_added = True + + if feed_added: + # Append vUID to versions + out_xml_versions_obj = out_xml.find('versions') + out_xml_versions_obj.set( + 'currentVersion', self.feed_version_name) + new_version_obj = ET.Element( + "version", {"type": "version", "uid": self.feed_version_name}) + out_xml_versions_obj.insert(0, new_version_obj) + + self._clear_handler(out_xml) + + # fist create backup + self._create_openclip_backup_file(self.out_file) + + self.log.info("Adding feed version: {}".format( + self.feed_version_name)) + + self.write_clip_data_to_file(self.out_file, out_xml) + + self.log.info("openClip Updated: {}".format(self.out_file)) + + def _feed_exists(self, xml_data, path): + # loop all available feed paths and check if + # the path is not already in file + for src_path in xml_data.iter('path'): + if path == src_path.text: + self.log.warning( + "Not appending file as it already is in .clip file") + return True + + def _create_openclip_backup_file(self, file): + bck_file = "{}.bak".format(file) + # if backup does not exist + if not os.path.isfile(bck_file): + shutil.copy2(file, bck_file) + else: + # in case it exists and is already multiplied + created = False + for _i in range(1, 99): + bck_file = "{name}.bak.{idx:0>2}".format( + name=file, + idx=_i) + # create numbered backup file + if not os.path.isfile(bck_file): + shutil.copy2(file, bck_file) + created = True + break + # in case numbered does not exists + if not created: + bck_file = "{}.bak.last".format(file) + shutil.copy2(file, bck_file) + + def _add_colorspace(self, feed_obj, profile_name): + feed_storage_obj = feed_obj.find("storageFormat") + feed_clr_obj = feed_storage_obj.find("colourSpace") + if feed_clr_obj is not None: + feed_clr_obj = ET.Element( + "colourSpace", {"type": "string"}) + feed_storage_obj.append(feed_clr_obj) + + feed_clr_obj.text = profile_name diff --git a/openpype/version.py b/openpype/version.py index 97aa585ca7..d447d27172 100644 --- a/openpype/version.py +++ b/openpype/version.py @@ -1,3 +1,10 @@ # -*- coding: utf-8 -*- """Package declaring Pype version.""" -__version__ = "3.9.3" +__version__ = "3.9.3-nightly.1-upp220408" + + +''' +includes: + - Flame: integrate batch groups: + https://github.com/pypeclub/OpenPype/pull/2928 +''' From 06d2e898654b9aca82c9a8c979938cc44ee1f766 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 8 Apr 2022 18:03:30 +0200 Subject: [PATCH 107/126] testing file --- openpype/hosts/flame/tests/flame_test.py | 30 ++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 openpype/hosts/flame/tests/flame_test.py diff --git a/openpype/hosts/flame/tests/flame_test.py b/openpype/hosts/flame/tests/flame_test.py new file mode 100644 index 0000000000..402983eeba --- /dev/null +++ b/openpype/hosts/flame/tests/flame_test.py @@ -0,0 +1,30 @@ +from openpype.lib import import_filepath + +plugin = import_filepath( + "/Users/pype.club/code/openpype/openpype/hosts/flame/api/test_plugin.py") + +openclip_file_path = "/Users/pype.club/FLAME_STORAGE/test_shot_fps_float/test.clip" +# feed_datas = [ +# { +# "path": "/Users/pype.club/pype_club_root/OP02_VFX_demo/shots/a/a0000001/publish/plate/plateMain/v007/op02vfx_a0000001_plateMain_v007_exr16fpdwaaCl.0997.exr", +# "version": "v007" +# }, +# { +# "path": "/Users/pype.club/pype_club_root/OP02_VFX_demo/shots/a/a0000001/publish/plate/plateMain/v008/op02vfx_a0000001_plateMain_v008_exr16fpdwaaCl.0997.exr", +# "version": "v008" +# } +# ] + +feed_datas = [ + { + "path": "/Users/pype.club/FLAME_STORAGE/test_shot_fps_float/v001/file_name_v001.1001.exr", + "version": "v001" + }, + { + "path": "/Users/pype.club/FLAME_STORAGE/test_shot_fps_float/v002/file_name_v002.1001.exr", + "version": "v002" + } +] +for feed_data in feed_datas: + oclip = plugin.OpenClipSolver(openclip_file_path, feed_data) + oclip.make() From b9a7d810cc20949452a4e5e069497b08ca72f8ae Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 8 Apr 2022 18:32:40 +0200 Subject: [PATCH 108/126] fix applying of env variables that are not defined by applications or tools --- openpype/lib/applications.py | 43 ++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/openpype/lib/applications.py b/openpype/lib/applications.py index 049658a548..07b91dda03 100644 --- a/openpype/lib/applications.py +++ b/openpype/lib/applications.py @@ -1399,8 +1399,27 @@ def prepare_app_environments(data, env_group=None, implementation_envs=True): app = data["app"] log = data["log"] + source_env = data["env"].copy() - _add_python_version_paths(app, data["env"], log) + _add_python_version_paths(app, source_env, log) + + # Use environments from local settings + filtered_local_envs = {} + system_settings = data["system_settings"] + whitelist_envs = system_settings["general"].get("local_env_white_list") + if whitelist_envs: + local_settings = get_local_settings() + local_envs = local_settings.get("environments") or {} + filtered_local_envs = { + key: value + for key, value in local_envs.items() + if key in whitelist_envs + } + + # Apply local environment variables for already existing values + for key, value in filtered_local_envs.items(): + if key in source_env: + source_env[key] = value # `added_env_keys` has debug purpose added_env_keys = {app.group.name, app.name} @@ -1438,19 +1457,6 @@ def prepare_app_environments(data, env_group=None, implementation_envs=True): ) ) - # Use environments from local settings - filtered_local_envs = {} - system_settings = data["system_settings"] - whitelist_envs = system_settings["general"].get("local_env_white_list") - if whitelist_envs: - local_settings = get_local_settings() - local_envs = local_settings.get("environments") or {} - filtered_local_envs = { - key: value - for key, value in local_envs.items() - if key in whitelist_envs - } - env_values = {} for _env_values in environments: if not _env_values: @@ -1458,6 +1464,10 @@ def prepare_app_environments(data, env_group=None, implementation_envs=True): # Choose right platform tool_env = parse_environments(_env_values, env_group) + + # Apply local environment variables + # - must happen between all values because they may be used during + # merge for key, value in filtered_local_envs.items(): if key in tool_env: tool_env[key] = value @@ -1465,7 +1475,8 @@ def prepare_app_environments(data, env_group=None, implementation_envs=True): # Merge dictionaries env_values = _merge_env(tool_env, env_values) - merged_env = _merge_env(env_values, data["env"]) + merged_env = _merge_env(env_values, source_env) + loaded_env = acre.compute(merged_env, cleanup=False) final_env = None @@ -1485,7 +1496,7 @@ def prepare_app_environments(data, env_group=None, implementation_envs=True): if final_env is None: final_env = loaded_env - keys_to_remove = set(data["env"].keys()) - set(final_env.keys()) + keys_to_remove = set(source_env.keys()) - set(final_env.keys()) # Update env data["env"].update(final_env) From 2af6e7140ec391ace6dd310f7744e6da09b74583 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 8 Apr 2022 18:51:55 +0200 Subject: [PATCH 109/126] changed label and added tooltip --- .../entities/schemas/system_schema/schema_general.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/settings/entities/schemas/system_schema/schema_general.json b/openpype/settings/entities/schemas/system_schema/schema_general.json index 997404b2e6..fcab4cd5d8 100644 --- a/openpype/settings/entities/schemas/system_schema/schema_general.json +++ b/openpype/settings/entities/schemas/system_schema/schema_general.json @@ -113,7 +113,8 @@ { "type": "list", "key": "local_env_white_list", - "label": "White list of local environment variables", + "label": "Local overrides of environment variable keys", + "tooltip": "Environment variable keys that can be changed per machine using Local settings UI.\nKey changes are applied only on applications and tools environments.", "use_label_wrap": true, "object_type": "text" }, From 9893903a5728bea368f033889bd352dd1854b1e3 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 8 Apr 2022 18:54:55 +0200 Subject: [PATCH 110/126] add label when there are not env keys to set --- .../local_settings/environments_widget.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/openpype/tools/settings/local_settings/environments_widget.py b/openpype/tools/settings/local_settings/environments_widget.py index 70631e8f29..14ca517851 100644 --- a/openpype/tools/settings/local_settings/environments_widget.py +++ b/openpype/tools/settings/local_settings/environments_widget.py @@ -37,12 +37,10 @@ class LocalEnvironmentsWidgets(QtWidgets.QWidget): content_widget = QtWidgets.QWidget(self) content_layout = QtWidgets.QGridLayout(content_widget) content_layout.setContentsMargins(0, 0, 0, 0) - content_layout.setColumnStretch(0, 0) - content_layout.setColumnStretch(1, 1) white_list_entity = ( self.system_settings_entity["general"]["local_env_white_list"] ) - + row = -1 for row, item in enumerate(white_list_entity): key = item.value label_widget = QtWidgets.QLabel(key, self) @@ -54,6 +52,21 @@ class LocalEnvironmentsWidgets(QtWidgets.QWidget): self._widgets_by_env_key[key] = input_widget + if row < 0: + label_widget = QtWidgets.QLabel( + ( + "Your studio does not allow to change" + " Environment variables locally." + ), + self + ) + content_layout.addWidget(label_widget, 0, 0) + content_layout.setColumnStretch(0, 1) + + else: + content_layout.setColumnStretch(0, 0) + content_layout.setColumnStretch(1, 1) + self._layout.addWidget(content_widget, 1) self._content_layout = content_layout From a103eba505d9358c0c7058f614e36dd14b2edd4b Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 8 Apr 2022 19:03:54 +0200 Subject: [PATCH 111/126] flame: fixing OpenClipSolver --- openpype/hosts/flame/api/lib.py | 18 +- openpype/hosts/flame/api/plugin.py | 183 +++------- openpype/hosts/flame/api/test_plugin.py | 428 ----------------------- openpype/hosts/flame/tests/flame_test.py | 30 -- 4 files changed, 56 insertions(+), 603 deletions(-) delete mode 100644 openpype/hosts/flame/api/test_plugin.py delete mode 100644 openpype/hosts/flame/tests/flame_test.py diff --git a/openpype/hosts/flame/api/lib.py b/openpype/hosts/flame/api/lib.py index 6ee0eb6a82..c7c444c1fb 100644 --- a/openpype/hosts/flame/api/lib.py +++ b/openpype/hosts/flame/api/lib.py @@ -784,9 +784,14 @@ class MediaInfoFile(object): # get clip data and make them single if there is multiple # clips data xml_data = self._make_single_clip_media_info(tmp_path) + self.log.debug("xml_data: {}".format(xml_data)) + self.log.debug("type: {}".format(type(xml_data))) # get all time related data and assign them self._get_time_info_from_origin(xml_data) + self.log.debug("start_frame: {}".format(self.start_frame)) + self.log.debug("fps: {}".format(self.fps)) + self.log.debug("drop frame: {}".format(self.drop_mode)) self.clip_data = xml_data @property @@ -913,10 +918,19 @@ class MediaInfoFile(object): self.log.warning(msg) @staticmethod - def write_clip_data_to_file(fpath, xml_data): + def write_clip_data_to_file(fpath, xml_element_data): + """ Write xml element of clip data to file + + Args: + fpath (string): file path + xml_element_data (xml.etree.ElementTree.Element): xml data + + Raises: + IOError: If data could not be written to file + """ try: # save it as new file - tree = cET.ElementTree(xml_data) + tree = cET.ElementTree(xml_element_data) tree.write( fpath, xml_declaration=True, method='xml', encoding='UTF-8' diff --git a/openpype/hosts/flame/api/plugin.py b/openpype/hosts/flame/api/plugin.py index bd0f9f1a81..42e6e19931 100644 --- a/openpype/hosts/flame/api/plugin.py +++ b/openpype/hosts/flame/api/plugin.py @@ -1,15 +1,11 @@ -import itertools import os import re import shutil -import sys -import xml.etree.cElementTree as cET from copy import deepcopy from xml.etree import ElementTree as ET import openpype.api as openpype import qargparse -import six from openpype import style from openpype.pipeline import LegacyCreator, LoaderPlugin from Qt import QtCore, QtWidgets @@ -658,8 +654,8 @@ class PublishableClip: # Publishing plugin functions -# Loader plugin functions +# Loader plugin functions class ClipLoader(LoaderPlugin): """A basic clip loader for Flame @@ -679,53 +675,37 @@ class ClipLoader(LoaderPlugin): ] -# TODO: inheritance from flame.api.lib.MediaInfoFile -class OpenClipSolver: - media_script_path = "/opt/Autodesk/mio/current/dl_get_media_info" - tmp_name = "_tmp.clip" - tmp_file = None +class OpenClipSolver(flib.MediaInfoFile): create_new_clip = False - out_feed_nb_ticks = None - out_feed_fps = None - out_feed_drop_mode = None - log = log def __init__(self, openclip_file_path, feed_data): - # test if media script paht exists - self._validate_media_script_path() + self.out_file = openclip_file_path # new feed variables: - feed_path = feed_data["path"] + feed_path = feed_data.pop("path") + + # initialize parent class + super(OpenClipSolver, self).__init__( + feed_path, + **feed_data + ) + + # get other metadata self.feed_version_name = feed_data["version"] self.feed_colorspace = feed_data.get("colorspace") - - if feed_data.get("logger"): - self.log = feed_data["logger"] + self.log.debug("feed_version_name: {}".format(self.feed_version_name)) # derivate other feed variables self.feed_basename = os.path.basename(feed_path) self.feed_dir = os.path.dirname(feed_path) self.feed_ext = os.path.splitext(self.feed_basename)[1][1:].lower() - - if not self._is_valid_tmp_file(openclip_file_path): - # openclip does not exist yet and will be created - self.tmp_file = self.out_file = openclip_file_path + self.log.debug("feed_ext: {}".format(self.feed_ext)) + self.log.debug("out_file: {}".format(self.out_file)) + if not self._is_valid_tmp_file(self.out_file): self.create_new_clip = True - else: - # update already created clip - # output a temp file - self.out_file = openclip_file_path - self.tmp_file = os.path.join(self.feed_dir, self.tmp_name) - - # remove previously generated temp files - # it will be regenerated - self._clear_tmp_file() - - self.log.info("Temp File: {}".format(self.tmp_file)) - def _is_valid_tmp_file(self, file): # check if file exists if os.path.isfile(file): @@ -740,7 +720,6 @@ class OpenClipSolver: return False def make(self): - self._generate_media_info_file() if self.create_new_clip: # New openClip @@ -748,69 +727,17 @@ class OpenClipSolver: else: self._update_open_clip() - def _validate_media_script_path(self): - if not os.path.isfile(self.media_script_path): - raise IOError("Media Scirpt does not exist: `{}`".format( - self.media_script_path)) - - def _generate_media_info_file(self): - # Create cmd arguments for gettig xml file info file - cmd_args = [ - self.media_script_path, - "-e", self.feed_ext, - "-o", self.tmp_file, - self.feed_dir - ] - - # execute creation of clip xml template data - try: - openpype.run_subprocess(cmd_args) - self._make_single_clip_media_info() - except TypeError: - self.log.error("Error creating self.tmp_file") - six.reraise(*sys.exc_info()) - - def _make_single_clip_media_info(self): - with open(self.tmp_file) as f: - lines = f.readlines() - _added_root = itertools.chain( - "", deepcopy(lines)[1:], "") - new_root = ET.fromstringlist(_added_root) - - # find the clip which is matching to my input name - xml_clips = new_root.findall("clip") - matching_clip = None - for xml_clip in xml_clips: - if xml_clip.find("name").text in self.feed_basename: - matching_clip = xml_clip - - if matching_clip is None: - # return warning there is missing clip - raise ET.ParseError( - "Missing clip in `{}`. Available clips {}".format( - self.feed_basename, [ - xml_clip.find("name").text - for xml_clip in xml_clips - ] - )) - - self._write_result_xml_to_file(self.tmp_file, matching_clip) - - def _clear_tmp_file(self): - if os.path.isfile(self.tmp_file): - os.remove(self.tmp_file) - def _clear_handler(self, xml_object): for handler in xml_object.findall("./handler"): - self.log.debug("Handler found") + self.log.info("Handler found") xml_object.remove(handler) def _create_new_open_clip(self): self.log.info("Building new openClip") + self.log.debug(">> self.clip_data: {}".format(self.clip_data)) - tmp_xml = ET.parse(self.tmp_file) - - tmp_xml_feeds = tmp_xml.find('tracks/track/feeds') + # clip data comming from MediaInfoFile + tmp_xml_feeds = self.clip_data.find('tracks/track/feeds') tmp_xml_feeds.set('currentVersion', self.feed_version_name) for tmp_feed in tmp_xml_feeds: tmp_feed.set('vuid', self.feed_version_name) @@ -821,46 +748,48 @@ class OpenClipSolver: self._clear_handler(tmp_feed) - tmp_xml_versions_obj = tmp_xml.find('versions') + tmp_xml_versions_obj = self.clip_data.find('versions') tmp_xml_versions_obj.set('currentVersion', self.feed_version_name) for xml_new_version in tmp_xml_versions_obj: xml_new_version.set('uid', self.feed_version_name) xml_new_version.set('type', 'version') - xml_data = self._fix_xml_data(tmp_xml) + self._clear_handler(self.clip_data) self.log.info("Adding feed version: {}".format(self.feed_basename)) - self._write_result_xml_to_file(self.out_file, xml_data) - - self.log.info("openClip Updated: {}".format(self.tmp_file)) + self.write_clip_data_to_file(self.out_file, self.clip_data) def _update_open_clip(self): self.log.info("Updating openClip ..") out_xml = ET.parse(self.out_file) - tmp_xml = ET.parse(self.tmp_file) + out_xml = out_xml.getroot() self.log.debug(">> out_xml: {}".format(out_xml)) - self.log.debug(">> tmp_xml: {}".format(tmp_xml)) + self.log.debug(">> self.clip_data: {}".format(self.clip_data)) # Get new feed from tmp file - tmp_xml_feed = tmp_xml.find('tracks/track/feeds/feed') + tmp_xml_feed = self.clip_data.find('tracks/track/feeds/feed') self._clear_handler(tmp_xml_feed) - self._get_time_info_from_origin(out_xml) - if self.out_feed_fps: + # update fps from MediaInfoFile class + if self.fps: tmp_feed_fps_obj = tmp_xml_feed.find( "startTimecode/rate") - tmp_feed_fps_obj.text = self.out_feed_fps - if self.out_feed_nb_ticks: + tmp_feed_fps_obj.text = str(self.fps) + + # update start_frame from MediaInfoFile class + if self.start_frame: tmp_feed_nb_ticks_obj = tmp_xml_feed.find( "startTimecode/nbTicks") - tmp_feed_nb_ticks_obj.text = self.out_feed_nb_ticks - if self.out_feed_drop_mode: + tmp_feed_nb_ticks_obj.text = str(self.start_frame) + + # update drop_mode from MediaInfoFile class + if self.drop_mode: tmp_feed_drop_mode_obj = tmp_xml_feed.find( "startTimecode/dropMode") - tmp_feed_drop_mode_obj.text = self.out_feed_drop_mode + tmp_feed_drop_mode_obj.text = str(self.drop_mode) new_path_obj = tmp_xml_feed.find( "spans/span/path") @@ -893,7 +822,7 @@ class OpenClipSolver: "version", {"type": "version", "uid": self.feed_version_name}) out_xml_versions_obj.insert(0, new_version_obj) - xml_data = self._fix_xml_data(out_xml) + self._clear_handler(out_xml) # fist create backup self._create_openclip_backup_file(self.out_file) @@ -901,30 +830,9 @@ class OpenClipSolver: self.log.info("Adding feed version: {}".format( self.feed_version_name)) - self._write_result_xml_to_file(self.out_file, xml_data) + self.write_clip_data_to_file(self.out_file, out_xml) - self.log.info("openClip Updated: {}".format(self.out_file)) - - self._clear_tmp_file() - - def _get_time_info_from_origin(self, xml_data): - try: - for out_track in xml_data.iter('track'): - for out_feed in out_track.iter('feed'): - out_feed_nb_ticks_obj = out_feed.find( - 'startTimecode/nbTicks') - self.out_feed_nb_ticks = out_feed_nb_ticks_obj.text - out_feed_fps_obj = out_feed.find( - 'startTimecode/rate') - self.out_feed_fps = out_feed_fps_obj.text - out_feed_drop_mode_obj = out_feed.find( - 'startTimecode/dropMode') - self.out_feed_drop_mode = out_feed_drop_mode_obj.text - break - else: - continue - except Exception as msg: - self.log.warning(msg) + self.log.debug("OpenClip Updated: {}".format(self.out_file)) def _feed_exists(self, xml_data, path): # loop all available feed paths and check if @@ -935,17 +843,6 @@ class OpenClipSolver: "Not appending file as it already is in .clip file") return True - def _fix_xml_data(self, xml_data): - xml_root = xml_data.getroot() - self._clear_handler(xml_root) - return xml_root - - def _write_result_xml_to_file(self, file, xml_data): - # save it as new file - tree = cET.ElementTree(xml_data) - tree.write(file, xml_declaration=True, - method='xml', encoding='UTF-8') - def _create_openclip_backup_file(self, file): bck_file = "{}.bak".format(file) # if backup does not exist diff --git a/openpype/hosts/flame/api/test_plugin.py b/openpype/hosts/flame/api/test_plugin.py deleted file mode 100644 index d75819a9e3..0000000000 --- a/openpype/hosts/flame/api/test_plugin.py +++ /dev/null @@ -1,428 +0,0 @@ -import os -import tempfile -import itertools -import contextlib -import xml.etree.cElementTree as cET -from copy import deepcopy -import shutil -from xml.etree import ElementTree as ET - -import openpype.api as openpype - -import logging - -log = logging.getLogger(__name__) - - -@contextlib.contextmanager -def maintained_temp_file_path(suffix=None): - _suffix = suffix or "" - - try: - # Store dumped json to temporary file - temporary_file = tempfile.mktemp( - suffix=_suffix, prefix="flame_maintained_") - yield temporary_file.replace("\\", "/") - - except IOError as _error: - raise IOError( - "Not able to create temp json file: {}".format(_error)) - - finally: - # Remove the temporary json - os.remove(temporary_file) - - -class MediaInfoFile(object): - """Class to get media info file clip data - - Raises: - IOError: MEDIA_SCRIPT_PATH path doesn't exists - TypeError: Not able to generate clip xml data file - ET.ParseError: Missing clip in xml clip data - IOError: Not able to save xml clip data to file - - Attributes: - str: `MEDIA_SCRIPT_PATH` path to flame binary - logging.Logger: `log` logger - - TODO: add method for getting metadata to dict - """ - MEDIA_SCRIPT_PATH = "/opt/Autodesk/mio/current/dl_get_media_info" - - log = log - - _clip_data = None - _start_frame = None - _fps = None - _drop_mode = None - - def __init__(self, path, **kwargs): - - # replace log if any - if kwargs.get("logger"): - self.log = kwargs["logger"] - - # test if `dl_get_media_info` paht exists - self._validate_media_script_path() - - # derivate other feed variables - self.feed_basename = os.path.basename(path) - self.feed_dir = os.path.dirname(path) - self.feed_ext = os.path.splitext(self.feed_basename)[1][1:].lower() - - with maintained_temp_file_path(".clip") as tmp_path: - self.log.info("Temp File: {}".format(tmp_path)) - self._generate_media_info_file(tmp_path) - - # get clip data and make them single if there is multiple - # clips data - xml_data = self._make_single_clip_media_info(tmp_path) - self.log.info("xml_data: {}".format(xml_data)) - self.log.info("type: {}".format(type(xml_data))) - - # get all time related data and assign them - self._get_time_info_from_origin(xml_data) - self.log.info("start_frame: {}".format(self.start_frame)) - self.log.info("fps: {}".format(self.fps)) - self.log.info("drop frame: {}".format(self.drop_mode)) - self.clip_data = xml_data - - @property - def clip_data(self): - """Clip's xml clip data - - Returns: - xml.etree.ElementTree: xml data - """ - return self._clip_data - - @clip_data.setter - def clip_data(self, data): - self._clip_data = data - - @property - def start_frame(self): - """ Clip's starting frame found in timecode - - Returns: - int: number of frames - """ - return self._start_frame - - @start_frame.setter - def start_frame(self, number): - self._start_frame = int(number) - - @property - def fps(self): - """ Clip's frame rate - - Returns: - float: frame rate - """ - return self._fps - - @fps.setter - def fps(self, fl_number): - self._fps = float(fl_number) - - @property - def drop_mode(self): - """ Clip's drop frame mode - - Returns: - str: drop frame flag - """ - return self._drop_mode - - @drop_mode.setter - def drop_mode(self, text): - self._drop_mode = str(text) - - def _validate_media_script_path(self): - if not os.path.isfile(self.MEDIA_SCRIPT_PATH): - raise IOError("Media Scirpt does not exist: `{}`".format( - self.MEDIA_SCRIPT_PATH)) - - def _generate_media_info_file(self, fpath): - # Create cmd arguments for gettig xml file info file - cmd_args = [ - self.MEDIA_SCRIPT_PATH, - "-e", self.feed_ext, - "-o", fpath, - self.feed_dir - ] - - try: - # execute creation of clip xml template data - openpype.run_subprocess(cmd_args) - except TypeError as error: - raise TypeError( - "Error creating `{}` due: {}".format(fpath, error)) - - def _make_single_clip_media_info(self, fpath): - with open(fpath) as f: - lines = f.readlines() - _added_root = itertools.chain( - "", deepcopy(lines)[1:], "") - new_root = ET.fromstringlist(_added_root) - - # find the clip which is matching to my input name - xml_clips = new_root.findall("clip") - matching_clip = None - for xml_clip in xml_clips: - if xml_clip.find("name").text in self.feed_basename: - matching_clip = xml_clip - - if matching_clip is None: - # return warning there is missing clip - raise ET.ParseError( - "Missing clip in `{}`. Available clips {}".format( - self.feed_basename, [ - xml_clip.find("name").text - for xml_clip in xml_clips - ] - )) - - return matching_clip - - def _get_time_info_from_origin(self, xml_data): - try: - for out_track in xml_data.iter('track'): - for out_feed in out_track.iter('feed'): - # start frame - out_feed_nb_ticks_obj = out_feed.find( - 'startTimecode/nbTicks') - self.start_frame = out_feed_nb_ticks_obj.text - - # fps - out_feed_fps_obj = out_feed.find( - 'startTimecode/rate') - self.fps = out_feed_fps_obj.text - - # drop frame mode - out_feed_drop_mode_obj = out_feed.find( - 'startTimecode/dropMode') - self.drop_mode = out_feed_drop_mode_obj.text - break - else: - continue - except Exception as msg: - self.log.warning(msg) - - @staticmethod - def write_clip_data_to_file(fpath, xml_data): - log.info(">>> type of xml_data: {}".format(type(xml_data))) - if isinstance(xml_data, ET.ElementTree): - xml_data = xml_data.getroot() - try: - # save it as new file - tree = cET.ElementTree(xml_data) - tree.write( - fpath, xml_declaration=True, - method='xml', encoding='UTF-8' - ) - except IOError as error: - raise IOError( - "Not able to write data to file: {}".format(error)) - - -class OpenClipSolver(MediaInfoFile): - create_new_clip = False - - log = log - - def __init__(self, openclip_file_path, feed_data): - self.out_file = openclip_file_path - - # new feed variables: - feed_path = feed_data.pop("path") - - # initialize parent class - super(OpenClipSolver, self).__init__( - feed_path, - **feed_data - ) - - # get other metadata - self.feed_version_name = feed_data["version"] - self.feed_colorspace = feed_data.get("colorspace") - self.log.info("feed_version_name: {}".format(self.feed_version_name)) - - # derivate other feed variables - self.feed_basename = os.path.basename(feed_path) - self.feed_dir = os.path.dirname(feed_path) - self.feed_ext = os.path.splitext(self.feed_basename)[1][1:].lower() - self.log.info("feed_ext: {}".format(self.feed_ext)) - self.log.info("out_file: {}".format(self.out_file)) - if not self._is_valid_tmp_file(self.out_file): - self.create_new_clip = True - - def _is_valid_tmp_file(self, file): - # check if file exists - if os.path.isfile(file): - # test also if file is not empty - with open(file) as f: - lines = f.readlines() - if len(lines) > 2: - return True - - # file is probably corrupted - os.remove(file) - return False - - def make(self): - - if self.create_new_clip: - # New openClip - self._create_new_open_clip() - else: - self._update_open_clip() - - def _clear_handler(self, xml_object): - for handler in xml_object.findall("./handler"): - self.log.info("Handler found") - xml_object.remove(handler) - - def _create_new_open_clip(self): - self.log.info("Building new openClip") - self.log.info(">> self.clip_data: {}".format(self.clip_data)) - - # clip data comming from MediaInfoFile - tmp_xml_feeds = self.clip_data.find('tracks/track/feeds') - tmp_xml_feeds.set('currentVersion', self.feed_version_name) - for tmp_feed in tmp_xml_feeds: - tmp_feed.set('vuid', self.feed_version_name) - - # add colorspace if any is set - if self.feed_colorspace: - self._add_colorspace(tmp_feed, self.feed_colorspace) - - self._clear_handler(tmp_feed) - - tmp_xml_versions_obj = self.clip_data.find('versions') - tmp_xml_versions_obj.set('currentVersion', self.feed_version_name) - for xml_new_version in tmp_xml_versions_obj: - xml_new_version.set('uid', self.feed_version_name) - xml_new_version.set('type', 'version') - - self._clear_handler(self.clip_data) - self.log.info("Adding feed version: {}".format(self.feed_basename)) - - self.write_clip_data_to_file(self.out_file, self.clip_data) - - def _update_open_clip(self): - self.log.info("Updating openClip ..") - - out_xml = ET.parse(self.out_file) - - self.log.info(">> out_xml: {}".format(out_xml)) - self.log.info(">> self.clip_data: {}".format(self.clip_data)) - - # Get new feed from tmp file - tmp_xml_feed = self.clip_data.find('tracks/track/feeds/feed') - - self._clear_handler(tmp_xml_feed) - - # update fps from MediaInfoFile class - if self.fps: - tmp_feed_fps_obj = tmp_xml_feed.find( - "startTimecode/rate") - tmp_feed_fps_obj.text = str(self.fps) - - # update start_frame from MediaInfoFile class - if self.start_frame: - tmp_feed_nb_ticks_obj = tmp_xml_feed.find( - "startTimecode/nbTicks") - tmp_feed_nb_ticks_obj.text = str(self.start_frame) - - # update drop_mode from MediaInfoFile class - if self.drop_mode: - tmp_feed_drop_mode_obj = tmp_xml_feed.find( - "startTimecode/dropMode") - tmp_feed_drop_mode_obj.text = str(self.drop_mode) - - new_path_obj = tmp_xml_feed.find( - "spans/span/path") - new_path = new_path_obj.text - - feed_added = False - if not self._feed_exists(out_xml, new_path): - tmp_xml_feed.set('vuid', self.feed_version_name) - # Append new temp file feed to .clip source out xml - out_track = out_xml.find("tracks/track") - # add colorspace if any is set - if self.feed_colorspace: - self._add_colorspace(tmp_xml_feed, self.feed_colorspace) - - out_feeds = out_track.find('feeds') - out_feeds.set('currentVersion', self.feed_version_name) - out_feeds.append(tmp_xml_feed) - - self.log.info( - "Appending new feed: {}".format( - self.feed_version_name)) - feed_added = True - - if feed_added: - # Append vUID to versions - out_xml_versions_obj = out_xml.find('versions') - out_xml_versions_obj.set( - 'currentVersion', self.feed_version_name) - new_version_obj = ET.Element( - "version", {"type": "version", "uid": self.feed_version_name}) - out_xml_versions_obj.insert(0, new_version_obj) - - self._clear_handler(out_xml) - - # fist create backup - self._create_openclip_backup_file(self.out_file) - - self.log.info("Adding feed version: {}".format( - self.feed_version_name)) - - self.write_clip_data_to_file(self.out_file, out_xml) - - self.log.info("openClip Updated: {}".format(self.out_file)) - - def _feed_exists(self, xml_data, path): - # loop all available feed paths and check if - # the path is not already in file - for src_path in xml_data.iter('path'): - if path == src_path.text: - self.log.warning( - "Not appending file as it already is in .clip file") - return True - - def _create_openclip_backup_file(self, file): - bck_file = "{}.bak".format(file) - # if backup does not exist - if not os.path.isfile(bck_file): - shutil.copy2(file, bck_file) - else: - # in case it exists and is already multiplied - created = False - for _i in range(1, 99): - bck_file = "{name}.bak.{idx:0>2}".format( - name=file, - idx=_i) - # create numbered backup file - if not os.path.isfile(bck_file): - shutil.copy2(file, bck_file) - created = True - break - # in case numbered does not exists - if not created: - bck_file = "{}.bak.last".format(file) - shutil.copy2(file, bck_file) - - def _add_colorspace(self, feed_obj, profile_name): - feed_storage_obj = feed_obj.find("storageFormat") - feed_clr_obj = feed_storage_obj.find("colourSpace") - if feed_clr_obj is not None: - feed_clr_obj = ET.Element( - "colourSpace", {"type": "string"}) - feed_storage_obj.append(feed_clr_obj) - - feed_clr_obj.text = profile_name diff --git a/openpype/hosts/flame/tests/flame_test.py b/openpype/hosts/flame/tests/flame_test.py deleted file mode 100644 index 402983eeba..0000000000 --- a/openpype/hosts/flame/tests/flame_test.py +++ /dev/null @@ -1,30 +0,0 @@ -from openpype.lib import import_filepath - -plugin = import_filepath( - "/Users/pype.club/code/openpype/openpype/hosts/flame/api/test_plugin.py") - -openclip_file_path = "/Users/pype.club/FLAME_STORAGE/test_shot_fps_float/test.clip" -# feed_datas = [ -# { -# "path": "/Users/pype.club/pype_club_root/OP02_VFX_demo/shots/a/a0000001/publish/plate/plateMain/v007/op02vfx_a0000001_plateMain_v007_exr16fpdwaaCl.0997.exr", -# "version": "v007" -# }, -# { -# "path": "/Users/pype.club/pype_club_root/OP02_VFX_demo/shots/a/a0000001/publish/plate/plateMain/v008/op02vfx_a0000001_plateMain_v008_exr16fpdwaaCl.0997.exr", -# "version": "v008" -# } -# ] - -feed_datas = [ - { - "path": "/Users/pype.club/FLAME_STORAGE/test_shot_fps_float/v001/file_name_v001.1001.exr", - "version": "v001" - }, - { - "path": "/Users/pype.club/FLAME_STORAGE/test_shot_fps_float/v002/file_name_v002.1001.exr", - "version": "v002" - } -] -for feed_data in feed_datas: - oclip = plugin.OpenClipSolver(openclip_file_path, feed_data) - oclip.make() From bb222642fa295e46357f1a6cc364ac34b5f6bce1 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 8 Apr 2022 19:06:32 +0200 Subject: [PATCH 112/126] reversing version rename --- openpype/version.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/openpype/version.py b/openpype/version.py index d447d27172..97aa585ca7 100644 --- a/openpype/version.py +++ b/openpype/version.py @@ -1,10 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring Pype version.""" -__version__ = "3.9.3-nightly.1-upp220408" - - -''' -includes: - - Flame: integrate batch groups: - https://github.com/pypeclub/OpenPype/pull/2928 -''' +__version__ = "3.9.3" From c5683f50d44f99469cc6393c68c2047dd40efd49 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 8 Apr 2022 20:07:53 +0200 Subject: [PATCH 113/126] flame: fixing skip condition --- openpype/hosts/flame/otio/flame_export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/flame/otio/flame_export.py b/openpype/hosts/flame/otio/flame_export.py index 0b9c9ce817..9f0bec62ea 100644 --- a/openpype/hosts/flame/otio/flame_export.py +++ b/openpype/hosts/flame/otio/flame_export.py @@ -558,7 +558,7 @@ def create_otio_timeline(sequence): len(track.segments) == 0 or track.hidden.get_value() ): - return None + continue # convert track to otio otio_track = create_otio_track( From 22918ac0d6c10a9ba827e0d2d8a94fa821e2f80c Mon Sep 17 00:00:00 2001 From: OpenPype Date: Sat, 9 Apr 2022 03:39:12 +0000 Subject: [PATCH 114/126] [Automated] Bump version --- CHANGELOG.md | 38 ++++++++++++++++++++++++-------------- openpype/version.py | 2 +- pyproject.toml | 2 +- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1e7d5d9e0..c216dd0595 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,27 @@ # Changelog +## [3.9.4-nightly.1](https://github.com/pypeclub/OpenPype/tree/HEAD) + +[Full Changelog](https://github.com/pypeclub/OpenPype/compare/3.9.3...HEAD) + +### 📖 Documentation + +- Documentation: Python requirements to 3.7.9 [\#3035](https://github.com/pypeclub/OpenPype/pull/3035) +- Website Docs: Remove unused pages [\#2974](https://github.com/pypeclub/OpenPype/pull/2974) + +**🚀 Enhancements** + +- Resolve environment variable in google drive credential path [\#3008](https://github.com/pypeclub/OpenPype/pull/3008) + +**🐛 Bug fixes** + +- Ftrack: Integrate ftrack api fix [\#3044](https://github.com/pypeclub/OpenPype/pull/3044) +- Webpublisher - removed wrong hardcoded family [\#3043](https://github.com/pypeclub/OpenPype/pull/3043) +- Unreal: Creator import fixes [\#3040](https://github.com/pypeclub/OpenPype/pull/3040) + ## [3.9.3](https://github.com/pypeclub/OpenPype/tree/3.9.3) (2022-04-07) -[Full Changelog](https://github.com/pypeclub/OpenPype/compare/3.9.2...3.9.3) +[Full Changelog](https://github.com/pypeclub/OpenPype/compare/CI/3.9.3-nightly.2...3.9.3) ### 📖 Documentation @@ -20,7 +39,6 @@ - Console Interpreter: Changed how console splitter size are reused on show [\#3016](https://github.com/pypeclub/OpenPype/pull/3016) - Deadline: Use more suitable name for sequence review logic [\#3015](https://github.com/pypeclub/OpenPype/pull/3015) - Nuke: add concurrency attr to deadline job [\#3005](https://github.com/pypeclub/OpenPype/pull/3005) -- Photoshop: create image without instance [\#3001](https://github.com/pypeclub/OpenPype/pull/3001) - Deadline: priority configurable in Maya jobs [\#2995](https://github.com/pypeclub/OpenPype/pull/2995) - Workfiles tool: Save as published workfiles [\#2937](https://github.com/pypeclub/OpenPype/pull/2937) @@ -32,10 +50,12 @@ - Harmony: Added creating subset name for workfile from template [\#3024](https://github.com/pypeclub/OpenPype/pull/3024) - AfterEffects: Added creating subset name for workfile from template [\#3023](https://github.com/pypeclub/OpenPype/pull/3023) - General: Add example addons to ignored [\#3022](https://github.com/pypeclub/OpenPype/pull/3022) +- SiteSync: fix transitive alternate sites, fix dropdown in Local Settings [\#3018](https://github.com/pypeclub/OpenPype/pull/3018) - Maya: Remove missing import [\#3017](https://github.com/pypeclub/OpenPype/pull/3017) - Ftrack: multiple reviewable componets [\#3012](https://github.com/pypeclub/OpenPype/pull/3012) - Tray publisher: Fixes after code movement [\#3010](https://github.com/pypeclub/OpenPype/pull/3010) - Nuke: fixing unicode type detection in effect loaders [\#3002](https://github.com/pypeclub/OpenPype/pull/3002) +- Fix - remove doubled dot in workfile created from template [\#2998](https://github.com/pypeclub/OpenPype/pull/2998) - Nuke: removing redundant Ftrack asset when farm publishing [\#2996](https://github.com/pypeclub/OpenPype/pull/2996) **Merged pull requests:** @@ -51,7 +71,6 @@ - Documentation: Added mention of adding My Drive as a root [\#2999](https://github.com/pypeclub/OpenPype/pull/2999) - Docs: Added MongoDB requirements [\#2951](https://github.com/pypeclub/OpenPype/pull/2951) -- Documentation: New publisher develop docs [\#2896](https://github.com/pypeclub/OpenPype/pull/2896) **🆕 New features** @@ -60,6 +79,7 @@ **🚀 Enhancements** +- Photoshop: create image without instance [\#3001](https://github.com/pypeclub/OpenPype/pull/3001) - TVPaint: Render scene family [\#3000](https://github.com/pypeclub/OpenPype/pull/3000) - Nuke: ReviewDataMov Read RAW attribute [\#2985](https://github.com/pypeclub/OpenPype/pull/2985) - General: `METADATA\_KEYS` constant as `frozenset` for optimal immutable lookup [\#2980](https://github.com/pypeclub/OpenPype/pull/2980) @@ -70,13 +90,11 @@ - TVPaint: Extractor to convert PNG into EXR [\#2942](https://github.com/pypeclub/OpenPype/pull/2942) - Workfiles: Open published workfiles [\#2925](https://github.com/pypeclub/OpenPype/pull/2925) - General: Default modules loaded dynamically [\#2923](https://github.com/pypeclub/OpenPype/pull/2923) -- Nuke: Add no-audio Tag [\#2911](https://github.com/pypeclub/OpenPype/pull/2911) - Nuke: improving readability [\#2903](https://github.com/pypeclub/OpenPype/pull/2903) **🐛 Bug fixes** - Hosts: Remove path existence checks in 'add\_implementation\_envs' [\#3004](https://github.com/pypeclub/OpenPype/pull/3004) -- Fix - remove doubled dot in workfile created from template [\#2998](https://github.com/pypeclub/OpenPype/pull/2998) - PS: fix renaming subset incorrectly in PS [\#2991](https://github.com/pypeclub/OpenPype/pull/2991) - Fix: Disable setuptools auto discovery [\#2990](https://github.com/pypeclub/OpenPype/pull/2990) - AEL: fix opening existing workfile if no scene opened [\#2989](https://github.com/pypeclub/OpenPype/pull/2989) @@ -120,21 +138,13 @@ **🚀 Enhancements** +- Nuke: Add no-audio Tag [\#2911](https://github.com/pypeclub/OpenPype/pull/2911) - General: Change how OPENPYPE\_DEBUG value is handled [\#2907](https://github.com/pypeclub/OpenPype/pull/2907) -- nuke: imageio adding ocio config version 1.2 [\#2897](https://github.com/pypeclub/OpenPype/pull/2897) -- Flame: support for comment with xml attribute overrides [\#2892](https://github.com/pypeclub/OpenPype/pull/2892) **🐛 Bug fixes** - General: Fix use of Anatomy roots [\#2904](https://github.com/pypeclub/OpenPype/pull/2904) - Fixing gap detection in extract review [\#2902](https://github.com/pypeclub/OpenPype/pull/2902) -- Pyblish Pype - ensure current state is correct when entering new group order [\#2899](https://github.com/pypeclub/OpenPype/pull/2899) -- SceneInventory: Fix import of load function [\#2894](https://github.com/pypeclub/OpenPype/pull/2894) -- Harmony - fixed creator issue [\#2891](https://github.com/pypeclub/OpenPype/pull/2891) - -**🔀 Refactored code** - -- General: Reduce style usage to OpenPype repository [\#2889](https://github.com/pypeclub/OpenPype/pull/2889) ## [3.9.0](https://github.com/pypeclub/OpenPype/tree/3.9.0) (2022-03-14) diff --git a/openpype/version.py b/openpype/version.py index 97aa585ca7..08dcbb5aed 100644 --- a/openpype/version.py +++ b/openpype/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring Pype version.""" -__version__ = "3.9.3" +__version__ = "3.9.4-nightly.1" diff --git a/pyproject.toml b/pyproject.toml index 006f6eb4e5..adec7ab158 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "OpenPype" -version = "3.9.3" # OpenPype +version = "3.9.4-nightly.1" # OpenPype description = "Open VFX and Animation pipeline with support." authors = ["OpenPype Team "] license = "MIT License" From 16e84073329b5dbc992dfadfd0818233f794cb0d Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 11 Apr 2022 10:50:34 +0200 Subject: [PATCH 115/126] flame: processing comments --- openpype/hosts/flame/api/plugin.py | 14 ++++++++------ openpype/hosts/flame/api/scripts/wiretap_com.py | 4 ++++ openpype/hosts/flame/otio/flame_export.py | 16 ++++++---------- .../plugins/publish/integrate_batch_group.py | 9 ++------- 4 files changed, 20 insertions(+), 23 deletions(-) diff --git a/openpype/hosts/flame/api/plugin.py b/openpype/hosts/flame/api/plugin.py index 42e6e19931..c87445fdd3 100644 --- a/openpype/hosts/flame/api/plugin.py +++ b/openpype/hosts/flame/api/plugin.py @@ -4,11 +4,12 @@ import shutil from copy import deepcopy from xml.etree import ElementTree as ET +from Qt import QtCore, QtWidgets + import openpype.api as openpype import qargparse from openpype import style from openpype.pipeline import LegacyCreator, LoaderPlugin -from Qt import QtCore, QtWidgets from . import constants from . import lib as flib @@ -712,12 +713,13 @@ class OpenClipSolver(flib.MediaInfoFile): # test also if file is not empty with open(file) as f: lines = f.readlines() - if len(lines) > 2: - return True - # file is probably corrupted - os.remove(file) - return False + if len(lines) > 2: + return True + + # file is probably corrupted + os.remove(file) + return False def make(self): diff --git a/openpype/hosts/flame/api/scripts/wiretap_com.py b/openpype/hosts/flame/api/scripts/wiretap_com.py index f78102c0a1..4825ff4386 100644 --- a/openpype/hosts/flame/api/scripts/wiretap_com.py +++ b/openpype/hosts/flame/api/scripts/wiretap_com.py @@ -461,6 +461,10 @@ class WireTapCom(object): def _subprocess_preexec_fn(): + """ Helper function + + Setting permission mask to 0777 + """ os.setpgrp() os.umask(0o000) diff --git a/openpype/hosts/flame/otio/flame_export.py b/openpype/hosts/flame/otio/flame_export.py index 9f0bec62ea..4fe05ec1d8 100644 --- a/openpype/hosts/flame/otio/flame_export.py +++ b/openpype/hosts/flame/otio/flame_export.py @@ -455,15 +455,10 @@ def add_otio_metadata(otio_item, item, **kwargs): otio_item.metadata.update({key: value}) -def _get_shot_tokens_values(clip, tokens, from_clip=False): +def _get_shot_tokens_values(clip, tokens): old_value = None output = {} - # in case it is segment from reel clip - # avoiding duplicity of segement data - if from_clip: - return {} - old_value = clip.shot_name.get_value() for token in tokens: @@ -480,7 +475,7 @@ def _get_shot_tokens_values(clip, tokens, from_clip=False): return output -def _get_segment_attributes(segment, from_clip=False): +def _get_segment_attributes(segment): log.debug("Segment name|hidden: {}|{}".format( segment.name.get_value(), segment.hidden @@ -503,9 +498,10 @@ def _get_segment_attributes(segment, from_clip=False): } # add all available shot tokens - shot_tokens = _get_shot_tokens_values(segment, [ - "", "", "", "", - ], from_clip) + shot_tokens = _get_shot_tokens_values( + segment, + ["", "", "", ""] + ) clip_data.update(shot_tokens) # populate shot source metadata diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py index cac99a25ac..3615f06a3d 100644 --- a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -299,10 +299,8 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): version_name = "v" version_padding = 3 - # return it as ordered dict - reutrn_dict = OrderedDict() # need to make sure the order of keys is correct - for item in ( + return OrderedDict( ("name", name), ("media_path", media_path), ("media_path_pattern", media_path_pattern), @@ -320,10 +318,7 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): ("version_mode", version_mode), ("version_name", version_name), ("version_padding", version_padding) - ): - reutrn_dict.update({item[0]: item[1]}) - - return reutrn_dict + ) def _get_shot_task_dir_path(self, instance, task_data): project_doc = instance.data["projectEntity"] From e8ffb06434b26b6f9245362dd71a457dcab828e8 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 11 Apr 2022 11:42:27 +0200 Subject: [PATCH 116/126] Fix publishing tvp workfile in webpublisher --- .../publish/collect_published_files.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py index 56b2ef6e20..8edaf4f67b 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -108,15 +108,18 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): instance.data["representations"] = self._get_single_repre( task_dir, task_data["files"], tags ) - file_url = os.path.join(task_dir, task_data["files"][0]) - no_of_frames = self._get_number_of_frames(file_url) - if no_of_frames: + if family != 'workfile': + file_url = os.path.join(task_dir, task_data["files"][0]) try: - frame_end = int(frame_start) + math.ceil(no_of_frames) - instance.data["frameEnd"] = math.ceil(frame_end) - 1 - self.log.debug("frameEnd:: {}".format( - instance.data["frameEnd"])) - except ValueError: + no_of_frames = self._get_number_of_frames(file_url) + if no_of_frames: + frame_end = int(frame_start) + \ + math.ceil(no_of_frames) + frame_end = math.ceil(frame_end) - 1 + instance.data["frameEnd"] = frame_end + self.log.debug("frameEnd:: {}".format( + instance.data["frameEnd"])) + except Exception: self.log.warning("Unable to count frames " "duration {}".format(no_of_frames)) From a198b7f3f6ef04f48fe43165f7ea17d32134876d Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 11 Apr 2022 12:02:08 +0200 Subject: [PATCH 117/126] add format arguments to concatenation arguments --- .../plugins/publish/extract_review_slate.py | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/openpype/plugins/publish/extract_review_slate.py b/openpype/plugins/publish/extract_review_slate.py index 505ae75169..49f0eac41d 100644 --- a/openpype/plugins/publish/extract_review_slate.py +++ b/openpype/plugins/publish/extract_review_slate.py @@ -158,13 +158,15 @@ class ExtractReviewSlate(openpype.api.Extractor): ]) if use_legacy_code: + format_args = [] codec_args = repre["_profile"].get('codec', []) output_args.extend(codec_args) # preset's output data output_args.extend(repre["_profile"].get('output', [])) else: # Codecs are copied from source for whole input - codec_args = self._get_codec_args(repre) + format_args, codec_args = self._get_format_codec_args(repre) + output_args.extend(format_args) output_args.extend(codec_args) # make sure colors are correct @@ -266,8 +268,14 @@ class ExtractReviewSlate(openpype.api.Extractor): "-safe", "0", "-i", conc_text_path, "-c", "copy", - output_path ] + # NOTE: Added because of OP Atom demuxers + # Add format arguments if there are any + # - keep format of output + if format_args: + concat_args.extend(format_args) + # Add final output path + concat_args.append(output_path) # ffmpeg concat subprocess self.log.debug( @@ -338,7 +346,7 @@ class ExtractReviewSlate(openpype.api.Extractor): return vf_back - def _get_codec_args(self, repre): + def _get_format_codec_args(self, repre): """Detect possible codec arguments from representation.""" codec_args = [] @@ -361,13 +369,9 @@ class ExtractReviewSlate(openpype.api.Extractor): return codec_args source_ffmpeg_cmd = repre.get("ffmpeg_cmd") - codec_args.extend( - get_ffmpeg_format_args(ffprobe_data, source_ffmpeg_cmd) - ) - codec_args.extend( - get_ffmpeg_codec_args( - ffprobe_data, source_ffmpeg_cmd, logger=self.log - ) + format_args = get_ffmpeg_format_args(ffprobe_data, source_ffmpeg_cmd) + codec_args = get_ffmpeg_codec_args( + ffprobe_data, source_ffmpeg_cmd, logger=self.log ) - return codec_args + return format_args, codec_args From aefaac6e5cd4530185656077567e38586e0586b4 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 11 Apr 2022 14:49:58 +0200 Subject: [PATCH 118/126] added ability to change general environments --- openpype/settings/lib.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/openpype/settings/lib.py b/openpype/settings/lib.py index 54502292dc..937329b417 100644 --- a/openpype/settings/lib.py +++ b/openpype/settings/lib.py @@ -1113,6 +1113,14 @@ def get_general_environments(): clear_metadata_from_settings(environments) + whitelist_envs = result["general"].get("local_env_white_list") + if whitelist_envs: + local_settings = get_local_settings() + local_envs = local_settings.get("environments") or {} + for key, value in local_envs.items(): + if key in whitelist_envs and key in environments: + environments[key] = value + return environments From 7a40cc269239e522155692c4f4899f8bcc9f6cc4 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 11 Apr 2022 17:19:36 +0200 Subject: [PATCH 119/126] fix orderdict --- openpype/hosts/flame/plugins/publish/integrate_batch_group.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py index 3615f06a3d..da9553cc2a 100644 --- a/openpype/hosts/flame/plugins/publish/integrate_batch_group.py +++ b/openpype/hosts/flame/plugins/publish/integrate_batch_group.py @@ -300,7 +300,7 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): version_padding = 3 # need to make sure the order of keys is correct - return OrderedDict( + return OrderedDict(( ("name", name), ("media_path", media_path), ("media_path_pattern", media_path_pattern), @@ -318,7 +318,7 @@ class IntegrateBatchGroup(pyblish.api.InstancePlugin): ("version_mode", version_mode), ("version_name", version_name), ("version_padding", version_padding) - ) + )) def _get_shot_task_dir_path(self, instance, task_data): project_doc = instance.data["projectEntity"] From 10ce2511f0be2a5627de0f1527e6128e66304dc1 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 11 Apr 2022 17:40:44 +0200 Subject: [PATCH 120/126] make sure temporary staging dir is removed --- openpype/plugins/publish/extract_review.py | 238 +++++++++++---------- 1 file changed, 127 insertions(+), 111 deletions(-) diff --git a/openpype/plugins/publish/extract_review.py b/openpype/plugins/publish/extract_review.py index 3ecea1f8bd..d569d82762 100644 --- a/openpype/plugins/publish/extract_review.py +++ b/openpype/plugins/publish/extract_review.py @@ -188,8 +188,7 @@ class ExtractReview(pyblish.api.InstancePlugin): outputs_per_repres = self._get_outputs_per_representations( instance, profile_outputs ) - fill_data = copy.deepcopy(instance.data["anatomyData"]) - for repre, outputs in outputs_per_repres: + for repre, outpu_defs in outputs_per_repres: # Check if input should be preconverted before processing # Store original staging dir (it's value may change) src_repre_staging_dir = repre["stagingDir"] @@ -241,126 +240,143 @@ class ExtractReview(pyblish.api.InstancePlugin): self.log ) - for _output_def in outputs: - output_def = copy.deepcopy(_output_def) - # Make sure output definition has "tags" key - if "tags" not in output_def: - output_def["tags"] = [] - - if "burnins" not in output_def: - output_def["burnins"] = [] - - # Create copy of representation - new_repre = copy.deepcopy(repre) - # Make sure new representation has origin staging dir - # - this is because source representation may change - # it's staging dir because of ffmpeg conversion - new_repre["stagingDir"] = src_repre_staging_dir - - # Remove "delete" tag from new repre if there is - if "delete" in new_repre["tags"]: - new_repre["tags"].remove("delete") - - # Add additional tags from output definition to representation - for tag in output_def["tags"]: - if tag not in new_repre["tags"]: - new_repre["tags"].append(tag) - - # Add burnin link from output definition to representation - for burnin in output_def["burnins"]: - if burnin not in new_repre.get("burnins", []): - if not new_repre.get("burnins"): - new_repre["burnins"] = [] - new_repre["burnins"].append(str(burnin)) - - self.log.debug( - "Linked burnins: `{}`".format(new_repre.get("burnins")) + try: + self._render_output_definitions( + instance, repre, src_repre_staging_dir, outpu_defs ) - self.log.debug( - "New representation tags: `{}`".format( - new_repre.get("tags")) + finally: + # Make sure temporary staging is cleaned up and representation + # has set origin stagingDir + if do_convert: + # Set staging dir of source representation back to previous + # value + repre["stagingDir"] = src_repre_staging_dir + if os.path.exists(new_staging_dir): + shutil.rmtree(new_staging_dir) + + def _render_output_definitions( + self, instance, repre, src_repre_staging_dir, outpu_defs + ): + fill_data = copy.deepcopy(instance.data["anatomyData"]) + for _output_def in outpu_defs: + output_def = copy.deepcopy(_output_def) + # Make sure output definition has "tags" key + if "tags" not in output_def: + output_def["tags"] = [] + + if "burnins" not in output_def: + output_def["burnins"] = [] + + # Create copy of representation + new_repre = copy.deepcopy(repre) + # Make sure new representation has origin staging dir + # - this is because source representation may change + # it's staging dir because of ffmpeg conversion + new_repre["stagingDir"] = src_repre_staging_dir + + # Remove "delete" tag from new repre if there is + if "delete" in new_repre["tags"]: + new_repre["tags"].remove("delete") + + # Add additional tags from output definition to representation + for tag in output_def["tags"]: + if tag not in new_repre["tags"]: + new_repre["tags"].append(tag) + + # Add burnin link from output definition to representation + for burnin in output_def["burnins"]: + if burnin not in new_repre.get("burnins", []): + if not new_repre.get("burnins"): + new_repre["burnins"] = [] + new_repre["burnins"].append(str(burnin)) + + self.log.debug( + "Linked burnins: `{}`".format(new_repre.get("burnins")) + ) + + self.log.debug( + "New representation tags: `{}`".format( + new_repre.get("tags")) + ) + + temp_data = self.prepare_temp_data(instance, repre, output_def) + files_to_clean = [] + if temp_data["input_is_sequence"]: + self.log.info("Filling gaps in sequence.") + files_to_clean = self.fill_sequence_gaps( + temp_data["origin_repre"]["files"], + new_repre["stagingDir"], + temp_data["frame_start"], + temp_data["frame_end"]) + + # create or update outputName + output_name = new_repre.get("outputName", "") + output_ext = new_repre["ext"] + if output_name: + output_name += "_" + output_name += output_def["filename_suffix"] + if temp_data["without_handles"]: + output_name += "_noHandles" + + # add outputName to anatomy format fill_data + fill_data.update({ + "output": output_name, + "ext": output_ext + }) + + try: # temporary until oiiotool is supported cross platform + ffmpeg_args = self._ffmpeg_arguments( + output_def, instance, new_repre, temp_data, fill_data ) - - temp_data = self.prepare_temp_data( - instance, repre, output_def) - files_to_clean = [] - if temp_data["input_is_sequence"]: - self.log.info("Filling gaps in sequence.") - files_to_clean = self.fill_sequence_gaps( - temp_data["origin_repre"]["files"], - new_repre["stagingDir"], - temp_data["frame_start"], - temp_data["frame_end"]) - - # create or update outputName - output_name = new_repre.get("outputName", "") - output_ext = new_repre["ext"] - if output_name: - output_name += "_" - output_name += output_def["filename_suffix"] - if temp_data["without_handles"]: - output_name += "_noHandles" - - # add outputName to anatomy format fill_data - fill_data.update({ - "output": output_name, - "ext": output_ext - }) - - try: # temporary until oiiotool is supported cross platform - ffmpeg_args = self._ffmpeg_arguments( - output_def, instance, new_repre, temp_data, fill_data + except ZeroDivisionError: + # TODO recalculate width and height using OIIO before + # conversion + if 'exr' in temp_data["origin_repre"]["ext"]: + self.log.warning( + ( + "Unsupported compression on input files." + " Skipping!!!" + ), + exc_info=True ) - except ZeroDivisionError: - if 'exr' in temp_data["origin_repre"]["ext"]: - self.log.debug("Unsupported compression on input " + - "files. Skipping!!!") - return - raise NotImplementedError + return + raise NotImplementedError - subprcs_cmd = " ".join(ffmpeg_args) + subprcs_cmd = " ".join(ffmpeg_args) - # run subprocess - self.log.debug("Executing: {}".format(subprcs_cmd)) + # run subprocess + self.log.debug("Executing: {}".format(subprcs_cmd)) - openpype.api.run_subprocess( - subprcs_cmd, shell=True, logger=self.log - ) + openpype.api.run_subprocess( + subprcs_cmd, shell=True, logger=self.log + ) - # delete files added to fill gaps - if files_to_clean: - for f in files_to_clean: - os.unlink(f) + # delete files added to fill gaps + if files_to_clean: + for f in files_to_clean: + os.unlink(f) - new_repre.update({ - "name": "{}_{}".format(output_name, output_ext), - "outputName": output_name, - "outputDef": output_def, - "frameStartFtrack": temp_data["output_frame_start"], - "frameEndFtrack": temp_data["output_frame_end"], - "ffmpeg_cmd": subprcs_cmd - }) + new_repre.update({ + "name": "{}_{}".format(output_name, output_ext), + "outputName": output_name, + "outputDef": output_def, + "frameStartFtrack": temp_data["output_frame_start"], + "frameEndFtrack": temp_data["output_frame_end"], + "ffmpeg_cmd": subprcs_cmd + }) - # Force to pop these key if are in new repre - new_repre.pop("preview", None) - new_repre.pop("thumbnail", None) - if "clean_name" in new_repre.get("tags", []): - new_repre.pop("outputName") + # Force to pop these key if are in new repre + new_repre.pop("preview", None) + new_repre.pop("thumbnail", None) + if "clean_name" in new_repre.get("tags", []): + new_repre.pop("outputName") - # adding representation - self.log.debug( - "Adding new representation: {}".format(new_repre) - ) - instance.data["representations"].append(new_repre) - - # Cleanup temp staging dir after procesisng of output definitions - if do_convert: - temp_dir = repre["stagingDir"] - shutil.rmtree(temp_dir) - # Set staging dir of source representation back to previous - # value - repre["stagingDir"] = src_repre_staging_dir + # adding representation + self.log.debug( + "Adding new representation: {}".format(new_repre) + ) + instance.data["representations"].append(new_repre) def input_is_sequence(self, repre): """Deduce from representation data if input is sequence.""" From 1bd174c37557dbe06c3c65d1cc621491bf7b2947 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 11 Apr 2022 17:50:27 +0200 Subject: [PATCH 121/126] simplified how to avoid adding attributes during oiio procesing --- openpype/lib/transcoding.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/openpype/lib/transcoding.py b/openpype/lib/transcoding.py index 8e79aba0ae..448c9eefe0 100644 --- a/openpype/lib/transcoding.py +++ b/openpype/lib/transcoding.py @@ -422,7 +422,12 @@ def convert_for_ffmpeg( compression = "none" # Prepare subprocess arguments - oiio_cmd = [get_oiio_tools_path()] + oiio_cmd = [ + get_oiio_tools_path(), + + # Don't add any additional attributes + "--nosoftwareattrib", + ] # Add input compression if available if compression: oiio_cmd.extend(["--compression", compression]) @@ -458,7 +463,6 @@ def convert_for_ffmpeg( "--frames", "{}-{}".format(input_frame_start, input_frame_end) ]) - ignore_attr_changes_added = False for attr_name, attr_value in input_info["attribs"].items(): if not isinstance(attr_value, str): continue @@ -466,10 +470,6 @@ def convert_for_ffmpeg( # Remove attributes that have string value longer than allowed length # for ffmpeg if len(attr_value) > MAX_FFMPEG_STRING_LEN: - if not ignore_attr_changes_added: - # Attrite changes won't be added to attributes itself - ignore_attr_changes_added = True - oiio_cmd.append("--sansattrib") # Set attribute to empty string logger.info(( "Removed attribute \"{}\" from metadata" From b40e5ba0b8bb697e15ff14e0f50ef45da61759c6 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 11 Apr 2022 17:50:53 +0200 Subject: [PATCH 122/126] added check for invalid characters in attribute value --- openpype/lib/transcoding.py | 38 +++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/openpype/lib/transcoding.py b/openpype/lib/transcoding.py index 448c9eefe0..c2fecf6628 100644 --- a/openpype/lib/transcoding.py +++ b/openpype/lib/transcoding.py @@ -17,6 +17,9 @@ from .vendor_bin_utils import ( # Max length of string that is supported by ffmpeg MAX_FFMPEG_STRING_LEN = 8196 +# Not allowed symbols in attributes for ffmpeg +NOT_ALLOWED_FFMPEG_CHARS = ("\"", ) + # OIIO known xml tags STRING_TAGS = { "format" @@ -367,11 +370,15 @@ def should_convert_for_ffmpeg(src_filepath): return None for attr_value in input_info["attribs"].values(): - if ( - isinstance(attr_value, str) - and len(attr_value) > MAX_FFMPEG_STRING_LEN - ): + if not isinstance(attr_value, str): + continue + + if len(attr_value) > MAX_FFMPEG_STRING_LEN: return True + + for char in NOT_ALLOWED_FFMPEG_CHARS: + if char in attr_value: + return True return False @@ -468,13 +475,28 @@ def convert_for_ffmpeg( continue # Remove attributes that have string value longer than allowed length - # for ffmpeg + # for ffmpeg or when containt unallowed symbols + erase_reason = "Missing reason" + erase_attribute = False if len(attr_value) > MAX_FFMPEG_STRING_LEN: + erase_reason = "has too long value ({} chars).".format( + len(attr_value) + ) + + if erase_attribute: + for char in NOT_ALLOWED_FFMPEG_CHARS: + if char in attr_value: + erase_attribute = True + erase_reason = ( + "contains unsupported character \"{}\"." + ).format(char) + break + + if erase_attribute: # Set attribute to empty string logger.info(( - "Removed attribute \"{}\" from metadata" - " because has too long value ({} chars)." - ).format(attr_name, len(attr_value))) + "Removed attribute \"{}\" from metadata because {}." + ).format(attr_name, erase_reason)) oiio_cmd.extend(["--eraseattrib", attr_name]) # Add last argument - path to output From 1f5be56ae2a2e930b2451fd91e120194b14d8d18 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 12 Apr 2022 14:39:56 +0200 Subject: [PATCH 123/126] added more logs --- .../hosts/nuke/plugins/publish/extract_slate_frame.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/nuke/plugins/publish/extract_slate_frame.py b/openpype/hosts/nuke/plugins/publish/extract_slate_frame.py index 6935afe144..fb52fc18b4 100644 --- a/openpype/hosts/nuke/plugins/publish/extract_slate_frame.py +++ b/openpype/hosts/nuke/plugins/publish/extract_slate_frame.py @@ -189,6 +189,7 @@ class ExtractSlateFrame(openpype.api.Extractor): for key, value in self.key_value_mapping.items(): enabled, template = value if not enabled: + self.log.debug("Key \"{}\" is disabled".format(key)) continue try: @@ -205,13 +206,19 @@ class ExtractSlateFrame(openpype.api.Extractor): except KeyError: self.log.warning( - "Template contains unknown key", + ( + "Template contains unknown key." + " Template \"{}\" Data: {}" + ).format(template, fill_data), exc_info=True ) continue try: node[key].setValue(value) + self.log.info("Change key \"{}\" to value \"{}\"".format( + key, value + )) except NameError: self.log.warning(( "Failed to set value \"{}\" on node attribute \"{}\"" From 786a6494eb32e9bbe0dd448482067647d6fddd12 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 12 Apr 2022 14:40:08 +0200 Subject: [PATCH 124/126] fixed settings --- .../defaults/project_settings/nuke.json | 26 ++++---- .../schemas/schema_nuke_publish.json | 66 ++++++++++--------- 2 files changed, 50 insertions(+), 42 deletions(-) diff --git a/openpype/settings/defaults/project_settings/nuke.json b/openpype/settings/defaults/project_settings/nuke.json index bdccb9b38e..ab015271ff 100644 --- a/openpype/settings/defaults/project_settings/nuke.json +++ b/openpype/settings/defaults/project_settings/nuke.json @@ -161,18 +161,20 @@ }, "ExtractSlateFrame": { "viewer_lut_raw": false, - "f_submission_note": [ - true, - "{comment}" - ], - "f_submitting_for": [ - true, - "{intent[value]}" - ], - "f_vfx_scope_of_work": [ - false, - "" - ] + "key_value_mapping": { + "f_submission_note": [ + true, + "{comment}" + ], + "f_submitting_for": [ + true, + "{intent[value]}" + ], + "f_vfx_scope_of_work": [ + false, + "" + ] + } }, "IncrementScriptVersion": { "enabled": true, diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_nuke_publish.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_nuke_publish.json index 3bf0eb3214..4a796f1933 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_nuke_publish.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_nuke_publish.json @@ -399,41 +399,47 @@ "word_wrap": true }, { - "type": "list-strict", - "key": "f_submission_note", - "label": "Submission Note:", - "object_types": [ + "type": "dict", + "key": "key_value_mapping", + "children": [ { - "type": "boolean" + "type": "list-strict", + "key": "f_submission_note", + "label": "Submission Note:", + "object_types": [ + { + "type": "boolean" + }, + { + "type": "text" + } + ] }, { - "type": "text" - } - ] - }, - { - "type": "list-strict", - "key": "f_submitting_for", - "label": "Submission For:", - "object_types": [ - { - "type": "boolean" + "type": "list-strict", + "key": "f_submitting_for", + "label": "Submission For:", + "object_types": [ + { + "type": "boolean" + }, + { + "type": "text" + } + ] }, { - "type": "text" - } - ] - }, - { - "type": "list-strict", - "key": "f_vfx_scope_of_work", - "label": "VFX Scope Of Work:", - "object_types": [ - { - "type": "boolean" - }, - { - "type": "text" + "type": "list-strict", + "key": "f_vfx_scope_of_work", + "label": "VFX Scope Of Work:", + "object_types": [ + { + "type": "boolean" + }, + { + "type": "text" + } + ] } ] } From f49d01559a25e084236a5b5ed936eb16a9006545 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 12 Apr 2022 14:40:27 +0200 Subject: [PATCH 125/126] fix database access in loader --- openpype/tools/utils/lib.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openpype/tools/utils/lib.py b/openpype/tools/utils/lib.py index efaf671915..5abbe01144 100644 --- a/openpype/tools/utils/lib.py +++ b/openpype/tools/utils/lib.py @@ -426,7 +426,10 @@ class FamilyConfigCache: # Make sure connection is installed # - accessing attribute which does not have auto-install self.dbcon.install() - asset_doc = self.dbcon.database[project_name].find_one( + database = getattr(self.dbcon, "database", None) + if database is None: + database = self.dbcon._database + asset_doc = database[project_name].find_one( {"type": "asset", "name": asset_name}, {"data.tasks": True} ) or {} From 0edeb4c6dc174c251e501f45e02ff37e6f95dbfc Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 12 Apr 2022 15:42:45 +0200 Subject: [PATCH 126/126] added init file for worker which triggers missing sound file dialog on worker open --- openpype/hosts/tvpaint/worker/init_file.tvpp | Bin 0 -> 59333 bytes openpype/hosts/tvpaint/worker/worker.py | 23 +++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 openpype/hosts/tvpaint/worker/init_file.tvpp diff --git a/openpype/hosts/tvpaint/worker/init_file.tvpp b/openpype/hosts/tvpaint/worker/init_file.tvpp new file mode 100644 index 0000000000000000000000000000000000000000..572d278fdb49c619d3cf040f59238377d1062d6a GIT binary patch literal 59333 zcmeI5U2Gjk6~`xy#|eL$!}Lal@Z5(1%?N=Qf`0dGjWgWwfZR9<*R;v@XejPJcO&bn2| z8yq`(_sp3yXU_a)c5Ub7@4tNO;a@y`;`g`T6~~`{_lG|c>gz8*_HSR9ApbjGG5w>D zq5E^*zm%SvOE= zj{5anGivQOF<%NM+B<^Z-b9@_?ipGRA|*agsrxwA@T7PE8CTSf z&Y`L*R{IGtjXGju+>Ekg@hIk^`8+X!dw@?OM_IL9hli?pSj=MtRWprMMT$ZU=K36b zT$Tp1(4J5o`UKHQ8sB&39@^Kt{{)`c*MHENIeu3h&GvEU_VInA?)?-$x-SFm7VT4n zeWuSYKI!z(iP|5l&)ZE<_i;Ap)C`}2NA~4iL9TV|=H%X<(sT0ZkBJ$#vM#FAS;9Da z4lB6y_owbD3!Q$QLk=>yIQ#RczbS5@!j^i5kK*ih;G;JNo%FuH1V2&fiko{Y7UGjQ zxsljbtLtGro$?v9=q-{eCMj~zyMmrU*NDFeFERV*_+x|oXAw{5nON_rJvoXe&h(8T z|DrgF`P#TT>M5ot`Xru@lj4kc4sVcYtU-bx`9Atd>OTVxPW?qZ(F^MR<719-?0OCN zDz1upe~pQAh*?BEV~7io+XYX0H~FZUp7;)~6racO7VKix9b~1I+yJc}?!!04QRMV6 zM+4tAb@_Oj_m484YMf918w-D?pB1m2Gk;cGd%=I>trGcPIb!a?KLk^=p z5B1tVM;=gbqYNfC(@GCcp%k025#W zOn?b60Vco%m;e)C0!)AjFaajO1egF5U;<2l2`~XBzyz286JP>NfC(@GCcp%k025#W zOn?b60Vco%m;e)C0!)AjFaajO1egF5U;<2l2`~XBzyz286JP>NfC(@GCcp%k025#W zOn?b60Vco%m;e)C0!)AjFaajO1egF5U;<3wz6p$6Oin%Z_KDxUebUqg5NH4P^Iu-+ zX1%m^wH&2smPdJ#rQ=O_^3W}}l5Vq|b>kN7&0ZekmUp7&jeh@j*6wY_p)8mEH_Lw| z-GR6NZRUUQ*OdQyFS*{ydRte^&8(euTsN$Fj{fIWDtk^DdV%XyD)evY^+MQ=lobb^ zL$|Npb>Hzk!ySaOfiM?&YKFTK);&koU8mMCyw$KKmDiJI>RMP099cnT%Uutvt`n&7 zMvg`(E3Q+m+QHIA9nz_x@S2fY%24{!@fwcw>%$~cuhDhPP(M^H%NM|>y#o-k64Gj= zx{Q?7z2O9J7+h1-$Y)Mf52aUg=(3}+wxMsB#d8N)_WJb(dRX0FyzSrH!r>fy@qY2V z9yW2XB%@AGFO`<7uKvOCs)Ks^^{3Hhj6*gPuSdOhURF&RPHMvor?laR)7l8a8EsU; z)7q$pv)ZVIXS7id&uXI)&M5d`=tH@cHsq)a~BPIxQ@IXiLNP4BI#C zz_2TZT{Y~QVb=}2F|+|?FfUH~&^G3k#=O#)7e{#*Z_F!=d8IM0H0A}e!C0Kbp>53T z8S~sd+iY32cvdYyZbn+Q;BhwlsFv-do2(`6B;Nsg zX*Tg{F`J&<#EZo2#rwl-R_tcgZr1E3-i78^yaJ6Tz1Gd9T@_yAR=h}COXMekZ;Mc5OoNmZ5oe|+Zy`oaEs&-}qY0)-;}VggLy z|CNCMihN(a7gF=RkiX@>_Q!9N|64yX{SYq`5KNfC(@GCcp%k025#WOn?b60Vco%m;e)C0!)AjFaajO1egF5U;<2l z2`~XBzyz286JP>NfC(@GCcp%k025#WOn?b60Vco%m;e)C0!)AjFaajO1egF5U;<2l z2`~XBzyz286JP>NfC(@GCcp%k025#WOn?b60Vco%m;e*Fb7$tv)I78mN)PTH{<}D| zyF0UVn%a-y8WD$~pzr{$L%2?e(#-tgG{wSq2zN1c=KPF`gKu(qk`x_E;*qJR>6%`i zU*79O+>#`lc*7=biIERK{P51?!oni#FF=(bO48iPw&Jwww6d(zb@I&F$$A|p*^I6y zsg}8O#H^kLayD0gH9I#?vzKrk!FA~T>yhWVvA-77eYac| z7iQFs2AALX=I&2_d+ylF?>%w*^Jo6>-Mu0{^ZwLJUwZblr~mcUm*)N{*S}WV^J8y> zSXrH>y?9K?73NSNBrQR$QCOaxo;?GMPf%afYbDu#A!P{Nqk)v6PV$snm_{iWCaqi0 zs~U#8MSePRMgD{8q(fH}Py12ir(;*(SLd$Cf53X7U|5{zfb~MtusHsJ z^+M&aIR1e3LhZ9S{($wW6IifbD1sK|QD?9y{($wWQ&<##zMCtk)}rm}(i+;V zuA$B98rrO`q0Ql&J{IQ;{SSJtA{iner6hx)d3Jclhy{}oWp zK1=^qy;cU}Sc}$_OvORAnzp_wjBsd86Cgg!VaP+Fks%T8A(@#MvcGH*h@Phh%&mGV2g{hu4J=d&6@o>YvhHN4Inu z4F{U=62SoipyM7u0zJSm(tmek6@xuWB%ONQsYAFOTJz;*Cyw%XZ71B?*@~O(-kP&I zKejY2A!AQHNtUkll6EUxZ01Uv-u2|5r-hNI+4CmcU_wHY9-s9IBVZ%z<{|#`!kJ9* zYYipC^v)?M$16_`vSi)x(%eEd7&x}$P8Xqf54