diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index f484016bfe..f59eb1edb1 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -35,6 +35,8 @@ body: label: Version description: What version are you running? Look to OpenPype Tray options: + - 3.17.7-nightly.3 + - 3.17.7-nightly.2 - 3.17.7-nightly.1 - 3.17.6 - 3.17.6-nightly.3 @@ -133,8 +135,6 @@ body: - 3.15.2-nightly.6 - 3.15.2-nightly.5 - 3.15.2-nightly.4 - - 3.15.2-nightly.3 - - 3.15.2-nightly.2 validations: required: true - type: dropdown diff --git a/openpype/client/__init__.py b/openpype/client/__init__.py index fe6dc97877..ba36d940e3 100644 --- a/openpype/client/__init__.py +++ b/openpype/client/__init__.py @@ -44,6 +44,8 @@ from .entities import ( get_thumbnail_id_from_source, get_workfile_info, + + get_asset_name_identifier, ) from .entity_links import ( @@ -108,4 +110,6 @@ __all__ = ( "get_linked_representation_id", "create_project", + + "get_asset_name_identifier", ) diff --git a/openpype/client/entities.py b/openpype/client/entities.py index 5d9654c611..cbaa943743 100644 --- a/openpype/client/entities.py +++ b/openpype/client/entities.py @@ -4,3 +4,22 @@ if not AYON_SERVER_ENABLED: from .mongo.entities import * else: from .server.entities import * + + +def get_asset_name_identifier(asset_doc): + """Get asset name identifier by asset document. + + This function is added because of AYON implementation where name + identifier is not just a name but full path. + + Asset document must have "name" key, and "data.parents" when in AYON mode. + + Args: + asset_doc (dict[str, Any]): Asset document. + """ + + if not AYON_SERVER_ENABLED: + return asset_doc["name"] + parents = list(asset_doc["data"]["parents"]) + parents.append(asset_doc["name"]) + return "/" + "/".join(parents) diff --git a/openpype/client/server/entities.py b/openpype/client/server/entities.py index b41727a797..75e58703be 100644 --- a/openpype/client/server/entities.py +++ b/openpype/client/server/entities.py @@ -182,6 +182,19 @@ def get_asset_by_name(project_name, asset_name, fields=None): return None +def _folders_query(project_name, con, fields, **kwargs): + if fields is None or "tasks" in fields: + folders = get_folders_with_tasks( + con, project_name, fields=fields, **kwargs + ) + + else: + folders = con.get_folders(project_name, fields=fields, **kwargs) + + for folder in folders: + yield folder + + def get_assets( project_name, asset_ids=None, @@ -201,20 +214,39 @@ def get_assets( fields = folder_fields_v3_to_v4(fields, con) kwargs = dict( folder_ids=asset_ids, - folder_names=asset_names, parent_ids=parent_ids, active=active, - fields=fields ) + if not asset_names: + for folder in _folders_query(project_name, con, fields, **kwargs): + yield convert_v4_folder_to_v3(folder, project_name) + return - if fields is None or "tasks" in fields: - folders = get_folders_with_tasks(con, project_name, **kwargs) + new_asset_names = set() + folder_paths = set() + for name in asset_names: + if "/" in name: + folder_paths.add(name) + else: + new_asset_names.add(name) - else: - folders = con.get_folders(project_name, **kwargs) + yielded_ids = set() + if folder_paths: + for folder in _folders_query( + project_name, con, fields, folder_paths=folder_paths, **kwargs + ): + yielded_ids.add(folder["id"]) + yield convert_v4_folder_to_v3(folder, project_name) - for folder in folders: - yield convert_v4_folder_to_v3(folder, project_name) + if not new_asset_names: + return + + for folder in _folders_query( + project_name, con, fields, folder_names=new_asset_names, **kwargs + ): + if folder["id"] not in yielded_ids: + yielded_ids.add(folder["id"]) + yield convert_v4_folder_to_v3(folder, project_name) def get_archived_assets( diff --git a/openpype/host/host.py b/openpype/host/host.py index 630fb873a8..afe06d1f55 100644 --- a/openpype/host/host.py +++ b/openpype/host/host.py @@ -170,7 +170,7 @@ class HostBase(object): if project_name: items.append(project_name) if asset_name: - items.append(asset_name) + items.append(asset_name.lstrip("/")) if task_name: items.append(task_name) if items: diff --git a/openpype/hosts/aftereffects/plugins/create/workfile_creator.py b/openpype/hosts/aftereffects/plugins/create/workfile_creator.py index 2e7b9d4a7e..5dc3d6592d 100644 --- a/openpype/hosts/aftereffects/plugins/create/workfile_creator.py +++ b/openpype/hosts/aftereffects/plugins/create/workfile_creator.py @@ -1,3 +1,4 @@ +from openpype import AYON_SERVER_ENABLED import openpype.hosts.aftereffects.api as api from openpype.client import get_asset_by_name from openpype.pipeline import ( @@ -43,6 +44,14 @@ class AEWorkfileCreator(AutoCreator): task_name = context.get_current_task_name() host_name = context.host_name + existing_asset_name = None + if existing_instance is not None: + if AYON_SERVER_ENABLED: + existing_asset_name = existing_instance.get("folderPath") + + if existing_asset_name is None: + existing_asset_name = existing_instance["asset"] + if existing_instance is None: asset_doc = get_asset_by_name(project_name, asset_name) subset_name = self.get_subset_name( @@ -50,10 +59,13 @@ class AEWorkfileCreator(AutoCreator): project_name, host_name ) data = { - "asset": asset_name, "task": task_name, "variant": self.default_variant } + if AYON_SERVER_ENABLED: + data["folderPath"] = asset_name + else: + data["asset"] = asset_name data.update(self.get_dynamic_data( self.default_variant, task_name, asset_doc, project_name, host_name, None @@ -68,7 +80,7 @@ class AEWorkfileCreator(AutoCreator): new_instance.data_to_store()) elif ( - existing_instance["asset"] != asset_name + existing_asset_name != asset_name or existing_instance["task"] != task_name ): asset_doc = get_asset_by_name(project_name, asset_name) @@ -76,6 +88,10 @@ class AEWorkfileCreator(AutoCreator): self.default_variant, task_name, asset_doc, project_name, host_name ) - existing_instance["asset"] = asset_name + if AYON_SERVER_ENABLED: + existing_instance["folderPath"] = asset_name + else: + existing_instance["asset"] = asset_name + existing_instance["task"] = task_name existing_instance["subset"] = subset_name diff --git a/openpype/hosts/aftereffects/plugins/publish/collect_workfile.py b/openpype/hosts/aftereffects/plugins/publish/collect_workfile.py index dc557f67fc..58d2757840 100644 --- a/openpype/hosts/aftereffects/plugins/publish/collect_workfile.py +++ b/openpype/hosts/aftereffects/plugins/publish/collect_workfile.py @@ -1,6 +1,8 @@ import os import pyblish.api + +from openpype.client import get_asset_name_identifier from openpype.pipeline.create import get_subset_name @@ -48,9 +50,11 @@ class CollectWorkfile(pyblish.api.ContextPlugin): asset_entity = context.data["assetEntity"] project_entity = context.data["projectEntity"] + asset_name = get_asset_name_identifier(asset_entity) + instance_data = { "active": True, - "asset": asset_entity["name"], + "asset": asset_name, "task": task, "frameStart": context.data['frameStart'], "frameEnd": context.data['frameEnd'], diff --git a/openpype/hosts/blender/api/plugin.py b/openpype/hosts/blender/api/plugin.py index 8d33187da3..568d8f6695 100644 --- a/openpype/hosts/blender/api/plugin.py +++ b/openpype/hosts/blender/api/plugin.py @@ -6,11 +6,11 @@ from typing import Dict, List, Optional import bpy +from openpype import AYON_SERVER_ENABLED from openpype.pipeline import ( Creator, CreatedInstance, LoaderPlugin, - get_current_task_name, ) from openpype.lib import BoolDef @@ -225,7 +225,12 @@ class BaseCreator(Creator): bpy.context.scene.collection.children.link(instances) # Create asset group - name = prepare_scene_name(instance_data["asset"], subset_name) + if AYON_SERVER_ENABLED: + asset_name = instance_data["folderPath"] + else: + asset_name = instance_data["asset"] + + name = prepare_scene_name(asset_name, subset_name) if self.create_as_asset_group: # Create instance as empty instance_node = bpy.data.objects.new(name=name, object_data=None) @@ -281,7 +286,14 @@ class BaseCreator(Creator): Args: update_list(List[UpdateData]): Changed instances - and their changes, as a list of tuples.""" + and their changes, as a list of tuples. + """ + + if AYON_SERVER_ENABLED: + asset_name_key = "folderPath" + else: + asset_name_key = "asset" + for created_instance, changes in update_list: data = created_instance.data_to_store() node = created_instance.transient_data["instance_node"] @@ -295,11 +307,12 @@ class BaseCreator(Creator): # Rename the instance node in the scene if subset or asset changed if ( - "subset" in changes.changed_keys - or "asset" in changes.changed_keys + "subset" in changes.changed_keys + or asset_name_key in changes.changed_keys ): + asset_name = data[asset_name_key] name = prepare_scene_name( - asset=data["asset"], subset=data["subset"] + asset=asset_name, subset=data["subset"] ) node.name = name diff --git a/openpype/hosts/blender/plugins/create/create_workfile.py b/openpype/hosts/blender/plugins/create/create_workfile.py index 9245434766..ceec3e0552 100644 --- a/openpype/hosts/blender/plugins/create/create_workfile.py +++ b/openpype/hosts/blender/plugins/create/create_workfile.py @@ -1,5 +1,6 @@ import bpy +from openpype import AYON_SERVER_ENABLED from openpype.pipeline import CreatedInstance, AutoCreator from openpype.client import get_asset_by_name from openpype.hosts.blender.api.plugin import BaseCreator @@ -24,7 +25,7 @@ class CreateWorkfile(BaseCreator, AutoCreator): def create(self): """Create workfile instances.""" - current_instance = next( + existing_instance = next( ( instance for instance in self.create_context.instances if instance.creator_identifier == self.identifier @@ -37,16 +38,27 @@ class CreateWorkfile(BaseCreator, AutoCreator): task_name = self.create_context.get_current_task_name() host_name = self.create_context.host_name - if not current_instance: + existing_asset_name = None + if existing_instance is not None: + if AYON_SERVER_ENABLED: + existing_asset_name = existing_instance.get("folderPath") + + if existing_asset_name is None: + existing_asset_name = existing_instance["asset"] + + if not existing_instance: asset_doc = get_asset_by_name(project_name, asset_name) subset_name = self.get_subset_name( task_name, task_name, asset_doc, project_name, host_name ) data = { - "asset": asset_name, "task": task_name, "variant": task_name, } + if AYON_SERVER_ENABLED: + data["folderPath"] = asset_name + else: + data["asset"] = asset_name data.update( self.get_dynamic_data( task_name, @@ -54,7 +66,7 @@ class CreateWorkfile(BaseCreator, AutoCreator): asset_doc, project_name, host_name, - current_instance, + existing_instance, ) ) self.log.info("Auto-creating workfile instance...") @@ -65,17 +77,21 @@ class CreateWorkfile(BaseCreator, AutoCreator): current_instance.transient_data["instance_node"] = instance_node self._add_instance_to_context(current_instance) elif ( - current_instance["asset"] != asset_name - or current_instance["task"] != task_name + existing_asset_name != asset_name + or existing_instance["task"] != task_name ): # Update instance context if it's different asset_doc = get_asset_by_name(project_name, asset_name) subset_name = self.get_subset_name( task_name, task_name, asset_doc, project_name, host_name ) - current_instance["asset"] = asset_name - current_instance["task"] = task_name - current_instance["subset"] = subset_name + if AYON_SERVER_ENABLED: + existing_instance["folderPath"] = asset_name + else: + existing_instance["asset"] = asset_name + + existing_instance["task"] = task_name + existing_instance["subset"] = subset_name def collect_instances(self): diff --git a/openpype/hosts/blender/plugins/publish/extract_abc.py b/openpype/hosts/blender/plugins/publish/extract_abc.py index 12d062d925..0e242e9d53 100644 --- a/openpype/hosts/blender/plugins/publish/extract_abc.py +++ b/openpype/hosts/blender/plugins/publish/extract_abc.py @@ -19,7 +19,10 @@ class ExtractABC(publish.Extractor, publish.OptionalPyblishPluginMixin): # Define extract output file path stagingdir = self.staging_dir(instance) - filename = f"{instance.name}.abc" + asset_name = instance.data["assetEntity"]["name"] + subset = instance.data["subset"] + instance_name = f"{asset_name}_{subset}" + filename = f"{instance_name}.abc" filepath = os.path.join(stagingdir, filename) # Perform extraction diff --git a/openpype/hosts/blender/plugins/publish/extract_abc_animation.py b/openpype/hosts/blender/plugins/publish/extract_abc_animation.py index 1c23fc8acb..6ef9b29693 100644 --- a/openpype/hosts/blender/plugins/publish/extract_abc_animation.py +++ b/openpype/hosts/blender/plugins/publish/extract_abc_animation.py @@ -23,7 +23,11 @@ class ExtractAnimationABC( # Define extract output file path stagingdir = self.staging_dir(instance) - filename = f"{instance.name}.abc" + asset_name = instance.data["assetEntity"]["name"] + subset = instance.data["subset"] + instance_name = f"{asset_name}_{subset}" + filename = f"{instance_name}.abc" + filepath = os.path.join(stagingdir, filename) # Perform extraction diff --git a/openpype/hosts/blender/plugins/publish/extract_blend.py b/openpype/hosts/blender/plugins/publish/extract_blend.py index a1b49dcd8f..94e87d537c 100644 --- a/openpype/hosts/blender/plugins/publish/extract_blend.py +++ b/openpype/hosts/blender/plugins/publish/extract_blend.py @@ -20,7 +20,10 @@ class ExtractBlend(publish.Extractor, publish.OptionalPyblishPluginMixin): # Define extract output file path stagingdir = self.staging_dir(instance) - filename = f"{instance.name}.blend" + asset_name = instance.data["assetEntity"]["name"] + subset = instance.data["subset"] + instance_name = f"{asset_name}_{subset}" + filename = f"{instance_name}.blend" filepath = os.path.join(stagingdir, filename) # Perform extraction diff --git a/openpype/hosts/blender/plugins/publish/extract_blend_animation.py b/openpype/hosts/blender/plugins/publish/extract_blend_animation.py index 2e0de9317e..11eb268271 100644 --- a/openpype/hosts/blender/plugins/publish/extract_blend_animation.py +++ b/openpype/hosts/blender/plugins/publish/extract_blend_animation.py @@ -23,7 +23,10 @@ class ExtractBlendAnimation( # Define extract output file path stagingdir = self.staging_dir(instance) - filename = f"{instance.name}.blend" + asset_name = instance.data["assetEntity"]["name"] + subset = instance.data["subset"] + instance_name = f"{asset_name}_{subset}" + filename = f"{instance_name}.blend" filepath = os.path.join(stagingdir, filename) # Perform extraction diff --git a/openpype/hosts/blender/plugins/publish/extract_camera_abc.py b/openpype/hosts/blender/plugins/publish/extract_camera_abc.py index 9d0b7f132b..df68668eae 100644 --- a/openpype/hosts/blender/plugins/publish/extract_camera_abc.py +++ b/openpype/hosts/blender/plugins/publish/extract_camera_abc.py @@ -21,7 +21,10 @@ class ExtractCameraABC(publish.Extractor, publish.OptionalPyblishPluginMixin): # Define extract output file path stagingdir = self.staging_dir(instance) - filename = f"{instance.name}.abc" + asset_name = instance.data["assetEntity"]["name"] + subset = instance.data["subset"] + instance_name = f"{asset_name}_{subset}" + filename = f"{instance_name}.abc" filepath = os.path.join(stagingdir, filename) # Perform extraction diff --git a/openpype/hosts/blender/plugins/publish/extract_camera_fbx.py b/openpype/hosts/blender/plugins/publish/extract_camera_fbx.py index 9bbcf047cc..ee046b7d11 100644 --- a/openpype/hosts/blender/plugins/publish/extract_camera_fbx.py +++ b/openpype/hosts/blender/plugins/publish/extract_camera_fbx.py @@ -20,7 +20,10 @@ class ExtractCamera(publish.Extractor, publish.OptionalPyblishPluginMixin): # Define extract output file path stagingdir = self.staging_dir(instance) - filename = f"{instance.name}.fbx" + asset_name = instance.data["assetEntity"]["name"] + subset = instance.data["subset"] + instance_name = f"{asset_name}_{subset}" + filename = f"{instance_name}.fbx" filepath = os.path.join(stagingdir, filename) # Perform extraction diff --git a/openpype/hosts/blender/plugins/publish/extract_fbx.py b/openpype/hosts/blender/plugins/publish/extract_fbx.py index 0ba82eca4e..4ae6501f7d 100644 --- a/openpype/hosts/blender/plugins/publish/extract_fbx.py +++ b/openpype/hosts/blender/plugins/publish/extract_fbx.py @@ -21,7 +21,10 @@ class ExtractFBX(publish.Extractor, publish.OptionalPyblishPluginMixin): # Define extract output file path stagingdir = self.staging_dir(instance) - filename = f"{instance.name}.fbx" + asset_name = instance.data["assetEntity"]["name"] + subset = instance.data["subset"] + instance_name = f"{asset_name}_{subset}" + filename = f"{instance_name}.fbx" filepath = os.path.join(stagingdir, filename) # Perform extraction diff --git a/openpype/hosts/blender/plugins/publish/extract_fbx_animation.py b/openpype/hosts/blender/plugins/publish/extract_fbx_animation.py index a705345edb..4fc8230a1b 100644 --- a/openpype/hosts/blender/plugins/publish/extract_fbx_animation.py +++ b/openpype/hosts/blender/plugins/publish/extract_fbx_animation.py @@ -145,7 +145,10 @@ class ExtractAnimationFBX( root.select_set(True) armature.select_set(True) - fbx_filename = f"{instance.name}_{armature.name}.fbx" + asset_name = instance.data["assetEntity"]["name"] + subset = instance.data["subset"] + instance_name = f"{asset_name}_{subset}" + fbx_filename = f"{instance_name}_{armature.name}.fbx" filepath = os.path.join(stagingdir, fbx_filename) override = plugin.create_blender_context( @@ -178,7 +181,7 @@ class ExtractAnimationFBX( pair[1].user_clear() bpy.data.actions.remove(pair[1]) - json_filename = f"{instance.name}.json" + json_filename = f"{instance_name}.json" json_path = os.path.join(stagingdir, json_filename) json_dict = { diff --git a/openpype/hosts/blender/plugins/publish/extract_layout.py b/openpype/hosts/blender/plugins/publish/extract_layout.py index 73d92961bc..41c6b0912c 100644 --- a/openpype/hosts/blender/plugins/publish/extract_layout.py +++ b/openpype/hosts/blender/plugins/publish/extract_layout.py @@ -224,7 +224,11 @@ class ExtractLayout(publish.Extractor, publish.OptionalPyblishPluginMixin): json_data.append(json_element) - json_filename = "{}.json".format(instance.name) + asset_name = instance.data["assetEntity"]["name"] + subset = instance.data["subset"] + instance_name = f"{asset_name}_{subset}" + json_filename = f"{instance_name}.json" + json_path = os.path.join(stagingdir, json_filename) with open(json_path, "w+") as file: diff --git a/openpype/hosts/blender/plugins/publish/extract_playblast.py b/openpype/hosts/blender/plugins/publish/extract_playblast.py index fe005c6593..a78aa14138 100644 --- a/openpype/hosts/blender/plugins/publish/extract_playblast.py +++ b/openpype/hosts/blender/plugins/publish/extract_playblast.py @@ -51,7 +51,10 @@ class ExtractPlayblast(publish.Extractor, publish.OptionalPyblishPluginMixin): # get output path stagingdir = self.staging_dir(instance) - filename = instance.name + asset_name = instance.data["assetEntity"]["name"] + subset = instance.data["subset"] + filename = f"{asset_name}_{subset}" + path = os.path.join(stagingdir, filename) self.log.debug(f"Outputting images to {path}") diff --git a/openpype/hosts/blender/plugins/publish/extract_thumbnail.py b/openpype/hosts/blender/plugins/publish/extract_thumbnail.py index 52e5d98fc4..e8a9c68dd1 100644 --- a/openpype/hosts/blender/plugins/publish/extract_thumbnail.py +++ b/openpype/hosts/blender/plugins/publish/extract_thumbnail.py @@ -27,7 +27,10 @@ class ExtractThumbnail(publish.Extractor): self.log.debug("Extracting capture..") stagingdir = self.staging_dir(instance) - filename = instance.name + asset_name = instance.data["assetEntity"]["name"] + subset = instance.data["subset"] + filename = f"{asset_name}_{subset}" + path = os.path.join(stagingdir, filename) self.log.debug(f"Outputting images to {path}") diff --git a/openpype/hosts/celaction/plugins/publish/collect_celaction_instances.py b/openpype/hosts/celaction/plugins/publish/collect_celaction_instances.py index c815c1edd4..875f15fcc5 100644 --- a/openpype/hosts/celaction/plugins/publish/collect_celaction_instances.py +++ b/openpype/hosts/celaction/plugins/publish/collect_celaction_instances.py @@ -1,6 +1,8 @@ import os import pyblish.api +from openpype.client import get_asset_name_identifier + class CollectCelactionInstances(pyblish.api.ContextPlugin): """ Adds the celaction render instances """ @@ -17,8 +19,10 @@ class CollectCelactionInstances(pyblish.api.ContextPlugin): asset_entity = context.data["assetEntity"] project_entity = context.data["projectEntity"] + asset_name = get_asset_name_identifier(asset_entity) + shared_instance_data = { - "asset": asset_entity["name"], + "asset": asset_name, "frameStart": asset_entity["data"]["frameStart"], "frameEnd": asset_entity["data"]["frameEnd"], "handleStart": asset_entity["data"]["handleStart"], diff --git a/openpype/hosts/flame/plugins/publish/collect_timeline_otio.py b/openpype/hosts/flame/plugins/publish/collect_timeline_otio.py index f8cfa9e963..20ac048986 100644 --- a/openpype/hosts/flame/plugins/publish/collect_timeline_otio.py +++ b/openpype/hosts/flame/plugins/publish/collect_timeline_otio.py @@ -1,5 +1,6 @@ import pyblish.api +from openpype.client import get_asset_name_identifier import openpype.hosts.flame.api as opfapi from openpype.hosts.flame.otio import flame_export from openpype.pipeline.create import get_subset_name @@ -33,13 +34,15 @@ class CollecTimelineOTIO(pyblish.api.ContextPlugin): project_settings=context.data["project_settings"] ) + asset_name = get_asset_name_identifier(asset_doc) + # adding otio timeline to context with opfapi.maintained_segment_selection(sequence) as selected_seg: otio_timeline = flame_export.create_otio_timeline(sequence) instance_data = { "name": subset_name, - "asset": asset_doc["name"], + "asset": asset_name, "subset": subset_name, "family": "workfile", "families": [] diff --git a/openpype/hosts/fusion/plugins/create/create_workfile.py b/openpype/hosts/fusion/plugins/create/create_workfile.py index 8acaaa172f..4092086ea4 100644 --- a/openpype/hosts/fusion/plugins/create/create_workfile.py +++ b/openpype/hosts/fusion/plugins/create/create_workfile.py @@ -1,6 +1,7 @@ from openpype.hosts.fusion.api import ( get_current_comp ) +from openpype import AYON_SERVER_ENABLED from openpype.client import get_asset_by_name from openpype.pipeline import ( AutoCreator, @@ -68,6 +69,13 @@ class FusionWorkfileCreator(AutoCreator): task_name = self.create_context.get_current_task_name() host_name = self.create_context.host_name + if existing_instance is None: + existing_instance_asset = None + elif AYON_SERVER_ENABLED: + existing_instance_asset = existing_instance["folderPath"] + else: + existing_instance_asset = existing_instance["asset"] + if existing_instance is None: asset_doc = get_asset_by_name(project_name, asset_name) subset_name = self.get_subset_name( @@ -75,10 +83,13 @@ class FusionWorkfileCreator(AutoCreator): project_name, host_name ) data = { - "asset": asset_name, "task": task_name, "variant": self.default_variant } + if AYON_SERVER_ENABLED: + data["folderPath"] = asset_name + else: + data["asset"] = asset_name data.update(self.get_dynamic_data( self.default_variant, task_name, asset_doc, project_name, host_name, None @@ -91,7 +102,7 @@ class FusionWorkfileCreator(AutoCreator): self._add_instance_to_context(new_instance) elif ( - existing_instance["asset"] != asset_name + existing_instance_asset != asset_name or existing_instance["task"] != task_name ): asset_doc = get_asset_by_name(project_name, asset_name) @@ -99,6 +110,9 @@ class FusionWorkfileCreator(AutoCreator): self.default_variant, task_name, asset_doc, project_name, host_name ) - existing_instance["asset"] = asset_name + if AYON_SERVER_ENABLED: + existing_instance["folderPath"] = asset_name + else: + existing_instance["asset"] = asset_name existing_instance["task"] = task_name existing_instance["subset"] = subset_name diff --git a/openpype/hosts/hiero/api/plugin.py b/openpype/hosts/hiero/api/plugin.py index 52f96261b2..b0c73e41fb 100644 --- a/openpype/hosts/hiero/api/plugin.py +++ b/openpype/hosts/hiero/api/plugin.py @@ -11,7 +11,6 @@ import qargparse from openpype.settings import get_current_project_settings from openpype.lib import Logger from openpype.pipeline import LoaderPlugin, LegacyCreator -from openpype.pipeline.context_tools import get_current_project_asset from openpype.pipeline.load import get_representation_path_from_context from . import lib @@ -32,7 +31,7 @@ def load_stylesheet(): class CreatorWidget(QtWidgets.QDialog): # output items - items = dict() + items = {} def __init__(self, name, info, ui_inputs, parent=None): super(CreatorWidget, self).__init__(parent) @@ -494,9 +493,8 @@ class ClipLoader: joint `data` key with asset.data dict into the representation """ - asset_name = self.context["representation"]["context"]["asset"] - asset_doc = get_current_project_asset(asset_name) - log.debug("__ asset_doc: {}".format(pformat(asset_doc))) + + asset_doc = self.context["asset"] self.data["assetData"] = asset_doc["data"] def _make_track_item(self, source_bin_item, audio=False): @@ -644,8 +642,8 @@ class PublishClip: Returns: hiero.core.TrackItem: hiero track item object with pype tag """ - vertical_clip_match = dict() - tag_data = dict() + vertical_clip_match = {} + tag_data = {} types = { "shot": "shot", "folder": "folder", @@ -707,9 +705,10 @@ class PublishClip: self._create_parents() def convert(self): - # solve track item data and add them to tag data - self._convert_to_tag_data() + tag_hierarchy_data = self._convert_to_tag_data() + + self.tag_data.update(tag_hierarchy_data) # if track name is in review track name and also if driving track name # is not in review track name: skip tag creation @@ -723,16 +722,23 @@ class PublishClip: if self.rename: # rename track item self.track_item.setName(new_name) - self.tag_data["asset"] = new_name + self.tag_data["asset_name"] = new_name else: - self.tag_data["asset"] = self.ti_name + self.tag_data["asset_name"] = self.ti_name self.tag_data["hierarchyData"]["shot"] = self.ti_name + # AYON unique identifier + folder_path = "/{}/{}".format( + tag_hierarchy_data["hierarchy"], + self.tag_data["asset_name"] + ) + self.tag_data["folderPath"] = folder_path if self.tag_data["heroTrack"] and self.review_layer: self.tag_data.update({"reviewTrack": self.review_layer}) else: self.tag_data.update({"reviewTrack": None}) + # TODO: remove debug print log.debug("___ self.tag_data: {}".format( pformat(self.tag_data) )) @@ -891,7 +897,7 @@ class PublishClip: tag_hierarchy_data = hero_data # add data to return data dict - self.tag_data.update(tag_hierarchy_data) + return tag_hierarchy_data def _solve_tag_hierarchy_data(self, hierarchy_formatting_data): """ Solve tag data from hierarchy data and templates. """ diff --git a/openpype/hosts/hiero/plugins/publish/collect_frame_tag_instances.py b/openpype/hosts/hiero/plugins/publish/collect_frame_tag_instances.py index 982a34efd6..79bf67b336 100644 --- a/openpype/hosts/hiero/plugins/publish/collect_frame_tag_instances.py +++ b/openpype/hosts/hiero/plugins/publish/collect_frame_tag_instances.py @@ -5,6 +5,8 @@ import json import pyblish.api +from openpype.client import get_asset_name_identifier + class CollectFrameTagInstances(pyblish.api.ContextPlugin): """Collect frames from tags. @@ -99,6 +101,9 @@ class CollectFrameTagInstances(pyblish.api.ContextPlugin): # first collect all available subset tag frames subset_data = {} + context_asset_doc = context.data["assetEntity"] + context_asset_name = get_asset_name_identifier(context_asset_doc) + for tag_data in sequence_tags: frame = int(tag_data["start"]) @@ -115,7 +120,7 @@ class CollectFrameTagInstances(pyblish.api.ContextPlugin): subset_data[subset] = { "frames": [frame], "format": tag_data["format"], - "asset": context.data["assetEntity"]["name"] + "asset": context_asset_name } return subset_data diff --git a/openpype/hosts/hiero/plugins/publish/precollect_instances.py b/openpype/hosts/hiero/plugins/publish/precollect_instances.py index 3f9da2cf60..590d7b7050 100644 --- a/openpype/hosts/hiero/plugins/publish/precollect_instances.py +++ b/openpype/hosts/hiero/plugins/publish/precollect_instances.py @@ -1,9 +1,12 @@ import pyblish + +from openpype import AYON_SERVER_ENABLED from openpype.pipeline.editorial import is_overlapping_otio_ranges + from openpype.hosts.hiero import api as phiero from openpype.hosts.hiero.api.otio import hiero_export -import hiero +import hiero # # developer reload modules from pprint import pformat @@ -80,25 +83,24 @@ class PrecollectInstances(pyblish.api.ContextPlugin): if k not in ("id", "applieswhole", "label") }) - asset = tag_data["asset"] + asset, asset_name = self._get_asset_data(tag_data) + subset = tag_data["subset"] # insert family into families - family = tag_data["family"] families = [str(f) for f in tag_data["families"]] - families.insert(0, str(family)) # form label - label = asset - if asset != clip_name: + label = "{} -".format(asset) + if asset_name != clip_name: label += " ({})".format(clip_name) label += " {}".format(subset) - label += " {}".format("[" + ", ".join(families) + "]") data.update({ "name": "{}_{}".format(asset, subset), "label": label, "asset": asset, + "asset_name": asset_name, "item": track_item, "families": families, "publish": tag_data["publish"], @@ -176,9 +178,9 @@ class PrecollectInstances(pyblish.api.ContextPlugin): }) def create_shot_instance(self, context, **data): + subset = "shotMain" master_layer = data.get("heroTrack") hierarchy_data = data.get("hierarchyData") - asset = data.get("asset") item = data.get("item") clip_name = item.name() @@ -189,23 +191,21 @@ class PrecollectInstances(pyblish.api.ContextPlugin): return asset = data["asset"] - subset = "shotMain" + asset_name = data["asset_name"] # insert family into families family = "shot" # form label - label = asset - if asset != clip_name: + label = "{} -".format(asset) + if asset_name != clip_name: label += " ({}) ".format(clip_name) label += " {}".format(subset) - label += " [{}]".format(family) data.update({ "name": "{}_{}".format(asset, subset), "label": label, "subset": subset, - "asset": asset, "family": family, "families": [] }) @@ -215,7 +215,33 @@ class PrecollectInstances(pyblish.api.ContextPlugin): self.log.debug( "_ instance.data: {}".format(pformat(instance.data))) + def _get_asset_data(self, data): + folder_path = data.pop("folderPath", None) + + if data.get("asset_name"): + asset_name = data["asset_name"] + else: + asset_name = data["asset"] + + # backward compatibility for clip tags + # which are missing folderPath key + # TODO remove this in future versions + if not folder_path: + hierarchy_path = data["hierarchy"] + folder_path = "/{}/{}".format( + hierarchy_path, + asset_name + ) + + if AYON_SERVER_ENABLED: + asset = folder_path + else: + asset = asset_name + + return asset, asset_name + def create_audio_instance(self, context, **data): + subset = "audioMain" master_layer = data.get("heroTrack") if not master_layer: @@ -230,23 +256,21 @@ class PrecollectInstances(pyblish.api.ContextPlugin): return asset = data["asset"] - subset = "audioMain" + asset_name = data["asset_name"] # insert family into families family = "audio" # form label - label = asset - if asset != clip_name: + label = "{} -".format(asset) + if asset_name != clip_name: label += " ({}) ".format(clip_name) label += " {}".format(subset) - label += " [{}]".format(family) data.update({ "name": "{}_{}".format(asset, subset), "label": label, "subset": subset, - "asset": asset, "family": family, "families": ["clip"] }) diff --git a/openpype/hosts/hiero/plugins/publish/precollect_workfile.py b/openpype/hosts/hiero/plugins/publish/precollect_workfile.py index 5a66581531..8abb0885c6 100644 --- a/openpype/hosts/hiero/plugins/publish/precollect_workfile.py +++ b/openpype/hosts/hiero/plugins/publish/precollect_workfile.py @@ -7,6 +7,7 @@ from qtpy.QtGui import QPixmap import hiero.ui +from openpype import AYON_SERVER_ENABLED from openpype.hosts.hiero.api.otio import hiero_export @@ -17,9 +18,11 @@ class PrecollectWorkfile(pyblish.api.ContextPlugin): order = pyblish.api.CollectorOrder - 0.491 def process(self, context): - asset = context.data["asset"] - subset = "workfile" + asset_name = asset + if AYON_SERVER_ENABLED: + asset_name = asset_name.split("/")[-1] + active_timeline = hiero.ui.activeSequence() project = active_timeline.project() fps = active_timeline.framerate().toFloat() @@ -27,7 +30,7 @@ class PrecollectWorkfile(pyblish.api.ContextPlugin): # adding otio timeline to context otio_timeline = hiero_export.create_otio_timeline() - # get workfile thumnail paths + # get workfile thumbnail paths tmp_staging = tempfile.mkdtemp(prefix="pyblish_tmp_") thumbnail_name = "workfile_thumbnail.png" thumbnail_path = os.path.join(tmp_staging, thumbnail_name) @@ -49,8 +52,8 @@ class PrecollectWorkfile(pyblish.api.ContextPlugin): } # get workfile paths - curent_file = project.path() - staging_dir, base_name = os.path.split(curent_file) + current_file = project.path() + staging_dir, base_name = os.path.split(current_file) # creating workfile representation workfile_representation = { @@ -59,13 +62,16 @@ class PrecollectWorkfile(pyblish.api.ContextPlugin): 'files': base_name, "stagingDir": staging_dir, } - + family = "workfile" instance_data = { - "name": "{}_{}".format(asset, subset), - "asset": asset, - "subset": "{}{}".format(asset, subset.capitalize()), + "label": "{} - {}Main".format( + asset, family), + "name": "{}_{}".format(asset_name, family), + "asset": context.data["asset"], + # TODO use 'get_subset_name' + "subset": "{}{}Main".format(asset_name, family.capitalize()), "item": project, - "family": "workfile", + "family": family, "families": [], "representations": [workfile_representation, thumb_representation] } @@ -78,7 +84,7 @@ class PrecollectWorkfile(pyblish.api.ContextPlugin): "activeProject": project, "activeTimeline": active_timeline, "otioTimeline": otio_timeline, - "currentFile": curent_file, + "currentFile": current_file, "colorspace": self.get_colorspace(project), "fps": fps } diff --git a/openpype/hosts/hiero/plugins/publish_old_workflow/collect_assetbuilds.py b/openpype/hosts/hiero/plugins/publish_old_workflow/collect_assetbuilds.py index 767f7c30f7..37370497a5 100644 --- a/openpype/hosts/hiero/plugins/publish_old_workflow/collect_assetbuilds.py +++ b/openpype/hosts/hiero/plugins/publish_old_workflow/collect_assetbuilds.py @@ -1,5 +1,6 @@ from pyblish import api -from openpype.client import get_assets + +from openpype.client import get_assets, get_asset_name_identifier class CollectAssetBuilds(api.ContextPlugin): @@ -19,10 +20,13 @@ class CollectAssetBuilds(api.ContextPlugin): def process(self, context): project_name = context.data["projectName"] asset_builds = {} - for asset in get_assets(project_name): - if asset["data"]["entityType"] == "AssetBuild": - self.log.debug("Found \"{}\" in database.".format(asset)) - asset_builds[asset["name"]] = asset + for asset_doc in get_assets(project_name): + if asset_doc["data"].get("entityType") != "AssetBuild": + continue + + asset_name = get_asset_name_identifier(asset_doc) + self.log.debug("Found \"{}\" in database.".format(asset_doc)) + asset_builds[asset_name] = asset_doc for instance in context: if instance.data["family"] != "clip": @@ -50,9 +54,7 @@ class CollectAssetBuilds(api.ContextPlugin): # Collect asset builds. data = {"assetbuilds": []} for name in asset_names: - data["assetbuilds"].append( - asset_builds[name] - ) + data["assetbuilds"].append(asset_builds[name]) self.log.debug( "Found asset builds: {}".format(data["assetbuilds"]) ) diff --git a/openpype/hosts/houdini/api/lib.py b/openpype/hosts/houdini/api/lib.py index 8b058e605d..513735092c 100644 --- a/openpype/hosts/houdini/api/lib.py +++ b/openpype/hosts/houdini/api/lib.py @@ -152,7 +152,9 @@ def get_output_parameter(node): return node.parm("ar_ass_file") elif node_type == "Redshift_Proxy_Output": return node.parm("RS_archive_file") - + elif node_type == "ifd": + if node.evalParm("soho_outputmode"): + return node.parm("soho_diskfile") raise TypeError("Node type '%s' not supported" % node_type) diff --git a/openpype/hosts/houdini/api/pipeline.py b/openpype/hosts/houdini/api/pipeline.py index 11135e20b2..d0f45c36b5 100644 --- a/openpype/hosts/houdini/api/pipeline.py +++ b/openpype/hosts/houdini/api/pipeline.py @@ -66,10 +66,6 @@ class HoudiniHost(HostBase, IWorkfileHost, ILoadHost, IPublishHost): register_event_callback("new", on_new) self._has_been_setup = True - # add houdini vendor packages - hou_pythonpath = os.path.join(HOUDINI_HOST_DIR, "vendor") - - sys.path.append(hou_pythonpath) # Set asset settings for the empty scene directly after launch of # Houdini so it initializes into the correct scene FPS, diff --git a/openpype/hosts/houdini/api/plugin.py b/openpype/hosts/houdini/api/plugin.py index 72565f7211..e162d0e461 100644 --- a/openpype/hosts/houdini/api/plugin.py +++ b/openpype/hosts/houdini/api/plugin.py @@ -6,6 +6,8 @@ from abc import ( ) import six import hou + +from openpype import AYON_SERVER_ENABLED from openpype.pipeline import ( CreatorError, LegacyCreator, @@ -142,12 +144,13 @@ class HoudiniCreatorBase(object): @staticmethod def create_instance_node( - node_name, parent, - node_type="geometry"): + asset_name, node_name, parent, node_type="geometry" + ): # type: (str, str, str) -> hou.Node """Create node representing instance. Arguments: + asset_name (str): Asset name. node_name (str): Name of the new node. parent (str): Name of the parent node. node_type (str, optional): Type of the node. @@ -182,8 +185,13 @@ class HoudiniCreator(NewCreator, HoudiniCreatorBase): if node_type is None: node_type = "geometry" + if AYON_SERVER_ENABLED: + asset_name = instance_data["folderPath"] + else: + asset_name = instance_data["asset"] + instance_node = self.create_instance_node( - subset_name, "/out", node_type) + asset_name, subset_name, "/out", node_type) self.customize_node_look(instance_node) diff --git a/openpype/hosts/houdini/plugins/create/create_arnold_ass.py b/openpype/hosts/houdini/plugins/create/create_arnold_ass.py index 12d08f7d83..437a14c723 100644 --- a/openpype/hosts/houdini/plugins/create/create_arnold_ass.py +++ b/openpype/hosts/houdini/plugins/create/create_arnold_ass.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- """Creator plugin for creating Arnold ASS files.""" from openpype.hosts.houdini.api import plugin +from openpype.lib import BoolDef class CreateArnoldAss(plugin.HoudiniCreator): @@ -21,6 +22,9 @@ class CreateArnoldAss(plugin.HoudiniCreator): instance_data.pop("active", None) instance_data.update({"node_type": "arnold"}) + creator_attributes = instance_data.setdefault( + "creator_attributes", dict()) + creator_attributes["farm"] = pre_create_data["farm"] instance = super(CreateArnoldAss, self).create( subset_name, @@ -52,3 +56,15 @@ class CreateArnoldAss(plugin.HoudiniCreator): # Lock any parameters in this list to_lock = ["ar_ass_export_enable", "family", "id"] self.lock_parameters(instance_node, to_lock) + + def get_instance_attr_defs(self): + return [ + BoolDef("farm", + label="Submitting to Farm", + default=False) + ] + + def get_pre_create_attr_defs(self): + attrs = super().get_pre_create_attr_defs() + # Use same attributes as for instance attributes + return attrs + self.get_instance_attr_defs() diff --git a/openpype/hosts/houdini/plugins/create/create_bgeo.py b/openpype/hosts/houdini/plugins/create/create_bgeo.py index 0f629cf9c9..4140919202 100644 --- a/openpype/hosts/houdini/plugins/create/create_bgeo.py +++ b/openpype/hosts/houdini/plugins/create/create_bgeo.py @@ -2,8 +2,8 @@ """Creator plugin for creating pointcache bgeo files.""" from openpype.hosts.houdini.api import plugin from openpype.pipeline import CreatedInstance, CreatorError -from openpype.lib import EnumDef import hou +from openpype.lib import EnumDef, BoolDef class CreateBGEO(plugin.HoudiniCreator): @@ -18,6 +18,9 @@ class CreateBGEO(plugin.HoudiniCreator): instance_data.pop("active", None) instance_data.update({"node_type": "geometry"}) + creator_attributes = instance_data.setdefault( + "creator_attributes", dict()) + creator_attributes["farm"] = pre_create_data["farm"] instance = super(CreateBGEO, self).create( subset_name, @@ -58,6 +61,13 @@ class CreateBGEO(plugin.HoudiniCreator): instance_node.setParms(parms) + def get_instance_attr_defs(self): + return [ + BoolDef("farm", + label="Submitting to Farm", + default=False) + ] + def get_pre_create_attr_defs(self): attrs = super().get_pre_create_attr_defs() bgeo_enum = [ @@ -89,7 +99,7 @@ class CreateBGEO(plugin.HoudiniCreator): return attrs + [ EnumDef("bgeo_type", bgeo_enum, label="BGEO Options"), - ] + ] + self.get_instance_attr_defs() def get_network_categories(self): return [ diff --git a/openpype/hosts/houdini/plugins/create/create_hda.py b/openpype/hosts/houdini/plugins/create/create_hda.py index ac075d2072..f670b55eb6 100644 --- a/openpype/hosts/houdini/plugins/create/create_hda.py +++ b/openpype/hosts/houdini/plugins/create/create_hda.py @@ -17,13 +17,13 @@ class CreateHDA(plugin.HoudiniCreator): icon = "gears" maintain_selection = False - def _check_existing(self, subset_name): + def _check_existing(self, asset_name, subset_name): # type: (str) -> bool """Check if existing subset name versions already exists.""" # Get all subsets of the current asset project_name = self.project_name asset_doc = get_asset_by_name( - project_name, self.data["asset"], fields=["_id"] + project_name, asset_name, fields=["_id"] ) subset_docs = get_subsets( project_name, asset_ids=[asset_doc["_id"]], fields=["name"] @@ -35,7 +35,8 @@ class CreateHDA(plugin.HoudiniCreator): return subset_name.lower() in existing_subset_names_low def create_instance_node( - self, node_name, parent, node_type="geometry"): + self, asset_name, node_name, parent, node_type="geometry" + ): parent_node = hou.node("/obj") if self.selected_nodes: @@ -61,7 +62,7 @@ class CreateHDA(plugin.HoudiniCreator): hda_file_name="$HIP/{}.hda".format(node_name) ) hda_node.layoutChildren() - elif self._check_existing(node_name): + elif self._check_existing(asset_name, node_name): raise plugin.OpenPypeCreatorError( ("subset {} is already published with different HDA" "definition.").format(node_name)) diff --git a/openpype/hosts/houdini/plugins/create/create_mantra_ifd.py b/openpype/hosts/houdini/plugins/create/create_mantra_ifd.py new file mode 100644 index 0000000000..7ea7d1042f --- /dev/null +++ b/openpype/hosts/houdini/plugins/create/create_mantra_ifd.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +"""Creator plugin for creating pointcache alembics.""" +from openpype.hosts.houdini.api import plugin +from openpype.pipeline import CreatedInstance +from openpype.lib import BoolDef + + +class CreateMantraIFD(plugin.HoudiniCreator): + """Mantra .ifd Archive""" + identifier = "io.openpype.creators.houdini.mantraifd" + label = "Mantra IFD" + family = "mantraifd" + icon = "gears" + + def create(self, subset_name, instance_data, pre_create_data): + import hou + instance_data.pop("active", None) + instance_data.update({"node_type": "ifd"}) + creator_attributes = instance_data.setdefault( + "creator_attributes", dict()) + creator_attributes["farm"] = pre_create_data["farm"] + instance = super(CreateMantraIFD, self).create( + subset_name, + instance_data, + pre_create_data) # type: CreatedInstance + + instance_node = hou.node(instance.get("instance_node")) + + filepath = "{}{}".format( + hou.text.expandString("$HIP/pyblish/"), + "{}.$F4.ifd".format(subset_name)) + parms = { + # Render frame range + "trange": 1, + # Arnold ROP settings + "soho_diskfile": filepath, + "soho_outputmode": 1 + } + + instance_node.setParms(parms) + + # Lock any parameters in this list + to_lock = ["soho_outputmode", "family", "id"] + self.lock_parameters(instance_node, to_lock) + + def get_instance_attr_defs(self): + return [ + BoolDef("farm", + label="Submitting to Farm", + default=False) + ] + + def get_pre_create_attr_defs(self): + attrs = super().get_pre_create_attr_defs() + # Use same attributes as for instance attributes + return attrs + self.get_instance_attr_defs() diff --git a/openpype/hosts/houdini/plugins/create/create_pointcache.py b/openpype/hosts/houdini/plugins/create/create_pointcache.py index 7eaf2aff2b..2d2f89cc48 100644 --- a/openpype/hosts/houdini/plugins/create/create_pointcache.py +++ b/openpype/hosts/houdini/plugins/create/create_pointcache.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- """Creator plugin for creating pointcache alembics.""" from openpype.hosts.houdini.api import plugin +from openpype.lib import BoolDef import hou + class CreatePointCache(plugin.HoudiniCreator): """Alembic ROP to pointcache""" identifier = "io.openpype.creators.houdini.pointcache" @@ -15,6 +17,9 @@ class CreatePointCache(plugin.HoudiniCreator): def create(self, subset_name, instance_data, pre_create_data): instance_data.pop("active", None) instance_data.update({"node_type": "alembic"}) + creator_attributes = instance_data.setdefault( + "creator_attributes", dict()) + creator_attributes["farm"] = pre_create_data["farm"] instance = super(CreatePointCache, self).create( subset_name, @@ -105,3 +110,15 @@ class CreatePointCache(plugin.HoudiniCreator): else: return min(outputs, key=lambda node: node.evalParm('outputidx')) + + def get_instance_attr_defs(self): + return [ + BoolDef("farm", + label="Submitting to Farm", + default=False) + ] + + def get_pre_create_attr_defs(self): + attrs = super().get_pre_create_attr_defs() + # Use same attributes as for instance attributes + return attrs + self.get_instance_attr_defs() diff --git a/openpype/hosts/houdini/plugins/create/create_redshift_proxy.py b/openpype/hosts/houdini/plugins/create/create_redshift_proxy.py index 3a4ab7008b..e1577c92e9 100644 --- a/openpype/hosts/houdini/plugins/create/create_redshift_proxy.py +++ b/openpype/hosts/houdini/plugins/create/create_redshift_proxy.py @@ -2,6 +2,7 @@ """Creator plugin for creating Redshift proxies.""" from openpype.hosts.houdini.api import plugin import hou +from openpype.lib import BoolDef class CreateRedshiftProxy(plugin.HoudiniCreator): @@ -24,6 +25,9 @@ class CreateRedshiftProxy(plugin.HoudiniCreator): # TODO: Somehow enforce so that it only shows the original limited # attributes of the Redshift_Proxy_Output node type instance_data.update({"node_type": "Redshift_Proxy_Output"}) + creator_attributes = instance_data.setdefault( + "creator_attributes", dict()) + creator_attributes["farm"] = pre_create_data["farm"] instance = super(CreateRedshiftProxy, self).create( subset_name, @@ -50,3 +54,15 @@ class CreateRedshiftProxy(plugin.HoudiniCreator): hou.ropNodeTypeCategory(), hou.sopNodeTypeCategory() ] + + def get_instance_attr_defs(self): + return [ + BoolDef("farm", + label="Submitting to Farm", + default=False) + ] + + def get_pre_create_attr_defs(self): + attrs = super().get_pre_create_attr_defs() + # Use same attributes as for instance attributes + return attrs + self.get_instance_attr_defs() diff --git a/openpype/hosts/houdini/plugins/create/create_vbd_cache.py b/openpype/hosts/houdini/plugins/create/create_vbd_cache.py index 69418f9575..53df9dda68 100644 --- a/openpype/hosts/houdini/plugins/create/create_vbd_cache.py +++ b/openpype/hosts/houdini/plugins/create/create_vbd_cache.py @@ -2,6 +2,7 @@ """Creator plugin for creating VDB Caches.""" from openpype.hosts.houdini.api import plugin from openpype.pipeline import CreatedInstance +from openpype.lib import BoolDef import hou @@ -19,15 +20,20 @@ class CreateVDBCache(plugin.HoudiniCreator): instance_data.pop("active", None) instance_data.update({"node_type": "geometry"}) - + creator_attributes = instance_data.setdefault( + "creator_attributes", dict()) + creator_attributes["farm"] = pre_create_data["farm"] instance = super(CreateVDBCache, self).create( subset_name, instance_data, pre_create_data) # type: CreatedInstance instance_node = hou.node(instance.get("instance_node")) + file_path = "{}{}".format( + hou.text.expandString("$HIP/pyblish/"), + "{}.$F4.vdb".format(subset_name)) parms = { - "sopoutput": "$HIP/pyblish/{}.$F4.vdb".format(subset_name), + "sopoutput": file_path, "initsim": True, "trange": 1 } @@ -103,3 +109,15 @@ class CreateVDBCache(plugin.HoudiniCreator): else: return min(outputs, key=lambda node: node.evalParm('outputidx')) + + def get_instance_attr_defs(self): + return [ + BoolDef("farm", + label="Submitting to Farm", + default=False) + ] + + def get_pre_create_attr_defs(self): + attrs = super().get_pre_create_attr_defs() + # Use same attributes as for instance attributes + return attrs + self.get_instance_attr_defs() diff --git a/openpype/hosts/houdini/plugins/create/create_workfile.py b/openpype/hosts/houdini/plugins/create/create_workfile.py index cc45a6c2a8..850f5c994e 100644 --- a/openpype/hosts/houdini/plugins/create/create_workfile.py +++ b/openpype/hosts/houdini/plugins/create/create_workfile.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- """Creator plugin for creating workfiles.""" +from openpype import AYON_SERVER_ENABLED from openpype.hosts.houdini.api import plugin from openpype.hosts.houdini.api.lib import read, imprint from openpype.hosts.houdini.api.pipeline import CONTEXT_CONTAINER @@ -30,16 +31,27 @@ class CreateWorkfile(plugin.HoudiniCreatorBase, AutoCreator): task_name = self.create_context.get_current_task_name() host_name = self.host_name + if current_instance is None: + current_instance_asset = None + elif AYON_SERVER_ENABLED: + current_instance_asset = current_instance["folderPath"] + else: + current_instance_asset = current_instance["asset"] + if current_instance is None: asset_doc = get_asset_by_name(project_name, asset_name) subset_name = self.get_subset_name( variant, task_name, asset_doc, project_name, host_name ) data = { - "asset": asset_name, "task": task_name, "variant": variant } + if AYON_SERVER_ENABLED: + data["folderPath"] = asset_name + else: + data["asset"] = asset_name + data.update( self.get_dynamic_data( variant, task_name, asset_doc, @@ -51,15 +63,18 @@ class CreateWorkfile(plugin.HoudiniCreatorBase, AutoCreator): ) self._add_instance_to_context(current_instance) elif ( - current_instance["asset"] != asset_name - or current_instance["task"] != task_name + current_instance_asset != asset_name + or current_instance["task"] != task_name ): # Update instance context if is not the same asset_doc = get_asset_by_name(project_name, asset_name) subset_name = self.get_subset_name( variant, task_name, asset_doc, project_name, host_name ) - current_instance["asset"] = asset_name + if AYON_SERVER_ENABLED: + current_instance["folderPath"] = asset_name + else: + current_instance["asset"] = asset_name current_instance["task"] = task_name current_instance["subset"] = subset_name diff --git a/openpype/hosts/houdini/plugins/publish/collect_cache_farm.py b/openpype/hosts/houdini/plugins/publish/collect_cache_farm.py new file mode 100644 index 0000000000..36ade32a35 --- /dev/null +++ b/openpype/hosts/houdini/plugins/publish/collect_cache_farm.py @@ -0,0 +1,75 @@ +import os +import pyblish.api +import hou +from openpype.hosts.houdini.api import lib + + +class CollectDataforCache(pyblish.api.InstancePlugin): + """Collect data for caching to Deadline.""" + + order = pyblish.api.CollectorOrder + 0.04 + families = ["ass", "pointcache", + "mantraifd", "redshiftproxy", + "vdbcache"] + hosts = ["houdini"] + targets = ["local", "remote"] + label = "Collect Data for Cache" + + def process(self, instance): + creator_attribute = instance.data["creator_attributes"] + farm_enabled = creator_attribute["farm"] + instance.data["farm"] = farm_enabled + if not farm_enabled: + self.log.debug("Caching on farm is disabled. " + "Skipping farm collecting.") + return + # Why do we need this particular collector to collect the expected + # output files from a ROP node. Don't we have a dedicated collector + # for that yet? + # Collect expected files + ropnode = hou.node(instance.data["instance_node"]) + output_parm = lib.get_output_parameter(ropnode) + expected_filepath = output_parm.eval() + instance.data.setdefault("files", list()) + instance.data.setdefault("expectedFiles", list()) + if instance.data.get("frames"): + files = self.get_files(instance, expected_filepath) + # list of files + instance.data["files"].extend(files) + else: + # single file + instance.data["files"].append(output_parm.eval()) + cache_files = {"_": instance.data["files"]} + # Convert instance family to pointcache if it is bgeo or abc + # because ??? + for family in instance.data["families"]: + if family == "bgeo" or "abc": + instance.data["family"] = "pointcache" + break + instance.data.update({ + "plugin": "Houdini", + "publish": True + }) + instance.data["families"].append("publish.hou") + instance.data["expectedFiles"].append(cache_files) + + self.log.debug("{}".format(instance.data)) + + def get_files(self, instance, output_parm): + """Get the files with the frame range data + + Args: + instance (_type_): instance + output_parm (_type_): path of output parameter + + Returns: + files: a list of files + """ + directory = os.path.dirname(output_parm) + + files = [ + os.path.join(directory, frame).replace("\\", "/") + for frame in instance.data["frames"] + ] + + return files diff --git a/openpype/hosts/houdini/plugins/publish/collect_chunk_size.py b/openpype/hosts/houdini/plugins/publish/collect_chunk_size.py new file mode 100644 index 0000000000..1c867e930a --- /dev/null +++ b/openpype/hosts/houdini/plugins/publish/collect_chunk_size.py @@ -0,0 +1,39 @@ +import pyblish.api +from openpype.lib import NumberDef +from openpype.pipeline import OpenPypePyblishPluginMixin + + +class CollectChunkSize(pyblish.api.InstancePlugin, + OpenPypePyblishPluginMixin): + """Collect chunk size for cache submission to Deadline.""" + + order = pyblish.api.CollectorOrder + 0.05 + families = ["ass", "pointcache", + "vdbcache", "mantraifd", + "redshiftproxy"] + hosts = ["houdini"] + targets = ["local", "remote"] + label = "Collect Chunk Size" + chunkSize = 999999 + + def process(self, instance): + # need to get the chunk size info from the setting + attr_values = self.get_attr_values_from_data(instance.data) + instance.data["chunkSize"] = attr_values.get("chunkSize") + + @classmethod + def apply_settings(cls, project_settings): + project_setting = project_settings["houdini"]["publish"]["CollectChunkSize"] # noqa + cls.chunkSize = project_setting["chunk_size"] + + @classmethod + def get_attribute_defs(cls): + return [ + NumberDef("chunkSize", + minimum=1, + maximum=999999, + decimals=0, + default=cls.chunkSize, + label="Frame Per Task") + + ] diff --git a/openpype/hosts/houdini/plugins/publish/collect_frames.py b/openpype/hosts/houdini/plugins/publish/collect_frames.py index cdef642174..089fae6b1b 100644 --- a/openpype/hosts/houdini/plugins/publish/collect_frames.py +++ b/openpype/hosts/houdini/plugins/publish/collect_frames.py @@ -16,7 +16,8 @@ class CollectFrames(pyblish.api.InstancePlugin): order = pyblish.api.CollectorOrder + 0.1 label = "Collect Frames" families = ["vdbcache", "imagesequence", "ass", - "redshiftproxy", "review", "bgeo"] + "mantraifd", "redshiftproxy", "review", + "bgeo"] def process(self, instance): diff --git a/openpype/hosts/houdini/plugins/publish/collect_usd_bootstrap.py b/openpype/hosts/houdini/plugins/publish/collect_usd_bootstrap.py index 14a8e3c056..462cf99b9c 100644 --- a/openpype/hosts/houdini/plugins/publish/collect_usd_bootstrap.py +++ b/openpype/hosts/houdini/plugins/publish/collect_usd_bootstrap.py @@ -1,6 +1,10 @@ import pyblish.api -from openpype.client import get_subset_by_name, get_asset_by_name +from openpype.client import ( + get_subset_by_name, + get_asset_by_name, + get_asset_name_identifier, +) import openpype.lib.usdlib as usdlib @@ -51,8 +55,9 @@ class CollectUsdBootstrap(pyblish.api.InstancePlugin): self.log.debug("Add bootstrap for: %s" % bootstrap) project_name = instance.context.data["projectName"] - asset = get_asset_by_name(project_name, instance.data["asset"]) - assert asset, "Asset must exist: %s" % asset + asset_name = instance.data["asset"] + asset_doc = get_asset_by_name(project_name, asset_name) + assert asset_doc, "Asset must exist: %s" % asset_name # Check which are not about to be created and don't exist yet required = {"shot": ["usdShot"], "asset": ["usdAsset"]}.get(bootstrap) @@ -67,19 +72,21 @@ class CollectUsdBootstrap(pyblish.api.InstancePlugin): required += list(layers) self.log.debug("Checking required bootstrap: %s" % required) - for subset in required: - if self._subset_exists(project_name, instance, subset, asset): + for subset_name in required: + if self._subset_exists( + project_name, instance, subset_name, asset_doc + ): continue self.log.debug( "Creating {0} USD bootstrap: {1} {2}".format( - bootstrap, asset["name"], subset + bootstrap, asset_name, subset_name ) ) - new = instance.context.create_instance(subset) - new.data["subset"] = subset - new.data["label"] = "{0} ({1})".format(subset, asset["name"]) + new = instance.context.create_instance(subset_name) + new.data["subset"] = subset_name + new.data["label"] = "{0} ({1})".format(subset_name, asset_name) new.data["family"] = "usd.bootstrap" new.data["comment"] = "Automated bootstrap USD file." new.data["publishFamilies"] = ["usd"] @@ -91,21 +98,23 @@ class CollectUsdBootstrap(pyblish.api.InstancePlugin): for key in ["asset"]: new.data[key] = instance.data[key] - def _subset_exists(self, project_name, instance, subset, asset): + def _subset_exists(self, project_name, instance, subset_name, asset_doc): """Return whether subset exists in current context or in database.""" # Allow it to be created during this publish session context = instance.context + + asset_doc_name = get_asset_name_identifier(asset_doc) for inst in context: if ( - inst.data["subset"] == subset - and inst.data["asset"] == asset["name"] + inst.data["subset"] == subset_name + and inst.data["asset"] == asset_doc_name ): return True # Or, if they already exist in the database we can # skip them too. if get_subset_by_name( - project_name, subset, asset["_id"], fields=["_id"] + project_name, subset_name, asset_doc["_id"], fields=["_id"] ): return True return False diff --git a/openpype/hosts/houdini/plugins/publish/extract_alembic.py b/openpype/hosts/houdini/plugins/publish/extract_alembic.py index bdd19b23d4..df2fdda241 100644 --- a/openpype/hosts/houdini/plugins/publish/extract_alembic.py +++ b/openpype/hosts/houdini/plugins/publish/extract_alembic.py @@ -14,8 +14,12 @@ class ExtractAlembic(publish.Extractor): label = "Extract Alembic" hosts = ["houdini"] families = ["abc", "camera"] + targets = ["local", "remote"] def process(self, instance): + if instance.data.get("farm"): + self.log.debug("Should be processed on farm, skipping.") + return ropnode = hou.node(instance.data["instance_node"]) diff --git a/openpype/hosts/houdini/plugins/publish/extract_ass.py b/openpype/hosts/houdini/plugins/publish/extract_ass.py index be60217055..28dd08f999 100644 --- a/openpype/hosts/houdini/plugins/publish/extract_ass.py +++ b/openpype/hosts/houdini/plugins/publish/extract_ass.py @@ -14,9 +14,12 @@ class ExtractAss(publish.Extractor): label = "Extract Ass" families = ["ass"] hosts = ["houdini"] + targets = ["local", "remote"] def process(self, instance): - + if instance.data.get("farm"): + self.log.debug("Should be processed on farm, skipping.") + return ropnode = hou.node(instance.data["instance_node"]) # Get the filename from the filename parameter diff --git a/openpype/hosts/houdini/plugins/publish/extract_bgeo.py b/openpype/hosts/houdini/plugins/publish/extract_bgeo.py index d13141b426..a3840f8f73 100644 --- a/openpype/hosts/houdini/plugins/publish/extract_bgeo.py +++ b/openpype/hosts/houdini/plugins/publish/extract_bgeo.py @@ -17,7 +17,9 @@ class ExtractBGEO(publish.Extractor): families = ["bgeo"] def process(self, instance): - + if instance.data.get("farm"): + self.log.debug("Should be processed on farm, skipping.") + return ropnode = hou.node(instance.data["instance_node"]) # Get the filename from the filename parameter diff --git a/openpype/hosts/houdini/plugins/publish/extract_mantra_ifd.py b/openpype/hosts/houdini/plugins/publish/extract_mantra_ifd.py new file mode 100644 index 0000000000..894260d1bf --- /dev/null +++ b/openpype/hosts/houdini/plugins/publish/extract_mantra_ifd.py @@ -0,0 +1,51 @@ +import os + +import pyblish.api + +from openpype.pipeline import publish + +import hou + + +class ExtractMantraIFD(publish.Extractor): + + order = pyblish.api.ExtractorOrder + label = "Extract Mantra ifd" + hosts = ["houdini"] + families = ["mantraifd"] + targets = ["local", "remote"] + + def process(self, instance): + if instance.data.get("farm"): + self.log.debug("Should be processed on farm, skipping.") + return + + ropnode = hou.node(instance.data.get("instance_node")) + output = ropnode.evalParm("soho_diskfile") + staging_dir = os.path.dirname(output) + instance.data["stagingDir"] = staging_dir + + files = instance.data["frames"] + missing_frames = [ + frame + for frame in instance.data["frames"] + if not os.path.exists( + os.path.normpath(os.path.join(staging_dir, frame))) + ] + if missing_frames: + raise RuntimeError("Failed to complete Mantra ifd extraction. " + "Missing output files: {}".format( + missing_frames)) + + if "representations" not in instance.data: + instance.data["representations"] = [] + + representation = { + 'name': 'ifd', + 'ext': 'ifd', + 'files': files, + "stagingDir": staging_dir, + "frameStart": instance.data["frameStart"], + "frameEnd": instance.data["frameEnd"], + } + instance.data["representations"].append(representation) diff --git a/openpype/hosts/houdini/plugins/publish/extract_redshift_proxy.py b/openpype/hosts/houdini/plugins/publish/extract_redshift_proxy.py index ef5991924f..218f3b9256 100644 --- a/openpype/hosts/houdini/plugins/publish/extract_redshift_proxy.py +++ b/openpype/hosts/houdini/plugins/publish/extract_redshift_proxy.py @@ -14,9 +14,12 @@ class ExtractRedshiftProxy(publish.Extractor): label = "Extract Redshift Proxy" families = ["redshiftproxy"] hosts = ["houdini"] + targets = ["local", "remote"] def process(self, instance): - + if instance.data.get("farm"): + self.log.debug("Should be processed on farm, skipping.") + return ropnode = hou.node(instance.data.get("instance_node")) # Get the filename from the filename parameter diff --git a/openpype/hosts/houdini/plugins/publish/extract_vdb_cache.py b/openpype/hosts/houdini/plugins/publish/extract_vdb_cache.py index 89af8e1756..8ac16704f0 100644 --- a/openpype/hosts/houdini/plugins/publish/extract_vdb_cache.py +++ b/openpype/hosts/houdini/plugins/publish/extract_vdb_cache.py @@ -16,7 +16,9 @@ class ExtractVDBCache(publish.Extractor): hosts = ["houdini"] def process(self, instance): - + if instance.data.get("farm"): + self.log.debug("Should be processed on farm, skipping.") + return ropnode = hou.node(instance.data["instance_node"]) # Get the filename from the filename parameter diff --git a/openpype/hosts/houdini/plugins/publish/increment_current_file.py b/openpype/hosts/houdini/plugins/publish/increment_current_file.py index 3569de7693..4788cca3cf 100644 --- a/openpype/hosts/houdini/plugins/publish/increment_current_file.py +++ b/openpype/hosts/houdini/plugins/publish/increment_current_file.py @@ -22,7 +22,8 @@ class IncrementCurrentFile(pyblish.api.ContextPlugin): "arnold_rop", "mantra_rop", "karma_rop", - "usdrender"] + "usdrender", + "publish.hou"] optional = True def process(self, context): diff --git a/openpype/hosts/houdini/plugins/publish/validate_houdini_license_category.py b/openpype/hosts/houdini/plugins/publish/validate_houdini_license_category.py index 5076acda60..108a700bbe 100644 --- a/openpype/hosts/houdini/plugins/publish/validate_houdini_license_category.py +++ b/openpype/hosts/houdini/plugins/publish/validate_houdini_license_category.py @@ -20,7 +20,7 @@ class ValidateHoudiniNotApprenticeLicense(pyblish.api.InstancePlugin): """ order = pyblish.api.ValidatorOrder - families = ["usd", "abc"] + families = ["usd", "abc", "fbx", "camera"] hosts = ["houdini"] label = "Houdini Apprentice License" diff --git a/openpype/hosts/houdini/plugins/publish/validate_subset_name.py b/openpype/hosts/houdini/plugins/publish/validate_subset_name.py index bb3648f361..7bed74ebb1 100644 --- a/openpype/hosts/houdini/plugins/publish/validate_subset_name.py +++ b/openpype/hosts/houdini/plugins/publish/validate_subset_name.py @@ -54,12 +54,13 @@ class ValidateSubsetName(pyblish.api.InstancePlugin, rop_node = hou.node(instance.data["instance_node"]) # Check subset name + asset_doc = instance.data["assetEntity"] subset_name = get_subset_name( family=instance.data["family"], variant=instance.data["variant"], task_name=instance.data["task"], - asset_doc=instance.data["assetEntity"], - dynamic_data={"asset": instance.data["asset"]} + asset_doc=asset_doc, + dynamic_data={"asset": asset_doc["name"]} ) if instance.data.get("subset") != subset_name: @@ -76,12 +77,13 @@ class ValidateSubsetName(pyblish.api.InstancePlugin, rop_node = hou.node(instance.data["instance_node"]) # Check subset name + asset_doc = instance.data["assetEntity"] subset_name = get_subset_name( family=instance.data["family"], variant=instance.data["variant"], task_name=instance.data["task"], - asset_doc=instance.data["assetEntity"], - dynamic_data={"asset": instance.data["asset"]} + asset_doc=asset_doc, + dynamic_data={"asset": asset_doc["name"]} ) instance.data["subset"] = subset_name diff --git a/openpype/hosts/houdini/startup/python3.10libs/pythonrc.py b/openpype/hosts/houdini/startup/python3.10libs/pythonrc.py new file mode 100644 index 0000000000..683ea6721c --- /dev/null +++ b/openpype/hosts/houdini/startup/python3.10libs/pythonrc.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +"""OpenPype startup script.""" +from openpype.pipeline import install_host +from openpype.hosts.houdini.api import HoudiniHost + + +def main(): + print("Installing OpenPype ...") + install_host(HoudiniHost()) + + +main() diff --git a/openpype/hosts/houdini/vendor/husdoutputprocessors/__init__.py b/openpype/hosts/houdini/vendor/husdoutputprocessors/__init__.py deleted file mode 100644 index 69e3be50da..0000000000 --- a/openpype/hosts/houdini/vendor/husdoutputprocessors/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/openpype/hosts/houdini/vendor/husdoutputprocessors/avalon_uri_processor.py b/openpype/hosts/houdini/vendor/husdoutputprocessors/avalon_uri_processor.py deleted file mode 100644 index 310d057a11..0000000000 --- a/openpype/hosts/houdini/vendor/husdoutputprocessors/avalon_uri_processor.py +++ /dev/null @@ -1,152 +0,0 @@ -import os -import hou -import husdoutputprocessors.base as base - -import colorbleed.usdlib as usdlib - -from openpype.client import get_asset_by_name -from openpype.pipeline import Anatomy, get_current_project_name - - -class AvalonURIOutputProcessor(base.OutputProcessorBase): - """Process Avalon URIs into their full path equivalents. - - """ - - _parameters = None - _param_prefix = 'avalonurioutputprocessor_' - _parms = { - "use_publish_paths": _param_prefix + "use_publish_paths" - } - - def __init__(self): - """ There is only one object of each output processor class that is - ever created in a Houdini session. Therefore be very careful - about what data gets put in this object. - """ - self._use_publish_paths = False - self._cache = dict() - - def displayName(self): - return 'Avalon URI Output Processor' - - def parameters(self): - - if not self._parameters: - parameters = hou.ParmTemplateGroup() - use_publish_path = hou.ToggleParmTemplate( - name=self._parms["use_publish_paths"], - label='Resolve Reference paths to publish paths', - default_value=False, - help=("When enabled any paths for Layers, References or " - "Payloads are resolved to published master versions.\n" - "This is usually only used by the publishing pipeline, " - "but can be used for testing too.")) - parameters.append(use_publish_path) - self._parameters = parameters.asDialogScript() - - return self._parameters - - def beginSave(self, config_node, t): - parm = self._parms["use_publish_paths"] - self._use_publish_paths = config_node.parm(parm).evalAtTime(t) - self._cache.clear() - - def endSave(self): - self._use_publish_paths = None - self._cache.clear() - - def processAsset(self, - asset_path, - asset_path_for_save, - referencing_layer_path, - asset_is_layer, - for_save): - """ - Args: - asset_path (str): The incoming file path you want to alter or not. - asset_path_for_save (bool): Whether the current path is a - referenced path in the USD file. When True, return the path - you want inside USD file. - referencing_layer_path (str): ??? - asset_is_layer (bool): Whether this asset is a USD layer file. - If this is False, the asset is something else (for example, - a texture or volume file). - for_save (bool): Whether the asset path is for a file to be saved - out. If so, then return actual written filepath. - - Returns: - The refactored asset path. - - """ - - # Retrieve from cache if this query occurred before (optimization) - cache_key = (asset_path, asset_path_for_save, asset_is_layer, for_save) - if cache_key in self._cache: - return self._cache[cache_key] - - relative_template = "{asset}_{subset}.{ext}" - uri_data = usdlib.parse_avalon_uri(asset_path) - if uri_data: - - if for_save: - # Set save output path to a relative path so other - # processors can potentially manage it easily? - path = relative_template.format(**uri_data) - - print("Avalon URI Resolver: %s -> %s" % (asset_path, path)) - self._cache[cache_key] = path - return path - - if self._use_publish_paths: - # Resolve to an Avalon published asset for embedded paths - path = self._get_usd_master_path(**uri_data) - else: - path = relative_template.format(**uri_data) - - print("Avalon URI Resolver: %s -> %s" % (asset_path, path)) - self._cache[cache_key] = path - return path - - self._cache[cache_key] = asset_path - return asset_path - - def _get_usd_master_path(self, - asset, - subset, - ext): - """Get the filepath for a .usd file of a subset. - - This will return the path to an unversioned master file generated by - `usd_master_file.py`. - - """ - - PROJECT = get_current_project_name() - anatomy = Anatomy(PROJECT) - asset_doc = get_asset_by_name(PROJECT, asset) - if not asset_doc: - raise RuntimeError("Invalid asset name: '%s'" % asset) - - template_obj = anatomy.templates_obj["publish"]["path"] - path = template_obj.format_strict({ - "project": PROJECT, - "asset": asset_doc["name"], - "subset": subset, - "representation": ext, - "version": 0 # stub version zero - }) - - # Remove the version folder - subset_folder = os.path.dirname(os.path.dirname(path)) - master_folder = os.path.join(subset_folder, "master") - fname = "{0}.{1}".format(subset, ext) - - return os.path.join(master_folder, fname).replace("\\", "/") - - -output_processor = AvalonURIOutputProcessor() - - -def usdOutputProcessor(): - return output_processor diff --git a/openpype/hosts/houdini/vendor/husdoutputprocessors/stagingdir_processor.py b/openpype/hosts/houdini/vendor/husdoutputprocessors/stagingdir_processor.py deleted file mode 100644 index d8e36d5aa8..0000000000 --- a/openpype/hosts/houdini/vendor/husdoutputprocessors/stagingdir_processor.py +++ /dev/null @@ -1,90 +0,0 @@ -import hou -import husdoutputprocessors.base as base -import os - - -class StagingDirOutputProcessor(base.OutputProcessorBase): - """Output all USD Rop file nodes into the Staging Directory - - Ignore any folders and paths set in the Configured Layers - and USD Rop node, just take the filename and save into a - single directory. - - """ - theParameters = None - parameter_prefix = "stagingdiroutputprocessor_" - stagingdir_parm_name = parameter_prefix + "stagingDir" - - def __init__(self): - self.staging_dir = None - - def displayName(self): - return 'StagingDir Output Processor' - - def parameters(self): - if not self.theParameters: - parameters = hou.ParmTemplateGroup() - rootdirparm = hou.StringParmTemplate( - self.stagingdir_parm_name, - 'Staging Directory', 1, - string_type=hou.stringParmType.FileReference, - file_type=hou.fileType.Directory - ) - parameters.append(rootdirparm) - self.theParameters = parameters.asDialogScript() - return self.theParameters - - def beginSave(self, config_node, t): - - # Use the Root Directory parameter if it is set. - root_dir_parm = config_node.parm(self.stagingdir_parm_name) - if root_dir_parm: - self.staging_dir = root_dir_parm.evalAtTime(t) - - if not self.staging_dir: - out_file_parm = config_node.parm('lopoutput') - if out_file_parm: - self.staging_dir = out_file_parm.evalAtTime(t) - if self.staging_dir: - (self.staging_dir, filename) = os.path.split(self.staging_dir) - - def endSave(self): - self.staging_dir = None - - def processAsset(self, asset_path, - asset_path_for_save, - referencing_layer_path, - asset_is_layer, - for_save): - """ - Args: - asset_path (str): The incoming file path you want to alter or not. - asset_path_for_save (bool): Whether the current path is a - referenced path in the USD file. When True, return the path - you want inside USD file. - referencing_layer_path (str): ??? - asset_is_layer (bool): Whether this asset is a USD layer file. - If this is False, the asset is something else (for example, - a texture or volume file). - for_save (bool): Whether the asset path is for a file to be saved - out. If so, then return actual written filepath. - - Returns: - The refactored asset path. - - """ - - # Treat save paths as being relative to the output path. - if for_save and self.staging_dir: - # Whenever we're processing a Save Path make sure to - # resolve it to the Staging Directory - filename = os.path.basename(asset_path) - return os.path.join(self.staging_dir, filename) - - return asset_path - - -output_processor = StagingDirOutputProcessor() -def usdOutputProcessor(): - return output_processor - diff --git a/openpype/hosts/maya/api/lib.py b/openpype/hosts/maya/api/lib.py index 078ed5192b..2a9defbf2d 100644 --- a/openpype/hosts/maya/api/lib.py +++ b/openpype/hosts/maya/api/lib.py @@ -102,8 +102,6 @@ _alembic_options = { INT_FPS = {15, 24, 25, 30, 48, 50, 60, 44100, 48000} FLOAT_FPS = {23.98, 23.976, 29.97, 47.952, 59.94} -RENDERLIKE_INSTANCE_FAMILIES = ["rendering", "vrayscene"] - DISPLAY_LIGHTS_ENUM = [ {"label": "Use Project Settings", "value": "project_settings"}, @@ -3032,194 +3030,6 @@ class shelf(): cmds.shelfLayout(self.name, p="ShelfLayout") -def _get_render_instances(): - """Return all 'render-like' instances. - - This returns list of instance sets that needs to receive information - about render layer changes. - - Returns: - list: list of instances - - """ - objectset = cmds.ls("*.id", long=True, exactType="objectSet", - recursive=True, objectsOnly=True) - - instances = [] - for objset in objectset: - if not cmds.attributeQuery("id", node=objset, exists=True): - continue - - id_attr = "{}.id".format(objset) - if cmds.getAttr(id_attr) != "pyblish.avalon.instance": - continue - - has_family = cmds.attributeQuery("family", - node=objset, - exists=True) - if not has_family: - continue - - if cmds.getAttr( - "{}.family".format(objset)) in RENDERLIKE_INSTANCE_FAMILIES: - instances.append(objset) - - return instances - - -renderItemObserverList = [] - - -class RenderSetupListObserver: - """Observer to catch changes in render setup layers.""" - - def listItemAdded(self, item): - print("--- adding ...") - self._add_render_layer(item) - - def listItemRemoved(self, item): - print("--- removing ...") - self._remove_render_layer(item.name()) - - def _add_render_layer(self, item): - render_sets = _get_render_instances() - layer_name = item.name() - - for render_set in render_sets: - members = cmds.sets(render_set, query=True) or [] - - namespace_name = "_{}".format(render_set) - if not cmds.namespace(exists=namespace_name): - index = 1 - namespace_name = "_{}".format(render_set) - try: - cmds.namespace(rm=namespace_name) - except RuntimeError: - # namespace is not empty, so we leave it untouched - pass - orignal_namespace_name = namespace_name - while(cmds.namespace(exists=namespace_name)): - namespace_name = "{}{}".format( - orignal_namespace_name, index) - index += 1 - - namespace = cmds.namespace(add=namespace_name) - - if members: - # if set already have namespaced members, use the same - # namespace as others. - namespace = members[0].rpartition(":")[0] - else: - namespace = namespace_name - - render_layer_set_name = "{}:{}".format(namespace, layer_name) - if render_layer_set_name in members: - continue - print(" - creating set for {}".format(layer_name)) - maya_set = cmds.sets(n=render_layer_set_name, empty=True) - cmds.sets(maya_set, forceElement=render_set) - rio = RenderSetupItemObserver(item) - print("- adding observer for {}".format(item.name())) - item.addItemObserver(rio.itemChanged) - renderItemObserverList.append(rio) - - def _remove_render_layer(self, layer_name): - render_sets = _get_render_instances() - - for render_set in render_sets: - members = cmds.sets(render_set, query=True) - if not members: - continue - - # all sets under set should have the same namespace - namespace = members[0].rpartition(":")[0] - render_layer_set_name = "{}:{}".format(namespace, layer_name) - - if render_layer_set_name in members: - print(" - removing set for {}".format(layer_name)) - cmds.delete(render_layer_set_name) - - -class RenderSetupItemObserver: - """Handle changes in render setup items.""" - - def __init__(self, item): - self.item = item - self.original_name = item.name() - - def itemChanged(self, *args, **kwargs): - """Item changed callback.""" - if self.item.name() == self.original_name: - return - - render_sets = _get_render_instances() - - for render_set in render_sets: - members = cmds.sets(render_set, query=True) - if not members: - continue - - # all sets under set should have the same namespace - namespace = members[0].rpartition(":")[0] - render_layer_set_name = "{}:{}".format( - namespace, self.original_name) - - if render_layer_set_name in members: - print(" <> renaming {} to {}".format(self.original_name, - self.item.name())) - cmds.rename(render_layer_set_name, - "{}:{}".format( - namespace, self.item.name())) - self.original_name = self.item.name() - - -renderListObserver = RenderSetupListObserver() - - -def add_render_layer_change_observer(): - import maya.app.renderSetup.model.renderSetup as renderSetup - - rs = renderSetup.instance() - render_sets = _get_render_instances() - - layers = rs.getRenderLayers() - for render_set in render_sets: - members = cmds.sets(render_set, query=True) - if not members: - continue - # all sets under set should have the same namespace - namespace = members[0].rpartition(":")[0] - for layer in layers: - render_layer_set_name = "{}:{}".format(namespace, layer.name()) - if render_layer_set_name not in members: - continue - rio = RenderSetupItemObserver(layer) - print("- adding observer for {}".format(layer.name())) - layer.addItemObserver(rio.itemChanged) - renderItemObserverList.append(rio) - - -def add_render_layer_observer(): - import maya.app.renderSetup.model.renderSetup as renderSetup - - print("> adding renderSetup observer ...") - rs = renderSetup.instance() - rs.addListObserver(renderListObserver) - pass - - -def remove_render_layer_observer(): - import maya.app.renderSetup.model.renderSetup as renderSetup - - print("< removing renderSetup observer ...") - rs = renderSetup.instance() - try: - rs.removeListObserver(renderListObserver) - except ValueError: - # no observer set yet - pass - - def update_content_on_context_change(): """ This will update scene content to match new asset on context change diff --git a/openpype/hosts/maya/api/lib_rendersettings.py b/openpype/hosts/maya/api/lib_rendersettings.py index 8b57c2e481..1f964589a9 100644 --- a/openpype/hosts/maya/api/lib_rendersettings.py +++ b/openpype/hosts/maya/api/lib_rendersettings.py @@ -70,8 +70,8 @@ class RenderSettings(object): def set_default_renderer_settings(self, renderer=None): """Set basic settings based on renderer.""" # Not all hosts can import this module. - from maya import cmds - import maya.mel as mel + from maya import cmds # noqa: F401 + import maya.mel as mel # noqa: F401 if not renderer: renderer = cmds.getAttr( @@ -126,6 +126,10 @@ class RenderSettings(object): """Sets settings for Arnold.""" from mtoa.core import createOptions # noqa from mtoa.aovs import AOVInterface # noqa + # Not all hosts can import this module. + from maya import cmds # noqa: F401 + import maya.mel as mel # noqa: F401 + createOptions() render_settings = self._project_settings["maya"]["RenderSettings"] arnold_render_presets = render_settings["arnold_renderer"] # noqa @@ -172,6 +176,10 @@ class RenderSettings(object): def _set_redshift_settings(self, width, height): """Sets settings for Redshift.""" + # Not all hosts can import this module. + from maya import cmds # noqa: F401 + import maya.mel as mel # noqa: F401 + render_settings = self._project_settings["maya"]["RenderSettings"] redshift_render_presets = render_settings["redshift_renderer"] @@ -224,6 +232,10 @@ class RenderSettings(object): def _set_renderman_settings(self, width, height, aov_separator): """Sets settings for Renderman""" + # Not all hosts can import this module. + from maya import cmds # noqa: F401 + import maya.mel as mel # noqa: F401 + rman_render_presets = ( self._project_settings ["maya"] @@ -285,6 +297,11 @@ class RenderSettings(object): def _set_vray_settings(self, aov_separator, width, height): # type: (str, int, int) -> None """Sets important settings for Vray.""" + # Not all hosts can import this module. + from maya import cmds # noqa: F401 + import maya.mel as mel # noqa: F401 + + settings = cmds.ls(type="VRaySettingsNode") node = settings[0] if settings else cmds.createNode("VRaySettingsNode") render_settings = self._project_settings["maya"]["RenderSettings"] @@ -357,6 +374,10 @@ class RenderSettings(object): @staticmethod def _set_global_output_settings(): + # Not all hosts can import this module. + from maya import cmds # noqa: F401 + import maya.mel as mel # noqa: F401 + # enable animation cmds.setAttr("defaultRenderGlobals.outFormatControl", 0) cmds.setAttr("defaultRenderGlobals.animation", 1) @@ -364,6 +385,10 @@ class RenderSettings(object): cmds.setAttr("defaultRenderGlobals.extensionPadding", 4) def _additional_attribs_setter(self, additional_attribs): + # Not all hosts can import this module. + from maya import cmds # noqa: F401 + import maya.mel as mel # noqa: F401 + for item in additional_attribs: attribute, value = item attribute = str(attribute) # ensure str conversion from settings diff --git a/openpype/hosts/maya/api/pipeline.py b/openpype/hosts/maya/api/pipeline.py index 6b791c9665..1ecfdfaa40 100644 --- a/openpype/hosts/maya/api/pipeline.py +++ b/openpype/hosts/maya/api/pipeline.py @@ -580,20 +580,11 @@ def on_save(): lib.set_id(node, new_id, overwrite=False) -def _update_render_layer_observers(): - # Helper to trigger update for all renderlayer observer logic - lib.remove_render_layer_observer() - lib.add_render_layer_observer() - lib.add_render_layer_change_observer() - - def on_open(): """On scene open let's assume the containers have changed.""" from openpype.widgets import popup - utils.executeDeferred(_update_render_layer_observers) - # Validate FPS after update_task_from_path to # ensure it is using correct FPS for the asset lib.validate_fps() @@ -630,7 +621,6 @@ def on_new(): with lib.suspended_refresh(): lib.set_context_settings() - utils.executeDeferred(_update_render_layer_observers) _remove_workfile_lock() diff --git a/openpype/hosts/maya/api/plugin.py b/openpype/hosts/maya/api/plugin.py index 212c4df492..e684a91fe2 100644 --- a/openpype/hosts/maya/api/plugin.py +++ b/openpype/hosts/maya/api/plugin.py @@ -7,6 +7,7 @@ import six from maya import cmds from maya.app.renderSetup.model import renderSetup +from openpype import AYON_SERVER_ENABLED from openpype.lib import BoolDef, Logger from openpype.settings import get_project_settings from openpype.pipeline import ( @@ -449,14 +450,16 @@ class RenderlayerCreator(NewCreator, MayaCreatorBase): # this instance will not have the `instance_node` data yet # until it's been saved/persisted at least once. project_name = self.create_context.get_current_project_name() - + asset_name = self.create_context.get_current_asset_name() instance_data = { - "asset": self.create_context.get_current_asset_name(), "task": self.create_context.get_current_task_name(), "variant": layer.name(), } - asset_doc = get_asset_by_name(project_name, - instance_data["asset"]) + if AYON_SERVER_ENABLED: + instance_data["folderPath"] = asset_name + else: + instance_data["asset"] = asset_name + asset_doc = get_asset_by_name(project_name, asset_name) subset_name = self.get_subset_name( layer.name(), instance_data["task"], diff --git a/openpype/hosts/maya/plugins/create/create_multishot_layout.py b/openpype/hosts/maya/plugins/create/create_multishot_layout.py index 0b027c02ea..8f5c423202 100644 --- a/openpype/hosts/maya/plugins/create/create_multishot_layout.py +++ b/openpype/hosts/maya/plugins/create/create_multishot_layout.py @@ -45,10 +45,14 @@ class CreateMultishotLayout(plugin.MayaCreator): above is done. """ - current_folder = get_folder_by_name( - project_name=get_current_project_name(), - folder_name=get_current_asset_name(), - ) + project_name = get_current_project_name() + folder_path = get_current_asset_name() + if "/" in folder_path: + current_folder = get_folder_by_path(project_name, folder_path) + else: + current_folder = get_folder_by_name( + project_name, folder_name=folder_path + ) current_path_parts = current_folder["path"].split("/") @@ -154,7 +158,7 @@ class CreateMultishotLayout(plugin.MayaCreator): # Create layout instance by the layout creator instance_data = { - "asset": shot["name"], + "folderPath": shot["path"], "variant": layout_creator.get_default_variant() } if layout_task: diff --git a/openpype/hosts/maya/plugins/create/create_review.py b/openpype/hosts/maya/plugins/create/create_review.py index f60e2406bc..18d661b186 100644 --- a/openpype/hosts/maya/plugins/create/create_review.py +++ b/openpype/hosts/maya/plugins/create/create_review.py @@ -2,6 +2,7 @@ import json from maya import cmds +from openpype import AYON_SERVER_ENABLED from openpype.hosts.maya.api import ( lib, plugin @@ -43,7 +44,11 @@ class CreateReview(plugin.MayaCreator): members = cmds.ls(selection=True) project_name = self.project_name - asset_doc = get_asset_by_name(project_name, instance_data["asset"]) + if AYON_SERVER_ENABLED: + asset_name = instance_data["folderPath"] + else: + asset_name = instance_data["asset"] + asset_doc = get_asset_by_name(project_name, asset_name) task_name = instance_data["task"] preset = lib.get_capture_preset( task_name, diff --git a/openpype/hosts/maya/plugins/create/create_unreal_skeletalmesh.py b/openpype/hosts/maya/plugins/create/create_unreal_skeletalmesh.py index 3c9a79156a..b4151bac99 100644 --- a/openpype/hosts/maya/plugins/create/create_unreal_skeletalmesh.py +++ b/openpype/hosts/maya/plugins/create/create_unreal_skeletalmesh.py @@ -51,7 +51,7 @@ class CreateUnrealSkeletalMesh(plugin.MayaCreator): # We reorganize the geometry that was originally added into the # set into either 'joints_SET' or 'geometry_SET' based on the # joint_hints from project settings - members = cmds.sets(instance_node, query=True) + members = cmds.sets(instance_node, query=True) or [] cmds.sets(clear=instance_node) geometry_set = cmds.sets(name="geometry_SET", empty=True) diff --git a/openpype/hosts/maya/plugins/create/create_workfile.py b/openpype/hosts/maya/plugins/create/create_workfile.py index d84753cd7f..198f9c4a36 100644 --- a/openpype/hosts/maya/plugins/create/create_workfile.py +++ b/openpype/hosts/maya/plugins/create/create_workfile.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- """Creator plugin for creating workfiles.""" +from openpype import AYON_SERVER_ENABLED from openpype.pipeline import CreatedInstance, AutoCreator -from openpype.client import get_asset_by_name +from openpype.client import get_asset_by_name, get_asset_name_identifier from openpype.hosts.maya.api import plugin from maya import cmds @@ -29,16 +30,27 @@ class CreateWorkfile(plugin.MayaCreatorBase, AutoCreator): task_name = self.create_context.get_current_task_name() host_name = self.create_context.host_name + if current_instance is None: + current_instance_asset = None + elif AYON_SERVER_ENABLED: + current_instance_asset = current_instance["folderPath"] + else: + current_instance_asset = current_instance["asset"] + if current_instance is None: asset_doc = get_asset_by_name(project_name, asset_name) subset_name = self.get_subset_name( variant, task_name, asset_doc, project_name, host_name ) data = { - "asset": asset_name, "task": task_name, "variant": variant } + if AYON_SERVER_ENABLED: + data["folderPath"] = asset_name + else: + data["asset"] = asset_name + data.update( self.get_dynamic_data( variant, task_name, asset_doc, @@ -50,15 +62,20 @@ class CreateWorkfile(plugin.MayaCreatorBase, AutoCreator): ) self._add_instance_to_context(current_instance) elif ( - current_instance["asset"] != asset_name - or current_instance["task"] != task_name + current_instance_asset != asset_name + or current_instance["task"] != task_name ): # Update instance context if is not the same asset_doc = get_asset_by_name(project_name, asset_name) subset_name = self.get_subset_name( variant, task_name, asset_doc, project_name, host_name ) - current_instance["asset"] = asset_name + asset_name = get_asset_name_identifier(asset_doc) + + if AYON_SERVER_ENABLED: + current_instance["folderPath"] = asset_name + else: + current_instance["asset"] = asset_name current_instance["task"] = task_name current_instance["subset"] = subset_name diff --git a/openpype/hosts/maya/plugins/publish/collect_review.py b/openpype/hosts/maya/plugins/publish/collect_review.py index 586939a3b8..0930da8f27 100644 --- a/openpype/hosts/maya/plugins/publish/collect_review.py +++ b/openpype/hosts/maya/plugins/publish/collect_review.py @@ -3,7 +3,7 @@ from maya import cmds, mel import pyblish.api from openpype.client import get_subset_by_name -from openpype.pipeline import legacy_io, KnownPublishError +from openpype.pipeline import KnownPublishError from openpype.hosts.maya.api import lib @@ -116,10 +116,10 @@ class CollectReview(pyblish.api.InstancePlugin): instance.data['remove'] = True else: - task = legacy_io.Session["AVALON_TASK"] - legacy_subset_name = task + 'Review' + project_name = instance.context.data["projectName"] asset_doc = instance.context.data['assetEntity'] - project_name = legacy_io.active_project() + task = instance.context.data["task"] + legacy_subset_name = task + 'Review' subset_doc = get_subset_by_name( project_name, legacy_subset_name, diff --git a/openpype/hosts/maya/plugins/publish/extract_unreal_skeletalmesh_abc.py b/openpype/hosts/maya/plugins/publish/extract_unreal_skeletalmesh_abc.py index 9c2f55a1ef..780ed2377c 100644 --- a/openpype/hosts/maya/plugins/publish/extract_unreal_skeletalmesh_abc.py +++ b/openpype/hosts/maya/plugins/publish/extract_unreal_skeletalmesh_abc.py @@ -13,16 +13,6 @@ from openpype.hosts.maya.api.lib import ( ) -@contextmanager -def renamed(original_name, renamed_name): - # type: (str, str) -> None - try: - cmds.rename(original_name, renamed_name) - yield - finally: - cmds.rename(renamed_name, original_name) - - class ExtractUnrealSkeletalMeshAbc(publish.Extractor): """Extract Unreal Skeletal Mesh as FBX from Maya. """ diff --git a/openpype/hosts/maya/plugins/publish/extract_unreal_skeletalmesh_fbx.py b/openpype/hosts/maya/plugins/publish/extract_unreal_skeletalmesh_fbx.py index 96175a07d7..4b36134694 100644 --- a/openpype/hosts/maya/plugins/publish/extract_unreal_skeletalmesh_fbx.py +++ b/openpype/hosts/maya/plugins/publish/extract_unreal_skeletalmesh_fbx.py @@ -62,6 +62,10 @@ class ExtractUnrealSkeletalMeshFbx(publish.Extractor): original_parent = to_extract[0].split("|")[1] parent_node = instance.data.get("asset") + # this needs to be done for AYON + # WARNING: since AYON supports duplicity of asset names, + # this needs to be refactored throughout the pipeline. + parent_node = parent_node.split("/")[-1] renamed_to_extract = [] for node in to_extract: diff --git a/openpype/hosts/maya/plugins/publish/validate_instance_in_context.py b/openpype/hosts/maya/plugins/publish/validate_instance_in_context.py index 4ded57137c..4222e63898 100644 --- a/openpype/hosts/maya/plugins/publish/validate_instance_in_context.py +++ b/openpype/hosts/maya/plugins/publish/validate_instance_in_context.py @@ -3,6 +3,7 @@ from __future__ import absolute_import import pyblish.api +from openpype import AYON_SERVER_ENABLED import openpype.hosts.maya.api.action from openpype.pipeline.publish import ( RepairAction, @@ -66,12 +67,16 @@ class ValidateInstanceInContext(pyblish.api.InstancePlugin, def repair(cls, instance): context_asset = cls.get_context_asset(instance) instance_node = instance.data["instance_node"] + if AYON_SERVER_ENABLED: + asset_name_attr = "folderPath" + else: + asset_name_attr = "asset" cmds.setAttr( - "{}.asset".format(instance_node), + "{}.{}".format(instance_node, asset_name_attr), context_asset, type="string" ) @staticmethod def get_context_asset(instance): - return instance.context.data["assetEntity"]["name"] + return instance.context.data["asset"] diff --git a/openpype/hosts/maya/plugins/publish/validate_model_name.py b/openpype/hosts/maya/plugins/publish/validate_model_name.py index f4c1aa39c7..11f59bb439 100644 --- a/openpype/hosts/maya/plugins/publish/validate_model_name.py +++ b/openpype/hosts/maya/plugins/publish/validate_model_name.py @@ -67,13 +67,15 @@ class ValidateModelName(pyblish.api.InstancePlugin, regex = cls.top_level_regex r = re.compile(regex) m = r.match(top_group) + project_name = instance.context.data["projectName"] + current_asset_name = instance.context.data["asset"] if m is None: cls.log.error("invalid name on: {}".format(top_group)) cls.log.error("name doesn't match regex {}".format(regex)) invalid.append(top_group) else: if "asset" in r.groupindex: - if m.group("asset") != legacy_io.Session["AVALON_ASSET"]: + if m.group("asset") != current_asset_name: cls.log.error("Invalid asset name in top level group.") return top_group if "subset" in r.groupindex: @@ -81,7 +83,7 @@ class ValidateModelName(pyblish.api.InstancePlugin, cls.log.error("Invalid subset name in top level group.") return top_group if "project" in r.groupindex: - if m.group("project") != legacy_io.Session["AVALON_PROJECT"]: + if m.group("project") != project_name: cls.log.error("Invalid project name in top level group.") return top_group diff --git a/openpype/hosts/maya/plugins/publish/validate_shader_name.py b/openpype/hosts/maya/plugins/publish/validate_shader_name.py index 36bb2c1fee..d6486dea7f 100644 --- a/openpype/hosts/maya/plugins/publish/validate_shader_name.py +++ b/openpype/hosts/maya/plugins/publish/validate_shader_name.py @@ -51,7 +51,7 @@ class ValidateShaderName(pyblish.api.InstancePlugin, descendants = cmds.ls(descendants, noIntermediate=True, long=True) shapes = cmds.ls(descendants, type=["nurbsSurface", "mesh"], long=True) - asset_name = instance.data.get("asset", None) + asset_name = instance.data.get("asset") # Check the number of connected shadingEngines per shape regex_compile = re.compile(cls.regex) diff --git a/openpype/hosts/maya/plugins/publish/validate_unreal_staticmesh_naming.py b/openpype/hosts/maya/plugins/publish/validate_unreal_staticmesh_naming.py index 58fa9d02bd..42d3dc3ac8 100644 --- a/openpype/hosts/maya/plugins/publish/validate_unreal_staticmesh_naming.py +++ b/openpype/hosts/maya/plugins/publish/validate_unreal_staticmesh_naming.py @@ -102,7 +102,8 @@ class ValidateUnrealStaticMeshName(pyblish.api.InstancePlugin, cl_r = re.compile(regex_collision) - mesh_name = "{}{}".format(instance.data["asset"], + asset_name = instance.data["assetEntity"]["name"] + mesh_name = "{}{}".format(asset_name, instance.data.get("variant", [])) for obj in collision_set: diff --git a/openpype/hosts/maya/tools/mayalookassigner/commands.py b/openpype/hosts/maya/tools/mayalookassigner/commands.py index 5cc4f84931..86df502ecd 100644 --- a/openpype/hosts/maya/tools/mayalookassigner/commands.py +++ b/openpype/hosts/maya/tools/mayalookassigner/commands.py @@ -4,7 +4,7 @@ from collections import defaultdict import maya.cmds as cmds -from openpype.client import get_assets +from openpype.client import get_assets, get_asset_name_identifier from openpype.pipeline import ( remove_container, registered_host, @@ -128,7 +128,8 @@ def create_items_from_nodes(nodes): project_name = get_current_project_name() asset_ids = set(id_hashes.keys()) - asset_docs = get_assets(project_name, asset_ids, fields=["name"]) + fields = {"_id", "name", "data.parents"} + asset_docs = get_assets(project_name, asset_ids, fields=fields) asset_docs_by_id = { str(asset_doc["_id"]): asset_doc for asset_doc in asset_docs @@ -156,8 +157,9 @@ def create_items_from_nodes(nodes): namespace = get_namespace_from_node(node) namespaces.add(namespace) + label = get_asset_name_identifier(asset_doc) asset_view_items.append({ - "label": asset_doc["name"], + "label": label, "asset": asset_doc, "looks": looks, "namespaces": namespaces diff --git a/openpype/hosts/maya/tools/mayalookassigner/widgets.py b/openpype/hosts/maya/tools/mayalookassigner/widgets.py index 82c37e2104..ef29a4c726 100644 --- a/openpype/hosts/maya/tools/mayalookassigner/widgets.py +++ b/openpype/hosts/maya/tools/mayalookassigner/widgets.py @@ -3,6 +3,7 @@ from collections import defaultdict from qtpy import QtWidgets, QtCore +from openpype.client import get_asset_name_identifier from openpype.tools.utils.models import TreeModel from openpype.tools.utils.lib import ( preserve_expanded_rows, @@ -126,7 +127,7 @@ class AssetOutliner(QtWidgets.QWidget): asset_namespaces = defaultdict(set) for item in items: asset_id = str(item["asset"]["_id"]) - asset_name = item["asset"]["name"] + asset_name = get_asset_name_identifier(item["asset"]) asset_namespaces[asset_name].add(item.get("namespace")) if asset_name in assets: diff --git a/openpype/hosts/nuke/api/lib.py b/openpype/hosts/nuke/api/lib.py index de5d13d347..88c587faf6 100644 --- a/openpype/hosts/nuke/api/lib.py +++ b/openpype/hosts/nuke/api/lib.py @@ -13,6 +13,7 @@ from collections import OrderedDict import nuke from qtpy import QtCore, QtWidgets +from openpype import AYON_SERVER_ENABLED from openpype.client import ( get_project, get_asset_by_name, @@ -1107,7 +1108,9 @@ def format_anatomy(data): Return: path (str) ''' - anatomy = Anatomy() + + project_name = get_current_project_name() + anatomy = Anatomy(project_name) log.debug("__ anatomy.templates: {}".format(anatomy.templates)) padding = None @@ -1125,8 +1128,10 @@ def format_anatomy(data): file = script_name() data["version"] = get_version_from_path(file) - project_name = anatomy.project_name - asset_name = data["asset"] + if AYON_SERVER_ENABLED: + asset_name = data["folderPath"] + else: + asset_name = data["asset"] task_name = data["task"] host_name = get_current_host_name() context_data = get_template_data_with_names( diff --git a/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py b/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py index 9c8bfae388..0539c1188f 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py +++ b/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py @@ -111,7 +111,6 @@ class ValidateNukeWriteNode( for value in values: if type(node_value) in (int, float): try: - if isinstance(value, list): value = color_gui_to_int(value) else: @@ -130,7 +129,7 @@ class ValidateNukeWriteNode( and key != "file" and key != "tile_color" ): - check.append([key, value, write_node[key].value()]) + check.append([key, node_value, write_node[key].value()]) if check: self._make_error(check) diff --git a/openpype/hosts/photoshop/lib.py b/openpype/hosts/photoshop/lib.py index 9f603a70d2..5c8dff947d 100644 --- a/openpype/hosts/photoshop/lib.py +++ b/openpype/hosts/photoshop/lib.py @@ -1,5 +1,6 @@ import re +from openpype import AYON_SERVER_ENABLED import openpype.hosts.photoshop.api as api from openpype.client import get_asset_by_name from openpype.lib import prepare_template_data @@ -43,6 +44,14 @@ class PSAutoCreator(AutoCreator): asset_name = context.get_current_asset_name() task_name = context.get_current_task_name() host_name = context.host_name + + if existing_instance is None: + existing_instance_asset = None + elif AYON_SERVER_ENABLED: + existing_instance_asset = existing_instance["folderPath"] + else: + existing_instance_asset = existing_instance["asset"] + if existing_instance is None: asset_doc = get_asset_by_name(project_name, asset_name) subset_name = self.get_subset_name( @@ -50,10 +59,13 @@ class PSAutoCreator(AutoCreator): project_name, host_name ) data = { - "asset": asset_name, "task": task_name, "variant": self.default_variant } + if AYON_SERVER_ENABLED: + data["folderPath"] = asset_name + else: + data["asset"] = asset_name data.update(self.get_dynamic_data( self.default_variant, task_name, asset_doc, project_name, host_name, None @@ -70,7 +82,7 @@ class PSAutoCreator(AutoCreator): new_instance.data_to_store()) elif ( - existing_instance["asset"] != asset_name + existing_instance_asset != asset_name or existing_instance["task"] != task_name ): asset_doc = get_asset_by_name(project_name, asset_name) @@ -78,7 +90,10 @@ class PSAutoCreator(AutoCreator): self.default_variant, task_name, asset_doc, project_name, host_name ) - existing_instance["asset"] = asset_name + if AYON_SERVER_ENABLED: + existing_instance["folderPath"] = asset_name + else: + existing_instance["asset"] = asset_name existing_instance["task"] = task_name existing_instance["subset"] = subset_name diff --git a/openpype/hosts/photoshop/plugins/create/create_flatten_image.py b/openpype/hosts/photoshop/plugins/create/create_flatten_image.py index afde77fdb4..24be9df0e0 100644 --- a/openpype/hosts/photoshop/plugins/create/create_flatten_image.py +++ b/openpype/hosts/photoshop/plugins/create/create_flatten_image.py @@ -1,5 +1,6 @@ from openpype.pipeline import CreatedInstance +from openpype import AYON_SERVER_ENABLED from openpype.lib import BoolDef import openpype.hosts.photoshop.api as api from openpype.hosts.photoshop.lib import PSAutoCreator, clean_subset_name @@ -37,6 +38,13 @@ class AutoImageCreator(PSAutoCreator): host_name = context.host_name asset_doc = get_asset_by_name(project_name, asset_name) + if existing_instance is None: + existing_instance_asset = None + elif AYON_SERVER_ENABLED: + existing_instance_asset = existing_instance["folderPath"] + else: + existing_instance_asset = existing_instance["asset"] + if existing_instance is None: subset_name = self.get_subset_name( self.default_variant, task_name, asset_doc, @@ -44,9 +52,12 @@ class AutoImageCreator(PSAutoCreator): ) data = { - "asset": asset_name, "task": task_name, } + if AYON_SERVER_ENABLED: + data["folderPath"] = asset_name + else: + data["asset"] = asset_name if not self.active_on_create: data["active"] = False @@ -62,15 +73,17 @@ class AutoImageCreator(PSAutoCreator): new_instance.data_to_store()) elif ( # existing instance from different context - existing_instance["asset"] != asset_name + existing_instance_asset != asset_name or existing_instance["task"] != task_name ): subset_name = self.get_subset_name( self.default_variant, task_name, asset_doc, project_name, host_name ) - - existing_instance["asset"] = asset_name + if AYON_SERVER_ENABLED: + existing_instance["folderPath"] = asset_name + else: + existing_instance["asset"] = asset_name existing_instance["task"] = task_name existing_instance["subset"] = subset_name diff --git a/openpype/hosts/photoshop/plugins/publish/collect_auto_image.py b/openpype/hosts/photoshop/plugins/publish/collect_auto_image.py index d4b5f480b1..4d7838c510 100644 --- a/openpype/hosts/photoshop/plugins/publish/collect_auto_image.py +++ b/openpype/hosts/photoshop/plugins/publish/collect_auto_image.py @@ -1,5 +1,6 @@ import pyblish.api +from openpype.client import get_asset_name_identifier from openpype.hosts.photoshop import api as photoshop from openpype.pipeline.create import get_subset_name @@ -27,7 +28,7 @@ class CollectAutoImage(pyblish.api.ContextPlugin): task_name = context.data["task"] host_name = context.data["hostName"] asset_doc = context.data["assetEntity"] - asset_name = asset_doc["name"] + asset_name = get_asset_name_identifier(asset_doc) auto_creator = proj_settings.get( "photoshop", {}).get( diff --git a/openpype/hosts/photoshop/plugins/publish/collect_auto_review.py b/openpype/hosts/photoshop/plugins/publish/collect_auto_review.py index 8964582a45..e5a2f326d7 100644 --- a/openpype/hosts/photoshop/plugins/publish/collect_auto_review.py +++ b/openpype/hosts/photoshop/plugins/publish/collect_auto_review.py @@ -7,6 +7,7 @@ Provides: """ import pyblish.api +from openpype.client import get_asset_name_identifier from openpype.hosts.photoshop import api as photoshop from openpype.pipeline.create import get_subset_name @@ -65,7 +66,8 @@ class CollectAutoReview(pyblish.api.ContextPlugin): task_name = context.data["task"] host_name = context.data["hostName"] asset_doc = context.data["assetEntity"] - asset_name = asset_doc["name"] + + asset_name = get_asset_name_identifier(asset_doc) subset_name = get_subset_name( family, diff --git a/openpype/hosts/photoshop/plugins/publish/collect_auto_workfile.py b/openpype/hosts/photoshop/plugins/publish/collect_auto_workfile.py index d3cc8d44d0..9ccb8f4f85 100644 --- a/openpype/hosts/photoshop/plugins/publish/collect_auto_workfile.py +++ b/openpype/hosts/photoshop/plugins/publish/collect_auto_workfile.py @@ -1,6 +1,7 @@ import os import pyblish.api +from openpype.client import get_asset_name_identifier from openpype.hosts.photoshop import api as photoshop from openpype.pipeline.create import get_subset_name @@ -69,8 +70,8 @@ class CollectAutoWorkfile(pyblish.api.ContextPlugin): task_name = context.data["task"] host_name = context.data["hostName"] asset_doc = context.data["assetEntity"] - asset_name = asset_doc["name"] + asset_name = get_asset_name_identifier(asset_doc) subset_name = get_subset_name( family, variant, diff --git a/openpype/hosts/resolve/api/plugin.py b/openpype/hosts/resolve/api/plugin.py index 5c4a92df89..197f288150 100644 --- a/openpype/hosts/resolve/api/plugin.py +++ b/openpype/hosts/resolve/api/plugin.py @@ -1,10 +1,11 @@ import re import uuid +import copy + import qargparse from qtpy import QtWidgets, QtCore from openpype.settings import get_current_project_settings -from openpype.pipeline.context_tools import get_current_project_asset from openpype.pipeline import ( LegacyCreator, LoaderPlugin, @@ -18,7 +19,7 @@ from .menu import load_stylesheet class CreatorWidget(QtWidgets.QDialog): # output items - items = dict() + items = {} def __init__(self, name, info, ui_inputs, parent=None): super(CreatorWidget, self).__init__(parent) @@ -100,7 +101,7 @@ class CreatorWidget(QtWidgets.QDialog): self.close() def value(self, data, new_data=None): - new_data = new_data or dict() + new_data = new_data or {} for k, v in data.items(): new_data[k] = { "target": None, @@ -289,7 +290,7 @@ class Spacer(QtWidgets.QWidget): class ClipLoader: active_bin = None - data = dict() + data = {} def __init__(self, loader_obj, context, **options): """ Initialize object @@ -386,8 +387,8 @@ class ClipLoader: joint `data` key with asset.data dict into the representation """ - asset_name = self.context["representation"]["context"]["asset"] - self.data["assetData"] = get_current_project_asset(asset_name)["data"] + + self.data["assetData"] = copy.deepcopy(self.context["asset"]["data"]) def load(self, files): """Load clip into timeline @@ -587,8 +588,8 @@ class PublishClip: Returns: hiero.core.TrackItem: hiero track item object with openpype tag """ - vertical_clip_match = dict() - tag_data = dict() + vertical_clip_match = {} + tag_data = {} types = { "shot": "shot", "folder": "folder", @@ -664,15 +665,23 @@ class PublishClip: new_name = self.tag_data.pop("newClipName") if self.rename: - self.tag_data["asset"] = new_name + self.tag_data["asset_name"] = new_name else: - self.tag_data["asset"] = self.ti_name + self.tag_data["asset_name"] = self.ti_name + # AYON unique identifier + folder_path = "/{}/{}".format( + self.tag_data["hierarchy"], + self.tag_data["asset_name"] + ) + self.tag_data["folder_path"] = folder_path + + # create new name for track item if not lib.pype_marker_workflow: # create compound clip workflow lib.create_compound_clip( self.timeline_item_data, - self.tag_data["asset"], + self.tag_data["asset_name"], self.mp_folder ) @@ -764,7 +773,7 @@ class PublishClip: # increasing steps by index of rename iteration self.count_steps *= self.rename_index - hierarchy_formatting_data = dict() + hierarchy_formatting_data = {} _data = self.timeline_item_default_data.copy() if self.ui_inputs: # adding tag metadata from ui @@ -853,8 +862,7 @@ class PublishClip: "parents": self.parents, "hierarchyData": hierarchy_formatting_data, "subset": self.subset, - "family": self.subset_family, - "families": ["clip"] + "family": self.subset_family } def _convert_to_entity(self, key): diff --git a/openpype/hosts/resolve/plugins/publish/extract_workfile.py b/openpype/hosts/resolve/plugins/publish/extract_workfile.py index 535f879b58..db63487405 100644 --- a/openpype/hosts/resolve/plugins/publish/extract_workfile.py +++ b/openpype/hosts/resolve/plugins/publish/extract_workfile.py @@ -26,6 +26,7 @@ class ExtractWorkfile(publish.Extractor): resolve_workfile_ext = ".drp" drp_file_name = name + resolve_workfile_ext + drp_file_path = os.path.normpath( os.path.join(staging_dir, drp_file_name)) diff --git a/openpype/hosts/resolve/plugins/publish/precollect_instances.py b/openpype/hosts/resolve/plugins/publish/precollect_instances.py index 8ec169ad65..bca6734848 100644 --- a/openpype/hosts/resolve/plugins/publish/precollect_instances.py +++ b/openpype/hosts/resolve/plugins/publish/precollect_instances.py @@ -9,6 +9,7 @@ from openpype.hosts.resolve.api.lib import ( get_publish_attribute, get_otio_clip_instance_data, ) +from openpype import AYON_SERVER_ENABLED class PrecollectInstances(pyblish.api.ContextPlugin): @@ -29,7 +30,7 @@ class PrecollectInstances(pyblish.api.ContextPlugin): for timeline_item_data in selected_timeline_items: - data = dict() + data = {} timeline_item = timeline_item_data["clip"]["item"] # get pype tag data @@ -60,24 +61,24 @@ class PrecollectInstances(pyblish.api.ContextPlugin): if k not in ("id", "applieswhole", "label") }) - asset = tag_data["asset"] + if AYON_SERVER_ENABLED: + asset = tag_data["folder_path"] + else: + asset = tag_data["asset_name"] + subset = tag_data["subset"] - # insert family into families - family = tag_data["family"] - families = [str(f) for f in tag_data["families"]] - families.insert(0, str(family)) - data.update({ - "name": "{} {} {}".format(asset, subset, families), + "name": "{}_{}".format(asset, subset), + "label": "{} {}".format(asset, subset), "asset": asset, "item": timeline_item, - "families": families, "publish": get_publish_attribute(timeline_item), "fps": context.data["fps"], "handleStart": handle_start, "handleEnd": handle_end, - "newAssetPublishing": True + "newAssetPublishing": True, + "families": ["clip"], }) # otio clip data @@ -135,7 +136,8 @@ class PrecollectInstances(pyblish.api.ContextPlugin): family = "shot" data.update({ - "name": "{} {} {}".format(asset, subset, family), + "name": "{}_{}".format(asset, subset), + "label": "{} {}".format(asset, subset), "subset": subset, "asset": asset, "family": family, diff --git a/openpype/hosts/resolve/plugins/publish/precollect_workfile.py b/openpype/hosts/resolve/plugins/publish/precollect_workfile.py index a2f3eaed7a..ccc5fd86ff 100644 --- a/openpype/hosts/resolve/plugins/publish/precollect_workfile.py +++ b/openpype/hosts/resolve/plugins/publish/precollect_workfile.py @@ -1,7 +1,9 @@ import pyblish.api from pprint import pformat +from openpype import AYON_SERVER_ENABLED from openpype.pipeline import get_current_asset_name + from openpype.hosts.resolve import api as rapi from openpype.hosts.resolve.otio import davinci_export @@ -13,9 +15,12 @@ class PrecollectWorkfile(pyblish.api.ContextPlugin): order = pyblish.api.CollectorOrder - 0.5 def process(self, context): + current_asset_name = asset_name = get_current_asset_name() - asset = get_current_asset_name() - subset = "workfile" + if AYON_SERVER_ENABLED: + asset_name = current_asset_name.split("/")[-1] + + subset = "workfileMain" project = rapi.get_current_project() fps = project.GetSetting("timelineFrameRate") video_tracks = rapi.get_video_track_names() @@ -24,9 +29,10 @@ class PrecollectWorkfile(pyblish.api.ContextPlugin): otio_timeline = davinci_export.create_otio_timeline(project) instance_data = { - "name": "{}_{}".format(asset, subset), - "asset": asset, - "subset": "{}{}".format(asset, subset.capitalize()), + "name": "{}_{}".format(asset_name, subset), + "label": "{} {}".format(current_asset_name, subset), + "asset": current_asset_name, + "subset": subset, "item": project, "family": "workfile", "families": [] diff --git a/openpype/hosts/standalonepublisher/plugins/publish/collect_harmony_scenes.py b/openpype/hosts/standalonepublisher/plugins/publish/collect_harmony_scenes.py index 48c36aa067..c435ca2096 100644 --- a/openpype/hosts/standalonepublisher/plugins/publish/collect_harmony_scenes.py +++ b/openpype/hosts/standalonepublisher/plugins/publish/collect_harmony_scenes.py @@ -60,6 +60,9 @@ class CollectHarmonyScenes(pyblish.api.InstancePlugin): # updating hierarchy data anatomy_data_new.update({ "asset": asset_data["name"], + "folder": { + "name": asset_data["name"], + }, "task": { "name": task, "type": task_type, diff --git a/openpype/hosts/standalonepublisher/plugins/publish/collect_harmony_zips.py b/openpype/hosts/standalonepublisher/plugins/publish/collect_harmony_zips.py index 40a969f8df..d90215e767 100644 --- a/openpype/hosts/standalonepublisher/plugins/publish/collect_harmony_zips.py +++ b/openpype/hosts/standalonepublisher/plugins/publish/collect_harmony_zips.py @@ -56,6 +56,9 @@ class CollectHarmonyZips(pyblish.api.InstancePlugin): anatomy_data_new.update( { "asset": asset_data["name"], + "folder": { + "name": asset_data["name"], + }, "task": { "name": task, "type": task_type, diff --git a/openpype/hosts/substancepainter/plugins/create/create_workfile.py b/openpype/hosts/substancepainter/plugins/create/create_workfile.py index d7f31f9dcf..c73277e405 100644 --- a/openpype/hosts/substancepainter/plugins/create/create_workfile.py +++ b/openpype/hosts/substancepainter/plugins/create/create_workfile.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- """Creator plugin for creating workfiles.""" +from openpype import AYON_SERVER_ENABLED from openpype.pipeline import CreatedInstance, AutoCreator from openpype.client import get_asset_by_name @@ -41,6 +42,13 @@ class CreateWorkfile(AutoCreator): if instance.creator_identifier == self.identifier ), None) + if current_instance is None: + current_instance_asset = None + elif AYON_SERVER_ENABLED: + current_instance_asset = current_instance["folderPath"] + else: + current_instance_asset = current_instance["asset"] + if current_instance is None: self.log.info("Auto-creating workfile instance...") asset_doc = get_asset_by_name(project_name, asset_name) @@ -48,22 +56,28 @@ class CreateWorkfile(AutoCreator): variant, task_name, asset_doc, project_name, host_name ) data = { - "asset": asset_name, "task": task_name, "variant": variant } + if AYON_SERVER_ENABLED: + data["folderPath"] = asset_name + else: + data["asset"] = asset_name current_instance = self.create_instance_in_context(subset_name, data) elif ( - current_instance["asset"] != asset_name - or current_instance["task"] != task_name + current_instance_asset != asset_name + or current_instance["task"] != task_name ): # Update instance context if is not the same asset_doc = get_asset_by_name(project_name, asset_name) subset_name = self.get_subset_name( variant, task_name, asset_doc, project_name, host_name ) - current_instance["asset"] = asset_name + if AYON_SERVER_ENABLED: + current_instance["folderPath"] = asset_name + else: + current_instance["asset"] = asset_name current_instance["task"] = task_name current_instance["subset"] = subset_name diff --git a/openpype/hosts/traypublisher/api/editorial.py b/openpype/hosts/traypublisher/api/editorial.py index e8f76bd314..613f1de768 100644 --- a/openpype/hosts/traypublisher/api/editorial.py +++ b/openpype/hosts/traypublisher/api/editorial.py @@ -53,11 +53,11 @@ class ShotMetadataSolver: try: # format to new shot name return shot_rename_template.format(**data) - except KeyError as _E: + except KeyError as _error: raise CreatorError(( "Make sure all keys in settings are correct:: \n\n" f"From template string {shot_rename_template} > " - f"`{_E}` has no equivalent in \n" + f"`{_error}` has no equivalent in \n" f"{list(data.keys())} input formatting keys!" )) @@ -100,7 +100,7 @@ class ShotMetadataSolver: "at your project settings..." )) - # QUESTION:how to refactory `match[-1]` to some better way? + # QUESTION:how to refactor `match[-1]` to some better way? output_data[token_key] = match[-1] return output_data @@ -130,10 +130,10 @@ class ShotMetadataSolver: parent_token["name"]: parent_token["value"].format(**data) for parent_token in hierarchy_parents } - except KeyError as _E: + except KeyError as _error: raise CreatorError(( "Make sure all keys in settings are correct : \n" - f"`{_E}` has no equivalent in \n{list(data.keys())}" + f"`{_error}` has no equivalent in \n{list(data.keys())}" )) _parent_tokens_type = { @@ -147,10 +147,10 @@ class ShotMetadataSolver: try: parent_name = _parent.format( **_parent_tokens_formatting_data) - except KeyError as _E: + except KeyError as _error: raise CreatorError(( "Make sure all keys in settings are correct : \n\n" - f"`{_E}` from template string " + f"`{_error}` from template string " f"{shot_hierarchy['parents_path']}, " f" has no equivalent in \n" f"{list(_parent_tokens_formatting_data.keys())} parents" @@ -319,8 +319,16 @@ class ShotMetadataSolver: tasks = self._generate_tasks_from_settings( project_doc) + # generate hierarchy path from parents + hierarchy_path = self._create_hierarchy_path(parents) + if hierarchy_path: + folder_path = f"/{hierarchy_path}/{shot_name}" + else: + folder_path = f"/{shot_name}" + return shot_name, { - "hierarchy": self._create_hierarchy_path(parents), + "hierarchy": hierarchy_path, + "folderPath": folder_path, "parents": parents, "tasks": tasks } diff --git a/openpype/hosts/traypublisher/api/plugin.py b/openpype/hosts/traypublisher/api/plugin.py index 36e041a32c..14c66fa08f 100644 --- a/openpype/hosts/traypublisher/api/plugin.py +++ b/openpype/hosts/traypublisher/api/plugin.py @@ -1,7 +1,9 @@ +from openpype import AYON_SERVER_ENABLED from openpype.client import ( get_assets, get_subsets, get_last_versions, + get_asset_name_identifier, ) from openpype.lib.attribute_definitions import ( FileDef, @@ -114,7 +116,10 @@ class SettingsCreator(TrayPublishCreator): # Fill 'version_to_use' if version control is enabled if self.allow_version_control: - asset_name = data["asset"] + if AYON_SERVER_ENABLED: + asset_name = data["folderPath"] + else: + asset_name = data["asset"] subset_docs_by_asset_id = self._prepare_next_versions( [asset_name], [subset_name]) version = subset_docs_by_asset_id[asset_name].get(subset_name) @@ -162,10 +167,10 @@ class SettingsCreator(TrayPublishCreator): asset_docs = get_assets( self.project_name, asset_names=asset_names, - fields=["_id", "name"] + fields=["_id", "name", "data.parents"] ) asset_names_by_id = { - asset_doc["_id"]: asset_doc["name"] + asset_doc["_id"]: get_asset_name_identifier(asset_doc) for asset_doc in asset_docs } subset_docs = list(get_subsets( diff --git a/openpype/hosts/traypublisher/plugins/create/create_colorspace_look.py b/openpype/hosts/traypublisher/plugins/create/create_colorspace_look.py index 5628d0973f..ac4c72a0ce 100644 --- a/openpype/hosts/traypublisher/plugins/create/create_colorspace_look.py +++ b/openpype/hosts/traypublisher/plugins/create/create_colorspace_look.py @@ -6,6 +6,7 @@ production type `ociolook`. All files are published as representation. """ from pathlib import Path +from openpype import AYON_SERVER_ENABLED from openpype.client import get_asset_by_name from openpype.lib.attribute_definitions import ( FileDef, EnumDef, TextDef, UISeparatorDef @@ -54,8 +55,12 @@ This creator publishes color space look file (LUT). # this should never happen raise CreatorError("Missing files from representation") + if AYON_SERVER_ENABLED: + asset_name = instance_data["folderPath"] + else: + asset_name = instance_data["asset"] asset_doc = get_asset_by_name( - self.project_name, instance_data["asset"]) + self.project_name, asset_name) subset_name = self.get_subset_name( variant=instance_data["variant"], diff --git a/openpype/hosts/traypublisher/plugins/create/create_editorial.py b/openpype/hosts/traypublisher/plugins/create/create_editorial.py index a2746f115f..26cce35d55 100644 --- a/openpype/hosts/traypublisher/plugins/create/create_editorial.py +++ b/openpype/hosts/traypublisher/plugins/create/create_editorial.py @@ -1,6 +1,7 @@ import os from copy import deepcopy import opentimelineio as otio +from openpype import AYON_SERVER_ENABLED from openpype.client import ( get_asset_by_name, get_project @@ -101,14 +102,23 @@ class EditorialShotInstanceCreator(EditorialClipInstanceCreatorBase): label = "Editorial Shot" def get_instance_attr_defs(self): - attr_defs = [ - TextDef( - "asset_name", - label="Asset name", + instance_attributes = [] + if AYON_SERVER_ENABLED: + instance_attributes.append( + TextDef( + "folderPath", + label="Folder path" + ) ) - ] - attr_defs.extend(CLIP_ATTR_DEFS) - return attr_defs + else: + instance_attributes.append( + TextDef( + "shotName", + label="Shot name" + ) + ) + instance_attributes.extend(CLIP_ATTR_DEFS) + return instance_attributes class EditorialPlateInstanceCreator(EditorialClipInstanceCreatorBase): @@ -214,8 +224,11 @@ or updating already created. Publishing will create OTIO file. i["family"] for i in self._creator_settings["family_presets"] ] } - # Create otio editorial instance - asset_name = instance_data["asset"] + if AYON_SERVER_ENABLED: + asset_name = instance_data["folderPath"] + else: + asset_name = instance_data["asset"] + asset_doc = get_asset_by_name(self.project_name, asset_name) if pre_create_data["fps"] == "from_selection": @@ -595,19 +608,23 @@ or updating already created. Publishing will create OTIO file. Returns: str: label string """ - shot_name = instance_data["shotName"] + if AYON_SERVER_ENABLED: + asset_name = instance_data["creator_attributes"]["folderPath"] + else: + asset_name = instance_data["creator_attributes"]["shotName"] + variant_name = instance_data["variant"] family = preset["family"] - # get variant name from preset or from inharitance + # get variant name from preset or from inheritance _variant_name = preset.get("variant") or variant_name # subset name subset_name = "{}{}".format( family, _variant_name.capitalize() ) - label = "{}_{}".format( - shot_name, + label = "{} {}".format( + asset_name, subset_name ) @@ -666,7 +683,10 @@ or updating already created. Publishing will create OTIO file. } ) - self._validate_name_uniqueness(shot_name) + # It should be validated only in openpype since we are supporting + # publishing to AYON with folder path and uniqueness is not an issue + if not AYON_SERVER_ENABLED: + self._validate_name_uniqueness(shot_name) timing_data = self._get_timing_data( otio_clip, @@ -677,35 +697,43 @@ or updating already created. Publishing will create OTIO file. # create creator attributes creator_attributes = { - "asset_name": shot_name, - "Parent hierarchy path": shot_metadata["hierarchy"], + "workfile_start_frame": workfile_start_frame, "fps": fps, "handle_start": int(handle_start), "handle_end": int(handle_end) } + # add timing data creator_attributes.update(timing_data) - # create shared new instance data + # create base instance data base_instance_data = { "shotName": shot_name, "variant": variant_name, - - # HACK: just for temporal bug workaround - # TODO: should loockup shot name for update - "asset": parent_asset_name, "task": "", - "newAssetPublishing": True, - - # parent time properties "trackStartFrame": track_start_frame, "timelineOffset": timeline_offset, - "isEditorial": True, # creator_attributes "creator_attributes": creator_attributes } + # update base instance data with context data + # and also update creator attributes with context data + if AYON_SERVER_ENABLED: + # TODO: this is here just to be able to publish + # to AYON with folder path + creator_attributes["folderPath"] = shot_metadata.pop("folderPath") + base_instance_data["folderPath"] = parent_asset_name + else: + creator_attributes.update({ + "shotName": shot_name, + "Parent hierarchy path": shot_metadata["hierarchy"] + }) + + base_instance_data["asset"] = parent_asset_name + # add creator attributes to shared instance data + base_instance_data["creator_attributes"] = creator_attributes # add hierarchy shot metadata base_instance_data.update(shot_metadata) diff --git a/openpype/hosts/traypublisher/plugins/create/create_movie_batch.py b/openpype/hosts/traypublisher/plugins/create/create_movie_batch.py index 3454b6e135..8fa65c7fff 100644 --- a/openpype/hosts/traypublisher/plugins/create/create_movie_batch.py +++ b/openpype/hosts/traypublisher/plugins/create/create_movie_batch.py @@ -2,6 +2,8 @@ import copy import os import re +from openpype import AYON_SERVER_ENABLED +from openpype.client import get_asset_name_identifier from openpype.lib import ( FileDef, BoolDef, @@ -64,8 +66,13 @@ class BatchMovieCreator(TrayPublishCreator): subset_name, task_name = self._get_subset_and_task( asset_doc, data["variant"], self.project_name) + asset_name = get_asset_name_identifier(asset_doc) + instance_data["task"] = task_name - instance_data["asset"] = asset_doc["name"] + if AYON_SERVER_ENABLED: + instance_data["folderPath"] = asset_name + else: + instance_data["asset"] = asset_name # Create new instance new_instance = CreatedInstance(self.family, subset_name, diff --git a/openpype/hosts/traypublisher/plugins/publish/collect_sequence_frame_data.py b/openpype/hosts/traypublisher/plugins/publish/collect_sequence_frame_data.py index 92cedf6b5b..5e60a94927 100644 --- a/openpype/hosts/traypublisher/plugins/publish/collect_sequence_frame_data.py +++ b/openpype/hosts/traypublisher/plugins/publish/collect_sequence_frame_data.py @@ -28,9 +28,9 @@ class CollectSequenceFrameData( return # editorial would fail since they might not be in database yet - is_editorial = instance.data.get("isEditorial") - if is_editorial: - self.log.debug("Instance is Editorial. Skipping.") + new_asset_publishing = instance.data.get("newAssetPublishing") + if new_asset_publishing: + self.log.debug("Instance is creating new asset. Skipping.") return frame_data = self.get_frame_data_from_repre_sequence(instance) diff --git a/openpype/hosts/traypublisher/plugins/publish/collect_shot_instances.py b/openpype/hosts/traypublisher/plugins/publish/collect_shot_instances.py index 78c1f14e4e..e00ac64244 100644 --- a/openpype/hosts/traypublisher/plugins/publish/collect_shot_instances.py +++ b/openpype/hosts/traypublisher/plugins/publish/collect_shot_instances.py @@ -2,6 +2,8 @@ from pprint import pformat import pyblish.api import opentimelineio as otio +from openpype import AYON_SERVER_ENABLED + class CollectShotInstance(pyblish.api.InstancePlugin): """ Collect shot instances @@ -119,8 +121,7 @@ class CollectShotInstance(pyblish.api.InstancePlugin): frame_end = _cr_attrs["frameEnd"] frame_dur = frame_end - frame_start - return { - "asset": _cr_attrs["asset_name"], + data = { "fps": float(_cr_attrs["fps"]), "handleStart": _cr_attrs["handle_start"], "handleEnd": _cr_attrs["handle_end"], @@ -133,6 +134,12 @@ class CollectShotInstance(pyblish.api.InstancePlugin): "sourceOut": _cr_attrs["sourceOut"], "workfileFrameStart": workfile_start_frame } + if AYON_SERVER_ENABLED: + data["asset"] = _cr_attrs["folderPath"] + else: + data["asset"] = _cr_attrs["shotName"] + + return data def _solve_hierarchy_context(self, instance): """ Adding hierarchy data to context shared data. @@ -148,7 +155,7 @@ class CollectShotInstance(pyblish.api.InstancePlugin): else {} ) - name = instance.data["asset"] + asset_name = instance.data["asset"] # get handles handle_start = int(instance.data["handleStart"]) @@ -170,7 +177,7 @@ class CollectShotInstance(pyblish.api.InstancePlugin): parents = instance.data.get('parents', []) - actual = {name: in_info} + actual = {asset_name: in_info} for parent in reversed(parents): parent_name = parent["entity_name"] diff --git a/openpype/hosts/traypublisher/plugins/publish/validate_frame_ranges.py b/openpype/hosts/traypublisher/plugins/publish/validate_frame_ranges.py index 4977a13374..56389a927f 100644 --- a/openpype/hosts/traypublisher/plugins/publish/validate_frame_ranges.py +++ b/openpype/hosts/traypublisher/plugins/publish/validate_frame_ranges.py @@ -31,9 +31,9 @@ class ValidateFrameRange(OptionalPyblishPluginMixin, return # editorial would fail since they might not be in database yet - is_editorial = instance.data.get("isEditorial") - if is_editorial: - self.log.debug("Instance is Editorial. Skipping.") + new_asset_publishing = instance.data.get("newAssetPublishing") + if new_asset_publishing: + self.log.debug("Instance is creating new asset. Skipping.") return if (self.skip_timelines_check and @@ -41,6 +41,7 @@ class ValidateFrameRange(OptionalPyblishPluginMixin, for pattern in self.skip_timelines_check)): self.log.info("Skipping for {} task".format(instance.data["task"])) + asset_doc = instance.data["assetEntity"] asset_data = asset_doc["data"] frame_start = asset_data["frameStart"] frame_end = asset_data["frameEnd"] diff --git a/openpype/hosts/tvpaint/plugins/create/create_render.py b/openpype/hosts/tvpaint/plugins/create/create_render.py index b7a7c208d9..667103432e 100644 --- a/openpype/hosts/tvpaint/plugins/create/create_render.py +++ b/openpype/hosts/tvpaint/plugins/create/create_render.py @@ -37,7 +37,8 @@ Todos: import collections from typing import Any, Optional, Union -from openpype.client import get_asset_by_name +from openpype import AYON_SERVER_ENABLED +from openpype.client import get_asset_by_name, get_asset_name_identifier from openpype.lib import ( prepare_template_data, AbstractAttrDef, @@ -784,18 +785,25 @@ class TVPaintAutoDetectRenderCreator(TVPaintCreator): project_name, host_name=self.create_context.host_name, ) + asset_name = get_asset_name_identifier(asset_doc) if existing_instance is not None: - existing_instance["asset"] = asset_doc["name"] + if AYON_SERVER_ENABLED: + existing_instance["folderPath"] = asset_name + else: + existing_instance["asset"] = asset_name existing_instance["task"] = task_name existing_instance["subset"] = subset_name return existing_instance instance_data: dict[str, str] = { - "asset": asset_doc["name"], "task": task_name, "family": creator.family, "variant": variant } + if AYON_SERVER_ENABLED: + instance_data["folderPath"] = asset_name + else: + instance_data["asset"] = asset_name pre_create_data: dict[str, str] = { "group_id": group_id, "mark_for_review": mark_for_review @@ -820,6 +828,8 @@ class TVPaintAutoDetectRenderCreator(TVPaintCreator): for layer_name in render_pass["layer_names"]: render_pass_by_layer_name[layer_name] = render_pass + asset_name = get_asset_name_identifier(asset_doc) + for layer in layers: layer_name = layer["name"] variant = layer_name @@ -838,17 +848,25 @@ class TVPaintAutoDetectRenderCreator(TVPaintCreator): ) if render_pass is not None: - render_pass["asset"] = asset_doc["name"] + if AYON_SERVER_ENABLED: + render_pass["folderPath"] = asset_name + else: + render_pass["asset"] = asset_name + render_pass["task"] = task_name render_pass["subset"] = subset_name continue instance_data: dict[str, str] = { - "asset": asset_doc["name"], "task": task_name, "family": creator.family, "variant": variant } + if AYON_SERVER_ENABLED: + instance_data["folderPath"] = asset_name + else: + instance_data["asset"] = asset_name + pre_create_data: dict[str, Any] = { "render_layer_instance_id": render_layer_instance.id, "layer_names": [layer_name], @@ -882,9 +900,13 @@ class TVPaintAutoDetectRenderCreator(TVPaintCreator): def create(self, subset_name, instance_data, pre_create_data): project_name: str = self.create_context.get_current_project_name() - asset_name: str = instance_data["asset"] + if AYON_SERVER_ENABLED: + asset_name: str = instance_data["folderPath"] + else: + asset_name: str = instance_data["asset"] task_name: str = instance_data["task"] - asset_doc: dict[str, Any] = get_asset_by_name(project_name, asset_name) + asset_doc: dict[str, Any] = get_asset_by_name( + project_name, asset_name) render_layers_by_group_id: dict[int, CreatedInstance] = {} render_passes_by_render_layer_id: dict[int, list[CreatedInstance]] = ( @@ -1061,7 +1083,6 @@ class TVPaintSceneRenderCreator(TVPaintAutoCreator): host_name ) data = { - "asset": asset_name, "task": task_name, "variant": self.default_variant, "creator_attributes": { @@ -1073,6 +1094,10 @@ class TVPaintSceneRenderCreator(TVPaintAutoCreator): self.default_pass_name ) } + if AYON_SERVER_ENABLED: + data["folderPath"] = asset_name + else: + data["asset"] = asset_name if not self.active_on_create: data["active"] = False @@ -1101,8 +1126,14 @@ class TVPaintSceneRenderCreator(TVPaintAutoCreator): asset_name = create_context.get_current_asset_name() task_name = create_context.get_current_task_name() + existing_name = None + if AYON_SERVER_ENABLED: + existing_name = existing_instance.get("folderPath") + if existing_name is None: + existing_name = existing_instance["asset"] + if ( - existing_instance["asset"] != asset_name + existing_name != asset_name or existing_instance["task"] != task_name ): asset_doc = get_asset_by_name(project_name, asset_name) @@ -1114,7 +1145,10 @@ class TVPaintSceneRenderCreator(TVPaintAutoCreator): host_name, existing_instance ) - existing_instance["asset"] = asset_name + if AYON_SERVER_ENABLED: + existing_instance["folderPath"] = asset_name + else: + existing_instance["asset"] = asset_name existing_instance["task"] = task_name existing_instance["subset"] = subset_name diff --git a/openpype/hosts/tvpaint/plugins/create/create_review.py b/openpype/hosts/tvpaint/plugins/create/create_review.py index 7bb7510a8e..5caf20f27d 100644 --- a/openpype/hosts/tvpaint/plugins/create/create_review.py +++ b/openpype/hosts/tvpaint/plugins/create/create_review.py @@ -1,3 +1,4 @@ +from openpype import AYON_SERVER_ENABLED from openpype.client import get_asset_by_name from openpype.pipeline import CreatedInstance from openpype.hosts.tvpaint.api.plugin import TVPaintAutoCreator @@ -33,6 +34,13 @@ class TVPaintReviewCreator(TVPaintAutoCreator): asset_name = create_context.get_current_asset_name() task_name = create_context.get_current_task_name() + if existing_instance is None: + existing_asset_name = None + elif AYON_SERVER_ENABLED: + existing_asset_name = existing_instance["folderPath"] + else: + existing_asset_name = existing_instance["asset"] + if existing_instance is None: asset_doc = get_asset_by_name(project_name, asset_name) subset_name = self.get_subset_name( @@ -43,10 +51,14 @@ class TVPaintReviewCreator(TVPaintAutoCreator): host_name ) data = { - "asset": asset_name, "task": task_name, "variant": self.default_variant } + if AYON_SERVER_ENABLED: + data["folderPath"] = asset_name + else: + data["asset"] = asset_name + if not self.active_on_create: data["active"] = False @@ -59,7 +71,7 @@ class TVPaintReviewCreator(TVPaintAutoCreator): self._add_instance_to_context(new_instance) elif ( - existing_instance["asset"] != asset_name + existing_asset_name != asset_name or existing_instance["task"] != task_name ): asset_doc = get_asset_by_name(project_name, asset_name) @@ -71,6 +83,9 @@ class TVPaintReviewCreator(TVPaintAutoCreator): host_name, existing_instance ) - existing_instance["asset"] = asset_name + if AYON_SERVER_ENABLED: + existing_instance["folderPath"] = asset_name + else: + existing_instance["asset"] = asset_name existing_instance["task"] = task_name existing_instance["subset"] = subset_name diff --git a/openpype/hosts/tvpaint/plugins/create/create_workfile.py b/openpype/hosts/tvpaint/plugins/create/create_workfile.py index c3982c0eca..4ce5d7fc96 100644 --- a/openpype/hosts/tvpaint/plugins/create/create_workfile.py +++ b/openpype/hosts/tvpaint/plugins/create/create_workfile.py @@ -1,3 +1,4 @@ +from openpype import AYON_SERVER_ENABLED from openpype.client import get_asset_by_name from openpype.pipeline import CreatedInstance from openpype.hosts.tvpaint.api.plugin import TVPaintAutoCreator @@ -29,6 +30,13 @@ class TVPaintWorkfileCreator(TVPaintAutoCreator): asset_name = create_context.get_current_asset_name() task_name = create_context.get_current_task_name() + if existing_instance is None: + existing_asset_name = None + elif AYON_SERVER_ENABLED: + existing_asset_name = existing_instance["folderPath"] + else: + existing_asset_name = existing_instance["asset"] + if existing_instance is None: asset_doc = get_asset_by_name(project_name, asset_name) subset_name = self.get_subset_name( @@ -39,10 +47,13 @@ class TVPaintWorkfileCreator(TVPaintAutoCreator): host_name ) data = { - "asset": asset_name, "task": task_name, "variant": self.default_variant } + if AYON_SERVER_ENABLED: + data["folderPath"] = asset_name + else: + data["asset"] = asset_name new_instance = CreatedInstance( self.family, subset_name, data, self @@ -53,7 +64,7 @@ class TVPaintWorkfileCreator(TVPaintAutoCreator): self._add_instance_to_context(new_instance) elif ( - existing_instance["asset"] != asset_name + existing_asset_name != asset_name or existing_instance["task"] != task_name ): asset_doc = get_asset_by_name(project_name, asset_name) @@ -65,6 +76,9 @@ class TVPaintWorkfileCreator(TVPaintAutoCreator): host_name, existing_instance ) - existing_instance["asset"] = asset_name + if AYON_SERVER_ENABLED: + existing_instance["folderPath"] = asset_name + else: + existing_instance["asset"] = asset_name existing_instance["task"] = task_name existing_instance["subset"] = subset_name diff --git a/openpype/hosts/tvpaint/plugins/publish/validate_asset_name.py b/openpype/hosts/tvpaint/plugins/publish/validate_asset_name.py index 9347960d3f..dc29e6c278 100644 --- a/openpype/hosts/tvpaint/plugins/publish/validate_asset_name.py +++ b/openpype/hosts/tvpaint/plugins/publish/validate_asset_name.py @@ -1,4 +1,5 @@ import pyblish.api +from openpype import AYON_SERVER_ENABLED from openpype.pipeline import ( PublishXmlValidationError, OptionalPyblishPluginMixin, @@ -24,12 +25,19 @@ class FixAssetNames(pyblish.api.Action): old_instance_items = list_instances() new_instance_items = [] for instance_item in old_instance_items: - instance_asset_name = instance_item.get("asset") + if AYON_SERVER_ENABLED: + instance_asset_name = instance_item.get("folderPath") + else: + instance_asset_name = instance_item.get("asset") + if ( instance_asset_name and instance_asset_name != context_asset_name ): - instance_item["asset"] = context_asset_name + if AYON_SERVER_ENABLED: + instance_item["folderPath"] = context_asset_name + else: + instance_item["asset"] = context_asset_name new_instance_items.append(instance_item) write_instances(new_instance_items) diff --git a/openpype/lib/applications.py b/openpype/lib/applications.py index ff5e27c122..4d75a01e1d 100644 --- a/openpype/lib/applications.py +++ b/openpype/lib/applications.py @@ -12,6 +12,7 @@ from abc import ABCMeta, abstractmethod import six from openpype import AYON_SERVER_ENABLED, PACKAGE_DIR +from openpype.client import get_asset_name_identifier from openpype.settings import ( get_system_settings, get_project_settings, @@ -1728,7 +1729,9 @@ def prepare_context_environments(data, env_group=None, modules_manager=None): "AVALON_APP_NAME": app.full_name } if asset_doc: - context_env["AVALON_ASSET"] = asset_doc["name"] + asset_name = get_asset_name_identifier(asset_doc) + context_env["AVALON_ASSET"] = asset_name + if task_name: context_env["AVALON_TASK"] = task_name diff --git a/openpype/modules/__init__.py b/openpype/modules/__init__.py index 1f345feea9..3097805353 100644 --- a/openpype/modules/__init__.py +++ b/openpype/modules/__init__.py @@ -10,6 +10,7 @@ from .interfaces import ( ) from .base import ( + AYONAddon, OpenPypeModule, OpenPypeAddOn, @@ -35,6 +36,7 @@ __all__ = ( "ISettingsChangeListener", "IHostAddon", + "AYONAddon", "OpenPypeModule", "OpenPypeAddOn", diff --git a/openpype/modules/asset_reporter/__init__.py b/openpype/modules/asset_reporter/__init__.py new file mode 100644 index 0000000000..6267b4824b --- /dev/null +++ b/openpype/modules/asset_reporter/__init__.py @@ -0,0 +1,8 @@ +from .module import ( + AssetReporterAction +) + + +__all__ = ( + "AssetReporterAction", +) diff --git a/openpype/modules/asset_reporter/module.py b/openpype/modules/asset_reporter/module.py new file mode 100644 index 0000000000..8c754cc3c0 --- /dev/null +++ b/openpype/modules/asset_reporter/module.py @@ -0,0 +1,27 @@ +import os.path + +from openpype import AYON_SERVER_ENABLED +from openpype.modules import OpenPypeModule, ITrayAction +from openpype.lib import run_detached_process, get_openpype_execute_args + + +class AssetReporterAction(OpenPypeModule, ITrayAction): + + label = "Asset Usage Report" + name = "asset_reporter" + + def tray_init(self): + pass + + def initialize(self, modules_settings): + self.enabled = not AYON_SERVER_ENABLED + + def on_action_trigger(self): + args = get_openpype_execute_args() + args += ["run", + os.path.join( + os.path.dirname(__file__), + "window.py")] + + print(" ".join(args)) + run_detached_process(args) diff --git a/openpype/modules/asset_reporter/window.py b/openpype/modules/asset_reporter/window.py new file mode 100644 index 0000000000..ed3bc298e1 --- /dev/null +++ b/openpype/modules/asset_reporter/window.py @@ -0,0 +1,418 @@ +"""Tool for generating asset usage report. + +This tool is used to generate asset usage report for a project. +It is using links between published version to find out where +the asset is used. + +""" + +import csv +import time + +import appdirs +import qtawesome +from pymongo.collection import Collection +from qtpy import QtCore, QtWidgets +from qtpy.QtGui import QClipboard, QColor + +from openpype import style +from openpype.client import OpenPypeMongoConnection +from openpype.lib import JSONSettingRegistry +from openpype.tools.utils import PlaceholderLineEdit, get_openpype_qt_app +from openpype.tools.utils.constants import PROJECT_NAME_ROLE +from openpype.tools.utils.models import ProjectModel, ProjectSortFilterProxy + + +class AssetReporterRegistry(JSONSettingRegistry): + """Class handling OpenPype general settings registry. + + This is used to store last selected project. + + Attributes: + vendor (str): Name used for path construction. + product (str): Additional name used for path construction. + + """ + + def __init__(self): + self.vendor = "ynput" + self.product = "openpype" + name = "asset_usage_reporter" + path = appdirs.user_data_dir(self.product, self.vendor) + super(AssetReporterRegistry, self).__init__(name, path) + + +class OverlayWidget(QtWidgets.QFrame): + """Overlay widget for choosing project. + + This code is taken from the Tray Publisher tool. + """ + project_selected = QtCore.Signal(str) + + def __init__(self, publisher_window): + super(OverlayWidget, self).__init__(publisher_window) + self.setObjectName("OverlayFrame") + + middle_frame = QtWidgets.QFrame(self) + middle_frame.setObjectName("ChooseProjectFrame") + + content_widget = QtWidgets.QWidget(middle_frame) + + header_label = QtWidgets.QLabel("Choose project", content_widget) + header_label.setObjectName("ChooseProjectLabel") + # Create project models and view + projects_model = ProjectModel() + projects_proxy = ProjectSortFilterProxy() + projects_proxy.setSourceModel(projects_model) + projects_proxy.setFilterKeyColumn(0) + + projects_view = QtWidgets.QListView(content_widget) + projects_view.setObjectName("ChooseProjectView") + projects_view.setModel(projects_proxy) + projects_view.setEditTriggers( + QtWidgets.QAbstractItemView.NoEditTriggers + ) + + confirm_btn = QtWidgets.QPushButton("Confirm", content_widget) + cancel_btn = QtWidgets.QPushButton("Cancel", content_widget) + cancel_btn.setVisible(False) + btns_layout = QtWidgets.QHBoxLayout() + btns_layout.addStretch(1) + btns_layout.addWidget(cancel_btn, 0) + btns_layout.addWidget(confirm_btn, 0) + + txt_filter = PlaceholderLineEdit(content_widget) + txt_filter.setPlaceholderText("Quick filter projects..") + txt_filter.setClearButtonEnabled(True) + txt_filter.addAction(qtawesome.icon("fa.filter", color="gray"), + QtWidgets.QLineEdit.LeadingPosition) + + content_layout = QtWidgets.QVBoxLayout(content_widget) + content_layout.setContentsMargins(0, 0, 0, 0) + content_layout.setSpacing(20) + content_layout.addWidget(header_label, 0) + content_layout.addWidget(txt_filter, 0) + content_layout.addWidget(projects_view, 1) + content_layout.addLayout(btns_layout, 0) + + middle_layout = QtWidgets.QHBoxLayout(middle_frame) + middle_layout.setContentsMargins(30, 30, 10, 10) + middle_layout.addWidget(content_widget) + + main_layout = QtWidgets.QHBoxLayout(self) + main_layout.setContentsMargins(10, 10, 10, 10) + main_layout.addStretch(1) + main_layout.addWidget(middle_frame, 2) + main_layout.addStretch(1) + + projects_view.doubleClicked.connect(self._on_double_click) + confirm_btn.clicked.connect(self._on_confirm_click) + cancel_btn.clicked.connect(self._on_cancel_click) + txt_filter.textChanged.connect(self._on_text_changed) + + self._projects_view = projects_view + self._projects_model = projects_model + self._projects_proxy = projects_proxy + self._cancel_btn = cancel_btn + self._confirm_btn = confirm_btn + self._txt_filter = txt_filter + + self._publisher_window = publisher_window + self._project_name = None + + def showEvent(self, event): + self._projects_model.refresh() + # Sort projects after refresh + self._projects_proxy.sort(0) + + setting_registry = AssetReporterRegistry() + try: + project_name = str(setting_registry.get_item("project_name")) + except ValueError: + project_name = None + + if project_name: + index = None + src_index = self._projects_model.find_project(project_name) + if src_index is not None: + index = self._projects_proxy.mapFromSource(src_index) + + if index is not None: + selection_model = self._projects_view.selectionModel() + selection_model.select( + index, + QtCore.QItemSelectionModel.SelectCurrent + ) + self._projects_view.setCurrentIndex(index) + + self._cancel_btn.setVisible(self._project_name is not None) + super(OverlayWidget, self).showEvent(event) + + def _on_double_click(self): + self.set_selected_project() + + def _on_confirm_click(self): + self.set_selected_project() + + def _on_cancel_click(self): + self._set_project(self._project_name) + + def _on_text_changed(self): + self._projects_proxy.setFilterRegularExpression( + self._txt_filter.text()) + + def set_selected_project(self): + index = self._projects_view.currentIndex() + + if project_name := index.data(PROJECT_NAME_ROLE): + self._set_project(project_name) + + def _set_project(self, project_name): + self._project_name = project_name + self.setVisible(False) + self.project_selected.emit(project_name) + + setting_registry = AssetReporterRegistry() + setting_registry.set_item("project_name", project_name) + + +class AssetReporterWindow(QtWidgets.QDialog): + default_width = 1000 + default_height = 800 + _content = None + + def __init__(self, parent=None, controller=None, reset_on_show=None): + super(AssetReporterWindow, self).__init__(parent) + + self._result = {} + self.setObjectName("AssetReporterWindow") + + self.setWindowTitle("Asset Usage Reporter") + + if parent is None: + on_top_flag = QtCore.Qt.WindowStaysOnTopHint + else: + on_top_flag = QtCore.Qt.Dialog + + self.setWindowFlags( + QtCore.Qt.WindowTitleHint + | QtCore.Qt.WindowMaximizeButtonHint + | QtCore.Qt.WindowMinimizeButtonHint + | QtCore.Qt.WindowCloseButtonHint + | on_top_flag + ) + self.table = QtWidgets.QTableWidget(self) + self.table.setColumnCount(3) + self.table.setColumnWidth(0, 400) + self.table.setColumnWidth(1, 300) + self.table.setHorizontalHeaderLabels(["Subset", "Used in", "Version"]) + + # self.text_area = QtWidgets.QTextEdit(self) + self.copy_button = QtWidgets.QPushButton('Copy to Clipboard', self) + self.save_button = QtWidgets.QPushButton('Save to CSV File', self) + + self.copy_button.clicked.connect(self.copy_to_clipboard) + self.save_button.clicked.connect(self.save_to_file) + + layout = QtWidgets.QVBoxLayout(self) + layout.addWidget(self.table) + # layout.addWidget(self.text_area) + layout.addWidget(self.copy_button) + layout.addWidget(self.save_button) + + self.resize(self.default_width, self.default_height) + self.setStyleSheet(style.load_stylesheet()) + + overlay_widget = OverlayWidget(self) + overlay_widget.project_selected.connect(self._on_project_select) + self._overlay_widget = overlay_widget + + def _on_project_select(self, project_name: str): + """Generate table when project is selected. + + This will generate the table and fill it with data. + Source data are held in memory in `_result` attribute that + is used to transform them into clipboard or csv file. + """ + self._project_name = project_name + self.process() + if not self._result: + self.set_content("no result generated") + return + + rows = sum(len(value) for key, value in self._result.items()) + self.table.setRowCount(rows) + + row = 0 + content = [] + for key, value in self._result.items(): + item = QtWidgets.QTableWidgetItem(key) + # this doesn't work as it is probably overriden by stylesheet? + # item.setBackground(QColor(32, 32, 32)) + self.table.setItem(row, 0, item) + for source in value: + self.table.setItem( + row, 1, QtWidgets.QTableWidgetItem(source["name"])) + self.table.setItem( + row, 2, QtWidgets.QTableWidgetItem( + str(source["version"]))) + row += 1 + + # generate clipboard content + content.append(key) + content.extend( + f"\t{source['name']} (v{source['version']})" for source in value # noqa: E501 + ) + self.set_content("\n".join(content)) + + def copy_to_clipboard(self): + clipboard = QtWidgets.QApplication.clipboard() + clipboard.setText(self._content, QClipboard.Clipboard) + + def save_to_file(self): + file_name, _ = QtWidgets.QFileDialog.getSaveFileName(self, 'Save File') + if file_name: + self._write_csv(file_name) + + def set_content(self, content): + self._content = content + + def get_content(self): + return self._content + + def _resize_overlay(self): + self._overlay_widget.resize( + self.width(), + self.height() + ) + + def resizeEvent(self, event): + super(AssetReporterWindow, self).resizeEvent(event) + self._resize_overlay() + + def _get_subset(self, version_id, project: Collection): + pipeline = [ + { + "$match": { + "_id": version_id + }, + }, { + "$lookup": { + "from": project.name, + "localField": "parent", + "foreignField": "_id", + "as": "parents" + } + } + ] + + result = project.aggregate(pipeline) + doc = next(result) + # print(doc) + return { + "name": f'{"/".join(doc["parents"][0]["data"]["parents"])}/{doc["parents"][0]["name"]}/{doc["name"]}', # noqa: E501 + "family": doc["data"].get("family") or doc["data"].get("families")[0] # noqa: E501 + } + + def process(self): + """Generate asset usage report data. + + This is the main method of the tool. It is using MongoDB + aggregation pipeline to find all published versions that + are used as input for other published versions. Then it + generates a map of assets and their usage. + + """ + start = time.perf_counter() + project = self._project_name + + # get all versions of published workfiles that has non-empty + # inputLinks and connect it with their respective documents + # using ID. + pipeline = [ + { + "$match": { + "data.inputLinks": { + "$exists": True, + "$ne": [] + }, + "data.families": {"$in": ["workfile"]} + } + }, { + "$lookup": { + "from": project, + "localField": "data.inputLinks.id", + "foreignField": "_id", + "as": "linked_docs" + } + } + ] + + client = OpenPypeMongoConnection.get_mongo_client() + db = client["avalon"] + + result = db[project].aggregate(pipeline) + + asset_map = [] + # this is creating the map - for every workfile and its linked + # documents, create a dictionary with "source" and "refs" keys + # and resolve the subset name and version from the document + for doc in result: + source = { + "source": self._get_subset(doc["parent"], db[project]), + } + source["source"].update({"version": doc["name"]}) + refs = [] + version = '' + for linked in doc["linked_docs"]: + try: + version = f'v{linked["name"]}' + except KeyError: + if linked["type"] == "hero_version": + version = "hero" + finally: + refs.append({ + "subset": self._get_subset( + linked["parent"], db[project]), + "version": version + }) + + source["refs"] = refs + asset_map.append(source) + + grouped = {} + + # this will group the assets by subset name and version + for asset in asset_map: + for ref in asset["refs"]: + key = f'{ref["subset"]["name"]} ({ref["version"]})' + if key in grouped: + grouped[key].append(asset["source"]) + else: + grouped[key] = [asset["source"]] + self._result = grouped + + end = time.perf_counter() + + print(f"Finished in {end - start:0.4f} seconds", 2) + + def _write_csv(self, file_name: str) -> None: + """Write CSV file with results.""" + with open(file_name, "w", newline="") as csvfile: + writer = csv.writer(csvfile, delimiter=";") + writer.writerow(["Subset", "Used in", "Version"]) + for key, value in self._result.items(): + writer.writerow([key, "", ""]) + for source in value: + writer.writerow(["", source["name"], source["version"]]) + + +def main(): + app_instance = get_openpype_qt_app() + window = AssetReporterWindow() + window.show() + app_instance.exec_() + + +if __name__ == "__main__": + main() diff --git a/openpype/modules/base.py b/openpype/modules/base.py index 4636906cec..1a2513b4b4 100644 --- a/openpype/modules/base.py +++ b/openpype/modules/base.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -"""Base class for Pype Modules.""" +"""Base class for AYON addons.""" import copy import os import sys @@ -11,6 +11,7 @@ import platform import threading import collections import traceback + from uuid import uuid4 from abc import ABCMeta, abstractmethod @@ -29,9 +30,12 @@ from openpype.settings import ( from openpype.settings.lib import ( get_studio_system_settings_overrides, - load_json_file + load_json_file, +) +from openpype.settings.ayon_settings import ( + is_dev_mode_enabled, + get_ayon_settings, ) -from openpype.settings.ayon_settings import is_dev_mode_enabled from openpype.lib import ( Logger, @@ -47,11 +51,11 @@ from .interfaces import ( ITrayService ) -# Files that will be always ignored on modules import +# Files that will be always ignored on addons import IGNORED_FILENAMES = ( "__pycache__", ) -# Files ignored on modules import from "./openpype/modules" +# Files ignored on addons import from "./openpype/modules" IGNORED_DEFAULT_FILENAMES = ( "__init__.py", "base.py", @@ -59,8 +63,8 @@ IGNORED_DEFAULT_FILENAMES = ( "example_addons", "default_modules", ) -# Modules that won't be loaded in AYON mode from "./openpype/modules" -# - the same modules are ignored in "./server_addon/create_ayon_addons.py" +# Addons that won't be loaded in AYON mode from "./openpype/modules" +# - the same addons are ignored in "./server_addon/create_ayon_addons.py" IGNORED_FILENAMES_IN_AYON = { "ftrack", "shotgrid", @@ -68,6 +72,10 @@ IGNORED_FILENAMES_IN_AYON = { "slack", "kitsu", } +IGNORED_HOSTS_IN_AYON = { + "flame", + "harmony", +} # Inherit from `object` for Python 2 hosts @@ -466,7 +474,7 @@ def _load_ayon_addons(openpype_modules, modules_key, log): attr = getattr(mod, attr_name) if ( inspect.isclass(attr) - and issubclass(attr, OpenPypeModule) + and issubclass(attr, AYONAddon) ): imported_modules.append(mod) break @@ -536,6 +544,11 @@ def _load_modules(): addons_dir = os.path.join(os.path.dirname(current_dir), "addons") module_dirs.append(addons_dir) + ignored_host_names = set(IGNORED_HOSTS_IN_AYON) + ignored_current_dir_filenames = set(IGNORED_DEFAULT_FILENAMES) + if AYON_SERVER_ENABLED: + ignored_current_dir_filenames |= IGNORED_FILENAMES_IN_AYON + processed_paths = set() for dirpath in frozenset(module_dirs): # Skip already processed paths @@ -551,9 +564,6 @@ def _load_modules(): is_in_current_dir = dirpath == current_dir is_in_host_dir = dirpath == hosts_dir - ignored_current_dir_filenames = set(IGNORED_DEFAULT_FILENAMES) - if AYON_SERVER_ENABLED: - ignored_current_dir_filenames |= IGNORED_FILENAMES_IN_AYON for filename in os.listdir(dirpath): # Ignore filenames @@ -566,6 +576,12 @@ def _load_modules(): ): continue + if ( + is_in_host_dir + and filename in ignored_host_names + ): + continue + fullpath = os.path.join(dirpath, filename) basename, ext = os.path.splitext(filename) @@ -633,26 +649,22 @@ def _load_modules(): @six.add_metaclass(ABCMeta) -class OpenPypeModule: - """Base class of pype module. +class AYONAddon(object): + """Base class of AYON addon. Attributes: - id (UUID): Module's id. - enabled (bool): Is module enabled. - name (str): Module name. - manager (ModulesManager): Manager that created the module. + id (UUID): Addon object id. + enabled (bool): Is addon enabled. + name (str): Addon name. + + Args: + manager (ModulesManager): Manager object who discovered addon. + settings (dict[str, Any]): AYON settings. """ - # Disable by default - enabled = False + enabled = True _id = None - @property - @abstractmethod - def name(self): - """Module's name.""" - pass - def __init__(self, manager, settings): self.manager = manager @@ -662,22 +674,45 @@ class OpenPypeModule: @property def id(self): + """Random id of addon object. + + Returns: + str: Object id. + """ + if self._id is None: self._id = uuid4() return self._id + @property @abstractmethod - def initialize(self, module_settings): - """Initialization of module attributes. + def name(self): + """Addon name. - It is not recommended to override __init__ that's why specific method - was implemented. + Returns: + str: Addon name. """ pass - def connect_with_modules(self, enabled_modules): - """Connect with other enabled modules.""" + def initialize(self, settings): + """Initialization of module attributes. + + It is not recommended to override __init__ that's why specific method + was implemented. + + Args: + settings (dict[str, Any]): Settings. + """ + + pass + + def connect_with_modules(self, enabled_addons): + """Connect with other enabled addons. + + Args: + enabled_addons (list[AYONAddon]): Addons that are enabled. + """ pass @@ -685,6 +720,9 @@ class OpenPypeModule: """Get global environments values of module. Environment variables that can be get only from system settings. + + Returns: + dict[str, str]: Environment variables. """ return {} @@ -697,7 +735,7 @@ class OpenPypeModule: Args: application (Application): Application that is launched. - env (dict): Current environment variables. + env (dict[str, str]): Current environment variables. """ pass @@ -713,7 +751,8 @@ class OpenPypeModule: to receive from 'host' object. Args: - host (ModuleType): Access to installed/registered host object. + host (Union[ModuleType, HostBase]): Access to installed/registered + host object. host_name (str): Name of host. project_name (str): Project name which is main part of host context. @@ -727,47 +766,66 @@ class OpenPypeModule: The best practise is to create click group for whole module which is used to separate commands. - class MyPlugin(OpenPypeModule): - ... - def cli(self, module_click_group): - module_click_group.add_command(cli_main) + Example: + class MyPlugin(AYONAddon): + ... + def cli(self, module_click_group): + module_click_group.add_command(cli_main) - @click.group(, help="") - def cli_main(): - pass + @click.group(, help="") + def cli_main(): + pass - @cli_main.command() - def mycommand(): - print("my_command") + @cli_main.command() + def mycommand(): + print("my_command") + + Args: + module_click_group (click.Group): Group to which can be added + commands. """ pass +class OpenPypeModule(AYONAddon): + """Base class of OpenPype module. + + Instead of 'AYONAddon' are passed in module settings. + + Args: + manager (ModulesManager): Manager object who discovered addon. + settings (dict[str, Any]): OpenPype settings. + """ + + # Disable by default + enabled = False + + class OpenPypeAddOn(OpenPypeModule): # Enable Addon by default enabled = True - def initialize(self, module_settings): - """Initialization is not be required for most of addons.""" - pass - class ModulesManager: """Manager of Pype modules helps to load and prepare them to work. Args: - modules_settings(dict): To be able create module manager with specified - data. For settings changes callbacks and testing purposes. + system_settings (Optional[dict[str, Any]]): OpenPype system settings. + ayon_settings (Optional[dict[str, Any]]): AYON studio settings. """ + # Helper attributes for report _report_total_key = "Total" + _system_settings = None + _ayon_settings = None - def __init__(self, _system_settings=None): + def __init__(self, system_settings=None, ayon_settings=None): self.log = logging.getLogger(self.__class__.__name__) - self._system_settings = _system_settings + self._system_settings = system_settings + self._ayon_settings = ayon_settings self.modules = [] self.modules_by_id = {} @@ -789,8 +847,9 @@ class ModulesManager: default (Any): Default output if module is not available. Returns: - Union[OpenPypeModule, None]: Module found by name or None. + Union[AYONAddon, None]: Module found by name or None. """ + return self.modules_by_name.get(module_name, default) def get_enabled_module(self, module_name, default=None): @@ -804,7 +863,7 @@ class ModulesManager: not enabled. Returns: - Union[OpenPypeModule, None]: Enabled module found by name or None. + Union[AYONAddon, None]: Enabled module found by name or None. """ module = self.get(module_name) @@ -819,11 +878,20 @@ class ModulesManager: import openpype_modules - self.log.debug("*** Pype modules initialization.") + self.log.debug("*** {} initialization.".format( + "AYON addons" + if AYON_SERVER_ENABLED + else "OpenPype modules" + )) # Prepare settings for modules - system_settings = getattr(self, "_system_settings", None) + system_settings = self._system_settings if system_settings is None: system_settings = get_system_settings() + + ayon_settings = self._ayon_settings + if AYON_SERVER_ENABLED and ayon_settings is None: + ayon_settings = get_ayon_settings() + modules_settings = system_settings["modules"] report = {} @@ -836,12 +904,13 @@ class ModulesManager: for name in dir(module): modules_item = getattr(module, name, None) # Filter globals that are not classes which inherit from - # OpenPypeModule + # AYONAddon if ( not inspect.isclass(modules_item) + or modules_item is AYONAddon or modules_item is OpenPypeModule or modules_item is OpenPypeAddOn - or not issubclass(modules_item, OpenPypeModule) + or not issubclass(modules_item, AYONAddon) ): continue @@ -866,10 +935,14 @@ class ModulesManager: module_classes.append(modules_item) for modules_item in module_classes: + is_openpype_module = issubclass(modules_item, OpenPypeModule) + settings = ( + modules_settings if is_openpype_module else ayon_settings + ) + name = modules_item.__name__ try: - name = modules_item.__name__ # Try initialize module - module = modules_item(self, modules_settings) + module = modules_item(self, settings) # Store initialized object self.modules.append(module) self.modules_by_id[module.id] = module @@ -924,8 +997,9 @@ class ModulesManager: """Enabled modules initialized by the manager. Returns: - list: Initialized and enabled modules. + list[AYONAddon]: Initialized and enabled modules. """ + return [ module for module in self.modules @@ -1108,7 +1182,7 @@ class ModulesManager: host_name (str): Host name for which is found host module. Returns: - OpenPypeModule: Found host module by name. + AYONAddon: Found host module by name. None: There was not found module inheriting IHostAddon which has host name set to passed 'host_name'. """ @@ -1129,12 +1203,11 @@ class ModulesManager: inheriting 'IHostAddon'. """ - host_names = { + return { module.host_name for module in self.get_enabled_modules() if isinstance(module, IHostAddon) } - return host_names def print_report(self): """Print out report of time spent on modules initialization parts. @@ -1290,6 +1363,10 @@ class TrayModulesManager(ModulesManager): callback can be defined with `doubleclick_callback` attribute. Missing feature how to define default callback. + + Args: + addon (AYONAddon): Addon object. + callback (FunctionType): Function callback. """ callback_name = "_".join([module.name, callback.__name__]) if callback_name not in self.doubleclick_callbacks: @@ -1310,11 +1387,17 @@ class TrayModulesManager(ModulesManager): self.tray_menu(tray_menu) def get_enabled_tray_modules(self): - output = [] - for module in self.modules: - if module.enabled and isinstance(module, ITrayModule): - output.append(module) - return output + """Enabled tray modules. + + Returns: + list[AYONAddon]: Enabled addons that inherit from tray interface. + """ + + return [ + module + for module in self.modules + if module.enabled and isinstance(module, ITrayModule) + ] def restart_tray(self): if self.tray_manager: diff --git a/openpype/modules/deadline/plugins/publish/submit_houdini_cache_deadline.py b/openpype/modules/deadline/plugins/publish/submit_houdini_cache_deadline.py new file mode 100644 index 0000000000..2b6231e916 --- /dev/null +++ b/openpype/modules/deadline/plugins/publish/submit_houdini_cache_deadline.py @@ -0,0 +1,190 @@ +import os +import getpass +from datetime import datetime + +import hou + +import attr +import pyblish.api +from openpype.lib import ( + TextDef, + NumberDef, +) +from openpype.pipeline import ( + legacy_io, + OpenPypePyblishPluginMixin +) +from openpype.tests.lib import is_in_tests +from openpype.lib import is_running_from_build +from openpype_modules.deadline import abstract_submit_deadline +from openpype_modules.deadline.abstract_submit_deadline import DeadlineJobInfo + + +@attr.s +class HoudiniPluginInfo(object): + Build = attr.ib(default=None) + IgnoreInputs = attr.ib(default=True) + ScriptJob = attr.ib(default=True) + SceneFile = attr.ib(default=None) # Input + SaveFile = attr.ib(default=True) + ScriptFilename = attr.ib(default=None) + OutputDriver = attr.ib(default=None) + Version = attr.ib(default=None) # Mandatory for Deadline + ProjectPath = attr.ib(default=None) + + +class HoudiniCacheSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline, # noqa + OpenPypePyblishPluginMixin): + """Submit Houdini scene to perform a local publish in Deadline. + + Publishing in Deadline can be helpful for scenes that publish very slow. + This way it can process in the background on another machine without the + Artist having to wait for the publish to finish on their local machine. + + Submission is done through the Deadline Web Service as + supplied via the environment variable AVALON_DEADLINE. + + """ + + label = "Submit Scene to Deadline" + order = pyblish.api.IntegratorOrder + hosts = ["houdini"] + families = ["publish.hou"] + targets = ["local"] + + priority = 50 + jobInfo = {} + pluginInfo = {} + group = None + + def get_job_info(self): + job_info = DeadlineJobInfo(Plugin="Houdini") + + job_info.update(self.jobInfo) + instance = self._instance + context = instance.context + assert all( + result["success"] for result in context.data["results"] + ), "Errors found, aborting integration.." + + # Deadline connection + AVALON_DEADLINE = legacy_io.Session.get( + "AVALON_DEADLINE", "http://localhost:8082" + ) + assert AVALON_DEADLINE, "Requires AVALON_DEADLINE" + + project_name = instance.context.data["projectName"] + filepath = context.data["currentFile"] + scenename = os.path.basename(filepath) + job_name = "{scene} - {instance} [PUBLISH]".format( + scene=scenename, instance=instance.name) + batch_name = "{code} - {scene}".format(code=project_name, + scene=scenename) + if is_in_tests(): + batch_name += datetime.now().strftime("%d%m%Y%H%M%S") + + job_info.Name = job_name + job_info.BatchName = batch_name + job_info.Plugin = instance.data["plugin"] + job_info.UserName = context.data.get("deadlineUser", getpass.getuser()) + rop_node = self.get_rop_node(instance) + if rop_node.type().name() != "alembic": + frames = "{start}-{end}x{step}".format( + start=int(instance.data["frameStart"]), + end=int(instance.data["frameEnd"]), + step=int(instance.data["byFrameStep"]), + ) + + job_info.Frames = frames + + job_info.Pool = instance.data.get("primaryPool") + job_info.SecondaryPool = instance.data.get("secondaryPool") + + attr_values = self.get_attr_values_from_data(instance.data) + + job_info.ChunkSize = instance.data["chunkSize"] + job_info.Comment = context.data.get("comment") + job_info.Priority = attr_values.get("priority", self.priority) + job_info.Group = attr_values.get("group", self.group) + + keys = [ + "FTRACK_API_KEY", + "FTRACK_API_USER", + "FTRACK_SERVER", + "OPENPYPE_SG_USER", + "AVALON_PROJECT", + "AVALON_ASSET", + "AVALON_TASK", + "AVALON_APP_NAME", + "OPENPYPE_DEV", + "OPENPYPE_LOG_NO_COLORS", + ] + + # Add OpenPype version if we are running from build. + if is_running_from_build(): + keys.append("OPENPYPE_VERSION") + # Add mongo url if it's enabled + if self._instance.context.data.get("deadlinePassMongoUrl"): + keys.append("OPENPYPE_MONGO") + + environment = dict({key: os.environ[key] for key in keys + if key in os.environ}, **legacy_io.Session) + + for key in keys: + value = environment.get(key) + if not value: + continue + job_info.EnvironmentKeyValue[key] = value + # to recognize render jobs + job_info.add_render_job_env_var() + + return job_info + + def get_plugin_info(self): + instance = self._instance + version = hou.applicationVersionString() + version = ".".join(version.split(".")[:2]) + rop = self.get_rop_node(instance) + plugin_info = HoudiniPluginInfo( + Build=None, + IgnoreInputs=True, + ScriptJob=True, + SceneFile=self.scene_path, + SaveFile=True, + OutputDriver=rop.path(), + Version=version, + ProjectPath=os.path.dirname(self.scene_path) + ) + + plugin_payload = attr.asdict(plugin_info) + + return plugin_payload + + def process(self, instance): + super(HoudiniCacheSubmitDeadline, self).process(instance) + output_dir = os.path.dirname(instance.data["files"][0]) + instance.data["outputDir"] = output_dir + instance.data["toBeRenderedOn"] = "deadline" + + def get_rop_node(self, instance): + rop = instance.data.get("instance_node") + rop_node = hou.node(rop) + + return rop_node + + @classmethod + def get_attribute_defs(cls): + defs = super(HoudiniCacheSubmitDeadline, cls).get_attribute_defs() + defs.extend([ + NumberDef("priority", + minimum=1, + maximum=250, + decimals=0, + default=cls.priority, + label="Priority"), + TextDef("group", + default=cls.group, + label="Group Name"), + ]) + + return defs diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 7d532923ff..26a605a744 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -132,6 +132,8 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline, cls.group = settings.get("group", cls.group) cls.strict_error_checking = settings.get("strict_error_checking", cls.strict_error_checking) + cls.jobInfo = settings.get("jobInfo", cls.jobInfo) + cls.pluginInfo = settings.get("pluginInfo", cls.pluginInfo) def get_job_info(self): job_info = DeadlineJobInfo(Plugin="MayaBatch") @@ -648,7 +650,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline, return job_info, attr.asdict(plugin_info) def _get_arnold_render_payload(self, data): - + from maya import cmds # Job Info job_info = copy.deepcopy(self.job_info) job_info.Name = self._job_info_label("Render") diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_cache_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_cache_job.py new file mode 100644 index 0000000000..b2ebebc303 --- /dev/null +++ b/openpype/modules/deadline/plugins/publish/submit_publish_cache_job.py @@ -0,0 +1,506 @@ +# -*- coding: utf-8 -*- +"""Submit publishing job to farm.""" +import os +import json +import re +from copy import deepcopy +import requests + +import pyblish.api + +from openpype import AYON_SERVER_ENABLED +from openpype.client import ( + get_last_version_by_subset_name, +) +from openpype.pipeline import publish, legacy_io +from openpype.lib import EnumDef, is_running_from_build +from openpype.tests.lib import is_in_tests +from openpype.pipeline.version_start import get_versioning_start + +from openpype.pipeline.farm.pyblish_functions import ( + create_skeleton_instance_cache, + create_instances_for_cache, + attach_instances_to_subset, + prepare_cache_representations, + create_metadata_path +) + + +class ProcessSubmittedCacheJobOnFarm(pyblish.api.InstancePlugin, + publish.OpenPypePyblishPluginMixin, + publish.ColormanagedPyblishPluginMixin): + """Process Cache Job submitted on farm + This is replicated version of submit publish job + specifically for cache(s). + + These jobs are dependent on a deadline job + submission prior to this plug-in. + + - In case of Deadline, it creates dependent job on farm publishing + rendered image sequence. + + Options in instance.data: + - deadlineSubmissionJob (dict, Required): The returned .json + data from the job submission to deadline. + + - outputDir (str, Required): The output directory where the metadata + file should be generated. It's assumed that this will also be + final folder containing the output files. + + - ext (str, Optional): The extension (including `.`) that is required + in the output filename to be picked up for image sequence + publishing. + + - expectedFiles (list or dict): explained below + + """ + + label = "Submit cache jobs to Deadline" + order = pyblish.api.IntegratorOrder + 0.2 + icon = "tractor" + + targets = ["local"] + + hosts = ["houdini"] + + families = ["publish.hou"] + + environ_job_filter = [ + "OPENPYPE_METADATA_FILE" + ] + + environ_keys = [ + "FTRACK_API_USER", + "FTRACK_API_KEY", + "FTRACK_SERVER", + "AVALON_APP_NAME", + "OPENPYPE_USERNAME", + "OPENPYPE_SG_USER", + "KITSU_LOGIN", + "KITSU_PWD" + ] + + # custom deadline attributes + deadline_department = "" + deadline_pool = "" + deadline_pool_secondary = "" + deadline_group = "" + deadline_chunk_size = 1 + deadline_priority = None + + # regex for finding frame number in string + R_FRAME_NUMBER = re.compile(r'.+\.(?P[0-9]+)\..+') + + plugin_pype_version = "3.0" + + # script path for publish_filesequence.py + publishing_script = None + + def _submit_deadline_post_job(self, instance, job): + """Submit publish job to Deadline. + + Deadline specific code separated from :meth:`process` for sake of + more universal code. Muster post job is sent directly by Muster + submitter, so this type of code isn't necessary for it. + + Returns: + (str): deadline_publish_job_id + """ + data = instance.data.copy() + subset = data["subset"] + job_name = "Publish - {subset}".format(subset=subset) + + anatomy = instance.context.data['anatomy'] + + # instance.data.get("subset") != instances[0]["subset"] + # 'Main' vs 'renderMain' + override_version = None + instance_version = instance.data.get("version") # take this if exists + if instance_version != 1: + override_version = instance_version + + output_dir = self._get_publish_folder( + anatomy, + deepcopy(instance.data["anatomyData"]), + instance.data.get("asset"), + instance.data["subset"], + instance.context, + instance.data["family"], + override_version + ) + + # Transfer the environment from the original job to this dependent + # job so they use the same environment + metadata_path, rootless_metadata_path = \ + create_metadata_path(instance, anatomy) + + environment = { + "AVALON_PROJECT": instance.context.data["projectName"], + "AVALON_ASSET": instance.context.data["asset"], + "AVALON_TASK": instance.context.data["task"], + "OPENPYPE_USERNAME": instance.context.data["user"], + "OPENPYPE_LOG_NO_COLORS": "1", + "IS_TEST": str(int(is_in_tests())) + } + + if AYON_SERVER_ENABLED: + environment["AYON_PUBLISH_JOB"] = "1" + environment["AYON_RENDER_JOB"] = "0" + environment["AYON_REMOTE_PUBLISH"] = "0" + environment["AYON_BUNDLE_NAME"] = os.environ["AYON_BUNDLE_NAME"] + deadline_plugin = "Ayon" + else: + environment["OPENPYPE_PUBLISH_JOB"] = "1" + environment["OPENPYPE_RENDER_JOB"] = "0" + environment["OPENPYPE_REMOTE_PUBLISH"] = "0" + deadline_plugin = "OpenPype" + # Add OpenPype version if we are running from build. + if is_running_from_build(): + self.environ_keys.append("OPENPYPE_VERSION") + + # add environments from self.environ_keys + for env_key in self.environ_keys: + if os.getenv(env_key): + environment[env_key] = os.environ[env_key] + + # pass environment keys from self.environ_job_filter + job_environ = job["Props"].get("Env", {}) + for env_j_key in self.environ_job_filter: + if job_environ.get(env_j_key): + environment[env_j_key] = job_environ[env_j_key] + + # Add mongo url if it's enabled + if instance.context.data.get("deadlinePassMongoUrl"): + mongo_url = os.environ.get("OPENPYPE_MONGO") + if mongo_url: + environment["OPENPYPE_MONGO"] = mongo_url + + priority = self.deadline_priority or instance.data.get("priority", 50) + + instance_settings = self.get_attr_values_from_data(instance.data) + initial_status = instance_settings.get("publishJobState", "Active") + # TODO: Remove this backwards compatibility of `suspend_publish` + if instance.data.get("suspend_publish"): + initial_status = "Suspended" + + args = [ + "--headless", + 'publish', + '"{}"'.format(rootless_metadata_path), + "--targets", "deadline", + "--targets", "farm" + ] + + if is_in_tests(): + args.append("--automatic-tests") + + # Generate the payload for Deadline submission + secondary_pool = ( + self.deadline_pool_secondary or instance.data.get("secondaryPool") + ) + payload = { + "JobInfo": { + "Plugin": deadline_plugin, + "BatchName": job["Props"]["Batch"], + "Name": job_name, + "UserName": job["Props"]["User"], + "Comment": instance.context.data.get("comment", ""), + + "Department": self.deadline_department, + "ChunkSize": self.deadline_chunk_size, + "Priority": priority, + "InitialStatus": initial_status, + + "Group": self.deadline_group, + "Pool": self.deadline_pool or instance.data.get("primaryPool"), + "SecondaryPool": secondary_pool, + # ensure the outputdirectory with correct slashes + "OutputDirectory0": output_dir.replace("\\", "/") + }, + "PluginInfo": { + "Version": self.plugin_pype_version, + "Arguments": " ".join(args), + "SingleFrameOnly": "True", + }, + # Mandatory for Deadline, may be empty + "AuxFiles": [], + } + + if job.get("_id"): + payload["JobInfo"]["JobDependency0"] = job["_id"] + + for index, (key_, value_) in enumerate(environment.items()): + payload["JobInfo"].update( + { + "EnvironmentKeyValue%d" + % index: "{key}={value}".format( + key=key_, value=value_ + ) + } + ) + # remove secondary pool + payload["JobInfo"].pop("SecondaryPool", None) + + self.log.debug("Submitting Deadline publish job ...") + + url = "{}/api/jobs".format(self.deadline_url) + response = requests.post(url, json=payload, timeout=10) + if not response.ok: + raise Exception(response.text) + + deadline_publish_job_id = response.json()["_id"] + + return deadline_publish_job_id + + def process(self, instance): + # type: (pyblish.api.Instance) -> None + """Process plugin. + + Detect type of render farm submission and create and post dependent + job in case of Deadline. It creates json file with metadata needed for + publishing in directory of render. + + Args: + instance (pyblish.api.Instance): Instance data. + + """ + if not instance.data.get("farm"): + self.log.debug("Skipping local instance.") + return + + anatomy = instance.context.data["anatomy"] + + instance_skeleton_data = create_skeleton_instance_cache(instance) + """ + if content of `expectedFiles` list are dictionaries, we will handle + it as list of AOVs, creating instance for every one of them. + + Example: + -------- + + expectedFiles = [ + { + "beauty": [ + "foo_v01.0001.exr", + "foo_v01.0002.exr" + ], + + "Z": [ + "boo_v01.0001.exr", + "boo_v01.0002.exr" + ] + } + ] + + This will create instances for `beauty` and `Z` subset + adding those files to their respective representations. + + If we have only list of files, we collect all file sequences. + More then one doesn't probably make sense, but we'll handle it + like creating one instance with multiple representations. + + Example: + -------- + + expectedFiles = [ + "foo_v01.0001.exr", + "foo_v01.0002.exr", + "xxx_v01.0001.exr", + "xxx_v01.0002.exr" + ] + + This will result in one instance with two representations: + `foo` and `xxx` + """ + + if isinstance(instance.data.get("expectedFiles")[0], dict): + instances = create_instances_for_cache( + instance, instance_skeleton_data) + else: + representations = prepare_cache_representations( + instance_skeleton_data, + instance.data.get("expectedFiles"), + anatomy + ) + + if "representations" not in instance_skeleton_data.keys(): + instance_skeleton_data["representations"] = [] + + # add representation + instance_skeleton_data["representations"] += representations + instances = [instance_skeleton_data] + + # attach instances to subset + if instance.data.get("attachTo"): + instances = attach_instances_to_subset( + instance.data.get("attachTo"), instances + ) + + r''' SUBMiT PUBLiSH JOB 2 D34DLiN3 + ____ + ' ' .---. .---. .--. .---. .--..--..--..--. .---. + | | --= \ | . \/ _|/ \| . \ || || \ |/ _| + | JOB | --= / | | || __| .. | | | |;_ || \ || __| + | | |____./ \.__|._||_.|___./|_____|||__|\__|\.___| + ._____. + + ''' + + render_job = None + submission_type = "" + if instance.data.get("toBeRenderedOn") == "deadline": + render_job = instance.data.pop("deadlineSubmissionJob", None) + submission_type = "deadline" + + if not render_job: + import getpass + + render_job = {} + self.log.debug("Faking job data ...") + render_job["Props"] = {} + # Render job doesn't exist because we do not have prior submission. + # We still use data from it so lets fake it. + # + # Batch name reflect original scene name + + if instance.data.get("assemblySubmissionJobs"): + render_job["Props"]["Batch"] = instance.data.get( + "jobBatchName") + else: + batch = os.path.splitext(os.path.basename( + instance.context.data.get("currentFile")))[0] + render_job["Props"]["Batch"] = batch + # User is deadline user + render_job["Props"]["User"] = instance.context.data.get( + "deadlineUser", getpass.getuser()) + + deadline_publish_job_id = None + if submission_type == "deadline": + # get default deadline webservice url from deadline module + self.deadline_url = instance.context.data["defaultDeadline"] + # if custom one is set in instance, use that + if instance.data.get("deadlineUrl"): + self.deadline_url = instance.data.get("deadlineUrl") + assert self.deadline_url, "Requires Deadline Webservice URL" + + deadline_publish_job_id = \ + self._submit_deadline_post_job(instance, render_job) + + # Inject deadline url to instances. + for inst in instances: + inst["deadlineUrl"] = self.deadline_url + + # publish job file + publish_job = { + "asset": instance_skeleton_data["asset"], + "frameStart": instance_skeleton_data["frameStart"], + "frameEnd": instance_skeleton_data["frameEnd"], + "fps": instance_skeleton_data["fps"], + "source": instance_skeleton_data["source"], + "user": instance.context.data["user"], + "version": instance.context.data["version"], # workfile version + "intent": instance.context.data.get("intent"), + "comment": instance.context.data.get("comment"), + "job": render_job or None, + "session": legacy_io.Session.copy(), + "instances": instances + } + + if deadline_publish_job_id: + publish_job["deadline_publish_job_id"] = deadline_publish_job_id + + metadata_path, rootless_metadata_path = \ + create_metadata_path(instance, anatomy) + + with open(metadata_path, "w") as f: + json.dump(publish_job, f, indent=4, sort_keys=True) + + def _get_publish_folder(self, anatomy, template_data, + asset, subset, context, + family, version=None): + """ + Extracted logic to pre-calculate real publish folder, which is + calculated in IntegrateNew inside of Deadline process. + This should match logic in: + 'collect_anatomy_instance_data' - to + get correct anatomy, family, version for subset and + 'collect_resources_path' + get publish_path + + Args: + anatomy (openpype.pipeline.anatomy.Anatomy): + template_data (dict): pre-calculated collected data for process + asset (string): asset name + subset (string): subset name (actually group name of subset) + family (string): for current deadline process it's always 'render' + TODO - for generic use family needs to be dynamically + calculated like IntegrateNew does + version (int): override version from instance if exists + + Returns: + (string): publish folder where rendered and published files will + be stored + based on 'publish' template + """ + + project_name = context.data["projectName"] + if not version: + version = get_last_version_by_subset_name( + project_name, + subset, + asset_name=asset + ) + if version: + version = int(version["name"]) + 1 + else: + version = get_versioning_start( + project_name, + template_data["app"], + task_name=template_data["task"]["name"], + task_type=template_data["task"]["type"], + family="render", + subset=subset, + project_settings=context.data["project_settings"] + ) + + host_name = context.data["hostName"] + task_info = template_data.get("task") or {} + + template_name = publish.get_publish_template_name( + project_name, + host_name, + family, + task_info.get("name"), + task_info.get("type"), + ) + + template_data["subset"] = subset + template_data["family"] = family + template_data["version"] = version + + render_templates = anatomy.templates_obj[template_name] + if "folder" in render_templates: + publish_folder = render_templates["folder"].format_strict( + template_data + ) + else: + # solve deprecated situation when `folder` key is not underneath + # `publish` anatomy + self.log.warning(( + "Deprecation warning: Anatomy does not have set `folder`" + " key underneath `publish` (in global of for project `{}`)." + ).format(project_name)) + + file_path = render_templates["path"].format_strict(template_data) + publish_folder = os.path.dirname(file_path) + + return publish_folder + + @classmethod + def get_attribute_defs(cls): + return [ + EnumDef("publishJobState", + label="Publish Job State", + items=["Active", "Suspended"], + default="Active") + ] diff --git a/openpype/modules/deadline/plugins/publish/validate_deadline_pools.py b/openpype/modules/deadline/plugins/publish/validate_deadline_pools.py index 949caff7d8..90b8241803 100644 --- a/openpype/modules/deadline/plugins/publish/validate_deadline_pools.py +++ b/openpype/modules/deadline/plugins/publish/validate_deadline_pools.py @@ -22,7 +22,8 @@ class ValidateDeadlinePools(OptionalPyblishPluginMixin, "render.frames_farm", "renderFarm", "renderlayer", - "maxrender"] + "maxrender", + "publish.hou"] optional = True # cache diff --git a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_instances.py b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_instances.py index 75f43cb22f..edad0b0132 100644 --- a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_instances.py +++ b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_instances.py @@ -61,6 +61,7 @@ class IntegrateFtrackInstance(pyblish.api.InstancePlugin): additional_metadata_keys = [] def process(self, instance): + # QUESTION: should this be operating even for `farm` target? self.log.debug("instance {}".format(instance)) instance_repres = instance.data.get("representations") @@ -143,70 +144,93 @@ class IntegrateFtrackInstance(pyblish.api.InstancePlugin): unmanaged_location_name = "ftrack.unmanaged" ftrack_server_location_name = "ftrack.server" + # check if any outputName keys are in review_representations + # also check if any outputName keys are in thumbnail_representations + synced_multiple_output_names = [] + for review_repre in review_representations: + review_output_name = review_repre.get("outputName") + if not review_output_name: + continue + for thumb_repre in thumbnail_representations: + thumb_output_name = thumb_repre.get("outputName") + if not thumb_output_name: + continue + if ( + thumb_output_name == review_output_name + # output name can be added also as tags during intermediate + # files creation + or thumb_output_name in review_repre.get("tags", []) + ): + synced_multiple_output_names.append( + thumb_repre["outputName"]) + self.log.debug("Multiple output names: {}".format( + synced_multiple_output_names + )) + multiple_synced_thumbnails = len(synced_multiple_output_names) > 1 + # Components data component_list = [] - # Components that will be duplicated to unmanaged location - src_components_to_add = [] + thumbnail_data_items = [] # Create thumbnail components - # TODO what if there is multiple thumbnails? - first_thumbnail_component = None - first_thumbnail_component_repre = None - - if not review_representations or has_movie_review: - for repre in thumbnail_representations: - repre_path = get_publish_repre_path(instance, repre, False) - if not repre_path: - self.log.warning( - "Published path is not set and source was removed." - ) - continue - - # Create copy of base comp item and append it - thumbnail_item = copy.deepcopy(base_component_item) - thumbnail_item["component_path"] = repre_path - thumbnail_item["component_data"] = { - "name": "thumbnail" - } - thumbnail_item["thumbnail"] = True - - # Create copy of item before setting location - if "delete" not in repre.get("tags", []): - src_components_to_add.append(copy.deepcopy(thumbnail_item)) - # Create copy of first thumbnail - if first_thumbnail_component is None: - first_thumbnail_component_repre = repre - first_thumbnail_component = thumbnail_item - # Set location - thumbnail_item["component_location_name"] = ( - ftrack_server_location_name - ) - - # Add item to component list - component_list.append(thumbnail_item) - - if first_thumbnail_component is not None: - metadata = self._prepare_image_component_metadata( - first_thumbnail_component_repre, - first_thumbnail_component["component_path"] + for repre in thumbnail_representations: + # get repre path from representation + # and return published_path if available + # the path is validated and if it does not exists it returns None + repre_path = get_publish_repre_path( + instance, + repre, + only_published=False ) + if not repre_path: + self.log.warning( + "Published path is not set or source was removed." + ) + continue - if metadata: - component_data = first_thumbnail_component["component_data"] - component_data["metadata"] = metadata + # Create copy of base comp item and append it + thumbnail_item = copy.deepcopy(base_component_item) + thumbnail_item.update({ + "component_path": repre_path, + "component_data": { + "name": ( + "thumbnail" if review_representations + else "ftrackreview-image" + ), + "metadata": self._prepare_image_component_metadata( + repre, + repre_path + ) + }, + "thumbnail": True, + "component_location_name": ftrack_server_location_name + }) - if review_representations: - component_data["name"] = "thumbnail" - else: - component_data["name"] = "ftrackreview-image" + # add thumbnail data to items for future synchronization + current_item_data = { + "sync_key": repre.get("outputName"), + "representation": repre, + "item": thumbnail_item + } + # Create copy of item before setting location + if "delete" not in repre.get("tags", []): + src_comp = self._create_src_component( + instance, + repre, + copy.deepcopy(thumbnail_item), + unmanaged_location_name + ) + component_list.append(src_comp) + + current_item_data["src_component"] = src_comp + + # Add item to component list + thumbnail_data_items.append(current_item_data) # Create review components # Change asset name of each new component for review - is_first_review_repre = True - not_first_components = [] - extended_asset_name = "" multiple_reviewable = len(review_representations) > 1 - for repre in review_representations: + for index, repre in enumerate(review_representations): if not self._is_repre_video(repre) and has_movie_review: self.log.debug("Movie repre has priority " "from {}".format(repre)) @@ -222,45 +246,50 @@ class IntegrateFtrackInstance(pyblish.api.InstancePlugin): # Create copy of base comp item and append it review_item = copy.deepcopy(base_component_item) - # get asset name and define extended name variant - asset_name = review_item["asset_data"]["name"] - extended_asset_name = "_".join( - (asset_name, repre["name"]) + # get first or synchronize thumbnail item + sync_thumbnail_item = None + sync_thumbnail_item_src = None + sync_thumbnail_data = self._get_matching_thumbnail_item( + repre, + thumbnail_data_items, + multiple_synced_thumbnails ) + if sync_thumbnail_data: + sync_thumbnail_item = sync_thumbnail_data.get("item") + sync_thumbnail_item_src = sync_thumbnail_data.get( + "src_component") - # reset extended if no need for extended asset name - if ( - self.keep_first_subset_name_for_review - and is_first_review_repre - ): - extended_asset_name = "" - else: - # only rename if multiple reviewable - if multiple_reviewable: - review_item["asset_data"]["name"] = extended_asset_name - else: - extended_asset_name = "" + """ + Renaming asset name only to those components which are explicitly + allowed in settings. Usually clients wanted to keep first component + as untouched product name with version and any other assetVersion + to be named with extended form. The renaming will only happen if + there is more than one reviewable component and extended name is + not empty. + """ + extended_asset_name = self._make_extended_component_name( + base_component_item, repre, index) - # rename all already created components - # only if first repre and extended name available - if is_first_review_repre and extended_asset_name: - # and rename all already created components - for _ci in component_list: - _ci["asset_data"]["name"] = extended_asset_name + if multiple_reviewable and extended_asset_name: + review_item["asset_data"]["name"] = extended_asset_name + # rename also thumbnail + if sync_thumbnail_item: + sync_thumbnail_item["asset_data"]["name"] = ( + extended_asset_name + ) + # rename also src_thumbnail + if sync_thumbnail_item_src: + sync_thumbnail_item_src["asset_data"]["name"] = ( + extended_asset_name + ) - # and rename all already created src components - for _sci in src_components_to_add: - _sci["asset_data"]["name"] = extended_asset_name - - # rename also first thumbnail component if any - if first_thumbnail_component is not None: - first_thumbnail_component[ - "asset_data"]["name"] = extended_asset_name - - # Change location - review_item["component_path"] = repre_path - # Change component data + # adding thumbnail component to component list + if sync_thumbnail_item: + component_list.append(copy.deepcopy(sync_thumbnail_item)) + if sync_thumbnail_item_src: + component_list.append(copy.deepcopy(sync_thumbnail_item_src)) + # add metadata to review component if self._is_repre_video(repre): component_name = "ftrackreview-mp4" metadata = self._prepare_video_component_metadata( @@ -273,28 +302,29 @@ class IntegrateFtrackInstance(pyblish.api.InstancePlugin): ) review_item["thumbnail"] = True - review_item["component_data"] = { - # Default component name is "main". - "name": component_name, - "metadata": metadata - } - - if is_first_review_repre: - is_first_review_repre = False - else: - # later detection for thumbnail duplication - not_first_components.append(review_item) + review_item.update({ + "component_path": repre_path, + "component_data": { + "name": component_name, + "metadata": metadata + }, + "component_location_name": ftrack_server_location_name + }) # Create copy of item before setting location if "delete" not in repre.get("tags", []): - src_components_to_add.append(copy.deepcopy(review_item)) + src_comp = self._create_src_component( + instance, + repre, + copy.deepcopy(review_item), + unmanaged_location_name + ) + component_list.append(src_comp) - # Set location - review_item["component_location_name"] = ( - ftrack_server_location_name - ) # Add item to component list component_list.append(review_item) + + if self.upload_reviewable_with_origin_name: origin_name_component = copy.deepcopy(review_item) filename = os.path.basename(repre_path) @@ -303,34 +333,6 @@ class IntegrateFtrackInstance(pyblish.api.InstancePlugin): ) component_list.append(origin_name_component) - # Duplicate thumbnail component for all not first reviews - if first_thumbnail_component is not None: - for component_item in not_first_components: - asset_name = component_item["asset_data"]["name"] - new_thumbnail_component = copy.deepcopy( - first_thumbnail_component - ) - new_thumbnail_component["asset_data"]["name"] = asset_name - new_thumbnail_component["component_location_name"] = ( - ftrack_server_location_name - ) - component_list.append(new_thumbnail_component) - - # Add source components for review and thubmnail components - for copy_src_item in src_components_to_add: - # Make sure thumbnail is disabled - copy_src_item["thumbnail"] = False - # Set location - copy_src_item["component_location_name"] = unmanaged_location_name - # Modify name of component to have suffix "_src" - component_data = copy_src_item["component_data"] - component_name = component_data["name"] - component_data["name"] = component_name + "_src" - component_data["metadata"] = self._prepare_component_metadata( - instance, repre, copy_src_item["component_path"], False - ) - component_list.append(copy_src_item) - # Add others representations as component for repre in other_representations: published_path = get_publish_repre_path(instance, repre, True) @@ -346,15 +348,17 @@ class IntegrateFtrackInstance(pyblish.api.InstancePlugin): ): other_item["asset_data"]["name"] = extended_asset_name - component_data = { - "name": repre["name"], - "metadata": self._prepare_component_metadata( - instance, repre, published_path, False - ) - } - other_item["component_data"] = component_data - other_item["component_location_name"] = unmanaged_location_name - other_item["component_path"] = published_path + other_item.update({ + "component_path": published_path, + "component_data": { + "name": repre["name"], + "metadata": self._prepare_component_metadata( + instance, repre, published_path, False + ) + }, + "component_location_name": unmanaged_location_name, + }) + component_list.append(other_item) def json_obj_parser(obj): @@ -370,6 +374,124 @@ class IntegrateFtrackInstance(pyblish.api.InstancePlugin): )) instance.data["ftrackComponentsList"] = component_list + def _get_matching_thumbnail_item( + self, + review_representation, + thumbnail_data_items, + are_multiple_synced_thumbnails + ): + """Return matching thumbnail item from list of thumbnail items. + + If a thumbnail item already exists, this should return it. + The benefit is that if an `outputName` key is found in + representation and is also used as a `sync_key` in a thumbnail + data item, it can sync with that item. + + Args: + review_representation (dict): Review representation + thumbnail_data_items (list): List of thumbnail data items + are_multiple_synced_thumbnails (bool): If there are multiple synced + thumbnails + + Returns: + dict: Thumbnail data item or empty dict + """ + output_name = review_representation.get("outputName") + tags = review_representation.get("tags", []) + matching_thumbnail_item = {} + for thumb_item in thumbnail_data_items: + if ( + are_multiple_synced_thumbnails + and ( + thumb_item["sync_key"] == output_name + # intermediate files can have preset name in tags + # this is usually aligned with `outputName` distributed + # during thumbnail creation in `need_thumbnail` tagging + # workflow + or thumb_item["sync_key"] in tags + ) + ): + # return only synchronized thumbnail if multiple + matching_thumbnail_item = thumb_item + break + elif not are_multiple_synced_thumbnails: + # return any first found thumbnail since we need thumbnail + # but dont care which one + matching_thumbnail_item = thumb_item + break + + if not matching_thumbnail_item: + # WARNING: this can only happen if multiple thumbnails + # workflow is broken, since it found multiple matching outputName + # in representation but they do not align with any thumbnail item + self.log.warning( + "No matching thumbnail item found for output name " + "'{}'".format(output_name) + ) + if not thumbnail_data_items: + self.log.warning( + "No thumbnail data items found" + ) + return {} + # as fallback return first thumbnail item + return thumbnail_data_items[0] + + return matching_thumbnail_item + + def _make_extended_component_name( + self, component_item, repre, iteration_index): + """ Returns the extended component name + + Name is based on the asset name and representation name. + + Args: + component_item (dict): The component item dictionary. + repre (dict): The representation dictionary. + iteration_index (int): The index of the iteration. + + Returns: + str: The extended component name. + + """ + # reset extended if no need for extended asset name + if self.keep_first_subset_name_for_review and iteration_index == 0: + return + + # get asset name and define extended name variant + asset_name = component_item["asset_data"]["name"] + return "_".join( + (asset_name, repre["name"]) + ) + + def _create_src_component( + self, instance, repre, component_item, location): + """Create src component for thumbnail. + + This will replicate the input component and change its name to + have suffix "_src". + + Args: + instance (pyblish.api.Instance): Instance + repre (dict): Representation + component_item (dict): Component item + location (str): Location name + + Returns: + dict: Component item + """ + # Make sure thumbnail is disabled + component_item["thumbnail"] = False + # Set location + component_item["component_location_name"] = location + # Modify name of component to have suffix "_src" + component_data = component_item["component_data"] + component_name = component_data["name"] + component_data["name"] = component_name + "_src" + component_data["metadata"] = self._prepare_component_metadata( + instance, repre, component_item["component_path"], False + ) + return component_item + def _collect_additional_metadata(self, streams): pass @@ -472,9 +594,6 @@ class IntegrateFtrackInstance(pyblish.api.InstancePlugin): stream_width = tmp_width stream_height = tmp_height - self.log.debug("FPS from stream is {} and duration is {}".format( - input_framerate, stream_duration - )) frame_out = float(stream_duration) * stream_fps break diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index 3eb49a39ee..e13bf97e54 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -189,7 +189,7 @@ class CreatePublishRoyalRenderJob(pyblish.api.InstancePlugin, environment = RREnvList({ "AVALON_PROJECT": anatomy_data["project"]["name"], - "AVALON_ASSET": anatomy_data["asset"], + "AVALON_ASSET": instance.context.data["asset"], "AVALON_TASK": anatomy_data["task"]["name"], "OPENPYPE_USERNAME": anatomy_data["user"] }) diff --git a/openpype/modules/timers_manager/timers_manager.py b/openpype/modules/timers_manager/timers_manager.py index 43286f7da4..674d834a1d 100644 --- a/openpype/modules/timers_manager/timers_manager.py +++ b/openpype/modules/timers_manager/timers_manager.py @@ -247,7 +247,7 @@ class TimersManager( return { "project_name": project_name, "asset_id": str(asset_doc["_id"]), - "asset_name": asset_doc["name"], + "asset_name": asset_name, "task_name": task_name, "task_type": task_type, "hierarchy": hierarchy_items diff --git a/openpype/pipeline/context_tools.py b/openpype/pipeline/context_tools.py index 034bbc0070..a607c90912 100644 --- a/openpype/pipeline/context_tools.py +++ b/openpype/pipeline/context_tools.py @@ -18,6 +18,7 @@ from openpype.client import ( get_asset_by_id, get_asset_by_name, version_is_latest, + get_asset_name_identifier, get_ayon_server_api_connection, ) from openpype.lib.events import emit_event @@ -44,7 +45,7 @@ from . import ( _is_installed = False _process_id = None -_registered_root = {"_": ""} +_registered_root = {"_": {}} _registered_host = {"_": None} # Keep modules manager (and it's modules) in memory # - that gives option to register modules' callbacks @@ -85,15 +86,22 @@ def register_root(path): def registered_root(): - """Return currently registered root""" - root = _registered_root["_"] - if root: - return root + """Return registered roots from current project anatomy. - root = legacy_io.Session.get("AVALON_PROJECTS") - if root: - return os.path.normpath(root) - return "" + Consider this does return roots only for current project and current + platforms, only if host was installer using 'install_host'. + + Deprecated: + Please use project 'Anatomy' to get roots. This function is still used + at current core functions of load logic, but that will change + in future and this function will be removed eventually. Using this + function at new places can cause problems in the future. + + Returns: + dict[str, str]: Root paths. + """ + + return _registered_root["_"] def install_host(host): @@ -592,14 +600,12 @@ def compute_session_changes( Dict[str, str]: Changes in the Session dictionary. """ - changes = {} - # Get asset document and asset if not asset_doc: task_name = None asset_name = None else: - asset_name = asset_doc["name"] + asset_name = get_asset_name_identifier(asset_doc) # Detect any changes compared session mapping = { diff --git a/openpype/pipeline/create/context.py b/openpype/pipeline/create/context.py index 25f03ddd3b..683699a0d1 100644 --- a/openpype/pipeline/create/context.py +++ b/openpype/pipeline/create/context.py @@ -11,7 +11,12 @@ from contextlib import contextmanager import pyblish.logic import pyblish.api -from openpype.client import get_assets, get_asset_by_name +from openpype import AYON_SERVER_ENABLED +from openpype.client import ( + get_assets, + get_asset_by_name, + get_asset_name_identifier, +) from openpype.settings import ( get_system_settings, get_project_settings @@ -922,9 +927,19 @@ class CreatedInstance: self._orig_data = copy.deepcopy(data) # Pop family and subset to prevent unexpected changes + # TODO change to 'productType' and 'productName' in AYON data.pop("family", None) data.pop("subset", None) + if AYON_SERVER_ENABLED: + asset_name = data.pop("asset", None) + if "folderPath" not in data: + data["folderPath"] = asset_name + + elif "folderPath" in data: + asset_name = data.pop("folderPath").split("/")[-1] + if "asset" not in data: + data["asset"] = asset_name # QUESTION Does it make sense to have data stored as ordered dict? self._data = collections.OrderedDict() @@ -1268,6 +1283,8 @@ class CreatedInstance: def has_set_asset(self): """Asset name is set in data.""" + if AYON_SERVER_ENABLED: + return "folderPath" in self._data return "asset" in self._data @property @@ -2003,8 +2020,14 @@ class CreateContext: project_name, self.host_name ) + asset_name = get_asset_name_identifier(asset_doc) + if AYON_SERVER_ENABLED: + asset_name_key = "folderPath" + else: + asset_name_key = "asset" + instance_data = { - "asset": asset_doc["name"], + asset_name_key: asset_name, "task": task_name, "family": creator.family, "variant": variant @@ -2229,34 +2252,51 @@ class CreateContext: task_names_by_asset_name = {} for instance in instances: task_name = instance.get("task") - asset_name = instance.get("asset") + if AYON_SERVER_ENABLED: + asset_name = instance.get("folderPath") + else: + asset_name = instance.get("asset") if asset_name: task_names_by_asset_name[asset_name] = set() if task_name: task_names_by_asset_name[asset_name].add(task_name) - asset_names = [ + asset_names = { asset_name for asset_name in task_names_by_asset_name.keys() if asset_name is not None - ] + } + fields = {"name", "data.tasks"} + if AYON_SERVER_ENABLED: + fields |= {"data.parents"} asset_docs = list(get_assets( self.project_name, asset_names=asset_names, - fields=["name", "data.tasks"] + fields=fields )) task_names_by_asset_name = {} + asset_docs_by_name = collections.defaultdict(list) for asset_doc in asset_docs: - asset_name = asset_doc["name"] + asset_name = get_asset_name_identifier(asset_doc) tasks = asset_doc.get("data", {}).get("tasks") or {} task_names_by_asset_name[asset_name] = set(tasks.keys()) + asset_docs_by_name[asset_doc["name"]].append(asset_doc) for instance in instances: if not instance.has_valid_asset or not instance.has_valid_task: continue - asset_name = instance["asset"] + if AYON_SERVER_ENABLED: + asset_name = instance["folderPath"] + if asset_name and "/" not in asset_name: + asset_docs = asset_docs_by_name.get(asset_name) + if len(asset_docs) == 1: + asset_name = get_asset_name_identifier(asset_docs[0]) + instance["folderPath"] = asset_name + else: + asset_name = instance["asset"] + if asset_name not in task_names_by_asset_name: instance.set_asset_invalid(True) continue diff --git a/openpype/pipeline/create/utils.py b/openpype/pipeline/create/utils.py index 2ef1f02bd6..ce4af8f474 100644 --- a/openpype/pipeline/create/utils.py +++ b/openpype/pipeline/create/utils.py @@ -1,6 +1,11 @@ import collections -from openpype.client import get_assets, get_subsets, get_last_versions +from openpype.client import ( + get_assets, + get_subsets, + get_last_versions, + get_asset_name_identifier, +) def get_last_versions_for_instances( @@ -52,10 +57,10 @@ def get_last_versions_for_instances( asset_docs = get_assets( project_name, asset_names=subset_names_by_asset_name.keys(), - fields=["name", "_id"] + fields=["name", "_id", "data.parents"] ) asset_names_by_id = { - asset_doc["_id"]: asset_doc["name"] + asset_doc["_id"]: get_asset_name_identifier(asset_doc) for asset_doc in asset_docs } if not asset_names_by_id: diff --git a/openpype/pipeline/farm/pyblish_functions.py b/openpype/pipeline/farm/pyblish_functions.py index 7ef3439dbd..ce0ef4d343 100644 --- a/openpype/pipeline/farm/pyblish_functions.py +++ b/openpype/pipeline/farm/pyblish_functions.py @@ -745,6 +745,238 @@ def get_resources(project_name, version, extension=None): return resources +def create_skeleton_instance_cache(instance): + # type: (pyblish.api.Instance, list, dict) -> dict + """Create skeleton instance from original instance data. + + This will create dictionary containing skeleton + - common - data used for publishing rendered instances. + This skeleton instance is then extended with additional data + and serialized to be processed by farm job. + + Args: + instance (pyblish.api.Instance): Original instance to + be used as a source of data. + + Returns: + dict: Dictionary with skeleton instance data. + + """ + # list of family names to transfer to new family if present + + context = instance.context + data = instance.data.copy() + anatomy = instance.context.data["anatomy"] # type: Anatomy + + # get time related data from instance (or context) + time_data = get_time_data_from_instance_or_context(instance) + + if data.get("extendFrames", False): + time_data.start, time_data.end = extend_frames( + data["asset"], + data["subset"], + time_data.start, + time_data.end, + ) + + source = data.get("source") or context.data.get("currentFile") + success, rootless_path = ( + anatomy.find_root_template_from_path(source) + ) + if success: + source = rootless_path + else: + # `rootless_path` is not set to `source` if none of roots match + log = Logger.get_logger("farm_publishing") + log.warning(("Could not find root path for remapping \"{}\". " + "This may cause issues.").format(source)) + + family = instance.data["family"] + # Make sure "render" is in the families to go through + # validating expected and rendered files + # during publishing job. + families = ["render", family] + + instance_skeleton_data = { + "family": family, + "subset": data["subset"], + "families": families, + "asset": data["asset"], + "frameStart": time_data.start, + "frameEnd": time_data.end, + "handleStart": time_data.handle_start, + "handleEnd": time_data.handle_end, + "frameStartHandle": time_data.start - time_data.handle_start, + "frameEndHandle": time_data.end + time_data.handle_end, + "comment": data.get("comment"), + "fps": time_data.fps, + "source": source, + "extendFrames": data.get("extendFrames"), + "overrideExistingFrame": data.get("overrideExistingFrame"), + "jobBatchName": data.get("jobBatchName", ""), + # map inputVersions `ObjectId` -> `str` so json supports it + "inputVersions": list(map(str, data.get("inputVersions", []))), + } + + # skip locking version if we are creating v01 + instance_version = data.get("version") # take this if exists + if instance_version != 1: + instance_skeleton_data["version"] = instance_version + + representations = get_transferable_representations(instance) + instance_skeleton_data["representations"] = representations + + persistent = instance.data.get("stagingDir_persistent") is True + instance_skeleton_data["stagingDir_persistent"] = persistent + + return instance_skeleton_data + + +def prepare_cache_representations(skeleton_data, exp_files, anatomy): + """Create representations for file sequences. + + This will return representations of expected files if they are not + in hierarchy of aovs. There should be only one sequence of files for + most cases, but if not - we create representation from each of them. + + Arguments: + skeleton_data (dict): instance data for which we are + setting representations + exp_files (list): list of expected files + anatomy (Anatomy) + Returns: + list of representations + + """ + representations = [] + collections, remainders = clique.assemble(exp_files) + + log = Logger.get_logger("farm_publishing") + + # create representation for every collected sequence + for collection in collections: + ext = collection.tail.lstrip(".") + + staging = os.path.dirname(list(collection)[0]) + success, rootless_staging_dir = ( + anatomy.find_root_template_from_path(staging) + ) + if success: + staging = rootless_staging_dir + else: + log.warning(( + "Could not find root path for remapping \"{}\"." + " This may cause issues on farm." + ).format(staging)) + + frame_start = int(skeleton_data.get("frameStartHandle")) + rep = { + "name": ext, + "ext": ext, + "files": [os.path.basename(f) for f in list(collection)], + "frameStart": frame_start, + "frameEnd": int(skeleton_data.get("frameEndHandle")), + # If expectedFile are absolute, we need only filenames + "stagingDir": staging, + "fps": skeleton_data.get("fps") + } + + representations.append(rep) + + return representations + + +def create_instances_for_cache(instance, skeleton): + """Create instance for cache. + + This will create new instance for every AOV it can detect in expected + files list. + + Args: + instance (pyblish.api.Instance): Original instance. + skeleton (dict): Skeleton data for instance (those needed) later + by collector. + + + Returns: + list of instances + + Throws: + ValueError: + + """ + anatomy = instance.context.data["anatomy"] + subset = skeleton["subset"] + family = skeleton["family"] + exp_files = instance.data["expectedFiles"] + log = Logger.get_logger("farm_publishing") + + instances = [] + # go through AOVs in expected files + for _, files in exp_files[0].items(): + cols, rem = clique.assemble(files) + # we shouldn't have any reminders. And if we do, it should + # be just one item for single frame renders. + if not cols and rem: + if len(rem) != 1: + raise ValueError("Found multiple non related files " + "to render, don't know what to do " + "with them.") + col = rem[0] + ext = os.path.splitext(col)[1].lstrip(".") + else: + # but we really expect only one collection. + # Nothing else make sense. + if len(cols) != 1: + raise ValueError("Only one image sequence type is expected.") # noqa: E501 + ext = cols[0].tail.lstrip(".") + col = list(cols[0]) + + if isinstance(col, (list, tuple)): + staging = os.path.dirname(col[0]) + else: + staging = os.path.dirname(col) + + try: + staging = remap_source(staging, anatomy) + except ValueError as e: + log.warning(e) + + new_instance = deepcopy(skeleton) + + new_instance["subset"] = subset + log.info("Creating data for: {}".format(subset)) + new_instance["family"] = family + new_instance["families"] = skeleton["families"] + # create representation + if isinstance(col, (list, tuple)): + files = [os.path.basename(f) for f in col] + else: + files = os.path.basename(col) + + rep = { + "name": ext, + "ext": ext, + "files": files, + "frameStart": int(skeleton["frameStartHandle"]), + "frameEnd": int(skeleton["frameEndHandle"]), + # If expectedFile are absolute, we need only filenames + "stagingDir": staging, + "fps": new_instance.get("fps"), + "tags": [], + } + + new_instance["representations"] = [rep] + + # if extending frames from existing version, copy files from there + # into our destination directory + if new_instance.get("extendFrames", False): + copy_extend_frames(new_instance, rep) + instances.append(new_instance) + log.debug("instances:{}".format(instances)) + return instances + + def copy_extend_frames(instance, representation): """Copy existing frames from latest version. diff --git a/openpype/pipeline/legacy_io.py b/openpype/pipeline/legacy_io.py index 60fa035c22..864102dff9 100644 --- a/openpype/pipeline/legacy_io.py +++ b/openpype/pipeline/legacy_io.py @@ -30,7 +30,7 @@ def install(): session = session_data_from_environment(context_keys=True) - session["schema"] = "openpype:session-3.0" + session["schema"] = "openpype:session-4.0" try: schema.validate(session) except schema.ValidationError as e: diff --git a/openpype/pipeline/mongodb.py b/openpype/pipeline/mongodb.py index 41a44c7373..c948983c3d 100644 --- a/openpype/pipeline/mongodb.py +++ b/openpype/pipeline/mongodb.py @@ -62,8 +62,6 @@ def auto_reconnect(func): SESSION_CONTEXT_KEYS = ( - # Root directory of projects on disk - "AVALON_PROJECTS", # Name of current Project "AVALON_PROJECT", # Name of current Asset diff --git a/openpype/pipeline/schema/session-4.0.json b/openpype/pipeline/schema/session-4.0.json new file mode 100644 index 0000000000..0dab48aa46 --- /dev/null +++ b/openpype/pipeline/schema/session-4.0.json @@ -0,0 +1,61 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + + "title": "openpype:session-4.0", + "description": "The Avalon environment", + + "type": "object", + + "additionalProperties": true, + + "required": [ + "AVALON_PROJECT" + ], + + "properties": { + "AVALON_PROJECT": { + "description": "Name of project", + "type": "string", + "pattern": "^\\w*$", + "example": "Hulk" + }, + "AVALON_ASSET": { + "description": "Name of asset", + "type": "string", + "pattern": "^[\\/\\w]*$", + "example": "Bruce" + }, + "AVALON_TASK": { + "description": "Name of task", + "type": "string", + "pattern": "^\\w*$", + "example": "modeling" + }, + "AVALON_APP": { + "description": "Name of host", + "type": "string", + "pattern": "^\\w*$", + "example": "maya" + }, + "AVALON_DB": { + "description": "Name of database", + "type": "string", + "pattern": "^\\w*$", + "example": "avalon", + "default": "avalon" + }, + "AVALON_LABEL": { + "description": "Nice name of Avalon, used in e.g. graphical user interfaces", + "type": "string", + "example": "MyLabel", + "default": "Avalon" + }, + "AVALON_TIMEOUT": { + "description": "Wherever there is a need for a timeout, this is the default value.", + "type": "string", + "pattern": "^[0-9]*$", + "default": "1000", + "example": "1000" + } + } +} diff --git a/openpype/plugins/publish/collect_anatomy_instance_data.py b/openpype/plugins/publish/collect_anatomy_instance_data.py index b4f4d6a16a..1b4b44e40e 100644 --- a/openpype/plugins/publish/collect_anatomy_instance_data.py +++ b/openpype/plugins/publish/collect_anatomy_instance_data.py @@ -30,7 +30,8 @@ import pyblish.api from openpype.client import ( get_assets, get_subsets, - get_last_versions + get_last_versions, + get_asset_name_identifier, ) from openpype.pipeline.version_start import get_versioning_start @@ -60,6 +61,9 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin): self.log.debug("Querying asset documents for instances.") context_asset_doc = context.data.get("assetEntity") + context_asset_name = None + if context_asset_doc: + context_asset_name = get_asset_name_identifier(context_asset_doc) instances_with_missing_asset_doc = collections.defaultdict(list) for instance in context: @@ -68,15 +72,15 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin): # There is possibility that assetEntity on instance is already set # which can happen in standalone publisher - if ( - instance_asset_doc - and instance_asset_doc["name"] == _asset_name - ): - continue + if instance_asset_doc: + instance_asset_name = get_asset_name_identifier( + instance_asset_doc) + if instance_asset_name == _asset_name: + continue # Check if asset name is the same as what is in context # - they may be different, e.g. in NukeStudio - if context_asset_doc and context_asset_doc["name"] == _asset_name: + if context_asset_name and context_asset_name == _asset_name: instance.data["assetEntity"] = context_asset_doc else: @@ -93,7 +97,7 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin): asset_docs = get_assets(project_name, asset_names=asset_names) asset_docs_by_name = { - asset_doc["name"]: asset_doc + get_asset_name_identifier(asset_doc): asset_doc for asset_doc in asset_docs } @@ -183,35 +187,29 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin): self.log.debug("Storing anatomy data to instance data.") project_doc = context.data["projectEntity"] - context_asset_doc = context.data.get("assetEntity") - project_task_types = project_doc["config"]["tasks"] for instance in context: + asset_doc = instance.data.get("assetEntity") anatomy_updates = { - "asset": instance.data["asset"], - "folder": { - "name": instance.data["asset"], - }, "family": instance.data["family"], "subset": instance.data["subset"], } - - # Hierarchy - asset_doc = instance.data.get("assetEntity") - if ( - asset_doc - and ( - not context_asset_doc - or asset_doc["_id"] != context_asset_doc["_id"] - ) - ): + if asset_doc: parents = asset_doc["data"].get("parents") or list() parent_name = project_doc["name"] if parents: parent_name = parents[-1] - anatomy_updates["hierarchy"] = "/".join(parents) - anatomy_updates["parent"] = parent_name + + hierarchy = "/".join(parents) + anatomy_updates.update({ + "asset": asset_doc["name"], + "hierarchy": hierarchy, + "parent": parent_name, + "folder": { + "name": asset_doc["name"], + }, + }) # Task task_type = None diff --git a/openpype/plugins/publish/collect_audio.py b/openpype/plugins/publish/collect_audio.py index 6aaadfc568..734a625852 100644 --- a/openpype/plugins/publish/collect_audio.py +++ b/openpype/plugins/publish/collect_audio.py @@ -6,6 +6,7 @@ from openpype.client import ( get_subsets, get_last_versions, get_representations, + get_asset_name_identifier, ) from openpype.pipeline.load import get_representation_path_with_anatomy @@ -121,12 +122,13 @@ class CollectAudio(pyblish.api.ContextPlugin): asset_docs = get_assets( project_name, asset_names=asset_names, - fields=["_id", "name"] + fields=["_id", "name", "data.parents"] ) - asset_id_by_name = {} - for asset_doc in asset_docs: - asset_id_by_name[asset_doc["name"]] = asset_doc["_id"] + asset_id_by_name = { + get_asset_name_identifier(asset_doc): asset_doc["_id"] + for asset_doc in asset_docs + } asset_ids = set(asset_id_by_name.values()) # Query subsets with name define by 'audio_subset_name' attr diff --git a/openpype/plugins/publish/collect_from_create_context.py b/openpype/plugins/publish/collect_from_create_context.py index 8806a13ca0..84f6141069 100644 --- a/openpype/plugins/publish/collect_from_create_context.py +++ b/openpype/plugins/publish/collect_from_create_context.py @@ -4,6 +4,7 @@ import os import pyblish.api +from openpype import AYON_SERVER_ENABLED from openpype.host import IPublishHost from openpype.pipeline import legacy_io, registered_host from openpype.pipeline.create import CreateContext @@ -38,6 +39,8 @@ class CollectFromCreateContext(pyblish.api.ContextPlugin): for created_instance in create_context.instances: instance_data = created_instance.data_to_store() + if AYON_SERVER_ENABLED: + instance_data["asset"] = instance_data.pop("folderPath") if instance_data["active"]: thumbnail_path = thumbnail_paths_by_instance_id.get( created_instance.id diff --git a/openpype/plugins/publish/collect_resources_path.py b/openpype/plugins/publish/collect_resources_path.py index 14c13310df..a7f12bdfdb 100644 --- a/openpype/plugins/publish/collect_resources_path.py +++ b/openpype/plugins/publish/collect_resources_path.py @@ -69,9 +69,9 @@ class CollectResourcesPath(pyblish.api.InstancePlugin): def process(self, instance): # editorial would fail since they might not be in database yet - is_editorial = instance.data.get("isEditorial") - if is_editorial: - self.log.debug("Instance is Editorial. Skipping.") + new_asset_publishing = instance.data.get("newAssetPublishing") + if new_asset_publishing: + self.log.debug("Instance is creating new asset. Skipping.") return anatomy = instance.context.data["anatomy"] diff --git a/openpype/plugins/publish/extract_hierarchy_to_ayon.py b/openpype/plugins/publish/extract_hierarchy_to_ayon.py index 0d9131718b..ef69369d67 100644 --- a/openpype/plugins/publish/extract_hierarchy_to_ayon.py +++ b/openpype/plugins/publish/extract_hierarchy_to_ayon.py @@ -8,7 +8,7 @@ from ayon_api import slugify_string from ayon_api.entity_hub import EntityHub from openpype import AYON_SERVER_ENABLED -from openpype.client import get_assets +from openpype.client import get_assets, get_asset_name_identifier from openpype.pipeline.template_data import ( get_asset_template_data, get_task_template_data, @@ -58,7 +58,7 @@ class ExtractHierarchyToAYON(pyblish.api.ContextPlugin): project_name, asset_names=instances_by_asset_name.keys() ) asset_docs_by_name = { - asset_doc["name"]: asset_doc + get_asset_name_identifier(asset_doc): asset_doc for asset_doc in asset_docs } for asset_name, instances in instances_by_asset_name.items(): @@ -191,15 +191,15 @@ class ExtractHierarchyToAYON(pyblish.api.ContextPlugin): """ # filter only the active publishing instances - active_folder_names = set() + active_folder_paths = set() for instance in context: if instance.data.get("publish") is not False: - active_folder_names.add(instance.data.get("asset")) + active_folder_paths.add(instance.data.get("asset")) - active_folder_names.discard(None) + active_folder_paths.discard(None) - self.log.debug("Active folder names: {}".format(active_folder_names)) - if not active_folder_names: + self.log.debug("Active folder paths: {}".format(active_folder_paths)) + if not active_folder_paths: return None project_item = None @@ -230,12 +230,13 @@ class ExtractHierarchyToAYON(pyblish.api.ContextPlugin): if not children_context: continue - for asset_name, asset_info in children_context.items(): + for asset, asset_info in children_context.items(): if ( - asset_name not in active_folder_names + asset not in active_folder_paths and not asset_info.get("childs") ): continue + asset_name = asset.split("/")[-1] item_id = uuid.uuid4().hex new_item = copy.deepcopy(asset_info) new_item["name"] = asset_name @@ -252,7 +253,7 @@ class ExtractHierarchyToAYON(pyblish.api.ContextPlugin): items_by_id[item_id] = new_item parent_id_by_item_id[item_id] = parent_id - if asset_name in active_folder_names: + if asset in active_folder_paths: valid_ids.add(item_id) hierarchy_queue.append((item_id, new_children_context)) diff --git a/openpype/plugins/publish/validate_editorial_asset_name.py b/openpype/plugins/publish/validate_editorial_asset_name.py index fca0d8e7f5..b5afc49f2e 100644 --- a/openpype/plugins/publish/validate_editorial_asset_name.py +++ b/openpype/plugins/publish/validate_editorial_asset_name.py @@ -2,7 +2,7 @@ from pprint import pformat import pyblish.api -from openpype.client import get_assets +from openpype.client import get_assets, get_asset_name_identifier class ValidateEditorialAssetName(pyblish.api.ContextPlugin): @@ -34,8 +34,11 @@ class ValidateEditorialAssetName(pyblish.api.ContextPlugin): self.log.debug("__ db_assets: {}".format(db_assets)) asset_db_docs = { - str(e["name"]): [str(p) for p in e["data"]["parents"]] - for e in db_assets} + get_asset_name_identifier(asset_doc): list( + asset_doc["data"]["parents"] + ) + for asset_doc in db_assets + } self.log.debug("__ project_entities: {}".format( pformat(asset_db_docs))) diff --git a/openpype/settings/ayon_settings.py b/openpype/settings/ayon_settings.py index 5171517232..6676e71a8e 100644 --- a/openpype/settings/ayon_settings.py +++ b/openpype/settings/ayon_settings.py @@ -1591,3 +1591,18 @@ def get_ayon_system_settings(default_values): return convert_system_settings( ayon_settings, default_values, addon_versions ) + + +def get_ayon_settings(project_name=None): + """AYON studio settings. + + Raw AYON settings values. + + Args: + project_name (Optional[str]): Project name. + + Returns: + dict[str, Any]: AYON settings. + """ + + return _AyonSettingsCache.get_value_by_project(project_name) diff --git a/openpype/settings/defaults/project_settings/deadline.json b/openpype/settings/defaults/project_settings/deadline.json index 2c5e0dc65d..a1df8ccba9 100644 --- a/openpype/settings/defaults/project_settings/deadline.json +++ b/openpype/settings/defaults/project_settings/deadline.json @@ -109,6 +109,14 @@ "chunk_size": 10, "group": "none" }, + "ProcessSubmittedCacheJobOnFarm": { + "enabled": true, + "deadline_department": "", + "deadline_pool": "", + "deadline_group": "", + "deadline_chunk_size": 1, + "deadline_priority": 50 + }, "ProcessSubmittedJobOnFarm": { "enabled": true, "deadline_department": "", diff --git a/openpype/settings/defaults/project_settings/houdini.json b/openpype/settings/defaults/project_settings/houdini.json index 0dd8443e44..0001b23967 100644 --- a/openpype/settings/defaults/project_settings/houdini.json +++ b/openpype/settings/defaults/project_settings/houdini.json @@ -137,6 +137,11 @@ } }, "publish": { + "CollectChunkSize": { + "enabled": true, + "optional": true, + "chunk_size": 999999 + }, "CollectAssetHandles": { "use_asset_handles": true }, diff --git a/openpype/settings/defaults/system_settings/modules.json b/openpype/settings/defaults/system_settings/modules.json index f524f01d45..bb943524f1 100644 --- a/openpype/settings/defaults/system_settings/modules.json +++ b/openpype/settings/defaults/system_settings/modules.json @@ -210,5 +210,8 @@ "darwin": "", "linux": "" } + }, + "asset_reporter": { + "enabled": false } } diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json b/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json index 64db852c89..e622dc43a2 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json @@ -584,6 +584,46 @@ } ] }, + { + "type": "dict", + "collapsible": true, + "key": "ProcessSubmittedCacheJobOnFarm", + "label": "ProcessSubmittedCacheJobOnFarm", + "checkbox_key": "enabled", + "is_group": true, + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + }, + { + "type": "text", + "key": "deadline_department", + "label": "Deadline department" + }, + { + "type": "text", + "key": "deadline_pool", + "label": "Deadline Pool" + }, + { + "type": "text", + "key": "deadline_group", + "label": "Deadline Group" + }, + { + "type": "number", + "key": "deadline_chunk_size", + "label": "Deadline Chunk Size" + }, + { + "type": "number", + "key": "deadline_priority", + "label": "Deadline Priotity" + } + ] + }, { "type": "dict", "collapsible": true, diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_houdini_publish.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_houdini_publish.json index 324cfd8d58..935c2a4c35 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_houdini_publish.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_houdini_publish.json @@ -55,6 +55,31 @@ } ] }, + { + "type": "dict", + "collapsible": true, + "checkbox_key": "enabled", + "key": "CollectChunkSize", + "label": "Collect Chunk Size", + "is_group": true, + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + }, + { + "type": "boolean", + "key": "optional", + "label": "Optional" + }, + { + "type": "number", + "key": "chunk_size", + "label": "Frames Per Task" + } + ] + }, { "type": "dict", "collapsible": true, diff --git a/openpype/settings/entities/schemas/system_schema/schema_modules.json b/openpype/settings/entities/schemas/system_schema/schema_modules.json index 952b38040c..5b189eae88 100644 --- a/openpype/settings/entities/schemas/system_schema/schema_modules.json +++ b/openpype/settings/entities/schemas/system_schema/schema_modules.json @@ -355,6 +355,20 @@ { "type": "dynamic_schema", "name": "system_settings/modules" + }, + { + "type": "dict", + "key": "asset_reporter", + "label": "Asset Usage Reporter", + "collapsible": true, + "checkbox_key": "enabled", + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + } + ] } ] } diff --git a/openpype/tools/ayon_launcher/models/actions.py b/openpype/tools/ayon_launcher/models/actions.py index 93ec115734..d7c4219dc2 100644 --- a/openpype/tools/ayon_launcher/models/actions.py +++ b/openpype/tools/ayon_launcher/models/actions.py @@ -402,12 +402,12 @@ class ActionsModel: ) def _prepare_session(self, project_name, folder_id, task_id): - folder_name = None + folder_path = None if folder_id: folder = self._controller.get_folder_entity( project_name, folder_id) if folder: - folder_name = folder["name"] + folder_path = folder["path"] task_name = None if task_id: @@ -417,7 +417,7 @@ class ActionsModel: return { "AVALON_PROJECT": project_name, - "AVALON_ASSET": folder_name, + "AVALON_ASSET": folder_path, "AVALON_TASK": task_name, } diff --git a/openpype/tools/ayon_loader/control.py b/openpype/tools/ayon_loader/control.py index c38973d0b3..8ec0d96e2e 100644 --- a/openpype/tools/ayon_loader/control.py +++ b/openpype/tools/ayon_loader/control.py @@ -290,7 +290,7 @@ class LoaderController(BackendLoaderController, FrontendLoaderController): project_name = context.get("project_name") asset_name = context.get("asset_name") if project_name and asset_name: - folder = ayon_api.get_folder_by_name( + folder = ayon_api.get_folder_by_path( project_name, asset_name, fields=["id"] ) if folder: diff --git a/openpype/tools/ayon_sceneinventory/control.py b/openpype/tools/ayon_sceneinventory/control.py index e98b0e307b..6111d7e43b 100644 --- a/openpype/tools/ayon_sceneinventory/control.py +++ b/openpype/tools/ayon_sceneinventory/control.py @@ -70,19 +70,12 @@ class SceneInventoryController: context = self.get_current_context() project_name = context["project_name"] - folder_path = context.get("folder_path") folder_name = context.get("asset_name") folder_id = None - if folder_path: - folder = ayon_api.get_folder_by_path(project_name, folder_path) + if folder_name: + folder = ayon_api.get_folder_by_path(project_name, folder_name) if folder: folder_id = folder["id"] - elif folder_name: - for folder in ayon_api.get_folders( - project_name, folder_names=[folder_name] - ): - folder_id = folder["id"] - break self._current_folder_id = folder_id self._current_folder_set = True diff --git a/openpype/tools/ayon_workfiles/control.py b/openpype/tools/ayon_workfiles/control.py index fbe6df3155..9d19571267 100644 --- a/openpype/tools/ayon_workfiles/control.py +++ b/openpype/tools/ayon_workfiles/control.py @@ -473,7 +473,7 @@ class BaseWorkfileController( current_file = self.get_current_workfile() folder_id = None if folder_name: - folder = ayon_api.get_folder_by_name(project_name, folder_name) + folder = ayon_api.get_folder_by_path(project_name, folder_name) if folder: folder_id = folder["id"] diff --git a/openpype/tools/creator/window.py b/openpype/tools/creator/window.py index 47f27a262a..117519e1d7 100644 --- a/openpype/tools/creator/window.py +++ b/openpype/tools/creator/window.py @@ -214,7 +214,7 @@ class CreatorWindow(QtWidgets.QDialog): asset_name = self._asset_name_input.text() # Early exit if no asset name - if not asset_name.strip(): + if not asset_name: self._build_menu() self.echo("Asset name is required ..") self._set_valid_state(False) diff --git a/openpype/tools/publisher/control.py b/openpype/tools/publisher/control.py index 3192fe949f..9e00d21750 100644 --- a/openpype/tools/publisher/control.py +++ b/openpype/tools/publisher/control.py @@ -12,10 +12,12 @@ from abc import ABCMeta, abstractmethod import six import pyblish.api +from openpype import AYON_SERVER_ENABLED from openpype.client import ( get_assets, get_asset_by_id, get_subsets, + get_asset_name_identifier, ) from openpype.lib.events import EventSystem from openpype.lib.attribute_definitions import ( @@ -73,6 +75,8 @@ class AssetDocsCache: "data.visualParent": True, "data.tasks": True } + if AYON_SERVER_ENABLED: + projection["data.parents"] = True def __init__(self, controller): self._controller = controller @@ -105,7 +109,7 @@ class AssetDocsCache: elif "tasks" not in asset_doc["data"]: asset_doc["data"]["tasks"] = {} - asset_name = asset_doc["name"] + asset_name = get_asset_name_identifier(asset_doc) asset_tasks = asset_doc["data"]["tasks"] task_names_by_asset_name[asset_name] = list(asset_tasks.keys()) asset_docs_by_name[asset_name] = asset_doc diff --git a/openpype/tools/publisher/widgets/assets_widget.py b/openpype/tools/publisher/widgets/assets_widget.py index c536f93c9b..32be514dd7 100644 --- a/openpype/tools/publisher/widgets/assets_widget.py +++ b/openpype/tools/publisher/widgets/assets_widget.py @@ -11,7 +11,8 @@ from openpype.tools.utils import ( from openpype.tools.utils.assets_widget import ( SingleSelectAssetsWidget, ASSET_ID_ROLE, - ASSET_NAME_ROLE + ASSET_NAME_ROLE, + ASSET_PATH_ROLE, ) @@ -31,6 +32,15 @@ class CreateWidgetAssetsWidget(SingleSelectAssetsWidget): self._last_filter_height = None + def get_selected_asset_name(self): + if AYON_SERVER_ENABLED: + selection_model = self._view.selectionModel() + indexes = selection_model.selectedRows() + for index in indexes: + return index.data(ASSET_PATH_ROLE) + return None + return super(CreateWidgetAssetsWidget, self).get_selected_asset_name() + def _check_header_height(self): """Catch header height changes. @@ -100,21 +110,24 @@ class AssetsHierarchyModel(QtGui.QStandardItemModel): self._controller = controller self._items_by_name = {} + self._items_by_path = {} self._items_by_asset_id = {} def reset(self): self.clear() self._items_by_name = {} + self._items_by_path = {} self._items_by_asset_id = {} assets_by_parent_id = self._controller.get_asset_hierarchy() items_by_name = {} + items_by_path = {} items_by_asset_id = {} _queue = collections.deque() - _queue.append((self.invisibleRootItem(), None)) + _queue.append((self.invisibleRootItem(), None, None)) while _queue: - parent_item, parent_id = _queue.popleft() + parent_item, parent_id, parent_path = _queue.popleft() children = assets_by_parent_id.get(parent_id) if not children: continue @@ -127,6 +140,11 @@ class AssetsHierarchyModel(QtGui.QStandardItemModel): for name in sorted(children_by_name.keys()): child = children_by_name[name] child_id = child["_id"] + if parent_path: + child_path = "{}/{}".format(parent_path, name) + else: + child_path = "/{}".format(name) + has_children = bool(assets_by_parent_id.get(child_id)) icon = get_asset_icon(child, has_children) @@ -138,15 +156,18 @@ class AssetsHierarchyModel(QtGui.QStandardItemModel): item.setData(icon, QtCore.Qt.DecorationRole) item.setData(child_id, ASSET_ID_ROLE) item.setData(name, ASSET_NAME_ROLE) + item.setData(child_path, ASSET_PATH_ROLE) items_by_name[name] = item + items_by_path[child_path] = item items_by_asset_id[child_id] = item items.append(item) - _queue.append((item, child_id)) + _queue.append((item, child_id, child_path)) parent_item.appendRows(items) self._items_by_name = items_by_name + self._items_by_path = items_by_path self._items_by_asset_id = items_by_asset_id def get_index_by_asset_id(self, asset_id): @@ -156,12 +177,20 @@ class AssetsHierarchyModel(QtGui.QStandardItemModel): return QtCore.QModelIndex() def get_index_by_asset_name(self, asset_name): - item = self._items_by_name.get(asset_name) + item = None + if AYON_SERVER_ENABLED: + item = self._items_by_path.get(asset_name) + + if item is None: + item = self._items_by_name.get(asset_name) + if item is None: return QtCore.QModelIndex() return item.index() def name_is_valid(self, item_name): + if AYON_SERVER_ENABLED and item_name in self._items_by_path: + return True return item_name in self._items_by_name @@ -296,7 +325,10 @@ class AssetsDialog(QtWidgets.QDialog): index = self._asset_view.currentIndex() asset_name = None if index.isValid(): - asset_name = index.data(ASSET_NAME_ROLE) + if AYON_SERVER_ENABLED: + asset_name = index.data(ASSET_PATH_ROLE) + else: + asset_name = index.data(ASSET_NAME_ROLE) self._selected_asset = asset_name self.done(1) diff --git a/openpype/tools/publisher/widgets/create_widget.py b/openpype/tools/publisher/widgets/create_widget.py index 64fed1d70c..73dcae51a5 100644 --- a/openpype/tools/publisher/widgets/create_widget.py +++ b/openpype/tools/publisher/widgets/create_widget.py @@ -816,8 +816,13 @@ class CreateWidget(QtWidgets.QWidget): # Where to define these data? # - what data show be stored? + if AYON_SERVER_ENABLED: + asset_key = "folderPath" + else: + asset_key = "asset" + instance_data = { - "asset": asset_name, + asset_key: asset_name, "task": task_name, "variant": variant, "family": family diff --git a/openpype/tools/publisher/widgets/widgets.py b/openpype/tools/publisher/widgets/widgets.py index 1860287fcf..ecccc4e0c8 100644 --- a/openpype/tools/publisher/widgets/widgets.py +++ b/openpype/tools/publisher/widgets/widgets.py @@ -540,6 +540,7 @@ class AssetsField(BaseClickableFrame): Does not change selected items (assets). """ self._name_input.setText(text) + self._name_input.end(False) def set_selected_items(self, asset_names=None): """Set asset names for selection of instances. @@ -1187,7 +1188,10 @@ class GlobalAttrsWidget(QtWidgets.QWidget): asset_names = [] for instance in self._current_instances: new_variant_value = instance.get("variant") - new_asset_name = instance.get("asset") + if AYON_SERVER_ENABLED: + new_asset_name = instance.get("folderPath") + else: + new_asset_name = instance.get("asset") new_task_name = instance.get("task") if variant_value is not None: new_variant_value = variant_value @@ -1219,7 +1223,11 @@ class GlobalAttrsWidget(QtWidgets.QWidget): instance["variant"] = variant_value if asset_name is not None: - instance["asset"] = asset_name + if AYON_SERVER_ENABLED: + instance["folderPath"] = asset_name + else: + instance["asset"] = asset_name + instance.set_asset_invalid(False) if task_name is not None: @@ -1317,7 +1325,10 @@ class GlobalAttrsWidget(QtWidgets.QWidget): variants.add(instance.get("variant") or self.unknown_value) families.add(instance.get("family") or self.unknown_value) - asset_name = instance.get("asset") or self.unknown_value + if AYON_SERVER_ENABLED: + asset_name = instance.get("folderPath") or self.unknown_value + else: + asset_name = instance.get("asset") or self.unknown_value task_name = instance.get("task") or "" asset_names.add(asset_name) asset_task_combinations.append((asset_name, task_name)) diff --git a/openpype/tools/utils/assets_widget.py b/openpype/tools/utils/assets_widget.py index a45d762c73..b83f4dfcaf 100644 --- a/openpype/tools/utils/assets_widget.py +++ b/openpype/tools/utils/assets_widget.py @@ -36,6 +36,7 @@ ASSET_ID_ROLE = QtCore.Qt.UserRole + 1 ASSET_NAME_ROLE = QtCore.Qt.UserRole + 2 ASSET_LABEL_ROLE = QtCore.Qt.UserRole + 3 ASSET_UNDERLINE_COLORS_ROLE = QtCore.Qt.UserRole + 4 +ASSET_PATH_ROLE = QtCore.Qt.UserRole + 5 class AssetsView(TreeViewSpinner, DeselectableTreeView): diff --git a/openpype/version.py b/openpype/version.py index 9d8724f926..a54f88871a 100644 --- a/openpype/version.py +++ b/openpype/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring Pype version.""" -__version__ = "3.17.7-nightly.1" +__version__ = "3.17.7-nightly.3" diff --git a/server_addon/create_ayon_addons.py b/server_addon/create_ayon_addons.py index fe8d278321..fc7a673dcc 100644 --- a/server_addon/create_ayon_addons.py +++ b/server_addon/create_ayon_addons.py @@ -33,6 +33,20 @@ IGNORE_FILE_PATTERNS: List[Pattern] = [ } ] +IGNORED_HOSTS = [ + "flame", + "harmony", +] + +IGNORED_MODULES = [ + "ftrack", + "shotgrid", + "sync_server", + "example_addons", + "slack", + "kitsu", +] + class ZipFileLongPaths(zipfile.ZipFile): """Allows longer paths in zip files. @@ -202,16 +216,6 @@ def create_openpype_package( str(pyproject_path), (private_dir / pyproject_path.name) ) - - ignored_hosts = [] - ignored_modules = [ - "ftrack", - "shotgrid", - "sync_server", - "example_addons", - "slack", - "kitsu", - ] # Subdirs that won't be added to output zip file ignored_subpaths = [ ["addons"], @@ -219,11 +223,11 @@ def create_openpype_package( ] ignored_subpaths.extend( ["hosts", host_name] - for host_name in ignored_hosts + for host_name in IGNORED_HOSTS ) ignored_subpaths.extend( ["modules", module_name] - for module_name in ignored_modules + for module_name in IGNORED_MODULES ) # Zip client @@ -297,6 +301,7 @@ def main( # Make sure output dir is created output_dir.mkdir(parents=True, exist_ok=True) + ignored_addons = set(IGNORED_HOSTS) | set(IGNORED_MODULES) for addon_dir in current_dir.iterdir(): if not addon_dir.is_dir(): continue @@ -304,6 +309,9 @@ def main( if addons and addon_dir.name not in addons: continue + if addon_dir.name in ignored_addons: + continue + server_dir = addon_dir / "server" if not server_dir.exists(): continue diff --git a/server_addon/deadline/server/settings/publish_plugins.py b/server_addon/deadline/server/settings/publish_plugins.py index 54b7ff57c1..7915308d2f 100644 --- a/server_addon/deadline/server/settings/publish_plugins.py +++ b/server_addon/deadline/server/settings/publish_plugins.py @@ -248,6 +248,17 @@ class AOVFilterSubmodel(BaseSettingsModel): ) +class ProcessCacheJobFarmModel(BaseSettingsModel): + """Process submitted job on farm.""" + + enabled: bool = Field(title="Enabled") + deadline_department: str = Field(title="Department") + deadline_pool: str = Field(title="Pool") + deadline_group: str = Field(title="Group") + deadline_chunk_size: int = Field(title="Chunk Size") + deadline_priority: int = Field(title="Priority") + + class ProcessSubmittedJobOnFarmModel(BaseSettingsModel): """Process submitted job on farm.""" @@ -311,6 +322,9 @@ class PublishPluginsModel(BaseSettingsModel): BlenderSubmitDeadline: BlenderSubmitDeadlineModel = Field( default_factory=BlenderSubmitDeadlineModel, title="Blender Submit Deadline") + ProcessSubmittedCacheJobOnFarm: ProcessCacheJobFarmModel = Field( + default_factory=ProcessCacheJobFarmModel, + title="Process submitted cache Job on farm.") ProcessSubmittedJobOnFarm: ProcessSubmittedJobOnFarmModel = Field( default_factory=ProcessSubmittedJobOnFarmModel, title="Process submitted job on farm.") @@ -426,6 +440,14 @@ DEFAULT_DEADLINE_PLUGINS_SETTINGS = { "chunk_size": 10, "group": "none" }, + "ProcessSubmittedCacheJobOnFarm": { + "enabled": True, + "deadline_department": "", + "deadline_pool": "", + "deadline_group": "", + "deadline_chunk_size": 1, + "deadline_priority": 50 + }, "ProcessSubmittedJobOnFarm": { "enabled": True, "deadline_department": "", diff --git a/server_addon/deadline/server/version.py b/server_addon/deadline/server/version.py index ae7362549b..bbab0242f6 100644 --- a/server_addon/deadline/server/version.py +++ b/server_addon/deadline/server/version.py @@ -1 +1 @@ -__version__ = "0.1.3" +__version__ = "0.1.4" diff --git a/tests/conftest.py b/tests/conftest.py index a862030fff..028c19bc5f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -85,3 +85,11 @@ def pytest_runtest_makereport(item, call): # be "setup", "call", "teardown" setattr(item, "rep_" + rep.when, rep) + + # In the event of module scoped fixtures, also mark failure in module. + module = item + while module is not None and not isinstance(module, pytest.Module): + module = module.parent + if module is not None: + if rep.when == 'call' and (rep.failed or rep.skipped): + module.module_test_failure = True diff --git a/tests/lib/testing_classes.py b/tests/lib/testing_classes.py index 7dae9a762f..7700381aa6 100644 --- a/tests/lib/testing_classes.py +++ b/tests/lib/testing_classes.py @@ -218,11 +218,7 @@ class ModuleUnitTest(BaseTest): yield mongo_client[self.TEST_OPENPYPE_NAME]["settings"] def is_test_failed(self, request): - # if request.node doesn't have rep_call, something failed - try: - return request.node.rep_call.failed - except AttributeError: - return True + return getattr(request.node, "module_test_failure", False) class PublishTest(ModuleUnitTest): diff --git a/website/docs/artist_hosts_houdini.md b/website/docs/artist_hosts_houdini.md index 940d5ac351..c4e8293c36 100644 --- a/website/docs/artist_hosts_houdini.md +++ b/website/docs/artist_hosts_houdini.md @@ -83,6 +83,30 @@ select your render camera. All the render outputs are stored in the pyblish/render directory within your project path.\ For Karma-specific render, it also outputs the USD render as default. +## Publishing cache to Deadline +Artist can publish cache to deadline which increases productivity as artist can use local machine +could be used for other tasks. +Caching on the farm is supported for: + +**Arnold ASS (.ass)** +**Pointcache (.bgeo and .abc)** +**VDB (.vdb)** +**Redshift Proxy (.rs)** + +To submit your cache to deadline, you need to create the instance(s) with clicking +**Submitting to Farm** and you can also enable **Use selection** to +select the object for caching in farm. +![Houdini Farm Cache Creator](assets/houdini_farm_cache_creator.png) + +When you go to Publish Tab and click the instance(s), you can set up your preferred +**Frame per task**. +![Houdini Farm Per Task](assets/houdini_frame_per_task.png) + +Once you hit **Publish**, the cache would be submitted and rendered in deadline. +When the render is finished, all the caches would be located in your publish folder. +You can see them in the Loader. +![Houdini Farm Per Task](assets/houdini_farm_cache_loader.png) + ## USD (experimental support) ### Publishing USD You can publish your Solaris Stage as USD file. diff --git a/website/docs/assets/houdini_farm_cache_creator.png b/website/docs/assets/houdini_farm_cache_creator.png new file mode 100644 index 0000000000..bf0af80336 Binary files /dev/null and b/website/docs/assets/houdini_farm_cache_creator.png differ diff --git a/website/docs/assets/houdini_farm_cache_loader.png b/website/docs/assets/houdini_farm_cache_loader.png new file mode 100644 index 0000000000..27b0122fab Binary files /dev/null and b/website/docs/assets/houdini_farm_cache_loader.png differ diff --git a/website/docs/assets/houdini_frame_per_task.png b/website/docs/assets/houdini_frame_per_task.png new file mode 100644 index 0000000000..a7d8b33114 Binary files /dev/null and b/website/docs/assets/houdini_frame_per_task.png differ