From 2072f2fbe2bdaa09c96dd3c849bff65785c3b3bb Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 4 Mar 2024 16:44:44 +0100 Subject: [PATCH] global plugins are using folder entity and task --- .../plugins/actions/open_file_explorer.py | 14 +- .../plugins/load/delete_old_versions.py | 36 +-- .../publish/collect_anatomy_context_data.py | 15 +- .../publish/collect_anatomy_instance_data.py | 280 ++++++++++++------ .../plugins/publish/collect_audio.py | 51 ++-- .../publish/collect_context_entities.py | 76 +++-- .../publish/collect_current_context.py | 8 +- .../plugins/publish/collect_frames_fix.py | 4 +- .../publish/collect_from_create_context.py | 4 +- .../publish/extract_hierarchy_to_ayon.py | 75 +++-- client/ayon_core/plugins/publish/integrate.py | 6 +- .../plugins/publish/integrate_thumbnail.py | 4 +- .../plugins/publish/validate_asset_docs.py | 24 +- .../publish/validate_editorial_asset_name.py | 108 ++++--- .../publish/validate_unique_subsets.py | 33 ++- 15 files changed, 457 insertions(+), 281 deletions(-) diff --git a/client/ayon_core/plugins/actions/open_file_explorer.py b/client/ayon_core/plugins/actions/open_file_explorer.py index c5c34cc4f0..c221752f11 100644 --- a/client/ayon_core/plugins/actions/open_file_explorer.py +++ b/client/ayon_core/plugins/actions/open_file_explorer.py @@ -5,7 +5,6 @@ from string import Formatter import ayon_api -from ayon_core.client import get_asset_by_name from ayon_core.pipeline import ( Anatomy, LauncherAction, @@ -27,10 +26,10 @@ class OpenTaskPath(LauncherAction): from qtpy import QtCore, QtWidgets project_name = session["AYON_PROJECT_NAME"] - asset_name = session["AYON_FOLDER_PATH"] + folder_path = session["AYON_FOLDER_PATH"] task_name = session.get("AYON_TASK_NAME", None) - path = self._get_workdir(project_name, asset_name, task_name) + path = self._get_workdir(project_name, folder_path, task_name) if not path: return @@ -61,11 +60,14 @@ class OpenTaskPath(LauncherAction): path = path.split(field, 1)[0] return path - def _get_workdir(self, project_name, asset_name, task_name): + def _get_workdir(self, project_name, folder_path, task_name): project_entity = ayon_api.get_project(project_name) - asset_doc = get_asset_by_name(project_name, asset_name) + folder_entity = ayon_api.get_folder_by_path(project_name, folder_path) + task_entity = ayon_api.get_task_by_name( + project_name, folder_entity["id"], task_name + ) - data = get_template_data(project_entity, asset_doc, task_name) + data = get_template_data(project_entity, folder_entity, task_entity) anatomy = Anatomy(project_name) workdir = anatomy.templates_obj["work"]["folder"].format(data) diff --git a/client/ayon_core/plugins/load/delete_old_versions.py b/client/ayon_core/plugins/load/delete_old_versions.py index 4fc61ebb8b..e0670caee5 100644 --- a/client/ayon_core/plugins/load/delete_old_versions.py +++ b/client/ayon_core/plugins/load/delete_old_versions.py @@ -1,5 +1,5 @@ # TODO This plugin is not converted for AYON - +# # import collections # import os # import uuid @@ -196,12 +196,14 @@ # msgBox.exec_() # # def get_data(self, context, versions_count): -# subset = context["subset"] -# asset = context["asset"] +# subset_doc = context["subset"] +# folder_entity = context["folder"] # project_name = context["project"]["name"] # anatomy = Anatomy(project_name) # -# versions = list(get_versions(project_name, subset_ids=[subset["_id"]])) +# versions = list(get_versions( +# project_name, subset_ids=[subset_doc["_id"]] +# )) # # versions_by_parent = collections.defaultdict(list) # for ent in versions: @@ -238,8 +240,10 @@ # versions_to_pop.append(version) # # for version in versions_to_pop: -# msg = "Asset: \"{}\" | Subset: \"{}\" | Version: \"{}\"".format( -# asset["name"], subset["name"], version["name"] +# msg = "Folder: \"{}\" | Subset: \"{}\" | Version: \"{}\"".format( +# folder_entity["path"], +# subset_doc["name"], +# version["name"] # ) # self.log.debug(( # "Skipping version. Already tagged as `deleted`. < {} >" @@ -254,7 +258,7 @@ # # if not version_ids: # msg = "Skipping processing. Nothing to delete on {}/{}".format( -# asset["name"], subset["name"] +# folder_entity["path"], subset_doc["name"] # ) # self.log.info(msg) # print(msg) @@ -310,17 +314,15 @@ # "Folder does not exist. Deleting it's files skipped: {}" # ).format(paths_msg)) # -# data = { +# return { # "dir_paths": dir_paths, # "file_paths_by_dir": file_paths_by_dir, # "versions": versions, -# "asset": asset, -# "subset": subset, +# "folder": folder_entity, +# "subset": subset_doc, # "archive_subset": versions_count == 0 # } # -# return data -# # def main(self, project_name, data, remove_publish_folder): # # Size of files. # size = 0 @@ -382,12 +384,12 @@ # data (dict): Data sent to subset loader with full context. # """ # -# # First check for ftrack id on asset document +# # First check for ftrack id on folder entity # # - skip if ther is none -# asset_ftrack_id = data["asset"]["data"].get("ftrackId") -# if not asset_ftrack_id: +# ftrack_id = data["folder"]["attrib"].get("ftrackId") +# if not ftrack_id: # self.log.info(( -# "Asset does not have filled ftrack id. Skipped delete" +# "Folder does not have filled ftrack id. Skipped delete" # " of ftrack version." # )) # return @@ -413,7 +415,7 @@ # " and asset.name is \"{}\"" # " and version in ({})" # ).format( -# asset_ftrack_id, +# ftrack_id, # product_name, # ",".join(versions) # ) diff --git a/client/ayon_core/plugins/publish/collect_anatomy_context_data.py b/client/ayon_core/plugins/publish/collect_anatomy_context_data.py index b5bb579498..cccf392e40 100644 --- a/client/ayon_core/plugins/publish/collect_anatomy_context_data.py +++ b/client/ayon_core/plugins/publish/collect_anatomy_context_data.py @@ -3,7 +3,8 @@ Requires: context -> anatomy context -> projectEntity - context -> assetEntity + context -> folderEntity + context -> taskEntity context -> task context -> username context -> datetimeData @@ -49,15 +50,15 @@ class CollectAnatomyContextData(pyblish.api.ContextPlugin): host_name = context.data["hostName"] project_settings = context.data["project_settings"] project_entity = context.data["projectEntity"] - asset_entity = context.data.get("assetEntity") - task_name = None - if asset_entity: - task_name = context.data["task"] + folder_entity = context.data.get("folderEntity") + task_entity = None + if folder_entity: + task_entity = context.data["taskEntity"] anatomy_data = get_template_data( project_entity, - asset_entity, - task_name, + folder_entity, + task_entity, host_name, project_settings ) diff --git a/client/ayon_core/plugins/publish/collect_anatomy_instance_data.py b/client/ayon_core/plugins/publish/collect_anatomy_instance_data.py index 8ad8a013c2..1b49d8288d 100644 --- a/client/ayon_core/plugins/publish/collect_anatomy_instance_data.py +++ b/client/ayon_core/plugins/publish/collect_anatomy_instance_data.py @@ -1,13 +1,17 @@ """ Requires: - context -> anatomyData + context -> projectName context -> projectEntity - context -> assetEntity + context -> anatomyData instance -> folderPath instance -> productName instance -> productType Optional: + context -> folderEntity + context -> taskEntity + instance -> task + instance -> taskEntity instance -> version instance -> resolutionWidth instance -> resolutionHeight @@ -15,7 +19,8 @@ Optional: Provides: instance -> projectEntity - instance -> assetEntity + instance -> folderEntity + instance -> taskEntity instance -> anatomyData instance -> version instance -> latestVersion @@ -26,12 +31,11 @@ import json import collections import pyblish.api +import ayon_api from ayon_core.client import ( - get_assets, get_subsets, get_last_versions, - get_asset_name_identifier, ) from ayon_core.pipeline.version_start import get_versioning_start @@ -51,73 +55,173 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin): self.log.debug("Collecting anatomy data for all instances.") project_name = context.data["projectName"] - self.fill_missing_asset_docs(context, project_name) + self.fill_missing_folder_entities(context, project_name) + self.fill_missing_task_entities(context, project_name) self.fill_latest_versions(context, project_name) self.fill_anatomy_data(context) self.log.debug("Anatomy Data collection finished.") - def fill_missing_asset_docs(self, context, project_name): - self.log.debug("Querying asset documents for instances.") + def fill_missing_folder_entities(self, context, project_name): + self.log.debug("Querying folder entities for instances.") - context_asset_doc = context.data.get("assetEntity") - context_asset_name = None - if context_asset_doc: - context_asset_name = get_asset_name_identifier(context_asset_doc) + context_folder_entity = context.data.get("folderEntity") + context_folder_path = None + if context_folder_entity: + context_folder_path = context_folder_entity["path"] - instances_with_missing_asset_doc = collections.defaultdict(list) + instances_missing_folder = collections.defaultdict(list) for instance in context: - instance_asset_doc = instance.data.get("assetEntity") - _asset_name = instance.data["folderPath"] + instance_folder_entity = instance.data.get("folderEntity") + _folder_path = instance.data["folderPath"] + _task_name = instance.data["task"] - # There is possibility that assetEntity on instance is already set - # which can happen in standalone publisher - if instance_asset_doc: - instance_asset_name = get_asset_name_identifier( - instance_asset_doc) - if instance_asset_name == _asset_name: + # There is possibility that folderEntity on instance is set + if instance_folder_entity: + instance_folder_path = instance_folder_entity["path"] + if instance_folder_path == _folder_path: continue - # Check if asset name is the same as what is in context - # - they may be different, e.g. in NukeStudio - if context_asset_name and context_asset_name == _asset_name: - instance.data["assetEntity"] = context_asset_doc + # Check if folder path is the same as what is in context + # - they may be different, e.g. during editorial publishing + if context_folder_path and context_folder_path == _folder_path: + instance.data["folderEntity"] = context_folder_entity else: - instances_with_missing_asset_doc[_asset_name].append(instance) + instances_missing_folder[_folder_path].append( + instance + ) - if not instances_with_missing_asset_doc: - self.log.debug("All instances already had right asset document.") + if not instances_missing_folder: + self.log.debug("All instances already had right folder entity.") return - asset_names = list(instances_with_missing_asset_doc.keys()) - self.log.debug("Querying asset documents with names: {}".format( - ", ".join(["\"{}\"".format(name) for name in asset_names]) + folder_paths = list(instances_missing_folder.keys()) + self.log.debug("Querying folder entities with paths: {}".format( + ", ".join(["\"{}\"".format(path) for path in folder_paths]) )) - asset_docs = get_assets(project_name, asset_names=asset_names) - asset_docs_by_name = { - get_asset_name_identifier(asset_doc): asset_doc - for asset_doc in asset_docs + folder_entities_by_path = { + folder_entity["path"]: folder_entity + for folder_entity in ayon_api.get_folders( + project_name, folder_paths=folder_paths + ) } - not_found_asset_names = [] - for asset_name, instances in instances_with_missing_asset_doc.items(): - asset_doc = asset_docs_by_name.get(asset_name) - if not asset_doc: - not_found_asset_names.append(asset_name) + not_found_folder_paths = [] + for folder_path, instances in instances_missing_folder.items(): + folder_entity = folder_entities_by_path.get(folder_path) + if not folder_entity: + not_found_folder_paths.append(folder_path) continue for _instance in instances: - _instance.data["assetEntity"] = asset_doc + _instance.data["folderEntity"] = folder_entity - if not_found_asset_names: - joined_asset_names = ", ".join( - ["\"{}\"".format(name) for name in not_found_asset_names] + if not_found_folder_paths: + joined_folder_paths = ", ".join( + ["\"{}\"".format(path) for path in not_found_folder_paths] ) self.log.warning(( - "Not found asset documents with names \"{}\"." - ).format(joined_asset_names)) + "Not found folder entities with paths \"{}\"." + ).format(joined_folder_paths)) + + def fill_missing_task_entities(self, context, project_name): + self.log.debug("Querying task entities for instances.") + + context_folder_entity = context.data.get("folderEntity") + context_folder_id = None + if context_folder_entity: + context_folder_id = context_folder_entity["id"] + context_task_entity = context.data.get("taskEntity") + context_task_name = None + if context_task_entity: + context_task_name = context_task_entity["path"] + + instances_missing_task = {} + folder_path_by_id = {} + for instance in context: + folder_entity = instance.data.get("folderEntity") + # Skip if instnace does not have filled folder entity + if not folder_entity: + continue + folder_id = folder_entity["id"] + folder_path_by_id[folder_id] = folder_entity["path"] + + task_entity = instance.data.get("taskEntity") + _task_name = instance.data["task"] + + # There is possibility that taskEntity on instance is set + if task_entity: + task_parent_id = task_entity["folderId"] + instance_task_name = task_entity["name"] + if ( + folder_id == task_parent_id + and instance_task_name == _task_name + ): + continue + + # Check if folder path is the same as what is in context + # - they may be different, e.g. in NukeStudio + if ( + context_folder_id == folder_id + and context_task_name == _task_name + ): + instance.data["taskEntity"] = context_task_entity + continue + + _by_folder_id = instances_missing_task.setdefault(folder_id, {}) + _by_task_name = _by_folder_id.setdefault(_task_name, []) + _by_task_name.append(instance) + + if not instances_missing_task: + self.log.debug("All instances already had right task entity.") + return + + self.log.debug("Querying task entities") + + all_folder_ids = set(instances_missing_task.keys()) + all_task_names = set() + for per_task in instances_missing_task.values(): + all_task_names |= set(per_task.keys()) + + task_entities = ayon_api.get_tasks( + project_name, + folder_ids=all_folder_ids, + task_names=all_task_names + ) + task_entity_by_ids = {} + for task_entity in task_entities: + folder_id = task_entity["folderId"] + task_name = task_entity["name"] + _by_folder_id = task_entity_by_ids.setdefault(folder_id, {}) + _by_folder_id[task_name] = task_entity + + not_found_task_paths = [] + for folder_id, by_task in instances_missing_task.items(): + for task_name, instances in by_task.items(): + task_entity = ( + task_entity_by_ids + .get(folder_id, {}) + .get(task_name) + ) + if not task_entity: + folder_path = folder_path_by_id[folder_id] + not_found_task_paths.append( + "/".join([folder_path, task_name]) + ) + continue + + for instance in instances: + instance.data["taskEntity"] = task_entity + + if not_found_task_paths: + joined_paths = ", ".join( + ["\"{}\"".format(path) for path in not_found_task_paths] + ) + self.log.warning(( + "Not found task entities with paths \"{}\"." + ).format(joined_paths)) def fill_latest_versions(self, context, project_name): """Try to find latest version for each instance's product name. @@ -140,13 +244,13 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin): latest_version = instance.data.get("latestVersion") instance.data["latestVersion"] = latest_version - # Skip instances without "assetEntity" - asset_doc = instance.data.get("assetEntity") - if not asset_doc: + # Skip instances without "folderEntity" + folder_entity = instance.data.get("folderEntity") + if not folder_entity: continue # Store folder ids and product names for queries - folder_id = asset_doc["_id"] + folder_id = folder_entity["id"] product_name = instance.data["productName"] # Prepare instance hierarchy for faster filling latest versions @@ -205,7 +309,7 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin): } }) - self._fill_asset_data(instance, project_entity, anatomy_data) + self._fill_folder_data(instance, project_entity, anatomy_data) self._fill_task_data(instance, task_types_by_name, anatomy_data) # Define version @@ -275,24 +379,28 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin): json.dumps(anatomy_data, indent=4) )) - def _fill_asset_data(self, instance, project_entity, anatomy_data): - # QUESTION should we make sure that all asset data are poped if asset - # data cannot be found? + def _fill_folder_data(self, instance, project_entity, anatomy_data): + # QUESTION should we make sure that all folder data are poped if + # folder data cannot be found? # - 'asset', 'hierarchy', 'parent', 'folder' - asset_doc = instance.data.get("assetEntity") - if asset_doc: - parents = asset_doc["data"].get("parents") or list() + folder_entity = instance.data.get("folderEntity") + if folder_entity: + folder_name = folder_entity["name"] + folder_path = folder_entity["path"] + hierarchy_parts = folder_path.split("/") + hierarchy_parts.pop(0) + hierarchy_parts.pop(-1) parent_name = project_entity["name"] - if parents: - parent_name = parents[-1] + if hierarchy_parts: + parent_name = hierarchy_parts[-1] - hierarchy = "/".join(parents) + hierarchy = "/".join(hierarchy_parts) anatomy_data.update({ - "asset": asset_doc["name"], + "asset": folder_name, "hierarchy": hierarchy, "parent": parent_name, "folder": { - "name": asset_doc["name"], + "name": folder_name, }, }) return @@ -305,13 +413,13 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin): if hierarchy: parent_name = hierarchy.split("/")[-1] - asset_name = instance.data["folderPath"].split("/")[-1] + folder_name = instance.data["folderPath"].split("/")[-1] anatomy_data.update({ - "asset": asset_name, + "asset": folder_name, "hierarchy": hierarchy, "parent": parent_name, "folder": { - "name": asset_name, + "name": folder_name, }, }) @@ -325,10 +433,10 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin): if not task_name: return - # Find task data based on asset entity - asset_doc = instance.data.get("assetEntity") - task_data = self._get_task_data_from_asset( - asset_doc, task_name, task_types_by_name + # Find task data based on folder entity + task_entity = instance.data.get("taskEntity") + task_data = self._get_task_data_from_entity( + task_entity, task_types_by_name ) if task_data: # Fill task data @@ -344,20 +452,20 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin): if not instance.data.get("newAssetPublishing"): return - # Try to find task data based on hierarchy context and asset name + # Try to find task data based on hierarchy context and folder path hierarchy_context = instance.context.data.get("hierarchyContext") - asset_name = instance.data.get("folderPath") - if not hierarchy_context or not asset_name: + folder_path = instance.data.get("folderPath") + if not hierarchy_context or not folder_path: return project_name = instance.context.data["projectName"] - if "/" not in asset_name: + if "/" not in folder_path: tasks_info = self._find_tasks_info_in_hierarchy( - hierarchy_context, asset_name + hierarchy_context, folder_path ) else: current_data = hierarchy_context.get(project_name, {}) - for key in asset_name.split("/"): + for key in folder_path.split("/"): if key: current_data = current_data.get("childs", {}).get(key, {}) tasks_info = current_data.get("tasks", {}) @@ -375,14 +483,13 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin): "short": task_code } - def _get_task_data_from_asset( - self, asset_doc, task_name, task_types_by_name + def _get_task_data_from_entity( + self, task_entity, task_types_by_name ): """ Args: - asset_doc (Union[dict[str, Any], None]): Asset document. - task_name (Union[str, None]): Task name. + task_entity (Union[dict[str, Any], None]): Task entity. task_types_by_name (dict[str, dict[str, Any]]): Project task types. @@ -390,28 +497,27 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin): Union[dict[str, str], None]: Task data or None if not found. """ - if not asset_doc or not task_name: + if not task_entity: return None - asset_tasks = asset_doc["data"]["tasks"] - task_type = asset_tasks.get(task_name, {}).get("type") + task_type = task_entity["taskType"] task_code = ( task_types_by_name .get(task_type, {}) .get("shortName") ) return { - "name": task_name, + "name": task_entity["name"], "type": task_type, "short": task_code } - def _find_tasks_info_in_hierarchy(self, hierarchy_context, asset_name): + def _find_tasks_info_in_hierarchy(self, hierarchy_context, folder_name): """Find tasks info for an asset in editorial hierarchy. Args: hierarchy_context (dict[str, Any]): Editorial hierarchy context. - asset_name (str): Asset name. + folder_name (str): Folder name. Returns: dict[str, dict[str, Any]]: Tasks info by name. @@ -421,8 +527,8 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin): hierarchy_queue.append(copy.deepcopy(hierarchy_context)) while hierarchy_queue: item = hierarchy_queue.popleft() - if asset_name in item: - return item[asset_name].get("tasks") or {} + if folder_name in item: + return item[folder_name].get("tasks") or {} for subitem in item.values(): hierarchy_queue.extend(subitem.get("childs") or []) diff --git a/client/ayon_core/plugins/publish/collect_audio.py b/client/ayon_core/plugins/publish/collect_audio.py index 357dad76d4..c818ebd147 100644 --- a/client/ayon_core/plugins/publish/collect_audio.py +++ b/client/ayon_core/plugins/publish/collect_audio.py @@ -1,18 +1,18 @@ import collections + +import ayon_api import pyblish.api from ayon_core.client import ( - get_assets, get_subsets, get_last_versions, get_representations, - get_asset_name_identifier, ) from ayon_core.pipeline.load import get_representation_path_with_anatomy class CollectAudio(pyblish.api.ContextPlugin): - """Collect asset's last published audio. + """Collect folders's last published audio. The audio product name searched for is defined in: project settings > Collect Audio @@ -23,7 +23,7 @@ class CollectAudio(pyblish.api.ContextPlugin): converted to context plugin which requires only 4 queries top. """ - label = "Collect Asset Audio" + label = "Collect Folder Audio" order = pyblish.api.CollectorOrder + 0.1 families = ["review"] hosts = [ @@ -64,30 +64,30 @@ class CollectAudio(pyblish.api.ContextPlugin): return # Add audio to instance if exists. - instances_by_asset_name = collections.defaultdict(list) + instances_by_folder_path = collections.defaultdict(list) for instance in filtered_instances: - asset_name = instance.data["folderPath"] - instances_by_asset_name[asset_name].append(instance) + folder_path = instance.data["folderPath"] + instances_by_folder_path[folder_path].append(instance) - asset_names = set(instances_by_asset_name.keys()) + folder_paths = set(instances_by_folder_path.keys()) self.log.debug(( - "Searching for audio product '{product}' in assets {assets}" + "Searching for audio product '{product}' in folders {folders}" ).format( product=self.audio_product_name, - assets=", ".join([ - '"{}"'.format(asset_name) - for asset_name in asset_names + folders=", ".join([ + '"{}"'.format(folder_path) + for folder_path in folder_paths ]) )) # Query all required documents project_name = context.data["projectName"] anatomy = context.data["anatomy"] - repre_docs_by_asset_names = self.query_representations( - project_name, asset_names) + repre_docs_by_folder_paths = self.query_representations( + project_name, folder_paths) - for asset_name, instances in instances_by_asset_name.items(): - repre_docs = repre_docs_by_asset_names[asset_name] + for folder_path, instances in instances_by_folder_path.items(): + repre_docs = repre_docs_by_folder_paths[folder_path] if not repre_docs: continue @@ -103,7 +103,7 @@ class CollectAudio(pyblish.api.ContextPlugin): self.log.debug("Audio Data added to instance ...") def query_representations(self, project_name, folder_paths): - """Query representations related to audio products for passed assets. + """Query representations related to audio products for passed folders. Args: project_name (str): Project in which we're looking for all @@ -113,25 +113,26 @@ class CollectAudio(pyblish.api.ContextPlugin): Returns: collections.defaultdict[str, List[Dict[Str, Any]]]: Representations - related to audio products by asset name. + related to audio products by folder path. """ output = collections.defaultdict(list) - # Query asset documents - asset_docs = get_assets( + # Query folder entities + folder_entities = ayon_api.get_folders( project_name, - asset_names=folder_paths, - fields=["_id", "name", "data.parents"] + folder_paths=folder_paths, + fields={"id", "path"} ) folder_id_by_path = { - get_asset_name_identifier(asset_doc): asset_doc["_id"] - for asset_doc in asset_docs + folder_entity["path"]: folder_entity["id"] + for folder_entity in folder_entities } folder_ids = set(folder_id_by_path.values()) # Query products with name define by 'audio_product_name' attr - # - one or none products with the name should be available on an asset + # - one or none products with the name should be available on + # an folder subset_docs = get_subsets( project_name, subset_names=[self.audio_product_name], diff --git a/client/ayon_core/plugins/publish/collect_context_entities.py b/client/ayon_core/plugins/publish/collect_context_entities.py index b5646576d6..f340178e4f 100644 --- a/client/ayon_core/plugins/publish/collect_context_entities.py +++ b/client/ayon_core/plugins/publish/collect_context_entities.py @@ -2,19 +2,20 @@ Requires: context -> projectName - context -> asset + context -> folderPath context -> task Provides: - context -> projectEntity - Project document from database. - context -> assetEntity - Asset document from database only if 'asset' is - set in context. + context -> projectEntity - Project entity from AYON server. + context -> folderEntity - Folder entity from AYON server only if + 'folderPath' is set in context data. + context -> taskEntity - Task entity from AYON server only if 'folderPath' + and 'task' are set in context data. """ import pyblish.api import ayon_api -from ayon_core.client import get_asset_by_name from ayon_core.pipeline import KnownPublishError @@ -26,45 +27,48 @@ class CollectContextEntities(pyblish.api.ContextPlugin): def process(self, context): project_name = context.data["projectName"] - asset_name = context.data["folderPath"] + folder_path = context.data["folderPath"] task_name = context.data["task"] project_entity = ayon_api.get_project(project_name) if not project_entity: raise KnownPublishError( - "Project '{0}' was not found.".format(project_name) + "Project '{}' was not found.".format(project_name) ) self.log.debug("Collected Project \"{}\"".format(project_entity)) context.data["projectEntity"] = project_entity - if not asset_name: + if not folder_path: self.log.info("Context is not set. Can't collect global data.") return - asset_entity = get_asset_by_name(project_name, asset_name) - assert asset_entity, ( - "No asset found by the name '{0}' in project '{1}'" - ).format(asset_name, project_name) + folder_entity = self._get_folder_entity(project_name, folder_path) + self.log.debug("Collected Folder \"{}\"".format(folder_entity)) - self.log.debug("Collected Asset \"{}\"".format(asset_entity)) + task_entity = self._get_task_entity( + project_name, folder_entity, task_name + ) + self.log.debug("Collected Task \"{}\"".format(task_entity)) - context.data["assetEntity"] = asset_entity + context.data["folderEntity"] = folder_entity + context.data["taskEntity"] = task_entity - data = asset_entity['data'] + folder_attributes = folder_entity["attrib"] # Task type - asset_tasks = data.get("tasks") or {} - task_info = asset_tasks.get(task_name) or {} - task_type = task_info.get("type") + task_type = None + if task_entity: + task_type = task_entity["taskType"] + context.data["taskType"] = task_type - frame_start = data.get("frameStart") + frame_start = folder_attributes.get("frameStart") if frame_start is None: frame_start = 1 self.log.warning("Missing frame start. Defaulting to 1.") - frame_end = data.get("frameEnd") + frame_end = folder_attributes.get("frameEnd") if frame_end is None: frame_end = 2 self.log.warning("Missing frame end. Defaulting to 2.") @@ -72,8 +76,8 @@ class CollectContextEntities(pyblish.api.ContextPlugin): context.data["frameStart"] = frame_start context.data["frameEnd"] = frame_end - handle_start = data.get("handleStart") or 0 - handle_end = data.get("handleEnd") or 0 + handle_start = folder_attributes.get("handleStart") or 0 + handle_end = folder_attributes.get("handleEnd") or 0 context.data["handleStart"] = int(handle_start) context.data["handleEnd"] = int(handle_end) @@ -83,4 +87,30 @@ class CollectContextEntities(pyblish.api.ContextPlugin): context.data["frameStartHandle"] = frame_start_h context.data["frameEndHandle"] = frame_end_h - context.data["fps"] = data["fps"] + context.data["fps"] = folder_attributes["fps"] + + def _get_folder_entity(self, project_name, folder_path): + if not folder_path: + return None + folder_entity = ayon_api.get_folder_by_path(project_name, folder_path) + if not folder_entity: + raise KnownPublishError( + "Folder '{}' was not found in project '{}'.".format( + folder_path, project_name + ) + ) + return folder_entity + + def _get_task_entity(self, project_name, folder_entity, task_name): + if not folder_entity or not task_name: + return None + task_entity = ayon_api.get_task_by_name( + project_name, folder_entity["id"], task_name + ) + if not task_entity: + task_path = "/".join([folder_entity["path"], task_name]) + raise KnownPublishError( + "Task '{}' was not found in project '{}'.".format( + task_path, project_name) + ) + return task_entity \ No newline at end of file diff --git a/client/ayon_core/plugins/publish/collect_current_context.py b/client/ayon_core/plugins/publish/collect_current_context.py index 76d30a913e..76181ffc39 100644 --- a/client/ayon_core/plugins/publish/collect_current_context.py +++ b/client/ayon_core/plugins/publish/collect_current_context.py @@ -21,14 +21,14 @@ class CollectCurrentContext(pyblish.api.ContextPlugin): def process(self, context): # Check if values are already set project_name = context.data.get("projectName") - asset_name = context.data.get("folderPath") + folder_path = context.data.get("folderPath") task_name = context.data.get("task") current_context = get_current_context() if not project_name: context.data["projectName"] = current_context["project_name"] - if not asset_name: + if not folder_path: context.data["folderPath"] = current_context["folder_path"] if not task_name: @@ -40,10 +40,10 @@ class CollectCurrentContext(pyblish.api.ContextPlugin): self.log.info(( "Collected project context\n" "Project: {project_name}\n" - "Asset: {asset_name}\n" + "Folder: {folder_path}\n" "Task: {task_name}" ).format( project_name=context.data["projectName"], - asset_name=context.data["folderPath"], + folder_path=context.data["folderPath"], task_name=context.data["task"] )) diff --git a/client/ayon_core/plugins/publish/collect_frames_fix.py b/client/ayon_core/plugins/publish/collect_frames_fix.py index 0fe86b8d70..bd2379984c 100644 --- a/client/ayon_core/plugins/publish/collect_frames_fix.py +++ b/client/ayon_core/plugins/publish/collect_frames_fix.py @@ -41,7 +41,7 @@ class CollectFramesFixDef( instance.data["frames_to_fix"] = frames_to_fix product_name = instance.data["productName"] - asset_name = instance.data["folderPath"] + folder_path = instance.data["folderPath"] project_entity = instance.data["projectEntity"] project_name = project_entity["name"] @@ -49,7 +49,7 @@ class CollectFramesFixDef( version = get_last_version_by_subset_name( project_name, product_name, - asset_name=asset_name + asset_name=folder_path ) if not version: self.log.warning( diff --git a/client/ayon_core/plugins/publish/collect_from_create_context.py b/client/ayon_core/plugins/publish/collect_from_create_context.py index 36d44def41..b99866fed9 100644 --- a/client/ayon_core/plugins/publish/collect_from_create_context.py +++ b/client/ayon_core/plugins/publish/collect_from_create_context.py @@ -53,11 +53,11 @@ class CollectFromCreateContext(pyblish.api.ContextPlugin): context.data.update(create_context.context_data_to_store()) context.data["newPublishing"] = True # Update context data - asset_name = create_context.get_current_folder_path() + folder_path = create_context.get_current_folder_path() task_name = create_context.get_current_task_name() for key, value in ( ("AYON_PROJECT_NAME", project_name), - ("AYON_FOLDER_PATH", asset_name), + ("AYON_FOLDER_PATH", folder_path), ("AYON_TASK_NAME", task_name) ): if value is None: diff --git a/client/ayon_core/plugins/publish/extract_hierarchy_to_ayon.py b/client/ayon_core/plugins/publish/extract_hierarchy_to_ayon.py index ef8a8b8dfc..59a15af299 100644 --- a/client/ayon_core/plugins/publish/extract_hierarchy_to_ayon.py +++ b/client/ayon_core/plugins/publish/extract_hierarchy_to_ayon.py @@ -4,12 +4,11 @@ import json import uuid import pyblish.api -from ayon_api import slugify_string +from ayon_api import slugify_string, get_folders, get_tasks from ayon_api.entity_hub import EntityHub -from ayon_core.client import get_assets, get_asset_name_identifier from ayon_core.pipeline.template_data import ( - get_asset_template_data, + get_folder_template_data, get_task_template_data, ) @@ -35,38 +34,74 @@ class ExtractHierarchyToAYON(pyblish.api.ContextPlugin): self._fill_instance_entities(context, project_name) def _fill_instance_entities(self, context, project_name): - instances_by_asset_name = collections.defaultdict(list) + instances_by_folder_path = collections.defaultdict(list) for instance in context: if instance.data.get("publish") is False: continue - instance_entity = instance.data.get("assetEntity") + instance_entity = instance.data.get("folderEntity") if instance_entity: continue - # Skip if instance asset does not match - instance_asset_name = instance.data.get("folderPath") - instances_by_asset_name[instance_asset_name].append(instance) + folder_path = instance.data.get("folderPath") + instances_by_folder_path[folder_path].append(instance) project_entity = context.data["projectEntity"] - asset_docs = get_assets( - project_name, asset_names=instances_by_asset_name.keys() + folder_entities = get_folders( + project_name, folder_paths=instances_by_folder_path.keys() ) - asset_docs_by_name = { - get_asset_name_identifier(asset_doc): asset_doc - for asset_doc in asset_docs + folder_entities_by_path = { + folder_entity["path"]: folder_entity + for folder_entity in folder_entities } - for asset_name, instances in instances_by_asset_name.items(): - asset_doc = asset_docs_by_name[asset_name] - asset_data = get_asset_template_data(asset_doc, project_name) + all_task_names = set() + folder_ids = set() + # Fill folderEntity and prepare data for task entities + for folder_path, instances in instances_by_folder_path.items(): + folder_entity = folder_entities_by_path[folder_path] + folder_ids.add(folder_entity["id"]) for instance in instances: task_name = instance.data.get("task") - template_data = get_task_template_data( - project_entity, asset_doc, task_name) - template_data.update(copy.deepcopy(asset_data)) + all_task_names.add(task_name) + + # Query task entities + # Discard 'None' task names + all_task_names.discard(None) + tasks_by_name_by_folder_id = { + folder_id: {} for folder_id in folder_ids + } + task_entities = [] + if all_task_names: + task_entities = get_tasks( + project_name, + task_names=all_task_names, + folder_ids=folder_ids, + ) + for task_entity in task_entities: + task_name = task_entity["name"] + folder_id = task_entity["folderId"] + tasks_by_name_by_folder_id[folder_id][task_name] = task_entity + + for folder_path, instances in instances_by_folder_path.items(): + folder_entity = folder_entities_by_path[folder_path] + folder_id = folder_entity["id"] + folder_data = get_folder_template_data( + folder_entity, project_name + ) + task_entities_by_name = tasks_by_name_by_folder_id[folder_id] + for instance in instances: + task_name = instance.data.get("task") + task_entity = task_entities_by_name.get(task_name) + template_data = {} + if task_entity: + template_data = get_task_template_data( + project_entity, task_entity + ) + template_data.update(copy.deepcopy(folder_data)) instance.data["anatomyData"].update(template_data) - instance.data["assetEntity"] = asset_doc + instance.data["folderEntity"] = folder_entity + instance.data["taskEntity"] = task_entity def _create_hierarchy(self, context, project_name): hierarchy_context = self._filter_hierarchy(context) diff --git a/client/ayon_core/plugins/publish/integrate.py b/client/ayon_core/plugins/publish/integrate.py index 12c702c93b..1e295d2763 100644 --- a/client/ayon_core/plugins/publish/integrate.py +++ b/client/ayon_core/plugins/publish/integrate.py @@ -413,14 +413,14 @@ class IntegrateAsset(pyblish.api.InstancePlugin): ) def prepare_subset(self, instance, op_session, project_name): - asset_doc = instance.data["assetEntity"] + folder_entity = instance.data["folderEntity"] product_name = instance.data["productName"] product_type = instance.data["productType"] self.log.debug("Product: {}".format(product_name)) # Get existing subset if it exists existing_subset_doc = get_subset_by_name( - project_name, product_name, asset_doc["_id"] + project_name, product_name, folder_entity["id"] ) # Define subset data @@ -441,7 +441,7 @@ class IntegrateAsset(pyblish.api.InstancePlugin): if existing_subset_doc: subset_id = existing_subset_doc["_id"] subset_doc = new_subset_document( - product_name, product_type, asset_doc["_id"], data, subset_id + product_name, product_type, folder_entity["id"], data, subset_id ) if existing_subset_doc is None: diff --git a/client/ayon_core/plugins/publish/integrate_thumbnail.py b/client/ayon_core/plugins/publish/integrate_thumbnail.py index 9eb649d5a0..362c5686ab 100644 --- a/client/ayon_core/plugins/publish/integrate_thumbnail.py +++ b/client/ayon_core/plugins/publish/integrate_thumbnail.py @@ -191,9 +191,9 @@ class IntegrateThumbnailsAYON(pyblish.api.ContextPlugin): version_name, version_id )) - asset_entity = instance.data["assetEntity"] + folder_id = instance.data["folderEntity"]["id"] folder_path = instance.data["folderPath"] - thumbnail_info_by_entity_id[asset_entity["_id"]] = { + thumbnail_info_by_entity_id[folder_id] = { "thumbnail_id": thumbnail_id, "entity_type": "asset", } diff --git a/client/ayon_core/plugins/publish/validate_asset_docs.py b/client/ayon_core/plugins/publish/validate_asset_docs.py index 22d957f6e2..95fe4252be 100644 --- a/client/ayon_core/plugins/publish/validate_asset_docs.py +++ b/client/ayon_core/plugins/publish/validate_asset_docs.py @@ -2,27 +2,27 @@ import pyblish.api from ayon_core.pipeline import PublishValidationError -class ValidateAssetDocs(pyblish.api.InstancePlugin): - """Validate existence of asset documents on instances. +class ValidateFolderEntities(pyblish.api.InstancePlugin): + """Validate existence of folder entity on instances. - Without asset document it is not possible to publish the instance. + Without folder entity it is not possible to publish the instance. - If context has set asset document the validation is skipped. + If context has set folder entity the validation is skipped. - Plugin was added because there are cases when context asset is not defined - e.g. in tray publisher. + Plugin was added because there are cases when context folder is not + defined e.g. in tray publisher. """ - label = "Validate Asset docs" + label = "Validate Folder entities" order = pyblish.api.ValidatorOrder def process(self, instance): - context_asset_doc = instance.context.data.get("assetEntity") - if context_asset_doc: + context_folder_entity = instance.context.data.get("folderEntity") + if context_folder_entity: return - if instance.data.get("assetEntity"): - self.log.debug("Instance has set asset document in its data.") + if instance.data.get("folderEntity"): + self.log.debug("Instance has set fodler entity in its data.") elif instance.data.get("newAssetPublishing"): # skip if it is editorial @@ -30,6 +30,6 @@ class ValidateAssetDocs(pyblish.api.InstancePlugin): else: raise PublishValidationError(( - "Instance \"{}\" doesn't have asset document " + "Instance \"{}\" doesn't have folder entity " "set which is needed for publishing." ).format(instance.data["name"])) diff --git a/client/ayon_core/plugins/publish/validate_editorial_asset_name.py b/client/ayon_core/plugins/publish/validate_editorial_asset_name.py index dd1a19f602..9b6794a0c4 100644 --- a/client/ayon_core/plugins/publish/validate_editorial_asset_name.py +++ b/client/ayon_core/plugins/publish/validate_editorial_asset_name.py @@ -1,19 +1,20 @@ from pprint import pformat +import ayon_api import pyblish.api -from ayon_core.client import get_assets, get_asset_name_identifier +from ayon_core.pipeline import KnownPublishError class ValidateEditorialAssetName(pyblish.api.ContextPlugin): - """ Validating if editorial's asset names are not already created in db. + """ Validating if editorial's folder names are not already created in db. Checking variations of names with different size of caps or with or without underscores. """ order = pyblish.api.ValidatorOrder - label = "Validate Editorial Asset Name" + label = "Validate Editorial Folder Name" hosts = [ "hiero", "resolve", @@ -23,62 +24,67 @@ class ValidateEditorialAssetName(pyblish.api.ContextPlugin): def process(self, context): - asset_and_parents = self.get_parents(context) - self.log.debug("__ asset_and_parents: {}".format(asset_and_parents)) + folder_and_parents = self.get_parents(context) + self.log.debug("__ folder_and_parents: {}".format(folder_and_parents)) project_name = context.data["projectName"] - db_assets = list(get_assets( - project_name, fields=["name", "data.parents"] + folder_entities = list(ayon_api.get_folders( + project_name, fields={"path"} )) - self.log.debug("__ db_assets: {}".format(db_assets)) + self.log.debug("__ folder_entities: {}".format(folder_entities)) - asset_db_docs = { - get_asset_name_identifier(asset_doc): list( - asset_doc["data"]["parents"] + existing_folder_paths = { + folder_entity["path"]: ( + folder_entity["path"].lstrip("/").rsplit("/")[0] ) - for asset_doc in db_assets + for folder_entity in folder_entities } self.log.debug("__ project_entities: {}".format( - pformat(asset_db_docs))) + pformat(existing_folder_paths))) - assets_missing_name = {} - assets_wrong_parent = {} - for asset in asset_and_parents.keys(): - if asset not in asset_db_docs.keys(): + folders_missing_name = {} + folders_wrong_parent = {} + for folder_path in folder_and_parents.keys(): + if folder_path not in existing_folder_paths.keys(): # add to some nonexistent list for next layer of check - assets_missing_name[asset] = asset_and_parents[asset] + folders_missing_name[folder_path] = ( + folder_and_parents[folder_path] + ) continue - if asset_and_parents[asset] != asset_db_docs[asset]: + existing_parents = existing_folder_paths[folder_path] + if folder_and_parents[folder_path] != existing_parents: # add to some nonexistent list for next layer of check - assets_wrong_parent[asset] = { - "required": asset_and_parents[asset], - "already_in_db": asset_db_docs[asset] + folders_wrong_parent[folder_path] = { + "required": folder_and_parents[folder_path], + "already_in_db": existing_folder_paths[folder_path] } continue - self.log.debug("correct asset: {}".format(asset)) + self.log.debug("correct folder: {}".format(folder_path)) - if assets_missing_name: + if folders_missing_name: wrong_names = {} self.log.debug( - ">> assets_missing_name: {}".format(assets_missing_name)) + ">> folders_missing_name: {}".format(folders_missing_name)) - # This will create set asset names - asset_names = { - a.lower().replace("_", "") for a in asset_db_docs + # This will create set of folder paths + folder_paths = { + folder_path.lower().replace("_", "") + for folder_path in existing_folder_paths } - for asset in assets_missing_name: - _asset = asset.lower().replace("_", "") - if _asset in asset_names: - wrong_names[asset].update( + for folder_path in folders_missing_name: + _folder_path = folder_path.lower().replace("_", "") + if _folder_path in folder_paths: + wrong_names[folder_path].update( { - "required_name": asset, + "required_name": folder_path, "used_variants_in_db": [ - a for a in asset_db_docs - if a.lower().replace("_", "") == _asset + p + for p in existing_folder_paths + if p.lower().replace("_", "") == _folder_path ] } ) @@ -87,33 +93,19 @@ class ValidateEditorialAssetName(pyblish.api.ContextPlugin): self.log.debug( ">> wrong_names: {}".format(wrong_names)) raise Exception( - "Some already existing asset name variants `{}`".format( + "Some already existing folder name variants `{}`".format( wrong_names)) - if assets_wrong_parent: + if folders_wrong_parent: self.log.debug( - ">> assets_wrong_parent: {}".format(assets_wrong_parent)) - raise Exception( - "Wrong parents on assets `{}`".format(assets_wrong_parent)) - - def _get_all_assets(self, input_dict): - """ Returns asset names in list. - - List contains all asset names including parents - """ - for key in input_dict.keys(): - # check if child key is available - if input_dict[key].get("childs"): - # loop deeper - self._get_all_assets( - input_dict[key]["childs"]) - else: - self.all_testing_assets.append(key) + ">> folders_wrong_parent: {}".format(folders_wrong_parent)) + raise KnownPublishError( + "Wrong parents on folders `{}`".format(folders_wrong_parent)) def get_parents(self, context): - return_dict = {} + output = {} for instance in context: - asset = instance.data["folderPath"] + folder_path = instance.data["folderPath"] families = instance.data.get("families", []) + [ instance.data["family"] ] @@ -123,8 +115,8 @@ class ValidateEditorialAssetName(pyblish.api.ContextPlugin): parents = instance.data["parents"] - return_dict[asset] = [ + output[folder_path] = [ str(p["entity_name"]) for p in parents if p["entity_type"].lower() != "project" ] - return return_dict + return output diff --git a/client/ayon_core/plugins/publish/validate_unique_subsets.py b/client/ayon_core/plugins/publish/validate_unique_subsets.py index 3144675c50..bda40b25b5 100644 --- a/client/ayon_core/plugins/publish/validate_unique_subsets.py +++ b/client/ayon_core/plugins/publish/validate_unique_subsets.py @@ -1,5 +1,7 @@ from collections import defaultdict + import pyblish.api + from ayon_core.pipeline.publish import ( PublishXmlValidationError, ) @@ -9,7 +11,7 @@ class ValidateSubsetUniqueness(pyblish.api.ContextPlugin): """Validate all product names are unique. This only validates whether the instances currently set to publish from - the workfile overlap one another for the asset + product they are publishing + the workfile overlap one another for the folder + product they are publishing to. This does not perform any check against existing publishes in the database @@ -17,7 +19,7 @@ class ValidateSubsetUniqueness(pyblish.api.ContextPlugin): versioning. A product may appear twice to publish from the workfile if one - of them is set to publish to another asset than the other. + of them is set to publish to another folder than the other. """ @@ -27,18 +29,18 @@ class ValidateSubsetUniqueness(pyblish.api.ContextPlugin): def process(self, context): - # Find instance per (asset,product) - instance_per_asset_product = defaultdict(list) + # Find instance per (folder,product) + instance_per_folder_product = defaultdict(list) for instance in context: # Ignore disabled instances if not instance.data.get('publish', True): continue - # Ignore instance without asset data - asset = instance.data.get("folderPath") - if asset is None: - self.log.warning("Instance found without `asset` data: " + # Ignore instance without folder data + folder_path = instance.data.get("folderPath") + if folder_path is None: + self.log.warning("Instance found without `folderPath` data: " "{}".format(instance.name)) continue @@ -50,16 +52,21 @@ class ValidateSubsetUniqueness(pyblish.api.ContextPlugin): ).format(instance.name)) continue - instance_per_asset_product[(asset, product_name)].append(instance) + instance_per_folder_product[(folder_path, product_name)].append( + instance + ) non_unique = [] - for (asset, product_name), instances in instance_per_asset_product.items(): - - # A single instance per asset, product is fine + for (folder_path, product_name), instances in ( + instance_per_folder_product.items() + ): + # A single instance per folder, product is fine if len(instances) < 2: continue - non_unique.append("{} > {}".format(asset, product_name)) + non_unique.append( + "{} > {}".format(folder_path, product_name) + ) if not non_unique: # All is ok