From 640c24d9b70665d1ecec6cf293759c213eaf4271 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Mon, 15 Jul 2024 15:33:39 +0200 Subject: [PATCH 01/90] add 'parents' to folder template data --- client/ayon_core/pipeline/template_data.py | 13 +++++++------ .../publish/collect_anatomy_instance_data.py | 5 ++++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/client/ayon_core/pipeline/template_data.py b/client/ayon_core/pipeline/template_data.py index d5f06d6a59..2c5346f14b 100644 --- a/client/ayon_core/pipeline/template_data.py +++ b/client/ayon_core/pipeline/template_data.py @@ -87,14 +87,14 @@ def get_folder_template_data(folder_entity, project_name): """ path = folder_entity["path"] - hierarchy_parts = path.split("/") + parents = path.split("/") # Remove empty string from the beginning - hierarchy_parts.pop(0) + parents.pop(0) # Remove last part which is folder name - folder_name = hierarchy_parts.pop(-1) - hierarchy = "/".join(hierarchy_parts) - if hierarchy_parts: - parent_name = hierarchy_parts[-1] + folder_name = parents.pop(-1) + hierarchy = "/".join(parents) + if parents: + parent_name = parents[-1] else: parent_name = project_name @@ -103,6 +103,7 @@ def get_folder_template_data(folder_entity, project_name): "name": folder_name, "type": folder_entity["folderType"], "path": path, + "parents": parents, }, "asset": folder_name, "hierarchy": hierarchy, diff --git a/client/ayon_core/plugins/publish/collect_anatomy_instance_data.py b/client/ayon_core/plugins/publish/collect_anatomy_instance_data.py index b6636696c1..6eeca6ad29 100644 --- a/client/ayon_core/plugins/publish/collect_anatomy_instance_data.py +++ b/client/ayon_core/plugins/publish/collect_anatomy_instance_data.py @@ -407,8 +407,10 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin): anatomy_data["hierarchy"] = hierarchy parent_name = project_entity["name"] + parents = [] if hierarchy: - parent_name = hierarchy.split("/")[-1] + parents = hierarchy.split("/") + parent_name = parents[-1] folder_name = instance.data["folderPath"].split("/")[-1] anatomy_data.update({ @@ -422,6 +424,7 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin): # Using 'Shot' is current default behavior of editorial # (or 'newHierarchyIntegration') publishing. "type": "Shot", + "parents": parents, }, }) From c08d0baa88839020caa7e9372fb286f100d56fc3 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Mon, 15 Jul 2024 15:57:55 +0200 Subject: [PATCH 02/90] simplify split --- client/ayon_core/pipeline/template_data.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/pipeline/template_data.py b/client/ayon_core/pipeline/template_data.py index 2c5346f14b..c7aa46fd62 100644 --- a/client/ayon_core/pipeline/template_data.py +++ b/client/ayon_core/pipeline/template_data.py @@ -87,9 +87,8 @@ def get_folder_template_data(folder_entity, project_name): """ path = folder_entity["path"] - parents = path.split("/") - # Remove empty string from the beginning - parents.pop(0) + # Remove empty string from the beginning and split by '/' + parents = path.lstrip("/").split("/") # Remove last part which is folder name folder_name = parents.pop(-1) hierarchy = "/".join(parents) From 282d1720ae958bbe8833ac64d6295e658a28b438 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 12 Sep 2024 16:39:09 +0200 Subject: [PATCH 03/90] Add staging directory functions and configurations - Added functions to handle custom staging directories - Updated imports and removed deprecated code - Created a new module for staging directory handling --- client/ayon_core/pipeline/__init__.py | 27 +-- .../ayon_core/pipeline/publish/constants.py | 1 - client/ayon_core/pipeline/publish/lib.py | 175 ++++++-------- client/ayon_core/pipeline/stagingdir.py | 220 ++++++++++++++++++ client/ayon_core/pipeline/tempdir.py | 90 +++++-- .../publish/collect_custom_staging_dir.py | 76 ------ .../plugins/publish/extract_burnin.py | 11 +- .../publish/extract_color_transcode.py | 15 +- .../plugins/publish/extract_review.py | 7 +- 9 files changed, 396 insertions(+), 226 deletions(-) create mode 100644 client/ayon_core/pipeline/stagingdir.py delete mode 100644 client/ayon_core/plugins/publish/collect_custom_staging_dir.py diff --git a/client/ayon_core/pipeline/__init__.py b/client/ayon_core/pipeline/__init__.py index 8fd00ee6b6..d5c3140d37 100644 --- a/client/ayon_core/pipeline/__init__.py +++ b/client/ayon_core/pipeline/__init__.py @@ -8,6 +8,10 @@ from .constants import ( from .anatomy import Anatomy +from .tempdir import get_temp_dir + +from .stagingdir import get_staging_dir + from .create import ( BaseCreator, Creator, @@ -116,10 +120,12 @@ __all__ = ( "AYON_CONTAINER_ID", "AYON_INSTANCE_ID", "HOST_WORKFILE_EXTENSIONS", - # --- Anatomy --- "Anatomy", - + # --- Temp dir --- + "get_temp_dir", + # --- Staging dir --- + "get_staging_dir", # --- Create --- "BaseCreator", "Creator", @@ -127,42 +133,34 @@ __all__ = ( "HiddenCreator", "CreatedInstance", "CreatorError", - "CreatorError", - # - legacy creation "LegacyCreator", "legacy_create", - "discover_creator_plugins", "discover_legacy_creator_plugins", "register_creator_plugin", "deregister_creator_plugin", "register_creator_plugin_path", "deregister_creator_plugin_path", - # --- Load --- "HeroVersionType", "IncompatibleLoaderError", "LoaderPlugin", "ProductLoaderPlugin", - "discover_loader_plugins", "register_loader_plugin", "deregister_loader_plugin_path", "register_loader_plugin_path", "deregister_loader_plugin", - "load_container", "remove_container", "update_container", "switch_container", - "loaders_from_representation", "get_representation_path", "get_representation_context", "get_repres_contexts", - # --- Publish --- "PublishValidationError", "PublishXmlValidationError", @@ -170,50 +168,41 @@ __all__ = ( "AYONPyblishPluginMixin", "OpenPypePyblishPluginMixin", "OptionalPyblishPluginMixin", - # --- Actions --- "LauncherAction", "InventoryAction", - "discover_launcher_actions", "register_launcher_action", "register_launcher_action_path", - "discover_inventory_actions", "register_inventory_action", "register_inventory_action_path", "deregister_inventory_action", "deregister_inventory_action_path", - # --- Process context --- "install_ayon_plugins", "install_openpype_plugins", "install_host", "uninstall_host", "is_installed", - "register_root", "registered_root", - "register_host", "registered_host", "deregister_host", "get_process_id", - "get_global_context", "get_current_context", "get_current_host_name", "get_current_project_name", "get_current_folder_path", "get_current_task_name", - # Workfile templates "discover_workfile_build_plugins", "register_workfile_build_plugin", "deregister_workfile_build_plugin", "register_workfile_build_plugin_path", "deregister_workfile_build_plugin_path", - # Backwards compatible function names "install", "uninstall", diff --git a/client/ayon_core/pipeline/publish/constants.py b/client/ayon_core/pipeline/publish/constants.py index 38f5ffef3f..5240628365 100644 --- a/client/ayon_core/pipeline/publish/constants.py +++ b/client/ayon_core/pipeline/publish/constants.py @@ -8,4 +8,3 @@ ValidateMeshOrder = pyblish.api.ValidatorOrder + 0.3 DEFAULT_PUBLISH_TEMPLATE = "default" DEFAULT_HERO_PUBLISH_TEMPLATE = "default" -TRANSIENT_DIR_TEMPLATE = "default" diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 8b82622e4c..9cfcd3f71a 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -2,7 +2,6 @@ import os import sys import inspect import copy -import tempfile import xml.etree.ElementTree from typing import Optional, Union, List @@ -18,15 +17,11 @@ from ayon_core.lib import ( ) from ayon_core.settings import get_project_settings from ayon_core.addon import AddonsManager -from ayon_core.pipeline import ( - tempdir, - Anatomy -) +from ayon_core.pipeline import get_staging_dir from ayon_core.pipeline.plugin_discover import DiscoverResult from .constants import ( DEFAULT_PUBLISH_TEMPLATE, DEFAULT_HERO_PUBLISH_TEMPLATE, - TRANSIENT_DIR_TEMPLATE ) @@ -581,58 +576,6 @@ def context_plugin_should_run(plugin, context): return False -def get_instance_staging_dir(instance): - """Unified way how staging dir is stored and created on instances. - - First check if 'stagingDir' is already set in instance data. - In case there already is new tempdir will not be created. - - It also supports `AYON_TMPDIR`, so studio can define own temp - shared repository per project or even per more granular context. - Template formatting is supported also with optional keys. Folder is - created in case it doesn't exists. - - Available anatomy formatting keys: - - root[work | ] - - project[name | code] - - Note: - Staging dir does not have to be necessarily in tempdir so be careful - about its usage. - - Args: - instance (pyblish.lib.Instance): Instance for which we want to get - staging dir. - - Returns: - str: Path to staging dir of instance. - """ - staging_dir = instance.data.get('stagingDir') - if staging_dir: - return staging_dir - - anatomy = instance.context.data.get("anatomy") - - # get customized tempdir path from `AYON_TMPDIR` env var - custom_temp_dir = tempdir.create_custom_tempdir( - anatomy.project_name, anatomy) - - if custom_temp_dir: - staging_dir = os.path.normpath( - tempfile.mkdtemp( - prefix="pyblish_tmp_", - dir=custom_temp_dir - ) - ) - else: - staging_dir = os.path.normpath( - tempfile.mkdtemp(prefix="pyblish_tmp_") - ) - instance.data['stagingDir'] = staging_dir - - return staging_dir - - def get_publish_repre_path(instance, repre, only_published=False): """Get representation path that can be used for integration. @@ -685,6 +628,8 @@ def get_publish_repre_path(instance, repre, only_published=False): return None +# deprecated: backward compatibility only +# TODO: remove in the future def get_custom_staging_dir_info( project_name, host_name, @@ -694,67 +639,85 @@ def get_custom_staging_dir_info( product_name, project_settings=None, anatomy=None, - log=None + log=None, ): - """Checks profiles if context should use special custom dir as staging. + from ayon_core.pipeline.stagingdir import get_staging_dir_config - Args: - project_name (str) - host_name (str) - product_type (str) - task_name (str) - task_type (str) - product_name (str) - project_settings(Dict[str, Any]): Prepared project settings. - anatomy (Dict[str, Any]) - log (Logger) (optional) + tr_data = get_staging_dir_config( + host_name, + project_name, + task_type, + task_name, + product_type, + product_name, + project_settings=project_settings, + anatomy=anatomy, + log=log, + ) + + if not tr_data: + return None, None + + return tr_data["template"], tr_data["persistence"] + + +def get_instance_staging_dir(instance): + """Unified way how staging dir is stored and created on instances. + + First check if 'stagingDir' is already set in instance data. + In case there already is new tempdir will not be created. Returns: - (tuple) - Raises: - ValueError - if misconfigured template should be used + str: Path to staging dir """ - settings = project_settings or get_project_settings(project_name) - custom_staging_dir_profiles = (settings["core"] - ["tools"] - ["publish"] - ["custom_staging_dir_profiles"]) - if not custom_staging_dir_profiles: - return None, None + staging_dir = instance.data.get("stagingDir") - if not log: - log = Logger.get_logger("get_custom_staging_dir_info") + if staging_dir: + return staging_dir - filtering_criteria = { - "hosts": host_name, - "families": product_type, - "task_names": task_name, - "task_types": task_type, - "subsets": product_name - } - profile = filter_profiles(custom_staging_dir_profiles, - filtering_criteria, - logger=log) + anatomy_data = instance.data["anatomyData"] + formatting_data = copy.deepcopy(anatomy_data) - if not profile or not profile["active"]: - return None, None + product_type = instance.data["productType"] + product_name = instance.data["productName"] - if not anatomy: - anatomy = Anatomy(project_name) + # context data based variables + project_entity = instance.context.data["projectEntity"] + folder_entity = instance.context.data["folderEntity"] + task_entity = instance.context.data["taskEntity"] + host_name = instance.context.data["hostName"] + project_settings = instance.context.data["project_settings"] + anatomy = instance.context.data["anatomy"] + current_file = instance.context.data.get("currentFile") - template_name = profile["template_name"] or TRANSIENT_DIR_TEMPLATE + # add current file as workfile name into formatting data + if current_file: + workfile = os.path.basename(current_file) + workfile_name, _ = os.path.splitext(workfile) + formatting_data["workfile_name"] = workfile_name - custom_staging_dir = anatomy.get_template_item( - "staging", template_name, "directory", default=None + dir_data = get_staging_dir( + host_name, + project_entity, + folder_entity, + task_entity, + product_type, + product_name, + anatomy, + project_settings=project_settings, + formatting_data=formatting_data, ) - if custom_staging_dir is None: - raise ValueError(( - "Anatomy of project \"{}\" does not have set" - " \"{}\" template key!" - ).format(project_name, template_name)) - is_persistent = profile["custom_staging_dir_persistent"] - return custom_staging_dir.template, is_persistent + staging_dir_path = dir_data["stagingDir"] + + # TODO: not sure if this is necessary + # path might be already created by get_staging_dir + if not os.path.exists(staging_dir_path): + os.makedirs(staging_dir_path) + + instance.data.update(dir_data) + + return staging_dir_path def get_published_workfile_instance(context): diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/stagingdir.py new file mode 100644 index 0000000000..e8fa1c4853 --- /dev/null +++ b/client/ayon_core/pipeline/stagingdir.py @@ -0,0 +1,220 @@ +from ayon_core.lib import Logger, filter_profiles, StringTemplate +from ayon_core.settings import get_project_settings +from .anatomy import Anatomy +from .tempdir import get_temp_dir +from ayon_core.pipeline.template_data import get_template_data + + +STAGING_DIR_TEMPLATES = "staging" + + +def get_staging_dir_config( + host_name, + project_name, + task_type, + task_name, + product_type, + product_name, + project_settings=None, + anatomy=None, + log=None, +): + """Get matching staging dir profile. + + Args: + host_name (str): Name of host. + project_name (str): Name of project. + task_type (str): Type of task. + task_name (str): Name of task. + product_type (str): Type of product. + product_name (str): Name of product. + project_settings(Dict[str, Any]): Prepared project settings. + anatomy (Dict[str, Any]) + log (Optional[logging.Logger]) + + Returns: + Dict or None: Data with directory template and is_persistent or None + Raises: + ValueError - if misconfigured template should be used + """ + settings = project_settings or get_project_settings(project_name) + + staging_dir_profiles = settings["core"]["tools"]["publish"][ + "custom_staging_dir_profiles" + ] + + if not staging_dir_profiles: + return None + + if not log: + log = Logger.get_logger("get_staging_dir_config") + + filtering_criteria = { + "hosts": host_name, + "task_types": task_type, + "task_names": task_name, + "product_types": product_type, + "product_names": product_name, + } + profile = filter_profiles( + staging_dir_profiles, filtering_criteria, logger=log) + + if not profile or not profile["active"]: + return None + + if not anatomy: + anatomy = Anatomy(project_name) + + # get template from template name + template_name = profile["template_name"] + _validate_template_name(project_name, template_name, anatomy) + + template = anatomy.templates[STAGING_DIR_TEMPLATES][template_name] + + if not template: + # template should always be found either from anatomy or from profile + raise ValueError( + "Staging dir profile is misconfigured! " + "No template was found for profile! " + "Check your project settings at: " + "'ayon+settings://core/tools/publish/custom_staging_dir_profiles'" + ) + + data_persistence = profile["custom_staging_dir_persistent"] + + return {"template": template, "persistence": data_persistence} + + +def _validate_template_name(project_name, template_name, anatomy): + """Check that staging dir section with appropriate template exist. + + Raises: + ValueError - if misconfigured template + """ + # TODO: only for backward compatibility of anatomy for older projects + if STAGING_DIR_TEMPLATES not in anatomy.templates: + raise ValueError( + ( + 'Anatomy of project "{}" does not have set' ' "{}" template section!' + ).format(project_name, template_name) + ) + + if template_name not in anatomy.templates[STAGING_DIR_TEMPLATES]: + raise ValueError( + ( + 'Anatomy of project "{}" does not have set' + ' "{}" template key at Staging Dir section!' + ).format(project_name, template_name) + ) + + +def get_staging_dir( + host_name, + project_entity, + folder_entity, + task_entity, + product_type, + product_name, + anatomy, + project_settings=None, + **kwargs +): + """Get staging dir data. + + If `force_temp` is set, staging dir will be created as tempdir. + If `always_get_some_dir` is set, staging dir will be created as tempdir if + no staging dir profile is found. + If `prefix` or `suffix` is not set, default values will be used. + + Arguments: + host_name (str): Name of host. + project_entity (Dict[str, Any]): Project entity. + folder_entity (Dict[str, Any]): Folder entity. + task_entity (Dict[str, Any]): Task entity. + product_type (str): Type of product. + product_name (str): Name of product. + anatomy (ayon_core.pipeline.Anatomy): Anatomy object. + project_settings (Optional[Dict[str, Any]]): Prepared project settings. + **kwargs: Arbitrary keyword arguments. See below. + + Keyword Arguments: + force_temp (bool): If True, staging dir will be created as tempdir. + always_return_path (bool): If True, staging dir will be created as + tempdir if no staging dir profile is found. + prefix (str): Prefix for staging dir. + suffix (str): Suffix for staging dir. + formatting_data (Dict[str, Any]): Data for formatting staging dir + template. + + Returns: + Dict[str, Any]: Staging dir data + """ + + log = kwargs.get("log") or Logger.get_logger("get_staging_dir") + always_return_path = kwargs.get("always_return_path") + + # make sure always_return_path is set to true by default + if always_return_path is None: + always_return_path = True + + if kwargs.get("force_temp"): + return get_temp_dir( + project_name=project_entity["name"], + anatomy=anatomy, + prefix=kwargs.get("prefix"), + suffix=kwargs.get("suffix"), + ) + + # making fewer queries to database + ctx_data = get_template_data( + project_entity, folder_entity, task_entity, host_name + ) + # add roots to ctx_data + ctx_data["root"] = anatomy.roots + + # add additional data + ctx_data.update({ + "product": { + "type": product_type, + "name": product_name + }, + "host": host_name, + }) + + # add additional data from kwargs + if kwargs.get("formatting_data"): + ctx_data.update(kwargs.get("formatting_data")) + + # get staging dir config + staging_dir_config = get_staging_dir_config( + host_name, + project_entity["name"], + task_entity["type"], + task_entity["name"], + product_type, + product_name, + project_settings=project_settings, + anatomy=anatomy, + log=log, + ) + + # if no preset matching and always_get_some_dir is set, return tempdir + if not staging_dir_config and always_return_path: + return { + "stagingDir": get_temp_dir( + project_name=project_name, + anatomy=anatomy, + prefix=kwargs.get("prefix"), + suffix=kwargs.get("suffix"), + ), + "stagingDir_persistent": False, + } + elif not staging_dir_config: + return None + + return { + "stagingDir": StringTemplate.format_template( + staging_dir_config["template"], ctx_data + ), + "stagingDir_persistent": staging_dir_config["persistence"], + } diff --git a/client/ayon_core/pipeline/tempdir.py b/client/ayon_core/pipeline/tempdir.py index 29d4659393..a6328135ee 100644 --- a/client/ayon_core/pipeline/tempdir.py +++ b/client/ayon_core/pipeline/tempdir.py @@ -3,11 +3,80 @@ Temporary folder operations """ import os +import tempfile +from pathlib import Path from ayon_core.lib import StringTemplate from ayon_core.pipeline import Anatomy -def create_custom_tempdir(project_name, anatomy=None): +def get_temp_dir( + project_name=None, anatomy=None, prefix=None, suffix=None, make_local=False +): + """Get temporary dir path. + + If `make_local` is set, tempdir will be created in local tempdir. + If `anatomy` is not set, default anatomy will be used. + If `prefix` or `suffix` is not set, default values will be used. + + It also supports `OPENPYPE_TMPDIR`, so studio can define own temp + shared repository per project or even per more granular context. + Template formatting is supported also with optional keys. Folder is + created in case it doesn't exists. + + Available anatomy formatting keys: + - root[work | ] + - project[name | code] + + Note: + Staging dir does not have to be necessarily in tempdir so be careful + about its usage. + + Args: + project_name (str)[optional]: Name of project. + anatomy (openpype.pipeline.Anatomy)[optional]: Anatomy object. + make_local (bool)[optional]: If True, temp dir will be created in + local tempdir. + suffix (str)[optional]: Suffix for tempdir. + prefix (str)[optional]: Prefix for tempdir. + + Returns: + str: Path to staging dir of instance. + """ + prefix = prefix or "ay_tmp_" + suffix = suffix or "" + + if make_local: + return _create_local_staging_dir(prefix, suffix) + + # make sure anatomy is set + if not anatomy: + anatomy = Anatomy(project_name) + + # get customized tempdir path from `OPENPYPE_TMPDIR` env var + custom_temp_dir = _create_custom_tempdir(anatomy.project_name, anatomy) + + return _create_local_staging_dir(prefix, suffix, custom_temp_dir) + + +def _create_local_staging_dir(prefix, suffix, dir=None): + """Create local staging dir + + Args: + prefix (str): prefix for tempdir + suffix (str): suffix for tempdir + + Returns: + str: path to tempdir + """ + # use pathlib for creating tempdir + staging_dir = Path(tempfile.mkdtemp( + prefix=prefix, suffix=suffix, dir=dir + )) + + return staging_dir.as_posix() + + +def _create_custom_tempdir(project_name, anatomy=None): """ Create custom tempdir Template path formatting is supporting: @@ -38,7 +107,7 @@ def create_custom_tempdir(project_name, anatomy=None): if anatomy is None: anatomy = Anatomy(project_name) # create base formate data - data = { + template_formatting_data = { "root": anatomy.roots, "project": { "name": anatomy.project_name, @@ -47,19 +116,14 @@ def create_custom_tempdir(project_name, anatomy=None): } # path is anatomy template custom_tempdir = StringTemplate.format_template( - env_tmpdir, data).normalized() + env_tmpdir, template_formatting_data) + + custom_tempdir_path = Path(custom_tempdir) else: # path is absolute - custom_tempdir = env_tmpdir + custom_tempdir_path = Path(env_tmpdir) - # create the dir path if it doesn't exists - if not os.path.exists(custom_tempdir): - try: - # create it if it doesn't exists - os.makedirs(custom_tempdir) - except IOError as error: - raise IOError( - "Path couldn't be created: {}".format(error)) + custom_tempdir_path.mkdir(parents=True, exist_ok=True) - return custom_tempdir + return custom_tempdir_path.as_posix() diff --git a/client/ayon_core/plugins/publish/collect_custom_staging_dir.py b/client/ayon_core/plugins/publish/collect_custom_staging_dir.py deleted file mode 100644 index 49c3a98dd2..0000000000 --- a/client/ayon_core/plugins/publish/collect_custom_staging_dir.py +++ /dev/null @@ -1,76 +0,0 @@ -""" -Requires: - anatomy - - -Provides: - instance.data -> stagingDir (folder path) - -> stagingDir_persistent (bool) -""" -import copy -import os.path - -import pyblish.api - -from ayon_core.pipeline.publish.lib import get_custom_staging_dir_info - - -class CollectCustomStagingDir(pyblish.api.InstancePlugin): - """Looks through profiles if stagingDir should be persistent and in special - location. - - Transient staging dir could be useful in specific use cases where is - desirable to have temporary renders in specific, persistent folders, could - be on disks optimized for speed for example. - - It is studio responsibility to clean up obsolete folders with data. - - Location of the folder is configured in `project_anatomy/templates/others`. - ('transient' key is expected, with 'folder' key) - - Which family/task type/product is applicable is configured in: - `project_settings/global/tools/publish/custom_staging_dir_profiles` - - """ - label = "Collect Custom Staging Directory" - order = pyblish.api.CollectorOrder + 0.4990 - - template_key = "transient" - - def process(self, instance): - product_type = instance.data["productType"] - product_name = instance.data["productName"] - host_name = instance.context.data["hostName"] - project_name = instance.context.data["projectName"] - project_settings = instance.context.data["project_settings"] - anatomy = instance.context.data["anatomy"] - task = instance.data["anatomyData"].get("task", {}) - - transient_tml, is_persistent = get_custom_staging_dir_info( - project_name, - host_name, - product_type, - product_name, - task.get("name"), - task.get("type"), - project_settings=project_settings, - anatomy=anatomy, - log=self.log) - - if transient_tml: - anatomy_data = copy.deepcopy(instance.data["anatomyData"]) - anatomy_data["root"] = anatomy.roots - scene_name = instance.context.data.get("currentFile") - if scene_name: - anatomy_data["scene_name"] = os.path.basename(scene_name) - transient_dir = transient_tml.format(**anatomy_data) - instance.data["stagingDir"] = transient_dir - - instance.data["stagingDir_persistent"] = is_persistent - result_str = "Adding '{}' as".format(transient_dir) - else: - result_str = "Not adding" - - self.log.debug("{} custom staging dir for instance with '{}'".format( - result_str, product_type - )) diff --git a/client/ayon_core/plugins/publish/extract_burnin.py b/client/ayon_core/plugins/publish/extract_burnin.py index 58a032a030..72578d9dc0 100644 --- a/client/ayon_core/plugins/publish/extract_burnin.py +++ b/client/ayon_core/plugins/publish/extract_burnin.py @@ -9,11 +9,13 @@ import clique import pyblish.api from ayon_core import resources, AYON_CORE_ROOT -from ayon_core.pipeline import publish +from ayon_core.pipeline import ( + publish, + get_temp_dir +) from ayon_core.lib import ( run_ayon_launcher_process, - get_transcode_temp_directory, convert_input_paths_for_ffmpeg, should_convert_for_ffmpeg ) @@ -250,7 +252,10 @@ class ExtractBurnin(publish.Extractor): # - change staging dir of source representation # - must be set back after output definitions processing if do_convert: - new_staging_dir = get_transcode_temp_directory() + new_staging_dir = get_temp_dir( + project_name=instance.context.data["projectName"], + make_local=True, + ) repre["stagingDir"] = new_staging_dir convert_input_paths_for_ffmpeg( diff --git a/client/ayon_core/plugins/publish/extract_color_transcode.py b/client/ayon_core/plugins/publish/extract_color_transcode.py index a28a761e7e..ba173867f8 100644 --- a/client/ayon_core/plugins/publish/extract_color_transcode.py +++ b/client/ayon_core/plugins/publish/extract_color_transcode.py @@ -3,15 +3,15 @@ import copy import clique import pyblish.api -from ayon_core.pipeline import publish +from ayon_core.pipeline import ( + publish, + get_temp_dir +) from ayon_core.lib import ( - is_oiio_supported, ) - from ayon_core.lib.transcoding import ( convert_colorspace, - get_transcode_temp_directory, ) from ayon_core.lib.profiles_filtering import filter_profiles @@ -104,7 +104,10 @@ class ExtractOIIOTranscode(publish.Extractor): new_repre = copy.deepcopy(repre) original_staging_dir = new_repre["stagingDir"] - new_staging_dir = get_transcode_temp_directory() + new_staging_dir = get_temp_dir( + project_name=instance.context.data["projectName"], + make_local=True, + ) new_repre["stagingDir"] = new_staging_dir if isinstance(new_repre["files"], list): @@ -254,7 +257,7 @@ class ExtractOIIOTranscode(publish.Extractor): (list) of [file.1001-1010#.exr] or [fileA.exr, fileB.exr] """ pattern = [clique.PATTERNS["frames"]] - collections, remainder = clique.assemble( + collections, _ = clique.assemble( files_to_convert, patterns=pattern, assume_padded_when_ambiguous=True) diff --git a/client/ayon_core/plugins/publish/extract_review.py b/client/ayon_core/plugins/publish/extract_review.py index 06b451bfbe..26cd2ef0b2 100644 --- a/client/ayon_core/plugins/publish/extract_review.py +++ b/client/ayon_core/plugins/publish/extract_review.py @@ -22,8 +22,8 @@ from ayon_core.lib.transcoding import ( should_convert_for_ffmpeg, get_review_layer_name, convert_input_paths_for_ffmpeg, - get_transcode_temp_directory, ) +from ayon_core.pipeline import get_temp_dir from ayon_core.pipeline.publish import ( KnownPublishError, get_publish_instance_label, @@ -310,7 +310,10 @@ class ExtractReview(pyblish.api.InstancePlugin): # - change staging dir of source representation # - must be set back after output definitions processing if do_convert: - new_staging_dir = get_transcode_temp_directory() + new_staging_dir = get_temp_dir( + project_name=instance.context.data["projectName"], + make_local=True, + ) repre["stagingDir"] = new_staging_dir convert_input_paths_for_ffmpeg( From 8b1674619ce7004069bed2fa10d8af39ffac3cb6 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 12 Sep 2024 16:40:49 +0200 Subject: [PATCH 04/90] Add staging directory functionality and a new plugin for managing staging directories in the pipeline. - Added import statement for 'os' in creator_plugins.py - Implemented method 'apply_staging_dir' to apply staging directory with persistence to instance's transient data in creator_plugins.py - Updated comments and added TODOs related to staging directories in various files - Created a new plugin 'CollectManagedStagingDir' to manage staging directories in publish/lib.py --- .../pipeline/create/creator_plugins.py | 55 +++++++++++++++++++ client/ayon_core/pipeline/publish/lib.py | 2 +- .../publish/collect_managed_staging_dir.py | 43 +++++++++++++++ 3 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 client/ayon_core/plugins/publish/collect_managed_staging_dir.py diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 61c10ee736..1360a74519 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import os import copy import collections from typing import TYPE_CHECKING, Optional @@ -14,6 +15,7 @@ from ayon_core.pipeline.plugin_discover import ( deregister_plugin, deregister_plugin_path ) +from ayon_core.pipeline import get_staging_dir from .constants import DEFAULT_VARIANT_VALUE from .product_name import get_product_name @@ -782,6 +784,59 @@ class Creator(BaseCreator): """ return self.pre_create_attr_defs + def apply_staging_dir(self, instance): + """Apply staging dir with persistence to instance's transient data. + + Method is called on instance creation and on instance update. + + Args: + instance (CreatedInstance): Instance for which should be staging + dir applied. + + Returns: + str: Path to staging dir. + """ + create_ctx = self.create_context + product_name = instance.get("productName") + product_type = instance.get("productType") + folder_path = instance.get("folderPath") + if not any([product_name, folder_path]): + return None + + version = instance.get("version") + if version is not None: + formatting_data = {"version": version} + + staging_dir_data = get_staging_dir( + create_ctx.host_name, + create_ctx.get_current_project_entity(), + create_ctx.get_current_folder_entity(), + create_ctx.get_current_task_entity(), + product_type, + product_name, + create_ctx.get_current_project_anatomy(), + create_ctx.get_current_project_settings(), + always_return_path=False, + log=self.log, + formatting_data=formatting_data, + ) + + if not staging_dir_data: + return None + + staging_dir_path = staging_dir_data["stagingDir"] + + # TODO: not sure if this is necessary + # path might be already created by get_staging_dir + if not os.path.exists(staging_dir_path): + os.makedirs(staging_dir_path) + + instance.transient_data.update(staging_dir_data) + + self.log.info(f"Applied staging dir to instance: {staging_dir_path}") + + return staging_dir_path + class HiddenCreator(BaseCreator): @abstractmethod diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 9cfcd3f71a..714794e8f8 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -628,7 +628,7 @@ def get_publish_repre_path(instance, repre, only_published=False): return None -# deprecated: backward compatibility only +# deprecated: backward compatibility only (2024-09-12) # TODO: remove in the future def get_custom_staging_dir_info( project_name, diff --git a/client/ayon_core/plugins/publish/collect_managed_staging_dir.py b/client/ayon_core/plugins/publish/collect_managed_staging_dir.py new file mode 100644 index 0000000000..ca6d5161c1 --- /dev/null +++ b/client/ayon_core/plugins/publish/collect_managed_staging_dir.py @@ -0,0 +1,43 @@ +""" +Requires: + anatomy + + +Provides: + instance.data -> stagingDir (folder path) + -> stagingDir_persistent (bool) +""" + +import pyblish.api + +from ayon_core.pipeline.publish import get_instance_staging_dir + + +class CollectManagedStagingDir(pyblish.api.InstancePlugin): + """Apply matching Staging Dir profile to a instance. + + Apply Staging dir via profiles could be useful in specific use cases + where is desirable to have temporary renders in specific, + persistent folders, could be on disks optimized for speed for example. + + It is studio's responsibility to clean up obsolete folders with data. + + Location of the folder is configured in: + `ayon+anatomy://_/templates/staging`. + + Which family/task type/subset is applicable is configured in: + `ayon+settings://core/tools/publish/custom_staging_dir_profiles` + """ + + label = "Collect Managed Staging Directory" + order = pyblish.api.CollectorOrder + 0.4990 + + def process(self, instance): + + staging_dir_path = get_instance_staging_dir(instance) + persistance = instance.data.get("stagingDir_persistent", False) + + self.log.info(( + f"Instance staging dir was set to `{staging_dir_path}` " + f"and persistence is set to `{persistance}`" + )) From 9e57f74b5c354a3d864eed71f545a5a7f93cc001 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 13 Sep 2024 16:20:46 +0200 Subject: [PATCH 05/90] Update variable names for clarity and consistency. - Renamed variables for better understanding and uniformity - Improved readability by using more descriptive names --- client/ayon_core/pipeline/publish/lib.py | 6 +++--- client/ayon_core/pipeline/stagingdir.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 714794e8f8..fb4db6ddf1 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -696,7 +696,7 @@ def get_instance_staging_dir(instance): workfile_name, _ = os.path.splitext(workfile) formatting_data["workfile_name"] = workfile_name - dir_data = get_staging_dir( + staging_dir_data = get_staging_dir( host_name, project_entity, folder_entity, @@ -708,14 +708,14 @@ def get_instance_staging_dir(instance): formatting_data=formatting_data, ) - staging_dir_path = dir_data["stagingDir"] + staging_dir_path = staging_dir_data["stagingDir"] # TODO: not sure if this is necessary # path might be already created by get_staging_dir if not os.path.exists(staging_dir_path): os.makedirs(staging_dir_path) - instance.data.update(dir_data) + instance.data.update(staging_dir_data) return staging_dir_path diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/stagingdir.py index e8fa1c4853..5ab9596528 100644 --- a/client/ayon_core/pipeline/stagingdir.py +++ b/client/ayon_core/pipeline/stagingdir.py @@ -202,7 +202,7 @@ def get_staging_dir( if not staging_dir_config and always_return_path: return { "stagingDir": get_temp_dir( - project_name=project_name, + project_name=project_entity["name"], anatomy=anatomy, prefix=kwargs.get("prefix"), suffix=kwargs.get("suffix"), From 2b5ab5439aa7d33b5c70ca11ce476cadae81ba0f Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Oct 2024 13:41:20 +0200 Subject: [PATCH 06/90] returning empty lines --- client/ayon_core/pipeline/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/client/ayon_core/pipeline/__init__.py b/client/ayon_core/pipeline/__init__.py index d5c3140d37..505c847c36 100644 --- a/client/ayon_core/pipeline/__init__.py +++ b/client/ayon_core/pipeline/__init__.py @@ -120,12 +120,16 @@ __all__ = ( "AYON_CONTAINER_ID", "AYON_INSTANCE_ID", "HOST_WORKFILE_EXTENSIONS", + # --- Anatomy --- "Anatomy", + # --- Temp dir --- "get_temp_dir", + # --- Staging dir --- "get_staging_dir", + # --- Create --- "BaseCreator", "Creator", @@ -134,6 +138,7 @@ __all__ = ( "CreatedInstance", "CreatorError", "CreatorError", + # - legacy creation "LegacyCreator", "legacy_create", @@ -143,6 +148,7 @@ __all__ = ( "deregister_creator_plugin", "register_creator_plugin_path", "deregister_creator_plugin_path", + # --- Load --- "HeroVersionType", "IncompatibleLoaderError", @@ -161,6 +167,7 @@ __all__ = ( "get_representation_path", "get_representation_context", "get_repres_contexts", + # --- Publish --- "PublishValidationError", "PublishXmlValidationError", @@ -168,6 +175,7 @@ __all__ = ( "AYONPyblishPluginMixin", "OpenPypePyblishPluginMixin", "OptionalPyblishPluginMixin", + # --- Actions --- "LauncherAction", "InventoryAction", @@ -179,6 +187,7 @@ __all__ = ( "register_inventory_action_path", "deregister_inventory_action", "deregister_inventory_action_path", + # --- Process context --- "install_ayon_plugins", "install_openpype_plugins", @@ -197,12 +206,14 @@ __all__ = ( "get_current_project_name", "get_current_folder_path", "get_current_task_name", + # Workfile templates "discover_workfile_build_plugins", "register_workfile_build_plugin", "deregister_workfile_build_plugin", "register_workfile_build_plugin_path", "deregister_workfile_build_plugin_path", + # Backwards compatible function names "install", "uninstall", From 2b765954a3451bc6cba358584b64fed9e8f633e6 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Oct 2024 13:54:05 +0200 Subject: [PATCH 07/90] returning empty lines --- client/ayon_core/pipeline/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/client/ayon_core/pipeline/__init__.py b/client/ayon_core/pipeline/__init__.py index 505c847c36..ea8b1617c6 100644 --- a/client/ayon_core/pipeline/__init__.py +++ b/client/ayon_core/pipeline/__init__.py @@ -137,11 +137,13 @@ __all__ = ( "HiddenCreator", "CreatedInstance", "CreatorError", + "CreatorError", # - legacy creation "LegacyCreator", "legacy_create", + "discover_creator_plugins", "discover_legacy_creator_plugins", "register_creator_plugin", @@ -154,15 +156,18 @@ __all__ = ( "IncompatibleLoaderError", "LoaderPlugin", "ProductLoaderPlugin", + "discover_loader_plugins", "register_loader_plugin", "deregister_loader_plugin_path", "register_loader_plugin_path", "deregister_loader_plugin", + "load_container", "remove_container", "update_container", "switch_container", + "loaders_from_representation", "get_representation_path", "get_representation_context", @@ -179,9 +184,11 @@ __all__ = ( # --- Actions --- "LauncherAction", "InventoryAction", + "discover_launcher_actions", "register_launcher_action", "register_launcher_action_path", + "discover_inventory_actions", "register_inventory_action", "register_inventory_action_path", @@ -194,12 +201,15 @@ __all__ = ( "install_host", "uninstall_host", "is_installed", + "register_root", "registered_root", + "register_host", "registered_host", "deregister_host", "get_process_id", + "get_global_context", "get_current_context", "get_current_host_name", From 396af0cf8610c2d2991c8a7842f164dcc49d98f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Thu, 24 Oct 2024 13:52:43 +0200 Subject: [PATCH 08/90] Update client/ayon_core/pipeline/create/creator_plugins.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/create/creator_plugins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 1360a74519..32ac2bd61f 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -800,7 +800,7 @@ class Creator(BaseCreator): product_name = instance.get("productName") product_type = instance.get("productType") folder_path = instance.get("folderPath") - if not any([product_name, folder_path]): + if not product_name or not folder_path: return None version = instance.get("version") From 9a860785bbce917cc2064db74a018ab012922d64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Thu, 24 Oct 2024 13:52:57 +0200 Subject: [PATCH 09/90] Update client/ayon_core/pipeline/create/creator_plugins.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/create/creator_plugins.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 32ac2bd61f..0de7707e38 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -828,8 +828,7 @@ class Creator(BaseCreator): # TODO: not sure if this is necessary # path might be already created by get_staging_dir - if not os.path.exists(staging_dir_path): - os.makedirs(staging_dir_path) + os.makedirs(staging_dir_path, exist_ok=True) instance.transient_data.update(staging_dir_data) From bd03634ed1a3237e49f1d3307a96722d0960b2a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Thu, 24 Oct 2024 13:53:38 +0200 Subject: [PATCH 10/90] Update client/ayon_core/pipeline/publish/lib.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/publish/lib.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index fb4db6ddf1..ba56c38c78 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -642,7 +642,15 @@ def get_custom_staging_dir_info( log=None, ): from ayon_core.pipeline.stagingdir import get_staging_dir_config - + warnings.warn( + ( + "Function 'get_custom_staging_dir_info' in" + " 'ayon_core.pipeline.publish' is deprecated. Please use" + " 'get_custom_staging_dir_info'" + " in 'ayon_core.pipeline.stagingdir'." + ), + DeprecationWarning, + ) tr_data = get_staging_dir_config( host_name, project_name, From fedf8e60c7b33f5873538773561269e686ee81b4 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Oct 2024 13:56:45 +0200 Subject: [PATCH 11/90] Add warnings module for future use Imported the 'warnings' module for potential future usage in the codebase. --- client/ayon_core/pipeline/publish/lib.py | 1 + 1 file changed, 1 insertion(+) diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index ba56c38c78..657af9570b 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -2,6 +2,7 @@ import os import sys import inspect import copy +import warnings import xml.etree.ElementTree from typing import Optional, Union, List From ea23f355f6dcb24f2dcb2806878afa27ddd11907 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Thu, 24 Oct 2024 14:00:36 +0200 Subject: [PATCH 12/90] Apply suggestions from code review Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/publish/lib.py | 3 +-- client/ayon_core/pipeline/stagingdir.py | 20 +++++++------------- client/ayon_core/pipeline/tempdir.py | 15 ++++++--------- 3 files changed, 14 insertions(+), 24 deletions(-) diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 657af9570b..6a31da82b2 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -721,8 +721,7 @@ def get_instance_staging_dir(instance): # TODO: not sure if this is necessary # path might be already created by get_staging_dir - if not os.path.exists(staging_dir_path): - os.makedirs(staging_dir_path) + os.makedirs(staging_dir_path, exist_ok=True) instance.data.update(staging_dir_data) diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/stagingdir.py index 5ab9596528..d0172c4848 100644 --- a/client/ayon_core/pipeline/stagingdir.py +++ b/client/ayon_core/pipeline/stagingdir.py @@ -1,9 +1,9 @@ from ayon_core.lib import Logger, filter_profiles, StringTemplate from ayon_core.settings import get_project_settings -from .anatomy import Anatomy -from .tempdir import get_temp_dir from ayon_core.pipeline.template_data import get_template_data +from .anatomy import Anatomy +from .tempdir import get_temp_dir STAGING_DIR_TEMPLATES = "staging" @@ -34,8 +34,10 @@ def get_staging_dir_config( Returns: Dict or None: Data with directory template and is_persistent or None + Raises: ValueError - if misconfigured template should be used + """ settings = project_settings or get_project_settings(project_name) @@ -91,14 +93,6 @@ def _validate_template_name(project_name, template_name, anatomy): Raises: ValueError - if misconfigured template """ - # TODO: only for backward compatibility of anatomy for older projects - if STAGING_DIR_TEMPLATES not in anatomy.templates: - raise ValueError( - ( - 'Anatomy of project "{}" does not have set' ' "{}" template section!' - ).format(project_name, template_name) - ) - if template_name not in anatomy.templates[STAGING_DIR_TEMPLATES]: raise ValueError( ( @@ -147,9 +141,9 @@ def get_staging_dir( template. Returns: - Dict[str, Any]: Staging dir data - """ + Optional[Dict[str, Any]]: Staging dir data + """ log = kwargs.get("log") or Logger.get_logger("get_staging_dir") always_return_path = kwargs.get("always_return_path") @@ -209,7 +203,7 @@ def get_staging_dir( ), "stagingDir_persistent": False, } - elif not staging_dir_config: + if not staging_dir_config: return None return { diff --git a/client/ayon_core/pipeline/tempdir.py b/client/ayon_core/pipeline/tempdir.py index a6328135ee..448e774e7c 100644 --- a/client/ayon_core/pipeline/tempdir.py +++ b/client/ayon_core/pipeline/tempdir.py @@ -23,24 +23,21 @@ def get_temp_dir( Template formatting is supported also with optional keys. Folder is created in case it doesn't exists. - Available anatomy formatting keys: - - root[work | ] - - project[name | code] - Note: Staging dir does not have to be necessarily in tempdir so be careful about its usage. Args: - project_name (str)[optional]: Name of project. - anatomy (openpype.pipeline.Anatomy)[optional]: Anatomy object. - make_local (bool)[optional]: If True, temp dir will be created in + project_name (str): Name of project. + anatomy (Optional[Anatomy]): Project Anatomy object. + suffix (Optional[str]): Suffix for tempdir. + prefix (Optional[str]): Prefix for tempdir. + make_local (Optional[bool]): If True, temp dir will be created in local tempdir. - suffix (str)[optional]: Suffix for tempdir. - prefix (str)[optional]: Prefix for tempdir. Returns: str: Path to staging dir of instance. + """ prefix = prefix or "ay_tmp_" suffix = suffix or "" From ea4ec677cac7a73f14225722f0dbc9c17328ff6a Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Oct 2024 14:40:30 +0200 Subject: [PATCH 13/90] reviewer suggestions for changes - Renamed function `get_staging_dir` to `get_staging_dir_info` for clarity. - Updated variable names in multiple files to use `template_data` instead of `formatting_data`. - Adjusted function parameters in the staging directory module for consistency and added new optional parameters. - Improved logging by passing logger instances instead of creating new loggers within functions. --- client/ayon_core/pipeline/__init__.py | 4 +- .../pipeline/create/creator_plugins.py | 18 ++++----- client/ayon_core/pipeline/publish/lib.py | 17 ++++----- client/ayon_core/pipeline/stagingdir.py | 34 ++++++++++------- client/ayon_core/pipeline/tempdir.py | 37 ++++++++----------- 5 files changed, 54 insertions(+), 56 deletions(-) diff --git a/client/ayon_core/pipeline/__init__.py b/client/ayon_core/pipeline/__init__.py index ea8b1617c6..4060501a92 100644 --- a/client/ayon_core/pipeline/__init__.py +++ b/client/ayon_core/pipeline/__init__.py @@ -10,7 +10,7 @@ from .anatomy import Anatomy from .tempdir import get_temp_dir -from .stagingdir import get_staging_dir +from .stagingdir import get_staging_dir_info from .create import ( BaseCreator, @@ -128,7 +128,7 @@ __all__ = ( "get_temp_dir", # --- Staging dir --- - "get_staging_dir", + "get_staging_dir_info", # --- Create --- "BaseCreator", diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 0de7707e38..124395ae16 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -15,7 +15,7 @@ from ayon_core.pipeline.plugin_discover import ( deregister_plugin, deregister_plugin_path ) -from ayon_core.pipeline import get_staging_dir +from ayon_core.pipeline import get_staging_dir_info from .constants import DEFAULT_VARIANT_VALUE from .product_name import get_product_name @@ -805,9 +805,9 @@ class Creator(BaseCreator): version = instance.get("version") if version is not None: - formatting_data = {"version": version} + template_data = {"version": version} - staging_dir_data = get_staging_dir( + staging_dir_info = get_staging_dir_info( create_ctx.host_name, create_ctx.get_current_project_entity(), create_ctx.get_current_folder_entity(), @@ -817,20 +817,20 @@ class Creator(BaseCreator): create_ctx.get_current_project_anatomy(), create_ctx.get_current_project_settings(), always_return_path=False, - log=self.log, - formatting_data=formatting_data, + logger=self.log, + template_data=template_data, ) - if not staging_dir_data: + if not staging_dir_info: return None - staging_dir_path = staging_dir_data["stagingDir"] + staging_dir_path = staging_dir_info["stagingDir"] # TODO: not sure if this is necessary - # path might be already created by get_staging_dir + # path might be already created by get_staging_dir_info os.makedirs(staging_dir_path, exist_ok=True) - instance.transient_data.update(staging_dir_data) + instance.transient_data.update(staging_dir_info) self.log.info(f"Applied staging dir to instance: {staging_dir_path}") diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 6a31da82b2..0f3a7c1d45 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -18,7 +18,7 @@ from ayon_core.lib import ( ) from ayon_core.settings import get_project_settings from ayon_core.addon import AddonsManager -from ayon_core.pipeline import get_staging_dir +from ayon_core.pipeline import get_staging_dir_info from ayon_core.pipeline.plugin_discover import DiscoverResult from .constants import ( DEFAULT_PUBLISH_TEMPLATE, @@ -685,7 +685,7 @@ def get_instance_staging_dir(instance): return staging_dir anatomy_data = instance.data["anatomyData"] - formatting_data = copy.deepcopy(anatomy_data) + template_data = copy.deepcopy(anatomy_data) product_type = instance.data["productType"] product_name = instance.data["productName"] @@ -703,9 +703,9 @@ def get_instance_staging_dir(instance): if current_file: workfile = os.path.basename(current_file) workfile_name, _ = os.path.splitext(workfile) - formatting_data["workfile_name"] = workfile_name + template_data["workfile_name"] = workfile_name - staging_dir_data = get_staging_dir( + staging_dir_info = get_staging_dir_info( host_name, project_entity, folder_entity, @@ -714,16 +714,15 @@ def get_instance_staging_dir(instance): product_name, anatomy, project_settings=project_settings, - formatting_data=formatting_data, + template_data=template_data, ) - staging_dir_path = staging_dir_data["stagingDir"] + staging_dir_path = staging_dir_info["stagingDir"] - # TODO: not sure if this is necessary - # path might be already created by get_staging_dir + # path might be already created by get_staging_dir_info os.makedirs(staging_dir_path, exist_ok=True) - instance.data.update(staging_dir_data) + instance.data.update(staging_dir_info) return staging_dir_path diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/stagingdir.py index d0172c4848..e9d425cf28 100644 --- a/client/ayon_core/pipeline/stagingdir.py +++ b/client/ayon_core/pipeline/stagingdir.py @@ -102,7 +102,7 @@ def _validate_template_name(project_name, template_name, anatomy): ) -def get_staging_dir( +def get_staging_dir_info( host_name, project_entity, folder_entity, @@ -111,9 +111,13 @@ def get_staging_dir( product_name, anatomy, project_settings=None, + template_data=None, + always_return_path=None, + force_tmp_dir=None, + logger=None, **kwargs ): - """Get staging dir data. + """Get staging dir info data. If `force_temp` is set, staging dir will be created as tempdir. If `always_get_some_dir` is set, staging dir will be created as tempdir if @@ -129,29 +133,31 @@ def get_staging_dir( product_name (str): Name of product. anatomy (ayon_core.pipeline.Anatomy): Anatomy object. project_settings (Optional[Dict[str, Any]]): Prepared project settings. + template_data (Optional[Dict[str, Any]]): Data for formatting staging + dir template. + always_return_path (Optional[bool]): If True, staging dir will be + created as tempdir if no staging dir profile is found. Input value + False will return None if no staging dir profile is found. + force_tmp_dir (Optional[bool]): If True, staging dir will be created as + tempdir. + logger (Optional[logging.Logger]): Logger instance. **kwargs: Arbitrary keyword arguments. See below. Keyword Arguments: - force_temp (bool): If True, staging dir will be created as tempdir. - always_return_path (bool): If True, staging dir will be created as - tempdir if no staging dir profile is found. prefix (str): Prefix for staging dir. suffix (str): Suffix for staging dir. - formatting_data (Dict[str, Any]): Data for formatting staging dir - template. Returns: - Optional[Dict[str, Any]]: Staging dir data + Optional[Dict[str, Any]]: Staging dir info data """ - log = kwargs.get("log") or Logger.get_logger("get_staging_dir") - always_return_path = kwargs.get("always_return_path") + log = logger or Logger.get_logger("get_staging_dir_info") # make sure always_return_path is set to true by default if always_return_path is None: always_return_path = True - if kwargs.get("force_temp"): + if force_tmp_dir: return get_temp_dir( project_name=project_entity["name"], anatomy=anatomy, @@ -175,9 +181,9 @@ def get_staging_dir( "host": host_name, }) - # add additional data from kwargs - if kwargs.get("formatting_data"): - ctx_data.update(kwargs.get("formatting_data")) + # add additional template formatting data + if template_data: + ctx_data.update(template_data) # get staging dir config staging_dir_config = get_staging_dir_config( diff --git a/client/ayon_core/pipeline/tempdir.py b/client/ayon_core/pipeline/tempdir.py index 448e774e7c..440ed882aa 100644 --- a/client/ayon_core/pipeline/tempdir.py +++ b/client/ayon_core/pipeline/tempdir.py @@ -10,29 +10,25 @@ from ayon_core.pipeline import Anatomy def get_temp_dir( - project_name=None, anatomy=None, prefix=None, suffix=None, make_local=False + project_name, anatomy=None, prefix=None, suffix=None, use_local_temp=False ): """Get temporary dir path. - If `make_local` is set, tempdir will be created in local tempdir. + If `use_local_temp` is set, tempdir will be created in local tempdir. If `anatomy` is not set, default anatomy will be used. If `prefix` or `suffix` is not set, default values will be used. - It also supports `OPENPYPE_TMPDIR`, so studio can define own temp + It also supports `AYON_TMPDIR`, so studio can define own temp shared repository per project or even per more granular context. Template formatting is supported also with optional keys. Folder is created in case it doesn't exists. - Note: - Staging dir does not have to be necessarily in tempdir so be careful - about its usage. - Args: project_name (str): Name of project. anatomy (Optional[Anatomy]): Project Anatomy object. suffix (Optional[str]): Suffix for tempdir. prefix (Optional[str]): Prefix for tempdir. - make_local (Optional[bool]): If True, temp dir will be created in + use_local_temp (Optional[bool]): If True, temp dir will be created in local tempdir. Returns: @@ -42,7 +38,7 @@ def get_temp_dir( prefix = prefix or "ay_tmp_" suffix = suffix or "" - if make_local: + if use_local_temp: return _create_local_staging_dir(prefix, suffix) # make sure anatomy is set @@ -55,19 +51,20 @@ def get_temp_dir( return _create_local_staging_dir(prefix, suffix, custom_temp_dir) -def _create_local_staging_dir(prefix, suffix, dir=None): +def _create_local_staging_dir(prefix, suffix, dirpath=None): """Create local staging dir Args: prefix (str): prefix for tempdir suffix (str): suffix for tempdir + dirpath (Optional[str]): path to tempdir Returns: str: path to tempdir """ # use pathlib for creating tempdir staging_dir = Path(tempfile.mkdtemp( - prefix=prefix, suffix=suffix, dir=dir + prefix=prefix, suffix=suffix, dir=dirpath )) return staging_dir.as_posix() @@ -89,31 +86,27 @@ def _create_custom_tempdir(project_name, anatomy=None): Returns: str | None: formatted path or None """ - env_tmpdir = os.getenv("AYON_TMPDIR") + env_tmpdir = os.getenv( + "AYON_TMPDIR", + ) if not env_tmpdir: - env_tmpdir = os.getenv("OPENPYPE_TMPDIR") - if not env_tmpdir: - return - print( - "DEPRECATION WARNING: Used 'OPENPYPE_TMPDIR' environment" - " variable. Please use 'AYON_TMPDIR' instead." - ) + return None custom_tempdir = None if "{" in env_tmpdir: if anatomy is None: anatomy = Anatomy(project_name) # create base formate data - template_formatting_data = { + template_data = { "root": anatomy.roots, "project": { "name": anatomy.project_name, "code": anatomy.project_code, - } + }, } # path is anatomy template custom_tempdir = StringTemplate.format_template( - env_tmpdir, template_formatting_data) + env_tmpdir, template_data) custom_tempdir_path = Path(custom_tempdir) From 1eb09045832f29024b0b2dba00adc6d71ace132f Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Oct 2024 14:48:47 +0200 Subject: [PATCH 14/90] Remove unnecessary root addition in get_staging_dir_info function. The code changes remove the unnecessary addition of roots to ctx_data in the get_staging_dir_info function. --- client/ayon_core/pipeline/stagingdir.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/stagingdir.py index e9d425cf28..818acef36a 100644 --- a/client/ayon_core/pipeline/stagingdir.py +++ b/client/ayon_core/pipeline/stagingdir.py @@ -169,8 +169,6 @@ def get_staging_dir_info( ctx_data = get_template_data( project_entity, folder_entity, task_entity, host_name ) - # add roots to ctx_data - ctx_data["root"] = anatomy.roots # add additional data ctx_data.update({ @@ -178,7 +176,7 @@ def get_staging_dir_info( "type": product_type, "name": product_name }, - "host": host_name, + "root": anatomy.roots }) # add additional template formatting data From fa9af2f8ded7e9c89519cfde2f01e04a3dd8b58a Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Oct 2024 15:06:11 +0200 Subject: [PATCH 15/90] Refactor Creator class method to handle missing info better. - Updated return type in docstring - Added a comment for clarity - Removed unnecessary return statements --- client/ayon_core/pipeline/create/creator_plugins.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 801feedd3d..4cbf432efd 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -843,14 +843,16 @@ class Creator(BaseCreator): dir applied. Returns: - str: Path to staging dir. + Optional[str]: Staging dir path or None if not applied. """ create_ctx = self.create_context product_name = instance.get("productName") product_type = instance.get("productType") folder_path = instance.get("folderPath") + + # this can only work if product name and folder path are available if not product_name or not folder_path: - return None + return version = instance.get("version") if version is not None: @@ -871,7 +873,7 @@ class Creator(BaseCreator): ) if not staging_dir_info: - return None + return staging_dir_path = staging_dir_info["stagingDir"] From aaadaffabe5aeb033f3b1f7e0fe3341c69356564 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Mon, 11 Nov 2024 16:57:41 +0800 Subject: [PATCH 16/90] refactoring the load container so that it can load the library project --- client/ayon_core/pipeline/load/utils.py | 4 +- .../ayon_core/tools/sceneinventory/control.py | 7 +- .../ayon_core/tools/sceneinventory/model.py | 2 +- .../tools/sceneinventory/models/containers.py | 131 +++++++++++------- client/ayon_core/tools/sceneinventory/view.py | 9 +- 5 files changed, 97 insertions(+), 56 deletions(-) diff --git a/client/ayon_core/pipeline/load/utils.py b/client/ayon_core/pipeline/load/utils.py index ee2c1af07f..6f69651a8f 100644 --- a/client/ayon_core/pipeline/load/utils.py +++ b/client/ayon_core/pipeline/load/utils.py @@ -465,7 +465,7 @@ def update_container(container, version=-1): from ayon_core.pipeline import get_current_project_name # Compute the different version from 'representation' - project_name = get_current_project_name() + project_name = container.get("project_name", get_current_project_name()) repre_id = container["representation"] if not _is_valid_representation_id(repre_id): raise ValueError( @@ -588,7 +588,7 @@ def switch_container(container, representation, loader_plugin=None): ) # Get the new representation to switch to - project_name = get_current_project_name() + project_name = container.get("project_name", get_current_project_name()) context = get_representation_context( project_name, representation["id"] diff --git a/client/ayon_core/tools/sceneinventory/control.py b/client/ayon_core/tools/sceneinventory/control.py index b890462506..8ce3a1bb7a 100644 --- a/client/ayon_core/tools/sceneinventory/control.py +++ b/client/ayon_core/tools/sceneinventory/control.py @@ -4,7 +4,7 @@ from ayon_core.lib.events import QueuedEventSystem from ayon_core.host import HostBase from ayon_core.pipeline import ( registered_host, - get_current_context, + get_current_context ) from ayon_core.tools.common_models import HierarchyModel, ProjectsModel @@ -110,8 +110,9 @@ class SceneInventoryController: representation_ids ) - def get_version_items(self, product_ids): - return self._containers_model.get_version_items(product_ids) + def get_version_items(self, product_ids, representation_ids): + return self._containers_model.get_version_items( + product_ids, representation_ids) # Site Sync methods def is_sitesync_enabled(self): diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index b7f79986ac..9d1202a906 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -150,7 +150,7 @@ class InventoryModel(QtGui.QStandardItemModel): if repre_info.is_valid } version_items_by_product_id = self._controller.get_version_items( - product_ids + product_ids, repre_id ) # SiteSync addon information progress_by_id = self._controller.get_representations_site_progress( diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index 871455c96b..693a5948c9 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -6,6 +6,7 @@ from ayon_api.graphql import GraphQlQuery from ayon_core.host import ILoadHost from ayon_core.tools.common_models.projects import StatusStates +from ayon_core.pipeline.context_tools import get_current_project_name # --- Implementation that should be in ayon-python-api --- @@ -93,13 +94,15 @@ class ContainerItem: loader_name, namespace, object_name, - item_id + item_id, + project_name ): self.representation_id = representation_id self.loader_name = loader_name self.object_name = object_name self.namespace = namespace self.item_id = item_id + self.project_name = project_name @classmethod def from_container_data(cls, container): @@ -109,6 +112,8 @@ class ContainerItem: namespace=container["namespace"], object_name=container["objectName"], item_id=uuid.uuid4().hex, + project_name=container.get( + "project_name", get_current_project_name()) ) @@ -222,6 +227,9 @@ class ContainersModel: def get_representation_info_items(self, representation_ids): output = {} missing_repre_ids = set() + missing_repre_ids_by_project = {} + containers = self._controller.get_containers() + for repre_id in representation_ids: try: uuid.UUID(repre_id) @@ -229,54 +237,60 @@ class ContainersModel: output[repre_id] = RepresentationInfo.new_invalid() continue + project_name = self._find_project_name(containers, repre_id) + if project_name is None: + project_name = self._controller.get_current_project_name() + repre_info = self._repre_info_by_id.get(repre_id) if repre_info is None: missing_repre_ids.add(repre_id) + missing_repre_ids_by_project.update({project_name: repre_id}) else: output[repre_id] = repre_info if not missing_repre_ids: return output - project_name = self._controller.get_current_project_name() - repre_hierarchy_by_id = get_representations_hierarchy( - project_name, missing_repre_ids - ) - for repre_id, repre_hierarchy in repre_hierarchy_by_id.items(): - kwargs = { - "folder_id": None, - "folder_path": None, - "product_id": None, - "product_name": None, - "product_type": None, - "product_group": None, - "version_id": None, - "representation_name": None, - } - folder = repre_hierarchy.folder - product = repre_hierarchy.product - version = repre_hierarchy.version - repre = repre_hierarchy.representation - if folder: - kwargs["folder_id"] = folder["id"] - kwargs["folder_path"] = folder["path"] - if product: - group = product["attrib"]["productGroup"] - kwargs["product_id"] = product["id"] - kwargs["product_name"] = product["name"] - kwargs["product_type"] = product["productType"] - kwargs["product_group"] = group - if version: - kwargs["version_id"] = version["id"] - if repre: - kwargs["representation_name"] = repre["name"] + for project_name, missing_ids in missing_repre_ids_by_project.items(): + repre_hierarchy_by_id = get_representations_hierarchy( + project_name, {missing_ids} + ) + for repre_id, repre_hierarchy in repre_hierarchy_by_id.items(): + kwargs = { + "folder_id": None, + "folder_path": None, + "product_id": None, + "product_name": None, + "product_type": None, + "product_group": None, + "version_id": None, + "representation_name": None, + } + folder = repre_hierarchy.folder + product = repre_hierarchy.product + version = repre_hierarchy.version + repre = repre_hierarchy.representation + if folder: + kwargs["folder_id"] = folder["id"] + kwargs["folder_path"] = folder["path"] + if product: + group = product["attrib"]["productGroup"] + kwargs["product_id"] = product["id"] + kwargs["product_name"] = product["name"] + kwargs["product_type"] = product["productType"] + kwargs["product_group"] = group + if version: + kwargs["version_id"] = version["id"] + if repre: + kwargs["representation_name"] = repre["name"] - repre_info = RepresentationInfo(**kwargs) - self._repre_info_by_id[repre_id] = repre_info - output[repre_id] = repre_info + repre_info = RepresentationInfo(**kwargs) + self._repre_info_by_id[repre_id] = repre_info + output[repre_id] = repre_info return output - def get_version_items(self, product_ids): + def get_version_items(self, product_ids, representation_ids): + project_ids_by_project_names = {} if not product_ids: return {} @@ -293,20 +307,37 @@ class ContainersModel: def version_sorted(entity): return entity["version"] + containers = self.get_containers() + for repre_id in representation_ids: + project_name = self._find_project_name(containers, repre_id) + if project_name is None: + project_name = self._controller.get_current_project_name() + repre_hierarchy_by_id = get_representations_hierarchy( + project_name, {repre_id} + ) + product_ids_list = set() + for repre_hierarchy in repre_hierarchy_by_id.values(): + product = repre_hierarchy.product + product_id = product["id"] + if product_id not in missing_ids: + continue + product_ids_list.add(product_id) + project_ids_by_project_names.update({project_name: product_ids_list}) - project_name = self._controller.get_current_project_name() version_entities_by_product_id = { product_id: [] for product_id in missing_ids } - - version_entities = list(ayon_api.get_versions( - project_name, - product_ids=missing_ids, - fields={"id", "version", "productId", "status"} - )) - version_entities.sort(key=version_sorted) - for version_entity in version_entities: + version_entities_list = [] + for project_name, missing_product_ids in project_ids_by_project_names.items(): + version_entities = list(ayon_api.get_versions( + project_name, + product_ids=missing_product_ids, + fields={"id", "version", "productId", "status"} + )) + version_entities_list.extend(version_entities) + version_entities_list.sort(key=version_sorted) + for version_entity in version_entities_list: product_id = version_entity["productId"] version_entities_by_product_id[product_id].append( version_entity @@ -337,12 +368,18 @@ class ContainersModel: self._version_items_by_product_id[product_id] = ( version_items_by_id ) - return { product_id: dict(self._version_items_by_product_id[product_id]) for product_id in product_ids } + def _find_project_name(self, containers, representation_id): + # Function to find the project name by representation + for container in containers: + if container.get('representation') == representation_id: + return container.get('project_name', get_current_project_name()) + return None + def _update_cache(self): if self._items_cache is not None: return diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index 22ba15fda8..5fc2113824 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -228,7 +228,10 @@ class SceneInventoryView(QtWidgets.QTreeView): return version_items_by_product_id = self._controller.get_version_items( - product_ids + product_ids, { + container_item.representation_id + for container_item in container_items_by_id.values() + } ) has_outdated = False has_loaded_hero_versions = False @@ -751,7 +754,7 @@ class SceneInventoryView(QtWidgets.QTreeView): active_version_id = active_repre_info.version_id active_product_id = active_repre_info.product_id version_items_by_product_id = self._controller.get_version_items( - product_ids + product_ids, repre_ids ) version_items = list( version_items_by_product_id[active_product_id].values() @@ -943,7 +946,7 @@ class SceneInventoryView(QtWidgets.QTreeView): if repre_info.is_valid } version_items_by_product_id = self._controller.get_version_items( - product_ids + product_ids, repre_ids ) update_containers = [] From dcb838e1454523b3cbee00f5fde54fdc0d36fb58 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Mon, 11 Nov 2024 19:17:00 +0800 Subject: [PATCH 17/90] resolve the project root during updating container --- client/ayon_core/pipeline/load/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/pipeline/load/utils.py b/client/ayon_core/pipeline/load/utils.py index 6f69651a8f..a6c5f0ce1f 100644 --- a/client/ayon_core/pipeline/load/utils.py +++ b/client/ayon_core/pipeline/load/utils.py @@ -542,9 +542,6 @@ def update_container(container, version=-1): ) ) - path = get_representation_path(new_representation) - if not path or not os.path.exists(path): - raise ValueError("Path {} doesn't exist".format(path)) project_entity = ayon_api.get_project(project_name) context = { "project": project_entity, @@ -553,6 +550,9 @@ def update_container(container, version=-1): "version": new_version, "representation": new_representation, } + path = get_representation_path_from_context(context) + if not path or not os.path.exists(path): + raise ValueError("Path {} doesn't exist".format(path)) return Loader().update(container, context) From 735409f9acb945c3af3cf61c9f2d35a2ce51de1e Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Tue, 12 Nov 2024 20:13:49 +0800 Subject: [PATCH 18/90] do not get the container item from self.get_container --- .../ayon_core/tools/sceneinventory/model.py | 5 +- .../tools/sceneinventory/models/containers.py | 75 ++++++++----------- client/ayon_core/tools/sceneinventory/view.py | 22 +++--- 3 files changed, 48 insertions(+), 54 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 3857ea1700..687d130f04 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -130,6 +130,7 @@ class InventoryModel(QtGui.QStandardItemModel): self._clear_items() items_by_repre_id = {} + project_names = set() for container_item in container_items: # if ( # selected is not None @@ -137,8 +138,10 @@ class InventoryModel(QtGui.QStandardItemModel): # ): # continue repre_id = container_item.representation_id + project_name = container_item.project_name items = items_by_repre_id.setdefault(repre_id, []) items.append(container_item) + project_names.add(project_name) repre_id = set(items_by_repre_id.keys()) repre_info_by_id = self._controller.get_representation_info_items( @@ -150,7 +153,7 @@ class InventoryModel(QtGui.QStandardItemModel): if repre_info.is_valid } version_items_by_product_id = self._controller.get_version_items( - product_ids, repre_id + product_ids, project_names ) # SiteSync addon information progress_by_id = self._controller.get_representations_site_progress( diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index 8a4beed52c..f5618d9f35 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -6,7 +6,6 @@ from ayon_api.graphql import GraphQlQuery from ayon_core.host import ILoadHost from ayon_core.tools.common_models.projects import StatusStates -from ayon_core.pipeline.context_tools import get_current_project_name # --- Implementation that should be in ayon-python-api --- @@ -112,8 +111,7 @@ class ContainerItem: namespace=container["namespace"], object_name=container["objectName"], item_id=uuid.uuid4().hex, - project_name=container.get( - "project_name", get_current_project_name()) + project_name=container.get("project_name", None) ) @@ -194,15 +192,21 @@ class ContainersModel: self._items_cache = None self._containers_by_id = {} self._container_items_by_id = {} + self._container_items_by_project = {} + self._project_name_by_repre_id = {} self._version_items_by_product_id = {} self._repre_info_by_id = {} + self._product_id_by_project = {} def reset(self): self._items_cache = None self._containers_by_id = {} self._container_items_by_id = {} + self._container_items_by_project = {} + self._project_name_by_repre_id = {} self._version_items_by_product_id = {} self._repre_info_by_id = {} + self._product_id_by_project = {} def get_containers(self): self._update_cache() @@ -226,10 +230,8 @@ class ContainersModel: def get_representation_info_items(self, representation_ids): output = {} - missing_repre_ids = set() missing_repre_ids_by_project = {} - containers = self._controller.get_containers() - + current_project_name = self._controller.get_current_project_name() for repre_id in representation_ids: try: uuid.UUID(repre_id) @@ -237,23 +239,23 @@ class ContainersModel: output[repre_id] = RepresentationInfo.new_invalid() continue - project_name = self._find_project_name(containers, repre_id) + project_name = self._project_name_by_repre_id.get(repre_id) if project_name is None: - project_name = self._controller.get_current_project_name() - + project_name = current_project_name repre_info = self._repre_info_by_id.get(repre_id) if repre_info is None: - missing_repre_ids.add(repre_id) - missing_repre_ids_by_project.update({project_name: repre_id}) + missing_repre_ids_by_project.setdefault( + project_name, set() + ).add(repre_id) else: output[repre_id] = repre_info - if not missing_repre_ids: + if not missing_repre_ids_by_project: return output for project_name, missing_ids in missing_repre_ids_by_project.items(): repre_hierarchy_by_id = get_representations_hierarchy( - project_name, {missing_ids} + project_name, missing_ids ) for repre_id, repre_hierarchy in repre_hierarchy_by_id.items(): kwargs = { @@ -286,19 +288,23 @@ class ContainersModel: repre_info = RepresentationInfo(**kwargs) self._repre_info_by_id[repre_id] = repre_info + self._product_id_by_project[project_name] = repre_info.product_id output[repre_id] = repre_info return output - def get_version_items(self, product_ids, representation_ids): - project_ids_by_project_names = {} + def get_version_items(self, product_ids, project_names): if not product_ids: return {} - missing_ids = { product_id for product_id in product_ids if product_id not in self._version_items_by_product_id } + + product_ids_by_project = { + project_name: self._product_id_by_project.get(project_name) + for project_name in project_names + } if missing_ids: status_items_by_name = { status_item.name: status_item @@ -307,34 +313,20 @@ class ContainersModel: def version_sorted(entity): return entity["version"] - containers = self.get_containers() - for repre_id in representation_ids: - project_name = self._find_project_name(containers, repre_id) - if project_name is None: - project_name = self._controller.get_current_project_name() - repre_hierarchy_by_id = get_representations_hierarchy( - project_name, {repre_id} - ) - product_ids_list = set() - for repre_hierarchy in repre_hierarchy_by_id.values(): - product = repre_hierarchy.product - product_id = product["id"] - if product_id not in missing_ids: - continue - product_ids_list.add(product_id) - project_ids_by_project_names.update({project_name: product_ids_list}) - + version_entities_list = [] version_entities_by_product_id = { product_id: [] for product_id in missing_ids } - version_entities_list = [] - for project_name, missing_product_ids in project_ids_by_project_names.items(): + for project_name, product_id in product_ids_by_project.items(): + if product_id not in missing_ids: + continue version_entities = list(ayon_api.get_versions( project_name, - product_ids=missing_product_ids, + product_ids={product_id}, fields={"id", "version", "productId", "status"} )) + version_entities_list.extend(version_entities) version_entities_list.sort(key=version_sorted) for version_entity in version_entities_list: @@ -342,7 +334,6 @@ class ContainersModel: version_entities_by_product_id[product_id].append( version_entity ) - for product_id, version_entities in ( version_entities_by_product_id.items() ): @@ -373,13 +364,6 @@ class ContainersModel: for product_id in product_ids } - def _find_project_name(self, containers, representation_id): - # Function to find the project name by representation - for container in containers: - if container.get('representation') == representation_id: - return container.get('project_name', get_current_project_name()) - return None - def _update_cache(self): if self._items_cache is not None: return @@ -395,6 +379,7 @@ class ContainersModel: container_items = [] containers_by_id = {} container_items_by_id = {} + project_name_by_repre_id = {} invalid_ids_mapping = {} for container in containers: try: @@ -418,8 +403,10 @@ class ContainersModel: containers_by_id[item.item_id] = container container_items_by_id[item.item_id] = item + project_name_by_repre_id[item.representation_id] = item.project_name container_items.append(item) self._containers_by_id = containers_by_id self._container_items_by_id = container_items_by_id + self._project_name_by_repre_id = project_name_by_repre_id self._items_cache = container_items diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index 5fc2113824..c5a25fa6dc 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -208,6 +208,7 @@ class SceneInventoryView(QtWidgets.QTreeView): filtered_items = [] product_ids = set() version_ids = set() + project_names = set() for container_item in container_items_by_id.values(): repre_id = container_item.representation_id repre_info = repre_info_by_id.get(repre_id) @@ -215,6 +216,7 @@ class SceneInventoryView(QtWidgets.QTreeView): filtered_items.append(container_item) version_ids.add(repre_info.version_id) product_ids.add(repre_info.product_id) + project_names.add(container_item.project_name) # remove remove_icon = qtawesome.icon("fa.remove", color=DEFAULT_COLOR) @@ -228,11 +230,7 @@ class SceneInventoryView(QtWidgets.QTreeView): return version_items_by_product_id = self._controller.get_version_items( - product_ids, { - container_item.representation_id - for container_item in container_items_by_id.values() - } - ) + product_ids, project_names) has_outdated = False has_loaded_hero_versions = False has_available_hero_version = False @@ -742,6 +740,10 @@ class SceneInventoryView(QtWidgets.QTreeView): container_item.representation_id for container_item in container_items_by_id.values() } + project_names = { + container_item.project_name + for container_item in container_items_by_id.values() + } repre_info_by_id = self._controller.get_representation_info_items( repre_ids ) @@ -754,8 +756,7 @@ class SceneInventoryView(QtWidgets.QTreeView): active_version_id = active_repre_info.version_id active_product_id = active_repre_info.product_id version_items_by_product_id = self._controller.get_version_items( - product_ids, repre_ids - ) + product_ids, project_names) version_items = list( version_items_by_product_id[active_product_id].values() ) @@ -937,6 +938,10 @@ class SceneInventoryView(QtWidgets.QTreeView): container_item.representation_id for container_item in containers_items_by_id.values() } + project_names = { + container_item.project_name + for container_item in containers_items_by_id.values() + } repre_info_by_id = self._controller.get_representation_info_items( repre_ids ) @@ -946,8 +951,7 @@ class SceneInventoryView(QtWidgets.QTreeView): if repre_info.is_valid } version_items_by_product_id = self._controller.get_version_items( - product_ids, repre_ids - ) + product_ids, project_names) update_containers = [] update_versions = [] From 0bec953dec2d786afb09a420e86f83980ddb19a1 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Thu, 14 Nov 2024 22:46:43 +0800 Subject: [PATCH 19/90] query the product id per project --- .../ayon_core/tools/sceneinventory/control.py | 4 +- .../ayon_core/tools/sceneinventory/model.py | 21 +++-- .../tools/sceneinventory/models/containers.py | 30 +++---- client/ayon_core/tools/sceneinventory/view.py | 86 ++++++++++++------- 4 files changed, 85 insertions(+), 56 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/control.py b/client/ayon_core/tools/sceneinventory/control.py index 8ce3a1bb7a..640911df80 100644 --- a/client/ayon_core/tools/sceneinventory/control.py +++ b/client/ayon_core/tools/sceneinventory/control.py @@ -110,9 +110,9 @@ class SceneInventoryController: representation_ids ) - def get_version_items(self, product_ids, representation_ids): + def get_version_items(self, project_name, product_ids): return self._containers_model.get_version_items( - product_ids, representation_ids) + project_name, product_ids) # Site Sync methods def is_sitesync_enabled(self): diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 687d130f04..162a0d4b71 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -152,9 +152,20 @@ class InventoryModel(QtGui.QStandardItemModel): for repre_info in repre_info_by_id.values() if repre_info.is_valid } - version_items_by_product_id = self._controller.get_version_items( - product_ids, project_names - ) + + project_products = {project_name: set() for project_name in project_names} + for representation_id, items in items_by_repre_id.items(): + repre_info = repre_info_by_id.get(representation_id) + if repre_info and repre_info.is_valid: + product_id = repre_info.product_id + for item in items: + project_name = item.project_name + project_products[project_name].add(product_id) + version_items_by_product_id = {} + for project_name, product_ids in project_products.items(): + version_items_by_product_id.update(self._controller.get_version_items( + project_name, product_ids + )) # SiteSync addon information progress_by_id = self._controller.get_representations_site_progress( repre_id @@ -236,7 +247,6 @@ class InventoryModel(QtGui.QStandardItemModel): for container_item in container_items: object_name = container_item.object_name or "" unique_name = repre_name + object_name - item = QtGui.QStandardItem() item.setColumnCount(root_item.columnCount()) item.setData(container_item.namespace, QtCore.Qt.DisplayRole) @@ -251,7 +261,6 @@ class InventoryModel(QtGui.QStandardItemModel): item.setData(True, IS_CONTAINER_ITEM_ROLE) item.setData(unique_name, ITEM_UNIQUE_NAME_ROLE) container_model_items.append(item) - if not container_model_items: continue @@ -290,7 +299,7 @@ class InventoryModel(QtGui.QStandardItemModel): group_item.setData(active_site_icon, ACTIVE_SITE_ICON_ROLE) group_item.setData(remote_site_icon, REMOTE_SITE_ICON_ROLE) group_item.setData(False, IS_CONTAINER_ITEM_ROLE) - + print(group_item) if version_color is not None: group_item.setData(version_color, VERSION_COLOR_ROLE) diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index f5618d9f35..4592b489e1 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -249,7 +249,6 @@ class ContainersModel: ).add(repre_id) else: output[repre_id] = repre_info - if not missing_repre_ids_by_project: return output @@ -292,7 +291,7 @@ class ContainersModel: output[repre_id] = repre_info return output - def get_version_items(self, product_ids, project_names): + def get_version_items(self, project_name, product_ids): if not product_ids: return {} missing_ids = { @@ -301,10 +300,7 @@ class ContainersModel: if product_id not in self._version_items_by_product_id } - product_ids_by_project = { - project_name: self._product_id_by_project.get(project_name) - for project_name in project_names - } + current_product_id = self._product_id_by_project.get(project_name) if missing_ids: status_items_by_name = { status_item.name: status_item @@ -313,24 +309,22 @@ class ContainersModel: def version_sorted(entity): return entity["version"] - version_entities_list = [] + if current_product_id not in missing_ids: + return version_entities_by_product_id = { product_id: [] for product_id in missing_ids } - for project_name, product_id in product_ids_by_project.items(): + version_entities = list(ayon_api.get_versions( + project_name, + product_ids={current_product_id}, + fields={"id", "version", "productId", "status"} + )) + version_entities.sort(key=version_sorted) + for version_entity in version_entities: + product_id = version_entity["productId"] if product_id not in missing_ids: continue - version_entities = list(ayon_api.get_versions( - project_name, - product_ids={product_id}, - fields={"id", "version", "productId", "status"} - )) - - version_entities_list.extend(version_entities) - version_entities_list.sort(key=version_sorted) - for version_entity in version_entities_list: - product_id = version_entity["productId"] version_entities_by_product_id[product_id].append( version_entity ) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index c5a25fa6dc..12a7ab2285 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -206,18 +206,20 @@ class SceneInventoryView(QtWidgets.QTreeView): # Exclude items that are "NOT FOUND" since setting versions, updating # and removal won't work for those items. filtered_items = [] - product_ids = set() + project_products = {} version_ids = set() - project_names = set() for container_item in container_items_by_id.values(): repre_id = container_item.representation_id + project_name = container_item.project_name repre_info = repre_info_by_id.get(repre_id) if repre_info and repre_info.is_valid: filtered_items.append(container_item) version_ids.add(repre_info.version_id) - product_ids.add(repre_info.product_id) - project_names.add(container_item.project_name) - + product_id = repre_info.product_id + if project_name not in project_products: + project_products[project_name] = set() + project_products[project_name].add(product_id) + print("p_products", project_products) # remove remove_icon = qtawesome.icon("fa.remove", color=DEFAULT_COLOR) remove_action = QtWidgets.QAction(remove_icon, "Remove items", menu) @@ -228,9 +230,12 @@ class SceneInventoryView(QtWidgets.QTreeView): # Keep remove action for invalid items menu.addAction(remove_action) return - - version_items_by_product_id = self._controller.get_version_items( - product_ids, project_names) + version_items_by_product_id = {} + for project_name, product_ids in project_products.items(): + version_items_by_product_id.update( + self._controller.get_version_items( + project_name, product_ids) + ) has_outdated = False has_loaded_hero_versions = False has_available_hero_version = False @@ -736,14 +741,11 @@ class SceneInventoryView(QtWidgets.QTreeView): container_items_by_id = self._controller.get_container_items_by_id( item_ids ) + print(container_items_by_id, "container") repre_ids = { container_item.representation_id for container_item in container_items_by_id.values() } - project_names = { - container_item.project_name - for container_item in container_items_by_id.values() - } repre_info_by_id = self._controller.get_representation_info_items( repre_ids ) @@ -752,11 +754,25 @@ class SceneInventoryView(QtWidgets.QTreeView): repre_info.product_id for repre_info in repre_info_by_id.values() } + project_products = {} + for container_item in container_items_by_id.values(): + repre_id = container_item.representation_id + project_name = container_item.project_name + repre_info = repre_info_by_id.get(repre_id) + if repre_info and repre_info.is_valid: + if project_name not in project_products: + project_products[project_name] = set() + product_id = repre_info.product_id + project_products[project_name].add(product_id) + print("proj_product", project_products) active_repre_info = repre_info_by_id[active_repre_id] active_version_id = active_repre_info.version_id active_product_id = active_repre_info.product_id - version_items_by_product_id = self._controller.get_version_items( - product_ids, project_names) + version_items_by_product_id = {} + for project_name, product_ids in project_products.items(): + version_items_by_product_id.update( + self._controller.get_version_items( + project_name, product_ids)) version_items = list( version_items_by_product_id[active_product_id].values() ) @@ -931,27 +947,37 @@ class SceneInventoryView(QtWidgets.QTreeView): self._update_containers_to_version(item_ids, version=-1) def _on_switch_to_versioned(self, item_ids): - containers_items_by_id = self._controller.get_container_items_by_id( - item_ids - ) + # Get container items by ID + containers_items_by_id = self._controller.get_container_items_by_id(item_ids) repre_ids = { container_item.representation_id for container_item in containers_items_by_id.values() } - project_names = { - container_item.project_name - for container_item in containers_items_by_id.values() + # Extract project names and their corresponding representation IDs + project_name_to_repre_ids = {} + for container_item in containers_items_by_id.values(): + project_name = container_item.project_name + repre_id = container_item.representation_id + if project_name not in project_name_to_repre_ids: + project_name_to_repre_ids[project_name] = set() + project_name_to_repre_ids[project_name].add(repre_id) + + # Get representation info items by ID + repre_info_by_id = self._controller.get_representation_info_items(repre_ids) + + # Create a dictionary to map project names to sets of product IDs + project_products = { + project_name: set() for project_name in project_name_to_repre_ids.keys() } - repre_info_by_id = self._controller.get_representation_info_items( - repre_ids - ) - product_ids = { - repre_info.product_id - for repre_info in repre_info_by_id.values() - if repre_info.is_valid - } - version_items_by_product_id = self._controller.get_version_items( - product_ids, project_names) + + print("project_products", project_products) + version_items_by_product_id = {} + for project_name, product_ids in project_name_to_repre_ids.items(): + version_items_by_product_id.update( + self._controller.get_version_items( + project_name, product_ids + ) + ) update_containers = [] update_versions = [] From 574ea3580da0ad9531c77e9297660c35e5e20f5b Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Fri, 15 Nov 2024 21:58:59 +0800 Subject: [PATCH 20/90] codes clean up & make sure it supports mulitple asset loading per project --- .../ayon_core/tools/sceneinventory/control.py | 4 +- .../ayon_core/tools/sceneinventory/model.py | 45 +++--- .../tools/sceneinventory/models/containers.py | 34 ++--- client/ayon_core/tools/sceneinventory/view.py | 138 +++++++++++------- 4 files changed, 127 insertions(+), 94 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/control.py b/client/ayon_core/tools/sceneinventory/control.py index 640911df80..310e41b117 100644 --- a/client/ayon_core/tools/sceneinventory/control.py +++ b/client/ayon_core/tools/sceneinventory/control.py @@ -105,9 +105,9 @@ class SceneInventoryController: def get_container_items_by_id(self, item_ids): return self._containers_model.get_container_items_by_id(item_ids) - def get_representation_info_items(self, representation_ids): + def get_representation_info_items(self, project_name, representation_ids): return self._containers_model.get_representation_info_items( - representation_ids + project_name, representation_ids ) def get_version_items(self, project_name, product_ids): diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 162a0d4b71..a37e3a2b40 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -129,43 +129,54 @@ class InventoryModel(QtGui.QStandardItemModel): self._clear_items() - items_by_repre_id = {} - project_names = set() + items_by_repre_id = collections.defaultdict(list) + repre_ids_by_project = collections.defaultdict(set) for container_item in container_items: # if ( # selected is not None # and container_item.item_id not in selected # ): # continue + project_name = ( + container_item.project_name or + self._controller.get_current_project_name() + ) repre_id = container_item.representation_id - project_name = container_item.project_name - items = items_by_repre_id.setdefault(repre_id, []) - items.append(container_item) - project_names.add(project_name) + items_by_repre_id[repre_id].append(container_item) + repre_ids_by_project[project_name].add(repre_id) repre_id = set(items_by_repre_id.keys()) - repre_info_by_id = self._controller.get_representation_info_items( - repre_id - ) + repre_info_by_id = {} + for project_name, repre_ids in repre_ids_by_project.items(): + repre_info = self._controller.get_representation_info_items( + project_name, repre_ids + ) + repre_info_by_id.update(repre_info) product_ids = { repre_info.product_id for repre_info in repre_info_by_id.values() if repre_info.is_valid } - project_products = {project_name: set() for project_name in project_names} - for representation_id, items in items_by_repre_id.items(): + project_products = collections.defaultdict(set) + for container_item in container_items: + representation_id = container_item.representation_id + project_name = ( + container_item.project_name or + self._controller.get_current_project_name() + ) repre_info = repre_info_by_id.get(representation_id) if repre_info and repre_info.is_valid: product_id = repre_info.product_id - for item in items: - project_name = item.project_name - project_products[project_name].add(product_id) + project_products[project_name].add(product_id) + version_items_by_product_id = {} for project_name, product_ids in project_products.items(): - version_items_by_product_id.update(self._controller.get_version_items( + version_items = self._controller.get_version_items( project_name, product_ids - )) + ) + version_items_by_product_id.update(version_items) + # SiteSync addon information progress_by_id = self._controller.get_representations_site_progress( repre_id @@ -299,7 +310,7 @@ class InventoryModel(QtGui.QStandardItemModel): group_item.setData(active_site_icon, ACTIVE_SITE_ICON_ROLE) group_item.setData(remote_site_icon, REMOTE_SITE_ICON_ROLE) group_item.setData(False, IS_CONTAINER_ITEM_ROLE) - print(group_item) + if version_color is not None: group_item.setData(version_color, VERSION_COLOR_ROLE) diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index 4592b489e1..52b568af03 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -193,20 +193,18 @@ class ContainersModel: self._containers_by_id = {} self._container_items_by_id = {} self._container_items_by_project = {} - self._project_name_by_repre_id = {} self._version_items_by_product_id = {} self._repre_info_by_id = {} - self._product_id_by_project = {} + self._product_ids_by_project = {} def reset(self): self._items_cache = None self._containers_by_id = {} self._container_items_by_id = {} self._container_items_by_project = {} - self._project_name_by_repre_id = {} self._version_items_by_product_id = {} self._repre_info_by_id = {} - self._product_id_by_project = {} + self._product_ids_by_project = {} def get_containers(self): self._update_cache() @@ -228,20 +226,17 @@ class ContainersModel: for item_id in item_ids } - def get_representation_info_items(self, representation_ids): + def get_representation_info_items(self, project_name, representation_ids): output = {} missing_repre_ids_by_project = {} - current_project_name = self._controller.get_current_project_name() + if project_name is None: + project_name = self._controller.get_current_project_name() for repre_id in representation_ids: try: uuid.UUID(repre_id) except ValueError: output[repre_id] = RepresentationInfo.new_invalid() continue - - project_name = self._project_name_by_repre_id.get(repre_id) - if project_name is None: - project_name = current_project_name repre_info = self._repre_info_by_id.get(repre_id) if repre_info is None: missing_repre_ids_by_project.setdefault( @@ -256,6 +251,7 @@ class ContainersModel: repre_hierarchy_by_id = get_representations_hierarchy( project_name, missing_ids ) + self._product_ids_by_project[project_name] = set() for repre_id, repre_hierarchy in repre_hierarchy_by_id.items(): kwargs = { "folder_id": None, @@ -287,20 +283,22 @@ class ContainersModel: repre_info = RepresentationInfo(**kwargs) self._repre_info_by_id[repre_id] = repre_info - self._product_id_by_project[project_name] = repre_info.product_id + self._product_ids_by_project[project_name].add( + repre_info.product_id) output[repre_id] = repre_info return output def get_version_items(self, project_name, product_ids): if not product_ids: return {} + if project_name is None: + project_name = self._controller.get_current_project_name() missing_ids = { product_id for product_id in product_ids if product_id not in self._version_items_by_product_id } - - current_product_id = self._product_id_by_project.get(project_name) + current_product_ids = self._product_ids_by_project.get(project_name) if missing_ids: status_items_by_name = { status_item.name: status_item @@ -309,22 +307,19 @@ class ContainersModel: def version_sorted(entity): return entity["version"] - if current_product_id not in missing_ids: - return + current_missing_ids = current_product_ids.intersection(missing_ids) version_entities_by_product_id = { product_id: [] - for product_id in missing_ids + for product_id in current_missing_ids } version_entities = list(ayon_api.get_versions( project_name, - product_ids={current_product_id}, + product_ids=current_missing_ids, fields={"id", "version", "productId", "status"} )) version_entities.sort(key=version_sorted) for version_entity in version_entities: product_id = version_entity["productId"] - if product_id not in missing_ids: - continue version_entities_by_product_id[product_id].append( version_entity ) @@ -402,5 +397,4 @@ class ContainersModel: self._containers_by_id = containers_by_id self._container_items_by_id = container_items_by_id - self._project_name_by_repre_id = project_name_by_repre_id self._items_cache = container_items diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index 12a7ab2285..1e0b96570e 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -192,11 +192,20 @@ class SceneInventoryView(QtWidgets.QTreeView): container_item = container_items_by_id[item_id] active_repre_id = container_item.representation_id break + repre_ids_by_project = collections.defaultdict(set) + for container_item in container_items_by_id.values(): + repre_id = container_item.representation_id + project_name = ( + container_item.project_name or + self._controller.get_current_project_name() + ) + repre_ids_by_project[project_name].add(repre_id) + repre_info_by_id = {} + for project_name, repre_ids in repre_ids_by_project.items(): + repre_info = self._controller.get_representation_info_items( + project_name, repre_ids) + repre_info_by_id.update(repre_info) - repre_info_by_id = self._controller.get_representation_info_items({ - container_item.representation_id - for container_item in container_items_by_id.values() - }) valid_repre_ids = { repre_id for repre_id, repre_info in repre_info_by_id.items() @@ -206,20 +215,20 @@ class SceneInventoryView(QtWidgets.QTreeView): # Exclude items that are "NOT FOUND" since setting versions, updating # and removal won't work for those items. filtered_items = [] - project_products = {} + product_ids_by_project = collections.defaultdict(set) version_ids = set() for container_item in container_items_by_id.values(): repre_id = container_item.representation_id - project_name = container_item.project_name + project_name = ( + container_item.project_name or + self._controller.get_current_project_name() + ) repre_info = repre_info_by_id.get(repre_id) if repre_info and repre_info.is_valid: filtered_items.append(container_item) version_ids.add(repre_info.version_id) product_id = repre_info.product_id - if project_name not in project_products: - project_products[project_name] = set() - project_products[project_name].add(product_id) - print("p_products", project_products) + product_ids_by_project[project_name].add(product_id) # remove remove_icon = qtawesome.icon("fa.remove", color=DEFAULT_COLOR) remove_action = QtWidgets.QAction(remove_icon, "Remove items", menu) @@ -231,11 +240,12 @@ class SceneInventoryView(QtWidgets.QTreeView): menu.addAction(remove_action) return version_items_by_product_id = {} - for project_name, product_ids in project_products.items(): - version_items_by_product_id.update( - self._controller.get_version_items( - project_name, product_ids) + for project_name, product_ids in product_ids_by_project.items(): + version_items = self._controller.get_version_items( + project_name, product_ids ) + version_items_by_product_id.update(version_items +) has_outdated = False has_loaded_hero_versions = False has_available_hero_version = False @@ -741,38 +751,47 @@ class SceneInventoryView(QtWidgets.QTreeView): container_items_by_id = self._controller.get_container_items_by_id( item_ids ) - print(container_items_by_id, "container") - repre_ids = { - container_item.representation_id - for container_item in container_items_by_id.values() - } - repre_info_by_id = self._controller.get_representation_info_items( - repre_ids - ) + repre_ids_by_project = collections.defaultdict(set) + for container_item in container_items_by_id.values(): + repre_id = container_item.representation_id + project_name = ( + container_item.project_name or + self._controller.get_current_project_name() + ) + repre_ids_by_project[project_name].add(repre_id) + repre_info_by_id = {} + for project_name, repre_ids in repre_ids_by_project.items(): + repre_info = self._controller.get_representation_info_items( + project_name, repre_ids + ) + repre_info_by_id.update(repre_info) product_ids = { repre_info.product_id for repre_info in repre_info_by_id.values() } - project_products = {} + product_ids_by_project = collections.defaultdict(set) for container_item in container_items_by_id.values(): repre_id = container_item.representation_id - project_name = container_item.project_name + project_name = ( + container_item.project_name or + self._controller.get_current_project_name() + ) repre_info = repre_info_by_id.get(repre_id) - if repre_info and repre_info.is_valid: - if project_name not in project_products: - project_products[project_name] = set() - product_id = repre_info.product_id - project_products[project_name].add(product_id) - print("proj_product", project_products) + if not repre_info or not repre_info.is_valid: + continue + product_ids_by_project[project_name].add( + repre_info.product_id + ) active_repre_info = repre_info_by_id[active_repre_id] active_version_id = active_repre_info.version_id active_product_id = active_repre_info.product_id version_items_by_product_id = {} - for project_name, product_ids in project_products.items(): - version_items_by_product_id.update( - self._controller.get_version_items( - project_name, product_ids)) + for project_name, project_product_ids in product_ids_by_project.items(): + version_items = self._controller.get_version_items( + project_name, project_product_ids + ) + version_items_by_product_id.update(version_items) version_items = list( version_items_by_product_id[active_product_id].values() ) @@ -949,35 +968,44 @@ class SceneInventoryView(QtWidgets.QTreeView): def _on_switch_to_versioned(self, item_ids): # Get container items by ID containers_items_by_id = self._controller.get_container_items_by_id(item_ids) - repre_ids = { - container_item.representation_id - for container_item in containers_items_by_id.values() - } # Extract project names and their corresponding representation IDs - project_name_to_repre_ids = {} + repre_ids_by_project = collections.defaultdict(set) for container_item in containers_items_by_id.values(): project_name = container_item.project_name + project_name = ( + container_item.project_name or + self._controller.get_current_project_name() + ) repre_id = container_item.representation_id - if project_name not in project_name_to_repre_ids: - project_name_to_repre_ids[project_name] = set() - project_name_to_repre_ids[project_name].add(repre_id) + repre_ids_by_project[project_name].add(repre_id) # Get representation info items by ID - repre_info_by_id = self._controller.get_representation_info_items(repre_ids) + repre_info_by_id = {} + for project_name, repre_ids in repre_ids_by_project.items(): + repre_info = self._controller.get_representation_info_items( + project_name, repre_ids) + repre_info_by_id.update(repre_info) - # Create a dictionary to map project names to sets of product IDs - project_products = { - project_name: set() for project_name in project_name_to_repre_ids.keys() - } - - print("project_products", project_products) - version_items_by_product_id = {} - for project_name, product_ids in project_name_to_repre_ids.items(): - version_items_by_product_id.update( - self._controller.get_version_items( - project_name, product_ids - ) + product_ids_by_project = collections.defaultdict(set) + for container_item in containers_items_by_id.values(): + repre_id = container_item.representation_id + project_name = ( + container_item.project_name or + self._controller.get_current_project_name() ) + repre_info = repre_info_by_id.get(repre_id) + if not repre_info or not repre_info.is_valid: + continue + product_ids_by_project[project_name].add( + repre_info.product_id + ) + + version_items_by_product_id = {} + for project_name, product_ids in product_ids_by_project.items(): + version_items = self._controller.get_version_items( + project_name, product_ids + ) + version_items_by_product_id.update(version_items) update_containers = [] update_versions = [] From 8965a8859435473c0171053cc2908dd011c05ad8 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Mon, 18 Nov 2024 17:59:20 +0800 Subject: [PATCH 21/90] clean up code and add project name row into scene inventory --- .../ayon_core/tools/sceneinventory/model.py | 8 +- .../tools/sceneinventory/models/containers.py | 87 +++++++++---------- 2 files changed, 47 insertions(+), 48 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index a37e3a2b40..03627e60b9 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -36,6 +36,7 @@ REMOTE_SITE_ICON_ROLE = QtCore.Qt.UserRole + 23 # This value hold unique value of container that should be used to identify # containers inbetween refresh. ITEM_UNIQUE_NAME_ROLE = QtCore.Qt.UserRole + 24 +PROJECT_NAME_ROLE = QtCore.Qt.UserRole + 25 class InventoryModel(QtGui.QStandardItemModel): @@ -52,6 +53,7 @@ class InventoryModel(QtGui.QStandardItemModel): "Object name", "Active site", "Remote site", + "Project" ] name_col = column_labels.index("Name") version_col = column_labels.index("Version") @@ -63,6 +65,7 @@ class InventoryModel(QtGui.QStandardItemModel): object_name_col = column_labels.index("Object name") active_site_col = column_labels.index("Active site") remote_site_col = column_labels.index("Remote site") + project_col = column_labels.index("Project") display_role_by_column = { name_col: QtCore.Qt.DisplayRole, version_col: VERSION_LABEL_ROLE, @@ -72,6 +75,7 @@ class InventoryModel(QtGui.QStandardItemModel): product_group_col: PRODUCT_GROUP_NAME_ROLE, loader_col: LOADER_NAME_ROLE, object_name_col: OBJECT_NAME_ROLE, + project_col: PROJECT_NAME_ROLE, active_site_col: ACTIVE_SITE_PROGRESS_ROLE, remote_site_col: REMOTE_SITE_PROGRESS_ROLE, } @@ -85,7 +89,7 @@ class InventoryModel(QtGui.QStandardItemModel): foreground_role_by_column = { name_col: NAME_COLOR_ROLE, version_col: VERSION_COLOR_ROLE, - status_col: STATUS_COLOR_ROLE + status_col: STATUS_COLOR_ROLE, } width_by_column = { name_col: 250, @@ -95,6 +99,7 @@ class InventoryModel(QtGui.QStandardItemModel): product_type_col: 150, product_group_col: 120, loader_col: 150, + project_col: 150, } OUTDATED_COLOR = QtGui.QColor(235, 30, 30) @@ -269,6 +274,7 @@ class InventoryModel(QtGui.QStandardItemModel): item.setData(version_label, VERSION_LABEL_ROLE) item.setData(container_item.loader_name, LOADER_NAME_ROLE) item.setData(container_item.object_name, OBJECT_NAME_ROLE) + item.setData(container_item.project_name, PROJECT_NAME_ROLE) item.setData(True, IS_CONTAINER_ITEM_ROLE) item.setData(unique_name, ITEM_UNIQUE_NAME_ROLE) container_model_items.append(item) diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index 52b568af03..b8b9aa400a 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -228,9 +228,7 @@ class ContainersModel: def get_representation_info_items(self, project_name, representation_ids): output = {} - missing_repre_ids_by_project = {} - if project_name is None: - project_name = self._controller.get_current_project_name() + missing_repre_ids = set() for repre_id in representation_ids: try: uuid.UUID(repre_id) @@ -239,60 +237,55 @@ class ContainersModel: continue repre_info = self._repre_info_by_id.get(repre_id) if repre_info is None: - missing_repre_ids_by_project.setdefault( - project_name, set() - ).add(repre_id) + missing_repre_ids.add(repre_id) else: output[repre_id] = repre_info - if not missing_repre_ids_by_project: + if not missing_repre_ids: return output - for project_name, missing_ids in missing_repre_ids_by_project.items(): - repre_hierarchy_by_id = get_representations_hierarchy( - project_name, missing_ids - ) - self._product_ids_by_project[project_name] = set() - for repre_id, repre_hierarchy in repre_hierarchy_by_id.items(): - kwargs = { - "folder_id": None, - "folder_path": None, - "product_id": None, - "product_name": None, - "product_type": None, - "product_group": None, - "version_id": None, - "representation_name": None, - } - folder = repre_hierarchy.folder - product = repre_hierarchy.product - version = repre_hierarchy.version - repre = repre_hierarchy.representation - if folder: - kwargs["folder_id"] = folder["id"] - kwargs["folder_path"] = folder["path"] - if product: - group = product["attrib"]["productGroup"] - kwargs["product_id"] = product["id"] - kwargs["product_name"] = product["name"] - kwargs["product_type"] = product["productType"] - kwargs["product_group"] = group - if version: - kwargs["version_id"] = version["id"] - if repre: - kwargs["representation_name"] = repre["name"] + repre_hierarchy_by_id = get_representations_hierarchy( + project_name, missing_repre_ids + ) + self._product_ids_by_project[project_name] = set() + for repre_id, repre_hierarchy in repre_hierarchy_by_id.items(): + kwargs = { + "folder_id": None, + "folder_path": None, + "product_id": None, + "product_name": None, + "product_type": None, + "product_group": None, + "version_id": None, + "representation_name": None, + } + folder = repre_hierarchy.folder + product = repre_hierarchy.product + version = repre_hierarchy.version + repre = repre_hierarchy.representation + if folder: + kwargs["folder_id"] = folder["id"] + kwargs["folder_path"] = folder["path"] + if product: + group = product["attrib"]["productGroup"] + kwargs["product_id"] = product["id"] + kwargs["product_name"] = product["name"] + kwargs["product_type"] = product["productType"] + kwargs["product_group"] = group + if version: + kwargs["version_id"] = version["id"] + if repre: + kwargs["representation_name"] = repre["name"] - repre_info = RepresentationInfo(**kwargs) - self._repre_info_by_id[repre_id] = repre_info - self._product_ids_by_project[project_name].add( - repre_info.product_id) - output[repre_id] = repre_info + repre_info = RepresentationInfo(**kwargs) + self._repre_info_by_id[repre_id] = repre_info + self._product_ids_by_project[project_name].add( + repre_info.product_id) + output[repre_id] = repre_info return output def get_version_items(self, project_name, product_ids): if not product_ids: return {} - if project_name is None: - project_name = self._controller.get_current_project_name() missing_ids = { product_id for product_id in product_ids From 0866c002019834e6a4aeda22becb9e5c208f4515 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Mon, 18 Nov 2024 19:18:50 +0800 Subject: [PATCH 22/90] code tweaks - kuba's comment --- .../ayon_core/tools/sceneinventory/model.py | 12 ++------ .../tools/sceneinventory/models/containers.py | 5 +++- client/ayon_core/tools/sceneinventory/view.py | 29 ++++--------------- 3 files changed, 12 insertions(+), 34 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 03627e60b9..5f9e6bb77c 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -53,7 +53,7 @@ class InventoryModel(QtGui.QStandardItemModel): "Object name", "Active site", "Remote site", - "Project" + "Project", ] name_col = column_labels.index("Name") version_col = column_labels.index("Version") @@ -142,10 +142,7 @@ class InventoryModel(QtGui.QStandardItemModel): # and container_item.item_id not in selected # ): # continue - project_name = ( - container_item.project_name or - self._controller.get_current_project_name() - ) + project_name = container_item.project_name repre_id = container_item.representation_id items_by_repre_id[repre_id].append(container_item) repre_ids_by_project[project_name].add(repre_id) @@ -166,10 +163,7 @@ class InventoryModel(QtGui.QStandardItemModel): project_products = collections.defaultdict(set) for container_item in container_items: representation_id = container_item.representation_id - project_name = ( - container_item.project_name or - self._controller.get_current_project_name() - ) + project_name = container_item.project_name repre_info = repre_info_by_id.get(representation_id) if repre_info and repre_info.is_valid: product_id = repre_info.product_id diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index b8b9aa400a..c12a05fd99 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -5,6 +5,7 @@ import ayon_api from ayon_api.graphql import GraphQlQuery from ayon_core.host import ILoadHost +from ayon_core.pipeline import get_current_project_name from ayon_core.tools.common_models.projects import StatusStates @@ -111,7 +112,9 @@ class ContainerItem: namespace=container["namespace"], object_name=container["objectName"], item_id=uuid.uuid4().hex, - project_name=container.get("project_name", None) + project_name=container.get( + "project_name", get_current_project_name() + ) ) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index 1e0b96570e..a049fd1e0b 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -195,10 +195,7 @@ class SceneInventoryView(QtWidgets.QTreeView): repre_ids_by_project = collections.defaultdict(set) for container_item in container_items_by_id.values(): repre_id = container_item.representation_id - project_name = ( - container_item.project_name or - self._controller.get_current_project_name() - ) + project_name = container_item.project_name repre_ids_by_project[project_name].add(repre_id) repre_info_by_id = {} for project_name, repre_ids in repre_ids_by_project.items(): @@ -219,10 +216,7 @@ class SceneInventoryView(QtWidgets.QTreeView): version_ids = set() for container_item in container_items_by_id.values(): repre_id = container_item.representation_id - project_name = ( - container_item.project_name or - self._controller.get_current_project_name() - ) + project_name = container_item.project_name repre_info = repre_info_by_id.get(repre_id) if repre_info and repre_info.is_valid: filtered_items.append(container_item) @@ -754,10 +748,7 @@ class SceneInventoryView(QtWidgets.QTreeView): repre_ids_by_project = collections.defaultdict(set) for container_item in container_items_by_id.values(): repre_id = container_item.representation_id - project_name = ( - container_item.project_name or - self._controller.get_current_project_name() - ) + project_name = container_item.project_name repre_ids_by_project[project_name].add(repre_id) repre_info_by_id = {} for project_name, repre_ids in repre_ids_by_project.items(): @@ -773,10 +764,7 @@ class SceneInventoryView(QtWidgets.QTreeView): product_ids_by_project = collections.defaultdict(set) for container_item in container_items_by_id.values(): repre_id = container_item.representation_id - project_name = ( - container_item.project_name or - self._controller.get_current_project_name() - ) + project_name = container_item.project_name repre_info = repre_info_by_id.get(repre_id) if not repre_info or not repre_info.is_valid: continue @@ -972,10 +960,6 @@ class SceneInventoryView(QtWidgets.QTreeView): repre_ids_by_project = collections.defaultdict(set) for container_item in containers_items_by_id.values(): project_name = container_item.project_name - project_name = ( - container_item.project_name or - self._controller.get_current_project_name() - ) repre_id = container_item.representation_id repre_ids_by_project[project_name].add(repre_id) @@ -989,10 +973,7 @@ class SceneInventoryView(QtWidgets.QTreeView): product_ids_by_project = collections.defaultdict(set) for container_item in containers_items_by_id.values(): repre_id = container_item.representation_id - project_name = ( - container_item.project_name or - self._controller.get_current_project_name() - ) + project_name = container_item.project_name repre_info = repre_info_by_id.get(repre_id) if not repre_info or not repre_info.is_valid: continue From 004e9626ee5c9e3899bf72fee0af4c88e2b75b8a Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Tue, 19 Nov 2024 00:19:48 +0800 Subject: [PATCH 23/90] big roy comment - refactoring the dict per repre_id per project --- client/ayon_core/tools/sceneinventory/model.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 5f9e6bb77c..7630e9ee45 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -135,7 +135,8 @@ class InventoryModel(QtGui.QStandardItemModel): self._clear_items() items_by_repre_id = collections.defaultdict(list) - repre_ids_by_project = collections.defaultdict(set) + item_by_repre_id_by_project_id = collections.defaultdict( + lambda: collections.defaultdict(set)) for container_item in container_items: # if ( # selected is not None @@ -145,20 +146,17 @@ class InventoryModel(QtGui.QStandardItemModel): project_name = container_item.project_name repre_id = container_item.representation_id items_by_repre_id[repre_id].append(container_item) - repre_ids_by_project[project_name].add(repre_id) + item_by_repre_id_by_project_id[project_name][repre_id].add(container_item) - repre_id = set(items_by_repre_id.keys()) repre_info_by_id = {} - for project_name, repre_ids in repre_ids_by_project.items(): + repre_id = set() + for project_name, repre_ids in item_by_repre_id_by_project_id.items(): + repre_ids = set(items_by_repre_id.keys()) repre_info = self._controller.get_representation_info_items( project_name, repre_ids ) repre_info_by_id.update(repre_info) - product_ids = { - repre_info.product_id - for repre_info in repre_info_by_id.values() - if repre_info.is_valid - } + repre_id.update(repre_ids) project_products = collections.defaultdict(set) for container_item in container_items: From f003d8af1d654460ab802a7b1a604f092a6d163d Mon Sep 17 00:00:00 2001 From: Robin De Lillo Date: Mon, 18 Nov 2024 15:29:40 -0500 Subject: [PATCH 24/90] Apply suggestions from code review Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/publish/lib.py | 28 +++++++++--------------- client/ayon_core/pipeline/stagingdir.py | 15 ++++++++----- client/ayon_core/pipeline/tempdir.py | 4 +--- 3 files changed, 21 insertions(+), 26 deletions(-) diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 67a65aec09..3fad15f1a2 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -687,33 +687,25 @@ def get_instance_staging_dir(instance): anatomy_data = instance.data["anatomyData"] template_data = copy.deepcopy(anatomy_data) - product_type = instance.data["productType"] - product_name = instance.data["productName"] - # context data based variables - project_entity = instance.context.data["projectEntity"] - folder_entity = instance.context.data["folderEntity"] - task_entity = instance.context.data["taskEntity"] - host_name = instance.context.data["hostName"] - project_settings = instance.context.data["project_settings"] - anatomy = instance.context.data["anatomy"] - current_file = instance.context.data.get("currentFile") + context = instance.context # add current file as workfile name into formatting data + current_file = context.data.get("currentFile") if current_file: workfile = os.path.basename(current_file) workfile_name, _ = os.path.splitext(workfile) template_data["workfile_name"] = workfile_name staging_dir_info = get_staging_dir_info( - host_name, - project_entity, - folder_entity, - task_entity, - product_type, - product_name, - anatomy, - project_settings=project_settings, + context.data["hostName"], + context.data["projectEntity"], + instance.data.get("folderEntity"), + instance.data.get("taskEntity"), + instance.data["productType"], + instance.data["productName"], + anatomy=context.data["anatomy"], + project_settings=context.data["project_settings"], template_data=template_data, ) diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/stagingdir.py index 818acef36a..1c658ac817 100644 --- a/client/ayon_core/pipeline/stagingdir.py +++ b/client/ayon_core/pipeline/stagingdir.py @@ -9,12 +9,12 @@ STAGING_DIR_TEMPLATES = "staging" def get_staging_dir_config( - host_name, project_name, task_type, task_name, product_type, product_name, + host_name, project_settings=None, anatomy=None, log=None, @@ -24,8 +24,8 @@ def get_staging_dir_config( Args: host_name (str): Name of host. project_name (str): Name of project. - task_type (str): Type of task. - task_name (str): Name of task. + task_type (Optional[str]): Type of task. + task_name (Optional[str]): Name of task. product_type (str): Type of product. product_name (str): Name of product. project_settings(Dict[str, Any]): Prepared project settings. @@ -103,13 +103,13 @@ def _validate_template_name(project_name, template_name, anatomy): def get_staging_dir_info( - host_name, project_entity, folder_entity, task_entity, product_type, product_name, - anatomy, + host_name, + anatomy=None, project_settings=None, template_data=None, always_return_path=None, @@ -157,6 +157,11 @@ def get_staging_dir_info( if always_return_path is None: always_return_path = True + if anatomy is None: + anatomy = Anatomy( + project_entity["name"], project_entity=project_entity + ) + if force_tmp_dir: return get_temp_dir( project_name=project_entity["name"], diff --git a/client/ayon_core/pipeline/tempdir.py b/client/ayon_core/pipeline/tempdir.py index 8a9334ecc2..b5f4a31ee7 100644 --- a/client/ayon_core/pipeline/tempdir.py +++ b/client/ayon_core/pipeline/tempdir.py @@ -86,9 +86,7 @@ def _create_custom_tempdir(project_name, anatomy=None): Returns: str | None: formatted path or None """ - env_tmpdir = os.getenv( - "AYON_TMPDIR", - ) + env_tmpdir = os.getenv("AYON_TMPDIR") if not env_tmpdir: return From e96133b34892bf0529f0bcc81e56822d5177ffbc Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Mon, 18 Nov 2024 16:34:31 -0500 Subject: [PATCH 25/90] Address feedback from PR. --- .../pipeline/create/creator_plugins.py | 2 +- client/ayon_core/pipeline/publish/lib.py | 1 + client/ayon_core/pipeline/stagingdir.py | 54 +++++++++---------- client/ayon_core/pipeline/tempdir.py | 11 ++-- 4 files changed, 31 insertions(+), 37 deletions(-) diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 4cbf432efd..667f70c27d 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -873,7 +873,7 @@ class Creator(BaseCreator): ) if not staging_dir_info: - return + return None staging_dir_path = staging_dir_info["stagingDir"] diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 3fad15f1a2..8d56deec04 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -707,6 +707,7 @@ def get_instance_staging_dir(instance): anatomy=context.data["anatomy"], project_settings=context.data["project_settings"], template_data=template_data, + always_return_path=True, ) staging_dir_path = staging_dir_info["stagingDir"] diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/stagingdir.py index 1c658ac817..07ef122337 100644 --- a/client/ayon_core/pipeline/stagingdir.py +++ b/client/ayon_core/pipeline/stagingdir.py @@ -77,7 +77,7 @@ def get_staging_dir_config( # template should always be found either from anatomy or from profile raise ValueError( "Staging dir profile is misconfigured! " - "No template was found for profile! " + f"No template was found for profile: {profile}! " "Check your project settings at: " "'ayon+settings://core/tools/publish/custom_staging_dir_profiles'" ) @@ -112,10 +112,11 @@ def get_staging_dir_info( anatomy=None, project_settings=None, template_data=None, - always_return_path=None, - force_tmp_dir=None, + always_return_path=True, + force_tmp_dir=False, logger=None, - **kwargs + prefix=None, + suffix=None, ): """Get staging dir info data. @@ -141,11 +142,8 @@ def get_staging_dir_info( force_tmp_dir (Optional[bool]): If True, staging dir will be created as tempdir. logger (Optional[logging.Logger]): Logger instance. - **kwargs: Arbitrary keyword arguments. See below. - - Keyword Arguments: - prefix (str): Prefix for staging dir. - suffix (str): Suffix for staging dir. + prefix (Optional[str]) Optional prefix for staging dir name. + suffix (Optional[str]): Optional suffix for staging dir name. Returns: Optional[Dict[str, Any]]: Staging dir info data @@ -153,10 +151,6 @@ def get_staging_dir_info( """ log = logger or Logger.get_logger("get_staging_dir_info") - # make sure always_return_path is set to true by default - if always_return_path is None: - always_return_path = True - if anatomy is None: anatomy = Anatomy( project_entity["name"], project_entity=project_entity @@ -166,11 +160,11 @@ def get_staging_dir_info( return get_temp_dir( project_name=project_entity["name"], anatomy=anatomy, - prefix=kwargs.get("prefix"), - suffix=kwargs.get("suffix"), + prefix=prefix, + suffix=suffix, ) - # making fewer queries to database + # making few queries to database ctx_data = get_template_data( project_entity, folder_entity, task_entity, host_name ) @@ -192,8 +186,8 @@ def get_staging_dir_info( staging_dir_config = get_staging_dir_config( host_name, project_entity["name"], - task_entity["type"], - task_entity["name"], + task_entity.get("type"), + task_entity.get("name"), product_type, product_name, project_settings=project_settings, @@ -201,19 +195,19 @@ def get_staging_dir_info( log=log, ) - # if no preset matching and always_get_some_dir is set, return tempdir - if not staging_dir_config and always_return_path: - return { - "stagingDir": get_temp_dir( - project_name=project_entity["name"], - anatomy=anatomy, - prefix=kwargs.get("prefix"), - suffix=kwargs.get("suffix"), - ), - "stagingDir_persistent": False, - } if not staging_dir_config: - return None + if always_return_path: # no config found but force an output + return { + "stagingDir": get_temp_dir( + project_name=project_entity["name"], + anatomy=anatomy, + prefix=kwargs.get("prefix"), + suffix=kwargs.get("suffix"), + ), + "stagingDir_persistent": False, + } + else: + return None return { "stagingDir": StringTemplate.format_template( diff --git a/client/ayon_core/pipeline/tempdir.py b/client/ayon_core/pipeline/tempdir.py index b5f4a31ee7..af2ff44a8f 100644 --- a/client/ayon_core/pipeline/tempdir.py +++ b/client/ayon_core/pipeline/tempdir.py @@ -5,6 +5,7 @@ Temporary folder operations import os import tempfile from pathlib import Path + from ayon_core.lib import StringTemplate from ayon_core.pipeline import Anatomy @@ -48,7 +49,7 @@ def get_temp_dir( # get customized tempdir path from `OPENPYPE_TMPDIR` env var custom_temp_dir = _create_custom_tempdir(anatomy.project_name, anatomy) - return _create_local_staging_dir(prefix, suffix, custom_temp_dir) + return _create_local_staging_dir(prefix, suffix, dirpath=custom_temp_dir) def _create_local_staging_dir(prefix, suffix, dirpath=None): @@ -70,7 +71,7 @@ def _create_local_staging_dir(prefix, suffix, dirpath=None): return staging_dir.as_posix() -def _create_custom_tempdir(project_name, anatomy=None): +def _create_custom_tempdir(project_name, anatomy): """ Create custom tempdir Template path formatting is supporting: @@ -81,19 +82,17 @@ def _create_custom_tempdir(project_name, anatomy=None): Args: project_name (str): project name - anatomy (ayon_core.pipeline.Anatomy)[optional]: Anatomy object + anatomy (ayon_core.pipeline.Anatomy): Anatomy object Returns: str | None: formatted path or None """ env_tmpdir = os.getenv("AYON_TMPDIR") if not env_tmpdir: - return + return None custom_tempdir = None if "{" in env_tmpdir: - if anatomy is None: - anatomy = Anatomy(project_name) # create base formate data template_data = { "root": anatomy.roots, From 7375538f22d84b8a4493fc84f77cfd4d12cd8fbc Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Mon, 18 Nov 2024 16:43:31 -0500 Subject: [PATCH 26/90] Fix lint. --- client/ayon_core/pipeline/stagingdir.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/stagingdir.py index 07ef122337..2dd5c2f3eb 100644 --- a/client/ayon_core/pipeline/stagingdir.py +++ b/client/ayon_core/pipeline/stagingdir.py @@ -201,8 +201,8 @@ def get_staging_dir_info( "stagingDir": get_temp_dir( project_name=project_entity["name"], anatomy=anatomy, - prefix=kwargs.get("prefix"), - suffix=kwargs.get("suffix"), + prefix=prefix, + suffix=suffix, ), "stagingDir_persistent": False, } From 3078ba2a239c9780bf784d07849d42c2e02ea0c5 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Tue, 19 Nov 2024 15:45:27 +0800 Subject: [PATCH 27/90] kuba's comment - add current project name as argument in the get_container_data and make sure the scene inventory won't show Entity N/A as name and version --- client/ayon_core/tools/sceneinventory/control.py | 2 +- client/ayon_core/tools/sceneinventory/model.py | 16 +++++++++------- .../tools/sceneinventory/models/containers.py | 8 ++++---- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/control.py b/client/ayon_core/tools/sceneinventory/control.py index 310e41b117..8c02881b82 100644 --- a/client/ayon_core/tools/sceneinventory/control.py +++ b/client/ayon_core/tools/sceneinventory/control.py @@ -4,7 +4,7 @@ from ayon_core.lib.events import QueuedEventSystem from ayon_core.host import HostBase from ayon_core.pipeline import ( registered_host, - get_current_context + get_current_context, ) from ayon_core.tools.common_models import HierarchyModel, ProjectsModel diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 7630e9ee45..5f9e6bb77c 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -135,8 +135,7 @@ class InventoryModel(QtGui.QStandardItemModel): self._clear_items() items_by_repre_id = collections.defaultdict(list) - item_by_repre_id_by_project_id = collections.defaultdict( - lambda: collections.defaultdict(set)) + repre_ids_by_project = collections.defaultdict(set) for container_item in container_items: # if ( # selected is not None @@ -146,17 +145,20 @@ class InventoryModel(QtGui.QStandardItemModel): project_name = container_item.project_name repre_id = container_item.representation_id items_by_repre_id[repre_id].append(container_item) - item_by_repre_id_by_project_id[project_name][repre_id].add(container_item) + repre_ids_by_project[project_name].add(repre_id) + repre_id = set(items_by_repre_id.keys()) repre_info_by_id = {} - repre_id = set() - for project_name, repre_ids in item_by_repre_id_by_project_id.items(): - repre_ids = set(items_by_repre_id.keys()) + for project_name, repre_ids in repre_ids_by_project.items(): repre_info = self._controller.get_representation_info_items( project_name, repre_ids ) repre_info_by_id.update(repre_info) - repre_id.update(repre_ids) + product_ids = { + repre_info.product_id + for repre_info in repre_info_by_id.values() + if repre_info.is_valid + } project_products = collections.defaultdict(set) for container_item in container_items: diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index c12a05fd99..e135cb0031 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -5,7 +5,6 @@ import ayon_api from ayon_api.graphql import GraphQlQuery from ayon_core.host import ILoadHost -from ayon_core.pipeline import get_current_project_name from ayon_core.tools.common_models.projects import StatusStates @@ -105,7 +104,7 @@ class ContainerItem: self.project_name = project_name @classmethod - def from_container_data(cls, container): + def from_container_data(cls, current_project_name, container): return cls( representation_id=container["representation"], loader_name=container["loader"], @@ -113,7 +112,7 @@ class ContainerItem: object_name=container["objectName"], item_id=uuid.uuid4().hex, project_name=container.get( - "project_name", get_current_project_name() + "project_name", current_project_name ) ) @@ -368,7 +367,8 @@ class ContainersModel: invalid_ids_mapping = {} for container in containers: try: - item = ContainerItem.from_container_data(container) + current_project_name = self._controller.get_current_project_name() + item = ContainerItem.from_container_data(current_project_name, container) repre_id = item.representation_id try: uuid.UUID(repre_id) From b8ba7f47b0dee929fd1259e49fc791fa13ec3ed3 Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Tue, 19 Nov 2024 16:49:08 -0500 Subject: [PATCH 28/90] Fix staging data computation. --- client/ayon_core/pipeline/create/creator_plugins.py | 8 +++++--- client/ayon_core/pipeline/publish/lib.py | 4 ++-- client/ayon_core/pipeline/stagingdir.py | 6 +++--- .../plugins/publish/collect_managed_staging_dir.py | 4 ++++ client/ayon_core/plugins/publish/extract_burnin.py | 2 +- .../ayon_core/plugins/publish/extract_color_transcode.py | 2 +- client/ayon_core/plugins/publish/extract_review.py | 2 +- 7 files changed, 17 insertions(+), 11 deletions(-) diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 667f70c27d..93e1f6f5cb 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -857,16 +857,18 @@ class Creator(BaseCreator): version = instance.get("version") if version is not None: template_data = {"version": version} + else: + template_data = {} staging_dir_info = get_staging_dir_info( - create_ctx.host_name, create_ctx.get_current_project_entity(), create_ctx.get_current_folder_entity(), create_ctx.get_current_task_entity(), product_type, product_name, - create_ctx.get_current_project_anatomy(), - create_ctx.get_current_project_settings(), + create_ctx.host_name, + anatomy=create_ctx.get_current_project_anatomy(), + project_settings=create_ctx.get_current_project_settings(), always_return_path=False, logger=self.log, template_data=template_data, diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 8d56deec04..4c36f473d1 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -653,12 +653,12 @@ def get_custom_staging_dir_info( DeprecationWarning, ) tr_data = get_staging_dir_config( - host_name, project_name, task_type, task_name, product_type, product_name, + host_name, project_settings=project_settings, anatomy=anatomy, log=log, @@ -698,12 +698,12 @@ def get_instance_staging_dir(instance): template_data["workfile_name"] = workfile_name staging_dir_info = get_staging_dir_info( - context.data["hostName"], context.data["projectEntity"], instance.data.get("folderEntity"), instance.data.get("taskEntity"), instance.data["productType"], instance.data["productName"], + context.data["hostName"], anatomy=context.data["anatomy"], project_settings=context.data["project_settings"], template_data=template_data, diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/stagingdir.py index 2dd5c2f3eb..c7cc95ff55 100644 --- a/client/ayon_core/pipeline/stagingdir.py +++ b/client/ayon_core/pipeline/stagingdir.py @@ -184,12 +184,12 @@ def get_staging_dir_info( # get staging dir config staging_dir_config = get_staging_dir_config( - host_name, project_entity["name"], - task_entity.get("type"), + task_entity.get("taskType"), task_entity.get("name"), product_type, product_name, + host_name, project_settings=project_settings, anatomy=anatomy, log=log, @@ -211,7 +211,7 @@ def get_staging_dir_info( return { "stagingDir": StringTemplate.format_template( - staging_dir_config["template"], ctx_data + staging_dir_config["template"]["directory"], ctx_data ), "stagingDir_persistent": staging_dir_config["persistence"], } diff --git a/client/ayon_core/plugins/publish/collect_managed_staging_dir.py b/client/ayon_core/plugins/publish/collect_managed_staging_dir.py index ca6d5161c1..1034b9a716 100644 --- a/client/ayon_core/plugins/publish/collect_managed_staging_dir.py +++ b/client/ayon_core/plugins/publish/collect_managed_staging_dir.py @@ -33,7 +33,11 @@ class CollectManagedStagingDir(pyblish.api.InstancePlugin): order = pyblish.api.CollectorOrder + 0.4990 def process(self, instance): + """ Collect the staging data and stores it to the instance. + Args: + instance (object): The instance to inspect. + """ staging_dir_path = get_instance_staging_dir(instance) persistance = instance.data.get("stagingDir_persistent", False) diff --git a/client/ayon_core/plugins/publish/extract_burnin.py b/client/ayon_core/plugins/publish/extract_burnin.py index 3eb4254a5e..8e8764fc33 100644 --- a/client/ayon_core/plugins/publish/extract_burnin.py +++ b/client/ayon_core/plugins/publish/extract_burnin.py @@ -254,7 +254,7 @@ class ExtractBurnin(publish.Extractor): if do_convert: new_staging_dir = get_temp_dir( project_name=instance.context.data["projectName"], - make_local=True, + use_local_temp=True, ) repre["stagingDir"] = new_staging_dir diff --git a/client/ayon_core/plugins/publish/extract_color_transcode.py b/client/ayon_core/plugins/publish/extract_color_transcode.py index 4f0053c426..3c11a016ec 100644 --- a/client/ayon_core/plugins/publish/extract_color_transcode.py +++ b/client/ayon_core/plugins/publish/extract_color_transcode.py @@ -106,7 +106,7 @@ class ExtractOIIOTranscode(publish.Extractor): original_staging_dir = new_repre["stagingDir"] new_staging_dir = get_temp_dir( project_name=instance.context.data["projectName"], - make_local=True, + use_local_temp=True, ) new_repre["stagingDir"] = new_staging_dir diff --git a/client/ayon_core/plugins/publish/extract_review.py b/client/ayon_core/plugins/publish/extract_review.py index 26cd2ef0b2..7c38b0453b 100644 --- a/client/ayon_core/plugins/publish/extract_review.py +++ b/client/ayon_core/plugins/publish/extract_review.py @@ -312,7 +312,7 @@ class ExtractReview(pyblish.api.InstancePlugin): if do_convert: new_staging_dir = get_temp_dir( project_name=instance.context.data["projectName"], - make_local=True, + use_local_temp=True, ) repre["stagingDir"] = new_staging_dir From d9c1a299b97ff8a4c5c6612a2644d41e7d564a40 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Thu, 21 Nov 2024 17:23:36 +0800 Subject: [PATCH 29/90] loading the asset per repre_id and per project --- .../ayon_core/tools/sceneinventory/model.py | 246 +++++++++--------- .../tools/sceneinventory/models/containers.py | 22 +- 2 files changed, 127 insertions(+), 141 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 5f9e6bb77c..29818e387f 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -131,50 +131,39 @@ class InventoryModel(QtGui.QStandardItemModel): """Refresh the model""" # for debugging or testing, injecting items from outside container_items = self._controller.get_container_items() - self._clear_items() - - items_by_repre_id = collections.defaultdict(list) - repre_ids_by_project = collections.defaultdict(set) - for container_item in container_items: + repre_id = set() + repre_id_by_project_id = collections.defaultdict(set) + version_items_by_product_id = collections.defaultdict(dict) + repre_info_by_id_by_project = collections.defaultdict(list) + item_by_repre_id_by_project_id = collections.defaultdict( + lambda: collections.defaultdict(set)) + for project_name, container_item in container_items.items(): # if ( # selected is not None # and container_item.item_id not in selected # ): # continue - project_name = container_item.project_name - repre_id = container_item.representation_id - items_by_repre_id[repre_id].append(container_item) - repre_ids_by_project[project_name].add(repre_id) - - repre_id = set(items_by_repre_id.keys()) - repre_info_by_id = {} - for project_name, repre_ids in repre_ids_by_project.items(): + for item in container_item.values(): + representation_id = item.representation_id + if item.project_name != project_name: + continue + repre_id.add(representation_id) + item_by_repre_id_by_project_id[project_name][representation_id].add(item) repre_info = self._controller.get_representation_info_items( - project_name, repre_ids + project_name, repre_id ) - repre_info_by_id.update(repre_info) - product_ids = { - repre_info.product_id - for repre_info in repre_info_by_id.values() - if repre_info.is_valid - } + repre_info_by_id_by_project[project_name] = repre_info - project_products = collections.defaultdict(set) - for container_item in container_items: - representation_id = container_item.representation_id - project_name = container_item.project_name - repre_info = repre_info_by_id.get(representation_id) - if repre_info and repre_info.is_valid: - product_id = repre_info.product_id - project_products[project_name].add(product_id) - - version_items_by_product_id = {} - for project_name, product_ids in project_products.items(): + product_ids = { + repre_info.product_id + for repre_info in repre_info.values() + if repre_info.is_valid + } version_items = self._controller.get_version_items( project_name, product_ids ) - version_items_by_product_id.update(version_items) + version_items_by_product_id[project_name] = version_items # SiteSync addon information progress_by_id = self._controller.get_representations_site_progress( @@ -216,112 +205,113 @@ class InventoryModel(QtGui.QStandardItemModel): root_item = self.invisibleRootItem() group_items = [] - for repre_id, container_items in items_by_repre_id.items(): - repre_info = repre_info_by_id[repre_id] - version_color = None - if not repre_info.is_valid: - version_label = "N/A" - group_name = "< Entity N/A >" - item_icon = invalid_item_icon - is_latest = False - is_hero = False - status_name = None + for project_name, items_by_repre_id in item_by_repre_id_by_project_id.items(): + for repre_id, container_items in items_by_repre_id.items(): + repre_info = repre_info_by_id_by_project[project_name][repre_id] + version_color = None + if not repre_info.is_valid: + version_label = "N/A" + group_name = "< Entity N/A >" + item_icon = invalid_item_icon + is_latest = False + is_hero = False + status_name = None - else: - group_name = "{}_{}: ({})".format( - repre_info.folder_path.rsplit("/")[-1], - repre_info.product_name, - repre_info.representation_name + else: + group_name = "{}_{}: ({})".format( + repre_info.folder_path.rsplit("/")[-1], + repre_info.product_name, + repre_info.representation_name + ) + item_icon = valid_item_icon + + version_items = ( + version_items_by_product_id[project_name][repre_info.product_id] + ) + version_item = version_items[repre_info.version_id] + version_label = format_version(version_item.version) + is_hero = version_item.version < 0 + is_latest = version_item.is_latest + if not version_item.is_latest: + version_color = self.OUTDATED_COLOR + status_name = version_item.status + + status_color, status_short, status_icon = self._get_status_data( + status_name ) - item_icon = valid_item_icon - version_items = ( - version_items_by_product_id[repre_info.product_id] + repre_name = ( + repre_info.representation_name or "" ) - version_item = version_items[repre_info.version_id] - version_label = format_version(version_item.version) - is_hero = version_item.version < 0 - is_latest = version_item.is_latest - if not version_item.is_latest: - version_color = self.OUTDATED_COLOR - status_name = version_item.status + container_model_items = [] + for container_item in container_items: + object_name = container_item.object_name or "" + unique_name = repre_name + object_name + item = QtGui.QStandardItem() + item.setColumnCount(root_item.columnCount()) + item.setData(container_item.namespace, QtCore.Qt.DisplayRole) + item.setData(self.GRAYOUT_COLOR, NAME_COLOR_ROLE) + item.setData(self.GRAYOUT_COLOR, VERSION_COLOR_ROLE) + item.setData(item_icon, QtCore.Qt.DecorationRole) + item.setData(repre_info.product_id, PRODUCT_ID_ROLE) + item.setData(container_item.item_id, ITEM_ID_ROLE) + item.setData(version_label, VERSION_LABEL_ROLE) + item.setData(container_item.loader_name, LOADER_NAME_ROLE) + item.setData(container_item.object_name, OBJECT_NAME_ROLE) + item.setData(container_item.project_name, PROJECT_NAME_ROLE) + item.setData(True, IS_CONTAINER_ITEM_ROLE) + item.setData(unique_name, ITEM_UNIQUE_NAME_ROLE) + container_model_items.append(item) + if not container_model_items: + continue - status_color, status_short, status_icon = self._get_status_data( - status_name - ) + progress = progress_by_id[repre_id] + active_site_progress = "{}%".format( + max(progress["active_site"], 0) * 100 + ) + remote_site_progress = "{}%".format( + max(progress["remote_site"], 0) * 100 + ) - repre_name = ( - repre_info.representation_name or "" - ) - container_model_items = [] - for container_item in container_items: - object_name = container_item.object_name or "" - unique_name = repre_name + object_name - item = QtGui.QStandardItem() - item.setColumnCount(root_item.columnCount()) - item.setData(container_item.namespace, QtCore.Qt.DisplayRole) - item.setData(self.GRAYOUT_COLOR, NAME_COLOR_ROLE) - item.setData(self.GRAYOUT_COLOR, VERSION_COLOR_ROLE) - item.setData(item_icon, QtCore.Qt.DecorationRole) - item.setData(repre_info.product_id, PRODUCT_ID_ROLE) - item.setData(container_item.item_id, ITEM_ID_ROLE) - item.setData(version_label, VERSION_LABEL_ROLE) - item.setData(container_item.loader_name, LOADER_NAME_ROLE) - item.setData(container_item.object_name, OBJECT_NAME_ROLE) - item.setData(container_item.project_name, PROJECT_NAME_ROLE) - item.setData(True, IS_CONTAINER_ITEM_ROLE) - item.setData(unique_name, ITEM_UNIQUE_NAME_ROLE) - container_model_items.append(item) - if not container_model_items: - continue + group_item = QtGui.QStandardItem() + group_item.setColumnCount(root_item.columnCount()) + group_item.setData(group_name, QtCore.Qt.DisplayRole) + group_item.setData(group_name, ITEM_UNIQUE_NAME_ROLE) + group_item.setData(group_item_icon, QtCore.Qt.DecorationRole) + group_item.setData(group_item_font, QtCore.Qt.FontRole) + group_item.setData(repre_info.product_id, PRODUCT_ID_ROLE) + group_item.setData(repre_info.product_type, PRODUCT_TYPE_ROLE) + group_item.setData(product_type_icon, PRODUCT_TYPE_ICON_ROLE) + group_item.setData(is_latest, VERSION_IS_LATEST_ROLE) + group_item.setData(is_hero, VERSION_IS_HERO_ROLE) + group_item.setData(version_label, VERSION_LABEL_ROLE) + group_item.setData(len(container_items), COUNT_ROLE) + group_item.setData(status_name, STATUS_NAME_ROLE) + group_item.setData(status_short, STATUS_SHORT_ROLE) + group_item.setData(status_color, STATUS_COLOR_ROLE) + group_item.setData(status_icon, STATUS_ICON_ROLE) - progress = progress_by_id[repre_id] - active_site_progress = "{}%".format( - max(progress["active_site"], 0) * 100 - ) - remote_site_progress = "{}%".format( - max(progress["remote_site"], 0) * 100 - ) - - group_item = QtGui.QStandardItem() - group_item.setColumnCount(root_item.columnCount()) - group_item.setData(group_name, QtCore.Qt.DisplayRole) - group_item.setData(group_name, ITEM_UNIQUE_NAME_ROLE) - group_item.setData(group_item_icon, QtCore.Qt.DecorationRole) - group_item.setData(group_item_font, QtCore.Qt.FontRole) - group_item.setData(repre_info.product_id, PRODUCT_ID_ROLE) - group_item.setData(repre_info.product_type, PRODUCT_TYPE_ROLE) - group_item.setData(product_type_icon, PRODUCT_TYPE_ICON_ROLE) - group_item.setData(is_latest, VERSION_IS_LATEST_ROLE) - group_item.setData(is_hero, VERSION_IS_HERO_ROLE) - group_item.setData(version_label, VERSION_LABEL_ROLE) - group_item.setData(len(container_items), COUNT_ROLE) - group_item.setData(status_name, STATUS_NAME_ROLE) - group_item.setData(status_short, STATUS_SHORT_ROLE) - group_item.setData(status_color, STATUS_COLOR_ROLE) - group_item.setData(status_icon, STATUS_ICON_ROLE) - - group_item.setData( - active_site_progress, ACTIVE_SITE_PROGRESS_ROLE - ) - group_item.setData( - remote_site_progress, REMOTE_SITE_PROGRESS_ROLE - ) - group_item.setData(active_site_icon, ACTIVE_SITE_ICON_ROLE) - group_item.setData(remote_site_icon, REMOTE_SITE_ICON_ROLE) - group_item.setData(False, IS_CONTAINER_ITEM_ROLE) - - if version_color is not None: - group_item.setData(version_color, VERSION_COLOR_ROLE) - - if repre_info.product_group: group_item.setData( - repre_info.product_group, PRODUCT_GROUP_NAME_ROLE + active_site_progress, ACTIVE_SITE_PROGRESS_ROLE ) - group_item.setData(group_icon, PRODUCT_GROUP_ICON_ROLE) + group_item.setData( + remote_site_progress, REMOTE_SITE_PROGRESS_ROLE + ) + group_item.setData(active_site_icon, ACTIVE_SITE_ICON_ROLE) + group_item.setData(remote_site_icon, REMOTE_SITE_ICON_ROLE) + group_item.setData(False, IS_CONTAINER_ITEM_ROLE) - group_item.appendRows(container_model_items) - group_items.append(group_item) + if version_color is not None: + group_item.setData(version_color, VERSION_COLOR_ROLE) + + if repre_info.product_group: + group_item.setData( + repre_info.product_group, PRODUCT_GROUP_NAME_ROLE + ) + group_item.setData(group_icon, PRODUCT_GROUP_ICON_ROLE) + + group_item.appendRows(container_model_items) + group_items.append(group_item) if group_items: root_item.appendRows(group_items) diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index e135cb0031..ca67fb59f9 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -191,7 +191,7 @@ class VersionItem: class ContainersModel: def __init__(self, controller): self._controller = controller - self._items_cache = None + self._project_cache = None self._containers_by_id = {} self._container_items_by_id = {} self._container_items_by_project = {} @@ -200,7 +200,7 @@ class ContainersModel: self._product_ids_by_project = {} def reset(self): - self._items_cache = None + self._project_cache = None self._containers_by_id = {} self._container_items_by_id = {} self._container_items_by_project = {} @@ -220,7 +220,7 @@ class ContainersModel: def get_container_items(self): self._update_cache() - return list(self._items_cache) + return self._project_cache def get_container_items_by_id(self, item_ids): return { @@ -248,7 +248,6 @@ class ContainersModel: repre_hierarchy_by_id = get_representations_hierarchy( project_name, missing_repre_ids ) - self._product_ids_by_project[project_name] = set() for repre_id, repre_hierarchy in repre_hierarchy_by_id.items(): kwargs = { "folder_id": None, @@ -280,8 +279,6 @@ class ContainersModel: repre_info = RepresentationInfo(**kwargs) self._repre_info_by_id[repre_id] = repre_info - self._product_ids_by_project[project_name].add( - repre_info.product_id) output[repre_id] = repre_info return output @@ -293,7 +290,6 @@ class ContainersModel: for product_id in product_ids if product_id not in self._version_items_by_product_id } - current_product_ids = self._product_ids_by_project.get(project_name) if missing_ids: status_items_by_name = { status_item.name: status_item @@ -302,14 +298,13 @@ class ContainersModel: def version_sorted(entity): return entity["version"] - current_missing_ids = current_product_ids.intersection(missing_ids) version_entities_by_product_id = { product_id: [] - for product_id in current_missing_ids + for product_id in missing_ids } version_entities = list(ayon_api.get_versions( project_name, - product_ids=current_missing_ids, + product_ids=missing_ids, fields={"id", "version", "productId", "status"} )) version_entities.sort(key=version_sorted) @@ -349,7 +344,7 @@ class ContainersModel: } def _update_cache(self): - if self._items_cache is not None: + if self._project_cache is not None: return host = self._controller.get_host() @@ -363,7 +358,7 @@ class ContainersModel: container_items = [] containers_by_id = {} container_items_by_id = {} - project_name_by_repre_id = {} + project_cache = collections.defaultdict(dict) invalid_ids_mapping = {} for container in containers: try: @@ -388,9 +383,10 @@ class ContainersModel: containers_by_id[item.item_id] = container container_items_by_id[item.item_id] = item - project_name_by_repre_id[item.representation_id] = item.project_name + project_cache[item.project_name] = container_items_by_id container_items.append(item) self._containers_by_id = containers_by_id self._container_items_by_id = container_items_by_id self._items_cache = container_items + self._project_cache = project_cache From 7a224914ea7dc59bcfa158cd6ead2404c4074a9e Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Thu, 21 Nov 2024 17:43:17 +0800 Subject: [PATCH 30/90] remove unused variable --- client/ayon_core/tools/sceneinventory/model.py | 1 - 1 file changed, 1 deletion(-) diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 29818e387f..640a8017ab 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -133,7 +133,6 @@ class InventoryModel(QtGui.QStandardItemModel): container_items = self._controller.get_container_items() self._clear_items() repre_id = set() - repre_id_by_project_id = collections.defaultdict(set) version_items_by_product_id = collections.defaultdict(dict) repre_info_by_id_by_project = collections.defaultdict(list) item_by_repre_id_by_project_id = collections.defaultdict( From a70135bb5156fc3883047561edcda702f81f8731 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Thu, 21 Nov 2024 22:35:25 +0800 Subject: [PATCH 31/90] implement switch and set version by repre_id/product_id per project --- .../ayon_core/tools/sceneinventory/model.py | 20 +- .../tools/sceneinventory/models/containers.py | 11 +- client/ayon_core/tools/sceneinventory/view.py | 190 +++++++++--------- 3 files changed, 113 insertions(+), 108 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 640a8017ab..75af957cfa 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -133,24 +133,26 @@ class InventoryModel(QtGui.QStandardItemModel): container_items = self._controller.get_container_items() self._clear_items() repre_id = set() + repre_ids_by_project = collections.defaultdict(set) version_items_by_product_id = collections.defaultdict(dict) - repre_info_by_id_by_project = collections.defaultdict(list) + repre_info_by_id_by_project = collections.defaultdict(dict) item_by_repre_id_by_project_id = collections.defaultdict( lambda: collections.defaultdict(set)) - for project_name, container_item in container_items.items(): + for container_item in container_items: # if ( # selected is not None # and container_item.item_id not in selected # ): # continue - for item in container_item.values(): - representation_id = item.representation_id - if item.project_name != project_name: - continue - repre_id.add(representation_id) - item_by_repre_id_by_project_id[project_name][representation_id].add(item) + project_name = container_item.project_name + representation_id = container_item.representation_id + repre_id.add(representation_id) + repre_ids_by_project[project_name].add(representation_id) + item_by_repre_id_by_project_id[project_name][representation_id].add(container_item) + + for project_name, representation_ids in repre_ids_by_project.items(): repre_info = self._controller.get_representation_info_items( - project_name, repre_id + project_name, representation_ids ) repre_info_by_id_by_project[project_name] = repre_info diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index ca67fb59f9..dc41bdc8fa 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -191,7 +191,7 @@ class VersionItem: class ContainersModel: def __init__(self, controller): self._controller = controller - self._project_cache = None + self._items_cache = None self._containers_by_id = {} self._container_items_by_id = {} self._container_items_by_project = {} @@ -200,7 +200,7 @@ class ContainersModel: self._product_ids_by_project = {} def reset(self): - self._project_cache = None + self._items_cache = None self._containers_by_id = {} self._container_items_by_id = {} self._container_items_by_project = {} @@ -220,7 +220,7 @@ class ContainersModel: def get_container_items(self): self._update_cache() - return self._project_cache + return list(self._items_cache) def get_container_items_by_id(self, item_ids): return { @@ -344,7 +344,7 @@ class ContainersModel: } def _update_cache(self): - if self._project_cache is not None: + if self._items_cache is not None: return host = self._controller.get_host() @@ -358,7 +358,6 @@ class ContainersModel: container_items = [] containers_by_id = {} container_items_by_id = {} - project_cache = collections.defaultdict(dict) invalid_ids_mapping = {} for container in containers: try: @@ -383,10 +382,8 @@ class ContainersModel: containers_by_id[item.item_id] = container container_items_by_id[item.item_id] = item - project_cache[item.project_name] = container_items_by_id container_items.append(item) self._containers_by_id = containers_by_id self._container_items_by_id = container_items_by_id self._items_cache = container_items - self._project_cache = project_cache diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index a049fd1e0b..24e0195e31 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -197,26 +197,29 @@ class SceneInventoryView(QtWidgets.QTreeView): repre_id = container_item.representation_id project_name = container_item.project_name repre_ids_by_project[project_name].add(repre_id) - repre_info_by_id = {} + + repre_info_by_project = collections.defaultdict(dict) for project_name, repre_ids in repre_ids_by_project.items(): repre_info = self._controller.get_representation_info_items( project_name, repre_ids) - repre_info_by_id.update(repre_info) - - valid_repre_ids = { - repre_id - for repre_id, repre_info in repre_info_by_id.items() - if repre_info.is_valid - } - + repre_info_by_project[project_name].update(repre_info) # Exclude items that are "NOT FOUND" since setting versions, updating # and removal won't work for those items. filtered_items = [] - product_ids_by_project = collections.defaultdict(set) version_ids = set() + valid_repre_ids = set() + product_ids_by_project = collections.defaultdict(set) for container_item in container_items_by_id.values(): - repre_id = container_item.representation_id project_name = container_item.project_name + repre_info_by_id = repre_info_by_project.get(project_name) + repre_id = container_item.representation_id + all_valid_repre_ids = { + repre_id + for repre_id, repre_info in repre_info_by_id.items() + if repre_info.is_valid + } + valid_repre_ids.update(all_valid_repre_ids) + repre_info = repre_info_by_id.get(repre_id) if repre_info and repre_info.is_valid: filtered_items.append(container_item) @@ -233,47 +236,47 @@ class SceneInventoryView(QtWidgets.QTreeView): # Keep remove action for invalid items menu.addAction(remove_action) return - version_items_by_product_id = {} + version_items_by_product_id_by_project = collections.defaultdict(dict) for project_name, product_ids in product_ids_by_project.items(): version_items = self._controller.get_version_items( project_name, product_ids ) - version_items_by_product_id.update(version_items -) + version_items_by_product_id_by_project[project_name] = version_items has_outdated = False has_loaded_hero_versions = False has_available_hero_version = False has_outdated_approved = False last_version_by_product_id = {} - for product_id, version_items_by_id in ( - version_items_by_product_id.items() - ): - _has_outdated_approved = False - _last_approved_version_item = None - for version_item in version_items_by_id.values(): - if version_item.is_hero: - has_available_hero_version = True - - elif version_item.is_last_approved: - _last_approved_version_item = version_item - _has_outdated_approved = True - - if version_item.version_id not in version_ids: - continue - - if version_item.is_hero: - has_loaded_hero_versions = True - elif not version_item.is_latest: - has_outdated = True - - if ( - _has_outdated_approved - and _last_approved_version_item is not None + for version_items_by_product_id in version_items_by_product_id_by_project.values(): + for product_id, version_items_by_id in ( + version_items_by_product_id.items() ): - last_version_by_product_id[product_id] = ( - _last_approved_version_item - ) - has_outdated_approved = True + _has_outdated_approved = False + _last_approved_version_item = None + for version_item in version_items_by_id.values(): + if version_item.is_hero: + has_available_hero_version = True + + elif version_item.is_last_approved: + _last_approved_version_item = version_item + _has_outdated_approved = True + + if version_item.version_id not in version_ids: + continue + + if version_item.is_hero: + has_loaded_hero_versions = True + elif not version_item.is_latest: + has_outdated = True + + if ( + _has_outdated_approved + and _last_approved_version_item is not None + ): + last_version_by_product_id[product_id] = ( + _last_approved_version_item + ) + has_outdated_approved = True switch_to_versioned = None if has_loaded_hero_versions: @@ -294,8 +297,9 @@ class SceneInventoryView(QtWidgets.QTreeView): approved_version_by_item_id = {} if has_outdated_approved: for container_item in container_items_by_id.values(): + project_name = container_item.project_name repre_id = container_item.representation_id - repre_info = repre_info_by_id.get(repre_id) + repre_info = repre_info_by_project[project_name][repre_id] if not repre_info or not repre_info.is_valid: continue version_item = last_version_by_product_id.get( @@ -750,54 +754,53 @@ class SceneInventoryView(QtWidgets.QTreeView): repre_id = container_item.representation_id project_name = container_item.project_name repre_ids_by_project[project_name].add(repre_id) - repre_info_by_id = {} + + versions = set() + repre_info_by_project = collections.defaultdict(dict) + version_items_by_product_id_by_project = collections.defaultdict(dict) for project_name, repre_ids in repre_ids_by_project.items(): repre_info = self._controller.get_representation_info_items( project_name, repre_ids ) - repre_info_by_id.update(repre_info) + repre_info_by_project[project_name].update(repre_info) - product_ids = { - repre_info.product_id - for repre_info in repre_info_by_id.values() - } - product_ids_by_project = collections.defaultdict(set) for container_item in container_items_by_id.values(): - repre_id = container_item.representation_id project_name = container_item.project_name + repre_info_by_id = repre_info_by_project.get(project_name) + repre_id = container_item.representation_id repre_info = repre_info_by_id.get(repre_id) - if not repre_info or not repre_info.is_valid: - continue - product_ids_by_project[project_name].add( + product_ids = { repre_info.product_id - ) - active_repre_info = repre_info_by_id[active_repre_id] - active_version_id = active_repre_info.version_id - active_product_id = active_repre_info.product_id - version_items_by_product_id = {} - for project_name, project_product_ids in product_ids_by_project.items(): + for repre_info in repre_info_by_id.values() + if repre_info.is_valid + } version_items = self._controller.get_version_items( - project_name, project_product_ids + project_name, product_ids ) - version_items_by_product_id.update(version_items) - version_items = list( - version_items_by_product_id[active_product_id].values() - ) - versions = {version_item.version for version_item in version_items} + version_items_by_product_id_by_project[project_name] = version_items + active_repre_info = repre_info_by_id[active_repre_id] + active_version_id = active_repre_info.version_id + active_product_id = active_repre_info.product_id + version_items = list( + version_items_by_product_id_by_project[project_name][active_product_id].values() + ) + all_versions = {version_item.version for version_item in version_items} + versions.update(all_versions) product_ids_by_version = collections.defaultdict(set) - for version_items_by_id in version_items_by_product_id.values(): - for version_item in version_items_by_id.values(): - version = version_item.version - _prod_version = version - if _prod_version < 0: - _prod_version = -1 - product_ids_by_version[_prod_version].add( - version_item.product_id - ) - if version in versions: - continue - versions.add(version) - version_items.append(version_item) + for version_items_by_product_id in version_items_by_product_id_by_project.values(): + for version_items_by_id in version_items_by_product_id.values(): + for version_item in version_items_by_id.values(): + version = version_item.version + _prod_version = version + if _prod_version < 0: + _prod_version = -1 + product_ids_by_version[_prod_version].add( + version_item.product_id + ) + if version in versions: + continue + versions.add(version) + version_items.append(version_item) def version_sorter(item): hero_value = 0 @@ -862,8 +865,9 @@ class SceneInventoryView(QtWidgets.QTreeView): filtered_item_ids = set() for container_item in container_items_by_id.values(): + project_name = container_item.project_name repre_id = container_item.representation_id - repre_info = repre_info_by_id[repre_id] + repre_info = repre_info_by_project[project_name][repre_id] if repre_info.product_id in product_ids: filtered_item_ids.add(container_item.item_id) @@ -964,37 +968,39 @@ class SceneInventoryView(QtWidgets.QTreeView): repre_ids_by_project[project_name].add(repre_id) # Get representation info items by ID - repre_info_by_id = {} + repre_info_by_project = collections.defaultdict(dict) for project_name, repre_ids in repre_ids_by_project.items(): repre_info = self._controller.get_representation_info_items( project_name, repre_ids) - repre_info_by_id.update(repre_info) + repre_info_by_project[project_name].update(repre_info) product_ids_by_project = collections.defaultdict(set) + version_items_by_product_id_by_project = collections.defaultdict(dict) for container_item in containers_items_by_id.values(): - repre_id = container_item.representation_id project_name = container_item.project_name + repre_info_by_id = repre_info_by_project.get(project_name) + repre_id = container_item.representation_id repre_info = repre_info_by_id.get(repre_id) - if not repre_info or not repre_info.is_valid: - continue - product_ids_by_project[project_name].add( + product_ids = { repre_info.product_id - ) - - version_items_by_product_id = {} - for project_name, product_ids in product_ids_by_project.items(): + for repre_info in repre_info_by_id.values() + if repre_info.is_valid + } version_items = self._controller.get_version_items( project_name, product_ids ) - version_items_by_product_id.update(version_items) + version_items_by_product_id_by_project[project_name] = version_items update_containers = [] update_versions = [] for item_id, container_item in containers_items_by_id.items(): repre_id = container_item.representation_id + project_name = container_item.project_name repre_info = repre_info_by_id[repre_id] product_id = repre_info.product_id - version_items_id = version_items_by_product_id[product_id] + version_items_id = ( + version_items_by_product_id_by_project[project_name][product_id] + ) version_item = version_items_id.get(repre_info.version_id, {}) if not version_item or not version_item.is_hero: continue From 77e5317ee3bae6b2aa792a9525cc3433e24ddeca Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Thu, 21 Nov 2024 22:37:55 +0800 Subject: [PATCH 32/90] remove unused variable --- client/ayon_core/tools/sceneinventory/view.py | 1 - 1 file changed, 1 deletion(-) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index 24e0195e31..025bff6e9f 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -974,7 +974,6 @@ class SceneInventoryView(QtWidgets.QTreeView): project_name, repre_ids) repre_info_by_project[project_name].update(repre_info) - product_ids_by_project = collections.defaultdict(set) version_items_by_product_id_by_project = collections.defaultdict(dict) for container_item in containers_items_by_id.values(): project_name = container_item.project_name From 26251bb9b4ebae496ab7aab0a7a3e0a1a95c1611 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 21 Nov 2024 18:29:31 +0100 Subject: [PATCH 33/90] simplified parsing of template --- client/ayon_core/lib/path_templates.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/client/ayon_core/lib/path_templates.py b/client/ayon_core/lib/path_templates.py index dc88ec956b..1a99ae459d 100644 --- a/client/ayon_core/lib/path_templates.py +++ b/client/ayon_core/lib/path_templates.py @@ -1,6 +1,7 @@ import os import re import numbers +from string import Formatter KEY_PATTERN = re.compile(r"(\{.*?[^{0]*\})") KEY_PADDING_PATTERN = re.compile(r"([^:]+)\S+[><]\S+") @@ -48,16 +49,16 @@ class StringTemplate: self._template = template parts = [] - last_end_idx = 0 - for item in KEY_PATTERN.finditer(template): - start, end = item.span() - if start > last_end_idx: - parts.append(template[last_end_idx:start]) - parts.append(FormattingPart(template[start:end])) - last_end_idx = end + formatter = Formatter() - if last_end_idx < len(template): - parts.append(template[last_end_idx:len(template)]) + for item in formatter.parse(template): + literal_text, field_name, format_spec, conversion = item + if literal_text: + parts.append(literal_text) + if field_name: + parts.append( + FormattingPart(field_name, format_spec, conversion) + ) new_parts = [] for part in parts: From e4875cc5096a5381861b45f45c28eb6c2a68215e Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 21 Nov 2024 18:37:34 +0100 Subject: [PATCH 34/90] fill FormattingPart init --- client/ayon_core/lib/path_templates.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/lib/path_templates.py b/client/ayon_core/lib/path_templates.py index 1a99ae459d..3871b97849 100644 --- a/client/ayon_core/lib/path_templates.py +++ b/client/ayon_core/lib/path_templates.py @@ -437,8 +437,21 @@ class FormattingPart: Args: template(str): String containing the formatting key. """ - def __init__(self, template): - self._template = template + def __init__(self, field_name, format_spec, conversion): + format_spec_v = "" + if format_spec: + format_spec_v = f":{format_spec}" + conversion_v = "" + if conversion: + conversion_v = f"!{conversion}" + + self._field_name = field_name + self._format_spec = format_spec_v + self._conversion = conversion_v + + template_base = f"{field_name}{format_spec_v}{conversion_v}" + self._template_base = template_base + self._template = f"{{{template_base}}}" @property def template(self): From d15148f001f17f4488119c402a54005be34e0d38 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 21 Nov 2024 18:38:49 +0100 Subject: [PATCH 35/90] support list in StringTemplate --- client/ayon_core/lib/path_templates.py | 122 +++++++++++++++++-------- 1 file changed, 82 insertions(+), 40 deletions(-) diff --git a/client/ayon_core/lib/path_templates.py b/client/ayon_core/lib/path_templates.py index 3871b97849..9b545f2851 100644 --- a/client/ayon_core/lib/path_templates.py +++ b/client/ayon_core/lib/path_templates.py @@ -1,10 +1,10 @@ import os import re +import copy import numbers +from typing import List from string import Formatter -KEY_PATTERN = re.compile(r"(\{.*?[^{0]*\})") -KEY_PADDING_PATTERN = re.compile(r"([^:]+)\S+[><]\S+") SUB_DICT_PATTERN = re.compile(r"([^\[\]]+)") OPTIONAL_PATTERN = re.compile(r"(<.*?[^{0]*>)[^0-9]*?") @@ -369,11 +369,10 @@ class TemplatePartResult: @staticmethod def split_keys_to_subdicts(values): output = {} + formatter = Formatter() for key, value in values.items(): - key_padding = list(KEY_PADDING_PATTERN.findall(key)) - if key_padding: - key = key_padding[0] - key_subdict = list(SUB_DICT_PATTERN.findall(key)) + _, field_name, _, _ = next(formatter.parse(f"{{{key}}}")) + key_subdict = list(SUB_DICT_PATTERN.findall(field_name)) data = output last_key = key_subdict.pop(-1) for subkey in key_subdict: @@ -502,6 +501,16 @@ class FormattingPart: return False return not queue + @staticmethod + def keys_to_template_base(keys: List[str]): + if not keys: + return None + # Create copy of keys + keys = list(keys) + template_base = keys.pop(0) + joined_keys = "".join([f"[{key}]" for key in keys]) + return f"{template_base}{joined_keys}" + def format(self, data, result): """Format the formattings string. @@ -509,7 +518,7 @@ class FormattingPart: data(dict): Data that should be used for formatting. result(TemplatePartResult): Object where result is stored. """ - key = self.template[1:-1] + key = self._template_base if key in result.realy_used_values: result.add_output(result.realy_used_values[key]) return result @@ -521,17 +530,38 @@ class FormattingPart: return result # check if key expects subdictionary keys (e.g. project[name]) - existence_check = key - key_padding = list(KEY_PADDING_PATTERN.findall(existence_check)) - if key_padding: - existence_check = key_padding[0] - key_subdict = list(SUB_DICT_PATTERN.findall(existence_check)) + key_subdict = list(SUB_DICT_PATTERN.findall(self._field_name)) value = data missing_key = False invalid_type = False used_keys = [] + keys_to_value = None + used_value = None + for sub_key in key_subdict: + if isinstance(value, list): + if not sub_key.lstrip("-").isdigit(): + invalid_type = True + break + sub_key = int(sub_key) + if sub_key < 0: + sub_key = len(value) + sub_key + + invalid = 0 > sub_key < len(data) + if invalid: + used_keys.append(sub_key) + missing_key = True + break + + used_keys.append(sub_key) + if keys_to_value is None: + keys_to_value = list(used_keys) + keys_to_value.pop(-1) + used_value = copy.deepcopy(value) + value = value[sub_key] + continue + if ( value is None or (hasattr(value, "items") and sub_key not in value) @@ -547,45 +577,57 @@ class FormattingPart: used_keys.append(sub_key) value = value.get(sub_key) - if missing_key or invalid_type: - if len(used_keys) == 0: - invalid_key = key_subdict[0] - else: - invalid_key = used_keys[0] - for idx, sub_key in enumerate(used_keys): - if idx == 0: - continue - invalid_key += "[{0}]".format(sub_key) + field_name = key_subdict[0] + if used_keys: + field_name = self.keys_to_template_base(used_keys) + if missing_key or invalid_type: if missing_key: - result.add_missing_key(invalid_key) + result.add_missing_key(field_name) elif invalid_type: - result.add_invalid_type(invalid_key, value) + result.add_invalid_type(field_name, value) result.add_output(self.template) return result - if self.validate_value_type(value): - fill_data = {} - first_value = True - for used_key in reversed(used_keys): - if first_value: - first_value = False - fill_data[used_key] = value - else: - _fill_data = {used_key: fill_data} - fill_data = _fill_data - - formatted_value = self.template.format(**fill_data) - result.add_realy_used_value(key, formatted_value) - result.add_used_value(existence_check, formatted_value) - result.add_output(formatted_value) + if not self.validate_value_type(value): + result.add_invalid_type(key, value) + result.add_output(self.template) return result - result.add_invalid_type(key, value) - result.add_output(self.template) + fill_data = root_fill_data = {} + parent_fill_data = None + parent_key = None + fill_value = data + value_filled = False + for used_key in used_keys: + if isinstance(fill_value, list): + parent_fill_data[parent_key] = fill_value + value_filled = True + break + fill_value = fill_value[used_key] + parent_fill_data = fill_data + fill_data = parent_fill_data.setdefault(used_key, {}) + parent_key = used_key + if not value_filled: + parent_fill_data[used_keys[-1]] = value + + template = f"{{{field_name}{self._format_spec}{self._conversion}}}" + formatted_value = template.format(**root_fill_data) + used_key = key + if keys_to_value is not None: + used_key = self.keys_to_template_base(keys_to_value) + + if used_value is None: + if isinstance(value, numbers.Number): + used_value = value + else: + used_value = formatted_value + result.add_realy_used_value(self._field_name, used_value) + result.add_used_value(used_key, used_value) + result.add_output(formatted_value) return result From e625901aebe0bab4a97e11ca4725c95ac29a86dc Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Fri, 22 Nov 2024 16:05:17 +0800 Subject: [PATCH 36/90] big roy's comment - code tweak --- client/ayon_core/tools/sceneinventory/model.py | 6 +++--- .../tools/sceneinventory/models/containers.py | 2 +- client/ayon_core/tools/sceneinventory/view.py | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 75af957cfa..849d8b8d17 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -132,7 +132,7 @@ class InventoryModel(QtGui.QStandardItemModel): # for debugging or testing, injecting items from outside container_items = self._controller.get_container_items() self._clear_items() - repre_id = set() + repre_ids = set() repre_ids_by_project = collections.defaultdict(set) version_items_by_product_id = collections.defaultdict(dict) repre_info_by_id_by_project = collections.defaultdict(dict) @@ -146,7 +146,7 @@ class InventoryModel(QtGui.QStandardItemModel): # continue project_name = container_item.project_name representation_id = container_item.representation_id - repre_id.add(representation_id) + repre_ids.add(representation_id) repre_ids_by_project[project_name].add(representation_id) item_by_repre_id_by_project_id[project_name][representation_id].add(container_item) @@ -168,7 +168,7 @@ class InventoryModel(QtGui.QStandardItemModel): # SiteSync addon information progress_by_id = self._controller.get_representations_site_progress( - repre_id + repre_ids ) sites_info = self._controller.get_sites_information() site_icons = { diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index dc41bdc8fa..aea94b97ef 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -359,9 +359,9 @@ class ContainersModel: containers_by_id = {} container_items_by_id = {} invalid_ids_mapping = {} + current_project_name = self._controller.get_current_project_name() for container in containers: try: - current_project_name = self._controller.get_current_project_name() item = ContainerItem.from_container_data(current_project_name, container) repre_id = item.representation_id try: diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index 025bff6e9f..d4381f55cd 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -755,7 +755,7 @@ class SceneInventoryView(QtWidgets.QTreeView): project_name = container_item.project_name repre_ids_by_project[project_name].add(repre_id) - versions = set() + version_ids = set() repre_info_by_project = collections.defaultdict(dict) version_items_by_product_id_by_project = collections.defaultdict(dict) for project_name, repre_ids in repre_ids_by_project.items(): @@ -784,8 +784,8 @@ class SceneInventoryView(QtWidgets.QTreeView): version_items = list( version_items_by_product_id_by_project[project_name][active_product_id].values() ) - all_versions = {version_item.version for version_item in version_items} - versions.update(all_versions) + version_ids.update(version_item.version for version_item in version_items) + product_ids_by_version = collections.defaultdict(set) for version_items_by_product_id in version_items_by_product_id_by_project.values(): for version_items_by_id in version_items_by_product_id.values(): @@ -797,9 +797,9 @@ class SceneInventoryView(QtWidgets.QTreeView): product_ids_by_version[_prod_version].add( version_item.product_id ) - if version in versions: + if version in version_ids: continue - versions.add(version) + version_ids.add(version) version_items.append(version_item) def version_sorter(item): From 7edc759842024453cf60702b1bf3b5c431c96fae Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Fri, 22 Nov 2024 17:57:49 +0100 Subject: [PATCH 37/90] remove unused variables --- client/ayon_core/tools/sceneinventory/models/containers.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index aea94b97ef..ad78061468 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -194,19 +194,15 @@ class ContainersModel: self._items_cache = None self._containers_by_id = {} self._container_items_by_id = {} - self._container_items_by_project = {} self._version_items_by_product_id = {} self._repre_info_by_id = {} - self._product_ids_by_project = {} def reset(self): self._items_cache = None self._containers_by_id = {} self._container_items_by_id = {} - self._container_items_by_project = {} self._version_items_by_product_id = {} self._repre_info_by_id = {} - self._product_ids_by_project = {} def get_containers(self): self._update_cache() From ef26dc2dc2cb50957710e25c2e7cc9aacad8d7da Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Fri, 22 Nov 2024 17:58:02 +0100 Subject: [PATCH 38/90] added empty lines for readability --- client/ayon_core/tools/sceneinventory/models/containers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index ad78061468..b1cbb38587 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -294,10 +294,12 @@ class ContainersModel: def version_sorted(entity): return entity["version"] + version_entities_by_product_id = { product_id: [] for product_id in missing_ids } + version_entities = list(ayon_api.get_versions( project_name, product_ids=missing_ids, @@ -309,6 +311,7 @@ class ContainersModel: version_entities_by_product_id[product_id].append( version_entity ) + for product_id, version_entities in ( version_entities_by_product_id.items() ): @@ -334,6 +337,7 @@ class ContainersModel: self._version_items_by_product_id[product_id] = ( version_items_by_id ) + return { product_id: dict(self._version_items_by_product_id[product_id]) for product_id in product_ids From 7935ed3284fca091eed45eb13acfd3ea456e0145 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Fri, 22 Nov 2024 17:58:23 +0100 Subject: [PATCH 39/90] site sync expects project name --- client/ayon_core/tools/sceneinventory/control.py | 14 ++++++++++---- .../tools/sceneinventory/models/sitesync.py | 10 +++++++--- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/control.py b/client/ayon_core/tools/sceneinventory/control.py index 8c02881b82..12dfc72e77 100644 --- a/client/ayon_core/tools/sceneinventory/control.py +++ b/client/ayon_core/tools/sceneinventory/control.py @@ -124,14 +124,20 @@ class SceneInventoryController: def get_site_provider_icons(self): return self._sitesync_model.get_site_provider_icons() - def get_representations_site_progress(self, representation_ids): + def get_representations_site_progress( + self, project_name, representation_ids + ): return self._sitesync_model.get_representations_site_progress( - representation_ids + project_name, representation_ids ) - def resync_representations(self, representation_ids, site_type): + def resync_representations( + self, project_name, representation_ids, site_type + ): return self._sitesync_model.resync_representations( - representation_ids, site_type + project_name, + representation_ids, + site_type ) # Switch dialog methods diff --git a/client/ayon_core/tools/sceneinventory/models/sitesync.py b/client/ayon_core/tools/sceneinventory/models/sitesync.py index 1a1f08bf02..1738ec2c15 100644 --- a/client/ayon_core/tools/sceneinventory/models/sitesync.py +++ b/client/ayon_core/tools/sceneinventory/models/sitesync.py @@ -54,7 +54,9 @@ class SiteSyncModel: "remote_site_provider": self._get_remote_site_provider() } - def get_representations_site_progress(self, representation_ids): + def get_representations_site_progress( + self, project_name, representation_ids + ): """Get progress of representations sync.""" representation_ids = set(representation_ids) @@ -68,7 +70,6 @@ class SiteSyncModel: if not self.is_sitesync_enabled(): return output - project_name = self._controller.get_current_project_name() sitesync_addon = self._get_sitesync_addon() repre_entities = ayon_api.get_representations( project_name, representation_ids @@ -86,10 +87,13 @@ class SiteSyncModel: return output - def resync_representations(self, representation_ids, site_type): + def resync_representations( + self, project_name, representation_ids, site_type + ): """ Args: + project_name (str): Project name. representation_ids (Iterable[str]): Representation ids. site_type (Literal[active_site, remote_site]): Site type. """ From b25715ee2bfc801486da97259890eb6425e1e35d Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Fri, 22 Nov 2024 17:59:47 +0100 Subject: [PATCH 40/90] use project name in site sync calls --- .../ayon_core/tools/sceneinventory/model.py | 13 +++++---- .../tools/sceneinventory/models/sitesync.py | 2 +- client/ayon_core/tools/sceneinventory/view.py | 29 +++++++++++++------ 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 849d8b8d17..cf9814a8e1 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -132,7 +132,6 @@ class InventoryModel(QtGui.QStandardItemModel): # for debugging or testing, injecting items from outside container_items = self._controller.get_container_items() self._clear_items() - repre_ids = set() repre_ids_by_project = collections.defaultdict(set) version_items_by_product_id = collections.defaultdict(dict) repre_info_by_id_by_project = collections.defaultdict(dict) @@ -146,7 +145,6 @@ class InventoryModel(QtGui.QStandardItemModel): # continue project_name = container_item.project_name representation_id = container_item.representation_id - repre_ids.add(representation_id) repre_ids_by_project[project_name].add(representation_id) item_by_repre_id_by_project_id[project_name][representation_id].add(container_item) @@ -167,9 +165,13 @@ class InventoryModel(QtGui.QStandardItemModel): version_items_by_product_id[project_name] = version_items # SiteSync addon information - progress_by_id = self._controller.get_representations_site_progress( - repre_ids - ) + progress_by_project = {} + for project_name, repre_ids in repre_ids_by_project.items(): + progress_by_id = self._controller.get_representations_site_progress( + project_name, repre_ids + ) + progress_by_project[project_name] = progress_by_id + sites_info = self._controller.get_sites_information() site_icons = { provider: get_qt_icon(icon_def) @@ -207,6 +209,7 @@ class InventoryModel(QtGui.QStandardItemModel): root_item = self.invisibleRootItem() group_items = [] for project_name, items_by_repre_id in item_by_repre_id_by_project_id.items(): + progress_by_id = progress_by_project[project_name] for repre_id, container_items in items_by_repre_id.items(): repre_info = repre_info_by_id_by_project[project_name][repre_id] version_color = None diff --git a/client/ayon_core/tools/sceneinventory/models/sitesync.py b/client/ayon_core/tools/sceneinventory/models/sitesync.py index 1738ec2c15..c8e1ac2cd3 100644 --- a/client/ayon_core/tools/sceneinventory/models/sitesync.py +++ b/client/ayon_core/tools/sceneinventory/models/sitesync.py @@ -103,7 +103,7 @@ class SiteSyncModel: active_site = self._get_active_site() remote_site = self._get_remote_site() progress = self.get_representations_site_progress( - representation_ids + project_name, representation_ids ) for repre_id in representation_ids: repre_progress = progress.get(repre_id) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index d4381f55cd..9bd2d65cd0 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -413,12 +413,13 @@ class SceneInventoryView(QtWidgets.QTreeView): self._handle_sitesync(menu, valid_repre_ids) - def _handle_sitesync(self, menu, repre_ids): + def _handle_sitesync(self, menu, repre_ids_by_project_name): """Adds actions for download/upload when SyncServer is enabled Args: menu (OptionMenu) - repre_ids (list) of object_ids + repre_ids_by_project_name (Dict[str, Set[str]]): Representation + ids by project name. Returns: (OptionMenu) @@ -427,7 +428,7 @@ class SceneInventoryView(QtWidgets.QTreeView): if not self._controller.is_sitesync_enabled(): return - if not repre_ids: + if not repre_ids_by_project_name: return menu.addSeparator() @@ -439,7 +440,10 @@ class SceneInventoryView(QtWidgets.QTreeView): menu ) download_active_action.triggered.connect( - lambda: self._add_sites(repre_ids, "active_site")) + lambda: self._add_sites( + repre_ids_by_project_name, "active_site" + ) + ) upload_icon = qtawesome.icon("fa.upload", color=DEFAULT_COLOR) upload_remote_action = QtWidgets.QAction( @@ -448,23 +452,30 @@ class SceneInventoryView(QtWidgets.QTreeView): menu ) upload_remote_action.triggered.connect( - lambda: self._add_sites(repre_ids, "remote_site")) + lambda: self._add_sites( + repre_ids_by_project_name, "remote_site" + ) + ) menu.addAction(download_active_action) menu.addAction(upload_remote_action) - def _add_sites(self, repre_ids, site_type): + def _add_sites(self, repre_ids_by_project_name, site_type): """(Re)sync all 'repre_ids' to specific site. It checks if opposite site has fully available content to limit accidents. (ReSync active when no remote >> losing active content) Args: - repre_ids (list) + repre_ids_by_project_name (Dict[str, Set[str]]): Representation + ids by project name. site_type (Literal[active_site, remote_site]): Site type. - """ - self._controller.resync_representations(repre_ids, site_type) + """ + for project_name, repre_ids in repre_ids_by_project_name.items(): + self._controller.resync_representations( + project_name, repre_ids, site_type + ) self.data_changed.emit() From a30698eb4b12381ec7c01f7ca1c98ae0e12fab3c Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Fri, 22 Nov 2024 18:02:49 +0100 Subject: [PATCH 41/90] refactor view codebase --- client/ayon_core/tools/sceneinventory/view.py | 105 ++++++++++-------- 1 file changed, 60 insertions(+), 45 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index 9bd2d65cd0..b3322ffc60 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -198,34 +198,41 @@ class SceneInventoryView(QtWidgets.QTreeView): project_name = container_item.project_name repre_ids_by_project[project_name].add(repre_id) - repre_info_by_project = collections.defaultdict(dict) + repre_info_by_project = {} + repre_ids_by_project_name = {} + version_ids_by_project = {} + product_ids_by_project = {} for project_name, repre_ids in repre_ids_by_project.items(): - repre_info = self._controller.get_representation_info_items( - project_name, repre_ids) - repre_info_by_project[project_name].update(repre_info) + repres_info = self._controller.get_representation_info_items( + project_name, repre_ids + ) + + repre_info_by_project[project_name] = repres_info + repre_ids = set() + version_ids = set() + product_ids = set() + for repre_id, repre_info in repres_info.items(): + if not repre_info.is_valid: + continue + repre_ids.add(repre_id) + version_ids.add(repre_info.version_id) + product_ids.add(repre_info.product_id) + + repre_ids_by_project_name[project_name] = repre_ids + version_ids_by_project[project_name] = version_ids + product_ids_by_project[project_name] = product_ids + # Exclude items that are "NOT FOUND" since setting versions, updating # and removal won't work for those items. filtered_items = [] - version_ids = set() - valid_repre_ids = set() - product_ids_by_project = collections.defaultdict(set) for container_item in container_items_by_id.values(): project_name = container_item.project_name - repre_info_by_id = repre_info_by_project.get(project_name) repre_id = container_item.representation_id - all_valid_repre_ids = { - repre_id - for repre_id, repre_info in repre_info_by_id.items() - if repre_info.is_valid - } - valid_repre_ids.update(all_valid_repre_ids) - + repre_info_by_id = repre_info_by_project.get(project_name, {}) repre_info = repre_info_by_id.get(repre_id) if repre_info and repre_info.is_valid: filtered_items.append(container_item) - version_ids.add(repre_info.version_id) - product_id = repre_info.product_id - product_ids_by_project[project_name].add(product_id) + # remove remove_icon = qtawesome.icon("fa.remove", color=DEFAULT_COLOR) remove_action = QtWidgets.QAction(remove_icon, "Remove items", menu) @@ -236,18 +243,23 @@ class SceneInventoryView(QtWidgets.QTreeView): # Keep remove action for invalid items menu.addAction(remove_action) return - version_items_by_product_id_by_project = collections.defaultdict(dict) - for project_name, product_ids in product_ids_by_project.items(): - version_items = self._controller.get_version_items( + + version_items_by_project = { + project_name: self._controller.get_version_items( project_name, product_ids ) - version_items_by_product_id_by_project[project_name] = version_items + for project_name, product_ids in product_ids_by_project.items() + } + has_outdated = False has_loaded_hero_versions = False has_available_hero_version = False has_outdated_approved = False last_version_by_product_id = {} - for version_items_by_product_id in version_items_by_product_id_by_project.values(): + for project_name, version_items_by_product_id in ( + version_items_by_project.items() + ): + version_ids = version_ids_by_project[project_name] for product_id, version_items_by_id in ( version_items_by_product_id.items() ): @@ -411,7 +423,7 @@ class SceneInventoryView(QtWidgets.QTreeView): menu.addAction(remove_action) - self._handle_sitesync(menu, valid_repre_ids) + self._handle_sitesync(menu, repre_ids_by_project_name) def _handle_sitesync(self, menu, repre_ids_by_project_name): """Adds actions for download/upload when SyncServer is enabled @@ -979,44 +991,47 @@ class SceneInventoryView(QtWidgets.QTreeView): repre_ids_by_project[project_name].add(repre_id) # Get representation info items by ID - repre_info_by_project = collections.defaultdict(dict) + repres_info_by_project = {} + version_items_by_project = {} for project_name, repre_ids in repre_ids_by_project.items(): - repre_info = self._controller.get_representation_info_items( - project_name, repre_ids) - repre_info_by_project[project_name].update(repre_info) + repre_info_by_id = self._controller.get_representation_info_items( + project_name, repre_ids + ) + repres_info_by_project[project_name] = repre_info_by_id - version_items_by_product_id_by_project = collections.defaultdict(dict) - for container_item in containers_items_by_id.values(): - project_name = container_item.project_name - repre_info_by_id = repre_info_by_project.get(project_name) - repre_id = container_item.representation_id - repre_info = repre_info_by_id.get(repre_id) product_ids = { repre_info.product_id for repre_info in repre_info_by_id.values() if repre_info.is_valid } - version_items = self._controller.get_version_items( + version_items_by_product_id = self._controller.get_version_items( project_name, product_ids ) - version_items_by_product_id_by_project[project_name] = version_items + version_items_by_project[project_name] = ( + version_items_by_product_id + ) update_containers = [] update_versions = [] - for item_id, container_item in containers_items_by_id.items(): - repre_id = container_item.representation_id + for container_item in containers_items_by_id.values(): project_name = container_item.project_name + repre_id = container_item.representation_id + + repre_info_by_id = repres_info_by_project[project_name] repre_info = repre_info_by_id[repre_id] - product_id = repre_info.product_id - version_items_id = ( - version_items_by_product_id_by_project[project_name][product_id] + + version_items_by_product_id = ( + version_items_by_project[project_name] ) - version_item = version_items_id.get(repre_info.version_id, {}) + product_id = repre_info.product_id + version_items_by_id = version_items_by_product_id[product_id] + version_item = version_items_by_id.get(repre_info.version_id, {}) if not version_item or not version_item.is_hero: continue + version = abs(version_item.version) version_found = False - for version_item in version_items_id.values(): + for version_item in version_items_by_id.values(): if version_item.is_hero: continue if version_item.version == version: @@ -1029,8 +1044,8 @@ class SceneInventoryView(QtWidgets.QTreeView): update_containers.append(container_item.item_id) update_versions.append(version) - # Specify version per item to update to - self._update_containers(update_containers, update_versions) + # Specify version per item to update to + self._update_containers(update_containers, update_versions) def _update_containers(self, item_ids, versions): """Helper to update items to given version (or version per item) From 562f2edacee460709963213c1aeae155416c097f Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Fri, 22 Nov 2024 18:22:27 +0100 Subject: [PATCH 42/90] site sync is fully project specific --- .../ayon_core/tools/sceneinventory/control.py | 4 +- .../ayon_core/tools/sceneinventory/model.py | 52 +++++--- .../tools/sceneinventory/models/sitesync.py | 121 +++++++++--------- 3 files changed, 98 insertions(+), 79 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/control.py b/client/ayon_core/tools/sceneinventory/control.py index 12dfc72e77..4f23b8e942 100644 --- a/client/ayon_core/tools/sceneinventory/control.py +++ b/client/ayon_core/tools/sceneinventory/control.py @@ -118,8 +118,8 @@ class SceneInventoryController: def is_sitesync_enabled(self): return self._sitesync_model.is_sitesync_enabled() - def get_sites_information(self): - return self._sitesync_model.get_sites_information() + def get_sites_information(self, project_name): + return self._sitesync_model.get_sites_information(project_name) def get_site_provider_icons(self): return self._sitesync_model.get_site_provider_icons() diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index cf9814a8e1..95272470aa 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -133,10 +133,10 @@ class InventoryModel(QtGui.QStandardItemModel): container_items = self._controller.get_container_items() self._clear_items() repre_ids_by_project = collections.defaultdict(set) - version_items_by_product_id = collections.defaultdict(dict) + version_items_by_project = collections.defaultdict(dict) repre_info_by_id_by_project = collections.defaultdict(dict) - item_by_repre_id_by_project_id = collections.defaultdict( - lambda: collections.defaultdict(set)) + item_by_repre_id_by_project = collections.defaultdict( + lambda: collections.defaultdict(list)) for container_item in container_items: # if ( # selected is not None @@ -146,7 +146,11 @@ class InventoryModel(QtGui.QStandardItemModel): project_name = container_item.project_name representation_id = container_item.representation_id repre_ids_by_project[project_name].add(representation_id) - item_by_repre_id_by_project_id[project_name][representation_id].add(container_item) + ( + item_by_repre_id_by_project + [project_name] + [representation_id] + ).append(container_item) for project_name, representation_ids in repre_ids_by_project.items(): repre_info = self._controller.get_representation_info_items( @@ -162,17 +166,20 @@ class InventoryModel(QtGui.QStandardItemModel): version_items = self._controller.get_version_items( project_name, product_ids ) - version_items_by_product_id[project_name] = version_items + version_items_by_project[project_name] = version_items # SiteSync addon information - progress_by_project = {} - for project_name, repre_ids in repre_ids_by_project.items(): - progress_by_id = self._controller.get_representations_site_progress( + progress_by_project = { + project_name: self._controller.get_representations_site_progress( project_name, repre_ids ) - progress_by_project[project_name] = progress_by_id + for project_name, repre_ids in repre_ids_by_project.items() + } - sites_info = self._controller.get_sites_information() + sites_info_by_project_name = { + project_name: self._controller.get_sites_information(project_name) + for project_name in repre_ids_by_project.keys() + } site_icons = { provider: get_qt_icon(icon_def) for provider, icon_def in ( @@ -203,15 +210,26 @@ class InventoryModel(QtGui.QStandardItemModel): group_item_font = QtGui.QFont() group_item_font.setBold(True) - active_site_icon = site_icons.get(sites_info["active_site_provider"]) - remote_site_icon = site_icons.get(sites_info["remote_site_provider"]) - root_item = self.invisibleRootItem() group_items = [] - for project_name, items_by_repre_id in item_by_repre_id_by_project_id.items(): + for project_name, items_by_repre_id in ( + item_by_repre_id_by_project.items() + ): + sites_info = sites_info_by_project_name[project_name] + active_site_icon = site_icons.get( + sites_info["active_site_provider"] + ) + remote_site_icon = site_icons.get( + sites_info["remote_site_provider"] + ) + progress_by_id = progress_by_project[project_name] + repre_info_by_id = repre_info_by_id_by_project[project_name] + version_items_by_product_id = ( + version_items_by_project[project_name] + ) for repre_id, container_items in items_by_repre_id.items(): - repre_info = repre_info_by_id_by_project[project_name][repre_id] + repre_info = repre_info_by_id[repre_id] version_color = None if not repre_info.is_valid: version_label = "N/A" @@ -230,7 +248,7 @@ class InventoryModel(QtGui.QStandardItemModel): item_icon = valid_item_icon version_items = ( - version_items_by_product_id[project_name][repre_info.product_id] + version_items_by_product_id[repre_info.product_id] ) version_item = version_items[repre_info.version_id] version_label = format_version(version_item.version) @@ -266,8 +284,6 @@ class InventoryModel(QtGui.QStandardItemModel): item.setData(True, IS_CONTAINER_ITEM_ROLE) item.setData(unique_name, ITEM_UNIQUE_NAME_ROLE) container_model_items.append(item) - if not container_model_items: - continue progress = progress_by_id[repre_id] active_site_progress = "{}%".format( diff --git a/client/ayon_core/tools/sceneinventory/models/sitesync.py b/client/ayon_core/tools/sceneinventory/models/sitesync.py index c8e1ac2cd3..546d2b15c0 100644 --- a/client/ayon_core/tools/sceneinventory/models/sitesync.py +++ b/client/ayon_core/tools/sceneinventory/models/sitesync.py @@ -11,18 +11,18 @@ class SiteSyncModel: self._sitesync_addon = NOT_SET self._sitesync_enabled = None - self._active_site = NOT_SET - self._remote_site = NOT_SET - self._active_site_provider = NOT_SET - self._remote_site_provider = NOT_SET + self._active_site = {} + self._remote_site = {} + self._active_site_provider = {} + self._remote_site_provider = {} def reset(self): self._sitesync_addon = NOT_SET self._sitesync_enabled = None - self._active_site = NOT_SET - self._remote_site = NOT_SET - self._active_site_provider = NOT_SET - self._remote_site_provider = NOT_SET + self._active_site = {} + self._remote_site = {} + self._active_site_provider = {} + self._remote_site_provider = {} def is_sitesync_enabled(self): """Site sync is enabled. @@ -46,12 +46,16 @@ class SiteSyncModel: sitesync_addon = self._get_sitesync_addon() return sitesync_addon.get_site_icons() - def get_sites_information(self): + def get_sites_information(self, project_name): return { - "active_site": self._get_active_site(), - "active_site_provider": self._get_active_site_provider(), - "remote_site": self._get_remote_site(), - "remote_site_provider": self._get_remote_site_provider() + "active_site": self._get_active_site(project_name), + "remote_site": self._get_remote_site(project_name), + "active_site_provider": self._get_active_site_provider( + project_name + ), + "remote_site_provider": self._get_remote_site_provider( + project_name + ) } def get_representations_site_progress( @@ -74,8 +78,8 @@ class SiteSyncModel: repre_entities = ayon_api.get_representations( project_name, representation_ids ) - active_site = self._get_active_site() - remote_site = self._get_remote_site() + active_site = self._get_active_site(project_name) + remote_site = self._get_remote_site(project_name) for repre_entity in repre_entities: repre_output = output[repre_entity["id"]] @@ -97,11 +101,9 @@ class SiteSyncModel: representation_ids (Iterable[str]): Representation ids. site_type (Literal[active_site, remote_site]): Site type. """ - - project_name = self._controller.get_current_project_name() sitesync_addon = self._get_sitesync_addon() - active_site = self._get_active_site() - remote_site = self._get_remote_site() + active_site = self._get_active_site(project_name) + remote_site = self._get_remote_site(project_name) progress = self.get_representations_site_progress( project_name, representation_ids ) @@ -136,48 +138,49 @@ class SiteSyncModel: self._sitesync_addon = sitesync_addon self._sitesync_enabled = sync_enabled - def _get_active_site(self): - if self._active_site is NOT_SET: - self._cache_sites() - return self._active_site + def _get_active_site(self, project_name): + if project_name not in self._active_site: + self._cache_sites(project_name) + return self._active_site[project_name] - def _get_remote_site(self): - if self._remote_site is NOT_SET: - self._cache_sites() - return self._remote_site + def _get_remote_site(self, project_name): + if project_name not in self._remote_site: + self._cache_sites(project_name) + return self._remote_site[project_name] - def _get_active_site_provider(self): - if self._active_site_provider is NOT_SET: - self._cache_sites() - return self._active_site_provider + def _get_active_site_provider(self, project_name): + if project_name not in self._active_site_provider: + self._cache_sites(project_name) + return self._active_site_provider[project_name] - def _get_remote_site_provider(self): - if self._remote_site_provider is NOT_SET: - self._cache_sites() - return self._remote_site_provider + def _get_remote_site_provider(self, project_name): + if project_name not in self._remote_site_provider: + self._cache_sites(project_name) + return self._remote_site_provider[project_name] - def _cache_sites(self): - active_site = None - remote_site = None - active_site_provider = None - remote_site_provider = None - if self.is_sitesync_enabled(): - sitesync_addon = self._get_sitesync_addon() - project_name = self._controller.get_current_project_name() - active_site = sitesync_addon.get_active_site(project_name) - remote_site = sitesync_addon.get_remote_site(project_name) - active_site_provider = "studio" - remote_site_provider = "studio" - if active_site != "studio": - active_site_provider = sitesync_addon.get_provider_for_site( - project_name, active_site - ) - if remote_site != "studio": - remote_site_provider = sitesync_addon.get_provider_for_site( - project_name, remote_site - ) + def _cache_sites(self, project_name): + self._active_site[project_name] = None + self._remote_site[project_name] = None + self._active_site_provider[project_name] = None + self._remote_site_provider[project_name] = None + if not self.is_sitesync_enabled(): + return - self._active_site = active_site - self._remote_site = remote_site - self._active_site_provider = active_site_provider - self._remote_site_provider = remote_site_provider + sitesync_addon = self._get_sitesync_addon() + active_site = sitesync_addon.get_active_site(project_name) + remote_site = sitesync_addon.get_remote_site(project_name) + active_site_provider = "studio" + remote_site_provider = "studio" + if active_site != "studio": + active_site_provider = sitesync_addon.get_provider_for_site( + project_name, active_site + ) + if remote_site != "studio": + remote_site_provider = sitesync_addon.get_provider_for_site( + project_name, remote_site + ) + + self._active_site[project_name] = active_site + self._remote_site[project_name] = remote_site + self._active_site_provider[project_name] = active_site_provider + self._remote_site_provider[project_name] = remote_site_provider From eea31b676d91c7ae746f7f1d2417737273c6cb05 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Fri, 22 Nov 2024 18:25:46 +0100 Subject: [PATCH 43/90] don't slow down project name getter --- client/ayon_core/pipeline/load/utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/pipeline/load/utils.py b/client/ayon_core/pipeline/load/utils.py index a6c5f0ce1f..de8e1676e7 100644 --- a/client/ayon_core/pipeline/load/utils.py +++ b/client/ayon_core/pipeline/load/utils.py @@ -465,7 +465,9 @@ def update_container(container, version=-1): from ayon_core.pipeline import get_current_project_name # Compute the different version from 'representation' - project_name = container.get("project_name", get_current_project_name()) + project_name = container.get("project_name") + if project_name is None: + project_name = get_current_project_name() repre_id = container["representation"] if not _is_valid_representation_id(repre_id): raise ValueError( @@ -588,7 +590,9 @@ def switch_container(container, representation, loader_plugin=None): ) # Get the new representation to switch to - project_name = container.get("project_name", get_current_project_name()) + project_name = container.get("project_name") + if project_name is None: + project_name = get_current_project_name() context = get_representation_context( project_name, representation["id"] From e21c7a157e5009e521d804fb0621bf55d750cba2 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Fri, 22 Nov 2024 18:46:56 +0100 Subject: [PATCH 44/90] use project name to get correct status icons --- .../ayon_core/tools/sceneinventory/control.py | 5 +- .../ayon_core/tools/sceneinventory/model.py | 48 ++++++++++++------- 2 files changed, 34 insertions(+), 19 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/control.py b/client/ayon_core/tools/sceneinventory/control.py index 4f23b8e942..60d9bc77a9 100644 --- a/client/ayon_core/tools/sceneinventory/control.py +++ b/client/ayon_core/tools/sceneinventory/control.py @@ -86,8 +86,9 @@ class SceneInventoryController: self._current_folder_set = True return self._current_folder_id - def get_project_status_items(self): - project_name = self.get_current_project_name() + def get_project_status_items(self, project_name=None): + if project_name is None: + project_name = self.get_current_project_name() return self._projects_model.get_project_status_items( project_name, None ) diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 95272470aa..79af0e5cf5 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -121,8 +121,8 @@ class InventoryModel(QtGui.QStandardItemModel): self._default_icon_color = get_default_entity_icon_color() - self._last_project_statuses = {} - self._last_status_icons_by_name = {} + self._last_project_statuses = collections.defaultdict(dict) + self._last_status_icons_by_name = collections.defaultdict(dict) def outdated(self, item): return item.get("isOutdated", True) @@ -131,7 +131,10 @@ class InventoryModel(QtGui.QStandardItemModel): """Refresh the model""" # for debugging or testing, injecting items from outside container_items = self._controller.get_container_items() + self._clear_items() + + project_names = set() repre_ids_by_project = collections.defaultdict(set) version_items_by_project = collections.defaultdict(dict) repre_info_by_id_by_project = collections.defaultdict(dict) @@ -145,6 +148,7 @@ class InventoryModel(QtGui.QStandardItemModel): # continue project_name = container_item.project_name representation_id = container_item.representation_id + project_names.add(project_name) repre_ids_by_project[project_name].add(representation_id) ( item_by_repre_id_by_project @@ -178,7 +182,7 @@ class InventoryModel(QtGui.QStandardItemModel): sites_info_by_project_name = { project_name: self._controller.get_sites_information(project_name) - for project_name in repre_ids_by_project.keys() + for project_name in project_names } site_icons = { provider: get_qt_icon(icon_def) @@ -186,11 +190,17 @@ class InventoryModel(QtGui.QStandardItemModel): self._controller.get_site_provider_icons().items() ) } - self._last_project_statuses = { - status_item.name: status_item - for status_item in self._controller.get_project_status_items() - } - self._last_status_icons_by_name = {} + last_project_statuses = collections.defaultdict(dict) + for project_name in project_names: + status_items_by_name = { + status_item.name: status_item + for status_item in self._controller.get_project_status_items( + project_name + ) + } + last_project_statuses[project_name] = status_items_by_name + self._last_project_statuses = last_project_statuses + self._last_status_icons_by_name = collections.defaultdict(dict) group_item_icon = qtawesome.icon( "fa.folder", color=self._default_icon_color @@ -258,9 +268,9 @@ class InventoryModel(QtGui.QStandardItemModel): version_color = self.OUTDATED_COLOR status_name = version_item.status - status_color, status_short, status_icon = self._get_status_data( - status_name - ) + ( + status_color, status_short, status_icon + ) = self._get_status_data(project_name, status_name) repre_name = ( repre_info.representation_name or "" @@ -392,17 +402,21 @@ class InventoryModel(QtGui.QStandardItemModel): root_item = self.invisibleRootItem() root_item.removeRows(0, root_item.rowCount()) - def _get_status_data(self, status_name): - status_item = self._last_project_statuses.get(status_name) - status_icon = self._get_status_icon(status_name, status_item) + def _get_status_data(self, project_name, status_name): + status_item = self._last_project_statuses[project_name].get( + status_name + ) + status_icon = self._get_status_icon( + project_name, status_name, status_item + ) status_color = status_short = None if status_item is not None: status_color = status_item.color status_short = status_item.short return status_color, status_short, status_icon - def _get_status_icon(self, status_name, status_item): - icon = self._last_status_icons_by_name.get(status_name) + def _get_status_icon(self, project_name, status_name, status_item): + icon = self._last_status_icons_by_name[project_name].get(status_name) if icon is not None: return icon @@ -415,7 +429,7 @@ class InventoryModel(QtGui.QStandardItemModel): }) if icon is None: icon = QtGui.QIcon() - self._last_status_icons_by_name[status_name] = icon + self._last_status_icons_by_name[project_name][status_name] = icon return icon From 87907b550b1410a6b6912f611287123e310e7f33 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Fri, 22 Nov 2024 19:05:29 +0100 Subject: [PATCH 45/90] fix switch version --- client/ayon_core/tools/sceneinventory/view.py | 92 +++++++++++-------- 1 file changed, 53 insertions(+), 39 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index b3322ffc60..9112db2ef3 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -772,60 +772,71 @@ class SceneInventoryView(QtWidgets.QTreeView): container_items_by_id = self._controller.get_container_items_by_id( item_ids ) + project_names = set() repre_ids_by_project = collections.defaultdict(set) for container_item in container_items_by_id.values(): repre_id = container_item.representation_id project_name = container_item.project_name + project_names.add(project_name) repre_ids_by_project[project_name].add(repre_id) - version_ids = set() - repre_info_by_project = collections.defaultdict(dict) - version_items_by_product_id_by_project = collections.defaultdict(dict) + active_project_name = None + active_repre_info = None + repre_info_by_project = {} + version_items_by_project = {} for project_name, repre_ids in repre_ids_by_project.items(): - repre_info = self._controller.get_representation_info_items( + repres_info = self._controller.get_representation_info_items( project_name, repre_ids ) - repre_info_by_project[project_name].update(repre_info) + if active_repre_info is None: + active_project_name = project_name + active_repre_info = repres_info.get(active_repre_id) - for container_item in container_items_by_id.values(): - project_name = container_item.project_name - repre_info_by_id = repre_info_by_project.get(project_name) - repre_id = container_item.representation_id - repre_info = repre_info_by_id.get(repre_id) product_ids = { repre_info.product_id - for repre_info in repre_info_by_id.values() + for repre_info in repres_info.values() if repre_info.is_valid } - version_items = self._controller.get_version_items( + version_items_by_product_id = self._controller.get_version_items( project_name, product_ids ) - version_items_by_product_id_by_project[project_name] = version_items - active_repre_info = repre_info_by_id[active_repre_id] - active_version_id = active_repre_info.version_id - active_product_id = active_repre_info.product_id - version_items = list( - version_items_by_product_id_by_project[project_name][active_product_id].values() + + repre_info_by_project[project_name] = repres_info + version_items_by_project[project_name] = version_items_by_product_id + + active_version_id = active_repre_info.version_id + active_product_id = active_repre_info.product_id + + versions = set() + product_ids = set() + version_items = [] + product_ids_by_version_by_project = {} + for project_name, version_items_by_product_id in ( + version_items_by_project.items() + ): + product_ids_by_version = collections.defaultdict(set) + product_ids_by_version_by_project[project_name] = ( + product_ids_by_version ) - version_ids.update(version_item.version for version_item in version_items) + versions |= { + version_item.version + for version_item in version_items_by_product_id.values() + } + for version_item in version_items_by_product_id.values(): + version = version_item.version + _prod_version = version + if _prod_version < 0: + _prod_version = -1 + product_ids_by_version[_prod_version].add( + version_item.product_id + ) + product_ids.add(version_item.product_id) + if version in versions: + continue + versions.add(version) + version_items.append((project_name, version_item)) - product_ids_by_version = collections.defaultdict(set) - for version_items_by_product_id in version_items_by_product_id_by_project.values(): - for version_items_by_id in version_items_by_product_id.values(): - for version_item in version_items_by_id.values(): - version = version_item.version - _prod_version = version - if _prod_version < 0: - _prod_version = -1 - product_ids_by_version[_prod_version].add( - version_item.product_id - ) - if version in version_ids: - continue - version_ids.add(version) - version_items.append(version_item) - - def version_sorter(item): + def version_sorter(_, item): hero_value = 0 i_version = item.version if i_version < 0: @@ -844,7 +855,8 @@ class SceneInventoryView(QtWidgets.QTreeView): version_options = [] active_version_idx = 0 - for idx, version_item in enumerate(version_items): + for idx, item in enumerate(version_items): + project_name, version_item = item version = version_item.version label = format_version(version) if version_item.version_id == active_version_id: @@ -884,11 +896,13 @@ class SceneInventoryView(QtWidgets.QTreeView): product_version = -1 version = HeroVersionType(version) - product_ids = product_ids_by_version[product_version] - filtered_item_ids = set() for container_item in container_items_by_id.values(): project_name = container_item.project_name + product_ids_by_version = ( + product_ids_by_version_by_project[project_name] + ) + product_ids = product_ids_by_version[product_version] repre_id = container_item.representation_id repre_info = repre_info_by_project[project_name][repre_id] if repre_info.product_id in product_ids: From b28f4b0ff1ea9662c169f508302659f459352075 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Fri, 22 Nov 2024 19:10:05 +0100 Subject: [PATCH 46/90] comment out unused variables --- client/ayon_core/tools/sceneinventory/view.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index 9112db2ef3..5892e4f983 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -780,7 +780,7 @@ class SceneInventoryView(QtWidgets.QTreeView): project_names.add(project_name) repre_ids_by_project[project_name].add(repre_id) - active_project_name = None + # active_project_name = None active_repre_info = None repre_info_by_project = {} version_items_by_project = {} @@ -789,7 +789,7 @@ class SceneInventoryView(QtWidgets.QTreeView): project_name, repre_ids ) if active_repre_info is None: - active_project_name = project_name + # active_project_name = project_name active_repre_info = repres_info.get(active_repre_id) product_ids = { @@ -805,7 +805,7 @@ class SceneInventoryView(QtWidgets.QTreeView): version_items_by_project[project_name] = version_items_by_product_id active_version_id = active_repre_info.version_id - active_product_id = active_repre_info.product_id + # active_product_id = active_repre_info.product_id versions = set() product_ids = set() From 463ad79a062b1fe3e23b3351189ef1200de678fb Mon Sep 17 00:00:00 2001 From: Robin De Lillo Date: Mon, 25 Nov 2024 10:14:38 -0500 Subject: [PATCH 47/90] Apply suggestions from code review Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/stagingdir.py | 46 +++++++++++++------------ 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/stagingdir.py index c7cc95ff55..4395f1a5d5 100644 --- a/client/ayon_core/pipeline/stagingdir.py +++ b/client/ayon_core/pipeline/stagingdir.py @@ -1,7 +1,7 @@ from ayon_core.lib import Logger, filter_profiles, StringTemplate from ayon_core.settings import get_project_settings -from ayon_core.pipeline.template_data import get_template_data +from .template_data import get_template_data from .anatomy import Anatomy from .tempdir import get_temp_dir @@ -71,7 +71,7 @@ def get_staging_dir_config( template_name = profile["template_name"] _validate_template_name(project_name, template_name, anatomy) - template = anatomy.templates[STAGING_DIR_TEMPLATES][template_name] + template = anatomy.get_template_item("staging", template_name) if not template: # template should always be found either from anatomy or from profile @@ -93,7 +93,7 @@ def _validate_template_name(project_name, template_name, anatomy): Raises: ValueError - if misconfigured template """ - if template_name not in anatomy.templates[STAGING_DIR_TEMPLATES]: + if template_name not in anatomy.templates["staging"]: raise ValueError( ( 'Anatomy of project "{}" does not have set' @@ -195,23 +195,25 @@ def get_staging_dir_info( log=log, ) - if not staging_dir_config: - if always_return_path: # no config found but force an output - return { - "stagingDir": get_temp_dir( - project_name=project_entity["name"], - anatomy=anatomy, - prefix=prefix, - suffix=suffix, - ), - "stagingDir_persistent": False, - } - else: - return None + if staging_dir_config: + return { + "stagingDir": StringTemplate.format_template( + staging_dir_config["template"]["directory"], + ctx_data + ), + "stagingDir_persistent": staging_dir_config["persistence"], + } - return { - "stagingDir": StringTemplate.format_template( - staging_dir_config["template"]["directory"], ctx_data - ), - "stagingDir_persistent": staging_dir_config["persistence"], - } + # no config found but force an output + if always_return_path: + return { + "stagingDir": get_temp_dir( + project_name=project_entity["name"], + anatomy=anatomy, + prefix=prefix, + suffix=suffix, + ), + "stagingDir_persistent": False, + } + + return None From 0a13574509b517e3d3dd796e7f73ee4d42ce10a4 Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Mon, 25 Nov 2024 10:20:12 -0500 Subject: [PATCH 48/90] Rename stagingdir to staging_dir. --- client/ayon_core/pipeline/__init__.py | 2 +- client/ayon_core/pipeline/publish/lib.py | 2 +- client/ayon_core/pipeline/{stagingdir.py => staging_dir.py} | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) rename client/ayon_core/pipeline/{stagingdir.py => staging_dir.py} (99%) diff --git a/client/ayon_core/pipeline/__init__.py b/client/ayon_core/pipeline/__init__.py index c58e385d79..41bcd0dbd1 100644 --- a/client/ayon_core/pipeline/__init__.py +++ b/client/ayon_core/pipeline/__init__.py @@ -9,7 +9,7 @@ from .anatomy import Anatomy from .tempdir import get_temp_dir -from .stagingdir import get_staging_dir_info +from .staging_dir import get_staging_dir_info from .create import ( BaseCreator, diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 4c36f473d1..c0dfe8c910 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -642,7 +642,7 @@ def get_custom_staging_dir_info( anatomy=None, log=None, ): - from ayon_core.pipeline.stagingdir import get_staging_dir_config + from ayon_core.pipeline.staging_dir import get_staging_dir_config warnings.warn( ( "Function 'get_custom_staging_dir_info' in" diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/staging_dir.py similarity index 99% rename from client/ayon_core/pipeline/stagingdir.py rename to client/ayon_core/pipeline/staging_dir.py index 4395f1a5d5..0e993ecae1 100644 --- a/client/ayon_core/pipeline/stagingdir.py +++ b/client/ayon_core/pipeline/staging_dir.py @@ -5,8 +5,6 @@ from .template_data import get_template_data from .anatomy import Anatomy from .tempdir import get_temp_dir -STAGING_DIR_TEMPLATES = "staging" - def get_staging_dir_config( project_name, From 2d6911513feab68c4aaf3bba66051bf0babbf196 Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Mon, 25 Nov 2024 10:23:43 -0500 Subject: [PATCH 49/90] Fix lint. --- client/ayon_core/pipeline/staging_dir.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/staging_dir.py b/client/ayon_core/pipeline/staging_dir.py index 0e993ecae1..e46426057d 100644 --- a/client/ayon_core/pipeline/staging_dir.py +++ b/client/ayon_core/pipeline/staging_dir.py @@ -203,7 +203,7 @@ def get_staging_dir_info( } # no config found but force an output - if always_return_path: + if always_return_path: return { "stagingDir": get_temp_dir( project_name=project_entity["name"], From 2066fe61a124e0639735cf533ac33894287271cf Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Mon, 25 Nov 2024 10:55:04 -0500 Subject: [PATCH 50/90] Fix anatomy template. --- client/ayon_core/pipeline/staging_dir.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/staging_dir.py b/client/ayon_core/pipeline/staging_dir.py index e46426057d..86aaf3002f 100644 --- a/client/ayon_core/pipeline/staging_dir.py +++ b/client/ayon_core/pipeline/staging_dir.py @@ -196,7 +196,7 @@ def get_staging_dir_info( if staging_dir_config: return { "stagingDir": StringTemplate.format_template( - staging_dir_config["template"]["directory"], + str(staging_dir_config["template"]["directory"]), ctx_data ), "stagingDir_persistent": staging_dir_config["persistence"], From a60796eb73f631ed14f8c0c4bcd193b05c40a5c3 Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Mon, 25 Nov 2024 14:27:13 -0500 Subject: [PATCH 51/90] Adjust missing taskEntity. --- client/ayon_core/pipeline/staging_dir.py | 1 + 1 file changed, 1 insertion(+) diff --git a/client/ayon_core/pipeline/staging_dir.py b/client/ayon_core/pipeline/staging_dir.py index 86aaf3002f..c8e3251e7b 100644 --- a/client/ayon_core/pipeline/staging_dir.py +++ b/client/ayon_core/pipeline/staging_dir.py @@ -147,6 +147,7 @@ def get_staging_dir_info( Optional[Dict[str, Any]]: Staging dir info data """ + task_entity = task_entity or {} log = logger or Logger.get_logger("get_staging_dir_info") if anatomy is None: From 47fa5e56027d7ac182b78b64391cea64f61fa032 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Tue, 26 Nov 2024 20:21:21 +0800 Subject: [PATCH 52/90] check on the active product id before adding version_items --- client/ayon_core/tools/sceneinventory/view.py | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index 5892e4f983..93c889d037 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -805,7 +805,7 @@ class SceneInventoryView(QtWidgets.QTreeView): version_items_by_project[project_name] = version_items_by_product_id active_version_id = active_repre_info.version_id - # active_product_id = active_repre_info.product_id + active_product_id = active_repre_info.product_id versions = set() product_ids = set() @@ -820,21 +820,23 @@ class SceneInventoryView(QtWidgets.QTreeView): ) versions |= { version_item.version - for version_item in version_items_by_product_id.values() + for version_item in + version_items_by_product_id[active_product_id].values() } - for version_item in version_items_by_product_id.values(): - version = version_item.version - _prod_version = version - if _prod_version < 0: - _prod_version = -1 - product_ids_by_version[_prod_version].add( - version_item.product_id - ) - product_ids.add(version_item.product_id) - if version in versions: - continue - versions.add(version) - version_items.append((project_name, version_item)) + for version_item_by_id in version_items_by_product_id.values(): + for version_item in version_item_by_id.values(): + version = version_item.version + _prod_version = version + if _prod_version < 0: + _prod_version = -1 + product_ids_by_version[_prod_version].add( + version_item.product_id + ) + product_ids.add(version_item.product_id) + if version in versions: + continue + versions.add(version) + version_items.append((project_name, version_item)) def version_sorter(_, item): hero_value = 0 From fafcbe8992e5d22efacdfb72b53bca39c16b5126 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Tue, 26 Nov 2024 20:24:57 +0800 Subject: [PATCH 53/90] check on the active product id before adding version_items --- client/ayon_core/tools/sceneinventory/view.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index 93c889d037..741587c064 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -805,7 +805,7 @@ class SceneInventoryView(QtWidgets.QTreeView): version_items_by_project[project_name] = version_items_by_product_id active_version_id = active_repre_info.version_id - active_product_id = active_repre_info.product_id + # active_product_id = active_repre_info.product_id versions = set() product_ids = set() @@ -818,11 +818,6 @@ class SceneInventoryView(QtWidgets.QTreeView): product_ids_by_version_by_project[project_name] = ( product_ids_by_version ) - versions |= { - version_item.version - for version_item in - version_items_by_product_id[active_product_id].values() - } for version_item_by_id in version_items_by_product_id.values(): for version_item in version_item_by_id.values(): version = version_item.version @@ -838,8 +833,9 @@ class SceneInventoryView(QtWidgets.QTreeView): versions.add(version) version_items.append((project_name, version_item)) - def version_sorter(_, item): + def version_sorter(items): hero_value = 0 + item = items[-1] i_version = item.version if i_version < 0: hero_value = 1 From 10e66c4c39fb6505823255e1e0b040e1c4dacb69 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Tue, 26 Nov 2024 20:30:28 +0800 Subject: [PATCH 54/90] comsetic fix --- client/ayon_core/tools/sceneinventory/model.py | 9 ++++++--- .../ayon_core/tools/sceneinventory/models/containers.py | 3 ++- client/ayon_core/tools/sceneinventory/view.py | 7 +++++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 79af0e5cf5..235b125eab 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -273,7 +273,8 @@ class InventoryModel(QtGui.QStandardItemModel): ) = self._get_status_data(project_name, status_name) repre_name = ( - repre_info.representation_name or "" + repre_info.representation_name or + "" ) container_model_items = [] for container_item in container_items: @@ -281,7 +282,8 @@ class InventoryModel(QtGui.QStandardItemModel): unique_name = repre_name + object_name item = QtGui.QStandardItem() item.setColumnCount(root_item.columnCount()) - item.setData(container_item.namespace, QtCore.Qt.DisplayRole) + item.setData(container_item.namespace, + QtCore.Qt.DisplayRole) item.setData(self.GRAYOUT_COLOR, NAME_COLOR_ROLE) item.setData(self.GRAYOUT_COLOR, VERSION_COLOR_ROLE) item.setData(item_icon, QtCore.Qt.DecorationRole) @@ -290,7 +292,8 @@ class InventoryModel(QtGui.QStandardItemModel): item.setData(version_label, VERSION_LABEL_ROLE) item.setData(container_item.loader_name, LOADER_NAME_ROLE) item.setData(container_item.object_name, OBJECT_NAME_ROLE) - item.setData(container_item.project_name, PROJECT_NAME_ROLE) + item.setData(container_item.project_name, + PROJECT_NAME_ROLE) item.setData(True, IS_CONTAINER_ITEM_ROLE) item.setData(unique_name, ITEM_UNIQUE_NAME_ROLE) container_model_items.append(item) diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index b1cbb38587..ec1ed39e87 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -362,7 +362,8 @@ class ContainersModel: current_project_name = self._controller.get_current_project_name() for container in containers: try: - item = ContainerItem.from_container_data(current_project_name, container) + item = ContainerItem.from_container_data( + current_project_name, container) repre_id = item.representation_id try: uuid.UUID(repre_id) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index 741587c064..ba23e115c0 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -802,7 +802,9 @@ class SceneInventoryView(QtWidgets.QTreeView): ) repre_info_by_project[project_name] = repres_info - version_items_by_project[project_name] = version_items_by_product_id + version_items_by_project[project_name] = ( + version_items_by_product_id + ) active_version_id = active_repre_info.version_id # active_product_id = active_repre_info.product_id @@ -994,7 +996,8 @@ class SceneInventoryView(QtWidgets.QTreeView): def _on_switch_to_versioned(self, item_ids): # Get container items by ID - containers_items_by_id = self._controller.get_container_items_by_id(item_ids) + containers_items_by_id = self._controller.get_container_items_by_id( + item_ids) # Extract project names and their corresponding representation IDs repre_ids_by_project = collections.defaultdict(set) for container_item in containers_items_by_id.values(): From 5d7aeaf0a707efac9347213c56a8d27cd741c88a Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 28 Nov 2024 11:50:48 +0100 Subject: [PATCH 55/90] better variable name --- client/ayon_core/tools/sceneinventory/view.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index ba23e115c0..43c6c8e2d0 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -820,8 +820,8 @@ class SceneInventoryView(QtWidgets.QTreeView): product_ids_by_version_by_project[project_name] = ( product_ids_by_version ) - for version_item_by_id in version_items_by_product_id.values(): - for version_item in version_item_by_id.values(): + for version_items_by_id in version_items_by_product_id.values(): + for version_item in version_items_by_id.values(): version = version_item.version _prod_version = version if _prod_version < 0: From 1776df18e2375e66a0520fddd9d8ef919d6bcdb3 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 28 Nov 2024 11:50:59 +0100 Subject: [PATCH 56/90] don't store project name to version items --- client/ayon_core/tools/sceneinventory/view.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index 43c6c8e2d0..a95a51ae37 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -833,11 +833,10 @@ class SceneInventoryView(QtWidgets.QTreeView): if version in versions: continue versions.add(version) - version_items.append((project_name, version_item)) + version_items.append(version_item) - def version_sorter(items): + def version_sorter(item): hero_value = 0 - item = items[-1] i_version = item.version if i_version < 0: hero_value = 1 @@ -855,8 +854,7 @@ class SceneInventoryView(QtWidgets.QTreeView): version_options = [] active_version_idx = 0 - for idx, item in enumerate(version_items): - project_name, version_item = item + for idx, version_item in enumerate(version_items): version = version_item.version label = format_version(version) if version_item.version_id == active_version_id: From 5c115ce166b70391048b83597d06635783da1118 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 28 Nov 2024 12:04:11 +0100 Subject: [PATCH 57/90] switch dialog can work per project --- .../sceneinventory/switch_dialog/dialog.py | 18 ++++++++------ client/ayon_core/tools/sceneinventory/view.py | 24 ++++++++++++++----- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/switch_dialog/dialog.py b/client/ayon_core/tools/sceneinventory/switch_dialog/dialog.py index 4977ad13c6..a6d88ed44a 100644 --- a/client/ayon_core/tools/sceneinventory/switch_dialog/dialog.py +++ b/client/ayon_core/tools/sceneinventory/switch_dialog/dialog.py @@ -46,8 +46,13 @@ class SwitchAssetDialog(QtWidgets.QDialog): switched = QtCore.Signal() - def __init__(self, controller, parent=None, items=None): - super(SwitchAssetDialog, self).__init__(parent) + def __init__(self, controller, project_name, items, parent=None): + super().__init__(parent) + + current_project_name = controller.get_current_project_name() + folder_id = None + if current_project_name == project_name: + folder_id = controller.get_current_folder_id() self.setWindowTitle("Switch selected items ...") @@ -147,11 +152,10 @@ class SwitchAssetDialog(QtWidgets.QDialog): self._init_repre_name = None self._fill_check = False + self._project_name = project_name + self._folder_id = folder_id - self._project_name = controller.get_current_project_name() - self._folder_id = controller.get_current_folder_id() - - self._current_folder_btn.setEnabled(self._folder_id is not None) + self._current_folder_btn.setEnabled(folder_id is not None) self._controller = controller @@ -159,7 +163,7 @@ class SwitchAssetDialog(QtWidgets.QDialog): self._prepare_content_data() def showEvent(self, event): - super(SwitchAssetDialog, self).showEvent(event) + super().showEvent(event) self._show_timer.start() def refresh(self, init_refresh=False): diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index a95a51ae37..918de6f7a4 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -912,14 +912,26 @@ class SceneInventoryView(QtWidgets.QTreeView): def _show_switch_dialog(self, item_ids): """Display Switch dialog""" - containers_by_id = self._controller.get_containers_by_item_ids( + container_items_by_id = self._controller.get_container_items_by_id( item_ids ) - dialog = SwitchAssetDialog( - self._controller, self, list(containers_by_id.values()) - ) - dialog.switched.connect(self.data_changed.emit) - dialog.show() + container_ids_by_project_name = collections.defaultdict(set) + for container_id, container_item in container_items_by_id.values(): + project_name = container_item.project_name + container_ids_by_project_name[project_name].add(container_id) + + for project_name, container_ids in container_ids_by_project_name.items(): + containers_by_id = self._controller.get_containers_by_item_ids( + container_ids + ) + dialog = SwitchAssetDialog( + self._controller, + project_name, + list(containers_by_id.values()), + self + ) + dialog.switched.connect(self.data_changed.emit) + dialog.show() def _show_remove_warning_dialog(self, item_ids): """Prompt a dialog to inform the user the action will remove items""" From 2eb97b972e0adbaef92d3a381b127d52aefa7b0d Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 28 Nov 2024 12:12:04 +0100 Subject: [PATCH 58/90] show project name on group instead of items --- client/ayon_core/tools/sceneinventory/model.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/model.py b/client/ayon_core/tools/sceneinventory/model.py index 235b125eab..885553acaf 100644 --- a/client/ayon_core/tools/sceneinventory/model.py +++ b/client/ayon_core/tools/sceneinventory/model.py @@ -292,8 +292,6 @@ class InventoryModel(QtGui.QStandardItemModel): item.setData(version_label, VERSION_LABEL_ROLE) item.setData(container_item.loader_name, LOADER_NAME_ROLE) item.setData(container_item.object_name, OBJECT_NAME_ROLE) - item.setData(container_item.project_name, - PROJECT_NAME_ROLE) item.setData(True, IS_CONTAINER_ITEM_ROLE) item.setData(unique_name, ITEM_UNIQUE_NAME_ROLE) container_model_items.append(item) @@ -323,6 +321,7 @@ class InventoryModel(QtGui.QStandardItemModel): group_item.setData(status_short, STATUS_SHORT_ROLE) group_item.setData(status_color, STATUS_COLOR_ROLE) group_item.setData(status_icon, STATUS_ICON_ROLE) + group_item.setData(project_name, PROJECT_NAME_ROLE) group_item.setData( active_site_progress, ACTIVE_SITE_PROGRESS_ROLE From 1ab7a652a3e38fe995105a3658be0ee48cc25a82 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 28 Nov 2024 12:13:55 +0100 Subject: [PATCH 59/90] fix formatting --- client/ayon_core/tools/sceneinventory/view.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index 918de6f7a4..fd67c43ac7 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -920,7 +920,9 @@ class SceneInventoryView(QtWidgets.QTreeView): project_name = container_item.project_name container_ids_by_project_name[project_name].add(container_id) - for project_name, container_ids in container_ids_by_project_name.items(): + for project_name, container_ids in ( + container_ids_by_project_name.items() + ): containers_by_id = self._controller.get_containers_by_item_ids( container_ids ) From 230ae53e4e2d1fbf2ce1c2765b657a6d64365d69 Mon Sep 17 00:00:00 2001 From: ynbot Date: Sat, 30 Nov 2024 14:31:16 +0000 Subject: [PATCH 60/90] [Automated] Update assign_pr_to_project caller workflow --- .github/workflows/assign_pr_to_project.yml | 35 +++++++++++++++++----- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/.github/workflows/assign_pr_to_project.yml b/.github/workflows/assign_pr_to_project.yml index 14e1a02075..e61d281c2a 100644 --- a/.github/workflows/assign_pr_to_project.yml +++ b/.github/workflows/assign_pr_to_project.yml @@ -3,25 +3,46 @@ on: workflow_dispatch: inputs: pr_number: - type: number + type: string description: "Run workflow for this PR number" required: true project_id: - type: number + type: string description: "Github Project Number" required: true - default: 16 + default: "16" pull_request: types: - opened +env: + GH_TOKEN: ${{ github.token }} + jobs: + get-pr-repo: + runs-on: ubuntu-latest + outputs: + pr_repo_name: ${{ steps.get-repo-name.outputs.repo_name || github.event.pull_request.head.repo.full_name }} + + # INFO `github.event.pull_request.head.repo.full_name` is not available on manual triggered (dispatched) runs + steps: + - name: Get PR repo name + if: ${{ github.event_name == 'workflow_dispatch' }} + id: get-repo-name + run: | + repo_name=$(gh pr view ${{ inputs.pr_number }} --json headRepository,headRepositoryOwner --repo ${{ github.repository }} | jq -r '.headRepositoryOwner.login + "/" + .headRepository.name') + echo "repo_name=$repo_name" >> $GITHUB_OUTPUT + auto-assign-pr: - if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + needs: + - get-pr-repo + if: ${{ needs.get-pr-repo.outputs.pr_repo_name == github.repository }} uses: ynput/ops-repo-automation/.github/workflows/pr_to_project.yml@main with: repo: "${{ github.repository }}" - project_id: "${{ inputs.project_id }}" - pull_request_number: "${{ github.event.pull_request.number || inputs.pr_number }}" + project_id: ${{ inputs.project_id != '' && fromJSON(inputs.project_id) || 16 }} + pull_request_number: ${{ github.event.pull_request.number || fromJSON(inputs.pr_number) }} secrets: - token: ${{ secrets.YNPUT_BOT_TOKEN }} + # INFO fallback to default `github.token` is required for PRs from forks + # INFO organization secrets won't be available to forks + token: ${{ secrets.YNPUT_BOT_TOKEN || github.token}} From f5a67f099d291f29789230931bc29f9b9184b4e7 Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Mon, 2 Dec 2024 14:58:42 -0500 Subject: [PATCH 61/90] Append {version} regex to staging dir. --- .../pipeline/create/creator_plugins.py | 17 +++++++++++--- client/ayon_core/pipeline/publish/lib.py | 8 ++++--- client/ayon_core/pipeline/staging_dir.py | 23 +++++++++++-------- 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 93e1f6f5cb..37f3e5b943 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Optional, Dict, Any from abc import ABC, abstractmethod from ayon_core.settings import get_project_settings -from ayon_core.lib import Logger +from ayon_core.lib import Logger, get_version_from_path from ayon_core.pipeline.plugin_discover import ( discover, register_plugin, @@ -860,6 +860,14 @@ class Creator(BaseCreator): else: template_data = {} + # TODO: confirm feature + anatomy_data_settings = self.project_settings["core"]["publish"]["CollectAnatomyInstanceData"] + follow_workfile_version = anatomy_data_settings["follow_workfile_version"] + if follow_workfile_version: + current_workfile = self.create_context.get_current_workfile_path() + workfile_version = get_version_from_path(current_workfile) + template_data = {"version": int(workfile_version)} + staging_dir_info = get_staging_dir_info( create_ctx.get_current_project_entity(), create_ctx.get_current_folder_entity(), @@ -877,12 +885,15 @@ class Creator(BaseCreator): if not staging_dir_info: return None - staging_dir_path = staging_dir_info["stagingDir"] + staging_dir_path = staging_dir_info.dir # path might be already created by get_staging_dir_info os.makedirs(staging_dir_path, exist_ok=True) - instance.transient_data.update(staging_dir_info) + instance.transient_data.update({ + "stagingDir": staging_dir_path, + "stagingDir_persistent": staging_dir_info.persistent, + }) self.log.info(f"Applied staging dir to instance: {staging_dir_path}") diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index c0dfe8c910..b86e439b72 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -710,12 +710,14 @@ def get_instance_staging_dir(instance): always_return_path=True, ) - staging_dir_path = staging_dir_info["stagingDir"] + staging_dir_path = staging_dir_info.dir # path might be already created by get_staging_dir_info os.makedirs(staging_dir_path, exist_ok=True) - - instance.data.update(staging_dir_info) + instance.data.update({ + "stagingDir": staging_dir_path, + "stagingDir_persistent": staging_dir_info.persistent, + }) return staging_dir_path diff --git a/client/ayon_core/pipeline/staging_dir.py b/client/ayon_core/pipeline/staging_dir.py index c8e3251e7b..fa216712b2 100644 --- a/client/ayon_core/pipeline/staging_dir.py +++ b/client/ayon_core/pipeline/staging_dir.py @@ -1,3 +1,5 @@ +from collections import namedtuple + from ayon_core.lib import Logger, filter_profiles, StringTemplate from ayon_core.settings import get_project_settings @@ -6,6 +8,9 @@ from .anatomy import Anatomy from .tempdir import get_temp_dir +StagingDir = namedtuple("StagingDir", ["dir", "persistent"]) + + def get_staging_dir_config( project_name, task_type, @@ -144,7 +149,7 @@ def get_staging_dir_info( suffix (Optional[str]): Optional suffix for staging dir name. Returns: - Optional[Dict[str, Any]]: Staging dir info data + Optional[StagingDir]: Staging dir info data """ task_entity = task_entity or {} @@ -195,24 +200,24 @@ def get_staging_dir_info( ) if staging_dir_config: - return { - "stagingDir": StringTemplate.format_template( + return StagingDir( + StringTemplate.format_template( str(staging_dir_config["template"]["directory"]), ctx_data ), - "stagingDir_persistent": staging_dir_config["persistence"], - } + staging_dir_config["persistence"], + ) # no config found but force an output if always_return_path: - return { - "stagingDir": get_temp_dir( + return StagingDir( + get_temp_dir( project_name=project_entity["name"], anatomy=anatomy, prefix=prefix, suffix=suffix, ), - "stagingDir_persistent": False, - } + False, + ) return None From fa014fa93cdd311ea7086d5cc3216deb14c3c14d Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Mon, 2 Dec 2024 15:02:08 -0500 Subject: [PATCH 62/90] Fix lint. --- client/ayon_core/pipeline/create/creator_plugins.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 37f3e5b943..87f67a3e80 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -861,8 +861,9 @@ class Creator(BaseCreator): template_data = {} # TODO: confirm feature - anatomy_data_settings = self.project_settings["core"]["publish"]["CollectAnatomyInstanceData"] - follow_workfile_version = anatomy_data_settings["follow_workfile_version"] + publish_settings = self.project_settings["core"]["publish"] + anatomy_settings = publish_settings["CollectAnatomyInstanceData"] + follow_workfile_version = anatomy_settings["follow_workfile_version"] if follow_workfile_version: current_workfile = self.create_context.get_current_workfile_path() workfile_version = get_version_from_path(current_workfile) From 9a0e490233151c116ea0ec2e88ad93fde1d7adda Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Tue, 3 Dec 2024 16:52:19 +0800 Subject: [PATCH 63/90] use items() for key, value --- client/ayon_core/tools/sceneinventory/view.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/tools/sceneinventory/view.py b/client/ayon_core/tools/sceneinventory/view.py index fd67c43ac7..bb95e37d4e 100644 --- a/client/ayon_core/tools/sceneinventory/view.py +++ b/client/ayon_core/tools/sceneinventory/view.py @@ -916,7 +916,7 @@ class SceneInventoryView(QtWidgets.QTreeView): item_ids ) container_ids_by_project_name = collections.defaultdict(set) - for container_id, container_item in container_items_by_id.values(): + for container_id, container_item in container_items_by_id.items(): project_name = container_item.project_name container_ids_by_project_name[project_name].add(container_id) From 648c2c52fe0ac20e1f8bcdb74c19aa6089db0fff Mon Sep 17 00:00:00 2001 From: Robin De Lillo Date: Tue, 3 Dec 2024 10:50:09 -0500 Subject: [PATCH 64/90] Apply suggestions from code review Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- .../pipeline/create/creator_plugins.py | 11 +++-- client/ayon_core/pipeline/staging_dir.py | 43 ++++++++++--------- client/ayon_core/pipeline/tempdir.py | 3 +- 3 files changed, 31 insertions(+), 26 deletions(-) diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 87f67a3e80..780cb71fca 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -862,8 +862,11 @@ class Creator(BaseCreator): # TODO: confirm feature publish_settings = self.project_settings["core"]["publish"] - anatomy_settings = publish_settings["CollectAnatomyInstanceData"] - follow_workfile_version = anatomy_settings["follow_workfile_version"] + follow_workfile_version = ( + publish_settings + ["CollectAnatomyInstanceData"] + ["follow_workfile_version"] + ) if follow_workfile_version: current_workfile = self.create_context.get_current_workfile_path() workfile_version = get_version_from_path(current_workfile) @@ -871,8 +874,8 @@ class Creator(BaseCreator): staging_dir_info = get_staging_dir_info( create_ctx.get_current_project_entity(), - create_ctx.get_current_folder_entity(), - create_ctx.get_current_task_entity(), + create_ctx.get_folder_entity(folder_path), + create_ctx.get_task_entity(folder_path, instance.get("task")), product_type, product_name, create_ctx.host_name, diff --git a/client/ayon_core/pipeline/staging_dir.py b/client/ayon_core/pipeline/staging_dir.py index fa216712b2..3c1c7c1ab2 100644 --- a/client/ayon_core/pipeline/staging_dir.py +++ b/client/ayon_core/pipeline/staging_dir.py @@ -8,7 +8,10 @@ from .anatomy import Anatomy from .tempdir import get_temp_dir -StagingDir = namedtuple("StagingDir", ["dir", "persistent"]) +@dataclass +class StagingDir: + directory: str + persistent: bool def get_staging_dir_config( @@ -78,10 +81,9 @@ def get_staging_dir_config( if not template: # template should always be found either from anatomy or from profile - raise ValueError( - "Staging dir profile is misconfigured! " - f"No template was found for profile: {profile}! " - "Check your project settings at: " + raise KeyError( + f"Staging template '{template_name}' was not found." + "Check project anatomy or settings at: " "'ayon+settings://core/tools/publish/custom_staging_dir_profiles'" ) @@ -98,10 +100,8 @@ def _validate_template_name(project_name, template_name, anatomy): """ if template_name not in anatomy.templates["staging"]: raise ValueError( - ( - 'Anatomy of project "{}" does not have set' - ' "{}" template key at Staging Dir section!' - ).format(project_name, template_name) + f'Anatomy of project "{project_name}" does not have set' + f' "{template_name}" template key at Staging Dir category!' ) @@ -131,14 +131,14 @@ def get_staging_dir_info( Arguments: host_name (str): Name of host. project_entity (Dict[str, Any]): Project entity. - folder_entity (Dict[str, Any]): Folder entity. - task_entity (Dict[str, Any]): Task entity. + folder_entity (Optional[Dict[str, Any]]): Folder entity. + task_entity (Optional[Dict[str, Any]]): Task entity. product_type (str): Type of product. product_name (str): Name of product. - anatomy (ayon_core.pipeline.Anatomy): Anatomy object. + anatomy (Optional[Anatomy]): Anatomy object. project_settings (Optional[Dict[str, Any]]): Prepared project settings. - template_data (Optional[Dict[str, Any]]): Data for formatting staging - dir template. + template_data (Optional[Dict[str, Any]]): Additional data for + formatting staging dir template. always_return_path (Optional[bool]): If True, staging dir will be created as tempdir if no staging dir profile is found. Input value False will return None if no staging dir profile is found. @@ -152,7 +152,6 @@ def get_staging_dir_info( Optional[StagingDir]: Staging dir info data """ - task_entity = task_entity or {} log = logger or Logger.get_logger("get_staging_dir_info") if anatomy is None: @@ -185,12 +184,16 @@ def get_staging_dir_info( # add additional template formatting data if template_data: ctx_data.update(template_data) + task_name = task_type = None + if task_entity: + task_name = task_entity["name"] + task_type = task_entity["taskType"] # get staging dir config staging_dir_config = get_staging_dir_config( project_entity["name"], - task_entity.get("taskType"), - task_entity.get("name"), + task_type, + task_name , product_type, product_name, host_name, @@ -200,11 +203,9 @@ def get_staging_dir_info( ) if staging_dir_config: + dir_template = staging_dir_config["template"]["directory"] return StagingDir( - StringTemplate.format_template( - str(staging_dir_config["template"]["directory"]), - ctx_data - ), + dir_template.format_strict(ctx_data), staging_dir_config["persistence"], ) diff --git a/client/ayon_core/pipeline/tempdir.py b/client/ayon_core/pipeline/tempdir.py index af2ff44a8f..52995d3f6a 100644 --- a/client/ayon_core/pipeline/tempdir.py +++ b/client/ayon_core/pipeline/tempdir.py @@ -36,7 +36,8 @@ def get_temp_dir( str: Path to staging dir of instance. """ - prefix = prefix or "ay_tmp_" + if prefix is None: + prefix = "ay_tmp_" suffix = suffix or "" if use_local_temp: From 0672f5c8bb2125ebaafed227f48111e1b1396aeb Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Tue, 3 Dec 2024 11:38:29 -0500 Subject: [PATCH 65/90] Address feedback from PR. --- .../pipeline/create/creator_plugins.py | 57 ++++++++++++------- client/ayon_core/pipeline/publish/lib.py | 2 +- client/ayon_core/pipeline/staging_dir.py | 9 +-- 3 files changed, 44 insertions(+), 24 deletions(-) diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 780cb71fca..6ccafe1bc7 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -833,17 +833,15 @@ class Creator(BaseCreator): """ return self.pre_create_attr_defs - def apply_staging_dir(self, instance): - """Apply staging dir with persistence to instance's transient data. - - Method is called on instance creation and on instance update. + def get_staging_dir(self, instance): + """Return the staging dir and persistence from instance. Args: instance (CreatedInstance): Instance for which should be staging - dir applied. + dir gathered. Returns: - Optional[str]: Staging dir path or None if not applied. + Optional[namedtuple]: Staging dir path and persistence or None """ create_ctx = self.create_context product_name = instance.get("productName") @@ -852,25 +850,32 @@ class Creator(BaseCreator): # this can only work if product name and folder path are available if not product_name or not folder_path: - return + return None - version = instance.get("version") - if version is not None: - template_data = {"version": version} - else: - template_data = {} - - # TODO: confirm feature publish_settings = self.project_settings["core"]["publish"] follow_workfile_version = ( publish_settings ["CollectAnatomyInstanceData"] ["follow_workfile_version"] ) - if follow_workfile_version: + + # Gather version number provided from the instance. + version = instance.get("version") + + # If follow workfile, gather version from workfile path. + if version is None and follow_workfile_version: current_workfile = self.create_context.get_current_workfile_path() workfile_version = get_version_from_path(current_workfile) - template_data = {"version": int(workfile_version)} + version = int(workfile_version) + + # Fill-up version with next version available. + elif version is None: + versions = self.get_next_versions_for_instances( + [instance] + ) + version, = tuple(versions.values()) + + template_data = {"version": version} staging_dir_info = get_staging_dir_info( create_ctx.get_current_project_entity(), @@ -886,12 +891,26 @@ class Creator(BaseCreator): template_data=template_data, ) - if not staging_dir_info: + return staging_dir_info or None + + def apply_staging_dir(self, instance): + """Apply staging dir with persistence to instance's transient data. + + Method is called on instance creation and on instance update. + + Args: + instance (CreatedInstance): Instance for which should be staging + dir applied. + + Returns: + Optional[str]: Staging dir path or None if not applied. + """ + staging_dir_info = self.get_staging_dir(instance) + if staging_dir_info is None: return None - staging_dir_path = staging_dir_info.dir - # path might be already created by get_staging_dir_info + staging_dir_path = staging_dir_info.directory os.makedirs(staging_dir_path, exist_ok=True) instance.transient_data.update({ diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index b86e439b72..2ba40d7687 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -710,7 +710,7 @@ def get_instance_staging_dir(instance): always_return_path=True, ) - staging_dir_path = staging_dir_info.dir + staging_dir_path = staging_dir_info.directory # path might be already created by get_staging_dir_info os.makedirs(staging_dir_path, exist_ok=True) diff --git a/client/ayon_core/pipeline/staging_dir.py b/client/ayon_core/pipeline/staging_dir.py index 3c1c7c1ab2..0317e55720 100644 --- a/client/ayon_core/pipeline/staging_dir.py +++ b/client/ayon_core/pipeline/staging_dir.py @@ -1,6 +1,6 @@ -from collections import namedtuple +from dataclasses import dataclass -from ayon_core.lib import Logger, filter_profiles, StringTemplate +from ayon_core.lib import Logger, filter_profiles from ayon_core.settings import get_project_settings from .template_data import get_template_data @@ -42,7 +42,7 @@ def get_staging_dir_config( Dict or None: Data with directory template and is_persistent or None Raises: - ValueError - if misconfigured template should be used + KeyError - if misconfigured template should be used """ settings = project_settings or get_project_settings(project_name) @@ -129,12 +129,12 @@ def get_staging_dir_info( If `prefix` or `suffix` is not set, default values will be used. Arguments: - host_name (str): Name of host. project_entity (Dict[str, Any]): Project entity. folder_entity (Optional[Dict[str, Any]]): Folder entity. task_entity (Optional[Dict[str, Any]]): Task entity. product_type (str): Type of product. product_name (str): Name of product. + host_name (str): Name of host. anatomy (Optional[Anatomy]): Anatomy object. project_settings (Optional[Dict[str, Any]]): Prepared project settings. template_data (Optional[Dict[str, Any]]): Additional data for @@ -184,6 +184,7 @@ def get_staging_dir_info( # add additional template formatting data if template_data: ctx_data.update(template_data) + task_name = task_type = None if task_entity: task_name = task_entity["name"] From ed7752a4df7fdfc2c1a27de98ccf890c9a5395ea Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Tue, 3 Dec 2024 16:56:53 -0500 Subject: [PATCH 66/90] Fix extract_otio_review --- client/ayon_core/plugins/publish/extract_otio_review.py | 1 + 1 file changed, 1 insertion(+) diff --git a/client/ayon_core/plugins/publish/extract_otio_review.py b/client/ayon_core/plugins/publish/extract_otio_review.py index fb9b269258..c8d2086865 100644 --- a/client/ayon_core/plugins/publish/extract_otio_review.py +++ b/client/ayon_core/plugins/publish/extract_otio_review.py @@ -78,6 +78,7 @@ class ExtractOTIOReview( if otio_review_clips is None: self.log.info(f"Instance `{instance}` has no otioReviewClips") + return # add plugin wide attributes self.representation_files = [] From 9bcc9b40191d0ecdbd435e5a58e714f6dfbd86fe Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:08:04 +0100 Subject: [PATCH 67/90] fix 'realy' typo to 'really' --- client/ayon_core/lib/path_templates.py | 35 +++++++++++++++++++------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/client/ayon_core/lib/path_templates.py b/client/ayon_core/lib/path_templates.py index 9b545f2851..bc4ed648b7 100644 --- a/client/ayon_core/lib/path_templates.py +++ b/client/ayon_core/lib/path_templates.py @@ -292,7 +292,7 @@ class TemplatePartResult: # Used values stored by key with all modifirs # - value is already formatted string # Example: {"version:0>3": "001"} - self._realy_used_values = {} + self._really_used_values: Dict[str, Any] = {} # Concatenated string output after formatting self._output = "" # Is this result from optional part @@ -314,7 +314,7 @@ class TemplatePartResult: if other.optional and not other.solved: return self._used_values.update(other.used_values) - self._realy_used_values.update(other.realy_used_values) + self._really_used_values.update(other.really_used_values) else: raise TypeError("Cannot add data from \"{}\" to \"{}\"".format( @@ -359,8 +359,17 @@ class TemplatePartResult: return self._invalid_optional_types @property - def realy_used_values(self): - return self._realy_used_values + def really_used_values(self) -> Dict[str, Any]: + return self._really_used_values + + @property + def realy_used_values(self) -> Dict[str, Any]: + warnings.warn( + "Property 'realy_used_values' is deprecated." + " Use 'really_used_values' instead.", + DeprecationWarning + ) + return self._really_used_values @property def used_values(self): @@ -391,8 +400,16 @@ class TemplatePartResult: return self.split_keys_to_subdicts(new_used_values) - def add_realy_used_value(self, key, value): - self._realy_used_values[key] = value + def add_really_used_value(self, key: str, value: Any): + self._really_used_values[key] = value + + def add_realy_used_value(self, key: str, value: Any): + warnings.warn( + "Method 'add_realy_used_value' is deprecated." + " Use 'add_really_used_value' instead.", + DeprecationWarning + ) + self.add_really_used_value(key, value) def add_used_value(self, key, value): self._used_values[key] = value @@ -519,8 +536,8 @@ class FormattingPart: result(TemplatePartResult): Object where result is stored. """ key = self._template_base - if key in result.realy_used_values: - result.add_output(result.realy_used_values[key]) + if key in result.really_used_values: + result.add_output(result.really_used_values[key]) return result # ensure key is properly formed [({})] properly closed. @@ -625,7 +642,7 @@ class FormattingPart: used_value = value else: used_value = formatted_value - result.add_realy_used_value(self._field_name, used_value) + result.add_really_used_value(self._field_name, used_value) result.add_used_value(used_key, used_value) result.add_output(formatted_value) return result From a80bbfbd5764b33bf51e44b0e92152e6311b58e0 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:13:33 +0100 Subject: [PATCH 68/90] added basic typehints --- client/ayon_core/lib/path_templates.py | 174 +++++++++++++++---------- 1 file changed, 103 insertions(+), 71 deletions(-) diff --git a/client/ayon_core/lib/path_templates.py b/client/ayon_core/lib/path_templates.py index bc4ed648b7..3e66344ff4 100644 --- a/client/ayon_core/lib/path_templates.py +++ b/client/ayon_core/lib/path_templates.py @@ -2,8 +2,13 @@ import os import re import copy import numbers -from typing import List +import warnings from string import Formatter +import typing +from typing import List, Dict, Any, Set, Optional + +if typing.TYPE_CHECKING: + from typing import Union SUB_DICT_PATTERN = re.compile(r"([^\[\]]+)") OPTIONAL_PATTERN = re.compile(r"(<.*?[^{0]*>)[^0-9]*?") @@ -19,9 +24,7 @@ class TemplateUnsolved(Exception): def __init__(self, template, missing_keys, invalid_types): invalid_type_items = [] for _key, _type in invalid_types.items(): - invalid_type_items.append( - "\"{0}\" {1}".format(_key, str(_type)) - ) + invalid_type_items.append(f"\"{_key}\" {str(_type)}") invalid_types_msg = "" if invalid_type_items: @@ -34,20 +37,21 @@ class TemplateUnsolved(Exception): missing_keys_msg = self.missing_keys_msg.format( ", ".join(missing_keys) ) - super(TemplateUnsolved, self).__init__( + super().__init__( self.msg.format(template, missing_keys_msg, invalid_types_msg) ) class StringTemplate: """String that can be formatted.""" - def __init__(self, template): + def __init__(self, template: str): if not isinstance(template, str): - raise TypeError("<{}> argument must be a string, not {}.".format( - self.__class__.__name__, str(type(template)) - )) + raise TypeError( + f"<{self.__class__.__name__}> argument must be a string," + f" not {str(type(template))}." + ) - self._template = template + self._template: str = template parts = [] formatter = Formatter() @@ -78,15 +82,17 @@ class StringTemplate: if substr: new_parts.append(substr) - self._parts = self.find_optional_parts(new_parts) + self._parts: List["Union[str, OptionalPart, FormattingPart]"] = ( + self.find_optional_parts(new_parts) + ) - def __str__(self): + def __str__(self) -> str: return self.template - def __repr__(self): - return "<{}> {}".format(self.__class__.__name__, self.template) + def __repr__(self) -> str: + return f"<{self.__class__.__name__}> {self.template}" - def __contains__(self, other): + def __contains__(self, other: str) -> bool: return other in self.template def replace(self, *args, **kwargs): @@ -94,10 +100,10 @@ class StringTemplate: return self @property - def template(self): + def template(self) -> str: return self._template - def format(self, data): + def format(self, data: Dict[str, Any]) -> "TemplateResult": """ Figure out with whole formatting. Separate advanced keys (*Like '{project[name]}') from string which must @@ -109,6 +115,7 @@ class StringTemplate: Returns: TemplateResult: Filled or partially filled template containing all data needed or missing for filling template. + """ result = TemplatePartResult() for part in self._parts: @@ -136,23 +143,29 @@ class StringTemplate: invalid_types ) - def format_strict(self, *args, **kwargs): - result = self.format(*args, **kwargs) + def format_strict(self, data: Dict[str, Any]) -> "TemplateResult": + result = self.format(data) result.validate() return result @classmethod - def format_template(cls, template, data): + def format_template( + cls, template: str, data: Dict[str, Any] + ) -> "TemplateResult": objected_template = cls(template) return objected_template.format(data) @classmethod - def format_strict_template(cls, template, data): + def format_strict_template( + cls, template: str, data: Dict[str, Any] + ) -> "TemplateResult": objected_template = cls(template) return objected_template.format_strict(data) @staticmethod - def find_optional_parts(parts): + def find_optional_parts( + parts: List["Union[str, FormattingPart]"] + ) -> List["Union[str, OptionalPart, FormattingPart]"]: new_parts = [] tmp_parts = {} counted_symb = -1 @@ -217,11 +230,11 @@ class TemplateResult(str): of number. """ - used_values = None - solved = None - template = None - missing_keys = None - invalid_types = None + used_values: Dict[str, Any] = None + solved: bool = None + template: str = None + missing_keys: List[str] = None + invalid_types: Dict[str, Any] = None def __new__( cls, filled_template, template, solved, @@ -249,7 +262,7 @@ class TemplateResult(str): self.invalid_types ) - def copy(self): + def copy(self) -> "TemplateResult": cls = self.__class__ return cls( str(self), @@ -260,7 +273,7 @@ class TemplateResult(str): self.invalid_types ) - def normalized(self): + def normalized(self) -> "TemplateResult": """Convert to normalized path.""" cls = self.__class__ @@ -276,27 +289,28 @@ class TemplateResult(str): class TemplatePartResult: """Result to store result of template parts.""" - def __init__(self, optional=False): + def __init__(self, optional: bool = False): # Missing keys or invalid value types of required keys - self._missing_keys = set() - self._invalid_types = {} + self._missing_keys: Set[str] = set() + self._invalid_types: Dict[str, Any] = {} # Missing keys or invalid value types of optional keys - self._missing_optional_keys = set() - self._invalid_optional_types = {} + self._missing_optional_keys: Set[str] = set() + self._invalid_optional_types: Dict[str, Any] = {} # Used values stored by key with origin type # - key without any padding or key modifiers # - value from filling data # Example: {"version": 1} - self._used_values = {} + self._used_values: Dict[str, Any] = {} # Used values stored by key with all modifirs # - value is already formatted string # Example: {"version:0>3": "001"} self._really_used_values: Dict[str, Any] = {} # Concatenated string output after formatting - self._output = "" + self._output: str = "" # Is this result from optional part - self._optional = True + # TODO find out why we don't use 'optional' from args + self._optional: bool = True def add_output(self, other): if isinstance(other, str): @@ -322,7 +336,7 @@ class TemplatePartResult: ) @property - def solved(self): + def solved(self) -> bool: if self.optional: if ( len(self.missing_optional_keys) > 0 @@ -335,27 +349,27 @@ class TemplatePartResult: ) @property - def optional(self): + def optional(self) -> bool: return self._optional @property - def output(self): + def output(self) -> str: return self._output @property - def missing_keys(self): + def missing_keys(self) -> Set[str]: return self._missing_keys @property - def missing_optional_keys(self): + def missing_optional_keys(self) -> Set[str]: return self._missing_optional_keys @property - def invalid_types(self): + def invalid_types(self) -> Dict[str, Any]: return self._invalid_types @property - def invalid_optional_types(self): + def invalid_optional_types(self) -> Dict[str, Any]: return self._invalid_optional_types @property @@ -372,11 +386,11 @@ class TemplatePartResult: return self._really_used_values @property - def used_values(self): + def used_values(self) -> Dict[str, Any]: return self._used_values @staticmethod - def split_keys_to_subdicts(values): + def split_keys_to_subdicts(values: Dict[str, Any]) -> Dict[str, Any]: output = {} formatter = Formatter() for key, value in values.items(): @@ -391,7 +405,7 @@ class TemplatePartResult: data[last_key] = value return output - def get_clean_used_values(self): + def get_clean_used_values(self) -> Dict[str, Any]: new_used_values = {} for key, value in self.used_values.items(): if isinstance(value, FormatObject): @@ -411,16 +425,16 @@ class TemplatePartResult: ) self.add_really_used_value(key, value) - def add_used_value(self, key, value): + def add_used_value(self, key: str, value: Any): self._used_values[key] = value - def add_missing_key(self, key): + def add_missing_key(self, key: str): if self._optional: self._missing_optional_keys.add(key) else: self._missing_keys.add(key) - def add_invalid_type(self, key, value): + def add_invalid_type(self, key: str, value: Any): if self._optional: self._invalid_optional_types[key] = type(value) else: @@ -438,10 +452,10 @@ class FormatObject: def __format__(self, *args, **kwargs): return self.value.__format__(*args, **kwargs) - def __str__(self): + def __str__(self) -> str: return str(self.value) - def __repr__(self): + def __repr__(self) -> str: return self.__str__() @@ -451,9 +465,17 @@ class FormattingPart: Containt only single key to format e.g. "{project[name]}". Args: - template(str): String containing the formatting key. + field_name (str): Name of key. + format_spec (str): Format specification. + conversion (Union[str, None]): Conversion type. + """ - def __init__(self, field_name, format_spec, conversion): + def __init__( + self, + field_name: str, + format_spec: str, + conversion: "Union[str, None]", + ): format_spec_v = "" if format_spec: format_spec_v = f":{format_spec}" @@ -461,26 +483,26 @@ class FormattingPart: if conversion: conversion_v = f"!{conversion}" - self._field_name = field_name - self._format_spec = format_spec_v - self._conversion = conversion_v + self._field_name: str = field_name + self._format_spec: str = format_spec_v + self._conversion: str = conversion_v template_base = f"{field_name}{format_spec_v}{conversion_v}" - self._template_base = template_base - self._template = f"{{{template_base}}}" + self._template_base: str = template_base + self._template: str = f"{{{template_base}}}" @property - def template(self): + def template(self) -> str: return self._template - def __repr__(self): + def __repr__(self) -> str: return "".format(self._template) - def __str__(self): + def __str__(self) -> str: return self._template @staticmethod - def validate_value_type(value): + def validate_value_type(value: Any) -> bool: """Check if value can be used for formatting of single key.""" if isinstance(value, (numbers.Number, FormatObject)): return True @@ -491,7 +513,7 @@ class FormattingPart: return False @staticmethod - def validate_key_is_matched(key): + def validate_key_is_matched(key: str) -> bool: """Validate that opening has closing at correct place. Future-proof, only square brackets are currently used in keys. @@ -528,12 +550,15 @@ class FormattingPart: joined_keys = "".join([f"[{key}]" for key in keys]) return f"{template_base}{joined_keys}" - def format(self, data, result): + def format( + self, data: Dict[str, Any], result: TemplatePartResult + ) -> TemplatePartResult: """Format the formattings string. Args: data(dict): Data that should be used for formatting. result(TemplatePartResult): Object where result is stored. + """ key = self._template_base if key in result.really_used_values: @@ -658,20 +683,27 @@ class OptionalPart: 'FormattingPart'. """ - def __init__(self, parts): - self._parts = parts + def __init__( + self, + parts: List["Union[str, OptionalPart, FormattingPart]"] + ): + self._parts: List["Union[str, OptionalPart, FormattingPart]"] = parts @property - def parts(self): + def parts(self) -> List["Union[str, OptionalPart, FormattingPart]"]: return self._parts - def __str__(self): + def __str__(self) -> str: return "<{}>".format("".join([str(p) for p in self._parts])) - def __repr__(self): + def __repr__(self) -> str: return "".format("".join([str(p) for p in self._parts])) - def format(self, data, result): + def format( + self, + data: Dict[str, Any], + result: TemplatePartResult, + ) -> TemplatePartResult: new_result = TemplatePartResult(True) for part in self._parts: if isinstance(part, str): From 04daa9306c3a0f5d70d24fa75fe936a415c6532f Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:14:59 +0100 Subject: [PATCH 69/90] remove unused import --- client/ayon_core/lib/path_templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/lib/path_templates.py b/client/ayon_core/lib/path_templates.py index 3e66344ff4..e3cae78a87 100644 --- a/client/ayon_core/lib/path_templates.py +++ b/client/ayon_core/lib/path_templates.py @@ -5,7 +5,7 @@ import numbers import warnings from string import Formatter import typing -from typing import List, Dict, Any, Set, Optional +from typing import List, Dict, Any, Set if typing.TYPE_CHECKING: from typing import Union From cc23f407afb8d78816dd8c124b0236cf9b8dd2ad Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Wed, 4 Dec 2024 11:23:29 -0500 Subject: [PATCH 70/90] Address feedback from PR. --- client/ayon_core/pipeline/staging_dir.py | 11 ++++------- client/ayon_core/pipeline/tempdir.py | 6 ++---- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/client/ayon_core/pipeline/staging_dir.py b/client/ayon_core/pipeline/staging_dir.py index 0317e55720..ea22d99389 100644 --- a/client/ayon_core/pipeline/staging_dir.py +++ b/client/ayon_core/pipeline/staging_dir.py @@ -173,13 +173,10 @@ def get_staging_dir_info( ) # add additional data - ctx_data.update({ - "product": { - "type": product_type, - "name": product_name - }, - "root": anatomy.roots - }) + ctx_data["product"] = { + "type": product_type, + "name": product_name + } # add additional template formatting data if template_data: diff --git a/client/ayon_core/pipeline/tempdir.py b/client/ayon_core/pipeline/tempdir.py index 52995d3f6a..fe057b7fc7 100644 --- a/client/ayon_core/pipeline/tempdir.py +++ b/client/ayon_core/pipeline/tempdir.py @@ -65,11 +65,9 @@ def _create_local_staging_dir(prefix, suffix, dirpath=None): str: path to tempdir """ # use pathlib for creating tempdir - staging_dir = Path(tempfile.mkdtemp( + return tempfile.mkdtemp( prefix=prefix, suffix=suffix, dir=dirpath - )) - - return staging_dir.as_posix() + ) def _create_custom_tempdir(project_name, anatomy): From 9f590cd2cec1656c19c407bc77a19dfe728f3fc1 Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Wed, 4 Dec 2024 16:37:25 -0500 Subject: [PATCH 71/90] Implement review representations in OTIO subset resources. --- .../publish/collect_otio_subset_resources.py | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/client/ayon_core/plugins/publish/collect_otio_subset_resources.py b/client/ayon_core/plugins/publish/collect_otio_subset_resources.py index c142036b83..f7b1c9d9b2 100644 --- a/client/ayon_core/plugins/publish/collect_otio_subset_resources.py +++ b/client/ayon_core/plugins/publish/collect_otio_subset_resources.py @@ -149,6 +149,7 @@ class CollectOtioSubsetResources( self.log.info( "frame_start-frame_end: {}-{}".format(frame_start, frame_end)) + review_repre = None if is_sequence: # file sequence way @@ -177,6 +178,11 @@ class CollectOtioSubsetResources( repre = self._create_representation( frame_start, frame_end, collection=collection) + if "review" in instance.data["families"]: + review_repre = self._create_representation( + frame_start, frame_end, collection=collection, + delete=True, review=True) + else: _trim = False dirname, filename = os.path.split(media_ref.target_url) @@ -191,17 +197,26 @@ class CollectOtioSubsetResources( repre = self._create_representation( frame_start, frame_end, file=filename, trim=_trim) + if "review" in instance.data["families"]: + review_repre = self._create_representation( + frame_start, frame_end, + file=filename, delete=True, review=True) + instance.data["originalDirname"] = self.staging_dir + # add representation to instance data if repre: colorspace = instance.data.get("colorspace") # add colorspace data to representation self.set_representation_colorspace( repre, instance.context, colorspace) - # add representation to instance data instance.data["representations"].append(repre) + # add review representation to instance data + if review_repre: + instance.data["representations"].append(review_repre) + self.log.debug(instance.data) def _create_representation(self, start, end, **kwargs): @@ -221,7 +236,8 @@ class CollectOtioSubsetResources( representation_data = { "frameStart": start, "frameEnd": end, - "stagingDir": self.staging_dir + "stagingDir": self.staging_dir, + "tags": [], } if kwargs.get("collection"): @@ -247,8 +263,10 @@ class CollectOtioSubsetResources( "frameEnd": end, }) - if kwargs.get("trim") is True: - representation_data["tags"] = ["trim"] + for tag_name in ("trim", "delete", "review"): + if kwargs.get(tag_name) is True: + representation_data["tags"].append(tag_name) + return representation_data def get_template_name(self, instance): From 156d3e6a1cd1e9807486ba8c5278f382b3c15058 Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Wed, 4 Dec 2024 16:45:20 -0500 Subject: [PATCH 72/90] Fix lint. --- .../ayon_core/plugins/publish/collect_otio_subset_resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/plugins/publish/collect_otio_subset_resources.py b/client/ayon_core/plugins/publish/collect_otio_subset_resources.py index f7b1c9d9b2..cc1ef3edef 100644 --- a/client/ayon_core/plugins/publish/collect_otio_subset_resources.py +++ b/client/ayon_core/plugins/publish/collect_otio_subset_resources.py @@ -215,7 +215,7 @@ class CollectOtioSubsetResources( # add review representation to instance data if review_repre: - instance.data["representations"].append(review_repre) + instance.data["representations"].append(review_repre) self.log.debug(instance.data) From 023e0722f8935b84238292f283a90916e920bedc Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:43:18 +0100 Subject: [PATCH 73/90] capture all possible errors that can happen during UUID conversion --- client/ayon_core/tools/loader/control.py | 2 +- client/ayon_core/tools/sceneinventory/models/containers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/tools/loader/control.py b/client/ayon_core/tools/loader/control.py index 2da77337fb..412e6677f0 100644 --- a/client/ayon_core/tools/loader/control.py +++ b/client/ayon_core/tools/loader/control.py @@ -382,7 +382,7 @@ class LoaderController(BackendLoaderController, FrontendLoaderController): try: uuid.UUID(repre_id) repre_ids.add(repre_id) - except ValueError: + except (ValueError, TypeError, AttributeError): pass product_ids = self._products_model.get_product_ids_by_repre_ids( diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index 9059485dff..572a96976b 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -230,7 +230,7 @@ class ContainersModel: for repre_id in representation_ids: try: uuid.UUID(repre_id) - except ValueError: + except (ValueError, TypeError, AttributeError): output[repre_id] = RepresentationInfo.new_invalid() continue repre_info = self._repre_info_by_id.get(repre_id) From 2292ecbac11da62427c2007665f587123503cc66 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:45:19 +0100 Subject: [PATCH 74/90] log about invalid representation id --- client/ayon_core/tools/sceneinventory/models/containers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index 572a96976b..08b86f6456 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -4,6 +4,7 @@ import collections import ayon_api from ayon_api.graphql import GraphQlQuery +from ayon_core.lib import Logger from ayon_core.host import ILoadHost from ayon_core.tools.common_models.projects import StatusStates @@ -196,6 +197,7 @@ class ContainersModel: self._container_items_by_id = {} self._version_items_by_product_id = {} self._repre_info_by_id = {} + self._log = Logger.get_logger("ContainersModel") def reset(self): self._items_cache = None @@ -368,6 +370,10 @@ class ContainersModel: try: uuid.UUID(repre_id) except (ValueError, TypeError, AttributeError): + self._log.warning( + "Container contains invalid representation id." + f"\n{container}" + ) # Fake not existing representation id so container # is shown in UI but as invalid item.representation_id = invalid_ids_mapping.setdefault( From 373df562543b1fed3c8d00d0b425cd6cbddf61aa Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:45:35 +0100 Subject: [PATCH 75/90] fix calling of missing method --- client/ayon_core/tools/sceneinventory/models/containers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index 08b86f6456..f25ef2b94c 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -380,10 +380,10 @@ class ContainersModel: repre_id, uuid.uuid4().hex ) - except Exception as e: + except Exception: # skip item if required data are missing - self._controller.log_error( - f"Failed to create item: {e}" + self._log.warning( + f"Failed to create container item", exc_info=True ) continue From b6d3ddc1c8f288e98d68f528334c8c61394f3ecd Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:54:47 +0100 Subject: [PATCH 76/90] more safeguard for invalid containers --- client/ayon_core/tools/loader/control.py | 14 +++++++------- .../tools/sceneinventory/models/containers.py | 17 +++++++++++------ 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/client/ayon_core/tools/loader/control.py b/client/ayon_core/tools/loader/control.py index 412e6677f0..4ce220f282 100644 --- a/client/ayon_core/tools/loader/control.py +++ b/client/ayon_core/tools/loader/control.py @@ -372,14 +372,14 @@ class LoaderController(BackendLoaderController, FrontendLoaderController): repre_ids = set() for container in containers: - repre_id = container.get("representation") - # Ignore invalid representation ids. - # - invalid representation ids may be available if e.g. is - # opened scene from OpenPype whe 'ObjectId' was used instead - # of 'uuid'. - # NOTE: Server call would crash if there is any invalid id. - # That would cause crash we won't get any information. try: + repre_id = container.get("representation") + # Ignore invalid representation ids. + # - invalid representation ids may be available if e.g. is + # opened scene from OpenPype whe 'ObjectId' was used instead + # of 'uuid'. + # NOTE: Server call would crash if there is any invalid id. + # That would cause crash we won't get any information. uuid.UUID(repre_id) repre_ids.add(repre_id) except (ValueError, TypeError, AttributeError): diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index f25ef2b94c..c761121d4d 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -350,12 +350,14 @@ class ContainersModel: return host = self._controller.get_host() - if isinstance(host, ILoadHost): - containers = list(host.get_containers()) - elif hasattr(host, "ls"): - containers = list(host.ls()) - else: - containers = [] + containers = [] + try: + if isinstance(host, ILoadHost): + containers = list(host.get_containers()) + elif hasattr(host, "ls"): + containers = list(host.ls()) + except Exception: + self._log.error("Failed to get containers", exc_info=True) container_items = [] containers_by_id = {} @@ -363,6 +365,9 @@ class ContainersModel: invalid_ids_mapping = {} current_project_name = self._controller.get_current_project_name() for container in containers: + if not container: + continue + try: item = ContainerItem.from_container_data( current_project_name, container) From 3c697b92f57fa597364da39b7e73a03ad963e563 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:57:22 +0100 Subject: [PATCH 77/90] remove unnecessary f-string --- client/ayon_core/tools/sceneinventory/models/containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/tools/sceneinventory/models/containers.py b/client/ayon_core/tools/sceneinventory/models/containers.py index c761121d4d..f841f87c8e 100644 --- a/client/ayon_core/tools/sceneinventory/models/containers.py +++ b/client/ayon_core/tools/sceneinventory/models/containers.py @@ -388,7 +388,7 @@ class ContainersModel: except Exception: # skip item if required data are missing self._log.warning( - f"Failed to create container item", exc_info=True + "Failed to create container item", exc_info=True ) continue From a26e9207d2957402c3de2a23fc0633d85e094675 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 5 Dec 2024 15:10:33 +0100 Subject: [PATCH 78/90] fix long line --- client/ayon_core/tools/loader/control.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/tools/loader/control.py b/client/ayon_core/tools/loader/control.py index 4ce220f282..16cf7c31c7 100644 --- a/client/ayon_core/tools/loader/control.py +++ b/client/ayon_core/tools/loader/control.py @@ -376,8 +376,8 @@ class LoaderController(BackendLoaderController, FrontendLoaderController): repre_id = container.get("representation") # Ignore invalid representation ids. # - invalid representation ids may be available if e.g. is - # opened scene from OpenPype whe 'ObjectId' was used instead - # of 'uuid'. + # opened scene from OpenPype whe 'ObjectId' was used + # instead of 'uuid'. # NOTE: Server call would crash if there is any invalid id. # That would cause crash we won't get any information. uuid.UUID(repre_id) From 72862554b4df26b304efb609200b901748c01c4f Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 5 Dec 2024 16:13:46 +0100 Subject: [PATCH 79/90] Fix missing slate frame frames_to_render should contain also additional slate frame --- client/ayon_core/pipeline/farm/pyblish_functions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 559561c827..c399855044 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -384,6 +384,7 @@ def prepare_representations( frame_end = frames_to_render[-1] if skeleton_data.get("slate"): frame_start -= 1 + frames_to_render.insert(0, frame_start) files = _get_real_files_to_rendered(collection, frames_to_render) From 49c5b875a8a275adb6264474df31a2fc4be3584a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 5 Dec 2024 16:19:58 +0100 Subject: [PATCH 80/90] Refactor removed filtering by frame Remainder has now real way to find frame pattern from single file. Doesn't make sense to filter on single file. --- client/ayon_core/pipeline/farm/pyblish_functions.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 559561c827..147763391b 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -435,13 +435,10 @@ def prepare_representations( " This may cause issues on farm." ).format(staging)) - files = _get_real_files_to_rendered( - [os.path.basename(remainder)], frames_to_render) - rep = { "name": ext, "ext": ext, - "files": files[0], + "files": os.path.basename(remainder), "stagingDir": staging, } From 2a20a9d169b2ed49eada6679166bd61a5db8889e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 5 Dec 2024 16:20:57 +0100 Subject: [PATCH 81/90] Refactor filtering based on frame_to_render Previous implementation was naive and could be dangerous. Updated docstrings. Renamed. --- .../pipeline/farm/pyblish_functions.py | 47 ++++++++++++------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 147763391b..722dc4b5c6 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -385,8 +385,7 @@ def prepare_representations( if skeleton_data.get("slate"): frame_start -= 1 - files = _get_real_files_to_rendered(collection, frames_to_render) - + files = _get_real_files_to_render(collection, frames_to_render) # explicitly disable review by user preview = preview and not do_not_add_review rep = { @@ -492,31 +491,47 @@ def _get_real_frames_to_render(frames): return frames_to_render -def _get_real_files_to_rendered(collection, frames_to_render): - """Use expected files based on real frames_to_render. +def _get_real_files_to_render(collection, frames_to_render): + """Filter files with frames that should be really rendered. + + 'expected_files' are collected from DCC based on timeline setting. This is + being calculated differently in each DCC. Filtering here is on single place + + But artists might explicitly set frames they want to render in Publisher UI + This range would override and filter previously prepared expected files + from DCC. - Artists might explicitly set frames they want to render via Publisher UI. - This uses this value to filter out files Args: - frames_to_render (list): of str '1001' + collection (clique.Collection): absolute paths + frames_to_render (list[int]): of int 1001 + Returns: + (list[str]) + + Example: + -------- + + expectedFiles = [ + "foo_v01.0001.exr", + "foo_v01.0002.exr", + ] + frames_to_render = '0001' + >> + ["foo_v01.0001.exr"] - only explicitly requested frame returned """ - files = [os.path.basename(f) for f in list(collection)] - file_name, extracted_frame = list(collect_frames(files).items())[0] - - if not extracted_frame: - return files - - found_frame_pattern_length = len(extracted_frame) + found_frame_pattern_length = collection.padding normalized_frames_to_render = { str(frame_to_render).zfill(found_frame_pattern_length) for frame_to_render in frames_to_render } + head_name = os.path.basename(collection.head) + + file_names = [os.path.basename(f) for f in collection] return [ file_name - for file_name in files + for file_name in file_names if any( - frame in file_name + f"{head_name}{frame}{collection.tail}" == file_name for frame in normalized_frames_to_render ) ] From b832c850c33cba13bb63cf6b7b58feaba5297510 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Thu, 5 Dec 2024 16:57:47 +0100 Subject: [PATCH 82/90] Update client/ayon_core/plugins/publish/collect_otio_subset_resources.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- .../plugins/publish/collect_otio_subset_resources.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/plugins/publish/collect_otio_subset_resources.py b/client/ayon_core/plugins/publish/collect_otio_subset_resources.py index cc1ef3edef..2d8e91fe09 100644 --- a/client/ayon_core/plugins/publish/collect_otio_subset_resources.py +++ b/client/ayon_core/plugins/publish/collect_otio_subset_resources.py @@ -199,8 +199,8 @@ class CollectOtioSubsetResources( if "review" in instance.data["families"]: review_repre = self._create_representation( - frame_start, frame_end, - file=filename, delete=True, review=True) + frame_start, frame_end, + file=filename, delete=True, review=True) instance.data["originalDirname"] = self.staging_dir From 81c71a757fae3ea0120b397017192336becb4a5a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 5 Dec 2024 17:15:40 +0100 Subject: [PATCH 83/90] Prepare normalized expected file names Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/farm/pyblish_functions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 722dc4b5c6..a7780ba97c 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -525,7 +525,10 @@ def _get_real_files_to_render(collection, frames_to_render): } head_name = os.path.basename(collection.head) - + normalized_filenames = { + f"{head_name}{frame}{collection.tail}" + for frame in normalized_frames_to_render + } file_names = [os.path.basename(f) for f in collection] return [ file_name From c886be22bdf2494e952c39d4f38959e35c732150 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 5 Dec 2024 17:19:19 +0100 Subject: [PATCH 84/90] Refactor logic to use set --- client/ayon_core/pipeline/farm/pyblish_functions.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index a7780ba97c..1f6e17972a 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -533,10 +533,7 @@ def _get_real_files_to_render(collection, frames_to_render): return [ file_name for file_name in file_names - if any( - f"{head_name}{frame}{collection.tail}" == file_name - for frame in normalized_frames_to_render - ) + if file_name in normalized_filenames ] From 06f6b519f0d45aa5e9dfadf3f184106d98166eb7 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 9 Dec 2024 12:42:07 +0100 Subject: [PATCH 85/90] Refactor logic to be even simpler --- .../pipeline/farm/pyblish_functions.py | 25 ++++++------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 1f6e17972a..902aa41af4 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -518,23 +518,14 @@ def _get_real_files_to_render(collection, frames_to_render): >> ["foo_v01.0001.exr"] - only explicitly requested frame returned """ - found_frame_pattern_length = collection.padding - normalized_frames_to_render = { - str(frame_to_render).zfill(found_frame_pattern_length) - for frame_to_render in frames_to_render - } - - head_name = os.path.basename(collection.head) - normalized_filenames = { - f"{head_name}{frame}{collection.tail}" - for frame in normalized_frames_to_render - } - file_names = [os.path.basename(f) for f in collection] - return [ - file_name - for file_name in file_names - if file_name in normalized_filenames - ] + included_frames = set(collection.indexes).intersection(frames_to_render) + real_collection = clique.Collection( + collection.head, + collection.tail, + collection.padding, + indexes=included_frames + ) + return list(real_collection) def create_instances_for_aov(instance, skeleton, aov_filter, From 6836a7f79b546e7dc500d2c8ba911469e3035012 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 9 Dec 2024 13:51:09 +0100 Subject: [PATCH 86/90] Fix docstring --- client/ayon_core/pipeline/farm/pyblish_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 902aa41af4..425b616adc 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -514,7 +514,7 @@ def _get_real_files_to_render(collection, frames_to_render): "foo_v01.0001.exr", "foo_v01.0002.exr", ] - frames_to_render = '0001' + frames_to_render = 1 >> ["foo_v01.0001.exr"] - only explicitly requested frame returned """ From d65865563f7aa34ede47b17e2dfc671a03f84255 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 9 Dec 2024 14:53:17 +0100 Subject: [PATCH 87/90] Fix expected files must be only file names --- client/ayon_core/pipeline/farm/pyblish_functions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 425b616adc..71068cb093 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -525,7 +525,8 @@ def _get_real_files_to_render(collection, frames_to_render): collection.padding, indexes=included_frames ) - return list(real_collection) + real_full_paths = list(real_collection) + return [os.path.basename(file_url) for file_url in real_full_paths] def create_instances_for_aov(instance, skeleton, aov_filter, From 4bf03f0a51ed5b73034b537f55d6c853fb572b3a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 9 Dec 2024 15:39:08 +0100 Subject: [PATCH 88/90] Remove unused import 'collect_frames' from pyblish_functions --- client/ayon_core/pipeline/farm/pyblish_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index 616e25596c..e48d99602e 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -7,7 +7,7 @@ from copy import deepcopy import attr import ayon_api import clique -from ayon_core.lib import Logger, collect_frames +from ayon_core.lib import Logger from ayon_core.pipeline import ( get_current_project_name, get_representation_path, From 55eb8e497fc30f3de22229fea904b223136cf33f Mon Sep 17 00:00:00 2001 From: Ynbot Date: Mon, 9 Dec 2024 15:13:06 +0000 Subject: [PATCH 89/90] [Automated] Add generated package files from main --- client/ayon_core/version.py | 2 +- package.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/version.py b/client/ayon_core/version.py index b2ece45120..0d3b533f57 100644 --- a/client/ayon_core/version.py +++ b/client/ayon_core/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring AYON addon 'core' version.""" -__version__ = "1.0.10+dev" +__version__ = "1.0.11" diff --git a/package.py b/package.py index 58ae5c08d9..464fbb007b 100644 --- a/package.py +++ b/package.py @@ -1,6 +1,6 @@ name = "core" title = "Core" -version = "1.0.10+dev" +version = "1.0.11" client_dir = "ayon_core" diff --git a/pyproject.toml b/pyproject.toml index d7cf9fa6ed..ab452816ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ [tool.poetry] name = "ayon-core" -version = "1.0.10+dev" +version = "1.0.11" description = "" authors = ["Ynput Team "] readme = "README.md" From a84a3cad33be7e6fc29d6b5539f1e33f556374e4 Mon Sep 17 00:00:00 2001 From: Ynbot Date: Mon, 9 Dec 2024 15:13:44 +0000 Subject: [PATCH 90/90] [Automated] Update version in package.py for develop --- client/ayon_core/version.py | 2 +- package.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/version.py b/client/ayon_core/version.py index 0d3b533f57..a4ae75914c 100644 --- a/client/ayon_core/version.py +++ b/client/ayon_core/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring AYON addon 'core' version.""" -__version__ = "1.0.11" +__version__ = "1.0.11+dev" diff --git a/package.py b/package.py index 464fbb007b..b8d88fc2ad 100644 --- a/package.py +++ b/package.py @@ -1,6 +1,6 @@ name = "core" title = "Core" -version = "1.0.11" +version = "1.0.11+dev" client_dir = "ayon_core" diff --git a/pyproject.toml b/pyproject.toml index ab452816ad..bdfaf797e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ [tool.poetry] name = "ayon-core" -version = "1.0.11" +version = "1.0.11+dev" description = "" authors = ["Ynput Team "] readme = "README.md"