From dee1d51640decb7e14b84a041a2e38389316499c Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Wed, 20 Aug 2025 18:13:30 +0200 Subject: [PATCH] cache entities --- .../ayon_core/tools/loader/models/actions.py | 185 +++++++++++++++--- 1 file changed, 163 insertions(+), 22 deletions(-) diff --git a/client/ayon_core/tools/loader/models/actions.py b/client/ayon_core/tools/loader/models/actions.py index c41119ac45..1e8bfe7ae1 100644 --- a/client/ayon_core/tools/loader/models/actions.py +++ b/client/ayon_core/tools/loader/models/actions.py @@ -5,6 +5,7 @@ import traceback import inspect import collections import uuid +from typing import Callable, Any import ayon_api @@ -53,6 +54,14 @@ class LoaderActionsModel: self._repre_loaders = NestedCacheItem( levels=1, lifetime=self.loaders_cache_lifetime) + self._projects_cache = NestedCacheItem(levels=1, lifetime=60) + self._folders_cache = NestedCacheItem(levels=2, lifetime=300) + self._tasks_cache = NestedCacheItem(levels=2, lifetime=300) + self._products_cache = NestedCacheItem(levels=2, lifetime=300) + self._versions_cache = NestedCacheItem(levels=2, lifetime=1200) + self._representations_cache = NestedCacheItem(levels=2, lifetime=1200) + self._repre_parents_cache = NestedCacheItem(levels=2, lifetime=1200) + def reset(self): """Reset the model with all cached items.""" @@ -61,6 +70,12 @@ class LoaderActionsModel: self._product_loaders.reset() self._repre_loaders.reset() + self._folders_cache.reset() + self._tasks_cache.reset() + self._products_cache.reset() + self._versions_cache.reset() + self._representations_cache.reset() + self._repre_parents_cache.reset() def get_action_items( self, @@ -358,8 +373,8 @@ class LoaderActionsModel: if not project_name and not version_ids: return version_context_by_id, repre_context_by_id - version_entities = ayon_api.get_versions( - project_name, version_ids=version_ids + version_entities = self._get_versions( + project_name, version_ids ) version_entities_by_id = {} version_entities_by_product_id = collections.defaultdict(list) @@ -370,18 +385,18 @@ class LoaderActionsModel: version_entities_by_product_id[product_id].append(version_entity) _product_ids = set(version_entities_by_product_id.keys()) - _product_entities = ayon_api.get_products( - project_name, product_ids=_product_ids + _product_entities = self._get_products( + project_name, _product_ids ) product_entities_by_id = {p["id"]: p for p in _product_entities} _folder_ids = {p["folderId"] for p in product_entities_by_id.values()} - _folder_entities = ayon_api.get_folders( - project_name, folder_ids=_folder_ids + _folder_entities = self._get_folders( + project_name, _folder_ids ) folder_entities_by_id = {f["id"]: f for f in _folder_entities} - project_entity = ayon_api.get_project(project_name) + project_entity = self._get_project(project_name) for version_id, version_entity in version_entities_by_id.items(): product_id = version_entity["productId"] @@ -395,8 +410,15 @@ class LoaderActionsModel: "version": version_entity, } - repre_entities = ayon_api.get_representations( - project_name, version_ids=version_ids) + all_repre_ids = set() + for repre_ids in self._get_repre_ids_by_version_ids( + project_name, version_ids + ).values(): + all_repre_ids |= repre_ids + + repre_entities = self._get_representations( + project_name, all_repre_ids + ) for repre_entity in repre_entities: version_id = repre_entity["versionId"] version_entity = version_entities_by_id[version_id] @@ -439,34 +461,35 @@ class LoaderActionsModel: if not project_name and not repre_ids: return version_context_by_id, repre_context_by_id - repre_entities = list(ayon_api.get_representations( - project_name, representation_ids=repre_ids - )) + repre_entities = self._get_representations( + project_name, repre_ids + ) version_ids = {r["versionId"] for r in repre_entities} - version_entities = ayon_api.get_versions( - project_name, version_ids=version_ids + version_entities = self._get_versions( + project_name, version_ids ) version_entities_by_id = { v["id"]: v for v in version_entities } product_ids = {v["productId"] for v in version_entities_by_id.values()} - product_entities = ayon_api.get_products( - project_name, product_ids=product_ids + product_entities = self._get_products( + project_name, product_ids + ) product_entities_by_id = { p["id"]: p for p in product_entities } folder_ids = {p["folderId"] for p in product_entities_by_id.values()} - folder_entities = ayon_api.get_folders( - project_name, folder_ids=folder_ids + folder_entities = self._get_folders( + project_name, folder_ids ) folder_entities_by_id = { f["id"]: f for f in folder_entities } - project_entity = ayon_api.get_project(project_name) + project_entity = self._get_project(project_name) version_context_by_id = {} for version_id, version_entity in version_entities_by_id.items(): @@ -498,6 +521,124 @@ class LoaderActionsModel: } return version_context_by_id, repre_context_by_id + def _get_project(self, project_name: str) -> dict[str, Any]: + cache = self._projects_cache[project_name] + if not cache.is_valid: + cache.update_data(ayon_api.get_project(project_name)) + return cache.get_data() + + def _get_folders( + self, project_name: str, folder_ids: set[str] + ) -> list[dict[str, Any]]: + """Get folders by ids.""" + return self._get_entities( + project_name, + folder_ids, + self._folders_cache, + ayon_api.get_folders, + "folder_ids", + ) + + def _get_products( + self, project_name: str, product_ids: set[str] + ) -> list[dict[str, Any]]: + """Get products by ids.""" + return self._get_entities( + project_name, + product_ids, + self._products_cache, + ayon_api.get_products, + "product_ids", + ) + + def _get_versions( + self, project_name: str, version_ids: set[str] + ) -> list[dict[str, Any]]: + """Get versions by ids.""" + return self._get_entities( + project_name, + version_ids, + self._versions_cache, + ayon_api.get_versions, + "version_ids", + ) + + def _get_representations( + self, project_name: str, representation_ids: set[str] + ) -> list[dict[str, Any]]: + """Get representations by ids.""" + return self._get_entities( + project_name, + representation_ids, + self._representations_cache, + ayon_api.get_representations, + "representation_ids", + ) + + def _get_repre_ids_by_version_ids( + self, project_name: str, version_ids: set[str] + ) -> dict[str, set[str]]: + output = {} + if not version_ids: + return output + + project_cache = self._repre_parents_cache[project_name] + missing_ids = set() + for version_id in version_ids: + cache = project_cache[version_id] + if cache.is_valid: + output[version_id] = cache.get_data() + else: + missing_ids.add(version_id) + + if missing_ids: + repre_cache = self._representations_cache[project_name] + repres_by_parent_id = collections.defaultdict(list) + for repre in ayon_api.get_representations( + project_name, version_ids=missing_ids + ): + version_id = repre["versionId"] + repre_cache[repre["id"]].update_data(repre) + repres_by_parent_id[version_id].append(repre) + + for version_id, repres in repres_by_parent_id.items(): + repre_ids = { + repre["id"] + for repre in repres + } + output[version_id] = set(repre_ids) + project_cache[version_id].update_data(repre_ids) + + return output + + def _get_entities( + self, + project_name: str, + entity_ids: set[str], + cache: NestedCacheItem, + getter: Callable, + filter_arg: str, + ) -> list[dict[str, Any]]: + entities = [] + if not entity_ids: + return entities + + missing_ids = set() + project_cache = cache[project_name] + for entity_id in entity_ids: + entity_cache = project_cache[entity_id] + if entity_cache.is_valid: + entities.append(entity_cache.get_data()) + else: + missing_ids.add(entity_id) + + if missing_ids: + for entity in getter(project_name, **{filter_arg: missing_ids}): + entities.append(entity) + entity_id = entity["id"] + project_cache[entity_id].update_data(entity) + return entities + def _get_action_items_for_contexts( self, project_name, @@ -601,12 +742,12 @@ class LoaderActionsModel: project_name, version_ids=version_ids )) product_ids = {v["productId"] for v in version_entities} - product_entities = ayon_api.get_products( - project_name, product_ids=product_ids + product_entities = self._get_products( + project_name, product_ids ) product_entities_by_id = {p["id"]: p for p in product_entities} folder_ids = {p["folderId"] for p in product_entities_by_id.values()} - folder_entities = ayon_api.get_folders( + folder_entities = self._get_folders( project_name, folder_ids=folder_ids ) folder_entities_by_id = {f["id"]: f for f in folder_entities}