cache entities

client/ayon_core/tools/loader/models/actions.py

@@ -5,6 +5,7 @@ import traceback
 import inspect
 import collections
 import uuid
+from typing import Callable, Any
 
 import ayon_api
 
@@ -53,6 +54,14 @@ class LoaderActionsModel:
         self._repre_loaders = NestedCacheItem(
             levels=1, lifetime=self.loaders_cache_lifetime)
 
+        self._projects_cache = NestedCacheItem(levels=1, lifetime=60)
+        self._folders_cache = NestedCacheItem(levels=2, lifetime=300)
+        self._tasks_cache = NestedCacheItem(levels=2, lifetime=300)
+        self._products_cache = NestedCacheItem(levels=2, lifetime=300)
+        self._versions_cache = NestedCacheItem(levels=2, lifetime=1200)
+        self._representations_cache = NestedCacheItem(levels=2, lifetime=1200)
+        self._repre_parents_cache = NestedCacheItem(levels=2, lifetime=1200)
+
     def reset(self):
         """Reset the model with all cached items."""
 
@@ -61,6 +70,12 @@ class LoaderActionsModel:
         self._product_loaders.reset()
         self._repre_loaders.reset()
 
+        self._folders_cache.reset()
+        self._tasks_cache.reset()
+        self._products_cache.reset()
+        self._versions_cache.reset()
+        self._representations_cache.reset()
+        self._repre_parents_cache.reset()
 
     def get_action_items(
         self,
@@ -358,8 +373,8 @@ class LoaderActionsModel:
         if not project_name and not version_ids:
             return version_context_by_id, repre_context_by_id
 
-        version_entities = ayon_api.get_versions(
-            project_name, version_ids=version_ids
+        version_entities = self._get_versions(
+            project_name, version_ids
         )
         version_entities_by_id = {}
         version_entities_by_product_id = collections.defaultdict(list)
@@ -370,18 +385,18 @@ class LoaderActionsModel:
             version_entities_by_product_id[product_id].append(version_entity)
 
         _product_ids = set(version_entities_by_product_id.keys())
-        _product_entities = ayon_api.get_products(
-            project_name, product_ids=_product_ids
+        _product_entities = self._get_products(
+            project_name, _product_ids
         )
         product_entities_by_id = {p["id"]: p for p in _product_entities}
 
         _folder_ids = {p["folderId"] for p in product_entities_by_id.values()}
-        _folder_entities = ayon_api.get_folders(
-            project_name, folder_ids=_folder_ids
+        _folder_entities = self._get_folders(
+            project_name, _folder_ids
         )
         folder_entities_by_id = {f["id"]: f for f in _folder_entities}
 
-        project_entity = ayon_api.get_project(project_name)
+        project_entity = self._get_project(project_name)
 
         for version_id, version_entity in version_entities_by_id.items():
             product_id = version_entity["productId"]
@@ -395,8 +410,15 @@ class LoaderActionsModel:
                 "version": version_entity,
             }
 
-        repre_entities = ayon_api.get_representations(
-            project_name, version_ids=version_ids)
+        all_repre_ids = set()
+        for repre_ids in self._get_repre_ids_by_version_ids(
+            project_name, version_ids
+        ).values():
+            all_repre_ids |= repre_ids
+
+        repre_entities = self._get_representations(
+            project_name, all_repre_ids
+        )
         for repre_entity in repre_entities:
             version_id = repre_entity["versionId"]
             version_entity = version_entities_by_id[version_id]
@@ -439,34 +461,35 @@ class LoaderActionsModel:
         if not project_name and not repre_ids:
             return version_context_by_id, repre_context_by_id
 
-        repre_entities = list(ayon_api.get_representations(
-            project_name, representation_ids=repre_ids
-        ))
+        repre_entities = self._get_representations(
+            project_name, repre_ids
+        )
         version_ids = {r["versionId"] for r in repre_entities}
-        version_entities = ayon_api.get_versions(
-            project_name, version_ids=version_ids
+        version_entities = self._get_versions(
+            project_name, version_ids
         )
         version_entities_by_id = {
             v["id"]: v for v in version_entities
         }
 
         product_ids = {v["productId"] for v in version_entities_by_id.values()}
-        product_entities = ayon_api.get_products(
-            project_name, product_ids=product_ids
+        product_entities = self._get_products(
+            project_name, product_ids
+
         )
         product_entities_by_id = {
             p["id"]: p for p in product_entities
         }
 
         folder_ids = {p["folderId"] for p in product_entities_by_id.values()}
-        folder_entities = ayon_api.get_folders(
-            project_name, folder_ids=folder_ids
+        folder_entities = self._get_folders(
+            project_name, folder_ids
         )
         folder_entities_by_id = {
             f["id"]: f for f in folder_entities
         }
 
-        project_entity = ayon_api.get_project(project_name)
+        project_entity = self._get_project(project_name)
 
         version_context_by_id = {}
         for version_id, version_entity in version_entities_by_id.items():
@@ -498,6 +521,124 @@ class LoaderActionsModel:
             }
         return version_context_by_id, repre_context_by_id
 
+    def _get_project(self, project_name: str) -> dict[str, Any]:
+        cache = self._projects_cache[project_name]
+        if not cache.is_valid:
+            cache.update_data(ayon_api.get_project(project_name))
+        return cache.get_data()
+
+    def _get_folders(
+        self, project_name: str, folder_ids: set[str]
+    ) -> list[dict[str, Any]]:
+        """Get folders by ids."""
+        return self._get_entities(
+            project_name,
+            folder_ids,
+            self._folders_cache,
+            ayon_api.get_folders,
+            "folder_ids",
+        )
+
+    def _get_products(
+        self, project_name: str, product_ids: set[str]
+    ) -> list[dict[str, Any]]:
+        """Get products by ids."""
+        return self._get_entities(
+            project_name,
+            product_ids,
+            self._products_cache,
+            ayon_api.get_products,
+            "product_ids",
+        )
+
+    def _get_versions(
+        self, project_name: str, version_ids: set[str]
+    ) -> list[dict[str, Any]]:
+        """Get versions by ids."""
+        return self._get_entities(
+            project_name,
+            version_ids,
+            self._versions_cache,
+            ayon_api.get_versions,
+            "version_ids",
+        )
+
+    def _get_representations(
+        self, project_name: str, representation_ids: set[str]
+    ) -> list[dict[str, Any]]:
+        """Get representations by ids."""
+        return self._get_entities(
+            project_name,
+            representation_ids,
+            self._representations_cache,
+            ayon_api.get_representations,
+            "representation_ids",
+        )
+
+    def _get_repre_ids_by_version_ids(
+        self, project_name: str, version_ids: set[str]
+    ) -> dict[str, set[str]]:
+        output = {}
+        if not version_ids:
+            return output
+
+        project_cache = self._repre_parents_cache[project_name]
+        missing_ids = set()
+        for version_id in version_ids:
+            cache = project_cache[version_id]
+            if cache.is_valid:
+                output[version_id] = cache.get_data()
+            else:
+                missing_ids.add(version_id)
+
+        if missing_ids:
+            repre_cache = self._representations_cache[project_name]
+            repres_by_parent_id = collections.defaultdict(list)
+            for repre in ayon_api.get_representations(
+                project_name, version_ids=missing_ids
+            ):
+                version_id = repre["versionId"]
+                repre_cache[repre["id"]].update_data(repre)
+                repres_by_parent_id[version_id].append(repre)
+
+            for version_id, repres in repres_by_parent_id.items():
+                repre_ids = {
+                    repre["id"]
+                    for repre in repres
+                }
+                output[version_id] = set(repre_ids)
+                project_cache[version_id].update_data(repre_ids)
+
+        return output
+
+    def _get_entities(
+        self,
+        project_name: str,
+        entity_ids: set[str],
+        cache: NestedCacheItem,
+        getter: Callable,
+        filter_arg: str,
+    ) -> list[dict[str, Any]]:
+        entities = []
+        if not entity_ids:
+            return entities
+
+        missing_ids = set()
+        project_cache = cache[project_name]
+        for entity_id in entity_ids:
+            entity_cache = project_cache[entity_id]
+            if entity_cache.is_valid:
+                entities.append(entity_cache.get_data())
+            else:
+                missing_ids.add(entity_id)
+
+        if missing_ids:
+            for entity in getter(project_name, **{filter_arg: missing_ids}):
+                entities.append(entity)
+                entity_id = entity["id"]
+                project_cache[entity_id].update_data(entity)
+        return entities
+
     def _get_action_items_for_contexts(
         self,
         project_name,
@@ -601,12 +742,12 @@ class LoaderActionsModel:
             project_name, version_ids=version_ids
         ))
         product_ids = {v["productId"] for v in version_entities}
-        product_entities = ayon_api.get_products(
-            project_name, product_ids=product_ids
+        product_entities = self._get_products(
+            project_name, product_ids
         )
         product_entities_by_id = {p["id"]: p for p in product_entities}
         folder_ids = {p["folderId"] for p in product_entities_by_id.values()}
-        folder_entities = ayon_api.get_folders(
+        folder_entities = self._get_folders(
             project_name, folder_ids=folder_ids
         )
         folder_entities_by_id = {f["id"]: f for f in folder_entities}