Global: Optimize anatomy formatting by only formatting used templates instead (#4784)

* TemplatesDict can create different type of template

* anatomy templates can be formatted on their own

* return objected templates on get item

* '_rootless_path' is public classmethod 'rootless_path_from_result'

* 'AnatomyStringTemplate' expect anatomy templates

* remove key getters

* fix typo 'create_ojected_templates' -> 'create_objected_templates'

* Fix type of argument

* Fix long line

* Optimize formatting to use single template formatting instead of formatting full anatomy

* Optimize formatting to use single template formatting instead of formatting full anatomy

* Optimize formatting to use single template formatting instead of formatting full anatomy

* Optimize formatting to use single template formatting instead of formatting full anatomy

* Optimize formatting to use single template formatting instead of formatting full anatomy

* Optimize formatting to use single template formatting instead of formatting full anatomy

* Optimize formatting to use single template formatting instead of formatting full anatomy

* Use format strict + code cosmetics

* Get template from the formatted data

* Update openpype/plugins/publish/integrate_legacy.py

Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com>

* Cosmetics

* Move template obj definition for path up + rename to `path_template_obj`

* Refactor more cases from `anatomy.format` to template obj `.format_strict`

* Refactor more cases from `anatomy.format` to template obj `.format_strict`

* Refactor more cases from `anatomy.format` to template obj `.format_strict`

---------

Co-authored-by: Jakub Trllo <jakub.trllo@gmail.com>
Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com>

openpype/hosts/houdini/vendor/husdoutputprocessors/avalon_uri_processor.py

@@ -128,14 +128,14 @@ class AvalonURIOutputProcessor(base.OutputProcessorBase):
         if not asset_doc:
             raise RuntimeError("Invalid asset name: '%s'" % asset)
 
-        formatted_anatomy = anatomy.format({
+        template_obj = anatomy.templates_obj["publish"]["path"]
+        path = template_obj.format_strict({
             "project": PROJECT,
             "asset": asset_doc["name"],
             "subset": subset,
             "representation": ext,
             "version": 0  # stub version zero
         })
-        path = formatted_anatomy["publish"]["path"]
 
         # Remove the version folder
         subset_folder = os.path.dirname(os.path.dirname(path))

openpype/hosts/standalonepublisher/plugins/publish/extract_workfile_location.py

@@ -27,11 +27,12 @@ class ExtractWorkfileUrl(pyblish.api.ContextPlugin):
                 rep_name = instance.data.get("representations")[0].get("name")
                 template_data["representation"] = rep_name
                 template_data["ext"] = rep_name
-                anatomy_filled = anatomy.format(template_data)
-                template_filled = anatomy_filled["publish"]["path"]
+                template_obj = anatomy.templates_obj["publish"]["path"]
+                template_filled = template_obj.format_strict(template_data)
                 filepath = os.path.normpath(template_filled)
                 self.log.info("Using published scene for render {}".format(
                     filepath))
+                break
 
         if not filepath:
             self.log.info("Texture batch doesn't contain workfile.")

openpype/hosts/unreal/hooks/pre_workfile_preparation.py

@@ -61,10 +61,10 @@ class UnrealPrelaunchHook(PreLaunchHook):
             project_name=project_doc["name"]
         )
         # Fill templates
-        filled_anatomy = anatomy.format(workdir_data)
+        template_obj = anatomy.templates_obj[workfile_template_key]["file"]
 
         # Return filename
-        return filled_anatomy[workfile_template_key]["file"]
+        return template_obj.format_strict(workdir_data)
 
     def exec_plugin_install(self, engine_path: Path, env: dict = None):
         # set up the QThread and worker with necessary signals

openpype/lib/usdlib.py

@@ -327,7 +327,8 @@ def get_usd_master_path(asset, subset, representation):
     else:
         asset_doc = get_asset_by_name(project_name, asset, fields=["name"])
 
-    formatted_result = anatomy.format(
+    template_obj = anatomy.templates_obj["publish"]["path"]
+    path = template_obj.format_strict(
         {
             "project": {
                 "name": project_name,
@@ -340,7 +341,6 @@ def get_usd_master_path(asset, subset, representation):
         }
     )
 
-    path = formatted_result["publish"]["path"]
     # Remove the version folder
     subset_folder = os.path.dirname(os.path.dirname(path))
     master_folder = os.path.join(subset_folder, "master")

openpype/modules/deadline/abstract_submit_deadline.py

@@ -534,8 +534,8 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin):
         template_data["comment"] = None
 
         anatomy = instance.context.data['anatomy']
-        anatomy_filled = anatomy.format(template_data)
-        template_filled = anatomy_filled["publish"]["path"]
+        template_obj = anatomy.templates_obj["publish"]["path"]
+        template_filled = template_obj.format_strict(template_data)
         file_path = os.path.normpath(template_filled)
 
         self.log.info("Using published scene for render {}".format(file_path))

openpype/modules/deadline/plugins/publish/submit_publish_job.py

@@ -1202,10 +1202,11 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
         template_data["family"] = "render"
         template_data["version"] = version
 
-        anatomy_filled = anatomy.format(template_data)
-
-        if "folder" in anatomy.templates["render"]:
-            publish_folder = anatomy_filled["render"]["folder"]
+        render_templates = anatomy.templates_obj["render"]
+        if "folder" in render_templates:
+            publish_folder = render_templates["folder"].format_strict(
+                template_data
+            )
         else:
             # solve deprecated situation when `folder` key is not underneath
             # `publish` anatomy
@@ -1215,8 +1216,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
                 " key underneath `publish` (in global of for project `{}`)."
             ).format(project_name))
 
-            file_path = anatomy_filled["render"]["path"]
-            # Directory
+            file_path = render_templates["path"].format_strict(template_data)
             publish_folder = os.path.dirname(file_path)
 
         return publish_folder

openpype/pipeline/context_tools.py

@@ -463,9 +463,7 @@ def get_workdir_from_session(session=None, template_key=None):
         session = legacy_io.Session
     project_name = session["AVALON_PROJECT"]
     host_name = session["AVALON_APP"]
-    anatomy = Anatomy(project_name)
     template_data = get_template_data_from_session(session)
-    anatomy_filled = anatomy.format(template_data)
 
     if not template_key:
         task_type = template_data["task"]["type"]
@@ -474,7 +472,10 @@ def get_workdir_from_session(session=None, template_key=None):
             host_name,
             project_name=project_name
         )
-    path = anatomy_filled[template_key]["folder"]
+
+    anatomy = Anatomy(project_name)
+    template_obj = anatomy.templates_obj[template_key]["folder"]
+    path = template_obj.format_strict(template_data)
     if path:
         path = os.path.normpath(path)
     return path

openpype/pipeline/delivery.py

@@ -1,5 +1,6 @@
 """Functions useful for delivery of published representations."""
 import os
+import copy
 import shutil
 import glob
 import clique
@@ -146,12 +147,11 @@ def deliver_single_file(
         report_items["Source file was not found"].append(msg)
         return report_items, 0
 
-    anatomy_filled = anatomy.format(anatomy_data)
     if format_dict:
-        template_result = anatomy_filled["delivery"][template_name]
-        delivery_path = template_result.rootless.format(**format_dict)
-    else:
-        delivery_path = anatomy_filled["delivery"][template_name]
+        anatomy_data = copy.deepcopy(anatomy_data)
+        anatomy_data["root"] = format_dict["root"]
+    template_obj = anatomy.templates_obj["delivery"][template_name]
+    delivery_path = template_obj.format_strict(anatomy_data)
 
     # Backwards compatibility when extension contained `.`
     delivery_path = delivery_path.replace("..", ".")
@@ -269,14 +269,12 @@ def deliver_sequence(
 
     frame_indicator = "@####@"
 
+    anatomy_data = copy.deepcopy(anatomy_data)
     anatomy_data["frame"] = frame_indicator
-    anatomy_filled = anatomy.format(anatomy_data)
-
     if format_dict:
-        template_result = anatomy_filled["delivery"][template_name]
-        delivery_path = template_result.rootless.format(**format_dict)
-    else:
-        delivery_path = anatomy_filled["delivery"][template_name]
+        anatomy_data["root"] = format_dict["root"]
+    template_obj = anatomy.templates_obj["delivery"][template_name]
+    delivery_path = template_obj.format_strict(anatomy_data)
 
     delivery_path = os.path.normpath(delivery_path.replace("\\", "/"))
     delivery_folder = os.path.dirname(delivery_path)

openpype/pipeline/workfile/path_resolving.py

@@ -132,9 +132,9 @@ def get_workdir_with_workdir_data(
             project_settings
         )
 
-    anatomy_filled = anatomy.format(workdir_data)
+    template_obj = anatomy.templates_obj[template_key]["folder"]
     # Output is TemplateResult object which contain useful data
-    output = anatomy_filled[template_key]["folder"]
+    output = template_obj.format_strict(workdir_data)
     if output:
         return output.normalized()
     return output

openpype/plugins/publish/collect_resources_path.py

@@ -83,10 +83,11 @@ class CollectResourcesPath(pyblish.api.InstancePlugin):
                 "hierarchy": instance.data["hierarchy"]
             })
 
-        anatomy_filled = anatomy.format(template_data)
-
-        if "folder" in anatomy.templates["publish"]:
-            publish_folder = anatomy_filled["publish"]["folder"]
+        publish_templates = anatomy.templates_obj["publish"]
+        if "folder" in publish_templates:
+            publish_folder = publish_templates["folder"].format_strict(
+                template_data
+            )
         else:
             # solve deprecated situation when `folder` key is not underneath
             # `publish` anatomy
@@ -95,8 +96,7 @@ class CollectResourcesPath(pyblish.api.InstancePlugin):
                 " key underneath `publish` (in global of for project `{}`)."
             ).format(anatomy.project_name))
 
-            file_path = anatomy_filled["publish"]["path"]
-            # Directory
+            file_path = publish_templates["path"].format_strict(template_data)
             publish_folder = os.path.dirname(file_path)
 
         publish_folder = os.path.normpath(publish_folder)

openpype/plugins/publish/integrate.py

@@ -665,8 +665,7 @@ class IntegrateAsset(pyblish.api.InstancePlugin):
         # - template_data (Dict[str, Any]): source data used to fill template
         #   - to add required data to 'repre_context' not used for
         #       formatting
-        # - anatomy_filled (Dict[str, Any]): filled anatomy of last file
-        #   - to fill 'publishDir' on instance.data -> not ideal
+        path_template_obj = anatomy.templates_obj[template_name]["path"]
 
         # Treat template with 'orignalBasename' in special way
         if "{originalBasename}" in template:
@@ -700,8 +699,7 @@ class IntegrateAsset(pyblish.api.InstancePlugin):
                 template_data["originalBasename"], _ = os.path.splitext(
                     src_file_name)
 
-                anatomy_filled = anatomy.format(template_data)
-                dst = anatomy_filled[template_name]["path"]
+                dst = path_template_obj.format_strict(template_data)
                 src = os.path.join(stagingdir, src_file_name)
                 transfers.append((src, dst))
                 if repre_context is None:
@@ -761,8 +759,9 @@ class IntegrateAsset(pyblish.api.InstancePlugin):
                     template_data["udim"] = index
                 else:
                     template_data["frame"] = index
-                anatomy_filled = anatomy.format(template_data)
-                template_filled = anatomy_filled[template_name]["path"]
+                template_filled = path_template_obj.format_strict(
+                    template_data
+                )
                 dst_filepaths.append(template_filled)
                 if repre_context is None:
                     self.log.debug(
@@ -798,8 +797,7 @@ class IntegrateAsset(pyblish.api.InstancePlugin):
             if is_udim:
                 template_data["udim"] = repre["udim"][0]
             # Construct destination filepath from template
-            anatomy_filled = anatomy.format(template_data)
-            template_filled = anatomy_filled[template_name]["path"]
+            template_filled = path_template_obj.format_strict(template_data)
             repre_context = template_filled.used_values
             dst = os.path.normpath(template_filled)
 
@@ -810,11 +808,9 @@ class IntegrateAsset(pyblish.api.InstancePlugin):
         # todo: Are we sure the assumption each representation
         #       ends up in the same folder is valid?
         if not instance.data.get("publishDir"):
-            instance.data["publishDir"] = (
-                anatomy_filled
-                [template_name]
-                ["folder"]
-            )
+            template_obj = anatomy.templates_obj[template_name]["folder"]
+            template_filled = template_obj.format_strict(template_data)
+            instance.data["publishDir"] = template_filled
 
         for key in self.db_representation_context_keys:
             # Also add these values to the context even if not used by the

openpype/plugins/publish/integrate_hero_version.py

@@ -291,6 +291,7 @@ class IntegrateHeroVersion(pyblish.api.InstancePlugin):
                 ))
         try:
             src_to_dst_file_paths = []
+            path_template_obj = anatomy.templates_obj[template_key]["path"]
             for repre_info in published_repres.values():
 
                 # Skip if new repre does not have published repre files
@@ -303,9 +304,7 @@ class IntegrateHeroVersion(pyblish.api.InstancePlugin):
                 anatomy_data.pop("version", None)
 
                 # Get filled path to repre context
-                anatomy_filled = anatomy.format(anatomy_data)
-                template_filled = anatomy_filled[template_key]["path"]
-
+                template_filled = path_template_obj.format_strict(anatomy_data)
                 repre_data = {
                     "path": str(template_filled),
                     "template": hero_template
@@ -343,8 +342,9 @@ class IntegrateHeroVersion(pyblish.api.InstancePlugin):
                     # Get head and tail for collection
                     frame_splitter = "_-_FRAME_SPLIT_-_"
                     anatomy_data["frame"] = frame_splitter
-                    _anatomy_filled = anatomy.format(anatomy_data)
-                    _template_filled = _anatomy_filled[template_key]["path"]
+                    _template_filled = path_template_obj.format_strict(
+                        anatomy_data
+                    )
                     head, tail = _template_filled.split(frame_splitter)
                     padding = int(
                         anatomy.templates[template_key]["frame_padding"]
@@ -520,24 +520,24 @@ class IntegrateHeroVersion(pyblish.api.InstancePlugin):
             })
 
         if "folder" in anatomy.templates[template_key]:
-            anatomy_filled = anatomy.format(template_data)
-            publish_folder = anatomy_filled[template_key]["folder"]
+            template_obj = anatomy.templates_obj[template_key]["folder"]
+            publish_folder = template_obj.format_strict(template_data)
         else:
             # This is for cases of Deprecated anatomy without `folder`
             # TODO remove when all clients have solved this issue
-            template_data.update({
-                "frame": "FRAME_TEMP",
-                "representation": "TEMP"
-            })
-            anatomy_filled = anatomy.format(template_data)
-            # solve deprecated situation when `folder` key is not underneath
-            # `publish` anatomy
             self.log.warning((
                 "Deprecation warning: Anatomy does not have set `folder`"
                 " key underneath `publish` (in global of for project `{}`)."
             ).format(anatomy.project_name))
+            # solve deprecated situation when `folder` key is not underneath
+            # `publish` anatomy
+            template_data.update({
+                "frame": "FRAME_TEMP",
+                "representation": "TEMP"
+            })
+            template_obj = anatomy.templates_obj[template_key]["path"]
+            file_path = template_obj.format_strict(template_data)
 
-            file_path = anatomy_filled[template_key]["path"]
             # Directory
             publish_folder = os.path.dirname(file_path)
 

openpype/plugins/publish/integrate_legacy.py

@@ -480,8 +480,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                     else:
                         template_data["udim"] = src_padding_exp % i
 
-                    anatomy_filled = anatomy.format(template_data)
-                    template_filled = anatomy_filled[template_name]["path"]
+                    template_obj = anatomy.templates_obj[template_name]["path"]
+                    template_filled = template_obj.format_strict(template_data)
                     if repre_context is None:
                         repre_context = template_filled.used_values
                     test_dest_files.append(
@@ -587,8 +587,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                 if repre.get("udim"):
                     template_data["udim"] = repre["udim"][0]
                 src = os.path.join(stagingdir, fname)
-                anatomy_filled = anatomy.format(template_data)
-                template_filled = anatomy_filled[template_name]["path"]
+                template_obj = anatomy.templates_obj[template_name]["path"]
+                template_filled = template_obj.format_strict(template_data)
                 repre_context = template_filled.used_values
                 dst = os.path.normpath(template_filled)
 
@@ -600,9 +600,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
 
             if not instance.data.get("publishDir"):
                 instance.data["publishDir"] = (
-                    anatomy_filled
-                    [template_name]
-                    ["folder"]
+                    anatomy.templates_obj[template_name]["folder"]
+                        .format_strict(template_data)
                 )
             if repre.get("udim"):
                 repre_context["udim"] = repre.get("udim")  # store list

openpype/plugins/publish/integrate_thumbnail.py

@@ -271,9 +271,9 @@ class IntegrateThumbnails(pyblish.api.ContextPlugin):
                 "thumbnail_type": "thumbnail"
             })
 
-            anatomy_filled = anatomy.format(template_data)
-            thumbnail_template = anatomy.templates["publish"]["thumbnail"]
-            template_filled = anatomy_filled["publish"]["thumbnail"]
+            template_obj = anatomy.templates_obj["publish"]["thumbnail"]
+            template_filled = template_obj.format_strict(template_data)
+            thumbnail_template = template_filled.template
 
             dst_full_path = os.path.normpath(str(template_filled))
             self.log.debug("Copying file .. {} -> {}".format(

openpype/tools/push_to_project/control_integrate.py

@@ -1050,8 +1050,8 @@ class ProjectPushItemProcess:
                 repre_format_data["ext"] = ext[1:]
                 break
 
-            tmp_result = anatomy.format(formatting_data)
-            folder_path = tmp_result[template_name]["folder"]
+            template_obj = anatomy.templates_obj[template_name]["folder"]
+            folder_path = template_obj.format_strict(formatting_data)
             repre_context = folder_path.used_values
             folder_path_rootless = folder_path.rootless
             repre_filepaths = []

openpype/tools/texture_copy/app.py

@@ -47,8 +47,8 @@ class TextureCopy:
             "hierarchy": hierarchy
         }
         anatomy = Anatomy(project_name)
-        anatomy_filled = anatomy.format(template_data)
-        return anatomy_filled['texture']['path']
+        template_obj = anatomy.templates_obj["texture"]["path"]
+        return template_obj.format_strict(template_data)
 
     def _get_version(self, path):
         versions = [0]

openpype/tools/workfiles/save_as_dialog.py

@@ -60,8 +60,8 @@ class CommentMatcher(object):
         temp_data["version"] = "<<version>>"
         temp_data["ext"] = "<<ext>>"
 
-        formatted = anatomy.format(temp_data)
-        fname_pattern = formatted[template_key]["file"]
+        template_obj = anatomy.templates_obj[template_key]["file"]
+        fname_pattern = template_obj.format_strict(temp_data)
         fname_pattern = re.escape(fname_pattern)
 
         # Replace comment and version with something we can match with regex
@@ -375,8 +375,8 @@ class SaveAsDialog(QtWidgets.QDialog):
 
         data["ext"] = data["ext"].lstrip(".")
 
-        anatomy_filled = self.anatomy.format(data)
-        return anatomy_filled[self.template_key]["file"]
+        template_obj = self.anatomy.templates_obj[self.template_key]["file"]
+        return template_obj.format_strict(data)
 
     def refresh(self):
         extensions = list(self._extensions)