Merge pull request #778 from ynput/enhancement/AY-6086_adding-parents-to-template-keys

Chore: Add 'parents' to folder template data

client/ayon_core/lib/path_templates.py

@@ -1,9 +1,15 @@
 import os
 import re
+import copy
 import numbers
+import warnings
+from string import Formatter
+import typing
+from typing import List, Dict, Any, Set
+
+if typing.TYPE_CHECKING:
+    from typing import Union
 
-KEY_PATTERN = re.compile(r"(\{.*?[^{0]*\})")
-KEY_PADDING_PATTERN = re.compile(r"([^:]+)\S+[><]\S+")
 SUB_DICT_PATTERN = re.compile(r"([^\[\]]+)")
 OPTIONAL_PATTERN = re.compile(r"(<.*?[^{0]*>)[^0-9]*?")
 
@@ -18,9 +24,7 @@ class TemplateUnsolved(Exception):
     def __init__(self, template, missing_keys, invalid_types):
         invalid_type_items = []
         for _key, _type in invalid_types.items():
-            invalid_type_items.append(
-                "\"{0}\" {1}".format(_key, str(_type))
-            )
+            invalid_type_items.append(f"\"{_key}\" {str(_type)}")
 
         invalid_types_msg = ""
         if invalid_type_items:
@@ -33,31 +37,32 @@ class TemplateUnsolved(Exception):
             missing_keys_msg = self.missing_keys_msg.format(
                 ", ".join(missing_keys)
             )
-        super(TemplateUnsolved, self).__init__(
+        super().__init__(
             self.msg.format(template, missing_keys_msg, invalid_types_msg)
         )
 
 
 class StringTemplate:
     """String that can be formatted."""
-    def __init__(self, template):
+    def __init__(self, template: str):
         if not isinstance(template, str):
-            raise TypeError("<{}> argument must be a string, not {}.".format(
-                self.__class__.__name__, str(type(template))
-            ))
+            raise TypeError(
+                f"<{self.__class__.__name__}> argument must be a string,"
+                f" not {str(type(template))}."
+            )
 
-        self._template = template
+        self._template: str = template
         parts = []
-        last_end_idx = 0
-        for item in KEY_PATTERN.finditer(template):
-            start, end = item.span()
-            if start > last_end_idx:
-                parts.append(template[last_end_idx:start])
-            parts.append(FormattingPart(template[start:end]))
-            last_end_idx = end
+        formatter = Formatter()
 
-        if last_end_idx < len(template):
-            parts.append(template[last_end_idx:len(template)])
+        for item in formatter.parse(template):
+            literal_text, field_name, format_spec, conversion = item
+            if literal_text:
+                parts.append(literal_text)
+            if field_name:
+                parts.append(
+                    FormattingPart(field_name, format_spec, conversion)
+                )
 
         new_parts = []
         for part in parts:
@@ -77,15 +82,17 @@ class StringTemplate:
             if substr:
                 new_parts.append(substr)
 
-        self._parts = self.find_optional_parts(new_parts)
+        self._parts: List["Union[str, OptionalPart, FormattingPart]"] = (
+            self.find_optional_parts(new_parts)
+        )
 
-    def __str__(self):
+    def __str__(self) -> str:
         return self.template
 
-    def __repr__(self):
-        return "<{}> {}".format(self.__class__.__name__, self.template)
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__}> {self.template}"
 
-    def __contains__(self, other):
+    def __contains__(self, other: str) -> bool:
         return other in self.template
 
     def replace(self, *args, **kwargs):
@@ -93,10 +100,10 @@ class StringTemplate:
         return self
 
     @property
-    def template(self):
+    def template(self) -> str:
         return self._template
 
-    def format(self, data):
+    def format(self, data: Dict[str, Any]) -> "TemplateResult":
         """ Figure out with whole formatting.
 
         Separate advanced keys (*Like '{project[name]}') from string which must
@@ -108,6 +115,7 @@ class StringTemplate:
         Returns:
             TemplateResult: Filled or partially filled template containing all
                 data needed or missing for filling template.
+
         """
         result = TemplatePartResult()
         for part in self._parts:
@@ -135,23 +143,29 @@ class StringTemplate:
             invalid_types
         )
 
-    def format_strict(self, *args, **kwargs):
-        result = self.format(*args, **kwargs)
+    def format_strict(self, data: Dict[str, Any]) -> "TemplateResult":
+        result = self.format(data)
         result.validate()
         return result
 
     @classmethod
-    def format_template(cls, template, data):
+    def format_template(
+        cls, template: str, data: Dict[str, Any]
+    ) -> "TemplateResult":
         objected_template = cls(template)
         return objected_template.format(data)
 
     @classmethod
-    def format_strict_template(cls, template, data):
+    def format_strict_template(
+        cls, template: str, data: Dict[str, Any]
+    ) -> "TemplateResult":
         objected_template = cls(template)
         return objected_template.format_strict(data)
 
     @staticmethod
-    def find_optional_parts(parts):
+    def find_optional_parts(
+        parts: List["Union[str, FormattingPart]"]
+    ) -> List["Union[str, OptionalPart, FormattingPart]"]:
         new_parts = []
         tmp_parts = {}
         counted_symb = -1
@@ -216,11 +230,11 @@ class TemplateResult(str):
             of number.
     """
 
-    used_values = None
-    solved = None
-    template = None
-    missing_keys = None
-    invalid_types = None
+    used_values: Dict[str, Any] = None
+    solved: bool = None
+    template: str = None
+    missing_keys: List[str] = None
+    invalid_types: Dict[str, Any] = None
 
     def __new__(
         cls, filled_template, template, solved,
@@ -248,7 +262,7 @@ class TemplateResult(str):
                 self.invalid_types
             )
 
-    def copy(self):
+    def copy(self) -> "TemplateResult":
         cls = self.__class__
         return cls(
             str(self),
@@ -259,7 +273,7 @@ class TemplateResult(str):
             self.invalid_types
         )
 
-    def normalized(self):
+    def normalized(self) -> "TemplateResult":
         """Convert to normalized path."""
 
         cls = self.__class__
@@ -275,27 +289,28 @@ class TemplateResult(str):
 
 class TemplatePartResult:
     """Result to store result of template parts."""
-    def __init__(self, optional=False):
+    def __init__(self, optional: bool = False):
         # Missing keys or invalid value types of required keys
-        self._missing_keys = set()
-        self._invalid_types = {}
+        self._missing_keys: Set[str] = set()
+        self._invalid_types: Dict[str, Any] = {}
         # Missing keys or invalid value types of optional keys
-        self._missing_optional_keys = set()
-        self._invalid_optional_types = {}
+        self._missing_optional_keys: Set[str] = set()
+        self._invalid_optional_types: Dict[str, Any] = {}
 
         # Used values stored by key with origin type
         #   - key without any padding or key modifiers
         #   - value from filling data
         #   Example: {"version": 1}
-        self._used_values = {}
+        self._used_values: Dict[str, Any] = {}
         # Used values stored by key with all modifirs
         #   - value is already formatted string
         #   Example: {"version:0>3": "001"}
-        self._realy_used_values = {}
+        self._really_used_values: Dict[str, Any] = {}
         # Concatenated string output after formatting
-        self._output = ""
+        self._output: str = ""
         # Is this result from optional part
-        self._optional = True
+        # TODO find out why we don't use 'optional' from args
+        self._optional: bool = True
 
     def add_output(self, other):
         if isinstance(other, str):
@@ -313,7 +328,7 @@ class TemplatePartResult:
             if other.optional and not other.solved:
                 return
             self._used_values.update(other.used_values)
-            self._realy_used_values.update(other.realy_used_values)
+            self._really_used_values.update(other.really_used_values)
 
         else:
             raise TypeError("Cannot add data from \"{}\" to \"{}\"".format(
@@ -321,7 +336,7 @@ class TemplatePartResult:
             )
 
     @property
-    def solved(self):
+    def solved(self) -> bool:
         if self.optional:
             if (
                 len(self.missing_optional_keys) > 0
@@ -334,45 +349,53 @@ class TemplatePartResult:
         )
 
     @property
-    def optional(self):
+    def optional(self) -> bool:
         return self._optional
 
     @property
-    def output(self):
+    def output(self) -> str:
         return self._output
 
     @property
-    def missing_keys(self):
+    def missing_keys(self) -> Set[str]:
         return self._missing_keys
 
     @property
-    def missing_optional_keys(self):
+    def missing_optional_keys(self) -> Set[str]:
         return self._missing_optional_keys
 
     @property
-    def invalid_types(self):
+    def invalid_types(self) -> Dict[str, Any]:
         return self._invalid_types
 
     @property
-    def invalid_optional_types(self):
+    def invalid_optional_types(self) -> Dict[str, Any]:
         return self._invalid_optional_types
 
     @property
-    def realy_used_values(self):
-        return self._realy_used_values
+    def really_used_values(self) -> Dict[str, Any]:
+        return self._really_used_values
 
     @property
-    def used_values(self):
+    def realy_used_values(self) -> Dict[str, Any]:
+        warnings.warn(
+            "Property 'realy_used_values' is deprecated."
+            " Use 'really_used_values' instead.",
+            DeprecationWarning
+        )
+        return self._really_used_values
+
+    @property
+    def used_values(self) -> Dict[str, Any]:
         return self._used_values
 
     @staticmethod
-    def split_keys_to_subdicts(values):
+    def split_keys_to_subdicts(values: Dict[str, Any]) -> Dict[str, Any]:
         output = {}
+        formatter = Formatter()
         for key, value in values.items():
-            key_padding = list(KEY_PADDING_PATTERN.findall(key))
-            if key_padding:
-                key = key_padding[0]
-            key_subdict = list(SUB_DICT_PATTERN.findall(key))
+            _, field_name, _, _ = next(formatter.parse(f"{{{key}}}"))
+            key_subdict = list(SUB_DICT_PATTERN.findall(field_name))
             data = output
             last_key = key_subdict.pop(-1)
             for subkey in key_subdict:
@@ -382,7 +405,7 @@ class TemplatePartResult:
             data[last_key] = value
         return output
 
-    def get_clean_used_values(self):
+    def get_clean_used_values(self) -> Dict[str, Any]:
         new_used_values = {}
         for key, value in self.used_values.items():
             if isinstance(value, FormatObject):
@@ -391,19 +414,27 @@ class TemplatePartResult:
 
         return self.split_keys_to_subdicts(new_used_values)
 
-    def add_realy_used_value(self, key, value):
-        self._realy_used_values[key] = value
+    def add_really_used_value(self, key: str, value: Any):
+        self._really_used_values[key] = value
 
-    def add_used_value(self, key, value):
+    def add_realy_used_value(self, key: str, value: Any):
+        warnings.warn(
+            "Method 'add_realy_used_value' is deprecated."
+            " Use 'add_really_used_value' instead.",
+            DeprecationWarning
+        )
+        self.add_really_used_value(key, value)
+
+    def add_used_value(self, key: str, value: Any):
         self._used_values[key] = value
 
-    def add_missing_key(self, key):
+    def add_missing_key(self, key: str):
         if self._optional:
             self._missing_optional_keys.add(key)
         else:
             self._missing_keys.add(key)
 
-    def add_invalid_type(self, key, value):
+    def add_invalid_type(self, key: str, value: Any):
         if self._optional:
             self._invalid_optional_types[key] = type(value)
         else:
@@ -421,10 +452,10 @@ class FormatObject:
     def __format__(self, *args, **kwargs):
         return self.value.__format__(*args, **kwargs)
 
-    def __str__(self):
+    def __str__(self) -> str:
         return str(self.value)
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return self.__str__()
 
 
@@ -434,23 +465,44 @@ class FormattingPart:
     Containt only single key to format e.g. "{project[name]}".
 
     Args:
-        template(str): String containing the formatting key.
+        field_name (str): Name of key.
+        format_spec (str): Format specification.
+        conversion (Union[str, None]): Conversion type.
+
     """
-    def __init__(self, template):
-        self._template = template
+    def __init__(
+        self,
+        field_name: str,
+        format_spec: str,
+        conversion: "Union[str, None]",
+    ):
+        format_spec_v = ""
+        if format_spec:
+            format_spec_v = f":{format_spec}"
+        conversion_v = ""
+        if conversion:
+            conversion_v = f"!{conversion}"
+
+        self._field_name: str = field_name
+        self._format_spec: str = format_spec_v
+        self._conversion: str = conversion_v
+
+        template_base = f"{field_name}{format_spec_v}{conversion_v}"
+        self._template_base: str = template_base
+        self._template: str = f"{{{template_base}}}"
 
     @property
-    def template(self):
+    def template(self) -> str:
         return self._template
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return "<Format:{}>".format(self._template)
 
-    def __str__(self):
+    def __str__(self) -> str:
         return self._template
 
     @staticmethod
-    def validate_value_type(value):
+    def validate_value_type(value: Any) -> bool:
         """Check if value can be used for formatting of single key."""
         if isinstance(value, (numbers.Number, FormatObject)):
             return True
@@ -461,7 +513,7 @@ class FormattingPart:
         return False
 
     @staticmethod
-    def validate_key_is_matched(key):
+    def validate_key_is_matched(key: str) -> bool:
         """Validate that opening has closing at correct place.
         Future-proof, only square brackets are currently used in keys.
 
@@ -488,16 +540,29 @@ class FormattingPart:
                     return False
         return not queue
 
-    def format(self, data, result):
+    @staticmethod
+    def keys_to_template_base(keys: List[str]):
+        if not keys:
+            return None
+        # Create copy of keys
+        keys = list(keys)
+        template_base = keys.pop(0)
+        joined_keys = "".join([f"[{key}]" for key in keys])
+        return f"{template_base}{joined_keys}"
+
+    def format(
+        self, data: Dict[str, Any], result: TemplatePartResult
+    ) -> TemplatePartResult:
         """Format the formattings string.
 
         Args:
             data(dict): Data that should be used for formatting.
             result(TemplatePartResult): Object where result is stored.
+
         """
-        key = self.template[1:-1]
-        if key in result.realy_used_values:
-            result.add_output(result.realy_used_values[key])
+        key = self._template_base
+        if key in result.really_used_values:
+            result.add_output(result.really_used_values[key])
             return result
 
         # ensure key is properly formed [({})] properly closed.
@@ -507,17 +572,38 @@ class FormattingPart:
             return result
 
         # check if key expects subdictionary keys (e.g. project[name])
-        existence_check = key
-        key_padding = list(KEY_PADDING_PATTERN.findall(existence_check))
-        if key_padding:
-            existence_check = key_padding[0]
-        key_subdict = list(SUB_DICT_PATTERN.findall(existence_check))
+        key_subdict = list(SUB_DICT_PATTERN.findall(self._field_name))
 
         value = data
         missing_key = False
         invalid_type = False
         used_keys = []
+        keys_to_value = None
+        used_value = None
+
         for sub_key in key_subdict:
+            if isinstance(value, list):
+                if not sub_key.lstrip("-").isdigit():
+                    invalid_type = True
+                    break
+                sub_key = int(sub_key)
+                if sub_key < 0:
+                    sub_key = len(value) + sub_key
+
+                invalid = 0 > sub_key < len(data)
+                if invalid:
+                    used_keys.append(sub_key)
+                    missing_key = True
+                    break
+
+                used_keys.append(sub_key)
+                if keys_to_value is None:
+                    keys_to_value = list(used_keys)
+                    keys_to_value.pop(-1)
+                    used_value = copy.deepcopy(value)
+                value = value[sub_key]
+                continue
+
             if (
                 value is None
                 or (hasattr(value, "items") and sub_key not in value)
@@ -533,45 +619,57 @@ class FormattingPart:
             used_keys.append(sub_key)
             value = value.get(sub_key)
 
-        if missing_key or invalid_type:
-            if len(used_keys) == 0:
-                invalid_key = key_subdict[0]
-            else:
-                invalid_key = used_keys[0]
-                for idx, sub_key in enumerate(used_keys):
-                    if idx == 0:
-                        continue
-                    invalid_key += "[{0}]".format(sub_key)
+        field_name = key_subdict[0]
+        if used_keys:
+            field_name = self.keys_to_template_base(used_keys)
 
+        if missing_key or invalid_type:
             if missing_key:
-                result.add_missing_key(invalid_key)
+                result.add_missing_key(field_name)
 
             elif invalid_type:
-                result.add_invalid_type(invalid_key, value)
+                result.add_invalid_type(field_name, value)
 
             result.add_output(self.template)
             return result
 
-        if self.validate_value_type(value):
-            fill_data = {}
-            first_value = True
-            for used_key in reversed(used_keys):
-                if first_value:
-                    first_value = False
-                    fill_data[used_key] = value
-                else:
-                    _fill_data = {used_key: fill_data}
-                    fill_data = _fill_data
-
-            formatted_value = self.template.format(**fill_data)
-            result.add_realy_used_value(key, formatted_value)
-            result.add_used_value(existence_check, formatted_value)
-            result.add_output(formatted_value)
+        if not self.validate_value_type(value):
+            result.add_invalid_type(key, value)
+            result.add_output(self.template)
             return result
 
-        result.add_invalid_type(key, value)
-        result.add_output(self.template)
+        fill_data = root_fill_data = {}
+        parent_fill_data = None
+        parent_key = None
+        fill_value = data
+        value_filled = False
+        for used_key in used_keys:
+            if isinstance(fill_value, list):
+                parent_fill_data[parent_key] = fill_value
+                value_filled = True
+                break
+            fill_value = fill_value[used_key]
+            parent_fill_data = fill_data
+            fill_data = parent_fill_data.setdefault(used_key, {})
+            parent_key = used_key
 
+        if not value_filled:
+            parent_fill_data[used_keys[-1]] = value
+
+        template = f"{{{field_name}{self._format_spec}{self._conversion}}}"
+        formatted_value = template.format(**root_fill_data)
+        used_key = key
+        if keys_to_value is not None:
+            used_key = self.keys_to_template_base(keys_to_value)
+
+        if used_value is None:
+            if isinstance(value, numbers.Number):
+                used_value = value
+            else:
+                used_value = formatted_value
+        result.add_really_used_value(self._field_name, used_value)
+        result.add_used_value(used_key, used_value)
+        result.add_output(formatted_value)
         return result
 
 
@@ -585,20 +683,27 @@ class OptionalPart:
             'FormattingPart'.
     """
 
-    def __init__(self, parts):
-        self._parts = parts
+    def __init__(
+        self,
+        parts: List["Union[str, OptionalPart, FormattingPart]"]
+    ):
+        self._parts: List["Union[str, OptionalPart, FormattingPart]"] = parts
 
     @property
-    def parts(self):
+    def parts(self) -> List["Union[str, OptionalPart, FormattingPart]"]:
         return self._parts
 
-    def __str__(self):
+    def __str__(self) -> str:
         return "<{}>".format("".join([str(p) for p in self._parts]))
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return "<Optional:{}>".format("".join([str(p) for p in self._parts]))
 
-    def format(self, data, result):
+    def format(
+        self,
+        data: Dict[str, Any],
+        result: TemplatePartResult,
+    ) -> TemplatePartResult:
         new_result = TemplatePartResult(True)
         for part in self._parts:
             if isinstance(part, str):

client/ayon_core/pipeline/template_data.py

@@ -87,14 +87,13 @@ def get_folder_template_data(folder_entity, project_name):
     """
 
     path = folder_entity["path"]
-    hierarchy_parts = path.split("/")
-    # Remove empty string from the beginning
-    hierarchy_parts.pop(0)
+    # Remove empty string from the beginning and split by '/'
+    parents = path.lstrip("/").split("/")
     # Remove last part which is folder name
-    folder_name = hierarchy_parts.pop(-1)
-    hierarchy = "/".join(hierarchy_parts)
-    if hierarchy_parts:
-        parent_name = hierarchy_parts[-1]
+    folder_name = parents.pop(-1)
+    hierarchy = "/".join(parents)
+    if parents:
+        parent_name = parents[-1]
     else:
         parent_name = project_name
 
@@ -103,6 +102,7 @@ def get_folder_template_data(folder_entity, project_name):
             "name": folder_name,
             "type": folder_entity["folderType"],
             "path": path,
+            "parents": parents,
         },
         "asset": folder_name,
         "hierarchy": hierarchy,

client/ayon_core/plugins/publish/collect_anatomy_instance_data.py

@@ -434,6 +434,7 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin):
                     #   Using 'Shot' is current default behavior of editorial
                     #   (or 'newHierarchyIntegration') publishing.
                     "type": "Shot",
+                    "parents": parents,
                 },
             })