From a9506a14806fe92b1d86bb68a3cd84a5749b4141 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Sat, 19 Feb 2022 20:31:31 +0100 Subject: [PATCH 01/10] extracted template formatting logic from anatomy --- openpype/lib/path_templates.py | 745 +++++++++++++++++++++++++++++++++ 1 file changed, 745 insertions(+) create mode 100644 openpype/lib/path_templates.py diff --git a/openpype/lib/path_templates.py b/openpype/lib/path_templates.py new file mode 100644 index 0000000000..6f68cc4ce9 --- /dev/null +++ b/openpype/lib/path_templates.py @@ -0,0 +1,745 @@ +import os +import re +import copy +import numbers +import collections + +import six + +from .log import PypeLogger + +log = PypeLogger.get_logger(__name__) + + +KEY_PATTERN = re.compile(r"(\{.*?[^{0]*\})") +KEY_PADDING_PATTERN = re.compile(r"([^:]+)\S+[><]\S+") +SUB_DICT_PATTERN = re.compile(r"([^\[\]]+)") +OPTIONAL_PATTERN = re.compile(r"(<.*?[^{0]*>)[^0-9]*?") + + +def merge_dict(main_dict, enhance_dict): + """Merges dictionaries by keys. + + Function call itself if value on key is again dictionary. + + Args: + main_dict (dict): First dict to merge second one into. + enhance_dict (dict): Second dict to be merged. + + Returns: + dict: Merged result. + + .. note:: does not overrides whole value on first found key + but only values differences from enhance_dict + + """ + for key, value in enhance_dict.items(): + if key not in main_dict: + main_dict[key] = value + elif isinstance(value, dict) and isinstance(main_dict[key], dict): + main_dict[key] = merge_dict(main_dict[key], value) + else: + main_dict[key] = value + return main_dict + + +class TemplateMissingKey(Exception): + """Exception for cases when key does not exist in template.""" + + msg = "Template key does not exist: `{}`." + + def __init__(self, parents): + parent_join = "".join(["[\"{0}\"]".format(key) for key in parents]) + super(TemplateMissingKey, self).__init__( + self.msg.format(parent_join) + ) + + +class TemplateUnsolved(Exception): + """Exception for unsolved template when strict is set to True.""" + + msg = "Template \"{0}\" is unsolved.{1}{2}" + invalid_types_msg = " Keys with invalid DataType: `{0}`." + missing_keys_msg = " Missing keys: \"{0}\"." + + def __init__(self, template, missing_keys, invalid_types): + invalid_type_items = [] + for _key, _type in invalid_types.items(): + invalid_type_items.append( + "\"{0}\" {1}".format(_key, str(_type)) + ) + + invalid_types_msg = "" + if invalid_type_items: + invalid_types_msg = self.invalid_types_msg.format( + ", ".join(invalid_type_items) + ) + + missing_keys_msg = "" + if missing_keys: + missing_keys_msg = self.missing_keys_msg.format( + ", ".join(missing_keys) + ) + super(TemplateUnsolved, self).__init__( + self.msg.format(template, missing_keys_msg, invalid_types_msg) + ) + + +class StringTemplate(object): + """String that can be formatted.""" + def __init__(self, template): + if not isinstance(template, six.string_types): + raise TypeError("<{}> argument must be a string, not {}.".format( + self.__class__.__name__, str(type(template)) + )) + + self._template = template + parts = [] + last_end_idx = 0 + for item in KEY_PATTERN.finditer(template): + start, end = item.span() + if start > last_end_idx: + parts.append(template[last_end_idx:start]) + parts.append(FormattingPart(template[start:end])) + last_end_idx = end + + if last_end_idx < len(template): + parts.append(template[last_end_idx:len(template)]) + + new_parts = [] + for part in parts: + if not isinstance(part, six.string_types): + new_parts.append(part) + continue + + substr = "" + for char in part: + if char not in ("<", ">"): + substr += char + else: + if substr: + new_parts.append(substr) + new_parts.append(char) + substr = "" + if substr: + new_parts.append(substr) + + self._parts = self.find_optional_parts(new_parts) + + @property + def template(self): + return self._template + + def format(self, data): + """ Figure out with whole formatting. + + Separate advanced keys (*Like '{project[name]}') from string which must + be formatted separatelly in case of missing or incomplete keys in data. + + Args: + data (dict): Containing keys to be filled into template. + + Returns: + TemplateResult: Filled or partially filled template containing all + data needed or missing for filling template. + """ + result = TemplatePartResult() + for part in self._parts: + if isinstance(part, six.string_types): + result.add_output(part) + else: + part.format(data, result) + + invalid_types = result.invalid_types + invalid_types.update(result.invalid_optional_types) + invalid_types = result.split_keys_to_subdicts(invalid_types) + + missing_keys = result.missing_keys + missing_keys |= result.missing_optional_keys + + solved = result.solved + used_values = result.split_keys_to_subdicts(result.used_values) + + return TemplateResult( + result.output, + self.template, + solved, + used_values, + missing_keys, + invalid_types + ) + + def format_strict(self, *args, **kwargs): + result = self.format(*args, **kwargs) + result.validate() + return result + + @staticmethod + def find_optional_parts(parts): + new_parts = [] + tmp_parts = {} + counted_symb = -1 + for part in parts: + if part == "<": + counted_symb += 1 + tmp_parts[counted_symb] = [] + + elif part == ">": + if counted_symb > -1: + parts = tmp_parts.pop(counted_symb) + counted_symb -= 1 + if parts: + # Remove optional start char + parts.pop(0) + if counted_symb < 0: + out_parts = new_parts + else: + out_parts = tmp_parts[counted_symb] + # Store temp parts + out_parts.append(OptionalPart(parts)) + continue + + if counted_symb < 0: + new_parts.append(part) + else: + tmp_parts[counted_symb].append(part) + + if tmp_parts: + for idx in sorted(tmp_parts.keys()): + new_parts.extend(tmp_parts[idx]) + return new_parts + + +class TemplatesDict(object): + def __init__(self, templates=None): + self._raw_templates = None + self._templates = None + self.set_templates(templates) + + def set_templates(self, templates): + if templates is None: + self._raw_templates = None + self._templates = None + elif isinstance(templates, dict): + self._raw_templates = copy.deepcopy(templates) + self._templates = self.create_ojected_templates(templates) + else: + raise TypeError("<{}> argument must be a dict, not {}.".format( + self.__class__.__name__, str(type(templates)) + )) + + def __getitem__(self, key): + return self.templates[key] + + def get(self, key, *args, **kwargs): + return self.templates.get(key, *args, **kwargs) + + @property + def raw_templates(self): + return self._raw_templates + + @property + def templates(self): + return self._templates + + @classmethod + def create_ojected_templates(cls, templates): + if not isinstance(templates, dict): + raise TypeError("Expected dict object, got {}".format( + str(type(templates)) + )) + + objected_templates = copy.deepcopy(templates) + inner_queue = collections.deque() + inner_queue.append(objected_templates) + while inner_queue: + item = inner_queue.popleft() + if not isinstance(item, dict): + continue + for key in tuple(item.keys()): + value = item[key] + if isinstance(value, six.string_types): + item[key] = StringTemplate(value) + elif isinstance(value, dict): + inner_queue.append(value) + return objected_templates + + def _format_value(self, value, data): + if isinstance(value, StringTemplate): + return value.format(data) + + if isinstance(value, dict): + return self._solve_dict(value, data) + return value + + def _solve_dict(self, templates, data): + """ Solves templates with entered data. + + Args: + templates (dict): All templates which will be formatted. + data (dict): Containing keys to be filled into template. + + Returns: + dict: With `TemplateResult` in values containing filled or + partially filled templates. + """ + output = collections.defaultdict(dict) + for key, value in templates.items(): + output[key] = self._format_value(value, data) + + return output + + def format(self, in_data, only_keys=True, strict=True): + """ Solves templates based on entered data. + + Args: + data (dict): Containing keys to be filled into template. + only_keys (bool, optional): Decides if environ will be used to + fill templates or only keys in data. + + Returns: + TemplatesResultDict: Output `TemplateResult` have `strict` + attribute set to True so accessing unfilled keys in templates + will raise exceptions with explaned error. + """ + # Create a copy of inserted data + data = copy.deepcopy(in_data) + + # Add environment variable to data + if only_keys is False: + for key, val in os.environ.items(): + env_key = "$" + key + if env_key not in data: + data[env_key] = val + + solved = self._solve_dict(self.templates, data) + + output = TemplatesResultDict(solved) + output.strict = strict + return output + + +class TemplateResult(str): + """Result of template format with most of information in. + + Args: + used_values (dict): Dictionary of template filling data with + only used keys. + solved (bool): For check if all required keys were filled. + template (str): Original template. + missing_keys (list): Missing keys that were not in the data. Include + missing optional keys. + invalid_types (dict): When key was found in data, but value had not + allowed DataType. Allowed data types are `numbers`, + `str`(`basestring`) and `dict`. Dictionary may cause invalid type + when value of key in data is dictionary but template expect string + of number. + """ + used_values = None + solved = None + template = None + missing_keys = None + invalid_types = None + + def __new__( + cls, filled_template, template, solved, + used_values, missing_keys, invalid_types + ): + new_obj = super(TemplateResult, cls).__new__(cls, filled_template) + new_obj.used_values = used_values + new_obj.solved = solved + new_obj.template = template + new_obj.missing_keys = list(set(missing_keys)) + new_obj.invalid_types = invalid_types + return new_obj + + def validate(self): + if not self.solved: + raise TemplateUnsolved( + self.template, + self.missing_keys, + self.invalid_types + ) + + +class TemplatesResultDict(dict): + """Holds and wrap TemplateResults for easy bug report.""" + + def __init__(self, in_data, key=None, parent=None, strict=None): + super(TemplatesResultDict, self).__init__() + for _key, _value in in_data.items(): + if isinstance(_value, dict): + _value = self.__class__(_value, _key, self) + self[_key] = _value + + self.key = key + self.parent = parent + self.strict = strict + if self.parent is None and strict is None: + self.strict = True + + def __getitem__(self, key): + if key not in self.keys(): + hier = self.hierarchy() + hier.append(key) + raise TemplateMissingKey(hier) + + value = super(TemplatesResultDict, self).__getitem__(key) + if isinstance(value, self.__class__): + return value + + # Raise exception when expected solved templates and it is not. + if self.raise_on_unsolved and hasattr(value, "validate"): + value.validate() + return value + + @property + def raise_on_unsolved(self): + """To affect this change `strict` attribute.""" + if self.strict is not None: + return self.strict + return self.parent.raise_on_unsolved + + def hierarchy(self): + """Return dictionary keys one by one to root parent.""" + if self.parent is None: + return [] + + hier_keys = [] + par_hier = self.parent.hierarchy() + if par_hier: + hier_keys.extend(par_hier) + hier_keys.append(self.key) + + return hier_keys + + @property + def missing_keys(self): + """Return missing keys of all children templates.""" + missing_keys = set() + for value in self.values(): + missing_keys |= value.missing_keys + return missing_keys + + @property + def invalid_types(self): + """Return invalid types of all children templates.""" + invalid_types = {} + for value in self.values(): + invalid_types = merge_dict(invalid_types, value.invalid_types) + return invalid_types + + @property + def used_values(self): + """Return used values for all children templates.""" + used_values = {} + for value in self.values(): + used_values = merge_dict(used_values, value.used_values) + return used_values + + def get_solved(self): + """Get only solved key from templates.""" + result = {} + for key, value in self.items(): + if isinstance(value, self.__class__): + value = value.get_solved() + if not value: + continue + result[key] = value + + elif ( + not hasattr(value, "solved") or + value.solved + ): + result[key] = value + return self.__class__(result, key=self.key, parent=self.parent) + + +class TemplatePartResult: + """Result to store result of template parts.""" + def __init__(self, optional=False): + # Missing keys or invalid value types of required keys + self._missing_keys = set() + self._invalid_types = {} + # Missing keys or invalid value types of optional keys + self._missing_optional_keys = set() + self._invalid_optional_types = {} + + # Used values stored by key + # - key without any padding or key modifiers + # - value from filling data + # Example: {"version": 1} + self._used_values = {} + # Used values stored by key with all modifirs + # - value is already formatted string + # Example: {"version:0>3": "001"} + self._realy_used_values = {} + # Concatenated string output after formatting + self._output = "" + # Is this result from optional part + self._optional = True + + def add_output(self, other): + if isinstance(other, six.string_types): + self._output += other + + elif isinstance(other, TemplatePartResult): + self._output += other.output + + self._missing_keys |= other.missing_keys + self._missing_optional_keys |= other.missing_optional_keys + + self._invalid_types.update(other.invalid_types) + self._invalid_optional_types.update(other.invalid_optional_types) + + if other.optional and not other.solved: + return + self._used_values.update(other.used_values) + self._realy_used_values.update(other.realy_used_values) + + else: + raise TypeError("Cannot add data from \"{}\" to \"{}\"".format( + str(type(other)), self.__class__.__name__) + ) + + @property + def solved(self): + if self.optional: + if ( + len(self.missing_optional_keys) > 0 + or len(self.invalid_optional_types) > 0 + ): + return False + return ( + len(self.missing_keys) == 0 + and len(self.invalid_types) == 0 + ) + + @property + def optional(self): + return self._optional + + @property + def output(self): + return self._output + + @property + def missing_keys(self): + return self._missing_keys + + @property + def missing_optional_keys(self): + return self._missing_optional_keys + + @property + def invalid_types(self): + return self._invalid_types + + @property + def invalid_optional_types(self): + return self._invalid_optional_types + + @property + def realy_used_values(self): + return self._realy_used_values + + @property + def used_values(self): + return self._used_values + + @staticmethod + def split_keys_to_subdicts(values): + output = {} + for key, value in values.items(): + key_padding = list(KEY_PADDING_PATTERN.findall(key)) + if key_padding: + key = key_padding[0] + key_subdict = list(SUB_DICT_PATTERN.findall(key)) + data = output + last_key = key_subdict.pop(-1) + for subkey in key_subdict: + if subkey not in data: + data[subkey] = {} + data = data[subkey] + data[last_key] = value + return output + + def add_realy_used_value(self, key, value): + self._realy_used_values[key] = value + + def add_used_value(self, key, value): + self._used_values[key] = value + + def add_missing_key(self, key): + if self._optional: + self._missing_optional_keys.add(key) + else: + self._missing_keys.add(key) + + def add_invalid_type(self, key, value): + if self._optional: + self._invalid_optional_types[key] = type(value) + else: + self._invalid_types[key] = type(value) + + +class FormatObject(object): + def __init__(self): + self.value = "" + + def __format__(self, *args, **kwargs): + return self.value.__format__(*args, **kwargs) + + def __str__(self): + return str(self.value) + + def __repr__(self): + return self.__str__() + + +class FormattingPart: + """String with formatting template. + + Containt only single key to format e.g. "{project[name]}". + + Args: + template(str): String containing the formatting key. + """ + def __init__(self, template): + self._template = template + + @property + def template(self): + return self._template + + def __repr__(self): + return "".format(self._template) + + def __str__(self): + return self._template + + @staticmethod + def validate_value_type(value): + """Check if value can be used for formatting of single key.""" + if isinstance(value, (numbers.Number, FormatObject)): + return True + + for inh_class in type(value).mro(): + if inh_class in six.string_types: + return True + return False + + def format(self, data, result): + """Format the formattings string. + + Args: + data(dict): Data that should be used for formatting. + result(TemplatePartResult): Object where result is stored. + """ + key = self.template[1:-1] + if key in result.realy_used_values: + result.add_output(result.realy_used_values[key]) + return result + + # check if key expects subdictionary keys (e.g. project[name]) + existence_check = key + key_padding = list(KEY_PADDING_PATTERN.findall(existence_check)) + if key_padding: + existence_check = key_padding[0] + key_subdict = list(SUB_DICT_PATTERN.findall(existence_check)) + + value = data + missing_key = False + invalid_type = False + used_keys = [] + for sub_key in key_subdict: + if ( + value is None + or (hasattr(value, "items") and sub_key not in value) + ): + missing_key = True + used_keys.append(sub_key) + break + + if not hasattr(value, "items"): + invalid_type = True + break + + used_keys.append(sub_key) + value = value.get(sub_key) + + if missing_key or invalid_type: + if len(used_keys) == 0: + invalid_key = key_subdict[0] + else: + invalid_key = used_keys[0] + for idx, sub_key in enumerate(used_keys): + if idx == 0: + continue + invalid_key += "[{0}]".format(sub_key) + + if missing_key: + result.add_missing_key(invalid_key) + + elif invalid_type: + result.add_invalid_type(invalid_key, value) + + result.add_output(self.template) + return result + + if self.validate_value_type(value): + fill_data = {} + first_value = True + for used_key in reversed(used_keys): + if first_value: + first_value = False + fill_data[used_key] = value + else: + _fill_data = {used_key: fill_data} + fill_data = _fill_data + + formatted_value = self.template.format(**fill_data) + result.add_realy_used_value(key, formatted_value) + result.add_used_value(existence_check, value) + result.add_output(formatted_value) + return result + + result.add_invalid_type(key, value) + result.add_output(self.template) + + return result + + +class OptionalPart: + """Template part which contains optional formatting strings. + + If this part can't be filled the result is empty string. + + Args: + parts(list): Parts of template. Can contain 'str', 'OptionalPart' or + 'FormattingPart'. + """ + def __init__(self, parts): + self._parts = parts + + @property + def parts(self): + return self._parts + + def __str__(self): + return "<{}>".format("".join([str(p) for p in self._parts])) + + def __repr__(self): + return "".format("".join([str(p) for p in self._parts])) + + def format(self, data, result): + new_result = TemplatePartResult(True) + for part in self._parts: + if isinstance(part, six.string_types): + new_result.add_output(part) + else: + part.format(data, new_result) + + if new_result.solved: + result.add_output(new_result) + return result From a80b15d7912cb92b419699a16e0442d6dc8909f4 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Sat, 19 Feb 2022 20:33:51 +0100 Subject: [PATCH 02/10] use new templates logic in anatomy templates --- openpype/lib/__init__.py | 18 +- openpype/lib/anatomy.py | 635 +++++---------------------------- openpype/lib/path_templates.py | 4 + 3 files changed, 117 insertions(+), 540 deletions(-) diff --git a/openpype/lib/__init__.py b/openpype/lib/__init__.py index ebe7648ad7..4d956d9876 100644 --- a/openpype/lib/__init__.py +++ b/openpype/lib/__init__.py @@ -16,6 +16,15 @@ sys.path.insert(0, python_version_dir) site.addsitedir(python_version_dir) +from .path_templates import ( + merge_dict, + TemplateMissingKey, + TemplateUnsolved, + StringTemplate, + TemplatesDict, + FormatObject, +) + from .env_tools import ( env_value_to_bool, get_paths_from_environ, @@ -41,7 +50,6 @@ from .mongo import ( OpenPypeMongoConnection ) from .anatomy import ( - merge_dict, Anatomy ) @@ -183,6 +191,13 @@ from .openpype_version import ( terminal = Terminal __all__ = [ + "merge_dict", + "TemplateMissingKey", + "TemplateUnsolved", + "StringTemplate", + "TemplatesDict", + "FormatObject", + "get_openpype_execute_args", "get_pype_execute_args", "get_linux_launcher_args", @@ -285,7 +300,6 @@ __all__ = [ "terminal", - "merge_dict", "Anatomy", "get_datetime_data", diff --git a/openpype/lib/anatomy.py b/openpype/lib/anatomy.py index fa81a18ff7..f817646cd7 100644 --- a/openpype/lib/anatomy.py +++ b/openpype/lib/anatomy.py @@ -9,6 +9,14 @@ from openpype.settings.lib import ( get_default_anatomy_settings, get_anatomy_settings ) +from .path_templates import ( + merge_dict, + TemplateUnsolved, + TemplateResult, + TemplatesResultDict, + TemplatesDict, + FormatObject, +) from .log import PypeLogger log = PypeLogger().get_logger(__name__) @@ -19,32 +27,6 @@ except NameError: StringType = str -def merge_dict(main_dict, enhance_dict): - """Merges dictionaries by keys. - - Function call itself if value on key is again dictionary. - - Args: - main_dict (dict): First dict to merge second one into. - enhance_dict (dict): Second dict to be merged. - - Returns: - dict: Merged result. - - .. note:: does not overrides whole value on first found key - but only values differences from enhance_dict - - """ - for key, value in enhance_dict.items(): - if key not in main_dict: - main_dict[key] = value - elif isinstance(value, dict) and isinstance(main_dict[key], dict): - main_dict[key] = merge_dict(main_dict[key], value) - else: - main_dict[key] = value - return main_dict - - class ProjectNotSet(Exception): """Exception raised when is created Anatomy without project name.""" @@ -59,7 +41,7 @@ class RootCombinationError(Exception): # TODO better error message msg = ( "Combination of root with and" - " without root name in Templates. {}" + " without root name in AnatomyTemplates. {}" ).format(joined_roots) super(RootCombinationError, self).__init__(msg) @@ -68,7 +50,7 @@ class RootCombinationError(Exception): class Anatomy: """Anatomy module helps to keep project settings. - Wraps key project specifications, Templates and Roots. + Wraps key project specifications, AnatomyTemplates and Roots. Args: project_name (str): Project name to look on overrides. @@ -93,7 +75,7 @@ class Anatomy: get_anatomy_settings(project_name, site_name) ) self._site_name = site_name - self._templates_obj = Templates(self) + self._templates_obj = AnatomyTemplates(self) self._roots_obj = Roots(self) # Anatomy used as dictionary @@ -158,12 +140,12 @@ class Anatomy: @property def templates(self): - """Wrap property `templates` of Anatomy's Templates instance.""" + """Wrap property `templates` of Anatomy's AnatomyTemplates instance.""" return self._templates_obj.templates @property def templates_obj(self): - """Return `Templates` object of current Anatomy instance.""" + """Return `AnatomyTemplates` object of current Anatomy instance.""" return self._templates_obj def format(self, *args, **kwargs): @@ -375,203 +357,45 @@ class Anatomy: return rootless_path.format(**data) -class TemplateMissingKey(Exception): - """Exception for cases when key does not exist in Anatomy.""" - - msg = "Anatomy key does not exist: `anatomy{0}`." - - def __init__(self, parents): - parent_join = "".join(["[\"{0}\"]".format(key) for key in parents]) - super(TemplateMissingKey, self).__init__( - self.msg.format(parent_join) - ) - - -class TemplateUnsolved(Exception): +class AnatomyTemplateUnsolved(TemplateUnsolved): """Exception for unsolved template when strict is set to True.""" msg = "Anatomy template \"{0}\" is unsolved.{1}{2}" - invalid_types_msg = " Keys with invalid DataType: `{0}`." - missing_keys_msg = " Missing keys: \"{0}\"." - def __init__(self, template, missing_keys, invalid_types): - invalid_type_items = [] - for _key, _type in invalid_types.items(): - invalid_type_items.append( - "\"{0}\" {1}".format(_key, str(_type)) - ) - invalid_types_msg = "" - if invalid_type_items: - invalid_types_msg = self.invalid_types_msg.format( - ", ".join(invalid_type_items) - ) +class AnatomyTemplateResult(TemplateResult): + rootless = None - missing_keys_msg = "" - if missing_keys: - missing_keys_msg = self.missing_keys_msg.format( - ", ".join(missing_keys) - ) - super(TemplateUnsolved, self).__init__( - self.msg.format(template, missing_keys_msg, invalid_types_msg) + def __new__(cls, result, rootless_path): + new_obj = super(AnatomyTemplateResult, cls).__new__( + cls, + str(result), + result.template, + result.solved, + result.used_values, + result.missing_keys, + result.invalid_types ) - - -class TemplateResult(str): - """Result (formatted template) of anatomy with most of information in. - - Args: - used_values (dict): Dictionary of template filling data with - only used keys. - solved (bool): For check if all required keys were filled. - template (str): Original template. - missing_keys (list): Missing keys that were not in the data. Include - missing optional keys. - invalid_types (dict): When key was found in data, but value had not - allowed DataType. Allowed data types are `numbers`, - `str`(`basestring`) and `dict`. Dictionary may cause invalid type - when value of key in data is dictionary but template expect string - of number. - """ - - def __new__( - cls, filled_template, template, solved, rootless_path, - used_values, missing_keys, invalid_types - ): - new_obj = super(TemplateResult, cls).__new__(cls, filled_template) - new_obj.used_values = used_values - new_obj.solved = solved - new_obj.template = template new_obj.rootless = rootless_path - new_obj.missing_keys = list(set(missing_keys)) - _invalid_types = {} - for invalid_type in invalid_types: - for key, val in invalid_type.items(): - if key in _invalid_types: - continue - _invalid_types[key] = val - new_obj.invalid_types = _invalid_types return new_obj - -class TemplatesDict(dict): - """Holds and wrap TemplateResults for easy bug report.""" - - def __init__(self, in_data, key=None, parent=None, strict=None): - super(TemplatesDict, self).__init__() - for _key, _value in in_data.items(): - if isinstance(_value, dict): - _value = self.__class__(_value, _key, self) - self[_key] = _value - - self.key = key - self.parent = parent - self.strict = strict - if self.parent is None and strict is None: - self.strict = True - - def __getitem__(self, key): - # Raise error about missing key in anatomy.yaml - if key not in self.keys(): - hier = self.hierarchy() - hier.append(key) - raise TemplateMissingKey(hier) - - value = super(TemplatesDict, self).__getitem__(key) - if isinstance(value, self.__class__): - return value - - # Raise exception when expected solved templates and it is not. - if ( - self.raise_on_unsolved - and (hasattr(value, "solved") and not value.solved) - ): - raise TemplateUnsolved( - value.template, value.missing_keys, value.invalid_types + def validate(self): + if not self.solved: + raise AnatomyTemplateUnsolved( + self.template, + self.missing_keys, + self.invalid_types ) - return value - - @property - def raise_on_unsolved(self): - """To affect this change `strict` attribute.""" - if self.strict is not None: - return self.strict - return self.parent.raise_on_unsolved - - def hierarchy(self): - """Return dictionary keys one by one to root parent.""" - if self.parent is None: - return [] - - hier_keys = [] - par_hier = self.parent.hierarchy() - if par_hier: - hier_keys.extend(par_hier) - hier_keys.append(self.key) - - return hier_keys - - @property - def missing_keys(self): - """Return missing keys of all children templates.""" - missing_keys = [] - for value in self.values(): - missing_keys.extend(value.missing_keys) - return list(set(missing_keys)) - - @property - def invalid_types(self): - """Return invalid types of all children templates.""" - invalid_types = {} - for value in self.values(): - for invalid_type in value.invalid_types: - _invalid_types = {} - for key, val in invalid_type.items(): - if key in invalid_types: - continue - _invalid_types[key] = val - invalid_types = merge_dict(invalid_types, _invalid_types) - return invalid_types - - @property - def used_values(self): - """Return used values for all children templates.""" - used_values = {} - for value in self.values(): - used_values = merge_dict(used_values, value.used_values) - return used_values - - def get_solved(self): - """Get only solved key from templates.""" - result = {} - for key, value in self.items(): - if isinstance(value, self.__class__): - value = value.get_solved() - if not value: - continue - result[key] = value - - elif ( - not hasattr(value, "solved") or - value.solved - ): - result[key] = value - return self.__class__(result, key=self.key, parent=self.parent) -class Templates: - key_pattern = re.compile(r"(\{.*?[^{0]*\})") - key_padding_pattern = re.compile(r"([^:]+)\S+[><]\S+") - sub_dict_pattern = re.compile(r"([^\[\]]+)") - optional_pattern = re.compile(r"(<.*?[^{0]*>)[^0-9]*?") - +class AnatomyTemplates(TemplatesDict): inner_key_pattern = re.compile(r"(\{@.*?[^{}0]*\})") inner_key_name_pattern = re.compile(r"\{@(.*?[^{}0]*)\}") def __init__(self, anatomy): + super(AnatomyTemplates, self).__init__() self.anatomy = anatomy self.loaded_project = None - self._templates = None def __getitem__(self, key): return self.templates[key] @@ -596,13 +420,51 @@ class Templates: self._templates = None if self._templates is None: - self._templates = self._discover() + self._discover() self.loaded_project = self.project_name return self._templates + def _format_value(self, value, data): + if isinstance(value, RootItem): + return self._solve_dict(value, data) + + result = super(AnatomyTemplates, self)._format_value(value, data) + if isinstance(result, TemplateResult): + rootless_path = self._rootless_path(result, data) + result = AnatomyTemplateResult(result, rootless_path) + return result + + def set_templates(self, templates): + if not templates: + self._raw_templates = None + self._templates = None + else: + self._raw_templates = copy.deepcopy(templates) + templates = copy.deepcopy(templates) + v_queue = collections.deque() + v_queue.append(templates) + while v_queue: + item = v_queue.popleft() + if not isinstance(item, dict): + continue + + for key in tuple(item.keys()): + value = item[key] + if isinstance(value, dict): + v_queue.append(value) + + elif ( + isinstance(value, StringType) + and "{task}" in value + ): + item[key] = value.replace("{task}", "{task[name]}") + + solved_templates = self.solve_template_inner_links(templates) + self._templates = self.create_ojected_templates(solved_templates) + def default_templates(self): """Return default templates data with solved inner keys.""" - return Templates.solve_template_inner_links( + return self.solve_template_inner_links( self.anatomy["templates"] ) @@ -613,7 +475,7 @@ class Templates: TODO: create templates if not exist. Returns: - TemplatesDict: Contain templates data for current project of + TemplatesResultDict: Contain templates data for current project of default templates. """ @@ -624,7 +486,7 @@ class Templates: " Trying to use default." ).format(self.project_name)) - return Templates.solve_template_inner_links(self.anatomy["templates"]) + self.set_templates(self.anatomy["templates"]) @classmethod def replace_inner_keys(cls, matches, value, key_values, key): @@ -791,149 +653,6 @@ class Templates: return keys_by_subkey - def _filter_optional(self, template, data): - """Filter invalid optional keys. - - Invalid keys may be missing keys of with invalid value DataType. - - Args: - template (str): Anatomy template which will be formatted. - data (dict): Containing keys to be filled into template. - - Result: - tuple: Contain origin template without missing optional keys and - without optional keys identificator ("<" and ">"), information - about missing optional keys and invalid types of optional keys. - - """ - - # Remove optional missing keys - missing_keys = [] - invalid_types = [] - for optional_group in self.optional_pattern.findall(template): - _missing_keys = [] - _invalid_types = [] - for optional_key in self.key_pattern.findall(optional_group): - key = str(optional_key[1:-1]) - key_padding = list( - self.key_padding_pattern.findall(key) - ) - if key_padding: - key = key_padding[0] - - validation_result = self._validate_data_key( - key, data - ) - missing_key = validation_result["missing_key"] - invalid_type = validation_result["invalid_type"] - - valid = True - if missing_key is not None: - _missing_keys.append(missing_key) - valid = False - - if invalid_type is not None: - _invalid_types.append(invalid_type) - valid = False - - if valid: - try: - optional_key.format(**data) - except KeyError: - _missing_keys.append(key) - valid = False - - valid = len(_invalid_types) == 0 and len(_missing_keys) == 0 - missing_keys.extend(_missing_keys) - invalid_types.extend(_invalid_types) - replacement = "" - if valid: - replacement = optional_group[1:-1] - - template = template.replace(optional_group, replacement) - return (template, missing_keys, invalid_types) - - def _validate_data_key(self, key, data): - """Check and prepare missing keys and invalid types of template.""" - result = { - "missing_key": None, - "invalid_type": None - } - - # check if key expects subdictionary keys (e.g. project[name]) - key_subdict = list(self.sub_dict_pattern.findall(key)) - used_keys = [] - if len(key_subdict) <= 1: - if key not in data: - result["missing_key"] = key - return result - - used_keys.append(key) - value = data[key] - - else: - value = data - missing_key = False - invalid_type = False - for sub_key in key_subdict: - if ( - value is None - or (hasattr(value, "items") and sub_key not in value) - ): - missing_key = True - used_keys.append(sub_key) - break - - elif not hasattr(value, "items"): - invalid_type = True - break - - used_keys.append(sub_key) - value = value.get(sub_key) - - if missing_key or invalid_type: - if len(used_keys) == 0: - invalid_key = key_subdict[0] - else: - invalid_key = used_keys[0] - for idx, sub_key in enumerate(used_keys): - if idx == 0: - continue - invalid_key += "[{0}]".format(sub_key) - - if missing_key: - result["missing_key"] = invalid_key - - elif invalid_type: - result["invalid_type"] = {invalid_key: type(value)} - - return result - - if isinstance(value, (numbers.Number, Roots, RootItem)): - return result - - for inh_class in type(value).mro(): - if inh_class == StringType: - return result - - result["missing_key"] = key - result["invalid_type"] = {key: type(value)} - return result - - def _merge_used_values(self, current_used, keys, value): - key = keys[0] - _keys = keys[1:] - if len(_keys) == 0: - current_used[key] = value - else: - next_dict = {} - if key in current_used: - next_dict = current_used[key] - current_used[key] = self._merge_used_values( - next_dict, _keys, value - ) - return current_used - def _dict_to_subkeys_list(self, subdict, pre_keys=None): if pre_keys is None: pre_keys = [] @@ -956,9 +675,11 @@ class Templates: return {key_list[0]: value} return {key_list[0]: self._keys_to_dicts(key_list[1:], value)} - def _rootless_path( - self, template, used_values, final_data, missing_keys, invalid_types - ): + def _rootless_path(self, result, final_data): + used_values = result.used_values + missing_keys = result.missing_keys + template = result.template + invalid_types = result.invalid_types if ( "root" not in used_values or "root" in missing_keys @@ -974,210 +695,48 @@ class Templates: if not root_keys: return - roots_dict = {} + output = str(result) for used_root_keys in root_keys: if not used_root_keys: continue + used_value = used_values root_key = None for key in used_root_keys: + used_value = used_value[key] if root_key is None: root_key = key else: root_key += "[{}]".format(key) root_key = "{" + root_key + "}" - - roots_dict = merge_dict( - roots_dict, - self._keys_to_dicts(used_root_keys, root_key) - ) - - final_data["root"] = roots_dict["root"] - return template.format(**final_data) - - def _format(self, orig_template, data): - """ Figure out with whole formatting. - - Separate advanced keys (*Like '{project[name]}') from string which must - be formatted separatelly in case of missing or incomplete keys in data. - - Args: - template (str): Anatomy template which will be formatted. - data (dict): Containing keys to be filled into template. - - Returns: - TemplateResult: Filled or partially filled template containing all - data needed or missing for filling template. - """ - task_data = data.get("task") - if ( - isinstance(task_data, StringType) - and "{task[name]}" in orig_template - ): - # Change task to dictionary if template expect dictionary - data["task"] = {"name": task_data} - - template, missing_optional, invalid_optional = ( - self._filter_optional(orig_template, data) - ) - # Remove optional missing keys - used_values = {} - invalid_required = [] - missing_required = [] - replace_keys = [] - - for group in self.key_pattern.findall(template): - orig_key = group[1:-1] - key = str(orig_key) - key_padding = list(self.key_padding_pattern.findall(key)) - if key_padding: - key = key_padding[0] - - validation_result = self._validate_data_key(key, data) - missing_key = validation_result["missing_key"] - invalid_type = validation_result["invalid_type"] - - if invalid_type is not None: - invalid_required.append(invalid_type) - replace_keys.append(key) - continue - - if missing_key is not None: - missing_required.append(missing_key) - replace_keys.append(key) - continue - - try: - value = group.format(**data) - key_subdict = list(self.sub_dict_pattern.findall(key)) - if len(key_subdict) <= 1: - used_values[key] = value - - else: - used_values = self._merge_used_values( - used_values, key_subdict, value - ) - - except (TypeError, KeyError): - missing_required.append(key) - replace_keys.append(key) - - final_data = copy.deepcopy(data) - for key in replace_keys: - key_subdict = list(self.sub_dict_pattern.findall(key)) - if len(key_subdict) <= 1: - final_data[key] = "{" + key + "}" - continue - - replace_key_dst = "---".join(key_subdict) - replace_key_dst_curly = "{" + replace_key_dst + "}" - replace_key_src_curly = "{" + key + "}" - template = template.replace( - replace_key_src_curly, replace_key_dst_curly - ) - final_data[replace_key_dst] = replace_key_src_curly - - solved = len(missing_required) == 0 and len(invalid_required) == 0 - - missing_keys = missing_required + missing_optional - invalid_types = invalid_required + invalid_optional - - filled_template = template.format(**final_data) - # WARNING `_rootless_path` change values in `final_data` please keep - # in midn when changing order - rootless_path = self._rootless_path( - template, used_values, final_data, missing_keys, invalid_types - ) - if rootless_path is None: - rootless_path = filled_template - - result = TemplateResult( - filled_template, orig_template, solved, rootless_path, - used_values, missing_keys, invalid_types - ) - return result - - def solve_dict(self, templates, data): - """ Solves templates with entered data. - - Args: - templates (dict): All Anatomy templates which will be formatted. - data (dict): Containing keys to be filled into template. - - Returns: - dict: With `TemplateResult` in values containing filled or - partially filled templates. - """ - output = collections.defaultdict(dict) - for key, orig_value in templates.items(): - if isinstance(orig_value, StringType): - # Replace {task} by '{task[name]}' for backward compatibility - if '{task}' in orig_value: - orig_value = orig_value.replace('{task}', '{task[name]}') - - output[key] = self._format(orig_value, data) - continue - - # Check if orig_value has items attribute (any dict inheritance) - if not hasattr(orig_value, "items"): - # TODO we should handle this case - output[key] = orig_value - continue - - for s_key, s_value in self.solve_dict(orig_value, data).items(): - output[key][s_key] = s_value + output = output.replace(str(used_value), root_key) return output + def format(self, data, strict=True): + roots = self.roots + if roots: + data["root"] = roots + result = super(AnatomyTemplates, self).format(data) + result.strict = strict + return result + def format_all(self, in_data, only_keys=True): """ Solves templates based on entered data. Args: data (dict): Containing keys to be filled into template. - only_keys (bool, optional): Decides if environ will be used to - fill templates or only keys in data. Returns: - TemplatesDict: Output `TemplateResult` have `strict` attribute - set to False so accessing unfilled keys in templates won't - raise any exceptions. + TemplatesResultDict: Output `TemplateResult` have `strict` + attribute set to False so accessing unfilled keys in templates + won't raise any exceptions. """ - output = self.format(in_data, only_keys) - output.strict = False - return output - - def format(self, in_data, only_keys=True): - """ Solves templates based on entered data. - - Args: - data (dict): Containing keys to be filled into template. - only_keys (bool, optional): Decides if environ will be used to - fill templates or only keys in data. - - Returns: - TemplatesDict: Output `TemplateResult` have `strict` attribute - set to True so accessing unfilled keys in templates will - raise exceptions with explaned error. - """ - # Create a copy of inserted data - data = copy.deepcopy(in_data) - - # Add environment variable to data - if only_keys is False: - for key, val in os.environ.items(): - data["$" + key] = val - - # override root value - roots = self.roots - if roots: - data["root"] = roots - solved = self.solve_dict(self.templates, data) - - return TemplatesDict(solved) + return self.format(in_data, strict=False) -class RootItem: +class RootItem(FormatObject): """Represents one item or roots. Holds raw data of root item specification. Raw data contain value diff --git a/openpype/lib/path_templates.py b/openpype/lib/path_templates.py index 6f68cc4ce9..b51951851f 100644 --- a/openpype/lib/path_templates.py +++ b/openpype/lib/path_templates.py @@ -584,6 +584,10 @@ class TemplatePartResult: class FormatObject(object): + """Object that can be used for formatting. + + This is base that is valid for to be used in 'StringTemplate' value. + """ def __init__(self): self.value = "" From 1fa2e86d1840a8a8801fdb8455930442550f792d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Sat, 19 Feb 2022 20:36:13 +0100 Subject: [PATCH 03/10] reorganized init --- openpype/lib/__init__.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/openpype/lib/__init__.py b/openpype/lib/__init__.py index 4d956d9876..6ec10a2209 100644 --- a/openpype/lib/__init__.py +++ b/openpype/lib/__init__.py @@ -16,15 +16,6 @@ sys.path.insert(0, python_version_dir) site.addsitedir(python_version_dir) -from .path_templates import ( - merge_dict, - TemplateMissingKey, - TemplateUnsolved, - StringTemplate, - TemplatesDict, - FormatObject, -) - from .env_tools import ( env_value_to_bool, get_paths_from_environ, @@ -44,6 +35,16 @@ from .execute import ( CREATE_NO_WINDOW ) from .log import PypeLogger, timeit + +from .path_templates import ( + merge_dict, + TemplateMissingKey, + TemplateUnsolved, + StringTemplate, + TemplatesDict, + FormatObject, +) + from .mongo import ( get_default_components, validate_mongo_connection, @@ -191,13 +192,6 @@ from .openpype_version import ( terminal = Terminal __all__ = [ - "merge_dict", - "TemplateMissingKey", - "TemplateUnsolved", - "StringTemplate", - "TemplatesDict", - "FormatObject", - "get_openpype_execute_args", "get_pype_execute_args", "get_linux_launcher_args", @@ -298,6 +292,13 @@ __all__ = [ "get_version_from_path", "get_last_version_from_path", + "merge_dict", + "TemplateMissingKey", + "TemplateUnsolved", + "StringTemplate", + "TemplatesDict", + "FormatObject", + "terminal", "Anatomy", From 26549b34650a45037084107356f94a12b3146441 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Sat, 19 Feb 2022 20:51:28 +0100 Subject: [PATCH 04/10] hound fixes --- openpype/lib/anatomy.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openpype/lib/anatomy.py b/openpype/lib/anatomy.py index f817646cd7..8f2f09a803 100644 --- a/openpype/lib/anatomy.py +++ b/openpype/lib/anatomy.py @@ -10,10 +10,8 @@ from openpype.settings.lib import ( get_anatomy_settings ) from .path_templates import ( - merge_dict, TemplateUnsolved, TemplateResult, - TemplatesResultDict, TemplatesDict, FormatObject, ) @@ -69,7 +67,10 @@ class Anatomy: " to load data for specific project." )) + from .avalon_context import get_project_code + self.project_name = project_name + self.project_code = get_project_code(project_name) self._data = self._prepare_anatomy_data( get_anatomy_settings(project_name, site_name) From f985e58bb9687d19e63c05dce061063fce35e77e Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Thu, 24 Feb 2022 12:27:13 +0100 Subject: [PATCH 05/10] Removed forgotten lines --- openpype/lib/anatomy.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/openpype/lib/anatomy.py b/openpype/lib/anatomy.py index 8f2f09a803..3bcd6169e4 100644 --- a/openpype/lib/anatomy.py +++ b/openpype/lib/anatomy.py @@ -67,10 +67,7 @@ class Anatomy: " to load data for specific project." )) - from .avalon_context import get_project_code - self.project_name = project_name - self.project_code = get_project_code(project_name) self._data = self._prepare_anatomy_data( get_anatomy_settings(project_name, site_name) From 8d7f56c9e822cc3b71c9d928cf8da153c68d1804 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 24 Feb 2022 15:11:40 +0100 Subject: [PATCH 06/10] added more methods to StringTemplate --- openpype/lib/path_templates.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/openpype/lib/path_templates.py b/openpype/lib/path_templates.py index b51951851f..3b0e9ad3cc 100644 --- a/openpype/lib/path_templates.py +++ b/openpype/lib/path_templates.py @@ -126,6 +126,19 @@ class StringTemplate(object): self._parts = self.find_optional_parts(new_parts) + def __str__(self): + return self.template + + def __repr__(self): + return "<{}> {}".format(self.__class__.__name__, self.template) + + def __contains__(self, other): + return other in self.template + + def replace(self, *args, **kwargs): + self._template = self.template.replace(*args, **kwargs) + return self + @property def template(self): return self._template From 899e59b05919a8ebd3830fd381a81aded412d5fc Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 24 Feb 2022 15:12:09 +0100 Subject: [PATCH 07/10] fix which template key is used for getting last workfile --- openpype/lib/applications.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/openpype/lib/applications.py b/openpype/lib/applications.py index 393c83e9be..f6182c1846 100644 --- a/openpype/lib/applications.py +++ b/openpype/lib/applications.py @@ -28,7 +28,8 @@ from .profiles_filtering import filter_profiles from .local_settings import get_openpype_username from .avalon_context import ( get_workdir_data, - get_workdir_with_workdir_data + get_workdir_with_workdir_data, + get_workfile_template_key ) from .python_module_tools import ( @@ -1587,14 +1588,15 @@ def _prepare_last_workfile(data, workdir): last_workfile_path = data.get("last_workfile_path") or "" if not last_workfile_path: extensions = avalon.api.HOST_WORKFILE_EXTENSIONS.get(app.host_name) - if extensions: anatomy = data["anatomy"] + project_settings = data["project_settings"] + task_type = workdir_data["task"]["type"] + template_key = get_workfile_template_key( + task_type, app.host_name, project_settings=project_settings + ) # Find last workfile - file_template = anatomy.templates["work"]["file"] - # Replace {task} by '{task[name]}' for backward compatibility - if '{task}' in file_template: - file_template = file_template.replace('{task}', '{task[name]}') + file_template = str(anatomy.templates[template_key]["file"]) workdir_data.update({ "version": 1, From 43839e63f131e0bd3fc32d1d91fdb61b1db1e96d Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 24 Feb 2022 15:36:15 +0100 Subject: [PATCH 08/10] TemplatesDict does not return objected templates with 'templates' attribute --- openpype/lib/anatomy.py | 61 +++++++++++-------- openpype/lib/path_templates.py | 11 +++- .../action_create_folders.py | 1 - 3 files changed, 46 insertions(+), 27 deletions(-) diff --git a/openpype/lib/anatomy.py b/openpype/lib/anatomy.py index 3bcd6169e4..3d56c1f1ba 100644 --- a/openpype/lib/anatomy.py +++ b/openpype/lib/anatomy.py @@ -402,7 +402,9 @@ class AnatomyTemplates(TemplatesDict): return self.templates.get(key, default) def reset(self): + self._raw_templates = None self._templates = None + self._objected_templates = None @property def project_name(self): @@ -414,13 +416,21 @@ class AnatomyTemplates(TemplatesDict): @property def templates(self): + self._validate_discovery() + return self._templates + + @property + def objected_templates(self): + self._validate_discovery() + return self._objected_templates + + def _validate_discovery(self): if self.project_name != self.loaded_project: - self._templates = None + self.reset() if self._templates is None: self._discover() self.loaded_project = self.project_name - return self._templates def _format_value(self, value, data): if isinstance(value, RootItem): @@ -434,31 +444,34 @@ class AnatomyTemplates(TemplatesDict): def set_templates(self, templates): if not templates: - self._raw_templates = None - self._templates = None - else: - self._raw_templates = copy.deepcopy(templates) - templates = copy.deepcopy(templates) - v_queue = collections.deque() - v_queue.append(templates) - while v_queue: - item = v_queue.popleft() - if not isinstance(item, dict): - continue + self.reset() + return - for key in tuple(item.keys()): - value = item[key] - if isinstance(value, dict): - v_queue.append(value) + self._raw_templates = copy.deepcopy(templates) + templates = copy.deepcopy(templates) + v_queue = collections.deque() + v_queue.append(templates) + while v_queue: + item = v_queue.popleft() + if not isinstance(item, dict): + continue - elif ( - isinstance(value, StringType) - and "{task}" in value - ): - item[key] = value.replace("{task}", "{task[name]}") + for key in tuple(item.keys()): + value = item[key] + if isinstance(value, dict): + v_queue.append(value) - solved_templates = self.solve_template_inner_links(templates) - self._templates = self.create_ojected_templates(solved_templates) + elif ( + isinstance(value, StringType) + and "{task}" in value + ): + item[key] = value.replace("{task}", "{task[name]}") + + solved_templates = self.solve_template_inner_links(templates) + self._templates = solved_templates + self._objected_templates = self.create_ojected_templates( + solved_templates + ) def default_templates(self): """Return default templates data with solved inner keys.""" diff --git a/openpype/lib/path_templates.py b/openpype/lib/path_templates.py index 3b0e9ad3cc..370ffdd27c 100644 --- a/openpype/lib/path_templates.py +++ b/openpype/lib/path_templates.py @@ -227,15 +227,18 @@ class TemplatesDict(object): def __init__(self, templates=None): self._raw_templates = None self._templates = None + self._objected_templates = None self.set_templates(templates) def set_templates(self, templates): if templates is None: self._raw_templates = None self._templates = None + self._objected_templates = None elif isinstance(templates, dict): self._raw_templates = copy.deepcopy(templates) - self._templates = self.create_ojected_templates(templates) + self._templates = templates + self._objected_templates = self.create_ojected_templates(templates) else: raise TypeError("<{}> argument must be a dict, not {}.".format( self.__class__.__name__, str(type(templates)) @@ -255,6 +258,10 @@ class TemplatesDict(object): def templates(self): return self._templates + @property + def objected_templates(self): + return self._objected_templates + @classmethod def create_ojected_templates(cls, templates): if not isinstance(templates, dict): @@ -325,7 +332,7 @@ class TemplatesDict(object): if env_key not in data: data[env_key] = val - solved = self._solve_dict(self.templates, data) + solved = self._solve_dict(self.objected_templates, data) output = TemplatesResultDict(solved) output.strict = strict diff --git a/openpype/modules/default_modules/ftrack/event_handlers_user/action_create_folders.py b/openpype/modules/default_modules/ftrack/event_handlers_user/action_create_folders.py index 8bbef9ad73..d15a865124 100644 --- a/openpype/modules/default_modules/ftrack/event_handlers_user/action_create_folders.py +++ b/openpype/modules/default_modules/ftrack/event_handlers_user/action_create_folders.py @@ -97,7 +97,6 @@ class CreateFolders(BaseAction): all_entities = self.get_notask_children(entity) anatomy = Anatomy(project_name) - project_settings = get_project_settings(project_name) work_keys = ["work", "folder"] work_template = anatomy.templates From e9c67e35bc2d294d1aa9a4319f4d67022079d1af Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 24 Feb 2022 17:21:52 +0100 Subject: [PATCH 09/10] fixed used values passed to TemplateResult --- openpype/lib/path_templates.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/openpype/lib/path_templates.py b/openpype/lib/path_templates.py index 370ffdd27c..62bfdf774a 100644 --- a/openpype/lib/path_templates.py +++ b/openpype/lib/path_templates.py @@ -171,7 +171,7 @@ class StringTemplate(object): missing_keys |= result.missing_optional_keys solved = result.solved - used_values = result.split_keys_to_subdicts(result.used_values) + used_values = result.get_clean_used_values() return TemplateResult( result.output, @@ -485,7 +485,7 @@ class TemplatePartResult: self._missing_optional_keys = set() self._invalid_optional_types = {} - # Used values stored by key + # Used values stored by key with origin type # - key without any padding or key modifiers # - value from filling data # Example: {"version": 1} @@ -584,6 +584,15 @@ class TemplatePartResult: data[last_key] = value return output + def get_clean_used_values(self): + new_used_values = {} + for key, value in self.used_values.items(): + if isinstance(value, FormatObject): + value = str(value) + new_used_values[key] = value + + return self.split_keys_to_subdicts(new_used_values) + def add_realy_used_value(self, key, value): self._realy_used_values[key] = value @@ -724,7 +733,7 @@ class FormattingPart: formatted_value = self.template.format(**fill_data) result.add_realy_used_value(key, formatted_value) - result.add_used_value(existence_check, value) + result.add_used_value(existence_check, formatted_value) result.add_output(formatted_value) return result From ed526947ebb717d830da9f876e57b78ec6c6bed3 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Tue, 1 Mar 2022 15:32:46 +0100 Subject: [PATCH 10/10] fix adding 'root' key to format data --- openpype/lib/anatomy.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openpype/lib/anatomy.py b/openpype/lib/anatomy.py index 3d56c1f1ba..3fbc05ee88 100644 --- a/openpype/lib/anatomy.py +++ b/openpype/lib/anatomy.py @@ -726,10 +726,11 @@ class AnatomyTemplates(TemplatesDict): return output def format(self, data, strict=True): + copy_data = copy.deepcopy(data) roots = self.roots if roots: - data["root"] = roots - result = super(AnatomyTemplates, self).format(data) + copy_data["root"] = roots + result = super(AnatomyTemplates, self).format(copy_data) result.strict = strict return result