diff --git a/client/ayon_core/lib/__init__.py b/client/ayon_core/lib/__init__.py index 5ccc8d03e5..d5629cbf3d 100644 --- a/client/ayon_core/lib/__init__.py +++ b/client/ayon_core/lib/__init__.py @@ -11,6 +11,7 @@ from .local_settings import ( get_launcher_storage_dir, get_addons_resources_dir, get_local_site_id, + get_ayon_user_entity, get_ayon_username, ) from .ayon_connection import initialize_ayon_connection @@ -73,6 +74,7 @@ from .log import ( ) from .path_templates import ( + DefaultKeysDict, TemplateUnsolved, StringTemplate, FormatObject, @@ -148,6 +150,7 @@ __all__ = [ "get_launcher_storage_dir", "get_addons_resources_dir", "get_local_site_id", + "get_ayon_user_entity", "get_ayon_username", "initialize_ayon_connection", @@ -228,6 +231,7 @@ __all__ = [ "get_version_from_path", "get_last_version_from_path", + "DefaultKeysDict", "TemplateUnsolved", "StringTemplate", "FormatObject", diff --git a/client/ayon_core/lib/local_settings.py b/client/ayon_core/lib/local_settings.py index 1edfc3c1b6..8a17b7af38 100644 --- a/client/ayon_core/lib/local_settings.py +++ b/client/ayon_core/lib/local_settings.py @@ -5,6 +5,7 @@ import json import platform import configparser import warnings +import copy from datetime import datetime from abc import ABC, abstractmethod from functools import lru_cache @@ -13,6 +14,8 @@ from typing import Optional, Any import platformdirs import ayon_api +from .cache import NestedCacheItem, CacheItem + _PLACEHOLDER = object() @@ -23,6 +26,7 @@ class RegistryItemNotFound(ValueError): class _Cache: username = None + user_entities_by_name = NestedCacheItem() def _get_ayon_appdirs(*args: str) -> str: @@ -569,6 +573,68 @@ def get_local_site_id(): return site_id +def _get_ayon_service_username() -> Optional[str]: + # TODO @iLLiCiTiT - do not use private attribute of 'ServerAPI', rather + # use public method to get username from connection stack. + con = ayon_api.get_server_api_connection() + user_stack = getattr(con, "_as_user_stack", None) + if user_stack is None: + return None + return user_stack.username + + +def get_ayon_user_entity(username: Optional[str] = None) -> dict[str, Any]: + """AYON user entity used for templates and publishing. + + Note: + Usually only service and admin users can receive the full user entity. + + Args: + username (Optional[str]): Username of the user. If not passed, then + the current user in 'ayon_api' is used. + + Returns: + dict[str, Any]: User entity. + + """ + service_username = _get_ayon_service_username() + # Handle service user handling first + if service_username: + if username is None: + username = service_username + cache: CacheItem = _Cache.user_entities_by_name[username] + if not cache.is_valid: + if username == service_username: + user = ayon_api.get_user() + else: + user = ayon_api.get_user(username) + cache.update_data(user) + return copy.deepcopy(cache.get_data()) + + # Cache current user + current_user = None + if _Cache.username is None: + current_user = ayon_api.get_user() + _Cache.username = current_user["name"] + + if username is None: + username = _Cache.username + + cache: CacheItem = _Cache.user_entities_by_name[username] + if not cache.is_valid: + user = None + if username == _Cache.username: + if current_user is None: + current_user = ayon_api.get_user() + user = current_user + + if user is None: + user = ayon_api.get_user(username) + cache.update_data(user) + + return copy.deepcopy(cache.get_data()) + + def get_ayon_username(): """AYON username used for templates and publishing. @@ -578,20 +644,5 @@ def get_ayon_username(): str: Username. """ - # Look for username in the connection stack - # - this is used when service is working as other user - # (e.g. in background sync) - # TODO @iLLiCiTiT - do not use private attribute of 'ServerAPI', rather - # use public method to get username from connection stack. - con = ayon_api.get_server_api_connection() - user_stack = getattr(con, "_as_user_stack", None) - if user_stack is not None: - username = user_stack.username - if username is not None: - return username - - # Cache the username to avoid multiple API calls - # - it is not expected that user would change - if _Cache.username is None: - _Cache.username = ayon_api.get_user()["name"] - return _Cache.username + user = get_ayon_user_entity() + return user["name"] diff --git a/client/ayon_core/lib/path_templates.py b/client/ayon_core/lib/path_templates.py index c6e9e14eac..aba2f296e3 100644 --- a/client/ayon_core/lib/path_templates.py +++ b/client/ayon_core/lib/path_templates.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import os import re import copy @@ -5,11 +7,7 @@ import numbers import warnings import platform from string import Formatter -import typing -from typing import List, Dict, Any, Set - -if typing.TYPE_CHECKING: - from typing import Union +from typing import Any, Union, Iterable SUB_DICT_PATTERN = re.compile(r"([^\[\]]+)") OPTIONAL_PATTERN = re.compile(r"(<.*?[^{0]*>)[^0-9]*?") @@ -44,6 +42,54 @@ class TemplateUnsolved(Exception): ) +class DefaultKeysDict(dict): + """Dictionary that supports the default key to use for str conversion. + + Is helpful for changes of a key in a template from string to dictionary + for example '{folder}' -> '{folder[name]}'. + >>> data = DefaultKeysDict( + >>> "name", + >>> {"folder": {"name": "FolderName"}} + >>> ) + >>> print("{folder[name]}".format_map(data)) + FolderName + >>> print("{folder}".format_map(data)) + FolderName + + Args: + default_key (Union[str, Iterable[str]]): Default key to use for str + conversion. Can also expect multiple keys for more nested + dictionary. + + """ + def __init__( + self, default_keys: Union[str, Iterable[str]], *args, **kwargs + ) -> None: + if isinstance(default_keys, str): + default_keys = [default_keys] + else: + default_keys = list(default_keys) + if not default_keys: + raise ValueError( + "Default key must be set. Got empty default keys." + ) + + self._default_keys = default_keys + super().__init__(*args, **kwargs) + + def __str__(self) -> str: + return str(self.get_default_value()) + + def get_default_keys(self) -> list[str]: + return list(self._default_keys) + + def get_default_value(self) -> Any: + value = self + for key in self._default_keys: + value = value[key] + return value + + class StringTemplate: """String that can be formatted.""" def __init__(self, template: str): @@ -84,7 +130,7 @@ class StringTemplate: if substr: new_parts.append(substr) - self._parts: List["Union[str, OptionalPart, FormattingPart]"] = ( + self._parts: list[Union[str, OptionalPart, FormattingPart]] = ( self.find_optional_parts(new_parts) ) @@ -105,7 +151,7 @@ class StringTemplate: def template(self) -> str: return self._template - def format(self, data: Dict[str, Any]) -> "TemplateResult": + def format(self, data: dict[str, Any]) -> "TemplateResult": """ Figure out with whole formatting. Separate advanced keys (*Like '{project[name]}') from string which must @@ -145,29 +191,29 @@ class StringTemplate: invalid_types ) - def format_strict(self, data: Dict[str, Any]) -> "TemplateResult": + def format_strict(self, data: dict[str, Any]) -> "TemplateResult": result = self.format(data) result.validate() return result @classmethod def format_template( - cls, template: str, data: Dict[str, Any] + cls, template: str, data: dict[str, Any] ) -> "TemplateResult": objected_template = cls(template) return objected_template.format(data) @classmethod def format_strict_template( - cls, template: str, data: Dict[str, Any] + cls, template: str, data: dict[str, Any] ) -> "TemplateResult": objected_template = cls(template) return objected_template.format_strict(data) @staticmethod def find_optional_parts( - parts: List["Union[str, FormattingPart]"] - ) -> List["Union[str, OptionalPart, FormattingPart]"]: + parts: list[Union[str, FormattingPart]] + ) -> list[Union[str, OptionalPart, FormattingPart]]: new_parts = [] tmp_parts = {} counted_symb = -1 @@ -192,7 +238,7 @@ class StringTemplate: len(parts) == 1 and isinstance(parts[0], str) ): - value = "<{}>".format(parts[0]) + value = f"<{parts[0]}>" else: value = OptionalPart(parts) @@ -223,7 +269,7 @@ class TemplateResult(str): only used keys. solved (bool): For check if all required keys were filled. template (str): Original template. - missing_keys (Iterable[str]): Missing keys that were not in the data. + missing_keys (list[str]): Missing keys that were not in the data. Include missing optional keys. invalid_types (dict): When key was found in data, but value had not allowed DataType. Allowed data types are `numbers`, @@ -232,11 +278,11 @@ class TemplateResult(str): of number. """ - used_values: Dict[str, Any] = None + used_values: dict[str, Any] = None solved: bool = None template: str = None - missing_keys: List[str] = None - invalid_types: Dict[str, Any] = None + missing_keys: list[str] = None + invalid_types: dict[str, Any] = None def __new__( cls, filled_template, template, solved, @@ -296,21 +342,21 @@ class TemplatePartResult: """Result to store result of template parts.""" def __init__(self, optional: bool = False): # Missing keys or invalid value types of required keys - self._missing_keys: Set[str] = set() - self._invalid_types: Dict[str, Any] = {} + self._missing_keys: set[str] = set() + self._invalid_types: dict[str, Any] = {} # Missing keys or invalid value types of optional keys - self._missing_optional_keys: Set[str] = set() - self._invalid_optional_types: Dict[str, Any] = {} + self._missing_optional_keys: set[str] = set() + self._invalid_optional_types: dict[str, Any] = {} # Used values stored by key with origin type # - key without any padding or key modifiers # - value from filling data # Example: {"version": 1} - self._used_values: Dict[str, Any] = {} + self._used_values: dict[str, Any] = {} # Used values stored by key with all modifirs # - value is already formatted string # Example: {"version:0>3": "001"} - self._really_used_values: Dict[str, Any] = {} + self._really_used_values: dict[str, Any] = {} # Concatenated string output after formatting self._output: str = "" # Is this result from optional part @@ -336,8 +382,9 @@ class TemplatePartResult: self._really_used_values.update(other.really_used_values) else: - raise TypeError("Cannot add data from \"{}\" to \"{}\"".format( - str(type(other)), self.__class__.__name__) + raise TypeError( + f"Cannot add data from \"{type(other)}\"" + f" to \"{self.__class__.__name__}\"" ) @property @@ -362,40 +409,41 @@ class TemplatePartResult: return self._output @property - def missing_keys(self) -> Set[str]: + def missing_keys(self) -> set[str]: return self._missing_keys @property - def missing_optional_keys(self) -> Set[str]: + def missing_optional_keys(self) -> set[str]: return self._missing_optional_keys @property - def invalid_types(self) -> Dict[str, Any]: + def invalid_types(self) -> dict[str, Any]: return self._invalid_types @property - def invalid_optional_types(self) -> Dict[str, Any]: + def invalid_optional_types(self) -> dict[str, Any]: return self._invalid_optional_types @property - def really_used_values(self) -> Dict[str, Any]: + def really_used_values(self) -> dict[str, Any]: return self._really_used_values @property - def realy_used_values(self) -> Dict[str, Any]: + def realy_used_values(self) -> dict[str, Any]: warnings.warn( "Property 'realy_used_values' is deprecated." " Use 'really_used_values' instead.", - DeprecationWarning + DeprecationWarning, + stacklevel=2, ) return self._really_used_values @property - def used_values(self) -> Dict[str, Any]: + def used_values(self) -> dict[str, Any]: return self._used_values @staticmethod - def split_keys_to_subdicts(values: Dict[str, Any]) -> Dict[str, Any]: + def split_keys_to_subdicts(values: dict[str, Any]) -> dict[str, Any]: output = {} formatter = Formatter() for key, value in values.items(): @@ -410,7 +458,7 @@ class TemplatePartResult: data[last_key] = value return output - def get_clean_used_values(self) -> Dict[str, Any]: + def get_clean_used_values(self) -> dict[str, Any]: new_used_values = {} for key, value in self.used_values.items(): if isinstance(value, FormatObject): @@ -426,7 +474,8 @@ class TemplatePartResult: warnings.warn( "Method 'add_realy_used_value' is deprecated." " Use 'add_really_used_value' instead.", - DeprecationWarning + DeprecationWarning, + stacklevel=2, ) self.add_really_used_value(key, value) @@ -479,7 +528,7 @@ class FormattingPart: self, field_name: str, format_spec: str, - conversion: "Union[str, None]", + conversion: Union[str, None], ): format_spec_v = "" if format_spec: @@ -546,7 +595,7 @@ class FormattingPart: return not queue @staticmethod - def keys_to_template_base(keys: List[str]): + def keys_to_template_base(keys: list[str]): if not keys: return None # Create copy of keys @@ -556,7 +605,7 @@ class FormattingPart: return f"{template_base}{joined_keys}" def format( - self, data: Dict[str, Any], result: TemplatePartResult + self, data: dict[str, Any], result: TemplatePartResult ) -> TemplatePartResult: """Format the formattings string. @@ -635,6 +684,12 @@ class FormattingPart: result.add_output(self.template) return result + if isinstance(value, DefaultKeysDict): + try: + value = value.get_default_value() + except KeyError: + pass + if not self.validate_value_type(value): result.add_invalid_type(key, value) result.add_output(self.template) @@ -687,23 +742,25 @@ class OptionalPart: def __init__( self, - parts: List["Union[str, OptionalPart, FormattingPart]"] + parts: list[Union[str, OptionalPart, FormattingPart]] ): - self._parts: List["Union[str, OptionalPart, FormattingPart]"] = parts + self._parts: list[Union[str, OptionalPart, FormattingPart]] = parts @property - def parts(self) -> List["Union[str, OptionalPart, FormattingPart]"]: + def parts(self) -> list[Union[str, OptionalPart, FormattingPart]]: return self._parts def __str__(self) -> str: - return "<{}>".format("".join([str(p) for p in self._parts])) + joined_parts = "".join([str(p) for p in self._parts]) + return f"<{joined_parts}>" def __repr__(self) -> str: - return "".format("".join([str(p) for p in self._parts])) + joined_parts = "".join([str(p) for p in self._parts]) + return f"" def format( self, - data: Dict[str, Any], + data: dict[str, Any], result: TemplatePartResult, ) -> TemplatePartResult: new_result = TemplatePartResult(True) diff --git a/client/ayon_core/pipeline/template_data.py b/client/ayon_core/pipeline/template_data.py index 0a95a98be8..dc7e95c788 100644 --- a/client/ayon_core/pipeline/template_data.py +++ b/client/ayon_core/pipeline/template_data.py @@ -1,27 +1,50 @@ +from __future__ import annotations + +from typing import Optional, Any + import ayon_api from ayon_core.settings import get_studio_settings -from ayon_core.lib.local_settings import get_ayon_username +from ayon_core.lib import DefaultKeysDict +from ayon_core.lib.local_settings import get_ayon_user_entity -def get_general_template_data(settings=None, username=None): +def get_general_template_data( + settings: Optional[dict[str, Any]] = None, + username: Optional[str] = None, + user_entity: Optional[dict[str, Any]] = None, +): """General template data based on system settings or machine. Output contains formatting keys: - - 'studio[name]' - Studio name filled from system settings - - 'studio[code]' - Studio code filled from system settings - - 'user' - User's name using 'get_ayon_username' + - 'studio[name]' - Studio name filled from system settings + - 'studio[code]' - Studio code filled from system settings + - 'user[name]' - User's name + - 'user[attrib][...]' - User's attributes + - 'user[data][...]' - User's data Args: settings (Dict[str, Any]): Studio or project settings. username (Optional[str]): AYON Username. - """ + user_entity (Optional[dict[str, Any]]): User entity. + """ if not settings: settings = get_studio_settings() - if username is None: - username = get_ayon_username() + if user_entity is None: + user_entity = get_ayon_user_entity(username) + + # Use dictionary with default value for backwards compatibility + # - we did support '{user}' now it should be '{user[name]}' + user_data = DefaultKeysDict( + "name", + { + "name": user_entity["name"], + "attrib": user_entity["attrib"], + "data": user_entity["data"], + } + ) core_settings = settings["core"] return { @@ -29,7 +52,7 @@ def get_general_template_data(settings=None, username=None): "name": core_settings["studio_name"], "code": core_settings["studio_code"] }, - "user": username + "user": user_data, } @@ -150,7 +173,8 @@ def get_template_data( task_entity=None, host_name=None, settings=None, - username=None + username=None, + user_entity=None, ): """Prepare data for templates filling from entered documents and info. @@ -173,13 +197,18 @@ def get_template_data( host_name (Optional[str]): Used to fill '{app}' key. settings (Union[Dict, None]): Prepared studio or project settings. They're queried if not passed (may be slower). - username (Optional[str]): AYON Username. + username (Optional[str]): DEPRECATED AYON Username. + user_entity (Optional[dict[str, Any]): AYON user entity. Returns: Dict[str, Any]: Data prepared for filling workdir template. """ - template_data = get_general_template_data(settings, username=username) + template_data = get_general_template_data( + settings, + username=username, + user_entity=user_entity, + ) template_data.update(get_project_template_data(project_entity)) if folder_entity: template_data.update(get_folder_template_data( diff --git a/client/ayon_core/plugins/publish/collect_anatomy_context_data.py b/client/ayon_core/plugins/publish/collect_anatomy_context_data.py index cccf392e40..5d2ecec433 100644 --- a/client/ayon_core/plugins/publish/collect_anatomy_context_data.py +++ b/client/ayon_core/plugins/publish/collect_anatomy_context_data.py @@ -16,6 +16,7 @@ Provides: import json import pyblish.api +from ayon_core.lib import get_ayon_user_entity from ayon_core.pipeline.template_data import get_template_data @@ -55,17 +56,18 @@ class CollectAnatomyContextData(pyblish.api.ContextPlugin): if folder_entity: task_entity = context.data["taskEntity"] + username = context.data["user"] + user_entity = get_ayon_user_entity(username) anatomy_data = get_template_data( project_entity, folder_entity, task_entity, - host_name, - project_settings + host_name=host_name, + settings=project_settings, + user_entity=user_entity, ) anatomy_data.update(context.data.get("datetimeData") or {}) - username = context.data["user"] - anatomy_data["user"] = username # Backwards compatibility for 'username' key anatomy_data["username"] = username diff --git a/client/ayon_core/plugins/publish/integrate.py b/client/ayon_core/plugins/publish/integrate.py index f1e066018c..d18e546392 100644 --- a/client/ayon_core/plugins/publish/integrate.py +++ b/client/ayon_core/plugins/publish/integrate.py @@ -121,7 +121,6 @@ class IntegrateAsset(pyblish.api.InstancePlugin): "version", "representation", "username", - "user", "output", # OpenPype keys - should be removed "asset", # folder[name] @@ -796,6 +795,14 @@ class IntegrateAsset(pyblish.api.InstancePlugin): if value is not None: repre_context[key] = value + # Keep only username + # NOTE This is to avoid storing all user attributes and data + # to representation + if "user" not in repre_context: + repre_context["user"] = { + "name": template_data["user"]["name"] + } + # Use previous representation's id if there is a name match existing = existing_repres_by_name.get(repre["name"].lower()) repre_id = None diff --git a/client/ayon_core/plugins/publish/integrate_hero_version.py b/client/ayon_core/plugins/publish/integrate_hero_version.py index 90e6f15568..a591cfe880 100644 --- a/client/ayon_core/plugins/publish/integrate_hero_version.py +++ b/client/ayon_core/plugins/publish/integrate_hero_version.py @@ -89,7 +89,6 @@ class IntegrateHeroVersion( "family", "representation", "username", - "user", "output" ] # QUESTION/TODO this process should happen on server if crashed due to @@ -364,6 +363,14 @@ class IntegrateHeroVersion( if value is not None: repre_context[key] = value + # Keep only username + # NOTE This is to avoid storing all user attributes and data + # to representation + if "user" not in repre_context: + repre_context["user"] = { + "name": anatomy_data["user"]["name"] + } + # Prepare new repre repre_entity = copy.deepcopy(repre_info["representation"]) repre_entity.pop("id", None) diff --git a/client/ayon_core/tools/push_to_project/models/integrate.py b/client/ayon_core/tools/push_to_project/models/integrate.py index ef49838152..68a0e2affb 100644 --- a/client/ayon_core/tools/push_to_project/models/integrate.py +++ b/client/ayon_core/tools/push_to_project/models/integrate.py @@ -5,7 +5,7 @@ import itertools import sys import traceback import uuid -from typing import Optional, Dict +from typing import Optional, Dict, Any import ayon_api from ayon_api.utils import create_entity_id @@ -650,10 +650,10 @@ class ProjectPushItemProcess: def _create_folder( self, - src_folder_entity, - project_entity, - parent_folder_entity, - folder_name + src_folder_entity: dict[str, Any], + project_entity: dict[str, Any], + parent_folder_entity: dict[str, Any], + folder_name: str ): parent_id = None if parent_folder_entity: @@ -702,10 +702,14 @@ class ProjectPushItemProcess: if new_folder_name != folder_name: folder_label = folder_name - # TODO find out how to define folder type + src_folder_type = src_folder_entity["folderType"] + dst_folder_type = self._get_dst_folder_type( + project_entity, + src_folder_type + ) folder_entity = new_folder_entity( folder_name, - "Folder", + dst_folder_type, parent_id=parent_id, attribs=new_folder_attrib ) @@ -727,6 +731,22 @@ class ProjectPushItemProcess: folder_entity["path"] = "/".join([parent_path, folder_name]) return folder_entity + def _get_dst_folder_type( + self, + project_entity: dict[str, Any], + src_folder_type: str + ) -> str: + """Get new folder type.""" + for folder_type in project_entity["folderTypes"]: + if folder_type["name"].lower() == src_folder_type.lower(): + return folder_type["name"] + + self._status.set_failed( + f"'{src_folder_type}' folder type is not configured in " + f"project Anatomy." + ) + raise PushToProjectError(self._status.fail_reason) + def _fill_or_create_destination_folder(self): dst_project_name = self._item.dst_project_name dst_folder_id = self._item.dst_folder_id @@ -1206,7 +1226,7 @@ class ProjectPushItemProcess: if value_to_update: repre_context[context_key] = value_to_update if "task" not in formatting_data: - repre_context.pop("task") + repre_context.pop("task", None) return repre_context