diff --git a/openpype/modules/__init__.py b/openpype/modules/__init__.py index 583480b049..68b5f6c247 100644 --- a/openpype/modules/__init__.py +++ b/openpype/modules/__init__.py @@ -1,21 +1,35 @@ # -*- coding: utf-8 -*- from .base import ( OpenPypeModule, + OpenPypeAddOn, OpenPypeInterface, load_modules, ModulesManager, - TrayModulesManager + TrayModulesManager, + + BaseModuleSettingsDef, + ModuleSettingsDef, + JsonFilesSettingsDef, + + get_module_settings_defs ) __all__ = ( "OpenPypeModule", + "OpenPypeAddOn", "OpenPypeInterface", "load_modules", "ModulesManager", - "TrayModulesManager" + "TrayModulesManager", + + "BaseModuleSettingsDef", + "ModuleSettingsDef", + "JsonFilesSettingsDef", + + "get_module_settings_defs" ) diff --git a/openpype/modules/base.py b/openpype/modules/base.py index d43d5635d1..01c3cebe60 100644 --- a/openpype/modules/base.py +++ b/openpype/modules/base.py @@ -2,9 +2,11 @@ """Base class for Pype Modules.""" import os import sys +import json import time import inspect import logging +import platform import threading import collections from uuid import uuid4 @@ -12,7 +14,18 @@ from abc import ABCMeta, abstractmethod import six import openpype -from openpype.settings import get_system_settings +from openpype.settings import ( + get_system_settings, + SYSTEM_SETTINGS_KEY, + PROJECT_SETTINGS_KEY, + SCHEMA_KEY_SYSTEM_SETTINGS, + SCHEMA_KEY_PROJECT_SETTINGS +) + +from openpype.settings.lib import ( + get_studio_system_settings_overrides, + load_json_file +) from openpype.lib import PypeLogger @@ -115,11 +128,51 @@ def get_default_modules_dir(): return os.path.join(current_dir, "default_modules") +def get_dynamic_modules_dirs(): + """Possible paths to OpenPype Addons of Modules. + + Paths are loaded from studio settings under: + `modules -> addon_paths -> {platform name}` + + Path may contain environment variable as a formatting string. + + They are not validated or checked their existence. + + Returns: + list: Paths loaded from studio overrides. + """ + output = [] + value = get_studio_system_settings_overrides() + for key in ("modules", "addon_paths", platform.system().lower()): + if key not in value: + return output + value = value[key] + + for path in value: + if not path: + continue + + try: + path = path.format(**os.environ) + except Exception: + pass + output.append(path) + return output + + def get_module_dirs(): """List of paths where OpenPype modules can be found.""" - dirpaths = [ - get_default_modules_dir() - ] + _dirpaths = [] + _dirpaths.append(get_default_modules_dir()) + _dirpaths.extend(get_dynamic_modules_dirs()) + + dirpaths = [] + for path in _dirpaths: + if not path: + continue + normalized = os.path.normpath(path) + if normalized not in dirpaths: + dirpaths.append(normalized) return dirpaths @@ -165,6 +218,9 @@ def _load_interfaces(): os.path.join(get_default_modules_dir(), "interfaces.py") ) for dirpath in dirpaths: + if not os.path.exists(dirpath): + continue + for filename in os.listdir(dirpath): if filename in ("__pycache__", ): continue @@ -272,12 +328,19 @@ def _load_modules(): # TODO add more logic how to define if folder is module or not # - check manifest and content of manifest - if os.path.isdir(fullpath): - import_module_from_dirpath(dirpath, filename, modules_key) + try: + if os.path.isdir(fullpath): + import_module_from_dirpath(dirpath, filename, modules_key) - elif ext in (".py", ): - module = import_filepath(fullpath) - setattr(openpype_modules, basename, module) + elif ext in (".py", ): + module = import_filepath(fullpath) + setattr(openpype_modules, basename, module) + + except Exception: + log.error( + "Failed to import '{}'.".format(fullpath), + exc_info=True + ) class _OpenPypeInterfaceMeta(ABCMeta): @@ -368,7 +431,16 @@ class OpenPypeModule: class OpenPypeAddOn(OpenPypeModule): - pass + # Enable Addon by default + enabled = True + + def initialize(self, module_settings): + """Initialization is not be required for most of addons.""" + pass + + def connect_with_modules(self, enabled_modules): + """Do not require to implement connection with modules for addon.""" + pass class ModulesManager: @@ -920,3 +992,424 @@ class TrayModulesManager(ModulesManager): ), exc_info=True ) + + +def get_module_settings_defs(): + """Check loaded addons/modules for existence of thei settings definition. + + Check if OpenPype addon/module as python module has class that inherit + from `ModuleSettingsDef` in python module variables (imported + in `__init__py`). + + Returns: + list: All valid and not abstract settings definitions from imported + openpype addons and modules. + """ + # Make sure modules are loaded + load_modules() + + import openpype_modules + + settings_defs = [] + + log = PypeLogger.get_logger("ModuleSettingsLoad") + + for raw_module in openpype_modules: + for attr_name in dir(raw_module): + attr = getattr(raw_module, attr_name) + if ( + not inspect.isclass(attr) + or attr is ModuleSettingsDef + or not issubclass(attr, ModuleSettingsDef) + ): + continue + + if inspect.isabstract(attr): + # Find missing implementations by convetion on `abc` module + not_implemented = [] + for attr_name in dir(attr): + attr = getattr(attr, attr_name, None) + abs_method = getattr( + attr, "__isabstractmethod__", None + ) + if attr and abs_method: + not_implemented.append(attr_name) + + # Log missing implementations + log.warning(( + "Skipping abstract Class: {} in module {}." + " Missing implementations: {}" + ).format( + attr_name, raw_module.__name__, ", ".join(not_implemented) + )) + continue + + settings_defs.append(attr) + + return settings_defs + + +@six.add_metaclass(ABCMeta) +class BaseModuleSettingsDef: + """Definition of settings for OpenPype module or AddOn.""" + _id = None + + @property + def id(self): + """ID created on initialization. + + ID should be per created object. Helps to store objects. + """ + if self._id is None: + self._id = uuid4() + return self._id + + @abstractmethod + def get_settings_schemas(self, schema_type): + """Setting schemas for passed schema type. + + These are main schemas by dynamic schema keys. If they're using + sub schemas or templates they should be loaded with + `get_dynamic_schemas`. + + Returns: + dict: Schema by `dynamic_schema` keys. + """ + pass + + @abstractmethod + def get_dynamic_schemas(self, schema_type): + """Settings schemas and templates that can be used anywhere. + + It is recommended to add prefix specific for addon/module to keys + (e.g. "my_addon/real_schema_name"). + + Returns: + dict: Schemas and templates by their keys. + """ + pass + + @abstractmethod + def get_defaults(self, top_key): + """Default values for passed top key. + + Top keys are (currently) "system_settings" or "project_settings". + + Should return exactly what was passed with `save_defaults`. + + Returns: + dict: Default values by path to first key in OpenPype defaults. + """ + pass + + @abstractmethod + def save_defaults(self, top_key, data): + """Save default values for passed top key. + + Top keys are (currently) "system_settings" or "project_settings". + + Passed data are by path to first key defined in main schemas. + """ + pass + + +class ModuleSettingsDef(BaseModuleSettingsDef): + """Settings definiton with separated system and procect settings parts. + + Reduce conditions that must be checked and adds predefined methods for + each case. + """ + def get_defaults(self, top_key): + """Split method into 2 methods by top key.""" + if top_key == SYSTEM_SETTINGS_KEY: + return self.get_default_system_settings() or {} + elif top_key == PROJECT_SETTINGS_KEY: + return self.get_default_project_settings() or {} + return {} + + def save_defaults(self, top_key, data): + """Split method into 2 methods by top key.""" + if top_key == SYSTEM_SETTINGS_KEY: + self.save_system_defaults(data) + elif top_key == PROJECT_SETTINGS_KEY: + self.save_project_defaults(data) + + def get_settings_schemas(self, schema_type): + """Split method into 2 methods by schema type.""" + if schema_type == SCHEMA_KEY_SYSTEM_SETTINGS: + return self.get_system_settings_schemas() or {} + elif schema_type == SCHEMA_KEY_PROJECT_SETTINGS: + return self.get_project_settings_schemas() or {} + return {} + + def get_dynamic_schemas(self, schema_type): + """Split method into 2 methods by schema type.""" + if schema_type == SCHEMA_KEY_SYSTEM_SETTINGS: + return self.get_system_dynamic_schemas() or {} + elif schema_type == SCHEMA_KEY_PROJECT_SETTINGS: + return self.get_project_dynamic_schemas() or {} + return {} + + @abstractmethod + def get_system_settings_schemas(self): + """Schemas and templates usable in system settings schemas. + + Returns: + dict: Schemas and templates by it's names. Names must be unique + across whole OpenPype. + """ + pass + + @abstractmethod + def get_project_settings_schemas(self): + """Schemas and templates usable in project settings schemas. + + Returns: + dict: Schemas and templates by it's names. Names must be unique + across whole OpenPype. + """ + pass + + @abstractmethod + def get_system_dynamic_schemas(self): + """System schemas by dynamic schema name. + + If dynamic schema name is not available in then schema will not used. + + Returns: + dict: Schemas or list of schemas by dynamic schema name. + """ + pass + + @abstractmethod + def get_project_dynamic_schemas(self): + """Project schemas by dynamic schema name. + + If dynamic schema name is not available in then schema will not used. + + Returns: + dict: Schemas or list of schemas by dynamic schema name. + """ + pass + + @abstractmethod + def get_default_system_settings(self): + """Default system settings values. + + Returns: + dict: Default values by path to first key. + """ + pass + + @abstractmethod + def get_default_project_settings(self): + """Default project settings values. + + Returns: + dict: Default values by path to first key. + """ + pass + + @abstractmethod + def save_system_defaults(self, data): + """Save default system settings values. + + Passed data are by path to first key defined in main schemas. + """ + pass + + @abstractmethod + def save_project_defaults(self, data): + """Save default project settings values. + + Passed data are by path to first key defined in main schemas. + """ + pass + + +class JsonFilesSettingsDef(ModuleSettingsDef): + """Preimplemented settings definition using json files and file structure. + + Expected file structure: + ┕ root + │ + │ # Default values + ┝ defaults + │ ┝ system_settings.json + │ ┕ project_settings.json + │ + │ # Schemas for `dynamic_template` type + ┝ dynamic_schemas + │ ┝ system_dynamic_schemas.json + │ ┕ project_dynamic_schemas.json + │ + │ # Schemas that can be used anywhere (enhancement for `dynamic_schemas`) + ┕ schemas + ┝ system_schemas + │ ┝ # Any schema or template files + │ ┕ ... + ┕ project_schemas + ┝ # Any schema or template files + ┕ ... + + Schemas can be loaded with prefix to avoid duplicated schema/template names + across all OpenPype addons/modules. Prefix can be defined with class + attribute `schema_prefix`. + + Only think which must be implemented in `get_settings_root_path` which + should return directory path to `root` (in structure graph above). + """ + # Possible way how to define `schemas` prefix + schema_prefix = "" + + @abstractmethod + def get_settings_root_path(self): + """Directory path where settings and it's schemas are located.""" + pass + + def __init__(self): + settings_root_dir = self.get_settings_root_path() + defaults_dir = os.path.join( + settings_root_dir, "defaults" + ) + dynamic_schemas_dir = os.path.join( + settings_root_dir, "dynamic_schemas" + ) + schemas_dir = os.path.join( + settings_root_dir, "schemas" + ) + + self.system_defaults_filepath = os.path.join( + defaults_dir, "system_settings.json" + ) + self.project_defaults_filepath = os.path.join( + defaults_dir, "project_settings.json" + ) + + self.system_dynamic_schemas_filepath = os.path.join( + dynamic_schemas_dir, "system_dynamic_schemas.json" + ) + self.project_dynamic_schemas_filepath = os.path.join( + dynamic_schemas_dir, "project_dynamic_schemas.json" + ) + + self.system_schemas_dir = os.path.join( + schemas_dir, "system_schemas" + ) + self.project_schemas_dir = os.path.join( + schemas_dir, "project_schemas" + ) + + def _load_json_file_data(self, path): + if os.path.exists(path): + return load_json_file(path) + return {} + + def get_default_system_settings(self): + """Default system settings values. + + Returns: + dict: Default values by path to first key. + """ + return self._load_json_file_data(self.system_defaults_filepath) + + def get_default_project_settings(self): + """Default project settings values. + + Returns: + dict: Default values by path to first key. + """ + return self._load_json_file_data(self.project_defaults_filepath) + + def _save_data_to_filepath(self, path, data): + dirpath = os.path.dirname(path) + if not os.path.exists(dirpath): + os.makedirs(dirpath) + + with open(path, "w") as file_stream: + json.dump(data, file_stream, indent=4) + + def save_system_defaults(self, data): + """Save default system settings values. + + Passed data are by path to first key defined in main schemas. + """ + self._save_data_to_filepath(self.system_defaults_filepath, data) + + def save_project_defaults(self, data): + """Save default project settings values. + + Passed data are by path to first key defined in main schemas. + """ + self._save_data_to_filepath(self.project_defaults_filepath, data) + + def get_system_dynamic_schemas(self): + """System schemas by dynamic schema name. + + If dynamic schema name is not available in then schema will not used. + + Returns: + dict: Schemas or list of schemas by dynamic schema name. + """ + return self._load_json_file_data(self.system_dynamic_schemas_filepath) + + def get_project_dynamic_schemas(self): + """Project schemas by dynamic schema name. + + If dynamic schema name is not available in then schema will not used. + + Returns: + dict: Schemas or list of schemas by dynamic schema name. + """ + return self._load_json_file_data(self.project_dynamic_schemas_filepath) + + def _load_files_from_path(self, path): + output = {} + if not path or not os.path.exists(path): + return output + + if os.path.isfile(path): + filename = os.path.basename(path) + basename, ext = os.path.splitext(filename) + if ext == ".json": + if self.schema_prefix: + key = "{}/{}".format(self.schema_prefix, basename) + else: + key = basename + output[key] = self._load_json_file_data(path) + return output + + path = os.path.normpath(path) + for root, _, files in os.walk(path, topdown=False): + for filename in files: + basename, ext = os.path.splitext(filename) + if ext != ".json": + continue + + json_path = os.path.join(root, filename) + store_key = os.path.join( + root.replace(path, ""), basename + ).replace("\\", "/") + if self.schema_prefix: + store_key = "{}/{}".format(self.schema_prefix, store_key) + output[store_key] = self._load_json_file_data(json_path) + + return output + + def get_system_settings_schemas(self): + """Schemas and templates usable in system settings schemas. + + Returns: + dict: Schemas and templates by it's names. Names must be unique + across whole OpenPype. + """ + return self._load_files_from_path(self.system_schemas_dir) + + def get_project_settings_schemas(self): + """Schemas and templates usable in project settings schemas. + + Returns: + dict: Schemas and templates by it's names. Names must be unique + across whole OpenPype. + """ + return self._load_files_from_path(self.project_schemas_dir) diff --git a/openpype/settings/__init__.py b/openpype/settings/__init__.py index b5810deef4..0adb5db0bd 100644 --- a/openpype/settings/__init__.py +++ b/openpype/settings/__init__.py @@ -1,7 +1,21 @@ +from .constants import ( + GLOBAL_SETTINGS_KEY, + SYSTEM_SETTINGS_KEY, + PROJECT_SETTINGS_KEY, + PROJECT_ANATOMY_KEY, + LOCAL_SETTING_KEY, + + SCHEMA_KEY_SYSTEM_SETTINGS, + SCHEMA_KEY_PROJECT_SETTINGS, + + KEY_ALLOWED_SYMBOLS, + KEY_REGEX +) from .exceptions import ( SaveWarningExc ) from .lib import ( + get_general_environments, get_system_settings, get_project_settings, get_current_project_settings, @@ -16,8 +30,21 @@ from .entities import ( __all__ = ( + "GLOBAL_SETTINGS_KEY", + "SYSTEM_SETTINGS_KEY", + "PROJECT_SETTINGS_KEY", + "PROJECT_ANATOMY_KEY", + "LOCAL_SETTING_KEY", + + "SCHEMA_KEY_SYSTEM_SETTINGS", + "SCHEMA_KEY_PROJECT_SETTINGS", + + "KEY_ALLOWED_SYMBOLS", + "KEY_REGEX", + "SaveWarningExc", + "get_general_environments", "get_system_settings", "get_project_settings", "get_current_project_settings", diff --git a/openpype/settings/constants.py b/openpype/settings/constants.py index a53e88a91e..2ea19ead4b 100644 --- a/openpype/settings/constants.py +++ b/openpype/settings/constants.py @@ -14,13 +14,17 @@ METADATA_KEYS = ( M_DYNAMIC_KEY_LABEL ) -# File where studio's system overrides are stored +# Keys where studio's system overrides are stored GLOBAL_SETTINGS_KEY = "global_settings" SYSTEM_SETTINGS_KEY = "system_settings" PROJECT_SETTINGS_KEY = "project_settings" PROJECT_ANATOMY_KEY = "project_anatomy" LOCAL_SETTING_KEY = "local_settings" +# Schema hub names +SCHEMA_KEY_SYSTEM_SETTINGS = "system_schema" +SCHEMA_KEY_PROJECT_SETTINGS = "projects_schema" + DEFAULT_PROJECT_KEY = "__default_project__" KEY_ALLOWED_SYMBOLS = "a-zA-Z0-9-_ " @@ -39,6 +43,9 @@ __all__ = ( "PROJECT_ANATOMY_KEY", "LOCAL_SETTING_KEY", + "SCHEMA_KEY_SYSTEM_SETTINGS", + "SCHEMA_KEY_PROJECT_SETTINGS", + "DEFAULT_PROJECT_KEY", "KEY_ALLOWED_SYMBOLS", diff --git a/openpype/settings/defaults/system_settings/modules.json b/openpype/settings/defaults/system_settings/modules.json index a0ba607edc..229b867327 100644 --- a/openpype/settings/defaults/system_settings/modules.json +++ b/openpype/settings/defaults/system_settings/modules.json @@ -1,4 +1,9 @@ { + "addon_paths": { + "windows": [], + "darwin": [], + "linux": [] + }, "avalon": { "AVALON_TIMEOUT": 1000, "AVALON_THUMBNAIL_ROOT": { diff --git a/openpype/settings/entities/base_entity.py b/openpype/settings/entities/base_entity.py index 851684520b..0e8274d374 100644 --- a/openpype/settings/entities/base_entity.py +++ b/openpype/settings/entities/base_entity.py @@ -104,6 +104,12 @@ class BaseItemEntity(BaseEntity): self.is_group = False # Entity's value will be stored into file with name of it's key self.is_file = False + # Default values are not stored to an openpype file + # - these must not be set through schemas directly + self.dynamic_schema_id = None + self.is_dynamic_schema_node = False + self.is_in_dynamic_schema_node = False + # Reference to parent entity which has `is_group` == True # - stays as None if none of parents is group self.group_item = None @@ -255,13 +261,22 @@ class BaseItemEntity(BaseEntity): ) # Group item can be only once in on hierarchy branch. - if self.is_group and self.group_item: + if self.is_group and self.group_item is not None: raise SchemeGroupHierarchyBug(self) + # Group item can be only once in on hierarchy branch. + if self.group_item is not None and self.is_dynamic_schema_node: + reason = ( + "Dynamic schema is inside grouped item {}." + " Change group hierarchy or remove dynamic" + " schema to be able work properly." + ).format(self.group_item.path) + raise EntitySchemaError(self, reason) + # Validate that env group entities will be stored into file. # - env group entities must store metadata which is not possible if # metadata would be outside of file - if not self.file_item and self.is_env_group: + if self.file_item is None and self.is_env_group: reason = ( "Environment item is not inside file" " item so can't store metadata for defaults." @@ -478,7 +493,15 @@ class BaseItemEntity(BaseEntity): @abstractmethod def settings_value(self): - """Value of an item without key.""" + """Value of an item without key without dynamic items.""" + pass + + @abstractmethod + def collect_dynamic_schema_entities(self): + """Collect entities that are on top of dynamically added schemas. + + This method make sence only when defaults are saved. + """ pass @abstractmethod @@ -808,6 +831,12 @@ class ItemEntity(BaseItemEntity): self.is_dynamic_item = is_dynamic_item self.is_file = self.schema_data.get("is_file", False) + # These keys have underscore as they must not be set in schemas + self.dynamic_schema_id = self.schema_data.get( + "_dynamic_schema_id", None + ) + self.is_dynamic_schema_node = self.dynamic_schema_id is not None + self.is_group = self.schema_data.get("is_group", False) self.is_in_dynamic_item = bool( not self.is_dynamic_item @@ -837,10 +866,20 @@ class ItemEntity(BaseItemEntity): self._require_restart_on_change = require_restart_on_change # File item reference - if self.parent.is_file: - self.file_item = self.parent - elif self.parent.file_item: - self.file_item = self.parent.file_item + if not self.is_dynamic_schema_node: + self.is_in_dynamic_schema_node = ( + self.parent.is_dynamic_schema_node + or self.parent.is_in_dynamic_schema_node + ) + + if ( + not self.is_dynamic_schema_node + and not self.is_in_dynamic_schema_node + ): + if self.parent.is_file: + self.file_item = self.parent + elif self.parent.file_item: + self.file_item = self.parent.file_item # Group item reference if self.parent.is_group: @@ -891,6 +930,18 @@ class ItemEntity(BaseItemEntity): def root_key(self): return self.root_item.root_key + @abstractmethod + def collect_dynamic_schema_entities(self, collector): + """Collect entities that are on top of dynamically added schemas. + + This method make sence only when defaults are saved. + + Args: + collector(DynamicSchemaValueCollector): Object where dynamic + entities are stored. + """ + pass + def schema_validations(self): if not self.label and self.use_label_wrap: reason = ( @@ -899,7 +950,12 @@ class ItemEntity(BaseItemEntity): ) raise EntitySchemaError(self, reason) - if self.is_file and self.file_item is not None: + if ( + not self.is_dynamic_schema_node + and not self.is_in_dynamic_schema_node + and self.is_file + and self.file_item is not None + ): reason = ( "Entity has set `is_file` to true but" " it's parent is already marked as file item." diff --git a/openpype/settings/entities/dict_conditional.py b/openpype/settings/entities/dict_conditional.py index 988464d059..d7b416921c 100644 --- a/openpype/settings/entities/dict_conditional.py +++ b/openpype/settings/entities/dict_conditional.py @@ -469,6 +469,10 @@ class DictConditionalEntity(ItemEntity): return True return False + def collect_dynamic_schema_entities(self, collector): + if self.is_dynamic_schema_node: + collector.add_entity(self) + def settings_value(self): if self._override_state is OverrideState.NOT_DEFINED: return NOT_SET @@ -482,13 +486,7 @@ class DictConditionalEntity(ItemEntity): output = {} for key, child_obj in children_items: - child_value = child_obj.settings_value() - if not child_obj.is_file and not child_obj.file_item: - for _key, _value in child_value.items(): - new_key = "/".join([key, _key]) - output[new_key] = _value - else: - output[key] = child_value + output[key] = child_obj.settings_value() return output if self.is_group: diff --git a/openpype/settings/entities/dict_immutable_keys_entity.py b/openpype/settings/entities/dict_immutable_keys_entity.py index 73b08f101a..57e21ff5f3 100644 --- a/openpype/settings/entities/dict_immutable_keys_entity.py +++ b/openpype/settings/entities/dict_immutable_keys_entity.py @@ -330,15 +330,32 @@ class DictImmutableKeysEntity(ItemEntity): return True return False + def collect_dynamic_schema_entities(self, collector): + for child_obj in self.non_gui_children.values(): + child_obj.collect_dynamic_schema_entities(collector) + + if self.is_dynamic_schema_node: + collector.add_entity(self) + def settings_value(self): if self._override_state is OverrideState.NOT_DEFINED: return NOT_SET if self._override_state is OverrideState.DEFAULTS: + is_dynamic_schema_node = ( + self.is_dynamic_schema_node or self.is_in_dynamic_schema_node + ) output = {} for key, child_obj in self.non_gui_children.items(): + if child_obj.is_dynamic_schema_node: + continue + child_value = child_obj.settings_value() - if not child_obj.is_file and not child_obj.file_item: + if ( + not is_dynamic_schema_node + and not child_obj.is_file + and not child_obj.file_item + ): for _key, _value in child_value.items(): new_key = "/".join([key, _key]) output[new_key] = _value diff --git a/openpype/settings/entities/dict_mutable_keys_entity.py b/openpype/settings/entities/dict_mutable_keys_entity.py index c3df935269..f75fb23d82 100644 --- a/openpype/settings/entities/dict_mutable_keys_entity.py +++ b/openpype/settings/entities/dict_mutable_keys_entity.py @@ -261,7 +261,7 @@ class DictMutableKeysEntity(EndpointEntity): raise EntitySchemaError(self, reason) # TODO Ability to store labels should be defined with different key - if self.collapsible_key and not self.file_item: + if self.collapsible_key and self.file_item is None: reason = ( "Modifiable dictionary with collapsible keys is not under" " file item so can't store metadata." diff --git a/openpype/settings/entities/input_entities.py b/openpype/settings/entities/input_entities.py index 336d1f5c1e..6afc59f6f2 100644 --- a/openpype/settings/entities/input_entities.py +++ b/openpype/settings/entities/input_entities.py @@ -49,6 +49,10 @@ class EndpointEntity(ItemEntity): super(EndpointEntity, self).schema_validations() + def collect_dynamic_schema_entities(self, collector): + if self.is_dynamic_schema_node: + collector.add_entity(self) + @abstractmethod def _settings_value(self): pass @@ -121,7 +125,11 @@ class InputEntity(EndpointEntity): def schema_validations(self): # Input entity must have file parent. - if not self.file_item: + if ( + not self.is_dynamic_schema_node + and not self.is_in_dynamic_schema_node + and self.file_item is None + ): raise EntitySchemaError(self, "Missing parent file entity.") super(InputEntity, self).schema_validations() diff --git a/openpype/settings/entities/item_entities.py b/openpype/settings/entities/item_entities.py index ac6b3e76dd..c7c9c3097e 100644 --- a/openpype/settings/entities/item_entities.py +++ b/openpype/settings/entities/item_entities.py @@ -115,6 +115,9 @@ class PathEntity(ItemEntity): def set(self, value): self.child_obj.set(value) + def collect_dynamic_schema_entities(self, *args, **kwargs): + self.child_obj.collect_dynamic_schema_entities(*args, **kwargs) + def settings_value(self): if self._override_state is OverrideState.NOT_DEFINED: return NOT_SET @@ -236,7 +239,12 @@ class ListStrictEntity(ItemEntity): def schema_validations(self): # List entity must have file parent. - if not self.file_item and not self.is_file: + if ( + not self.is_dynamic_schema_node + and not self.is_in_dynamic_schema_node + and not self.is_file + and self.file_item is None + ): raise EntitySchemaError( self, "Missing file entity in hierarchy." ) @@ -279,6 +287,10 @@ class ListStrictEntity(ItemEntity): for idx, item in enumerate(new_value): self.children[idx].set(item) + def collect_dynamic_schema_entities(self, collector): + if self.is_dynamic_schema_node: + collector.add_entity(self) + def settings_value(self): if self._override_state is OverrideState.NOT_DEFINED: return NOT_SET diff --git a/openpype/settings/entities/lib.py b/openpype/settings/entities/lib.py index 01f61d8bdf..f207322dee 100644 --- a/openpype/settings/entities/lib.py +++ b/openpype/settings/entities/lib.py @@ -3,6 +3,7 @@ import re import json import copy import inspect +import collections import contextlib from .exceptions import ( @@ -10,6 +11,12 @@ from .exceptions import ( SchemaDuplicatedEnvGroupKeys ) +from openpype.settings.constants import ( + SYSTEM_SETTINGS_KEY, + PROJECT_SETTINGS_KEY, + SCHEMA_KEY_SYSTEM_SETTINGS, + SCHEMA_KEY_PROJECT_SETTINGS +) try: STRING_TYPE = basestring except Exception: @@ -24,6 +31,10 @@ TEMPLATE_METADATA_KEYS = ( DEFAULT_VALUES_KEY, ) +SCHEMA_EXTEND_TYPES = ( + "schema", "template", "schema_template", "dynamic_schema" +) + template_key_pattern = re.compile(r"(\{.*?[^{0]*\})") @@ -102,8 +113,8 @@ class OverrideState: class SchemasHub: - def __init__(self, schema_subfolder, reset=True): - self._schema_subfolder = schema_subfolder + def __init__(self, schema_type, reset=True): + self._schema_type = schema_type self._loaded_types = {} self._gui_types = tuple() @@ -112,25 +123,56 @@ class SchemasHub: self._loaded_templates = {} self._loaded_schemas = {} + # Attributes for modules settings + self._dynamic_schemas_defs_by_id = {} + self._dynamic_schemas_by_id = {} + # Store validating and validated dynamic template or schemas self._validating_dynamic = set() self._validated_dynamic = set() - # It doesn't make sence to reload types on each reset as they can't be - # changed - self._load_types() - # Trigger reset if reset: self.reset() + @property + def schema_type(self): + return self._schema_type + def reset(self): + self._load_modules_settings_defs() + self._load_types() self._load_schemas() + def _load_modules_settings_defs(self): + from openpype.modules import get_module_settings_defs + + module_settings_defs = get_module_settings_defs() + for module_settings_def_cls in module_settings_defs: + module_settings_def = module_settings_def_cls() + def_id = module_settings_def.id + self._dynamic_schemas_defs_by_id[def_id] = module_settings_def + @property def gui_types(self): return self._gui_types + def resolve_dynamic_schema(self, dynamic_key): + output = [] + for def_id, def_keys in self._dynamic_schemas_by_id.items(): + if dynamic_key in def_keys: + def_schema = def_keys[dynamic_key] + if not def_schema: + continue + + if isinstance(def_schema, dict): + def_schema = [def_schema] + + for item in def_schema: + item["_dynamic_schema_id"] = def_id + output.extend(def_schema) + return output + def get_template_name(self, item_def, default=None): """Get template name from passed item definition. @@ -260,7 +302,7 @@ class SchemasHub: list: Resolved schema data. """ schema_type = schema_data["type"] - if schema_type not in ("schema", "template", "schema_template"): + if schema_type not in SCHEMA_EXTEND_TYPES: return [schema_data] if schema_type == "schema": @@ -268,6 +310,9 @@ class SchemasHub: self.get_schema(schema_data["name"]) ) + if schema_type == "dynamic_schema": + return self.resolve_dynamic_schema(schema_data["name"]) + template_name = schema_data["name"] template_def = self.get_template(template_name) @@ -368,14 +413,16 @@ class SchemasHub: self._crashed_on_load = {} self._loaded_templates = {} self._loaded_schemas = {} + self._dynamic_schemas_by_id = {} dirpath = os.path.join( os.path.dirname(os.path.abspath(__file__)), "schemas", - self._schema_subfolder + self.schema_type ) loaded_schemas = {} loaded_templates = {} + dynamic_schemas_by_id = {} for root, _, filenames in os.walk(dirpath): for filename in filenames: basename, ext = os.path.splitext(filename) @@ -425,8 +472,34 @@ class SchemasHub: ) loaded_schemas[basename] = schema_data + defs_iter = self._dynamic_schemas_defs_by_id.items() + for def_id, module_settings_def in defs_iter: + dynamic_schemas_by_id[def_id] = ( + module_settings_def.get_dynamic_schemas(self.schema_type) + ) + module_schemas = module_settings_def.get_settings_schemas( + self.schema_type + ) + for key, schema_data in module_schemas.items(): + if isinstance(schema_data, list): + if key in loaded_templates: + raise KeyError( + "Duplicated template key \"{}\"".format(key) + ) + loaded_templates[key] = schema_data + else: + if key in loaded_schemas: + raise KeyError( + "Duplicated schema key \"{}\"".format(key) + ) + loaded_schemas[key] = schema_data + self._loaded_templates = loaded_templates self._loaded_schemas = loaded_schemas + self._dynamic_schemas_by_id = dynamic_schemas_by_id + + def get_dynamic_modules_settings_defs(self, schema_def_id): + return self._dynamic_schemas_defs_by_id.get(schema_def_id) def _fill_template(self, child_data, template_def): """Fill template based on schema definition and template definition. @@ -660,3 +733,38 @@ class SchemasHub: if found_idx is not None: metadata_item = template_def.pop(found_idx) return metadata_item + + +class DynamicSchemaValueCollector: + # Map schema hub type to store keys + schema_hub_type_map = { + SCHEMA_KEY_SYSTEM_SETTINGS: SYSTEM_SETTINGS_KEY, + SCHEMA_KEY_PROJECT_SETTINGS: PROJECT_SETTINGS_KEY + } + + def __init__(self, schema_hub): + self._schema_hub = schema_hub + self._dynamic_entities = [] + + def add_entity(self, entity): + self._dynamic_entities.append(entity) + + def create_hierarchy(self): + output = collections.defaultdict(dict) + for entity in self._dynamic_entities: + output[entity.dynamic_schema_id][entity.path] = ( + entity.settings_value() + ) + return output + + def save_values(self): + hierarchy = self.create_hierarchy() + + for schema_def_id, schema_def_value in hierarchy.items(): + schema_def = self._schema_hub.get_dynamic_modules_settings_defs( + schema_def_id + ) + top_key = self.schema_hub_type_map.get( + self._schema_hub.schema_type + ) + schema_def.save_defaults(top_key, schema_def_value) diff --git a/openpype/settings/entities/root_entities.py b/openpype/settings/entities/root_entities.py index 4a06d2d591..05d20ee60b 100644 --- a/openpype/settings/entities/root_entities.py +++ b/openpype/settings/entities/root_entities.py @@ -9,8 +9,11 @@ from .base_entity import BaseItemEntity from .lib import ( NOT_SET, WRAPPER_TYPES, + SCHEMA_KEY_SYSTEM_SETTINGS, + SCHEMA_KEY_PROJECT_SETTINGS, OverrideState, - SchemasHub + SchemasHub, + DynamicSchemaValueCollector ) from .exceptions import ( SchemaError, @@ -28,6 +31,7 @@ from openpype.settings.lib import ( DEFAULTS_DIR, get_default_settings, + reset_default_settings, get_studio_system_settings_overrides, save_studio_settings, @@ -265,6 +269,16 @@ class RootEntity(BaseItemEntity): output[key] = child_obj.value return output + def collect_dynamic_schema_entities(self): + output = DynamicSchemaValueCollector(self.schema_hub) + if self._override_state is not OverrideState.DEFAULTS: + return output + + for child_obj in self.non_gui_children.values(): + child_obj.collect_dynamic_schema_entities(output) + + return output + def settings_value(self): """Value for current override state with metadata. @@ -276,6 +290,8 @@ class RootEntity(BaseItemEntity): if self._override_state is not OverrideState.DEFAULTS: output = {} for key, child_obj in self.non_gui_children.items(): + if child_obj.is_dynamic_schema_node: + continue value = child_obj.settings_value() if value is not NOT_SET: output[key] = value @@ -374,6 +390,7 @@ class RootEntity(BaseItemEntity): if self._override_state is OverrideState.DEFAULTS: self._save_default_values() + reset_default_settings() elif self._override_state is OverrideState.STUDIO: self._save_studio_values() @@ -421,6 +438,9 @@ class RootEntity(BaseItemEntity): with open(output_path, "w") as file_stream: json.dump(value, file_stream, indent=4) + dynamic_values_item = self.collect_dynamic_schema_entities() + dynamic_values_item.save_values() + @abstractmethod def _save_studio_values(self): """Save studio override values.""" @@ -476,7 +496,7 @@ class SystemSettings(RootEntity): ): if schema_hub is None: # Load system schemas - schema_hub = SchemasHub("system_schema") + schema_hub = SchemasHub(SCHEMA_KEY_SYSTEM_SETTINGS) super(SystemSettings, self).__init__(schema_hub, reset) @@ -607,7 +627,7 @@ class ProjectSettings(RootEntity): if schema_hub is None: # Load system schemas - schema_hub = SchemasHub("projects_schema") + schema_hub = SchemasHub(SCHEMA_KEY_PROJECT_SETTINGS) super(ProjectSettings, self).__init__(schema_hub, reset) diff --git a/openpype/settings/entities/schemas/README.md b/openpype/settings/entities/schemas/README.md index 05605f8ce1..a89bbb20da 100644 --- a/openpype/settings/entities/schemas/README.md +++ b/openpype/settings/entities/schemas/README.md @@ -112,6 +112,22 @@ ``` - It is possible to define default values for unfilled fields to do so one of items in list must be dictionary with key `"__default_values__"` and value as dictionary with default key: values (as in example above). +### dynamic_schema +- dynamic templates that can be defined by class of `ModuleSettingsDef` +- example: +``` +{ + "type": "dynamic_schema", + "name": "project_settings/global" +} +``` +- all valid `ModuleSettingsDef` classes where calling of `get_settings_schemas` + will return dictionary where is key "project_settings/global" with schemas + will extend and replace this item +- works almost the same way as templates + - one item can be replaced by multiple items (or by 0 items) +- goal is to dynamically loaded settings of OpenPype addons without having + their schemas or default values in main repository ## Basic Dictionary inputs - these inputs wraps another inputs into {key: value} relation diff --git a/openpype/settings/entities/schemas/projects_schema/schema_main.json b/openpype/settings/entities/schemas/projects_schema/schema_main.json index 575cfc9e72..c9eca5dedd 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_main.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_main.json @@ -125,6 +125,10 @@ { "type": "schema", "name": "schema_project_unreal" + }, + { + "type": "dynamic_schema", + "name": "project_settings/global" } ] } diff --git a/openpype/settings/entities/schemas/system_schema/schema_modules.json b/openpype/settings/entities/schemas/system_schema/schema_modules.json index dd85f9351a..31d8e04731 100644 --- a/openpype/settings/entities/schemas/system_schema/schema_modules.json +++ b/openpype/settings/entities/schemas/system_schema/schema_modules.json @@ -5,6 +5,18 @@ "collapsible": true, "is_file": true, "children": [ + { + "type": "path", + "key": "addon_paths", + "label": "OpenPype AddOn Paths", + "use_label_wrap": true, + "multiplatform": true, + "multipath": true, + "require_restart": true + }, + { + "type": "separator" + }, { "type": "dict", "key": "avalon", diff --git a/openpype/settings/lib.py b/openpype/settings/lib.py index 4a363910b8..60ed54bd4a 100644 --- a/openpype/settings/lib.py +++ b/openpype/settings/lib.py @@ -329,6 +329,45 @@ def reset_default_settings(): _DEFAULT_SETTINGS = None +def _get_default_settings(): + from openpype.modules import get_module_settings_defs + + defaults = load_openpype_default_settings() + + module_settings_defs = get_module_settings_defs() + for module_settings_def_cls in module_settings_defs: + module_settings_def = module_settings_def_cls() + system_defaults = module_settings_def.get_defaults( + SYSTEM_SETTINGS_KEY + ) or {} + for path, value in system_defaults.items(): + if not path: + continue + + subdict = defaults["system_settings"] + path_items = list(path.split("/")) + last_key = path_items.pop(-1) + for key in path_items: + subdict = subdict[key] + subdict[last_key] = value + + project_defaults = module_settings_def.get_defaults( + PROJECT_SETTINGS_KEY + ) or {} + for path, value in project_defaults.items(): + if not path: + continue + + subdict = defaults + path_items = list(path.split("/")) + last_key = path_items.pop(-1) + for key in path_items: + subdict = subdict[key] + subdict[last_key] = value + + return defaults + + def get_default_settings(): """Get default settings. @@ -338,12 +377,10 @@ def get_default_settings(): Returns: dict: Loaded default settings. """ - # TODO add cacher - return load_openpype_default_settings() - # global _DEFAULT_SETTINGS - # if _DEFAULT_SETTINGS is None: - # _DEFAULT_SETTINGS = load_jsons_from_dir(DEFAULTS_DIR) - # return copy.deepcopy(_DEFAULT_SETTINGS) + global _DEFAULT_SETTINGS + if _DEFAULT_SETTINGS is None: + _DEFAULT_SETTINGS = _get_default_settings() + return copy.deepcopy(_DEFAULT_SETTINGS) def load_json_file(fpath): @@ -380,8 +417,8 @@ def load_jsons_from_dir(path, *args, **kwargs): "data1": "CONTENT OF FILE" }, "folder2": { - "data1": { - "subfolder1": "CONTENT OF FILE" + "subfolder1": { + "data2": "CONTENT OF FILE" } } }