Merge pull request #1959 from pypeclub/feature/module_settings

Global: Settings defined by Addons/Modules

openpype/modules/__init__.py

@@ -1,21 +1,35 @@
 # -*- coding: utf-8 -*-
 from .base import (
     OpenPypeModule,
+    OpenPypeAddOn,
     OpenPypeInterface,
 
     load_modules,
 
     ModulesManager,
-    TrayModulesManager
+    TrayModulesManager,
+
+    BaseModuleSettingsDef,
+    ModuleSettingsDef,
+    JsonFilesSettingsDef,
+
+    get_module_settings_defs
 )
 
 
 __all__ = (
     "OpenPypeModule",
+    "OpenPypeAddOn",
     "OpenPypeInterface",
 
     "load_modules",
 
     "ModulesManager",
-    "TrayModulesManager"
+    "TrayModulesManager",
+
+    "BaseModuleSettingsDef",
+    "ModuleSettingsDef",
+    "JsonFilesSettingsDef",
+
+    "get_module_settings_defs"
 )

openpype/modules/base.py

@@ -2,9 +2,11 @@
 """Base class for Pype Modules."""
 import os
 import sys
+import json
 import time
 import inspect
 import logging
+import platform
 import threading
 import collections
 from uuid import uuid4
@@ -12,7 +14,18 @@ from abc import ABCMeta, abstractmethod
 import six
 
 import openpype
-from openpype.settings import get_system_settings
+from openpype.settings import (
+    get_system_settings,
+    SYSTEM_SETTINGS_KEY,
+    PROJECT_SETTINGS_KEY,
+    SCHEMA_KEY_SYSTEM_SETTINGS,
+    SCHEMA_KEY_PROJECT_SETTINGS
+)
+
+from openpype.settings.lib import (
+    get_studio_system_settings_overrides,
+    load_json_file
+)
 from openpype.lib import PypeLogger
 
 
@@ -115,11 +128,51 @@ def get_default_modules_dir():
     return os.path.join(current_dir, "default_modules")
 
 
+def get_dynamic_modules_dirs():
+    """Possible paths to OpenPype Addons of Modules.
+
+    Paths are loaded from studio settings under:
+        `modules -> addon_paths -> {platform name}`
+
+    Path may contain environment variable as a formatting string.
+
+    They are not validated or checked their existence.
+
+    Returns:
+        list: Paths loaded from studio overrides.
+    """
+    output = []
+    value = get_studio_system_settings_overrides()
+    for key in ("modules", "addon_paths", platform.system().lower()):
+        if key not in value:
+            return output
+        value = value[key]
+
+    for path in value:
+        if not path:
+            continue
+
+        try:
+            path = path.format(**os.environ)
+        except Exception:
+            pass
+        output.append(path)
+    return output
+
+
 def get_module_dirs():
     """List of paths where OpenPype modules can be found."""
-    dirpaths = [
-        get_default_modules_dir()
-    ]
+    _dirpaths = []
+    _dirpaths.append(get_default_modules_dir())
+    _dirpaths.extend(get_dynamic_modules_dirs())
+
+    dirpaths = []
+    for path in _dirpaths:
+        if not path:
+            continue
+        normalized = os.path.normpath(path)
+        if normalized not in dirpaths:
+            dirpaths.append(normalized)
     return dirpaths
 
 
@@ -165,6 +218,9 @@ def _load_interfaces():
         os.path.join(get_default_modules_dir(), "interfaces.py")
     )
     for dirpath in dirpaths:
+        if not os.path.exists(dirpath):
+            continue
+
         for filename in os.listdir(dirpath):
             if filename in ("__pycache__", ):
                 continue
@@ -272,12 +328,19 @@ def _load_modules():
 
             # TODO add more logic how to define if folder is module or not
             # - check manifest and content of manifest
-            if os.path.isdir(fullpath):
-                import_module_from_dirpath(dirpath, filename, modules_key)
+            try:
+                if os.path.isdir(fullpath):
+                    import_module_from_dirpath(dirpath, filename, modules_key)
 
-            elif ext in (".py", ):
-                module = import_filepath(fullpath)
-                setattr(openpype_modules, basename, module)
+                elif ext in (".py", ):
+                    module = import_filepath(fullpath)
+                    setattr(openpype_modules, basename, module)
+
+            except Exception:
+                log.error(
+                    "Failed to import '{}'.".format(fullpath),
+                    exc_info=True
+                )
 
 
 class _OpenPypeInterfaceMeta(ABCMeta):
@@ -368,7 +431,16 @@ class OpenPypeModule:
 
 
 class OpenPypeAddOn(OpenPypeModule):
-    pass
+    # Enable Addon by default
+    enabled = True
+
+    def initialize(self, module_settings):
+        """Initialization is not be required for most of addons."""
+        pass
+
+    def connect_with_modules(self, enabled_modules):
+        """Do not require to implement connection with modules for addon."""
+        pass
 
 
 class ModulesManager:
@@ -920,3 +992,424 @@ class TrayModulesManager(ModulesManager):
                         ),
                         exc_info=True
                     )
+
+
+def get_module_settings_defs():
+    """Check loaded addons/modules for existence of thei settings definition.
+
+    Check if OpenPype addon/module as python module has class that inherit
+    from `ModuleSettingsDef` in python module variables (imported
+    in `__init__py`).
+
+    Returns:
+        list: All valid and not abstract settings definitions from imported
+            openpype addons and modules.
+    """
+    # Make sure modules are loaded
+    load_modules()
+
+    import openpype_modules
+
+    settings_defs = []
+
+    log = PypeLogger.get_logger("ModuleSettingsLoad")
+
+    for raw_module in openpype_modules:
+        for attr_name in dir(raw_module):
+            attr = getattr(raw_module, attr_name)
+            if (
+                not inspect.isclass(attr)
+                or attr is ModuleSettingsDef
+                or not issubclass(attr, ModuleSettingsDef)
+            ):
+                continue
+
+            if inspect.isabstract(attr):
+                # Find missing implementations by convetion on `abc` module
+                not_implemented = []
+                for attr_name in dir(attr):
+                    attr = getattr(attr, attr_name, None)
+                    abs_method = getattr(
+                        attr, "__isabstractmethod__", None
+                    )
+                    if attr and abs_method:
+                        not_implemented.append(attr_name)
+
+                # Log missing implementations
+                log.warning((
+                    "Skipping abstract Class: {} in module {}."
+                    " Missing implementations: {}"
+                ).format(
+                    attr_name, raw_module.__name__, ", ".join(not_implemented)
+                ))
+                continue
+
+            settings_defs.append(attr)
+
+    return settings_defs
+
+
+@six.add_metaclass(ABCMeta)
+class BaseModuleSettingsDef:
+    """Definition of settings for OpenPype module or AddOn."""
+    _id = None
+
+    @property
+    def id(self):
+        """ID created on initialization.
+
+        ID should be per created object. Helps to store objects.
+        """
+        if self._id is None:
+            self._id = uuid4()
+        return self._id
+
+    @abstractmethod
+    def get_settings_schemas(self, schema_type):
+        """Setting schemas for passed schema type.
+
+        These are main schemas by dynamic schema keys. If they're using
+        sub schemas or templates they should be loaded with
+        `get_dynamic_schemas`.
+
+        Returns:
+            dict: Schema by `dynamic_schema` keys.
+        """
+        pass
+
+    @abstractmethod
+    def get_dynamic_schemas(self, schema_type):
+        """Settings schemas and templates that can be used anywhere.
+
+        It is recommended to add prefix specific for addon/module to keys
+        (e.g. "my_addon/real_schema_name").
+
+        Returns:
+            dict: Schemas and templates by their keys.
+        """
+        pass
+
+    @abstractmethod
+    def get_defaults(self, top_key):
+        """Default values for passed top key.
+
+        Top keys are (currently) "system_settings" or "project_settings".
+
+        Should return exactly what was passed with `save_defaults`.
+
+        Returns:
+            dict: Default values by path to first key in OpenPype defaults.
+        """
+        pass
+
+    @abstractmethod
+    def save_defaults(self, top_key, data):
+        """Save default values for passed top key.
+
+        Top keys are (currently) "system_settings" or "project_settings".
+
+        Passed data are by path to first key defined in main schemas.
+        """
+        pass
+
+
+class ModuleSettingsDef(BaseModuleSettingsDef):
+    """Settings definiton with separated system and procect settings parts.
+
+    Reduce conditions that must be checked and adds predefined methods for
+    each case.
+    """
+    def get_defaults(self, top_key):
+        """Split method into 2 methods by top key."""
+        if top_key == SYSTEM_SETTINGS_KEY:
+            return self.get_default_system_settings() or {}
+        elif top_key == PROJECT_SETTINGS_KEY:
+            return self.get_default_project_settings() or {}
+        return {}
+
+    def save_defaults(self, top_key, data):
+        """Split method into 2 methods by top key."""
+        if top_key == SYSTEM_SETTINGS_KEY:
+            self.save_system_defaults(data)
+        elif top_key == PROJECT_SETTINGS_KEY:
+            self.save_project_defaults(data)
+
+    def get_settings_schemas(self, schema_type):
+        """Split method into 2 methods by schema type."""
+        if schema_type == SCHEMA_KEY_SYSTEM_SETTINGS:
+            return self.get_system_settings_schemas() or {}
+        elif schema_type == SCHEMA_KEY_PROJECT_SETTINGS:
+            return self.get_project_settings_schemas() or {}
+        return {}
+
+    def get_dynamic_schemas(self, schema_type):
+        """Split method into 2 methods by schema type."""
+        if schema_type == SCHEMA_KEY_SYSTEM_SETTINGS:
+            return self.get_system_dynamic_schemas() or {}
+        elif schema_type == SCHEMA_KEY_PROJECT_SETTINGS:
+            return self.get_project_dynamic_schemas() or {}
+        return {}
+
+    @abstractmethod
+    def get_system_settings_schemas(self):
+        """Schemas and templates usable in system settings schemas.
+
+        Returns:
+            dict: Schemas and templates by it's names. Names must be unique
+                across whole OpenPype.
+        """
+        pass
+
+    @abstractmethod
+    def get_project_settings_schemas(self):
+        """Schemas and templates usable in project settings schemas.
+
+        Returns:
+            dict: Schemas and templates by it's names. Names must be unique
+                across whole OpenPype.
+        """
+        pass
+
+    @abstractmethod
+    def get_system_dynamic_schemas(self):
+        """System schemas by dynamic schema name.
+
+        If dynamic schema name is not available in then schema will not used.
+
+        Returns:
+            dict: Schemas or list of schemas by dynamic schema name.
+        """
+        pass
+
+    @abstractmethod
+    def get_project_dynamic_schemas(self):
+        """Project schemas by dynamic schema name.
+
+        If dynamic schema name is not available in then schema will not used.
+
+        Returns:
+            dict: Schemas or list of schemas by dynamic schema name.
+        """
+        pass
+
+    @abstractmethod
+    def get_default_system_settings(self):
+        """Default system settings values.
+
+        Returns:
+            dict: Default values by path to first key.
+        """
+        pass
+
+    @abstractmethod
+    def get_default_project_settings(self):
+        """Default project settings values.
+
+        Returns:
+            dict: Default values by path to first key.
+        """
+        pass
+
+    @abstractmethod
+    def save_system_defaults(self, data):
+        """Save default system settings values.
+
+        Passed data are by path to first key defined in main schemas.
+        """
+        pass
+
+    @abstractmethod
+    def save_project_defaults(self, data):
+        """Save default project settings values.
+
+        Passed data are by path to first key defined in main schemas.
+        """
+        pass
+
+
+class JsonFilesSettingsDef(ModuleSettingsDef):
+    """Preimplemented settings definition using json files and file structure.
+
+    Expected file structure:
+    ┕ root
+      │
+      │ # Default values
+      ┝ defaults
+      │ ┝ system_settings.json
+      │ ┕ project_settings.json
+      │
+      │ # Schemas for `dynamic_template` type
+      ┝ dynamic_schemas
+      │ ┝ system_dynamic_schemas.json
+      │ ┕ project_dynamic_schemas.json
+      │
+      │ # Schemas that can be used anywhere (enhancement for `dynamic_schemas`)
+      ┕ schemas
+        ┝ system_schemas
+        │ ┝ <system schema.json> # Any schema or template files
+        │ ┕ ...
+        ┕ project_schemas
+          ┝ <system schema.json> # Any schema or template files
+          ┕ ...
+
+    Schemas can be loaded with prefix to avoid duplicated schema/template names
+    across all OpenPype addons/modules. Prefix can be defined with class
+    attribute `schema_prefix`.
+
+    Only think which must be implemented in `get_settings_root_path` which
+    should return directory path to `root` (in structure graph above).
+    """
+    # Possible way how to define `schemas` prefix
+    schema_prefix = ""
+
+    @abstractmethod
+    def get_settings_root_path(self):
+        """Directory path where settings and it's schemas are located."""
+        pass
+
+    def __init__(self):
+        settings_root_dir = self.get_settings_root_path()
+        defaults_dir = os.path.join(
+            settings_root_dir, "defaults"
+        )
+        dynamic_schemas_dir = os.path.join(
+            settings_root_dir, "dynamic_schemas"
+        )
+        schemas_dir = os.path.join(
+            settings_root_dir, "schemas"
+        )
+
+        self.system_defaults_filepath = os.path.join(
+            defaults_dir, "system_settings.json"
+        )
+        self.project_defaults_filepath = os.path.join(
+            defaults_dir, "project_settings.json"
+        )
+
+        self.system_dynamic_schemas_filepath = os.path.join(
+            dynamic_schemas_dir, "system_dynamic_schemas.json"
+        )
+        self.project_dynamic_schemas_filepath = os.path.join(
+            dynamic_schemas_dir, "project_dynamic_schemas.json"
+        )
+
+        self.system_schemas_dir = os.path.join(
+            schemas_dir, "system_schemas"
+        )
+        self.project_schemas_dir = os.path.join(
+            schemas_dir, "project_schemas"
+        )
+
+    def _load_json_file_data(self, path):
+        if os.path.exists(path):
+            return load_json_file(path)
+        return {}
+
+    def get_default_system_settings(self):
+        """Default system settings values.
+
+        Returns:
+            dict: Default values by path to first key.
+        """
+        return self._load_json_file_data(self.system_defaults_filepath)
+
+    def get_default_project_settings(self):
+        """Default project settings values.
+
+        Returns:
+            dict: Default values by path to first key.
+        """
+        return self._load_json_file_data(self.project_defaults_filepath)
+
+    def _save_data_to_filepath(self, path, data):
+        dirpath = os.path.dirname(path)
+        if not os.path.exists(dirpath):
+            os.makedirs(dirpath)
+
+        with open(path, "w") as file_stream:
+            json.dump(data, file_stream, indent=4)
+
+    def save_system_defaults(self, data):
+        """Save default system settings values.
+
+        Passed data are by path to first key defined in main schemas.
+        """
+        self._save_data_to_filepath(self.system_defaults_filepath, data)
+
+    def save_project_defaults(self, data):
+        """Save default project settings values.
+
+        Passed data are by path to first key defined in main schemas.
+        """
+        self._save_data_to_filepath(self.project_defaults_filepath, data)
+
+    def get_system_dynamic_schemas(self):
+        """System schemas by dynamic schema name.
+
+        If dynamic schema name is not available in then schema will not used.
+
+        Returns:
+            dict: Schemas or list of schemas by dynamic schema name.
+        """
+        return self._load_json_file_data(self.system_dynamic_schemas_filepath)
+
+    def get_project_dynamic_schemas(self):
+        """Project schemas by dynamic schema name.
+
+        If dynamic schema name is not available in then schema will not used.
+
+        Returns:
+            dict: Schemas or list of schemas by dynamic schema name.
+        """
+        return self._load_json_file_data(self.project_dynamic_schemas_filepath)
+
+    def _load_files_from_path(self, path):
+        output = {}
+        if not path or not os.path.exists(path):
+            return output
+
+        if os.path.isfile(path):
+            filename = os.path.basename(path)
+            basename, ext = os.path.splitext(filename)
+            if ext == ".json":
+                if self.schema_prefix:
+                    key = "{}/{}".format(self.schema_prefix, basename)
+                else:
+                    key = basename
+                output[key] = self._load_json_file_data(path)
+            return output
+
+        path = os.path.normpath(path)
+        for root, _, files in os.walk(path, topdown=False):
+            for filename in files:
+                basename, ext = os.path.splitext(filename)
+                if ext != ".json":
+                    continue
+
+                json_path = os.path.join(root, filename)
+                store_key = os.path.join(
+                    root.replace(path, ""), basename
+                ).replace("\\", "/")
+                if self.schema_prefix:
+                    store_key = "{}/{}".format(self.schema_prefix, store_key)
+                output[store_key] = self._load_json_file_data(json_path)
+
+        return output
+
+    def get_system_settings_schemas(self):
+        """Schemas and templates usable in system settings schemas.
+
+        Returns:
+            dict: Schemas and templates by it's names. Names must be unique
+                across whole OpenPype.
+        """
+        return self._load_files_from_path(self.system_schemas_dir)
+
+    def get_project_settings_schemas(self):
+        """Schemas and templates usable in project settings schemas.
+
+        Returns:
+            dict: Schemas and templates by it's names. Names must be unique
+                across whole OpenPype.
+        """
+        return self._load_files_from_path(self.project_schemas_dir)

openpype/settings/__init__.py

@@ -1,7 +1,21 @@
+from .constants import (
+    GLOBAL_SETTINGS_KEY,
+    SYSTEM_SETTINGS_KEY,
+    PROJECT_SETTINGS_KEY,
+    PROJECT_ANATOMY_KEY,
+    LOCAL_SETTING_KEY,
+
+    SCHEMA_KEY_SYSTEM_SETTINGS,
+    SCHEMA_KEY_PROJECT_SETTINGS,
+
+    KEY_ALLOWED_SYMBOLS,
+    KEY_REGEX
+)
 from .exceptions import (
     SaveWarningExc
 )
 from .lib import (
+    get_general_environments,
     get_system_settings,
     get_project_settings,
     get_current_project_settings,
@@ -16,8 +30,21 @@ from .entities import (
 
 
 __all__ = (
+    "GLOBAL_SETTINGS_KEY",
+    "SYSTEM_SETTINGS_KEY",
+    "PROJECT_SETTINGS_KEY",
+    "PROJECT_ANATOMY_KEY",
+    "LOCAL_SETTING_KEY",
+
+    "SCHEMA_KEY_SYSTEM_SETTINGS",
+    "SCHEMA_KEY_PROJECT_SETTINGS",
+
+    "KEY_ALLOWED_SYMBOLS",
+    "KEY_REGEX",
+
     "SaveWarningExc",
 
+    "get_general_environments",
     "get_system_settings",
     "get_project_settings",
     "get_current_project_settings",

openpype/settings/constants.py

@@ -14,13 +14,17 @@ METADATA_KEYS = (
     M_DYNAMIC_KEY_LABEL
 )
 
-# File where studio's system overrides are stored
+# Keys where studio's system overrides are stored
 GLOBAL_SETTINGS_KEY = "global_settings"
 SYSTEM_SETTINGS_KEY = "system_settings"
 PROJECT_SETTINGS_KEY = "project_settings"
 PROJECT_ANATOMY_KEY = "project_anatomy"
 LOCAL_SETTING_KEY = "local_settings"
 
+# Schema hub names
+SCHEMA_KEY_SYSTEM_SETTINGS = "system_schema"
+SCHEMA_KEY_PROJECT_SETTINGS = "projects_schema"
+
 DEFAULT_PROJECT_KEY = "__default_project__"
 
 KEY_ALLOWED_SYMBOLS = "a-zA-Z0-9-_ "
@@ -39,6 +43,9 @@ __all__ = (
     "PROJECT_ANATOMY_KEY",
     "LOCAL_SETTING_KEY",
 
+    "SCHEMA_KEY_SYSTEM_SETTINGS",
+    "SCHEMA_KEY_PROJECT_SETTINGS",
+
     "DEFAULT_PROJECT_KEY",
 
     "KEY_ALLOWED_SYMBOLS",

openpype/settings/defaults/system_settings/modules.json

@@ -1,4 +1,9 @@
 {
+    "addon_paths": {
+        "windows": [],
+        "darwin": [],
+        "linux": []
+    },
     "avalon": {
         "AVALON_TIMEOUT": 1000,
         "AVALON_THUMBNAIL_ROOT": {

openpype/settings/entities/base_entity.py

@@ -104,6 +104,12 @@ class BaseItemEntity(BaseEntity):
         self.is_group = False
         # Entity's value will be stored into file with name of it's key
         self.is_file = False
+        # Default values are not stored to an openpype file
+        # - these must not be set through schemas directly
+        self.dynamic_schema_id = None
+        self.is_dynamic_schema_node = False
+        self.is_in_dynamic_schema_node = False
+
         # Reference to parent entity which has `is_group` == True
         #   - stays as None if none of parents is group
         self.group_item = None
@@ -255,13 +261,22 @@ class BaseItemEntity(BaseEntity):
             )
 
         # Group item can be only once in on hierarchy branch.
-        if self.is_group and self.group_item:
+        if self.is_group and self.group_item is not None:
             raise SchemeGroupHierarchyBug(self)
 
+        # Group item can be only once in on hierarchy branch.
+        if self.group_item is not None and self.is_dynamic_schema_node:
+            reason = (
+                "Dynamic schema is inside grouped item {}."
+                " Change group hierarchy or remove dynamic"
+                " schema to be able work properly."
+            ).format(self.group_item.path)
+            raise EntitySchemaError(self, reason)
+
         # Validate that env group entities will be stored into file.
         #   - env group entities must store metadata which is not possible if
         #       metadata would be outside of file
-        if not self.file_item and self.is_env_group:
+        if self.file_item is None and self.is_env_group:
             reason = (
                 "Environment item is not inside file"
                 " item so can't store metadata for defaults."
@@ -478,7 +493,15 @@ class BaseItemEntity(BaseEntity):
 
     @abstractmethod
     def settings_value(self):
-        """Value of an item without key."""
+        """Value of an item without key without dynamic items."""
+        pass
+
+    @abstractmethod
+    def collect_dynamic_schema_entities(self):
+        """Collect entities that are on top of dynamically added schemas.
+
+        This method make sence only when defaults are saved.
+        """
         pass
 
     @abstractmethod
@@ -808,6 +831,12 @@ class ItemEntity(BaseItemEntity):
         self.is_dynamic_item = is_dynamic_item
 
         self.is_file = self.schema_data.get("is_file", False)
+        # These keys have underscore as they must not be set in schemas
+        self.dynamic_schema_id = self.schema_data.get(
+            "_dynamic_schema_id", None
+        )
+        self.is_dynamic_schema_node = self.dynamic_schema_id is not None
+
         self.is_group = self.schema_data.get("is_group", False)
         self.is_in_dynamic_item = bool(
             not self.is_dynamic_item
@@ -837,10 +866,20 @@ class ItemEntity(BaseItemEntity):
         self._require_restart_on_change = require_restart_on_change
 
         # File item reference
-        if self.parent.is_file:
-            self.file_item = self.parent
-        elif self.parent.file_item:
-            self.file_item = self.parent.file_item
+        if not self.is_dynamic_schema_node:
+            self.is_in_dynamic_schema_node = (
+                self.parent.is_dynamic_schema_node
+                or self.parent.is_in_dynamic_schema_node
+            )
+
+        if (
+            not self.is_dynamic_schema_node
+            and not self.is_in_dynamic_schema_node
+        ):
+            if self.parent.is_file:
+                self.file_item = self.parent
+            elif self.parent.file_item:
+                self.file_item = self.parent.file_item
 
         # Group item reference
         if self.parent.is_group:
@@ -891,6 +930,18 @@ class ItemEntity(BaseItemEntity):
     def root_key(self):
         return self.root_item.root_key
 
+    @abstractmethod
+    def collect_dynamic_schema_entities(self, collector):
+        """Collect entities that are on top of dynamically added schemas.
+
+        This method make sence only when defaults are saved.
+
+        Args:
+            collector(DynamicSchemaValueCollector): Object where dynamic
+                entities are stored.
+        """
+        pass
+
     def schema_validations(self):
         if not self.label and self.use_label_wrap:
             reason = (
@@ -899,7 +950,12 @@ class ItemEntity(BaseItemEntity):
             )
             raise EntitySchemaError(self, reason)
 
-        if self.is_file and self.file_item is not None:
+        if (
+            not self.is_dynamic_schema_node
+            and not self.is_in_dynamic_schema_node
+            and self.is_file
+            and self.file_item is not None
+        ):
             reason = (
                 "Entity has set `is_file` to true but"
                 " it's parent is already marked as file item."

openpype/settings/entities/dict_conditional.py

@@ -469,6 +469,10 @@ class DictConditionalEntity(ItemEntity):
                     return True
         return False
 
+    def collect_dynamic_schema_entities(self, collector):
+        if self.is_dynamic_schema_node:
+            collector.add_entity(self)
+
     def settings_value(self):
         if self._override_state is OverrideState.NOT_DEFINED:
             return NOT_SET
@@ -482,13 +486,7 @@ class DictConditionalEntity(ItemEntity):
 
             output = {}
             for key, child_obj in children_items:
-                child_value = child_obj.settings_value()
-                if not child_obj.is_file and not child_obj.file_item:
-                    for _key, _value in child_value.items():
-                        new_key = "/".join([key, _key])
-                        output[new_key] = _value
-                else:
-                    output[key] = child_value
+                output[key] = child_obj.settings_value()
             return output
 
         if self.is_group:

openpype/settings/entities/dict_immutable_keys_entity.py

@@ -330,15 +330,32 @@ class DictImmutableKeysEntity(ItemEntity):
                     return True
         return False
 
+    def collect_dynamic_schema_entities(self, collector):
+        for child_obj in self.non_gui_children.values():
+            child_obj.collect_dynamic_schema_entities(collector)
+
+        if self.is_dynamic_schema_node:
+            collector.add_entity(self)
+
     def settings_value(self):
         if self._override_state is OverrideState.NOT_DEFINED:
             return NOT_SET
 
         if self._override_state is OverrideState.DEFAULTS:
+            is_dynamic_schema_node = (
+                self.is_dynamic_schema_node or self.is_in_dynamic_schema_node
+            )
             output = {}
             for key, child_obj in self.non_gui_children.items():
+                if child_obj.is_dynamic_schema_node:
+                    continue
+
                 child_value = child_obj.settings_value()
-                if not child_obj.is_file and not child_obj.file_item:
+                if (
+                    not is_dynamic_schema_node
+                    and not child_obj.is_file
+                    and not child_obj.file_item
+                ):
                     for _key, _value in child_value.items():
                         new_key = "/".join([key, _key])
                         output[new_key] = _value

openpype/settings/entities/dict_mutable_keys_entity.py

@@ -261,7 +261,7 @@ class DictMutableKeysEntity(EndpointEntity):
             raise EntitySchemaError(self, reason)
 
         # TODO Ability to store labels should be defined with different key
-        if self.collapsible_key and not self.file_item:
+        if self.collapsible_key and self.file_item is None:
             reason = (
                 "Modifiable dictionary with collapsible keys is not under"
                 " file item so can't store metadata."

openpype/settings/entities/input_entities.py

@@ -49,6 +49,10 @@ class EndpointEntity(ItemEntity):
 
         super(EndpointEntity, self).schema_validations()
 
+    def collect_dynamic_schema_entities(self, collector):
+        if self.is_dynamic_schema_node:
+            collector.add_entity(self)
+
     @abstractmethod
     def _settings_value(self):
         pass
@@ -121,7 +125,11 @@ class InputEntity(EndpointEntity):
 
     def schema_validations(self):
         # Input entity must have file parent.
-        if not self.file_item:
+        if (
+            not self.is_dynamic_schema_node
+            and not self.is_in_dynamic_schema_node
+            and self.file_item is None
+        ):
             raise EntitySchemaError(self, "Missing parent file entity.")
 
         super(InputEntity, self).schema_validations()

openpype/settings/entities/item_entities.py

@@ -115,6 +115,9 @@ class PathEntity(ItemEntity):
     def set(self, value):
         self.child_obj.set(value)
 
+    def collect_dynamic_schema_entities(self, *args, **kwargs):
+        self.child_obj.collect_dynamic_schema_entities(*args, **kwargs)
+
     def settings_value(self):
         if self._override_state is OverrideState.NOT_DEFINED:
             return NOT_SET
@@ -236,7 +239,12 @@ class ListStrictEntity(ItemEntity):
 
     def schema_validations(self):
         # List entity must have file parent.
-        if not self.file_item and not self.is_file:
+        if (
+            not self.is_dynamic_schema_node
+            and not self.is_in_dynamic_schema_node
+            and not self.is_file
+            and self.file_item is None
+        ):
             raise EntitySchemaError(
                 self, "Missing file entity in hierarchy."
             )
@@ -279,6 +287,10 @@ class ListStrictEntity(ItemEntity):
         for idx, item in enumerate(new_value):
             self.children[idx].set(item)
 
+    def collect_dynamic_schema_entities(self, collector):
+        if self.is_dynamic_schema_node:
+            collector.add_entity(self)
+
     def settings_value(self):
         if self._override_state is OverrideState.NOT_DEFINED:
             return NOT_SET

openpype/settings/entities/lib.py

@@ -3,6 +3,7 @@ import re
 import json
 import copy
 import inspect
+import collections
 import contextlib
 
 from .exceptions import (
@@ -10,6 +11,12 @@ from .exceptions import (
     SchemaDuplicatedEnvGroupKeys
 )
 
+from openpype.settings.constants import (
+    SYSTEM_SETTINGS_KEY,
+    PROJECT_SETTINGS_KEY,
+    SCHEMA_KEY_SYSTEM_SETTINGS,
+    SCHEMA_KEY_PROJECT_SETTINGS
+)
 try:
     STRING_TYPE = basestring
 except Exception:
@@ -24,6 +31,10 @@ TEMPLATE_METADATA_KEYS = (
     DEFAULT_VALUES_KEY,
 )
 
+SCHEMA_EXTEND_TYPES = (
+    "schema", "template", "schema_template", "dynamic_schema"
+)
+
 template_key_pattern = re.compile(r"(\{.*?[^{0]*\})")
 
 
@@ -102,8 +113,8 @@ class OverrideState:
 
 
 class SchemasHub:
-    def __init__(self, schema_subfolder, reset=True):
-        self._schema_subfolder = schema_subfolder
+    def __init__(self, schema_type, reset=True):
+        self._schema_type = schema_type
 
         self._loaded_types = {}
         self._gui_types = tuple()
@@ -112,25 +123,56 @@ class SchemasHub:
         self._loaded_templates = {}
         self._loaded_schemas = {}
 
+        # Attributes for modules settings
+        self._dynamic_schemas_defs_by_id = {}
+        self._dynamic_schemas_by_id = {}
+
         # Store validating and validated dynamic template or schemas
         self._validating_dynamic = set()
         self._validated_dynamic = set()
 
-        # It doesn't make sence to reload types on each reset as they can't be
-        #   changed
-        self._load_types()
-
         # Trigger reset
         if reset:
             self.reset()
 
+    @property
+    def schema_type(self):
+        return self._schema_type
+
     def reset(self):
+        self._load_modules_settings_defs()
+        self._load_types()
         self._load_schemas()
 
+    def _load_modules_settings_defs(self):
+        from openpype.modules import get_module_settings_defs
+
+        module_settings_defs = get_module_settings_defs()
+        for module_settings_def_cls in module_settings_defs:
+            module_settings_def = module_settings_def_cls()
+            def_id = module_settings_def.id
+            self._dynamic_schemas_defs_by_id[def_id] = module_settings_def
+
     @property
     def gui_types(self):
         return self._gui_types
 
+    def resolve_dynamic_schema(self, dynamic_key):
+        output = []
+        for def_id, def_keys in self._dynamic_schemas_by_id.items():
+            if dynamic_key in def_keys:
+                def_schema = def_keys[dynamic_key]
+                if not def_schema:
+                    continue
+
+                if isinstance(def_schema, dict):
+                    def_schema = [def_schema]
+
+                for item in def_schema:
+                    item["_dynamic_schema_id"] = def_id
+                output.extend(def_schema)
+        return output
+
     def get_template_name(self, item_def, default=None):
         """Get template name from passed item definition.
 
@@ -260,7 +302,7 @@ class SchemasHub:
             list: Resolved schema data.
         """
         schema_type = schema_data["type"]
-        if schema_type not in ("schema", "template", "schema_template"):
+        if schema_type not in SCHEMA_EXTEND_TYPES:
             return [schema_data]
 
         if schema_type == "schema":
@@ -268,6 +310,9 @@ class SchemasHub:
                 self.get_schema(schema_data["name"])
             )
 
+        if schema_type == "dynamic_schema":
+            return self.resolve_dynamic_schema(schema_data["name"])
+
         template_name = schema_data["name"]
         template_def = self.get_template(template_name)
 
@@ -368,14 +413,16 @@ class SchemasHub:
         self._crashed_on_load = {}
         self._loaded_templates = {}
         self._loaded_schemas = {}
+        self._dynamic_schemas_by_id = {}
 
         dirpath = os.path.join(
             os.path.dirname(os.path.abspath(__file__)),
             "schemas",
-            self._schema_subfolder
+            self.schema_type
         )
         loaded_schemas = {}
         loaded_templates = {}
+        dynamic_schemas_by_id = {}
         for root, _, filenames in os.walk(dirpath):
             for filename in filenames:
                 basename, ext = os.path.splitext(filename)
@@ -425,8 +472,34 @@ class SchemasHub:
                         )
                     loaded_schemas[basename] = schema_data
 
+        defs_iter = self._dynamic_schemas_defs_by_id.items()
+        for def_id, module_settings_def in defs_iter:
+            dynamic_schemas_by_id[def_id] = (
+                module_settings_def.get_dynamic_schemas(self.schema_type)
+            )
+            module_schemas = module_settings_def.get_settings_schemas(
+                self.schema_type
+            )
+            for key, schema_data in module_schemas.items():
+                if isinstance(schema_data, list):
+                    if key in loaded_templates:
+                        raise KeyError(
+                            "Duplicated template key \"{}\"".format(key)
+                        )
+                    loaded_templates[key] = schema_data
+                else:
+                    if key in loaded_schemas:
+                        raise KeyError(
+                            "Duplicated schema key \"{}\"".format(key)
+                        )
+                    loaded_schemas[key] = schema_data
+
         self._loaded_templates = loaded_templates
         self._loaded_schemas = loaded_schemas
+        self._dynamic_schemas_by_id = dynamic_schemas_by_id
+
+    def get_dynamic_modules_settings_defs(self, schema_def_id):
+        return self._dynamic_schemas_defs_by_id.get(schema_def_id)
 
     def _fill_template(self, child_data, template_def):
         """Fill template based on schema definition and template definition.
@@ -660,3 +733,38 @@ class SchemasHub:
         if found_idx is not None:
             metadata_item = template_def.pop(found_idx)
         return metadata_item
+
+
+class DynamicSchemaValueCollector:
+    # Map schema hub type to store keys
+    schema_hub_type_map = {
+        SCHEMA_KEY_SYSTEM_SETTINGS: SYSTEM_SETTINGS_KEY,
+        SCHEMA_KEY_PROJECT_SETTINGS: PROJECT_SETTINGS_KEY
+    }
+
+    def __init__(self, schema_hub):
+        self._schema_hub = schema_hub
+        self._dynamic_entities = []
+
+    def add_entity(self, entity):
+        self._dynamic_entities.append(entity)
+
+    def create_hierarchy(self):
+        output = collections.defaultdict(dict)
+        for entity in self._dynamic_entities:
+            output[entity.dynamic_schema_id][entity.path] = (
+                entity.settings_value()
+            )
+        return output
+
+    def save_values(self):
+        hierarchy = self.create_hierarchy()
+
+        for schema_def_id, schema_def_value in hierarchy.items():
+            schema_def = self._schema_hub.get_dynamic_modules_settings_defs(
+                schema_def_id
+            )
+            top_key = self.schema_hub_type_map.get(
+                self._schema_hub.schema_type
+            )
+            schema_def.save_defaults(top_key, schema_def_value)

openpype/settings/entities/root_entities.py

@@ -9,8 +9,11 @@ from .base_entity import BaseItemEntity
 from .lib import (
     NOT_SET,
     WRAPPER_TYPES,
+    SCHEMA_KEY_SYSTEM_SETTINGS,
+    SCHEMA_KEY_PROJECT_SETTINGS,
     OverrideState,
-    SchemasHub
+    SchemasHub,
+    DynamicSchemaValueCollector
 )
 from .exceptions import (
     SchemaError,
@@ -28,6 +31,7 @@ from openpype.settings.lib import (
     DEFAULTS_DIR,
 
     get_default_settings,
+    reset_default_settings,
 
     get_studio_system_settings_overrides,
     save_studio_settings,
@@ -265,6 +269,16 @@ class RootEntity(BaseItemEntity):
             output[key] = child_obj.value
         return output
 
+    def collect_dynamic_schema_entities(self):
+        output = DynamicSchemaValueCollector(self.schema_hub)
+        if self._override_state is not OverrideState.DEFAULTS:
+            return output
+
+        for child_obj in self.non_gui_children.values():
+            child_obj.collect_dynamic_schema_entities(output)
+
+        return output
+
     def settings_value(self):
         """Value for current override state with metadata.
 
@@ -276,6 +290,8 @@ class RootEntity(BaseItemEntity):
         if self._override_state is not OverrideState.DEFAULTS:
             output = {}
             for key, child_obj in self.non_gui_children.items():
+                if child_obj.is_dynamic_schema_node:
+                    continue
                 value = child_obj.settings_value()
                 if value is not NOT_SET:
                     output[key] = value
@@ -374,6 +390,7 @@ class RootEntity(BaseItemEntity):
 
         if self._override_state is OverrideState.DEFAULTS:
             self._save_default_values()
+            reset_default_settings()
 
         elif self._override_state is OverrideState.STUDIO:
             self._save_studio_values()
@@ -421,6 +438,9 @@ class RootEntity(BaseItemEntity):
             with open(output_path, "w") as file_stream:
                 json.dump(value, file_stream, indent=4)
 
+        dynamic_values_item = self.collect_dynamic_schema_entities()
+        dynamic_values_item.save_values()
+
     @abstractmethod
     def _save_studio_values(self):
         """Save studio override values."""
@@ -476,7 +496,7 @@ class SystemSettings(RootEntity):
     ):
         if schema_hub is None:
             # Load system schemas
-            schema_hub = SchemasHub("system_schema")
+            schema_hub = SchemasHub(SCHEMA_KEY_SYSTEM_SETTINGS)
 
         super(SystemSettings, self).__init__(schema_hub, reset)
 
@@ -607,7 +627,7 @@ class ProjectSettings(RootEntity):
 
         if schema_hub is None:
             # Load system schemas
-            schema_hub = SchemasHub("projects_schema")
+            schema_hub = SchemasHub(SCHEMA_KEY_PROJECT_SETTINGS)
 
         super(ProjectSettings, self).__init__(schema_hub, reset)
 

openpype/settings/entities/schemas/README.md

@@ -112,6 +112,22 @@
 ```
 - It is possible to define default values for unfilled fields to do so one of items in list must be dictionary with key `"__default_values__"` and value as dictionary with default key: values (as in example above).
 
+### dynamic_schema
+- dynamic templates that can be defined by class of `ModuleSettingsDef`
+- example:
+```
+{
+    "type": "dynamic_schema",
+    "name": "project_settings/global"
+}
+```
+- all valid `ModuleSettingsDef` classes where calling of `get_settings_schemas`
+    will return dictionary where is key "project_settings/global" with schemas
+    will extend and replace this item
+- works almost the same way as templates
+    - one item can be replaced by multiple items (or by 0 items)
+- goal is to dynamically loaded settings of OpenPype addons without having
+    their schemas or default values in main repository
 
 ## Basic Dictionary inputs
 - these inputs wraps another inputs into {key: value} relation

openpype/settings/entities/schemas/projects_schema/schema_main.json

@@ -125,6 +125,10 @@
                 {
                     "type": "schema",
                     "name": "schema_project_unreal"
+                },
+                {
+                    "type": "dynamic_schema",
+                    "name": "project_settings/global"
                 }
             ]
         }

openpype/settings/entities/schemas/system_schema/schema_modules.json

@@ -5,6 +5,18 @@
     "collapsible": true,
     "is_file": true,
     "children": [
+        {
+            "type": "path",
+            "key": "addon_paths",
+            "label": "OpenPype AddOn Paths",
+            "use_label_wrap": true,
+            "multiplatform": true,
+            "multipath": true,
+            "require_restart": true
+        },
+        {
+            "type": "separator"
+        },
         {
             "type": "dict",
             "key": "avalon",

openpype/settings/lib.py

@@ -329,6 +329,45 @@ def reset_default_settings():
     _DEFAULT_SETTINGS = None
 
 
+def _get_default_settings():
+    from openpype.modules import get_module_settings_defs
+
+    defaults = load_openpype_default_settings()
+
+    module_settings_defs = get_module_settings_defs()
+    for module_settings_def_cls in module_settings_defs:
+        module_settings_def = module_settings_def_cls()
+        system_defaults = module_settings_def.get_defaults(
+            SYSTEM_SETTINGS_KEY
+        ) or {}
+        for path, value in system_defaults.items():
+            if not path:
+                continue
+
+            subdict = defaults["system_settings"]
+            path_items = list(path.split("/"))
+            last_key = path_items.pop(-1)
+            for key in path_items:
+                subdict = subdict[key]
+            subdict[last_key] = value
+
+        project_defaults = module_settings_def.get_defaults(
+            PROJECT_SETTINGS_KEY
+        ) or {}
+        for path, value in project_defaults.items():
+            if not path:
+                continue
+
+            subdict = defaults
+            path_items = list(path.split("/"))
+            last_key = path_items.pop(-1)
+            for key in path_items:
+                subdict = subdict[key]
+            subdict[last_key] = value
+
+    return defaults
+
+
 def get_default_settings():
     """Get default settings.
 
@@ -338,12 +377,10 @@ def get_default_settings():
     Returns:
         dict: Loaded default settings.
     """
-    # TODO add cacher
-    return load_openpype_default_settings()
-    # global _DEFAULT_SETTINGS
-    # if _DEFAULT_SETTINGS is None:
-    #     _DEFAULT_SETTINGS = load_jsons_from_dir(DEFAULTS_DIR)
-    # return copy.deepcopy(_DEFAULT_SETTINGS)
+    global _DEFAULT_SETTINGS
+    if _DEFAULT_SETTINGS is None:
+        _DEFAULT_SETTINGS = _get_default_settings()
+    return copy.deepcopy(_DEFAULT_SETTINGS)
 
 
 def load_json_file(fpath):
@@ -380,8 +417,8 @@ def load_jsons_from_dir(path, *args, **kwargs):
             "data1": "CONTENT OF FILE"
         },
         "folder2": {
-            "data1": {
-                "subfolder1": "CONTENT OF FILE"
+            "subfolder1": {
+                "data2": "CONTENT OF FILE"
             }
         }
     }