Merge branch 'develop' into feature/host_settings

pype/settings/lib.py

@@ -6,9 +6,11 @@ import copy
 log = logging.getLogger(__name__)
 
 # Metadata keys for work with studio and project overrides
-OVERRIDEN_KEY = "__overriden_keys__"
+M_OVERRIDEN_KEY = "__overriden_keys__"
+# Metadata key for storing information about environments
+M_ENVIRONMENT_KEY = "__environment_keys__"
 # NOTE key popping not implemented yet
-POP_KEY = "__pop_key__"
+M_POP_KEY = "__pop_key__"
 
 # Folder where studio overrides are stored
 STUDIO_OVERRIDES_PATH = os.environ["PYPE_PROJECT_CONFIGS"]
@@ -111,6 +113,32 @@ def load_json(fpath):
     return {}
 
 
+def find_environments(data):
+    if not data or not isinstance(data, dict):
+        return
+
+    output = {}
+    if M_ENVIRONMENT_KEY in data:
+        metadata = data.pop(M_ENVIRONMENT_KEY)
+        for env_group_key, env_keys in metadata.items():
+            output[env_group_key] = {}
+            for key in env_keys:
+                output[env_group_key][key] = data[key]
+
+    for value in data.values():
+        result = find_environments(value)
+        if not result:
+            continue
+
+        for env_group_key, env_values in result.items():
+            if env_group_key not in output:
+                output[env_group_key] = {}
+
+            for env_key, env_value in env_values.items():
+                output[env_group_key][env_key] = env_value
+    return output
+
+
 def subkey_merge(_dict, value, keys):
     key = keys.pop(0)
     if not keys:
@@ -223,13 +251,13 @@ def project_anatomy_overrides(project_name):
 
 
 def merge_overrides(global_dict, override_dict):
-    if OVERRIDEN_KEY in override_dict:
-        overriden_keys = set(override_dict.pop(OVERRIDEN_KEY))
+    if M_OVERRIDEN_KEY in override_dict:
+        overriden_keys = set(override_dict.pop(M_OVERRIDEN_KEY))
     else:
         overriden_keys = set()
 
     for key, value in override_dict.items():
-        if value == POP_KEY:
+        if value == M_POP_KEY:
             global_dict.pop(key)
 
         elif (
@@ -271,6 +299,8 @@ def project_settings(project_name):
 
 
 def environments():
-    default_values = default_settings()[ENVIRONMENTS_KEY]
-    studio_values = studio_system_settings()
-    return apply_overrides(default_values, studio_values)
+    envs = copy.deepcopy(default_settings()[ENVIRONMENTS_KEY])
+    envs_from_system_settings = find_environments(system_settings())
+    for env_group_key, values in envs_from_system_settings.items():
+        envs[env_group_key] = values
+    return envs

pype/tools/settings/settings/README.md

@@ -19,6 +19,7 @@
 - GUI schemas are huge json files, to be able to split whole configuration into multiple schema there's type `schema`
 - system configuration schemas are stored in `~/tools/settings/settings/gui_schemas/system_schema/` and project configurations in `~/tools/settings/settings/gui_schemas/projects_schema/`
 - each schema name is filename of json file except extension (without ".json")
+- if content is dictionary content will be used as `schema` else will be used as `schema_template`
 
 ### schema
 - can have only key `"children"` which is list of strings, each string should represent another schema (order matters) string represebts name of the schema
@@ -31,6 +32,87 @@
 }
 ```
 
+### schema_template
+- allows to define schema "templates" to not duplicate same content multiple times
+```javascript
+// EXAMPLE json file content (filename: example_template.json)
+[
+    {
+        "__default_values__": {
+            "multipath_executables": true
+        }
+    }, {
+        "type": "raw-json",
+        "label": "{host_label} Environments",
+        "key": "{host_name}_environments",
+        "env_group_key": "{host_name}"
+    }, {
+        "type": "path-widget",
+        "key": "{host_name}_executables",
+        "label": "{host_label} - Full paths to executables",
+        "multiplatform": "{multipath_executables}",
+        "multipath": true
+    }
+]
+```
+```javascript
+// EXAMPLE usage of the template in schema
+{
+    "type": "dict",
+    "key": "schema_template_examples",
+    "label": "Schema template examples",
+    "children": [
+        {
+            "type": "schema_template",
+            // filename of template (example_template.json)
+            "name": "example_template",
+            "template_data": {
+                "host_label": "Maya 2019",
+                "host_name": "maya_2019",
+                "multipath_executables": false
+            }
+        }, {
+            "type": "schema_template",
+            "name": "example_template",
+            "template_data": {
+                "host_label": "Maya 2020",
+                "host_name": "maya_2020"
+            }
+        }
+    ]
+}
+```
+- item in schema mush contain `"type"` and `"name"` keys but it is also expected that `"template_data"` will be entered too
+- all items in the list, except `__default_values__`, will replace `schema_template` item in schema
+- template may contain another template or schema
+- it is expected that schema template will have unfilled fields as in example
+    - unfilled fields are allowed only in values of schema dictionary
+```javascript
+{
+    ...
+    // Allowed
+    "key": "{to_fill}"
+    ...
+    // Not allowed
+    "{to_fill}": "value"
+    ...
+}
+```
+- Unfilled fields can be also used for non string values, in that case value must contain only one key and value for fill must contain right type.
+```javascript
+{
+    ...
+    // Allowed
+    "multiplatform": "{executable_multiplatform}"
+    ...
+    // Not allowed
+    "multiplatform": "{executable_multiplatform}_enhanced_string"
+    ...
+}
+```
+- It is possible to define default values for unfilled fields to do so one of items in list must be dictionary with key `"__default_values__"` and value as dictionary with default key: values (as in example above).
+
+
 ## Basic Dictionary inputs
 - these inputs wraps another inputs into {key: value} relation
 

pype/tools/settings/settings/gui_schemas/system_schema/example_schema.json

@@ -5,6 +5,40 @@
     "is_file": true,
     "children": [
         {
+            "type": "dict",
+            "key": "schema_template_exaples",
+            "label": "Schema template examples",
+            "children": [
+                {
+                    "type": "schema_template",
+                    "name": "example_template",
+                    "template_data": {
+                        "host_label": "Maya 2019",
+                        "host_name": "maya_2019",
+                        "multipath_executables": false
+                    }
+                }, {
+                    "type": "schema_template",
+                    "name": "example_template",
+                    "template_data": {
+                        "host_label": "Maya 2020",
+                        "host_name": "maya_2020"
+                    }
+                }
+            ]
+        }, {
+            "key": "env_group_test",
+            "label": "EnvGroup Test",
+            "type": "dict",
+            "children": [
+                {
+                    "key": "key_to_store_in_system_settings",
+                    "label": "Testing environment group",
+                    "type": "raw-json",
+                    "env_group_key": "test_group"
+                }
+            ]
+        }, {
             "key": "dict_wrapper",
             "type": "dict-invisible",
             "children": [

pype/tools/settings/settings/gui_schemas/system_schema/example_template.json

@@ -0,0 +1,18 @@
+[
+    {
+        "__default_values__": {
+            "multipath_executables": true
+        }
+    }, {
+        "type": "raw-json",
+        "label": "{host_label} Environments",
+        "key": "{host_name}_environments",
+        "env_group_key": "{host_name}"
+    }, {
+        "type": "path-widget",
+        "key": "{host_name}_executables",
+        "label": "{host_label} - Full paths to executables",
+        "multiplatform": "{multipath_executables}",
+        "multipath": true
+    }
+]

pype/tools/settings/settings/widgets/base.py

@@ -47,6 +47,7 @@ class SystemWidget(QtWidgets.QWidget):
         self._ignore_value_changes = False
 
         self.input_fields = []
+        self.environ_fields = []
 
         scroll_widget = QtWidgets.QScrollArea(self)
         scroll_widget.setObjectName("GroupWidget")
@@ -130,10 +131,14 @@ class SystemWidget(QtWidgets.QWidget):
         for input_field in self.input_fields:
             input_field.hierarchical_style_update()
 
+    def add_environ_field(self, input_field):
+        self.environ_fields.append(input_field)
+
     def reset(self):
         reset_default_settings()
 
         self.input_fields.clear()
+        self.environ_fields.clear()
         while self.content_layout.count() != 0:
             widget = self.content_layout.itemAt(0).widget()
             self.content_layout.removeWidget(widget)
@@ -214,7 +219,7 @@ class SystemWidget(QtWidgets.QWidget):
             all_values = _all_values
 
         # Skip first key
-        all_values = all_values["system"]
+        all_values = lib.convert_gui_data_with_metadata(all_values["system"])
 
         prject_defaults_dir = os.path.join(
             DEFAULTS_DIR, SYSTEM_SETTINGS_KEY
@@ -246,16 +251,19 @@ class SystemWidget(QtWidgets.QWidget):
     def _update_values(self):
         self.ignore_value_changes = True
 
-        default_values = {
+        default_values = lib.convert_data_to_gui_data({
             "system": default_settings()[SYSTEM_SETTINGS_KEY]
-        }
+        })
         for input_field in self.input_fields:
             input_field.update_default_values(default_values)
 
         if self._hide_studio_overrides:
             system_values = lib.NOT_SET
         else:
-            system_values = {"system": studio_system_settings()}
+            system_values = lib.convert_overrides_to_gui_data(
+                {"system": studio_system_settings()}
+            )
+
         for input_field in self.input_fields:
             input_field.update_studio_values(system_values)
 
@@ -730,17 +738,20 @@ class ProjectWidget(QtWidgets.QWidget):
     def _update_values(self):
         self.ignore_value_changes = True
 
-        default_values = {"project": default_settings()}
+        default_values = default_values = lib.convert_data_to_gui_data(
+            {"project": default_settings()}
+        )
         for input_field in self.input_fields:
             input_field.update_default_values(default_values)
 
         if self._hide_studio_overrides:
             studio_values = lib.NOT_SET
         else:
-            studio_values = {"project": {
+            studio_values = lib.convert_overrides_to_gui_data({"project": {
                 PROJECT_SETTINGS_KEY: studio_project_settings(),
                 PROJECT_ANATOMY_KEY: studio_project_anatomy()
-            }}
+            }})
+
         for input_field in self.input_fields:
             input_field.update_studio_values(studio_values)
 

pype/tools/settings/settings/widgets/item_types.py

@@ -24,12 +24,32 @@ class SettingObject:
     # Will allow to show actions for the item type (disabled for proxies) else
     # item is skipped and try to trigger actions on it's parent.
     allow_actions = True
+    # If item can store environment values
+    allow_to_environment = False
     # All item types must have implemented Qt signal which is emitted when
     # it's or it's children value has changed,
     value_changed = None
     # Item will expand to full width in grid layout
     expand_in_grid = False
 
+    def merge_metadata(self, current_metadata, new_metadata):
+        for key, value in new_metadata.items():
+            if key not in current_metadata:
+                current_metadata[key] = value
+
+            elif key == "groups":
+                current_metadata[key].extend(value)
+
+            elif key == "environments":
+                for group_key, subvalue in value.items():
+                    if group_key not in current_metadata[key]:
+                        current_metadata[key][group_key] = []
+                    current_metadata[key][group_key].extend(subvalue)
+
+            else:
+                raise KeyError("Unknown metadata key: \"{}\"".format(key))
+        return current_metadata
+
     def _set_default_attributes(self):
         """Create and reset attributes required for all item types.
 
@@ -49,6 +69,9 @@ class SettingObject:
         self._as_widget = False
         self._is_group = False
 
+        # If value should be stored to environments
+        self._env_group_key = None
+
         self._any_parent_as_widget = None
         self._any_parent_is_group = None
 
@@ -84,6 +107,20 @@ class SettingObject:
         self._is_group = input_data.get("is_group", False)
         # TODO not implemented yet
         self._is_nullable = input_data.get("is_nullable", False)
+        self._env_group_key = input_data.get("env_group_key")
+
+        if self.is_environ:
+            if not self.allow_to_environment:
+                raise TypeError((
+                    "Item {} does not allow to store environment values"
+                ).format(input_data["type"]))
+
+            if self.as_widget:
+                raise TypeError((
+                    "Item is used as widget and"
+                    " marked to store environments at the same time."
+                ))
+            self.add_environ_field(self)
 
         any_parent_as_widget = parent.as_widget
         if not any_parent_as_widget:
@@ -140,6 +177,17 @@ class SettingObject:
         """
         return self._has_studio_override or self._parent.has_studio_override
 
+    @property
+    def is_environ(self):
+        return self._env_group_key is not None
+
+    @property
+    def env_group_key(self):
+        return self._env_group_key
+
+    def add_environ_field(self, input_field):
+        self._parent.add_environ_field(input_field)
+
     @property
     def as_widget(self):
         """Item is used as widget in parent item.
@@ -269,6 +317,13 @@ class SettingObject:
         """Output for saving changes or overrides."""
         return {self.key: self.item_value()}
 
+    def environment_value(self):
+        raise NotImplementedError(
+            "{} Method `environment_value` not implemented!".format(
+                repr(self)
+            )
+        )
+
     @classmethod
     def style_state(
         cls, has_studio_override, is_invalid, is_overriden, is_modified
@@ -1133,6 +1188,7 @@ class RawJsonInput(QtWidgets.QPlainTextEdit):
 class RawJsonWidget(QtWidgets.QWidget, InputObject):
     default_input_value = "{}"
     value_changed = QtCore.Signal(object)
+    allow_to_environment = True
 
     def __init__(
         self, input_data, parent,
@@ -1182,7 +1238,21 @@ class RawJsonWidget(QtWidgets.QWidget, InputObject):
     def item_value(self):
         if self.is_invalid:
             return NOT_SET
-        return self.input_field.json_value()
+
+        value = self.input_field.json_value()
+        if not self.is_environ:
+            return value
+
+        output = {}
+        for key, value in value.items():
+            output[key.upper()] = value
+
+        output[METADATA_KEY] = {
+            "environments": {
+                self.env_group_key: list(output.keys())
+            }
+        }
+        return output
 
 
 class ListItem(QtWidgets.QWidget, SettingObject):
@@ -2543,6 +2613,33 @@ class DictWidget(QtWidgets.QWidget, SettingObject):
             output.update(input_field.config_value())
         return output
 
+    def _override_values(self, project_overrides):
+        values = {}
+        groups = []
+        for input_field in self.input_fields:
+            if project_overrides:
+                value, is_group = input_field.overrides()
+            else:
+                value, is_group = input_field.studio_overrides()
+            if value is NOT_SET:
+                continue
+
+            if METADATA_KEY in value and METADATA_KEY in values:
+                new_metadata = value.pop(METADATA_KEY)
+                values[METADATA_KEY] = self.merge_metadata(
+                    values[METADATA_KEY], new_metadata
+                )
+
+            values.update(value)
+            if is_group:
+                groups.extend(value.keys())
+
+        if groups:
+            if METADATA_KEY not in values:
+                values[METADATA_KEY] = {}
+            values[METADATA_KEY]["groups"] = groups
+        return {self.key: values}, self.is_group
+
     def studio_overrides(self):
         if (
             not (self.as_widget or self.any_parent_as_widget)
@@ -2550,34 +2647,12 @@ class DictWidget(QtWidgets.QWidget, SettingObject):
             and not self.child_has_studio_override
         ):
             return NOT_SET, False
-
-        values = {}
-        groups = []
-        for input_field in self.input_fields:
-            value, is_group = input_field.studio_overrides()
-            if value is not NOT_SET:
-                values.update(value)
-                if is_group:
-                    groups.extend(value.keys())
-        if groups:
-            values[METADATA_KEY] = {"groups": groups}
-        return {self.key: values}, self.is_group
+        return self._override_values(False)
 
     def overrides(self):
         if not self.is_overriden and not self.child_overriden:
             return NOT_SET, False
-
-        values = {}
-        groups = []
-        for input_field in self.input_fields:
-            value, is_group = input_field.overrides()
-            if value is not NOT_SET:
-                values.update(value)
-                if is_group:
-                    groups.extend(value.keys())
-        if groups:
-            values[METADATA_KEY] = {"groups": groups}
-        return {self.key: values}, self.is_group
+        return self._override_values(True)
 
 
 class DictInvisible(QtWidgets.QWidget, SettingObject):
@@ -2792,6 +2867,33 @@ class DictInvisible(QtWidgets.QWidget, SettingObject):
             )
         self._was_overriden = bool(self._is_overriden)
 
+    def _override_values(self, project_overrides):
+        values = {}
+        groups = []
+        for input_field in self.input_fields:
+            if project_overrides:
+                value, is_group = input_field.overrides()
+            else:
+                value, is_group = input_field.studio_overrides()
+            if value is NOT_SET:
+                continue
+
+            if METADATA_KEY in value and METADATA_KEY in values:
+                new_metadata = value.pop(METADATA_KEY)
+                values[METADATA_KEY] = self.merge_metadata(
+                    values[METADATA_KEY], new_metadata
+                )
+
+            values.update(value)
+            if is_group:
+                groups.extend(value.keys())
+
+        if groups:
+            if METADATA_KEY not in values:
+                values[METADATA_KEY] = {}
+            values[METADATA_KEY]["groups"] = groups
+        return {self.key: values}, self.is_group
+
     def studio_overrides(self):
         if (
             not (self.as_widget or self.any_parent_as_widget)
@@ -2799,34 +2901,12 @@ class DictInvisible(QtWidgets.QWidget, SettingObject):
             and not self.child_has_studio_override
         ):
             return NOT_SET, False
-
-        values = {}
-        groups = []
-        for input_field in self.input_fields:
-            value, is_group = input_field.studio_overrides()
-            if value is not NOT_SET:
-                values.update(value)
-                if is_group:
-                    groups.extend(value.keys())
-        if groups:
-            values[METADATA_KEY] = {"groups": groups}
-        return {self.key: values}, self.is_group
+        return self._override_values(False)
 
     def overrides(self):
         if not self.is_overriden and not self.child_overriden:
             return NOT_SET, False
-
-        values = {}
-        groups = []
-        for input_field in self.input_fields:
-            value, is_group = input_field.overrides()
-            if value is not NOT_SET:
-                values.update(value)
-                if is_group:
-                    groups.extend(value.keys())
-        if groups:
-            values[METADATA_KEY] = {"groups": groups}
-        return {self.key: values}, self.is_group
+        return self._override_values(True)
 
 
 class PathWidget(QtWidgets.QWidget, SettingObject):

pype/tools/settings/settings/widgets/lib.py

@@ -1,7 +1,8 @@
 import os
+import re
 import json
 import copy
-from pype.settings.lib import OVERRIDEN_KEY
+from pype.settings.lib import M_OVERRIDEN_KEY, M_ENVIRONMENT_KEY
 from queue import Queue
 
 
@@ -11,10 +12,50 @@ class TypeToKlass:
 
 
 NOT_SET = type("NOT_SET", (), {"__bool__": lambda obj: False})()
-METADATA_KEY = type("METADATA_KEY", (), {})
+METADATA_KEY = type("METADATA_KEY", (), {})()
 OVERRIDE_VERSION = 1
 CHILD_OFFSET = 15
 
+key_pattern = re.compile(r"(\{.*?[^{0]*\})")
+
+
+def convert_gui_data_with_metadata(data, ignored_keys=None):
+    if not data or not isinstance(data, dict):
+        return data
+
+    if ignored_keys is None:
+        ignored_keys = tuple()
+
+    output = {}
+    if METADATA_KEY in data:
+        metadata = data.pop(METADATA_KEY)
+        for key, value in metadata.items():
+            if key in ignored_keys or key == "groups":
+                continue
+
+            if key == "environments":
+                output[M_ENVIRONMENT_KEY] = value
+            else:
+                raise KeyError("Unknown metadata key \"{}\"".format(key))
+
+    for key, value in data.items():
+        output[key] = convert_gui_data_with_metadata(value, ignored_keys)
+    return output
+
+
+def convert_data_to_gui_data(data, first=True):
+    if not data or not isinstance(data, dict):
+        return data
+
+    output = {}
+    if M_ENVIRONMENT_KEY in data:
+        data.pop(M_ENVIRONMENT_KEY)
+
+    for key, value in data.items():
+        output[key] = convert_data_to_gui_data(value, False)
+
+    return output
+
 
 def convert_gui_data_to_overrides(data, first=True):
     if not data or not isinstance(data, dict):
@@ -23,14 +64,15 @@ def convert_gui_data_to_overrides(data, first=True):
     output = {}
     if first:
         output["__override_version__"] = OVERRIDE_VERSION
+        data = convert_gui_data_with_metadata(data)
 
     if METADATA_KEY in data:
         metadata = data.pop(METADATA_KEY)
         for key, value in metadata.items():
             if key == "groups":
-                output[OVERRIDEN_KEY] = value
+                output[M_OVERRIDEN_KEY] = value
             else:
-                KeyError("Unknown metadata key \"{}\"".format(key))
+                raise KeyError("Unknown metadata key \"{}\"".format(key))
 
     for key, value in data.items():
         output[key] = convert_gui_data_to_overrides(value, False)
@@ -41,9 +83,12 @@ def convert_overrides_to_gui_data(data, first=True):
     if not data or not isinstance(data, dict):
         return data
 
+    if first:
+        data = convert_data_to_gui_data(data)
+
     output = {}
-    if OVERRIDEN_KEY in data:
-        groups = data.pop(OVERRIDEN_KEY)
+    if M_OVERRIDEN_KEY in data:
+        groups = data.pop(M_OVERRIDEN_KEY)
         if METADATA_KEY not in output:
             output[METADATA_KEY] = {}
         output[METADATA_KEY]["groups"] = groups
@@ -54,7 +99,111 @@ def convert_overrides_to_gui_data(data, first=True):
     return output
 
 
-def _fill_inner_schemas(schema_data, schema_collection):
+def _fill_schema_template_data(
+    template, template_data, required_keys=None, missing_keys=None
+):
+    first = False
+    if required_keys is None:
+        first = True
+        required_keys = set()
+        missing_keys = set()
+
+        _template = []
+        default_values = {}
+        for item in template:
+            if isinstance(item, dict) and "__default_values__" in item:
+                default_values = item["__default_values__"]
+            else:
+                _template.append(item)
+        template = _template
+
+        for key, value in default_values.items():
+            if key not in template_data:
+                template_data[key] = value
+
+    if not template:
+        output = template
+
+    elif isinstance(template, list):
+        output = []
+        for item in template:
+            output.append(_fill_schema_template_data(
+                item, template_data, required_keys, missing_keys
+            ))
+
+    elif isinstance(template, dict):
+        output = {}
+        for key, value in template.items():
+            output[key] = _fill_schema_template_data(
+                value, template_data, required_keys, missing_keys
+            )
+
+    elif isinstance(template, str):
+        # TODO find much better way how to handle filling template data
+        for replacement_string in key_pattern.findall(template):
+            key = str(replacement_string[1:-1])
+            required_keys.add(key)
+            if key not in template_data:
+                missing_keys.add(key)
+                continue
+
+            value = template_data[key]
+            if replacement_string == template:
+                # Replace the value with value from templates data
+                # - with this is possible to set value with different type
+                template = value
+            else:
+                # Only replace the key in string
+                template = template.replace(replacement_string, value)
+        output = template
+
+    else:
+        output = template
+
+    if first and missing_keys:
+        raise SchemaTemplateMissingKeys(missing_keys, required_keys)
+
+    return output
+
+
+def _fill_schema_template(child_data, schema_collection, schema_templates):
+    template_name = child_data["name"]
+    template = schema_templates.get(template_name)
+    if template is None:
+        if template_name in schema_collection:
+            raise KeyError((
+                "Schema \"{}\" is used as `schema_template`"
+            ).format(template_name))
+        raise KeyError("Schema template \"{}\" was not found".format(
+            template_name
+        ))
+
+    template_data = child_data.get("template_data") or {}
+    try:
+        filled_child = _fill_schema_template_data(
+            template, template_data
+        )
+
+    except SchemaTemplateMissingKeys as exc:
+        raise SchemaTemplateMissingKeys(
+            exc.missing_keys, exc.required_keys, template_name
+        )
+
+    output = []
+    for item in filled_child:
+        filled_item = _fill_inner_schemas(
+            item, schema_collection, schema_templates
+        )
+        if filled_item["type"] == "schema_template":
+            output.extend(_fill_schema_template(
+                filled_item, schema_collection, schema_templates
+            ))
+        else:
+            output.append(filled_item)
+    return output
+
+
+def _fill_inner_schemas(schema_data, schema_collection, schema_templates):
     if schema_data["type"] == "schema":
         raise ValueError("First item in schema data can't be schema.")
 
@@ -64,21 +213,62 @@ def _fill_inner_schemas(schema_data, schema_collection):
 
     new_children = []
     for child in children:
-        if child["type"] != "schema":
-            new_child = _fill_inner_schemas(child, schema_collection)
-            new_children.append(new_child)
+        child_type = child["type"]
+        if child_type == "schema":
+            schema_name = child["name"]
+            if schema_name not in schema_collection:
+                if schema_name in schema_templates:
+                    raise KeyError((
+                        "Schema template \"{}\" is used as `schema`"
+                    ).format(schema_name))
+                raise KeyError(
+                    "Schema \"{}\" was not found".format(schema_name)
+                )
+
+            filled_child = _fill_inner_schemas(
+                schema_collection[schema_name],
+                schema_collection,
+                schema_templates
+            )
+
+        elif child_type == "schema_template":
+            for filled_child in _fill_schema_template(
+                child, schema_collection, schema_templates
+            ):
+                new_children.append(filled_child)
             continue
 
-        new_child = _fill_inner_schemas(
-            schema_collection[child["name"]],
-            schema_collection
-        )
-        new_children.append(new_child)
+        else:
+            filled_child = _fill_inner_schemas(
+                child, schema_collection, schema_templates
+            )
+
+        new_children.append(filled_child)
 
     schema_data["children"] = new_children
     return schema_data
 
 
+class SchemaTemplateMissingKeys(Exception):
+    def __init__(self, missing_keys, required_keys, template_name=None):
+        self.missing_keys = missing_keys
+        self.required_keys = required_keys
+        if template_name:
+            msg = f"Schema template \"{template_name}\" require more keys.\n"
+        else:
+            msg = ""
+        msg += "Required keys: {}\nMissing keys: {}".format(
+            self.join_keys(required_keys),
+            self.join_keys(missing_keys)
+        )
+        super(SchemaTemplateMissingKeys, self).__init__(msg)
+
+    def join_keys(self, keys):
+        return ", ".join([
+            f"\"{key}\"" for key in keys
+        ])
+
+
 class SchemaMissingFileInfo(Exception):
     def __init__(self, invalid):
         full_path_keys = []
@@ -120,6 +310,21 @@ class SchemaDuplicatedKeys(Exception):
         super(SchemaDuplicatedKeys, self).__init__(msg)
 
 
+class SchemaDuplicatedEnvGroupKeys(Exception):
+    def __init__(self, invalid):
+        items = []
+        for key_path, keys in invalid.items():
+            joined_keys = ", ".join([
+                "\"{}\"".format(key) for key in keys
+            ])
+            items.append("\"{}\" ({})".format(key_path, joined_keys))
+
+        msg = (
+            "Schema items contain duplicated environment group keys. {}"
+        ).format(" || ".join(items))
+        super(SchemaDuplicatedEnvGroupKeys, self).__init__(msg)
+
+
 def file_keys_from_schema(schema_data):
     output = []
     item_type = schema_data["type"]
@@ -277,10 +482,50 @@ def validate_keys_are_unique(schema_data, keys=None):
         raise SchemaDuplicatedKeys(invalid)
 
 
+def validate_environment_groups_uniquenes(
+    schema_data, env_groups=None, keys=None
+):
+    is_first = False
+    if env_groups is None:
+        is_first = True
+        env_groups = {}
+        keys = []
+
+    my_keys = copy.deepcopy(keys)
+    key = schema_data.get("key")
+    if key:
+        my_keys.append(key)
+
+    env_group_key = schema_data.get("env_group_key")
+    if env_group_key:
+        if env_group_key not in env_groups:
+            env_groups[env_group_key] = []
+        env_groups[env_group_key].append("/".join(my_keys))
+
+    children = schema_data.get("children")
+    if not children:
+        return
+
+    for child in children:
+        validate_environment_groups_uniquenes(
+            child, env_groups, copy.deepcopy(my_keys)
+        )
+
+    if is_first:
+        invalid = {}
+        for env_group_key, key_paths in env_groups.items():
+            if len(key_paths) > 1:
+                invalid[env_group_key] = key_paths
+
+        if invalid:
+            raise SchemaDuplicatedEnvGroupKeys(invalid)
+
+
 def validate_schema(schema_data):
     validate_all_has_ending_file(schema_data)
     validate_is_group_is_unique_in_hierarchy(schema_data)
     validate_keys_are_unique(schema_data)
+    validate_environment_groups_uniquenes(schema_data)
 
 
 def gui_schema(subfolder, main_schema_name):
@@ -292,23 +537,30 @@ def gui_schema(subfolder, main_schema_name):
     )
 
     loaded_schemas = {}
-    for filename in os.listdir(dirpath):
-        basename, ext = os.path.splitext(filename)
-        if ext != ".json":
-            continue
+    loaded_schema_templates = {}
+    for root, _, filenames in os.walk(dirpath):
+        for filename in filenames:
+            basename, ext = os.path.splitext(filename)
+            if ext != ".json":
+                continue
 
-        filepath = os.path.join(dirpath, filename)
-        with open(filepath, "r") as json_stream:
-            try:
-                schema_data = json.load(json_stream)
-            except Exception as e:
-                raise Exception((f"Unable to parse JSON file {json_stream}\n "
-                                 f" - {e}")) from e
-        loaded_schemas[basename] = schema_data
+            filepath = os.path.join(root, filename)
+            with open(filepath, "r") as json_stream:
+                try:
+                    schema_data = json.load(json_stream)
+                except Exception as exc:
+                    raise Exception((
+                        f"Unable to parse JSON file {filepath}\n{exc}"
+                    )) from exc
+            if isinstance(schema_data, list):
+                loaded_schema_templates[basename] = schema_data
+            else:
+                loaded_schemas[basename] = schema_data
 
     main_schema = _fill_inner_schemas(
         loaded_schemas[main_schema_name],
-        loaded_schemas
+        loaded_schemas,
+        loaded_schema_templates
     )
     validate_schema(main_schema)
     return main_schema