Merge pull request #588 from pypeclub/feature/environments_in_settings

Environments in settings GUI

pype/settings/lib.py

@@ -6,9 +6,11 @@ import copy
 log = logging.getLogger(__name__)
 
 # Metadata keys for work with studio and project overrides
-OVERRIDEN_KEY = "__overriden_keys__"
+M_OVERRIDEN_KEY = "__overriden_keys__"
+# Metadata key for storing information about environments
+M_ENVIRONMENT_KEY = "__environment_keys__"
 # NOTE key popping not implemented yet
-POP_KEY = "__pop_key__"
+M_POP_KEY = "__pop_key__"
 
 # Folder where studio overrides are stored
 STUDIO_OVERRIDES_PATH = os.environ["PYPE_PROJECT_CONFIGS"]
@@ -111,6 +113,32 @@ def load_json(fpath):
     return {}
 
 
+def find_environments(data):
+    if not data or not isinstance(data, dict):
+        return
+
+    output = {}
+    if M_ENVIRONMENT_KEY in data:
+        metadata = data.pop(M_ENVIRONMENT_KEY)
+        for env_group_key, env_keys in metadata.items():
+            output[env_group_key] = {}
+            for key in env_keys:
+                output[env_group_key][key] = data[key]
+
+    for value in data.values():
+        result = find_environments(value)
+        if not result:
+            continue
+
+        for env_group_key, env_values in result.items():
+            if env_group_key not in output:
+                output[env_group_key] = {}
+
+            for env_key, env_value in env_values.items():
+                output[env_group_key][env_key] = env_value
+    return output
+
+
 def subkey_merge(_dict, value, keys):
     key = keys.pop(0)
     if not keys:
@@ -223,13 +251,13 @@ def project_anatomy_overrides(project_name):
 
 
 def merge_overrides(global_dict, override_dict):
-    if OVERRIDEN_KEY in override_dict:
-        overriden_keys = set(override_dict.pop(OVERRIDEN_KEY))
+    if M_OVERRIDEN_KEY in override_dict:
+        overriden_keys = set(override_dict.pop(M_OVERRIDEN_KEY))
     else:
         overriden_keys = set()
 
     for key, value in override_dict.items():
-        if value == POP_KEY:
+        if value == M_POP_KEY:
             global_dict.pop(key)
 
         elif (
@@ -271,6 +299,8 @@ def project_settings(project_name):
 
 
 def environments():
-    default_values = default_settings()[ENVIRONMENTS_KEY]
-    studio_values = studio_system_settings()
-    return apply_overrides(default_values, studio_values)
+    envs = copy.deepcopy(default_settings()[ENVIRONMENTS_KEY])
+    envs_from_system_settings = find_environments(system_settings())
+    for env_group_key, values in envs_from_system_settings.items():
+        envs[env_group_key] = values
+    return envs

pype/tools/settings/settings/gui_schemas/system_schema/1_examples.json

@@ -5,6 +5,18 @@
     "is_file": true,
     "children": [
         {
+            "key": "env_group_test",
+            "label": "EnvGroup Test",
+            "type": "dict",
+            "children": [
+                {
+                    "key": "key_to_store_in_system_settings",
+                    "label": "Testing environment group",
+                    "type": "raw-json",
+                    "env_group_key": "test_group"
+                }
+            ]
+        }, {
             "key": "dict_wrapper",
             "type": "dict-invisible",
             "children": [

pype/tools/settings/settings/widgets/base.py

@@ -47,6 +47,7 @@ class SystemWidget(QtWidgets.QWidget):
         self._ignore_value_changes = False
 
         self.input_fields = []
+        self.environ_fields = []
 
         scroll_widget = QtWidgets.QScrollArea(self)
         scroll_widget.setObjectName("GroupWidget")
@@ -130,10 +131,14 @@ class SystemWidget(QtWidgets.QWidget):
         for input_field in self.input_fields:
             input_field.hierarchical_style_update()
 
+    def add_environ_field(self, input_field):
+        self.environ_fields.append(input_field)
+
     def reset(self):
         reset_default_settings()
 
         self.input_fields.clear()
+        self.environ_fields.clear()
         while self.content_layout.count() != 0:
             widget = self.content_layout.itemAt(0).widget()
             self.content_layout.removeWidget(widget)
@@ -214,7 +219,7 @@ class SystemWidget(QtWidgets.QWidget):
             all_values = _all_values
 
         # Skip first key
-        all_values = all_values["system"]
+        all_values = lib.convert_gui_data_with_metadata(all_values["system"])
 
         prject_defaults_dir = os.path.join(
             DEFAULTS_DIR, SYSTEM_SETTINGS_KEY
@@ -246,16 +251,19 @@ class SystemWidget(QtWidgets.QWidget):
     def _update_values(self):
         self.ignore_value_changes = True
 
-        default_values = {
+        default_values = lib.convert_data_to_gui_data({
             "system": default_settings()[SYSTEM_SETTINGS_KEY]
-        }
+        })
         for input_field in self.input_fields:
             input_field.update_default_values(default_values)
 
         if self._hide_studio_overrides:
             system_values = lib.NOT_SET
         else:
-            system_values = {"system": studio_system_settings()}
+            system_values = lib.convert_overrides_to_gui_data(
+                {"system": studio_system_settings()}
+            )
+
         for input_field in self.input_fields:
             input_field.update_studio_values(system_values)
 
@@ -730,17 +738,20 @@ class ProjectWidget(QtWidgets.QWidget):
     def _update_values(self):
         self.ignore_value_changes = True
 
-        default_values = {"project": default_settings()}
+        default_values = default_values = lib.convert_data_to_gui_data(
+            {"project": default_settings()}
+        )
         for input_field in self.input_fields:
             input_field.update_default_values(default_values)
 
         if self._hide_studio_overrides:
             studio_values = lib.NOT_SET
         else:
-            studio_values = {"project": {
+            studio_values = lib.convert_overrides_to_gui_data({"project": {
                 PROJECT_SETTINGS_KEY: studio_project_settings(),
                 PROJECT_ANATOMY_KEY: studio_project_anatomy()
-            }}
+            }})
+
         for input_field in self.input_fields:
             input_field.update_studio_values(studio_values)
 

pype/tools/settings/settings/widgets/item_types.py

@@ -24,12 +24,32 @@ class SettingObject:
     # Will allow to show actions for the item type (disabled for proxies) else
     # item is skipped and try to trigger actions on it's parent.
     allow_actions = True
+    # If item can store environment values
+    allow_to_environment = False
     # All item types must have implemented Qt signal which is emitted when
     # it's or it's children value has changed,
     value_changed = None
     # Item will expand to full width in grid layout
     expand_in_grid = False
 
+    def merge_metadata(self, current_metadata, new_metadata):
+        for key, value in new_metadata.items():
+            if key not in current_metadata:
+                current_metadata[key] = value
+
+            elif key == "groups":
+                current_metadata[key].extend(value)
+
+            elif key == "environments":
+                for group_key, subvalue in value.items():
+                    if group_key not in current_metadata[key]:
+                        current_metadata[key][group_key] = []
+                    current_metadata[key][group_key].extend(subvalue)
+
+            else:
+                raise KeyError("Unknown metadata key: \"{}\"".format(key))
+        return current_metadata
+
     def _set_default_attributes(self):
         """Create and reset attributes required for all item types.
 
@@ -49,6 +69,9 @@ class SettingObject:
         self._as_widget = False
         self._is_group = False
 
+        # If value should be stored to environments
+        self._env_group_key = None
+
         self._any_parent_as_widget = None
         self._any_parent_is_group = None
 
@@ -84,6 +107,20 @@ class SettingObject:
         self._is_group = input_data.get("is_group", False)
         # TODO not implemented yet
         self._is_nullable = input_data.get("is_nullable", False)
+        self._env_group_key = input_data.get("env_group_key")
+
+        if self.is_environ:
+            if not self.allow_to_environment:
+                raise TypeError((
+                    "Item {} does not allow to store environment values"
+                ).format(input_data["type"]))
+
+            if self.as_widget:
+                raise TypeError((
+                    "Item is used as widget and"
+                    " marked to store environments at the same time."
+                ))
+            self.add_environ_field(self)
 
         any_parent_as_widget = parent.as_widget
         if not any_parent_as_widget:
@@ -140,6 +177,17 @@ class SettingObject:
         """
         return self._has_studio_override or self._parent.has_studio_override
 
+    @property
+    def is_environ(self):
+        return self._env_group_key is not None
+
+    @property
+    def env_group_key(self):
+        return self._env_group_key
+
+    def add_environ_field(self, input_field):
+        self._parent.add_environ_field(input_field)
+
     @property
     def as_widget(self):
         """Item is used as widget in parent item.
@@ -269,6 +317,13 @@ class SettingObject:
         """Output for saving changes or overrides."""
         return {self.key: self.item_value()}
 
+    def environment_value(self):
+        raise NotImplementedError(
+            "{} Method `environment_value` not implemented!".format(
+                repr(self)
+            )
+        )
+
     @classmethod
     def style_state(
         cls, has_studio_override, is_invalid, is_overriden, is_modified
@@ -1133,6 +1188,7 @@ class RawJsonInput(QtWidgets.QPlainTextEdit):
 class RawJsonWidget(QtWidgets.QWidget, InputObject):
     default_input_value = "{}"
     value_changed = QtCore.Signal(object)
+    allow_to_environment = True
 
     def __init__(
         self, input_data, parent,
@@ -1182,7 +1238,21 @@ class RawJsonWidget(QtWidgets.QWidget, InputObject):
     def item_value(self):
         if self.is_invalid:
             return NOT_SET
-        return self.input_field.json_value()
+
+        value = self.input_field.json_value()
+        if not self.is_environ:
+            return value
+
+        output = {}
+        for key, value in value.items():
+            output[key.upper()] = value
+
+        output[METADATA_KEY] = {
+            "environments": {
+                self.env_group_key: list(output.keys())
+            }
+        }
+        return output
 
 
 class ListItem(QtWidgets.QWidget, SettingObject):
@@ -2543,6 +2613,33 @@ class DictWidget(QtWidgets.QWidget, SettingObject):
             output.update(input_field.config_value())
         return output
 
+    def _override_values(self, project_overrides):
+        values = {}
+        groups = []
+        for input_field in self.input_fields:
+            if project_overrides:
+                value, is_group = input_field.overrides()
+            else:
+                value, is_group = input_field.studio_overrides()
+            if value is NOT_SET:
+                continue
+
+            if METADATA_KEY in value and METADATA_KEY in values:
+                new_metadata = value.pop(METADATA_KEY)
+                values[METADATA_KEY] = self.merge_metadata(
+                    values[METADATA_KEY], new_metadata
+                )
+
+            values.update(value)
+            if is_group:
+                groups.extend(value.keys())
+
+        if groups:
+            if METADATA_KEY not in values:
+                values[METADATA_KEY] = {}
+            values[METADATA_KEY]["groups"] = groups
+        return {self.key: values}, self.is_group
+
     def studio_overrides(self):
         if (
             not (self.as_widget or self.any_parent_as_widget)
@@ -2550,34 +2647,12 @@ class DictWidget(QtWidgets.QWidget, SettingObject):
             and not self.child_has_studio_override
         ):
             return NOT_SET, False
-
-        values = {}
-        groups = []
-        for input_field in self.input_fields:
-            value, is_group = input_field.studio_overrides()
-            if value is not NOT_SET:
-                values.update(value)
-                if is_group:
-                    groups.extend(value.keys())
-        if groups:
-            values[METADATA_KEY] = {"groups": groups}
-        return {self.key: values}, self.is_group
+        return self._override_values(False)
 
     def overrides(self):
         if not self.is_overriden and not self.child_overriden:
             return NOT_SET, False
-
-        values = {}
-        groups = []
-        for input_field in self.input_fields:
-            value, is_group = input_field.overrides()
-            if value is not NOT_SET:
-                values.update(value)
-                if is_group:
-                    groups.extend(value.keys())
-        if groups:
-            values[METADATA_KEY] = {"groups": groups}
-        return {self.key: values}, self.is_group
+        return self._override_values(True)
 
 
 class DictInvisible(QtWidgets.QWidget, SettingObject):
@@ -2792,6 +2867,33 @@ class DictInvisible(QtWidgets.QWidget, SettingObject):
             )
         self._was_overriden = bool(self._is_overriden)
 
+    def _override_values(self, project_overrides):
+        values = {}
+        groups = []
+        for input_field in self.input_fields:
+            if project_overrides:
+                value, is_group = input_field.overrides()
+            else:
+                value, is_group = input_field.studio_overrides()
+            if value is NOT_SET:
+                continue
+
+            if METADATA_KEY in value and METADATA_KEY in values:
+                new_metadata = value.pop(METADATA_KEY)
+                values[METADATA_KEY] = self.merge_metadata(
+                    values[METADATA_KEY], new_metadata
+                )
+
+            values.update(value)
+            if is_group:
+                groups.extend(value.keys())
+
+        if groups:
+            if METADATA_KEY not in values:
+                values[METADATA_KEY] = {}
+            values[METADATA_KEY]["groups"] = groups
+        return {self.key: values}, self.is_group
+
     def studio_overrides(self):
         if (
             not (self.as_widget or self.any_parent_as_widget)
@@ -2799,34 +2901,12 @@ class DictInvisible(QtWidgets.QWidget, SettingObject):
             and not self.child_has_studio_override
         ):
             return NOT_SET, False
-
-        values = {}
-        groups = []
-        for input_field in self.input_fields:
-            value, is_group = input_field.studio_overrides()
-            if value is not NOT_SET:
-                values.update(value)
-                if is_group:
-                    groups.extend(value.keys())
-        if groups:
-            values[METADATA_KEY] = {"groups": groups}
-        return {self.key: values}, self.is_group
+        return self._override_values(False)
 
     def overrides(self):
         if not self.is_overriden and not self.child_overriden:
             return NOT_SET, False
-
-        values = {}
-        groups = []
-        for input_field in self.input_fields:
-            value, is_group = input_field.overrides()
-            if value is not NOT_SET:
-                values.update(value)
-                if is_group:
-                    groups.extend(value.keys())
-        if groups:
-            values[METADATA_KEY] = {"groups": groups}
-        return {self.key: values}, self.is_group
+        return self._override_values(True)
 
 
 class PathWidget(QtWidgets.QWidget, SettingObject):

pype/tools/settings/settings/widgets/lib.py

@@ -1,7 +1,7 @@
 import os
 import json
 import copy
-from pype.settings.lib import OVERRIDEN_KEY
+from pype.settings.lib import M_OVERRIDEN_KEY, M_ENVIRONMENT_KEY
 from queue import Queue
 
 
@@ -11,11 +11,49 @@ class TypeToKlass:
 
 
 NOT_SET = type("NOT_SET", (), {"__bool__": lambda obj: False})()
-METADATA_KEY = type("METADATA_KEY", (), {})
+METADATA_KEY = type("METADATA_KEY", (), {})()
 OVERRIDE_VERSION = 1
 CHILD_OFFSET = 15
 
 
+def convert_gui_data_with_metadata(data, ignored_keys=None):
+    if not data or not isinstance(data, dict):
+        return data
+
+    if ignored_keys is None:
+        ignored_keys = tuple()
+
+    output = {}
+    if METADATA_KEY in data:
+        metadata = data.pop(METADATA_KEY)
+        for key, value in metadata.items():
+            if key in ignored_keys or key == "groups":
+                continue
+
+            if key == "environments":
+                output[M_ENVIRONMENT_KEY] = value
+            else:
+                raise KeyError("Unknown metadata key \"{}\"".format(key))
+
+    for key, value in data.items():
+        output[key] = convert_gui_data_with_metadata(value, ignored_keys)
+    return output
+
+
+def convert_data_to_gui_data(data, first=True):
+    if not data or not isinstance(data, dict):
+        return data
+
+    output = {}
+    if M_ENVIRONMENT_KEY in data:
+        data.pop(M_ENVIRONMENT_KEY)
+
+    for key, value in data.items():
+        output[key] = convert_data_to_gui_data(value, False)
+
+    return output
+
+
 def convert_gui_data_to_overrides(data, first=True):
     if not data or not isinstance(data, dict):
         return data
@@ -23,14 +61,15 @@ def convert_gui_data_to_overrides(data, first=True):
     output = {}
     if first:
         output["__override_version__"] = OVERRIDE_VERSION
+        data = convert_gui_data_with_metadata(data)
 
     if METADATA_KEY in data:
         metadata = data.pop(METADATA_KEY)
         for key, value in metadata.items():
             if key == "groups":
-                output[OVERRIDEN_KEY] = value
+                output[M_OVERRIDEN_KEY] = value
             else:
-                KeyError("Unknown metadata key \"{}\"".format(key))
+                raise KeyError("Unknown metadata key \"{}\"".format(key))
 
     for key, value in data.items():
         output[key] = convert_gui_data_to_overrides(value, False)
@@ -41,9 +80,12 @@ def convert_overrides_to_gui_data(data, first=True):
     if not data or not isinstance(data, dict):
         return data
 
+    if first:
+        data = convert_data_to_gui_data(data)
+
     output = {}
-    if OVERRIDEN_KEY in data:
-        groups = data.pop(OVERRIDEN_KEY)
+    if M_OVERRIDEN_KEY in data:
+        groups = data.pop(M_OVERRIDEN_KEY)
         if METADATA_KEY not in output:
             output[METADATA_KEY] = {}
         output[METADATA_KEY]["groups"] = groups
@@ -120,6 +162,21 @@ class SchemaDuplicatedKeys(Exception):
         super(SchemaDuplicatedKeys, self).__init__(msg)
 
 
+class SchemaDuplicatedEnvGroupKeys(Exception):
+    def __init__(self, invalid):
+        items = []
+        for key_path, keys in invalid.items():
+            joined_keys = ", ".join([
+                "\"{}\"".format(key) for key in keys
+            ])
+            items.append("\"{}\" ({})".format(key_path, joined_keys))
+
+        msg = (
+            "Schema items contain duplicated environment group keys. {}"
+        ).format(" || ".join(items))
+        super(SchemaDuplicatedEnvGroupKeys, self).__init__(msg)
+
+
 def file_keys_from_schema(schema_data):
     output = []
     item_type = schema_data["type"]
@@ -277,10 +334,50 @@ def validate_keys_are_unique(schema_data, keys=None):
         raise SchemaDuplicatedKeys(invalid)
 
 
+def validate_environment_groups_uniquenes(
+    schema_data, env_groups=None, keys=None
+):
+    is_first = False
+    if env_groups is None:
+        is_first = True
+        env_groups = {}
+        keys = []
+
+    my_keys = copy.deepcopy(keys)
+    key = schema_data.get("key")
+    if key:
+        my_keys.append(key)
+
+    env_group_key = schema_data.get("env_group_key")
+    if env_group_key:
+        if env_group_key not in env_groups:
+            env_groups[env_group_key] = []
+        env_groups[env_group_key].append("/".join(my_keys))
+
+    children = schema_data.get("children")
+    if not children:
+        return
+
+    for child in children:
+        validate_environment_groups_uniquenes(
+            child, env_groups, copy.deepcopy(my_keys)
+        )
+
+    if is_first:
+        invalid = {}
+        for env_group_key, key_paths in env_groups.items():
+            if len(key_paths) > 1:
+                invalid[env_group_key] = key_paths
+
+        if invalid:
+            raise SchemaDuplicatedEnvGroupKeys(invalid)
+
+
 def validate_schema(schema_data):
     validate_all_has_ending_file(schema_data)
     validate_is_group_is_unique_in_hierarchy(schema_data)
     validate_keys_are_unique(schema_data)
+    validate_environment_groups_uniquenes(schema_data)
 
 
 def gui_schema(subfolder, main_schema_name):