diff --git a/pype/settings/lib.py b/pype/settings/lib.py index 848bdeea92..96c3829388 100644 --- a/pype/settings/lib.py +++ b/pype/settings/lib.py @@ -6,9 +6,11 @@ import copy log = logging.getLogger(__name__) # Metadata keys for work with studio and project overrides -OVERRIDEN_KEY = "__overriden_keys__" +M_OVERRIDEN_KEY = "__overriden_keys__" +# Metadata key for storing information about environments +M_ENVIRONMENT_KEY = "__environment_keys__" # NOTE key popping not implemented yet -POP_KEY = "__pop_key__" +M_POP_KEY = "__pop_key__" # Folder where studio overrides are stored STUDIO_OVERRIDES_PATH = os.environ["PYPE_PROJECT_CONFIGS"] @@ -111,6 +113,32 @@ def load_json(fpath): return {} +def find_environments(data): + if not data or not isinstance(data, dict): + return + + output = {} + if M_ENVIRONMENT_KEY in data: + metadata = data.pop(M_ENVIRONMENT_KEY) + for env_group_key, env_keys in metadata.items(): + output[env_group_key] = {} + for key in env_keys: + output[env_group_key][key] = data[key] + + for value in data.values(): + result = find_environments(value) + if not result: + continue + + for env_group_key, env_values in result.items(): + if env_group_key not in output: + output[env_group_key] = {} + + for env_key, env_value in env_values.items(): + output[env_group_key][env_key] = env_value + return output + + def subkey_merge(_dict, value, keys): key = keys.pop(0) if not keys: @@ -223,13 +251,13 @@ def project_anatomy_overrides(project_name): def merge_overrides(global_dict, override_dict): - if OVERRIDEN_KEY in override_dict: - overriden_keys = set(override_dict.pop(OVERRIDEN_KEY)) + if M_OVERRIDEN_KEY in override_dict: + overriden_keys = set(override_dict.pop(M_OVERRIDEN_KEY)) else: overriden_keys = set() for key, value in override_dict.items(): - if value == POP_KEY: + if value == M_POP_KEY: global_dict.pop(key) elif ( @@ -271,6 +299,8 @@ def project_settings(project_name): def environments(): - default_values = default_settings()[ENVIRONMENTS_KEY] - studio_values = studio_system_settings() - return apply_overrides(default_values, studio_values) + envs = copy.deepcopy(default_settings()[ENVIRONMENTS_KEY]) + envs_from_system_settings = find_environments(system_settings()) + for env_group_key, values in envs_from_system_settings.items(): + envs[env_group_key] = values + return envs diff --git a/pype/tools/settings/settings/gui_schemas/system_schema/1_examples.json b/pype/tools/settings/settings/gui_schemas/system_schema/1_examples.json index dd0f7f20d1..06ce23321a 100644 --- a/pype/tools/settings/settings/gui_schemas/system_schema/1_examples.json +++ b/pype/tools/settings/settings/gui_schemas/system_schema/1_examples.json @@ -5,6 +5,18 @@ "is_file": true, "children": [ { + "key": "env_group_test", + "label": "EnvGroup Test", + "type": "dict", + "children": [ + { + "key": "key_to_store_in_system_settings", + "label": "Testing environment group", + "type": "raw-json", + "env_group_key": "test_group" + } + ] + }, { "key": "dict_wrapper", "type": "dict-invisible", "children": [ diff --git a/pype/tools/settings/settings/widgets/base.py b/pype/tools/settings/settings/widgets/base.py index 243a8448e5..7cbe7c2f6f 100644 --- a/pype/tools/settings/settings/widgets/base.py +++ b/pype/tools/settings/settings/widgets/base.py @@ -47,6 +47,7 @@ class SystemWidget(QtWidgets.QWidget): self._ignore_value_changes = False self.input_fields = [] + self.environ_fields = [] scroll_widget = QtWidgets.QScrollArea(self) scroll_widget.setObjectName("GroupWidget") @@ -130,10 +131,14 @@ class SystemWidget(QtWidgets.QWidget): for input_field in self.input_fields: input_field.hierarchical_style_update() + def add_environ_field(self, input_field): + self.environ_fields.append(input_field) + def reset(self): reset_default_settings() self.input_fields.clear() + self.environ_fields.clear() while self.content_layout.count() != 0: widget = self.content_layout.itemAt(0).widget() self.content_layout.removeWidget(widget) @@ -214,7 +219,7 @@ class SystemWidget(QtWidgets.QWidget): all_values = _all_values # Skip first key - all_values = all_values["system"] + all_values = lib.convert_gui_data_with_metadata(all_values["system"]) prject_defaults_dir = os.path.join( DEFAULTS_DIR, SYSTEM_SETTINGS_KEY @@ -246,16 +251,19 @@ class SystemWidget(QtWidgets.QWidget): def _update_values(self): self.ignore_value_changes = True - default_values = { + default_values = lib.convert_data_to_gui_data({ "system": default_settings()[SYSTEM_SETTINGS_KEY] - } + }) for input_field in self.input_fields: input_field.update_default_values(default_values) if self._hide_studio_overrides: system_values = lib.NOT_SET else: - system_values = {"system": studio_system_settings()} + system_values = lib.convert_overrides_to_gui_data( + {"system": studio_system_settings()} + ) + for input_field in self.input_fields: input_field.update_studio_values(system_values) @@ -730,17 +738,20 @@ class ProjectWidget(QtWidgets.QWidget): def _update_values(self): self.ignore_value_changes = True - default_values = {"project": default_settings()} + default_values = default_values = lib.convert_data_to_gui_data( + {"project": default_settings()} + ) for input_field in self.input_fields: input_field.update_default_values(default_values) if self._hide_studio_overrides: studio_values = lib.NOT_SET else: - studio_values = {"project": { + studio_values = lib.convert_overrides_to_gui_data({"project": { PROJECT_SETTINGS_KEY: studio_project_settings(), PROJECT_ANATOMY_KEY: studio_project_anatomy() - }} + }}) + for input_field in self.input_fields: input_field.update_studio_values(studio_values) diff --git a/pype/tools/settings/settings/widgets/item_types.py b/pype/tools/settings/settings/widgets/item_types.py index 4124c32ba8..1127d611d7 100644 --- a/pype/tools/settings/settings/widgets/item_types.py +++ b/pype/tools/settings/settings/widgets/item_types.py @@ -24,12 +24,32 @@ class SettingObject: # Will allow to show actions for the item type (disabled for proxies) else # item is skipped and try to trigger actions on it's parent. allow_actions = True + # If item can store environment values + allow_to_environment = False # All item types must have implemented Qt signal which is emitted when # it's or it's children value has changed, value_changed = None # Item will expand to full width in grid layout expand_in_grid = False + def merge_metadata(self, current_metadata, new_metadata): + for key, value in new_metadata.items(): + if key not in current_metadata: + current_metadata[key] = value + + elif key == "groups": + current_metadata[key].extend(value) + + elif key == "environments": + for group_key, subvalue in value.items(): + if group_key not in current_metadata[key]: + current_metadata[key][group_key] = [] + current_metadata[key][group_key].extend(subvalue) + + else: + raise KeyError("Unknown metadata key: \"{}\"".format(key)) + return current_metadata + def _set_default_attributes(self): """Create and reset attributes required for all item types. @@ -49,6 +69,9 @@ class SettingObject: self._as_widget = False self._is_group = False + # If value should be stored to environments + self._env_group_key = None + self._any_parent_as_widget = None self._any_parent_is_group = None @@ -84,6 +107,20 @@ class SettingObject: self._is_group = input_data.get("is_group", False) # TODO not implemented yet self._is_nullable = input_data.get("is_nullable", False) + self._env_group_key = input_data.get("env_group_key") + + if self.is_environ: + if not self.allow_to_environment: + raise TypeError(( + "Item {} does not allow to store environment values" + ).format(input_data["type"])) + + if self.as_widget: + raise TypeError(( + "Item is used as widget and" + " marked to store environments at the same time." + )) + self.add_environ_field(self) any_parent_as_widget = parent.as_widget if not any_parent_as_widget: @@ -140,6 +177,17 @@ class SettingObject: """ return self._has_studio_override or self._parent.has_studio_override + @property + def is_environ(self): + return self._env_group_key is not None + + @property + def env_group_key(self): + return self._env_group_key + + def add_environ_field(self, input_field): + self._parent.add_environ_field(input_field) + @property def as_widget(self): """Item is used as widget in parent item. @@ -269,6 +317,13 @@ class SettingObject: """Output for saving changes or overrides.""" return {self.key: self.item_value()} + def environment_value(self): + raise NotImplementedError( + "{} Method `environment_value` not implemented!".format( + repr(self) + ) + ) + @classmethod def style_state( cls, has_studio_override, is_invalid, is_overriden, is_modified @@ -1133,6 +1188,7 @@ class RawJsonInput(QtWidgets.QPlainTextEdit): class RawJsonWidget(QtWidgets.QWidget, InputObject): default_input_value = "{}" value_changed = QtCore.Signal(object) + allow_to_environment = True def __init__( self, input_data, parent, @@ -1182,7 +1238,21 @@ class RawJsonWidget(QtWidgets.QWidget, InputObject): def item_value(self): if self.is_invalid: return NOT_SET - return self.input_field.json_value() + + value = self.input_field.json_value() + if not self.is_environ: + return value + + output = {} + for key, value in value.items(): + output[key.upper()] = value + + output[METADATA_KEY] = { + "environments": { + self.env_group_key: list(output.keys()) + } + } + return output class ListItem(QtWidgets.QWidget, SettingObject): @@ -2543,6 +2613,33 @@ class DictWidget(QtWidgets.QWidget, SettingObject): output.update(input_field.config_value()) return output + def _override_values(self, project_overrides): + values = {} + groups = [] + for input_field in self.input_fields: + if project_overrides: + value, is_group = input_field.overrides() + else: + value, is_group = input_field.studio_overrides() + if value is NOT_SET: + continue + + if METADATA_KEY in value and METADATA_KEY in values: + new_metadata = value.pop(METADATA_KEY) + values[METADATA_KEY] = self.merge_metadata( + values[METADATA_KEY], new_metadata + ) + + values.update(value) + if is_group: + groups.extend(value.keys()) + + if groups: + if METADATA_KEY not in values: + values[METADATA_KEY] = {} + values[METADATA_KEY]["groups"] = groups + return {self.key: values}, self.is_group + def studio_overrides(self): if ( not (self.as_widget or self.any_parent_as_widget) @@ -2550,34 +2647,12 @@ class DictWidget(QtWidgets.QWidget, SettingObject): and not self.child_has_studio_override ): return NOT_SET, False - - values = {} - groups = [] - for input_field in self.input_fields: - value, is_group = input_field.studio_overrides() - if value is not NOT_SET: - values.update(value) - if is_group: - groups.extend(value.keys()) - if groups: - values[METADATA_KEY] = {"groups": groups} - return {self.key: values}, self.is_group + return self._override_values(False) def overrides(self): if not self.is_overriden and not self.child_overriden: return NOT_SET, False - - values = {} - groups = [] - for input_field in self.input_fields: - value, is_group = input_field.overrides() - if value is not NOT_SET: - values.update(value) - if is_group: - groups.extend(value.keys()) - if groups: - values[METADATA_KEY] = {"groups": groups} - return {self.key: values}, self.is_group + return self._override_values(True) class DictInvisible(QtWidgets.QWidget, SettingObject): @@ -2792,6 +2867,33 @@ class DictInvisible(QtWidgets.QWidget, SettingObject): ) self._was_overriden = bool(self._is_overriden) + def _override_values(self, project_overrides): + values = {} + groups = [] + for input_field in self.input_fields: + if project_overrides: + value, is_group = input_field.overrides() + else: + value, is_group = input_field.studio_overrides() + if value is NOT_SET: + continue + + if METADATA_KEY in value and METADATA_KEY in values: + new_metadata = value.pop(METADATA_KEY) + values[METADATA_KEY] = self.merge_metadata( + values[METADATA_KEY], new_metadata + ) + + values.update(value) + if is_group: + groups.extend(value.keys()) + + if groups: + if METADATA_KEY not in values: + values[METADATA_KEY] = {} + values[METADATA_KEY]["groups"] = groups + return {self.key: values}, self.is_group + def studio_overrides(self): if ( not (self.as_widget or self.any_parent_as_widget) @@ -2799,34 +2901,12 @@ class DictInvisible(QtWidgets.QWidget, SettingObject): and not self.child_has_studio_override ): return NOT_SET, False - - values = {} - groups = [] - for input_field in self.input_fields: - value, is_group = input_field.studio_overrides() - if value is not NOT_SET: - values.update(value) - if is_group: - groups.extend(value.keys()) - if groups: - values[METADATA_KEY] = {"groups": groups} - return {self.key: values}, self.is_group + return self._override_values(False) def overrides(self): if not self.is_overriden and not self.child_overriden: return NOT_SET, False - - values = {} - groups = [] - for input_field in self.input_fields: - value, is_group = input_field.overrides() - if value is not NOT_SET: - values.update(value) - if is_group: - groups.extend(value.keys()) - if groups: - values[METADATA_KEY] = {"groups": groups} - return {self.key: values}, self.is_group + return self._override_values(True) class PathWidget(QtWidgets.QWidget, SettingObject): diff --git a/pype/tools/settings/settings/widgets/lib.py b/pype/tools/settings/settings/widgets/lib.py index cf2bd7f8af..87f6554612 100644 --- a/pype/tools/settings/settings/widgets/lib.py +++ b/pype/tools/settings/settings/widgets/lib.py @@ -1,7 +1,7 @@ import os import json import copy -from pype.settings.lib import OVERRIDEN_KEY +from pype.settings.lib import M_OVERRIDEN_KEY, M_ENVIRONMENT_KEY from queue import Queue @@ -11,11 +11,49 @@ class TypeToKlass: NOT_SET = type("NOT_SET", (), {"__bool__": lambda obj: False})() -METADATA_KEY = type("METADATA_KEY", (), {}) +METADATA_KEY = type("METADATA_KEY", (), {})() OVERRIDE_VERSION = 1 CHILD_OFFSET = 15 +def convert_gui_data_with_metadata(data, ignored_keys=None): + if not data or not isinstance(data, dict): + return data + + if ignored_keys is None: + ignored_keys = tuple() + + output = {} + if METADATA_KEY in data: + metadata = data.pop(METADATA_KEY) + for key, value in metadata.items(): + if key in ignored_keys or key == "groups": + continue + + if key == "environments": + output[M_ENVIRONMENT_KEY] = value + else: + raise KeyError("Unknown metadata key \"{}\"".format(key)) + + for key, value in data.items(): + output[key] = convert_gui_data_with_metadata(value, ignored_keys) + return output + + +def convert_data_to_gui_data(data, first=True): + if not data or not isinstance(data, dict): + return data + + output = {} + if M_ENVIRONMENT_KEY in data: + data.pop(M_ENVIRONMENT_KEY) + + for key, value in data.items(): + output[key] = convert_data_to_gui_data(value, False) + + return output + + def convert_gui_data_to_overrides(data, first=True): if not data or not isinstance(data, dict): return data @@ -23,14 +61,15 @@ def convert_gui_data_to_overrides(data, first=True): output = {} if first: output["__override_version__"] = OVERRIDE_VERSION + data = convert_gui_data_with_metadata(data) if METADATA_KEY in data: metadata = data.pop(METADATA_KEY) for key, value in metadata.items(): if key == "groups": - output[OVERRIDEN_KEY] = value + output[M_OVERRIDEN_KEY] = value else: - KeyError("Unknown metadata key \"{}\"".format(key)) + raise KeyError("Unknown metadata key \"{}\"".format(key)) for key, value in data.items(): output[key] = convert_gui_data_to_overrides(value, False) @@ -41,9 +80,12 @@ def convert_overrides_to_gui_data(data, first=True): if not data or not isinstance(data, dict): return data + if first: + data = convert_data_to_gui_data(data) + output = {} - if OVERRIDEN_KEY in data: - groups = data.pop(OVERRIDEN_KEY) + if M_OVERRIDEN_KEY in data: + groups = data.pop(M_OVERRIDEN_KEY) if METADATA_KEY not in output: output[METADATA_KEY] = {} output[METADATA_KEY]["groups"] = groups @@ -120,6 +162,21 @@ class SchemaDuplicatedKeys(Exception): super(SchemaDuplicatedKeys, self).__init__(msg) +class SchemaDuplicatedEnvGroupKeys(Exception): + def __init__(self, invalid): + items = [] + for key_path, keys in invalid.items(): + joined_keys = ", ".join([ + "\"{}\"".format(key) for key in keys + ]) + items.append("\"{}\" ({})".format(key_path, joined_keys)) + + msg = ( + "Schema items contain duplicated environment group keys. {}" + ).format(" || ".join(items)) + super(SchemaDuplicatedEnvGroupKeys, self).__init__(msg) + + def file_keys_from_schema(schema_data): output = [] item_type = schema_data["type"] @@ -277,10 +334,50 @@ def validate_keys_are_unique(schema_data, keys=None): raise SchemaDuplicatedKeys(invalid) +def validate_environment_groups_uniquenes( + schema_data, env_groups=None, keys=None +): + is_first = False + if env_groups is None: + is_first = True + env_groups = {} + keys = [] + + my_keys = copy.deepcopy(keys) + key = schema_data.get("key") + if key: + my_keys.append(key) + + env_group_key = schema_data.get("env_group_key") + if env_group_key: + if env_group_key not in env_groups: + env_groups[env_group_key] = [] + env_groups[env_group_key].append("/".join(my_keys)) + + children = schema_data.get("children") + if not children: + return + + for child in children: + validate_environment_groups_uniquenes( + child, env_groups, copy.deepcopy(my_keys) + ) + + if is_first: + invalid = {} + for env_group_key, key_paths in env_groups.items(): + if len(key_paths) > 1: + invalid[env_group_key] = key_paths + + if invalid: + raise SchemaDuplicatedEnvGroupKeys(invalid) + + def validate_schema(schema_data): validate_all_has_ending_file(schema_data) validate_is_group_is_unique_in_hierarchy(schema_data) validate_keys_are_unique(schema_data) + validate_environment_groups_uniquenes(schema_data) def gui_schema(subfolder, main_schema_name):