diff --git a/pype/resources/app_icons/hiero.png b/pype/resources/app_icons/hiero.png new file mode 100644 index 0000000000..04bbf6265b Binary files /dev/null and b/pype/resources/app_icons/hiero.png differ diff --git a/pype/settings/lib.py b/pype/settings/lib.py index 848bdeea92..96c3829388 100644 --- a/pype/settings/lib.py +++ b/pype/settings/lib.py @@ -6,9 +6,11 @@ import copy log = logging.getLogger(__name__) # Metadata keys for work with studio and project overrides -OVERRIDEN_KEY = "__overriden_keys__" +M_OVERRIDEN_KEY = "__overriden_keys__" +# Metadata key for storing information about environments +M_ENVIRONMENT_KEY = "__environment_keys__" # NOTE key popping not implemented yet -POP_KEY = "__pop_key__" +M_POP_KEY = "__pop_key__" # Folder where studio overrides are stored STUDIO_OVERRIDES_PATH = os.environ["PYPE_PROJECT_CONFIGS"] @@ -111,6 +113,32 @@ def load_json(fpath): return {} +def find_environments(data): + if not data or not isinstance(data, dict): + return + + output = {} + if M_ENVIRONMENT_KEY in data: + metadata = data.pop(M_ENVIRONMENT_KEY) + for env_group_key, env_keys in metadata.items(): + output[env_group_key] = {} + for key in env_keys: + output[env_group_key][key] = data[key] + + for value in data.values(): + result = find_environments(value) + if not result: + continue + + for env_group_key, env_values in result.items(): + if env_group_key not in output: + output[env_group_key] = {} + + for env_key, env_value in env_values.items(): + output[env_group_key][env_key] = env_value + return output + + def subkey_merge(_dict, value, keys): key = keys.pop(0) if not keys: @@ -223,13 +251,13 @@ def project_anatomy_overrides(project_name): def merge_overrides(global_dict, override_dict): - if OVERRIDEN_KEY in override_dict: - overriden_keys = set(override_dict.pop(OVERRIDEN_KEY)) + if M_OVERRIDEN_KEY in override_dict: + overriden_keys = set(override_dict.pop(M_OVERRIDEN_KEY)) else: overriden_keys = set() for key, value in override_dict.items(): - if value == POP_KEY: + if value == M_POP_KEY: global_dict.pop(key) elif ( @@ -271,6 +299,8 @@ def project_settings(project_name): def environments(): - default_values = default_settings()[ENVIRONMENTS_KEY] - studio_values = studio_system_settings() - return apply_overrides(default_values, studio_values) + envs = copy.deepcopy(default_settings()[ENVIRONMENTS_KEY]) + envs_from_system_settings = find_environments(system_settings()) + for env_group_key, values in envs_from_system_settings.items(): + envs[env_group_key] = values + return envs diff --git a/pype/tools/settings/settings/README.md b/pype/tools/settings/settings/README.md index e8b7fcdb57..4f4e9d305a 100644 --- a/pype/tools/settings/settings/README.md +++ b/pype/tools/settings/settings/README.md @@ -19,6 +19,7 @@ - GUI schemas are huge json files, to be able to split whole configuration into multiple schema there's type `schema` - system configuration schemas are stored in `~/tools/settings/settings/gui_schemas/system_schema/` and project configurations in `~/tools/settings/settings/gui_schemas/projects_schema/` - each schema name is filename of json file except extension (without ".json") +- if content is dictionary content will be used as `schema` else will be used as `schema_template` ### schema - can have only key `"children"` which is list of strings, each string should represent another schema (order matters) string represebts name of the schema @@ -31,6 +32,87 @@ } ``` +### schema_template +- allows to define schema "templates" to not duplicate same content multiple times +```javascript +// EXAMPLE json file content (filename: example_template.json) +[ + { + "__default_values__": { + "multipath_executables": true + } + }, { + "type": "raw-json", + "label": "{host_label} Environments", + "key": "{host_name}_environments", + "env_group_key": "{host_name}" + }, { + "type": "path-widget", + "key": "{host_name}_executables", + "label": "{host_label} - Full paths to executables", + "multiplatform": "{multipath_executables}", + "multipath": true + } +] +``` +```javascript +// EXAMPLE usage of the template in schema +{ + "type": "dict", + "key": "schema_template_examples", + "label": "Schema template examples", + "children": [ + { + "type": "schema_template", + // filename of template (example_template.json) + "name": "example_template", + "template_data": { + "host_label": "Maya 2019", + "host_name": "maya_2019", + "multipath_executables": false + } + }, { + "type": "schema_template", + "name": "example_template", + "template_data": { + "host_label": "Maya 2020", + "host_name": "maya_2020" + } + } + ] +} +``` +- item in schema mush contain `"type"` and `"name"` keys but it is also expected that `"template_data"` will be entered too +- all items in the list, except `__default_values__`, will replace `schema_template` item in schema +- template may contain another template or schema +- it is expected that schema template will have unfilled fields as in example + - unfilled fields are allowed only in values of schema dictionary +```javascript +{ + ... + // Allowed + "key": "{to_fill}" + ... + // Not allowed + "{to_fill}": "value" + ... +} +``` +- Unfilled fields can be also used for non string values, in that case value must contain only one key and value for fill must contain right type. +```javascript +{ + ... + // Allowed + "multiplatform": "{executable_multiplatform}" + ... + // Not allowed + "multiplatform": "{executable_multiplatform}_enhanced_string" + ... +} +``` +- It is possible to define default values for unfilled fields to do so one of items in list must be dictionary with key `"__default_values__"` and value as dictionary with default key: values (as in example above). + + ## Basic Dictionary inputs - these inputs wraps another inputs into {key: value} relation diff --git a/pype/tools/settings/settings/gui_schemas/system_schema/1_examples.json b/pype/tools/settings/settings/gui_schemas/system_schema/example_schema.json similarity index 93% rename from pype/tools/settings/settings/gui_schemas/system_schema/1_examples.json rename to pype/tools/settings/settings/gui_schemas/system_schema/example_schema.json index dd0f7f20d1..7612e54116 100644 --- a/pype/tools/settings/settings/gui_schemas/system_schema/1_examples.json +++ b/pype/tools/settings/settings/gui_schemas/system_schema/example_schema.json @@ -5,6 +5,40 @@ "is_file": true, "children": [ { + "type": "dict", + "key": "schema_template_exaples", + "label": "Schema template examples", + "children": [ + { + "type": "schema_template", + "name": "example_template", + "template_data": { + "host_label": "Maya 2019", + "host_name": "maya_2019", + "multipath_executables": false + } + }, { + "type": "schema_template", + "name": "example_template", + "template_data": { + "host_label": "Maya 2020", + "host_name": "maya_2020" + } + } + ] + }, { + "key": "env_group_test", + "label": "EnvGroup Test", + "type": "dict", + "children": [ + { + "key": "key_to_store_in_system_settings", + "label": "Testing environment group", + "type": "raw-json", + "env_group_key": "test_group" + } + ] + }, { "key": "dict_wrapper", "type": "dict-invisible", "children": [ diff --git a/pype/tools/settings/settings/gui_schemas/system_schema/example_template.json b/pype/tools/settings/settings/gui_schemas/system_schema/example_template.json new file mode 100644 index 0000000000..48a3c955b9 --- /dev/null +++ b/pype/tools/settings/settings/gui_schemas/system_schema/example_template.json @@ -0,0 +1,18 @@ +[ + { + "__default_values__": { + "multipath_executables": true + } + }, { + "type": "raw-json", + "label": "{host_label} Environments", + "key": "{host_name}_environments", + "env_group_key": "{host_name}" + }, { + "type": "path-widget", + "key": "{host_name}_executables", + "label": "{host_label} - Full paths to executables", + "multiplatform": "{multipath_executables}", + "multipath": true + } +] diff --git a/pype/tools/settings/settings/widgets/base.py b/pype/tools/settings/settings/widgets/base.py index 243a8448e5..7cbe7c2f6f 100644 --- a/pype/tools/settings/settings/widgets/base.py +++ b/pype/tools/settings/settings/widgets/base.py @@ -47,6 +47,7 @@ class SystemWidget(QtWidgets.QWidget): self._ignore_value_changes = False self.input_fields = [] + self.environ_fields = [] scroll_widget = QtWidgets.QScrollArea(self) scroll_widget.setObjectName("GroupWidget") @@ -130,10 +131,14 @@ class SystemWidget(QtWidgets.QWidget): for input_field in self.input_fields: input_field.hierarchical_style_update() + def add_environ_field(self, input_field): + self.environ_fields.append(input_field) + def reset(self): reset_default_settings() self.input_fields.clear() + self.environ_fields.clear() while self.content_layout.count() != 0: widget = self.content_layout.itemAt(0).widget() self.content_layout.removeWidget(widget) @@ -214,7 +219,7 @@ class SystemWidget(QtWidgets.QWidget): all_values = _all_values # Skip first key - all_values = all_values["system"] + all_values = lib.convert_gui_data_with_metadata(all_values["system"]) prject_defaults_dir = os.path.join( DEFAULTS_DIR, SYSTEM_SETTINGS_KEY @@ -246,16 +251,19 @@ class SystemWidget(QtWidgets.QWidget): def _update_values(self): self.ignore_value_changes = True - default_values = { + default_values = lib.convert_data_to_gui_data({ "system": default_settings()[SYSTEM_SETTINGS_KEY] - } + }) for input_field in self.input_fields: input_field.update_default_values(default_values) if self._hide_studio_overrides: system_values = lib.NOT_SET else: - system_values = {"system": studio_system_settings()} + system_values = lib.convert_overrides_to_gui_data( + {"system": studio_system_settings()} + ) + for input_field in self.input_fields: input_field.update_studio_values(system_values) @@ -730,17 +738,20 @@ class ProjectWidget(QtWidgets.QWidget): def _update_values(self): self.ignore_value_changes = True - default_values = {"project": default_settings()} + default_values = default_values = lib.convert_data_to_gui_data( + {"project": default_settings()} + ) for input_field in self.input_fields: input_field.update_default_values(default_values) if self._hide_studio_overrides: studio_values = lib.NOT_SET else: - studio_values = {"project": { + studio_values = lib.convert_overrides_to_gui_data({"project": { PROJECT_SETTINGS_KEY: studio_project_settings(), PROJECT_ANATOMY_KEY: studio_project_anatomy() - }} + }}) + for input_field in self.input_fields: input_field.update_studio_values(studio_values) diff --git a/pype/tools/settings/settings/widgets/item_types.py b/pype/tools/settings/settings/widgets/item_types.py index 4124c32ba8..1127d611d7 100644 --- a/pype/tools/settings/settings/widgets/item_types.py +++ b/pype/tools/settings/settings/widgets/item_types.py @@ -24,12 +24,32 @@ class SettingObject: # Will allow to show actions for the item type (disabled for proxies) else # item is skipped and try to trigger actions on it's parent. allow_actions = True + # If item can store environment values + allow_to_environment = False # All item types must have implemented Qt signal which is emitted when # it's or it's children value has changed, value_changed = None # Item will expand to full width in grid layout expand_in_grid = False + def merge_metadata(self, current_metadata, new_metadata): + for key, value in new_metadata.items(): + if key not in current_metadata: + current_metadata[key] = value + + elif key == "groups": + current_metadata[key].extend(value) + + elif key == "environments": + for group_key, subvalue in value.items(): + if group_key not in current_metadata[key]: + current_metadata[key][group_key] = [] + current_metadata[key][group_key].extend(subvalue) + + else: + raise KeyError("Unknown metadata key: \"{}\"".format(key)) + return current_metadata + def _set_default_attributes(self): """Create and reset attributes required for all item types. @@ -49,6 +69,9 @@ class SettingObject: self._as_widget = False self._is_group = False + # If value should be stored to environments + self._env_group_key = None + self._any_parent_as_widget = None self._any_parent_is_group = None @@ -84,6 +107,20 @@ class SettingObject: self._is_group = input_data.get("is_group", False) # TODO not implemented yet self._is_nullable = input_data.get("is_nullable", False) + self._env_group_key = input_data.get("env_group_key") + + if self.is_environ: + if not self.allow_to_environment: + raise TypeError(( + "Item {} does not allow to store environment values" + ).format(input_data["type"])) + + if self.as_widget: + raise TypeError(( + "Item is used as widget and" + " marked to store environments at the same time." + )) + self.add_environ_field(self) any_parent_as_widget = parent.as_widget if not any_parent_as_widget: @@ -140,6 +177,17 @@ class SettingObject: """ return self._has_studio_override or self._parent.has_studio_override + @property + def is_environ(self): + return self._env_group_key is not None + + @property + def env_group_key(self): + return self._env_group_key + + def add_environ_field(self, input_field): + self._parent.add_environ_field(input_field) + @property def as_widget(self): """Item is used as widget in parent item. @@ -269,6 +317,13 @@ class SettingObject: """Output for saving changes or overrides.""" return {self.key: self.item_value()} + def environment_value(self): + raise NotImplementedError( + "{} Method `environment_value` not implemented!".format( + repr(self) + ) + ) + @classmethod def style_state( cls, has_studio_override, is_invalid, is_overriden, is_modified @@ -1133,6 +1188,7 @@ class RawJsonInput(QtWidgets.QPlainTextEdit): class RawJsonWidget(QtWidgets.QWidget, InputObject): default_input_value = "{}" value_changed = QtCore.Signal(object) + allow_to_environment = True def __init__( self, input_data, parent, @@ -1182,7 +1238,21 @@ class RawJsonWidget(QtWidgets.QWidget, InputObject): def item_value(self): if self.is_invalid: return NOT_SET - return self.input_field.json_value() + + value = self.input_field.json_value() + if not self.is_environ: + return value + + output = {} + for key, value in value.items(): + output[key.upper()] = value + + output[METADATA_KEY] = { + "environments": { + self.env_group_key: list(output.keys()) + } + } + return output class ListItem(QtWidgets.QWidget, SettingObject): @@ -2543,6 +2613,33 @@ class DictWidget(QtWidgets.QWidget, SettingObject): output.update(input_field.config_value()) return output + def _override_values(self, project_overrides): + values = {} + groups = [] + for input_field in self.input_fields: + if project_overrides: + value, is_group = input_field.overrides() + else: + value, is_group = input_field.studio_overrides() + if value is NOT_SET: + continue + + if METADATA_KEY in value and METADATA_KEY in values: + new_metadata = value.pop(METADATA_KEY) + values[METADATA_KEY] = self.merge_metadata( + values[METADATA_KEY], new_metadata + ) + + values.update(value) + if is_group: + groups.extend(value.keys()) + + if groups: + if METADATA_KEY not in values: + values[METADATA_KEY] = {} + values[METADATA_KEY]["groups"] = groups + return {self.key: values}, self.is_group + def studio_overrides(self): if ( not (self.as_widget or self.any_parent_as_widget) @@ -2550,34 +2647,12 @@ class DictWidget(QtWidgets.QWidget, SettingObject): and not self.child_has_studio_override ): return NOT_SET, False - - values = {} - groups = [] - for input_field in self.input_fields: - value, is_group = input_field.studio_overrides() - if value is not NOT_SET: - values.update(value) - if is_group: - groups.extend(value.keys()) - if groups: - values[METADATA_KEY] = {"groups": groups} - return {self.key: values}, self.is_group + return self._override_values(False) def overrides(self): if not self.is_overriden and not self.child_overriden: return NOT_SET, False - - values = {} - groups = [] - for input_field in self.input_fields: - value, is_group = input_field.overrides() - if value is not NOT_SET: - values.update(value) - if is_group: - groups.extend(value.keys()) - if groups: - values[METADATA_KEY] = {"groups": groups} - return {self.key: values}, self.is_group + return self._override_values(True) class DictInvisible(QtWidgets.QWidget, SettingObject): @@ -2792,6 +2867,33 @@ class DictInvisible(QtWidgets.QWidget, SettingObject): ) self._was_overriden = bool(self._is_overriden) + def _override_values(self, project_overrides): + values = {} + groups = [] + for input_field in self.input_fields: + if project_overrides: + value, is_group = input_field.overrides() + else: + value, is_group = input_field.studio_overrides() + if value is NOT_SET: + continue + + if METADATA_KEY in value and METADATA_KEY in values: + new_metadata = value.pop(METADATA_KEY) + values[METADATA_KEY] = self.merge_metadata( + values[METADATA_KEY], new_metadata + ) + + values.update(value) + if is_group: + groups.extend(value.keys()) + + if groups: + if METADATA_KEY not in values: + values[METADATA_KEY] = {} + values[METADATA_KEY]["groups"] = groups + return {self.key: values}, self.is_group + def studio_overrides(self): if ( not (self.as_widget or self.any_parent_as_widget) @@ -2799,34 +2901,12 @@ class DictInvisible(QtWidgets.QWidget, SettingObject): and not self.child_has_studio_override ): return NOT_SET, False - - values = {} - groups = [] - for input_field in self.input_fields: - value, is_group = input_field.studio_overrides() - if value is not NOT_SET: - values.update(value) - if is_group: - groups.extend(value.keys()) - if groups: - values[METADATA_KEY] = {"groups": groups} - return {self.key: values}, self.is_group + return self._override_values(False) def overrides(self): if not self.is_overriden and not self.child_overriden: return NOT_SET, False - - values = {} - groups = [] - for input_field in self.input_fields: - value, is_group = input_field.overrides() - if value is not NOT_SET: - values.update(value) - if is_group: - groups.extend(value.keys()) - if groups: - values[METADATA_KEY] = {"groups": groups} - return {self.key: values}, self.is_group + return self._override_values(True) class PathWidget(QtWidgets.QWidget, SettingObject): diff --git a/pype/tools/settings/settings/widgets/lib.py b/pype/tools/settings/settings/widgets/lib.py index cf2bd7f8af..569e7bfbb7 100644 --- a/pype/tools/settings/settings/widgets/lib.py +++ b/pype/tools/settings/settings/widgets/lib.py @@ -1,7 +1,8 @@ import os +import re import json import copy -from pype.settings.lib import OVERRIDEN_KEY +from pype.settings.lib import M_OVERRIDEN_KEY, M_ENVIRONMENT_KEY from queue import Queue @@ -11,10 +12,50 @@ class TypeToKlass: NOT_SET = type("NOT_SET", (), {"__bool__": lambda obj: False})() -METADATA_KEY = type("METADATA_KEY", (), {}) +METADATA_KEY = type("METADATA_KEY", (), {})() OVERRIDE_VERSION = 1 CHILD_OFFSET = 15 +key_pattern = re.compile(r"(\{.*?[^{0]*\})") + + +def convert_gui_data_with_metadata(data, ignored_keys=None): + if not data or not isinstance(data, dict): + return data + + if ignored_keys is None: + ignored_keys = tuple() + + output = {} + if METADATA_KEY in data: + metadata = data.pop(METADATA_KEY) + for key, value in metadata.items(): + if key in ignored_keys or key == "groups": + continue + + if key == "environments": + output[M_ENVIRONMENT_KEY] = value + else: + raise KeyError("Unknown metadata key \"{}\"".format(key)) + + for key, value in data.items(): + output[key] = convert_gui_data_with_metadata(value, ignored_keys) + return output + + +def convert_data_to_gui_data(data, first=True): + if not data or not isinstance(data, dict): + return data + + output = {} + if M_ENVIRONMENT_KEY in data: + data.pop(M_ENVIRONMENT_KEY) + + for key, value in data.items(): + output[key] = convert_data_to_gui_data(value, False) + + return output + def convert_gui_data_to_overrides(data, first=True): if not data or not isinstance(data, dict): @@ -23,14 +64,15 @@ def convert_gui_data_to_overrides(data, first=True): output = {} if first: output["__override_version__"] = OVERRIDE_VERSION + data = convert_gui_data_with_metadata(data) if METADATA_KEY in data: metadata = data.pop(METADATA_KEY) for key, value in metadata.items(): if key == "groups": - output[OVERRIDEN_KEY] = value + output[M_OVERRIDEN_KEY] = value else: - KeyError("Unknown metadata key \"{}\"".format(key)) + raise KeyError("Unknown metadata key \"{}\"".format(key)) for key, value in data.items(): output[key] = convert_gui_data_to_overrides(value, False) @@ -41,9 +83,12 @@ def convert_overrides_to_gui_data(data, first=True): if not data or not isinstance(data, dict): return data + if first: + data = convert_data_to_gui_data(data) + output = {} - if OVERRIDEN_KEY in data: - groups = data.pop(OVERRIDEN_KEY) + if M_OVERRIDEN_KEY in data: + groups = data.pop(M_OVERRIDEN_KEY) if METADATA_KEY not in output: output[METADATA_KEY] = {} output[METADATA_KEY]["groups"] = groups @@ -54,7 +99,111 @@ def convert_overrides_to_gui_data(data, first=True): return output -def _fill_inner_schemas(schema_data, schema_collection): +def _fill_schema_template_data( + template, template_data, required_keys=None, missing_keys=None +): + first = False + if required_keys is None: + first = True + required_keys = set() + missing_keys = set() + + _template = [] + default_values = {} + for item in template: + if isinstance(item, dict) and "__default_values__" in item: + default_values = item["__default_values__"] + else: + _template.append(item) + template = _template + + for key, value in default_values.items(): + if key not in template_data: + template_data[key] = value + + if not template: + output = template + + elif isinstance(template, list): + output = [] + for item in template: + output.append(_fill_schema_template_data( + item, template_data, required_keys, missing_keys + )) + + elif isinstance(template, dict): + output = {} + for key, value in template.items(): + output[key] = _fill_schema_template_data( + value, template_data, required_keys, missing_keys + ) + + elif isinstance(template, str): + # TODO find much better way how to handle filling template data + for replacement_string in key_pattern.findall(template): + key = str(replacement_string[1:-1]) + required_keys.add(key) + if key not in template_data: + missing_keys.add(key) + continue + + value = template_data[key] + if replacement_string == template: + # Replace the value with value from templates data + # - with this is possible to set value with different type + template = value + else: + # Only replace the key in string + template = template.replace(replacement_string, value) + output = template + + else: + output = template + + if first and missing_keys: + raise SchemaTemplateMissingKeys(missing_keys, required_keys) + + return output + + +def _fill_schema_template(child_data, schema_collection, schema_templates): + template_name = child_data["name"] + template = schema_templates.get(template_name) + if template is None: + if template_name in schema_collection: + raise KeyError(( + "Schema \"{}\" is used as `schema_template`" + ).format(template_name)) + raise KeyError("Schema template \"{}\" was not found".format( + template_name + )) + + template_data = child_data.get("template_data") or {} + try: + filled_child = _fill_schema_template_data( + template, template_data + ) + + except SchemaTemplateMissingKeys as exc: + raise SchemaTemplateMissingKeys( + exc.missing_keys, exc.required_keys, template_name + ) + + output = [] + for item in filled_child: + filled_item = _fill_inner_schemas( + item, schema_collection, schema_templates + ) + if filled_item["type"] == "schema_template": + output.extend(_fill_schema_template( + filled_item, schema_collection, schema_templates + )) + else: + output.append(filled_item) + return output + + +def _fill_inner_schemas(schema_data, schema_collection, schema_templates): if schema_data["type"] == "schema": raise ValueError("First item in schema data can't be schema.") @@ -64,21 +213,62 @@ def _fill_inner_schemas(schema_data, schema_collection): new_children = [] for child in children: - if child["type"] != "schema": - new_child = _fill_inner_schemas(child, schema_collection) - new_children.append(new_child) + child_type = child["type"] + if child_type == "schema": + schema_name = child["name"] + if schema_name not in schema_collection: + if schema_name in schema_templates: + raise KeyError(( + "Schema template \"{}\" is used as `schema`" + ).format(schema_name)) + raise KeyError( + "Schema \"{}\" was not found".format(schema_name) + ) + + filled_child = _fill_inner_schemas( + schema_collection[schema_name], + schema_collection, + schema_templates + ) + + elif child_type == "schema_template": + for filled_child in _fill_schema_template( + child, schema_collection, schema_templates + ): + new_children.append(filled_child) continue - new_child = _fill_inner_schemas( - schema_collection[child["name"]], - schema_collection - ) - new_children.append(new_child) + else: + filled_child = _fill_inner_schemas( + child, schema_collection, schema_templates + ) + + new_children.append(filled_child) schema_data["children"] = new_children return schema_data +class SchemaTemplateMissingKeys(Exception): + def __init__(self, missing_keys, required_keys, template_name=None): + self.missing_keys = missing_keys + self.required_keys = required_keys + if template_name: + msg = f"Schema template \"{template_name}\" require more keys.\n" + else: + msg = "" + msg += "Required keys: {}\nMissing keys: {}".format( + self.join_keys(required_keys), + self.join_keys(missing_keys) + ) + super(SchemaTemplateMissingKeys, self).__init__(msg) + + def join_keys(self, keys): + return ", ".join([ + f"\"{key}\"" for key in keys + ]) + + class SchemaMissingFileInfo(Exception): def __init__(self, invalid): full_path_keys = [] @@ -120,6 +310,21 @@ class SchemaDuplicatedKeys(Exception): super(SchemaDuplicatedKeys, self).__init__(msg) +class SchemaDuplicatedEnvGroupKeys(Exception): + def __init__(self, invalid): + items = [] + for key_path, keys in invalid.items(): + joined_keys = ", ".join([ + "\"{}\"".format(key) for key in keys + ]) + items.append("\"{}\" ({})".format(key_path, joined_keys)) + + msg = ( + "Schema items contain duplicated environment group keys. {}" + ).format(" || ".join(items)) + super(SchemaDuplicatedEnvGroupKeys, self).__init__(msg) + + def file_keys_from_schema(schema_data): output = [] item_type = schema_data["type"] @@ -277,10 +482,50 @@ def validate_keys_are_unique(schema_data, keys=None): raise SchemaDuplicatedKeys(invalid) +def validate_environment_groups_uniquenes( + schema_data, env_groups=None, keys=None +): + is_first = False + if env_groups is None: + is_first = True + env_groups = {} + keys = [] + + my_keys = copy.deepcopy(keys) + key = schema_data.get("key") + if key: + my_keys.append(key) + + env_group_key = schema_data.get("env_group_key") + if env_group_key: + if env_group_key not in env_groups: + env_groups[env_group_key] = [] + env_groups[env_group_key].append("/".join(my_keys)) + + children = schema_data.get("children") + if not children: + return + + for child in children: + validate_environment_groups_uniquenes( + child, env_groups, copy.deepcopy(my_keys) + ) + + if is_first: + invalid = {} + for env_group_key, key_paths in env_groups.items(): + if len(key_paths) > 1: + invalid[env_group_key] = key_paths + + if invalid: + raise SchemaDuplicatedEnvGroupKeys(invalid) + + def validate_schema(schema_data): validate_all_has_ending_file(schema_data) validate_is_group_is_unique_in_hierarchy(schema_data) validate_keys_are_unique(schema_data) + validate_environment_groups_uniquenes(schema_data) def gui_schema(subfolder, main_schema_name): @@ -292,23 +537,30 @@ def gui_schema(subfolder, main_schema_name): ) loaded_schemas = {} - for filename in os.listdir(dirpath): - basename, ext = os.path.splitext(filename) - if ext != ".json": - continue + loaded_schema_templates = {} + for root, _, filenames in os.walk(dirpath): + for filename in filenames: + basename, ext = os.path.splitext(filename) + if ext != ".json": + continue - filepath = os.path.join(dirpath, filename) - with open(filepath, "r") as json_stream: - try: - schema_data = json.load(json_stream) - except Exception as e: - raise Exception((f"Unable to parse JSON file {json_stream}\n " - f" - {e}")) from e - loaded_schemas[basename] = schema_data + filepath = os.path.join(root, filename) + with open(filepath, "r") as json_stream: + try: + schema_data = json.load(json_stream) + except Exception as exc: + raise Exception(( + f"Unable to parse JSON file {filepath}\n{exc}" + )) from exc + if isinstance(schema_data, list): + loaded_schema_templates[basename] = schema_data + else: + loaded_schemas[basename] = schema_data main_schema = _fill_inner_schemas( loaded_schemas[main_schema_name], - loaded_schemas + loaded_schemas, + loaded_schema_templates ) validate_schema(main_schema) return main_schema