From dfe666e61fe3eaeb7a1e69dcdacb0645b6092ab1 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Sat, 3 Oct 2020 01:15:19 +0200 Subject: [PATCH 01/16] renamed `load_json` to `load_json_file` --- pype/settings/lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/settings/lib.py b/pype/settings/lib.py index 96c3829388..129a218dba 100644 --- a/pype/settings/lib.py +++ b/pype/settings/lib.py @@ -59,7 +59,7 @@ def default_settings(): return _DEFAULT_SETTINGS -def load_json(fpath): +def load_json_file(fpath): # Load json data with open(fpath, "r") as opened_file: lines = opened_file.read().splitlines() From 9683208f0ca217ab17a300412bd10fa8977923c3 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Sat, 3 Oct 2020 01:15:52 +0200 Subject: [PATCH 02/16] simplified `load_json_file` as jsons are expected to be saved with gui --- pype/settings/lib.py | 41 +---------------------------------------- 1 file changed, 1 insertion(+), 40 deletions(-) diff --git a/pype/settings/lib.py b/pype/settings/lib.py index 129a218dba..57f90ff0f7 100644 --- a/pype/settings/lib.py +++ b/pype/settings/lib.py @@ -61,48 +61,9 @@ def default_settings(): def load_json_file(fpath): # Load json data - with open(fpath, "r") as opened_file: - lines = opened_file.read().splitlines() - - # prepare json string - standard_json = "" - for line in lines: - # Remove all whitespace on both sides - line = line.strip() - - # Skip blank lines - if len(line) == 0: - continue - - standard_json += line - - # Check if has extra commas - extra_comma = False - if ",]" in standard_json or ",}" in standard_json: - extra_comma = True - standard_json = standard_json.replace(",]", "]") - standard_json = standard_json.replace(",}", "}") - - if extra_comma: - log.error("Extra comma in json file: \"{}\"".format(fpath)) - - # return empty dict if file is empty - if standard_json == "": - return {} - - # Try to parse string - try: - return json.loads(standard_json) - - except json.decoder.JSONDecodeError: - # Return empty dict if it is first time that decode error happened - return {} - - # Repreduce the exact same exception but traceback contains better - # information about position of error in the loaded json try: with open(fpath, "r") as opened_file: - json.load(opened_file) + return json.load(opened_file) except json.decoder.JSONDecodeError: log.warning( From 11766b71a58b65067db9902bfd81e08092d321b1 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Sat, 3 Oct 2020 01:16:41 +0200 Subject: [PATCH 03/16] changed `load_json` to `load_json_file` in code --- pype/settings/lib.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pype/settings/lib.py b/pype/settings/lib.py index 57f90ff0f7..6d36101194 100644 --- a/pype/settings/lib.py +++ b/pype/settings/lib.py @@ -153,25 +153,25 @@ def load_jsons_from_dir(path, *args, **kwargs): def studio_system_settings(): if os.path.exists(SYSTEM_SETTINGS_PATH): - return load_json(SYSTEM_SETTINGS_PATH) + return load_json_file(SYSTEM_SETTINGS_PATH) return {} def studio_environments(): if os.path.exists(ENVIRONMENTS_PATH): - return load_json(ENVIRONMENTS_PATH) + return load_json_file(ENVIRONMENTS_PATH) return {} def studio_project_settings(): if os.path.exists(PROJECT_SETTINGS_PATH): - return load_json(PROJECT_SETTINGS_PATH) + return load_json_file(PROJECT_SETTINGS_PATH) return {} def studio_project_anatomy(): if os.path.exists(PROJECT_ANATOMY_PATH): - return load_json(PROJECT_ANATOMY_PATH) + return load_json_file(PROJECT_ANATOMY_PATH) return {} @@ -198,7 +198,7 @@ def project_settings_overrides(project_name): path_to_json = path_to_project_overrides(project_name) if not os.path.exists(path_to_json): return {} - return load_json(path_to_json) + return load_json_file(path_to_json) def project_anatomy_overrides(project_name): @@ -208,7 +208,7 @@ def project_anatomy_overrides(project_name): path_to_json = path_to_project_anatomy(project_name) if not os.path.exists(path_to_json): return {} - return load_json(path_to_json) + return load_json_file(path_to_json) def merge_overrides(global_dict, override_dict): From f44cb798ff3a639d2cd3b6945a5530f71c350ee1 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Sat, 3 Oct 2020 01:17:08 +0200 Subject: [PATCH 04/16] moved `load_jsons_from_dir` --- pype/settings/lib.py | 76 ++++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/pype/settings/lib.py b/pype/settings/lib.py index 6d36101194..fb76bdc2b3 100644 --- a/pype/settings/lib.py +++ b/pype/settings/lib.py @@ -74,6 +74,44 @@ def load_json_file(fpath): return {} +def load_jsons_from_dir(path, *args, **kwargs): + output = {} + + path = os.path.normpath(path) + if not os.path.exists(path): + # TODO warning + return output + + sub_keys = list(kwargs.pop("subkeys", args)) + for sub_key in tuple(sub_keys): + _path = os.path.join(path, sub_key) + if not os.path.exists(_path): + break + + path = _path + sub_keys.pop(0) + + base_len = len(path) + 1 + for base, _directories, filenames in os.walk(path): + base_items_str = base[base_len:] + if not base_items_str: + base_items = [] + else: + base_items = base_items_str.split(os.path.sep) + + for filename in filenames: + basename, ext = os.path.splitext(filename) + if ext == ".json": + full_path = os.path.join(base, filename) + value = load_json_file(full_path) + dict_keys = base_items + [basename] + output = subkey_merge(output, value, dict_keys) + + for sub_key in sub_keys: + output = output[sub_key] + return output + + def find_environments(data): if not data or not isinstance(data, dict): return @@ -113,44 +151,6 @@ def subkey_merge(_dict, value, keys): return _dict -def load_jsons_from_dir(path, *args, **kwargs): - output = {} - - path = os.path.normpath(path) - if not os.path.exists(path): - # TODO warning - return output - - sub_keys = list(kwargs.pop("subkeys", args)) - for sub_key in tuple(sub_keys): - _path = os.path.join(path, sub_key) - if not os.path.exists(_path): - break - - path = _path - sub_keys.pop(0) - - base_len = len(path) + 1 - for base, _directories, filenames in os.walk(path): - base_items_str = base[base_len:] - if not base_items_str: - base_items = [] - else: - base_items = base_items_str.split(os.path.sep) - - for filename in filenames: - basename, ext = os.path.splitext(filename) - if ext == ".json": - full_path = os.path.join(base, filename) - value = load_json(full_path) - dict_keys = base_items + [basename] - output = subkey_merge(output, value, dict_keys) - - for sub_key in sub_keys: - output = output[sub_key] - return output - - def studio_system_settings(): if os.path.exists(SYSTEM_SETTINGS_PATH): return load_json_file(SYSTEM_SETTINGS_PATH) From 62c157ab2e4e1de4d64b7715d46812e18289b55c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Sat, 3 Oct 2020 01:19:17 +0200 Subject: [PATCH 05/16] path to project settings returns path to studio overrides if project name is None --- pype/settings/lib.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pype/settings/lib.py b/pype/settings/lib.py index fb76bdc2b3..ca134d0611 100644 --- a/pype/settings/lib.py +++ b/pype/settings/lib.py @@ -175,7 +175,9 @@ def studio_project_anatomy(): return {} -def path_to_project_overrides(project_name): +def path_to_project_settings(project_name): + if not project_name: + return PROJECT_SETTINGS_PATH return os.path.join( STUDIO_OVERRIDES_PATH, project_name, @@ -184,6 +186,8 @@ def path_to_project_overrides(project_name): def path_to_project_anatomy(project_name): + if not project_name: + return PROJECT_ANATOMY_PATH return os.path.join( STUDIO_OVERRIDES_PATH, project_name, @@ -195,7 +199,7 @@ def project_settings_overrides(project_name): if not project_name: return {} - path_to_json = path_to_project_overrides(project_name) + path_to_json = path_to_project_settings(project_name) if not os.path.exists(path_to_json): return {} return load_json_file(path_to_json) From 85844ba4cfdce91844c3211d13ad22ac0c6d77ed Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Sat, 3 Oct 2020 01:19:36 +0200 Subject: [PATCH 06/16] added functions for saving settings --- pype/settings/lib.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/pype/settings/lib.py b/pype/settings/lib.py index ca134d0611..0e6aa65adc 100644 --- a/pype/settings/lib.py +++ b/pype/settings/lib.py @@ -195,6 +195,38 @@ def path_to_project_anatomy(project_name): ) +def save_studio_settings(data): + dirpath = os.path.dirname(SYSTEM_SETTINGS_PATH) + if not os.path.exists(dirpath): + os.makedirs(dirpath) + + print("Saving studio overrides") + with open(SYSTEM_SETTINGS_PATH, "w") as file_stream: + json.dump(data, file_stream, indent=4) + + +def save_project_settings(project_name, overrides): + project_overrides_json_path = path_to_project_settings(project_name) + dirpath = os.path.dirname(project_overrides_json_path) + if not os.path.exists(dirpath): + os.makedirs(dirpath) + + print("Saving overrides of project \"{}\"".format(project_name)) + with open(project_overrides_json_path, "w") as file_stream: + json.dump(overrides, file_stream, indent=4) + + +def save_project_anatomy(project_name, anatomy_data): + project_anatomy_json_path = path_to_project_anatomy(project_name) + dirpath = os.path.dirname(project_anatomy_json_path) + if not os.path.exists(dirpath): + os.makedirs(dirpath) + + print("Saving anatomy of project \"{}\"".format(project_name)) + with open(project_anatomy_json_path, "w") as file_stream: + json.dump(anatomy_data, file_stream, indent=4) + + def project_settings_overrides(project_name): if not project_name: return {} From 2dea080841f2fdffdebec9b64742dfb8c5740a7b Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Sat, 3 Oct 2020 01:19:44 +0200 Subject: [PATCH 07/16] STUDIO_OVERRIDES_PATH is safer --- pype/settings/lib.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pype/settings/lib.py b/pype/settings/lib.py index 0e6aa65adc..abb704f033 100644 --- a/pype/settings/lib.py +++ b/pype/settings/lib.py @@ -13,7 +13,7 @@ M_ENVIRONMENT_KEY = "__environment_keys__" M_POP_KEY = "__pop_key__" # Folder where studio overrides are stored -STUDIO_OVERRIDES_PATH = os.environ["PYPE_PROJECT_CONFIGS"] +STUDIO_OVERRIDES_PATH = os.getenv("PYPE_PROJECT_CONFIGS") # File where studio's system overrides are stored SYSTEM_SETTINGS_KEY = "system_settings" @@ -70,7 +70,6 @@ def load_json_file(fpath): "File has invalid json format \"{}\"".format(fpath), exc_info=True ) - return {} From d8b6733b86e03d36c8d8192bcdfe829dfb2c6a5e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Sat, 3 Oct 2020 01:20:01 +0200 Subject: [PATCH 08/16] using saving functions in settings tool --- pype/tools/settings/settings/widgets/base.py | 130 +++++-------------- 1 file changed, 31 insertions(+), 99 deletions(-) diff --git a/pype/tools/settings/settings/widgets/base.py b/pype/tools/settings/settings/widgets/base.py index 7cbe7c2f6f..5feab765e5 100644 --- a/pype/tools/settings/settings/widgets/base.py +++ b/pype/tools/settings/settings/widgets/base.py @@ -3,11 +3,8 @@ import json from Qt import QtWidgets, QtCore, QtGui from pype.settings.lib import ( SYSTEM_SETTINGS_KEY, - SYSTEM_SETTINGS_PATH, PROJECT_SETTINGS_KEY, - PROJECT_SETTINGS_PATH, PROJECT_ANATOMY_KEY, - PROJECT_ANATOMY_PATH, DEFAULTS_DIR, @@ -21,8 +18,9 @@ from pype.settings.lib import ( project_settings_overrides, project_anatomy_overrides, - path_to_project_overrides, - path_to_project_anatomy + save_studio_settings, + save_project_settings, + save_project_anatomy ) from .widgets import UnsavedChangesDialog from . import lib @@ -183,13 +181,7 @@ class SystemWidget(QtWidgets.QWidget): values = lib.convert_gui_data_to_overrides(_data.get("system", {})) - dirpath = os.path.dirname(SYSTEM_SETTINGS_PATH) - if not os.path.exists(dirpath): - os.makedirs(dirpath) - - print("Saving data to:", SYSTEM_SETTINGS_PATH) - with open(SYSTEM_SETTINGS_PATH, "w") as file_stream: - json.dump(values, file_stream, indent=4) + save_studio_settings(values) self._update_values() @@ -621,29 +613,25 @@ class ProjectWidget(QtWidgets.QWidget): if item.child_invalid: has_invalid = True - if has_invalid: - invalid_items = [] - for item in self.input_fields: - invalid_items.extend(item.get_invalid()) - msg_box = QtWidgets.QMessageBox( - QtWidgets.QMessageBox.Warning, - "Invalid input", - "There is invalid value in one of inputs." - " Please lead red color and fix them." - ) - msg_box.setStandardButtons(QtWidgets.QMessageBox.Ok) - msg_box.exec_() + if not has_invalid: + return self._save_overrides() - first_invalid_item = invalid_items[0] - self.scroll_widget.ensureWidgetVisible(first_invalid_item) - if first_invalid_item.isVisible(): - first_invalid_item.setFocus(True) - return + invalid_items = [] + for item in self.input_fields: + invalid_items.extend(item.get_invalid()) + msg_box = QtWidgets.QMessageBox( + QtWidgets.QMessageBox.Warning, + "Invalid input", + "There is invalid value in one of inputs." + " Please lead red color and fix them." + ) + msg_box.setStandardButtons(QtWidgets.QMessageBox.Ok) + msg_box.exec_() - if self.project_name is None: - self._save_studio_overrides() - else: - self._save_overrides() + first_invalid_item = invalid_items[0] + self.scroll_widget.ensureWidgetVisible(first_invalid_item) + if first_invalid_item.isVisible(): + first_invalid_item.setFocus(True) def _on_refresh(self): self.reset() @@ -665,75 +653,19 @@ class ProjectWidget(QtWidgets.QWidget): ) # Saving overrides data - project_overrides_data = output_data.get( - PROJECT_SETTINGS_KEY, {} - ) - project_overrides_json_path = path_to_project_overrides( - self.project_name - ) - dirpath = os.path.dirname(project_overrides_json_path) - if not os.path.exists(dirpath): - os.makedirs(dirpath) - - print("Saving data to:", project_overrides_json_path) - with open(project_overrides_json_path, "w") as file_stream: - json.dump(project_overrides_data, file_stream, indent=4) + project_overrides_data = output_data.get(PROJECT_SETTINGS_KEY, {}) + save_project_settings(self.project_name, project_overrides_data) # Saving anatomy data - project_anatomy_data = output_data.get( - PROJECT_ANATOMY_KEY, {} - ) - project_anatomy_json_path = path_to_project_anatomy( - self.project_name - ) - dirpath = os.path.dirname(project_anatomy_json_path) - if not os.path.exists(dirpath): - os.makedirs(dirpath) + project_anatomy_data = output_data.get(PROJECT_ANATOMY_KEY, {}) + save_project_anatomy(self.project_name, project_anatomy_data) - print("Saving data to:", project_anatomy_json_path) - with open(project_anatomy_json_path, "w") as file_stream: - json.dump(project_anatomy_data, file_stream, indent=4) - - # Refill values with overrides - self._on_project_change() - - def _save_studio_overrides(self): - data = {} - for input_field in self.input_fields: - value, is_group = input_field.studio_overrides() - if value is not lib.NOT_SET: - data.update(value) - - output_data = lib.convert_gui_data_to_overrides( - data.get("project", {}) - ) - - # Project overrides data - project_overrides_data = output_data.get( - PROJECT_SETTINGS_KEY, {} - ) - dirpath = os.path.dirname(PROJECT_SETTINGS_PATH) - if not os.path.exists(dirpath): - os.makedirs(dirpath) - - print("Saving data to:", PROJECT_SETTINGS_PATH) - with open(PROJECT_SETTINGS_PATH, "w") as file_stream: - json.dump(project_overrides_data, file_stream, indent=4) - - # Project Anatomy data - project_anatomy_data = output_data.get( - PROJECT_ANATOMY_KEY, {} - ) - dirpath = os.path.dirname(PROJECT_ANATOMY_PATH) - if not os.path.exists(dirpath): - os.makedirs(dirpath) - - print("Saving data to:", PROJECT_ANATOMY_PATH) - with open(PROJECT_ANATOMY_PATH, "w") as file_stream: - json.dump(project_anatomy_data, file_stream, indent=4) - - # Update saved values - self._update_values() + if self.project_name: + # Refill values with overrides + self._on_project_change() + else: + # Update saved values + self._update_values() def _update_values(self): self.ignore_value_changes = True From 4ed67bed349a4f73a574507599542d934c83fc52 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Sat, 3 Oct 2020 01:31:02 +0200 Subject: [PATCH 09/16] make sure settings.lib import won't crash if PYPE_PROJECT_CONFIGS is not set --- pype/settings/lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/settings/lib.py b/pype/settings/lib.py index abb704f033..a29d71ac4d 100644 --- a/pype/settings/lib.py +++ b/pype/settings/lib.py @@ -13,7 +13,7 @@ M_ENVIRONMENT_KEY = "__environment_keys__" M_POP_KEY = "__pop_key__" # Folder where studio overrides are stored -STUDIO_OVERRIDES_PATH = os.getenv("PYPE_PROJECT_CONFIGS") +STUDIO_OVERRIDES_PATH = os.getenv("PYPE_PROJECT_CONFIGS") or "" # File where studio's system overrides are stored SYSTEM_SETTINGS_KEY = "system_settings" From 84554e0568de6aceda745c34594d0af938de3399 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 5 Oct 2020 10:35:03 +0200 Subject: [PATCH 10/16] fixed saving of project settings --- pype/tools/settings/settings/widgets/base.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pype/tools/settings/settings/widgets/base.py b/pype/tools/settings/settings/widgets/base.py index 5feab765e5..e342d375f5 100644 --- a/pype/tools/settings/settings/widgets/base.py +++ b/pype/tools/settings/settings/widgets/base.py @@ -643,8 +643,12 @@ class ProjectWidget(QtWidgets.QWidget): def _save_overrides(self): data = {} + studio_overrides = bool(self.project_name is None) for item in self.input_fields: - value, is_group = item.overrides() + if studio_overrides: + value, is_group = item.studio_overrides() + else: + value, is_group = item.overrides() if value is not lib.NOT_SET: data.update(value) @@ -670,7 +674,7 @@ class ProjectWidget(QtWidgets.QWidget): def _update_values(self): self.ignore_value_changes = True - default_values = default_values = lib.convert_data_to_gui_data( + default_values = lib.convert_data_to_gui_data( {"project": default_settings()} ) for input_field in self.input_fields: From 15021cbb94b14981cedb3739dabf9e3aaf609850 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 5 Oct 2020 11:24:05 +0200 Subject: [PATCH 11/16] safer work with default settings --- pype/settings/lib.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/settings/lib.py b/pype/settings/lib.py index a29d71ac4d..0c4f21e7ee 100644 --- a/pype/settings/lib.py +++ b/pype/settings/lib.py @@ -278,13 +278,13 @@ def apply_overrides(source_data, override_data): def system_settings(): - default_values = default_settings()[SYSTEM_SETTINGS_KEY] + default_values = copy.deepcopy(default_settings()[SYSTEM_SETTINGS_KEY]) studio_values = studio_system_settings() return apply_overrides(default_values, studio_values) def project_settings(project_name): - default_values = default_settings()[PROJECT_SETTINGS_KEY] + default_values = copy.deepcopy(default_settings()[PROJECT_SETTINGS_KEY]) studio_values = studio_project_settings() studio_overrides = apply_overrides(default_values, studio_values) From f7db222c7679700e247875e0490ec6d3e7d70de7 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 5 Oct 2020 11:24:27 +0200 Subject: [PATCH 12/16] print more specific data --- pype/settings/lib.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pype/settings/lib.py b/pype/settings/lib.py index 0c4f21e7ee..a8ae0bb753 100644 --- a/pype/settings/lib.py +++ b/pype/settings/lib.py @@ -199,7 +199,9 @@ def save_studio_settings(data): if not os.path.exists(dirpath): os.makedirs(dirpath) - print("Saving studio overrides") + print("Saving studio overrides. Output path: {}".format( + SYSTEM_SETTINGS_PATH + )) with open(SYSTEM_SETTINGS_PATH, "w") as file_stream: json.dump(data, file_stream, indent=4) @@ -210,7 +212,9 @@ def save_project_settings(project_name, overrides): if not os.path.exists(dirpath): os.makedirs(dirpath) - print("Saving overrides of project \"{}\"".format(project_name)) + print("Saving overrides of project \"{}\". Output path: {}".format( + project_name, project_overrides_json_path + )) with open(project_overrides_json_path, "w") as file_stream: json.dump(overrides, file_stream, indent=4) @@ -221,7 +225,9 @@ def save_project_anatomy(project_name, anatomy_data): if not os.path.exists(dirpath): os.makedirs(dirpath) - print("Saving anatomy of project \"{}\"".format(project_name)) + print("Saving anatomy of project \"{}\". Output path: {}".format( + project_name, project_anatomy_json_path + )) with open(project_anatomy_json_path, "w") as file_stream: json.dump(anatomy_data, file_stream, indent=4) From fe6e3ad1ea2b50ce2648ac877ceab3e2fd5b4365 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 5 Oct 2020 11:24:39 +0200 Subject: [PATCH 13/16] added few dostrings --- pype/settings/lib.py | 101 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 97 insertions(+), 4 deletions(-) diff --git a/pype/settings/lib.py b/pype/settings/lib.py index a8ae0bb753..b82e4a651b 100644 --- a/pype/settings/lib.py +++ b/pype/settings/lib.py @@ -74,6 +74,35 @@ def load_json_file(fpath): def load_jsons_from_dir(path, *args, **kwargs): + """Load all json files with content from entered path. + + Enterd path hiearchy: + |_ folder1 + | |_ data1.json + |_ folder2 + |_ subfolder1 + |_ data2.json + + Will result in: + ```javascript + { + "folder1": { + "data1": "CONTENT OF FILE" + }, + "folder2": { + "data1": { + "subfolder1": "CONTENT OF FILE" + } + } + } + ``` + + Args: + path (str): Path to folder where jsons should be. + + Returns: + dict: loaded data + """ output = {} path = os.path.normpath(path) @@ -112,6 +141,15 @@ def load_jsons_from_dir(path, *args, **kwargs): def find_environments(data): + """ Find environemnt values from system settings by it's metadata. + + Args: + data(dict): System settings data or dictionary which may contain + environments metadata. + + Returns: + dict: Key as Environment key and value for `acre` module. + """ if not data or not isinstance(data, dict): return @@ -151,24 +189,28 @@ def subkey_merge(_dict, value, keys): def studio_system_settings(): + """Studio overrides of system settings.""" if os.path.exists(SYSTEM_SETTINGS_PATH): return load_json_file(SYSTEM_SETTINGS_PATH) return {} def studio_environments(): + """Environment values from defaults.""" if os.path.exists(ENVIRONMENTS_PATH): return load_json_file(ENVIRONMENTS_PATH) return {} def studio_project_settings(): + """Studio overrides of default project settings.""" if os.path.exists(PROJECT_SETTINGS_PATH): return load_json_file(PROJECT_SETTINGS_PATH) return {} def studio_project_anatomy(): + """Studio overrides of default project anatomy data.""" if os.path.exists(PROJECT_ANATOMY_PATH): return load_json_file(PROJECT_ANATOMY_PATH) return {} @@ -195,6 +237,14 @@ def path_to_project_anatomy(project_name): def save_studio_settings(data): + """Save studio overrides of system settings. + + Saving must corespond with loading. For loading should be used function + `studio_system_settings`. + + Args: + data(dict): Data of studio overrides with override metadata. + """ dirpath = os.path.dirname(SYSTEM_SETTINGS_PATH) if not os.path.exists(dirpath): os.makedirs(dirpath) @@ -207,6 +257,17 @@ def save_studio_settings(data): def save_project_settings(project_name, overrides): + """Save studio overrides of project settings. + + Data are saved for specific project or as defaults for all projects. + Saving must corespond with loading. For loading should be used functions + `project_settings_overrides` and `studio_project_settings`. + + Args: + project_name(str, null): Project name for which overrides are + or None for global settings. + data(dict): Data of project overrides with override metadata. + """ project_overrides_json_path = path_to_project_settings(project_name) dirpath = os.path.dirname(project_overrides_json_path) if not os.path.exists(dirpath): @@ -220,6 +281,17 @@ def save_project_settings(project_name, overrides): def save_project_anatomy(project_name, anatomy_data): + """Save studio overrides of project anatomy. + + Data are saved for specific project or as defaults for all projects. + Saving must corespond with loading. For loading should be used functions + `project_anatomy_overrides` and `studio_project_anatomy`. + + Args: + project_name(str, null): Project name for which overrides are + or None for global settings. + data(dict): Data of project overrides with override metadata. + """ project_anatomy_json_path = path_to_project_anatomy(project_name) dirpath = os.path.dirname(project_anatomy_json_path) if not os.path.exists(dirpath): @@ -233,6 +305,14 @@ def save_project_anatomy(project_name, anatomy_data): def project_settings_overrides(project_name): + """Studio overrides of project settings for specific project. + + Args: + project_name(str): Name of project for which data should be loaded. + + Returns: + dict: Only overrides for entered project, may be empty dictionary. + """ if not project_name: return {} @@ -243,6 +323,14 @@ def project_settings_overrides(project_name): def project_anatomy_overrides(project_name): + """Studio overrides of project anatomy for specific project. + + Args: + project_name(str): Name of project for which data should be loaded. + + Returns: + dict: Only overrides for entered project, may be empty dictionary. + """ if not project_name: return {} @@ -253,6 +341,7 @@ def project_anatomy_overrides(project_name): def merge_overrides(global_dict, override_dict): + """Merge override data to source data by metadata stored in.""" if M_OVERRIDEN_KEY in override_dict: overriden_keys = set(override_dict.pop(M_OVERRIDEN_KEY)) else: @@ -262,10 +351,7 @@ def merge_overrides(global_dict, override_dict): if value == M_POP_KEY: global_dict.pop(key) - elif ( - key in overriden_keys - or key not in global_dict - ): + elif (key in overriden_keys or key not in global_dict): global_dict[key] = value elif isinstance(value, dict) and isinstance(global_dict[key], dict): @@ -284,12 +370,14 @@ def apply_overrides(source_data, override_data): def system_settings(): + """System settings with applied studio overrides.""" default_values = copy.deepcopy(default_settings()[SYSTEM_SETTINGS_KEY]) studio_values = studio_system_settings() return apply_overrides(default_values, studio_values) def project_settings(project_name): + """Project settings with applied studio and project overrides.""" default_values = copy.deepcopy(default_settings()[PROJECT_SETTINGS_KEY]) studio_values = studio_project_settings() @@ -301,6 +389,11 @@ def project_settings(project_name): def environments(): + """Environments from defaults and extracted from system settings. + + Returns: + dict: Output should be ready for `acre` module. + """ envs = copy.deepcopy(default_settings()[ENVIRONMENTS_KEY]) envs_from_system_settings = find_environments(system_settings()) for env_group_key, values in envs_from_system_settings.items(): From df0597799402898c14ed0f14e6c1dbb97b3a15e3 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 5 Oct 2020 12:00:10 +0200 Subject: [PATCH 14/16] modified docstring in save functions to be more clear --- pype/settings/lib.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/pype/settings/lib.py b/pype/settings/lib.py index b82e4a651b..2afdff1dd0 100644 --- a/pype/settings/lib.py +++ b/pype/settings/lib.py @@ -239,8 +239,9 @@ def path_to_project_anatomy(project_name): def save_studio_settings(data): """Save studio overrides of system settings. - Saving must corespond with loading. For loading should be used function - `studio_system_settings`. + Do not use to store whole system settings data with defaults but only it's + overrides with metadata defining how overrides should be applied in load + function. For loading should be used function `studio_system_settings`. Args: data(dict): Data of studio overrides with override metadata. @@ -260,8 +261,12 @@ def save_project_settings(project_name, overrides): """Save studio overrides of project settings. Data are saved for specific project or as defaults for all projects. - Saving must corespond with loading. For loading should be used functions - `project_settings_overrides` and `studio_project_settings`. + + Do not use to store whole project settings data with defaults but only it's + overrides with metadata defining how overrides should be applied in load + function. For loading should be used functions `studio_project_settings` + for global project settings and `project_settings_overrides` for + project specific settings. Args: project_name(str, null): Project name for which overrides are @@ -283,9 +288,11 @@ def save_project_settings(project_name, overrides): def save_project_anatomy(project_name, anatomy_data): """Save studio overrides of project anatomy. - Data are saved for specific project or as defaults for all projects. - Saving must corespond with loading. For loading should be used functions - `project_anatomy_overrides` and `studio_project_anatomy`. + Do not use to store whole project anatomy data with defaults but only it's + overrides with metadata defining how overrides should be applied in load + function. For loading should be used functions `studio_project_anatomy` + for global project settings and `project_anatomy_overrides` for + project specific settings. Args: project_name(str, null): Project name for which overrides are From 40c4b512bbbf2d916f840887b1018b9224a88e79 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 5 Oct 2020 20:17:27 +0200 Subject: [PATCH 15/16] modified first docstring by comments --- pype/settings/lib.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pype/settings/lib.py b/pype/settings/lib.py index 2afdff1dd0..c2c4d3a363 100644 --- a/pype/settings/lib.py +++ b/pype/settings/lib.py @@ -74,9 +74,12 @@ def load_json_file(fpath): def load_jsons_from_dir(path, *args, **kwargs): - """Load all json files with content from entered path. + """Load all .json files with content from entered folder path. - Enterd path hiearchy: + Data are loaded recursively from a directory and recreate the + hierarchy as a dictionary. + + Entered path hiearchy: |_ folder1 | |_ data1.json |_ folder2 @@ -98,10 +101,10 @@ def load_jsons_from_dir(path, *args, **kwargs): ``` Args: - path (str): Path to folder where jsons should be. + path (str): Path to the root folder where the json hierarchy starts. Returns: - dict: loaded data + dict: Loaded data. """ output = {} From b489016fd0c9886363216266e92045b2eafc3299 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Mon, 19 Oct 2020 13:27:48 +0200 Subject: [PATCH 16/16] Small fixes in docstrings --- pype/settings/lib.py | 42 +++++++++++++++--------------------------- 1 file changed, 15 insertions(+), 27 deletions(-) diff --git a/pype/settings/lib.py b/pype/settings/lib.py index c2c4d3a363..a61238f973 100644 --- a/pype/settings/lib.py +++ b/pype/settings/lib.py @@ -23,9 +23,6 @@ SYSTEM_SETTINGS_PATH = os.path.join( # File where studio's environment overrides are stored ENVIRONMENTS_KEY = "environments" -ENVIRONMENTS_PATH = os.path.join( - STUDIO_OVERRIDES_PATH, ENVIRONMENTS_KEY + ".json" -) # File where studio's default project overrides are stored PROJECT_SETTINGS_KEY = "project_settings" @@ -198,13 +195,6 @@ def studio_system_settings(): return {} -def studio_environments(): - """Environment values from defaults.""" - if os.path.exists(ENVIRONMENTS_PATH): - return load_json_file(ENVIRONMENTS_PATH) - return {} - - def studio_project_settings(): """Studio overrides of default project settings.""" if os.path.exists(PROJECT_SETTINGS_PATH): @@ -289,13 +279,7 @@ def save_project_settings(project_name, overrides): def save_project_anatomy(project_name, anatomy_data): - """Save studio overrides of project anatomy. - - Do not use to store whole project anatomy data with defaults but only it's - overrides with metadata defining how overrides should be applied in load - function. For loading should be used functions `studio_project_anatomy` - for global project settings and `project_anatomy_overrides` for - project specific settings. + """Save studio overrides of project anatomy data. Args: project_name(str, null): Project name for which overrides are @@ -350,8 +334,9 @@ def project_anatomy_overrides(project_name): return load_json_file(path_to_json) -def merge_overrides(global_dict, override_dict): - """Merge override data to source data by metadata stored in.""" +def merge_overrides(source_dict, override_dict): + """Merge data from override_dict to source_dict.""" + if M_OVERRIDEN_KEY in override_dict: overriden_keys = set(override_dict.pop(M_OVERRIDEN_KEY)) else: @@ -359,17 +344,17 @@ def merge_overrides(global_dict, override_dict): for key, value in override_dict.items(): if value == M_POP_KEY: - global_dict.pop(key) + source_dict.pop(key) - elif (key in overriden_keys or key not in global_dict): - global_dict[key] = value + elif (key in overriden_keys or key not in source_dict): + source_dict[key] = value - elif isinstance(value, dict) and isinstance(global_dict[key], dict): - global_dict[key] = merge_overrides(global_dict[key], value) + elif isinstance(value, dict) and isinstance(source_dict[key], dict): + source_dict[key] = merge_overrides(source_dict[key], value) else: - global_dict[key] = value - return global_dict + source_dict[key] = value + return source_dict def apply_overrides(source_data, override_data): @@ -399,7 +384,10 @@ def project_settings(project_name): def environments(): - """Environments from defaults and extracted from system settings. + """Calculated environment based on defaults and system settings. + + Any default environment also found in the system settings will be fully + overriden by the one from the system settings. Returns: dict: Output should be ready for `acre` module.