From ab5fadccbb18e6c53687fc1e51b8feb9867a391b Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 11 Dec 2020 16:05:21 +0100 Subject: [PATCH 01/17] extracted constants to constants.py --- pype/settings/constants.py | 36 +++++++++++++ pype/settings/lib.py | 54 +++++--------------- pype/tools/settings/settings/widgets/base.py | 7 +-- 3 files changed, 54 insertions(+), 43 deletions(-) create mode 100644 pype/settings/constants.py diff --git a/pype/settings/constants.py b/pype/settings/constants.py new file mode 100644 index 0000000000..4a9f0e2287 --- /dev/null +++ b/pype/settings/constants.py @@ -0,0 +1,36 @@ +# Metadata keys for work with studio and project overrides +M_OVERRIDEN_KEY = "__overriden_keys__" +# Metadata key for storing information about environments +M_ENVIRONMENT_KEY = "__environment_keys__" +# Metadata key for storing dynamic created labels +M_DYNAMIC_KEY_LABEL = "__dynamic_keys_labels__" +# NOTE key popping not implemented yet +M_POP_KEY = "__pop_key__" + +METADATA_KEYS = ( + M_OVERRIDEN_KEY, + M_ENVIRONMENT_KEY, + M_DYNAMIC_KEY_LABEL, + M_POP_KEY +) + +# File where studio's system overrides are stored +SYSTEM_SETTINGS_KEY = "system_settings" +ENVIRONMENTS_KEY = "environments" +PROJECT_SETTINGS_KEY = "project_settings" +PROJECT_ANATOMY_KEY = "project_anatomy" + + +__all__ = ( + "M_OVERRIDEN_KEY", + "M_ENVIRONMENT_KEY", + "M_DYNAMIC_KEY_LABEL", + "M_POP_KEY", + + "METADATA_KEYS", + + "SYSTEM_SETTINGS_KEY", + "ENVIRONMENTS_KEY", + "PROJECT_SETTINGS_KEY", + "PROJECT_ANATOMY_KEY" +) diff --git a/pype/settings/lib.py b/pype/settings/lib.py index dfd4707dbf..62febfc1d2 100644 --- a/pype/settings/lib.py +++ b/pype/settings/lib.py @@ -2,52 +2,26 @@ import os import json import logging import copy +from .constants import ( + M_OVERRIDEN_KEY, + M_ENVIRONMENT_KEY, + M_DYNAMIC_KEY_LABEL, + M_POP_KEY, + + METADATA_KEYS, + + SYSTEM_SETTINGS_KEY, + ENVIRONMENTS_KEY, + PROJECT_SETTINGS_KEY, + PROJECT_ANATOMY_KEY +) +) log = logging.getLogger(__name__) # Py2 + Py3 json decode exception JSON_EXC = getattr(json.decoder, "JSONDecodeError", ValueError) -# Metadata keys for work with studio and project overrides -M_OVERRIDEN_KEY = "__overriden_keys__" -# Metadata key for storing information about environments -M_ENVIRONMENT_KEY = "__environment_keys__" -# Metadata key for storing dynamic created labels -M_DYNAMIC_KEY_LABEL = "__dynamic_keys_labels__" -# NOTE key popping not implemented yet -M_POP_KEY = "__pop_key__" - -METADATA_KEYS = ( - M_OVERRIDEN_KEY, - M_ENVIRONMENT_KEY, - M_DYNAMIC_KEY_LABEL, - M_POP_KEY -) - -# Folder where studio overrides are stored -STUDIO_OVERRIDES_PATH = os.getenv("PYPE_PROJECT_CONFIGS") or "" - -# File where studio's system overrides are stored -SYSTEM_SETTINGS_KEY = "system_settings" -SYSTEM_SETTINGS_PATH = os.path.join( - STUDIO_OVERRIDES_PATH, SYSTEM_SETTINGS_KEY + ".json" -) - -# File where studio's environment overrides are stored -ENVIRONMENTS_KEY = "environments" - -# File where studio's default project overrides are stored -PROJECT_SETTINGS_KEY = "project_settings" -PROJECT_SETTINGS_FILENAME = PROJECT_SETTINGS_KEY + ".json" -PROJECT_SETTINGS_PATH = os.path.join( - STUDIO_OVERRIDES_PATH, PROJECT_SETTINGS_FILENAME -) - -PROJECT_ANATOMY_KEY = "project_anatomy" -PROJECT_ANATOMY_FILENAME = PROJECT_ANATOMY_KEY + ".json" -PROJECT_ANATOMY_PATH = os.path.join( - STUDIO_OVERRIDES_PATH, PROJECT_ANATOMY_FILENAME -) # Path to default settings DEFAULTS_DIR = os.path.join(os.path.dirname(__file__), "defaults") diff --git a/pype/tools/settings/settings/widgets/base.py b/pype/tools/settings/settings/widgets/base.py index 4ff7ec26e7..ddd814e38e 100644 --- a/pype/tools/settings/settings/widgets/base.py +++ b/pype/tools/settings/settings/widgets/base.py @@ -2,11 +2,12 @@ import os import copy import json from Qt import QtWidgets, QtCore, QtGui -from pype.settings.lib import ( +from pype.settings.constants import ( SYSTEM_SETTINGS_KEY, PROJECT_SETTINGS_KEY, - PROJECT_ANATOMY_KEY, - + PROJECT_ANATOMY_KEY +) +from pype.settings.lib import ( DEFAULTS_DIR, reset_default_settings, From 5d9fb93558e9dfa436afb9d3ad156cfeb89c48d9 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 11 Dec 2020 16:06:26 +0100 Subject: [PATCH 02/17] implemented abstract settings handler SettingsHandler --- pype/settings/handlers.py | 89 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 pype/settings/handlers.py diff --git a/pype/settings/handlers.py b/pype/settings/handlers.py new file mode 100644 index 0000000000..b2381ed4f6 --- /dev/null +++ b/pype/settings/handlers.py @@ -0,0 +1,89 @@ +from abc import ABCMeta, abstractmethod +import six + + +@six.add_metaclass(ABCMeta) +class SettingsHandler: + @abstractmethod + def save_studio_settings(self, data): + """Save studio overrides of system settings. + + Do not use to store whole system settings data with defaults but only + it's overrides with metadata defining how overrides should be applied + in load function. For loading should be used function + `studio_system_settings`. + + Args: + data(dict): Data of studio overrides with override metadata. + """ + pass + + @abstractmethod + def save_project_settings(self, project_name, overrides): + """Save studio overrides of project settings. + + Data are saved for specific project or as defaults for all projects. + + Do not use to store whole project settings data with defaults but only + it's overrides with metadata defining how overrides should be applied + in load function. For loading should be used function + `get_studio_project_settings_overrides` for global project settings + and `get_project_settings_overrides` for project specific settings. + + Args: + project_name(str, null): Project name for which overrides are + or None for global settings. + data(dict): Data of project overrides with override metadata. + """ + pass + + @abstractmethod + def save_project_anatomy(self, project_name, anatomy_data): + """Save studio overrides of project anatomy data. + + Args: + project_name(str, null): Project name for which overrides are + or None for global settings. + data(dict): Data of project overrides with override metadata. + """ + pass + + @abstractmethod + def get_studio_system_settings_overrides(self): + """Studio overrides of system settings.""" + pass + + @abstractmethod + def get_studio_project_settings_overrides(self): + """Studio overrides of default project settings.""" + pass + + @abstractmethod + def get_studio_project_anatomy_overrides(self): + """Studio overrides of default project anatomy data.""" + pass + + @abstractmethod + def get_project_settings_overrides(self, project_name): + """Studio overrides of project settings for specific project. + + Args: + project_name(str): Name of project for which data should be loaded. + + Returns: + dict: Only overrides for entered project, may be empty dictionary. + """ + pass + + @abstractmethod + def get_project_anatomy_overrides(self, project_name): + """Studio overrides of project anatomy for specific project. + + Args: + project_name(str): Name of project for which data should be loaded. + + Returns: + dict: Only overrides for entered project, may be empty dictionary. + """ + pass + From de92f0a8d3c06f3c282b827d6eb80e1a188d26fd Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 11 Dec 2020 16:07:47 +0100 Subject: [PATCH 03/17] extracted current storing process to SettingsFileHandler --- pype/settings/handlers.py | 208 +++++++++++++++++++ pype/settings/lib.py | 163 ++------------- pype/tools/settings/settings/widgets/base.py | 49 +++-- 3 files changed, 248 insertions(+), 172 deletions(-) diff --git a/pype/settings/handlers.py b/pype/settings/handlers.py index b2381ed4f6..c1829273b1 100644 --- a/pype/settings/handlers.py +++ b/pype/settings/handlers.py @@ -1,5 +1,24 @@ +import os +import json +import logging from abc import ABCMeta, abstractmethod import six +import pype +from .constants import ( + M_OVERRIDEN_KEY, + M_ENVIRONMENT_KEY, + M_DYNAMIC_KEY_LABEL, + M_POP_KEY, + + METADATA_KEYS, + + SYSTEM_SETTINGS_KEY, + ENVIRONMENTS_KEY, + PROJECT_SETTINGS_KEY, + PROJECT_ANATOMY_KEY +) + +JSON_EXC = getattr(json.decoder, "JSONDecodeError", ValueError) @six.add_metaclass(ABCMeta) @@ -87,3 +106,192 @@ class SettingsHandler: """ pass + +class SettingsFileHandler(SettingsHandler): + def __init__(self): + self.log = logging.getLogger("SettingsFileHandler") + + # Folder where studio overrides are stored + studio_overrides_dir = os.getenv("PYPE_PROJECT_CONFIGS") + if not studio_overrides_dir: + studio_overrides_dir = os.path.dirname(os.path.dirname( + os.path.abspath(pype.__file__) + )) + system_settings_path = os.path.join( + studio_overrides_dir, SYSTEM_SETTINGS_KEY + ".json" + ) + + # File where studio's default project overrides are stored + project_settings_filename = PROJECT_SETTINGS_KEY + ".json" + project_settings_path = os.path.join( + studio_overrides_dir, project_settings_filename + ) + + project_anatomy_filename = PROJECT_ANATOMY_KEY + ".json" + project_anatomy_path = os.path.join( + studio_overrides_dir, project_anatomy_filename + ) + + self.studio_overrides_dir = studio_overrides_dir + self.system_settings_path = system_settings_path + + self.project_settings_filename = project_settings_filename + self.project_anatomy_filename = project_anatomy_filename + + self.project_settings_path = project_settings_path + self.project_anatomy_path = project_anatomy_path + + def load_json_file(self, fpath): + # Load json data + try: + with open(fpath, "r") as opened_file: + return json.load(opened_file) + + except JSON_EXC: + self.log.warning( + "File has invalid json format \"{}\"".format(fpath), + exc_info=True + ) + return {} + + def path_to_project_settings(self, project_name): + if not project_name: + return self.project_settings_path + return os.path.join( + self.studio_overrides_dir, + project_name, + self.project_settings_filename + ) + + def path_to_project_anatomy(self, project_name): + if not project_name: + return self.project_anatomy_path + return os.path.join( + self.studio_overrides_dir, + project_name, + self.project_anatomy_filename + ) + + def save_studio_settings(self, data): + """Save studio overrides of system settings. + + Do not use to store whole system settings data with defaults but only + it's overrides with metadata defining how overrides should be applied + in load function. For loading should be used function + `studio_system_settings`. + + Args: + data(dict): Data of studio overrides with override metadata. + """ + dirpath = os.path.dirname(self.system_settings_path) + if not os.path.exists(dirpath): + os.makedirs(dirpath) + + self.log.debug( + "Saving studio overrides. Output path: {}".format( + self.system_settings_path + ) + ) + with open(self.system_settings_path, "w") as file_stream: + json.dump(data, file_stream, indent=4) + + def save_project_settings(self, project_name, overrides): + """Save studio overrides of project settings. + + Data are saved for specific project or as defaults for all projects. + + Do not use to store whole project settings data with defaults but only + it's overrides with metadata defining how overrides should be applied + in load function. For loading should be used function + `get_studio_project_settings_overrides` for global project settings + and `get_project_settings_overrides` for project specific settings. + + Args: + project_name(str, null): Project name for which overrides are + or None for global settings. + data(dict): Data of project overrides with override metadata. + """ + project_overrides_json_path = self.path_to_project_settings( + project_name + ) + dirpath = os.path.dirname(project_overrides_json_path) + if not os.path.exists(dirpath): + os.makedirs(dirpath) + + self.log.debug( + "Saving overrides of project \"{}\". Output path: {}".format( + project_name, project_overrides_json_path + ) + ) + with open(project_overrides_json_path, "w") as file_stream: + json.dump(overrides, file_stream, indent=4) + + def save_project_anatomy(self, project_name, anatomy_data): + """Save studio overrides of project anatomy data. + + Args: + project_name(str, null): Project name for which overrides are + or None for global settings. + data(dict): Data of project overrides with override metadata. + """ + project_anatomy_json_path = self.path_to_project_anatomy(project_name) + dirpath = os.path.dirname(project_anatomy_json_path) + if not os.path.exists(dirpath): + os.makedirs(dirpath) + + self.log.debug( + "Saving anatomy of project \"{}\". Output path: {}".format( + project_name, project_anatomy_json_path + ) + ) + with open(project_anatomy_json_path, "w") as file_stream: + json.dump(anatomy_data, file_stream, indent=4) + + def get_studio_system_settings_overrides(self): + """Studio overrides of system settings.""" + if os.path.exists(self.system_settings_path): + return self.load_json_file(self.system_settings_path) + return {} + + def get_studio_project_settings_overrides(self): + """Studio overrides of default project settings.""" + if os.path.exists(self.project_settings_path): + return self.load_json_file(self.project_settings_path) + return {} + + def get_studio_project_anatomy_overrides(self): + """Studio overrides of default project anatomy data.""" + if os.path.exists(self.project_anatomy_path): + return self.load_json_file(self.project_anatomy_path) + return {} + + def get_project_settings_overrides(self, project_name): + """Studio overrides of project settings for specific project. + + Args: + project_name(str): Name of project for which data should be loaded. + + Returns: + dict: Only overrides for entered project, may be empty dictionary. + """ + path_to_json = self.path_to_project_settings(project_name) + if not os.path.exists(path_to_json): + return {} + return self.load_json_file(path_to_json) + + def get_project_anatomy_overrides(self, project_name): + """Studio overrides of project anatomy for specific project. + + Args: + project_name(str): Name of project for which data should be loaded. + + Returns: + dict: Only overrides for entered project, may be empty dictionary. + """ + if not project_name: + return {} + + path_to_json = self.path_to_project_anatomy(project_name) + if not os.path.exists(path_to_json): + return {} + return self.load_json_file(path_to_json) diff --git a/pype/settings/lib.py b/pype/settings/lib.py index 62febfc1d2..0852c92a24 100644 --- a/pype/settings/lib.py +++ b/pype/settings/lib.py @@ -15,6 +15,8 @@ from .constants import ( PROJECT_SETTINGS_KEY, PROJECT_ANATOMY_KEY ) +from .handlers import ( + SettingsFileHandler ) log = logging.getLogger(__name__) @@ -29,6 +31,9 @@ DEFAULTS_DIR = os.path.join(os.path.dirname(__file__), "defaults") # Variable where cache of default settings are stored _DEFAULT_SETTINGS = None +# Handler of studio overrides +SETTINGS_HANDLER = SettingsFileHandler() + class DuplicatedEnvGroups(Exception): def __init__(self, duplicated): @@ -220,150 +225,6 @@ def subkey_merge(_dict, value, keys): return _dict -def path_to_project_settings(project_name): - if not project_name: - return PROJECT_SETTINGS_PATH - return os.path.join( - STUDIO_OVERRIDES_PATH, - project_name, - PROJECT_SETTINGS_FILENAME - ) - - -def path_to_project_anatomy(project_name): - if not project_name: - return PROJECT_ANATOMY_PATH - return os.path.join( - STUDIO_OVERRIDES_PATH, - project_name, - PROJECT_ANATOMY_FILENAME - ) - - -def save_studio_settings(data): - """Save studio overrides of system settings. - - Do not use to store whole system settings data with defaults but only it's - overrides with metadata defining how overrides should be applied in load - function. For loading should be used function `studio_system_settings`. - - Args: - data(dict): Data of studio overrides with override metadata. - """ - dirpath = os.path.dirname(SYSTEM_SETTINGS_PATH) - if not os.path.exists(dirpath): - os.makedirs(dirpath) - - print("Saving studio overrides. Output path: {}".format( - SYSTEM_SETTINGS_PATH - )) - with open(SYSTEM_SETTINGS_PATH, "w") as file_stream: - json.dump(data, file_stream, indent=4) - - -def save_project_settings(project_name, overrides): - """Save studio overrides of project settings. - - Data are saved for specific project or as defaults for all projects. - - Do not use to store whole project settings data with defaults but only it's - overrides with metadata defining how overrides should be applied in load - function. For loading should be used function - `get_studio_project_settings_overrides` for global project settings - and `get_project_settings_overrides` for project specific settings. - - Args: - project_name(str, null): Project name for which overrides are - or None for global settings. - data(dict): Data of project overrides with override metadata. - """ - project_overrides_json_path = path_to_project_settings(project_name) - dirpath = os.path.dirname(project_overrides_json_path) - if not os.path.exists(dirpath): - os.makedirs(dirpath) - - print("Saving overrides of project \"{}\". Output path: {}".format( - project_name, project_overrides_json_path - )) - with open(project_overrides_json_path, "w") as file_stream: - json.dump(overrides, file_stream, indent=4) - - -def save_project_anatomy(project_name, anatomy_data): - """Save studio overrides of project anatomy data. - - Args: - project_name(str, null): Project name for which overrides are - or None for global settings. - data(dict): Data of project overrides with override metadata. - """ - project_anatomy_json_path = path_to_project_anatomy(project_name) - dirpath = os.path.dirname(project_anatomy_json_path) - if not os.path.exists(dirpath): - os.makedirs(dirpath) - - print("Saving anatomy of project \"{}\". Output path: {}".format( - project_name, project_anatomy_json_path - )) - with open(project_anatomy_json_path, "w") as file_stream: - json.dump(anatomy_data, file_stream, indent=4) - - -def get_studio_system_settings_overrides(): - """Studio overrides of system settings.""" - if os.path.exists(SYSTEM_SETTINGS_PATH): - return load_json_file(SYSTEM_SETTINGS_PATH) - return {} - - -def get_studio_project_settings_overrides(): - """Studio overrides of default project settings.""" - if os.path.exists(PROJECT_SETTINGS_PATH): - return load_json_file(PROJECT_SETTINGS_PATH) - return {} - - -def get_studio_project_anatomy_overrides(): - """Studio overrides of default project anatomy data.""" - if os.path.exists(PROJECT_ANATOMY_PATH): - return load_json_file(PROJECT_ANATOMY_PATH) - return {} - - -def get_project_settings_overrides(project_name): - """Studio overrides of project settings for specific project. - - Args: - project_name(str): Name of project for which data should be loaded. - - Returns: - dict: Only overrides for entered project, may be empty dictionary. - """ - - path_to_json = path_to_project_settings(project_name) - if not os.path.exists(path_to_json): - return {} - return load_json_file(path_to_json) - - -def get_project_anatomy_overrides(project_name): - """Studio overrides of project anatomy for specific project. - - Args: - project_name(str): Name of project for which data should be loaded. - - Returns: - dict: Only overrides for entered project, may be empty dictionary. - """ - if not project_name: - return {} - - path_to_json = path_to_project_anatomy(project_name) - if not os.path.exists(path_to_json): - return {} - return load_json_file(path_to_json) - - def merge_overrides(source_dict, override_dict): """Merge data from override_dict to source_dict.""" @@ -397,7 +258,7 @@ def apply_overrides(source_data, override_data): def get_system_settings(clear_metadata=True): """System settings with applied studio overrides.""" default_values = get_default_settings()[SYSTEM_SETTINGS_KEY] - studio_values = get_studio_system_settings_overrides() + studio_values = SETTINGS_HANDLER.get_studio_system_settings_overrides() result = apply_overrides(default_values, studio_values) if clear_metadata: clear_metadata_from_settings(result) @@ -407,7 +268,7 @@ def get_system_settings(clear_metadata=True): def get_default_project_settings(clear_metadata=True): """Project settings with applied studio's default project overrides.""" default_values = get_default_settings()[PROJECT_SETTINGS_KEY] - studio_values = get_studio_project_settings_overrides() + studio_values = SETTINGS_HANDLER.get_studio_project_settings_overrides() result = apply_overrides(default_values, studio_values) if clear_metadata: clear_metadata_from_settings(result) @@ -417,7 +278,7 @@ def get_default_project_settings(clear_metadata=True): def get_default_anatomy_settings(clear_metadata=True): """Project anatomy data with applied studio's default project overrides.""" default_values = get_default_settings()[PROJECT_ANATOMY_KEY] - studio_values = get_studio_project_anatomy_overrides() + studio_values = SETTINGS_HANDLER.get_studio_project_anatomy_overrides() result = apply_overrides(default_values, studio_values) if clear_metadata: clear_metadata_from_settings(result) @@ -433,7 +294,9 @@ def get_anatomy_settings(project_name, clear_metadata=True): ) studio_overrides = get_default_anatomy_settings(False) - project_overrides = get_project_anatomy_overrides(project_name) + project_overrides = SETTINGS_HANDLER.get_project_anatomy_overrides( + project_name + ) result = apply_overrides(studio_overrides, project_overrides) if clear_metadata: @@ -450,7 +313,9 @@ def get_project_settings(project_name, clear_metadata=True): ) studio_overrides = get_default_project_settings(False) - project_overrides = get_project_settings_overrides(project_name) + project_overrides = SETTINGS_HANDLER.get_project_settings_overrides( + project_name + ) result = apply_overrides(studio_overrides, project_overrides) if clear_metadata: diff --git a/pype/tools/settings/settings/widgets/base.py b/pype/tools/settings/settings/widgets/base.py index ddd814e38e..7c09abad73 100644 --- a/pype/tools/settings/settings/widgets/base.py +++ b/pype/tools/settings/settings/widgets/base.py @@ -9,21 +9,11 @@ from pype.settings.constants import ( ) from pype.settings.lib import ( DEFAULTS_DIR, + SETTINGS_HANDLER, reset_default_settings, get_default_settings, - get_studio_system_settings_overrides, - get_studio_project_settings_overrides, - get_studio_project_anatomy_overrides, - - get_project_settings_overrides, - get_project_anatomy_overrides, - - save_studio_settings, - save_project_settings, - save_project_anatomy, - apply_overrides, find_environments, DuplicatedEnvGroups @@ -370,7 +360,7 @@ class SystemWidget(SettingsCategoryWidget): if not self.duplicated_env_group_validation(overrides=values): return - save_studio_settings(values) + SETTINGS_HANDLER.save_studio_settings(values) def update_values(self): default_values = lib.convert_data_to_gui_data({ @@ -382,8 +372,11 @@ class SystemWidget(SettingsCategoryWidget): if self._hide_studio_overrides: system_values = lib.NOT_SET else: + studio_system_overrides = ( + SETTINGS_HANDLER.get_studio_system_settings_overrides() + ) system_values = lib.convert_overrides_to_gui_data( - {self.main_schema_key: get_studio_system_settings_overrides()} + {self.main_schema_key: studio_system_overrides} ) for input_field in self.input_fields: @@ -547,8 +540,12 @@ class ProjectWidget(SettingsCategoryWidget): _project_anatomy = lib.NOT_SET self.is_overidable = False else: - _project_overrides = get_project_settings_overrides(project_name) - _project_anatomy = get_project_anatomy_overrides(project_name) + _project_overrides = ( + SETTINGS_HANDLER.get_project_settings_overrides(project_name) + ) + _project_anatomy = ( + SETTINGS_HANDLER.get_project_anatomy_overrides(project_name) + ) self.is_overidable = True overrides = {self.main_schema_key: { @@ -582,11 +579,15 @@ class ProjectWidget(SettingsCategoryWidget): # Saving overrides data project_overrides_data = output_data.get(PROJECT_SETTINGS_KEY, {}) - save_project_settings(self.project_name, project_overrides_data) + SETTINGS_HANDLER.save_project_settings( + self.project_name, project_overrides_data + ) # Saving anatomy data project_anatomy_data = output_data.get(PROJECT_ANATOMY_KEY, {}) - save_project_anatomy(self.project_name, project_anatomy_data) + SETTINGS_HANDLER.save_project_anatomy( + self.project_name, project_anatomy_data + ) def update_values(self): if self.project_name is not None: @@ -602,14 +603,16 @@ class ProjectWidget(SettingsCategoryWidget): if self._hide_studio_overrides: studio_values = lib.NOT_SET else: + project_settings_value = ( + SETTINGS_HANDLER.get_studio_project_settings_overrides() + ) + project_anatomy_value = ( + SETTINGS_HANDLER.get_studio_project_anatomy_overrides() + ) studio_values = lib.convert_overrides_to_gui_data({ self.main_schema_key: { - PROJECT_SETTINGS_KEY: ( - get_studio_project_settings_overrides() - ), - PROJECT_ANATOMY_KEY: ( - get_studio_project_anatomy_overrides() - ) + PROJECT_SETTINGS_KEY: project_settings_value, + PROJECT_ANATOMY_KEY: project_anatomy_value } }) From 50434ecb6022bdbf55ba36f7db0bb6aa8bfde2b0 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 11 Dec 2020 16:08:04 +0100 Subject: [PATCH 04/17] use absolute path to defaults --- pype/settings/lib.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pype/settings/lib.py b/pype/settings/lib.py index 0852c92a24..e9f1a15feb 100644 --- a/pype/settings/lib.py +++ b/pype/settings/lib.py @@ -26,7 +26,10 @@ JSON_EXC = getattr(json.decoder, "JSONDecodeError", ValueError) # Path to default settings -DEFAULTS_DIR = os.path.join(os.path.dirname(__file__), "defaults") +DEFAULTS_DIR = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "defaults" +) # Variable where cache of default settings are stored _DEFAULT_SETTINGS = None From 19b2c77621e2a8cfb91ab5322e24d0ee12486e2a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 11 Dec 2020 16:23:00 +0100 Subject: [PATCH 05/17] implemented naive singleton for mongo connection for pype's purposes --- pype/lib/__init__.py | 3 +- pype/lib/mongo.py | 86 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 85 insertions(+), 4 deletions(-) diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index 02b5178311..e856956822 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -7,7 +7,8 @@ from .log import PypeLogger, timeit from .mongo import ( decompose_url, compose_url, - get_default_components + get_default_components, + PypeMongoConnection ) from .anatomy import Anatomy diff --git a/pype/lib/mongo.py b/pype/lib/mongo.py index f950572c6e..f82c8b5e23 100644 --- a/pype/lib/mongo.py +++ b/pype/lib/mongo.py @@ -1,9 +1,13 @@ import os +import sys +import time +import logging +import pymongo -try: - from urllib.parse import urlparse, parse_qs -except ImportError: +if sys.version_info[0] == 2: from urlparse import urlparse, parse_qs +else: + from urllib.parse import urlparse, parse_qs class MongoEnvNotSet(Exception): @@ -79,3 +83,79 @@ def get_default_components(): "URL for Mongo logging connection is not set." ) return decompose_url(mongo_url) + + +def extract_port_from_url(url): + parsed_url = urlparse(url) + if parsed_url.scheme is None: + _url = "mongodb://{}".format(url) + parsed_url = urlparse(_url) + return parsed_url.port + + +class PypeMongoConnection: + """Singleton MongoDB connection. + + Keeps MongoDB connections by url. + """ + mongo_clients = {} + log = logging.getLogger("PypeMongoConnection") + + @classmethod + def get_mongo_client(cls, mongo_url=None): + if mongo_url is None: + mongo_url = os.environ["PYPE_MONGO"] + + connection = cls.mongo_clients.get(mongo_url) + if connection: + # Naive validation of existing connection + try: + connection.server_info() + except Exception: + connection = None + + if not connection: + cls.log.debug("Creating mongo connection to {}".format(mongo_url)) + connection = cls.create_connection(mongo_url) + cls.mongo_clients[mongo_url] = connection + + return connection + + @classmethod + def create_connection(cls, mongo_url, timeout=None): + if timeout is None: + timeout = int(os.environ.get("AVALON_TIMEOUT") or 1000) + + kwargs = { + "host": mongo_url, + "serverSelectionTimeoutMS": timeout + } + + port = extract_port_from_url(mongo_url) + if port is not None: + kwargs["port"] = int(port) + + mongo_client = pymongo.MongoClient(**kwargs) + + for _retry in range(3): + try: + t1 = time.time() + mongo_client.server_info() + + except Exception: + cls.log.warning("Retrying...") + time.sleep(1) + timeout *= 1.5 + + else: + break + + else: + raise IOError(( + "ERROR: Couldn't connect to {} in less than {:.3f}ms" + ).format(mongo_url, timeout)) + + cls.log.info("Connected to {}, delay {:.3f}s".format( + mongo_url, time.time() - t1 + )) + return mongo_client From b6120e3a23e8d8eb9e9dde55d065ddf872711986 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 11 Dec 2020 16:33:17 +0100 Subject: [PATCH 06/17] prepared structure of MongoSettingsHandler --- pype/settings/handlers.py | 80 +++++++++++++++++++++++++++++++++++---- 1 file changed, 72 insertions(+), 8 deletions(-) diff --git a/pype/settings/handlers.py b/pype/settings/handlers.py index c1829273b1..300d4492eb 100644 --- a/pype/settings/handlers.py +++ b/pype/settings/handlers.py @@ -5,15 +5,7 @@ from abc import ABCMeta, abstractmethod import six import pype from .constants import ( - M_OVERRIDEN_KEY, - M_ENVIRONMENT_KEY, - M_DYNAMIC_KEY_LABEL, - M_POP_KEY, - - METADATA_KEYS, - SYSTEM_SETTINGS_KEY, - ENVIRONMENTS_KEY, PROJECT_SETTINGS_KEY, PROJECT_ANATOMY_KEY ) @@ -295,3 +287,75 @@ class SettingsFileHandler(SettingsHandler): if not os.path.exists(path_to_json): return {} return self.load_json_file(path_to_json) + + +class MongoSettingsHandler(SettingsHandler): + def __init__(self): + pass + + @classmethod + def save_studio_settings(self, data): + """Save studio overrides of system settings. + + Do not use to store whole system settings data with defaults but only it's + overrides with metadata defining how overrides should be applied in load + function. For loading should be used function `studio_system_settings`. + + Args: + data(dict): Data of studio overrides with override metadata. + """ + + def save_project_settings(self, project_name, overrides): + """Save studio overrides of project settings. + + Data are saved for specific project or as defaults for all projects. + + Do not use to store whole project settings data with defaults but only it's + overrides with metadata defining how overrides should be applied in load + function. For loading should be used function + `get_studio_project_settings_overrides` for global project settings + and `get_project_settings_overrides` for project specific settings. + + Args: + project_name(str, null): Project name for which overrides are + or None for global settings. + data(dict): Data of project overrides with override metadata. + """ + + def save_project_anatomy(self, project_name, anatomy_data): + """Save studio overrides of project anatomy data. + + Args: + project_name(str, null): Project name for which overrides are + or None for global settings. + data(dict): Data of project overrides with override metadata. + """ + + def get_studio_system_settings_overrides(self): + """Studio overrides of system settings.""" + + def get_studio_project_settings_overrides(self): + """Studio overrides of default project settings.""" + + def get_studio_project_anatomy_overrides(self): + """Studio overrides of default project anatomy data.""" + + def get_project_settings_overrides(self, project_name): + """Studio overrides of project settings for specific project. + + Args: + project_name(str): Name of project for which data should be loaded. + + Returns: + dict: Only overrides for entered project, may be empty dictionary. + """ + + def get_project_anatomy_overrides(self, project_name): + """Studio overrides of project anatomy for specific project. + + Args: + project_name(str): Name of project for which data should be loaded. + + Returns: + dict: Only overrides for entered project, may be empty dictionary. + """ \ No newline at end of file From 9299eb55e968d8a735af500240df83fc3c34b07c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 11 Dec 2020 16:34:31 +0100 Subject: [PATCH 07/17] removed ENVIRONEMNTS_KEY as not used --- pype/settings/constants.py | 2 -- pype/settings/lib.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/pype/settings/constants.py b/pype/settings/constants.py index 4a9f0e2287..d21817fb8e 100644 --- a/pype/settings/constants.py +++ b/pype/settings/constants.py @@ -16,7 +16,6 @@ METADATA_KEYS = ( # File where studio's system overrides are stored SYSTEM_SETTINGS_KEY = "system_settings" -ENVIRONMENTS_KEY = "environments" PROJECT_SETTINGS_KEY = "project_settings" PROJECT_ANATOMY_KEY = "project_anatomy" @@ -30,7 +29,6 @@ __all__ = ( "METADATA_KEYS", "SYSTEM_SETTINGS_KEY", - "ENVIRONMENTS_KEY", "PROJECT_SETTINGS_KEY", "PROJECT_ANATOMY_KEY" ) diff --git a/pype/settings/lib.py b/pype/settings/lib.py index e9f1a15feb..009d855e02 100644 --- a/pype/settings/lib.py +++ b/pype/settings/lib.py @@ -5,13 +5,11 @@ import copy from .constants import ( M_OVERRIDEN_KEY, M_ENVIRONMENT_KEY, - M_DYNAMIC_KEY_LABEL, M_POP_KEY, METADATA_KEYS, SYSTEM_SETTINGS_KEY, - ENVIRONMENTS_KEY, PROJECT_SETTINGS_KEY, PROJECT_ANATOMY_KEY ) From 1a89b125bec4564b29b37a3eb52786476368bb34 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 11 Dec 2020 17:27:46 +0100 Subject: [PATCH 08/17] filled basic implementation of mongo settings handler --- pype/settings/handlers.py | 93 +++++++++++++++++++++++++++++++++++---- 1 file changed, 85 insertions(+), 8 deletions(-) diff --git a/pype/settings/handlers.py b/pype/settings/handlers.py index 300d4492eb..cc6e7f9d10 100644 --- a/pype/settings/handlers.py +++ b/pype/settings/handlers.py @@ -290,29 +290,60 @@ class SettingsFileHandler(SettingsHandler): class MongoSettingsHandler(SettingsHandler): + """Settings handler that use mongo for storing and loading of settings.""" + system_settings_key = "system_settings" + project_anatomy_key = "project_anatomy" + project_settings_key = "project_settings" + def __init__(self): - pass + # Get mongo connection + from pype.lib import PypeMongoConnection + settings_collection = PypeMongoConnection.get_mongo_client() + + # TODO prepare version of pype + # - pype version should define how are settings saved and loaded + + # TODO modify to not use hardcoded keys + database_name = "pype" + collection_name = "settings" + + self.settings_collection = settings_collection + + self.database_name = database_name + self.collection_name = collection_name + + self.collection = settings_collection[database_name][collection_name] @classmethod def save_studio_settings(self, data): """Save studio overrides of system settings. - Do not use to store whole system settings data with defaults but only it's - overrides with metadata defining how overrides should be applied in load - function. For loading should be used function `studio_system_settings`. + Do not use to store whole system settings data with defaults but only + it's overrides with metadata defining how overrides should be applied + in load function. For loading should be used function + `studio_system_settings`. Args: data(dict): Data of studio overrides with override metadata. """ + self.collection.replace_one( + { + "type": self.system_settings_key + }, + { + "type": self.system_settings_key, + "value": data + } + ) def save_project_settings(self, project_name, overrides): """Save studio overrides of project settings. Data are saved for specific project or as defaults for all projects. - Do not use to store whole project settings data with defaults but only it's - overrides with metadata defining how overrides should be applied in load - function. For loading should be used function + Do not use to store whole project settings data with defaults but only + it's overrides with metadata defining how overrides should be applied + in load function. For loading should be used function `get_studio_project_settings_overrides` for global project settings and `get_project_settings_overrides` for project specific settings. @@ -322,6 +353,18 @@ class MongoSettingsHandler(SettingsHandler): data(dict): Data of project overrides with override metadata. """ + self.collection.replace_one( + { + "type": self.project_settings_key, + "project_name": project_name + }, + { + "type": self.project_settings_key, + "project_name": project_name, + "value": overrides + } + ) + def save_project_anatomy(self, project_name, anatomy_data): """Save studio overrides of project anatomy data. @@ -330,15 +373,37 @@ class MongoSettingsHandler(SettingsHandler): or None for global settings. data(dict): Data of project overrides with override metadata. """ + self.collection.replace_one( + { + "type": self.project_anatomy_key, + "project_name": project_name + }, + { + "type": self.project_anatomy_key, + "project_name": project_name, + "value": anatomy_data + } + ) def get_studio_system_settings_overrides(self): """Studio overrides of system settings.""" + return self.collection.find_one({ + "type": self.system_settings_key + }) or {} def get_studio_project_settings_overrides(self): """Studio overrides of default project settings.""" + return self.collection.find_one({ + "type": self.project_settings_key, + "project_name": None + }) or {} def get_studio_project_anatomy_overrides(self): """Studio overrides of default project anatomy data.""" + return self.collection.find_one({ + "type": self.project_anatomy_key, + "project_name": None + }) or {} def get_project_settings_overrides(self, project_name): """Studio overrides of project settings for specific project. @@ -349,6 +414,12 @@ class MongoSettingsHandler(SettingsHandler): Returns: dict: Only overrides for entered project, may be empty dictionary. """ + if not project_name: + return {} + return self.collection.find_one({ + "type": self.project_settings_key, + "project_name": project_name + }) or {} def get_project_anatomy_overrides(self, project_name): """Studio overrides of project anatomy for specific project. @@ -358,4 +429,10 @@ class MongoSettingsHandler(SettingsHandler): Returns: dict: Only overrides for entered project, may be empty dictionary. - """ \ No newline at end of file + """ + if not project_name: + return {} + return self.collection.find_one({ + "type": self.project_anatomy_key, + "project_name": project_name + }) or {} From c6fbd4797945d0458673484b9ad20007790138f6 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 11 Dec 2020 17:28:32 +0100 Subject: [PATCH 09/17] moved back settings lib functions and _SETTINGS_HANDLER is set with wrapper --- pype/settings/lib.py | 71 ++++++++++++++++++-- pype/tools/settings/settings/widgets/base.py | 50 +++++++------- pype/tools/settings/settings/widgets/lib.py | 2 +- 3 files changed, 89 insertions(+), 34 deletions(-) diff --git a/pype/settings/lib.py b/pype/settings/lib.py index 009d855e02..d1d050cdaf 100644 --- a/pype/settings/lib.py +++ b/pype/settings/lib.py @@ -1,5 +1,6 @@ import os import json +import functools import logging import copy from .constants import ( @@ -14,7 +15,8 @@ from .constants import ( PROJECT_ANATOMY_KEY ) from .handlers import ( - SettingsFileHandler + SettingsFileHandler, + MongoSettingsHandler ) log = logging.getLogger(__name__) @@ -33,7 +35,62 @@ DEFAULTS_DIR = os.path.join( _DEFAULT_SETTINGS = None # Handler of studio overrides -SETTINGS_HANDLER = SettingsFileHandler() +_SETTINGS_HANDLER = None + + +def require_handler(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + global _SETTINGS_HANDLER + if _SETTINGS_HANDLER is None: + _SETTINGS_HANDLER = create_settings_handler() + return func(*args, **kwargs) + return wrapper + + +def create_settings_handler(): + # This may be logic which handler is used (in future) + return MongoSettingsHandler() + + +@require_handler +def save_studio_settings(data): + _SETTINGS_HANDLER.save_studio_settings(data) + + +@require_handler +def save_project_settings(project_name, overrides): + _SETTINGS_HANDLER.save_project_settings(project_name, overrides) + + +@require_handler +def save_project_anatomy(project_name, anatomy_data): + _SETTINGS_HANDLER.save_project_anatomy(project_name, anatomy_data) + + +@require_handler +def get_studio_system_settings_overrides(): + _SETTINGS_HANDLER.get_studio_system_settings_overrides() + + +@require_handler +def get_studio_project_settings_overrides(): + _SETTINGS_HANDLER.get_studio_project_settings_overrides() + + +@require_handler +def get_studio_project_anatomy_overrides(): + _SETTINGS_HANDLER.get_studio_project_anatomy_overrides() + + +@require_handler +def get_project_settings_overrides(project_name): + _SETTINGS_HANDLER.get_project_settings_overrides(project_name) + + +@require_handler +def get_project_anatomy_overrides(project_name): + _SETTINGS_HANDLER.get_project_anatomy_overrides(project_name) class DuplicatedEnvGroups(Exception): @@ -259,7 +316,7 @@ def apply_overrides(source_data, override_data): def get_system_settings(clear_metadata=True): """System settings with applied studio overrides.""" default_values = get_default_settings()[SYSTEM_SETTINGS_KEY] - studio_values = SETTINGS_HANDLER.get_studio_system_settings_overrides() + studio_values = get_studio_system_settings_overrides() result = apply_overrides(default_values, studio_values) if clear_metadata: clear_metadata_from_settings(result) @@ -269,7 +326,7 @@ def get_system_settings(clear_metadata=True): def get_default_project_settings(clear_metadata=True): """Project settings with applied studio's default project overrides.""" default_values = get_default_settings()[PROJECT_SETTINGS_KEY] - studio_values = SETTINGS_HANDLER.get_studio_project_settings_overrides() + studio_values = get_studio_project_settings_overrides() result = apply_overrides(default_values, studio_values) if clear_metadata: clear_metadata_from_settings(result) @@ -279,7 +336,7 @@ def get_default_project_settings(clear_metadata=True): def get_default_anatomy_settings(clear_metadata=True): """Project anatomy data with applied studio's default project overrides.""" default_values = get_default_settings()[PROJECT_ANATOMY_KEY] - studio_values = SETTINGS_HANDLER.get_studio_project_anatomy_overrides() + studio_values = get_studio_project_anatomy_overrides() result = apply_overrides(default_values, studio_values) if clear_metadata: clear_metadata_from_settings(result) @@ -295,7 +352,7 @@ def get_anatomy_settings(project_name, clear_metadata=True): ) studio_overrides = get_default_anatomy_settings(False) - project_overrides = SETTINGS_HANDLER.get_project_anatomy_overrides( + project_overrides = get_project_anatomy_overrides( project_name ) @@ -314,7 +371,7 @@ def get_project_settings(project_name, clear_metadata=True): ) studio_overrides = get_default_project_settings(False) - project_overrides = SETTINGS_HANDLER.get_project_settings_overrides( + project_overrides = get_project_settings_overrides( project_name ) diff --git a/pype/tools/settings/settings/widgets/base.py b/pype/tools/settings/settings/widgets/base.py index 7c09abad73..8e16c3614c 100644 --- a/pype/tools/settings/settings/widgets/base.py +++ b/pype/tools/settings/settings/widgets/base.py @@ -7,13 +7,24 @@ from pype.settings.constants import ( PROJECT_SETTINGS_KEY, PROJECT_ANATOMY_KEY ) + from pype.settings.lib import ( DEFAULTS_DIR, - SETTINGS_HANDLER, reset_default_settings, get_default_settings, + get_studio_system_settings_overrides, + get_studio_project_settings_overrides, + get_studio_project_anatomy_overrides, + + get_project_settings_overrides, + get_project_anatomy_overrides, + + save_studio_settings, + save_project_settings, + save_project_anatomy, + apply_overrides, find_environments, DuplicatedEnvGroups @@ -360,7 +371,7 @@ class SystemWidget(SettingsCategoryWidget): if not self.duplicated_env_group_validation(overrides=values): return - SETTINGS_HANDLER.save_studio_settings(values) + save_studio_settings(values) def update_values(self): default_values = lib.convert_data_to_gui_data({ @@ -372,11 +383,8 @@ class SystemWidget(SettingsCategoryWidget): if self._hide_studio_overrides: system_values = lib.NOT_SET else: - studio_system_overrides = ( - SETTINGS_HANDLER.get_studio_system_settings_overrides() - ) system_values = lib.convert_overrides_to_gui_data( - {self.main_schema_key: studio_system_overrides} + {self.main_schema_key: get_studio_system_settings_overrides()} ) for input_field in self.input_fields: @@ -540,12 +548,8 @@ class ProjectWidget(SettingsCategoryWidget): _project_anatomy = lib.NOT_SET self.is_overidable = False else: - _project_overrides = ( - SETTINGS_HANDLER.get_project_settings_overrides(project_name) - ) - _project_anatomy = ( - SETTINGS_HANDLER.get_project_anatomy_overrides(project_name) - ) + _project_overrides = get_project_settings_overrides(project_name) + _project_anatomy = get_project_anatomy_overrides(project_name) self.is_overidable = True overrides = {self.main_schema_key: { @@ -579,15 +583,11 @@ class ProjectWidget(SettingsCategoryWidget): # Saving overrides data project_overrides_data = output_data.get(PROJECT_SETTINGS_KEY, {}) - SETTINGS_HANDLER.save_project_settings( - self.project_name, project_overrides_data - ) + save_project_settings(self.project_name, project_overrides_data) # Saving anatomy data project_anatomy_data = output_data.get(PROJECT_ANATOMY_KEY, {}) - SETTINGS_HANDLER.save_project_anatomy( - self.project_name, project_anatomy_data - ) + save_project_anatomy(self.project_name, project_anatomy_data) def update_values(self): if self.project_name is not None: @@ -603,16 +603,14 @@ class ProjectWidget(SettingsCategoryWidget): if self._hide_studio_overrides: studio_values = lib.NOT_SET else: - project_settings_value = ( - SETTINGS_HANDLER.get_studio_project_settings_overrides() - ) - project_anatomy_value = ( - SETTINGS_HANDLER.get_studio_project_anatomy_overrides() - ) studio_values = lib.convert_overrides_to_gui_data({ self.main_schema_key: { - PROJECT_SETTINGS_KEY: project_settings_value, - PROJECT_ANATOMY_KEY: project_anatomy_value + PROJECT_SETTINGS_KEY: ( + get_studio_project_settings_overrides() + ), + PROJECT_ANATOMY_KEY: ( + get_studio_project_anatomy_overrides() + ) } }) diff --git a/pype/tools/settings/settings/widgets/lib.py b/pype/tools/settings/settings/widgets/lib.py index 9a6331009b..9c56f51726 100644 --- a/pype/tools/settings/settings/widgets/lib.py +++ b/pype/tools/settings/settings/widgets/lib.py @@ -2,7 +2,7 @@ import os import re import json import copy -from pype.settings.lib import ( +from pype.settings.constants import ( M_OVERRIDEN_KEY, M_ENVIRONMENT_KEY, M_DYNAMIC_KEY_LABEL From 2816f50473cd0afcf70ab0e19a6a4afd6a388945 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 11 Dec 2020 17:34:38 +0100 Subject: [PATCH 10/17] few minor fixes --- pype/settings/handlers.py | 1 - pype/settings/lib.py | 16 ++++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/pype/settings/handlers.py b/pype/settings/handlers.py index cc6e7f9d10..d43ab8b30d 100644 --- a/pype/settings/handlers.py +++ b/pype/settings/handlers.py @@ -314,7 +314,6 @@ class MongoSettingsHandler(SettingsHandler): self.collection = settings_collection[database_name][collection_name] - @classmethod def save_studio_settings(self, data): """Save studio overrides of system settings. diff --git a/pype/settings/lib.py b/pype/settings/lib.py index d1d050cdaf..9aa4a6a85d 100644 --- a/pype/settings/lib.py +++ b/pype/settings/lib.py @@ -55,42 +55,42 @@ def create_settings_handler(): @require_handler def save_studio_settings(data): - _SETTINGS_HANDLER.save_studio_settings(data) + return _SETTINGS_HANDLER.save_studio_settings(data) @require_handler def save_project_settings(project_name, overrides): - _SETTINGS_HANDLER.save_project_settings(project_name, overrides) + return _SETTINGS_HANDLER.save_project_settings(project_name, overrides) @require_handler def save_project_anatomy(project_name, anatomy_data): - _SETTINGS_HANDLER.save_project_anatomy(project_name, anatomy_data) + return _SETTINGS_HANDLER.save_project_anatomy(project_name, anatomy_data) @require_handler def get_studio_system_settings_overrides(): - _SETTINGS_HANDLER.get_studio_system_settings_overrides() + return _SETTINGS_HANDLER.get_studio_system_settings_overrides() @require_handler def get_studio_project_settings_overrides(): - _SETTINGS_HANDLER.get_studio_project_settings_overrides() + return _SETTINGS_HANDLER.get_studio_project_settings_overrides() @require_handler def get_studio_project_anatomy_overrides(): - _SETTINGS_HANDLER.get_studio_project_anatomy_overrides() + return _SETTINGS_HANDLER.get_studio_project_anatomy_overrides() @require_handler def get_project_settings_overrides(project_name): - _SETTINGS_HANDLER.get_project_settings_overrides(project_name) + return _SETTINGS_HANDLER.get_project_settings_overrides(project_name) @require_handler def get_project_anatomy_overrides(project_name): - _SETTINGS_HANDLER.get_project_anatomy_overrides(project_name) + return _SETTINGS_HANDLER.get_project_anatomy_overrides(project_name) class DuplicatedEnvGroups(Exception): From 87e463ab6c42e814c67b6d4f954a32d4e0249ac2 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 14 Dec 2020 09:54:29 +0100 Subject: [PATCH 11/17] move setting environments after PYPE_MONGO is set --- pype.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pype.py b/pype.py index 2580bafb55..e90c257595 100644 --- a/pype.py +++ b/pype.py @@ -113,7 +113,7 @@ def boot(): """ print(art) - set_environments() + # find pype versions bootstrap = BootstrapRepos() pype_versions = bootstrap.find_pype() @@ -139,6 +139,7 @@ def boot(): else: os.environ["PYPE_MONGO"] = pype_mongo + set_environments() if getattr(sys, 'frozen', False): if not pype_versions: import igniter From 638d9c2843518031d6b8530d2ce05924b975a317 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 14 Dec 2020 10:08:57 +0100 Subject: [PATCH 12/17] implemented class for caches --- pype/settings/handlers.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/pype/settings/handlers.py b/pype/settings/handlers.py index d43ab8b30d..798739e4d0 100644 --- a/pype/settings/handlers.py +++ b/pype/settings/handlers.py @@ -1,6 +1,7 @@ import os import json import logging +import datetime from abc import ABCMeta, abstractmethod import six import pype @@ -289,6 +290,34 @@ class SettingsFileHandler(SettingsHandler): return self.load_json_file(path_to_json) +class CacheValues: + cache_lifetime = 10 + + def __init__(self): + self.data = None + self.creation_time = None + + def update_data(self, data): + self.data = data + self.creation_time = datetime.datetime.now() + + def update_from_document(self, document): + value = "{}" + if document: + value = document.get("value") or value + self.data = json.loads(value) + + def to_json_string(self): + return json.dumps(self.data or {}) + + @property + def is_outdated(self): + if self.creation_time is None: + return True + delta = (datetime.datetime.now() - self.creation_time).seconds + return delta > self.cache_lifetime + + class MongoSettingsHandler(SettingsHandler): """Settings handler that use mongo for storing and loading of settings.""" system_settings_key = "system_settings" From 2bad8adc9fd157fa67cd523515d7b11a51fe1e26 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 14 Dec 2020 10:09:11 +0100 Subject: [PATCH 13/17] mongo handler is using caches --- pype/settings/handlers.py | 133 ++++++++++++++++++++++++-------------- 1 file changed, 85 insertions(+), 48 deletions(-) diff --git a/pype/settings/handlers.py b/pype/settings/handlers.py index 798739e4d0..724548fd12 100644 --- a/pype/settings/handlers.py +++ b/pype/settings/handlers.py @@ -1,6 +1,7 @@ import os import json import logging +import collections import datetime from abc import ABCMeta, abstractmethod import six @@ -320,9 +321,6 @@ class CacheValues: class MongoSettingsHandler(SettingsHandler): """Settings handler that use mongo for storing and loading of settings.""" - system_settings_key = "system_settings" - project_anatomy_key = "project_anatomy" - project_settings_key = "project_settings" def __init__(self): # Get mongo connection @@ -343,6 +341,10 @@ class MongoSettingsHandler(SettingsHandler): self.collection = settings_collection[database_name][collection_name] + self.system_settings_cache = CacheValues() + self.project_settings_cache = collections.defaultdict(CacheValues) + self.project_anatomy_cache = collections.defaultdict(CacheValues) + def save_studio_settings(self, data): """Save studio overrides of system settings. @@ -354,14 +356,17 @@ class MongoSettingsHandler(SettingsHandler): Args: data(dict): Data of studio overrides with override metadata. """ + self.system_settings_cache.update_data(data) + self.collection.replace_one( { - "type": self.system_settings_key + "type": SYSTEM_SETTINGS_KEY }, { - "type": self.system_settings_key, - "value": data - } + "type": SYSTEM_SETTINGS_KEY, + "value": self.system_settings_cache.to_json_string() + }, + upsert=True ) def save_project_settings(self, project_name, overrides): @@ -380,17 +385,11 @@ class MongoSettingsHandler(SettingsHandler): or None for global settings. data(dict): Data of project overrides with override metadata. """ + data_cache = self.project_settings_cache[project_name] + data_cache.update_data(overrides) - self.collection.replace_one( - { - "type": self.project_settings_key, - "project_name": project_name - }, - { - "type": self.project_settings_key, - "project_name": project_name, - "value": overrides - } + self._save_project_data( + project_name, PROJECT_SETTINGS_KEY, data_cache ) def save_project_anatomy(self, project_name, anatomy_data): @@ -401,37 +400,62 @@ class MongoSettingsHandler(SettingsHandler): or None for global settings. data(dict): Data of project overrides with override metadata. """ + data_cache = self.project_anatomy_cache[project_name] + data_cache.update_data(anatomy_data) + + self._save_project_data( + project_name, PROJECT_ANATOMY_KEY, data_cache + ) + + def _save_project_data(self, project_name, doc_type, data_cache): + is_default = bool(project_name is None) + replace_filter = { + "type": doc_type, + "is_default": is_default + } + replace_data = { + "type": doc_type, + "value": data_cache.to_json_string(), + "is_default": is_default + } + if not is_default: + replace_filter["project_name"] = project_name + replace_data["project_name"] = project_name + self.collection.replace_one( - { - "type": self.project_anatomy_key, - "project_name": project_name - }, - { - "type": self.project_anatomy_key, - "project_name": project_name, - "value": anatomy_data - } + replace_filter, + replace_data, + upsert=True ) def get_studio_system_settings_overrides(self): """Studio overrides of system settings.""" - return self.collection.find_one({ - "type": self.system_settings_key - }) or {} + if self.system_settings_cache.is_outdated: + document = self.collection.find_one({ + "type": SYSTEM_SETTINGS_KEY + }) + + self.system_settings_cache.update_from_document(document) + return self.system_settings_cache.data + + def _get_project_settings_overrides(self, project_name): + if self.project_settings_cache[project_name].is_outdated: + document_filter = { + "type": PROJECT_SETTINGS_KEY, + } + if project_name is None: + document_filter["is_default"] = True + else: + document_filter["project_name"] = project_name + document = self.collection.find_one(document_filter) + self.project_settings_cache[project_name].update_from_document( + document + ) + return self.project_settings_cache[project_name].data def get_studio_project_settings_overrides(self): """Studio overrides of default project settings.""" - return self.collection.find_one({ - "type": self.project_settings_key, - "project_name": None - }) or {} - - def get_studio_project_anatomy_overrides(self): - """Studio overrides of default project anatomy data.""" - return self.collection.find_one({ - "type": self.project_anatomy_key, - "project_name": None - }) or {} + return self._get_project_settings_overrides(None) def get_project_settings_overrides(self, project_name): """Studio overrides of project settings for specific project. @@ -444,10 +468,26 @@ class MongoSettingsHandler(SettingsHandler): """ if not project_name: return {} - return self.collection.find_one({ - "type": self.project_settings_key, - "project_name": project_name - }) or {} + return self._get_project_settings_overrides(project_name) + + def _get_project_anatomy_overrides(self, project_name): + if self.project_anatomy_cache[project_name].is_outdated: + document_filter = { + "type": PROJECT_ANATOMY_KEY, + } + if project_name is None: + document_filter["is_default"] = True + else: + document_filter["project_name"] = project_name + document = self.collection.find_one(document_filter) + self.project_anatomy_cache[project_name].update_from_document( + document + ) + return self.project_anatomy_cache[project_name].data + + def get_studio_project_anatomy_overrides(self): + """Studio overrides of default project anatomy data.""" + return self._get_project_anatomy_overrides(None) def get_project_anatomy_overrides(self, project_name): """Studio overrides of project anatomy for specific project. @@ -460,7 +500,4 @@ class MongoSettingsHandler(SettingsHandler): """ if not project_name: return {} - return self.collection.find_one({ - "type": self.project_anatomy_key, - "project_name": project_name - }) or {} + return self._get_project_anatomy_overrides(project_name) From 51ad0b5b9678f3a39403680c4def943719634128 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 14 Dec 2020 10:14:34 +0100 Subject: [PATCH 14/17] hanler is imported in `create_settings_handler` --- pype/settings/lib.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pype/settings/lib.py b/pype/settings/lib.py index 9aa4a6a85d..0b9c5813be 100644 --- a/pype/settings/lib.py +++ b/pype/settings/lib.py @@ -14,10 +14,6 @@ from .constants import ( PROJECT_SETTINGS_KEY, PROJECT_ANATOMY_KEY ) -from .handlers import ( - SettingsFileHandler, - MongoSettingsHandler -) log = logging.getLogger(__name__) @@ -49,7 +45,9 @@ def require_handler(func): def create_settings_handler(): - # This may be logic which handler is used (in future) + from .handlers import MongoSettingsHandler + # Handler can't be created in global space on initialization but only when + # needed. Plus here may be logic: Which handler is used (in future). return MongoSettingsHandler() From 4d3e74e039a37726d40eb32be25ef8bc9bbc13a2 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 14 Dec 2020 10:14:57 +0100 Subject: [PATCH 15/17] load_json_file is used from lib --- pype/settings/handlers.py | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/pype/settings/handlers.py b/pype/settings/handlers.py index 724548fd12..332392cf42 100644 --- a/pype/settings/handlers.py +++ b/pype/settings/handlers.py @@ -11,6 +11,7 @@ from .constants import ( PROJECT_SETTINGS_KEY, PROJECT_ANATOMY_KEY ) +from .lib import load_json_file JSON_EXC = getattr(json.decoder, "JSONDecodeError", ValueError) @@ -135,19 +136,6 @@ class SettingsFileHandler(SettingsHandler): self.project_settings_path = project_settings_path self.project_anatomy_path = project_anatomy_path - def load_json_file(self, fpath): - # Load json data - try: - with open(fpath, "r") as opened_file: - return json.load(opened_file) - - except JSON_EXC: - self.log.warning( - "File has invalid json format \"{}\"".format(fpath), - exc_info=True - ) - return {} - def path_to_project_settings(self, project_name): if not project_name: return self.project_settings_path @@ -244,19 +232,19 @@ class SettingsFileHandler(SettingsHandler): def get_studio_system_settings_overrides(self): """Studio overrides of system settings.""" if os.path.exists(self.system_settings_path): - return self.load_json_file(self.system_settings_path) + return load_json_file(self.system_settings_path) return {} def get_studio_project_settings_overrides(self): """Studio overrides of default project settings.""" if os.path.exists(self.project_settings_path): - return self.load_json_file(self.project_settings_path) + return load_json_file(self.project_settings_path) return {} def get_studio_project_anatomy_overrides(self): """Studio overrides of default project anatomy data.""" if os.path.exists(self.project_anatomy_path): - return self.load_json_file(self.project_anatomy_path) + return load_json_file(self.project_anatomy_path) return {} def get_project_settings_overrides(self, project_name): @@ -271,7 +259,7 @@ class SettingsFileHandler(SettingsHandler): path_to_json = self.path_to_project_settings(project_name) if not os.path.exists(path_to_json): return {} - return self.load_json_file(path_to_json) + return load_json_file(path_to_json) def get_project_anatomy_overrides(self, project_name): """Studio overrides of project anatomy for specific project. @@ -288,7 +276,7 @@ class SettingsFileHandler(SettingsHandler): path_to_json = self.path_to_project_anatomy(project_name) if not os.path.exists(path_to_json): return {} - return self.load_json_file(path_to_json) + return load_json_file(path_to_json) class CacheValues: From 0b8811c3211cbcb1ef1c5e1daf49790c9ed1a8e0 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 14 Dec 2020 10:20:13 +0100 Subject: [PATCH 16/17] added PypeMongoConnection to `__all__` --- pype/lib/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index e856956822..9444ef5195 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -131,6 +131,8 @@ __all__ = [ "decompose_url", "compose_url", "get_default_components", + "PypeMongoConnection", + "IniSettingRegistry", "JSONSettingRegistry", "PypeSettingsRegistry", From 2a6541eb9038af93160c9c78c8e4efbd00fd74bf Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 14 Dec 2020 11:23:16 +0100 Subject: [PATCH 17/17] return copy of data ot pure data --- pype/settings/handlers.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pype/settings/handlers.py b/pype/settings/handlers.py index 332392cf42..0df4c98820 100644 --- a/pype/settings/handlers.py +++ b/pype/settings/handlers.py @@ -1,5 +1,6 @@ import os import json +import copy import logging import collections import datetime @@ -286,6 +287,11 @@ class CacheValues: self.data = None self.creation_time = None + def data_copy(self): + if not self.data: + return {} + return copy.deepcopy(self.data) + def update_data(self, data): self.data = data self.creation_time = datetime.datetime.now() @@ -424,7 +430,7 @@ class MongoSettingsHandler(SettingsHandler): }) self.system_settings_cache.update_from_document(document) - return self.system_settings_cache.data + return self.system_settings_cache.data_copy() def _get_project_settings_overrides(self, project_name): if self.project_settings_cache[project_name].is_outdated: @@ -439,7 +445,7 @@ class MongoSettingsHandler(SettingsHandler): self.project_settings_cache[project_name].update_from_document( document ) - return self.project_settings_cache[project_name].data + return self.project_settings_cache[project_name].data_copy() def get_studio_project_settings_overrides(self): """Studio overrides of default project settings.""" @@ -471,7 +477,7 @@ class MongoSettingsHandler(SettingsHandler): self.project_anatomy_cache[project_name].update_from_document( document ) - return self.project_anatomy_cache[project_name].data + return self.project_anatomy_cache[project_name].data_copy() def get_studio_project_anatomy_overrides(self): """Studio overrides of default project anatomy data."""