Merge pull request #601 from pypeclub/feature/settings_api_moved_out_of_tool

Settings function moved out of tool

pype/settings/lib.py

@@ -13,7 +13,7 @@ M_ENVIRONMENT_KEY = "__environment_keys__"
 M_POP_KEY = "__pop_key__"
 
 # Folder where studio overrides are stored
-STUDIO_OVERRIDES_PATH = os.environ["PYPE_PROJECT_CONFIGS"]
+STUDIO_OVERRIDES_PATH = os.getenv("PYPE_PROJECT_CONFIGS") or ""
 
 # File where studio's system overrides are stored
 SYSTEM_SETTINGS_KEY = "system_settings"
@@ -23,9 +23,6 @@ SYSTEM_SETTINGS_PATH = os.path.join(
 
 # File where studio's environment overrides are stored
 ENVIRONMENTS_KEY = "environments"
-ENVIRONMENTS_PATH = os.path.join(
-    STUDIO_OVERRIDES_PATH, ENVIRONMENTS_KEY + ".json"
-)
 
 # File where studio's default project overrides are stored
 PROJECT_SETTINGS_KEY = "project_settings"
@@ -59,61 +56,100 @@ def default_settings():
     return _DEFAULT_SETTINGS
 
 
-def load_json(fpath):
+def load_json_file(fpath):
     # Load json data
-    with open(fpath, "r") as opened_file:
-        lines = opened_file.read().splitlines()
-
-    # prepare json string
-    standard_json = ""
-    for line in lines:
-        # Remove all whitespace on both sides
-        line = line.strip()
-
-        # Skip blank lines
-        if len(line) == 0:
-            continue
-
-        standard_json += line
-
-    # Check if has extra commas
-    extra_comma = False
-    if ",]" in standard_json or ",}" in standard_json:
-        extra_comma = True
-    standard_json = standard_json.replace(",]", "]")
-    standard_json = standard_json.replace(",}", "}")
-
-    if extra_comma:
-        log.error("Extra comma in json file: \"{}\"".format(fpath))
-
-    # return empty dict if file is empty
-    if standard_json == "":
-        return {}
-
-    # Try to parse string
-    try:
-        return json.loads(standard_json)
-
-    except json.decoder.JSONDecodeError:
-        # Return empty dict if it is first time that decode error happened
-        return {}
-
-    # Repreduce the exact same exception but traceback contains better
-    # information about position of error in the loaded json
     try:
         with open(fpath, "r") as opened_file:
-            json.load(opened_file)
+            return json.load(opened_file)
 
     except json.decoder.JSONDecodeError:
         log.warning(
             "File has invalid json format \"{}\"".format(fpath),
             exc_info=True
         )
-
     return {}
 
 
+def load_jsons_from_dir(path, *args, **kwargs):
+    """Load all .json files with content from entered folder path.
+
+    Data are loaded recursively from a directory and recreate the
+    hierarchy as a dictionary.
+
+    Entered path hiearchy:
+    |_ folder1
+    | |_ data1.json
+    |_ folder2
+      |_ subfolder1
+        |_ data2.json
+
+    Will result in:
+    ```javascript
+    {
+        "folder1": {
+            "data1": "CONTENT OF FILE"
+        },
+        "folder2": {
+            "data1": {
+                "subfolder1": "CONTENT OF FILE"
+            }
+        }
+    }
+    ```
+
+    Args:
+        path (str): Path to the root folder where the json hierarchy starts.
+
+    Returns:
+        dict: Loaded data.
+    """
+    output = {}
+
+    path = os.path.normpath(path)
+    if not os.path.exists(path):
+        # TODO warning
+        return output
+
+    sub_keys = list(kwargs.pop("subkeys", args))
+    for sub_key in tuple(sub_keys):
+        _path = os.path.join(path, sub_key)
+        if not os.path.exists(_path):
+            break
+
+        path = _path
+        sub_keys.pop(0)
+
+    base_len = len(path) + 1
+    for base, _directories, filenames in os.walk(path):
+        base_items_str = base[base_len:]
+        if not base_items_str:
+            base_items = []
+        else:
+            base_items = base_items_str.split(os.path.sep)
+
+        for filename in filenames:
+            basename, ext = os.path.splitext(filename)
+            if ext == ".json":
+                full_path = os.path.join(base, filename)
+                value = load_json_file(full_path)
+                dict_keys = base_items + [basename]
+                output = subkey_merge(output, value, dict_keys)
+
+    for sub_key in sub_keys:
+        output = output[sub_key]
+    return output
+
+
 def find_environments(data):
+    """ Find environemnt values from system settings by it's metadata.
+
+    Args:
+        data(dict): System settings data or dictionary which may contain
+            environments metadata.
+
+    Returns:
+        dict: Key as Environment key and value for `acre` module.
+    """
     if not data or not isinstance(data, dict):
         return
 
@@ -152,69 +188,30 @@ def subkey_merge(_dict, value, keys):
     return _dict
 
 
-def load_jsons_from_dir(path, *args, **kwargs):
-    output = {}
-
-    path = os.path.normpath(path)
-    if not os.path.exists(path):
-        # TODO warning
-        return output
-
-    sub_keys = list(kwargs.pop("subkeys", args))
-    for sub_key in tuple(sub_keys):
-        _path = os.path.join(path, sub_key)
-        if not os.path.exists(_path):
-            break
-
-        path = _path
-        sub_keys.pop(0)
-
-    base_len = len(path) + 1
-    for base, _directories, filenames in os.walk(path):
-        base_items_str = base[base_len:]
-        if not base_items_str:
-            base_items = []
-        else:
-            base_items = base_items_str.split(os.path.sep)
-
-        for filename in filenames:
-            basename, ext = os.path.splitext(filename)
-            if ext == ".json":
-                full_path = os.path.join(base, filename)
-                value = load_json(full_path)
-                dict_keys = base_items + [basename]
-                output = subkey_merge(output, value, dict_keys)
-
-    for sub_key in sub_keys:
-        output = output[sub_key]
-    return output
-
-
 def studio_system_settings():
+    """Studio overrides of system settings."""
     if os.path.exists(SYSTEM_SETTINGS_PATH):
-        return load_json(SYSTEM_SETTINGS_PATH)
+        return load_json_file(SYSTEM_SETTINGS_PATH)
-    return {}
-
-
-def studio_environments():
-    if os.path.exists(ENVIRONMENTS_PATH):
-        return load_json(ENVIRONMENTS_PATH)
     return {}
 
 
 def studio_project_settings():
+    """Studio overrides of default project settings."""
     if os.path.exists(PROJECT_SETTINGS_PATH):
-        return load_json(PROJECT_SETTINGS_PATH)
+        return load_json_file(PROJECT_SETTINGS_PATH)
     return {}
 
 
 def studio_project_anatomy():
+    """Studio overrides of default project anatomy data."""
     if os.path.exists(PROJECT_ANATOMY_PATH):
-        return load_json(PROJECT_ANATOMY_PATH)
+        return load_json_file(PROJECT_ANATOMY_PATH)
     return {}
 
 
-def path_to_project_overrides(project_name):
+def path_to_project_settings(project_name):
+    if not project_name:
+        return PROJECT_SETTINGS_PATH
     return os.path.join(
         STUDIO_OVERRIDES_PATH,
         project_name,
@@ -223,6 +220,8 @@ def path_to_project_overrides(project_name):
 
 
 def path_to_project_anatomy(project_name):
+    if not project_name:
+        return PROJECT_ANATOMY_PATH
     return os.path.join(
         STUDIO_OVERRIDES_PATH,
         project_name,
@@ -230,27 +229,114 @@ def path_to_project_anatomy(project_name):
     )
 
 
+def save_studio_settings(data):
+    """Save studio overrides of system settings.
+
+    Do not use to store whole system settings data with defaults but only it's
+    overrides with metadata defining how overrides should be applied in load
+    function. For loading should be used function `studio_system_settings`.
+
+    Args:
+        data(dict): Data of studio overrides with override metadata.
+    """
+    dirpath = os.path.dirname(SYSTEM_SETTINGS_PATH)
+    if not os.path.exists(dirpath):
+        os.makedirs(dirpath)
+
+    print("Saving studio overrides. Output path: {}".format(
+        SYSTEM_SETTINGS_PATH
+    ))
+    with open(SYSTEM_SETTINGS_PATH, "w") as file_stream:
+        json.dump(data, file_stream, indent=4)
+
+
+def save_project_settings(project_name, overrides):
+    """Save studio overrides of project settings.
+
+    Data are saved for specific project or as defaults for all projects.
+
+    Do not use to store whole project settings data with defaults but only it's
+    overrides with metadata defining how overrides should be applied in load
+    function. For loading should be used functions `studio_project_settings`
+    for global project settings and `project_settings_overrides` for
+    project specific settings.
+
+    Args:
+        project_name(str, null): Project name for which overrides are
+            or None for global settings.
+        data(dict): Data of project overrides with override metadata.
+    """
+    project_overrides_json_path = path_to_project_settings(project_name)
+    dirpath = os.path.dirname(project_overrides_json_path)
+    if not os.path.exists(dirpath):
+        os.makedirs(dirpath)
+
+    print("Saving overrides of project \"{}\". Output path: {}".format(
+        project_name, project_overrides_json_path
+    ))
+    with open(project_overrides_json_path, "w") as file_stream:
+        json.dump(overrides, file_stream, indent=4)
+
+
+def save_project_anatomy(project_name, anatomy_data):
+    """Save studio overrides of project anatomy data.
+
+    Args:
+        project_name(str, null): Project name for which overrides are
+            or None for global settings.
+        data(dict): Data of project overrides with override metadata.
+    """
+    project_anatomy_json_path = path_to_project_anatomy(project_name)
+    dirpath = os.path.dirname(project_anatomy_json_path)
+    if not os.path.exists(dirpath):
+        os.makedirs(dirpath)
+
+    print("Saving anatomy of project \"{}\". Output path: {}".format(
+        project_name, project_anatomy_json_path
+    ))
+    with open(project_anatomy_json_path, "w") as file_stream:
+        json.dump(anatomy_data, file_stream, indent=4)
+
+
 def project_settings_overrides(project_name):
+    """Studio overrides of project settings for specific project.
+
+    Args:
+        project_name(str): Name of project for which data should be loaded.
+
+    Returns:
+        dict: Only overrides for entered project, may be empty dictionary.
+    """
     if not project_name:
         return {}
 
-    path_to_json = path_to_project_overrides(project_name)
+    path_to_json = path_to_project_settings(project_name)
     if not os.path.exists(path_to_json):
         return {}
-    return load_json(path_to_json)
+    return load_json_file(path_to_json)
 
 
 def project_anatomy_overrides(project_name):
+    """Studio overrides of project anatomy for specific project.
+
+    Args:
+        project_name(str): Name of project for which data should be loaded.
+
+    Returns:
+        dict: Only overrides for entered project, may be empty dictionary.
+    """
     if not project_name:
         return {}
 
     path_to_json = path_to_project_anatomy(project_name)
     if not os.path.exists(path_to_json):
         return {}
-    return load_json(path_to_json)
+    return load_json_file(path_to_json)
 
 
-def merge_overrides(global_dict, override_dict):
+def merge_overrides(source_dict, override_dict):
+    """Merge data from override_dict to source_dict."""
+
     if M_OVERRIDEN_KEY in override_dict:
         overriden_keys = set(override_dict.pop(M_OVERRIDEN_KEY))
     else:
@@ -258,20 +344,17 @@ def merge_overrides(global_dict, override_dict):
 
     for key, value in override_dict.items():
         if value == M_POP_KEY:
-            global_dict.pop(key)
+            source_dict.pop(key)
 
-        elif (
+        elif (key in overriden_keys or key not in source_dict):
-            key in overriden_keys
+            source_dict[key] = value
-            or key not in global_dict
-        ):
-            global_dict[key] = value
 
-        elif isinstance(value, dict) and isinstance(global_dict[key], dict):
+        elif isinstance(value, dict) and isinstance(source_dict[key], dict):
-            global_dict[key] = merge_overrides(global_dict[key], value)
+            source_dict[key] = merge_overrides(source_dict[key], value)
 
         else:
-            global_dict[key] = value
+            source_dict[key] = value
-    return global_dict
+    return source_dict
 
 
 def apply_overrides(source_data, override_data):
@@ -282,13 +365,15 @@ def apply_overrides(source_data, override_data):
 
 
 def system_settings():
-    default_values = default_settings()[SYSTEM_SETTINGS_KEY]
+    """System settings with applied studio overrides."""
+    default_values = copy.deepcopy(default_settings()[SYSTEM_SETTINGS_KEY])
     studio_values = studio_system_settings()
     return apply_overrides(default_values, studio_values)
 
 
 def project_settings(project_name):
-    default_values = default_settings()[PROJECT_SETTINGS_KEY]
+    """Project settings with applied studio and project overrides."""
+    default_values = copy.deepcopy(default_settings()[PROJECT_SETTINGS_KEY])
     studio_values = studio_project_settings()
 
     studio_overrides = apply_overrides(default_values, studio_values)
@@ -299,6 +384,14 @@ def project_settings(project_name):
 
 
 def environments():
+    """Calculated environment based on defaults and system settings.
+
+    Any default environment also found in the system settings will be fully
+    overriden by the one from the system settings.
+
+    Returns:
+        dict: Output should be ready for `acre` module.
+    """
     envs = copy.deepcopy(default_settings()[ENVIRONMENTS_KEY])
     envs_from_system_settings = find_environments(system_settings())
     for env_group_key, values in envs_from_system_settings.items():

pype/tools/settings/settings/widgets/base.py

@@ -3,11 +3,8 @@ import json
 from Qt import QtWidgets, QtCore, QtGui
 from pype.settings.lib import (
     SYSTEM_SETTINGS_KEY,
-    SYSTEM_SETTINGS_PATH,
     PROJECT_SETTINGS_KEY,
-    PROJECT_SETTINGS_PATH,
     PROJECT_ANATOMY_KEY,
-    PROJECT_ANATOMY_PATH,
 
     DEFAULTS_DIR,
 
@@ -21,8 +18,9 @@ from pype.settings.lib import (
     project_settings_overrides,
     project_anatomy_overrides,
 
-    path_to_project_overrides,
+    save_studio_settings,
-    path_to_project_anatomy
+    save_project_settings,
+    save_project_anatomy
 )
 from .widgets import UnsavedChangesDialog
 from . import lib
@@ -183,13 +181,7 @@ class SystemWidget(QtWidgets.QWidget):
 
         values = lib.convert_gui_data_to_overrides(_data.get("system", {}))
 
-        dirpath = os.path.dirname(SYSTEM_SETTINGS_PATH)
+        save_studio_settings(values)
-        if not os.path.exists(dirpath):
-            os.makedirs(dirpath)
-
-        print("Saving data to:", SYSTEM_SETTINGS_PATH)
-        with open(SYSTEM_SETTINGS_PATH, "w") as file_stream:
-            json.dump(values, file_stream, indent=4)
 
         self._update_values()
 
@@ -621,29 +613,25 @@ class ProjectWidget(QtWidgets.QWidget):
             if item.child_invalid:
                 has_invalid = True
 
-        if has_invalid:
+        if not has_invalid:
-            invalid_items = []
+            return self._save_overrides()
-            for item in self.input_fields:
-                invalid_items.extend(item.get_invalid())
-            msg_box = QtWidgets.QMessageBox(
-                QtWidgets.QMessageBox.Warning,
-                "Invalid input",
-                "There is invalid value in one of inputs."
-                " Please lead red color and fix them."
-            )
-            msg_box.setStandardButtons(QtWidgets.QMessageBox.Ok)
-            msg_box.exec_()
 
-            first_invalid_item = invalid_items[0]
+        invalid_items = []
-            self.scroll_widget.ensureWidgetVisible(first_invalid_item)
+        for item in self.input_fields:
-            if first_invalid_item.isVisible():
+            invalid_items.extend(item.get_invalid())
-                first_invalid_item.setFocus(True)
+        msg_box = QtWidgets.QMessageBox(
-            return
+            QtWidgets.QMessageBox.Warning,
+            "Invalid input",
+            "There is invalid value in one of inputs."
+            " Please lead red color and fix them."
+        )
+        msg_box.setStandardButtons(QtWidgets.QMessageBox.Ok)
+        msg_box.exec_()
 
-        if self.project_name is None:
+        first_invalid_item = invalid_items[0]
-            self._save_studio_overrides()
+        self.scroll_widget.ensureWidgetVisible(first_invalid_item)
-        else:
+        if first_invalid_item.isVisible():
-            self._save_overrides()
+            first_invalid_item.setFocus(True)
 
     def _on_refresh(self):
         self.reset()
@@ -655,8 +643,12 @@ class ProjectWidget(QtWidgets.QWidget):
 
     def _save_overrides(self):
         data = {}
+        studio_overrides = bool(self.project_name is None)
         for item in self.input_fields:
-            value, is_group = item.overrides()
+            if studio_overrides:
+                value, is_group = item.studio_overrides()
+            else:
+                value, is_group = item.overrides()
             if value is not lib.NOT_SET:
                 data.update(value)
 
@@ -665,80 +657,24 @@ class ProjectWidget(QtWidgets.QWidget):
         )
 
         # Saving overrides data
-        project_overrides_data = output_data.get(
+        project_overrides_data = output_data.get(PROJECT_SETTINGS_KEY, {})
-            PROJECT_SETTINGS_KEY, {}
+        save_project_settings(self.project_name, project_overrides_data)
-        )
-        project_overrides_json_path = path_to_project_overrides(
-            self.project_name
-        )
-        dirpath = os.path.dirname(project_overrides_json_path)
-        if not os.path.exists(dirpath):
-            os.makedirs(dirpath)
-
-        print("Saving data to:", project_overrides_json_path)
-        with open(project_overrides_json_path, "w") as file_stream:
-            json.dump(project_overrides_data, file_stream, indent=4)
 
         # Saving anatomy data
-        project_anatomy_data = output_data.get(
+        project_anatomy_data = output_data.get(PROJECT_ANATOMY_KEY, {})
-            PROJECT_ANATOMY_KEY, {}
+        save_project_anatomy(self.project_name, project_anatomy_data)
-        )
-        project_anatomy_json_path = path_to_project_anatomy(
-            self.project_name
-        )
-        dirpath = os.path.dirname(project_anatomy_json_path)
-        if not os.path.exists(dirpath):
-            os.makedirs(dirpath)
 
-        print("Saving data to:", project_anatomy_json_path)
+        if self.project_name:
-        with open(project_anatomy_json_path, "w") as file_stream:
+            # Refill values with overrides
-            json.dump(project_anatomy_data, file_stream, indent=4)
+            self._on_project_change()
-
+        else:
-        # Refill values with overrides
+            # Update saved values
-        self._on_project_change()
+            self._update_values()
-
-    def _save_studio_overrides(self):
-        data = {}
-        for input_field in self.input_fields:
-            value, is_group = input_field.studio_overrides()
-            if value is not lib.NOT_SET:
-                data.update(value)
-
-        output_data = lib.convert_gui_data_to_overrides(
-            data.get("project", {})
-        )
-
-        # Project overrides data
-        project_overrides_data = output_data.get(
-            PROJECT_SETTINGS_KEY, {}
-        )
-        dirpath = os.path.dirname(PROJECT_SETTINGS_PATH)
-        if not os.path.exists(dirpath):
-            os.makedirs(dirpath)
-
-        print("Saving data to:", PROJECT_SETTINGS_PATH)
-        with open(PROJECT_SETTINGS_PATH, "w") as file_stream:
-            json.dump(project_overrides_data, file_stream, indent=4)
-
-        # Project Anatomy data
-        project_anatomy_data = output_data.get(
-            PROJECT_ANATOMY_KEY, {}
-        )
-        dirpath = os.path.dirname(PROJECT_ANATOMY_PATH)
-        if not os.path.exists(dirpath):
-            os.makedirs(dirpath)
-
-        print("Saving data to:", PROJECT_ANATOMY_PATH)
-        with open(PROJECT_ANATOMY_PATH, "w") as file_stream:
-            json.dump(project_anatomy_data, file_stream, indent=4)
-
-        # Update saved values
-        self._update_values()
 
     def _update_values(self):
         self.ignore_value_changes = True
 
-        default_values = default_values = lib.convert_data_to_gui_data(
+        default_values = lib.convert_data_to_gui_data(
             {"project": default_settings()}
         )
         for input_field in self.input_fields: