Merge pull request #811 from pypeclub/feature/settings_in_mongo

Settings store in mongo

pype.py

@@ -113,7 +113,7 @@ def boot():
         """
 
     print(art)
-    set_environments()
+
     # find pype versions
     bootstrap = BootstrapRepos()
     pype_versions = bootstrap.find_pype()
@@ -139,6 +139,7 @@ def boot():
         else:
             os.environ["PYPE_MONGO"] = pype_mongo
 
+    set_environments()
     if getattr(sys, 'frozen', False):
         if not pype_versions:
             import igniter

pype/lib/__init__.py

@@ -7,7 +7,8 @@ from .log import PypeLogger, timeit
 from .mongo import (
     decompose_url,
     compose_url,
-    get_default_components
+    get_default_components,
+    PypeMongoConnection
 )
 from .anatomy import Anatomy
 
@@ -130,6 +131,8 @@ __all__ = [
     "decompose_url",
     "compose_url",
     "get_default_components",
+    "PypeMongoConnection",
+
     "IniSettingRegistry",
     "JSONSettingRegistry",
     "PypeSettingsRegistry",

pype/lib/mongo.py

@@ -1,9 +1,13 @@
 import os
+import sys
+import time
+import logging
+import pymongo
 
-try:
+if sys.version_info[0] == 2:
-    from urllib.parse import urlparse, parse_qs
-except ImportError:
     from urlparse import urlparse, parse_qs
+else:
+    from urllib.parse import urlparse, parse_qs
 
 
 class MongoEnvNotSet(Exception):
@@ -79,3 +83,79 @@ def get_default_components():
             "URL for Mongo logging connection is not set."
         )
     return decompose_url(mongo_url)
+
+
+def extract_port_from_url(url):
+    parsed_url = urlparse(url)
+    if parsed_url.scheme is None:
+        _url = "mongodb://{}".format(url)
+        parsed_url = urlparse(_url)
+    return parsed_url.port
+
+
+class PypeMongoConnection:
+    """Singleton MongoDB connection.
+
+    Keeps MongoDB connections by url.
+    """
+    mongo_clients = {}
+    log = logging.getLogger("PypeMongoConnection")
+
+    @classmethod
+    def get_mongo_client(cls, mongo_url=None):
+        if mongo_url is None:
+            mongo_url = os.environ["PYPE_MONGO"]
+
+        connection = cls.mongo_clients.get(mongo_url)
+        if connection:
+            # Naive validation of existing connection
+            try:
+                connection.server_info()
+            except Exception:
+                connection = None
+
+        if not connection:
+            cls.log.debug("Creating mongo connection to {}".format(mongo_url))
+            connection = cls.create_connection(mongo_url)
+            cls.mongo_clients[mongo_url] = connection
+
+        return connection
+
+    @classmethod
+    def create_connection(cls, mongo_url, timeout=None):
+        if timeout is None:
+            timeout = int(os.environ.get("AVALON_TIMEOUT") or 1000)
+
+        kwargs = {
+            "host": mongo_url,
+            "serverSelectionTimeoutMS": timeout
+        }
+
+        port = extract_port_from_url(mongo_url)
+        if port is not None:
+            kwargs["port"] = int(port)
+
+        mongo_client = pymongo.MongoClient(**kwargs)
+
+        for _retry in range(3):
+            try:
+                t1 = time.time()
+                mongo_client.server_info()
+
+            except Exception:
+                cls.log.warning("Retrying...")
+                time.sleep(1)
+                timeout *= 1.5
+
+            else:
+                break
+
+        else:
+            raise IOError((
+                "ERROR: Couldn't connect to {} in less than {:.3f}ms"
+            ).format(mongo_url, timeout))
+
+        cls.log.info("Connected to {}, delay {:.3f}s".format(
+            mongo_url, time.time() - t1
+        ))
+        return mongo_client

pype/settings/constants.py

@@ -0,0 +1,34 @@
+# Metadata keys for work with studio and project overrides
+M_OVERRIDEN_KEY = "__overriden_keys__"
+# Metadata key for storing information about environments
+M_ENVIRONMENT_KEY = "__environment_keys__"
+# Metadata key for storing dynamic created labels
+M_DYNAMIC_KEY_LABEL = "__dynamic_keys_labels__"
+# NOTE key popping not implemented yet
+M_POP_KEY = "__pop_key__"
+
+METADATA_KEYS = (
+    M_OVERRIDEN_KEY,
+    M_ENVIRONMENT_KEY,
+    M_DYNAMIC_KEY_LABEL,
+    M_POP_KEY
+)
+
+# File where studio's system overrides are stored
+SYSTEM_SETTINGS_KEY = "system_settings"
+PROJECT_SETTINGS_KEY = "project_settings"
+PROJECT_ANATOMY_KEY = "project_anatomy"
+
+
+__all__ = (
+    "M_OVERRIDEN_KEY",
+    "M_ENVIRONMENT_KEY",
+    "M_DYNAMIC_KEY_LABEL",
+    "M_POP_KEY",
+
+    "METADATA_KEYS",
+
+    "SYSTEM_SETTINGS_KEY",
+    "PROJECT_SETTINGS_KEY",
+    "PROJECT_ANATOMY_KEY"
+)

pype/settings/handlers.py

@@ -0,0 +1,497 @@
+import os
+import json
+import copy
+import logging
+import collections
+import datetime
+from abc import ABCMeta, abstractmethod
+import six
+import pype
+from .constants import (
+    SYSTEM_SETTINGS_KEY,
+    PROJECT_SETTINGS_KEY,
+    PROJECT_ANATOMY_KEY
+)
+from .lib import load_json_file
+
+JSON_EXC = getattr(json.decoder, "JSONDecodeError", ValueError)
+
+
+@six.add_metaclass(ABCMeta)
+class SettingsHandler:
+    @abstractmethod
+    def save_studio_settings(self, data):
+        """Save studio overrides of system settings.
+
+        Do not use to store whole system settings data with defaults but only
+        it's overrides with metadata defining how overrides should be applied
+        in load function. For loading should be used function
+        `studio_system_settings`.
+
+        Args:
+            data(dict): Data of studio overrides with override metadata.
+        """
+        pass
+
+    @abstractmethod
+    def save_project_settings(self, project_name, overrides):
+        """Save studio overrides of project settings.
+
+        Data are saved for specific project or as defaults for all projects.
+
+        Do not use to store whole project settings data with defaults but only
+        it's overrides with metadata defining how overrides should be applied
+        in load function. For loading should be used function
+        `get_studio_project_settings_overrides` for global project settings
+        and `get_project_settings_overrides` for project specific settings.
+
+        Args:
+            project_name(str, null): Project name for which overrides are
+                or None for global settings.
+            data(dict): Data of project overrides with override metadata.
+        """
+        pass
+
+    @abstractmethod
+    def save_project_anatomy(self, project_name, anatomy_data):
+        """Save studio overrides of project anatomy data.
+
+        Args:
+            project_name(str, null): Project name for which overrides are
+                or None for global settings.
+            data(dict): Data of project overrides with override metadata.
+        """
+        pass
+
+    @abstractmethod
+    def get_studio_system_settings_overrides(self):
+        """Studio overrides of system settings."""
+        pass
+
+    @abstractmethod
+    def get_studio_project_settings_overrides(self):
+        """Studio overrides of default project settings."""
+        pass
+
+    @abstractmethod
+    def get_studio_project_anatomy_overrides(self):
+        """Studio overrides of default project anatomy data."""
+        pass
+
+    @abstractmethod
+    def get_project_settings_overrides(self, project_name):
+        """Studio overrides of project settings for specific project.
+
+        Args:
+            project_name(str): Name of project for which data should be loaded.
+
+        Returns:
+            dict: Only overrides for entered project, may be empty dictionary.
+        """
+        pass
+
+    @abstractmethod
+    def get_project_anatomy_overrides(self, project_name):
+        """Studio overrides of project anatomy for specific project.
+
+        Args:
+            project_name(str): Name of project for which data should be loaded.
+
+        Returns:
+            dict: Only overrides for entered project, may be empty dictionary.
+        """
+        pass
+
+
+class SettingsFileHandler(SettingsHandler):
+    def __init__(self):
+        self.log = logging.getLogger("SettingsFileHandler")
+
+        # Folder where studio overrides are stored
+        studio_overrides_dir = os.getenv("PYPE_PROJECT_CONFIGS")
+        if not studio_overrides_dir:
+            studio_overrides_dir = os.path.dirname(os.path.dirname(
+                os.path.abspath(pype.__file__)
+            ))
+        system_settings_path = os.path.join(
+            studio_overrides_dir, SYSTEM_SETTINGS_KEY + ".json"
+        )
+
+        # File where studio's default project overrides are stored
+        project_settings_filename = PROJECT_SETTINGS_KEY + ".json"
+        project_settings_path = os.path.join(
+            studio_overrides_dir, project_settings_filename
+        )
+
+        project_anatomy_filename = PROJECT_ANATOMY_KEY + ".json"
+        project_anatomy_path = os.path.join(
+            studio_overrides_dir, project_anatomy_filename
+        )
+
+        self.studio_overrides_dir = studio_overrides_dir
+        self.system_settings_path = system_settings_path
+
+        self.project_settings_filename = project_settings_filename
+        self.project_anatomy_filename = project_anatomy_filename
+
+        self.project_settings_path = project_settings_path
+        self.project_anatomy_path = project_anatomy_path
+
+    def path_to_project_settings(self, project_name):
+        if not project_name:
+            return self.project_settings_path
+        return os.path.join(
+            self.studio_overrides_dir,
+            project_name,
+            self.project_settings_filename
+        )
+
+    def path_to_project_anatomy(self, project_name):
+        if not project_name:
+            return self.project_anatomy_path
+        return os.path.join(
+            self.studio_overrides_dir,
+            project_name,
+            self.project_anatomy_filename
+        )
+
+    def save_studio_settings(self, data):
+        """Save studio overrides of system settings.
+
+        Do not use to store whole system settings data with defaults but only
+        it's overrides with metadata defining how overrides should be applied
+        in load function. For loading should be used function
+        `studio_system_settings`.
+
+        Args:
+            data(dict): Data of studio overrides with override metadata.
+        """
+        dirpath = os.path.dirname(self.system_settings_path)
+        if not os.path.exists(dirpath):
+            os.makedirs(dirpath)
+
+        self.log.debug(
+            "Saving studio overrides. Output path: {}".format(
+                self.system_settings_path
+            )
+        )
+        with open(self.system_settings_path, "w") as file_stream:
+            json.dump(data, file_stream, indent=4)
+
+    def save_project_settings(self, project_name, overrides):
+        """Save studio overrides of project settings.
+
+        Data are saved for specific project or as defaults for all projects.
+
+        Do not use to store whole project settings data with defaults but only
+        it's overrides with metadata defining how overrides should be applied
+        in load function. For loading should be used function
+        `get_studio_project_settings_overrides` for global project settings
+        and `get_project_settings_overrides` for project specific settings.
+
+        Args:
+            project_name(str, null): Project name for which overrides are
+                or None for global settings.
+            data(dict): Data of project overrides with override metadata.
+        """
+        project_overrides_json_path = self.path_to_project_settings(
+            project_name
+        )
+        dirpath = os.path.dirname(project_overrides_json_path)
+        if not os.path.exists(dirpath):
+            os.makedirs(dirpath)
+
+        self.log.debug(
+            "Saving overrides of project \"{}\". Output path: {}".format(
+                project_name, project_overrides_json_path
+            )
+        )
+        with open(project_overrides_json_path, "w") as file_stream:
+            json.dump(overrides, file_stream, indent=4)
+
+    def save_project_anatomy(self, project_name, anatomy_data):
+        """Save studio overrides of project anatomy data.
+
+        Args:
+            project_name(str, null): Project name for which overrides are
+                or None for global settings.
+            data(dict): Data of project overrides with override metadata.
+        """
+        project_anatomy_json_path = self.path_to_project_anatomy(project_name)
+        dirpath = os.path.dirname(project_anatomy_json_path)
+        if not os.path.exists(dirpath):
+            os.makedirs(dirpath)
+
+        self.log.debug(
+            "Saving anatomy of project \"{}\". Output path: {}".format(
+                project_name, project_anatomy_json_path
+            )
+        )
+        with open(project_anatomy_json_path, "w") as file_stream:
+            json.dump(anatomy_data, file_stream, indent=4)
+
+    def get_studio_system_settings_overrides(self):
+        """Studio overrides of system settings."""
+        if os.path.exists(self.system_settings_path):
+            return load_json_file(self.system_settings_path)
+        return {}
+
+    def get_studio_project_settings_overrides(self):
+        """Studio overrides of default project settings."""
+        if os.path.exists(self.project_settings_path):
+            return load_json_file(self.project_settings_path)
+        return {}
+
+    def get_studio_project_anatomy_overrides(self):
+        """Studio overrides of default project anatomy data."""
+        if os.path.exists(self.project_anatomy_path):
+            return load_json_file(self.project_anatomy_path)
+        return {}
+
+    def get_project_settings_overrides(self, project_name):
+        """Studio overrides of project settings for specific project.
+
+        Args:
+            project_name(str): Name of project for which data should be loaded.
+
+        Returns:
+            dict: Only overrides for entered project, may be empty dictionary.
+        """
+        path_to_json = self.path_to_project_settings(project_name)
+        if not os.path.exists(path_to_json):
+            return {}
+        return load_json_file(path_to_json)
+
+    def get_project_anatomy_overrides(self, project_name):
+        """Studio overrides of project anatomy for specific project.
+
+        Args:
+            project_name(str): Name of project for which data should be loaded.
+
+        Returns:
+            dict: Only overrides for entered project, may be empty dictionary.
+        """
+        if not project_name:
+            return {}
+
+        path_to_json = self.path_to_project_anatomy(project_name)
+        if not os.path.exists(path_to_json):
+            return {}
+        return load_json_file(path_to_json)
+
+
+class CacheValues:
+    cache_lifetime = 10
+
+    def __init__(self):
+        self.data = None
+        self.creation_time = None
+
+    def data_copy(self):
+        if not self.data:
+            return {}
+        return copy.deepcopy(self.data)
+
+    def update_data(self, data):
+        self.data = data
+        self.creation_time = datetime.datetime.now()
+
+    def update_from_document(self, document):
+        value = "{}"
+        if document:
+            value = document.get("value") or value
+        self.data = json.loads(value)
+
+    def to_json_string(self):
+        return json.dumps(self.data or {})
+
+    @property
+    def is_outdated(self):
+        if self.creation_time is None:
+            return True
+        delta = (datetime.datetime.now() - self.creation_time).seconds
+        return delta > self.cache_lifetime
+
+
+class MongoSettingsHandler(SettingsHandler):
+    """Settings handler that use mongo for storing and loading of settings."""
+
+    def __init__(self):
+        # Get mongo connection
+        from pype.lib import PypeMongoConnection
+        settings_collection = PypeMongoConnection.get_mongo_client()
+
+        # TODO prepare version of pype
+        # - pype version should define how are settings saved and loaded
+
+        # TODO modify to not use hardcoded keys
+        database_name = "pype"
+        collection_name = "settings"
+
+        self.settings_collection = settings_collection
+
+        self.database_name = database_name
+        self.collection_name = collection_name
+
+        self.collection = settings_collection[database_name][collection_name]
+
+        self.system_settings_cache = CacheValues()
+        self.project_settings_cache = collections.defaultdict(CacheValues)
+        self.project_anatomy_cache = collections.defaultdict(CacheValues)
+
+    def save_studio_settings(self, data):
+        """Save studio overrides of system settings.
+
+        Do not use to store whole system settings data with defaults but only
+        it's overrides with metadata defining how overrides should be applied
+        in load function. For loading should be used function
+        `studio_system_settings`.
+
+        Args:
+            data(dict): Data of studio overrides with override metadata.
+        """
+        self.system_settings_cache.update_data(data)
+
+        self.collection.replace_one(
+            {
+                "type": SYSTEM_SETTINGS_KEY
+            },
+            {
+                "type": SYSTEM_SETTINGS_KEY,
+                "value": self.system_settings_cache.to_json_string()
+            },
+            upsert=True
+        )
+
+    def save_project_settings(self, project_name, overrides):
+        """Save studio overrides of project settings.
+
+        Data are saved for specific project or as defaults for all projects.
+
+        Do not use to store whole project settings data with defaults but only
+        it's overrides with metadata defining how overrides should be applied
+        in load function. For loading should be used function
+        `get_studio_project_settings_overrides` for global project settings
+        and `get_project_settings_overrides` for project specific settings.
+
+        Args:
+            project_name(str, null): Project name for which overrides are
+                or None for global settings.
+            data(dict): Data of project overrides with override metadata.
+        """
+        data_cache = self.project_settings_cache[project_name]
+        data_cache.update_data(overrides)
+
+        self._save_project_data(
+            project_name, PROJECT_SETTINGS_KEY, data_cache
+        )
+
+    def save_project_anatomy(self, project_name, anatomy_data):
+        """Save studio overrides of project anatomy data.
+
+        Args:
+            project_name(str, null): Project name for which overrides are
+                or None for global settings.
+            data(dict): Data of project overrides with override metadata.
+        """
+        data_cache = self.project_anatomy_cache[project_name]
+        data_cache.update_data(anatomy_data)
+
+        self._save_project_data(
+            project_name, PROJECT_ANATOMY_KEY, data_cache
+        )
+
+    def _save_project_data(self, project_name, doc_type, data_cache):
+        is_default = bool(project_name is None)
+        replace_filter = {
+            "type": doc_type,
+            "is_default": is_default
+        }
+        replace_data = {
+            "type": doc_type,
+            "value": data_cache.to_json_string(),
+            "is_default": is_default
+        }
+        if not is_default:
+            replace_filter["project_name"] = project_name
+            replace_data["project_name"] = project_name
+
+        self.collection.replace_one(
+            replace_filter,
+            replace_data,
+            upsert=True
+        )
+
+    def get_studio_system_settings_overrides(self):
+        """Studio overrides of system settings."""
+        if self.system_settings_cache.is_outdated:
+            document = self.collection.find_one({
+                "type": SYSTEM_SETTINGS_KEY
+            })
+
+            self.system_settings_cache.update_from_document(document)
+        return self.system_settings_cache.data_copy()
+
+    def _get_project_settings_overrides(self, project_name):
+        if self.project_settings_cache[project_name].is_outdated:
+            document_filter = {
+                "type": PROJECT_SETTINGS_KEY,
+            }
+            if project_name is None:
+                document_filter["is_default"] = True
+            else:
+                document_filter["project_name"] = project_name
+            document = self.collection.find_one(document_filter)
+            self.project_settings_cache[project_name].update_from_document(
+                document
+            )
+        return self.project_settings_cache[project_name].data_copy()
+
+    def get_studio_project_settings_overrides(self):
+        """Studio overrides of default project settings."""
+        return self._get_project_settings_overrides(None)
+
+    def get_project_settings_overrides(self, project_name):
+        """Studio overrides of project settings for specific project.
+
+        Args:
+            project_name(str): Name of project for which data should be loaded.
+
+        Returns:
+            dict: Only overrides for entered project, may be empty dictionary.
+        """
+        if not project_name:
+            return {}
+        return self._get_project_settings_overrides(project_name)
+
+    def _get_project_anatomy_overrides(self, project_name):
+        if self.project_anatomy_cache[project_name].is_outdated:
+            document_filter = {
+                "type": PROJECT_ANATOMY_KEY,
+            }
+            if project_name is None:
+                document_filter["is_default"] = True
+            else:
+                document_filter["project_name"] = project_name
+            document = self.collection.find_one(document_filter)
+            self.project_anatomy_cache[project_name].update_from_document(
+                document
+            )
+        return self.project_anatomy_cache[project_name].data_copy()
+
+    def get_studio_project_anatomy_overrides(self):
+        """Studio overrides of default project anatomy data."""
+        return self._get_project_anatomy_overrides(None)
+
+    def get_project_anatomy_overrides(self, project_name):
+        """Studio overrides of project anatomy for specific project.
+
+        Args:
+            project_name(str): Name of project for which data should be loaded.
+
+        Returns:
+            dict: Only overrides for entered project, may be empty dictionary.
+        """
+        if not project_name:
+            return {}
+        return self._get_project_anatomy_overrides(project_name)

pype/settings/lib.py

@@ -1,60 +1,95 @@
 import os
 import json
+import functools
 import logging
 import copy
+from .constants import (
+    M_OVERRIDEN_KEY,
+    M_ENVIRONMENT_KEY,
+    M_POP_KEY,
+
+    METADATA_KEYS,
+
+    SYSTEM_SETTINGS_KEY,
+    PROJECT_SETTINGS_KEY,
+    PROJECT_ANATOMY_KEY
+)
 
 log = logging.getLogger(__name__)
 
 # Py2 + Py3 json decode exception
 JSON_EXC = getattr(json.decoder, "JSONDecodeError", ValueError)
 
-# Metadata keys for work with studio and project overrides
-M_OVERRIDEN_KEY = "__overriden_keys__"
-# Metadata key for storing information about environments
-M_ENVIRONMENT_KEY = "__environment_keys__"
-# Metadata key for storing dynamic created labels
-M_DYNAMIC_KEY_LABEL = "__dynamic_keys_labels__"
-# NOTE key popping not implemented yet
-M_POP_KEY = "__pop_key__"
-
-METADATA_KEYS = (
-    M_OVERRIDEN_KEY,
-    M_ENVIRONMENT_KEY,
-    M_DYNAMIC_KEY_LABEL,
-    M_POP_KEY
-)
-
-# Folder where studio overrides are stored
-STUDIO_OVERRIDES_PATH = os.getenv("PYPE_PROJECT_CONFIGS") or ""
-
-# File where studio's system overrides are stored
-SYSTEM_SETTINGS_KEY = "system_settings"
-SYSTEM_SETTINGS_PATH = os.path.join(
-    STUDIO_OVERRIDES_PATH, SYSTEM_SETTINGS_KEY + ".json"
-)
-
-# File where studio's environment overrides are stored
-ENVIRONMENTS_KEY = "environments"
-
-# File where studio's default project overrides are stored
-PROJECT_SETTINGS_KEY = "project_settings"
-PROJECT_SETTINGS_FILENAME = PROJECT_SETTINGS_KEY + ".json"
-PROJECT_SETTINGS_PATH = os.path.join(
-    STUDIO_OVERRIDES_PATH, PROJECT_SETTINGS_FILENAME
-)
-
-PROJECT_ANATOMY_KEY = "project_anatomy"
-PROJECT_ANATOMY_FILENAME = PROJECT_ANATOMY_KEY + ".json"
-PROJECT_ANATOMY_PATH = os.path.join(
-    STUDIO_OVERRIDES_PATH, PROJECT_ANATOMY_FILENAME
-)
 
 # Path to default settings
-DEFAULTS_DIR = os.path.join(os.path.dirname(__file__), "defaults")
+DEFAULTS_DIR = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)),
+    "defaults"
+)
 
 # Variable where cache of default settings are stored
 _DEFAULT_SETTINGS = None
 
+# Handler of studio overrides
+_SETTINGS_HANDLER = None
+
+
+def require_handler(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        global _SETTINGS_HANDLER
+        if _SETTINGS_HANDLER is None:
+            _SETTINGS_HANDLER = create_settings_handler()
+        return func(*args, **kwargs)
+    return wrapper
+
+
+def create_settings_handler():
+    from .handlers import MongoSettingsHandler
+    # Handler can't be created in global space on initialization but only when
+    # needed. Plus here may be logic: Which handler is used (in future).
+    return MongoSettingsHandler()
+
+
+@require_handler
+def save_studio_settings(data):
+    return _SETTINGS_HANDLER.save_studio_settings(data)
+
+
+@require_handler
+def save_project_settings(project_name, overrides):
+    return _SETTINGS_HANDLER.save_project_settings(project_name, overrides)
+
+
+@require_handler
+def save_project_anatomy(project_name, anatomy_data):
+    return _SETTINGS_HANDLER.save_project_anatomy(project_name, anatomy_data)
+
+
+@require_handler
+def get_studio_system_settings_overrides():
+    return _SETTINGS_HANDLER.get_studio_system_settings_overrides()
+
+
+@require_handler
+def get_studio_project_settings_overrides():
+    return _SETTINGS_HANDLER.get_studio_project_settings_overrides()
+
+
+@require_handler
+def get_studio_project_anatomy_overrides():
+    return _SETTINGS_HANDLER.get_studio_project_anatomy_overrides()
+
+
+@require_handler
+def get_project_settings_overrides(project_name):
+    return _SETTINGS_HANDLER.get_project_settings_overrides(project_name)
+
+
+@require_handler
+def get_project_anatomy_overrides(project_name):
+    return _SETTINGS_HANDLER.get_project_anatomy_overrides(project_name)
+
 
 class DuplicatedEnvGroups(Exception):
     def __init__(self, duplicated):
@@ -248,150 +283,6 @@ def subkey_merge(_dict, value, keys):
     return _dict
 
 
-def path_to_project_settings(project_name):
-    if not project_name:
-        return PROJECT_SETTINGS_PATH
-    return os.path.join(
-        STUDIO_OVERRIDES_PATH,
-        project_name,
-        PROJECT_SETTINGS_FILENAME
-    )
-
-
-def path_to_project_anatomy(project_name):
-    if not project_name:
-        return PROJECT_ANATOMY_PATH
-    return os.path.join(
-        STUDIO_OVERRIDES_PATH,
-        project_name,
-        PROJECT_ANATOMY_FILENAME
-    )
-
-
-def save_studio_settings(data):
-    """Save studio overrides of system settings.
-
-    Do not use to store whole system settings data with defaults but only it's
-    overrides with metadata defining how overrides should be applied in load
-    function. For loading should be used function `studio_system_settings`.
-
-    Args:
-        data(dict): Data of studio overrides with override metadata.
-    """
-    dirpath = os.path.dirname(SYSTEM_SETTINGS_PATH)
-    if not os.path.exists(dirpath):
-        os.makedirs(dirpath)
-
-    print("Saving studio overrides. Output path: {}".format(
-        SYSTEM_SETTINGS_PATH
-    ))
-    with open(SYSTEM_SETTINGS_PATH, "w") as file_stream:
-        json.dump(data, file_stream, indent=4)
-
-
-def save_project_settings(project_name, overrides):
-    """Save studio overrides of project settings.
-
-    Data are saved for specific project or as defaults for all projects.
-
-    Do not use to store whole project settings data with defaults but only it's
-    overrides with metadata defining how overrides should be applied in load
-    function. For loading should be used function
-    `get_studio_project_settings_overrides` for global project settings
-    and `get_project_settings_overrides` for project specific settings.
-
-    Args:
-        project_name(str, null): Project name for which overrides are
-            or None for global settings.
-        data(dict): Data of project overrides with override metadata.
-    """
-    project_overrides_json_path = path_to_project_settings(project_name)
-    dirpath = os.path.dirname(project_overrides_json_path)
-    if not os.path.exists(dirpath):
-        os.makedirs(dirpath)
-
-    print("Saving overrides of project \"{}\". Output path: {}".format(
-        project_name, project_overrides_json_path
-    ))
-    with open(project_overrides_json_path, "w") as file_stream:
-        json.dump(overrides, file_stream, indent=4)
-
-
-def save_project_anatomy(project_name, anatomy_data):
-    """Save studio overrides of project anatomy data.
-
-    Args:
-        project_name(str, null): Project name for which overrides are
-            or None for global settings.
-        data(dict): Data of project overrides with override metadata.
-    """
-    project_anatomy_json_path = path_to_project_anatomy(project_name)
-    dirpath = os.path.dirname(project_anatomy_json_path)
-    if not os.path.exists(dirpath):
-        os.makedirs(dirpath)
-
-    print("Saving anatomy of project \"{}\". Output path: {}".format(
-        project_name, project_anatomy_json_path
-    ))
-    with open(project_anatomy_json_path, "w") as file_stream:
-        json.dump(anatomy_data, file_stream, indent=4)
-
-
-def get_studio_system_settings_overrides():
-    """Studio overrides of system settings."""
-    if os.path.exists(SYSTEM_SETTINGS_PATH):
-        return load_json_file(SYSTEM_SETTINGS_PATH)
-    return {}
-
-
-def get_studio_project_settings_overrides():
-    """Studio overrides of default project settings."""
-    if os.path.exists(PROJECT_SETTINGS_PATH):
-        return load_json_file(PROJECT_SETTINGS_PATH)
-    return {}
-
-
-def get_studio_project_anatomy_overrides():
-    """Studio overrides of default project anatomy data."""
-    if os.path.exists(PROJECT_ANATOMY_PATH):
-        return load_json_file(PROJECT_ANATOMY_PATH)
-    return {}
-
-
-def get_project_settings_overrides(project_name):
-    """Studio overrides of project settings for specific project.
-
-    Args:
-        project_name(str): Name of project for which data should be loaded.
-
-    Returns:
-        dict: Only overrides for entered project, may be empty dictionary.
-    """
-
-    path_to_json = path_to_project_settings(project_name)
-    if not os.path.exists(path_to_json):
-        return {}
-    return load_json_file(path_to_json)
-
-
-def get_project_anatomy_overrides(project_name):
-    """Studio overrides of project anatomy for specific project.
-
-    Args:
-        project_name(str): Name of project for which data should be loaded.
-
-    Returns:
-        dict: Only overrides for entered project, may be empty dictionary.
-    """
-    if not project_name:
-        return {}
-
-    path_to_json = path_to_project_anatomy(project_name)
-    if not os.path.exists(path_to_json):
-        return {}
-    return load_json_file(path_to_json)
-
-
 def merge_overrides(source_dict, override_dict):
     """Merge data from override_dict to source_dict."""
 
@@ -461,7 +352,9 @@ def get_anatomy_settings(project_name, clear_metadata=True):
         )
 
     studio_overrides = get_default_anatomy_settings(False)
-    project_overrides = get_project_anatomy_overrides(project_name)
+    project_overrides = get_project_anatomy_overrides(
+        project_name
+    )
 
     result = apply_overrides(studio_overrides, project_overrides)
     if clear_metadata:
@@ -478,7 +371,9 @@ def get_project_settings(project_name, clear_metadata=True):
         )
 
     studio_overrides = get_default_project_settings(False)
-    project_overrides = get_project_settings_overrides(project_name)
+    project_overrides = get_project_settings_overrides(
+        project_name
+    )
 
     result = apply_overrides(studio_overrides, project_overrides)
     if clear_metadata:

pype/tools/settings/settings/widgets/base.py

@@ -2,11 +2,13 @@ import os
 import copy
 import json
 from Qt import QtWidgets, QtCore, QtGui
-from pype.settings.lib import (
+from pype.settings.constants import (
     SYSTEM_SETTINGS_KEY,
     PROJECT_SETTINGS_KEY,
-    PROJECT_ANATOMY_KEY,
+    PROJECT_ANATOMY_KEY
+)
 
+from pype.settings.lib import (
     DEFAULTS_DIR,
 
     reset_default_settings,

pype/tools/settings/settings/widgets/lib.py

@@ -2,7 +2,7 @@ import os
 import re
 import json
 import copy
-from pype.settings.lib import (
+from pype.settings.constants import (
     M_OVERRIDEN_KEY,
     M_ENVIRONMENT_KEY,
     M_DYNAMIC_KEY_LABEL