From 56c4ae7eb7b81b400693a75d277a0c1609fe5aa6 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 31 Mar 2021 14:57:36 +0200 Subject: [PATCH 01/12] removed studio soft key from general and replaced with pype_path key --- .../schemas/system_schema/schema_general.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pype/settings/entities/schemas/system_schema/schema_general.json b/pype/settings/entities/schemas/system_schema/schema_general.json index cf88043cd0..26dc251acf 100644 --- a/pype/settings/entities/schemas/system_schema/schema_general.json +++ b/pype/settings/entities/schemas/system_schema/schema_general.json @@ -19,20 +19,20 @@ "type": "splitter" }, { - "key": "studio_soft", - "type": "path", - "label": "Studio Software Location", - "multiplatform": true, - "multipath": false + "key": "environment", + "label": "Environment", + "type": "raw-json", + "env_group_key": "global" }, { "type": "splitter" }, { - "key": "environment", - "label": "Environment", - "type": "raw-json", - "env_group_key": "global" + "type": "path", + "key": "pype_path", + "label": "Pype Path", + "multiplatform": true, + "multipath": true } ] } From 85863015a38cfa6218d094c5c30fa3122878f085 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 31 Mar 2021 14:57:51 +0200 Subject: [PATCH 02/12] defined constant for global settings type --- pype/settings/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pype/settings/constants.py b/pype/settings/constants.py index f6077e826e..a53e88a91e 100644 --- a/pype/settings/constants.py +++ b/pype/settings/constants.py @@ -15,6 +15,7 @@ METADATA_KEYS = ( ) # File where studio's system overrides are stored +GLOBAL_SETTINGS_KEY = "global_settings" SYSTEM_SETTINGS_KEY = "system_settings" PROJECT_SETTINGS_KEY = "project_settings" PROJECT_ANATOMY_KEY = "project_anatomy" From 969b2dedef573575c00eb408541b7d3232783835 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 31 Mar 2021 14:58:19 +0200 Subject: [PATCH 03/12] resaved defaults --- pype/settings/defaults/system_settings/general.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pype/settings/defaults/system_settings/general.json b/pype/settings/defaults/system_settings/general.json index e03e00aca8..99db4e85c6 100644 --- a/pype/settings/defaults/system_settings/general.json +++ b/pype/settings/defaults/system_settings/general.json @@ -1,11 +1,6 @@ { "studio_name": "Studio name", "studio_code": "stu", - "studio_soft": { - "windows": "convert from \"STUDIO_SOFT\"", - "darwin": "", - "linux": "" - }, "environment": { "FFMPEG_PATH": { "windows": "{PYPE_ROOT}/vendor/bin/ffmpeg_exec/windows/bin", @@ -19,5 +14,10 @@ "PYPE_OCIO_CONFIG" ] } + }, + "pype_path": { + "windows": [], + "darwin": [], + "linux": [] } } \ No newline at end of file From cd9254b27d897fdf97313cb126528a958a819c84 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 31 Mar 2021 14:58:50 +0200 Subject: [PATCH 04/12] settings handler saves pype_path from system settings to global settings on save --- pype/settings/handlers.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/pype/settings/handlers.py b/pype/settings/handlers.py index 5a4e507547..0891ac39b5 100644 --- a/pype/settings/handlers.py +++ b/pype/settings/handlers.py @@ -8,6 +8,7 @@ from abc import ABCMeta, abstractmethod import six import pype from .constants import ( + GLOBAL_SETTINGS_KEY, SYSTEM_SETTINGS_KEY, PROJECT_SETTINGS_KEY, PROJECT_ANATOMY_KEY, @@ -401,6 +402,11 @@ class MongoSettingsHandler(SettingsHandler): self._prepare_project_settings_keys() return self._attribute_keys + def _prepare_global_settings(self, data): + if "general" not in data: + return {} + return data["general"].get("pype_path") or {} + def save_studio_settings(self, data): """Save studio overrides of system settings. @@ -412,8 +418,8 @@ class MongoSettingsHandler(SettingsHandler): Args: data(dict): Data of studio overrides with override metadata. """ + # Store system settings self.system_settings_cache.update_data(data) - self.collection.replace_one( { "type": SYSTEM_SETTINGS_KEY @@ -425,6 +431,22 @@ class MongoSettingsHandler(SettingsHandler): upsert=True ) + # Get global settings from system settings + global_settings = self._prepare_global_settings( + self.system_settings_cache.data + ) + # Store global settings + self.collection.replace_one( + { + "type": GLOBAL_SETTINGS_KEY + }, + { + "type": GLOBAL_SETTINGS_KEY, + "data": global_settings + }, + upsert=True + ) + def save_project_settings(self, project_name, overrides): """Save studio overrides of project settings. From a9442274ec4c533f8fd4aac1e672d0063ed33e70 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 31 Mar 2021 14:59:23 +0200 Subject: [PATCH 05/12] removed unused function load_environments from igniter --- igniter/tools.py | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/igniter/tools.py b/igniter/tools.py index 4ed4ae67f4..43cea8382a 100644 --- a/igniter/tools.py +++ b/igniter/tools.py @@ -185,38 +185,6 @@ def validate_path_string(path: str) -> (bool, str): return True, "valid path" -def load_environments(sections: list = None) -> dict: - """Load environments from Pype. - - This will load environments from database, process them with - :mod:`acre` and return them as flattened dictionary. - - Args: - sections (list, optional): load specific types - - Returns; - dict of str: loaded and processed environments. - - """ - import acre - - from pype import settings - - all_env = settings.get_environments() - merged_env = {} - - sections = sections or all_env.keys() - - for section in sections: - try: - parsed_env = acre.parse(all_env[section]) - except AttributeError: - continue - merged_env = acre.append(merged_env, parsed_env) - - return acre.compute(merged_env, cleanup=True) - - def get_pype_path_from_db(url: str) -> Union[str, None]: """Get Pype path from database. From e7781a454003481b59191a3093c4036532c3192c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 31 Mar 2021 15:00:41 +0200 Subject: [PATCH 06/12] added function to load global settings from mongo --- igniter/tools.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/igniter/tools.py b/igniter/tools.py index 43cea8382a..4d82783607 100644 --- a/igniter/tools.py +++ b/igniter/tools.py @@ -185,6 +185,44 @@ def validate_path_string(path: str) -> (bool, str): return True, "valid path" +def get_pype_global_settings(url: str) -> dict: + """Load global settings from Mongo database. + + We are loading data from database `pype` and collection `settings`. + There we expect document type `global_settings`. + + Returns: + dict: With settings data. Empty dictionary is returned if not found. + """ + try: + components = decompose_url(url) + except RuntimeError: + return {} + mongo_kwargs = { + "host": compose_url(**components), + "serverSelectionTimeoutMS": 2000 + } + port = components.get("port") + if port is not None: + mongo_kwargs["port"] = int(port) + + try: + # Create mongo connection + client = MongoClient(**mongo_kwargs) + # Access settings collection + col = client["pype"]["settings"] + # Query global settings + global_settings = col.find_one({"type": "global_settings"}) or {} + # Close Mongo connection + client.close() + + except Exception: + # TODO log traceback or message + return {} + + return global_settings.get("data") or {} + + def get_pype_path_from_db(url: str) -> Union[str, None]: """Get Pype path from database. From b98fd5d0a9c2f281821d8ade50a3ad4c4ff082c9 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 31 Mar 2021 15:02:18 +0200 Subject: [PATCH 07/12] modified `get_pype_path_from_db` to use `get_pype_global_settings` function --- igniter/tools.py | 31 ++----------------------------- 1 file changed, 2 insertions(+), 29 deletions(-) diff --git a/igniter/tools.py b/igniter/tools.py index 4d82783607..80fcd5a9e6 100644 --- a/igniter/tools.py +++ b/igniter/tools.py @@ -224,40 +224,13 @@ def get_pype_global_settings(url: str) -> dict: def get_pype_path_from_db(url: str) -> Union[str, None]: - """Get Pype path from database. - - We are loading data from database `pype` and collection `settings`. - There we expect document type `global_settings`. + """Get Pype path from global settings. Args: url (str): mongodb url. Returns: path to Pype or None if not found - """ - try: - components = decompose_url(url) - except RuntimeError: - return None - mongo_args = { - "host": compose_url(**components), - "serverSelectionTimeoutMS": 2000 - } - port = components.get("port") - if port is not None: - mongo_args["port"] = int(port) - - try: - client = MongoClient(**mongo_args) - except Exception: - return None - - db = client.pype - col = db.settings - - global_settings = col.find_one({"type": "global_settings"}, {"data": 1}) - if not global_settings: - return None - global_settings.get("data", {}) + global_settings = get_pype_global_settings(url) return global_settings.get("pype_path", {}).get(platform.system().lower()) From c51f31161eea1a664912dfe0f4f8dc384a471a5a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 31 Mar 2021 15:07:20 +0200 Subject: [PATCH 08/12] changed how paths are checked --- igniter/tools.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/igniter/tools.py b/igniter/tools.py index 80fcd5a9e6..7f5aa8d876 100644 --- a/igniter/tools.py +++ b/igniter/tools.py @@ -6,6 +6,7 @@ Functions ``compose_url()`` and ``decompose_url()`` are the same as in version is decided. """ +import os from typing import Dict, Union from urllib.parse import urlparse, parse_qs from pathlib import Path @@ -234,3 +235,17 @@ def get_pype_path_from_db(url: str) -> Union[str, None]: """ global_settings = get_pype_global_settings(url) return global_settings.get("pype_path", {}).get(platform.system().lower()) + paths = ( + global_settings + .get("pype_path", {}) + .get(platform.system().lower()) + ) or [] + # For cases when `pype_path` is a single path + if paths and isinstance(paths, str): + paths = [paths] + + # Loop over paths and return only existing + for path in paths: + if os.path.exists(path): + return path + return None From 75f9fd7d1c8845cffe30e5aef01a795d73b5b4cd Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 31 Mar 2021 15:26:44 +0200 Subject: [PATCH 09/12] close validation mongo connection --- igniter/tools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/igniter/tools.py b/igniter/tools.py index 7f5aa8d876..38d7fa2b26 100644 --- a/igniter/tools.py +++ b/igniter/tools.py @@ -131,6 +131,7 @@ def validate_mongo_connection(cnx: str) -> (bool, str): try: client = MongoClient(**mongo_args) client.server_info() + client.close() except ServerSelectionTimeoutError as e: return False, f"Cannot connect to server {cnx} - {e}" except ValueError: From 86755f97cc54dca0c4a94781f311ad13ea3a4c0c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 31 Mar 2021 15:46:05 +0200 Subject: [PATCH 10/12] fix global settings dict hierarchy --- pype/settings/handlers.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pype/settings/handlers.py b/pype/settings/handlers.py index 0891ac39b5..cc071f9fb5 100644 --- a/pype/settings/handlers.py +++ b/pype/settings/handlers.py @@ -403,9 +403,11 @@ class MongoSettingsHandler(SettingsHandler): return self._attribute_keys def _prepare_global_settings(self, data): - if "general" not in data: - return {} - return data["general"].get("pype_path") or {} + output = {} + # Add "pype_path" key to global settings if is set + if "general" in data and "pype_path" in data["general"]: + output["pype_path"] = data["general"]["pype_path"] + return output def save_studio_settings(self, data): """Save studio overrides of system settings. From fae5a6506ce294ce6b2b251bf70c9e6d6582b1cd Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 31 Mar 2021 16:16:44 +0200 Subject: [PATCH 11/12] added `url` argument back to docstring --- igniter/tools.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/igniter/tools.py b/igniter/tools.py index 38d7fa2b26..08157e8728 100644 --- a/igniter/tools.py +++ b/igniter/tools.py @@ -193,6 +193,9 @@ def get_pype_global_settings(url: str) -> dict: We are loading data from database `pype` and collection `settings`. There we expect document type `global_settings`. + Args: + url (str): MongoDB url. + Returns: dict: With settings data. Empty dictionary is returned if not found. """ From 3280d1a8aadde67a10817531cb2471cf530ef0eb Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 31 Mar 2021 16:16:54 +0200 Subject: [PATCH 12/12] removed forgotten line --- igniter/tools.py | 1 - 1 file changed, 1 deletion(-) diff --git a/igniter/tools.py b/igniter/tools.py index 08157e8728..11fe6b02f2 100644 --- a/igniter/tools.py +++ b/igniter/tools.py @@ -238,7 +238,6 @@ def get_pype_path_from_db(url: str) -> Union[str, None]: path to Pype or None if not found """ global_settings = get_pype_global_settings(url) - return global_settings.get("pype_path", {}).get(platform.system().lower()) paths = ( global_settings .get("pype_path", {})