Merge pull request #2401 from pypeclub/feature/OP-2013_MongoDB-Ability-to-specify-replica-set-groups

General: MongoDB ability to specify replica set groups

openpype/api.py

@@ -31,8 +31,6 @@ from .lib import (
 )
 
 from .lib.mongo import (
-    decompose_url,
-    compose_url,
     get_default_components
 )
 
@@ -84,8 +82,6 @@ __all__ = [
     "Anatomy",
     "config",
     "execute",
-    "decompose_url",
-    "compose_url",
     "get_default_components",
     "ApplicationManager",
     "BuildWorkfile",

openpype/lib/__init__.py

@@ -32,8 +32,6 @@ from .execute import (
 )
 from .log import PypeLogger, timeit
 from .mongo import (
-    decompose_url,
-    compose_url,
     get_default_components,
     validate_mongo_connection,
     OpenPypeMongoConnection
@@ -276,8 +274,6 @@ __all__ = [
     "get_datetime_data",
 
     "PypeLogger",
-    "decompose_url",
-    "compose_url",
     "get_default_components",
     "validate_mongo_connection",
     "OpenPypeMongoConnection",

openpype/lib/log.py

@@ -27,7 +27,7 @@ import copy
 from . import Terminal
 from .mongo import (
     MongoEnvNotSet,
-    decompose_url,
+    get_default_components,
     OpenPypeMongoConnection
 )
 try:
@@ -202,8 +202,9 @@ class PypeLogger:
     use_mongo_logging = None
     mongo_process_id = None
 
-    # Information about mongo url
-    log_mongo_url = None
+    # Backwards compatibility - was used in start.py
+    # TODO remove when all old builds are replaced with new one
+    #   not using 'log_mongo_url_components'
     log_mongo_url_components = None
 
     # Database name in Mongo
@@ -282,9 +283,9 @@ class PypeLogger:
         if not cls.use_mongo_logging:
             return
 
-        components = cls.log_mongo_url_components
+        components = get_default_components()
         kwargs = {
-            "host": cls.log_mongo_url,
+            "host": components["host"],
             "database_name": cls.log_database_name,
             "collection": cls.log_collection_name,
             "username": components["username"],
@@ -324,6 +325,7 @@ class PypeLogger:
         # Change initialization state to prevent runtime changes
         # if is executed during runtime
         cls.initialized = False
+        cls.log_mongo_url_components = get_default_components()
 
         # Define if should logging to mongo be used
         use_mongo_logging = bool(log4mongo is not None)
@@ -354,14 +356,8 @@ class PypeLogger:
         # Define if is in OPENPYPE_DEBUG mode
         cls.pype_debug = int(os.getenv("OPENPYPE_DEBUG") or "0")
 
-        # Mongo URL where logs will be stored
-        cls.log_mongo_url = os.environ.get("OPENPYPE_MONGO")
-
-        if not cls.log_mongo_url:
+        if not os.environ.get("OPENPYPE_MONGO"):
             cls.use_mongo_logging = False
-        else:
-            # Decompose url
-            cls.log_mongo_url_components = decompose_url(cls.log_mongo_url)
 
         # Mark as initialized
         cls.initialized = True
@@ -474,7 +470,7 @@ class PypeLogger:
         if not cls.initialized:
             cls.initialize()
 
-        return OpenPypeMongoConnection.get_mongo_client(cls.log_mongo_url)
+        return OpenPypeMongoConnection.get_mongo_client()
 
 
 def timeit(method):

openpype/lib/mongo.py

@@ -15,7 +15,19 @@ class MongoEnvNotSet(Exception):
     pass
 
 
-def decompose_url(url):
+def _decompose_url(url):
+    """Decompose mongo url to basic components.
+
+    Used for creation of MongoHandler which expect mongo url components as
+    separated kwargs. Components are at the end not used as we're setting
+    connection directly this is just a dumb components for MongoHandler
+    validation pass.
+    """
+    # Use first url from passed url
+    #   - this is beacuse it is possible to pass multiple urls for multiple
+    #       replica sets which would crash on urlparse otherwise
+    #   - please don't use comma in username of password
+    url = url.split(",")[0]
     components = {
         "scheme": None,
         "host": None,
@@ -48,42 +60,13 @@ def decompose_url(url):
     return components
 
 
-def compose_url(scheme=None,
-                host=None,
-                username=None,
-                password=None,
-                port=None,
-                auth_db=None):
-
-    url = "{scheme}://"
-
-    if username and password:
-        url += "{username}:{password}@"
-
-    url += "{host}"
-    if port:
-        url += ":{port}"
-
-    if auth_db:
-        url += "?authSource={auth_db}"
-
-    return url.format(**{
-        "scheme": scheme,
-        "host": host,
-        "username": username,
-        "password": password,
-        "port": port,
-        "auth_db": auth_db
-    })
-
-
 def get_default_components():
     mongo_url = os.environ.get("OPENPYPE_MONGO")
     if mongo_url is None:
         raise MongoEnvNotSet(
             "URL for Mongo logging connection is not set."
         )
-    return decompose_url(mongo_url)
+    return _decompose_url(mongo_url)
 
 
 def should_add_certificate_path_to_mongo_url(mongo_url):

start.py

@@ -1110,15 +1110,15 @@ def get_info(use_staging=None) -> list:
     # Reinitialize
     PypeLogger.initialize()
 
-    log_components = PypeLogger.log_mongo_url_components
-    if log_components["host"]:
-        inf.append(("Logging to MongoDB", log_components["host"]))
-        inf.append(("  - port", log_components["port"] or "<N/A>"))
+    mongo_components = get_default_components()
+    if mongo_components["host"]:
+        inf.append(("Logging to MongoDB", mongo_components["host"]))
+        inf.append(("  - port", mongo_components["port"] or "<N/A>"))
         inf.append(("  - database", PypeLogger.log_database_name))
         inf.append(("  - collection", PypeLogger.log_collection_name))
-        inf.append(("  - user", log_components["username"] or "<N/A>"))
-        if log_components["auth_db"]:
-            inf.append(("  - auth source", log_components["auth_db"]))
+        inf.append(("  - user", mongo_components["username"] or "<N/A>"))
+        if mongo_components["auth_db"]:
+            inf.append(("  - auth source", mongo_components["auth_db"]))
 
     maximum = max(len(i[0]) for i in inf)
     formatted = []