Cloud Mongo

There was a circular import when importing from pypeapp, which I couldnt resolve, so have copied the mongo url logic across.

pype/api.py

@@ -32,7 +32,10 @@ from .lib import (
     get_version_from_path,
     get_last_version_from_path,
     modified_environ,
-    add_tool_to_environment
+    add_tool_to_environment,
+    decompose_url,
+    compose_url,
+    get_default_components
 )
 
 # Special naming case for subprocess since its a built-in method.
@@ -44,6 +47,9 @@ __all__ = [
     "project_overrides_dir_path",
     "config",
     "execute",
+    "decompose_url",
+    "compose_url",
+    "get_default_components",
 
     # plugin classes
     "Extractor",

pype/lib.py

@@ -1386,3 +1386,84 @@ def ffprobe_streams(path_to_file):
     popen_output = popen.communicate()[0]
     log.debug("FFprobe output: {}".format(popen_output))
     return json.loads(popen_output)["streams"]
+
+
+import os
+
+try:
+    from urllib.parse import urlparse, parse_qs
+except ImportError:
+    from urlparse import urlparse, parse_qs
+
+
+def decompose_url(url):
+    components = {
+        "scheme": None,
+        "host": None,
+        "port": None,
+        "username": None,
+        "password": None,
+        "auth_db": ""
+    }
+
+    result = urlparse(url)
+
+    components["scheme"] = result.scheme
+    components["host"] = result.hostname
+    try:
+        components["port"] = result.port
+    except ValueError:
+        raise RuntimeError("invalid port specified")
+    components["username"] = result.username
+    components["password"] = result.password
+
+    try:
+        components["auth_db"] = parse_qs(result.query)['authSource'][0]
+    except KeyError:
+        # no auth db provided, mongo will use the one we are connecting to
+        pass
+
+    return components
+
+
+def compose_url(scheme=None,
+                host=None,
+                username=None,
+                password=None,
+                database=None,
+                collection=None,
+                port=None,
+                auth_db=""):
+
+    url = "{scheme}://"
+
+    if username and password:
+        url += "{username}:{password}@"
+
+    url += "{host}"
+
+    if database:
+        url += "/{database}"
+
+    if database and collection:
+        url += "/{collection}"
+
+    if port:
+        url += ":{port}"
+
+    url += auth_db
+
+    return url.format(**{
+        "scheme": scheme,
+        "host": host,
+        "username": username,
+        "password": password,
+        "database": database,
+        "collection": collection,
+        "port": port,
+        "auth_db": ""
+    })
+
+
+def get_default_components():
+    return decompose_url(os.environ["MONGO_URL"])

pype/modules/adobe_communicator/lib/io_nonsingleton.py

@@ -16,6 +16,7 @@ import contextlib
 
 from avalon import schema
 from avalon.vendor import requests
+from pype.api import get_default_components, compose_url
 
 # Third-party dependencies
 import pymongo
@@ -72,8 +73,15 @@ class DbConnector(object):
         self.Session.update(self._from_environment())
 
         timeout = int(self.Session["AVALON_TIMEOUT"])
+
+        components = get_default_components()
+        port = components.pop("port")
+        host = compose_url(**components)
         self._mongo_client = pymongo.MongoClient(
-            self.Session["AVALON_MONGO"], serverSelectionTimeoutMS=timeout)
+            host=host,
+            port=port,
+            serverSelectionTimeoutMS=timeout
+        )
 
         for retry in range(3):
             try:

pype/modules/avalon_apps/rest_api.py

@@ -8,10 +8,7 @@ from pype.modules.ftrack.lib.custom_db_connector import DbConnector
 
 
 class AvalonRestApi(RestApi):
-    dbcon = DbConnector(
-        os.environ["AVALON_MONGO"],
-        os.environ["AVALON_DB"]
-    )
+    dbcon = DbConnector(os.environ["AVALON_DB"])
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)

pype/modules/ftrack/ftrack_server/event_server_cli.py

@@ -13,10 +13,10 @@ import time
 import uuid
 
 import ftrack_api
+import pymongo
 from pype.modules.ftrack.lib import credentials
-from pype.modules.ftrack.ftrack_server.lib import (
-    ftrack_events_mongo_settings, check_ftrack_url
-)
+from pype.modules.ftrack.ftrack_server.lib import check_ftrack_url
+from pype.api import get_default_components, compose_url
 import socket_thread
 
 
@@ -30,22 +30,19 @@ class MongoPermissionsError(Exception):
 
 def check_mongo_url(host, port, log_error=False):
     """Checks if mongo server is responding"""
-    sock = None
     try:
-        sock = socket.create_connection(
-            (host, port),
-            timeout=1
-        )
-        return True
-    except socket.error as err:
+        client = pymongo.MongoClient(host=host, port=port)
+        # Force connection on a request as the connect=True parameter of
+        # MongoClient seems to be useless here
+        client.server_info()
+    except pymongo.errors.ServerSelectionTimeoutError as err:
         if log_error:
             print("Can't connect to MongoDB at {}:{} because: {}".format(
                 host, port, err
             ))
         return False
-    finally:
-        if sock is not None:
-            sock.close()
+
+    return True
 
 
 def validate_credentials(url, user, api):
@@ -190,9 +187,9 @@ def main_loop(ftrack_url):
 
     os.environ["FTRACK_EVENT_SUB_ID"] = str(uuid.uuid1())
     # Get mongo hostname and port for testing mongo connection
-    mongo_list = ftrack_events_mongo_settings()
-    mongo_hostname = mongo_list[0]
-    mongo_port = mongo_list[1]
+    components = get_default_components()
+    mongo_port = components.pop("port")
+    mongo_hostname = compose_url(**components)
 
     # Current file
     file_path = os.path.dirname(os.path.realpath(__file__))
@@ -275,7 +272,7 @@ def main_loop(ftrack_url):
         # Run threads only if Ftrack is accessible
         if not ftrack_accessible or not mongo_accessible:
             if not mongo_accessible and not printed_mongo_error:
-                mongo_url = mongo_hostname + ":" + mongo_port
+                mongo_url = "{}:{}".format(mongo_hostname, mongo_port)
                 print("Can't access Mongo {}".format(mongo_url))
 
             if not ftrack_accessible and not printed_ftrack_error:

pype/modules/ftrack/ftrack_server/lib.py

@@ -18,12 +18,8 @@ import ftrack_api.operation
 import ftrack_api._centralized_storage_scenario
 import ftrack_api.event
 from ftrack_api.logging import LazyLogMessage as L
-try:
-    from urllib.parse import urlparse, parse_qs
-except ImportError:
-    from urlparse import urlparse, parse_qs
 
-from pype.api import Logger
+from pype.api import Logger, compose_url, get_default_components
 
 from pype.modules.ftrack.lib.custom_db_connector import DbConnector
 
@@ -32,67 +28,10 @@ TOPIC_STATUS_SERVER = "pype.event.server.status"
 TOPIC_STATUS_SERVER_RESULT = "pype.event.server.status.result"
 
 
-def ftrack_events_mongo_settings():
-    host = None
-    port = None
-    username = None
-    password = None
-    collection = None
-    database = None
-    auth_db = ""
-
-    if os.environ.get('FTRACK_EVENTS_MONGO_URL'):
-        result = urlparse(os.environ['FTRACK_EVENTS_MONGO_URL'])
-
-        host = result.hostname
-        try:
-            port = result.port
-        except ValueError:
-            raise RuntimeError("invalid port specified")
-        username = result.username
-        password = result.password
-        try:
-            database = result.path.lstrip("/").split("/")[0]
-            collection = result.path.lstrip("/").split("/")[1]
-        except IndexError:
-            if not database:
-                raise RuntimeError("missing database name for logging")
-        try:
-            auth_db = parse_qs(result.query)['authSource'][0]
-        except KeyError:
-            # no auth db provided, mongo will use the one we are connecting to
-            pass
-    else:
-        host = os.environ.get('FTRACK_EVENTS_MONGO_HOST')
-        port = int(os.environ.get('FTRACK_EVENTS_MONGO_PORT', "0"))
-        database = os.environ.get('FTRACK_EVENTS_MONGO_DB')
-        username = os.environ.get('FTRACK_EVENTS_MONGO_USER')
-        password = os.environ.get('FTRACK_EVENTS_MONGO_PASSWORD')
-        collection = os.environ.get('FTRACK_EVENTS_MONGO_COL')
-        auth_db = os.environ.get('FTRACK_EVENTS_MONGO_AUTH_DB', 'avalon')
-
-    return host, port, database, username, password, collection, auth_db
-
-
 def get_ftrack_event_mongo_info():
-    host, port, database, username, password, collection, auth_db = (
-        ftrack_events_mongo_settings()
-    )
-    user_pass = ""
-    if username and password:
-        user_pass = "{}:{}@".format(username, password)
-
-    socket_path = "{}:{}".format(host, port)
-
-    dab = ""
-    if database:
-        dab = "/{}".format(database)
-
-    auth = ""
-    if auth_db:
-        auth = "?authSource={}".format(auth_db)
-
-    url = "mongodb://{}{}{}{}".format(user_pass, socket_path, dab, auth)
+    url = compose_url(get_default_components())
+    database = os.environ["FTRACK_EVENTS_MONGO_DB"]
+    collection = os.environ["FTRACK_EVENTS_MONGO_COL"]
 
     return url, database, collection
 
@@ -205,7 +144,6 @@ class ProcessEventHub(SocketBaseEventHub):
 
     def __init__(self, *args, **kwargs):
         self.dbcon = DbConnector(
-            mongo_url=self.url,
             database_name=self.database,
             table_name=self.table_name
         )
@@ -269,7 +207,7 @@ class ProcessEventHub(SocketBaseEventHub):
     def load_events(self):
         """Load not processed events sorted by stored date"""
         ago_date = datetime.datetime.now() - datetime.timedelta(days=3)
-        result = self.dbcon.delete_many({
+        self.dbcon.delete_many({
             "pype_data.stored": {"$lte": ago_date},
             "pype_data.is_processed": True
         })

pype/modules/ftrack/ftrack_server/sub_event_storer.py

@@ -25,7 +25,6 @@ class SessionFactory:
 
 url, database, table_name = get_ftrack_event_mongo_info()
 dbcon = DbConnector(
-    mongo_url=url,
     database_name=database,
     table_name=table_name
 )

pype/modules/ftrack/lib/custom_db_connector.py

@@ -13,6 +13,8 @@ import atexit
 # Third-party dependencies
 import pymongo
 
+from pype.api import get_default_components, compose_url
+
 
 class NotActiveTable(Exception):
     def __init__(self, *args, **kwargs):
@@ -63,13 +65,12 @@ class DbConnector:
     log = logging.getLogger(__name__)
     timeout = 1000
 
-    def __init__(self, mongo_url, database_name, table_name=None):
+    def __init__(self, database_name, table_name=None):
         self._mongo_client = None
         self._sentry_client = None
         self._sentry_logging_handler = None
         self._database = None
         self._is_installed = False
-        self._mongo_url = mongo_url
         self._database_name = database_name
 
         self.active_table = table_name
@@ -95,8 +96,12 @@ class DbConnector:
         atexit.register(self.uninstall)
         logging.basicConfig()
 
+        components = get_default_components()
+        port = components.pop("port")
+        host = compose_url(**components)
         self._mongo_client = pymongo.MongoClient(
-            self._mongo_url,
+            host=host,
+            port=port,
             serverSelectionTimeoutMS=self.timeout
         )
 
@@ -113,11 +118,11 @@ class DbConnector:
         else:
             raise IOError(
                 "ERROR: Couldn't connect to %s in "
-                "less than %.3f ms" % (self._mongo_url, self.timeout)
+                "less than %.3f ms" % (host, self.timeout)
             )
 
         self.log.info("Connected to %s, delay %.3f s" % (
-            self._mongo_url, time.time() - t1
+            host, time.time() - t1
         ))
 
         self._database = self._mongo_client[self._database_name]

pype/modules/ftrack/lib/io_nonsingleton.py

@@ -73,7 +73,10 @@ class DbConnector(object):
 
         timeout = int(self.Session["AVALON_TIMEOUT"])
         self._mongo_client = pymongo.MongoClient(
-            self.Session["AVALON_MONGO"], serverSelectionTimeoutMS=timeout)
+            host=os.environ["AVALON_MONGO_HOST"],
+            port=int(os.environ["AVALON_MONGO_PORT"]),
+            serverSelectionTimeoutMS=timeout
+        )
 
         for retry in range(3):
             try:

schema/session-2.0.json

@@ -56,13 +56,6 @@
             "pattern": "^\\w*$",
             "example": "maya2016"
         },
-        "AVALON_MONGO": {
-            "description": "Address to the asset database",
-            "type": "string",
-            "pattern": "^mongodb://[\\w/@:.]*$",
-            "example": "mongodb://localhost:27017",
-            "default": "mongodb://localhost:27017"
-        },
         "AVALON_DB": {
             "description": "Name of database",
             "type": "string",