Merge pull request #289 from tokejepsen/2.x/feature/cloud_mongo

Cloud Mongo

pype/api.py

@@ -32,7 +32,10 @@ from .lib import (
     get_version_from_path,
     get_last_version_from_path,
     modified_environ,
-    add_tool_to_environment
+    add_tool_to_environment,
+    decompose_url,
+    compose_url,
+    get_default_components
 )
 
 # Special naming case for subprocess since its a built-in method.
@@ -44,6 +47,9 @@ __all__ = [
     "project_overrides_dir_path",
     "config",
     "execute",
+    "decompose_url",
+    "compose_url",
+    "get_default_components",
 
     # plugin classes
     "Extractor",

pype/lib.py

@@ -17,6 +17,11 @@ import six
 import avalon.api
 from .api import config
 
+try:
+    from urllib.parse import urlparse
+except ImportError:
+    from urlparse import urlparse
+
 log = logging.getLogger(__name__)
 
 
@@ -1386,3 +1391,73 @@ def ffprobe_streams(path_to_file):
     popen_output = popen.communicate()[0]
     log.debug("FFprobe output: {}".format(popen_output))
     return json.loads(popen_output)["streams"]
+
+
+def decompose_url(url):
+    components = {
+        "scheme": None,
+        "host": None,
+        "port": None,
+        "username": None,
+        "password": None,
+        "query": None
+    }
+
+    result = urlparse(url)
+
+    components["scheme"] = result.scheme
+    components["host"] = result.hostname
+    try:
+        components["port"] = result.port
+    except ValueError:
+        raise RuntimeError("invalid port specified")
+
+    components["username"] = result.username
+    components["password"] = result.password
+    components["query"] = result.query
+
+    return components
+
+
+def compose_url(scheme=None,
+                host=None,
+                username=None,
+                password=None,
+                database=None,
+                collection=None,
+                port=None,
+                query=None):
+
+    url = "{scheme}://"
+
+    if username and password:
+        url += "{username}:{password}@"
+
+    url += "{host}"
+
+    if database:
+        url += "/{database}"
+
+    if database and collection:
+        url += "/{collection}"
+
+    if port:
+        url += ":{port}"
+
+    if query:
+        url += "?{}".format(query)
+
+    return url.format(**{
+        "scheme": scheme,
+        "host": host,
+        "username": username,
+        "password": password,
+        "database": database,
+        "collection": collection,
+        "port": port,
+        "query": query
+    })
+
+
+def get_default_components():
+    return decompose_url(os.environ["MONGO_URL"])

pype/modules/adobe_communicator/lib/io_nonsingleton.py

@@ -16,6 +16,7 @@ import contextlib
 
 from avalon import schema
 from avalon.vendor import requests
+from pype.api import get_default_components, compose_url
 
 # Third-party dependencies
 import pymongo
@@ -72,8 +73,15 @@ class DbConnector(object):
         self.Session.update(self._from_environment())
 
         timeout = int(self.Session["AVALON_TIMEOUT"])
+
+        components = get_default_components()
+        port = components.pop("port")
+        host = compose_url(**components)
         self._mongo_client = pymongo.MongoClient(
-            self.Session["AVALON_MONGO"], serverSelectionTimeoutMS=timeout)
+            host=host,
+            port=port,
+            serverSelectionTimeoutMS=timeout
+        )
 
         for retry in range(3):
             try:

pype/modules/avalon_apps/rest_api.py

@@ -8,10 +8,7 @@ from pype.modules.ftrack.lib.custom_db_connector import DbConnector
 
 
 class AvalonRestApi(RestApi):
-    dbcon = DbConnector(
-        os.environ["AVALON_MONGO"],
-        os.environ["AVALON_DB"]
-    )
+    dbcon = DbConnector(os.environ["AVALON_DB"])
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)

pype/modules/ftrack/ftrack_server/event_server_cli.py

@@ -13,10 +13,10 @@ import time
 import uuid
 
 import ftrack_api
+import pymongo
 from pype.modules.ftrack.lib import credentials
-from pype.modules.ftrack.ftrack_server.lib import (
-    ftrack_events_mongo_settings, check_ftrack_url
-)
+from pype.modules.ftrack.ftrack_server.lib import check_ftrack_url
+from pype.api import get_default_components, compose_url
 import socket_thread
 
 
@@ -30,22 +30,19 @@ class MongoPermissionsError(Exception):
 
 def check_mongo_url(host, port, log_error=False):
     """Checks if mongo server is responding"""
-    sock = None
     try:
-        sock = socket.create_connection(
-            (host, port),
-            timeout=1
-        )
-        return True
-    except socket.error as err:
+        client = pymongo.MongoClient(host=host, port=port)
+        # Force connection on a request as the connect=True parameter of
+        # MongoClient seems to be useless here
+        client.server_info()
+    except pymongo.errors.ServerSelectionTimeoutError as err:
         if log_error:
             print("Can't connect to MongoDB at {}:{} because: {}".format(
                 host, port, err
             ))
         return False
-    finally:
-        if sock is not None:
-            sock.close()
+
+    return True
 
 
 def validate_credentials(url, user, api):
@@ -190,9 +187,9 @@ def main_loop(ftrack_url):
 
     os.environ["FTRACK_EVENT_SUB_ID"] = str(uuid.uuid1())
     # Get mongo hostname and port for testing mongo connection
-    mongo_list = ftrack_events_mongo_settings()
-    mongo_hostname = mongo_list[0]
-    mongo_port = mongo_list[1]
+    components = get_default_components()
+    mongo_port = components.pop("port")
+    mongo_hostname = compose_url(**components)
 
     # Current file
     file_path = os.path.dirname(os.path.realpath(__file__))
@@ -275,7 +272,7 @@ def main_loop(ftrack_url):
         # Run threads only if Ftrack is accessible
         if not ftrack_accessible or not mongo_accessible:
             if not mongo_accessible and not printed_mongo_error:
-                mongo_url = mongo_hostname + ":" + mongo_port
+                mongo_url = "{}:{}".format(mongo_hostname, mongo_port)
                 print("Can't access Mongo {}".format(mongo_url))
 
             if not ftrack_accessible and not printed_ftrack_error:

pype/modules/ftrack/ftrack_server/lib.py

@@ -18,12 +18,8 @@ import ftrack_api.operation
 import ftrack_api._centralized_storage_scenario
 import ftrack_api.event
 from ftrack_api.logging import LazyLogMessage as L
-try:
-    from urllib.parse import urlparse, parse_qs
-except ImportError:
-    from urlparse import urlparse, parse_qs
 
-from pype.api import Logger
+from pype.api import Logger, compose_url, get_default_components
 
 from pype.modules.ftrack.lib.custom_db_connector import DbConnector
 
@@ -32,67 +28,10 @@ TOPIC_STATUS_SERVER = "pype.event.server.status"
 TOPIC_STATUS_SERVER_RESULT = "pype.event.server.status.result"
 
 
-def ftrack_events_mongo_settings():
-    host = None
-    port = None
-    username = None
-    password = None
-    collection = None
-    database = None
-    auth_db = ""
-
-    if os.environ.get('FTRACK_EVENTS_MONGO_URL'):
-        result = urlparse(os.environ['FTRACK_EVENTS_MONGO_URL'])
-
-        host = result.hostname
-        try:
-            port = result.port
-        except ValueError:
-            raise RuntimeError("invalid port specified")
-        username = result.username
-        password = result.password
-        try:
-            database = result.path.lstrip("/").split("/")[0]
-            collection = result.path.lstrip("/").split("/")[1]
-        except IndexError:
-            if not database:
-                raise RuntimeError("missing database name for logging")
-        try:
-            auth_db = parse_qs(result.query)['authSource'][0]
-        except KeyError:
-            # no auth db provided, mongo will use the one we are connecting to
-            pass
-    else:
-        host = os.environ.get('FTRACK_EVENTS_MONGO_HOST')
-        port = int(os.environ.get('FTRACK_EVENTS_MONGO_PORT', "0"))
-        database = os.environ.get('FTRACK_EVENTS_MONGO_DB')
-        username = os.environ.get('FTRACK_EVENTS_MONGO_USER')
-        password = os.environ.get('FTRACK_EVENTS_MONGO_PASSWORD')
-        collection = os.environ.get('FTRACK_EVENTS_MONGO_COL')
-        auth_db = os.environ.get('FTRACK_EVENTS_MONGO_AUTH_DB', 'avalon')
-
-    return host, port, database, username, password, collection, auth_db
-
-
 def get_ftrack_event_mongo_info():
-    host, port, database, username, password, collection, auth_db = (
-        ftrack_events_mongo_settings()
-    )
-    user_pass = ""
-    if username and password:
-        user_pass = "{}:{}@".format(username, password)
-
-    socket_path = "{}:{}".format(host, port)
-
-    dab = ""
-    if database:
-        dab = "/{}".format(database)
-
-    auth = ""
-    if auth_db:
-        auth = "?authSource={}".format(auth_db)
-
-    url = "mongodb://{}{}{}{}".format(user_pass, socket_path, dab, auth)
+    url = compose_url(get_default_components())
+    database = os.environ.get("FTRACK_EVENTS_MONGO_DB") or "pype"
+    collection = os.environ.get("FTRACK_EVENTS_MONGO_COL") or "ftrack_events"
 
     return url, database, collection
 
@@ -205,7 +144,6 @@ class ProcessEventHub(SocketBaseEventHub):
 
     def __init__(self, *args, **kwargs):
         self.dbcon = DbConnector(
-            mongo_url=self.url,
             database_name=self.database,
             table_name=self.table_name
         )
@@ -269,7 +207,7 @@ class ProcessEventHub(SocketBaseEventHub):
     def load_events(self):
         """Load not processed events sorted by stored date"""
         ago_date = datetime.datetime.now() - datetime.timedelta(days=3)
-        result = self.dbcon.delete_many({
+        self.dbcon.delete_many({
             "pype_data.stored": {"$lte": ago_date},
             "pype_data.is_processed": True
         })

pype/modules/ftrack/ftrack_server/sub_event_storer.py

@@ -25,7 +25,6 @@ class SessionFactory:
 
 url, database, table_name = get_ftrack_event_mongo_info()
 dbcon = DbConnector(
-    mongo_url=url,
     database_name=database,
     table_name=table_name
 )

pype/modules/ftrack/lib/custom_db_connector.py

@@ -13,6 +13,8 @@ import atexit
 # Third-party dependencies
 import pymongo
 
+from pype.api import get_default_components, compose_url
+
 
 class NotActiveTable(Exception):
     def __init__(self, *args, **kwargs):
@@ -63,13 +65,12 @@ class DbConnector:
     log = logging.getLogger(__name__)
     timeout = 1000
 
-    def __init__(self, mongo_url, database_name, table_name=None):
+    def __init__(self, database_name, table_name=None):
         self._mongo_client = None
         self._sentry_client = None
         self._sentry_logging_handler = None
         self._database = None
         self._is_installed = False
-        self._mongo_url = mongo_url
         self._database_name = database_name
 
         self.active_table = table_name
@@ -95,8 +96,12 @@ class DbConnector:
         atexit.register(self.uninstall)
         logging.basicConfig()
 
+        components = get_default_components()
+        port = components.pop("port")
+        host = compose_url(**components)
         self._mongo_client = pymongo.MongoClient(
-            self._mongo_url,
+            host=host,
+            port=port,
             serverSelectionTimeoutMS=self.timeout
         )
 
@@ -113,11 +118,11 @@ class DbConnector:
         else:
             raise IOError(
                 "ERROR: Couldn't connect to %s in "
-                "less than %.3f ms" % (self._mongo_url, self.timeout)
+                "less than %.3f ms" % (host, self.timeout)
             )
 
         self.log.info("Connected to %s, delay %.3f s" % (
-            self._mongo_url, time.time() - t1
+            host, time.time() - t1
         ))
 
         self._database = self._mongo_client[self._database_name]

pype/modules/ftrack/lib/io_nonsingleton.py

@@ -73,7 +73,10 @@ class DbConnector(object):
 
         timeout = int(self.Session["AVALON_TIMEOUT"])
         self._mongo_client = pymongo.MongoClient(
-            self.Session["AVALON_MONGO"], serverSelectionTimeoutMS=timeout)
+            host=os.environ["AVALON_MONGO_HOST"],
+            port=int(os.environ["AVALON_MONGO_PORT"]),
+            serverSelectionTimeoutMS=timeout
+        )
 
         for retry in range(3):
             try:

pype/modules/logging/gui/models.py

@@ -1,8 +1,7 @@
-import os
 import collections
 from Qt import QtCore
 from pype.api import Logger
-from pypeapp.lib.log import _bootstrap_mongo_log
+from pypeapp.lib.log import _bootstrap_mongo_log, COLLECTION
 
 log = Logger().get_logger("LogModel", "LoggingModule")
 
@@ -41,11 +40,10 @@ class LogModel(QtCore.QAbstractItemModel):
         super(LogModel, self).__init__(parent)
         self._root_node = Node()
 
-        collection = os.environ.get('PYPE_LOG_MONGO_COL')
         database = _bootstrap_mongo_log()
         self.dbcon = None
-        if collection in database.list_collection_names():
-            self.dbcon = database[collection]
+        if COLLECTION in database.list_collection_names():
+            self.dbcon = database[COLLECTION]
 
     def add_log(self, log):
         node = Node(log)

pype/modules/logging/tray/logging_module.py

@@ -1,20 +1,23 @@
 from Qt import QtWidgets
-
 from pype.api import Logger
-
 from ..gui.app import LogsWindow
 
-log = Logger().get_logger("LoggingModule", "logging")
-
 
 class LoggingModule:
     def __init__(self, main_parent=None, parent=None):
         self.parent = parent
+        self.log = Logger().get_logger(self.__class__.__name__, "logging")
 
-        self.window = LogsWindow()
+        try:
+            self.window = LogsWindow()
+            self.tray_menu = self._tray_menu
+        except Exception:
+            self.log.warning(
+                "Couldn't set Logging GUI due to error.", exc_info=True
+            )
 
     # Definition of Tray menu
-    def tray_menu(self, parent_menu):
+    def _tray_menu(self, parent_menu):
         # Menu for Tray App
         menu = QtWidgets.QMenu('Logging', parent_menu)
         # menu.setProperty('submenu', 'on')

schema/session-2.0.json

@@ -56,13 +56,6 @@
             "pattern": "^\\w*$",
             "example": "maya2016"
         },
-        "AVALON_MONGO": {
-            "description": "Address to the asset database",
-            "type": "string",
-            "pattern": "^mongodb://[\\w/@:.]*$",
-            "example": "mongodb://localhost:27017",
-            "default": "mongodb://localhost:27017"
-        },
         "AVALON_DB": {
             "description": "Name of database",
             "type": "string",