From 511b2d1e24b1d463c18136eecc18f228aa03c7fd Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Thu, 18 Jun 2020 22:33:29 +0100 Subject: [PATCH] Cloud Mongo There was a circular import when importing from pypeapp, which I couldnt resolve, so have copied the mongo url logic across. --- pype/api.py | 8 +- pype/lib.py | 81 +++++++++++++++++++ .../adobe_communicator/lib/io_nonsingleton.py | 10 ++- pype/modules/avalon_apps/rest_api.py | 5 +- .../ftrack/ftrack_server/event_server_cli.py | 31 ++++--- pype/modules/ftrack/ftrack_server/lib.py | 72 ++--------------- .../ftrack/ftrack_server/sub_event_storer.py | 1 - .../modules/ftrack/lib/custom_db_connector.py | 15 ++-- pype/modules/ftrack/lib/io_nonsingleton.py | 5 +- schema/session-2.0.json | 7 -- 10 files changed, 131 insertions(+), 104 deletions(-) diff --git a/pype/api.py b/pype/api.py index 200ca9daec..f279bb501a 100644 --- a/pype/api.py +++ b/pype/api.py @@ -32,7 +32,10 @@ from .lib import ( get_version_from_path, get_last_version_from_path, modified_environ, - add_tool_to_environment + add_tool_to_environment, + decompose_url, + compose_url, + get_default_components ) # Special naming case for subprocess since its a built-in method. @@ -44,6 +47,9 @@ __all__ = [ "project_overrides_dir_path", "config", "execute", + "decompose_url", + "compose_url", + "get_default_components", # plugin classes "Extractor", diff --git a/pype/lib.py b/pype/lib.py index 7c7a01d5cc..a41a082bc0 100644 --- a/pype/lib.py +++ b/pype/lib.py @@ -1386,3 +1386,84 @@ def ffprobe_streams(path_to_file): popen_output = popen.communicate()[0] log.debug("FFprobe output: {}".format(popen_output)) return json.loads(popen_output)["streams"] + + +import os + +try: + from urllib.parse import urlparse, parse_qs +except ImportError: + from urlparse import urlparse, parse_qs + + +def decompose_url(url): + components = { + "scheme": None, + "host": None, + "port": None, + "username": None, + "password": None, + "auth_db": "" + } + + result = urlparse(url) + + components["scheme"] = result.scheme + components["host"] = result.hostname + try: + components["port"] = result.port + except ValueError: + raise RuntimeError("invalid port specified") + components["username"] = result.username + components["password"] = result.password + + try: + components["auth_db"] = parse_qs(result.query)['authSource'][0] + except KeyError: + # no auth db provided, mongo will use the one we are connecting to + pass + + return components + + +def compose_url(scheme=None, + host=None, + username=None, + password=None, + database=None, + collection=None, + port=None, + auth_db=""): + + url = "{scheme}://" + + if username and password: + url += "{username}:{password}@" + + url += "{host}" + + if database: + url += "/{database}" + + if database and collection: + url += "/{collection}" + + if port: + url += ":{port}" + + url += auth_db + + return url.format(**{ + "scheme": scheme, + "host": host, + "username": username, + "password": password, + "database": database, + "collection": collection, + "port": port, + "auth_db": "" + }) + + +def get_default_components(): + return decompose_url(os.environ["MONGO_URL"]) diff --git a/pype/modules/adobe_communicator/lib/io_nonsingleton.py b/pype/modules/adobe_communicator/lib/io_nonsingleton.py index 6380e4eb23..d042d2f6d8 100644 --- a/pype/modules/adobe_communicator/lib/io_nonsingleton.py +++ b/pype/modules/adobe_communicator/lib/io_nonsingleton.py @@ -16,6 +16,7 @@ import contextlib from avalon import schema from avalon.vendor import requests +from pype.api import get_default_components, compose_url # Third-party dependencies import pymongo @@ -72,8 +73,15 @@ class DbConnector(object): self.Session.update(self._from_environment()) timeout = int(self.Session["AVALON_TIMEOUT"]) + + components = get_default_components() + port = components.pop("port") + host = compose_url(**components) self._mongo_client = pymongo.MongoClient( - self.Session["AVALON_MONGO"], serverSelectionTimeoutMS=timeout) + host=host, + port=port, + serverSelectionTimeoutMS=timeout + ) for retry in range(3): try: diff --git a/pype/modules/avalon_apps/rest_api.py b/pype/modules/avalon_apps/rest_api.py index a5dc326a8e..58cb3a47f3 100644 --- a/pype/modules/avalon_apps/rest_api.py +++ b/pype/modules/avalon_apps/rest_api.py @@ -8,10 +8,7 @@ from pype.modules.ftrack.lib.custom_db_connector import DbConnector class AvalonRestApi(RestApi): - dbcon = DbConnector( - os.environ["AVALON_MONGO"], - os.environ["AVALON_DB"] - ) + dbcon = DbConnector(os.environ["AVALON_DB"]) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/pype/modules/ftrack/ftrack_server/event_server_cli.py b/pype/modules/ftrack/ftrack_server/event_server_cli.py index 5709a88297..7ca04756df 100644 --- a/pype/modules/ftrack/ftrack_server/event_server_cli.py +++ b/pype/modules/ftrack/ftrack_server/event_server_cli.py @@ -13,10 +13,10 @@ import time import uuid import ftrack_api +import pymongo from pype.modules.ftrack.lib import credentials -from pype.modules.ftrack.ftrack_server.lib import ( - ftrack_events_mongo_settings, check_ftrack_url -) +from pype.modules.ftrack.ftrack_server.lib import check_ftrack_url +from pype.api import get_default_components, compose_url import socket_thread @@ -30,22 +30,19 @@ class MongoPermissionsError(Exception): def check_mongo_url(host, port, log_error=False): """Checks if mongo server is responding""" - sock = None try: - sock = socket.create_connection( - (host, port), - timeout=1 - ) - return True - except socket.error as err: + client = pymongo.MongoClient(host=host, port=port) + # Force connection on a request as the connect=True parameter of + # MongoClient seems to be useless here + client.server_info() + except pymongo.errors.ServerSelectionTimeoutError as err: if log_error: print("Can't connect to MongoDB at {}:{} because: {}".format( host, port, err )) return False - finally: - if sock is not None: - sock.close() + + return True def validate_credentials(url, user, api): @@ -190,9 +187,9 @@ def main_loop(ftrack_url): os.environ["FTRACK_EVENT_SUB_ID"] = str(uuid.uuid1()) # Get mongo hostname and port for testing mongo connection - mongo_list = ftrack_events_mongo_settings() - mongo_hostname = mongo_list[0] - mongo_port = mongo_list[1] + components = get_default_components() + mongo_port = components.pop("port") + mongo_hostname = compose_url(**components) # Current file file_path = os.path.dirname(os.path.realpath(__file__)) @@ -275,7 +272,7 @@ def main_loop(ftrack_url): # Run threads only if Ftrack is accessible if not ftrack_accessible or not mongo_accessible: if not mongo_accessible and not printed_mongo_error: - mongo_url = mongo_hostname + ":" + mongo_port + mongo_url = "{}:{}".format(mongo_hostname, mongo_port) print("Can't access Mongo {}".format(mongo_url)) if not ftrack_accessible and not printed_ftrack_error: diff --git a/pype/modules/ftrack/ftrack_server/lib.py b/pype/modules/ftrack/ftrack_server/lib.py index 129cd7173a..b7383dab07 100644 --- a/pype/modules/ftrack/ftrack_server/lib.py +++ b/pype/modules/ftrack/ftrack_server/lib.py @@ -18,12 +18,8 @@ import ftrack_api.operation import ftrack_api._centralized_storage_scenario import ftrack_api.event from ftrack_api.logging import LazyLogMessage as L -try: - from urllib.parse import urlparse, parse_qs -except ImportError: - from urlparse import urlparse, parse_qs -from pype.api import Logger +from pype.api import Logger, compose_url, get_default_components from pype.modules.ftrack.lib.custom_db_connector import DbConnector @@ -32,67 +28,10 @@ TOPIC_STATUS_SERVER = "pype.event.server.status" TOPIC_STATUS_SERVER_RESULT = "pype.event.server.status.result" -def ftrack_events_mongo_settings(): - host = None - port = None - username = None - password = None - collection = None - database = None - auth_db = "" - - if os.environ.get('FTRACK_EVENTS_MONGO_URL'): - result = urlparse(os.environ['FTRACK_EVENTS_MONGO_URL']) - - host = result.hostname - try: - port = result.port - except ValueError: - raise RuntimeError("invalid port specified") - username = result.username - password = result.password - try: - database = result.path.lstrip("/").split("/")[0] - collection = result.path.lstrip("/").split("/")[1] - except IndexError: - if not database: - raise RuntimeError("missing database name for logging") - try: - auth_db = parse_qs(result.query)['authSource'][0] - except KeyError: - # no auth db provided, mongo will use the one we are connecting to - pass - else: - host = os.environ.get('FTRACK_EVENTS_MONGO_HOST') - port = int(os.environ.get('FTRACK_EVENTS_MONGO_PORT', "0")) - database = os.environ.get('FTRACK_EVENTS_MONGO_DB') - username = os.environ.get('FTRACK_EVENTS_MONGO_USER') - password = os.environ.get('FTRACK_EVENTS_MONGO_PASSWORD') - collection = os.environ.get('FTRACK_EVENTS_MONGO_COL') - auth_db = os.environ.get('FTRACK_EVENTS_MONGO_AUTH_DB', 'avalon') - - return host, port, database, username, password, collection, auth_db - - def get_ftrack_event_mongo_info(): - host, port, database, username, password, collection, auth_db = ( - ftrack_events_mongo_settings() - ) - user_pass = "" - if username and password: - user_pass = "{}:{}@".format(username, password) - - socket_path = "{}:{}".format(host, port) - - dab = "" - if database: - dab = "/{}".format(database) - - auth = "" - if auth_db: - auth = "?authSource={}".format(auth_db) - - url = "mongodb://{}{}{}{}".format(user_pass, socket_path, dab, auth) + url = compose_url(get_default_components()) + database = os.environ["FTRACK_EVENTS_MONGO_DB"] + collection = os.environ["FTRACK_EVENTS_MONGO_COL"] return url, database, collection @@ -205,7 +144,6 @@ class ProcessEventHub(SocketBaseEventHub): def __init__(self, *args, **kwargs): self.dbcon = DbConnector( - mongo_url=self.url, database_name=self.database, table_name=self.table_name ) @@ -269,7 +207,7 @@ class ProcessEventHub(SocketBaseEventHub): def load_events(self): """Load not processed events sorted by stored date""" ago_date = datetime.datetime.now() - datetime.timedelta(days=3) - result = self.dbcon.delete_many({ + self.dbcon.delete_many({ "pype_data.stored": {"$lte": ago_date}, "pype_data.is_processed": True }) diff --git a/pype/modules/ftrack/ftrack_server/sub_event_storer.py b/pype/modules/ftrack/ftrack_server/sub_event_storer.py index c4d199407d..727d5aa515 100644 --- a/pype/modules/ftrack/ftrack_server/sub_event_storer.py +++ b/pype/modules/ftrack/ftrack_server/sub_event_storer.py @@ -25,7 +25,6 @@ class SessionFactory: url, database, table_name = get_ftrack_event_mongo_info() dbcon = DbConnector( - mongo_url=url, database_name=database, table_name=table_name ) diff --git a/pype/modules/ftrack/lib/custom_db_connector.py b/pype/modules/ftrack/lib/custom_db_connector.py index b307117127..e570e4da38 100644 --- a/pype/modules/ftrack/lib/custom_db_connector.py +++ b/pype/modules/ftrack/lib/custom_db_connector.py @@ -13,6 +13,8 @@ import atexit # Third-party dependencies import pymongo +from pype.api import get_default_components, compose_url + class NotActiveTable(Exception): def __init__(self, *args, **kwargs): @@ -63,13 +65,12 @@ class DbConnector: log = logging.getLogger(__name__) timeout = 1000 - def __init__(self, mongo_url, database_name, table_name=None): + def __init__(self, database_name, table_name=None): self._mongo_client = None self._sentry_client = None self._sentry_logging_handler = None self._database = None self._is_installed = False - self._mongo_url = mongo_url self._database_name = database_name self.active_table = table_name @@ -95,8 +96,12 @@ class DbConnector: atexit.register(self.uninstall) logging.basicConfig() + components = get_default_components() + port = components.pop("port") + host = compose_url(**components) self._mongo_client = pymongo.MongoClient( - self._mongo_url, + host=host, + port=port, serverSelectionTimeoutMS=self.timeout ) @@ -113,11 +118,11 @@ class DbConnector: else: raise IOError( "ERROR: Couldn't connect to %s in " - "less than %.3f ms" % (self._mongo_url, self.timeout) + "less than %.3f ms" % (host, self.timeout) ) self.log.info("Connected to %s, delay %.3f s" % ( - self._mongo_url, time.time() - t1 + host, time.time() - t1 )) self._database = self._mongo_client[self._database_name] diff --git a/pype/modules/ftrack/lib/io_nonsingleton.py b/pype/modules/ftrack/lib/io_nonsingleton.py index 6380e4eb23..73856557ea 100644 --- a/pype/modules/ftrack/lib/io_nonsingleton.py +++ b/pype/modules/ftrack/lib/io_nonsingleton.py @@ -73,7 +73,10 @@ class DbConnector(object): timeout = int(self.Session["AVALON_TIMEOUT"]) self._mongo_client = pymongo.MongoClient( - self.Session["AVALON_MONGO"], serverSelectionTimeoutMS=timeout) + host=os.environ["AVALON_MONGO_HOST"], + port=int(os.environ["AVALON_MONGO_PORT"]), + serverSelectionTimeoutMS=timeout + ) for retry in range(3): try: diff --git a/schema/session-2.0.json b/schema/session-2.0.json index d37f2ac822..7ad2c63bcf 100644 --- a/schema/session-2.0.json +++ b/schema/session-2.0.json @@ -56,13 +56,6 @@ "pattern": "^\\w*$", "example": "maya2016" }, - "AVALON_MONGO": { - "description": "Address to the asset database", - "type": "string", - "pattern": "^mongodb://[\\w/@:.]*$", - "example": "mongodb://localhost:27017", - "default": "mongodb://localhost:27017" - }, "AVALON_DB": { "description": "Name of database", "type": "string",