diff --git a/igniter/tools.py b/igniter/tools.py index c934289064..04d7451335 100644 --- a/igniter/tools.py +++ b/igniter/tools.py @@ -1,18 +1,12 @@ # -*- coding: utf-8 -*- -"""Tools used in **Igniter** GUI. - -Functions ``compose_url()`` and ``decompose_url()`` are the same as in -``openpype.lib`` and they are here to avoid importing OpenPype module before its -version is decided. - -""" -import sys +"""Tools used in **Igniter** GUI.""" import os -from typing import Dict, Union +from typing import Union from urllib.parse import urlparse, parse_qs from pathlib import Path import platform +import certifi from pymongo import MongoClient from pymongo.errors import ( ServerSelectionTimeoutError, @@ -22,89 +16,32 @@ from pymongo.errors import ( ) -def decompose_url(url: str) -> Dict: - """Decompose mongodb url to its separate components. - - Args: - url (str): Mongodb url. - - Returns: - dict: Dictionary of components. +def should_add_certificate_path_to_mongo_url(mongo_url): + """Check if should add ca certificate to mongo url. + Since 30.9.2021 cloud mongo requires newer certificates that are not + available on most of workstation. This adds path to certifi certificate + which is valid for it. To add the certificate path url must have scheme + 'mongodb+srv' or has 'ssl=true' or 'tls=true' in url query. """ - components = { - "scheme": None, - "host": None, - "port": None, - "username": None, - "password": None, - "auth_db": None - } + parsed = urlparse(mongo_url) + query = parse_qs(parsed.query) + lowered_query_keys = set(key.lower() for key in query.keys()) + add_certificate = False + # Check if url 'ssl' or 'tls' are set to 'true' + for key in ("ssl", "tls"): + if key in query and "true" in query["ssl"]: + add_certificate = True + break - result = urlparse(url) - if result.scheme is None: - _url = "mongodb://{}".format(url) - result = urlparse(_url) + # Check if url contains 'mongodb+srv' + if not add_certificate and parsed.scheme == "mongodb+srv": + add_certificate = True - components["scheme"] = result.scheme - components["host"] = result.hostname - try: - components["port"] = result.port - except ValueError: - raise RuntimeError("invalid port specified") - components["username"] = result.username - components["password"] = result.password - - try: - components["auth_db"] = parse_qs(result.query)['authSource'][0] - except KeyError: - # no auth db provided, mongo will use the one we are connecting to - pass - - return components - - -def compose_url(scheme: str = None, - host: str = None, - username: str = None, - password: str = None, - port: int = None, - auth_db: str = None) -> str: - """Compose mongodb url from its individual components. - - Args: - scheme (str, optional): - host (str, optional): - username (str, optional): - password (str, optional): - port (str, optional): - auth_db (str, optional): - - Returns: - str: mongodb url - - """ - - url = "{scheme}://" - - if username and password: - url += "{username}:{password}@" - - url += "{host}" - if port: - url += ":{port}" - - if auth_db: - url += "?authSource={auth_db}" - - return url.format(**{ - "scheme": scheme, - "host": host, - "username": username, - "password": password, - "port": port, - "auth_db": auth_db - }) + # Check if url does already contain certificate path + if add_certificate and "tlscafile" in lowered_query_keys: + add_certificate = False + return add_certificate def validate_mongo_connection(cnx: str) -> (bool, str): @@ -121,12 +58,18 @@ def validate_mongo_connection(cnx: str) -> (bool, str): if parsed.scheme not in ["mongodb", "mongodb+srv"]: return False, "Not mongodb schema" + kwargs = { + "serverSelectionTimeoutMS": 2000 + } + # Add certificate path if should be required + if should_add_certificate_path_to_mongo_url(cnx): + kwargs["ssl_ca_certs"] = certifi.where() + try: - client = MongoClient( - cnx, - serverSelectionTimeoutMS=2000 - ) + client = MongoClient(cnx, **kwargs) client.server_info() + with client.start_session(): + pass client.close() except ServerSelectionTimeoutError as e: return False, f"Cannot connect to server {cnx} - {e}" @@ -152,10 +95,7 @@ def validate_mongo_string(mongo: str) -> (bool, str): """ if not mongo: return True, "empty string" - parsed = urlparse(mongo) - if parsed.scheme in ["mongodb", "mongodb+srv"]: - return validate_mongo_connection(mongo) - return False, "not valid mongodb schema" + return validate_mongo_connection(mongo) def validate_path_string(path: str) -> (bool, str): @@ -195,21 +135,13 @@ def get_openpype_global_settings(url: str) -> dict: Returns: dict: With settings data. Empty dictionary is returned if not found. """ - try: - components = decompose_url(url) - except RuntimeError: - return {} - mongo_kwargs = { - "host": compose_url(**components), - "serverSelectionTimeoutMS": 2000 - } - port = components.get("port") - if port is not None: - mongo_kwargs["port"] = int(port) + kwargs = {} + if should_add_certificate_path_to_mongo_url(url): + kwargs["ssl_ca_certs"] = certifi.where() try: # Create mongo connection - client = MongoClient(**mongo_kwargs) + client = MongoClient(url, **kwargs) # Access settings collection col = client["openpype"]["settings"] # Query global settings diff --git a/openpype/lib/mongo.py b/openpype/lib/mongo.py index 8bfaba75d6..0fd4517b5b 100644 --- a/openpype/lib/mongo.py +++ b/openpype/lib/mongo.py @@ -3,6 +3,7 @@ import sys import time import logging import pymongo +import certifi if sys.version_info[0] == 2: from urlparse import urlparse, parse_qs @@ -85,12 +86,33 @@ def get_default_components(): return decompose_url(mongo_url) -def extract_port_from_url(url): - parsed_url = urlparse(url) - if parsed_url.scheme is None: - _url = "mongodb://{}".format(url) - parsed_url = urlparse(_url) - return parsed_url.port +def should_add_certificate_path_to_mongo_url(mongo_url): + """Check if should add ca certificate to mongo url. + + Since 30.9.2021 cloud mongo requires newer certificates that are not + available on most of workstation. This adds path to certifi certificate + which is valid for it. To add the certificate path url must have scheme + 'mongodb+srv' or has 'ssl=true' or 'tls=true' in url query. + """ + parsed = urlparse(mongo_url) + query = parse_qs(parsed.query) + lowered_query_keys = set(key.lower() for key in query.keys()) + add_certificate = False + # Check if url 'ssl' or 'tls' are set to 'true' + for key in ("ssl", "tls"): + if key in query and "true" in query["ssl"]: + add_certificate = True + break + + # Check if url contains 'mongodb+srv' + if not add_certificate and parsed.scheme == "mongodb+srv": + add_certificate = True + + # Check if url does already contain certificate path + if add_certificate and "tlscafile" in lowered_query_keys: + add_certificate = False + + return add_certificate def validate_mongo_connection(mongo_uri): @@ -106,26 +128,9 @@ def validate_mongo_connection(mongo_uri): passed so probably couldn't connect to mongo server. """ - parsed = urlparse(mongo_uri) - # Force validation of scheme - if parsed.scheme not in ["mongodb", "mongodb+srv"]: - raise pymongo.errors.InvalidURI(( - "Invalid URI scheme:" - " URI must begin with 'mongodb://' or 'mongodb+srv://'" - )) - # we have mongo connection string. Let's try if we can connect. - components = decompose_url(mongo_uri) - mongo_args = { - "host": compose_url(**components), - "serverSelectionTimeoutMS": 1000 - } - port = components.get("port") - if port is not None: - mongo_args["port"] = int(port) - - # Create connection - client = pymongo.MongoClient(**mongo_args) - client.server_info() + client = OpenPypeMongoConnection.create_connection( + mongo_uri, retry_attempts=1 + ) client.close() @@ -151,6 +156,8 @@ class OpenPypeMongoConnection: # Naive validation of existing connection try: connection.server_info() + with connection.start_session(): + pass except Exception: connection = None @@ -162,38 +169,53 @@ class OpenPypeMongoConnection: return connection @classmethod - def create_connection(cls, mongo_url, timeout=None): + def create_connection(cls, mongo_url, timeout=None, retry_attempts=None): + parsed = urlparse(mongo_url) + # Force validation of scheme + if parsed.scheme not in ["mongodb", "mongodb+srv"]: + raise pymongo.errors.InvalidURI(( + "Invalid URI scheme:" + " URI must begin with 'mongodb://' or 'mongodb+srv://'" + )) + if timeout is None: timeout = int(os.environ.get("AVALON_TIMEOUT") or 1000) kwargs = { - "host": mongo_url, "serverSelectionTimeoutMS": timeout } + if should_add_certificate_path_to_mongo_url(mongo_url): + kwargs["ssl_ca_certs"] = certifi.where() - port = extract_port_from_url(mongo_url) - if port is not None: - kwargs["port"] = int(port) + mongo_client = pymongo.MongoClient(mongo_url, **kwargs) - mongo_client = pymongo.MongoClient(**kwargs) + if retry_attempts is None: + retry_attempts = 3 - for _retry in range(3): + elif not retry_attempts: + retry_attempts = 1 + + last_exc = None + valid = False + t1 = time.time() + for attempt in range(1, retry_attempts + 1): try: - t1 = time.time() mongo_client.server_info() - - except Exception: - cls.log.warning("Retrying...") - time.sleep(1) - timeout *= 1.5 - - else: + with mongo_client.start_session(): + pass + valid = True break - else: - raise IOError(( - "ERROR: Couldn't connect to {} in less than {:.3f}ms" - ).format(mongo_url, timeout)) + except Exception as exc: + last_exc = exc + if attempt < retry_attempts: + cls.log.warning( + "Attempt {} failed. Retrying... ".format(attempt) + ) + time.sleep(1) + + if not valid: + raise last_exc cls.log.info("Connected to {}, delay {:.3f}s".format( mongo_url, time.time() - t1 diff --git a/openpype/modules/default_modules/ftrack/ftrack_server/event_server_cli.py b/openpype/modules/default_modules/ftrack/ftrack_server/event_server_cli.py index 075694d8f6..1a76905b38 100644 --- a/openpype/modules/default_modules/ftrack/ftrack_server/event_server_cli.py +++ b/openpype/modules/default_modules/ftrack/ftrack_server/event_server_cli.py @@ -17,7 +17,8 @@ from openpype.lib import ( get_pype_execute_args, OpenPypeMongoConnection, get_openpype_version, - get_build_version + get_build_version, + validate_mongo_connection ) from openpype_modules.ftrack import FTRACK_MODULE_DIR from openpype_modules.ftrack.lib import credentials @@ -36,11 +37,15 @@ class MongoPermissionsError(Exception): def check_mongo_url(mongo_uri, log_error=False): """Checks if mongo server is responding""" try: - client = pymongo.MongoClient(mongo_uri) - # Force connection on a request as the connect=True parameter of - # MongoClient seems to be useless here - client.server_info() - client.close() + validate_mongo_connection(mongo_uri) + + except pymongo.errors.InvalidURI as err: + if log_error: + print("Can't connect to MongoDB at {} because: {}".format( + mongo_uri, err + )) + return False + except pymongo.errors.ServerSelectionTimeoutError as err: if log_error: print("Can't connect to MongoDB at {} because: {}".format( diff --git a/start.py b/start.py index 689efbdac1..ada613b4eb 100644 --- a/start.py +++ b/start.py @@ -102,9 +102,6 @@ import subprocess import site from pathlib import Path -from igniter.tools import get_openpype_global_settings - - # OPENPYPE_ROOT is variable pointing to build (or code) directory # WARNING `OPENPYPE_ROOT` must be defined before igniter import # - igniter changes cwd which cause that filepath of this script won't lead @@ -192,6 +189,7 @@ else: import igniter # noqa: E402 from igniter import BootstrapRepos # noqa: E402 from igniter.tools import ( + get_openpype_global_settings, get_openpype_path_from_db, validate_mongo_connection ) # noqa