From de075189ca3d49354322d0f67fa0a43510a07504 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 29 Sep 2021 13:14:21 +0200 Subject: [PATCH 01/35] pass context data in dynamic data for creation --- openpype/hosts/tvpaint/api/plugin.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/tvpaint/api/plugin.py b/openpype/hosts/tvpaint/api/plugin.py index 3eb9a5be31..e148e44a27 100644 --- a/openpype/hosts/tvpaint/api/plugin.py +++ b/openpype/hosts/tvpaint/api/plugin.py @@ -3,4 +3,17 @@ from avalon.tvpaint import pipeline class Creator(PypeCreatorMixin, pipeline.Creator): - pass + @classmethod + def get_dynamic_data(cls, *args, **kwargs): + dynamic_data = super(Creator, cls).get_dynamic_data(*args, **kwargs) + + # Change asset and name by current workfile context + workfile_context = pipeline.get_current_workfile_context() + asset_name = workfile_context.get("asset") + task_name = workfile_context.get("task") + if "asset" not in dynamic_data and asset_name: + dynamic_data["asset"] = asset_name + + if "task" not in dynamic_data and task_name: + dynamic_data["task"] = task_name + return dynamic_data From 31ce3f9fec4946c1628e8ad970709b92574de3bd Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 1 Oct 2021 14:48:51 +0200 Subject: [PATCH 02/35] get rid of decompose_url and compose_url --- igniter/tools.py | 107 ++++------------------------------------------- 1 file changed, 7 insertions(+), 100 deletions(-) diff --git a/igniter/tools.py b/igniter/tools.py index c934289064..4e31601665 100644 --- a/igniter/tools.py +++ b/igniter/tools.py @@ -22,89 +22,6 @@ from pymongo.errors import ( ) -def decompose_url(url: str) -> Dict: - """Decompose mongodb url to its separate components. - - Args: - url (str): Mongodb url. - - Returns: - dict: Dictionary of components. - - """ - components = { - "scheme": None, - "host": None, - "port": None, - "username": None, - "password": None, - "auth_db": None - } - - result = urlparse(url) - if result.scheme is None: - _url = "mongodb://{}".format(url) - result = urlparse(_url) - - components["scheme"] = result.scheme - components["host"] = result.hostname - try: - components["port"] = result.port - except ValueError: - raise RuntimeError("invalid port specified") - components["username"] = result.username - components["password"] = result.password - - try: - components["auth_db"] = parse_qs(result.query)['authSource'][0] - except KeyError: - # no auth db provided, mongo will use the one we are connecting to - pass - - return components - - -def compose_url(scheme: str = None, - host: str = None, - username: str = None, - password: str = None, - port: int = None, - auth_db: str = None) -> str: - """Compose mongodb url from its individual components. - - Args: - scheme (str, optional): - host (str, optional): - username (str, optional): - password (str, optional): - port (str, optional): - auth_db (str, optional): - - Returns: - str: mongodb url - - """ - - url = "{scheme}://" - - if username and password: - url += "{username}:{password}@" - - url += "{host}" - if port: - url += ":{port}" - - if auth_db: - url += "?authSource={auth_db}" - - return url.format(**{ - "scheme": scheme, - "host": host, - "username": username, - "password": password, - "port": port, - "auth_db": auth_db - }) def validate_mongo_connection(cnx: str) -> (bool, str): @@ -121,12 +38,14 @@ def validate_mongo_connection(cnx: str) -> (bool, str): if parsed.scheme not in ["mongodb", "mongodb+srv"]: return False, "Not mongodb schema" + kwargs = { + "serverSelectionTimeoutMS": 2000 + } try: - client = MongoClient( - cnx, - serverSelectionTimeoutMS=2000 - ) + client = MongoClient(cnx, **kwargs) client.server_info() + with client.start_session(): + pass client.close() except ServerSelectionTimeoutError as e: return False, f"Cannot connect to server {cnx} - {e}" @@ -195,21 +114,9 @@ def get_openpype_global_settings(url: str) -> dict: Returns: dict: With settings data. Empty dictionary is returned if not found. """ - try: - components = decompose_url(url) - except RuntimeError: - return {} - mongo_kwargs = { - "host": compose_url(**components), - "serverSelectionTimeoutMS": 2000 - } - port = components.get("port") - if port is not None: - mongo_kwargs["port"] = int(port) - try: # Create mongo connection - client = MongoClient(**mongo_kwargs) + client = MongoClient(url) # Access settings collection col = client["openpype"]["settings"] # Query global settings From 6cad9d50eac6646f64a548af334773fb064af6d6 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 1 Oct 2021 14:52:09 +0200 Subject: [PATCH 03/35] skip scheme validation which happens in validate_mongo_connection --- igniter/tools.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/igniter/tools.py b/igniter/tools.py index 4e31601665..f465d43597 100644 --- a/igniter/tools.py +++ b/igniter/tools.py @@ -71,10 +71,7 @@ def validate_mongo_string(mongo: str) -> (bool, str): """ if not mongo: return True, "empty string" - parsed = urlparse(mongo) - if parsed.scheme in ["mongodb", "mongodb+srv"]: - return validate_mongo_connection(mongo) - return False, "not valid mongodb schema" + return validate_mongo_connection(mongo) def validate_path_string(path: str) -> (bool, str): From 76dd29daa72d48972a0fbb85b41fb7f2389e97c0 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 1 Oct 2021 14:55:40 +0200 Subject: [PATCH 04/35] implemented add_certificate_path_to_mongo_url which adds certificate path to mongo url --- igniter/tools.py | 58 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/igniter/tools.py b/igniter/tools.py index f465d43597..a8ff708f90 100644 --- a/igniter/tools.py +++ b/igniter/tools.py @@ -1,18 +1,12 @@ # -*- coding: utf-8 -*- -"""Tools used in **Igniter** GUI. - -Functions ``compose_url()`` and ``decompose_url()`` are the same as in -``openpype.lib`` and they are here to avoid importing OpenPype module before its -version is decided. - -""" -import sys +"""Tools used in **Igniter** GUI.""" import os -from typing import Dict, Union -from urllib.parse import urlparse, parse_qs +from typing import Union +from urllib.parse import urlparse, parse_qs, ParseResult from pathlib import Path import platform +import certifi from pymongo import MongoClient from pymongo.errors import ( ServerSelectionTimeoutError, @@ -22,6 +16,50 @@ from pymongo.errors import ( ) +def add_certificate_path_to_mongo_url(mongo_url): + """Check if should add ca certificate to mongo url. + + Since 30.9.2021 cloud mongo requires newer certificates that are not + available on most of workstation. This adds path to certifi certificate + which is valid for it. To add the certificate path url must have scheme + 'mongodb+srv' or has 'ssl=true' or 'tls=true' in url query. + """ + parsed = urlparse(mongo_url) + query = parse_qs(parsed.query) + lowered_query_keys = set(key.lower() for key in query.keys()) + add_certificate = False + # Check if url 'ssl' or 'tls' are set to 'true' + for key in ("ssl", "tls"): + if key in query and "true" in query["ssl"]: + add_certificate = True + break + + # Check if url contains 'mongodb+srv' + if not add_certificate and parsed.scheme == "mongodb+srv": + add_certificate = True + + # Check if url does already contain certificate path + if add_certificate and "tlscafile" in lowered_query_keys: + add_certificate = False + + # Add certificate path to mongo url + if add_certificate: + path = parsed.path + if not path: + path = "/admin" + query = parsed.query + tls_query = "tlscafile={}".format(certifi.where()) + if not query: + query = tls_query + else: + query = "&".join((query, tls_query)) + new_url = ParseResult( + parsed.scheme, parsed.netloc, path, + parsed.params, query, parsed.fragment + ) + mongo_url = new_url.geturl() + + return mongo_url def validate_mongo_connection(cnx: str) -> (bool, str): From be191b0c932f575ff772a8ab3b31a3c12edcb0ea Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 1 Oct 2021 14:55:57 +0200 Subject: [PATCH 05/35] use 'add_certificate_path_to_mongo_url' on openpype start --- igniter/tools.py | 3 +++ start.py | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/igniter/tools.py b/igniter/tools.py index a8ff708f90..ae680bf1f1 100644 --- a/igniter/tools.py +++ b/igniter/tools.py @@ -79,6 +79,9 @@ def validate_mongo_connection(cnx: str) -> (bool, str): kwargs = { "serverSelectionTimeoutMS": 2000 } + # Add certificate path if should be required + cnx = add_certificate_path_to_mongo_url(cnx) + try: client = MongoClient(cnx, **kwargs) client.server_info() diff --git a/start.py b/start.py index 689efbdac1..3c345200c3 100644 --- a/start.py +++ b/start.py @@ -193,6 +193,7 @@ import igniter # noqa: E402 from igniter import BootstrapRepos # noqa: E402 from igniter.tools import ( get_openpype_path_from_db, + add_certificate_path_to_mongo_url, validate_mongo_connection ) # noqa from igniter.bootstrap_repos import OpenPypeVersion # noqa: E402 @@ -585,7 +586,7 @@ def _determine_mongodb() -> str: except ValueError: raise RuntimeError("Missing MongoDB url") - return openpype_mongo + return add_certificate_path_to_mongo_url(openpype_mongo) def _initialize_environment(openpype_version: OpenPypeVersion) -> None: From d684cac9bf7e267e6293feeee0d296574f95f6b7 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 1 Oct 2021 14:56:28 +0200 Subject: [PATCH 06/35] added session check to ftrack mongo connection --- .../default_modules/ftrack/ftrack_server/event_server_cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openpype/modules/default_modules/ftrack/ftrack_server/event_server_cli.py b/openpype/modules/default_modules/ftrack/ftrack_server/event_server_cli.py index 075694d8f6..7061b34ab3 100644 --- a/openpype/modules/default_modules/ftrack/ftrack_server/event_server_cli.py +++ b/openpype/modules/default_modules/ftrack/ftrack_server/event_server_cli.py @@ -40,6 +40,8 @@ def check_mongo_url(mongo_uri, log_error=False): # Force connection on a request as the connect=True parameter of # MongoClient seems to be useless here client.server_info() + with client.start_session(): + pass client.close() except pymongo.errors.ServerSelectionTimeoutError as err: if log_error: From 4fd4b8e2e8bfbac7218050b7f15dce9bd4d51038 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 1 Oct 2021 16:33:06 +0200 Subject: [PATCH 07/35] fix import of `get_openpype_global_settings` --- start.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/start.py b/start.py index 3c345200c3..6861355600 100644 --- a/start.py +++ b/start.py @@ -102,9 +102,6 @@ import subprocess import site from pathlib import Path -from igniter.tools import get_openpype_global_settings - - # OPENPYPE_ROOT is variable pointing to build (or code) directory # WARNING `OPENPYPE_ROOT` must be defined before igniter import # - igniter changes cwd which cause that filepath of this script won't lead @@ -192,6 +189,7 @@ else: import igniter # noqa: E402 from igniter import BootstrapRepos # noqa: E402 from igniter.tools import ( + get_openpype_global_settings, get_openpype_path_from_db, add_certificate_path_to_mongo_url, validate_mongo_connection From 0b9160c2fef1e16c6250aa1fdc85a77186403657 Mon Sep 17 00:00:00 2001 From: jrsndlr Date: Fri, 1 Oct 2021 17:27:59 +0200 Subject: [PATCH 08/35] check if files exists by path with hashes --- openpype/lib/delivery.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/openpype/lib/delivery.py b/openpype/lib/delivery.py index 943cd9fcaf..5735cbc99d 100644 --- a/openpype/lib/delivery.py +++ b/openpype/lib/delivery.py @@ -1,9 +1,11 @@ """Functions useful for delivery action or loader""" import os import shutil +import glob import clique import collections + def collect_frames(files): """ Returns dict of source path and its frame, if from sequence @@ -228,7 +230,16 @@ def process_sequence( Returns: (collections.defaultdict , int) """ - if not os.path.exists(src_path): + + def hash_path_exist(myPath): + res = myPath.replace('#', '*') + glob_search_results = glob.glob(res) + if len(glob_search_results) > 0: + return True + else: + return False + + if not hash_path_exist(src_path): msg = "{} doesn't exist for {}".format(src_path, repre["_id"]) report_items["Source file was not found"].append(msg) From b7581c4c7dab7d78d6bf0d3954d7eec535c2fdec Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 1 Oct 2021 18:14:09 +0200 Subject: [PATCH 09/35] event server cli uses validate mongo url from openpype.lib --- .../ftrack/ftrack_server/event_server_cli.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/openpype/modules/default_modules/ftrack/ftrack_server/event_server_cli.py b/openpype/modules/default_modules/ftrack/ftrack_server/event_server_cli.py index 7061b34ab3..1a76905b38 100644 --- a/openpype/modules/default_modules/ftrack/ftrack_server/event_server_cli.py +++ b/openpype/modules/default_modules/ftrack/ftrack_server/event_server_cli.py @@ -17,7 +17,8 @@ from openpype.lib import ( get_pype_execute_args, OpenPypeMongoConnection, get_openpype_version, - get_build_version + get_build_version, + validate_mongo_connection ) from openpype_modules.ftrack import FTRACK_MODULE_DIR from openpype_modules.ftrack.lib import credentials @@ -36,13 +37,15 @@ class MongoPermissionsError(Exception): def check_mongo_url(mongo_uri, log_error=False): """Checks if mongo server is responding""" try: - client = pymongo.MongoClient(mongo_uri) - # Force connection on a request as the connect=True parameter of - # MongoClient seems to be useless here - client.server_info() - with client.start_session(): - pass - client.close() + validate_mongo_connection(mongo_uri) + + except pymongo.errors.InvalidURI as err: + if log_error: + print("Can't connect to MongoDB at {} because: {}".format( + mongo_uri, err + )) + return False + except pymongo.errors.ServerSelectionTimeoutError as err: if log_error: print("Can't connect to MongoDB at {} because: {}".format( From 06c3076c993b0414824fae7be8f31c7d2c44ef02 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 1 Oct 2021 18:15:06 +0200 Subject: [PATCH 10/35] ssl certificate filepath is not added to mongo connection string but is added as argument to MongoClient --- igniter/tools.py | 31 +++++++++---------------------- start.py | 3 +-- 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/igniter/tools.py b/igniter/tools.py index ae680bf1f1..cb06fdaee2 100644 --- a/igniter/tools.py +++ b/igniter/tools.py @@ -16,7 +16,7 @@ from pymongo.errors import ( ) -def add_certificate_path_to_mongo_url(mongo_url): +def should_add_certificate_path_to_mongo_url(mongo_url): """Check if should add ca certificate to mongo url. Since 30.9.2021 cloud mongo requires newer certificates that are not @@ -41,25 +41,7 @@ def add_certificate_path_to_mongo_url(mongo_url): # Check if url does already contain certificate path if add_certificate and "tlscafile" in lowered_query_keys: add_certificate = False - - # Add certificate path to mongo url - if add_certificate: - path = parsed.path - if not path: - path = "/admin" - query = parsed.query - tls_query = "tlscafile={}".format(certifi.where()) - if not query: - query = tls_query - else: - query = "&".join((query, tls_query)) - new_url = ParseResult( - parsed.scheme, parsed.netloc, path, - parsed.params, query, parsed.fragment - ) - mongo_url = new_url.geturl() - - return mongo_url + return add_certificate def validate_mongo_connection(cnx: str) -> (bool, str): @@ -80,7 +62,8 @@ def validate_mongo_connection(cnx: str) -> (bool, str): "serverSelectionTimeoutMS": 2000 } # Add certificate path if should be required - cnx = add_certificate_path_to_mongo_url(cnx) + if should_add_certificate_path_to_mongo_url(cnx): + kwargs["ssl_ca_certs"] = certifi.where() try: client = MongoClient(cnx, **kwargs) @@ -152,9 +135,13 @@ def get_openpype_global_settings(url: str) -> dict: Returns: dict: With settings data. Empty dictionary is returned if not found. """ + kwargs = {} + if should_add_certificate_path_to_mongo_url(url): + kwargs["ssl_ca_certs"] = certifi.where() + try: # Create mongo connection - client = MongoClient(url) + client = MongoClient(url, **kwargs) # Access settings collection col = client["openpype"]["settings"] # Query global settings diff --git a/start.py b/start.py index 6861355600..ada613b4eb 100644 --- a/start.py +++ b/start.py @@ -191,7 +191,6 @@ from igniter import BootstrapRepos # noqa: E402 from igniter.tools import ( get_openpype_global_settings, get_openpype_path_from_db, - add_certificate_path_to_mongo_url, validate_mongo_connection ) # noqa from igniter.bootstrap_repos import OpenPypeVersion # noqa: E402 @@ -584,7 +583,7 @@ def _determine_mongodb() -> str: except ValueError: raise RuntimeError("Missing MongoDB url") - return add_certificate_path_to_mongo_url(openpype_mongo) + return openpype_mongo def _initialize_environment(openpype_version: OpenPypeVersion) -> None: From 53fe16fffcea8225eaa949ced837fa31b05f85da Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 1 Oct 2021 18:15:30 +0200 Subject: [PATCH 11/35] created copy of 'should_add_certificate_path_to_mongo_url' in openpype.lib.mongo --- openpype/lib/mongo.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/openpype/lib/mongo.py b/openpype/lib/mongo.py index 8bfaba75d6..054f40a5b0 100644 --- a/openpype/lib/mongo.py +++ b/openpype/lib/mongo.py @@ -3,6 +3,7 @@ import sys import time import logging import pymongo +import certifi if sys.version_info[0] == 2: from urlparse import urlparse, parse_qs @@ -93,6 +94,35 @@ def extract_port_from_url(url): return parsed_url.port +def should_add_certificate_path_to_mongo_url(mongo_url): + """Check if should add ca certificate to mongo url. + + Since 30.9.2021 cloud mongo requires newer certificates that are not + available on most of workstation. This adds path to certifi certificate + which is valid for it. To add the certificate path url must have scheme + 'mongodb+srv' or has 'ssl=true' or 'tls=true' in url query. + """ + parsed = urlparse(mongo_url) + query = parse_qs(parsed.query) + lowered_query_keys = set(key.lower() for key in query.keys()) + add_certificate = False + # Check if url 'ssl' or 'tls' are set to 'true' + for key in ("ssl", "tls"): + if key in query and "true" in query["ssl"]: + add_certificate = True + break + + # Check if url contains 'mongodb+srv' + if not add_certificate and parsed.scheme == "mongodb+srv": + add_certificate = True + + # Check if url does already contain certificate path + if add_certificate and "tlscafile" in lowered_query_keys: + add_certificate = False + + return add_certificate + + def validate_mongo_connection(mongo_uri): """Check if provided mongodb URL is valid. From 5782d6d801b67afe62ea3427291e3a4f484728af Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 1 Oct 2021 18:17:27 +0200 Subject: [PATCH 12/35] added ability to change retry times of connecting to mongo --- openpype/lib/mongo.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/openpype/lib/mongo.py b/openpype/lib/mongo.py index 054f40a5b0..3630b1320f 100644 --- a/openpype/lib/mongo.py +++ b/openpype/lib/mongo.py @@ -192,7 +192,7 @@ class OpenPypeMongoConnection: return connection @classmethod - def create_connection(cls, mongo_url, timeout=None): + def create_connection(cls, mongo_url, timeout=None, retry_attempts=None): if timeout is None: timeout = int(os.environ.get("AVALON_TIMEOUT") or 1000) @@ -206,8 +206,13 @@ class OpenPypeMongoConnection: kwargs["port"] = int(port) mongo_client = pymongo.MongoClient(**kwargs) + if retry_attempts is None: + retry_attempts = 3 - for _retry in range(3): + elif not retry_attempts: + retry_attempts = 1 + + for _retry in range(retry_attempts): try: t1 = time.time() mongo_client.server_info() From 23cc014f3299f4f71923f214f2d15b39edb7fe9b Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 1 Oct 2021 18:25:45 +0200 Subject: [PATCH 13/35] removed useless extract_port_from_url --- openpype/lib/mongo.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/openpype/lib/mongo.py b/openpype/lib/mongo.py index 3630b1320f..7527fce3b9 100644 --- a/openpype/lib/mongo.py +++ b/openpype/lib/mongo.py @@ -86,14 +86,6 @@ def get_default_components(): return decompose_url(mongo_url) -def extract_port_from_url(url): - parsed_url = urlparse(url) - if parsed_url.scheme is None: - _url = "mongodb://{}".format(url) - parsed_url = urlparse(_url) - return parsed_url.port - - def should_add_certificate_path_to_mongo_url(mongo_url): """Check if should add ca certificate to mongo url. @@ -201,9 +193,6 @@ class OpenPypeMongoConnection: "serverSelectionTimeoutMS": timeout } - port = extract_port_from_url(mongo_url) - if port is not None: - kwargs["port"] = int(port) mongo_client = pymongo.MongoClient(**kwargs) if retry_attempts is None: From 0c00a50417220d21cdb9008fcc9f1413df08f59a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 1 Oct 2021 18:25:59 +0200 Subject: [PATCH 14/35] added validation of mongo url --- openpype/lib/mongo.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/openpype/lib/mongo.py b/openpype/lib/mongo.py index 7527fce3b9..64c46eac68 100644 --- a/openpype/lib/mongo.py +++ b/openpype/lib/mongo.py @@ -185,6 +185,14 @@ class OpenPypeMongoConnection: @classmethod def create_connection(cls, mongo_url, timeout=None, retry_attempts=None): + parsed = urlparse(mongo_url) + # Force validation of scheme + if parsed.scheme not in ["mongodb", "mongodb+srv"]: + raise pymongo.errors.InvalidURI(( + "Invalid URI scheme:" + " URI must begin with 'mongodb://' or 'mongodb+srv://'" + )) + if timeout is None: timeout = int(os.environ.get("AVALON_TIMEOUT") or 1000) From 3448e1ce78803b89980bd295c1f77ca2d4322984 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 1 Oct 2021 18:26:51 +0200 Subject: [PATCH 15/35] moved time start out of loop --- openpype/lib/mongo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/lib/mongo.py b/openpype/lib/mongo.py index 64c46eac68..cadf81b63b 100644 --- a/openpype/lib/mongo.py +++ b/openpype/lib/mongo.py @@ -209,9 +209,9 @@ class OpenPypeMongoConnection: elif not retry_attempts: retry_attempts = 1 + t1 = time.time() for _retry in range(retry_attempts): try: - t1 = time.time() mongo_client.server_info() except Exception: From 2714c644f25ad1d9fe4a85e730d6feb6c076ab93 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 1 Oct 2021 18:27:16 +0200 Subject: [PATCH 16/35] pass mongo url as argument instead of kwarg --- openpype/lib/mongo.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openpype/lib/mongo.py b/openpype/lib/mongo.py index cadf81b63b..a192427c81 100644 --- a/openpype/lib/mongo.py +++ b/openpype/lib/mongo.py @@ -197,12 +197,11 @@ class OpenPypeMongoConnection: timeout = int(os.environ.get("AVALON_TIMEOUT") or 1000) kwargs = { - "host": mongo_url, "serverSelectionTimeoutMS": timeout } + mongo_client = pymongo.MongoClient(mongo_url, **kwargs) - mongo_client = pymongo.MongoClient(**kwargs) if retry_attempts is None: retry_attempts = 3 From 95de5d15f55fb640d7d1bbf1ac100f4df2562559 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 1 Oct 2021 18:37:10 +0200 Subject: [PATCH 17/35] store exception and reraise it when connection is not successful --- openpype/lib/mongo.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/openpype/lib/mongo.py b/openpype/lib/mongo.py index a192427c81..fcfc4a62f3 100644 --- a/openpype/lib/mongo.py +++ b/openpype/lib/mongo.py @@ -208,23 +208,27 @@ class OpenPypeMongoConnection: elif not retry_attempts: retry_attempts = 1 + last_exc = None + valid = False t1 = time.time() - for _retry in range(retry_attempts): + for attempt in range(1, retry_attempts + 1): try: mongo_client.server_info() - - except Exception: - cls.log.warning("Retrying...") - time.sleep(1) - timeout *= 1.5 - - else: + with mongo_client.start_session(): + pass + valid = True break - else: - raise IOError(( - "ERROR: Couldn't connect to {} in less than {:.3f}ms" - ).format(mongo_url, timeout)) + except Exception as exc: + last_exc = exc + if attempt < retry_attempts: + cls.log.warning( + "Attempt {} failed. Retrying... ".format(attempt) + ) + time.sleep(1) + + if not valid: + raise last_exc cls.log.info("Connected to {}, delay {:.3f}s".format( mongo_url, time.time() - t1 From 4d4c01519ebb6a6b0e5912fe777922f2c97c1f42 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 1 Oct 2021 18:37:28 +0200 Subject: [PATCH 18/35] validate mongo connection uses `create_connection` --- openpype/lib/mongo.py | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/openpype/lib/mongo.py b/openpype/lib/mongo.py index fcfc4a62f3..d27ec99aa2 100644 --- a/openpype/lib/mongo.py +++ b/openpype/lib/mongo.py @@ -128,26 +128,9 @@ def validate_mongo_connection(mongo_uri): passed so probably couldn't connect to mongo server. """ - parsed = urlparse(mongo_uri) - # Force validation of scheme - if parsed.scheme not in ["mongodb", "mongodb+srv"]: - raise pymongo.errors.InvalidURI(( - "Invalid URI scheme:" - " URI must begin with 'mongodb://' or 'mongodb+srv://'" - )) - # we have mongo connection string. Let's try if we can connect. - components = decompose_url(mongo_uri) - mongo_args = { - "host": compose_url(**components), - "serverSelectionTimeoutMS": 1000 - } - port = components.get("port") - if port is not None: - mongo_args["port"] = int(port) - - # Create connection - client = pymongo.MongoClient(**mongo_args) - client.server_info() + client = OpenPypeMongoConnection.create_connection( + mongo_uri, retry_attempts=1 + ) client.close() From f4e58771a2c8cdeec7052be44ad89f80a28531fe Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 1 Oct 2021 18:37:39 +0200 Subject: [PATCH 19/35] add ssla ca certificate if should --- openpype/lib/mongo.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openpype/lib/mongo.py b/openpype/lib/mongo.py index d27ec99aa2..c758f0d73c 100644 --- a/openpype/lib/mongo.py +++ b/openpype/lib/mongo.py @@ -182,6 +182,8 @@ class OpenPypeMongoConnection: kwargs = { "serverSelectionTimeoutMS": timeout } + if should_add_certificate_path_to_mongo_url(mongo_url): + kwargs["ssl_ca_certs"] = certifi.where() mongo_client = pymongo.MongoClient(mongo_url, **kwargs) From 0dec31288b2c33afd51d7ef3a542546aa83a2f4c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 1 Oct 2021 18:48:26 +0200 Subject: [PATCH 20/35] added session validation to get_mongo_connection --- openpype/lib/mongo.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openpype/lib/mongo.py b/openpype/lib/mongo.py index c758f0d73c..0fd4517b5b 100644 --- a/openpype/lib/mongo.py +++ b/openpype/lib/mongo.py @@ -156,6 +156,8 @@ class OpenPypeMongoConnection: # Naive validation of existing connection try: connection.server_info() + with connection.start_session(): + pass except Exception: connection = None From 39f8e1dc4200fbf560d1fadb5bd2f1d407e92d31 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 1 Oct 2021 18:48:43 +0200 Subject: [PATCH 21/35] removed unused import --- igniter/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/igniter/tools.py b/igniter/tools.py index cb06fdaee2..04d7451335 100644 --- a/igniter/tools.py +++ b/igniter/tools.py @@ -2,7 +2,7 @@ """Tools used in **Igniter** GUI.""" import os from typing import Union -from urllib.parse import urlparse, parse_qs, ParseResult +from urllib.parse import urlparse, parse_qs from pathlib import Path import platform From 1e6cdd784f3fe497efbeaa920af2aaf7cd4cef87 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 4 Oct 2021 11:14:51 +0200 Subject: [PATCH 22/35] updating avalon-core submodul repo --- repos/avalon-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/avalon-core b/repos/avalon-core index 8aee68fa10..4b80f81e66 160000 --- a/repos/avalon-core +++ b/repos/avalon-core @@ -1 +1 @@ -Subproject commit 8aee68fa10ab4d79be1a91e7728a609748e7c3c6 +Subproject commit 4b80f81e66aca593784be8b299110a0b6541276f From 768dd94af96e321648b20b15499b92e2a800cb0e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 4 Oct 2021 13:11:35 +0200 Subject: [PATCH 23/35] Fix - broken import --- .../modules/default_modules/sync_server/providers/sftp.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/providers/sftp.py b/openpype/modules/default_modules/sync_server/providers/sftp.py index ce63d35c8c..afcc89767c 100644 --- a/openpype/modules/default_modules/sync_server/providers/sftp.py +++ b/openpype/modules/default_modules/sync_server/providers/sftp.py @@ -8,9 +8,7 @@ import platform from openpype.api import Logger from openpype.api import get_system_settings -from openpype.modules.default_modules.sync_server.providers.abstract_provider \ - import AbstractProvider - +from .abstract_provider import AbstractProvider log = Logger().get_logger("SyncServer") try: From d8c3535b6b3d3c3983995d314322828ecbd5c1c5 Mon Sep 17 00:00:00 2001 From: "clement.hector" Date: Mon, 4 Oct 2021 18:06:23 +0200 Subject: [PATCH 24/35] fix djv view for openpype3 --- openpype/plugins/load/open_djv.py | 38 +++++++++++++++---------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/openpype/plugins/load/open_djv.py b/openpype/plugins/load/open_djv.py index 39b54364d9..5b49bb58d0 100644 --- a/openpype/plugins/load/open_djv.py +++ b/openpype/plugins/load/open_djv.py @@ -1,26 +1,28 @@ import os -import subprocess from avalon import api +from openpype.api import ApplicationManager def existing_djv_path(): - djv_paths = os.environ.get("DJV_PATH") or "" - for path in djv_paths.split(os.pathsep): - if os.path.exists(path): - return path - return None + app_manager = ApplicationManager() + djv_list = [] + for app_name, app in app_manager.applications.items(): + if 'djv' in app_name and app.find_executable(): + djv_list.append(app_name) + + return djv_list class OpenInDJV(api.Loader): """Open Image Sequence with system default""" - djv_path = existing_djv_path() - families = ["*"] if djv_path else [] + djv_list = existing_djv_path() + families = ["*"] if djv_list else [] representations = [ "cin", "dpx", "avi", "dv", "gif", "flv", "mkv", "mov", "mpg", "mpeg", "mp4", "m4v", "mxf", "iff", "z", "ifl", "jpeg", "jpg", "jfif", "lut", "1dl", "exr", "pic", "png", "ppm", "pnm", "pgm", "pbm", "rla", "rpf", - "sgi", "rgba", "rgb", "bw", "tga", "tiff", "tif", "img" + "sgi", "rgba", "rgb", "bw", "tga", "tiff", "tif", "img", "h264", ] label = "Open in DJV" @@ -41,20 +43,18 @@ class OpenInDJV(api.Loader): ) if not remainder: - seqeunce = collections[0] - first_image = list(seqeunce)[0] + sequence = collections[0] + first_image = list(sequence)[0] else: first_image = self.fname filepath = os.path.normpath(os.path.join(directory, first_image)) self.log.info("Opening : {}".format(filepath)) - cmd = [ - # DJV path - os.path.normpath(self.djv_path), - # PATH TO COMPONENT - os.path.normpath(filepath) - ] + last_djv_version = sorted(self.djv_list)[-1] - # Run DJV with these commands - subprocess.Popen(cmd) + app_manager = ApplicationManager() + djv = app_manager.applications.get(last_djv_version) + djv.arguments.append(filepath) + + app_manager.launch(last_djv_version) From a2a350b434fbc9c0984be611c28812a22cf83412 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 5 Oct 2021 10:19:25 +0200 Subject: [PATCH 25/35] use constant for "intent" custom attribute --- .../ftrack/event_handlers_user/action_create_cust_attrs.py | 3 ++- openpype/modules/default_modules/ftrack/lib/__init__.py | 3 ++- openpype/modules/default_modules/ftrack/lib/constants.py | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/openpype/modules/default_modules/ftrack/event_handlers_user/action_create_cust_attrs.py b/openpype/modules/default_modules/ftrack/event_handlers_user/action_create_cust_attrs.py index 3869d8ad08..0bd243ab4c 100644 --- a/openpype/modules/default_modules/ftrack/event_handlers_user/action_create_cust_attrs.py +++ b/openpype/modules/default_modules/ftrack/event_handlers_user/action_create_cust_attrs.py @@ -10,6 +10,7 @@ from openpype_modules.ftrack.lib import ( CUST_ATTR_GROUP, CUST_ATTR_TOOLS, CUST_ATTR_APPLICATIONS, + CUST_ATTR_INTENT, default_custom_attributes_definition, app_definitions_from_app_manager, @@ -431,7 +432,7 @@ class CustomAttributes(BaseAction): intent_custom_attr_data = { "label": "Intent", - "key": "intent", + "key": CUST_ATTR_INTENT, "type": "enumerator", "entity_type": "assetversion", "group": CUST_ATTR_GROUP, diff --git a/openpype/modules/default_modules/ftrack/lib/__init__.py b/openpype/modules/default_modules/ftrack/lib/__init__.py index 433a1f7881..80b4db9dd6 100644 --- a/openpype/modules/default_modules/ftrack/lib/__init__.py +++ b/openpype/modules/default_modules/ftrack/lib/__init__.py @@ -3,7 +3,8 @@ from .constants import ( CUST_ATTR_AUTO_SYNC, CUST_ATTR_GROUP, CUST_ATTR_TOOLS, - CUST_ATTR_APPLICATIONS + CUST_ATTR_APPLICATIONS, + CUST_ATTR_INTENT ) from .settings import ( get_ftrack_event_mongo_info diff --git a/openpype/modules/default_modules/ftrack/lib/constants.py b/openpype/modules/default_modules/ftrack/lib/constants.py index 73d5112e6d..e6e2013d2b 100644 --- a/openpype/modules/default_modules/ftrack/lib/constants.py +++ b/openpype/modules/default_modules/ftrack/lib/constants.py @@ -10,3 +10,5 @@ CUST_ATTR_AUTO_SYNC = "avalon_auto_sync" CUST_ATTR_APPLICATIONS = "applications" # Environment tools custom attribute CUST_ATTR_TOOLS = "tools_env" +# Intent custom attribute name +CUST_ATTR_INTENT = "intent" From 253f7f97c4839293d5f60cdf55297c558e472578 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 5 Oct 2021 10:19:56 +0200 Subject: [PATCH 26/35] raise not found custom attributes only for attributes that can be set on ftrack --- .../default_modules/ftrack/ftrack_module.py | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/openpype/modules/default_modules/ftrack/ftrack_module.py b/openpype/modules/default_modules/ftrack/ftrack_module.py index c73f9b100d..1a45b1e4b7 100644 --- a/openpype/modules/default_modules/ftrack/ftrack_module.py +++ b/openpype/modules/default_modules/ftrack/ftrack_module.py @@ -230,7 +230,13 @@ class FtrackModule( return import ftrack_api - from openpype_modules.ftrack.lib import get_openpype_attr + from openpype_modules.ftrack.lib import ( + get_openpype_attr, + default_custom_attributes_definition, + CUST_ATTR_TOOLS, + CUST_ATTR_APPLICATIONS, + CUST_ATTR_INTENT + ) try: session = self.create_ftrack_session() @@ -255,6 +261,15 @@ class FtrackModule( project_id = project_entity["id"] + ca_defs = default_custom_attributes_definition() + hierarchical_attrs = ca_defs.get("is_hierarchical") or {} + project_attrs = ca_defs.get("show") or {} + ca_keys = ( + set(hierarchical_attrs.keys()) + + set(project_attrs.keys()) + + {CUST_ATTR_TOOLS, CUST_ATTR_APPLICATIONS, CUST_ATTR_INTENT} + ) + cust_attr, hier_attr = get_openpype_attr(session) cust_attr_by_key = {attr["key"]: attr for attr in cust_attr} hier_attrs_by_key = {attr["key"]: attr for attr in hier_attr} @@ -266,10 +281,11 @@ class FtrackModule( if not configuration: configuration = cust_attr_by_key.get(key) if not configuration: - self.log.warning( - "Custom attribute \"{}\" was not found.".format(key) - ) - missing[key] = value + if key in ca_keys: + self.log.warning( + "Custom attribute \"{}\" was not found.".format(key) + ) + missing[key] = value continue # TODO add add permissions check From c03ba80974bccf2085de207d9944eb3e9b6219ac Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 5 Oct 2021 10:22:55 +0200 Subject: [PATCH 27/35] fix set adding --- openpype/modules/default_modules/ftrack/ftrack_module.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/modules/default_modules/ftrack/ftrack_module.py b/openpype/modules/default_modules/ftrack/ftrack_module.py index 1a45b1e4b7..bfcecabafe 100644 --- a/openpype/modules/default_modules/ftrack/ftrack_module.py +++ b/openpype/modules/default_modules/ftrack/ftrack_module.py @@ -266,8 +266,8 @@ class FtrackModule( project_attrs = ca_defs.get("show") or {} ca_keys = ( set(hierarchical_attrs.keys()) - + set(project_attrs.keys()) - + {CUST_ATTR_TOOLS, CUST_ATTR_APPLICATIONS, CUST_ATTR_INTENT} + | set(project_attrs.keys()) + | {CUST_ATTR_TOOLS, CUST_ATTR_APPLICATIONS, CUST_ATTR_INTENT} ) cust_attr, hier_attr = get_openpype_attr(session) From c55ca5af2d2b4d6050b2b6bcd2d913028aa6ceee Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 5 Oct 2021 10:26:17 +0200 Subject: [PATCH 28/35] skip keys before they're checked --- .../modules/default_modules/ftrack/ftrack_module.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/openpype/modules/default_modules/ftrack/ftrack_module.py b/openpype/modules/default_modules/ftrack/ftrack_module.py index bfcecabafe..9cbf979239 100644 --- a/openpype/modules/default_modules/ftrack/ftrack_module.py +++ b/openpype/modules/default_modules/ftrack/ftrack_module.py @@ -277,15 +277,17 @@ class FtrackModule( failed = {} missing = {} for key, value in attributes_changes.items(): + if key not in ca_keys: + continue + configuration = hier_attrs_by_key.get(key) if not configuration: configuration = cust_attr_by_key.get(key) if not configuration: - if key in ca_keys: - self.log.warning( - "Custom attribute \"{}\" was not found.".format(key) - ) - missing[key] = value + self.log.warning( + "Custom attribute \"{}\" was not found.".format(key) + ) + missing[key] = value continue # TODO add add permissions check From a199a850b07c4d178d21954ece9b4a2560994225 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 5 Oct 2021 10:41:07 +0200 Subject: [PATCH 29/35] project model has it's refresh and use constants for roles --- openpype/tools/settings/settings/constants.py | 16 +++ openpype/tools/settings/settings/widgets.py | 123 +++++++++++------- 2 files changed, 90 insertions(+), 49 deletions(-) create mode 100644 openpype/tools/settings/settings/constants.py diff --git a/openpype/tools/settings/settings/constants.py b/openpype/tools/settings/settings/constants.py new file mode 100644 index 0000000000..5c20bf1afe --- /dev/null +++ b/openpype/tools/settings/settings/constants.py @@ -0,0 +1,16 @@ +from Qt import QtCore + + +DEFAULT_PROJECT_LABEL = "< Default >" +PROJECT_NAME_ROLE = QtCore.Qt.UserRole + 1 +PROJECT_IS_ACTIVE_ROLE = QtCore.Qt.UserRole + 2 +PROJECT_IS_SELECTED_ROLE = QtCore.Qt.UserRole + 3 + + +__all__ = ( + "DEFAULT_PROJECT_LABEL", + + "PROJECT_NAME_ROLE", + "PROJECT_IS_ACTIVE_ROLE", + "PROJECT_IS_SELECTED_ROLE" +) diff --git a/openpype/tools/settings/settings/widgets.py b/openpype/tools/settings/settings/widgets.py index a461f3e675..a94621c8d3 100644 --- a/openpype/tools/settings/settings/widgets.py +++ b/openpype/tools/settings/settings/widgets.py @@ -7,6 +7,12 @@ from avalon.mongodb import ( ) from openpype.settings.lib import get_system_settings +from .constants import ( + DEFAULT_PROJECT_LABEL, + PROJECT_NAME_ROLE, + PROJECT_IS_ACTIVE_ROLE, + PROJECT_IS_SELECTED_ROLE +) class SettingsLineEdit(QtWidgets.QLineEdit): @@ -602,10 +608,63 @@ class NiceCheckbox(QtWidgets.QFrame): return super(NiceCheckbox, self).mouseReleaseEvent(event) -class ProjectListModel(QtGui.QStandardItemModel): - sort_role = QtCore.Qt.UserRole + 10 - filter_role = QtCore.Qt.UserRole + 11 - selected_role = QtCore.Qt.UserRole + 12 +class ProjectModel(QtGui.QStandardItemModel): + def __init__(self, only_active, *args, **kwargs): + super(ProjectModel, self).__init__(*args, **kwargs) + + self.dbcon = None + + self._only_active = only_active + self._default_item = None + self._items_by_name = {} + + def set_dbcon(self, dbcon): + self.dbcon = dbcon + + def refresh(self): + new_items = [] + if self._default_item is None: + item = QtGui.QStandardItem(DEFAULT_PROJECT_LABEL) + item.setData(None, PROJECT_NAME_ROLE) + item.setData(True, PROJECT_IS_ACTIVE_ROLE) + item.setData(False, PROJECT_IS_SELECTED_ROLE) + new_items.append(item) + self._default_item = item + + project_names = set() + if self.dbcon is not None: + for project_doc in self.dbcon.projects( + projection={"name": 1, "data.active": 1}, + only_active=self._only_active + ): + project_name = project_doc["name"] + project_names.add(project_name) + if project_name in self._items_by_name: + item = self._items_by_name[project_name] + else: + item = QtGui.QStandardItem(project_name) + + self._items_by_name[project_name] = item + new_items.append(item) + + is_active = project_doc.get("data", {}).get("active", True) + item.setData(project_name, PROJECT_NAME_ROLE) + item.setData(is_active, PROJECT_IS_ACTIVE_ROLE) + item.setData(False, PROJECT_IS_SELECTED_ROLE) + + if not is_active: + font = item.font() + font.setItalic(True) + item.setFont(font) + + root_item = self.invisibleRootItem() + for project_name in tuple(self._items_by_name.keys()): + if project_name not in project_names: + item = self._items_by_name.pop(project_name) + root_item.removeRow(item.row()) + + if new_items: + root_item.appendRows(new_items) class ProjectListView(QtWidgets.QListView): @@ -619,7 +678,6 @@ class ProjectListView(QtWidgets.QListView): class ProjectListSortFilterProxy(QtCore.QSortFilterProxyModel): - def __init__(self, *args, **kwargs): super(ProjectListSortFilterProxy, self).__init__(*args, **kwargs) self._enable_filter = True @@ -630,7 +688,7 @@ class ProjectListSortFilterProxy(QtCore.QSortFilterProxyModel): index = self.sourceModel().index(source_row, 0, source_parent) is_active = bool(index.data(self.filterRole())) - is_selected = bool(index.data(ProjectListModel.selected_role)) + is_selected = bool(index.data(PROJECT_IS_SELECTED_ROLE)) return is_active or is_selected @@ -643,7 +701,6 @@ class ProjectListSortFilterProxy(QtCore.QSortFilterProxyModel): class ProjectListWidget(QtWidgets.QWidget): - default = "< Default >" project_changed = QtCore.Signal() def __init__(self, parent, only_active=False): @@ -657,13 +714,10 @@ class ProjectListWidget(QtWidgets.QWidget): label_widget = QtWidgets.QLabel("Projects") project_list = ProjectListView(self) - project_model = ProjectListModel() + project_model = ProjectModel(only_active) project_proxy = ProjectListSortFilterProxy() - project_proxy.setFilterRole(ProjectListModel.filter_role) - project_proxy.setSortRole(ProjectListModel.sort_role) - project_proxy.setSortCaseSensitivity(QtCore.Qt.CaseInsensitive) - + project_proxy.setFilterRole(PROJECT_IS_ACTIVE_ROLE) project_proxy.setSourceModel(project_model) project_list.setModel(project_proxy) @@ -693,13 +747,14 @@ class ProjectListWidget(QtWidgets.QWidget): project_list.left_mouse_released_at.connect(self.on_item_clicked) + self._default_project_item = None + self.project_list = project_list self.project_proxy = project_proxy self.project_model = project_model self.inactive_chk = inactive_chk self.dbcon = None - self._only_active = only_active def on_item_clicked(self, new_index): new_project_name = new_index.data(QtCore.Qt.DisplayRole) @@ -746,12 +801,12 @@ class ProjectListWidget(QtWidgets.QWidget): return not self._parent.entity.has_unsaved_changes def project_name(self): - if self.current_project == self.default: + if self.current_project == DEFAULT_PROJECT_LABEL: return None return self.current_project def select_default_project(self): - self.select_project(self.default) + self.select_project(DEFAULT_PROJECT_LABEL) def select_project(self, project_name): model = self.project_model @@ -759,10 +814,10 @@ class ProjectListWidget(QtWidgets.QWidget): found_items = model.findItems(project_name) if not found_items: - found_items = model.findItems(self.default) + found_items = model.findItems(DEFAULT_PROJECT_LABEL) index = model.indexFromItem(found_items[0]) - model.setData(index, True, ProjectListModel.selected_role) + model.setData(index, True, PROJECT_IS_SELECTED_ROLE) index = proxy.mapFromSource(index) @@ -777,9 +832,6 @@ class ProjectListWidget(QtWidgets.QWidget): selected_project = index.data(QtCore.Qt.DisplayRole) break - model = self.project_model - model.clear() - mongo_url = os.environ["OPENPYPE_MONGO"] # Force uninstall of whole avalon connection if url does not match @@ -797,35 +849,8 @@ class ProjectListWidget(QtWidgets.QWidget): self.dbcon = None self.current_project = None - items = [(self.default, True)] - - if self.dbcon: - - for doc in self.dbcon.projects( - projection={"name": 1, "data.active": 1}, - only_active=self._only_active - ): - items.append( - (doc["name"], doc.get("data", {}).get("active", True)) - ) - - for project_name, is_active in items: - - row = QtGui.QStandardItem(project_name) - row.setData(is_active, ProjectListModel.filter_role) - row.setData(False, ProjectListModel.selected_role) - - if is_active: - row.setData(project_name, ProjectListModel.sort_role) - - else: - row.setData("~" + project_name, ProjectListModel.sort_role) - - font = row.font() - font.setItalic(True) - row.setFont(font) - - model.appendRow(row) + self.project_model.set_dbcon(self.dbcon) + self.project_model.refresh() self.project_proxy.sort(0) From 134bae90d39689bf4acaddbe0c6d724831f61e0f Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 5 Oct 2021 10:41:56 +0200 Subject: [PATCH 30/35] implement custom sort method for project sorting --- openpype/tools/settings/settings/widgets.py | 24 ++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/openpype/tools/settings/settings/widgets.py b/openpype/tools/settings/settings/widgets.py index a94621c8d3..710884e9e5 100644 --- a/openpype/tools/settings/settings/widgets.py +++ b/openpype/tools/settings/settings/widgets.py @@ -677,11 +677,29 @@ class ProjectListView(QtWidgets.QListView): super(ProjectListView, self).mouseReleaseEvent(event) -class ProjectListSortFilterProxy(QtCore.QSortFilterProxyModel): +class ProjectSortFilterProxy(QtCore.QSortFilterProxyModel): def __init__(self, *args, **kwargs): - super(ProjectListSortFilterProxy, self).__init__(*args, **kwargs) + super(ProjectSortFilterProxy, self).__init__(*args, **kwargs) self._enable_filter = True + def lessThan(self, left_index, right_index): + if left_index.data(PROJECT_NAME_ROLE) is None: + return True + + if right_index.data(PROJECT_NAME_ROLE) is None: + return False + + left_is_active = left_index.data(PROJECT_IS_ACTIVE_ROLE) + right_is_active = right_index.data(PROJECT_IS_ACTIVE_ROLE) + if right_is_active == left_is_active: + return super(ProjectSortFilterProxy, self).lessThan( + left_index, right_index + ) + + if left_is_active: + return True + return False + def filterAcceptsRow(self, source_row, source_parent): if not self._enable_filter: return True @@ -715,7 +733,7 @@ class ProjectListWidget(QtWidgets.QWidget): project_list = ProjectListView(self) project_model = ProjectModel(only_active) - project_proxy = ProjectListSortFilterProxy() + project_proxy = ProjectSortFilterProxy() project_proxy.setFilterRole(PROJECT_IS_ACTIVE_ROLE) project_proxy.setSourceModel(project_model) From c014b698a693a7409b49a9a08ceb5877643d3630 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 5 Oct 2021 15:29:45 +0200 Subject: [PATCH 31/35] Fix - import pysftp only when necessary Blender 2.93 has issue with conflicting libraries, pysftp is actually not needed in provider running in a host, do not import it or explode when its not necessary --- .../sync_server/providers/sftp.py | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/providers/sftp.py b/openpype/modules/default_modules/sync_server/providers/sftp.py index afcc89767c..b1eacb32a7 100644 --- a/openpype/modules/default_modules/sync_server/providers/sftp.py +++ b/openpype/modules/default_modules/sync_server/providers/sftp.py @@ -11,11 +11,15 @@ from openpype.api import get_system_settings from .abstract_provider import AbstractProvider log = Logger().get_logger("SyncServer") +pysftp = None try: - import pysftp + import pysftp as _pysftp + + pysftp = _pysftp except (ImportError, SyntaxError): - if six.PY3: - six.reraise(*sys.exc_info()) + pass + # if six.PY3: + # six.reraise(*sys.exc_info()) # handle imports from Python 2 hosts - in those only basic methods are used log.warning("Import failed, imported from Python 2, operations will fail.") @@ -41,7 +45,7 @@ class SFTPHandler(AbstractProvider): self.project_name = project_name self.site_name = site_name self.root = None - self.conn = None + self._conn = None self.presets = presets if not self.presets: @@ -63,11 +67,17 @@ class SFTPHandler(AbstractProvider): self.sftp_key = provider_presets["sftp_key"] self.sftp_key_pass = provider_presets["sftp_key_pass"] - self.conn = self._get_conn() - self._tree = None self.active = True + @property + def conn(self): + """SFTP connection, cannot be used in all places though.""" + if not self._conn: + self._conn = self._get_conn() + + return self._conn + def is_active(self): """ Returns True if provider is activated, eg. has working credentials. @@ -321,7 +331,8 @@ class SFTPHandler(AbstractProvider): if not self.file_path_exists(path): raise FileNotFoundError("File {} to be deleted doesn't exist." .format(path)) - self.conn.remove(path) + conn = self._get_conn() + conn.remove(path) def list_folder(self, folder_path): """ @@ -394,6 +405,9 @@ class SFTPHandler(AbstractProvider): Returns: pysftp.Connection """ + if not pysftp: + raise ImportError + cnopts = pysftp.CnOpts() cnopts.hostkeys = None From 2c6efcc01737c4c0c8f8d41fc1e1e2bb8ce279ee Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 5 Oct 2021 15:36:39 +0200 Subject: [PATCH 32/35] Small refactor --- .../modules/default_modules/sync_server/providers/sftp.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/providers/sftp.py b/openpype/modules/default_modules/sync_server/providers/sftp.py index b1eacb32a7..3363ed40a5 100644 --- a/openpype/modules/default_modules/sync_server/providers/sftp.py +++ b/openpype/modules/default_modules/sync_server/providers/sftp.py @@ -13,9 +13,7 @@ log = Logger().get_logger("SyncServer") pysftp = None try: - import pysftp as _pysftp - - pysftp = _pysftp + import pysftp except (ImportError, SyntaxError): pass # if six.PY3: From 7c61510f302d798f3a984718182ee4e918b01872 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 5 Oct 2021 15:38:06 +0200 Subject: [PATCH 33/35] Small refactor --- openpype/modules/default_modules/sync_server/providers/sftp.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/providers/sftp.py b/openpype/modules/default_modules/sync_server/providers/sftp.py index 3363ed40a5..9036493d2a 100644 --- a/openpype/modules/default_modules/sync_server/providers/sftp.py +++ b/openpype/modules/default_modules/sync_server/providers/sftp.py @@ -16,8 +16,6 @@ try: import pysftp except (ImportError, SyntaxError): pass - # if six.PY3: - # six.reraise(*sys.exc_info()) # handle imports from Python 2 hosts - in those only basic methods are used log.warning("Import failed, imported from Python 2, operations will fail.") From 5f52935485c28f2d1ee43308043019820af88d85 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 5 Oct 2021 15:44:42 +0200 Subject: [PATCH 34/35] Small refactor for file deletion --- .../modules/default_modules/sync_server/providers/sftp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/providers/sftp.py b/openpype/modules/default_modules/sync_server/providers/sftp.py index 9036493d2a..07450265e2 100644 --- a/openpype/modules/default_modules/sync_server/providers/sftp.py +++ b/openpype/modules/default_modules/sync_server/providers/sftp.py @@ -327,8 +327,8 @@ class SFTPHandler(AbstractProvider): if not self.file_path_exists(path): raise FileNotFoundError("File {} to be deleted doesn't exist." .format(path)) - conn = self._get_conn() - conn.remove(path) + + self.conn.remove(path) def list_folder(self, folder_path): """ From 86deaebea87bd5ab71c1978ff74e93b58d19f551 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 5 Oct 2021 17:03:26 +0200 Subject: [PATCH 35/35] added second variant of "loop" behavior - "repeat" --- openpype/hosts/tvpaint/plugins/publish/extract_sequence.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/tvpaint/plugins/publish/extract_sequence.py b/openpype/hosts/tvpaint/plugins/publish/extract_sequence.py index 36f0b0c954..c45ff53c3c 100644 --- a/openpype/hosts/tvpaint/plugins/publish/extract_sequence.py +++ b/openpype/hosts/tvpaint/plugins/publish/extract_sequence.py @@ -606,7 +606,7 @@ class ExtractSequence(pyblish.api.Extractor): self._copy_image(eq_frame_filepath, new_filepath) layer_files_by_frame[frame_idx] = new_filepath - elif pre_behavior == "loop": + elif pre_behavior in ("loop", "repeat"): # Loop backwards from last frame of layer for frame_idx in reversed(range(mark_in_index, frame_start_index)): eq_frame_idx_offset = ( @@ -678,7 +678,7 @@ class ExtractSequence(pyblish.api.Extractor): self._copy_image(eq_frame_filepath, new_filepath) layer_files_by_frame[frame_idx] = new_filepath - elif post_behavior == "loop": + elif post_behavior in ("loop", "repeat"): # Loop backwards from last frame of layer for frame_idx in range(frame_end_index + 1, mark_out_index + 1): eq_frame_idx = frame_idx % frame_count