From f1c442eff9bdc1f4bbaa181fd9bc9eca4e8866dc Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 29 Dec 2020 16:47:32 +0100 Subject: [PATCH 01/12] removed TimedRotatingFileHandler and file logger creation methods --- pype/lib/log.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/pype/lib/log.py b/pype/lib/log.py index 47f379d952..80f4adbf31 100644 --- a/pype/lib/log.py +++ b/pype/lib/log.py @@ -21,7 +21,6 @@ import socket import sys import time import traceback -from logging.handlers import TimedRotatingFileHandler from . import Terminal from .mongo import ( @@ -274,41 +273,14 @@ class PypeLogger: def __init__(self): self.PYPE_DEBUG = int(os.environ.get("PYPE_DEBUG", "0")) - @staticmethod - def get_file_path(host='pype'): - ts = time.time() - log_name = datetime.datetime.fromtimestamp(ts).strftime( - '%Y-%m-%d' # '%Y-%m-%d_%H-%M-%S' - ) - logger_file_root = os.path.join( - os.path.expanduser("~"), - ".pype-setup" - ) - logger_file_path = os.path.join( - logger_file_root, - "{}-{}.{}".format(host, log_name, 'log') - ) - if not os.path.exists(logger_file_root): - os.mkdir(logger_file_root) - return logger_file_path - def _get_file_handler(self, host): - logger_file_path = PypeLogger.get_file_path(host) - formatter = PypeFormatter(self.FORMAT_FILE) - file_handler = TimedRotatingFileHandler( - logger_file_path, - when='midnight' - ) - file_handler.set_name("PypeFileHandler") - file_handler.setFormatter(formatter) - return file_handler def _get_mongo_handler(self): components = _log_mongo_components() From 0cb7573e53aaf772954a615f922e508636d5585d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 29 Dec 2020 17:00:19 +0100 Subject: [PATCH 02/12] usage of unicode is done through builtins attribute --- pype/lib/log.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pype/lib/log.py b/pype/lib/log.py index 80f4adbf31..55c6da0be6 100644 --- a/pype/lib/log.py +++ b/pype/lib/log.py @@ -40,11 +40,6 @@ except ImportError: else: _mongo_logging = True -try: - unicode - _unicode = True -except NameError: - _unicode = False PYPE_DEBUG = int(os.getenv("PYPE_DEBUG", "0")) @@ -113,6 +108,8 @@ def _bootstrap_mongo_log(components=None): LOG_COLLECTION_NAME, capped=True, max=5000, size=1073741824 ) return logdb +# Check for `unicode` in builtins +USE_UNICODE = hasattr(__builtins__, "unicode") class PypeStreamHandler(logging.StreamHandler): @@ -147,7 +144,8 @@ class PypeStreamHandler(logging.StreamHandler): msg = Terminal.log(msg) stream = self.stream fs = "%s\n" - if not _unicode: # if no unicode support... + # if no unicode support... + if not USE_UNICODE: stream.write(fs % msg) else: try: From 7a3e0b54c22655e93b870d0665d28a25ab199dd6 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 29 Dec 2020 17:00:30 +0100 Subject: [PATCH 03/12] removed unused logger from applicaitons.py --- pype/lib/applications.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pype/lib/applications.py b/pype/lib/applications.py index 253ffa0ad2..9f43ae5542 100644 --- a/pype/lib/applications.py +++ b/pype/lib/applications.py @@ -15,8 +15,6 @@ from .python_module_tools import ( classes_from_module ) -log = PypeLogger().get_logger(__name__) - class ApplicationNotFound(Exception): """Application was not found in ApplicationManager by name.""" From 44e0fcba336cb14a5fcf2f7b26e45c861cfbdc38 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 29 Dec 2020 17:01:05 +0100 Subject: [PATCH 04/12] exception is set with direct set of key which is faster and expplicit --- pype/lib/log.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pype/lib/log.py b/pype/lib/log.py index 55c6da0be6..dde8fa9452 100644 --- a/pype/lib/log.py +++ b/pype/lib/log.py @@ -232,13 +232,12 @@ class PypeMongoFormatter(logging.Formatter): } # Standard document decorated with exception info if record.exc_info is not None: - document.update({ - 'exception': { - 'message': str(record.exc_info[1]), - 'code': 0, - 'stackTrace': self.formatException(record.exc_info) - } - }) + document['exception'] = { + 'message': str(record.exc_info[1]), + 'code': 0, + 'stackTrace': self.formatException(record.exc_info) + } + # Standard document decorated with extra contextual information if len(self.DEFAULT_PROPERTIES) != len(record.__dict__): contextual_extra = set(record.__dict__).difference( From e10dfe96200c8972a89e5391ef1aee544357f6fb Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 29 Dec 2020 17:04:13 +0100 Subject: [PATCH 05/12] moved global variables under PypeLogger class --- pype/lib/log.py | 65 +++++++++++++++---------------------------------- 1 file changed, 19 insertions(+), 46 deletions(-) diff --git a/pype/lib/log.py b/pype/lib/log.py index dde8fa9452..6274914e4d 100644 --- a/pype/lib/log.py +++ b/pype/lib/log.py @@ -33,41 +33,7 @@ from .mongo import ( try: import log4mongo from log4mongo.handlers import MongoHandler - from bson.objectid import ObjectId - MONGO_PROCESS_ID = ObjectId() except ImportError: - _mongo_logging = False -else: - _mongo_logging = True - - - -PYPE_DEBUG = int(os.getenv("PYPE_DEBUG", "0")) -LOG_DATABASE_NAME = os.environ.get("PYPE_LOG_MONGO_DB") or "pype" -LOG_COLLECTION_NAME = os.environ.get("PYPE_LOG_MONGO_COL") or "logs" - -system_name, pc_name = platform.uname()[:2] -host_name = socket.gethostname() -try: - ip = socket.gethostbyname(host_name) -except socket.gaierror: - ip = "127.0.0.1" - -# Get process name -if len(sys.argv) > 0 and os.path.basename(sys.argv[0]) == "tray.py": - process_name = "Tray" -else: - try: - import psutil - process = psutil.Process(os.getpid()) - process_name = process.name() - - except ImportError: - process_name = os.environ.get("AVALON_APP_NAME") - if not process_name: - process_name = os.path.basename(sys.executable) - - def _log_mongo_components(): mongo_url = os.environ.get("PYPE_LOG_MONGO_URL") if mongo_url is not None: @@ -222,13 +188,7 @@ class PypeMongoFormatter(logging.Formatter): 'fileName': record.pathname, 'module': record.module, 'method': record.funcName, - 'lineNumber': record.lineno, - 'process_id': MONGO_PROCESS_ID, - 'hostname': host_name, - 'hostip': ip, - 'username': getpass.getuser(), - 'system_name': system_name, - 'process_name': process_name + 'lineNumber': record.lineno } # Standard document decorated with exception info if record.exc_info is not None: @@ -249,9 +209,6 @@ class PypeMongoFormatter(logging.Formatter): class PypeLogger: - - PYPE_DEBUG = 0 - DFT = '%(levelname)s >>> { %(name)s }: [ %(message)s ] ' DBG = " - { %(name)s }: [ %(message)s ] " INF = ">>> [ %(message)s ] " @@ -267,12 +224,28 @@ class PypeLogger: logging.CRITICAL: CRI, } - def __init__(self): - self.PYPE_DEBUG = int(os.environ.get("PYPE_DEBUG", "0")) + # Is static class initialized + bootstraped = False + initialized = False + _init_lock = threading.Lock() + # Defines if mongo logging should be used + use_mongo_logging = None + mongo_process_id = None + # Information about mongo url + log_mongo_url = None + log_mongo_url_components = None + log_database_name = None + log_collection_name = None + # PYPE_DEBUG + pype_debug = 0 + # Data same for all record documents + process_data = None + # Cached process name or ability to set different process name + _process_name = None From 9839ff0f0bab84675a8102cb42f9b2d11a5103dc Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 29 Dec 2020 17:07:52 +0100 Subject: [PATCH 06/12] moved all logic handling under PypeLogger class --- pype/lib/log.py | 312 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 225 insertions(+), 87 deletions(-) diff --git a/pype/lib/log.py b/pype/lib/log.py index 6274914e4d..24e3489a3e 100644 --- a/pype/lib/log.py +++ b/pype/lib/log.py @@ -21,59 +21,22 @@ import socket import sys import time import traceback +import threading +import copy from . import Terminal from .mongo import ( MongoEnvNotSet, decompose_url, - compose_url, - get_default_components + PypeMongoConnection ) - try: import log4mongo from log4mongo.handlers import MongoHandler except ImportError: -def _log_mongo_components(): - mongo_url = os.environ.get("PYPE_LOG_MONGO_URL") - if mongo_url is not None: - components = decompose_url(mongo_url) - else: - components = get_default_components() - return components + log4mongo = None + MongoHandler = type("NOT_SET", (), {}) - -def _bootstrap_mongo_log(components=None): - """ - This will check if database and collection for logging exist on server. - """ - import pymongo - - if components is None: - components = _log_mongo_components() - - if not components["host"]: - # fail silently - return - - timeout = int(os.environ.get("AVALON_TIMEOUT", 1000)) - kwargs = { - "host": compose_url(**components), - "serverSelectionTimeoutMS": timeout - } - - port = components.get("port") - if port is not None: - kwargs["port"] = int(port) - client = pymongo.MongoClient(**kwargs) - logdb = client[LOG_DATABASE_NAME] - - collist = logdb.list_collection_names() - if LOG_COLLECTION_NAME not in collist: - logdb.create_collection( - LOG_COLLECTION_NAME, capped=True, max=5000, size=1073741824 - ) - return logdb # Check for `unicode` in builtins USE_UNICODE = hasattr(__builtins__, "unicode") @@ -190,6 +153,8 @@ class PypeMongoFormatter(logging.Formatter): 'method': record.funcName, 'lineNumber': record.lineno } + document.update(PypeLogger.get_process_data()) + # Standard document decorated with exception info if record.exc_info is not None: document['exception'] = { @@ -247,21 +212,65 @@ class PypeLogger: # Cached process name or ability to set different process name _process_name = None + @classmethod + def get_logger(cls, name=None, host=None): + if not cls.initialized: + cls.initialize() + logger = logging.getLogger(name or "__main__") + if cls.pype_debug > 1: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + add_mongo_handler = cls.use_mongo_logging + add_console_handler = True + for handler in logger.handlers: + if isinstance(handler, MongoHandler): + add_mongo_handler = False + elif isinstance(handler, PypeStreamHandler): + add_console_handler = False - def _get_mongo_handler(self): - components = _log_mongo_components() - # Check existence of mongo connection before creating Mongo handler - if log4mongo.handlers._connection is None: - _bootstrap_mongo_log(components) + if add_console_handler: + logger.addHandler(cls._get_console_handler()) + if add_mongo_handler: + try: + handler = cls._get_mongo_handler() + if handler: + logger.addHandler(handler) + + except MongoEnvNotSet: + # Skip if mongo environments are not set yet + cls.use_mongo_logging = False + + except Exception: + lines = traceback.format_exception(*sys.exc_info()) + for line in lines: + if line.endswith("\n"): + line = line[:-1] + Terminal.echo(line) + cls.use_mongo_logging = False + + # Do not propagate logs to root logger + logger.propagate = False + + return logger + + @classmethod + def _get_mongo_handler(cls): + cls.bootstrap_mongo_log() + + if not cls.use_mongo_logging: + return + + components = cls.log_mongo_url_components kwargs = { - "host": compose_url(**components), - "database_name": LOG_DATABASE_NAME, - "collection": LOG_COLLECTION_NAME, + "host": cls.log_mongo_url, + "database_name": cls.log_database_name, + "collection": cls.log_collection_name, "username": components["username"], "password": components["password"], "capped": True, @@ -274,56 +283,185 @@ class PypeLogger: return MongoHandler(**kwargs) - def _get_console_handler(self): - - formatter = PypeFormatter(self.FORMAT_FILE) + @classmethod + def _get_console_handler(cls): + formatter = PypeFormatter(cls.FORMAT_FILE) console_handler = PypeStreamHandler() console_handler.set_name("PypeStreamHandler") console_handler.setFormatter(formatter) return console_handler - def get_logger(self, name=None, host=None): - logger = logging.getLogger(name or '__main__') - - if self.PYPE_DEBUG > 1: - logger.setLevel(logging.DEBUG) + @classmethod + def initialize(cls): + if not cls._init_lock.locked(): + with cls._init_lock: + cls._initialize() else: - logger.setLevel(logging.INFO) + # If lock is locked wait until is finished + while cls._init_lock.locked(): + time.sleep(0.1) - global _mongo_logging - add_mongo_handler = _mongo_logging - add_console_handler = True + @classmethod + def _initialize(cls): + # Change initialization state to prevent runtime changes + # if is executed during runtime + cls.initialized = False - for handler in logger.handlers: - if isinstance(handler, MongoHandler): - add_mongo_handler = False - elif isinstance(handler, PypeStreamHandler): - add_console_handler = False - - if add_console_handler: - logger.addHandler(self._get_console_handler()) - - if add_mongo_handler: + # Define if should logging to mongo be used + use_mongo_logging = bool(log4mongo is not None) + # Set mongo id for process (ONLY ONCE) + if use_mongo_logging and cls.mongo_process_id is None: try: - logger.addHandler(self._get_mongo_handler()) - - except MongoEnvNotSet: - # Skip if mongo environments are not set yet - _mongo_logging = False - + from bson.objectid import ObjectId except Exception: - lines = traceback.format_exception(*sys.exc_info()) - for line in lines: - if line.endswith("\n"): - line = line[:-1] - Terminal.echo(line) - _mongo_logging = False + use_mongo_logging = False + ObjectId = lambda *a, **kw: None - # Do not propagate logs to root logger - logger.propagate = False + # Check if mongo id was passed with environments and pop it + # - This is for subprocesses that are part of another process + # like Ftrack event server has 3 other subprocesses that should + # use same mongo id + mongo_id = os.environ.pop("PYPE_PROCESS_MONGO_ID", None) + if not mongo_id: + # Create new object id + mongo_id = ObjectId() + else: + # Convert string to ObjectId object + mongo_id = ObjectId(mongo_id) + cls.mongo_process_id = mongo_id - return logger + # Store result to class definition + cls.use_mongo_logging = use_mongo_logging + + # Define if is in PYPE_DEBUG mode + cls.pype_debug = int(os.getenv("PYPE_DEBUG") or "0") + + # Mongo URL where logs will be stored + cls.log_mongo_url = ( + os.environ.get("PYPE_LOG_MONGO_URL") + or os.environ.get("PYPE_MONGO") + ) + if not cls.log_mongo_url: + cls.use_mongo_logging = False + else: + # Decompose url + cls.log_mongo_url_components = decompose_url(cls.log_mongo_url) + + # Database name in Mongo + cls.log_database_name = ( + os.environ.get("PYPE_LOG_MONGO_DB") or "pype" + ) + # Collection name under database in Mongo + cls.log_collection_name = ( + os.environ.get("PYPE_LOG_MONGO_COL") or "logs" + ) + + # Mark as initialized + cls.initialized = True + + @classmethod + def get_process_data(cls): + if cls.process_data is not None: + return copy.deepcopy(cls.process_data) + + if not cls.initialized: + cls.initialize() + + host_name = socket.gethostname() + try: + host_ip = socket.gethostbyname(host_name) + except socket.gaierror: + host_ip = "127.0.0.1" + + process_name = cls.get_process_name() + + cls.process_data = { + "process_id": cls.mongo_process_id, + "hostname": host_name, + "hostip": host_ip, + "username": getpass.getuser(), + "system_name": platform.system(), + "process_name": process_name + } + return copy.deepcopy(cls.process_data) + + @classmethod + def set_process_name(cls, process_name): + # Just change the attribute + cls._process_name = process_name + if cls.process_data is not None: + cls.process_data["process_name"] = process_name + + @classmethod + def get_process_name(cls): + """Process name that is like "label" of a process. + + Pype's logging can be used from pype itseld of from hosts. Even in Pype + it's good to know if logs are from Pype tray or from pype's event + server. This should help to identify that information. + """ + if cls._process_name is not None: + return cls._process_name + + # Get process name + process_name = os.environ.get("AVALON_APP_NAME") + if not process_name: + try: + import psutil + process = psutil.Process(os.getpid()) + process_name = process.name() + + except ImportError: + pass + + if not process_name: + process_name = os.path.basename(sys.executable) + + cls._process_name = process_name + return cls._process_name + + @classmethod + def bootstrap_mongo_log(cls): + if cls.bootstraped: + return + + if not cls.initialized: + cls.initialize() + + if not cls.use_mongo_logging: + return + + client = log4mongo.handlers._connection + if not client: + client = cls.get_log_mongo_connection() + # Set the client inside log4mongo handlers to not create another + # mongo db connection. + log4mongo.handlers._connection = client + + logdb = client[cls.log_database_name] + + collist = logdb.list_collection_names() + if cls.log_collection_name not in collist: + logdb.create_collection( + cls.log_collection_name, + capped=True, + max=5000, + size=1073741824 + ) + cls.bootstraped = True + + @classmethod + def get_log_mongo_connection(cls): + """Mongo connection that allows to get to log collection. + + This is implemented to prevent multiple connections to mongo from same + process. + """ + if not cls.initialized: + cls.initialize() + + return PypeMongoConnection.get_mongo_client(cls.log_mongo_url) def timeit(method): From b66c3c7b3d109c8af807b86e5ae4cac551a9881a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 29 Dec 2020 17:08:29 +0100 Subject: [PATCH 07/12] pype is using PypeLogger to print information about logging --- pype.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/pype.py b/pype.py index 769e8c8f6f..3d700aebdb 100644 --- a/pype.py +++ b/pype.py @@ -218,7 +218,7 @@ def boot(): def get_info() -> list: """Print additional information to console.""" from pype.lib.mongo import get_default_components - from pype.lib.log import LOG_DATABASE_NAME, LOG_COLLECTION_NAME + from pype.lib.log import PypeLogger components = get_default_components() @@ -242,14 +242,18 @@ def get_info() -> list: infos.append(("Using Muster at", os.environ.get("MUSTER_REST_URL"))) - if components["host"]: - infos.append(("Logging to MongoDB", components["host"])) - infos.append((" - port", components["port"] or "")) - infos.append((" - database", LOG_DATABASE_NAME)) - infos.append((" - collection", LOG_COLLECTION_NAME)) - infos.append((" - user", components["username"] or "")) - if components["auth_db"]: - infos.append((" - auth source", components["auth_db"])) + # Reinitialize + PypeLogger.initialize() + + log_components = PypeLogger.log_mongo_url_components + if log_components["host"]: + infos.append(("Logging to MongoDB", log_components["host"])) + infos.append((" - port", log_components["port"] or "")) + infos.append((" - database", PypeLogger.log_database_name)) + infos.append((" - collection", PypeLogger.log_collection_name)) + infos.append((" - user", log_components["username"] or "")) + if log_components["auth_db"]: + infos.append((" - auth source", log_components["auth_db"])) maximum = max([len(i[0]) for i in infos]) formatted = [] From cc06b0e2cdc75f772a75ef0f4e15945f69f691e4 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 29 Dec 2020 17:08:50 +0100 Subject: [PATCH 08/12] fixed log viewer model --- pype/modules/log_viewer/tray/models.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/pype/modules/log_viewer/tray/models.py b/pype/modules/log_viewer/tray/models.py index 3591f3dde2..7112002a34 100644 --- a/pype/modules/log_viewer/tray/models.py +++ b/pype/modules/log_viewer/tray/models.py @@ -1,9 +1,8 @@ import collections from Qt import QtCore, QtGui -from pype.api import Logger -from pype.lib.log import _bootstrap_mongo_log, LOG_COLLECTION_NAME +from pype.lib.log import PypeLogger -log = Logger().get_logger("LogModel", "LoggingModule") +log = PypeLogger().get_logger("LogModel", "LoggingModule") class LogModel(QtGui.QStandardItemModel): @@ -44,9 +43,14 @@ class LogModel(QtGui.QStandardItemModel): self.dbcon = None # Crash if connection is not possible to skip this module - database = _bootstrap_mongo_log() - if LOG_COLLECTION_NAME in database.list_collection_names(): - self.dbcon = database[LOG_COLLECTION_NAME] + if not PypeLogger.initialized: + PypeLogger.initialize() + + connection = PypeLogger.get_log_mongo_connection() + if connection: + PypeLogger.bootstrap_mongo_log() + database = connection[PypeLogger.log_database_name] + self.dbcon = database[PypeLogger.log_collection_name] def headerData(self, section, orientation, role): if ( From 6655be9d8b0905a27108bb5e147ef7c552285a0a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 29 Dec 2020 17:10:02 +0100 Subject: [PATCH 09/12] set few process names for testing --- pype/modules/ftrack/ftrack_server/socket_thread.py | 3 +++ pype/modules/ftrack/ftrack_server/sub_user_server.py | 2 ++ pype/pype_commands.py | 3 +++ 3 files changed, 8 insertions(+) diff --git a/pype/modules/ftrack/ftrack_server/socket_thread.py b/pype/modules/ftrack/ftrack_server/socket_thread.py index e66e8bc775..6a5fe2c9d6 100644 --- a/pype/modules/ftrack/ftrack_server/socket_thread.py +++ b/pype/modules/ftrack/ftrack_server/socket_thread.py @@ -55,6 +55,8 @@ class SocketThread(threading.Thread): "Running Socked thread on {}:{}".format(*server_address) ) + env = os.environ.copy() + env["PYPE_PROCESS_MONGO_ID"] = str(Logger.mongo_process_id) self.subproc = subprocess.Popen( [ sys.executable, @@ -62,6 +64,7 @@ class SocketThread(threading.Thread): *self.additional_args, str(self.port) ], + env=env, stdin=subprocess.PIPE ) diff --git a/pype/modules/ftrack/ftrack_server/sub_user_server.py b/pype/modules/ftrack/ftrack_server/sub_user_server.py index 1dedbbc7d0..58d5982ac2 100644 --- a/pype/modules/ftrack/ftrack_server/sub_user_server.py +++ b/pype/modules/ftrack/ftrack_server/sub_user_server.py @@ -51,6 +51,8 @@ def main(args): if __name__ == "__main__": + Logger.set_process_name("Ftrack User server") + # Register interupt signal def signal_handler(sig, frame): log.info( diff --git a/pype/pype_commands.py b/pype/pype_commands.py index ecaaf68a47..36e5ac639f 100644 --- a/pype/pype_commands.py +++ b/pype/pype_commands.py @@ -3,6 +3,7 @@ import os import subprocess import sys +from pype.lib import PypeLogger class PypeCommands: @@ -12,6 +13,8 @@ class PypeCommands: """ @staticmethod def launch_tray(debug=False): + PypeLogger.set_process_name("Tray") + from pype.tools import tray tray.main() From 17a056f8c61797fa008e63ff1b1ab905d623780d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 29 Dec 2020 17:21:16 +0100 Subject: [PATCH 10/12] removed host argument on get_logger call --- pype/hosts/fusion/pipeline.py | 2 +- pype/hosts/fusion/utils.py | 2 +- pype/hosts/hiero/__init__.py | 2 +- pype/hosts/hiero/events.py | 2 +- pype/hosts/hiero/lib.py | 2 +- pype/hosts/hiero/menu.py | 2 +- pype/hosts/hiero/tags.py | 2 +- pype/hosts/hiero/workio.py | 2 +- pype/hosts/nuke/__init__.py | 2 +- pype/hosts/nuke/lib.py | 2 +- pype/hosts/nuke/menu.py | 2 +- pype/hosts/nuke/presets.py | 2 +- pype/hosts/premiere/__init__.py | 2 +- pype/hosts/premiere/lib.py | 2 +- pype/hosts/resolve/lib.py | 2 +- pype/hosts/resolve/pipeline.py | 2 +- pype/hosts/resolve/preload_console.py | 2 +- pype/hosts/resolve/utils.py | 2 +- pype/hosts/resolve/workio.py | 2 +- pype/lib/log.py | 11 ++++++++++- .../clockify/launcher_actions/ClockifyStart.py | 2 +- .../modules/clockify/launcher_actions/ClockifySync.py | 2 +- pype/modules/ftrack/tray/ftrack_tray.py | 2 +- pype/modules/log_viewer/tray/models.py | 4 +--- pype/plugins/launcher/actions/AssetCreator.py | 2 +- pype/plugins/nuke/load/actions.py | 2 +- 26 files changed, 35 insertions(+), 28 deletions(-) diff --git a/pype/hosts/fusion/pipeline.py b/pype/hosts/fusion/pipeline.py index 22a80076da..b2918675ea 100644 --- a/pype/hosts/fusion/pipeline.py +++ b/pype/hosts/fusion/pipeline.py @@ -9,7 +9,7 @@ from pyblish import api as pyblish from pype.api import Logger from pype import PLUGINS_DIR -log = Logger().get_logger(__name__, "fusion") +log = Logger().get_logger(__name__) AVALON_CONFIG = os.environ["AVALON_CONFIG"] diff --git a/pype/hosts/fusion/utils.py b/pype/hosts/fusion/utils.py index 15fa52e938..6eee54b5b1 100644 --- a/pype/hosts/fusion/utils.py +++ b/pype/hosts/fusion/utils.py @@ -9,7 +9,7 @@ import shutil from pype.api import Logger -log = Logger().get_logger(__name__, "fusion") +log = Logger().get_logger(__name__) def _sync_utility_scripts(env=None): diff --git a/pype/hosts/hiero/__init__.py b/pype/hosts/hiero/__init__.py index 50fd39440b..9eb3cd551a 100644 --- a/pype/hosts/hiero/__init__.py +++ b/pype/hosts/hiero/__init__.py @@ -31,7 +31,7 @@ __all__ = [ ] # get logger -log = Logger().get_logger(__name__, "hiero") +log = Logger().get_logger(__name__) ''' Creating all important host related variables ''' diff --git a/pype/hosts/hiero/events.py b/pype/hosts/hiero/events.py index d78f8d54d4..4f9851cbac 100644 --- a/pype/hosts/hiero/events.py +++ b/pype/hosts/hiero/events.py @@ -4,7 +4,7 @@ from pype.api import Logger from .lib import sync_avalon_data_to_workfile, launch_workfiles_app from .tags import add_tags_from_presets -log = Logger().get_logger(__name__, "hiero") +log = Logger().get_logger(__name__) def startupCompleted(event): diff --git a/pype/hosts/hiero/lib.py b/pype/hosts/hiero/lib.py index 0db9b92b3d..182d7e53c0 100644 --- a/pype/hosts/hiero/lib.py +++ b/pype/hosts/hiero/lib.py @@ -9,7 +9,7 @@ from avalon.vendor.Qt import (QtWidgets, QtGui) import pype.api as pype from pype.api import Logger, Anatomy -log = Logger().get_logger(__name__, "hiero") +log = Logger().get_logger(__name__) cached_process = None diff --git a/pype/hosts/hiero/menu.py b/pype/hosts/hiero/menu.py index 697381f3cb..853ee8cf7e 100644 --- a/pype/hosts/hiero/menu.py +++ b/pype/hosts/hiero/menu.py @@ -12,7 +12,7 @@ from .lib import ( set_workfiles ) -log = Logger().get_logger(__name__, "hiero") +log = Logger().get_logger(__name__) self = sys.modules[__name__] self._change_context_menu = None diff --git a/pype/hosts/hiero/tags.py b/pype/hosts/hiero/tags.py index 551dc1698d..af1a9b947a 100644 --- a/pype/hosts/hiero/tags.py +++ b/pype/hosts/hiero/tags.py @@ -8,7 +8,7 @@ from pprint import pformat from pype.api import Logger from avalon import io -log = Logger().get_logger(__name__, "hiero") +log = Logger().get_logger(__name__) def tag_data(): diff --git a/pype/hosts/hiero/workio.py b/pype/hosts/hiero/workio.py index f11a34c9a8..c3505ef1bc 100644 --- a/pype/hosts/hiero/workio.py +++ b/pype/hosts/hiero/workio.py @@ -4,7 +4,7 @@ from avalon import api from pype.api import Logger -log = Logger().get_logger(__name__, "hiero") +log = Logger().get_logger(__name__) def file_extensions(): diff --git a/pype/hosts/nuke/__init__.py b/pype/hosts/nuke/__init__.py index 787f69f635..d0f3577c2e 100644 --- a/pype/hosts/nuke/__init__.py +++ b/pype/hosts/nuke/__init__.py @@ -15,7 +15,7 @@ from . import lib self = sys.modules[__name__] self.workfiles_launched = False -log = Logger().get_logger(__name__, "nuke") +log = Logger().get_logger(__name__) AVALON_CONFIG = os.getenv("AVALON_CONFIG", "pype") diff --git a/pype/hosts/nuke/lib.py b/pype/hosts/nuke/lib.py index 50b9697d8e..ddfdd260e2 100644 --- a/pype/hosts/nuke/lib.py +++ b/pype/hosts/nuke/lib.py @@ -20,7 +20,7 @@ from .presets import ( from .utils import set_context_favorites -log = pype.Logger().get_logger(__name__, "nuke") +log = pype.Logger().get_logger(__name__) self = sys.modules[__name__] self._project = None diff --git a/pype/hosts/nuke/menu.py b/pype/hosts/nuke/menu.py index b1ef7f47c4..f35cebfde0 100644 --- a/pype/hosts/nuke/menu.py +++ b/pype/hosts/nuke/menu.py @@ -5,7 +5,7 @@ from pype.hosts.nuke import lib from ...lib import BuildWorkfile from pype.api import Logger -log = Logger().get_logger(__name__, "nuke") +log = Logger().get_logger(__name__) def install(): diff --git a/pype/hosts/nuke/presets.py b/pype/hosts/nuke/presets.py index 2a296afc88..e40459f400 100644 --- a/pype/hosts/nuke/presets.py +++ b/pype/hosts/nuke/presets.py @@ -1,7 +1,7 @@ from pype.api import Anatomy, config, Logger import nuke -log = Logger().get_logger(__name__, "nuke") +log = Logger().get_logger(__name__) def get_anatomy(**kwarg): diff --git a/pype/hosts/premiere/__init__.py b/pype/hosts/premiere/__init__.py index acba84496b..8a9a032c54 100644 --- a/pype/hosts/premiere/__init__.py +++ b/pype/hosts/premiere/__init__.py @@ -18,7 +18,7 @@ __all__ = [ "ls" ] -log = Logger().get_logger(__name__, "premiere") +log = Logger().get_logger(__name__) def install(): diff --git a/pype/hosts/premiere/lib.py b/pype/hosts/premiere/lib.py index d76d27270a..5282e2c747 100644 --- a/pype/hosts/premiere/lib.py +++ b/pype/hosts/premiere/lib.py @@ -10,7 +10,7 @@ from pype.widgets.message_window import message from pype import PLUGINS_DIR from pype.api import Logger -log = Logger().get_logger(__name__, "premiere") +log = Logger().get_logger(__name__) self = sys.modules[__name__] self._has_been_setup = False diff --git a/pype/hosts/resolve/lib.py b/pype/hosts/resolve/lib.py index deb4fa6339..69496a5bcb 100644 --- a/pype/hosts/resolve/lib.py +++ b/pype/hosts/resolve/lib.py @@ -5,7 +5,7 @@ from pprint import pformat from pype.api import Logger -log = Logger().get_logger(__name__, "resolve") +log = Logger().get_logger(__name__) self = sys.modules[__name__] self.pm = None diff --git a/pype/hosts/resolve/pipeline.py b/pype/hosts/resolve/pipeline.py index 92bef2e13b..1a1c513f44 100644 --- a/pype/hosts/resolve/pipeline.py +++ b/pype/hosts/resolve/pipeline.py @@ -9,7 +9,7 @@ from pyblish import api as pyblish import pype from pype.api import Logger -log = Logger().get_logger(__name__, "resolve") +log = Logger().get_logger(__name__) AVALON_CONFIG = os.environ["AVALON_CONFIG"] diff --git a/pype/hosts/resolve/preload_console.py b/pype/hosts/resolve/preload_console.py index 58975777b8..de55c3673c 100644 --- a/pype/hosts/resolve/preload_console.py +++ b/pype/hosts/resolve/preload_console.py @@ -3,7 +3,7 @@ import time from pype.hosts.resolve.utils import get_resolve_module from pype.api import Logger -log = Logger().get_logger(__name__, "resolve") +log = Logger().get_logger(__name__) wait_delay = 2.5 wait = 0.00 diff --git a/pype/hosts/resolve/utils.py b/pype/hosts/resolve/utils.py index cb042a88e0..2a3f78a2df 100644 --- a/pype/hosts/resolve/utils.py +++ b/pype/hosts/resolve/utils.py @@ -9,7 +9,7 @@ import os import shutil from pype.api import Logger -log = Logger().get_logger(__name__, "resolve") +log = Logger().get_logger(__name__) def get_resolve_module(): diff --git a/pype/hosts/resolve/workio.py b/pype/hosts/resolve/workio.py index da74ebdeb1..18936df018 100644 --- a/pype/hosts/resolve/workio.py +++ b/pype/hosts/resolve/workio.py @@ -9,7 +9,7 @@ from . import ( ) -log = Logger().get_logger(__name__, "resolve") +log = Logger().get_logger(__name__) exported_projet_ext = ".drp" diff --git a/pype/lib/log.py b/pype/lib/log.py index 24e3489a3e..395385fadf 100644 --- a/pype/lib/log.py +++ b/pype/lib/log.py @@ -213,7 +213,7 @@ class PypeLogger: _process_name = None @classmethod - def get_logger(cls, name=None, host=None): + def get_logger(cls, name=None, _host=None): if not cls.initialized: cls.initialize() @@ -257,6 +257,15 @@ class PypeLogger: # Do not propagate logs to root logger logger.propagate = False + if _host is not None: + # Warn about deprecated argument + # TODO remove backwards compatibility of host argument which is + # not used for more than a year + logger.warning( + "Logger \"{}\" is using argument `host` on `get_logger`" + " which is deprecated. Please remove as backwards" + " compatibility will be removed soon." + ) return logger @classmethod diff --git a/pype/modules/clockify/launcher_actions/ClockifyStart.py b/pype/modules/clockify/launcher_actions/ClockifyStart.py index f97360662f..d02005e2de 100644 --- a/pype/modules/clockify/launcher_actions/ClockifyStart.py +++ b/pype/modules/clockify/launcher_actions/ClockifyStart.py @@ -3,7 +3,7 @@ from pype.api import Logger from pype.modules.clockify.clockify_api import ClockifyAPI -log = Logger().get_logger(__name__, "clockify_start") +log = Logger().get_logger(__name__) class ClockifyStart(api.Action): diff --git a/pype/modules/clockify/launcher_actions/ClockifySync.py b/pype/modules/clockify/launcher_actions/ClockifySync.py index 422a346023..5f0e57b8c8 100644 --- a/pype/modules/clockify/launcher_actions/ClockifySync.py +++ b/pype/modules/clockify/launcher_actions/ClockifySync.py @@ -1,7 +1,7 @@ from avalon import api, io from pype.modules.clockify.clockify_api import ClockifyAPI from pype.api import Logger -log = Logger().get_logger(__name__, "clockify_sync") +log = Logger().get_logger(__name__) class ClockifySync(api.Action): diff --git a/pype/modules/ftrack/tray/ftrack_tray.py b/pype/modules/ftrack/tray/ftrack_tray.py index 56133208c2..3f6432e541 100644 --- a/pype/modules/ftrack/tray/ftrack_tray.py +++ b/pype/modules/ftrack/tray/ftrack_tray.py @@ -13,7 +13,7 @@ from . import login_dialog from pype.api import Logger, resources -log = Logger().get_logger("FtrackModule", "ftrack") +log = Logger().get_logger("FtrackModule") class FtrackTrayWrapper: diff --git a/pype/modules/log_viewer/tray/models.py b/pype/modules/log_viewer/tray/models.py index 7112002a34..6820d5bcb9 100644 --- a/pype/modules/log_viewer/tray/models.py +++ b/pype/modules/log_viewer/tray/models.py @@ -2,8 +2,6 @@ import collections from Qt import QtCore, QtGui from pype.lib.log import PypeLogger -log = PypeLogger().get_logger("LogModel", "LoggingModule") - class LogModel(QtGui.QStandardItemModel): COLUMNS = ( @@ -45,7 +43,7 @@ class LogModel(QtGui.QStandardItemModel): # Crash if connection is not possible to skip this module if not PypeLogger.initialized: PypeLogger.initialize() - + connection = PypeLogger.get_log_mongo_connection() if connection: PypeLogger.bootstrap_mongo_log() diff --git a/pype/plugins/launcher/actions/AssetCreator.py b/pype/plugins/launcher/actions/AssetCreator.py index 9787aae002..5e845fdb5e 100644 --- a/pype/plugins/launcher/actions/AssetCreator.py +++ b/pype/plugins/launcher/actions/AssetCreator.py @@ -2,7 +2,7 @@ from avalon import api, lib from pype.api import Logger -log = Logger().get_logger(__name__, "asset_creator") +log = Logger().get_logger(__name__) class AssetCreator(api.Action): diff --git a/pype/plugins/nuke/load/actions.py b/pype/plugins/nuke/load/actions.py index 96f5f3b8c4..d0c95b6b16 100644 --- a/pype/plugins/nuke/load/actions.py +++ b/pype/plugins/nuke/load/actions.py @@ -5,7 +5,7 @@ from avalon import api from pype.api import Logger -log = Logger().get_logger(__name__, "nuke") +log = Logger().get_logger(__name__) class SetFrameRangeLoader(api.Loader): From e5d5f45cc07bb776e17e3caacf5778789d52e845 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 29 Dec 2020 17:52:21 +0100 Subject: [PATCH 11/12] removed lambda usage --- pype/lib/log.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pype/lib/log.py b/pype/lib/log.py index 395385fadf..aa7d651170 100644 --- a/pype/lib/log.py +++ b/pype/lib/log.py @@ -325,20 +325,20 @@ class PypeLogger: from bson.objectid import ObjectId except Exception: use_mongo_logging = False - ObjectId = lambda *a, **kw: None # Check if mongo id was passed with environments and pop it # - This is for subprocesses that are part of another process # like Ftrack event server has 3 other subprocesses that should # use same mongo id - mongo_id = os.environ.pop("PYPE_PROCESS_MONGO_ID", None) - if not mongo_id: - # Create new object id - mongo_id = ObjectId() - else: - # Convert string to ObjectId object - mongo_id = ObjectId(mongo_id) - cls.mongo_process_id = mongo_id + if use_mongo_logging: + mongo_id = os.environ.pop("PYPE_PROCESS_MONGO_ID", None) + if not mongo_id: + # Create new object id + mongo_id = ObjectId() + else: + # Convert string to ObjectId object + mongo_id = ObjectId(mongo_id) + cls.mongo_process_id = mongo_id # Store result to class definition cls.use_mongo_logging = use_mongo_logging From e5bec0aad8e56d7030c9c6d9e39d088cea24574c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 29 Dec 2020 17:56:38 +0100 Subject: [PATCH 12/12] added few comments --- pype/lib/log.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pype/lib/log.py b/pype/lib/log.py index aa7d651170..b6cbff5648 100644 --- a/pype/lib/log.py +++ b/pype/lib/log.py @@ -303,6 +303,7 @@ class PypeLogger: @classmethod def initialize(cls): + # TODO update already created loggers on re-initialization if not cls._init_lock.locked(): with cls._init_lock: cls._initialize() @@ -371,6 +372,10 @@ class PypeLogger: @classmethod def get_process_data(cls): + """Data about current process which should be same for all records. + + Process data are used for each record sent to mongo database. + """ if cls.process_data is not None: return copy.deepcopy(cls.process_data) @@ -397,8 +402,10 @@ class PypeLogger: @classmethod def set_process_name(cls, process_name): + """Set process name for mongo logs.""" # Just change the attribute cls._process_name = process_name + # Update process data if are already set if cls.process_data is not None: cls.process_data["process_name"] = process_name @@ -432,6 +439,7 @@ class PypeLogger: @classmethod def bootstrap_mongo_log(cls): + """Prepare mongo logging.""" if cls.bootstraped: return