From f82e87787a88a534ac64eae549b526b0e84878b0 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 1 Apr 2021 21:44:30 +0200 Subject: [PATCH 1/6] DL - cleanup, add documentation --- .../custom/events/OpenPype/OpenPype.param | 4 +-- .../custom/plugins/OpenPype/OpenPype.options | 31 ------------------- .../custom/plugins/OpenPype/OpenPype.param | 13 ++------ .../custom/plugins/OpenPype/OpenPype.py | 7 ++--- vendor/deadline/readme.md | 31 ++++++++++++++++++- 5 files changed, 36 insertions(+), 50 deletions(-) diff --git a/vendor/deadline/custom/events/OpenPype/OpenPype.param b/vendor/deadline/custom/events/OpenPype/OpenPype.param index c9610ffd19..871ce47467 100644 --- a/vendor/deadline/custom/events/OpenPype/OpenPype.param +++ b/vendor/deadline/custom/events/OpenPype/OpenPype.param @@ -28,8 +28,8 @@ Default=DEBUG Description=Logging level where printing will start. [OpenPypeExecutable] -Type=MultiLineMultiFolder -Label=Path to OpenPype executable dir +Type=multilinemultifilename +Label=Path to OpenPype executable Category=Job Plugins CategoryOrder=1 CategoryIndex=1 diff --git a/vendor/deadline/custom/plugins/OpenPype/OpenPype.options b/vendor/deadline/custom/plugins/OpenPype/OpenPype.options index df75bbe0fb..6908fe865f 100644 --- a/vendor/deadline/custom/plugins/OpenPype/OpenPype.options +++ b/vendor/deadline/custom/plugins/OpenPype/OpenPype.options @@ -1,13 +1,3 @@ -[ScriptFile] -Type=filename -Label=Script File -Category=Python Options -CategoryOrder=0 -Index=0 -Description=The script file to be executed. -Required=false -DisableIfBlank=true - [Arguments] Type=string Label=Arguments @@ -18,24 +8,3 @@ Description=The arguments to pass to the script. If no arguments are required, l Required=false DisableIfBlank=true -[Version] -Type=enum -Values=3.0 -Label=Version -Category=Python Options -CategoryOrder=0 -Index=2 -Description=The version of Python to use. -Required=false -DisableIfBlank=true - -[SingleFramesOnly] -Type=boolean -Label=Single Frames Only -Category=Job Options -CategoryOrder=1 -Index=0 -Description=If enabled, the plugin will only render one frame at a time even if a single task contains a chunk of frames. -Required=true -DisableIfBlank=true -Default=false diff --git a/vendor/deadline/custom/plugins/OpenPype/OpenPype.param b/vendor/deadline/custom/plugins/OpenPype/OpenPype.param index 9bf9ba25e7..8bd6dce12d 100644 --- a/vendor/deadline/custom/plugins/OpenPype/OpenPype.param +++ b/vendor/deadline/custom/plugins/OpenPype/OpenPype.param @@ -7,18 +7,9 @@ Index=0 Default=OpenPype Plugin for Deadline Description=Not configurable -[ConcurrentTasks] -Type=label -Label=ConcurrentTasks -Category=About Plugin -CategoryOrder=-1 -Index=0 -Default=True -Description=Not configurable - -[OpenPype_Executable_3_0] +[OpenPypeExecutable] Type=multilinemultifilename -Label=OpenPype 3.0 Executable +Label=OpenPype Executable Category=OpenPype Executables CategoryOrder=0 Index=0 diff --git a/vendor/deadline/custom/plugins/OpenPype/OpenPype.py b/vendor/deadline/custom/plugins/OpenPype/OpenPype.py index e03bea9c0c..451d71fb63 100644 --- a/vendor/deadline/custom/plugins/OpenPype/OpenPype.py +++ b/vendor/deadline/custom/plugins/OpenPype/OpenPype.py @@ -53,14 +53,11 @@ class OpenPypeDeadlinePlugin(DeadlinePlugin): ".*Progress: (\d+)%.*").HandleCallback += self.HandleProgress def RenderExecutable(self): - version = self.GetPluginInfoEntry("Version") - - exeList = self.GetConfigEntry( - "OpenPype_Executable_" + version.replace(".", "_")) + exeList = self.GetConfigEntry("OpenPypeExecutable") exe = FileUtils.SearchFileList(exeList) if exe == "": self.FailRender( - "OpenPype " + version + " executable was not found " + + "OpenPype executable was not found " + "in the semicolon separated list \"" + exeList + "\". " + "The path to the render executable can be configured " + "from the Plugin Configuration in the Deadline Monitor.") diff --git a/vendor/deadline/readme.md b/vendor/deadline/readme.md index 34d4403bf5..803a290050 100644 --- a/vendor/deadline/readme.md +++ b/vendor/deadline/readme.md @@ -1,3 +1,32 @@ ## OpenPype Deadline repository overlay - This directory is overlay for Deadline repository. It means that you can copy whole hierarchy to Deadline repository and it should work. + This directory is overlay for Deadline repository. + It means that you can copy whole hierarchy to Deadline repository and it should work. + + Logic: + ----- + Event + ----- + For each rendering job OpenPype event is triggered, it stores path to OpenPype + executable (needs to be configured on `Deadline's Configure Events > OpenPype`) + job's extra key 'openpype_executables'. + + This value is used by `GlobalJobPreLoad` to call that executable to pull + environment's variables which are needed to add to ALL plugins process environments. + These env. vars are injected into rendering process. + + Event is necessary here as a middle man to allow configuring location of executable + which is ONLY then used by `GlobalJobPreLoad` (which doesnt have any user facing + configuration at all). + + `GlobalJobPreLoad` is triggered before each job, it contains backward compatible + logic to not modify old Pype2 or not OpenPype triggered jobs. + + Plugin + ------ + For each publishing job `OpenPypeDeadlinePlugin` is called, which calls + configured location of OpenPype executable (needs to be configured in + `Deadline's Configure Plugins > OpenPype`) + and triggers command. + + From 5887efeefcbbe9d62dedc0848431edeae14b754f Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 1 Apr 2021 21:56:50 +0200 Subject: [PATCH 2/6] DL fix missed plugin name --- openpype/modules/deadline/deadline_module.py | 2 +- openpype/modules/deadline/plugins/publish/submit_publish_job.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/modules/deadline/deadline_module.py b/openpype/modules/deadline/deadline_module.py index 9f2f9469c1..dfbc49248b 100644 --- a/openpype/modules/deadline/deadline_module.py +++ b/openpype/modules/deadline/deadline_module.py @@ -13,7 +13,7 @@ class DeadlineModule(PypeModule, IPluginPaths): self.deadline_url = deadline_settings["DEADLINE_REST_URL"] def get_global_environments(self): - """Deadline global environments for pype implementation.""" + """Deadline global environments for open implementation.""" return { "DEADLINE_REST_URL": self.deadline_url } diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index 03ac395815..a2e21fb766 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -98,7 +98,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): label = "Submit image sequence jobs to Deadline or Muster" order = pyblish.api.IntegratorOrder + 0.2 icon = "tractor" - deadline_plugin = "Pype" + deadline_plugin = "OpenPype" hosts = ["fusion", "maya", "nuke", "celaction", "aftereffects", "harmony"] From 73841259bfc10e67eda5b5b45be98c60e378bdc6 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 1 Apr 2021 22:16:46 +0200 Subject: [PATCH 3/6] Rebrand database access All occurences of 'pype' database should be replaced with 'openpype' --- openpype/lib/__init__.py | 4 ++-- openpype/lib/log.py | 6 +++--- openpype/lib/mongo.py | 4 ++-- .../modules/ftrack/ftrack_server/event_server_cli.py | 4 ++-- openpype/modules/ftrack/ftrack_server/lib.py | 6 +++--- openpype/modules/ftrack/lib/settings.py | 6 ++---- openpype/modules/ftrack/scripts/sub_event_storer.py | 6 +++--- openpype/settings/handlers.py | 12 ++++++------ start.py | 1 + 9 files changed, 24 insertions(+), 25 deletions(-) diff --git a/openpype/lib/__init__.py b/openpype/lib/__init__.py index e945c0a42b..554c0d8ec3 100644 --- a/openpype/lib/__init__.py +++ b/openpype/lib/__init__.py @@ -24,7 +24,7 @@ from .mongo import ( compose_url, get_default_components, validate_mongo_connection, - PypeMongoConnection + OpenPypeMongoConnection ) from .anatomy import ( merge_dict, @@ -213,7 +213,7 @@ __all__ = [ "compose_url", "get_default_components", "validate_mongo_connection", - "PypeMongoConnection", + "OpenPypeMongoConnection", "IniSettingRegistry", "JSONSettingRegistry", diff --git a/openpype/lib/log.py b/openpype/lib/log.py index 9675637700..9745279e28 100644 --- a/openpype/lib/log.py +++ b/openpype/lib/log.py @@ -28,7 +28,7 @@ from . import Terminal from .mongo import ( MongoEnvNotSet, decompose_url, - PypeMongoConnection + OpenPypeMongoConnection ) try: import log4mongo @@ -203,7 +203,7 @@ class PypeLogger: log_mongo_url_components = None # Database name in Mongo - log_database_name = "pype" + log_database_name = os.environ["OPENPYPE_DATABASE_NAME"] # Collection name under database in Mongo log_collection_name = "logs" @@ -470,7 +470,7 @@ class PypeLogger: if not cls.initialized: cls.initialize() - return PypeMongoConnection.get_mongo_client(cls.log_mongo_url) + return OpenPypeMongoConnection.get_mongo_client(cls.log_mongo_url) def timeit(method): diff --git a/openpype/lib/mongo.py b/openpype/lib/mongo.py index edb52a29ab..8bfaba75d6 100644 --- a/openpype/lib/mongo.py +++ b/openpype/lib/mongo.py @@ -129,13 +129,13 @@ def validate_mongo_connection(mongo_uri): client.close() -class PypeMongoConnection: +class OpenPypeMongoConnection: """Singleton MongoDB connection. Keeps MongoDB connections by url. """ mongo_clients = {} - log = logging.getLogger("PypeMongoConnection") + log = logging.getLogger("OpenPypeMongoConnection") @staticmethod def get_default_mongo_url(): diff --git a/openpype/modules/ftrack/ftrack_server/event_server_cli.py b/openpype/modules/ftrack/ftrack_server/event_server_cli.py index 1c382be18c..b5cc1bef3e 100644 --- a/openpype/modules/ftrack/ftrack_server/event_server_cli.py +++ b/openpype/modules/ftrack/ftrack_server/event_server_cli.py @@ -16,7 +16,7 @@ import ftrack_api import pymongo from openpype.lib import ( get_pype_execute_args, - PypeMongoConnection + OpenPypeMongoConnection ) from openpype.modules.ftrack import FTRACK_MODULE_DIR from openpype.modules.ftrack.lib import ( @@ -181,7 +181,7 @@ def main_loop(ftrack_url): os.environ["FTRACK_EVENT_SUB_ID"] = str(uuid.uuid1()) - mongo_uri = PypeMongoConnection.get_default_mongo_url() + mongo_uri = OpenPypeMongoConnection.get_default_mongo_url() # Current file scripts_dir = os.path.join(FTRACK_MODULE_DIR, "scripts") diff --git a/openpype/modules/ftrack/ftrack_server/lib.py b/openpype/modules/ftrack/ftrack_server/lib.py index f408b52f83..6d3bca305d 100644 --- a/openpype/modules/ftrack/ftrack_server/lib.py +++ b/openpype/modules/ftrack/ftrack_server/lib.py @@ -24,7 +24,7 @@ except ImportError: from ftrack_api._weakref import WeakMethod from openpype.modules.ftrack.lib import get_ftrack_event_mongo_info -from openpype.lib import PypeMongoConnection +from openpype.lib import OpenPypeMongoConnection from openpype.api import Logger TOPIC_STATUS_SERVER = "pype.event.server.status" @@ -144,14 +144,14 @@ class ProcessEventHub(SocketBaseEventHub): def prepare_dbcon(self): try: database_name, collection_name = get_ftrack_event_mongo_info() - mongo_client = PypeMongoConnection.get_mongo_client() + mongo_client = OpenPypeMongoConnection.get_mongo_client() self.dbcon = mongo_client[database_name][collection_name] self.mongo_client = mongo_client except pymongo.errors.AutoReconnect: self.pypelog.error(( "Mongo server \"{}\" is not responding, exiting." - ).format(PypeMongoConnection.get_default_mongo_url())) + ).format(OpenPypeMongoConnection.get_default_mongo_url())) sys.exit(0) except pymongo.errors.OperationFailure: diff --git a/openpype/modules/ftrack/lib/settings.py b/openpype/modules/ftrack/lib/settings.py index b77807c329..f6967411db 100644 --- a/openpype/modules/ftrack/lib/settings.py +++ b/openpype/modules/ftrack/lib/settings.py @@ -1,8 +1,6 @@ +import os from openpype.api import get_system_settings -PYPE_DATABASE_NAME = "pype" - - def get_ftrack_settings(): return get_system_settings()["modules"]["ftrack"] @@ -13,6 +11,6 @@ def get_ftrack_url_from_settings(): def get_ftrack_event_mongo_info(): ftrack_settings = get_ftrack_settings() - database_name = PYPE_DATABASE_NAME + database_name = os.environ["OPENPYPE_DATABASE_NAME"] collection_name = "ftrack_events" return database_name, collection_name diff --git a/openpype/modules/ftrack/scripts/sub_event_storer.py b/openpype/modules/ftrack/scripts/sub_event_storer.py index 7f903f1bc0..6ee61f5a08 100644 --- a/openpype/modules/ftrack/scripts/sub_event_storer.py +++ b/openpype/modules/ftrack/scripts/sub_event_storer.py @@ -14,7 +14,7 @@ from openpype.modules.ftrack.ftrack_server.lib import ( TOPIC_STATUS_SERVER_RESULT ) from openpype.modules.ftrack.lib import get_ftrack_event_mongo_info -from openpype.lib import PypeMongoConnection +from openpype.lib import OpenPypeMongoConnection from openpype.api import Logger log = Logger.get_logger("Event storer") @@ -35,11 +35,11 @@ ignore_topics = [] def install_db(): global dbcon try: - mongo_client = PypeMongoConnection.get_mongo_client() + mongo_client = OpenPypeMongoConnection.get_mongo_client() dbcon = mongo_client[database_name][collection_name] except pymongo.errors.AutoReconnect: log.error("Mongo server \"{}\" is not responding, exiting.".format( - PypeMongoConnection.get_default_mongo_url() + OpenPypeMongoConnection.get_default_mongo_url() )) sys.exit(0) diff --git a/openpype/settings/handlers.py b/openpype/settings/handlers.py index 59f091ea3a..b3e1b1b1e1 100644 --- a/openpype/settings/handlers.py +++ b/openpype/settings/handlers.py @@ -170,18 +170,18 @@ class MongoSettingsHandler(SettingsHandler): def __init__(self): # Get mongo connection - from openpype.lib import PypeMongoConnection + from openpype.lib import OpenPypeMongoConnection from avalon.api import AvalonMongoDB - settings_collection = PypeMongoConnection.get_mongo_client() + settings_collection = OpenPypeMongoConnection.get_mongo_client() self._anatomy_keys = None self._attribute_keys = None # TODO prepare version of pype # - pype version should define how are settings saved and loaded + database_name = os.environ["OPENPYPE_DATABASE_NAME"] # TODO modify to not use hardcoded keys - database_name = "pype" collection_name = "settings" self.settings_collection = settings_collection @@ -541,19 +541,19 @@ class MongoLocalSettingsHandler(LocalSettingsHandler): def __init__(self, local_site_id=None): # Get mongo connection from openpype.lib import ( - PypeMongoConnection, + OpenPypeMongoConnection, get_local_site_id ) if local_site_id is None: local_site_id = get_local_site_id() - settings_collection = PypeMongoConnection.get_mongo_client() + settings_collection = OpenPypeMongoConnection.get_mongo_client() # TODO prepare version of pype # - pype version should define how are settings saved and loaded + database_name = os.environ["OPENPYPE_DATABASE_NAME"] # TODO modify to not use hardcoded keys - database_name = "pype" collection_name = "settings" self.settings_collection = settings_collection diff --git a/start.py b/start.py index 3e3a586d8c..47a538934d 100644 --- a/start.py +++ b/start.py @@ -140,6 +140,7 @@ def set_openpype_global_environments() -> None: # Hardcoded default values os.environ["PYBLISH_GUI"] = "pyblish_pype" + os.environ["OPENPYPE_DATABASE_NAME"] = "openpype" # name of Pype database # Change scale factor only if is not set if "QT_AUTO_SCREEN_SCALE_FACTOR" not in os.environ: os.environ["QT_AUTO_SCREEN_SCALE_FACTOR"] = "1" From 306e862824962284b246f64607d35731a7198144 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 1 Apr 2021 22:27:59 +0200 Subject: [PATCH 4/6] Setting of openpype database must be higher Added missed ones --- repos/avalon-core | 2 +- start.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/repos/avalon-core b/repos/avalon-core index 09065aaf6b..a51844b29d 160000 --- a/repos/avalon-core +++ b/repos/avalon-core @@ -1 +1 @@ -Subproject commit 09065aaf6b8617a0af0a6f0f82bb1cabf6b0890e +Subproject commit a51844b29ddef45871b2aa574b7802d79893e6db diff --git a/start.py b/start.py index 47a538934d..1f946a705c 100644 --- a/start.py +++ b/start.py @@ -140,7 +140,6 @@ def set_openpype_global_environments() -> None: # Hardcoded default values os.environ["PYBLISH_GUI"] = "pyblish_pype" - os.environ["OPENPYPE_DATABASE_NAME"] = "openpype" # name of Pype database # Change scale factor only if is not set if "QT_AUTO_SCREEN_SCALE_FACTOR" not in os.environ: os.environ["QT_AUTO_SCREEN_SCALE_FACTOR"] = "1" @@ -562,6 +561,7 @@ def boot(): sys.exit(1) os.environ["OPENPYPE_MONGO"] = openpype_mongo + os.environ["OPENPYPE_DATABASE_NAME"] = "openpype" # name of Pype database # ------------------------------------------------------------------------ # Set environments - load OpenPype path from database (if set) From 3a07b7c682e6f4bb6a0d6d246cdf06460e8fb51b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 1 Apr 2021 22:35:16 +0200 Subject: [PATCH 5/6] Fix typo --- openpype/modules/deadline/deadline_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/deadline/deadline_module.py b/openpype/modules/deadline/deadline_module.py index dfbc49248b..2a2fba41d6 100644 --- a/openpype/modules/deadline/deadline_module.py +++ b/openpype/modules/deadline/deadline_module.py @@ -13,7 +13,7 @@ class DeadlineModule(PypeModule, IPluginPaths): self.deadline_url = deadline_settings["DEADLINE_REST_URL"] def get_global_environments(self): - """Deadline global environments for open implementation.""" + """Deadline global environments for OpenPype implementation.""" return { "DEADLINE_REST_URL": self.deadline_url } From 0b63fab90594f155c56d37fe60ad4527cd6deab1 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Thu, 1 Apr 2021 22:46:59 +0200 Subject: [PATCH 6/6] bump avalon --- repos/avalon-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/avalon-core b/repos/avalon-core index a51844b29d..63dc6e6c10 160000 --- a/repos/avalon-core +++ b/repos/avalon-core @@ -1 +1 @@ -Subproject commit a51844b29ddef45871b2aa574b7802d79893e6db +Subproject commit 63dc6e6c105ce4acbbee6f15812c6b2cd1cc69ba