Merge pull request #41 from pypeclub/fix/deadline_cleanup

Deadline - cleanup, add documentation

openpype/lib/__init__.py

@@ -24,7 +24,7 @@ from .mongo import (
     compose_url,
     get_default_components,
     validate_mongo_connection,
-    PypeMongoConnection
+    OpenPypeMongoConnection
 )
 from .anatomy import (
     merge_dict,
@@ -213,7 +213,7 @@ __all__ = [
     "compose_url",
     "get_default_components",
     "validate_mongo_connection",
-    "PypeMongoConnection",
+    "OpenPypeMongoConnection",
 
     "IniSettingRegistry",
     "JSONSettingRegistry",

openpype/lib/log.py

@@ -28,7 +28,7 @@ from . import Terminal
 from .mongo import (
     MongoEnvNotSet,
     decompose_url,
-    PypeMongoConnection
+    OpenPypeMongoConnection
 )
 try:
     import log4mongo
@@ -203,7 +203,7 @@ class PypeLogger:
     log_mongo_url_components = None
 
     # Database name in Mongo
-    log_database_name = "pype"
+    log_database_name = os.environ["OPENPYPE_DATABASE_NAME"]
     # Collection name under database in Mongo
     log_collection_name = "logs"
 
@@ -470,7 +470,7 @@ class PypeLogger:
         if not cls.initialized:
             cls.initialize()
 
-        return PypeMongoConnection.get_mongo_client(cls.log_mongo_url)
+        return OpenPypeMongoConnection.get_mongo_client(cls.log_mongo_url)
 
 
 def timeit(method):

openpype/lib/mongo.py

@@ -129,13 +129,13 @@ def validate_mongo_connection(mongo_uri):
     client.close()
 
 
-class PypeMongoConnection:
+class OpenPypeMongoConnection:
     """Singleton MongoDB connection.
 
     Keeps MongoDB connections by url.
     """
     mongo_clients = {}
-    log = logging.getLogger("PypeMongoConnection")
+    log = logging.getLogger("OpenPypeMongoConnection")
 
     @staticmethod
     def get_default_mongo_url():

openpype/modules/deadline/deadline_module.py

@@ -13,7 +13,7 @@ class DeadlineModule(PypeModule, IPluginPaths):
         self.deadline_url = deadline_settings["DEADLINE_REST_URL"]
 
     def get_global_environments(self):
-        """Deadline global environments for pype implementation."""
+        """Deadline global environments for OpenPype implementation."""
         return {
             "DEADLINE_REST_URL": self.deadline_url
         }

openpype/modules/deadline/plugins/publish/submit_publish_job.py

@@ -98,7 +98,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
     label = "Submit image sequence jobs to Deadline or Muster"
     order = pyblish.api.IntegratorOrder + 0.2
     icon = "tractor"
-    deadline_plugin = "Pype"
+    deadline_plugin = "OpenPype"
 
     hosts = ["fusion", "maya", "nuke", "celaction", "aftereffects", "harmony"]
 

openpype/modules/ftrack/ftrack_server/event_server_cli.py

@@ -16,7 +16,7 @@ import ftrack_api
 import pymongo
 from openpype.lib import (
     get_pype_execute_args,
-    PypeMongoConnection
+    OpenPypeMongoConnection
 )
 from openpype.modules.ftrack import FTRACK_MODULE_DIR
 from openpype.modules.ftrack.lib import (
@@ -181,7 +181,7 @@ def main_loop(ftrack_url):
 
     os.environ["FTRACK_EVENT_SUB_ID"] = str(uuid.uuid1())
 
-    mongo_uri = PypeMongoConnection.get_default_mongo_url()
+    mongo_uri = OpenPypeMongoConnection.get_default_mongo_url()
 
     # Current file
     scripts_dir = os.path.join(FTRACK_MODULE_DIR, "scripts")

openpype/modules/ftrack/ftrack_server/lib.py

@@ -24,7 +24,7 @@ except ImportError:
     from ftrack_api._weakref import WeakMethod
 from openpype.modules.ftrack.lib import get_ftrack_event_mongo_info
 
-from openpype.lib import PypeMongoConnection
+from openpype.lib import OpenPypeMongoConnection
 from openpype.api import Logger
 
 TOPIC_STATUS_SERVER = "pype.event.server.status"
@@ -144,14 +144,14 @@ class ProcessEventHub(SocketBaseEventHub):
     def prepare_dbcon(self):
         try:
             database_name, collection_name = get_ftrack_event_mongo_info()
-            mongo_client = PypeMongoConnection.get_mongo_client()
+            mongo_client = OpenPypeMongoConnection.get_mongo_client()
             self.dbcon = mongo_client[database_name][collection_name]
             self.mongo_client = mongo_client
 
         except pymongo.errors.AutoReconnect:
             self.pypelog.error((
                 "Mongo server \"{}\" is not responding, exiting."
-            ).format(PypeMongoConnection.get_default_mongo_url()))
+            ).format(OpenPypeMongoConnection.get_default_mongo_url()))
             sys.exit(0)
 
         except pymongo.errors.OperationFailure:

openpype/modules/ftrack/lib/settings.py

@@ -1,8 +1,6 @@
+import os
 from openpype.api import get_system_settings
 
-PYPE_DATABASE_NAME = "pype"
-
-
 def get_ftrack_settings():
     return get_system_settings()["modules"]["ftrack"]
 
@@ -13,6 +11,6 @@ def get_ftrack_url_from_settings():
 
 def get_ftrack_event_mongo_info():
     ftrack_settings = get_ftrack_settings()
-    database_name = PYPE_DATABASE_NAME
+    database_name = os.environ["OPENPYPE_DATABASE_NAME"]
     collection_name = "ftrack_events"
     return database_name, collection_name

openpype/modules/ftrack/scripts/sub_event_storer.py

@@ -14,7 +14,7 @@ from openpype.modules.ftrack.ftrack_server.lib import (
     TOPIC_STATUS_SERVER_RESULT
 )
 from openpype.modules.ftrack.lib import get_ftrack_event_mongo_info
-from openpype.lib import PypeMongoConnection
+from openpype.lib import OpenPypeMongoConnection
 from openpype.api import Logger
 
 log = Logger.get_logger("Event storer")
@@ -35,11 +35,11 @@ ignore_topics = []
 def install_db():
     global dbcon
     try:
-        mongo_client = PypeMongoConnection.get_mongo_client()
+        mongo_client = OpenPypeMongoConnection.get_mongo_client()
         dbcon = mongo_client[database_name][collection_name]
     except pymongo.errors.AutoReconnect:
         log.error("Mongo server \"{}\" is not responding, exiting.".format(
-            PypeMongoConnection.get_default_mongo_url()
+            OpenPypeMongoConnection.get_default_mongo_url()
         ))
         sys.exit(0)
 

openpype/settings/handlers.py

@@ -170,18 +170,18 @@ class MongoSettingsHandler(SettingsHandler):
 
     def __init__(self):
         # Get mongo connection
-        from openpype.lib import PypeMongoConnection
+        from openpype.lib import OpenPypeMongoConnection
         from avalon.api import AvalonMongoDB
 
-        settings_collection = PypeMongoConnection.get_mongo_client()
+        settings_collection = OpenPypeMongoConnection.get_mongo_client()
 
         self._anatomy_keys = None
         self._attribute_keys = None
         # TODO prepare version of pype
         # - pype version should define how are settings saved and loaded
 
+        database_name = os.environ["OPENPYPE_DATABASE_NAME"]
         # TODO modify to not use hardcoded keys
-        database_name = "pype"
         collection_name = "settings"
 
         self.settings_collection = settings_collection
@@ -541,19 +541,19 @@ class MongoLocalSettingsHandler(LocalSettingsHandler):
     def __init__(self, local_site_id=None):
         # Get mongo connection
         from openpype.lib import (
-            PypeMongoConnection,
+            OpenPypeMongoConnection,
             get_local_site_id
         )
 
         if local_site_id is None:
             local_site_id = get_local_site_id()
-        settings_collection = PypeMongoConnection.get_mongo_client()
+        settings_collection = OpenPypeMongoConnection.get_mongo_client()
 
         # TODO prepare version of pype
         # - pype version should define how are settings saved and loaded
 
+        database_name = os.environ["OPENPYPE_DATABASE_NAME"]
         # TODO modify to not use hardcoded keys
-        database_name = "pype"
         collection_name = "settings"
 
         self.settings_collection = settings_collection

start.py

@@ -561,6 +561,7 @@ def boot():
         sys.exit(1)
 
     os.environ["OPENPYPE_MONGO"] = openpype_mongo
+    os.environ["OPENPYPE_DATABASE_NAME"] = "openpype"  # name of Pype database
 
     # ------------------------------------------------------------------------
     # Set environments - load OpenPype path from database (if set)

vendor/deadline/custom/events/OpenPype/OpenPype.param

@@ -28,8 +28,8 @@ Default=DEBUG
 Description=Logging level where printing will start.
 
 [OpenPypeExecutable]
-Type=MultiLineMultiFolder
-Label=Path to OpenPype executable dir
+Type=multilinemultifilename
+Label=Path to OpenPype executable
 Category=Job Plugins
 CategoryOrder=1
 CategoryIndex=1

vendor/deadline/custom/plugins/OpenPype/OpenPype.options

@@ -1,13 +1,3 @@
-[ScriptFile]
-Type=filename
-Label=Script File
-Category=Python Options
-CategoryOrder=0
-Index=0
-Description=The script file to be executed.
-Required=false
-DisableIfBlank=true
-
 [Arguments]
 Type=string
 Label=Arguments
@@ -18,24 +8,3 @@ Description=The arguments to pass to the script. If no arguments are required, l
 Required=false
 DisableIfBlank=true
 
-[Version]
-Type=enum
-Values=3.0
-Label=Version
-Category=Python Options
-CategoryOrder=0
-Index=2
-Description=The version of Python to use.
-Required=false
-DisableIfBlank=true
-
-[SingleFramesOnly]
-Type=boolean
-Label=Single Frames Only
-Category=Job Options
-CategoryOrder=1
-Index=0
-Description=If enabled, the plugin will only render one frame at a time even if a single task contains a chunk of frames.
-Required=true
-DisableIfBlank=true
-Default=false

vendor/deadline/custom/plugins/OpenPype/OpenPype.param

@@ -7,18 +7,9 @@ Index=0
 Default=OpenPype Plugin for Deadline
 Description=Not configurable
 
-[ConcurrentTasks]
-Type=label
-Label=ConcurrentTasks
-Category=About Plugin
-CategoryOrder=-1
-Index=0
-Default=True
-Description=Not configurable
-
-[OpenPype_Executable_3_0]
+[OpenPypeExecutable]
 Type=multilinemultifilename
-Label=OpenPype 3.0 Executable
+Label=OpenPype Executable
 Category=OpenPype Executables
 CategoryOrder=0
 Index=0

vendor/deadline/custom/plugins/OpenPype/OpenPype.py

@@ -53,14 +53,11 @@ class OpenPypeDeadlinePlugin(DeadlinePlugin):
             ".*Progress: (\d+)%.*").HandleCallback += self.HandleProgress
 
     def RenderExecutable(self):
-        version = self.GetPluginInfoEntry("Version")
-
-        exeList = self.GetConfigEntry(
-            "OpenPype_Executable_" + version.replace(".", "_"))
+        exeList = self.GetConfigEntry("OpenPypeExecutable")
         exe = FileUtils.SearchFileList(exeList)
         if exe == "":
             self.FailRender(
-                "OpenPype " + version + " executable was not found " +
+                "OpenPype executable was not found " +
                 "in the semicolon separated list \"" + exeList + "\". " +
                 "The path to the render executable can be configured " +
                 "from the Plugin Configuration in the Deadline Monitor.")

vendor/deadline/readme.md

@@ -1,3 +1,32 @@
 ## OpenPype Deadline repository overlay
 
- This directory is overlay for Deadline repository. It means that you can copy whole hierarchy to Deadline repository and it should work.
+ This directory is overlay for Deadline repository. 
+ It means that you can copy whole hierarchy to Deadline repository and it should work.
+ 
+ Logic:
+ -----
+ Event
+ -----
+ For each rendering job OpenPype event is triggered, it stores path to OpenPype
+ executable (needs to be configured on `Deadline's Configure Events > OpenPype`) 
+ job's extra key 'openpype_executables'.
+ 
+ This value is used by `GlobalJobPreLoad` to call that executable to pull
+ environment's variables which are needed to add to ALL plugins process environments.
+ These env. vars are injected into rendering process.
+ 
+ Event is necessary here as a middle man to allow configuring location of executable
+ which is ONLY then used by `GlobalJobPreLoad` (which doesnt have any user facing
+ configuration at all).
+ 
+ `GlobalJobPreLoad` is triggered before each job, it contains backward compatible
+ logic to not modify old Pype2 or not OpenPype triggered jobs.
+ 
+ Plugin
+ ------
+ For each publishing job `OpenPypeDeadlinePlugin` is called, which calls 
+ configured location of OpenPype executable (needs to be configured in 
+ `Deadline's Configure Plugins > OpenPype`) 
+ and triggers command.
+ 
+