From c4358e12b22c63a5485b9b2ad6552a8ec52655c3 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 31 Jan 2022 18:59:48 +0100 Subject: [PATCH 1/4] it is possible to tell that mongo url environment should be passed to job --- .../publish/collect_default_deadline_server.py | 4 ++++ .../publish/submit_aftereffects_deadline.py | 3 +++ .../plugins/publish/submit_harmony_deadline.py | 3 +++ .../publish/submit_houdini_remote_publish.py | 15 ++++++++++++--- .../publish/submit_houdini_render_deadline.py | 4 ++++ .../plugins/publish/submit_maya_deadline.py | 3 +++ .../plugins/publish/submit_nuke_deadline.py | 4 ++++ .../plugins/publish/submit_publish_job.py | 13 +++++++++---- 8 files changed, 42 insertions(+), 7 deletions(-) diff --git a/openpype/modules/default_modules/deadline/plugins/publish/collect_default_deadline_server.py b/openpype/modules/default_modules/deadline/plugins/publish/collect_default_deadline_server.py index 53231bd7e4..fc056342a8 100644 --- a/openpype/modules/default_modules/deadline/plugins/publish/collect_default_deadline_server.py +++ b/openpype/modules/default_modules/deadline/plugins/publish/collect_default_deadline_server.py @@ -9,6 +9,8 @@ class CollectDefaultDeadlineServer(pyblish.api.ContextPlugin): order = pyblish.api.CollectorOrder + 0.01 label = "Default Deadline Webservice" + pass_mongo_url = False + def process(self, context): try: deadline_module = context.data.get("openPypeModules")["deadline"] @@ -19,3 +21,5 @@ class CollectDefaultDeadlineServer(pyblish.api.ContextPlugin): # get default deadline webservice url from deadline module self.log.debug(deadline_module.deadline_urls) context.data["defaultDeadline"] = deadline_module.deadline_urls["default"] # noqa: E501 + + context.data["deadlinePassMongoUrl"] = self.pass_mongo_url diff --git a/openpype/modules/default_modules/deadline/plugins/publish/submit_aftereffects_deadline.py b/openpype/modules/default_modules/deadline/plugins/publish/submit_aftereffects_deadline.py index 69159fda1a..c5a691f8ae 100644 --- a/openpype/modules/default_modules/deadline/plugins/publish/submit_aftereffects_deadline.py +++ b/openpype/modules/default_modules/deadline/plugins/publish/submit_aftereffects_deadline.py @@ -67,6 +67,9 @@ class AfterEffectsSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline "OPENPYPE_DEV", "OPENPYPE_LOG_NO_COLORS" ] + # Add mongo url if it's enabled + if self._instance.context.get("deadlinePassMongoUrl"): + keys.append("OPENPYPE_MONGO") environment = dict({key: os.environ[key] for key in keys if key in os.environ}, **api.Session) diff --git a/openpype/modules/default_modules/deadline/plugins/publish/submit_harmony_deadline.py b/openpype/modules/default_modules/deadline/plugins/publish/submit_harmony_deadline.py index 37041a84b1..e52e76064e 100644 --- a/openpype/modules/default_modules/deadline/plugins/publish/submit_harmony_deadline.py +++ b/openpype/modules/default_modules/deadline/plugins/publish/submit_harmony_deadline.py @@ -276,6 +276,9 @@ class HarmonySubmitDeadline( "OPENPYPE_DEV", "OPENPYPE_LOG_NO_COLORS" ] + # Add mongo url if it's enabled + if self._instance.context.get("deadlinePassMongoUrl"): + keys.append("OPENPYPE_MONGO") environment = dict({key: os.environ[key] for key in keys if key in os.environ}, **api.Session) diff --git a/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_remote_publish.py b/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_remote_publish.py index c3228bfe52..199b7834b9 100644 --- a/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_remote_publish.py +++ b/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_remote_publish.py @@ -105,15 +105,21 @@ class HoudiniSubmitPublishDeadline(pyblish.api.ContextPlugin): # Clarify job name per submission (include instance name) payload["JobInfo"]["Name"] = job_name + " - %s" % instance self.submit_job( - payload, instances=[instance], deadline=AVALON_DEADLINE + context, + payload, + instances=[instance], + deadline=AVALON_DEADLINE ) else: # Submit a single job self.submit_job( - payload, instances=instance_names, deadline=AVALON_DEADLINE + context, + payload, + instances=instance_names, + deadline=AVALON_DEADLINE ) - def submit_job(self, payload, instances, deadline): + def submit_job(self, context, payload, instances, deadline): # Ensure we operate on a copy, a shallow copy is fine. payload = payload.copy() @@ -125,6 +131,9 @@ class HoudiniSubmitPublishDeadline(pyblish.api.ContextPlugin): # similar environment using it, e.g. "houdini17.5;pluginx2.3" "AVALON_TOOLS", ] + # Add mongo url if it's enabled + if context.get("deadlinePassMongoUrl"): + keys.append("OPENPYPE_MONGO") environment = dict( {key: os.environ[key] for key in keys if key in os.environ}, diff --git a/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_render_deadline.py b/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_render_deadline.py index fa146c0d30..2a8ed1653a 100644 --- a/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_render_deadline.py +++ b/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_render_deadline.py @@ -101,6 +101,10 @@ class HoudiniSubmitRenderDeadline(pyblish.api.InstancePlugin): # similar environment using it, e.g. "maya2018;vray4.x;yeti3.1.9" "AVALON_TOOLS", ] + # Add mongo url if it's enabled + if context.get("deadlinePassMongoUrl"): + keys.append("OPENPYPE_MONGO") + environment = dict({key: os.environ[key] for key in keys if key in os.environ}, **api.Session) diff --git a/openpype/modules/default_modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/default_modules/deadline/plugins/publish/submit_maya_deadline.py index 51a19e2aad..c5292a8dd0 100644 --- a/openpype/modules/default_modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/default_modules/deadline/plugins/publish/submit_maya_deadline.py @@ -498,6 +498,9 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): "OPENPYPE_DEV", "OPENPYPE_LOG_NO_COLORS" ] + # Add mongo url if it's enabled + if instance.context.get("deadlinePassMongoUrl"): + keys.append("OPENPYPE_MONGO") environment = dict({key: os.environ[key] for key in keys if key in os.environ}, **api.Session) diff --git a/openpype/modules/default_modules/deadline/plugins/publish/submit_nuke_deadline.py b/openpype/modules/default_modules/deadline/plugins/publish/submit_nuke_deadline.py index ae9cd985eb..5a4779de8a 100644 --- a/openpype/modules/default_modules/deadline/plugins/publish/submit_nuke_deadline.py +++ b/openpype/modules/default_modules/deadline/plugins/publish/submit_nuke_deadline.py @@ -249,6 +249,10 @@ class NukeSubmitDeadline(pyblish.api.InstancePlugin): "TOOL_ENV", "FOUNDRY_LICENSE" ] + # Add mongo url if it's enabled + if instance.context.get("deadlinePassMongoUrl"): + keys.append("OPENPYPE_MONGO") + # add allowed keys from preset if any if self.env_allowed_keys: keys += self.env_allowed_keys diff --git a/openpype/modules/default_modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/default_modules/deadline/plugins/publish/submit_publish_job.py index 516bd755d0..c9a8eedf30 100644 --- a/openpype/modules/default_modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/default_modules/deadline/plugins/publish/submit_publish_job.py @@ -227,6 +227,11 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): environment["OPENPYPE_USERNAME"] = instance.context.data["user"] environment["OPENPYPE_PUBLISH_JOB"] = "1" environment["OPENPYPE_RENDER_JOB"] = "0" + # Add mongo url if it's enabled + if instance.context.get("deadlinePassMongoUrl"): + mongo_url = os.environ.get("OPENPYPE_MONGO") + if mongo_url: + environment["OPENPYPE_MONGO"] = mongo_url args = [ 'publish', @@ -273,18 +278,18 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): else: payload["JobInfo"]["JobDependency0"] = job["_id"] - i = 0 - for index, key in enumerate(environment): + index = 0 + for key in environment: if key.upper() in self.enviro_filter: payload["JobInfo"].update( { "EnvironmentKeyValue%d" - % i: "{key}={value}".format( + % index: "{key}={value}".format( key=key, value=environment[key] ) } ) - i += 1 + index += 1 # remove secondary pool payload["JobInfo"].pop("SecondaryPool", None) From 9bfee3a404a2d2a0706863689795cb8754a0be5f Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 31 Jan 2022 19:07:30 +0100 Subject: [PATCH 2/4] adde settings defying if mongo url is passed --- .../defaults/project_settings/deadline.json | 3 +++ .../projects_schema/schema_project_deadline.json | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/openpype/settings/defaults/project_settings/deadline.json b/openpype/settings/defaults/project_settings/deadline.json index 9fb964b494..f4e46fcddc 100644 --- a/openpype/settings/defaults/project_settings/deadline.json +++ b/openpype/settings/defaults/project_settings/deadline.json @@ -1,6 +1,9 @@ { "deadline_servers": [], "publish": { + "CollectDefaultDeadlineServer": { + "pass_mongo_url": false + }, "ValidateExpectedFiles": { "enabled": true, "active": true, diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json b/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json index eb9eeb5448..7c44791160 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json @@ -17,6 +17,19 @@ "key": "publish", "label": "Publish plugins", "children": [ + { + "type": "dict", + "collapsible": true, + "key": "CollectDefaultDeadlineServer", + "label": "Default Deadline Webservice", + "children": [ + { + "type": "boolean", + "key": "pass_mongo_url", + "label": "Pass Mongo url to job" + } + ] + }, { "type": "dict", "collapsible": true, From cc84e86f7f3b8412977ae1a43caac0228107eff6 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 7 Feb 2022 13:03:06 +0100 Subject: [PATCH 3/4] Fix - should be context.data --- .../deadline/plugins/publish/submit_aftereffects_deadline.py | 2 +- .../deadline/plugins/publish/submit_harmony_deadline.py | 2 +- .../deadline/plugins/publish/submit_houdini_remote_publish.py | 2 +- .../deadline/plugins/publish/submit_houdini_render_deadline.py | 2 +- .../deadline/plugins/publish/submit_maya_deadline.py | 2 +- .../deadline/plugins/publish/submit_nuke_deadline.py | 2 +- .../deadline/plugins/publish/submit_publish_job.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/openpype/modules/default_modules/deadline/plugins/publish/submit_aftereffects_deadline.py b/openpype/modules/default_modules/deadline/plugins/publish/submit_aftereffects_deadline.py index c5a691f8ae..a43e38ff94 100644 --- a/openpype/modules/default_modules/deadline/plugins/publish/submit_aftereffects_deadline.py +++ b/openpype/modules/default_modules/deadline/plugins/publish/submit_aftereffects_deadline.py @@ -68,7 +68,7 @@ class AfterEffectsSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline "OPENPYPE_LOG_NO_COLORS" ] # Add mongo url if it's enabled - if self._instance.context.get("deadlinePassMongoUrl"): + if self._instance.context.data.get("deadlinePassMongoUrl"): keys.append("OPENPYPE_MONGO") environment = dict({key: os.environ[key] for key in keys diff --git a/openpype/modules/default_modules/deadline/plugins/publish/submit_harmony_deadline.py b/openpype/modules/default_modules/deadline/plugins/publish/submit_harmony_deadline.py index e52e76064e..6683c34dbd 100644 --- a/openpype/modules/default_modules/deadline/plugins/publish/submit_harmony_deadline.py +++ b/openpype/modules/default_modules/deadline/plugins/publish/submit_harmony_deadline.py @@ -277,7 +277,7 @@ class HarmonySubmitDeadline( "OPENPYPE_LOG_NO_COLORS" ] # Add mongo url if it's enabled - if self._instance.context.get("deadlinePassMongoUrl"): + if self._instance.context.data.get("deadlinePassMongoUrl"): keys.append("OPENPYPE_MONGO") environment = dict({key: os.environ[key] for key in keys diff --git a/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_remote_publish.py b/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_remote_publish.py index 199b7834b9..c683eb68a8 100644 --- a/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_remote_publish.py +++ b/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_remote_publish.py @@ -132,7 +132,7 @@ class HoudiniSubmitPublishDeadline(pyblish.api.ContextPlugin): "AVALON_TOOLS", ] # Add mongo url if it's enabled - if context.get("deadlinePassMongoUrl"): + if context.data.get("deadlinePassMongoUrl"): keys.append("OPENPYPE_MONGO") environment = dict( diff --git a/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_render_deadline.py b/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_render_deadline.py index 2a8ed1653a..2cd6b0e6b0 100644 --- a/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_render_deadline.py +++ b/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_render_deadline.py @@ -102,7 +102,7 @@ class HoudiniSubmitRenderDeadline(pyblish.api.InstancePlugin): "AVALON_TOOLS", ] # Add mongo url if it's enabled - if context.get("deadlinePassMongoUrl"): + if context.data.get("deadlinePassMongoUrl"): keys.append("OPENPYPE_MONGO") environment = dict({key: os.environ[key] for key in keys diff --git a/openpype/modules/default_modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/default_modules/deadline/plugins/publish/submit_maya_deadline.py index c5292a8dd0..a87c0c374a 100644 --- a/openpype/modules/default_modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/default_modules/deadline/plugins/publish/submit_maya_deadline.py @@ -499,7 +499,7 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): "OPENPYPE_LOG_NO_COLORS" ] # Add mongo url if it's enabled - if instance.context.get("deadlinePassMongoUrl"): + if instance.context.data.get("deadlinePassMongoUrl"): keys.append("OPENPYPE_MONGO") environment = dict({key: os.environ[key] for key in keys diff --git a/openpype/modules/default_modules/deadline/plugins/publish/submit_nuke_deadline.py b/openpype/modules/default_modules/deadline/plugins/publish/submit_nuke_deadline.py index 5a4779de8a..d6bd11620d 100644 --- a/openpype/modules/default_modules/deadline/plugins/publish/submit_nuke_deadline.py +++ b/openpype/modules/default_modules/deadline/plugins/publish/submit_nuke_deadline.py @@ -250,7 +250,7 @@ class NukeSubmitDeadline(pyblish.api.InstancePlugin): "FOUNDRY_LICENSE" ] # Add mongo url if it's enabled - if instance.context.get("deadlinePassMongoUrl"): + if instance.context.data.get("deadlinePassMongoUrl"): keys.append("OPENPYPE_MONGO") # add allowed keys from preset if any diff --git a/openpype/modules/default_modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/default_modules/deadline/plugins/publish/submit_publish_job.py index c9a8eedf30..f165545caf 100644 --- a/openpype/modules/default_modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/default_modules/deadline/plugins/publish/submit_publish_job.py @@ -228,7 +228,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): environment["OPENPYPE_PUBLISH_JOB"] = "1" environment["OPENPYPE_RENDER_JOB"] = "0" # Add mongo url if it's enabled - if instance.context.get("deadlinePassMongoUrl"): + if instance.context.data.get("deadlinePassMongoUrl"): mongo_url = os.environ.get("OPENPYPE_MONGO") if mongo_url: environment["OPENPYPE_MONGO"] = mongo_url From 9d1468de8b58af480e1e41821456ef490a56ae78 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 7 Feb 2022 13:13:44 +0100 Subject: [PATCH 4/4] Fix - context must be pushed through with RenderInstance Submission to DL checks context for flag for pushing through OPENPYPE_MONGO --- openpype/hosts/aftereffects/plugins/publish/collect_render.py | 2 +- openpype/hosts/harmony/plugins/publish/collect_farm_render.py | 1 + openpype/lib/abstract_collect_render.py | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/aftereffects/plugins/publish/collect_render.py b/openpype/hosts/aftereffects/plugins/publish/collect_render.py index 74d38751e1..3c26312b38 100644 --- a/openpype/hosts/aftereffects/plugins/publish/collect_render.py +++ b/openpype/hosts/aftereffects/plugins/publish/collect_render.py @@ -118,6 +118,7 @@ class CollectAERender(abstract_collect_render.AbstractCollectRender): instance.anatomyData = context.data["anatomyData"] instance.outputDir = self._get_output_dir(instance) + instance.context = context settings = get_project_settings(os.getenv("AVALON_PROJECT")) reviewable_subset_filter = \ @@ -142,7 +143,6 @@ class CollectAERender(abstract_collect_render.AbstractCollectRender): break self.log.info("New instance:: {}".format(instance)) - instances.append(instance) return instances diff --git a/openpype/hosts/harmony/plugins/publish/collect_farm_render.py b/openpype/hosts/harmony/plugins/publish/collect_farm_render.py index 31a249591e..4284d11235 100644 --- a/openpype/hosts/harmony/plugins/publish/collect_farm_render.py +++ b/openpype/hosts/harmony/plugins/publish/collect_farm_render.py @@ -176,6 +176,7 @@ class CollectFarmRender(openpype.lib.abstract_collect_render. ignoreFrameHandleCheck=True ) + render_instance.context = context self.log.debug(render_instance) instances.append(render_instance) diff --git a/openpype/lib/abstract_collect_render.py b/openpype/lib/abstract_collect_render.py index d9c8a0993d..3839aad45d 100644 --- a/openpype/lib/abstract_collect_render.py +++ b/openpype/lib/abstract_collect_render.py @@ -76,6 +76,7 @@ class RenderInstance(object): deadlineSubmissionJob = attr.ib(default=None) anatomyData = attr.ib(default=None) outputDir = attr.ib(default=None) + context = attr.ib(default=None) @frameStart.validator def check_frame_start(self, _, value):