resolve conflict

openpype/modules/deadline/plugins/publish/collect_deadline_server_from_instance.py

@@ -5,8 +5,6 @@ This is resolving index of server lists stored in `deadlineServers` instance
 attribute or using default server if that attribute doesn't exists.
 
 """
-from maya import cmds
-
 import pyblish.api
 from openpype.pipeline.publish import KnownPublishError
 
@@ -44,7 +42,8 @@ class CollectDeadlineServerFromInstance(pyblish.api.InstancePlugin):
             str: Selected Deadline Webservice URL.
 
         """
-
+        # Not all hosts can import this module.
+        from maya import cmds
         deadline_settings = (
             render_instance.context.data
             ["system_settings"]

openpype/modules/deadline/plugins/publish/submit_blender_deadline.py

@@ -0,0 +1,182 @@
+# -*- coding: utf-8 -*-
+"""Submitting render job to Deadline."""
+
+import os
+import getpass
+import attr
+from datetime import datetime
+
+from openpype.lib import is_running_from_build
+from openpype.pipeline import legacy_io
+from openpype.pipeline.farm.tools import iter_expected_files
+from openpype.tests.lib import is_in_tests
+
+from openpype_modules.deadline import abstract_submit_deadline
+from openpype_modules.deadline.abstract_submit_deadline import DeadlineJobInfo
+
+
+@attr.s
+class BlenderPluginInfo():
+    SceneFile = attr.ib(default=None)   # Input
+    Version = attr.ib(default=None)  # Mandatory for Deadline
+    SaveFile = attr.ib(default=True)
+
+
+class BlenderSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline):
+    label = "Submit Render to Deadline"
+    hosts = ["blender"]
+    families = ["render.farm"]
+
+    use_published = True
+    priority = 50
+    chunk_size = 1
+    jobInfo = {}
+    pluginInfo = {}
+    group = None
+
+    def get_job_info(self):
+        job_info = DeadlineJobInfo(Plugin="Blender")
+
+        job_info.update(self.jobInfo)
+
+        instance = self._instance
+        context = instance.context
+
+        # Always use the original work file name for the Job name even when
+        # rendering is done from the published Work File. The original work
+        # file name is clearer because it can also have subversion strings,
+        # etc. which are stripped for the published file.
+        src_filepath = context.data["currentFile"]
+        src_filename = os.path.basename(src_filepath)
+
+        if is_in_tests():
+            src_filename += datetime.now().strftime("%d%m%Y%H%M%S")
+
+        job_info.Name = f"{src_filename} - {instance.name}"
+        job_info.BatchName = src_filename
+        instance.data.get("blenderRenderPlugin", "Blender")
+        job_info.UserName = context.data.get("deadlineUser", getpass.getuser())
+
+        # Deadline requires integers in frame range
+        frames = "{start}-{end}x{step}".format(
+            start=int(instance.data["frameStartHandle"]),
+            end=int(instance.data["frameEndHandle"]),
+            step=int(instance.data["byFrameStep"]),
+        )
+        job_info.Frames = frames
+
+        job_info.Pool = instance.data.get("primaryPool")
+        job_info.SecondaryPool = instance.data.get("secondaryPool")
+        job_info.Comment = context.data.get("comment")
+        job_info.Priority = instance.data.get("priority", self.priority)
+
+        if self.group != "none" and self.group:
+            job_info.Group = self.group
+
+        attr_values = self.get_attr_values_from_data(instance.data)
+        render_globals = instance.data.setdefault("renderGlobals", {})
+        machine_list = attr_values.get("machineList", "")
+        if machine_list:
+            if attr_values.get("whitelist", True):
+                machine_list_key = "Whitelist"
+            else:
+                machine_list_key = "Blacklist"
+            render_globals[machine_list_key] = machine_list
+
+        job_info.Priority = attr_values.get("priority")
+        job_info.ChunkSize = attr_values.get("chunkSize")
+
+        # Add options from RenderGlobals
+        render_globals = instance.data.get("renderGlobals", {})
+        job_info.update(render_globals)
+
+        keys = [
+            "FTRACK_API_KEY",
+            "FTRACK_API_USER",
+            "FTRACK_SERVER",
+            "OPENPYPE_SG_USER",
+            "AVALON_PROJECT",
+            "AVALON_ASSET",
+            "AVALON_TASK",
+            "AVALON_APP_NAME",
+            "OPENPYPE_DEV"
+            "IS_TEST"
+        ]
+
+        # Add OpenPype version if we are running from build.
+        if is_running_from_build():
+            keys.append("OPENPYPE_VERSION")
+
+        # Add mongo url if it's enabled
+        if self._instance.context.data.get("deadlinePassMongoUrl"):
+            keys.append("OPENPYPE_MONGO")
+
+        environment = dict({key: os.environ[key] for key in keys
+                            if key in os.environ}, **legacy_io.Session)
+
+        for key in keys:
+            value = environment.get(key)
+            if not value:
+                continue
+            job_info.EnvironmentKeyValue[key] = value
+
+        # to recognize job from PYPE for turning Event On/Off
+        job_info.add_render_job_env_var()
+        job_info.EnvironmentKeyValue["OPENPYPE_LOG_NO_COLORS"] = "1"
+
+        # Adding file dependencies.
+        if self.asset_dependencies:
+            dependencies = instance.context.data["fileDependencies"]
+            for dependency in dependencies:
+                job_info.AssetDependency += dependency
+
+        # Add list of expected files to job
+        # ---------------------------------
+        exp = instance.data.get("expectedFiles")
+        for filepath in iter_expected_files(exp):
+            job_info.OutputDirectory += os.path.dirname(filepath)
+            job_info.OutputFilename += os.path.basename(filepath)
+
+        return job_info
+
+    def get_plugin_info(self):
+        # Not all hosts can import this module.
+        import bpy
+
+        plugin_info = BlenderPluginInfo(
+            SceneFile=self.scene_path,
+            Version=bpy.app.version_string,
+            SaveFile=True,
+        )
+
+        plugin_payload = attr.asdict(plugin_info)
+
+        # Patching with pluginInfo from settings
+        for key, value in self.pluginInfo.items():
+            plugin_payload[key] = value
+
+        return plugin_payload
+
+    def process_submission(self):
+        instance = self._instance
+
+        expected_files = instance.data["expectedFiles"]
+        if not expected_files:
+            raise RuntimeError("No Render Elements found!")
+
+        first_file = next(iter_expected_files(expected_files))
+        output_dir = os.path.dirname(first_file)
+        instance.data["outputDir"] = output_dir
+        instance.data["toBeRenderedOn"] = "deadline"
+
+        payload = self.assemble_payload()
+        return self.submit(payload)
+
+    def from_published_scene(self):
+        """
+        This is needed to set the correct path for the json metadata. Because
+        the rendering path is set in the blend file during the collection,
+        and the path is adjusted to use the published scene, this ensures that
+        the metadata and the rendered files are in the same location.
+        """
+        return super().from_published_scene(False)

openpype/modules/deadline/plugins/publish/submit_fusion_deadline.py

@@ -6,13 +6,15 @@ import requests
 
 import pyblish.api
 
+from openpype import AYON_SERVER_ENABLED
 from openpype.pipeline import legacy_io
 from openpype.pipeline.publish import (
     OpenPypePyblishPluginMixin
 )
 from openpype.lib import (
     BoolDef,
-    NumberDef
+    NumberDef,
+    is_running_from_build
 )
 
 
@@ -34,6 +36,8 @@ class FusionSubmitDeadline(
     targets = ["local"]
 
     # presets
+    plugin = None
+
     priority = 50
     chunk_size = 1
     concurrent_tasks = 1
@@ -173,7 +177,7 @@ class FusionSubmitDeadline(
                 "SecondaryPool": instance.data.get("secondaryPool"),
                 "Group": self.group,
 
-                "Plugin": "Fusion",
+                "Plugin": self.plugin,
                 "Frames": "{start}-{end}".format(
                     start=int(instance.data["frameStartHandle"]),
                     end=int(instance.data["frameEndHandle"])
@@ -216,16 +220,34 @@ class FusionSubmitDeadline(
 
         # Include critical variables with submission
         keys = [
-            # TODO: This won't work if the slaves don't have access to
-            # these paths, such as if slaves are running Linux and the
-            # submitter is on Windows.
-            "PYTHONPATH",
-            "OFX_PLUGIN_PATH",
-            "FUSION9_MasterPrefs"
+            "FTRACK_API_KEY",
+            "FTRACK_API_USER",
+            "FTRACK_SERVER",
+            "AVALON_PROJECT",
+            "AVALON_ASSET",
+            "AVALON_TASK",
+            "AVALON_APP_NAME",
+            "OPENPYPE_DEV",
+            "OPENPYPE_LOG_NO_COLORS",
+            "IS_TEST"
         ]
+
+        # Add OpenPype version if we are running from build.
+        if is_running_from_build():
+            keys.append("OPENPYPE_VERSION")
+
         environment = dict({key: os.environ[key] for key in keys
                             if key in os.environ}, **legacy_io.Session)
 
+        # to recognize render jobs
+        if AYON_SERVER_ENABLED:
+            environment["AYON_BUNDLE_NAME"] = os.environ["AYON_BUNDLE_NAME"]
+            render_job_label = "AYON_RENDER_JOB"
+        else:
+            render_job_label = "OPENPYPE_RENDER_JOB"
+
+        environment[render_job_label] = "1"
+
         payload["JobInfo"].update({
             "EnvironmentKeyValue%d" % index: "{key}={value}".format(
                 key=key,

openpype/modules/deadline/plugins/publish/submit_houdini_remote_publish.py

@@ -3,7 +3,6 @@ import json
 from datetime import datetime
 
 import requests
-import hou
 
 import pyblish.api
 
@@ -31,6 +30,8 @@ class HoudiniSubmitPublishDeadline(pyblish.api.ContextPlugin):
     targets = ["deadline"]
 
     def process(self, context):
+        # Not all hosts can import this module.
+        import hou
 
         # Ensure no errors so far
         assert all(

openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py

@@ -1,9 +1,8 @@
-import hou
-
 import os
 import attr
 import getpass
 from datetime import datetime
+
 import pyblish.api
 
 from openpype.pipeline import legacy_io
@@ -65,9 +64,11 @@ class HoudiniSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline):
             job_info.BatchName += datetime.now().strftime("%d%m%Y%H%M%S")
 
         # Deadline requires integers in frame range
+        start = instance.data["frameStartHandle"]
+        end = instance.data["frameEndHandle"]
         frames = "{start}-{end}x{step}".format(
-            start=int(instance.data["frameStart"]),
-            end=int(instance.data["frameEnd"]),
+            start=int(start),
+            end=int(end),
             step=int(instance.data["byFrameStep"]),
         )
         job_info.Frames = frames
@@ -117,6 +118,8 @@ class HoudiniSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline):
         return job_info
 
     def get_plugin_info(self):
+        # Not all hosts can import this module.
+        import hou
 
         instance = self._instance
         context = instance.context

openpype/modules/deadline/plugins/publish/submit_max_deadline.py

@@ -1,8 +1,8 @@
 import os
 import getpass
 import copy
-
 import attr
+
 from openpype.lib import (
     TextDef,
     BoolDef,
@@ -15,11 +15,6 @@ from openpype.pipeline import (
 from openpype.pipeline.publish.lib import (
     replace_with_published_scene_path
 )
-from openpype.hosts.max.api.lib import (
-    get_current_renderer,
-    get_multipass_setting
-)
-from openpype.hosts.max.api.lib_rendersettings import RenderSettings
 from openpype_modules.deadline import abstract_submit_deadline
 from openpype_modules.deadline.abstract_submit_deadline import DeadlineJobInfo
 from openpype.lib import is_running_from_build
@@ -191,6 +186,13 @@ class MaxSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline,
         self.submit(self.assemble_payload(job_info, plugin_info))
 
     def _use_published_name(self, data, project_settings):
+        # Not all hosts can import these modules.
+        from openpype.hosts.max.api.lib import (
+            get_current_renderer,
+            get_multipass_setting
+        )
+        from openpype.hosts.max.api.lib_rendersettings import RenderSettings
+
         instance = self._instance
         job_info = copy.deepcopy(self.job_info)
         plugin_info = copy.deepcopy(self.plugin_info)
@@ -238,9 +240,10 @@ class MaxSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline,
             plugin_data["redshift_SeparateAovFiles"] = instance.data.get(
                 "separateAovFiles")
         if instance.data["cameras"]:
-            plugin_info["Camera0"] = None
-            plugin_info["Camera"] = instance.data["cameras"][0]
-            plugin_info["Camera1"] = instance.data["cameras"][0]
+            camera = instance.data["cameras"][0]
+            plugin_info["Camera0"] = camera
+            plugin_info["Camera"] = camera
+            plugin_info["Camera1"] = camera
         self.log.debug("plugin data:{}".format(plugin_data))
         plugin_info.update(plugin_data)
 

openpype/modules/deadline/plugins/publish/submit_maya_deadline.py

@@ -28,8 +28,6 @@ from collections import OrderedDict
 
 import attr
 
-from maya import cmds
-
 from openpype.pipeline import (
     legacy_io,
     OpenPypePyblishPluginMixin
@@ -134,6 +132,8 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline,
         cls.group = settings.get("group", cls.group)
         cls.strict_error_checking = settings.get("strict_error_checking",
                                                  cls.strict_error_checking)
+        cls.jobInfo = settings.get("jobInfo", cls.jobInfo)
+        cls.pluginInfo = settings.get("pluginInfo", cls.pluginInfo)
 
     def get_job_info(self):
         job_info = DeadlineJobInfo(Plugin="MayaBatch")
@@ -246,6 +246,8 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline,
         return job_info
 
     def get_plugin_info(self):
+        # Not all hosts can import this module.
+        from maya import cmds
 
         instance = self._instance
         context = instance.context
@@ -288,7 +290,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline,
         return plugin_payload
 
     def process_submission(self):
-
+        from maya import cmds
         instance = self._instance
 
         filepath = self.scene_path  # publish if `use_publish` else workfile
@@ -648,7 +650,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline,
         return job_info, attr.asdict(plugin_info)
 
     def _get_arnold_render_payload(self, data):
-
+        from maya import cmds
         # Job Info
         job_info = copy.deepcopy(self.job_info)
         job_info.Name = self._job_info_label("Render")
@@ -675,7 +677,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline,
             str
 
         """
-
+        from maya import cmds
         # "vrayscene/<Scene>/<Scene>_<Layer>/<Layer>"
         vray_settings = cmds.ls(type="VRaySettingsNode")
         node = vray_settings[0]

openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py

@@ -2,8 +2,6 @@ import os
 import attr
 from datetime import datetime
 
-from maya import cmds
-
 from openpype import AYON_SERVER_ENABLED
 from openpype.pipeline import legacy_io, PublishXmlValidationError
 from openpype.tests.lib import is_in_tests
@@ -127,7 +125,8 @@ class MayaSubmitRemotePublishDeadline(
             job_info.EnvironmentKeyValue[key] = value
 
     def get_plugin_info(self):
-
+        # Not all hosts can import this module.
+        from maya import cmds
         scene = self._instance.context.data["currentFile"]
 
         plugin_info = MayaPluginInfo()

openpype/modules/deadline/plugins/publish/submit_nuke_deadline.py

@@ -7,8 +7,6 @@ from datetime import datetime
 import requests
 import pyblish.api
 
-import nuke
-
 from openpype import AYON_SERVER_ENABLED
 from openpype.pipeline import legacy_io
 from openpype.pipeline.publish import (
@@ -48,6 +46,7 @@ class NukeSubmitDeadline(pyblish.api.InstancePlugin,
     use_gpu = False
     env_allowed_keys = []
     env_search_replace_values = {}
+    workfile_dependency = True
 
     @classmethod
     def get_attribute_defs(cls):
@@ -83,6 +82,11 @@ class NukeSubmitDeadline(pyblish.api.InstancePlugin,
                 "suspend_publish",
                 default=False,
                 label="Suspend publish"
+            ),
+            BoolDef(
+                "workfile_dependency",
+                default=True,
+                label="Workfile Dependency"
             )
         ]
 
@@ -313,6 +317,13 @@ class NukeSubmitDeadline(pyblish.api.InstancePlugin,
             "AuxFiles": []
         }
 
+        # Add workfile dependency.
+        workfile_dependency = instance.data["attributeValues"].get(
+            "workfile_dependency", self.workfile_dependency
+        )
+        if workfile_dependency:
+            payload["JobInfo"].update({"AssetDependency0": script_path})
+
         # TODO: rewrite for baking with sequences
         if baking_submission:
             payload["JobInfo"].update({
@@ -485,6 +496,9 @@ class NukeSubmitDeadline(pyblish.api.InstancePlugin,
         Returning:
             list: captured groups list
         """
+        # Not all hosts can import this module.
+        import nuke
+
         captured_groups = []
         for lg_name, list_node_class in self.limit_groups.items():
             for node_class in list_node_class:

openpype/modules/deadline/plugins/publish/submit_publish_job.py

@@ -96,7 +96,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin,
     targets = ["local"]
 
     hosts = ["fusion", "max", "maya", "nuke", "houdini",
-             "celaction", "aftereffects", "harmony"]
+             "celaction", "aftereffects", "harmony", "blender"]
 
     families = ["render.farm", "render.frames_farm",
                 "prerender.farm", "prerender.frames_farm",
@@ -107,6 +107,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin,
                 "redshift_rop"]
 
     aov_filter = {"maya": [r".*([Bb]eauty).*"],
+                  "blender": [r".*([Bb]eauty).*"],
                   "aftereffects": [r".*"],  # for everything from AE
                   "harmony": [r".*"],  # for everything from AE
                   "celaction": [r".*"],
@@ -707,6 +708,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin,
         """
 
         project_name = context.data["projectName"]
+        host_name = context.data["hostName"]
         if not version:
             version = get_last_version_by_subset_name(
                 project_name,
@@ -718,7 +720,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin,
             else:
                 version = get_versioning_start(
                     project_name,
-                    template_data["app"],
+                    host_name,
                     task_name=template_data["task"]["name"],
                     task_type=template_data["task"]["type"],
                     family="render",

openpype/modules/deadline/repository/custom/plugins/Ayon/Ayon.py

@@ -85,7 +85,7 @@ class AyonDeadlinePlugin(DeadlinePlugin):
         }
 
         for env, val in environment.items():
-            self.SetProcessEnvironmentVariable(env, val)
+            self.SetEnvironmentVariable(env, val)
 
         exe_list = self.GetConfigEntry("AyonExecutable")
         # clean '\ ' for MacOS pasting
@@ -96,16 +96,16 @@ class AyonDeadlinePlugin(DeadlinePlugin):
         for path in exe_list.split(";"):
             if path.startswith("~"):
                 path = os.path.expanduser(path)
-                expanded_paths.append(path)
+            expanded_paths.append(path)
         exe = FileUtils.SearchFileList(";".join(expanded_paths))
 
         if exe == "":
             self.FailRender(
-                "Ayon executable was not found " +
-                "in the semicolon separated list " +
-                "\"" + ";".join(exe_list) + "\". " +
-                "The path to the render executable can be configured " +
-                "from the Plugin Configuration in the Deadline Monitor.")
+                "Ayon executable was not found in the semicolon separated "
+                "list: \"{}\". The path to the render executable can be "
+                "configured from the Plugin Configuration in the Deadline "
+                "Monitor.".format(exe_list)
+            )
         return exe
 
     def RenderArgument(self):

openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py

@@ -495,7 +495,10 @@ def inject_ayon_environment(deadlinePlugin):
             "AYON_BUNDLE_NAME": ayon_bundle_name,
         }
         for env, val in environment.items():
+            # Add the env var for the Render Plugin that is about to render
             deadlinePlugin.SetEnvironmentVariable(env, val)
+            # Add the env var for current calls to `DeadlinePlugin.RunProcess`
+            deadlinePlugin.SetProcessEnvironmentVariable(env, val)
 
         args_str = subprocess.list2cmdline(args)
         print(">>> Executing: {} {}".format(exe, args_str))