From 6a3992a3e11b949a5948319f78b403314ac83d87 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Thu, 5 Aug 2021 17:54:58 +0200 Subject: [PATCH 1/8] =?UTF-8?q?add=20support=20for=20multiple=20deadline?= =?UTF-8?q?=20servers=20=F0=9F=96=A5=EF=B8=8F=F0=9F=96=A5=EF=B8=8F?= =?UTF-8?q?=F0=9F=96=A5=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../maya/plugins/create/create_render.py | 172 ++++++++++++------ openpype/modules/deadline/deadline_module.py | 10 +- .../plugins/publish/submit_maya_deadline.py | 16 +- .../plugins/publish/submit_nuke_deadline.py | 16 +- .../plugins/publish/submit_publish_job.py | 23 ++- .../publish/validate_deadline_connection.py | 27 +-- .../validate_expected_and_rendered_files.py | 17 +- .../defaults/project_settings/deadline.json | 1 + .../defaults/project_settings/maya.json | 12 +- .../defaults/project_settings/unreal.json | 3 +- .../defaults/system_settings/modules.json | 4 +- openpype/settings/entities/__init__.py | 4 +- openpype/settings/entities/enum_entity.py | 51 ++++++ .../schema_project_deadline.json | 6 + .../schemas/schema_maya_create.json | 24 ++- .../schemas/system_schema/schema_modules.json | 8 +- 16 files changed, 274 insertions(+), 120 deletions(-) diff --git a/openpype/hosts/maya/plugins/create/create_render.py b/openpype/hosts/maya/plugins/create/create_render.py index cbca091365..76cac5fe25 100644 --- a/openpype/hosts/maya/plugins/create/create_render.py +++ b/openpype/hosts/maya/plugins/create/create_render.py @@ -4,6 +4,7 @@ import os import json import appdirs import requests +import six from maya import cmds import maya.app.renderSetup.model.renderSetup as renderSetup @@ -12,7 +13,13 @@ from openpype.hosts.maya.api import ( lib, plugin ) -from openpype.api import (get_system_settings, get_asset) +from openpype.api import ( + get_system_settings, + get_project_settings, + get_asset) +from openpype.modules import ModulesManager + +from avalon.api import Session class CreateRender(plugin.Creator): @@ -83,6 +90,19 @@ class CreateRender(plugin.Creator): def __init__(self, *args, **kwargs): """Constructor.""" super(CreateRender, self).__init__(*args, **kwargs) + project_settings = get_project_settings(Session["AVALON_PROJECT"]) + try: + self.deadline_servers = ( + project_settings["deadline"] + ["deadline_servers"] + ) + except AttributeError: + # Handle situation were we had only one url for deadline. + manager = ModulesManager() + deadline_module = manager.modules_by_name["deadline"] + # get default deadline webservice url from deadline module + deadline_url = deadline_module.deadline_url + self.deadline_servers = {"default": deadline_url} def process(self): """Entry point.""" @@ -94,10 +114,10 @@ class CreateRender(plugin.Creator): use_selection = self.options.get("useSelection") with lib.undo_chunk(): self._create_render_settings() - instance = super(CreateRender, self).process() + self.instance = super(CreateRender, self).process() # create namespace with instance index = 1 - namespace_name = "_{}".format(str(instance)) + namespace_name = "_{}".format(str(self.instance)) try: cmds.namespace(rm=namespace_name) except RuntimeError: @@ -105,12 +125,19 @@ class CreateRender(plugin.Creator): pass while cmds.namespace(exists=namespace_name): - namespace_name = "_{}{}".format(str(instance), index) + namespace_name = "_{}{}".format(str(self.instance), index) index += 1 namespace = cmds.namespace(add=namespace_name) - cmds.setAttr("{}.machineList".format(instance), lock=True) + # add Deadline server selection list + cmds.scriptJob( + attributeChange=[ + "{}.deadlineServers".format(self.instance), + self._deadline_webservice_changed + ]) + + cmds.setAttr("{}.machineList".format(self.instance), lock=True) self._rs = renderSetup.instance() layers = self._rs.getRenderLayers() if use_selection: @@ -122,7 +149,7 @@ class CreateRender(plugin.Creator): render_set = cmds.sets( n="{}:{}".format(namespace, layer.name())) sets.append(render_set) - cmds.sets(sets, forceElement=instance) + cmds.sets(sets, forceElement=self.instance) # if no render layers are present, create default one with # asterisk selector @@ -138,62 +165,56 @@ class CreateRender(plugin.Creator): renderer = 'renderman' self._set_default_renderer_settings(renderer) + return self.instance + + def _deadline_webservice_changed(self): + """Refresh Deadline server dependent options.""" + # get selected server + webservice = self.deadline_servers[ + self.server_aliases[ + cmds.getAttr("{}.deadlineServers".format(self.instance)) + ] + ] + pools = self._get_deadline_pools(webservice) + cmds.deleteAttr("{}.primaryPool".format(self.instance)) + cmds.deleteAttr("{}.secondaryPool".format(self.instance)) + cmds.addAttr(self.instance, longName="primaryPool", + attributeType="enum", + enumName=":".join(pools)) + cmds.addAttr(self.instance, longName="secondaryPool", + attributeType="enum", + enumName=":".join(["-"] + pools)) + + def _get_deadline_pools(self, webservice): + # type: (str) -> list + """Get pools from Deadline. + Args: + webservice (str): Server url. + Returns: + list: Pools. + Throws: + RuntimeError: If deadline webservice is unreachable. + + """ + argument = "{}/api/pools?NamesOnly=true".format(webservice) + try: + response = self._requests_get(argument) + except requests.exceptions.ConnectionError as exc: + msg = 'Cannot connect to deadline web service' + self.log.error(msg) + six.reraise(exc, RuntimeError('{} - {}'.format(msg, exc))) + if not response.ok: + self.log.warning("No pools retrieved") + return [] + + return response.json() def _create_render_settings(self): # get pools - pools = [] - - system_settings = get_system_settings()["modules"] - - deadline_enabled = system_settings["deadline"]["enabled"] - muster_enabled = system_settings["muster"]["enabled"] - deadline_url = system_settings["deadline"]["DEADLINE_REST_URL"] - muster_url = system_settings["muster"]["MUSTER_REST_URL"] - - if deadline_enabled and muster_enabled: - self.log.error( - "Both Deadline and Muster are enabled. " "Cannot support both." - ) - raise RuntimeError("Both Deadline and Muster are enabled") - - if deadline_enabled: - argument = "{}/api/pools?NamesOnly=true".format(deadline_url) - try: - response = self._requests_get(argument) - except requests.exceptions.ConnectionError as e: - msg = 'Cannot connect to deadline web service' - self.log.error(msg) - raise RuntimeError('{} - {}'.format(msg, e)) - if not response.ok: - self.log.warning("No pools retrieved") - else: - pools = response.json() - self.data["primaryPool"] = pools - # We add a string "-" to allow the user to not - # set any secondary pools - self.data["secondaryPool"] = ["-"] + pools - - if muster_enabled: - self.log.info(">>> Loading Muster credentials ...") - self._load_credentials() - self.log.info(">>> Getting pools ...") - try: - pools = self._get_muster_pools() - except requests.exceptions.HTTPError as e: - if e.startswith("401"): - self.log.warning("access token expired") - self._show_login() - raise RuntimeError("Access token expired") - except requests.exceptions.ConnectionError: - self.log.error("Cannot connect to Muster API endpoint.") - raise RuntimeError("Cannot connect to {}".format(muster_url)) - pool_names = [] - for pool in pools: - self.log.info(" - pool: {}".format(pool["name"])) - pool_names.append(pool["name"]) - - self.data["primaryPool"] = pool_names + pool_names = [] + self.server_aliases = self.deadline_servers.keys() + self.data["deadlineServers"] = self.server_aliases self.data["suspendPublishJob"] = False self.data["review"] = True self.data["extendFrames"] = False @@ -212,6 +233,41 @@ class CreateRender(plugin.Creator): # Disable for now as this feature is not working yet # self.data["assScene"] = False + system_settings = get_system_settings()["modules"] + + deadline_enabled = system_settings["deadline"]["enabled"] + muster_enabled = system_settings["muster"]["enabled"] + muster_url = system_settings["muster"]["MUSTER_REST_URL"] + + if deadline_enabled and muster_enabled: + self.log.error( + "Both Deadline and Muster are enabled. " "Cannot support both." + ) + raise RuntimeError("Both Deadline and Muster are enabled") + + if deadline_enabled: + pool_names = self._get_deadline_pools( + self.deadline_servers["default"]) + + if muster_enabled: + self.log.info(">>> Loading Muster credentials ...") + self._load_credentials() + self.log.info(">>> Getting pools ...") + try: + pools = self._get_muster_pools() + except requests.exceptions.HTTPError as e: + if e.startswith("401"): + self.log.warning("access token expired") + self._show_login() + raise RuntimeError("Access token expired") + except requests.exceptions.ConnectionError: + self.log.error("Cannot connect to Muster API endpoint.") + raise RuntimeError("Cannot connect to {}".format(muster_url)) + for pool in pools: + self.log.info(" - pool: {}".format(pool["name"])) + pool_names.append(pool["name"]) + + self.data["primaryPool"] = pool_names self.options = {"useSelection": False} # Force no content def _load_credentials(self): diff --git a/openpype/modules/deadline/deadline_module.py b/openpype/modules/deadline/deadline_module.py index 2a2fba41d6..8329b3151b 100644 --- a/openpype/modules/deadline/deadline_module.py +++ b/openpype/modules/deadline/deadline_module.py @@ -10,7 +10,15 @@ class DeadlineModule(PypeModule, IPluginPaths): # This module is always enabled deadline_settings = modules_settings[self.name] self.enabled = deadline_settings["enabled"] - self.deadline_url = deadline_settings["DEADLINE_REST_URL"] + deadline_url = deadline_settings.get("DEADLINE_REST_URL") + if not deadline_url: + deadline_url = deadline_settings.get("deadline_urls", {}).get("default") # noqa: E501 + if not deadline_url: + self.enabled = False + self.log.warning(("default Deadline Webservice URL " + "not specified. Disabling module.")) + return + self.deadline_url = deadline_url def get_global_environments(self): """Deadline global environments for OpenPype implementation.""" diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index a652da7786..f8577e24fa 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -36,6 +36,7 @@ from avalon import api import pyblish.api from openpype.hosts.maya.api import lib +from openpype.modules import ModulesManager # Documentation for keys available at: # https://docs.thinkboxsoftware.com @@ -264,12 +265,15 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): self._instance = instance self.payload_skeleton = copy.deepcopy(payload_skeleton_template) - self._deadline_url = ( - context.data["system_settings"] - ["modules"] - ["deadline"] - ["DEADLINE_REST_URL"] - ) + + manager = ModulesManager() + deadline_module = manager.modules_by_name["deadline"] + # get default deadline webservice url from deadline module + self.deadline_url = deadline_module.deadline_url + # if custom one is set in instance, use that + if instance.data.get("deadlineUrl"): + self.deadline_url = instance.data.get("deadlineUrl") + assert self.deadline_url, "Requires Deadline Webservice URL" self._job_info = ( context.data["project_settings"].get( diff --git a/openpype/modules/deadline/plugins/publish/submit_nuke_deadline.py b/openpype/modules/deadline/plugins/publish/submit_nuke_deadline.py index fed98d8a08..1624423715 100644 --- a/openpype/modules/deadline/plugins/publish/submit_nuke_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_nuke_deadline.py @@ -4,6 +4,7 @@ import getpass from avalon import api from avalon.vendor import requests +from openpype.modules import ModulesManager import re import pyblish.api import nuke @@ -42,13 +43,14 @@ class NukeSubmitDeadline(pyblish.api.InstancePlugin): node = instance[0] context = instance.context - deadline_url = ( - context.data["system_settings"] - ["modules"] - ["deadline"] - ["DEADLINE_REST_URL"] - ) - assert deadline_url, "Requires DEADLINE_REST_URL" + manager = ModulesManager() + deadline_module = manager.modules_by_name["deadline"] + # get default deadline webservice url from deadline module + self.deadline_url = deadline_module.deadline_url + # if custom one is set in instance, use that + if instance.data.get("deadlineUrl"): + self.deadline_url = instance.data.get("deadlineUrl") + assert self.deadline_url, "Requires Deadline Webservice URL" self.deadline_url = "{}/api/jobs".format(deadline_url) self._comment = context.data.get("comment", "") diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index 41f8337fd8..ed838e64ed 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -5,7 +5,7 @@ import os import json import re from copy import copy, deepcopy -import sys +from openpype.modules import ModulesManager import openpype.api from avalon import api, io @@ -615,14 +615,16 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): instance["families"] = families def process(self, instance): + # type: (pyblish.api.Instance) -> None """Process plugin. Detect type of renderfarm submission and create and post dependend job in case of Deadline. It creates json file with metadata needed for publishing in directory of render. - :param instance: Instance data - :type instance: dict + Args: + instance (pyblish.api.Instance): Instance data. + """ data = instance.data.copy() context = instance.context @@ -908,13 +910,14 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): } if submission_type == "deadline": - self.deadline_url = ( - context.data["system_settings"] - ["modules"] - ["deadline"] - ["DEADLINE_REST_URL"] - ) - assert self.deadline_url, "Requires DEADLINE_REST_URL" + manager = ModulesManager() + deadline_module = manager.modules_by_name["deadline"] + # get default deadline webservice url from deadline module + self.deadline_url = deadline_module.deadline_url + # if custom one is set in instance, use that + if instance.data.get("deadlineUrl"): + self.deadline_url = instance.data.get("deadlineUrl") + assert self.deadline_url, "Requires Deadline Webservice URL" self._submit_deadline_post_job(instance, render_job, instances) diff --git a/openpype/modules/deadline/plugins/publish/validate_deadline_connection.py b/openpype/modules/deadline/plugins/publish/validate_deadline_connection.py index 9b10619c0b..1dba94d822 100644 --- a/openpype/modules/deadline/plugins/publish/validate_deadline_connection.py +++ b/openpype/modules/deadline/plugins/publish/validate_deadline_connection.py @@ -1,11 +1,11 @@ import pyblish.api from avalon.vendor import requests -from openpype.plugin import contextplugin_should_run +from openpype.modules import ModulesManager import os -class ValidateDeadlineConnection(pyblish.api.ContextPlugin): +class ValidateDeadlineConnection(pyblish.api.InstancePlugin): """Validate Deadline Web Service is running""" label = "Validate Deadline Web Service" @@ -13,18 +13,19 @@ class ValidateDeadlineConnection(pyblish.api.ContextPlugin): hosts = ["maya", "nuke"] families = ["renderlayer"] - def process(self, context): + def process(self, instance): - # Workaround bug pyblish-base#250 - if not contextplugin_should_run(self, context): - return - - deadline_url = ( - context.data["system_settings"] - ["modules"] - ["deadline"] - ["DEADLINE_REST_URL"] - ) + manager = ModulesManager() + deadline_module = manager.modules_by_name["deadline"] + # get default deadline webservice url from deadline module + deadline_url = deadline_module.deadline_url + # if custom one is set in instance, use that + if instance.data.get("deadlineUrl"): + deadline_url = instance.data.get("deadlineUrl") + self.log.info( + "We have deadline URL on instance {}".format( + deadline_url)) + assert deadline_url, "Requires Deadline Webservice URL" # Check response response = self._requests_get(deadline_url) diff --git a/openpype/modules/deadline/plugins/publish/validate_expected_and_rendered_files.py b/openpype/modules/deadline/plugins/publish/validate_expected_and_rendered_files.py index c71b5106ec..ca82c54fb8 100644 --- a/openpype/modules/deadline/plugins/publish/validate_expected_and_rendered_files.py +++ b/openpype/modules/deadline/plugins/publish/validate_expected_and_rendered_files.py @@ -4,9 +4,9 @@ import pyblish.api from avalon.vendor import requests -from openpype.api import get_system_settings from openpype.lib.abstract_submit_deadline import requests_get from openpype.lib.delivery import collect_frames +from openpype.modules import ModulesManager class ValidateExpectedFiles(pyblish.api.InstancePlugin): @@ -129,13 +129,14 @@ class ValidateExpectedFiles(pyblish.api.InstancePlugin): Might be different than job info saved in metadata.json if user manually changes job pre/during rendering. """ - deadline_url = ( - get_system_settings() - ["modules"] - ["deadline"] - ["DEADLINE_REST_URL"] - ) - assert deadline_url, "Requires DEADLINE_REST_URL" + manager = ModulesManager() + deadline_module = manager.modules_by_name["deadline"] + # get default deadline webservice url from deadline module + deadline_url = deadline_module.deadline_url + # if custom one is set in instance, use that + if instance.data.get("deadlineUrl"): + deadline_url = instance.data.get("deadlineUrl") + assert deadline_url, "Requires Deadline Webservice URL" url = "{}/api/jobs?JobID={}".format(deadline_url, job_id) try: diff --git a/openpype/settings/defaults/project_settings/deadline.json b/openpype/settings/defaults/project_settings/deadline.json index 2dba20d63c..81d611af1e 100644 --- a/openpype/settings/defaults/project_settings/deadline.json +++ b/openpype/settings/defaults/project_settings/deadline.json @@ -1,4 +1,5 @@ { + "deadline_servers": [], "publish": { "ValidateExpectedFiles": { "enabled": true, diff --git a/openpype/settings/defaults/project_settings/maya.json b/openpype/settings/defaults/project_settings/maya.json index 1db6cdf9f1..e19c03b139 100644 --- a/openpype/settings/defaults/project_settings/maya.json +++ b/openpype/settings/defaults/project_settings/maya.json @@ -31,6 +31,12 @@ "Main" ] }, + "CreateRender": { + "enabled": true, + "defaults": [ + "Main" + ] + }, "CreateAnimation": { "enabled": true, "defaults": [ @@ -81,12 +87,6 @@ "Main" ] }, - "CreateRender": { - "enabled": true, - "defaults": [ - "Main" - ] - }, "CreateRenderSetup": { "enabled": true, "defaults": [ diff --git a/openpype/settings/defaults/project_settings/unreal.json b/openpype/settings/defaults/project_settings/unreal.json index 46b9ca2a18..dad61cd1f0 100644 --- a/openpype/settings/defaults/project_settings/unreal.json +++ b/openpype/settings/defaults/project_settings/unreal.json @@ -1,6 +1,5 @@ { "project_setup": { - "dev_mode": true, - "install_unreal_python_engine": false + "dev_mode": true } } \ No newline at end of file diff --git a/openpype/settings/defaults/system_settings/modules.json b/openpype/settings/defaults/system_settings/modules.json index 1b74b4695c..3a70b90590 100644 --- a/openpype/settings/defaults/system_settings/modules.json +++ b/openpype/settings/defaults/system_settings/modules.json @@ -140,7 +140,9 @@ }, "deadline": { "enabled": true, - "DEADLINE_REST_URL": "http://localhost:8082" + "deadline_urls": { + "default": "http://127.0.0.1:8082" + } }, "muster": { "enabled": false, diff --git a/openpype/settings/entities/__init__.py b/openpype/settings/entities/__init__.py index c0eef15e69..9cda702e9a 100644 --- a/openpype/settings/entities/__init__.py +++ b/openpype/settings/entities/__init__.py @@ -105,7 +105,8 @@ from .enum_entity import ( AppsEnumEntity, ToolsEnumEntity, TaskTypeEnumEntity, - ProvidersEnum + ProvidersEnum, + DeadlineUrlEnumEntity ) from .list_entity import ListEntity @@ -160,6 +161,7 @@ __all__ = ( "ToolsEnumEntity", "TaskTypeEnumEntity", "ProvidersEnum", + "DeadlineUrlEnumEntity", "ListEntity", diff --git a/openpype/settings/entities/enum_entity.py b/openpype/settings/entities/enum_entity.py index 4f6a2886bc..7b3de1ffe7 100644 --- a/openpype/settings/entities/enum_entity.py +++ b/openpype/settings/entities/enum_entity.py @@ -423,3 +423,54 @@ class ProvidersEnum(BaseEnumEntity): self._current_value = value_on_not_set self.value_on_not_set = value_on_not_set + + +class DeadlineUrlEnumEntity(BaseEnumEntity): + schema_types = ["deadline_url-enum"] + + def _item_initalization(self): + self.multiselection = self.schema_data.get("multiselection", True) + + self.enum_items = [] + self.valid_keys = set() + + if self.multiselection: + self.valid_value_types = (list,) + self.value_on_not_set = [] + else: + for key in self.valid_keys: + if self.value_on_not_set is NOT_SET: + self.value_on_not_set = key + break + + self.valid_value_types = (STRING_TYPE,) + + # GUI attribute + self.placeholder = self.schema_data.get("placeholder") + + def _get_enum_values(self): + system_settings_entity = self.get_entity_from_path("system_settings") + + valid_keys = set() + enum_items_list = [] + deadline_urls_entity = (system_settings_entity + ["modules"] + ["deadline"] + ["deadline_urls"] + ) + for server_name, url_entity in deadline_urls_entity.items(): + enum_items_list.append( + {server_name: "{}: {}".format(server_name, url_entity.value)}) + valid_keys.add(server_name) + return enum_items_list, valid_keys + + def set_override_state(self, *args, **kwargs): + super(DeadlineUrlEnumEntity, self).set_override_state(*args, **kwargs) + + self.enum_items, self.valid_keys = self._get_enum_values() + new_value = [] + for key in self._current_value: + if key in self.valid_keys: + new_value.append(key) + self._current_value = new_value + diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json b/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json index 27eeaef559..bd14d2ea9d 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json @@ -5,6 +5,12 @@ "collapsible": true, "is_file": true, "children": [ + { + "type": "deadline_url-enum", + "key": "deadline_servers", + "label": "Deadline Webservice URLs", + "multiselect": true + }, { "type": "dict", "collapsible": true, diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_create.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_create.json index d728f1def3..44a35af7c1 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_create.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_create.json @@ -29,6 +29,26 @@ } ] }, + { + "type": "dict", + "collapsible": true, + "key": "CreateRender", + "label": "Create Render", + "checkbox_key": "enabled", + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + }, + { + "type": "list", + "key": "defaults", + "label": "Default Subsets", + "object_type": "text" + } + ] + }, { "type": "schema_template", "name": "template_create_plugin", @@ -65,10 +85,6 @@ "key": "CreatePointCache", "label": "Create Cache" }, - { - "key": "CreateRender", - "label": "Create Render" - }, { "key": "CreateRenderSetup", "label": "Create Render Setup" diff --git a/openpype/settings/entities/schemas/system_schema/schema_modules.json b/openpype/settings/entities/schemas/system_schema/schema_modules.json index 7d734ff4fd..75c08b2cd9 100644 --- a/openpype/settings/entities/schemas/system_schema/schema_modules.json +++ b/openpype/settings/entities/schemas/system_schema/schema_modules.json @@ -130,9 +130,11 @@ "label": "Enabled" }, { - "type": "text", - "key": "DEADLINE_REST_URL", - "label": "Deadline Resl URL" + "type": "dict-modifiable", + "object_type": "text", + "key": "deadline_urls", + "required_keys": ["default"], + "label": "Deadline Webservice URLs" } ] }, From 8424821fbc090ddb82ae8028dbf76b7284bc01ae Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Fri, 6 Aug 2021 14:41:50 +0200 Subject: [PATCH 2/8] =?UTF-8?q?finalize=20support=20for=20multiple=20deadl?= =?UTF-8?q?ine=20servers=20=EF=B8=8F=EF=B8=8F=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../maya/plugins/create/create_render.py | 34 ++++++++--- .../maya/plugins/publish/collect_render.py | 61 ++++++++++++++++++- openpype/lib/abstract_submit_deadline.py | 12 ++-- .../collect_default_deadline_server.py | 17 ++++++ .../plugins/publish/submit_maya_deadline.py | 13 ++-- .../plugins/publish/submit_nuke_deadline.py | 9 +-- .../plugins/publish/submit_publish_job.py | 5 +- .../publish/validate_deadline_connection.py | 6 +- .../validate_expected_and_rendered_files.py | 10 ++- 9 files changed, 120 insertions(+), 47 deletions(-) create mode 100644 openpype/modules/deadline/plugins/publish/collect_default_deadline_server.py diff --git a/openpype/hosts/maya/plugins/create/create_render.py b/openpype/hosts/maya/plugins/create/create_render.py index 76cac5fe25..98ad3114d1 100644 --- a/openpype/hosts/maya/plugins/create/create_render.py +++ b/openpype/hosts/maya/plugins/create/create_render.py @@ -5,6 +5,7 @@ import json import appdirs import requests import six +import sys from maya import cmds import maya.app.renderSetup.model.renderSetup as renderSetup @@ -90,12 +91,20 @@ class CreateRender(plugin.Creator): def __init__(self, *args, **kwargs): """Constructor.""" super(CreateRender, self).__init__(*args, **kwargs) + deadline_settings = get_system_settings()["modules"]["deadline"] + if not deadline_settings["enabled"]: + self.deadline_servers = {} + return project_settings = get_project_settings(Session["AVALON_PROJECT"]) try: - self.deadline_servers = ( + default_servers = deadline_settings["deadline_urls"] + project_servers = ( project_settings["deadline"] ["deadline_servers"] ) + self.deadline_servers = dict( + (k, default_servers[k]) + for k in project_servers if k in default_servers) except AttributeError: # Handle situation were we had only one url for deadline. manager = ModulesManager() @@ -131,11 +140,12 @@ class CreateRender(plugin.Creator): namespace = cmds.namespace(add=namespace_name) # add Deadline server selection list - cmds.scriptJob( - attributeChange=[ - "{}.deadlineServers".format(self.instance), - self._deadline_webservice_changed - ]) + if self.deadline_servers: + cmds.scriptJob( + attributeChange=[ + "{}.deadlineServers".format(self.instance), + self._deadline_webservice_changed + ]) cmds.setAttr("{}.machineList".format(self.instance), lock=True) self._rs = renderSetup.instance() @@ -170,6 +180,7 @@ class CreateRender(plugin.Creator): def _deadline_webservice_changed(self): """Refresh Deadline server dependent options.""" # get selected server + from maya import cmds webservice = self.deadline_servers[ self.server_aliases[ cmds.getAttr("{}.deadlineServers".format(self.instance)) @@ -194,7 +205,7 @@ class CreateRender(plugin.Creator): list: Pools. Throws: RuntimeError: If deadline webservice is unreachable. - + """ argument = "{}/api/pools?NamesOnly=true".format(webservice) try: @@ -202,7 +213,10 @@ class CreateRender(plugin.Creator): except requests.exceptions.ConnectionError as exc: msg = 'Cannot connect to deadline web service' self.log.error(msg) - six.reraise(exc, RuntimeError('{} - {}'.format(msg, exc))) + six.reraise( + RuntimeError, + RuntimeError('{} - {}'.format(msg, exc)), + sys.exc_info()[2]) if not response.ok: self.log.warning("No pools retrieved") return [] @@ -268,6 +282,9 @@ class CreateRender(plugin.Creator): pool_names.append(pool["name"]) self.data["primaryPool"] = pool_names + # We add a string "-" to allow the user to not + # set any secondary pools + self.data["secondaryPool"] = ["-"] + pool_names self.options = {"useSelection": False} # Force no content def _load_credentials(self): @@ -388,6 +405,7 @@ class CreateRender(plugin.Creator): if renderer == "arnold": # set format to exr + cmds.setAttr( "defaultArnoldDriver.ai_translator", "exr", type="string") # enable animation diff --git a/openpype/hosts/maya/plugins/publish/collect_render.py b/openpype/hosts/maya/plugins/publish/collect_render.py index 647a46e240..85afd971c6 100644 --- a/openpype/hosts/maya/plugins/publish/collect_render.py +++ b/openpype/hosts/maya/plugins/publish/collect_render.py @@ -51,6 +51,12 @@ import pyblish.api from avalon import maya, api from openpype.hosts.maya.api.expected_files import ExpectedFiles from openpype.hosts.maya.api import lib +from openpype.api import ( + get_system_settings, + get_project_settings +) + +from avalon.api import Session class CollectMayaRender(pyblish.api.ContextPlugin): @@ -86,6 +92,10 @@ class CollectMayaRender(pyblish.api.ContextPlugin): asset = api.Session["AVALON_ASSET"] workspace = context.data["workspaceDir"] + deadline_settings = get_system_settings()["modules"]["deadline"] + + if deadline_settings["enabled"]: + deadline_url = self._collect_deadline_url(render_instance) self._rs = renderSetup.instance() current_layer = self._rs.getVisibleRenderLayer() maya_render_layers = { @@ -263,6 +273,9 @@ class CollectMayaRender(pyblish.api.ContextPlugin): "vrayUseReferencedAovs") or False } + if deadline_url: + data["deadlineUrl"] = deadline_url + if self.sync_workfile_version: data["version"] = context.data["version"] @@ -392,11 +405,13 @@ class CollectMayaRender(pyblish.api.ContextPlugin): rset = self.maya_layers[layer].renderSettingsCollectionInstance() return rset.getOverrides() - def get_render_attribute(self, attr, layer): + @staticmethod + def get_render_attribute(attr, layer): """Get attribute from render options. Args: - attr (str): name of attribute to be looked up. + attr (str): name of attribute to be looked up + layer (str): name of render layer Returns: Attribute value @@ -405,3 +420,45 @@ class CollectMayaRender(pyblish.api.ContextPlugin): return lib.get_attr_in_layer( "defaultRenderGlobals.{}".format(attr), layer=layer ) + + @staticmethod + def _collect_deadline_url(render_instance): + # type: (pyblish.api.Instance) -> str + """Get Deadline Webservice URL from render instance. + + This will get all configured Deadline Webservice URLs and create + subset of them based upon project configuration. It will then take + `deadlineServers` from render instance that is now basically `int` + index of that list. + + Args: + render_instance (pyblish.api.Instance): Render instance created + by Creator in Maya. + + Returns: + str: Selected Deadline Webservice URL. + + """ + + deadline_settings = get_system_settings()["modules"]["deadline"] + project_settings = get_project_settings(Session["AVALON_PROJECT"]) + try: + default_servers = deadline_settings["deadline_urls"] + project_servers = ( + project_settings["deadline"] + ["deadline_servers"] + ) + deadline_servers = dict( + (k, default_servers[k]) + for k in project_servers if k in default_servers) + except AttributeError: + # Handle situation were we had only one url for deadline. + deadline_url = render_instance.context.data["defaultDeadline"] + deadline_servers = {"default": deadline_url} + + deadline_url = deadline_servers[ + list(deadline_servers.keys())[ + int(render_instance.data.get("deadlineServers")) + ] + ] + return deadline_url diff --git a/openpype/lib/abstract_submit_deadline.py b/openpype/lib/abstract_submit_deadline.py index 4a052a4ee2..5b6e1743e0 100644 --- a/openpype/lib/abstract_submit_deadline.py +++ b/openpype/lib/abstract_submit_deadline.py @@ -415,13 +415,11 @@ class AbstractSubmitDeadline(pyblish.api.InstancePlugin): """Plugin entry point.""" self._instance = instance context = instance.context - self._deadline_url = ( - context.data["system_settings"] - ["modules"] - ["deadline"] - ["DEADLINE_REST_URL"] - ) - assert self._deadline_url, "Requires DEADLINE_REST_URL" + self._deadline_url = context.data.get("defaultDeadline") + self._deadline_url = instance.data.get( + "deadlineUrl", self._deadline_url) + + assert self._deadline_url, "Requires Deadline Webservice URL" file_path = None if self.use_published: diff --git a/openpype/modules/deadline/plugins/publish/collect_default_deadline_server.py b/openpype/modules/deadline/plugins/publish/collect_default_deadline_server.py new file mode 100644 index 0000000000..2bb18486ff --- /dev/null +++ b/openpype/modules/deadline/plugins/publish/collect_default_deadline_server.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +"""Collect default Deadline server.""" +from openpype.modules import ModulesManager +import pyblish.api + + +class CollectDefaultDeadlineServer(pyblish.api.ContextPlugin): + """Collect default Deadline Webservice URL.""" + + order = pyblish.api.CollectorOrder + 0.01 + label = "Default Deadline Webservice" + + def process(self, context): + manager = ModulesManager() + deadline_module = manager.modules_by_name["deadline"] + # get default deadline webservice url from deadline module + context.data["defaultDeadline"] = deadline_module.deadline_url diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index f8577e24fa..898a257112 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -36,7 +36,6 @@ from avalon import api import pyblish.api from openpype.hosts.maya.api import lib -from openpype.modules import ModulesManager # Documentation for keys available at: # https://docs.thinkboxsoftware.com @@ -266,10 +265,8 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): self._instance = instance self.payload_skeleton = copy.deepcopy(payload_skeleton_template) - manager = ModulesManager() - deadline_module = manager.modules_by_name["deadline"] # get default deadline webservice url from deadline module - self.deadline_url = deadline_module.deadline_url + self.deadline_url = instance.context.data.get("defaultDeadline") # if custom one is set in instance, use that if instance.data.get("deadlineUrl"): self.deadline_url = instance.data.get("deadlineUrl") @@ -291,8 +288,6 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): "pluginInfo", {}) ) - assert self._deadline_url, "Requires DEADLINE_REST_URL" - context = instance.context workspace = context.data["workspaceDir"] anatomy = context.data['anatomy'] @@ -674,7 +669,7 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): self.log.info( "Submitting tile job(s) [{}] ...".format(len(frame_payloads))) - url = "{}/api/jobs".format(self._deadline_url) + url = "{}/api/jobs".format(self.deadline_url) tiles_count = instance.data.get("tilesX") * instance.data.get("tilesY") # noqa: E501 for tile_job in frame_payloads: @@ -758,7 +753,7 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): self.log.debug(json.dumps(payload, indent=4, sort_keys=True)) # E.g. http://192.168.0.1:8082/api/jobs - url = "{}/api/jobs".format(self._deadline_url) + url = "{}/api/jobs".format(self.deadline_url) response = self._requests_post(url, json=payload) if not response.ok: raise Exception(response.text) @@ -968,7 +963,7 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): payload = self._get_arnold_export_payload(data) self.log.info("Submitting ass export job.") - url = "{}/api/jobs".format(self._deadline_url) + url = "{}/api/jobs".format(self.deadline_url) response = self._requests_post(url, json=payload) if not response.ok: self.log.error("Submition failed!") diff --git a/openpype/modules/deadline/plugins/publish/submit_nuke_deadline.py b/openpype/modules/deadline/plugins/publish/submit_nuke_deadline.py index 1624423715..1baef5c297 100644 --- a/openpype/modules/deadline/plugins/publish/submit_nuke_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_nuke_deadline.py @@ -4,7 +4,6 @@ import getpass from avalon import api from avalon.vendor import requests -from openpype.modules import ModulesManager import re import pyblish.api import nuke @@ -43,14 +42,12 @@ class NukeSubmitDeadline(pyblish.api.InstancePlugin): node = instance[0] context = instance.context - manager = ModulesManager() - deadline_module = manager.modules_by_name["deadline"] # get default deadline webservice url from deadline module - self.deadline_url = deadline_module.deadline_url + deadline_url = instance.context.data["defaultDeadline"] # if custom one is set in instance, use that if instance.data.get("deadlineUrl"): - self.deadline_url = instance.data.get("deadlineUrl") - assert self.deadline_url, "Requires Deadline Webservice URL" + deadline_url = instance.data.get("deadlineUrl") + assert deadline_url, "Requires Deadline Webservice URL" self.deadline_url = "{}/api/jobs".format(deadline_url) self._comment = context.data.get("comment", "") diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index ed838e64ed..19e3174384 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -5,7 +5,6 @@ import os import json import re from copy import copy, deepcopy -from openpype.modules import ModulesManager import openpype.api from avalon import api, io @@ -910,10 +909,8 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): } if submission_type == "deadline": - manager = ModulesManager() - deadline_module = manager.modules_by_name["deadline"] # get default deadline webservice url from deadline module - self.deadline_url = deadline_module.deadline_url + self.deadline_url = instance.context.data["defaultDeadline"] # if custom one is set in instance, use that if instance.data.get("deadlineUrl"): self.deadline_url = instance.data.get("deadlineUrl") diff --git a/openpype/modules/deadline/plugins/publish/validate_deadline_connection.py b/openpype/modules/deadline/plugins/publish/validate_deadline_connection.py index 1dba94d822..ff664d9f83 100644 --- a/openpype/modules/deadline/plugins/publish/validate_deadline_connection.py +++ b/openpype/modules/deadline/plugins/publish/validate_deadline_connection.py @@ -1,7 +1,6 @@ import pyblish.api from avalon.vendor import requests -from openpype.modules import ModulesManager import os @@ -14,11 +13,8 @@ class ValidateDeadlineConnection(pyblish.api.InstancePlugin): families = ["renderlayer"] def process(self, instance): - - manager = ModulesManager() - deadline_module = manager.modules_by_name["deadline"] # get default deadline webservice url from deadline module - deadline_url = deadline_module.deadline_url + deadline_url = instance.context.data["defaultDeadline"] # if custom one is set in instance, use that if instance.data.get("deadlineUrl"): deadline_url = instance.data.get("deadlineUrl") diff --git a/openpype/modules/deadline/plugins/publish/validate_expected_and_rendered_files.py b/openpype/modules/deadline/plugins/publish/validate_expected_and_rendered_files.py index ca82c54fb8..2d16bc965d 100644 --- a/openpype/modules/deadline/plugins/publish/validate_expected_and_rendered_files.py +++ b/openpype/modules/deadline/plugins/publish/validate_expected_and_rendered_files.py @@ -6,7 +6,6 @@ from avalon.vendor import requests from openpype.lib.abstract_submit_deadline import requests_get from openpype.lib.delivery import collect_frames -from openpype.modules import ModulesManager class ValidateExpectedFiles(pyblish.api.InstancePlugin): @@ -22,6 +21,7 @@ class ValidateExpectedFiles(pyblish.api.InstancePlugin): allow_user_override = True def process(self, instance): + self.instance = instance frame_list = self._get_frame_list(instance.data["render_job_id"]) for repre in instance.data["representations"]: @@ -129,13 +129,11 @@ class ValidateExpectedFiles(pyblish.api.InstancePlugin): Might be different than job info saved in metadata.json if user manually changes job pre/during rendering. """ - manager = ModulesManager() - deadline_module = manager.modules_by_name["deadline"] # get default deadline webservice url from deadline module - deadline_url = deadline_module.deadline_url + deadline_url = self.instance.context.data["defaultDeadline"] # if custom one is set in instance, use that - if instance.data.get("deadlineUrl"): - deadline_url = instance.data.get("deadlineUrl") + if self.instance.data.get("deadlineUrl"): + deadline_url = self.instance.data.get("deadlineUrl") assert deadline_url, "Requires Deadline Webservice URL" url = "{}/api/jobs?JobID={}".format(deadline_url, job_id) From dfb230566af0aa4682f08e491795f060daed3a46 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Fri, 6 Aug 2021 14:45:49 +0200 Subject: [PATCH 3/8] =?UTF-8?q?hound=20=F0=9F=90=95=E2=80=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- openpype/settings/entities/enum_entity.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/settings/entities/enum_entity.py b/openpype/settings/entities/enum_entity.py index 7b3de1ffe7..a31c1ba5ec 100644 --- a/openpype/settings/entities/enum_entity.py +++ b/openpype/settings/entities/enum_entity.py @@ -453,7 +453,8 @@ class DeadlineUrlEnumEntity(BaseEnumEntity): valid_keys = set() enum_items_list = [] - deadline_urls_entity = (system_settings_entity + deadline_urls_entity = ( + system_settings_entity ["modules"] ["deadline"] ["deadline_urls"] @@ -473,4 +474,3 @@ class DeadlineUrlEnumEntity(BaseEnumEntity): if key in self.valid_keys: new_value.append(key) self._current_value = new_value - From 55fd0ded900d92f61304ae58dc20572bf1f53b05 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Tue, 10 Aug 2021 19:11:02 +0200 Subject: [PATCH 4/8] rework collector and settings flow --- .../maya/plugins/publish/collect_render.py | 59 +++------------ openpype/modules/deadline/deadline_module.py | 16 ++--- .../collect_deadline_server_from_instance.py | 71 +++++++++++++++++++ .../collect_default_deadline_server.py | 11 +-- openpype/plugins/publish/collect_modules.py | 15 ++++ 5 files changed, 108 insertions(+), 64 deletions(-) create mode 100644 openpype/modules/deadline/plugins/publish/collect_deadline_server_from_instance.py create mode 100644 openpype/plugins/publish/collect_modules.py diff --git a/openpype/hosts/maya/plugins/publish/collect_render.py b/openpype/hosts/maya/plugins/publish/collect_render.py index 85afd971c6..e90efbc64d 100644 --- a/openpype/hosts/maya/plugins/publish/collect_render.py +++ b/openpype/hosts/maya/plugins/publish/collect_render.py @@ -51,12 +51,6 @@ import pyblish.api from avalon import maya, api from openpype.hosts.maya.api.expected_files import ExpectedFiles from openpype.hosts.maya.api import lib -from openpype.api import ( - get_system_settings, - get_project_settings -) - -from avalon.api import Session class CollectMayaRender(pyblish.api.ContextPlugin): @@ -70,6 +64,8 @@ class CollectMayaRender(pyblish.api.ContextPlugin): def process(self, context): """Entry point to collector.""" render_instance = None + deadline_url = None + for instance in context: if "rendering" in instance.data["families"]: render_instance = instance @@ -92,10 +88,15 @@ class CollectMayaRender(pyblish.api.ContextPlugin): asset = api.Session["AVALON_ASSET"] workspace = context.data["workspaceDir"] - deadline_settings = get_system_settings()["modules"]["deadline"] + deadline_settings = ( + context.data + ["system_settings"] + ["modules"] + ["deadline"] + ) if deadline_settings["enabled"]: - deadline_url = self._collect_deadline_url(render_instance) + deadline_url = render_instance.data.get("deadlineUrl") self._rs = renderSetup.instance() current_layer = self._rs.getVisibleRenderLayer() maya_render_layers = { @@ -420,45 +421,3 @@ class CollectMayaRender(pyblish.api.ContextPlugin): return lib.get_attr_in_layer( "defaultRenderGlobals.{}".format(attr), layer=layer ) - - @staticmethod - def _collect_deadline_url(render_instance): - # type: (pyblish.api.Instance) -> str - """Get Deadline Webservice URL from render instance. - - This will get all configured Deadline Webservice URLs and create - subset of them based upon project configuration. It will then take - `deadlineServers` from render instance that is now basically `int` - index of that list. - - Args: - render_instance (pyblish.api.Instance): Render instance created - by Creator in Maya. - - Returns: - str: Selected Deadline Webservice URL. - - """ - - deadline_settings = get_system_settings()["modules"]["deadline"] - project_settings = get_project_settings(Session["AVALON_PROJECT"]) - try: - default_servers = deadline_settings["deadline_urls"] - project_servers = ( - project_settings["deadline"] - ["deadline_servers"] - ) - deadline_servers = dict( - (k, default_servers[k]) - for k in project_servers if k in default_servers) - except AttributeError: - # Handle situation were we had only one url for deadline. - deadline_url = render_instance.context.data["defaultDeadline"] - deadline_servers = {"default": deadline_url} - - deadline_url = deadline_servers[ - list(deadline_servers.keys())[ - int(render_instance.data.get("deadlineServers")) - ] - ] - return deadline_url diff --git a/openpype/modules/deadline/deadline_module.py b/openpype/modules/deadline/deadline_module.py index 8329b3151b..17e7674f5b 100644 --- a/openpype/modules/deadline/deadline_module.py +++ b/openpype/modules/deadline/deadline_module.py @@ -11,20 +11,16 @@ class DeadlineModule(PypeModule, IPluginPaths): deadline_settings = modules_settings[self.name] self.enabled = deadline_settings["enabled"] deadline_url = deadline_settings.get("DEADLINE_REST_URL") - if not deadline_url: - deadline_url = deadline_settings.get("deadline_urls", {}).get("default") # noqa: E501 - if not deadline_url: + if deadline_url: + deadline_urls = {"default": deadline_url} + else: + deadline_urls = deadline_settings.get("deadline_urls") # noqa: E501 + + if not deadline_urls: self.enabled = False self.log.warning(("default Deadline Webservice URL " "not specified. Disabling module.")) return - self.deadline_url = deadline_url - - def get_global_environments(self): - """Deadline global environments for OpenPype implementation.""" - return { - "DEADLINE_REST_URL": self.deadline_url - } def connect_with_modules(self, *_a, **_kw): return diff --git a/openpype/modules/deadline/plugins/publish/collect_deadline_server_from_instance.py b/openpype/modules/deadline/plugins/publish/collect_deadline_server_from_instance.py new file mode 100644 index 0000000000..784616615d --- /dev/null +++ b/openpype/modules/deadline/plugins/publish/collect_deadline_server_from_instance.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +"""Collect Deadline servers from instance. + +This is resolving index of server lists stored in `deadlineServers` instance +attribute or using default server if that attribute doesn't exists. + +""" +import pyblish.api + + +class CollectDeadlineServerFromInstance(pyblish.api.InstancePlugin): + """Collect Deadline Webservice URL from instance.""" + + order = pyblish.api.CollectorOrder + label = "Deadline Webservice from the Instance" + families = ["rendering"] + + def process(self, instance): + instance.data["deadlineUrl"] = self._collect_deadline_url(instance) + self.log.info( + "Using {} for submission.".format(instance.data["deadlineUrl"])) + + @staticmethod + def _collect_deadline_url(render_instance): + # type: (pyblish.api.Instance) -> str + """Get Deadline Webservice URL from render instance. + + This will get all configured Deadline Webservice URLs and create + subset of them based upon project configuration. It will then take + `deadlineServers` from render instance that is now basically `int` + index of that list. + + Args: + render_instance (pyblish.api.Instance): Render instance created + by Creator in Maya. + + Returns: + str: Selected Deadline Webservice URL. + + """ + + deadline_settings = ( + render_instance.context.data + ["system_settings"] + ["modules"] + ["deadline"] + ) + + try: + default_servers = deadline_settings["deadline_urls"] + project_servers = ( + render_instance.context.data + ["project_settings"] + ["deadline"] + ["deadline_servers"] + ) + deadline_servers = { + k: default_servers[k] + for k in project_servers + if k in default_servers + } + + except AttributeError: + # Handle situation were we had only one url for deadline. + return render_instance.context.data["defaultDeadline"] + + return deadline_servers[ + list(deadline_servers.keys())[ + int(render_instance.data.get("deadlineServers")) + ] + ] diff --git a/openpype/modules/deadline/plugins/publish/collect_default_deadline_server.py b/openpype/modules/deadline/plugins/publish/collect_default_deadline_server.py index 2bb18486ff..80d6b6539b 100644 --- a/openpype/modules/deadline/plugins/publish/collect_default_deadline_server.py +++ b/openpype/modules/deadline/plugins/publish/collect_default_deadline_server.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- """Collect default Deadline server.""" -from openpype.modules import ModulesManager import pyblish.api @@ -11,7 +10,11 @@ class CollectDefaultDeadlineServer(pyblish.api.ContextPlugin): label = "Default Deadline Webservice" def process(self, context): - manager = ModulesManager() - deadline_module = manager.modules_by_name["deadline"] + try: + deadline_module = context.data.get("openPypeModules")["deadline"] + except AttributeError: + self.log.error("Cannot get OpenPype Deadline module.") + raise AssertionError("OpenPype Deadline module not found.") + # get default deadline webservice url from deadline module - context.data["defaultDeadline"] = deadline_module.deadline_url + context.data["defaultDeadline"] = deadline_module.deadline_urls["default"] diff --git a/openpype/plugins/publish/collect_modules.py b/openpype/plugins/publish/collect_modules.py new file mode 100644 index 0000000000..6a6d5da511 --- /dev/null +++ b/openpype/plugins/publish/collect_modules.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +"""Collect OpenPype modules.""" +from openpype.modules import ModulesManager +import pyblish.api + + +class CollectModules(pyblish.api.ContextPlugin): + """Collect OpenPype modules.""" + + order = pyblish.api.CollectorOrder + label = "OpenPype Modules" + + def process(self, context): + manager = ModulesManager() + context.data["openPypeModules"] = manager.modules From 50f638972413bf2af705b7e146c374b6227ad544 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Tue, 10 Aug 2021 19:21:16 +0200 Subject: [PATCH 5/8] fix creator --- openpype/hosts/maya/plugins/create/create_render.py | 3 +-- openpype/modules/deadline/deadline_module.py | 10 +++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/openpype/hosts/maya/plugins/create/create_render.py b/openpype/hosts/maya/plugins/create/create_render.py index 98ad3114d1..84582f4177 100644 --- a/openpype/hosts/maya/plugins/create/create_render.py +++ b/openpype/hosts/maya/plugins/create/create_render.py @@ -110,8 +110,7 @@ class CreateRender(plugin.Creator): manager = ModulesManager() deadline_module = manager.modules_by_name["deadline"] # get default deadline webservice url from deadline module - deadline_url = deadline_module.deadline_url - self.deadline_servers = {"default": deadline_url} + self.deadline_servers = deadline_module.deadline_urls def process(self): """Entry point.""" diff --git a/openpype/modules/deadline/deadline_module.py b/openpype/modules/deadline/deadline_module.py index 17e7674f5b..c687d579c6 100644 --- a/openpype/modules/deadline/deadline_module.py +++ b/openpype/modules/deadline/deadline_module.py @@ -6,17 +6,21 @@ from openpype.modules import ( class DeadlineModule(PypeModule, IPluginPaths): name = "deadline" + def __init__(self, manager, settings): + super().__init__(manager, settings) + self.deadline_urls = {} + def initialize(self, modules_settings): # This module is always enabled deadline_settings = modules_settings[self.name] self.enabled = deadline_settings["enabled"] deadline_url = deadline_settings.get("DEADLINE_REST_URL") if deadline_url: - deadline_urls = {"default": deadline_url} + self.deadline_urls = {"default": deadline_url} else: - deadline_urls = deadline_settings.get("deadline_urls") # noqa: E501 + self.deadline_urls = deadline_settings.get("deadline_urls") # noqa: E501 - if not deadline_urls: + if not self.deadline_urls: self.enabled = False self.log.warning(("default Deadline Webservice URL " "not specified. Disabling module.")) From a8bf58142e8f61fb1cb4e9cc2fd62465bae8a47f Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Wed, 11 Aug 2021 12:28:18 +0200 Subject: [PATCH 6/8] fix default states --- .../maya/plugins/create/create_render.py | 139 +++++++++++------- openpype/modules/deadline/deadline_module.py | 2 +- .../collect_default_deadline_server.py | 1 + openpype/plugins/publish/collect_modules.py | 2 +- 4 files changed, 85 insertions(+), 59 deletions(-) diff --git a/openpype/hosts/maya/plugins/create/create_render.py b/openpype/hosts/maya/plugins/create/create_render.py index 84582f4177..66e3fb89c9 100644 --- a/openpype/hosts/maya/plugins/create/create_render.py +++ b/openpype/hosts/maya/plugins/create/create_render.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- """Create ``Render`` instance in Maya.""" import os +from collections import OrderedDict import json import appdirs import requests @@ -102,9 +103,15 @@ class CreateRender(plugin.Creator): project_settings["deadline"] ["deadline_servers"] ) - self.deadline_servers = dict( - (k, default_servers[k]) - for k in project_servers if k in default_servers) + self.deadline_servers = { + k: default_servers[k] + for k in project_servers + if k in default_servers + } + + if not self.deadline_servers: + self.deadline_servers = default_servers + except AttributeError: # Handle situation were we had only one url for deadline. manager = ModulesManager() @@ -223,6 +230,7 @@ class CreateRender(plugin.Creator): return response.json() def _create_render_settings(self): + """Create instance settings.""" # get pools pool_names = [] @@ -259,13 +267,22 @@ class CreateRender(plugin.Creator): raise RuntimeError("Both Deadline and Muster are enabled") if deadline_enabled: - pool_names = self._get_deadline_pools( - self.deadline_servers["default"]) + # if default server is not between selected, use first one for + # initial list of pools. + try: + deadline_url = self.deadline_servers["default"] + except KeyError: + deadline_url = [ + self.deadline_servers[k] for k in self.deadline_servers.keys() + ][0] + + pool_names = self._get_deadline_pools(deadline_url) if muster_enabled: self.log.info(">>> Loading Muster credentials ...") self._load_credentials() self.log.info(">>> Getting pools ...") + pools = [] try: pools = self._get_muster_pools() except requests.exceptions.HTTPError as e: @@ -365,9 +382,7 @@ class CreateRender(plugin.Creator): """ if "verify" not in kwargs: - kwargs["verify"] = ( - False if os.getenv("OPENPYPE_DONT_VERIFY_SSL", True) else True - ) # noqa + kwargs["verify"] = not os.getenv("OPENPYPE_DONT_VERIFY_SSL", True) return requests.post(*args, **kwargs) def _requests_get(self, *args, **kwargs): @@ -384,9 +399,7 @@ class CreateRender(plugin.Creator): """ if "verify" not in kwargs: - kwargs["verify"] = ( - False if os.getenv("OPENPYPE_DONT_VERIFY_SSL", True) else True - ) # noqa + kwargs["verify"] = not os.getenv("OPENPYPE_DONT_VERIFY_SSL", True) return requests.get(*args, **kwargs) def _set_default_renderer_settings(self, renderer): @@ -407,12 +420,7 @@ class CreateRender(plugin.Creator): cmds.setAttr( "defaultArnoldDriver.ai_translator", "exr", type="string") - # enable animation - cmds.setAttr("defaultRenderGlobals.outFormatControl", 0) - cmds.setAttr("defaultRenderGlobals.animation", 1) - cmds.setAttr("defaultRenderGlobals.putFrameBeforeExt", 1) - cmds.setAttr("defaultRenderGlobals.extensionPadding", 4) - + self._set_global_output_settings() # resolution cmds.setAttr( "defaultResolution.width", @@ -422,43 +430,12 @@ class CreateRender(plugin.Creator): asset["data"].get("resolutionHeight")) if renderer == "vray": - vray_settings = cmds.ls(type="VRaySettingsNode") - if not vray_settings: - node = cmds.createNode("VRaySettingsNode") - else: - node = vray_settings[0] - - # set underscore as element separator instead of default `.` - cmds.setAttr( - "{}.fileNameRenderElementSeparator".format( - node), - "_" - ) - # set format to exr - cmds.setAttr( - "{}.imageFormatStr".format(node), 5) - - # animType - cmds.setAttr( - "{}.animType".format(node), 1) - - # resolution - cmds.setAttr( - "{}.width".format(node), - asset["data"].get("resolutionWidth")) - cmds.setAttr( - "{}.height".format(node), - asset["data"].get("resolutionHeight")) - + self._set_vray_settings(asset) if renderer == "redshift": - redshift_settings = cmds.ls(type="RedshiftOptions") - if not redshift_settings: - node = cmds.createNode("RedshiftOptions") - else: - node = redshift_settings[0] + _ = self._set_renderer_option( + "RedshiftOptions", "{}.imageFormat", 1 + ) - # set exr - cmds.setAttr("{}.imageFormat".format(node), 1) # resolution cmds.setAttr( "defaultResolution.width", @@ -467,8 +444,56 @@ class CreateRender(plugin.Creator): "defaultResolution.height", asset["data"].get("resolutionHeight")) - # enable animation - cmds.setAttr("defaultRenderGlobals.outFormatControl", 0) - cmds.setAttr("defaultRenderGlobals.animation", 1) - cmds.setAttr("defaultRenderGlobals.putFrameBeforeExt", 1) - cmds.setAttr("defaultRenderGlobals.extensionPadding", 4) + self._set_global_output_settings() + + @staticmethod + def _set_renderer_option(renderer_node, arg=None, value=None): + # type: (str, str, str) -> str + """Set option on renderer node. + + If renderer settings node doesn't exists, it is created first. + + Args: + renderer_node (str): Renderer name. + arg (str, optional): Argument name. + value (str, optional): Argument value. + + Returns: + str: Renderer settings node. + + """ + settings = cmds.ls(type=renderer_node) + result = settings[0] if settings else cmds.createNode(renderer_node) + cmds.setAttr(arg.format(result), value) + return result + + def _set_vray_settings(self, asset): + # type: (dict) -> None + """Sets important settings for Vray.""" + node = self._set_renderer_option( + "VRaySettingsNode", "{}.fileNameRenderElementSeparator", "_" + ) + + # set format to exr + cmds.setAttr( + "{}.imageFormatStr".format(node), 5) + + # animType + cmds.setAttr( + "{}.animType".format(node), 1) + + # resolution + cmds.setAttr( + "{}.width".format(node), + asset["data"].get("resolutionWidth")) + cmds.setAttr( + "{}.height".format(node), + asset["data"].get("resolutionHeight")) + + @staticmethod + def _set_global_output_settings(): + # enable animation + cmds.setAttr("defaultRenderGlobals.outFormatControl", 0) + cmds.setAttr("defaultRenderGlobals.animation", 1) + cmds.setAttr("defaultRenderGlobals.putFrameBeforeExt", 1) + cmds.setAttr("defaultRenderGlobals.extensionPadding", 4) diff --git a/openpype/modules/deadline/deadline_module.py b/openpype/modules/deadline/deadline_module.py index c687d579c6..a07cb1a660 100644 --- a/openpype/modules/deadline/deadline_module.py +++ b/openpype/modules/deadline/deadline_module.py @@ -7,8 +7,8 @@ class DeadlineModule(PypeModule, IPluginPaths): name = "deadline" def __init__(self, manager, settings): - super().__init__(manager, settings) self.deadline_urls = {} + super(DeadlineModule, self).__init__(manager, settings) def initialize(self, modules_settings): # This module is always enabled diff --git a/openpype/modules/deadline/plugins/publish/collect_default_deadline_server.py b/openpype/modules/deadline/plugins/publish/collect_default_deadline_server.py index 80d6b6539b..17851032cf 100644 --- a/openpype/modules/deadline/plugins/publish/collect_default_deadline_server.py +++ b/openpype/modules/deadline/plugins/publish/collect_default_deadline_server.py @@ -17,4 +17,5 @@ class CollectDefaultDeadlineServer(pyblish.api.ContextPlugin): raise AssertionError("OpenPype Deadline module not found.") # get default deadline webservice url from deadline module + self.log.debug(deadline_module.deadline_urls) context.data["defaultDeadline"] = deadline_module.deadline_urls["default"] diff --git a/openpype/plugins/publish/collect_modules.py b/openpype/plugins/publish/collect_modules.py index 6a6d5da511..bec0c2b436 100644 --- a/openpype/plugins/publish/collect_modules.py +++ b/openpype/plugins/publish/collect_modules.py @@ -12,4 +12,4 @@ class CollectModules(pyblish.api.ContextPlugin): def process(self, context): manager = ModulesManager() - context.data["openPypeModules"] = manager.modules + context.data["openPypeModules"] = manager.modules_by_name From 7fa840e8ae3954231ec0f43fba1dc81001719d48 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Wed, 11 Aug 2021 12:31:39 +0200 Subject: [PATCH 7/8] =?UTF-8?q?fix=20hound=20=F0=9F=90=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- openpype/hosts/maya/plugins/create/create_render.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/maya/plugins/create/create_render.py b/openpype/hosts/maya/plugins/create/create_render.py index 66e3fb89c9..4fd4b9d986 100644 --- a/openpype/hosts/maya/plugins/create/create_render.py +++ b/openpype/hosts/maya/plugins/create/create_render.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """Create ``Render`` instance in Maya.""" import os -from collections import OrderedDict import json import appdirs import requests @@ -273,7 +272,8 @@ class CreateRender(plugin.Creator): deadline_url = self.deadline_servers["default"] except KeyError: deadline_url = [ - self.deadline_servers[k] for k in self.deadline_servers.keys() + self.deadline_servers[k] + for k in self.deadline_servers.keys() ][0] pool_names = self._get_deadline_pools(deadline_url) From 29184a5bf17a6bd8ecc41082318ea9a17714c41f Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Wed, 11 Aug 2021 12:33:15 +0200 Subject: [PATCH 8/8] =?UTF-8?q?fix=20hound=20=F0=9F=90=95=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../deadline/plugins/publish/collect_default_deadline_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/deadline/plugins/publish/collect_default_deadline_server.py b/openpype/modules/deadline/plugins/publish/collect_default_deadline_server.py index 17851032cf..53231bd7e4 100644 --- a/openpype/modules/deadline/plugins/publish/collect_default_deadline_server.py +++ b/openpype/modules/deadline/plugins/publish/collect_default_deadline_server.py @@ -18,4 +18,4 @@ class CollectDefaultDeadlineServer(pyblish.api.ContextPlugin): # get default deadline webservice url from deadline module self.log.debug(deadline_module.deadline_urls) - context.data["defaultDeadline"] = deadline_module.deadline_urls["default"] + context.data["defaultDeadline"] = deadline_module.deadline_urls["default"] # noqa: E501