Frame13/ayon-core
1
0
Fork 1
mirror of https://github.com/ynput/ayon-core.git synced 2025-12-24 21:04:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'develop' into enhancement/OP-2956_move-host-install

Browse source
This commit is contained in:
Jakub Trllo 2022-04-14 12:45:27 +02:00
commit 26bd9e8ff3
26 changed files with 343 additions and 319 deletions
Download patch file Download diff file Expand all files Collapse all files

2
openpype/hosts/aftereffects/plugins/publish/collect_render.py
View file

@ -25,7 +25,7 @@ class AERenderInstance(RenderInstance):
class CollectAERender(abstract_collect_render.AbstractCollectRender):
order = pyblish.api.CollectorOrder + 0.498
order = pyblish.api.CollectorOrder + 0.400
label = "Collect After Effects Render Layers"
hosts = ["aftereffects"]

116
openpype/hosts/maya/plugins/create/create_render.py
View file

@ -4,8 +4,6 @@ import os
import json
import appdirs
import requests
import six
import sys
from maya import cmds
import maya.app.renderSetup.model.renderSetup as renderSetup
@ -14,6 +12,7 @@ from openpype.hosts.maya.api import (
lib,
plugin
)
from openpype.lib import requests_get
from openpype.api import (
get_system_settings,
get_project_settings,
@ -117,6 +116,8 @@ class CreateRender(plugin.Creator):
except KeyError:
self.aov_separator = "_"
manager = ModulesManager()
self.deadline_module = manager.modules_by_name["deadline"]
try:
default_servers = deadline_settings["deadline_urls"]
project_servers = (
@ -133,10 +134,8 @@ class CreateRender(plugin.Creator):
except AttributeError:
# Handle situation were we had only one url for deadline.
manager = ModulesManager()
deadline_module = manager.modules_by_name["deadline"]
# get default deadline webservice url from deadline module
self.deadline_servers = deadline_module.deadline_urls
self.deadline_servers = self.deadline_module.deadline_urls
def process(self):
"""Entry point."""
@ -205,48 +204,31 @@ class CreateRender(plugin.Creator):
def _deadline_webservice_changed(self):
"""Refresh Deadline server dependent options."""
# get selected server
from maya import cmds
webservice = self.deadline_servers[
self.server_aliases[
cmds.getAttr("{}.deadlineServers".format(self.instance))
]
]
pools = self._get_deadline_pools(webservice)
pools = self.deadline_module.get_deadline_pools(webservice, self.log)
cmds.deleteAttr("{}.primaryPool".format(self.instance))
cmds.deleteAttr("{}.secondaryPool".format(self.instance))
pool_setting = (self._project_settings["deadline"]
["publish"]
["CollectDeadlinePools"])
primary_pool = pool_setting["primary_pool"]
sorted_pools = self._set_default_pool(list(pools), primary_pool)
cmds.addAttr(self.instance, longName="primaryPool",
attributeType="enum",
enumName=":".join(pools))
cmds.addAttr(self.instance, longName="secondaryPool",
enumName=":".join(sorted_pools))
pools = ["-"] + pools
secondary_pool = pool_setting["secondary_pool"]
sorted_pools = self._set_default_pool(list(pools), secondary_pool)
cmds.addAttr("{}.secondaryPool".format(self.instance),
attributeType="enum",
enumName=":".join(["-"] + pools))
def _get_deadline_pools(self, webservice):
# type: (str) -> list
"""Get pools from Deadline.
Args:
webservice (str): Server url.
Returns:
list: Pools.
Throws:
RuntimeError: If deadline webservice is unreachable.
"""
argument = "{}/api/pools?NamesOnly=true".format(webservice)
try:
response = self._requests_get(argument)
except requests.exceptions.ConnectionError as exc:
msg = 'Cannot connect to deadline web service'
self.log.error(msg)
six.reraise(
RuntimeError,
RuntimeError('{} - {}'.format(msg, exc)),
sys.exc_info()[2])
if not response.ok:
self.log.warning("No pools retrieved")
return []
return response.json()
enumName=":".join(sorted_pools))
def _create_render_settings(self):
"""Create instance settings."""
@ -295,7 +277,8 @@ class CreateRender(plugin.Creator):
# use first one for initial list of pools.
deadline_url = next(iter(self.deadline_servers.values()))
pool_names = self._get_deadline_pools(deadline_url)
pool_names = self.deadline_module.get_deadline_pools(deadline_url,
self.log)
maya_submit_dl = self._project_settings.get(
"deadline", {}).get(
"publish", {}).get(
@ -326,12 +309,27 @@ class CreateRender(plugin.Creator):
self.log.info(" - pool: {}".format(pool["name"]))
pool_names.append(pool["name"])
self.data["primaryPool"] = pool_names
pool_setting = (self._project_settings["deadline"]
["publish"]
["CollectDeadlinePools"])
primary_pool = pool_setting["primary_pool"]
self.data["primaryPool"] = self._set_default_pool(pool_names,
primary_pool)
# We add a string "-" to allow the user to not
# set any secondary pools
self.data["secondaryPool"] = ["-"] + pool_names
pool_names = ["-"] + pool_names
secondary_pool = pool_setting["secondary_pool"]
self.data["secondaryPool"] = self._set_default_pool(pool_names,
secondary_pool)
self.options = {"useSelection": False} # Force no content
def _set_default_pool(self, pool_names, pool_value):
"""Reorder pool names, default should come first"""
if pool_value and pool_value in pool_names:
pool_names.remove(pool_value)
pool_names = [pool_value] + pool_names
return pool_names
def _load_credentials(self):
"""Load Muster credentials.
@ -366,7 +364,7 @@ class CreateRender(plugin.Creator):
"""
params = {"authToken": self._token}
api_entry = "/api/pools/list"
response = self._requests_get(self.MUSTER_REST_URL + api_entry,
response = requests_get(self.MUSTER_REST_URL + api_entry,
params=params)
if response.status_code != 200:
if response.status_code == 401:
@ -392,45 +390,11 @@ class CreateRender(plugin.Creator):
api_url = "{}/muster/show_login".format(
os.environ["OPENPYPE_WEBSERVER_URL"])
self.log.debug(api_url)
login_response = self._requests_get(api_url, timeout=1)
login_response = requests_get(api_url, timeout=1)
if login_response.status_code != 200:
self.log.error("Cannot show login form to Muster")
raise Exception("Cannot show login form to Muster")
def _requests_post(self, *args, **kwargs):
"""Wrap request post method.
Disabling SSL certificate validation if ``DONT_VERIFY_SSL`` environment
variable is found. This is useful when Deadline or Muster server are
running with self-signed certificates and their certificate is not
added to trusted certificates on client machines.
Warning:
Disabling SSL certificate validation is defeating one line
of defense SSL is providing and it is not recommended.
"""
if "verify" not in kwargs:
kwargs["verify"] = not os.getenv("OPENPYPE_DONT_VERIFY_SSL", True)
return requests.post(*args, **kwargs)
def _requests_get(self, *args, **kwargs):
"""Wrap request get method.
Disabling SSL certificate validation if ``DONT_VERIFY_SSL`` environment
variable is found. This is useful when Deadline or Muster server are
running with self-signed certificates and their certificate is not
added to trusted certificates on client machines.
Warning:
Disabling SSL certificate validation is defeating one line
of defense SSL is providing and it is not recommended.
"""
if "verify" not in kwargs:
kwargs["verify"] = not os.getenv("OPENPYPE_DONT_VERIFY_SSL", True)
return requests.get(*args, **kwargs)
def _set_default_renderer_settings(self, renderer):
"""Set basic settings based on renderer.

86
openpype/hosts/maya/plugins/create/create_vrayscene.py
View file

@ -4,8 +4,6 @@ import os
import json
import appdirs
import requests
import six
import sys
from maya import cmds
import maya.app.renderSetup.model.renderSetup as renderSetup
@ -19,6 +17,7 @@ from openpype.api import (
get_project_settings
)
from openpype.lib import requests_get
from openpype.pipeline import CreatorError
from openpype.modules import ModulesManager
@ -40,6 +39,10 @@ class CreateVRayScene(plugin.Creator):
self._rs = renderSetup.instance()
self.data["exportOnFarm"] = False
deadline_settings = get_system_settings()["modules"]["deadline"]
manager = ModulesManager()
self.deadline_module = manager.modules_by_name["deadline"]
if not deadline_settings["enabled"]:
self.deadline_servers = {}
return
@ -62,10 +65,8 @@ class CreateVRayScene(plugin.Creator):
except AttributeError:
# Handle situation were we had only one url for deadline.
manager = ModulesManager()
deadline_module = manager.modules_by_name["deadline"]
# get default deadline webservice url from deadline module
self.deadline_servers = deadline_module.deadline_urls
self.deadline_servers = self.deadline_module.deadline_urls
def process(self):
"""Entry point."""
@ -128,7 +129,7 @@ class CreateVRayScene(plugin.Creator):
cmds.getAttr("{}.deadlineServers".format(self.instance))
]
]
pools = self._get_deadline_pools(webservice)
pools = self.deadline_module.get_deadline_pools(webservice)
cmds.deleteAttr("{}.primaryPool".format(self.instance))
cmds.deleteAttr("{}.secondaryPool".format(self.instance))
cmds.addAttr(self.instance, longName="primaryPool",
@ -138,33 +139,6 @@ class CreateVRayScene(plugin.Creator):
attributeType="enum",
enumName=":".join(["-"] + pools))
def _get_deadline_pools(self, webservice):
# type: (str) -> list
"""Get pools from Deadline.
Args:
webservice (str): Server url.
Returns:
list: Pools.
Throws:
RuntimeError: If deadline webservice is unreachable.
"""
argument = "{}/api/pools?NamesOnly=true".format(webservice)
try:
response = self._requests_get(argument)
except requests.exceptions.ConnectionError as exc:
msg = 'Cannot connect to deadline web service'
self.log.error(msg)
six.reraise(
CreatorError,
CreatorError('{} - {}'.format(msg, exc)),
sys.exc_info()[2])
if not response.ok:
self.log.warning("No pools retrieved")
return []
return response.json()
def _create_vray_instance_settings(self):
# get pools
pools = []
@ -195,7 +169,7 @@ class CreateVRayScene(plugin.Creator):
for k in self.deadline_servers.keys()
][0]
pool_names = self._get_deadline_pools(deadline_url)
pool_names = self.deadline_module.get_deadline_pools(deadline_url)
if muster_enabled:
self.log.info(">>> Loading Muster credentials ...")
@ -259,8 +233,8 @@ class CreateVRayScene(plugin.Creator):
"""
params = {"authToken": self._token}
api_entry = "/api/pools/list"
response = self._requests_get(self.MUSTER_REST_URL + api_entry,
params=params)
response = requests_get(self.MUSTER_REST_URL + api_entry,
params=params)
if response.status_code != 200:
if response.status_code == 401:
self.log.warning("Authentication token expired.")
@ -285,45 +259,7 @@ class CreateVRayScene(plugin.Creator):
api_url = "{}/muster/show_login".format(
os.environ["OPENPYPE_WEBSERVER_URL"])
self.log.debug(api_url)
login_response = self._requests_get(api_url, timeout=1)
login_response = requests_get(api_url, timeout=1)
if login_response.status_code != 200:
self.log.error("Cannot show login form to Muster")
raise CreatorError("Cannot show login form to Muster")
def _requests_post(self, *args, **kwargs):
"""Wrap request post method.
Disabling SSL certificate validation if ``DONT_VERIFY_SSL`` environment
variable is found. This is useful when Deadline or Muster server are
running with self-signed certificates and their certificate is not
added to trusted certificates on client machines.
Warning:
Disabling SSL certificate validation is defeating one line
of defense SSL is providing and it is not recommended.
"""
if "verify" not in kwargs:
kwargs["verify"] = (
False if os.getenv("OPENPYPE_DONT_VERIFY_SSL", True) else True
) # noqa
return requests.post(*args, **kwargs)
def _requests_get(self, *args, **kwargs):
"""Wrap request get method.
Disabling SSL certificate validation if ``DONT_VERIFY_SSL`` environment
variable is found. This is useful when Deadline or Muster server are
running with self-signed certificates and their certificate is not
added to trusted certificates on client machines.
Warning:
Disabling SSL certificate validation is defeating one line
of defense SSL is providing and it is not recommended.
"""
if "verify" not in kwargs:
kwargs["verify"] = (
False if os.getenv("OPENPYPE_DONT_VERIFY_SSL", True) else True
) # noqa
return requests.get(*args, **kwargs)

6
openpype/hosts/maya/plugins/publish/collect_render.py
View file

@ -386,6 +386,12 @@ class CollectMayaRender(pyblish.api.ContextPlugin):
overrides = self.parse_options(str(render_globals))
data.update(**overrides)
# get string values for pools
primary_pool = overrides["renderGlobals"]["Pool"]
secondary_pool = overrides["renderGlobals"].get("SecondaryPool")
data["primaryPool"] = primary_pool
data["secondaryPool"] = secondary_pool
# Define nice label
label = "{0} ({1})".format(expected_layer_name, data["asset"])
label += " [{0}-{1}]".format(

19
openpype/hosts/maya/plugins/publish/submit_maya_muster.py
View file

@ -4,13 +4,13 @@ import getpass
import platform
import appdirs
import requests
from maya import cmds
from avalon import api
import pyblish.api
from openpype.lib import requests_post
from openpype.hosts.maya.api import lib
from openpype.api import get_system_settings
@ -184,7 +184,7 @@ class MayaSubmitMuster(pyblish.api.InstancePlugin):
"select": "name"
}
api_entry = '/api/templates/list'
response = self._requests_post(
response = requests_post(
self.MUSTER_REST_URL + api_entry, params=params)
if response.status_code != 200:
self.log.error(
@ -235,7 +235,7 @@ class MayaSubmitMuster(pyblish.api.InstancePlugin):
"name": "submit"
}
api_entry = '/api/queue/actions'
response = self._requests_post(
response = requests_post(
self.MUSTER_REST_URL + api_entry, params=params, json=payload)
if response.status_code != 200:
@ -549,16 +549,3 @@ class MayaSubmitMuster(pyblish.api.InstancePlugin):
% (value, int(value))
)
def _requests_post(self, *args, **kwargs):
""" Wrapper for requests, disabling SSL certificate validation if
DONT_VERIFY_SSL environment variable is found. This is useful when
Deadline or Muster server are running with self-signed certificates
and their certificate is not added to trusted certificates on
client machines.
WARNING: disabling SSL certificate validation is defeating one line
of defense SSL is providing and it is not recommended.
"""
if 'verify' not in kwargs:
kwargs['verify'] = False if os.getenv("OPENPYPE_DONT_VERIFY_SSL", True) else True # noqa
return requests.post(*args, **kwargs)

34
openpype/hosts/maya/plugins/publish/validate_muster_connection.py
View file

@ -2,9 +2,9 @@ import os
import json
import appdirs
import requests
import pyblish.api
from openpype.lib import requests_get
from openpype.plugin import contextplugin_should_run
import openpype.hosts.maya.api.action
@ -51,7 +51,7 @@ class ValidateMusterConnection(pyblish.api.ContextPlugin):
'authToken': self._token
}
api_entry = '/api/pools/list'
response = self._requests_get(
response = requests_get(
MUSTER_REST_URL + api_entry, params=params)
assert response.status_code == 200, "invalid response from server"
assert response.json()['ResponseData'], "invalid data in response"
@ -88,35 +88,7 @@ class ValidateMusterConnection(pyblish.api.ContextPlugin):
api_url = "{}/muster/show_login".format(
os.environ["OPENPYPE_WEBSERVER_URL"])
cls.log.debug(api_url)
response = cls._requests_get(api_url, timeout=1)
response = requests_get(api_url, timeout=1)
if response.status_code != 200:
cls.log.error('Cannot show login form to Muster')
raise Exception('Cannot show login form to Muster')
def _requests_post(self, *args, **kwargs):
""" Wrapper for requests, disabling SSL certificate validation if
DONT_VERIFY_SSL environment variable is found. This is useful when
Deadline or Muster server are running with self-signed certificates
and their certificate is not added to trusted certificates on
client machines.
WARNING: disabling SSL certificate validation is defeating one line
of defense SSL is providing and it is not recommended.
"""
if 'verify' not in kwargs:
kwargs['verify'] = False if os.getenv("OPENPYPE_DONT_VERIFY_SSL", True) else True # noqa
return requests.post(*args, **kwargs)
def _requests_get(self, *args, **kwargs):
""" Wrapper for requests, disabling SSL certificate validation if
DONT_VERIFY_SSL environment variable is found. This is useful when
Deadline or Muster server are running with self-signed certificates
and their certificate is not added to trusted certificates on
client machines.
WARNING: disabling SSL certificate validation is defeating one line
of defense SSL is providing and it is not recommended.
"""
if 'verify' not in kwargs:
kwargs['verify'] = False if os.getenv("OPENPYPE_DONT_VERIFY_SSL", True) else True # noqa
return requests.get(*args, **kwargs)

2
openpype/hosts/nuke/plugins/inventory/select_containers.py
View file

@ -1,5 +1,5 @@
from openpype.pipeline import InventoryAction
from openpype.hosts.nuke.api.commands import viewer_update_and_undo_stop
from openpype.hosts.nuke.api.command import viewer_update_and_undo_stop
class SelectContainers(InventoryAction):

2
openpype/hosts/nuke/plugins/load/load_backdrop.py
View file

@ -14,7 +14,7 @@ from openpype.hosts.nuke.api.lib import (
get_avalon_knob_data,
set_avalon_knob_data
)
from openpype.hosts.nuke.api.commands import viewer_update_and_undo_stop
from openpype.hosts.nuke.api.command import viewer_update_and_undo_stop
from openpype.hosts.nuke.api import containerise, update_container

9
openpype/lib/__init__.py
View file

@ -221,6 +221,12 @@ from .openpype_version import (
is_current_version_higher_than_expected
)
from .connections import (
requests_get,
requests_post
)
terminal = Terminal
__all__ = [
@ -390,4 +396,7 @@ __all__ = [
"is_running_from_build",
"is_running_staging",
"is_current_version_studio_latest",
"requests_get",
"requests_post"
]

38
openpype/lib/connections.py Normal file
View file

@ -0,0 +1,38 @@
import requests
import os
def requests_post(*args, **kwargs):
"""Wrap request post method.
Disabling SSL certificate validation if ``DONT_VERIFY_SSL`` environment
variable is found. This is useful when Deadline or Muster server are
running with self-signed certificates and their certificate is not
added to trusted certificates on client machines.
Warning:
Disabling SSL certificate validation is defeating one line
of defense SSL is providing and it is not recommended.
"""
if "verify" not in kwargs:
kwargs["verify"] = not os.getenv("OPENPYPE_DONT_VERIFY_SSL", True)
return requests.post(*args, **kwargs)
def requests_get(*args, **kwargs):
"""Wrap request get method.
Disabling SSL certificate validation if ``DONT_VERIFY_SSL`` environment
variable is found. This is useful when Deadline or Muster server are
running with self-signed certificates and their certificate is not
added to trusted certificates on client machines.
Warning:
Disabling SSL certificate validation is defeating one line
of defense SSL is providing and it is not recommended.
"""
if "verify" not in kwargs:
kwargs["verify"] = not os.getenv("OPENPYPE_DONT_VERIFY_SSL", True)
return requests.get(*args, **kwargs)

43
openpype/modules/deadline/deadline_module.py
View file

@ -1,8 +1,19 @@
import os
import requests
import six
import sys
from openpype.lib import requests_get, PypeLogger
from openpype.modules import OpenPypeModule
from openpype_interfaces import IPluginPaths
class DeadlineWebserviceError(Exception):
"""
Exception to throw when connection to Deadline server fails.
"""
class DeadlineModule(OpenPypeModule, IPluginPaths):
name = "deadline"
@ -32,3 +43,35 @@ class DeadlineModule(OpenPypeModule, IPluginPaths):
return {
"publish": [os.path.join(current_dir, "plugins", "publish")]
}
@staticmethod
def get_deadline_pools(webservice, log=None):
# type: (str) -> list
"""Get pools from Deadline.
Args:
webservice (str): Server url.
log (Logger)
Returns:
list: Pools.
Throws:
RuntimeError: If deadline webservice is unreachable.
"""
if not log:
log = PypeLogger.get_logger(__name__)
argument = "{}/api/pools?NamesOnly=true".format(webservice)
try:
response = requests_get(argument)
except requests.exceptions.ConnectionError as exc:
msg = 'Cannot connect to DL web service {}'.format(webservice)
log.error(msg)
six.reraise(
DeadlineWebserviceError,
DeadlineWebserviceError('{} - {}'.format(msg, exc)),
sys.exc_info()[2])
if not response.ok:
log.warning("No pools retrieved")
return []
return response.json()

2
openpype/modules/deadline/plugins/publish/collect_deadline_server_from_instance.py
View file

@ -11,7 +11,7 @@ import pyblish.api
class CollectDeadlineServerFromInstance(pyblish.api.InstancePlugin):
"""Collect Deadline Webservice URL from instance."""
order = pyblish.api.CollectorOrder + 0.02
order = pyblish.api.CollectorOrder + 0.415
label = "Deadline Webservice from the Instance"
families = ["rendering"]

2
openpype/modules/deadline/plugins/publish/collect_default_deadline_server.py
View file

@ -6,7 +6,7 @@ import pyblish.api
class CollectDefaultDeadlineServer(pyblish.api.ContextPlugin):
"""Collect default Deadline Webservice URL."""
order = pyblish.api.CollectorOrder + 0.01
order = pyblish.api.CollectorOrder + 0.410
label = "Default Deadline Webservice"
pass_mongo_url = False

23
openpype/modules/deadline/plugins/publish/collect_pools.py Normal file
View file

@ -0,0 +1,23 @@
# -*- coding: utf-8 -*-
"""Collect Deadline pools. Choose default one from Settings
"""
import pyblish.api
class CollectDeadlinePools(pyblish.api.InstancePlugin):
"""Collect pools from instance if present, from Setting otherwise."""
order = pyblish.api.CollectorOrder + 0.420
label = "Collect Deadline Pools"
families = ["rendering", "render.farm", "renderFarm", "renderlayer"]
primary_pool = None
secondary_pool = None
def process(self, instance):
if not instance.data.get("primaryPool"):
instance.data["primaryPool"] = self.primary_pool or "none"
if not instance.data.get("secondaryPool"):
instance.data["secondaryPool"] = self.secondary_pool or "none"

31
openpype/modules/deadline/plugins/publish/help/validate_deadline_pools.xml Normal file
View file

@ -0,0 +1,31 @@
<?xml version="1.0" encoding="UTF-8"?>
<root>
<error id="main">
<title>Scene setting</title>
<description>
## Invalid Deadline pools found
Configured pools don't match what is set in Deadline.
{invalid_value_str}
### How to repair?
If your instance had deadline pools set on creation, remove or
change them.
In other cases inform admin to change them in Settings.
Available deadline pools {pools_str}.
</description>
<detail>
### __Detailed Info__
This error is shown when deadline pool is not on Deadline anymore. It
could happen in case of republish old workfile which was created with
previous deadline pools,
or someone changed pools on Deadline side, but didn't modify Openpype
Settings.
</detail>
</error>
</root>

6
openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py
View file

@ -37,8 +37,6 @@ class AfterEffectsSubmitDeadline(
priority = 50
chunk_size = 1000000
primary_pool = None
secondary_pool = None
group = None
department = None
multiprocess = True
@ -62,8 +60,8 @@ class AfterEffectsSubmitDeadline(
dln_job_info.Frames = frame_range
dln_job_info.Priority = self.priority
dln_job_info.Pool = self.primary_pool
dln_job_info.SecondaryPool = self.secondary_pool
dln_job_info.Pool = self._instance.data.get("primaryPool")
dln_job_info.SecondaryPool = self._instance.data.get("secondaryPool")
dln_job_info.Group = self.group
dln_job_info.Department = self.department
dln_job_info.ChunkSize = self.chunk_size

6
openpype/modules/deadline/plugins/publish/submit_harmony_deadline.py
View file

@ -241,8 +241,6 @@ class HarmonySubmitDeadline(
optional = True
use_published = False
primary_pool = ""
secondary_pool = ""
priority = 50
chunk_size = 1000000
group = "none"
@ -259,8 +257,8 @@ class HarmonySubmitDeadline(
# for now, get those from presets. Later on it should be
# configurable in Harmony UI directly.
job_info.Priority = self.priority
job_info.Pool = self.primary_pool
job_info.SecondaryPool = self.secondary_pool
job_info.Pool = self._instance.data.get("primaryPool")
job_info.SecondaryPool = self._instance.data.get("secondaryPool")
job_info.ChunkSize = self.chunk_size
job_info.BatchName = os.path.basename(self._instance.data["source"])
job_info.Department = self.department

5
openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py
View file

@ -7,7 +7,7 @@ from avalon import api
import pyblish.api
import hou
# import hou ???
class HoudiniSubmitRenderDeadline(pyblish.api.InstancePlugin):
@ -71,7 +71,8 @@ class HoudiniSubmitRenderDeadline(pyblish.api.InstancePlugin):
"UserName": deadline_user,
"Plugin": "Houdini",
"Pool": "houdini_redshift", # todo: remove hardcoded pool
"Pool": instance.data.get("primaryPool"),
"secondaryPool": instance.data.get("secondaryPool"),
"Frames": frames,
"ChunkSize": instance.data.get("chunkSize", 10),

47
openpype/modules/deadline/plugins/publish/submit_maya_deadline.py
View file

@ -35,6 +35,7 @@ from maya import cmds
from avalon import api
import pyblish.api
from openpype.lib import requests_post
from openpype.hosts.maya.api import lib
# Documentation for keys available at:
@ -700,7 +701,7 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
tiles_count = instance.data.get("tilesX") * instance.data.get("tilesY") # noqa: E501
for tile_job in frame_payloads:
response = self._requests_post(url, json=tile_job)
response = requests_post(url, json=tile_job)
if not response.ok:
raise Exception(response.text)
@ -763,7 +764,7 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
job_idx, len(assembly_payloads)
))
self.log.debug(json.dumps(ass_job, indent=4, sort_keys=True))
response = self._requests_post(url, json=ass_job)
response = requests_post(url, json=ass_job)
if not response.ok:
raise Exception(response.text)
@ -781,7 +782,7 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
# E.g. http://192.168.0.1:8082/api/jobs
url = "{}/api/jobs".format(self.deadline_url)
response = self._requests_post(url, json=payload)
response = requests_post(url, json=payload)
if not response.ok:
raise Exception(response.text)
instance.data["deadlineSubmissionJob"] = response.json()
@ -989,7 +990,7 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
self.log.info("Submitting ass export job.")
url = "{}/api/jobs".format(self.deadline_url)
response = self._requests_post(url, json=payload)
response = requests_post(url, json=payload)
if not response.ok:
self.log.error("Submition failed!")
self.log.error(response.status_code)
@ -1013,44 +1014,6 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
% (value, int(value))
)
def _requests_post(self, *args, **kwargs):
"""Wrap request post method.
Disabling SSL certificate validation if ``DONT_VERIFY_SSL`` environment
variable is found. This is useful when Deadline or Muster server are
running with self-signed certificates and their certificate is not
added to trusted certificates on client machines.
Warning:
Disabling SSL certificate validation is defeating one line
of defense SSL is providing and it is not recommended.
"""
if 'verify' not in kwargs:
kwargs['verify'] = not os.getenv("OPENPYPE_DONT_VERIFY_SSL", True)
# add 10sec timeout before bailing out
kwargs['timeout'] = 10
return requests.post(*args, **kwargs)
def _requests_get(self, *args, **kwargs):
"""Wrap request get method.
Disabling SSL certificate validation if ``DONT_VERIFY_SSL`` environment
variable is found. This is useful when Deadline or Muster server are
running with self-signed certificates and their certificate is not
added to trusted certificates on client machines.
Warning:
Disabling SSL certificate validation is defeating one line
of defense SSL is providing and it is not recommended.
"""
if 'verify' not in kwargs:
kwargs['verify'] = not os.getenv("OPENPYPE_DONT_VERIFY_SSL", True)
# add 10sec timeout before bailing out
kwargs['timeout'] = 10
return requests.get(*args, **kwargs)
def format_vray_output_filename(self, filename, template, dir=False):
"""Format the expected output file of the Export job.

6
openpype/modules/deadline/plugins/publish/submit_nuke_deadline.py
View file

@ -28,8 +28,6 @@ class NukeSubmitDeadline(pyblish.api.InstancePlugin):
priority = 50
chunk_size = 1
concurrent_tasks = 1
primary_pool = ""
secondary_pool = ""
group = ""
department = ""
limit_groups = {}
@ -187,8 +185,8 @@ class NukeSubmitDeadline(pyblish.api.InstancePlugin):
"Department": self.department,
"Pool": self.primary_pool,
"SecondaryPool": self.secondary_pool,
"Pool": instance.data.get("primaryPool"),
"SecondaryPool": instance.data.get("secondaryPool"),
"Group": self.group,
"Plugin": "Nuke",

43
openpype/modules/deadline/plugins/publish/submit_publish_job.py
View file

@ -8,6 +8,7 @@ from copy import copy, deepcopy
import requests
import clique
import openpype.api
from openpype.pipeline.farm.patterning import match_aov_pattern
from avalon import api, io
@ -107,7 +108,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
families = ["render.farm", "prerender.farm",
"renderlayer", "imagesequence", "vrayscene"]
aov_filter = {"maya": [r".*(?:[\._-])*([Bb]eauty)(?:[\.|_])*.*"],
aov_filter = {"maya": [r".*([Bb]eauty).*"],
"aftereffects": [r".*"], # for everything from AE
"harmony": [r".*"], # for everything from AE
"celaction": [r".*"]}
@ -129,7 +130,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
"OPENPYPE_PUBLISH_JOB"
]
# custom deadline atributes
# custom deadline attributes
deadline_department = ""
deadline_pool = ""
deadline_pool_secondary = ""
@ -259,8 +260,8 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
"Priority": priority,
"Group": self.deadline_group,
"Pool": self.deadline_pool,
"SecondaryPool": self.deadline_pool_secondary,
"Pool": instance.data.get("primaryPool"),
"SecondaryPool": instance.data.get("secondaryPool"),
"OutputDirectory0": output_dir
},
@ -449,12 +450,15 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
app = os.environ.get("AVALON_APP", "")
preview = False
if app in self.aov_filter.keys():
for aov_pattern in self.aov_filter[app]:
if re.match(aov_pattern, aov):
preview = True
break
if isinstance(col, list):
render_file_name = os.path.basename(col[0])
else:
render_file_name = os.path.basename(col)
aov_patterns = self.aov_filter
preview = match_aov_pattern(app, aov_patterns, render_file_name)
# toggle preview on if multipart is on
if instance_data.get("multipartExr"):
preview = True
@ -532,22 +536,17 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
# should be review made.
# - "review" tag is never added when is set to 'False'
if instance["useSequenceForReview"]:
# if filtered aov name is found in filename, toggle it for
# preview video rendering
for app in self.aov_filter.keys():
if os.environ.get("AVALON_APP", "") == app:
# iteratre all aov filters
for aov in self.aov_filter[app]:
if re.match(
aov,
list(collection)[0]
):
preview = True
break
# toggle preview on if multipart is on
if instance.get("multipartExr", False):
preview = True
else:
render_file_name = list(collection)[0]
host_name = os.environ.get("AVALON_APP", "")
# if filtered aov name is found in filename, toggle it for
# preview video rendering
preview = match_aov_pattern(
host_name, self.aov_filter, render_file_name
)
staging = os.path.dirname(list(collection)[0])
success, rootless_staging_dir = (

48
openpype/modules/deadline/plugins/publish/validate_deadline_pools.py Normal file
View file

@ -0,0 +1,48 @@
import pyblish.api
from openpype.pipeline import (
PublishXmlValidationError,
OptionalPyblishPluginMixin
)
from openpype.modules.deadline.deadline_module import DeadlineModule
class ValidateDeadlinePools(OptionalPyblishPluginMixin,
pyblish.api.InstancePlugin):
"""Validate primaryPool and secondaryPool on instance.
Values are on instance based on value insertion when Creating instance or
by Settings in CollectDeadlinePools.
"""
label = "Validate Deadline Pools"
order = pyblish.api.ValidatorOrder
families = ["rendering", "render.farm", "renderFarm", "renderlayer"]
optional = True
def process(self, instance):
# get default deadline webservice url from deadline module
deadline_url = instance.context.data["defaultDeadline"]
self.log.info("deadline_url::{}".format(deadline_url))
pools = DeadlineModule.get_deadline_pools(deadline_url, log=self.log)
self.log.info("pools::{}".format(pools))
formatting_data = {
"pools_str": ",".join(pools)
}
primary_pool = instance.data.get("primaryPool")
if primary_pool and primary_pool not in pools:
msg = "Configured primary '{}' not present on Deadline".format(
instance.data["primaryPool"])
formatting_data["invalid_value_str"] = msg
raise PublishXmlValidationError(self, msg,
formatting_data=formatting_data)
secondary_pool = instance.data.get("secondaryPool")
if secondary_pool and secondary_pool not in pools:
msg = "Configured secondary '{}' not present on Deadline".format(
instance.data["secondaryPool"])
formatting_data["invalid_value_str"] = msg
raise PublishXmlValidationError(self, msg,
formatting_data=formatting_data)

0
openpype/pipeline/farm/__init__.py Normal file
View file

24
openpype/pipeline/farm/patterning.py Normal file
View file

@ -0,0 +1,24 @@
# -*- coding: utf-8 -*-
import re
def match_aov_pattern(host_name, aov_patterns, render_file_name):
"""Matching against a `AOV` pattern in the render files.
In order to match the AOV name we must compare
against the render filename string that we are
grabbing the render filename string from the collection
that we have grabbed from `exp_files`.
Args:
app (str): Host name.
aov_patterns (dict): AOV patterns from AOV filters.
render_file_name (str): Incoming file name to match against.
Returns:
bool: Review state for rendered file (render_file_name).
"""
aov_pattern = aov_patterns.get(host_name, [])
if not aov_pattern:
return False
return any(re.match(p, render_file_name) for p in aov_pattern)

14
openpype/settings/defaults/project_settings/deadline.json
View file

@ -4,6 +4,10 @@
"CollectDefaultDeadlineServer": {
"pass_mongo_url": false
},
"CollectDeadlinePools": {
"primary_pool": "",
"secondary_pool": ""
},
"ValidateExpectedFiles": {
"enabled": true,
"active": true,
@ -38,8 +42,6 @@
"priority": 50,
"chunk_size": 10,
"concurrent_tasks": 1,
"primary_pool": "",
"secondary_pool": "",
"group": "",
"department": "",
"use_gpu": true,
@ -54,8 +56,6 @@
"use_published": true,
"priority": 50,
"chunk_size": 10000,
"primary_pool": "",
"secondary_pool": "",
"group": "",
"department": ""
},
@ -66,8 +66,6 @@
"use_published": true,
"priority": 50,
"chunk_size": 10000,
"primary_pool": "",
"secondary_pool": "",
"group": "",
"department": "",
"multiprocess": true
@ -83,7 +81,7 @@
"skip_integration_repre_list": [],
"aov_filter": {
"maya": [
".+(?:\\.|_)([Bb]eauty)(?:\\.|_).*"
".*([Bb]eauty).*"
],
"nuke": [
".*"
@ -100,4 +98,4 @@
}
}
}
}
}

48
openpype/settings/entities/schemas/projects_schema/schema_project_deadline.json
View file

@ -30,6 +30,24 @@
}
]
},
{
"type": "dict",
"collapsible": true,
"key": "CollectDeadlinePools",
"label": "Default Deadline Pools",
"children": [
{
"type": "text",
"key": "primary_pool",
"label": "Primary Pool"
},
{
"type": "text",
"key": "secondary_pool",
"label": "Secondary Pool"
}
]
},
{
"type": "dict",
"collapsible": true,
@ -223,16 +241,6 @@
{
"type": "splitter"
},
{
"type": "text",
"key": "primary_pool",
"label": "Primary Pool"
},
{
"type": "text",
"key": "secondary_pool",
"label": "Secondary Pool"
},
{
"type": "text",
"key": "group",
@ -313,16 +321,6 @@
"key": "chunk_size",
"label": "Chunk Size"
},
{
"type": "text",
"key": "primary_pool",
"label": "Primary Pool"
},
{
"type": "text",
"key": "secondary_pool",
"label": "Secondary Pool"
},
{
"type": "text",
"key": "group",
@ -372,16 +370,6 @@
"key": "chunk_size",
"label": "Chunk Size"
},
{
"type": "text",
"key": "primary_pool",
"label": "Primary Pool"
},
{
"type": "text",
"key": "secondary_pool",
"label": "Secondary Pool"
},
{
"type": "text",
"key": "group",