Frame13/ayon-core
1
0
Fork 1
mirror of https://github.com/ynput/ayon-core.git synced 2025-12-25 13:24:54 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

submitting vray export job to deadline

Browse source
This commit is contained in:
Ondřej Samohel 2020-05-21 18:40:50 +02:00
parent 5241359106
commit be9302580f
No known key found for this signature in database
GPG key ID: 8A29C663C672C2B7
5 changed files with 350 additions and 137 deletions
Download patch file Download diff file Expand all files Collapse all files

88
pype/plugins/maya/create/create_render.py
View file

@ -1,3 +1,5 @@
# -*- coding: utf-8 -*-
"""Create ``Render`` instance in Maya."""
import os
import json
import appdirs
@ -11,7 +13,38 @@ import avalon.maya
class CreateRender(avalon.maya.Creator):
"""Create render layer for export"""
"""Create *render* instance.
Render instances are not actually published, they hold options for
collecting of render data. It render instance is present, it will trigger
collection of render layers, AOVs, cameras for either direct submission
to render farm or export as various standalone formats (like V-Rays
``vrscenes`` or Arnolds ``ass`` files) and then submitting them to render
farm.
Instance has following attributes::
primaryPool (list of str): Primary list of slave machine pool to use.
secondaryPool (list of str): Optional secondary list of slave pools.
suspendPublishJob (bool): Suspend the job after it is submitted.
extendFrames (bool): Use already existing frames from previous version
to extend current render.
overrideExistingFrame (bool): Overwrite already existing frames.
priority (int): Submitted job priority
framesPerTask (int): How many frames per task to render. This is
basically job division on render farm.
whitelist (list of str): White list of slave machines
machineList (list of str): Specific list of slave machines to use
useMayaBatch (bool): Use Maya batch mode to render as opposite to
Maya interactive mode. This consumes different licenses.
vrscene (bool): Submit as ``vrscene`` file for standalone V-Ray
renderer.
ass (bool): Submit as ``ass`` file for standalone Arnold renderer.
See Also:
https://pype.club/docs/artist_hosts_maya#creating-basic-render-setup
"""
label = "Render"
family = "rendering"
@ -42,9 +75,11 @@ class CreateRender(avalon.maya.Creator):
}
def __init__(self, *args, **kwargs):
"""Constructor."""
super(CreateRender, self).__init__(*args, **kwargs)
def process(self):
"""Entry point."""
exists = cmds.ls(self.name)
if exists:
return cmds.warning("%s already exists." % exists[0])
@ -145,17 +180,21 @@ class CreateRender(avalon.maya.Creator):
self.data["whitelist"] = False
self.data["machineList"] = ""
self.data["useMayaBatch"] = True
self.data["vrayScene"] = False
self.data["assScene"] = False
self.options = {"useSelection": False} # Force no content
def _load_credentials(self):
"""
Load Muster credentials from file and set `MUSTER_USER`,
`MUSTER_PASSWORD`, `MUSTER_REST_URL` is loaded from presets.
"""Load Muster credentials.
.. todo::
Load Muster credentials from file and set ```MUSTER_USER``,
```MUSTER_PASSWORD``, ``MUSTER_REST_URL`` is loaded from presets.
Raises:
RuntimeError: If loaded credentials are invalid.
AttributeError: If ``MUSTER_REST_URL`` is not set.
Show login dialog if access token is invalid or missing.
"""
app_dir = os.path.normpath(appdirs.user_data_dir("pype-app", "pype"))
file_name = "muster_cred.json"
@ -172,8 +211,11 @@ class CreateRender(avalon.maya.Creator):
raise AttributeError("Muster REST API url not set")
def _get_muster_pools(self):
"""
Get render pools from muster
"""Get render pools from Muster.
Raises:
Exception: If pool list cannot be obtained from Muster.
"""
params = {"authToken": self._token}
api_entry = "/api/pools/list"
@ -209,14 +251,17 @@ class CreateRender(avalon.maya.Creator):
raise Exception("Cannot show login form to Muster")
def _requests_post(self, *args, **kwargs):
""" Wrapper for requests, disabling SSL certificate validation if
DONT_VERIFY_SSL environment variable is found. This is useful when
Deadline or Muster server are running with self-signed certificates
and their certificate is not added to trusted certificates on
client machines.
"""Wrap request post method.
WARNING: disabling SSL certificate validation is defeating one line
Disabling SSL certificate validation if ``DONT_VERIFY_SSL`` environment
variable is found. This is useful when Deadline or Muster server are
running with self-signed certificates and their certificate is not
added to trusted certificates on client machines.
Warning:
Disabling SSL certificate validation is defeating one line
of defense SSL is providing and it is not recommended.
"""
if "verify" not in kwargs:
kwargs["verify"] = (
@ -225,14 +270,17 @@ class CreateRender(avalon.maya.Creator):
return requests.post(*args, **kwargs)
def _requests_get(self, *args, **kwargs):
""" Wrapper for requests, disabling SSL certificate validation if
DONT_VERIFY_SSL environment variable is found. This is useful when
Deadline or Muster server are running with self-signed certificates
and their certificate is not added to trusted certificates on
client machines.
"""Wrap request get method.
WARNING: disabling SSL certificate validation is defeating one line
Disabling SSL certificate validation if ``DONT_VERIFY_SSL`` environment
variable is found. This is useful when Deadline or Muster server are
running with self-signed certificates and their certificate is not
added to trusted certificates on client machines.
Warning:
Disabling SSL certificate validation is defeating one line
of defense SSL is providing and it is not recommended.
"""
if "verify" not in kwargs:
kwargs["verify"] = (

17
pype/plugins/maya/publish/collect_render.py
View file

@ -125,7 +125,7 @@ class CollectMayaRender(pyblish.api.ContextPlugin):
if "family" not in cmds.listAttr(s):
continue
attachTo.append(
attach_to.append(
{
"version": None, # we need integrator for that
"subset": s,
@ -170,15 +170,15 @@ class CollectMayaRender(pyblish.api.ContextPlugin):
if isinstance(exp_files[0], dict):
for aov, files in exp_files[0].items():
full_paths = []
for ef in files:
full_path = os.path.join(workspace, "renders", ef)
for e in files:
full_path = os.path.join(workspace, "renders", e)
full_path = full_path.replace("\\", "/")
full_paths.append(full_path)
aov_dict[aov] = full_paths
else:
full_paths = []
for ef in exp_files:
full_path = os.path.join(workspace, "renders", ef)
for e in exp_files:
full_path = os.path.join(workspace, "renders", e)
full_path = full_path.replace("\\", "/")
full_paths.append(full_path)
aov_dict["beauty"] = full_paths
@ -255,6 +255,13 @@ class CollectMayaRender(pyblish.api.ContextPlugin):
data[attr] = value
# handle standalone renderers
if render_instance.data.get("vrayScene") is True:
data["families"].append("vrayscene")
if render_instance.data.get("ass") is True:
data["families"].append("assScene")
# Include (optional) global settings
# Get global overrides and translate to Deadline values
overrides = self.parse_options(str(render_globals))

2
pype/plugins/maya/publish/collect_vray_scene.py
View file

@ -15,7 +15,7 @@ class CollectVRayScene(pyblish.api.ContextPlugin):
order = pyblish.api.CollectorOrder
label = "Collect VRay Scene"
hosts = ["maya"]
hosts = ["foo"]
def process(self, context):
"""Collector entry point."""

374
pype/plugins/maya/publish/submit_maya_deadline.py
View file

@ -3,26 +3,60 @@
This module is taking care of submitting job from Maya to Deadline. It
creates job and set correct environments. Its behavior is controlled by
`DEADLINE_REST_URL` environment variable - pointing to Deadline Web Service
and `MayaSubmitDeadline.use_published (bool)` property telling Deadline to
``DEADLINE_REST_URL`` environment variable - pointing to Deadline Web Service
and :data:`MayaSubmitDeadline.use_published` property telling Deadline to
use published scene workfile or not.
If ``vrscene`` or ``assscene`` are detected in families, it will first
submit job to export these files and then dependent job to render them.
Attributes:
payload_skeleton (dict): Skeleton payload data sent as job to Deadline.
Default values are for ``MayaBatch`` plugin.
"""
import os
import json
import getpass
import re
import copy
import clique
import requests
from maya import cmds
from avalon import api
from avalon.vendor import requests
import pyblish.api
import pype.maya.lib as lib
# Documentation for keys available at:
# https://docs.thinkboxsoftware.com
# /products/deadline/8.0/1_User%20Manual/manual
# /manual-submission.html#job-info-file-options
payload_skeleton = {
"JobInfo": {
"BatchName": None, # Top-level group name
"Name": None, # Job name, as seen in Monitor
"UserName": None,
"Plugin": "MayaBatch",
"Frames": "{start}-{end}x{step}",
"Comment": None,
},
"PluginInfo": {
"SceneFile": None, # Input
"OutputFilePath": None, # Output directory and filename
"OutputFilePrefix": None,
"Version": cmds.about(version=True), # Mandatory for Deadline
"UsingRenderLayers": True,
"RenderLayer": None, # Render only this layer
"Renderer": None,
"ProjectPath": None, # Resolve relative references
},
"AuxFiles": [] # Mandatory for Deadline, may be empty
}
def get_renderer_variables(renderlayer=None):
"""Retrieve the extension which has been set in the VRay settings.
@ -91,7 +125,15 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
"""Submit available render layers to Deadline.
Renders are submitted to a Deadline Web Service as
supplied via the environment variable DEADLINE_REST_URL
supplied via the environment variable ``DEADLINE_REST_URL``.
Note:
If Deadline configuration is not detected, this plugin will
be disabled.
Attributes:
use_published (bool): Use published scene to render instead of the
one in work area.
"""
@ -108,10 +150,11 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
use_published = True
def process(self, instance):
DEADLINE_REST_URL = os.environ.get("DEADLINE_REST_URL",
"http://localhost:8082")
assert DEADLINE_REST_URL, "Requires DEADLINE_REST_URL"
"""Plugin entry point."""
self._instance = instance
self._deadline_url = os.environ.get(
"DEADLINE_REST_URL", "http://localhost:8082")
assert self._deadline_url, "Requires DEADLINE_REST_URL"
context = instance.context
workspace = context.data["workspaceDir"]
@ -119,6 +162,7 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
filepath = None
# Handle render/export from published scene or not ------------------
if self.use_published:
for i in context:
if "workfile" in i.data["families"]:
@ -166,11 +210,11 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
orig_scene, new_scene
))
allInstances = []
all_instances = []
for result in context.data["results"]:
if (result["instance"] is not None and
result["instance"] not in allInstances):
allInstances.append(result["instance"])
result["instance"] not in all_instances): # noqa: E128
all_instances.append(result["instance"])
# fallback if nothing was set
if not filepath:
@ -179,6 +223,7 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
self.log.debug(filepath)
# Gather needed data ------------------------------------------------
filename = os.path.basename(filepath)
comment = context.data.get("comment", "")
dirname = os.path.join(workspace, "renders")
@ -198,91 +243,49 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
output_filename_0 = filename_0
# Create render folder ----------------------------------------------
try:
# Ensure render folder exists
os.makedirs(dirname)
except OSError:
pass
# Documentation for keys available at:
# https://docs.thinkboxsoftware.com
# /products/deadline/8.0/1_User%20Manual/manual
# /manual-submission.html#job-info-file-options
payload = {
"JobInfo": {
# Top-level group name
"BatchName": filename,
# Fill in common data to payload ------------------------------------
payload_data = {}
payload_data["filename"] = filename
payload_data["filepath"] = filepath
payload_data["jobname"] = jobname
payload_data["deadline_user"] = deadline_user
payload_data["comment"] = comment
payload_data["output_filename_0"] = output_filename_0
payload_data["render_variables"] = render_variables
payload_data["renderlayer"] = renderlayer
payload_data["workspace"] = workspace
# Asset dependency to wait for at least the scene file to sync.
"AssetDependency0": filepath,
frame_pattern = payload_skeleton["JobInfo"]["Frames"]
payload_skeleton["JobInfo"]["Frames"] = frame_pattern.format(
start=int(self._instance.data["frameStartHandle"]),
end=int(self._instance.data["frameEndHandle"]),
step=int(self._instance.data["byFrameStep"]))
# Job name, as seen in Monitor
"Name": jobname,
payload_skeleton["JobInfo"]["Plugin"] = self._instance.data.get(
"mayaRenderPlugin", "MayaBatch")
# Arbitrary username, for visualisation in Monitor
"UserName": deadline_user,
payload_skeleton["JobInfo"]["BatchName"] = filename
# Job name, as seen in Monitor
payload_skeleton["JobInfo"]["Name"] = jobname
# Arbitrary username, for visualisation in Monitor
payload_skeleton["JobInfo"]["UserName"] = deadline_user
# Optional, enable double-click to preview rendered
# frames from Deadline Monitor
payload_skeleton["JobInfo"]["OutputDirectory0"] = \
os.path.dirname(output_filename_0)
payload_skeleton["JobInfo"]["OutputFilename0"] = \
output_filename_0.replace("\\", "/")
"Plugin": instance.data.get("mayaRenderPlugin", "MayaBatch"),
"Frames": "{start}-{end}x{step}".format(
start=int(instance.data["frameStartHandle"]),
end=int(instance.data["frameEndHandle"]),
step=int(instance.data["byFrameStep"]),
),
"Comment": comment,
# Optional, enable double-click to preview rendered
# frames from Deadline Monitor
"OutputDirectory0": os.path.dirname(output_filename_0),
"OutputFilename0": output_filename_0.replace("\\", "/")
},
"PluginInfo": {
# Input
"SceneFile": filepath,
# Output directory and filename
"OutputFilePath": dirname.replace("\\", "/"),
"OutputFilePrefix": render_variables["filename_prefix"],
# Mandatory for Deadline
"Version": cmds.about(version=True),
# Only render layers are considered renderable in this pipeline
"UsingRenderLayers": True,
# Render only this layer
"RenderLayer": renderlayer,
# Determine which renderer to use from the file itself
"Renderer": instance.data["renderer"],
# Resolve relative references
"ProjectPath": workspace,
},
# Mandatory for Deadline, may be empty
"AuxFiles": []
}
exp = instance.data.get("expectedFiles")
OutputFilenames = {}
expIndex = 0
if isinstance(exp[0], dict):
# we have aovs and we need to iterate over them
for aov, files in exp[0].items():
col = clique.assemble(files)[0][0]
outputFile = col.format('{head}{padding}{tail}')
payload['JobInfo']['OutputFilename' + str(expIndex)] = outputFile # noqa: E501
OutputFilenames[expIndex] = outputFile
expIndex += 1
else:
col = clique.assemble(files)[0][0]
outputFile = col.format('{head}{padding}{tail}')
payload['JobInfo']['OutputFilename' + str(expIndex)] = outputFile
# OutputFilenames[expIndex] = outputFile
payload_skeleton["JobInfo"]["Comment"] = comment
# Handle environments -----------------------------------------------
# We need those to pass them to pype for it to set correct context
keys = [
"FTRACK_API_KEY",
@ -298,27 +301,67 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
environment = dict({key: os.environ[key] for key in keys
if key in os.environ}, **api.Session)
payload["JobInfo"].update({
payload_skeleton["JobInfo"].update({
"EnvironmentKeyValue%d" % index: "{key}={value}".format(
key=key,
value=environment[key]
) for index, key in enumerate(environment)
})
# Include optional render globals
# Add options from RenderGlobals-------------------------------------
render_globals = instance.data.get("renderGlobals", {})
payload["JobInfo"].update(render_globals)
payload_skeleton["JobInfo"].update(render_globals)
# Submit preceeding export jobs -------------------------------------
export_job = None
if "vrayscene" in instance.data["families"]:
export_job = self._submit_export(payload_data, "vray")
if "assscene" in instance.data["families"]:
export_job = self._submit_export(payload_data, "arnold")
# Prepare main render job -------------------------------------------
if "vrayscene" in instance.data["families"]:
payload = self._get_vray_render_payload(payload_data)
elif "assscene" in instance.data["families"]:
pass
else:
payload = self._get_maya_payload(payload_data)
# Add export job as dependency --------------------------------------
if export_job:
payload["JobInfo"]["JobDependency0"] = export_job
# Add list of expected files to job ---------------------------------
exp = instance.data.get("expectedFiles")
output_filenames = {}
exp_index = 0
if isinstance(exp[0], dict):
# we have aovs and we need to iterate over them
for aov, files in exp[0].items():
col = clique.assemble(files)[0][0]
output_file = col.format('{head}{padding}{tail}')
payload['JobInfo']['OutputFilename' + str(exp_index)] = output_file # noqa: E501
output_filenames[exp_index] = output_file
exp_index += 1
else:
col = clique.assemble(files)[0][0]
output_file = col.format('{head}{padding}{tail}')
payload['JobInfo']['OutputFilename' + str(exp_index)] = output_file
# OutputFilenames[exp_index] = output_file
plugin = payload["JobInfo"]["Plugin"]
self.log.info("using render plugin : {}".format(plugin))
self.preflight_check(instance)
# Submit job to farm ------------------------------------------------
self.log.info("Submitting ...")
self.log.info(json.dumps(payload, indent=4, sort_keys=True))
self.log.debug(json.dumps(payload, indent=4, sort_keys=True))
# E.g. http://192.168.0.1:8082/api/jobs
url = "{}/api/jobs".format(DEADLINE_REST_URL)
url = "{}/api/jobs".format(self._deadline_url)
response = self._requests_post(url, json=payload)
if not response.ok:
raise Exception(response.text)
@ -327,9 +370,118 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
instance.data["outputDir"] = os.path.dirname(filename_0)
instance.data["deadlineSubmissionJob"] = response.json()
def preflight_check(self, instance):
"""Ensure the startFrame, endFrame and byFrameStep are integers"""
def _get_maya_payload(self, data):
payload = copy.deepcopy(payload_skeleton)
job_info_ext = {
# Asset dependency to wait for at least the scene file to sync.
"AssetDependency0": data["filepath"],
}
plugin_info = {
"SceneFile": data["filepath"],
# Output directory and filename
"OutputFilePath": data["dirname"].replace("\\", "/"),
"OutputFilePrefix": data["render_variables"]["filename_prefix"], # noqa: E501
# Only render layers are considered renderable in this pipeline
"UsingRenderLayers": True,
# Render only this layer
"RenderLayer": data["renderlayer"],
# Determine which renderer to use from the file itself
"Renderer": self._instance.data["renderer"],
# Resolve relative references
"ProjectPath": data["workspace"],
}
payload["JobInfo"].update(job_info_ext)
payload["PluginInfo"].update(plugin_info)
return payload
def _get_vray_export_payload(self, data):
payload = copy.deepcopy(payload_skeleton)
job_info_ext = {
# Job name, as seen in Monitor
"Name": "Export {} [{}-{}]".format(
data["jobname"],
int(self._instance.data["frameStartHandle"]),
int(self._instance.data["frameEndHandle"])),
"Plugin": "MayaBatch",
"FramesPerTask": self._instance.data.get("framesPerTask", 1)
}
plugin_info = {
# Renderer
"Renderer": "vray",
# Input
"SceneFile": data["filepath"],
"SkipExistingFrames": True,
"UsingRenderLayers": True,
"UseLegacyRenderLayers": True
}
payload["JobInfo"].update(job_info_ext)
payload["PluginInfo"].update(plugin_info)
return payload
def _get_vray_render_payload(self, data):
payload = copy.deepcopy(payload_skeleton)
first_file = data["output_filename_0"]
ext, _ = os.path.splitext(first_file)
first_file = first_file.replace(ext, "vrscene")
first_file = first_file.replace(
"#" * data["render_variables"]["padding"],
"{:04d}".format(int(self._instance.data["frameStartHandle"])))
job_info_ext = {
"Name": "Render {} [{}-{}]".format(
data["jobname"],
int(self._instance.data["frameStartHandle"]),
int(self._instance.data["frameEndHandle"])),
"Plugin": "Vray",
"OverrideTaskExtraInfoNames": False,
}
plugin_info = {
"InputFilename": first_file,
"SeparateFilesPerFrame": True,
"VRayEngine": "V-Ray",
"Width": self._instance.data["resolutionWidth"],
"Height": self._instance.data["resolutionHeight"],
}
payload["JobInfo"].update(job_info_ext)
payload["PluginInfo"].update(plugin_info)
return payload
def _submit_export(self, data, format):
if format == "vray":
payload = self._get_vray_export_payload(data)
self.log.info("Submitting vrscene export job.")
elif format == "ass":
payload = self._get_arnold_export_payload(data)
self.log.info("Submitting ass export job.")
url = "{}/api/jobs".format(self._deadline_url)
response = self._requests_post(url, json=payload)
if not response.ok:
self.log.error("Submition failed!")
self.log.error(response.status_code)
self.log.error(response.content)
self.log.debug(payload)
raise RuntimeError(response.text)
dependency = response.json()
return dependency["_id"]
def preflight_check(self, instance):
"""Ensure the startFrame, endFrame and byFrameStep are integers."""
for key in ("frameStartHandle", "frameEndHandle", "byFrameStep"):
value = instance.data[key]
@ -342,14 +494,17 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
)
def _requests_post(self, *args, **kwargs):
""" Wrapper for requests, disabling SSL certificate validation if
DONT_VERIFY_SSL environment variable is found. This is useful when
Deadline or Muster server are running with self-signed certificates
and their certificate is not added to trusted certificates on
client machines.
"""Wrap request post method.
WARNING: disabling SSL certificate validation is defeating one line
Disabling SSL certificate validation if ``DONT_VERIFY_SSL`` environment
variable is found. This is useful when Deadline or Muster server are
running with self-signed certificates and their certificate is not
added to trusted certificates on client machines.
Warning:
Disabling SSL certificate validation is defeating one line
of defense SSL is providing and it is not recommended.
"""
if 'verify' not in kwargs:
kwargs['verify'] = False if os.getenv("PYPE_DONT_VERIFY_SSL", True) else True # noqa
@ -358,14 +513,17 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
return requests.post(*args, **kwargs)
def _requests_get(self, *args, **kwargs):
""" Wrapper for requests, disabling SSL certificate validation if
DONT_VERIFY_SSL environment variable is found. This is useful when
Deadline or Muster server are running with self-signed certificates
and their certificate is not added to trusted certificates on
client machines.
"""Wrap request get method.
WARNING: disabling SSL certificate validation is defeating one line
Disabling SSL certificate validation if ``DONT_VERIFY_SSL`` environment
variable is found. This is useful when Deadline or Muster server are
running with self-signed certificates and their certificate is not
added to trusted certificates on client machines.
Warning:
Disabling SSL certificate validation is defeating one line
of defense SSL is providing and it is not recommended.
"""
if 'verify' not in kwargs:
kwargs['verify'] = False if os.getenv("PYPE_DONT_VERIFY_SSL", True) else True # noqa

6
pype/plugins/maya/publish/submit_vray_deadline.py
View file

@ -24,7 +24,7 @@ class VraySubmitDeadline(pyblish.api.InstancePlugin):
label = "Submit to Deadline ( vrscene )"
order = pyblish.api.IntegratorOrder
hosts = ["maya"]
families = ["vrayscene"]
families = ["vrayscene_foo"]
if not os.environ.get("DEADLINE_REST_URL"):
optional = False
active = False
@ -130,7 +130,7 @@ class VraySubmitDeadline(pyblish.api.InstancePlugin):
start_frame = int(instance.data["frameStart"])
end_frame = int(instance.data["frameEnd"])
ext = instance.data.get("ext", "exr")
ext = instance.data.get("ext") or "exr"
# Create output directory for renders
render_ouput = self.format_output_filename(instance,
@ -238,7 +238,7 @@ class VraySubmitDeadline(pyblish.api.InstancePlugin):
for index, k in enumerate(env)}
def format_output_filename(self, instance, filename, template, dir=False):
"""Format the expected output file of the Export job
"""Format the expected output file of the Export job.
Example:
<Scene>/<Scene>_<Layer>/<Layer>