Frame13/ayon-core
1
0
Fork 1
mirror of https://github.com/ynput/ayon-core.git synced 2025-12-24 21:04:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #398 from pypeclub/feature/maya-tile-rendering

Browse source 
Maya: Publishing of tile renderings on Deadline
This commit is contained in:
Milan Kolar 2020-08-10 16:49:51 +02:00 • committed by GitHub
commit 76a6ac6bd6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 188 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

1
pype/plugins/celaction/publish/submit_celaction_deadline.py
View file

@ -34,6 +34,7 @@ class ExtractCelactionDeadline(pyblish.api.InstancePlugin):
]
def process(self, instance):
instance.data["toBeRenderedOn"] = "deadline"
context = instance.context
DEADLINE_REST_URL = os.environ.get("DEADLINE_REST_URL")

2
pype/plugins/fusion/publish/submit_deadline.py
View file

@ -22,7 +22,7 @@ class FusionSubmitDeadline(pyblish.api.InstancePlugin):
families = ["saver.deadline"]
def process(self, instance):
instance.data["toBeRenderedOn"] = "deadline"
context = instance.context
key = "__hasRun{}".format(self.__class__.__name__)

104
pype/plugins/global/publish/submit_publish_job.py
View file

@ -233,7 +233,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
return (metadata_path, roothless_mtdt_p)
def _submit_deadline_post_job(self, instance, job):
def _submit_deadline_post_job(self, instance, job, instances):
"""Submit publish job to Deadline.
Deadline specific code separated from :meth:`process` for sake of
@ -253,7 +253,6 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
"Plugin": "Python",
"BatchName": job["Props"]["Batch"],
"Name": job_name,
"JobDependency0": job["_id"],
"UserName": job["Props"]["User"],
"Comment": instance.context.data.get("comment", ""),
@ -276,12 +275,33 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
# Mandatory for Deadline, may be empty
"AuxFiles": [],
}
"""
In this part we will add file dependencies instead of job dependencies.
This way we don't need to take care of tile assembly job, getting its
id or name. We expect it to produce specific file with specific name
and we are just waiting for them.
"""
if instance.data.get("tileRendering"):
self.log.info("Adding tile assembly results as dependencies...")
asset_index = 0
for inst in instances:
for represenation in inst.get("representations", []):
if isinstance(represenation["files"], (list, tuple)):
for file in represenation["files"]:
dependency = os.path.join(output_dir, file)
payload["JobInfo"]["AssetDependency{}".format(asset_index)] = dependency # noqa: E501
else:
dependency = os.path.join(
output_dir, represenation["files"])
payload["JobInfo"]["AssetDependency{}".format(asset_index)] = dependency # noqa: E501
asset_index += 1
else:
payload["JobInfo"]["JobDependency0"] = job["_id"]
# Transfer the environment from the original job to this dependent
# job so they use the same environment
metadata_path, roothless_metadata_path = self._create_metadata_path(
instance)
environment = job["Props"].get("Env", {})
environment["PYPE_METADATA_FILE"] = roothless_metadata_path
environment["AVALON_PROJECT"] = io.Session["AVALON_PROJECT"]
@ -422,7 +442,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
# but we really expect only one collection.
# Nothing else make sense.
assert len(cols) == 1, "only one image sequence type is expected" # noqa: E501
_, ext = os.path.splitext(cols[0].tail)
ext = cols[0].tail.lstrip(".")
col = list(cols[0])
self.log.debug(col)
@ -628,25 +648,6 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
if hasattr(instance, "_log"):
data['_log'] = instance._log
render_job = data.pop("deadlineSubmissionJob", None)
submission_type = "deadline"
if not render_job:
# No deadline job. Try Muster: musterSubmissionJob
render_job = data.pop("musterSubmissionJob", None)
submission_type = "muster"
assert render_job, (
"Can't continue without valid Deadline "
"or Muster submission prior to this "
"plug-in."
)
if submission_type == "deadline":
self.DEADLINE_REST_URL = os.environ.get(
"DEADLINE_REST_URL", "http://localhost:8082"
)
assert self.DEADLINE_REST_URL, "Requires DEADLINE_REST_URL"
self._submit_deadline_post_job(instance, render_job)
asset = data.get("asset") or api.Session["AVALON_ASSET"]
subset = data.get("subset")
@ -861,6 +862,61 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
at.get("subset"), at.get("version")))
instances = new_instances
r''' SUBMiT PUBLiSH JOB 2 D34DLiN3
____
' ' .---. .---. .--. .---. .--..--..--..--. .---.
| | --= \ | . \/ _|/ \| . \ || || \ |/ _|
| JOB | --= / | | || __| .. | | | |;_ || \ || __|
| | |____./ \.__|._||_.|___./|_____|||__|\__|\.___|
._____.
'''
render_job = None
if instance.data.get("toBeRenderedOn") == "deadline":
render_job = data.pop("deadlineSubmissionJob", None)
submission_type = "deadline"
if instance.data.get("toBeRenderedOn") == "muster":
render_job = data.pop("musterSubmissionJob", None)
submission_type = "muster"
if not render_job and instance.data.get("tileRendering") is False:
raise AssertionError(("Cannot continue without valid Deadline "
"or Muster submission."))
if not render_job:
import getpass
render_job = {}
self.log.info("Faking job data ...")
render_job["Props"] = {}
# Render job doesn't exist because we do not have prior submission.
# We still use data from it so lets fake it.
#
# Batch name reflect original scene name
render_job["Props"]["Batch"] = os.path.splitext(os.path.basename(
context.data.get("currentFile")))[0]
# User is deadline user
render_job["Props"]["User"] = context.data.get(
"deadlineUser", getpass.getuser())
# Priority is now not handled at all
render_job["Props"]["Pri"] = instance.data.get("priority")
render_job["Props"]["Env"] = {
"FTRACK_API_USER": os.environ.get("FTRACK_API_USER"),
"FTRACK_API_KEY": os.environ.get("FTRACK_API_KEY"),
"FTRACK_SERVER": os.environ.get("FTRACK_SERVER"),
}
if submission_type == "deadline":
self.DEADLINE_REST_URL = os.environ.get(
"DEADLINE_REST_URL", "http://localhost:8082"
)
assert self.DEADLINE_REST_URL, "Requires DEADLINE_REST_URL"
self._submit_deadline_post_job(instance, render_job, instances)
# publish job file
publish_job = {
"asset": asset,
@ -872,7 +928,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
"version": context.data["version"], # this is workfile version
"intent": context.data.get("intent"),
"comment": context.data.get("comment"),
"job": render_job,
"job": render_job or None,
"session": api.Session.copy(),
"instances": instances
}

8
pype/plugins/maya/create/create_render.py
View file

@ -40,6 +40,9 @@ class CreateRender(avalon.maya.Creator):
vrscene (bool): Submit as ``vrscene`` file for standalone V-Ray
renderer.
ass (bool): Submit as ``ass`` file for standalone Arnold renderer.
tileRendering (bool): Instance is set to tile rendering mode. We
won't submit actuall render, but we'll make publish job to wait
for Tile Assemly job done and then publish.
See Also:
https://pype.club/docs/artist_hosts_maya#creating-basic-render-setup
@ -181,6 +184,7 @@ class CreateRender(avalon.maya.Creator):
self.data["machineList"] = ""
self.data["useMayaBatch"] = False
self.data["vrayScene"] = False
self.data["tileRendering"] = False
# Disable for now as this feature is not working yet
# self.data["assScene"] = False
@ -189,8 +193,8 @@ class CreateRender(avalon.maya.Creator):
def _load_credentials(self):
"""Load Muster credentials.
Load Muster credentials from file and set ```MUSTER_USER``,
```MUSTER_PASSWORD``, ``MUSTER_REST_URL`` is loaded from presets.
Load Muster credentials from file and set ``MUSTER_USER``,
``MUSTER_PASSWORD``, ``MUSTER_REST_URL`` is loaded from presets.
Raises:
RuntimeError: If loaded credentials are invalid.

2
pype/plugins/maya/publish/collect_render.py
View file

@ -242,6 +242,8 @@ class CollectMayaRender(pyblish.api.ContextPlugin):
"resolutionWidth": cmds.getAttr("defaultResolution.width"),
"resolutionHeight": cmds.getAttr("defaultResolution.height"),
"pixelAspect": cmds.getAttr("defaultResolution.pixelAspect"),
"tileRendering": render_instance.data.get("tileRendering") or False, # noqa: E501
"priority": render_instance.data.get("priority")
}
# Apply each user defined attribute as data

32
pype/plugins/maya/publish/submit_maya_deadline.py
View file

@ -45,6 +45,7 @@ payload_skeleton = {
"Plugin": "MayaPype",
"Frames": "{start}-{end}x{step}",
"Comment": None,
"Priority": 50,
},
"PluginInfo": {
"SceneFile": None, # Input
@ -86,7 +87,8 @@ def get_renderer_variables(renderlayer, root):
gin="#" * int(padding),
lut=True,
layer=renderlayer or lib.get_current_renderlayer())[0]
filename_0 = filename_0.replace('_<RenderPass>', '_beauty')
filename_0 = re.sub('_<RenderPass>', '_beauty',
filename_0, flags=re.IGNORECASE)
prefix_attr = "defaultRenderGlobals.imageFilePrefix"
if renderer == "vray":
renderlayer = renderlayer.split("_")[-1]
@ -165,6 +167,7 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
def process(self, instance):
"""Plugin entry point."""
instance.data["toBeRenderedOn"] = "deadline"
self._instance = instance
self._deadline_url = os.environ.get(
"DEADLINE_REST_URL", "http://localhost:8082")
@ -173,6 +176,7 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
context = instance.context
workspace = context.data["workspaceDir"]
anatomy = context.data['anatomy']
instance.data["toBeRenderedOn"] = "deadline"
filepath = None
@ -299,6 +303,9 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
payload_skeleton["JobInfo"]["Name"] = jobname
# Arbitrary username, for visualisation in Monitor
payload_skeleton["JobInfo"]["UserName"] = deadline_user
# Set job priority
payload_skeleton["JobInfo"]["Priority"] = self._instance.data.get(
"priority", 50)
# Optional, enable double-click to preview rendered
# frames from Deadline Monitor
payload_skeleton["JobInfo"]["OutputDirectory0"] = \
@ -386,7 +393,7 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
payload['JobInfo']['OutputFilename' + str(exp_index)] = rem[0] # noqa: E501
output_file = rem[0]
else:
output_file = col.format('{head}{padding}{tail}')
output_file = col[0].format('{head}{padding}{tail}')
payload['JobInfo']['OutputFilename' + str(exp_index)] = output_file # noqa: E501
output_filenames[exp_index] = output_file
exp_index += 1
@ -400,7 +407,7 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
"with them.")
payload['JobInfo']['OutputFilename' + str(exp_index)] = rem[0] # noqa: E501
else:
output_file = col.format('{head}{padding}{tail}')
output_file = col[0].format('{head}{padding}{tail}')
payload['JobInfo']['OutputFilename' + str(exp_index)] = output_file # noqa: E501
plugin = payload["JobInfo"]["Plugin"]
@ -409,18 +416,21 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
self.preflight_check(instance)
# Submit job to farm ------------------------------------------------
self.log.info("Submitting ...")
self.log.debug(json.dumps(payload, indent=4, sort_keys=True))
if not instance.data.get("tileRendering"):
self.log.info("Submitting ...")
self.log.debug(json.dumps(payload, indent=4, sort_keys=True))
# E.g. http://192.168.0.1:8082/api/jobs
url = "{}/api/jobs".format(self._deadline_url)
response = self._requests_post(url, json=payload)
if not response.ok:
raise Exception(response.text)
# E.g. http://192.168.0.1:8082/api/jobs
url = "{}/api/jobs".format(self._deadline_url)
response = self._requests_post(url, json=payload)
if not response.ok:
raise Exception(response.text)
instance.data["deadlineSubmissionJob"] = response.json()
else:
self.log.info("Skipping submission, tile rendering enabled.")
# Store output dir for unified publisher (filesequence)
instance.data["outputDir"] = os.path.dirname(output_filename_0)
instance.data["deadlineSubmissionJob"] = response.json()
def _get_maya_payload(self, data):
payload = copy.deepcopy(payload_skeleton)

1
pype/plugins/maya/publish/submit_maya_muster.py
View file

@ -249,6 +249,7 @@ class MayaSubmitMuster(pyblish.api.InstancePlugin):
Authenticate with Muster, collect all data, prepare path for post
render publish job and submit job to farm.
"""
instance.data["toBeRenderedOn"] = "muster"
# setup muster environment
self.MUSTER_REST_URL = os.environ.get("MUSTER_REST_URL")

69
pype/plugins/maya/publish/validate_deadline_tile_submission.py Normal file
View file

@ -0,0 +1,69 @@
# -*- coding: utf-8 -*-
"""Validate settings from Deadline Submitter.
This is useful mainly for tile rendering, where jobs on farm are created by
submitter script from Maya.
Unfortunately Deadline doesn't expose frame number for tiles job so that
cannot be validated, even if it is important setting. Also we cannot
determine if 'Region Rendering' (tile rendering) is enabled or not because
of the same thing.
"""
import os
from maya import mel
from maya import cmds
import pyblish.api
from pype.hosts.maya import lib
class ValidateDeadlineTileSubmission(pyblish.api.InstancePlugin):
"""Validate Deadline Submission settings are OK for tile rendering."""
label = "Validate Deadline Tile Submission"
order = pyblish.api.ValidatorOrder
hosts = ["maya"]
families = ["renderlayer"]
if not os.environ.get("DEADLINE_REST_URL"):
active = False
def process(self, instance):
"""Entry point."""
# try if Deadline submitter was loaded
if mel.eval("exists SubmitJobToDeadline") == 0:
# if not, try to load it manually
try:
mel.eval("source DeadlineMayaClient;")
except RuntimeError:
raise AssertionError("Deadline Maya client cannot be loaded")
mel.eval("DeadlineMayaClient();")
assert mel.eval("exists SubmitJobToDeadline") == 1, (
"Deadline Submission script cannot be initialized.")
if instance.data.get("tileRendering"):
job_name = cmds.getAttr("defaultRenderGlobals.deadlineJobName")
scene_name = os.path.splitext(os.path.basename(
instance.context.data.get("currentFile")))[0]
if job_name != scene_name:
self.log.warning(("Job submitted through Deadline submitter "
"has different name then current scene "
"{} / {}").format(job_name, scene_name))
if cmds.getAttr("defaultRenderGlobals.deadlineTileSingleJob") == 1:
layer = instance.data['setMembers']
anim_override = lib.get_attr_in_layer(
"defaultRenderGlobals.animation", layer=layer)
assert anim_override, (
"Animation must be enabled in "
"Render Settings even when rendering single frame."
)
start_frame = cmds.getAttr("defaultRenderGlobals.startFrame")
end_frame = cmds.getAttr("defaultRenderGlobals.endFrame")
assert start_frame == end_frame, (
"Start frame and end frame are not equals. When "
"'Submit All Tles As A Single Job' is selected, only "
"single frame is expected to be rendered. It must match "
"the one specified in Deadline Submitter under "
"'Region Rendering'"
)

6
pype/plugins/maya/publish/validate_frame_range.py
View file

@ -29,6 +29,12 @@ class ValidateFrameRange(pyblish.api.InstancePlugin):
def process(self, instance):
context = instance.context
if instance.data.get("tileRendering"):
self.log.info((
"Skipping frame range validation because "
"tile rendering is enabled."
))
return
frame_start_handle = int(context.data.get("frameStartHandle"))
frame_end_handle = int(context.data.get("frameEndHandle"))

1
pype/plugins/nuke/publish/submit_nuke_deadline.py
View file

@ -28,6 +28,7 @@ class NukeSubmitDeadline(pyblish.api.InstancePlugin):
deadline_chunk_size = 1
def process(self, instance):
instance.data["toBeRenderedOn"] = "deadline"
families = instance.data["families"]
node = instance[0]