mirror of
https://github.com/ynput/ayon-core.git
synced 2025-12-24 21:04:40 +01:00
tile rendering support in pype
This commit is contained in:
Ondřej Samohel
parent
c57086b13d
commit
507be49269
4 changed files with 280 additions and 19 deletions
|
|
@ -718,7 +718,8 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
|
|||
"pixelAspect": data.get("pixelAspect", 1),
|
||||
"resolutionWidth": data.get("resolutionWidth", 1920),
|
||||
"resolutionHeight": data.get("resolutionHeight", 1080),
|
||||
"multipartExr": data.get("multipartExr", False)
|
||||
"multipartExr": data.get("multipartExr", False),
|
||||
"jobBatchName": data.get("jobBatchName", "")
|
||||
}
|
||||
|
||||
if "prerender" in instance.data["families"]:
|
||||
|
|
@ -895,8 +896,12 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
|
|||
# We still use data from it so lets fake it.
|
||||
#
|
||||
# Batch name reflect original scene name
|
||||
render_job["Props"]["Batch"] = os.path.splitext(os.path.basename(
|
||||
context.data.get("currentFile")))[0]
|
||||
|
||||
if instance.data.get("assemblySubmissionJob"):
|
||||
render_job["Props"]["Batch"] = instance.data.get("jobBatchName")
|
||||
else:
|
||||
render_job["Props"]["Batch"] = os.path.splitext(
|
||||
os.path.basename(context.data.get("currentFile")))[0]
|
||||
# User is deadline user
|
||||
render_job["Props"]["User"] = context.data.get(
|
||||
"deadlineUser", getpass.getuser())
|
||||
|
|
|
|||
|
|
@ -185,6 +185,8 @@ class CreateRender(avalon.maya.Creator):
|
|||
self.data["useMayaBatch"] = False
|
||||
self.data["vrayScene"] = False
|
||||
self.data["tileRendering"] = False
|
||||
self.data["tilesX"] = 2
|
||||
self.data["tilesY"] = 2
|
||||
# Disable for now as this feature is not working yet
|
||||
# self.data["assScene"] = False
|
||||
|
||||
|
|
|
|||
|
|
@ -243,6 +243,8 @@ class CollectMayaRender(pyblish.api.ContextPlugin):
|
|||
"resolutionHeight": cmds.getAttr("defaultResolution.height"),
|
||||
"pixelAspect": cmds.getAttr("defaultResolution.pixelAspect"),
|
||||
"tileRendering": render_instance.data.get("tileRendering") or False, # noqa: E501
|
||||
"tilesX": render_instance.data.get("tilesX") or 2,
|
||||
"tilesY": render_instance.data.get("tilesY") or 2,
|
||||
"priority": render_instance.data.get("priority")
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -16,11 +16,14 @@ Attributes:
|
|||
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
import os
|
||||
import json
|
||||
import getpass
|
||||
import copy
|
||||
import re
|
||||
import hashlib
|
||||
from datetime import datetime
|
||||
|
||||
import clique
|
||||
import requests
|
||||
|
|
@ -61,6 +64,91 @@ payload_skeleton = {
|
|||
}
|
||||
|
||||
|
||||
def _format_tiles(filename, index, tiles_x, tiles_y, width, height, prefix):
|
||||
"""Generate tile entries for Deadline tile job.
|
||||
|
||||
Returns two dictionaries - one that can be directly used in Deadline
|
||||
job, second that can be used for Deadline Assembly job configuration
|
||||
file.
|
||||
|
||||
This will format tile names:
|
||||
|
||||
Example::
|
||||
{
|
||||
"OutputFilename0Tile0": "_tile_1x1_4x4_Main_beauty.1001.exr",
|
||||
"OutputFilename0Tile1": "_tile_2x1_4x4_Main_beauty.1001.exr"
|
||||
}
|
||||
|
||||
And add tile prefixes like:
|
||||
|
||||
Example::
|
||||
Image prefix is:
|
||||
`maya/<Scene>/<RenderLayer>/<RenderLayer>_<RenderPass>`
|
||||
|
||||
Result for tile 0 for 4x4 will be:
|
||||
`maya/<Scene>/<RenderLayer>/_tile_1x1_4x4_<RenderLayer>_<RenderPass>`
|
||||
|
||||
Calculating coordinates is tricky as in Job they are defined as top,
|
||||
left, bottom, right with zero being in top-left corner. But Assembler
|
||||
configuration file takes tile coordinates as X, Y, Width and Height and
|
||||
zero is bottom left corner.
|
||||
|
||||
Args:
|
||||
filename (str): Filename to process as tiles.
|
||||
index (int): Index of that file if it is sequence.
|
||||
tiles_x (int): Number of tiles in X.
|
||||
tiles_y (int): Number if tikes in Y.
|
||||
width (int): Width resolution of final image.
|
||||
height (int): Height resolution of final image.
|
||||
prefix (str): Image prefix.
|
||||
|
||||
Returns:
|
||||
(dict, dict): Tuple of two dictionaires - first can be used to
|
||||
extend JobInfo, second has tiles x, y, width and height
|
||||
used for assembler configuration.
|
||||
|
||||
"""
|
||||
tile = 0
|
||||
out = {"JobInfo": {}, "PluginInfo": {}}
|
||||
cfg = {}
|
||||
w_space = width / tiles_x
|
||||
h_space = height / tiles_y
|
||||
|
||||
for tile_x in range(1, tiles_x + 1):
|
||||
for tile_y in range(1, tiles_y + 1):
|
||||
tile_prefix = "_tile_{}x{}_{}x{}_".format(
|
||||
tile_x, tile_y,
|
||||
tiles_x,
|
||||
tiles_y
|
||||
)
|
||||
out_tile_index = "OutputFilename{}Tile{}".format(
|
||||
str(index), tile
|
||||
)
|
||||
new_filename = "{}/{}{}".format(
|
||||
os.path.dirname(filename),
|
||||
tile_prefix,
|
||||
os.path.basename(filename)
|
||||
)
|
||||
out["JobInfo"][out_tile_index] = new_filename
|
||||
out["PluginInfo"]["RegionPrefix{}".format(str(tile))] = tile_prefix.join( # noqa: E501
|
||||
prefix.rsplit("/", 1))
|
||||
|
||||
out["PluginInfo"]["RegionTop{}".format(tile)] = int(height) - (tile_y * h_space) # noqa: E501
|
||||
out["PluginInfo"]["RegionBottom{}".format(tile)] = int(height) - ((tile_y - 1) * h_space) - 1 # noqa: E501
|
||||
out["PluginInfo"]["RegionLeft{}".format(tile)] = (tile_x - 1) * w_space # noqa: E501
|
||||
out["PluginInfo"]["RegionRight{}".format(tile)] = (tile_x * w_space) - 1 # noqa: E501
|
||||
|
||||
cfg["Tile{}".format(tile)] = new_filename
|
||||
cfg["Tile{}Tile".format(tile)] = new_filename
|
||||
cfg["Tile{}X".format(tile)] = (tile_x - 1) * w_space
|
||||
cfg["Tile{}Y".format(tile)] = (tile_y - 1) * h_space
|
||||
cfg["Tile{}Width".format(tile)] = tile_x * w_space
|
||||
cfg["Tile{}Height".format(tile)] = tile_y * h_space
|
||||
|
||||
tile += 1
|
||||
return out, cfg
|
||||
|
||||
|
||||
def get_renderer_variables(renderlayer, root):
|
||||
"""Retrieve the extension which has been set in the VRay settings.
|
||||
|
||||
|
|
@ -164,6 +252,7 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
|
|||
optional = True
|
||||
|
||||
use_published = True
|
||||
tile_assembler_plugin = "DraftTileAssembler"
|
||||
|
||||
def process(self, instance):
|
||||
"""Plugin entry point."""
|
||||
|
|
@ -309,7 +398,7 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
|
|||
# Optional, enable double-click to preview rendered
|
||||
# frames from Deadline Monitor
|
||||
payload_skeleton["JobInfo"]["OutputDirectory0"] = \
|
||||
os.path.dirname(output_filename_0)
|
||||
os.path.dirname(output_filename_0).replace("\\", "/")
|
||||
payload_skeleton["JobInfo"]["OutputFilename0"] = \
|
||||
output_filename_0.replace("\\", "/")
|
||||
|
||||
|
|
@ -376,9 +465,8 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
|
|||
|
||||
# Add list of expected files to job ---------------------------------
|
||||
exp = instance.data.get("expectedFiles")
|
||||
|
||||
output_filenames = {}
|
||||
exp_index = 0
|
||||
output_filenames = {}
|
||||
|
||||
if isinstance(exp[0], dict):
|
||||
# we have aovs and we need to iterate over them
|
||||
|
|
@ -390,33 +478,202 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
|
|||
assert len(rem) == 1, ("Found multiple non related files "
|
||||
"to render, don't know what to do "
|
||||
"with them.")
|
||||
payload['JobInfo']['OutputFilename' + str(exp_index)] = rem[0] # noqa: E501
|
||||
output_file = rem[0]
|
||||
if not instance.data.get("tileRendering"):
|
||||
payload['JobInfo']['OutputFilename' + str(exp_index)] = output_file # noqa: E501
|
||||
else:
|
||||
output_file = col[0].format('{head}{padding}{tail}')
|
||||
payload['JobInfo']['OutputFilename' + str(exp_index)] = output_file # noqa: E501
|
||||
output_filenames[exp_index] = output_file
|
||||
if not instance.data.get("tileRendering"):
|
||||
payload['JobInfo']['OutputFilename' + str(exp_index)] = output_file # noqa: E501
|
||||
|
||||
output_filenames['OutputFilename' + str(exp_index)] = output_file # noqa: E501
|
||||
exp_index += 1
|
||||
else:
|
||||
col, rem = clique.assemble(files)
|
||||
col, rem = clique.assemble(exp)
|
||||
if not col and rem:
|
||||
# we couldn't find any collections but have
|
||||
# individual files.
|
||||
assert len(rem) == 1, ("Found multiple non related files "
|
||||
"to render, don't know what to do "
|
||||
"with them.")
|
||||
payload['JobInfo']['OutputFilename' + str(exp_index)] = rem[0] # noqa: E501
|
||||
|
||||
output_file = rem[0]
|
||||
if not instance.data.get("tileRendering"):
|
||||
payload['JobInfo']['OutputFilename' + str(exp_index)] = output_file # noqa: E501
|
||||
else:
|
||||
output_file = col[0].format('{head}{padding}{tail}')
|
||||
payload['JobInfo']['OutputFilename' + str(exp_index)] = output_file # noqa: E501
|
||||
if not instance.data.get("tileRendering"):
|
||||
payload['JobInfo']['OutputFilename' + str(exp_index)] = output_file # noqa: E501
|
||||
|
||||
output_filenames['OutputFilename' + str(exp_index)] = output_file
|
||||
|
||||
plugin = payload["JobInfo"]["Plugin"]
|
||||
self.log.info("using render plugin : {}".format(plugin))
|
||||
|
||||
# Store output dir for unified publisher (filesequence)
|
||||
instance.data["outputDir"] = os.path.dirname(output_filename_0)
|
||||
|
||||
self.preflight_check(instance)
|
||||
|
||||
# Submit job to farm ------------------------------------------------
|
||||
if not instance.data.get("tileRendering"):
|
||||
# Prepare tiles data ------------------------------------------------
|
||||
if instance.data.get("tileRendering"):
|
||||
# if we have sequence of files, we need to create tile job for
|
||||
# every frame
|
||||
|
||||
payload["JobInfo"]["TileJob"] = True
|
||||
payload["JobInfo"]["TileJobTilesInX"] = instance.data.get("tilesX")
|
||||
payload["JobInfo"]["TileJobTilesInY"] = instance.data.get("tilesY")
|
||||
payload["PluginInfo"]["ImageHeight"] = instance.data.get("resolutionHeight") # noqa: E501
|
||||
payload["PluginInfo"]["ImageWidth"] = instance.data.get("resolutionWidth") # noqa: E501
|
||||
payload["PluginInfo"]["RegionRendering"] = True
|
||||
|
||||
assembly_payload = {
|
||||
"AuxFiles": [],
|
||||
"JobInfo": {
|
||||
"BatchName": payload["JobInfo"]["BatchName"],
|
||||
"Frames": 0,
|
||||
"Name": "{} - Tile Assembly Job".format(
|
||||
payload["JobInfo"]["Name"]),
|
||||
"OutputDirectory0":
|
||||
payload["JobInfo"]["OutputDirectory0"].replace(
|
||||
"\\", "/"),
|
||||
"Plugin": self.tile_assembler_plugin,
|
||||
"MachineLimit": 1
|
||||
},
|
||||
"PluginInfo": {
|
||||
"CleanupTiles": 1,
|
||||
"ErrorOnMissing": True
|
||||
}
|
||||
}
|
||||
assembly_payload["JobInfo"].update(output_filenames)
|
||||
|
||||
frame_payloads = []
|
||||
assembly_payloads = {}
|
||||
|
||||
R_FRAME_NUMBER = re.compile(r".+\.(?P<frame>[0-9]+)\..+") # noqa: N806, E501
|
||||
REPL_FRAME_NUMBER = re.compile(r"(.+\.)([0-9]+)(\..+)") # noqa: N806, E501
|
||||
|
||||
if isinstance(exp[0], dict):
|
||||
# we have aovs and we need to iterate over them
|
||||
# get files from `beauty`
|
||||
files = exp[0].get("beauty")
|
||||
if not files:
|
||||
# if beauty doesn't exists, use first aov we found
|
||||
files = exp[0].get(list(exp[0].keys())[0])
|
||||
else:
|
||||
files = exp
|
||||
|
||||
file_index = 1
|
||||
for file in files:
|
||||
frame = re.search(R_FRAME_NUMBER, file).group("frame")
|
||||
new_payload = copy.copy(payload)
|
||||
new_payload["JobInfo"]["Name"] = \
|
||||
"{} (Frame {} - {} tiles)".format(
|
||||
new_payload["JobInfo"]["Name"],
|
||||
frame,
|
||||
instance.data.get("tilesX") * instance.data.get("tilesY") # noqa: E501
|
||||
)
|
||||
new_payload["JobInfo"]["TileJobFrame"] = frame
|
||||
|
||||
tiles_data = _format_tiles(
|
||||
file, 0,
|
||||
instance.data.get("tilesX"),
|
||||
instance.data.get("tilesY"),
|
||||
instance.data.get("resolutionWidth"),
|
||||
instance.data.get("resolutionHeight"),
|
||||
payload["PluginInfo"]["OutputFilePrefix"]
|
||||
)[0]
|
||||
new_payload["JobInfo"].update(tiles_data["JobInfo"])
|
||||
new_payload["PluginInfo"].update(tiles_data["PluginInfo"])
|
||||
|
||||
job_hash = hashlib.sha256("{}_{}".format(file_index, file))
|
||||
new_payload["JobInfo"]["ExtraInfo0"] = job_hash.hexdigest()
|
||||
new_payload["JobInfo"]["ExtraInfo1"] = file
|
||||
|
||||
frame_payloads.append(new_payload)
|
||||
|
||||
new_assembly_payload = copy.copy(assembly_payload)
|
||||
new_assembly_payload["JobInfo"]["OutputFilename0"] = re.sub(
|
||||
REPL_FRAME_NUMBER,
|
||||
"\\1{}\\3".format("#" * len(frame)), file)
|
||||
|
||||
new_assembly_payload["JobInfo"]["ExtraInfo0"] = job_hash.hexdigest() # noqa: E501
|
||||
new_assembly_payload["JobInfo"]["ExtraInfo1"] = file
|
||||
assembly_payloads[job_hash.hexdigest()] = new_assembly_payload
|
||||
file_index += 1
|
||||
|
||||
self.log.info(
|
||||
"Submitting tile job(s) [{}] ...".format(len(frame_payloads)))
|
||||
|
||||
url = "{}/api/jobs".format(self._deadline_url)
|
||||
tiles_count = instance.data.get("tilesX") * instance.data.get("tilesY") # noqa: E501
|
||||
|
||||
for tile_job in frame_payloads:
|
||||
response = self._requests_post(url, json=tile_job)
|
||||
if not response.ok:
|
||||
raise Exception(response.text)
|
||||
|
||||
job_id = response.json()["_id"]
|
||||
hash = response.json()["Props"]["Ex0"]
|
||||
file = response.json()["Props"]["Ex1"]
|
||||
assembly_payloads[hash]["JobInfo"]["JobDependency0"] = job_id
|
||||
|
||||
# write assembly job config files
|
||||
now = datetime.now()
|
||||
|
||||
config_file = os.path.join(
|
||||
os.path.dirname(output_filename_0),
|
||||
"{}_config_{}.txt".format(
|
||||
os.path.splitext(file)[0],
|
||||
now.strftime("%Y_%m_%d_%H_%M_%S")
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
if not os.path.isdir(os.path.dirname(config_file)):
|
||||
os.makedirs(os.path.dirname(config_file))
|
||||
except OSError:
|
||||
# directory is not available
|
||||
self.log.warning(
|
||||
"Path is unreachable: `{}`".format(
|
||||
os.path.dirname(config_file)))
|
||||
|
||||
with open(config_file, "w") as cf:
|
||||
print("TileCount={}".format(tiles_count), file=cf)
|
||||
print("ImageFileName={}".format(file), file=cf)
|
||||
print("ImageWidth={}".format(
|
||||
instance.data.get("resolutionWidth")), file=cf)
|
||||
print("ImageHeight={}".format(
|
||||
instance.data.get("resolutionHeight")), file=cf)
|
||||
|
||||
tiles = _format_tiles(
|
||||
file, 0,
|
||||
instance.data.get("tilesX"),
|
||||
instance.data.get("tilesY"),
|
||||
instance.data.get("resolutionWidth"),
|
||||
instance.data.get("resolutionHeight"),
|
||||
payload["PluginInfo"]["OutputFilePrefix"]
|
||||
)[1]
|
||||
sorted(tiles)
|
||||
for k, v in tiles.items():
|
||||
print("{}={}".format(k, v), file=cf)
|
||||
|
||||
self.log.debug(json.dumps(assembly_payloads,
|
||||
indent=4, sort_keys=True))
|
||||
self.log.info(
|
||||
"Submitting assembly job(s) [{}] ...".format(len(assembly_payloads))) # noqa: E501
|
||||
url = "{}/api/jobs".format(self._deadline_url)
|
||||
response = self._requests_post(url, json={
|
||||
"Jobs": list(assembly_payloads.values()),
|
||||
"AuxFiles": []
|
||||
})
|
||||
if not response.ok:
|
||||
raise Exception(response)
|
||||
|
||||
instance.data["assemblySubmissionJob"] = assembly_payloads
|
||||
instance.data["jobBatchName"] = payload["JobInfo"]["BatchName"]
|
||||
else:
|
||||
# Submit job to farm --------------------------------------------
|
||||
self.log.info("Submitting ...")
|
||||
self.log.debug(json.dumps(payload, indent=4, sort_keys=True))
|
||||
|
||||
|
|
@ -426,11 +683,6 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
|
|||
if not response.ok:
|
||||
raise Exception(response.text)
|
||||
instance.data["deadlineSubmissionJob"] = response.json()
|
||||
else:
|
||||
self.log.info("Skipping submission, tile rendering enabled.")
|
||||
|
||||
# Store output dir for unified publisher (filesequence)
|
||||
instance.data["outputDir"] = os.path.dirname(output_filename_0)
|
||||
|
||||
def _get_maya_payload(self, data):
|
||||
payload = copy.deepcopy(payload_skeleton)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue