More refactoring/cleanup (WIP)

This commit is contained in:
Roy Nieterau 2022-08-29 23:21:30 +02:00
parent 7af7f71eda
commit f91e33c038

View file

@ -70,7 +70,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline):
tile_assembler_plugin = "OpenPypeTileAssembler"
priority = 50
tile_priority = 50
limit_groups = []
limit = [] # limit groups
jobInfo = {}
pluginInfo = {}
group = "none"
@ -112,23 +112,18 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline):
job_info.ChunkSize = instance.data.get("chunkSize", 10)
job_info.Comment = context.data.get("comment")
job_info.Priority = instance.data.get("priority", self.priority)
job_info.FramesPerTask = instance.data.get("framesPerTask", 1)
if self.group != "none" and self.group:
job_info.Group = self.group
if self.limit_groups:
job_info.LimitGroups = ",".join(self.limit_groups)
if self.limit:
job_info.LimitGroups = ",".join(self.limit)
# Optional, enable double-click to preview rendered
# frames from Deadline Monitor
self.payload_skeleton["JobInfo"]["OutputDirectory0"] = \
os.path.dirname(output_filename_0).replace("\\", "/")
self.payload_skeleton["JobInfo"]["OutputFilename0"] = \
output_filename_0.replace("\\", "/")
# Add options from RenderGlobals-------------------------------------
# Add options from RenderGlobals
render_globals = instance.data.get("renderGlobals", {})
self.payload_skeleton["JobInfo"].update(render_globals)
for key, value in render_globals:
setattr(job_info, key, value)
keys = [
"FTRACK_API_KEY",
@ -140,7 +135,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline):
"AVALON_TASK",
"AVALON_APP_NAME",
"OPENPYPE_DEV",
"OPENPYPE_LOG_NO_COLORS",
"OPENPYPE_VERSION"
]
# Add mongo url if it's enabled
@ -150,10 +144,8 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline):
environment = dict({key: os.environ[key] for key in keys
if key in os.environ}, **legacy_io.Session)
# TODO: Taken from old publish class - test whether still needed
environment["OPENPYPE_LOG_NO_COLORS"] = "1"
environment["OPENPYPE_MAYA_VERSION"] = cmds.about(v=True)
# to recognize job from PYPE for turning Event On/Off
environment["OPENPYPE_RENDER_JOB"] = "1"
@ -166,7 +158,10 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline):
)
# to recognize job from PYPE for turning Event On/Off
job_info.EnvironmentKeyValue = "OPENPYPE_RENDER_JOB=1"
job_info.EnvironmentKeyValue = "OPENPYPE_LOG_NO_COLORS=1"
# Optional, enable double-click to preview rendered
# frames from Deadline Monitor
for i, filepath in enumerate(instance.data["files"]):
dirname = os.path.dirname(filepath)
fname = os.path.basename(filepath)
@ -213,14 +208,13 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline):
instance = self._instance
context = instance.context
renderlayer = instance.data['setMembers'] # rs_beauty
# Output driver to render
plugin_info = DeadlinePluginInfo(
SceneFile=context.data["currentFile"],
SceneFile=self.scene_path,
Version=cmds.about(version=True),
RenderLayer=renderlayer,
RenderSetupIncludeLights=instance.data.get("renderSetupIncludeLights") # noqa
RenderLayer=instance.data['setMembers'],
RenderSetupIncludeLights=instance.data.get("renderSetupIncludeLights"), # noqa
ProjectPath=context.data["workspaceDir"],
UsingRenderLayers=True,
)
plugin_payload = attr.asdict(plugin_info)
@ -236,12 +230,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline):
instance = self._instance
context = instance.context
# Generated by AbstractSubmitDeadline. The `job_info`, `plugin_info`
# and `aux_files` are the skeleton payloads that are the basis for
# all the maya submissions
job_info = self.job_info
plugin_info = self.plugin_info
aux_files = self.aux_files
filepath = self.scene_path # publish if `use_publish` else workfile
# TODO: Avoid the need for this logic here, needed for submit publish
@ -250,18 +238,9 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline):
instance.data["outputDir"] = output_dir
instance.data["toBeRenderedOn"] = "deadline"
self.limit_groups = self.limit
# Patch workfile (only when use_published is enabled)
if self.use_published:
patches = (
context.data["project_settings"].get(
"deadline", {}).get(
"publish", {}).get(
"MayaSubmitDeadline", {}).get(
"scene_patches", {})
)
self._patch_workfile(filepath, patches)
self._patch_workfile()
# Gather needed data ------------------------------------------------
workspace = context.data["workspaceDir"]
@ -271,22 +250,6 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline):
.get('default_render_image_folder')
filename = os.path.basename(filepath)
dirname = os.path.join(workspace, default_render_file)
renderlayer = instance.data['setMembers'] # rs_beauty
# Get the variables depending on the renderer
# TODO: Find replacement logic for `get_renderer_variables` through
# what is collected for the render or is implemented in maya
# api `lib_renderproducts`
render_variables = get_renderer_variables(renderlayer, dirname)
filename_0 = render_variables["filename_0"]
if self.use_published:
new_scene = os.path.splitext(filename)[0]
orig_scene = os.path.splitext(
os.path.basename(context.data["currentFile"]))[0]
filename_0 = render_variables["filename_0"].replace(
orig_scene, new_scene)
output_filename_0 = filename_0
# this is needed because renderman handles directory and file
# prefixes separately
@ -301,16 +264,18 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline):
pass
# Fill in common data to payload ------------------------------------
# TODO: Replace these with collected data from CollectRender
payload_data = {
"filename": filename,
"filepath": filepath,
"jobname": jobname,
"output_filename_0": output_filename_0,
"renderlayer": renderlayer,
"workspace": workspace,
"dirname": dirname,
}
# Store output dir for unified publisher (filesequence)
instance.data["outputDir"] = os.path.dirname(output_filename_0)
# Submit preceding export jobs -------------------------------------
export_job = None
assert not all(x in instance.data["families"]
@ -333,17 +298,16 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline):
# Add export job as dependency --------------------------------------
if export_job:
payload["JobInfo"]["JobDependency0"] = export_job
# Store output dir for unified publisher (filesequence)
instance.data["outputDir"] = os.path.dirname(output_filename_0)
job_info, _ = payload
job_info.JobDependency = export_job
if instance.data.get("tileRendering"):
# Prepare tiles data
self._tile_render(instance, payload)
else:
# Submit main render job
self.submit(payload)
job_info, plugin_info = payload
self.submit(self.assemble_payload(job_info, plugin_info))
def _tile_render(self, instance, payload):
@ -546,18 +510,12 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline):
instance.data["jobBatchName"]))
def _get_maya_payload(self, data):
payload = copy.deepcopy(self.payload_skeleton)
if not self.asset_dependencies:
job_info_ext = {}
job_info = copy.deepcopy(self.job_info)
else:
job_info_ext = {
# Asset dependency to wait for at least the scene file to sync.
"AssetDependency0": data["filepath"],
}
renderer = self._instance.data["renderer"]
if self.asset_dependencies:
# Asset dependency to wait for at least the scene file to sync.
job_info.AssetDependency = self.scene_path
# Get layer prefix
render_products = self._instance.data["renderProducts"]
@ -569,6 +527,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline):
# Renderman 22, and so if we are using renderman > 21 we need to set
# renderer string on the job to `renderman22`. We will have to change
# this when Deadline releases new version handling this.
renderer = self._instance.data["renderer"]
if renderer == "renderman":
try:
from rfm2.config import cfg # noqa
@ -580,29 +539,20 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline):
renderer = "renderman22"
plugin_info = {
"SceneFile": data["filepath"],
# Output directory and filename
"OutputFilePath": data["dirname"].replace("\\", "/"),
"OutputFilePrefix": layer_prefix,
# Only render layers are considered renderable in this pipeline
"UsingRenderLayers": True,
# Render only this layer
"RenderLayer": data["renderlayer"],
# Determine which renderer to use from the file itself
"Renderer": renderer,
# Resolve relative references
"ProjectPath": data["workspace"],
}
payload["JobInfo"].update(job_info_ext)
payload["PluginInfo"].update(plugin_info)
return payload
return job_info, plugin_info
def _get_vray_export_payload(self, data):
payload = copy.deepcopy(self.payload_skeleton)
job_info = copy.deepcopy(self.job_info)
job_info.Name = self._job_info_label("Export")
# Get V-Ray settings info to compute output path
vray_settings = cmds.ls(type="VRaySettingsNode")
node = vray_settings[0]
template = cmds.getAttr("{}.vrscene_filename".format(node))
@ -610,34 +560,15 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline):
first_file = self.format_vray_output_filename(scene, template)
first_file = "{}/{}".format(data["workspace"], first_file)
output = os.path.dirname(first_file)
job_info_ext = {
# Job name, as seen in Monitor
"Name": "Export {} [{}-{}]".format(
data["jobname"],
int(self._instance.data["frameStartHandle"]),
int(self._instance.data["frameEndHandle"])),
"Plugin": self._instance.data.get(
"mayaRenderPlugin", "MayaPype"),
"FramesPerTask": self._instance.data.get("framesPerTask", 1)
}
plugin_info_ext = {
# Renderer
plugin_info = {
"Renderer": "vray",
# Input
"SceneFile": data["filepath"],
"SkipExistingFrames": True,
"UsingRenderLayers": True,
"UseLegacyRenderLayers": True,
"RenderLayer": data["renderlayer"],
"ProjectPath": data["workspace"],
"OutputFilePath": output
}
payload["JobInfo"].update(job_info_ext)
payload["PluginInfo"].update(plugin_info_ext)
return payload
return job_info, plugin_info
def _get_arnold_export_payload(self, data):
@ -653,76 +584,55 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline):
script = os.path.normpath(module_path)
payload = copy.deepcopy(self.payload_skeleton)
job_info_ext = {
# Job name, as seen in Monitor
"Name": "Export {} [{}-{}]".format(
data["jobname"],
int(self._instance.data["frameStartHandle"]),
int(self._instance.data["frameEndHandle"])),
job_info = copy.deepcopy(self.job_info)
plugin_info = copy.deepcopy(self.plugin_info)
"Plugin": "Python",
"FramesPerTask": self._instance.data.get("framesPerTask", 1),
"Frames": 1
job_info.Name = self._job_info_label("Export")
# Force a single frame Python job
job_info.Plugin = "Python"
job_info.Frames = 1
# add required env vars for the export script
envs = {
"AVALON_APP_NAME": os.environ.get("AVALON_APP_NAME"),
"OPENPYPE_ASS_EXPORT_RENDER_LAYER": data["renderlayer"],
"OPENPYPE_ASS_EXPORT_SCENE_FILE": self.scene_path,
"OPENPYPE_ASS_EXPORT_OUTPUT": payload['JobInfo']['OutputFilename0'], # noqa
"OPENPYPE_ASS_EXPORT_START": int(self._instance.data["frameStartHandle"]), # noqa
"OPENPYPE_ASS_EXPORT_END": int(self._instance.data["frameEndHandle"]), # noqa
"OPENPYPE_ASS_EXPORT_STEP": 1
}
for key, value in envs.items():
job_info.EnvironmentKeyValue = "{key}={value}".format(key=key,
value=value)
plugin_info_ext = {
plugin_info.update({
"Version": "3.6",
"ScriptFile": script,
"Arguments": "",
"SingleFrameOnly": "True",
}
payload["JobInfo"].update(job_info_ext)
payload["PluginInfo"].update(plugin_info_ext)
})
envs = [
v
for k, v in payload["JobInfo"].items()
if k.startswith("EnvironmentKeyValue")
]
# add app name to environment
envs.append(
"AVALON_APP_NAME={}".format(os.environ.get("AVALON_APP_NAME")))
envs.append(
"OPENPYPE_ASS_EXPORT_RENDER_LAYER={}".format(data["renderlayer"]))
envs.append(
"OPENPYPE_ASS_EXPORT_SCENE_FILE={}".format(data["filepath"]))
envs.append(
"OPENPYPE_ASS_EXPORT_OUTPUT={}".format(
payload['JobInfo']['OutputFilename0']))
envs.append(
"OPENPYPE_ASS_EXPORT_START={}".format(
int(self._instance.data["frameStartHandle"])))
envs.append(
"OPENPYPE_ASS_EXPORT_END={}".format(
int(self._instance.data["frameEndHandle"])))
envs.append(
"OPENPYPE_ASS_EXPORT_STEP={}".format(1))
for i, e in enumerate(envs):
payload["JobInfo"]["EnvironmentKeyValue{}".format(i)] = e
return payload
return job_info, plugin_info
def _get_vray_render_payload(self, data):
payload = copy.deepcopy(self.payload_skeleton)
# Job Info
job_info = copy.deepcopy(self.job_info)
job_info.Name = self._job_info_label("Render")
job_info.Plugin = "Vray"
job_info.OverrideTaskExtraInfoNames = False
# Plugin Info
vray_settings = cmds.ls(type="VRaySettingsNode")
node = vray_settings[0]
template = cmds.getAttr("{}.vrscene_filename".format(node))
# "vrayscene/<Scene>/<Scene>_<Layer>/<Layer>"
scene, _ = os.path.splitext(data["filename"])
scene, _ = os.path.splitext(self.scene_path)
first_file = self.format_vray_output_filename(scene, template)
first_file = "{}/{}".format(data["workspace"], first_file)
job_info_ext = {
"Name": "Render {} [{}-{}]".format(
data["jobname"],
int(self._instance.data["frameStartHandle"]),
int(self._instance.data["frameEndHandle"])),
"Plugin": "Vray",
"OverrideTaskExtraInfoNames": False,
}
plugin_info = {
"InputFilename": first_file,
@ -731,35 +641,28 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline):
"Width": self._instance.data["resolutionWidth"],
"Height": self._instance.data["resolutionHeight"],
"OutputFilePath": payload["JobInfo"]["OutputDirectory0"],
"OutputFileName": payload["JobInfo"]["OutputFilename0"]
"OutputFilePath": job_info.OutputDirectory[0],
"OutputFileName": job_info.OutputFilename[0]
}
payload["JobInfo"].update(job_info_ext)
payload["PluginInfo"].update(plugin_info)
return payload
return job_info, plugin_info
def _get_arnold_render_payload(self, data):
payload = copy.deepcopy(self.payload_skeleton)
# Job Info
job_info = copy.deepcopy(self.job_info)
job_info.Name = self._job_info_label("Render")
job_info.Plugin = "Arnold"
job_info.OverrideTaskExtraInfoNames = False
# Plugin Info
ass_file, _ = os.path.splitext(data["output_filename_0"])
first_file = ass_file + ".ass"
job_info_ext = {
"Name": "Render {} [{}-{}]".format(
data["jobname"],
int(self._instance.data["frameStartHandle"]),
int(self._instance.data["frameEndHandle"])),
"Plugin": "Arnold",
"OverrideTaskExtraInfoNames": False,
}
plugin_info = {
"ArnoldFile": first_file,
}
payload["JobInfo"].update(job_info_ext)
payload["PluginInfo"].update(plugin_info)
return payload
return job_info, plugin_info
def format_vray_output_filename(self, filename, template, dir=False):
"""Format the expected output file of the Export job.
@ -804,7 +707,7 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline):
return result
def _patch_workfile(self, file, patches):
def _patch_workfile(self):
# type: (str, dict) -> [str, None]
"""Patch Maya scene.
@ -818,19 +721,25 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline):
"line": "line to insert"
}
Args:
file (str): File to patch.
patches (dict): Dictionary defining patches.
Returns:
str: Patched file path or None
"""
if not patches or os.path.splitext(file)[1].lower() != ".ma":
project_settings = self._instance.context.data["project_settings"]
patches = (
project_settings.get(
"deadline", {}).get(
"publish", {}).get(
"MayaSubmitDeadline", {}).get(
"scene_patches", {})
)
if not patches:
return
if not os.path.splitext(self.scene_path)[1].lower() != ".ma":
self.log.debug("Skipping workfile patch since workfile is not "
".ma file")
return
compiled_regex = [re.compile(p["regex"]) for p in patches]
with open(file, "r+") as pf:
with open(self.scene_path, "r+") as pf:
scene_data = pf.readlines()
for ln, line in enumerate(scene_data):
for i, r in enumerate(compiled_regex):
@ -839,10 +748,17 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline):
pf.seek(0)
pf.writelines(scene_data)
pf.truncate()
self.log.info(
"Applied {} patch to scene.".format(
patches[i]["name"]))
return file
self.log.info("Applied {} patch to scene.".format(
patches[i]["name"]
))
def _job_info_label(self, label):
return "{label} {job.Name} [{start}-{end}]".format(
label=label,
job=self.job_info,
start=int(self._instance.data["frameStartHandle"]),
end=int(self._instance.data["frameEndHandle"]),
)
def _format_tiles(