mirror of
https://github.com/ynput/ayon-core.git
synced 2025-12-24 21:04:40 +01:00
fixed path usage, renderer handling and few code style issues
This commit is contained in:
Ondřej Samohel
parent
8947bb5153
commit
1afabb00e2
2 changed files with 152 additions and 96 deletions
|
|
@ -1,3 +1,6 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
"""Submit publishing job to farm."""
|
||||
|
||||
import os
|
||||
import json
|
||||
import re
|
||||
|
|
@ -10,7 +13,7 @@ import pyblish.api
|
|||
|
||||
|
||||
def _get_script():
|
||||
"""Get path to the image sequence script"""
|
||||
"""Get path to the image sequence script."""
|
||||
try:
|
||||
from pype.scripts import publish_filesequence
|
||||
except Exception:
|
||||
|
|
@ -23,8 +26,8 @@ def _get_script():
|
|||
return os.path.normpath(module_path)
|
||||
|
||||
|
||||
# Logic to retrieve latest files concerning extendFrames
|
||||
def get_latest_version(asset_name, subset_name, family):
|
||||
"""Retrieve latest files concerning extendFrame feature."""
|
||||
# Get asset
|
||||
asset_name = io.find_one(
|
||||
{"type": "asset", "name": asset_name}, projection={"name": True}
|
||||
|
|
@ -58,9 +61,7 @@ def get_latest_version(asset_name, subset_name, family):
|
|||
|
||||
|
||||
def get_resources(version, extension=None):
|
||||
"""
|
||||
Get the files from the specific version
|
||||
"""
|
||||
"""Get the files from the specific version."""
|
||||
query = {"type": "representation", "parent": version["_id"]}
|
||||
if extension:
|
||||
query["name"] = extension
|
||||
|
|
@ -80,14 +81,25 @@ def get_resources(version, extension=None):
|
|||
return resources
|
||||
|
||||
|
||||
def get_resource_files(resources, frame_range, override=True):
|
||||
def get_resource_files(resources, frame_range=None):
|
||||
"""Get resource files at given path.
|
||||
|
||||
If `frame_range` is specified those outside will be removed.
|
||||
|
||||
Arguments:
|
||||
resources (list): List of resources
|
||||
frame_range (list): Frame range to apply override
|
||||
|
||||
Returns:
|
||||
list of str: list of collected resources
|
||||
|
||||
"""
|
||||
res_collections, _ = clique.assemble(resources)
|
||||
assert len(res_collections) == 1, "Multiple collections found"
|
||||
res_collection = res_collections[0]
|
||||
|
||||
# Remove any frames
|
||||
if override:
|
||||
if frame_range is not None:
|
||||
for frame in frame_range:
|
||||
if frame not in res_collection.indexes:
|
||||
continue
|
||||
|
|
@ -147,7 +159,8 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
|
|||
"FTRACK_SERVER",
|
||||
"PYPE_SETUP_PATH",
|
||||
"PYPE_METADATA_FILE",
|
||||
"AVALON_PROJECT"
|
||||
"AVALON_PROJECT",
|
||||
"PYPE_LOG_NO_COLORS"
|
||||
]
|
||||
|
||||
# pool used to do the publishing job
|
||||
|
|
@ -169,10 +182,12 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
|
|||
families_transfer = ["render3d", "render2d", "ftrack", "slate"]
|
||||
|
||||
def _submit_deadline_post_job(self, instance, job):
|
||||
"""
|
||||
"""Submit publish job to Deadline.
|
||||
|
||||
Deadline specific code separated from :meth:`process` for sake of
|
||||
more universal code. Muster post job is sent directly by Muster
|
||||
submitter, so this type of code isn't necessary for it.
|
||||
|
||||
"""
|
||||
data = instance.data.copy()
|
||||
subset = data["subset"]
|
||||
|
|
@ -225,6 +240,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
|
|||
environment = job["Props"].get("Env", {})
|
||||
environment["PYPE_METADATA_FILE"] = metadata_path
|
||||
environment["AVALON_PROJECT"] = io.Session["AVALON_PROJECT"]
|
||||
environment["PYPE_LOG_NO_COLORS"] = "1"
|
||||
i = 0
|
||||
for index, key in enumerate(environment):
|
||||
if key.upper() in self.enviro_filter:
|
||||
|
|
@ -250,16 +266,20 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
|
|||
raise Exception(response.text)
|
||||
|
||||
def _copy_extend_frames(self, instance, representation):
|
||||
"""
|
||||
"""Copy existing frames from latest version.
|
||||
|
||||
This will copy all existing frames from subset's latest version back
|
||||
to render directory and rename them to what renderer is expecting.
|
||||
|
||||
:param instance: instance to get required data from
|
||||
:type instance: pyblish.plugin.Instance
|
||||
"""
|
||||
Arguments:
|
||||
instance (pyblish.plugin.Instance): instance to get required
|
||||
data from
|
||||
representation (dict): presentation to operate on
|
||||
|
||||
"""
|
||||
import speedcopy
|
||||
|
||||
anatomy = instance.context.data["anatomy"]
|
||||
self.log.info("Preparing to copy ...")
|
||||
start = instance.data.get("startFrame")
|
||||
end = instance.data.get("endFrame")
|
||||
|
|
@ -297,9 +317,11 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
|
|||
# type
|
||||
assert fn is not None, "padding string wasn't found"
|
||||
# list of tuples (source, destination)
|
||||
staging = representation.get("stagingDir")
|
||||
staging = anatomy.fill_roots(staging)
|
||||
resource_files.append(
|
||||
(frame,
|
||||
os.path.join(representation.get("stagingDir"),
|
||||
os.path.join(staging,
|
||||
"{}{}{}".format(pre,
|
||||
fn.group("frame"),
|
||||
post)))
|
||||
|
|
@ -319,19 +341,20 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
|
|||
"Finished copying %i files" % len(resource_files))
|
||||
|
||||
def _create_instances_for_aov(self, instance_data, exp_files):
|
||||
"""
|
||||
"""Create instance for each AOV found.
|
||||
|
||||
This will create new instance for every aov it can detect in expected
|
||||
files list.
|
||||
|
||||
:param instance_data: skeleton data for instance (those needed) later
|
||||
by collector
|
||||
:type instance_data: pyblish.plugin.Instance
|
||||
:param exp_files: list of expected files divided by aovs
|
||||
:type exp_files: list
|
||||
:returns: list of instances
|
||||
:rtype: list(publish.plugin.Instance)
|
||||
"""
|
||||
Arguments:
|
||||
instance_data (pyblish.plugin.Instance): skeleton data for instance
|
||||
(those needed) later by collector
|
||||
exp_files (list): list of expected files divided by aovs
|
||||
|
||||
Returns:
|
||||
list of instances
|
||||
|
||||
"""
|
||||
task = os.environ["AVALON_TASK"]
|
||||
subset = instance_data["subset"]
|
||||
instances = []
|
||||
|
|
@ -354,7 +377,19 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
|
|||
|
||||
subset_name = '{}_{}'.format(group_name, aov)
|
||||
|
||||
anatomy = instance_data.context.data["anatomy"]
|
||||
|
||||
staging = os.path.dirname(list(cols[0])[0])
|
||||
success, rootless_staging_dir = (
|
||||
anatomy.roots_obj.find_root_template_from_path(staging)
|
||||
)
|
||||
if success:
|
||||
staging = rootless_staging_dir
|
||||
else:
|
||||
self.log.warning((
|
||||
"Could not find root path for remapping \"{}\"."
|
||||
" This may cause issues on farm."
|
||||
).format(staging))
|
||||
|
||||
self.log.info("Creating data for: {}".format(subset_name))
|
||||
|
||||
|
|
@ -398,22 +433,25 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
|
|||
return instances
|
||||
|
||||
def _get_representations(self, instance, exp_files):
|
||||
"""
|
||||
"""Create representations for file sequences.
|
||||
|
||||
This will return representations of expected files if they are not
|
||||
in hierarchy of aovs. There should be only one sequence of files for
|
||||
most cases, but if not - we create representation from each of them.
|
||||
|
||||
:param instance: instance for which we are setting representations
|
||||
:type instance: pyblish.plugin.Instance
|
||||
:param exp_files: list of expected files
|
||||
:type exp_files: list
|
||||
:returns: list of representations
|
||||
:rtype: list(dict)
|
||||
"""
|
||||
Arguments:
|
||||
instance (pyblish.plugin.Instance): instance for which we are
|
||||
setting representations
|
||||
exp_files (list): list of expected files
|
||||
|
||||
Returns:
|
||||
list of representations
|
||||
|
||||
"""
|
||||
representations = []
|
||||
collections, remainders = clique.assemble(exp_files)
|
||||
bake_render_path = instance.get("bakeRenderPath")
|
||||
anatomy = instance.context.data["anatomy"]
|
||||
|
||||
# create representation for every collected sequence
|
||||
for collection in collections:
|
||||
|
|
@ -435,6 +473,18 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
|
|||
if bake_render_path:
|
||||
preview = False
|
||||
|
||||
staging = os.path.dirname(list(collection)[0])
|
||||
success, rootless_staging_dir = (
|
||||
anatomy.roots_obj.find_root_template_from_path(staging)
|
||||
)
|
||||
if success:
|
||||
staging = rootless_staging_dir
|
||||
else:
|
||||
self.log.warning((
|
||||
"Could not find root path for remapping \"{}\"."
|
||||
" This may cause issues on farm."
|
||||
).format(staging))
|
||||
|
||||
rep = {
|
||||
"name": ext,
|
||||
"ext": ext,
|
||||
|
|
@ -442,7 +492,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
|
|||
"frameStart": int(instance.get("frameStartHandle")),
|
||||
"frameEnd": int(instance.get("frameEndHandle")),
|
||||
# If expectedFile are absolute, we need only filenames
|
||||
"stagingDir": os.path.dirname(list(collection)[0]),
|
||||
"stagingDir": staging,
|
||||
"anatomy_template": "render",
|
||||
"fps": instance.get("fps"),
|
||||
"tags": ["review", "preview"] if preview else [],
|
||||
|
|
@ -458,6 +508,19 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
|
|||
# add reminders as representations
|
||||
for remainder in remainders:
|
||||
ext = remainder.split(".")[-1]
|
||||
|
||||
staging = os.path.dirname(remainder)
|
||||
success, rootless_staging_dir = (
|
||||
anatomy.roots_obj.find_root_template_from_path(staging)
|
||||
)
|
||||
if success:
|
||||
staging = rootless_staging_dir
|
||||
else:
|
||||
self.log.warning((
|
||||
"Could not find root path for remapping \"{}\"."
|
||||
" This may cause issues on farm."
|
||||
).format(staging))
|
||||
|
||||
rep = {
|
||||
"name": ext,
|
||||
"ext": ext,
|
||||
|
|
@ -490,7 +553,8 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
|
|||
instance["families"] = families
|
||||
|
||||
def process(self, instance):
|
||||
"""
|
||||
"""Process plugin.
|
||||
|
||||
Detect type of renderfarm submission and create and post dependend job
|
||||
in case of Deadline. It creates json file with metadata needed for
|
||||
publishing in directory of render.
|
||||
|
|
@ -631,6 +695,12 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
|
|||
)
|
||||
if success:
|
||||
repre["stagingDir"] = rootless_staging_dir
|
||||
else:
|
||||
self.log.warning((
|
||||
"Could not find root path for remapping \"{}\"."
|
||||
" This may cause issues on farm."
|
||||
).format(staging_dir))
|
||||
repre["stagingDir"] = staging_dir
|
||||
|
||||
if "publish_on_farm" in repre.get("tags"):
|
||||
# create representations attribute of not there
|
||||
|
|
@ -774,12 +844,21 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
|
|||
with open(metadata_path, "w") as f:
|
||||
json.dump(publish_job, f, indent=4, sort_keys=True)
|
||||
|
||||
def _extend_frames(self, asset, subset, start, end, override):
|
||||
"""
|
||||
This will get latest version of asset and update frame range based
|
||||
on minimum and maximuma values
|
||||
"""
|
||||
def _extend_frames(self, asset, subset, start, end):
|
||||
"""Get latest version of asset nad update frame range.
|
||||
|
||||
Based on minimum and maximuma values.
|
||||
|
||||
Arguments:
|
||||
asset (str): asset name
|
||||
subset (str): subset name
|
||||
start (int): start frame
|
||||
end (int): end frame
|
||||
|
||||
Returns:
|
||||
(int, int): upddate frame start/end
|
||||
|
||||
"""
|
||||
# Frame comparison
|
||||
prev_start = None
|
||||
prev_end = None
|
||||
|
|
|
|||
|
|
@ -1,6 +1,17 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
"""Submitting render job to Deadline.
|
||||
|
||||
This module is taking care of submitting job from Maya to Deadline. It
|
||||
creates job and set correct environments. Its behavior is controlled by
|
||||
`DEADLINE_REST_URL` environment variable - pointing to Deadline Web Service
|
||||
and `MayaSubmitDeadline.use_published (bool)` property telling Deadline to
|
||||
use published scene workfile or not.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import getpass
|
||||
import re
|
||||
import clique
|
||||
|
||||
from maya import cmds
|
||||
|
|
@ -14,7 +25,7 @@ import pype.maya.lib as lib
|
|||
|
||||
|
||||
def get_renderer_variables(renderlayer=None):
|
||||
"""Retrieve the extension which has been set in the VRay settings
|
||||
"""Retrieve the extension which has been set in the VRay settings.
|
||||
|
||||
Will return None if the current renderer is not VRay
|
||||
For Maya 2016.5 and up the renderSetup creates renderSetupLayer node which
|
||||
|
|
@ -25,8 +36,8 @@ def get_renderer_variables(renderlayer=None):
|
|||
|
||||
Returns:
|
||||
dict
|
||||
"""
|
||||
|
||||
"""
|
||||
renderer = lib.get_renderer(renderlayer or lib.get_current_renderlayer())
|
||||
render_attrs = lib.RENDER_ATTRS.get(renderer, lib.RENDER_ATTRS["default"])
|
||||
|
||||
|
|
@ -34,7 +45,7 @@ def get_renderer_variables(renderlayer=None):
|
|||
render_attrs["padding"]))
|
||||
|
||||
filename_0 = cmds.renderSettings(fullPath=True, firstImageName=True)[0]
|
||||
|
||||
prefix_attr = "defaultRenderGlobals.imageFilePrefix"
|
||||
if renderer == "vray":
|
||||
# Maya's renderSettings function does not return V-Ray file extension
|
||||
# so we get the extension from vraySettings
|
||||
|
|
@ -46,62 +57,33 @@ def get_renderer_variables(renderlayer=None):
|
|||
if extension is None:
|
||||
extension = "png"
|
||||
|
||||
filename_prefix = "<Scene>/<Scene>_<Layer>/<Layer>"
|
||||
if extension == "exr (multichannel)" or extension == "exr (deep)":
|
||||
extension = "exr"
|
||||
|
||||
prefix_attr = "vraySettings.fileNamePrefix"
|
||||
elif renderer == "renderman":
|
||||
prefix_attr = "rmanGlobals.imageFileFormat"
|
||||
elif renderer == "redshift":
|
||||
# mapping redshift extension dropdown values to strings
|
||||
ext_mapping = ["iff", "exr", "tif", "png", "tga", "jpg"]
|
||||
extension = ext_mapping[
|
||||
cmds.getAttr("redshiftOptions.imageFormat")
|
||||
]
|
||||
else:
|
||||
# Get the extension, getAttr defaultRenderGlobals.imageFormat
|
||||
# returns an index number.
|
||||
filename_base = os.path.basename(filename_0)
|
||||
extension = os.path.splitext(filename_base)[-1].strip(".")
|
||||
filename_prefix = cmds.getAttr("defaultRenderGlobals.imageFilePrefix")
|
||||
|
||||
filename_prefix = cmds.getAttr(prefix_attr)
|
||||
return {"ext": extension,
|
||||
"filename_prefix": filename_prefix,
|
||||
"padding": padding,
|
||||
"filename_0": filename_0}
|
||||
|
||||
|
||||
def preview_fname(folder, scene, layer, padding, ext):
|
||||
"""Return output file path with #### for padding.
|
||||
|
||||
Deadline requires the path to be formatted with # in place of numbers.
|
||||
For example `/path/to/render.####.png`
|
||||
|
||||
Args:
|
||||
folder (str): The root output folder (image path)
|
||||
scene (str): The scene name
|
||||
layer (str): The layer name to be rendered
|
||||
padding (int): The padding length
|
||||
ext(str): The output file extension
|
||||
|
||||
Returns:
|
||||
str
|
||||
|
||||
"""
|
||||
|
||||
fileprefix = cmds.getAttr("defaultRenderGlobals.imageFilePrefix")
|
||||
output = fileprefix + ".{number}.{ext}"
|
||||
# RenderPass is currently hardcoded to "beauty" because its not important
|
||||
# for the deadline submission, but we will need something to replace
|
||||
# "<RenderPass>".
|
||||
mapping = {
|
||||
"<Scene>": "{scene}",
|
||||
"<RenderLayer>": "{layer}",
|
||||
"RenderPass": "beauty"
|
||||
}
|
||||
for key, value in mapping.items():
|
||||
output = output.replace(key, value)
|
||||
output = output.format(
|
||||
scene=scene,
|
||||
layer=layer,
|
||||
number="#" * padding,
|
||||
ext=ext
|
||||
)
|
||||
|
||||
return os.path.join(folder, output)
|
||||
|
||||
|
||||
class MayaSubmitDeadline(pyblish.api.InstancePlugin):
|
||||
"""Submit available render layers to Deadline
|
||||
"""Submit available render layers to Deadline.
|
||||
|
||||
Renders are submitted to a Deadline Web Service as
|
||||
supplied via the environment variable DEADLINE_REST_URL
|
||||
|
|
@ -194,22 +176,17 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
|
|||
|
||||
filename = os.path.basename(filepath)
|
||||
comment = context.data.get("comment", "")
|
||||
scene = os.path.splitext(filename)[0]
|
||||
dirname = os.path.join(workspace, "renders")
|
||||
renderlayer = instance.data['setMembers'] # rs_beauty
|
||||
renderlayer_name = instance.data['subset'] # beauty
|
||||
# renderlayer_globals = instance.data["renderGlobals"]
|
||||
# legacy_layers = renderlayer_globals["UseLegacyRenderLayers"]
|
||||
deadline_user = context.data.get("deadlineUser", getpass.getuser())
|
||||
jobname = "%s - %s" % (filename, instance.name)
|
||||
|
||||
# Get the variables depending on the renderer
|
||||
render_variables = get_renderer_variables(renderlayer)
|
||||
output_filename_0 = preview_fname(folder=dirname,
|
||||
scene=scene,
|
||||
layer=renderlayer_name,
|
||||
padding=render_variables["padding"],
|
||||
ext=render_variables["ext"])
|
||||
output_filename_0 = re.sub(
|
||||
"(/d+{{{}}})".format(render_variables["padding"]),
|
||||
"#" * render_variables["padding"],
|
||||
render_variables["filename_0"])
|
||||
|
||||
try:
|
||||
# Ensure render folder exists
|
||||
|
|
@ -284,7 +261,7 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
|
|||
for aov, files in exp[0].items():
|
||||
col = clique.assemble(files)[0][0]
|
||||
outputFile = col.format('{head}{padding}{tail}')
|
||||
payload['JobInfo']['OutputFilename' + str(expIndex)] = outputFile
|
||||
payload['JobInfo']['OutputFilename' + str(expIndex)] = outputFile # noqa: E501
|
||||
OutputFilenames[expIndex] = outputFile
|
||||
expIndex += 1
|
||||
else:
|
||||
|
|
@ -293,7 +270,6 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
|
|||
payload['JobInfo']['OutputFilename' + str(expIndex)] = outputFile
|
||||
# OutputFilenames[expIndex] = outputFile
|
||||
|
||||
|
||||
# We need those to pass them to pype for it to set correct context
|
||||
keys = [
|
||||
"FTRACK_API_KEY",
|
||||
|
|
@ -334,7 +310,8 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
|
|||
raise Exception(response.text)
|
||||
|
||||
# Store output dir for unified publisher (filesequence)
|
||||
instance.data["outputDir"] = os.path.dirname(output_filename_0)
|
||||
instance.data["outputDir"] = os.path.dirname(
|
||||
render_variables["filename_0"])
|
||||
instance.data["deadlineSubmissionJob"] = response.json()
|
||||
|
||||
def preflight_check(self, instance):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue