From 1afabb00e2e8b649c2c3b7c7eefc813b56983162 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Thu, 7 May 2020 15:04:37 +0200 Subject: [PATCH] fixed path usage, renderer handling and few code style issues --- .../global/publish/submit_publish_job.py | 155 +++++++++++++----- .../maya/publish/submit_maya_deadline.py | 93 ++++------- 2 files changed, 152 insertions(+), 96 deletions(-) diff --git a/pype/plugins/global/publish/submit_publish_job.py b/pype/plugins/global/publish/submit_publish_job.py index 6d04c8cb01..8688d161e2 100644 --- a/pype/plugins/global/publish/submit_publish_job.py +++ b/pype/plugins/global/publish/submit_publish_job.py @@ -1,3 +1,6 @@ +# -*- coding: utf-8 -*- +"""Submit publishing job to farm.""" + import os import json import re @@ -10,7 +13,7 @@ import pyblish.api def _get_script(): - """Get path to the image sequence script""" + """Get path to the image sequence script.""" try: from pype.scripts import publish_filesequence except Exception: @@ -23,8 +26,8 @@ def _get_script(): return os.path.normpath(module_path) -# Logic to retrieve latest files concerning extendFrames def get_latest_version(asset_name, subset_name, family): + """Retrieve latest files concerning extendFrame feature.""" # Get asset asset_name = io.find_one( {"type": "asset", "name": asset_name}, projection={"name": True} @@ -58,9 +61,7 @@ def get_latest_version(asset_name, subset_name, family): def get_resources(version, extension=None): - """ - Get the files from the specific version - """ + """Get the files from the specific version.""" query = {"type": "representation", "parent": version["_id"]} if extension: query["name"] = extension @@ -80,14 +81,25 @@ def get_resources(version, extension=None): return resources -def get_resource_files(resources, frame_range, override=True): +def get_resource_files(resources, frame_range=None): + """Get resource files at given path. + If `frame_range` is specified those outside will be removed. + + Arguments: + resources (list): List of resources + frame_range (list): Frame range to apply override + + Returns: + list of str: list of collected resources + + """ res_collections, _ = clique.assemble(resources) assert len(res_collections) == 1, "Multiple collections found" res_collection = res_collections[0] # Remove any frames - if override: + if frame_range is not None: for frame in frame_range: if frame not in res_collection.indexes: continue @@ -147,7 +159,8 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): "FTRACK_SERVER", "PYPE_SETUP_PATH", "PYPE_METADATA_FILE", - "AVALON_PROJECT" + "AVALON_PROJECT", + "PYPE_LOG_NO_COLORS" ] # pool used to do the publishing job @@ -169,10 +182,12 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): families_transfer = ["render3d", "render2d", "ftrack", "slate"] def _submit_deadline_post_job(self, instance, job): - """ + """Submit publish job to Deadline. + Deadline specific code separated from :meth:`process` for sake of more universal code. Muster post job is sent directly by Muster submitter, so this type of code isn't necessary for it. + """ data = instance.data.copy() subset = data["subset"] @@ -225,6 +240,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): environment = job["Props"].get("Env", {}) environment["PYPE_METADATA_FILE"] = metadata_path environment["AVALON_PROJECT"] = io.Session["AVALON_PROJECT"] + environment["PYPE_LOG_NO_COLORS"] = "1" i = 0 for index, key in enumerate(environment): if key.upper() in self.enviro_filter: @@ -250,16 +266,20 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): raise Exception(response.text) def _copy_extend_frames(self, instance, representation): - """ + """Copy existing frames from latest version. + This will copy all existing frames from subset's latest version back to render directory and rename them to what renderer is expecting. - :param instance: instance to get required data from - :type instance: pyblish.plugin.Instance - """ + Arguments: + instance (pyblish.plugin.Instance): instance to get required + data from + representation (dict): presentation to operate on + """ import speedcopy + anatomy = instance.context.data["anatomy"] self.log.info("Preparing to copy ...") start = instance.data.get("startFrame") end = instance.data.get("endFrame") @@ -297,9 +317,11 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): # type assert fn is not None, "padding string wasn't found" # list of tuples (source, destination) + staging = representation.get("stagingDir") + staging = anatomy.fill_roots(staging) resource_files.append( (frame, - os.path.join(representation.get("stagingDir"), + os.path.join(staging, "{}{}{}".format(pre, fn.group("frame"), post))) @@ -319,19 +341,20 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): "Finished copying %i files" % len(resource_files)) def _create_instances_for_aov(self, instance_data, exp_files): - """ + """Create instance for each AOV found. + This will create new instance for every aov it can detect in expected files list. - :param instance_data: skeleton data for instance (those needed) later - by collector - :type instance_data: pyblish.plugin.Instance - :param exp_files: list of expected files divided by aovs - :type exp_files: list - :returns: list of instances - :rtype: list(publish.plugin.Instance) - """ + Arguments: + instance_data (pyblish.plugin.Instance): skeleton data for instance + (those needed) later by collector + exp_files (list): list of expected files divided by aovs + Returns: + list of instances + + """ task = os.environ["AVALON_TASK"] subset = instance_data["subset"] instances = [] @@ -354,7 +377,19 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): subset_name = '{}_{}'.format(group_name, aov) + anatomy = instance_data.context.data["anatomy"] + staging = os.path.dirname(list(cols[0])[0]) + success, rootless_staging_dir = ( + anatomy.roots_obj.find_root_template_from_path(staging) + ) + if success: + staging = rootless_staging_dir + else: + self.log.warning(( + "Could not find root path for remapping \"{}\"." + " This may cause issues on farm." + ).format(staging)) self.log.info("Creating data for: {}".format(subset_name)) @@ -398,22 +433,25 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): return instances def _get_representations(self, instance, exp_files): - """ + """Create representations for file sequences. + This will return representations of expected files if they are not in hierarchy of aovs. There should be only one sequence of files for most cases, but if not - we create representation from each of them. - :param instance: instance for which we are setting representations - :type instance: pyblish.plugin.Instance - :param exp_files: list of expected files - :type exp_files: list - :returns: list of representations - :rtype: list(dict) - """ + Arguments: + instance (pyblish.plugin.Instance): instance for which we are + setting representations + exp_files (list): list of expected files + Returns: + list of representations + + """ representations = [] collections, remainders = clique.assemble(exp_files) bake_render_path = instance.get("bakeRenderPath") + anatomy = instance.context.data["anatomy"] # create representation for every collected sequence for collection in collections: @@ -435,6 +473,18 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): if bake_render_path: preview = False + staging = os.path.dirname(list(collection)[0]) + success, rootless_staging_dir = ( + anatomy.roots_obj.find_root_template_from_path(staging) + ) + if success: + staging = rootless_staging_dir + else: + self.log.warning(( + "Could not find root path for remapping \"{}\"." + " This may cause issues on farm." + ).format(staging)) + rep = { "name": ext, "ext": ext, @@ -442,7 +492,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): "frameStart": int(instance.get("frameStartHandle")), "frameEnd": int(instance.get("frameEndHandle")), # If expectedFile are absolute, we need only filenames - "stagingDir": os.path.dirname(list(collection)[0]), + "stagingDir": staging, "anatomy_template": "render", "fps": instance.get("fps"), "tags": ["review", "preview"] if preview else [], @@ -458,6 +508,19 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): # add reminders as representations for remainder in remainders: ext = remainder.split(".")[-1] + + staging = os.path.dirname(remainder) + success, rootless_staging_dir = ( + anatomy.roots_obj.find_root_template_from_path(staging) + ) + if success: + staging = rootless_staging_dir + else: + self.log.warning(( + "Could not find root path for remapping \"{}\"." + " This may cause issues on farm." + ).format(staging)) + rep = { "name": ext, "ext": ext, @@ -490,7 +553,8 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): instance["families"] = families def process(self, instance): - """ + """Process plugin. + Detect type of renderfarm submission and create and post dependend job in case of Deadline. It creates json file with metadata needed for publishing in directory of render. @@ -631,6 +695,12 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): ) if success: repre["stagingDir"] = rootless_staging_dir + else: + self.log.warning(( + "Could not find root path for remapping \"{}\"." + " This may cause issues on farm." + ).format(staging_dir)) + repre["stagingDir"] = staging_dir if "publish_on_farm" in repre.get("tags"): # create representations attribute of not there @@ -774,12 +844,21 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): with open(metadata_path, "w") as f: json.dump(publish_job, f, indent=4, sort_keys=True) - def _extend_frames(self, asset, subset, start, end, override): - """ - This will get latest version of asset and update frame range based - on minimum and maximuma values - """ + def _extend_frames(self, asset, subset, start, end): + """Get latest version of asset nad update frame range. + Based on minimum and maximuma values. + + Arguments: + asset (str): asset name + subset (str): subset name + start (int): start frame + end (int): end frame + + Returns: + (int, int): upddate frame start/end + + """ # Frame comparison prev_start = None prev_end = None diff --git a/pype/plugins/maya/publish/submit_maya_deadline.py b/pype/plugins/maya/publish/submit_maya_deadline.py index 7547f34ba1..239bad8f83 100644 --- a/pype/plugins/maya/publish/submit_maya_deadline.py +++ b/pype/plugins/maya/publish/submit_maya_deadline.py @@ -1,6 +1,17 @@ +# -*- coding: utf-8 -*- +"""Submitting render job to Deadline. + +This module is taking care of submitting job from Maya to Deadline. It +creates job and set correct environments. Its behavior is controlled by +`DEADLINE_REST_URL` environment variable - pointing to Deadline Web Service +and `MayaSubmitDeadline.use_published (bool)` property telling Deadline to +use published scene workfile or not. +""" + import os import json import getpass +import re import clique from maya import cmds @@ -14,7 +25,7 @@ import pype.maya.lib as lib def get_renderer_variables(renderlayer=None): - """Retrieve the extension which has been set in the VRay settings + """Retrieve the extension which has been set in the VRay settings. Will return None if the current renderer is not VRay For Maya 2016.5 and up the renderSetup creates renderSetupLayer node which @@ -25,8 +36,8 @@ def get_renderer_variables(renderlayer=None): Returns: dict - """ + """ renderer = lib.get_renderer(renderlayer or lib.get_current_renderlayer()) render_attrs = lib.RENDER_ATTRS.get(renderer, lib.RENDER_ATTRS["default"]) @@ -34,7 +45,7 @@ def get_renderer_variables(renderlayer=None): render_attrs["padding"])) filename_0 = cmds.renderSettings(fullPath=True, firstImageName=True)[0] - + prefix_attr = "defaultRenderGlobals.imageFilePrefix" if renderer == "vray": # Maya's renderSettings function does not return V-Ray file extension # so we get the extension from vraySettings @@ -46,62 +57,33 @@ def get_renderer_variables(renderlayer=None): if extension is None: extension = "png" - filename_prefix = "/_/" + if extension == "exr (multichannel)" or extension == "exr (deep)": + extension = "exr" + + prefix_attr = "vraySettings.fileNamePrefix" + elif renderer == "renderman": + prefix_attr = "rmanGlobals.imageFileFormat" + elif renderer == "redshift": + # mapping redshift extension dropdown values to strings + ext_mapping = ["iff", "exr", "tif", "png", "tga", "jpg"] + extension = ext_mapping[ + cmds.getAttr("redshiftOptions.imageFormat") + ] else: # Get the extension, getAttr defaultRenderGlobals.imageFormat # returns an index number. filename_base = os.path.basename(filename_0) extension = os.path.splitext(filename_base)[-1].strip(".") - filename_prefix = cmds.getAttr("defaultRenderGlobals.imageFilePrefix") + filename_prefix = cmds.getAttr(prefix_attr) return {"ext": extension, "filename_prefix": filename_prefix, "padding": padding, "filename_0": filename_0} -def preview_fname(folder, scene, layer, padding, ext): - """Return output file path with #### for padding. - - Deadline requires the path to be formatted with # in place of numbers. - For example `/path/to/render.####.png` - - Args: - folder (str): The root output folder (image path) - scene (str): The scene name - layer (str): The layer name to be rendered - padding (int): The padding length - ext(str): The output file extension - - Returns: - str - - """ - - fileprefix = cmds.getAttr("defaultRenderGlobals.imageFilePrefix") - output = fileprefix + ".{number}.{ext}" - # RenderPass is currently hardcoded to "beauty" because its not important - # for the deadline submission, but we will need something to replace - # "". - mapping = { - "": "{scene}", - "": "{layer}", - "RenderPass": "beauty" - } - for key, value in mapping.items(): - output = output.replace(key, value) - output = output.format( - scene=scene, - layer=layer, - number="#" * padding, - ext=ext - ) - - return os.path.join(folder, output) - - class MayaSubmitDeadline(pyblish.api.InstancePlugin): - """Submit available render layers to Deadline + """Submit available render layers to Deadline. Renders are submitted to a Deadline Web Service as supplied via the environment variable DEADLINE_REST_URL @@ -194,22 +176,17 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): filename = os.path.basename(filepath) comment = context.data.get("comment", "") - scene = os.path.splitext(filename)[0] dirname = os.path.join(workspace, "renders") renderlayer = instance.data['setMembers'] # rs_beauty - renderlayer_name = instance.data['subset'] # beauty - # renderlayer_globals = instance.data["renderGlobals"] - # legacy_layers = renderlayer_globals["UseLegacyRenderLayers"] deadline_user = context.data.get("deadlineUser", getpass.getuser()) jobname = "%s - %s" % (filename, instance.name) # Get the variables depending on the renderer render_variables = get_renderer_variables(renderlayer) - output_filename_0 = preview_fname(folder=dirname, - scene=scene, - layer=renderlayer_name, - padding=render_variables["padding"], - ext=render_variables["ext"]) + output_filename_0 = re.sub( + "(/d+{{{}}})".format(render_variables["padding"]), + "#" * render_variables["padding"], + render_variables["filename_0"]) try: # Ensure render folder exists @@ -284,7 +261,7 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): for aov, files in exp[0].items(): col = clique.assemble(files)[0][0] outputFile = col.format('{head}{padding}{tail}') - payload['JobInfo']['OutputFilename' + str(expIndex)] = outputFile + payload['JobInfo']['OutputFilename' + str(expIndex)] = outputFile # noqa: E501 OutputFilenames[expIndex] = outputFile expIndex += 1 else: @@ -293,7 +270,6 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): payload['JobInfo']['OutputFilename' + str(expIndex)] = outputFile # OutputFilenames[expIndex] = outputFile - # We need those to pass them to pype for it to set correct context keys = [ "FTRACK_API_KEY", @@ -334,7 +310,8 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin): raise Exception(response.text) # Store output dir for unified publisher (filesequence) - instance.data["outputDir"] = os.path.dirname(output_filename_0) + instance.data["outputDir"] = os.path.dirname( + render_variables["filename_0"]) instance.data["deadlineSubmissionJob"] = response.json() def preflight_check(self, instance):