From 4ae098e4d2a4434849c7f42152102d8bc59a31af Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Sat, 1 Dec 2018 01:39:38 +0100 Subject: [PATCH] minor cleanups --- .../_publish_unused/collect_deadline_user.py | 60 ---- .../global/publish/collect_templates.py | 3 - pype/plugins/global/publish/integrate.py | 1 + .../global/publish/submit_publish_job.py | 330 ------------------ .../global/publish/validate_templates.py | 42 --- 5 files changed, 1 insertion(+), 435 deletions(-) delete mode 100644 pype/plugins/global/_publish_unused/collect_deadline_user.py delete mode 100644 pype/plugins/global/publish/submit_publish_job.py delete mode 100644 pype/plugins/global/publish/validate_templates.py diff --git a/pype/plugins/global/_publish_unused/collect_deadline_user.py b/pype/plugins/global/_publish_unused/collect_deadline_user.py deleted file mode 100644 index f4d13a0545..0000000000 --- a/pype/plugins/global/_publish_unused/collect_deadline_user.py +++ /dev/null @@ -1,60 +0,0 @@ -import os -import subprocess - -import pyblish.api - -CREATE_NO_WINDOW = 0x08000000 - - -def deadline_command(cmd): - # Find Deadline - path = os.environ.get("DEADLINE_PATH", None) - assert path is not None, "Variable 'DEADLINE_PATH' must be set" - - executable = os.path.join(path, "deadlinecommand") - if os.name == "nt": - executable += ".exe" - assert os.path.exists( - executable), "Deadline executable not found at %s" % executable - assert cmd, "Must have a command" - - query = (executable, cmd) - - process = subprocess.Popen(query, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - creationflags=CREATE_NO_WINDOW) - out, err = process.communicate() - - return out - - -class CollectDeadlineUser(pyblish.api.ContextPlugin): - """Retrieve the local active Deadline user""" - - order = pyblish.api.CollectorOrder + 0.499 - label = "Deadline User" - - hosts = ['maya', 'fusion', 'nuke'] - families = [ - "renderlayer", - "saver.deadline", - "imagesequence" - ] - - - def process(self, context): - """Inject the current working file""" - user = None - try: - user = deadline_command("GetCurrentUserName").strip() - except: - self.log.warning("Deadline command seems not to be working") - - if not user: - self.log.warning("No Deadline user found. " - "Do you have Deadline installed?") - return - - self.log.info("Found Deadline user: {}".format(user)) - context.data['deadlineUser'] = user diff --git a/pype/plugins/global/publish/collect_templates.py b/pype/plugins/global/publish/collect_templates.py index f2a3da7df4..48b6c448e3 100644 --- a/pype/plugins/global/publish/collect_templates.py +++ b/pype/plugins/global/publish/collect_templates.py @@ -18,6 +18,3 @@ class CollectTemplates(pyblish.api.ContextPlugin): type=["anatomy"] ) context.data['anatomy'] = templates.anatomy - for key in templates.anatomy: - self.log.info(str(key) + ": " + str(templates.anatomy[key])) - # return diff --git a/pype/plugins/global/publish/integrate.py b/pype/plugins/global/publish/integrate.py index 87ffa2aaa3..e20f59133c 100644 --- a/pype/plugins/global/publish/integrate.py +++ b/pype/plugins/global/publish/integrate.py @@ -233,6 +233,7 @@ class IntegrateAsset(pyblish.api.InstancePlugin): "root": root, "project": PROJECT, "projectcode": "prjX", + 'task': api.Session["AVALON_TASK"], "silo": asset['silo'], "asset": ASSET, "family": instance.data['family'], diff --git a/pype/plugins/global/publish/submit_publish_job.py b/pype/plugins/global/publish/submit_publish_job.py deleted file mode 100644 index cb852f7c43..0000000000 --- a/pype/plugins/global/publish/submit_publish_job.py +++ /dev/null @@ -1,330 +0,0 @@ -import os -import json -import re - -from avalon import api, io -from avalon.vendor import requests, clique - -import pyblish.api - - -def _get_script(): - """Get path to the image sequence script""" - try: - from pype.fusion.scripts import publish_filesequence - except Exception as e: - raise RuntimeError("Expected module 'publish_imagesequence'" - "to be available") - - module_path = publish_filesequence.__file__ - if module_path.endswith(".pyc"): - module_path = module_path[:-len(".pyc")] + ".py" - - return module_path - - -# Logic to retrieve latest files concerning extendFrames -def get_latest_version(asset_name, subset_name, family): - # Get asset - asset_name = io.find_one({"type": "asset", - "name": asset_name}, - projection={"name": True}) - - subset = io.find_one({"type": "subset", - "name": subset_name, - "parent": asset_name["_id"]}, - projection={"_id": True, "name": True}) - - # Check if subsets actually exists (pre-run check) - assert subset, "No subsets found, please publish with `extendFrames` off" - - # Get version - version_projection = {"name": True, - "data.startFrame": True, - "data.endFrame": True, - "parent": True} - - version = io.find_one({"type": "version", - "parent": subset["_id"], - "data.families": family}, - projection=version_projection, - sort=[("name", -1)]) - - assert version, "No version found, this is a bug" - - return version - - -def get_resources(version, extension=None): - """ - Get the files from the specific version - """ - query = {"type": "representation", "parent": version["_id"]} - if extension: - query["name"] = extension - - representation = io.find_one(query) - assert representation, "This is a bug" - - directory = api.get_representation_path(representation) - print("Source: ", directory) - resources = sorted([os.path.normpath(os.path.join(directory, fname)) - for fname in os.listdir(directory)]) - - return resources - - -def get_resource_files(resources, frame_range, override=True): - - res_collections, _ = clique.assemble(resources) - assert len(res_collections) == 1, "Multiple collections found" - res_collection = res_collections[0] - - # Remove any frames - if override: - for frame in frame_range: - if frame not in res_collection.indexes: - continue - res_collection.indexes.remove(frame) - - return list(res_collection) - - -class SubmitDependentImageSequenceJobDeadline(pyblish.api.InstancePlugin): - """Submit image sequence publish jobs to Deadline. - - These jobs are dependent on a deadline job submission prior to this - plug-in. - - Renders are submitted to a Deadline Web Service as - supplied via the environment variable AVALON_DEADLINE - - Options in instance.data: - - deadlineSubmission (dict, Required): The returned .json - data from the job submission to deadline. - - - outputDir (str, Required): The output directory where the metadata - file should be generated. It's assumed that this will also be - final folder containing the output files. - - - ext (str, Optional): The extension (including `.`) that is required - in the output filename to be picked up for image sequence - publishing. - - - publishJobState (str, Optional): "Active" or "Suspended" - This defaults to "Suspended" - - This requires a "startFrame" and "endFrame" to be present in instance.data - or in context.data. - - """ - - label = "Submit image sequence jobs to Deadline" - order = pyblish.api.IntegratorOrder + 0.1 - - hosts = ["fusion", "maya", "nuke"] - - families = [ - "render.deadline", - "renderlayer", - "imagesequence" - ] - - def process(self, instance): - - # AVALON_DEADLINE = api.Session.get("AVALON_DEADLINE", - # "http://localhost:8082") - # assert AVALON_DEADLINE, "Requires AVALON_DEADLINE" - - try: - deadline_url = os.environ["DEADLINE_REST_URL"] - except KeyError: - self.log.error("Deadline REST API url not found.") - - # Get a submission job - job = instance.data.get("deadlineSubmissionJob") - if not job: - raise RuntimeError("Can't continue without valid deadline " - "submission prior to this plug-in.") - - data = instance.data.copy() - subset = data["subset"] - state = data.get("publishJobState", "Suspended") - job_name = "{batch} - {subset} [publish image sequence]".format( - batch=job["Props"]["Name"], - subset=subset - ) - - # Add in start/end frame - context = instance.context - start = instance.data.get("startFrame", context.data["startFrame"]) - end = instance.data.get("endFrame", context.data["endFrame"]) - resources = [] - - # Add in regex for sequence filename - # This assumes the output files start with subset name and ends with - # a file extension. - if "ext" in instance.data: - ext = re.escape(instance.data["ext"]) - else: - ext = "\.\D+" - - regex = "^{subset}.*\d+{ext}$".format(subset=re.escape(subset), - ext=ext) - - # Write metadata for publish job - render_job = data.pop("deadlineSubmissionJob") - metadata = { - "regex": regex, - "startFrame": start, - "endFrame": end, - "families": ["imagesequence"], - - # Optional metadata (for debugging) - "metadata": { - "instance": data, - "job": job, - "session": api.Session.copy() - } - } - - # Ensure output dir exists - output_dir = instance.data["outputDir"] - if not os.path.isdir(output_dir): - os.makedirs(output_dir) - - if data.get("extendFrames", False): - - family = "imagesequence" - override = data["overrideExistingFrame"] - - # override = data.get("overrideExistingFrame", False) - out_file = render_job.get("OutFile") - if not out_file: - raise RuntimeError("OutFile not found in render job!") - - extension = os.path.splitext(out_file[0])[1] - _ext = extension[1:] - - # Frame comparison - prev_start = None - prev_end = None - resource_range = range(int(start), int(end)+1) - - # Gather all the subset files (one subset per render pass!) - subset_names = [data["subset"]] - subset_names.extend(data.get("renderPasses", [])) - - for subset_name in subset_names: - version = get_latest_version(asset_name=data["asset"], - subset_name=subset_name, - family=family) - - # Set prev start / end frames for comparison - if not prev_start and not prev_end: - prev_start = version["data"]["startFrame"] - prev_end = version["data"]["endFrame"] - - subset_resources = get_resources(version, _ext) - resource_files = get_resource_files(subset_resources, - resource_range, - override) - - resources.extend(resource_files) - - updated_start = min(start, prev_start) - updated_end = max(end, prev_end) - - # Update metadata and instance start / end frame - self.log.info("Updating start / end frame : " - "{} - {}".format(updated_start, updated_end)) - - # TODO : Improve logic to get new frame range for the - # publish job (publish_filesequence.py) - # The current approach is not following Pyblish logic which is based - # on Collect / Validate / Extract. - - # ---- Collect Plugins --- - # Collect Extend Frames - Only run if extendFrames is toggled - # # # Store in instance: - # # # Previous rendered files per subset based on frames - # # # --> Add to instance.data[resources] - # # # Update publish frame range - - # ---- Validate Plugins --- - # Validate Extend Frames - # # # Check if instance has the requirements to extend frames - # There might have been some things which can be added to the list - # Please do so when fixing this. - - # Start frame - metadata["startFrame"] = updated_start - metadata["metadata"]["instance"]["startFrame"] = updated_start - - # End frame - metadata["endFrame"] = updated_end - metadata["metadata"]["instance"]["endFrame"] = updated_end - - metadata_filename = "{}_metadata.json".format(subset) - metadata_path = os.path.join(output_dir, metadata_filename) - with open(metadata_path, "w") as f: - json.dump(metadata, f, indent=4, sort_keys=True) - - # Generate the payload for Deadline submission - payload = { - "JobInfo": { - "Plugin": "Python", - "BatchName": job["Props"]["Batch"], - "Name": job_name, - "JobType": "Normal", - "JobDependency0": job["_id"], - "UserName": job["Props"]["User"], - "Comment": instance.context.data.get("comment", ""), - "InitialStatus": state - }, - "PluginInfo": { - "Version": "3.6", - "ScriptFile": _get_script(), - "Arguments": '--path "{}"'.format(metadata_path), - "SingleFrameOnly": "True" - }, - - # Mandatory for Deadline, may be empty - "AuxFiles": [] - } - - # Transfer the environment from the original job to this dependent - # job so they use the same environment - environment = job["Props"].get("Env", {}) - payload["JobInfo"].update({ - "EnvironmentKeyValue%d" % index: "{key}={value}".format( - key=key, - value=environment[key] - ) for index, key in enumerate(environment) - }) - - # Avoid copied pools and remove secondary pool - payload["JobInfo"]["Pool"] = "none" - payload["JobInfo"].pop("SecondaryPool", None) - - self.log.info("Submitting..") - self.log.info(json.dumps(payload, indent=4, sort_keys=True)) - - url = "{}/api/jobs".format(deadline_url) - response = requests.post(url, json=payload) - if not response.ok: - raise Exception(response.text) - - # Copy files from previous render if extendFrame is True - if data.get("extendFrames", False): - - self.log.info("Preparing to copy ..") - import shutil - - dest_path = data["outputDir"] - for source in resources: - src_file = os.path.basename(source) - dest = os.path.join(dest_path, src_file) - shutil.copy(source, dest) - - self.log.info("Finished copying %i files" % len(resources)) diff --git a/pype/plugins/global/publish/validate_templates.py b/pype/plugins/global/publish/validate_templates.py deleted file mode 100644 index f806104bb2..0000000000 --- a/pype/plugins/global/publish/validate_templates.py +++ /dev/null @@ -1,42 +0,0 @@ -import pyblish.api -from app.api import ( - Templates -) - -class ValidateTemplates(pyblish.api.ContextPlugin): - """Check if all templates were filed""" - - label = "Validate Templates" - order = pyblish.api.ValidatorOrder - 0.1 - hosts = ["maya", "houdini", "nuke"] - - def process(self, context): - - anatomy = context.data["anatomy"] - if not anatomy: - raise RuntimeError("Did not find templates") - else: - data = { "project": {"name": "D001_projectsx", - "code": "prjX"}, - "representation": "exr", - "VERSION": 3, - "SUBVERSION": 10, - "task": "animation", - "asset": "sh001", - "hierarchy": "ep101/sq01/sh010"} - - - anatomy = context.data["anatomy"].format(data) - self.log.info(anatomy.work.path) - - data = { "project": {"name": "D001_projectsy", - "code": "prjY"}, - "representation": "abc", - "VERSION": 1, - "SUBVERSION": 5, - "task": "lookdev", - "asset": "bob", - "hierarchy": "ep101/sq01/bob"} - - anatomy = context.data["anatomy"].format(data) - self.log.info(anatomy.work.file)