Frame13/ayon-core
1
0
Fork 1
mirror of https://github.com/ynput/ayon-core.git synced 2025-12-24 21:04:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

tile support for separate AOVs and better dep handling

Browse source
This commit is contained in:
Ondřej Samohel 2020-08-19 16:10:41 +02:00
parent 42a19156f4
commit a200be829b
No known key found for this signature in database
GPG key ID: 8A29C663C672C2B7
2 changed files with 34 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

26
pype/plugins/global/publish/submit_publish_job.py
View file

@ -275,26 +275,14 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
# Mandatory for Deadline, may be empty
"AuxFiles": [],
}
"""
In this part we will add file dependencies instead of job dependencies.
This way we don't need to take care of tile assembly job, getting its
id or name. We expect it to produce specific file with specific name
and we are just waiting for them.
"""
# add assembly jobs as dependencies
if instance.data.get("tileRendering"):
self.log.info("Adding tile assembly results as dependencies...")
asset_index = 0
for inst in instances:
for represenation in inst.get("representations", []):
if isinstance(represenation["files"], (list, tuple)):
for file in represenation["files"]:
dependency = os.path.join(output_dir, file)
payload["JobInfo"]["AssetDependency{}".format(asset_index)] = dependency # noqa: E501
else:
dependency = os.path.join(
output_dir, represenation["files"])
payload["JobInfo"]["AssetDependency{}".format(asset_index)] = dependency # noqa: E501
asset_index += 1
self.log.info("Adding tile assembly jobs as dependencies...")
job_index = 0
for assembly_id in instance.data.get("assemblySubmissionJobs"):
payload["JobInfo"]["JobDependency{}".format(job_index)] = assembly_id # noqa: E501
job_index += 1
else:
payload["JobInfo"]["JobDependency0"] = job["_id"]

35
pype/plugins/maya/publish/submit_maya_deadline.py
View file

@ -24,6 +24,7 @@ import copy
import re
import hashlib
from datetime import datetime
import itertools
import clique
import requests
@ -548,7 +549,7 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
assembly_payload["JobInfo"].update(output_filenames)
frame_payloads = []
assembly_payloads = {}
assembly_payloads = []
R_FRAME_NUMBER = re.compile(r".+\.(?P<frame>[0-9]+)\..+") # noqa: N806, E501
REPL_FRAME_NUMBER = re.compile(r"(.+\.)([0-9]+)(\..+)") # noqa: N806, E501
@ -557,11 +558,19 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
# we have aovs and we need to iterate over them
# get files from `beauty`
files = exp[0].get("beauty")
# assembly files are used for assembly jobs as we need to put
# together all AOVs
assembly_files = list(
itertools.chain.from_iterable(
[f for _, f in exp[0].items()]))
if not files:
# if beauty doesn't exists, use first aov we found
files = exp[0].get(list(exp[0].keys())[0])
else:
files = exp
assembly_files = files
frame_jobs = {}
file_index = 1
for file in files:
@ -590,11 +599,16 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
new_payload["PluginInfo"].update(tiles_data["PluginInfo"])
job_hash = hashlib.sha256("{}_{}".format(file_index, file))
frame_jobs[frame] = job_hash.hexdigest()
new_payload["JobInfo"]["ExtraInfo0"] = job_hash.hexdigest()
new_payload["JobInfo"]["ExtraInfo1"] = file
frame_payloads.append(new_payload)
file_index += 1
file_index = 1
for file in assembly_files:
frame = re.search(R_FRAME_NUMBER, file).group("frame")
new_assembly_payload = copy.deepcopy(assembly_payload)
new_assembly_payload["JobInfo"]["Name"] = \
"{} (Frame {})".format(
@ -604,9 +618,9 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
REPL_FRAME_NUMBER,
"\\1{}\\3".format("#" * len(frame)), file)
new_assembly_payload["JobInfo"]["ExtraInfo0"] = job_hash.hexdigest() # noqa: E501
new_assembly_payload["JobInfo"]["ExtraInfo0"] = frame_jobs[frame] # noqa: E501
new_assembly_payload["JobInfo"]["ExtraInfo1"] = file
assembly_payloads[job_hash.hexdigest()] = new_assembly_payload
assembly_payloads.append(new_assembly_payload)
file_index += 1
self.log.info(
@ -622,9 +636,13 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
job_id = response.json()["_id"]
hash = response.json()["Props"]["Ex0"]
file = response.json()["Props"]["Ex1"]
assembly_payloads[hash]["JobInfo"]["JobDependency0"] = job_id
for assembly_job in assembly_payloads:
if assembly_job["JobInfo"]["ExtraInfo0"] == hash:
assembly_job["JobInfo"]["JobDependency0"] = job_id
for assembly_job in assembly_payloads:
file = assembly_job["JobInfo"]["ExtraInfo1"]
# write assembly job config files
now = datetime.now()
@ -646,7 +664,7 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
os.path.dirname(config_file)))
# add config file as job auxFile
assembly_payloads[hash]["AuxFiles"] = [config_file]
assembly_job["AuxFiles"] = [config_file]
with open(config_file, "w") as cf:
print("TileCount={}".format(tiles_count), file=cf)
@ -670,7 +688,7 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
job_idx = 1
instance.data["assemblySubmissionJobs"] = []
for _k, ass_job in assembly_payloads.items():
for ass_job in assembly_payloads:
self.log.info("submitting assembly job {} of {}".format(
job_idx, len(assembly_payloads)
))
@ -679,7 +697,8 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
if not response.ok:
raise Exception(response.text)
instance.data["assemblySubmissionJobs"].append(ass_job)
instance.data["assemblySubmissionJobs"].append(
response.json()["_id"])
job_idx += 1
instance.data["jobBatchName"] = payload["JobInfo"]["BatchName"]