Frame13/ayon-core
1
0
Fork 1
mirror of https://github.com/ynput/ayon-core.git synced 2025-12-25 05:14:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merged in bugfix/PYPE-104-clean-pythonpath-for-deadline (pull request #36)

Browse source 
Bugfix/PYPE-104 clean pythonpath for deadline

Approved-by: Milan Kolar <milan@orbi.tools>
This commit is contained in:
Milan Kolar 2019-01-02 10:51:17 +00:00
commit ffba5904d7
14 changed files with 190 additions and 49 deletions
Download patch file Download diff file Expand all files Collapse all files

2
pype/plugins/ftrack/collect_ftrack_api.py
View file

@ -25,6 +25,6 @@ class CollectFtrackApi(pyblish.api.ContextPlugin):
result = session.query('Task where\
project.full_name is "{0}" and\
name is "{1}" and\
parent.name is "{2}"'.format(project, task, asset))
parent.name is "{2}"'.format(project, task, asset)).one()
context.data["ftrackTask"] = result

2
pype/plugins/ftrack/integrate_ftrack_api.py
View file

@ -82,6 +82,8 @@ class IntegrateFtrackApi(pyblish.api.InstancePlugin):
self.query("Asset", asset_data)
).first()
self.log.info(asset_entity)
# Extracting metadata, and adding after entity creation. This is
# due to a ftrack_api bug where you can't add metadata on creation.
asset_metadata = asset_data.pop("metadata", {})

13
pype/plugins/ftrack/integrate_ftrack_instances.py
View file

@ -45,7 +45,6 @@ class IntegrateFtrackInstance(pyblish.api.InstancePlugin):
ft_session = instance.context.data["ftrackSession"]
for file in instance.data['destination_list']:
self.log.debug('file {}'.format(file))
@ -56,21 +55,21 @@ class IntegrateFtrackInstance(pyblish.api.InstancePlugin):
if ext in ['.mov']:
location = ft_session.query(
'Location where name is "ftrack.server"').one()
'Location where name is "ftrack.server"').one()
component_data = {
"name": "ftrackreview-mp4", # Default component name is "main".
"metadata": {'ftr_meta': json.dumps({
'frameIn': int(instance.data["startFrame"]),
'frameOut': int(instance.data["startFrame"]),
'frameRate': 25})}
}
'frameIn': int(instance.data["startFrame"]),
'frameOut': int(instance.data["startFrame"]),
'frameRate': 25})}
}
elif ext in [".jpg"]:
component_data = {
"name": "thumbnail" # Default component name is "main".
}
thumbnail = True
location = ft_session.query(
'Location where name is "ftrack.server"').one()
'Location where name is "ftrack.server"').one()
else:
component_data = {
"name": ext[1:] # Default component name is "main".

26
pype/plugins/global/publish/cleanup.py
View file

@ -18,16 +18,16 @@ class CleanUp(pyblish.api.InstancePlugin):
import tempfile
# staging_dir = instance.data.get("stagingDir", None)
# if not staging_dir or not os.path.exists(staging_dir):
# self.log.info("No staging directory found: %s" % staging_dir)
# return
#
# temp_root = tempfile.gettempdir()
# if not os.path.normpath(staging_dir).startswith(temp_root):
# self.log.info("Skipping cleanup. Staging directory is not in the "
# "temp folder: %s" % staging_dir)
# return
#
# self.log.info("Removing temporary folder ...")
# shutil.rmtree(staging_dir)
staging_dir = instance.data.get("stagingDir", None)
if not staging_dir or not os.path.exists(staging_dir):
self.log.info("No staging directory found: %s" % staging_dir)
return
temp_root = tempfile.gettempdir()
if not os.path.normpath(staging_dir).startswith(temp_root):
self.log.info("Skipping cleanup. Staging directory is not in the "
"temp folder: %s" % staging_dir)
return
self.log.info("Removing temporary folder ...")
shutil.rmtree(staging_dir)

12
pype/plugins/global/publish/collect_filesequences.py
View file

@ -88,7 +88,6 @@ class CollectFileSequences(pyblish.api.ContextPlugin):
order = pyblish.api.CollectorOrder
targets = ["filesequence"]
label = "File Sequences"
hosts = ['maya']
def process(self, context):
@ -149,7 +148,7 @@ class CollectFileSequences(pyblish.api.ContextPlugin):
raise RuntimeError("Invalid sequence")
# Get family from the data
families = data.get("families", ["imagesequence"])
families = data.get("families", ["render"])
assert isinstance(families, (list, tuple)), "Must be iterable"
assert families, "Must have at least a single family"
@ -168,6 +167,9 @@ class CollectFileSequences(pyblish.api.ContextPlugin):
start = data.get("startFrame", indices[0])
end = data.get("endFrame", indices[-1])
# root = os.path.normpath(root)
# self.log.info("Source: {}}".format(data.get("source", "")))
instance.data.update({
"name": str(collection),
"family": families[0], # backwards compatibility / pyblish
@ -177,9 +179,13 @@ class CollectFileSequences(pyblish.api.ContextPlugin):
"stagingDir": root,
"files": [list(collection)],
"startFrame": start,
"endFrame": end
"endFrame": end,
"source": data.get('source', '')
})
instance.append(collection)
if data.get('user'):
context.data["user"] = data['user']
self.log.debug("Collected instance:\n"
"{}".format(pprint.pformat(instance.data)))

2
pype/plugins/global/publish/collect_json.py
View file

@ -3,7 +3,7 @@ import json
import re
import pyblish.api
from config.vendor import clique
from pype.vendor import clique
class CollectJSON(pyblish.api.ContextPlugin):

3
pype/plugins/global/publish/extract_json.py
View file

@ -4,7 +4,7 @@ import datetime
import time
import pyblish.api
from config.vendor import clique
from pype.vendor import clique
class ExtractJSON(pyblish.api.ContextPlugin):
@ -12,6 +12,7 @@ class ExtractJSON(pyblish.api.ContextPlugin):
order = pyblish.api.IntegratorOrder
label = "JSON"
hosts = ['nuke', 'maya']
def process(self, context):

16
pype/plugins/global/publish/integrate.py
View file

@ -25,7 +25,6 @@ class IntegrateAsset(pyblish.api.InstancePlugin):
order = pyblish.api.IntegratorOrder
families = ["animation",
"camera",
"imagesequence",
"look",
"mayaAscii",
"model",
@ -36,7 +35,8 @@ class IntegrateAsset(pyblish.api.InstancePlugin):
"vrayproxy",
"yetiRig",
"yeticache",
"review"]
"review",
"scene"]
def process(self, instance):
@ -351,11 +351,17 @@ class IntegrateAsset(pyblish.api.InstancePlugin):
families.append(instance_family)
families += current_families
self.log.debug("Registered roor: {}".format(api.registered_root()))
# create relative source path for DB
relative_path = os.path.relpath(context.data["currentFile"],
api.registered_root())
source = os.path.join("{root}", relative_path).replace("\\", "/")
try:
source = instance.data['source']
except KeyError:
source = context.data["currentFile"]
relative_path = os.path.relpath(source, api.registered_root())
source = os.path.join("{root}", relative_path).replace("\\", "/")
self.log.debug("Source: {}".format(source))
version_data = {"families": families,
"time": context.data["time"],
"author": context.data["user"],

17
pype/plugins/nuke/publish/integrate_rendered_frames.py → pype/plugins/global/publish/integrate_rendered_frames.py
View file

@ -24,7 +24,7 @@ class IntegrateFrames(pyblish.api.InstancePlugin):
label = "Integrate Frames"
order = pyblish.api.IntegratorOrder
families = ["prerendered.frames"]
families = ["prerendered.frames", "imagesequence", "render"]
def process(self, instance):
@ -96,7 +96,7 @@ class IntegrateFrames(pyblish.api.InstancePlugin):
self.log.info("Verifying version from assumed destination")
assumed_data = instance.data["assumedTemplateData"]
assumed_version = assumed_data["version"]
assumed_version = assumed_data["VERSION"]
if assumed_version != next_version:
raise AttributeError("Assumed version 'v{0:03d}' does not match"
"next version in database "
@ -231,7 +231,7 @@ class IntegrateFrames(pyblish.api.InstancePlugin):
"asset": ASSET,
"family": instance.data['family'],
"subset": subset["name"],
"version": version["name"],
"VERSION": version["name"],
"hierarchy": hierarchy,
"representation": ext[1:]
}
@ -344,10 +344,13 @@ class IntegrateFrames(pyblish.api.InstancePlugin):
families.append(instance_family)
families += current_families
# create relative source path for DB
relative_path = os.path.relpath(context.data["currentFile"],
api.registered_root())
source = os.path.join("{root}", relative_path).replace("\\", "/")
try:
source = instance.data['source']
except KeyError:
source = context.data["currentFile"]
relative_path = os.path.relpath(source, api.registered_root())
source = os.path.join("{root}", relative_path).replace("\\", "/")
version_data = {"families": families,
"time": context.data["time"],

18
pype/plugins/global/publish/submit_publish_job.py
View file

@ -120,7 +120,7 @@ class SubmitDependentImageSequenceJobDeadline(pyblish.api.InstancePlugin):
"""
label = "Submit image sequence jobs to Deadline"
order = pyblish.api.IntegratorOrder + 0.1
order = pyblish.api.IntegratorOrder + 0.2
hosts = ["fusion", "maya", "nuke"]
@ -148,6 +148,7 @@ class SubmitDependentImageSequenceJobDeadline(pyblish.api.InstancePlugin):
"submission prior to this plug-in.")
data = instance.data.copy()
asset = data.get("asset") or api.Session["AVALON_ASSET"]
subset = data["subset"]
state = data.get("publishJobState", "Suspended")
job_name = "{batch} - {subset} [publish image sequence]".format(
@ -172,13 +173,24 @@ class SubmitDependentImageSequenceJobDeadline(pyblish.api.InstancePlugin):
regex = "^{subset}.*\d+{ext}$".format(subset=re.escape(subset),
ext=ext)
try:
source = data['source']
except KeyError:
source = context.data["currentFile"]
relative_path = os.path.relpath(source, api.registered_root())
source = os.path.join("{root}", relative_path).replace("\\", "/")
# Write metadata for publish job
render_job = data.pop("deadlineSubmissionJob")
metadata = {
"asset": asset,
"regex": regex,
"startFrame": start,
"endFrame": end,
"families": ["imagesequence"],
"families": ["render"],
"source": source,
"user": context.data["user"],
# Optional metadata (for debugging)
"metadata": {
@ -195,7 +207,7 @@ class SubmitDependentImageSequenceJobDeadline(pyblish.api.InstancePlugin):
if data.get("extendFrames", False):
family = "imagesequence"
family = "render"
override = data["overrideExistingFrame"]
# override = data.get("overrideExistingFrame", False)

2
pype/plugins/maya/publish/collect_current_file.py
View file

@ -1,6 +1,8 @@
from maya import cmds
import pyblish.api
import os
from pype.maya import lib
class CollectMayaCurrentFile(pyblish.api.ContextPlugin):

52
pype/plugins/maya/publish/collect_scene.py Normal file
View file

@ -0,0 +1,52 @@
from maya import cmds
import pyblish.api
import avalon.api
import os
from pype.maya import lib
class CollectMayaScene(pyblish.api.ContextPlugin):
"""Inject the current working file into context"""
order = pyblish.api.CollectorOrder - 0.1
label = "Maya Scene"
hosts = ['maya']
def process(self, context):
"""Inject the current working file"""
current_file = context.data['currentFile']
folder, file = os.path.split(current_file)
filename, ext = os.path.splitext(file)
task = avalon.api.Session["AVALON_TASK"]
data = {}
for key, value in lib.collect_animation_data().items():
data[key] = value
# create instance
instance = context.create_instance(name=filename)
subset = 'scene' + task.capitalize()
data.update({
"subset": subset,
"asset": os.getenv("AVALON_ASSET", None),
"label": subset,
"publish": True,
"family": 'scene',
"representation": "ma",
"setMembers": [current_file],
"stagingDir": folder
})
data['files'] = [file]
instance.data.update(data)
self.log.info('Collected instance: {}'.format(file))
self.log.info('Scene path: {}'.format(current_file))
self.log.info('stagin Dir: {}'.format(folder))
self.log.info('subset: {}'.format(filename))

70
pype/plugins/maya/publish/submit_deadline.py
View file

@ -51,7 +51,7 @@ def get_renderer_variables(renderlayer=None):
# returns an index number.
filename_base = os.path.basename(filename_0)
extension = os.path.splitext(filename_base)[-1].strip(".")
filename_prefix = "<Scene>/<Scene>_<RenderLayer>/<RenderLayer>"
filename_prefix = "<Scene>/<RenderLayer>/<RenderLayer>"
return {"ext": extension,
"filename_prefix": filename_prefix,
@ -78,7 +78,7 @@ def preview_fname(folder, scene, layer, padding, ext):
"""
# Following hardcoded "<Scene>/<Scene>_<Layer>/<Layer>"
output = "{scene}/{scene}_{layer}/{layer}.{number}.{ext}".format(
output = "{scene}/{layer}/{layer}.{number}.{ext}".format(
scene=scene,
layer=layer,
number="#" * padding,
@ -97,12 +97,14 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
"""
label = "Submit to Deadline"
order = pyblish.api.IntegratorOrder
order = pyblish.api.IntegratorOrder + 0.1
hosts = ["maya"]
families = ["renderlayer"]
def process(self, instance):
self.log.debug('Starting deadline submitter')
try:
deadline_url = os.environ["DEADLINE_REST_URL"]
except KeyError:
@ -110,11 +112,29 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
# AVALON_DEADLINE = api.Session.get("AVALON_DEADLINE",
# "http://localhost:8082")
# assert AVALON_DEADLINE, "Requires AVALON_DEADLINE"
# assert AVALON_DEADLINE, "Requires AVALON_DEADLINE
context = instance.context
filepath = None
allInstances = []
for result in context.data["results"]:
if (result["instance"] is not None and
result["instance"] not in allInstances):
allInstances.append(result["instance"])
for inst in allInstances:
print(inst)
if inst.data['family'] == 'scene':
filepath = inst.data['destination_list'][0]
if not filepath:
filepath = context.data["currentFile"]
self.log.debug(filepath)
workspace = context.data["workspaceDir"]
filepath = context.data["currentFile"]
filename = os.path.basename(filepath)
comment = context.data.get("comment", "")
scene = os.path.splitext(filename)[0]
@ -209,6 +229,14 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
# running Linux and the submitter is on Windows.
"PYTHONPATH",
"MTOA_EXTENSIONS_PATH",
"MTOA_EXTENSIONS",
"DYLD_LIBRARY_PATH",
"MAYA_RENDER_DESC_PATH",
"MAYA_MODULE_PATH",
"ARNOLD_PLUGIN_PATH",
"AVALON_SCHEMA",
# todo: This is a temporary fix for yeti variables
"PEREGRINEL_LICENSE",
"REDSHIFT_MAYAEXTENSIONSPATH",
@ -216,14 +244,43 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
"VRAY_FOR_MAYA2018_PLUGINS_X64",
"VRAY_PLUGINS_X64",
"VRAY_USE_THREAD_AFFINITY",
"MAYA_MODULE_PATH"
"MAYA_MODULE_PATH",
"TOOL_ENV"
]
environment = dict({key: os.environ[key] for key in keys
if key in os.environ}, **api.Session)
for path in os.environ:
if path.lower().startswith('pype_'):
environment[path] = os.environ[path]
PATHS = os.environ["PATH"].split(";")
environment["PATH"] = ";".join([p for p in PATHS
if p.startswith("P:")])
clean_pythonpath = ''
for path in environment['PYTHONPATH'].split(os.pathsep):
# self.log.debug('checking path for UTF: {}'.format(path))
try:
path.decode('UTF-8', 'strict')
# path = path.lower().replace("k:/", r"\\kre-c01\\share\\").replace("p:/", r"\\kre-p01\\share\\")
clean_pythonpath += path + os.pathsep
except UnicodeDecodeError:
self.log.debug('path contains non UTF characters')
environment['PYTHONPATH'] = clean_pythonpath
for key in environment:
remapped_key = ''
list_paths = environment[key].split(os.pathsep)
if len(list_paths) > 1:
for path in list_paths:
path = path.replace("K:/", "\\\\kre-c01\\share\\").replace("P:/", "\\\\kre-p01\\share\\")
path = path.replace("K:\\", "\\\\kre-c01\\share\\").replace("P:\\", "\\\\kre-p01\\share\\")
remapped_key += path + os.pathsep
else:
path = list_paths[0].replace("K:/", "\\\\kre-c01\\share\\").replace("P:/", "\\\\kre-p01\\share\\")
path = path.replace("K:\\", "\\\\kre-c01\\share\\").replace("P:\\", "\\\\kre-p01\\share\\")
remapped_key = path
environment[key] = remapped_key
payload["JobInfo"].update({
"EnvironmentKeyValue%d" % index: "{key}={value}".format(
@ -251,6 +308,7 @@ class MayaSubmitDeadline(pyblish.api.InstancePlugin):
raise Exception(response.text)
# Store output dir for unified publisher (filesequence)
instance.data['source'] = filepath
instance.data["outputDir"] = os.path.dirname(output_filename_0)
instance.data["deadlineSubmissionJob"] = response.json()

4
pype/plugins/maya/publish/validate_rendersettings.py
View file

@ -34,8 +34,8 @@ class ValidateRenderSettings(pyblish.api.InstancePlugin):
actions = [pype.api.RepairAction]
DEFAULT_PADDING = 4
RENDERER_PREFIX = {"vray": "<Scene>/<Scene>_<Layer>/<Layer>"}
DEFAULT_PREFIX = "<Scene>/<Scene>_<RenderLayer>/<RenderLayer>"
RENDERER_PREFIX = {"vray": "<Scene>/<Layer>/<Layer>"}
DEFAULT_PREFIX = "<Scene>/<RenderLayer>/<RenderLayer>"
def process(self, instance):