update ftrack publishing to the new api

This commit is contained in:
Milan Kolar 2018-11-29 01:48:42 +01:00
parent 1f6e1abce6
commit 8080b81a8c
14 changed files with 77 additions and 871 deletions

View file

@ -2,7 +2,8 @@ import os
import json
import base64
import ftrack_api
import ftrack_api_old as ftrack_api
reload(ftrack_api)
import pyblish.api

View file

@ -7,7 +7,7 @@ import clique
class IntegrateFtrackApi(pyblish.api.InstancePlugin):
""" Commit components to server. """
order = pyblish.api.IntegratorOrder
order = pyblish.api.IntegratorOrder+0.499
label = "Integrate Ftrack Api"
families = ["ftrack"]

View file

@ -0,0 +1,66 @@
import pyblish.api
import os
class IntegrateFtrackInstance(pyblish.api.InstancePlugin):
"""Collect ftrack component data
Add ftrack component list to instance.
"""
order = pyblish.api.IntegratorOrder + 0.48
label = 'Integrate Ftrack Component'
family_mapping = { 'camera': 'cam',
'look': 'look',
'mayaAscii':'scene',
'model':'geo',
'rig':'rig',
'setdress':'setdress',
'pointcache':'cache',
'review':'mov'}
def process(self, instance):
self.log.debug('instance {}'.format(instance))
asset_name = instance.data["subset"]
assumed_data = instance.data["assumedTemplateData"]
assumed_version = assumed_data["version"]
version_number = int(assumed_version)
family = instance.data['family'].lower()
asset_type = ''
asset_type = self.family_mapping[family]
componentList = []
transfers = instance.data["transfers"]
ft_session = instance.context.data["ftrackSession"]
location = ft_session.query('Location where name is "ftrack.unmanaged"').one()
self.log.debug('location {}'.format(location))
for src, dest in transfers:
filename, ext = os.path.splitext(src)
self.log.debug('source filename: ' + filename)
self.log.debug('source ext: ' + ext)
componentList.append({"assettype_data": {
"short": asset_type,
},
"assetversion_data": {
"version": version_number,
},
"component_data": {
"name": ext[1:], # Default component name is "main".
},
"component_path": dest,
'component_location': location,
"component_overwrite": False,
}
)
self.log.debug('componentsList: {}'.format(str(componentList)))
instance.data["ftrackComponentsList"] = componentList

View file

@ -1,64 +0,0 @@
import pyblish.api
@pyblish.api.log
class CollectFtrackAsset(pyblish.api.ContextPlugin):
""" Adds ftrack asset information to the instance
"""
order = pyblish.api.CollectorOrder + 0.45
label = 'Asset Attributes'
def process(self, context):
for instance in context:
self.log.info('instance {}'.format(instance))
# skipping instance if ftrackData isn't present
if not context.has_data('ftrackData'):
self.log.info('No ftrackData present. Skipping this instance')
continue
# skipping instance if ftrackComponents isn't present
if not instance.has_data('ftrackComponents'):
self.log.info('No ftrackComponents present\
Skipping this instance')
continue
ftrack_data = context.data['ftrackData'].copy()
if not instance.data.get("ftrackAssetName"):
asset_name = instance.data["subset"]
instance.data['ftrackAssetName'] = asset_name
# task type filtering
task_type = ftrack_data['Task']['type'].lower()
asset_type = ''
family = instance.data['family'].lower()
self.log.debug('task type {}'.format(task_type))
if family == 'camera':
asset_type = 'cam'
elif family == 'look':
asset_type = 'look'
elif family == 'mayaAscii':
asset_type = 'scene'
elif family == 'model':
asset_type = 'geo'
elif family == 'rig':
asset_type = 'rig'
elif family == 'setdress':
asset_type = 'setdress'
elif family == 'pointcache':
asset_type = 'cache'
elif family == 'previz':
asset_type = 'mov'
if asset_type:
instance.data['ftrackAssetType'] = asset_type
self.log.debug('asset type: {}'.format(asset_type))
self.log.debug('asset name: {}'.format(asset_name))

View file

@ -1,17 +0,0 @@
import pyblish.api
class CollectFtrackInstance(pyblish.api.InstancePlugin):
"""Collect ftrack component data
Add empty ftrack components to instance.
"""
order = pyblish.api.CollectorOrder + 0.4
label = 'Collect Ftrack Components'
def process(self, instance):
components = {}
instance.data['ftrackComponents'] = components

View file

@ -1,90 +0,0 @@
import os
import ftrack
import pyblish.api
@pyblish.api.log
class CollectFtrackData(pyblish.api.ContextPlugin):
"""Collects ftrack data from FTRACK_CONNECT_EVENT
Arguments:
version (int): version number of the publish
"""
order = pyblish.api.CollectorOrder
hosts = ['*']
version = (0, 1, 0)
def process(self, context):
# accounting for preexiting data
if "ftrackData" in context.data:
data = context.data["ftrackData"]
self.log.info('Found ftrack data: \n\n%s' % data)
return
# getting task id
taskid = ''
taskid = os.environ.get('FTRACK_TASKID', '')
if taskid:
ftrack_data = self.get_data(taskid)
# set ftrack data
context.set_data('ftrackData', value=ftrack_data)
self.log.info('Found ftrack data: \n\n%s' % ftrack_data)
def get_data(self, taskid):
task_codes = {
'Animation': 'anim',
'Layout': 'layout',
'FX': 'fx',
'Compositing': 'comp',
'Motion Graphics': 'mograph',
'Lighting': 'light',
'Modelling': 'geo',
'Rigging': 'rig',
'Art': 'art',
}
try:
task = ftrack.Task(id=taskid)
except ValueError:
task = None
parents = task.getParents()
project = ftrack.Project(task.get('showid'))
taskType = task.getType().getName()
entityType = task.getObjectType()
ctx = {
'Project': {
'name': project.get('fullname'),
'code': project.get('name'),
'id': task.get('showid'),
'root': project.getRoot(),
},
entityType: {
'type': taskType,
'name': task.getName(),
'id': task.getId(),
'code': task_codes.get(taskType, None)
}
}
for parent in parents:
tempdic = {}
if parent.get('entityType') == 'task' and parent.getObjectType():
objectType = parent.getObjectType()
tempdic['name'] = parent.getName()
tempdic['description'] = parent.getDescription()
tempdic['id'] = parent.getId()
if objectType == 'Asset Build':
tempdic['type'] = parent.getType().get('name')
objectType = objectType.replace(' ', '_')
ctx[objectType] = tempdic
return ctx

View file

@ -1,91 +0,0 @@
import pyblish.api
import ftrack
class IntegrateFtrack(pyblish.api.InstancePlugin):
""" Creating components in Ftrack. """
order = pyblish.api.IntegratorOrder + 0.4
label = "Ftrack"
def process(self, instance):
# Skipping instance if ftrackData isn"t present.
if not instance.context.has_data("ftrackData"):
msg = "No ftrackData present. "
msg += "Skipping this instance: \"%s\"" % instance
self.log.info(msg)
return
# Skipping instance if ftrackComponents isn"t present.
if not instance.has_data("ftrackComponents"):
msg = "No ftrackComponents present. "
msg += "Skipping this instance: \"%s\"" % instance
self.log.info(msg)
return
asset_version = instance.data["ftrackAssetVersion"]
version = ftrack.AssetVersion(asset_version["id"])
# Get existing component names.
existing_component_names = []
for component in version.getComponents():
existing_component_names.append(component.getName())
components = instance.data("ftrackComponents")
for component_name in instance.data("ftrackComponents"):
# Get path to component.
path = ""
if "path" in components[component_name]:
path = components[component_name]["path"]
# If no path existing to the component, we skip to the next.
if not path:
continue
# Assuming default picked location unless others specified.
location = ftrack.pickLocation()
if "location" in components[component_name]:
location = components[component_name]["location"]
# Get existing component.
component = None
if component_name in existing_component_names:
component = version.getComponent(name=component_name)
msg = "Found existing \"{0}\" component."
self.log.info(msg.format(component_name))
# To overwrite we have to delete the existing component and
# create a new one. This is to ensure the data gets to the
# location correctly.
if "overwrite" in components[component_name] and component:
msg = "Removing component in location: \"{0}\"."
self.log.info(msg.format(location.getName()))
location.removeComponent(component)
msg = "Deleting \"{0}\" component."
self.log.info(msg.format(component_name))
component.delete()
component = None
if not component:
msg = "Creating \"{0}\" component, with data path: \"{1}\"."
self.log.info(msg.format(component_name, path))
component = version.createComponent(name=component_name,
path=path,
location=location)
cid = component.getId()
instance.data["ftrackComponents"][component_name]["id"] = cid
# make reviewable
if "reviewable" in components[component_name]:
upload = components[component_name]['reviewable']#True
for component in version.getComponents():
if component_name in ("ftrackreview-mp4",
"ftrackreview-webm"):
upload = False
break
if upload:
ftrack.Review.makeReviewable(version, path)

View file

@ -1,115 +0,0 @@
import pyblish.api
import ftrack
@pyblish.api.log
class ExtractFtrack(pyblish.api.InstancePlugin):
""" Creating any Asset or AssetVersion in Ftrack.
"""
order = pyblish.api.IntegratorOrder + 0.3
label = 'Ftrack Extract'
def process(self, instance):
# Skipping instance if ftrackData isn"t present.
if not instance.context.has_data("ftrackData"):
msg = "No ftrackData present. "
msg += "Skipping this instance: \"%s\"" % instance
self.log.info(msg)
return
# Skipping instance if ftrackComponents isn"t present.
if not instance.has_data("ftrackComponents"):
msg = "No ftrackComponents present. "
msg += "Skipping this instance: \"%s\"" % instance
self.log.info(msg)
return
ftrack_data = instance.context.data('ftrackData').copy()
task = ftrack.Task(ftrack_data['Task']['id'])
parent = task.getParent()
asset_data = None
create_version = False
# creating asset
if instance.data('ftrackAssetCreate'):
asset = None
# creating asset from ftrackAssetName
if instance.has_data('ftrackAssetName'):
asset_name = instance.data('ftrackAssetName')
if instance.has_data('ftrackAssetType'):
asset_type = instance.data('ftrackAssetType')
else:
asset_type = ftrack_data['Task']['code']
asset = parent.createAsset(name=asset_name,
assetType=asset_type, task=task)
msg = "Creating new asset cause ftrackAssetName"
msg += " (\"%s\") doesn't exist." % asset_name
self.log.info(msg)
else:
# creating a new asset
asset_name = ftrack_data['Task']['type']
asset_type = ftrack_data['Task']['code']
asset = parent.createAsset(name=asset_type,
assetType=asset_type, task=task)
msg = "Creating asset cause no asset is present."
self.log.info(msg)
create_version = True
# adding asset to ftrack data
asset_data = {'id': asset.getId(),
'name': asset.getName()}
if not asset_data:
asset_data = instance.data('ftrackAsset')
instance.set_data('ftrackAsset', value=asset_data)
# creating version
version = None
if instance.data('ftrackAssetVersionCreate') or create_version:
asset = ftrack.Asset(asset_data['id'])
taskid = ftrack_data['Task']['id']
assumed_data = instance.data["assumedTemplateData"]
assumed_version = assumed_data["version"]
version_number = int(assumed_version)
version = self.GetVersionByNumber(asset, version_number)
if not version:
version = asset.createVersion(comment='', taskid=taskid)
version.set('version', value=version_number)
msg = 'Creating new asset version by %s.' % version_number
self.log.info(msg)
else:
msg = 'Using existing asset version by %s.' % version_number
self.log.info(msg)
asset_version = {'id': version.getId(), 'number': version_number}
instance.set_data('ftrackAssetVersion', value=asset_version)
version.publish()
else:
# using existing version
asset_version = instance.data('ftrackAssetVersion')
version = ftrack.AssetVersion(asset_version['id'])
# adding asset version to ftrack data
instance.set_data('ftrackAssetVersion', value=asset_version)
def GetVersionByNumber(self, asset, number):
for version in asset.getVersions():
try:
if version.getVersion() == int(number):
return version
except:
return None

View file

@ -1,30 +0,0 @@
import pyblish.api
import os
class IntegrateFtrackInstance(pyblish.api.InstancePlugin):
"""Collect ftrack component data
Add empty ftrack components to instance.
"""
order = pyblish.api.IntegratorOrder + 0.31
label = 'Integrate Ftrack Instance'
def process(self, instance):
transfers = instance.data["transfers"]
for src, dest in transfers:
self.log.info("Copying file .. {} -> {}".format(src, dest))
filename, ext = os.path.splitext(src)
self.log.debug('source filename: ' + filename)
self.log.debug('source ext: ' + ext)
components = instance.data['ftrackComponents']
components[str(ext)[1:]] = {'path': dest}
self.log.debug('components: {}'.format(str(components)))

View file

@ -1,150 +0,0 @@
import pyblish.api
import ftrack
@pyblish.api.log
class ValidateFtrack(pyblish.api.InstancePlugin):
""" Validate the existence of Asset, AssetVersion and Components.
"""
order = pyblish.api.ValidatorOrder + 0.1
optional = True
label = 'Ftrack'
def process(self, instance):
context = instance.context
# Skipping instance if ftrackData isn"t present.
if not context.has_data("ftrackData"):
msg = "No ftrackData present. "
msg += "Skipping this instance: \"%s\"" % instance
self.log.info(msg)
return
# Skipping instance if ftrackComponents isn"t present.
if not instance.has_data("ftrackComponents"):
msg = "No ftrackComponents present. "
msg += "Skipping this instance: \"%s\"" % instance
self.log.info(msg)
return
ftrack_data = context.data('ftrackData').copy()
task = ftrack.Task(ftrack_data['Task']['id'])
# checking asset
create_asset = True
asset = None
if instance.has_data('ftrackAssetType'):
asset_type = instance.data('ftrackAssetType')
else:
asset_type = ftrack_data['Task']['code']
assets = task.getAssets(assetTypes=[asset_type])
if len(assets) == 0:
instance.set_data('ftrackAssetCreate', value=True)
return
if instance.has_data('ftrackAssetName'):
# searching for existing asset
asset_name = instance.data('ftrackAssetName')
for a in assets:
if asset_name.lower() == a.getName().lower():
asset = a
create_asset = False
msg = 'Found existing asset from ftrackAssetName'
self.log.info(msg)
else:
# if only one asset exists, then we'll use that asset
msg = "Can't compute asset."
if len(assets) == 1:
print('FOUND ONE ASSET')
for a in assets:
print a
asset = a
create_asset = False
assert asset, msg
self.log.info('Found existing asset by default.')
elif len(assets) > 1:
asset_name = ftrack_data['Task']['type']
for a in assets:
msg += " Multiple assets on shot: \n\n%s" % a
if asset_name.lower() == a.getName().lower():
asset = a
create_asset = False
assert asset, msg
self.log.info('Found existing asset by default.')
else:
self.log.info('No asset found, new one will be created.')
# adding asset to ftrack data
if asset:
asset_data = {'id': asset.getId(),
'name': asset.getName()}
instance.set_data('ftrackAsset', value=asset_data)
instance.set_data('ftrackAssetCreate', value=create_asset)
# if we are creating a new asset,
# then we don't need to validate the rest
if create_asset:
return
# checking version
assumed_data = instance.data["assumedTemplateData"]
assumed_version = assumed_data["version"]
version_number = int(assumed_version)
create_version = True
version = None
# search for existing version
for v in asset.getVersions():
if int(v.getVersion()) == version_number:
msg = "AssetVersion exists but is not visible in UI."
assert v.get('ispublished'), msg
version = v
# ftrack_data['AssetVersion'] = {'id': v.getId(),
# 'number': version_number}
asset_version = {'id': v.getId(), 'number': version_number}
instance.set_data('ftrackAssetVersion', value=asset_version)
create_version = False
msg = 'Found existing version number: %s' % version_number
self.log.info(msg)
instance.set_data('ftrackAssetVersionCreate', value=create_version)
# if we are creating a new asset version,
# then we don't need to validate the rest
if create_version:
return
# checking components
online_components = version.getComponents()
ftrack_components = instance.data('ftrackComponents').copy()
for local_c in ftrack_components:
for online_c in online_components:
if local_c == online_c.getName():
# warning about existing components
msg = 'Component "%s" already exists. ' % local_c
msg += 'To replace it delete it in the browser first.'
if not ftrack_components[local_c].get("overwrite", False):
self.log.warning(msg)
# validating review components
if 'reviewable' in ftrack_components[local_c]:
msg = 'Reviewable component already exists in the\
version. To replace it\
delete it in the webUI first'
assert online_c.getName() not in (
'ftrackreview-mp4', 'ftrackreview-webm'), msg

View file

@ -1,26 +0,0 @@
import os
import ftrack_api_27 as ftrack_api
import pyblish.api
class PyblishFtrackCollectFtrackApi(pyblish.api.ContextPlugin):
""" Collects an ftrack session and the current task id. """
order = pyblish.api.CollectorOrder
label = "Ftrack"
def process(self, context):
# Collect session
session = ftrack_api.Session()
context.data["ftrackSession"] = session
# Collect task
taskid = ""
taskid = os.environ.get("FTRACK_TASKID", "")
context.data["ftrackTask"] = session.get("Task", taskid)
self.log.info("collected: {}".format(context.data["ftrackTask"]))

View file

@ -1,281 +0,0 @@
import os
import pyblish.api
from pype.vendor import clique
class PyblishFtrackIntegrateFtrackApi(pyblish.api.InstancePlugin):
""" Commit components to server. """
order = pyblish.api.IntegratorOrder
label = "Ftrack"
families = ["ftrack"]
def query(self, entitytype, data):
""" Generate a query expression from data supplied.
If a value is not a string, we'll add the id of the entity to the
query.
Args:
entitytype (str): The type of entity to query.
data (dict): The data to identify the entity.
exclusions (list): All keys to exclude from the query.
Returns:
str: String query to use with "session.query"
"""
queries = []
for key, value in data.iteritems():
if not isinstance(value, (basestring, int)):
if "id" in value.keys():
queries.append(
"{0}.id is \"{1}\"".format(key, value["id"])
)
else:
queries.append("{0} is \"{1}\"".format(key, value))
query = (
"select id from " + entitytype + " where " + " and ".join(queries)
)
self.log.debug(query)
return query
def process(self, instance):
session = instance.context.data["ftrackSession"]
task = instance.context.data["ftrackTask"]
info_msg = "Created new {entity_type} with data: {data}"
info_msg += ", metadata: {metadata}."
# Iterate over components and publish
for data in instance.data.get("ftrackComponentsList", []):
# AssetType
# Get existing entity.
assettype_data = {"short": "upload"}
assettype_data.update(data.get("assettype_data", {}))
assettype_entity = session.query(
self.query("AssetType", assettype_data)
).first()
# Create a new entity if none exits.
if not assettype_entity:
assettype_entity = session.create("AssetType", assettype_data)
self.log.info(
"Created new AssetType with data: ".format(assettype_data)
)
# Asset
# Get existing entity.
asset_data = {
"name": task["name"],
"type": assettype_entity,
"parent": task["parent"],
}
asset_data.update(data.get("asset_data", {}))
asset_entity = session.query(
self.query("Asset", asset_data)
).first()
# Extracting metadata, and adding after entity creation. This is
# due to a ftrack_api bug where you can't add metadata on creation.
asset_metadata = asset_data.pop("metadata", {})
# Create a new entity if none exits.
if not asset_entity:
asset_entity = session.create("Asset", asset_data)
self.log.info(
info_msg.format(
entity_type="Asset",
data=asset_data,
metadata=asset_metadata
)
)
# Adding metadata
existing_asset_metadata = asset_entity["metadata"]
existing_asset_metadata.update(asset_metadata)
asset_entity["metadata"] = existing_asset_metadata
# AssetVersion
# Get existing entity.
assetversion_data = {
"version": 0,
"asset": asset_entity,
"task": task
}
assetversion_data.update(data.get("assetversion_data", {}))
assetversion_entity = session.query(
self.query("AssetVersion", assetversion_data)
).first()
# Extracting metadata, and adding after entity creation. This is
# due to a ftrack_api bug where you can't add metadata on creation.
assetversion_metadata = assetversion_data.pop("metadata", {})
# Create a new entity if none exits.
if not assetversion_entity:
assetversion_entity = session.create(
"AssetVersion", assetversion_data
)
self.log.info(
info_msg.format(
entity_type="AssetVersion",
data=assetversion_data,
metadata=assetversion_metadata
)
)
# Adding metadata
existing_assetversion_metadata = assetversion_entity["metadata"]
existing_assetversion_metadata.update(assetversion_metadata)
assetversion_entity["metadata"] = existing_assetversion_metadata
# Have to commit the version and asset, because location can't
# determine the final location without.
session.commit()
# Component
# Get existing entity.
component_data = {
"name": "main",
"version": assetversion_entity
}
component_data.update(data.get("component_data", {}))
component_entity = session.query(
self.query("Component", component_data)
).first()
component_overwrite = data.get("component_overwrite", False)
location = data.get("component_location", session.pick_location())
# Overwrite existing component data if requested.
if component_entity and component_overwrite:
origin_location = session.query(
"Location where name is \"ftrack.origin\""
).one()
# Removing existing members from location
components = list(component_entity.get("members", []))
components += [component_entity]
for component in components:
for loc in component["component_locations"]:
if location["id"] == loc["location_id"]:
location.remove_component(
component, recursive=False
)
# Deleting existing members on component entity
for member in component_entity.get("members", []):
session.delete(member)
del(member)
session.commit()
# Reset members in memory
if "members" in component_entity.keys():
component_entity["members"] = []
# Add components to origin location
try:
collection = clique.parse(data["component_path"])
except ValueError:
# Assume its a single file
# Changing file type
name, ext = os.path.splitext(data["component_path"])
component_entity["file_type"] = ext
origin_location.add_component(
component_entity, data["component_path"]
)
else:
# Changing file type
component_entity["file_type"] = collection.format("{tail}")
# Create member components for sequence.
for member_path in collection:
size = 0
try:
size = os.path.getsize(member_path)
except OSError:
pass
name = collection.match(member_path).group("index")
member_data = {
"name": name,
"container": component_entity,
"size": size,
"file_type": os.path.splitext(member_path)[-1]
}
component = session.create(
"FileComponent", member_data
)
origin_location.add_component(
component, member_path, recursive=False
)
component_entity["members"].append(component)
# Add components to location.
location.add_component(
component_entity, origin_location, recursive=True
)
data["component"] = component
msg = "Overwriting Component with path: {0}, data: {1}, "
msg += "location: {2}"
self.log.info(
msg.format(
data["component_path"],
component_data,
location
)
)
# Extracting metadata, and adding after entity creation. This is
# due to a ftrack_api bug where you can't add metadata on creation.
component_metadata = component_data.pop("metadata", {})
# Create new component if none exists.
if not component_entity:
component_entity = assetversion_entity.create_component(
data["component_path"],
data=component_data,
location=location
)
data["component"] = component_entity
msg = "Created new Component with path: {0}, data: {1}"
msg += ", metadata: {2}, location: {3}"
self.log.info(
msg.format(
data["component_path"],
component_data,
component_metadata,
location
)
)
# Adding metadata
existing_component_metadata = component_entity["metadata"]
existing_component_metadata.update(component_metadata)
component_entity["metadata"] = existing_component_metadata
# Inform user about no changes to the database.
if component_entity and not component_overwrite:
data["component"] = component_entity
self.log.info(
"Found existing component, and no request to overwrite. "
"Nothing has been changed."
)
else:
# Commit changes.
session.commit()

View file

@ -109,6 +109,9 @@ class CollectInstances(pyblish.api.ContextPlugin):
# Store the exact members of the object set
instance.data["setMembers"] = members
# make ftrack publishable
instance.data["families"] = ['ftrack']
# Define nice label
name = cmds.ls(objset, long=False)[0] # use short name
label = "{0} ({1})".format(name,

View file

@ -161,21 +161,21 @@ class Session(object):
if api_key is None:
api_key = os.environ.get(
'ftrack_api_old_KEY',
'FTRACK_API_KEY',
# Backwards compatibility
os.environ.get('ftrack_api_oldKEY')
os.environ.get('FTRACK_APIKEY')
)
if not api_key:
raise TypeError(
'Required "api_key" not specified. Pass as argument or set in '
'environment variable ftrack_api_old_KEY.'
'environment variable FTRACK_API_KEY.'
)
self._api_key = api_key
if api_user is None:
api_user = os.environ.get('ftrack_api_old_USER')
api_user = os.environ.get('FTRACK_API_USER')
if not api_user:
try:
api_user = getpass.getuser()
@ -185,7 +185,7 @@ class Session(object):
if not api_user:
raise TypeError(
'Required "api_user" not specified. Pass as argument, set in '
'environment variable ftrack_api_old_USER or one of the standard '
'environment variable FTRACK_API_USER or one of the standard '
'environment variables used by Python\'s getpass module.'
)