Merge branch 'refs/heads/develop' into feature/PYPE-570-maya-renderlayer-creator

This commit is contained in:
Ondrej Samohel 2019-12-06 14:39:30 +01:00
commit 6c978adea8
24 changed files with 4771 additions and 5307 deletions

View file

@ -5,7 +5,8 @@ import json
import arrow
import logging
import ftrack_api
from pype.ftrack import BaseAction, get_ca_mongoid
from pype.ftrack import BaseAction
from pype.ftrack.lib.avalon_sync import CustAttrIdKey
from pypeapp import config
from ftrack_api.exception import NoResultFoundError
@ -171,7 +172,6 @@ class CustomAttributes(BaseAction):
def avalon_mongo_id_attributes(self, session):
# Attribute Name and Label
cust_attr_name = get_ca_mongoid()
cust_attr_label = 'Avalon/Mongo Id'
# Types that don't need object_type_id
@ -207,7 +207,7 @@ class CustomAttributes(BaseAction):
group = self.get_group('avalon')
data = {}
data['key'] = cust_attr_name
data['key'] = CustAttrIdKey
data['label'] = cust_attr_label
data['type'] = custom_attribute_type
data['default'] = ''

View file

@ -142,6 +142,13 @@ class CreateProjectFolders(BaseAction):
else:
data['project_id'] = parent['project']['id']
existing_entity = self.session.query((
"TypedContext where name is \"{}\" and "
"parent_id is \"{}\" and project_id is \"{}\""
).format(name, data['parent_id'], data['project_id'])).first()
if existing_entity:
return existing_entity
new_ent = self.session.create(ent_type, data)
self.session.commit()
return new_ent

View file

@ -2,12 +2,9 @@ import os
import json
from ruamel import yaml
import ftrack_api
from pype.ftrack import BaseAction
from pypeapp import config
from pype.ftrack.lib import get_avalon_attr
from ftrack_api import session as fa_session
from pype.ftrack.lib.avalon_sync import get_avalon_attr
class PrepareProject(BaseAction):
@ -55,6 +52,8 @@ class PrepareProject(BaseAction):
attributes_to_set = {}
for attr in hier_cust_attrs:
key = attr["key"]
if key.startswith("avalon_"):
continue
attributes_to_set[key] = {
"label": attr["label"],
"object": attr,
@ -65,6 +64,8 @@ class PrepareProject(BaseAction):
if attr["entity_type"].lower() != "show":
continue
key = attr["key"]
if key.startswith("avalon_"):
continue
attributes_to_set[key] = {
"label": attr["label"],
"object": attr,

File diff suppressed because it is too large Load diff

View file

@ -1,7 +1,5 @@
import os
import ftrack_api
from pype.ftrack import BaseAction
from ftrack_api import session as fa_session
class ActionAskWhereIRun(BaseAction):

File diff suppressed because it is too large Load diff

View file

@ -1,6 +1,6 @@
import ftrack_api
from pype.ftrack import BaseEvent, get_ca_mongoid
from pype.ftrack.events.event_sync_to_avalon import SyncToAvalon
from pype.ftrack.lib import BaseEvent
from pype.ftrack.lib.avalon_sync import CustAttrIdKey
from pype.ftrack.events.event_sync_to_avalon import SyncToAvalonEvent
class DelAvalonIdFromNew(BaseEvent):
@ -11,7 +11,8 @@ class DelAvalonIdFromNew(BaseEvent):
Priority of this event must be less than SyncToAvalon event
'''
priority = SyncToAvalon.priority - 1
priority = SyncToAvalonEvent.priority - 1
ignore_me = True
def launch(self, session, event):
created = []
@ -28,7 +29,7 @@ class DelAvalonIdFromNew(BaseEvent):
elif (
entity.get('action', None) == 'update' and
get_ca_mongoid() in entity['keys'] and
CustAttrIdKey in entity['keys'] and
entity_id in created
):
ftrack_entity = session.get(
@ -37,13 +38,11 @@ class DelAvalonIdFromNew(BaseEvent):
)
cust_attr = ftrack_entity['custom_attributes'][
get_ca_mongoid()
CustAttrIdKey
]
if cust_attr != '':
ftrack_entity['custom_attributes'][
get_ca_mongoid()
] = ''
ftrack_entity['custom_attributes'][CustAttrIdKey] = ''
session.commit()
except Exception:
@ -53,5 +52,4 @@ class DelAvalonIdFromNew(BaseEvent):
def register(session, plugins_presets):
'''Register plugin. Called when used as an plugin.'''
DelAvalonIdFromNew(session, plugins_presets).register()

View file

@ -1,213 +0,0 @@
import os
import sys
from pype.ftrack.lib.io_nonsingleton import DbConnector
import ftrack_api
from pype.ftrack import BaseEvent, lib
from bson.objectid import ObjectId
class SyncHierarchicalAttrs(BaseEvent):
# After sync to avalon event!
priority = 101
db_con = DbConnector()
ca_mongoid = lib.get_ca_mongoid()
def launch(self, session, event):
# Filter entities and changed values if it makes sence to run script
processable = []
processable_ent = {}
for ent in event['data']['entities']:
# Ignore entities that are not tasks or projects
if ent['entityType'].lower() not in ['task', 'show']:
continue
action = ent.get("action")
# skip if remove (Entity does not exist in Ftrack)
if action == "remove":
continue
# When entity was add we don't care about keys
if action != "add":
keys = ent.get('keys')
if not keys:
continue
entity = session.get(self._get_entity_type(ent), ent['entityId'])
processable.append(ent)
processable_ent[ent['entityId']] = {
"entity": entity,
"action": action,
"link": entity["link"]
}
if not processable:
return True
# Find project of entities
ft_project = None
for entity_dict in processable_ent.values():
try:
base_proj = entity_dict['link'][0]
except Exception:
continue
ft_project = session.get(base_proj['type'], base_proj['id'])
break
# check if project is set to auto-sync
if (
ft_project is None or
'avalon_auto_sync' not in ft_project['custom_attributes'] or
ft_project['custom_attributes']['avalon_auto_sync'] is False
):
return True
# Get hierarchical custom attributes from "avalon" group
custom_attributes = {}
query = 'CustomAttributeGroup where name is "avalon"'
all_avalon_attr = session.query(query).one()
for cust_attr in all_avalon_attr['custom_attribute_configurations']:
if 'avalon_' in cust_attr['key']:
continue
if not cust_attr['is_hierarchical']:
continue
custom_attributes[cust_attr['key']] = cust_attr
if not custom_attributes:
return True
self.db_con.install()
self.db_con.Session['AVALON_PROJECT'] = ft_project['full_name']
for ent in processable:
entity_dict = processable_ent[ent['entityId']]
entity = entity_dict["entity"]
ent_path = "/".join([ent["name"] for ent in entity_dict['link']])
action = entity_dict["action"]
keys_to_process = {}
if action == "add":
# Store all custom attributes when entity was added
for key in custom_attributes:
keys_to_process[key] = entity['custom_attributes'][key]
else:
# Update only updated keys
for key in ent['keys']:
if key in custom_attributes:
keys_to_process[key] = entity['custom_attributes'][key]
processed_keys = self.get_hierarchical_values(
keys_to_process, entity
)
# Do the processing of values
self.update_hierarchical_attribute(entity, processed_keys, ent_path)
self.db_con.uninstall()
return True
def get_hierarchical_values(self, keys_dict, entity):
# check already set values
_set_keys = []
for key, value in keys_dict.items():
if value is not None:
_set_keys.append(key)
# pop set values from keys_dict
set_keys = {}
for key in _set_keys:
set_keys[key] = keys_dict.pop(key)
# find if entity has set values and pop them out
keys_to_pop = []
for key in keys_dict.keys():
_val = entity["custom_attributes"][key]
if _val:
keys_to_pop.append(key)
set_keys[key] = _val
for key in keys_to_pop:
keys_dict.pop(key)
# if there are not keys to find value return found
if not keys_dict:
return set_keys
# end recursion if entity is project
if entity.entity_type.lower() == "project":
for key, value in keys_dict.items():
set_keys[key] = value
else:
result = self.get_hierarchical_values(keys_dict, entity["parent"])
for key, value in result.items():
set_keys[key] = value
return set_keys
def update_hierarchical_attribute(self, entity, keys_dict, ent_path):
# TODO store all keys at once for entity
custom_attributes = entity.get('custom_attributes')
if not custom_attributes:
return
mongoid = custom_attributes.get(self.ca_mongoid)
if not mongoid:
return
try:
mongoid = ObjectId(mongoid)
except Exception:
return
mongo_entity = self.db_con.find_one({'_id': mongoid})
if not mongo_entity:
return
changed_keys = {}
data = mongo_entity.get('data') or {}
for key, value in keys_dict.items():
cur_value = data.get(key)
if cur_value:
if cur_value == value:
continue
changed_keys[key] = value
data[key] = value
if not changed_keys:
return
self.log.debug(
"{} - updated hierarchical attributes: {}".format(
ent_path, str(changed_keys)
)
)
self.db_con.update_many(
{'_id': mongoid},
{'$set': {'data': data}}
)
for child in entity.get('children', []):
_keys_dict = {}
for key, value in keys_dict.items():
if key not in child.get('custom_attributes', {}):
continue
child_value = child['custom_attributes'][key]
if child_value is not None:
continue
_keys_dict[key] = value
if not _keys_dict:
continue
child_path = "/".join([ent["name"] for ent in child['link']])
self.update_hierarchical_attribute(child, _keys_dict, child_path)
def register(session, plugins_presets):
'''Register plugin. Called when used as an plugin.'''
SyncHierarchicalAttrs(session, plugins_presets).register()

File diff suppressed because it is too large Load diff

View file

@ -1,4 +1,3 @@
import ftrack_api
from pype.ftrack import BaseEvent
@ -26,28 +25,34 @@ class ThumbnailEvents(BaseEvent):
# Update task thumbnail from published version
# if (entity['entityType'] == 'assetversion' and
# entity['action'] == 'encoded'):
if (
entity['entityType'] == 'assetversion'
and 'thumbid' in (entity.get('keys') or [])
elif (
entity['entityType'] == 'assetversion' and
entity['action'] != 'remove' and
'thumbid' in (entity.get('keys') or [])
):
version = session.get('AssetVersion', entity['entityId'])
if not version:
continue
thumbnail = version.get('thumbnail')
if thumbnail:
parent = version['asset']['parent']
task = version['task']
parent['thumbnail_id'] = version['thumbnail_id']
if parent.entity_type.lower() == "project":
name = parent["full_name"]
else:
name = parent["name"]
msg = '>>> Updating thumbnail for shot [ {} ]'.format(name)
if not thumbnail:
continue
if task:
task['thumbnail_id'] = version['thumbnail_id']
msg += " and task [ {} ]".format(task["name"])
parent = version['asset']['parent']
task = version['task']
parent['thumbnail_id'] = version['thumbnail_id']
if parent.entity_type.lower() == "project":
name = parent["full_name"]
else:
name = parent["name"]
msg = '>>> Updating thumbnail for shot [ {} ]'.format(name)
self.log.info(msg)
if task:
task['thumbnail_id'] = version['thumbnail_id']
msg += " and task [ {} ]".format(task["name"])
self.log.info(msg)
try:
session.commit()
@ -57,5 +62,4 @@ class ThumbnailEvents(BaseEvent):
def register(session, plugins_presets):
'''Register plugin. Called when used as an plugin.'''
ThumbnailEvents(session, plugins_presets).register()

View file

@ -1,12 +1,15 @@
import ftrack_api
from pype.ftrack import BaseEvent, lib
from pype.ftrack.lib.io_nonsingleton import DbConnector
from bson.objectid import ObjectId
from pypeapp import config
from pypeapp import Anatomy
import subprocess
import os
import re
import subprocess
from pype.ftrack import BaseEvent
from pype.ftrack.lib.avalon_sync import CustAttrIdKey
from pype.ftrack.lib.io_nonsingleton import DbConnector
from bson.objectid import ObjectId
from pypeapp import config
from pypeapp import Anatomy
class UserAssigmentEvent(BaseEvent):
@ -36,7 +39,6 @@ class UserAssigmentEvent(BaseEvent):
"""
db_con = DbConnector()
ca_mongoid = lib.get_ca_mongoid()
def error(self, *err):
for e in err:
@ -105,7 +107,7 @@ class UserAssigmentEvent(BaseEvent):
self.db_con.Session['AVALON_PROJECT'] = task['project']['full_name']
avalon_entity = None
parent_id = parent['custom_attributes'].get(self.ca_mongoid)
parent_id = parent['custom_attributes'].get(CustAttrIdKey)
if parent_id:
parent_id = ObjectId(parent_id)
avalon_entity = self.db_con.find_one({

View file

@ -41,7 +41,7 @@ class ProcessEventHub(ftrack_api.event.hub.EventHub):
def prepare_dbcon(self):
try:
self.dbcon.install()
self.dbcon._database.collection_names()
self.dbcon._database.list_collection_names()
except pymongo.errors.AutoReconnect:
log.error("Mongo server \"{}\" is not responding, exiting.".format(
os.environ["AVALON_MONGO"]

View file

@ -21,11 +21,23 @@ class StorerEventHub(ftrack_api.event.hub.EventHub):
def _handle_packet(self, code, packet_identifier, path, data):
"""Override `_handle_packet` which extend heartbeat"""
if self._code_name_mapping[code] == "heartbeat":
code_name = self._code_name_mapping[code]
if code_name == "heartbeat":
# Reply with heartbeat.
self.sock.sendall(b"storer")
return self._send_packet(self._code_name_mapping['heartbeat'])
elif code_name == "connect":
event = ftrack_api.event.base.Event(
topic="pype.storer.started",
data={},
source={
"id": self.id,
"user": {"username": self._api_user}
}
)
self._event_queue.put(event)
return super(StorerEventHub, self)._handle_packet(
code, packet_identifier, path, data
)

View file

@ -5,6 +5,7 @@ import signal
import socket
import pymongo
import ftrack_api
from ftrack_server import FtrackServer
from pype.ftrack.ftrack_server.lib import get_ftrack_event_mongo_info
from pype.ftrack.lib.custom_db_connector import DbConnector
@ -15,6 +16,13 @@ log = Logger().get_logger("Event storer")
url, database, table_name = get_ftrack_event_mongo_info()
class SessionClass:
def __init__(self):
self.session = None
session_obj = SessionClass()
dbcon = DbConnector(
mongo_url=url,
database_name=database,
@ -24,10 +32,11 @@ dbcon = DbConnector(
# ignore_topics = ["ftrack.meta.connected"]
ignore_topics = []
def install_db():
try:
dbcon.install()
dbcon._database.collection_names()
dbcon._database.list_collection_names()
except pymongo.errors.AutoReconnect:
log.error("Mongo server \"{}\" is not responding, exiting.".format(
os.environ["AVALON_MONGO"]
@ -49,7 +58,7 @@ def launch(event):
try:
# dbcon.insert_one(event_data)
dbcon.update({"id": event_id}, event_data, upsert=True)
dbcon.replace_one({"id": event_id}, event_data, upsert=True)
log.debug("Event: {} stored".format(event_id))
except pymongo.errors.AutoReconnect:
@ -65,10 +74,71 @@ def launch(event):
)
def trigger_sync(event):
session = session_obj.session
if session is None:
log.warning("Session is not set. Can't trigger Sync to avalon action.")
return True
projects = session.query("Project").all()
if not projects:
return True
query = {
"pype_data.is_processed": False,
"topic": "ftrack.action.launch",
"data.actionIdentifier": "sync.to.avalon.server"
}
set_dict = {
"$set": {"pype_data.is_processed": True}
}
dbcon.update_many(query, set_dict)
selections = []
for project in projects:
if project["status"] != "active":
continue
auto_sync = project["custom_attributes"].get("avalon_auto_sync")
if not auto_sync:
continue
selections.append({
"entityId": project["id"],
"entityType": "show"
})
if not selections:
return
user = session.query(
"User where username is \"{}\"".format(session.api_user)
).one()
user_data = {
"username": user["username"],
"id": user["id"]
}
for selection in selections:
event_data = {
"actionIdentifier": "sync.to.avalon.server",
"selection": [selection]
}
session.event_hub.publish(
ftrack_api.event.base.Event(
topic="ftrack.action.launch",
data=event_data,
source=dict(user=user_data)
),
on_error="ignore"
)
def register(session):
'''Registers the event, subscribing the discover and launch topics.'''
install_db()
session.event_hub.subscribe("topic=*", launch)
session.event_hub.subscribe("topic=pype.storer.started", trigger_sync)
def main(args):
@ -85,6 +155,7 @@ def main(args):
try:
session = StorerSession(auto_connect_event_hub=True, sock=sock)
session_obj.session = session
register(session)
server = FtrackServer("event")
log.debug("Launched Ftrack Event storer")

View file

@ -1,4 +1,4 @@
from .avalon_sync import *
from . import avalon_sync
from .credentials import *
from .ftrack_app_handler import *
from .ftrack_event_handler import *

File diff suppressed because it is too large Load diff

View file

@ -2,7 +2,6 @@ import functools
import time
from pypeapp import Logger
import ftrack_api
from ftrack_api import session as fa_session
from pype.ftrack.ftrack_server import session_processor
@ -243,7 +242,7 @@ class BaseHandler(object):
_entities is None or
_entities[0].get(
'link', None
) == fa_session.ftrack_api.symbol.NOT_SET
) == ftrack_api.symbol.NOT_SET
):
_entities = self._get_entities(event)
@ -447,7 +446,7 @@ class BaseHandler(object):
'applicationId=ftrack.client.web and user.id="{0}"'
).format(user_id)
self.session.event_hub.publish(
fa_session.ftrack_api.event.base.Event(
ftrack_api.event.base.Event(
topic='ftrack.action.trigger-user-interface',
data=dict(
type='message',
@ -495,8 +494,8 @@ class BaseHandler(object):
if not user:
raise TypeError((
'Ftrack user with {} "{}" was not found!'.format(key, value)
))
'Ftrack user with {} "{}" was not found!'
).format(key, value))
user_id = user['id']
@ -505,7 +504,7 @@ class BaseHandler(object):
).format(user_id)
self.session.event_hub.publish(
fa_session.ftrack_api.event.base.Event(
ftrack_api.event.base.Event(
topic='ftrack.action.trigger-user-interface',
data=dict(
type='widget',
@ -533,7 +532,7 @@ class BaseHandler(object):
else:
first = False
subtitle = {'type': 'label', 'value':'<h3>{}</h3>'.format(key)}
subtitle = {'type': 'label', 'value': '<h3>{}</h3>'.format(key)}
items.append(subtitle)
if isinstance(value, list):
for item in value:
@ -593,7 +592,7 @@ class BaseHandler(object):
# Create and trigger event
session.event_hub.publish(
fa_session.ftrack_api.event.base.Event(
ftrack_api.event.base.Event(
topic=topic,
data=_event_data,
source=dict(user=_user_data)
@ -614,7 +613,7 @@ class BaseHandler(object):
if not source and event:
source = event.get("source")
# Create and trigger event
event = fa_session.ftrack_api.event.base.Event(
event = ftrack_api.event.base.Event(
topic=topic,
data=event_data,
source=source

View file

@ -28,7 +28,8 @@ class IntegrateFtrackInstance(pyblish.api.InstancePlugin):
'plate': 'img',
'audio': 'audio',
'workfile': 'scene',
'animation': 'cache'
'animation': 'cache',
'image': 'img'
}
def process(self, instance):

View file

@ -70,7 +70,9 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
"audio",
"yetiRig",
"yeticache",
"source"
"source",
"matchmove",
"image"
]
exclude_families = ["clip"]

View file

@ -0,0 +1,30 @@
from avalon import api
from maya import mel
class MatchmoveLoader(api.Loader):
"""
This will run matchmove script to create track in scene.
Supported script types are .py and .mel
"""
families = ["matchmove"]
representations = ["py", "mel"]
defaults = ["Camera", "Object", "Mocap"]
label = "Run matchmove script"
icon = "empire"
color = "orange"
def load(self, context, name, namespace, data):
if self.fname.lower().endswith(".py"):
exec(open(self.fname).read())
elif self.fname.lower().endswith(".mel"):
mel.eval('source "{}"'.format(self.fname))
else:
self.log.error("Unsupported script type")
return True

View file

@ -0,0 +1,24 @@
from avalon import api
class MatchmoveLoader(api.Loader):
"""
This will run matchmove script to create track in script.
"""
families = ["matchmove"]
representations = ["py"]
defaults = ["Camera", "Object"]
label = "Run matchmove script"
icon = "empire"
color = "orange"
def load(self, context, name, namespace, data):
if self.fname.lower().endswith(".py"):
exec(open(self.fname).read())
else:
self.log.error("Unsupported script type")
return True

View file

@ -45,66 +45,71 @@ class CollectContextDataSAPublish(pyblish.api.ContextPlugin):
with open(input_json_path, "r") as f:
in_data = json.load(f)
asset_name = in_data['asset']
family_preset_key = in_data.get('family_preset_key', '')
family = in_data['family']
subset = in_data['subset']
asset_name = in_data["asset"]
family_preset_key = in_data.get("family_preset_key", "")
family = in_data["family"]
subset = in_data["subset"]
# Load presets
presets = context.data.get("presets")
if not presets:
from pypeapp import config
presets = config.get_presets()
# Get from presets anatomy key that will be used for getting template
# - default integrate new is used if not set
anatomy_key = presets.get(
"standalone_publish", {}).get(
"families", {}).get(
family_preset_key, {}).get(
"anatomy_template"
anatomy_key = (
presets.get("standalone_publish", {})
.get("families", {})
.get(family_preset_key, {})
.get("anatomy_template")
)
project = io.find_one({'type': 'project'})
asset = io.find_one({
'type': 'asset',
'name': asset_name
})
context.data['project'] = project
context.data['asset'] = asset
project = io.find_one({"type": "project"})
asset = io.find_one({"type": "asset", "name": asset_name})
context.data["project"] = project
context.data["asset"] = asset
instance = context.create_instance(subset)
instance.data.update({
"subset": subset,
"asset": asset_name,
"label": subset,
"name": subset,
"family": family,
"frameStart": in_data.get("representations", [None])[0].get("frameStart", None),
"frameEnd": in_data.get("representations", [None])[0].get("frameEnd", None),
"families": [family, 'ftrack'],
})
instance.data.update(
{
"subset": subset,
"asset": asset_name,
"label": subset,
"name": subset,
"family": family,
"version": in_data.get("version", 1),
"frameStart": in_data.get("representations", [None])[0].get(
"frameStart", None
),
"frameEnd": in_data.get("representations", [None])[0].get(
"frameEnd", None
),
"families": [family, "ftrack"],
}
)
self.log.info("collected instance: {}".format(instance.data))
self.log.info("parsing data: {}".format(in_data))
instance.data['destination_list'] = list()
instance.data['representations'] = list()
instance.data['source'] = 'standalone publisher'
instance.data["destination_list"] = list()
instance.data["representations"] = list()
instance.data["source"] = "standalone publisher"
for component in in_data['representations']:
for component in in_data["representations"]:
component['destination'] = component['files']
component['stagingDir'] = component['stagingDir']
component["destination"] = component["files"]
component["stagingDir"] = component["stagingDir"]
# Do not set anatomy_template if not specified
if anatomy_key:
component['anatomy_template'] = anatomy_key
if isinstance(component['files'], list):
collections, remainder = clique.assemble(component['files'])
component["anatomy_template"] = anatomy_key
if isinstance(component["files"], list):
collections, remainder = clique.assemble(component["files"])
self.log.debug("collecting sequence: {}".format(collections))
instance.data["frameStart"] = int(component["frameStart"])
instance.data["frameEnd"] = int(component["frameEnd"])
instance.data['fps'] = int(component['fps'])
instance.data["fps"] = int(component["fps"])
if component["preview"]:
instance.data["families"].append("review")

View file

@ -0,0 +1,29 @@
"""
Requires:
Nothing
Provides:
Instance
"""
import pyblish.api
import logging
log = logging.getLogger("collector")
class CollectMatchmovePublish(pyblish.api.InstancePlugin):
"""
Collector with only one reason for its existence - remove 'ftrack'
family implicitly added by Standalone Publisher
"""
label = "Collect Matchmove - SA Publish"
order = pyblish.api.CollectorOrder
family = ["matchmove"]
hosts = ["standalonepublisher"]
def process(self, instance):
if "ftrack" in instance.data["families"]:
instance.data["families"].remove("ftrack")

View file

@ -1,4 +1,5 @@
from . import QtCore
import re
class RecursiveSortFilterProxyModel(QtCore.QSortFilterProxyModel):